From 7f292e6f7a585db640b6ec87c7989fbdec1a3c23 Mon Sep 17 00:00:00 2001 From: zjk <15778488+zjks@user.noreply.gitee.com> Date: Wed, 30 Apr 2025 17:12:22 +0800 Subject: [PATCH 1/3] init ais-bench_workload --- ais-bench_workload/README.md | 47 ++ ais-bench_workload/build/build.sh | 153 +++++++ .../build/download_and_build.sh | 91 ++++ ...64\346\230\216\346\226\207\346\241\243.md" | 429 ++++++++++++++++++ ...64\346\230\216\346\226\207\346\241\243.md" | 225 +++++++++ ...55\345\273\272\346\214\207\345\257\274.md" | 160 +++++++ ...04\345\273\272\346\225\231\347\250\213.md" | 235 ++++++++++ ...\351\227\250\346\214\207\345\257\274.docx" | Bin 0 -> 488919 bytes ...rm\347\232\204\346\226\271\346\263\225.md" | 163 +++++++ ...11\345\205\250\345\243\260\346\230\216.md" | 10 + ais-bench_workload/img/faq1.png | Bin 0 -> 30108 bytes ais-bench_workload/src/ais_utils_adapter.py | 28 ++ .../src/common/calc_glm2_result.py | 46 ++ .../src/common/calc_llm_result.py | 55 +++ ais-bench_workload/src/common/calc_power.sh | 149 ++++++ .../src/common/calc_resourceinfo.sh | 88 ++++ ais-bench_workload/src/common/calc_result.py | 43 ++ .../src/common/cluster_common.sh | 267 +++++++++++ .../src/common/cluster_common_2.0.sh | 128 ++++++ ais-bench_workload/src/common/common.sh | 61 +++ ais-bench_workload/src/common/log_util.sh | 60 +++ .../src/common/modelarts_handler.py | 269 +++++++++++ .../src/common/modelarts_handler_v2.py | 201 ++++++++ ais-bench_workload/src/common/node_common.sh | 113 +++++ ais-bench_workload/src/common/patch_common.sh | 67 +++ .../src/common/sshpass_common.sh | 212 +++++++++ .../src/common/train_modelarts.py | 106 +++++ .../src/train/huawei/common/mindspore_env.sh | 16 + .../src/train/huawei/common/tensorflow_env.sh | 40 ++ .../huawei/train_mindspore_bert/build.sh | 39 ++ .../train_mindspore_bert/config/config.sh | 18 + .../config/modelarts_config.py | 159 +++++++ .../config/modelarts_config.py.r1.3 | 159 +++++++ .../modelarts_r1.10.patch | 182 ++++++++ .../train_mindspore_bert/modelarts_r1.3.patch | 199 ++++++++ .../train_mindspore_bert/modelarts_r1.5.patch | 192 ++++++++ .../train_mindspore_bert/modelarts_r1.7.patch | 182 ++++++++ .../train_mindspore_bert/modelarts_r1.8.patch | 184 ++++++++ .../train_mindspore_bert/modelarts_r1.9.patch | 182 ++++++++ .../train_mindspore_bert/modelarts_r2.0.patch | 180 ++++++++ .../train_mindspore_bert/modelarts_r2.1.patch | 180 ++++++++ .../train_mindspore_bert/modelarts_r2.2.patch | 180 ++++++++ .../huawei/train_mindspore_bert/patch.sh | 140 ++++++ .../huawei/train_mindspore_bert/r1.10.patch | 76 ++++ .../huawei/train_mindspore_bert/r1.5.patch | 75 +++ .../huawei/train_mindspore_bert/r1.6.patch | 77 ++++ .../huawei/train_mindspore_bert/r1.7.patch | 77 ++++ .../huawei/train_mindspore_bert/r1.8.patch | 78 ++++ .../huawei/train_mindspore_bert/r1.9.patch | 80 ++++ .../huawei/train_mindspore_bert/r2.0.patch | 77 ++++ .../huawei/train_mindspore_bert/r2.1.patch | 77 ++++ .../huawei/train_mindspore_bert/r2.2.patch | 77 ++++ .../huawei/train_mindspore_bert/r2.3.patch | 77 ++++ .../train_mindspore_bert/scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../scripts/modelarts_run.sh | 39 ++ .../train_mindspore_bert/scripts/run_node.sh | 113 +++++ .../huawei/train_mindspore_deeplabv3/build.sh | 31 ++ .../config/config.sh | 16 + ...77\347\224\250\350\257\264\346\230\216.md" | 75 +++ .../huawei/train_mindspore_deeplabv3/patch.sh | 77 ++++ .../train_mindspore_deeplabv3/r1.9.patch | 56 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 78 ++++ .../scripts/run_node.sh | 106 +++++ .../train_mindspore_deepspeech2/build.sh | 31 ++ .../config/config.sh | 6 + ...77\347\224\250\350\257\264\346\230\216.md" | 109 +++++ .../train_mindspore_deepspeech2/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 77 ++++ .../scripts/run_node.sh | 81 ++++ .../train_mindspore_faster_rcnn/build.sh | 31 ++ .../config/config.sh | 16 + ...77\347\224\250\350\257\264\346\230\216.md" | 72 +++ .../train_mindspore_faster_rcnn/patch.sh | 71 +++ .../train_mindspore_faster_rcnn/r1.9.patch | 71 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../scripts/run_node.sh | 104 +++++ .../huawei/train_mindspore_glm2/README.md | 201 ++++++++ .../huawei/train_mindspore_glm2/build.sh | 33 ++ .../train_mindspore_glm2/config/config.sh | 33 ++ .../huawei/train_mindspore_glm2/patch.sh | 65 +++ .../huawei/train_mindspore_glm2/r2.2.patch | 231 ++++++++++ .../train_mindspore_glm2/scripts/benchmark.sh | 44 ++ .../scripts/cluster_offline_run.sh | 134 ++++++ .../scripts/run_glm2_6b_finetune.yaml | 250 ++++++++++ .../scripts/run_glm2_6b_finetune_eval.yaml | 250 ++++++++++ .../train_mindspore_glm2/scripts/run_node.sh | 167 +++++++ .../huawei/train_mindspore_gnmt_v2/build.sh | 36 ++ .../train_mindspore_gnmt_v2/config/config.sh | 18 + .../huawei/train_mindspore_gnmt_v2/patch.sh | 65 +++ .../huawei/train_mindspore_gnmt_v2/r1.9.patch | 72 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../scripts/run_node.sh | 87 ++++ .../huawei/train_mindspore_llama/README.md | 156 +++++++ .../huawei/train_mindspore_llama/build.sh | 34 ++ .../train_mindspore_llama/config/config.sh | 38 ++ .../huawei/train_mindspore_llama/patch.sh | 65 +++ .../huawei/train_mindspore_llama/r2.2.patch | 231 ++++++++++ .../scripts/benchmark.sh | 46 ++ .../scripts/cluster_offline_run.sh | 146 ++++++ .../scripts/pre_conf_yaml.py | 121 +++++ .../train_mindspore_llama/scripts/run_node.sh | 181 ++++++++ .../train_mindspore_pangu_alpha/build.sh | 31 ++ .../config/config.sh | 17 + .../train_mindspore_pangu_alpha/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 78 ++++ .../scripts/run_node.sh | 83 ++++ .../huawei/train_mindspore_resnet/build.sh | 36 ++ .../train_mindspore_resnet/config/config.sh | 14 + .../config/modelarts_config.py | 134 ++++++ .../config/modelarts_config.py.r1.3 | 134 ++++++ .../modelarts_r1.10.patch | 162 +++++++ .../modelarts_r1.3.patch | 230 ++++++++++ .../modelarts_r1.5.patch | 162 +++++++ .../modelarts_r1.7.patch | 162 +++++++ .../modelarts_r1.8.patch | 166 +++++++ .../modelarts_r1.9.patch | 163 +++++++ .../modelarts_r2.0.patch | 164 +++++++ .../modelarts_r2.1.patch | 164 +++++++ .../modelarts_r2.2.patch | 164 +++++++ .../huawei/train_mindspore_resnet/patch.sh | 134 ++++++ .../huawei/train_mindspore_resnet/r1.10.patch | 85 ++++ .../huawei/train_mindspore_resnet/r1.5.patch | 87 ++++ .../huawei/train_mindspore_resnet/r1.6.patch | 88 ++++ .../huawei/train_mindspore_resnet/r1.7.patch | 83 ++++ .../huawei/train_mindspore_resnet/r1.8.patch | 83 ++++ .../huawei/train_mindspore_resnet/r1.9.patch | 82 ++++ .../huawei/train_mindspore_resnet/r2.0.patch | 91 ++++ .../huawei/train_mindspore_resnet/r2.1.patch | 91 ++++ .../huawei/train_mindspore_resnet/r2.2.patch | 91 ++++ .../huawei/train_mindspore_resnet/r2.3.patch | 91 ++++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 81 ++++ .../scripts/modelarts_run.sh | 42 ++ .../scripts/run_node.sh | 110 +++++ .../train/huawei/train_mindspore_ssd/build.sh | 36 ++ .../train_mindspore_ssd/config/config.sh | 20 + .../train/huawei/train_mindspore_ssd/patch.sh | 65 +++ .../huawei/train_mindspore_ssd/r1.9.patch | 76 ++++ .../train_mindspore_ssd/scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../train_mindspore_ssd/scripts/run_node.sh | 107 +++++ .../huawei/train_mindspore_widedeep/build.sh | 31 ++ .../train_mindspore_widedeep/config/config.sh | 12 + ...77\347\224\250\350\257\264\346\230\216.md" | 63 +++ .../huawei/train_mindspore_widedeep/patch.sh | 71 +++ .../train_mindspore_widedeep/r1.5.patch | 73 +++ .../train_mindspore_widedeep/r1.9.patch | 78 ++++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 78 ++++ .../scripts/run_node.sh | 107 +++++ .../train_tensorflow_bert_base/build.sh | 31 ++ .../config/config.sh | 12 + .../train_tensorflow_bert_base/patch.sh | 66 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 79 ++++ .../scripts/run_node.sh | 96 ++++ .../train_tensorflow_densenet121/build.sh | 31 ++ .../config/config.sh | 11 + .../train_tensorflow_densenet121/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 79 ++++ .../scripts/run_node.sh | 78 ++++ .../train_tensorflow_mobilenetv2/build.sh | 31 ++ .../config/config.sh | 14 + .../train_tensorflow_mobilenetv2/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../scripts/run_node.sh | 86 ++++ .../train_tensorflow_nezha_large/build.sh | 31 ++ .../config/config.sh | 12 + .../train_tensorflow_nezha_large/master.patch | 28 ++ .../train_tensorflow_nezha_large/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 79 ++++ .../scripts/run_node.sh | 99 ++++ .../train_tensorflow_resnet101/build.sh | 31 ++ .../config/config.sh | 13 + .../train_tensorflow_resnet101/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 82 ++++ .../scripts/run_node.sh | 84 ++++ .../train_tensorflow_resnet50/README.txt | 7 + .../huawei/train_tensorflow_resnet50/build.sh | 31 ++ .../config/config.sh | 15 + ...77\347\224\250\350\257\264\346\230\216.md" | 71 +++ .../train_tensorflow_resnet50/master.patch | 48 ++ .../huawei/train_tensorflow_resnet50/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../scripts/run_node.sh | 74 +++ .../train_tensorflow_resnext50/build.sh | 31 ++ .../config/config.sh | 11 + .../train_tensorflow_resnext50/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 79 ++++ .../scripts/run_node.sh | 87 ++++ .../train_tensorflow_ssd_resnet34/build.sh | 31 ++ .../config/config.sh | 16 + .../train_tensorflow_ssd_resnet34/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 83 ++++ .../scripts/run_node.sh | 78 ++++ .../huawei/train_tensorflow_vgg16/build.sh | 31 ++ .../train_tensorflow_vgg16/config/config.sh | 13 + .../huawei/train_tensorflow_vgg16/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 80 ++++ .../scripts/run_node.sh | 83 ++++ .../huawei/train_tensorflow_yolov3/build.sh | 31 ++ .../train_tensorflow_yolov3/config/config.sh | 11 + .../train_tensorflow_yolov3/master.patch | 43 ++ .../huawei/train_tensorflow_yolov3/patch.sh | 65 +++ .../scripts/benchmark.sh | 31 ++ .../scripts/cluster_offline_run.sh | 78 ++++ .../scripts/run_node.sh | 105 +++++ .../nvidia/train_tensorflow_bert/README.MD | 80 ++++ .../nvidia/train_tensorflow_bert/build.sh | 21 + .../config/config_pretrain.sh | 13 + .../nvidia/train_tensorflow_bert/master.patch | 35 ++ .../nvidia/train_tensorflow_bert/patch.sh | 72 +++ .../scripts/benchmark.sh | 61 +++ .../train_tensorflow_bert/scripts/run.sh | 48 ++ .../nvidia/train_tensorflow_resnet/README.MD | 55 +++ .../nvidia/train_tensorflow_resnet/build.sh | 21 + .../config/config_imagenet2012.sh | 13 + .../nvidia/train_tensorflow_resnet/patch.sh | 79 ++++ .../train_tensorflow_resnet/r1.13.0.patch | 59 +++ .../scripts/benchmark.sh | 65 +++ .../train_tensorflow_resnet/scripts/run.sh | 39 ++ 235 files changed, 19148 insertions(+) create mode 100644 ais-bench_workload/README.md create mode 100644 ais-bench_workload/build/build.sh create mode 100644 ais-bench_workload/build/download_and_build.sh create mode 100644 "ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" create mode 100644 "ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" create mode 100644 "ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" create mode 100644 "ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" create mode 100644 "ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" create mode 100644 "ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" create mode 100644 "ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" create mode 100644 ais-bench_workload/img/faq1.png create mode 100644 ais-bench_workload/src/ais_utils_adapter.py create mode 100644 ais-bench_workload/src/common/calc_glm2_result.py create mode 100644 ais-bench_workload/src/common/calc_llm_result.py create mode 100644 ais-bench_workload/src/common/calc_power.sh create mode 100644 ais-bench_workload/src/common/calc_resourceinfo.sh create mode 100644 ais-bench_workload/src/common/calc_result.py create mode 100644 ais-bench_workload/src/common/cluster_common.sh create mode 100644 ais-bench_workload/src/common/cluster_common_2.0.sh create mode 100644 ais-bench_workload/src/common/common.sh create mode 100644 ais-bench_workload/src/common/log_util.sh create mode 100644 ais-bench_workload/src/common/modelarts_handler.py create mode 100644 ais-bench_workload/src/common/modelarts_handler_v2.py create mode 100644 ais-bench_workload/src/common/node_common.sh create mode 100644 ais-bench_workload/src/common/patch_common.sh create mode 100644 ais-bench_workload/src/common/sshpass_common.sh create mode 100644 ais-bench_workload/src/common/train_modelarts.py create mode 100644 ais-bench_workload/src/train/huawei/common/mindspore_env.sh create mode 100644 ais-bench_workload/src/train/huawei/common/tensorflow_env.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh create mode 100644 "ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh create mode 100644 "ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh create mode 100644 "ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh create mode 100644 "ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh create mode 100644 "ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh create mode 100644 ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh create mode 100644 ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh diff --git a/ais-bench_workload/README.md b/ais-bench_workload/README.md new file mode 100644 index 0000000..b26cd20 --- /dev/null +++ b/ais-bench_workload/README.md @@ -0,0 +1,47 @@ +# ais-bench-workload + +## 介绍 + +ais-bench-workload目录主要承载基于AISBench测试基准的模型负载代码以及为AISBench测试基准贡献的高易用性子工具，用于AI服务器的性能测试。 + +### AISBench场景介绍 + +AISBench标准化性能测试软件，又称AI Server Benchmark软件，是根据AI标准（IEEE 2937及 T/CESA 1169-2021）对AI服务器进行性能测试的工具软件。 + +AISBench软件包括如下2个测试场景： + +- 网络测试模式 - 适用于正式测试场景 + +```mermaid + +graph LR + subgraph stubs服务器-被测试者-厂商设备 + ais-bench-stubs -- 本地拉起 --> 负载代码 + end + subgraph tester服务器-测试者 + ais-bench-tester --网络交互通信 --> ais-bench-stubs + end + +``` + +- 本地离线测试模式 - 适用于本地裸机测试场景，不需要联网,不需要连接tester服务器 + +```mermaid + +graph LR + subgraph stubs服务器-被测试者-厂商设备 + ais-bench-stubs --本地拉起 --> 负载代码 + end + +``` + +### AISBench工具介绍 + +AISBench包括如下工具： + +| 工具名 | 工具及资料获取 | +| -------------------- | ------------------------------------------------------------ | +| AISBench测试基准工具 | 请从[人工智能系统性能基准工作组](https://www.aisbench.com/tool)获取。 | +| AISBench模型负载工具 | 训练负载：https://gitee.com/aisbench/training
推理负载：https://gitee.com/aisbench/inference
+| AISBench推理工具 | https://gitee.com/ascend/tools/tree/master/ais-bench_workload/tool/ais_bench | + diff --git a/ais-bench_workload/build/build.sh b/ais-bench_workload/build/build.sh new file mode 100644 index 0000000..96c28bd --- /dev/null +++ b/ais-bench_workload/build/build.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) +ROOT_PATH=$(readlink -f $CURDIR/../) +OUTPUT_PATH="$ROOT_PATH/output/" + +export VERSION="1.0" + +check_command_exist() +{ + command=$1 + if type $command >/dev/null 2>&1;then + return 0 + else + return 1 + fi +} + +function check_env() +{ + check_command_exist git || { echo "git running failed" ; return 1; } + return $ret_ok +} + +function get_and_check_args() +{ + # check args num is valid + [ $# -ge 4 ] || { echo "args should >=3 not valid";return 1; } + + STUBS_PACKETS=$1 + STUBS_NAME=$(basename $STUBS_PACKETS) + STUBS_SUBNAME=${STUBS_NAME%.tar.gz} + + # according type get application base dir + basedir="$2" + manufactory_group=`ls $ROOT_PATH/src/$basedir` + echo $manufactory_group | grep -wq "$3" || { echo "$3 not invliad in [$manufactory_group] return";return 1; } + MANUFACTORY=$3 + + target_group=`ls $ROOT_PATH/src/$basedir/$MANUFACTORY` + echo $target_group | grep -wq "$4" || { echo "$4 not invliad in [$target_group] return";return 1; } + TARGETDIR=$4 + + shift + shift + shift + shift + scripts_args="$@" +} + +# copy doc files to packet +copy_doc_files() +{ + local branch_args="$1" + local run_type="$2" + + [ -d $OUTPUT_BASE_DIR/code/doc/ ] || mkdir -p $OUTPUT_BASE_DIR/code/doc/ + cp $ROOT_PATH/doc/*.md $OUTPUT_BASE_DIR/code/doc/ + + [[ "$PACKET_TYPE" == "inference" ]] && { cp $OUTPUT_BASE_DIR/code/doc/ais-bench_workload_inference*.md $OUTPUT_BASE_DIR/README.md;return; } + + # train modelarts mode + [[ "$PACKET_TYPE" == "train" && "$run_type" == "modelarts" ]] && { cp $OUTPUT_BASE_DIR/code/doc/ais-bench_workload_train_modelarts*.md $OUTPUT_BASE_DIR/README.md;return; } + # default as train offline mode + cp $OUTPUT_BASE_DIR/code/doc/ais-bench_workload_train_offline*.md $OUTPUT_BASE_DIR/README.md +} + +function build_packet() +{ + get_and_check_args "$@" || { echo "get check args failed ret:$ret";return $ret_error; } + + PACKET_TYPE="$2" + + BUILD_TMP_PATH=$CURDIR/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + # untar packfile + tar xvf $STUBS_PACKETS -C $BUILD_TMP_PATH || { echo "tar file failed ret";return $ret_error; } + + # exec build.sh and cp files + TARGET_PATH=$ROOT_PATH/src/$PACKET_TYPE/$MANUFACTORY/$TARGETDIR + if [ -f $TARGET_PATH/build.sh ];then + bash $TARGET_PATH/build.sh $scripts_args || { echo "warn build target failed"; return $ret_error; } + fi + + if [ ! -d $TARGET_PATH/output ];then + echo "targetdir:$TARGET_PATH not find output return" + return $ret_error + fi + + cd $BUILD_TMP_PATH + # get untar dir + OUTPUT_BASE_DIR=`find ./ -name "Ais-Benchmark-Stubs*" -type d` + if [[ ! -d "$OUTPUT_BASE_DIR" || ! -d "$OUTPUT_BASE_DIR/code" ]];then + echo "find no path:$OUTPUT_BASE_DIR return" + return $ret_error + fi + + cp $TARGET_PATH/output/* -rf $OUTPUT_BASE_DIR/code + chmod -R u+x $OUTPUT_BASE_DIR/code + + # for stubs old versions add adapter new ais_utils.py file + if [ ! -f $OUTPUT_BASE_DIR/code/ais_utils.py ] && [ -f $OUTPUT_BASE_DIR/code/libset_result.so ];then + cp ${ROOT_PATH}/src/ais_utils_adapter.py $OUTPUT_BASE_DIR/code/ais_utils.py + fi + + copy_doc_files $scripts_args + + #PLATFORM=`uname -i` + # OUTPUT_PACKET_NAME="$PACKET_TYPE"_"$MANUFACTORY"_"$TARGETDIR-Ais-Bench-$PLATFORM-${scripts_args// /_}" + OUTPUT_PACKET_NAME="$PACKET_TYPE"_"$MANUFACTORY"_"$TARGETDIR-$STUBS_SUBNAME-${scripts_args// /_}" + rm -rf $OUTPUT_PATH/$OUTPUT_PACKET_NAME.tar.gz + mv $OUTPUT_BASE_DIR $OUTPUT_PACKET_NAME + tar -czf $OUTPUT_PATH/$OUTPUT_PACKET_NAME.tar.gz $OUTPUT_PACKET_NAME + ret=$? + if [ $ret != 0 ];then + echo "tar out packet failed ret:$ret" + return $ret_error + fi + return $ret_ok +} + +function main() +{ + [[ $1 == *"tar.gz" && -f $1 ]] || { echo "args1:$1 not valid file" ; return 1; } + + if [ "$2" != "inference" -a "$2" != "train" ];then + echo "target not valid in:[$1] not match [train inference]" + return $ret_error + fi + + [ -d $OUTPUT_PATH ] || { mkdir -p $OUTPUT_PATH; } + + target=$2 + echo "target:$target now building" + check_env || { ret=$?;echo "check env failed ret:$ret";return $ret; } + + if [ "$target" == "inference" -o "$target" == "train" ];then + build_packet "$@" || { echo "build build_inference failed:$?";return 1; } + else + echo "target:$target return" + return 1 + fi + echo "target:$target now build done" + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/build/download_and_build.sh b/ais-bench_workload/build/download_and_build.sh new file mode 100644 index 0000000..012e4c3 --- /dev/null +++ b/ais-bench_workload/build/download_and_build.sh @@ -0,0 +1,91 @@ +##!/bin/bash + +CURDIR=$(dirname $(readlink -f $0)) + +function check_command_exist() +{ + command=$1 + if type $command >/dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +check_file_valid() +{ + if [ ! -f "$1" ]; then + return 1 + fi + return 0 +} + +function try_download_stubs_packet(){ + #mkdir -p $INFER_BASE_PATH/opensource/gflags/src/ + #cmd="wget $1 --no-check-certificate -O $2" + cmd="curl -k -o $2 $1" + timeout 60 $cmd #>/dev/null 2>&1 + ret=$? + if [ "$ret" == 0 -a -s "$2" ];then + echo "download cmd:$cmd targetfile:$2 OK" + else + echo "downlaod targetfile by $cmd Failed please check network or manual download to target file" + return 1 + fi +} + +# unrar_files() +# { +# local target_file_=$1 +# local tmp_dir_=$2 +# check_command_exist start && { start winrar e $target_file_ $tmp_dir_;return 0; } +# check_command_exist unrar && { unrar e $target_file_ $tmp_dir_ ;return 0; } +# return 1; +# } + +main() +{ + local version="$1" + local type="$2" + [ "$2" != "modelarts" ] && type="" + + stubs_packet_x86_64_url="https://www.aisbench.com/assets/public/article_attachment/1/file/93fa30f8-ffac-4b99-9da6-78caa62d3df7.gz" + stubs_bak_x86_64_url="https://aisbenchtest.obs.cn-north-4.myhuaweicloud.com/stubs_package/Ais-Benchmark-Stubs-x86_64-1.0.tar.gz" # obs link + stubs_x86_64_pkg_name="Ais-Benchmark-Stubs-x86_64-1.0.tar.gz" + stubs_packet_aarch64_url="https://www.aisbench.com/assets/public/article_attachment/1/file/0796bd4e-136d-45fe-b863-24661679a283.gz" + stubs_bak_aarch64_url="https://aisbenchtest.obs.cn-north-4.myhuaweicloud.com/stubs_package/Ais-Benchmark-Stubs-aarch64-1.0.tar.gz" # obs link + stubs_aarch64_pkg_name="Ais-Benchmark-Stubs-aarch64-1.0.tar.gz" + + tmp_dir="$CURDIR/buildtmpstubs" + [ -d $tmp_dir ] && rm -rf $tmp_dir + mkdir -p $tmp_dir + + check_command_exist git || { echo "git cmd not valid"; return -1; } + check_command_exist tar || { echo "tar cmd not valid"; return -1; } + + target_file="$tmp_dir/${stubs_x86_64_pkg_name}" + try_download_stubs_packet $stubs_packet_x86_64_url $target_file || { + echo "donwload x86_64 stubs failed, using bak_url ${stubs_bak_x86_64_url}"; + try_download_stubs_packet $stubs_bak_x86_64_url $target_file || { echo "donwload x86_64 stubs failed again!";return 1; } + } + target_file="$tmp_dir/${stubs_aarch64_pkg_name}" + try_download_stubs_packet $stubs_packet_aarch64_url $target_file || { + echo "donwload aarch64 stubs failed, using bak_url ${stubs_bak_aarch64_url}"; + try_download_stubs_packet $stubs_bak_aarch64_url $target_file || { echo "donwload aarch64 stubs failed again!";return 1; } + } + + sleep 5 + x86_stubs="$tmp_dir/Ais-Benchmark-Stubs-x86_64-1.0.tar.gz" + check_file_valid "$x86_stubs" || { echo "x86_stubs:${x86_stubs} not valid path" ; return 1; } + + arm_stubs="$tmp_dir/Ais-Benchmark-Stubs-aarch64-1.0.tar.gz" + check_file_valid "$arm_stubs" || { echo "arm_stubs:${arm_stubs} not valid path" ; return 1; } + + bash -x $CURDIR/build.sh $x86_stubs train huawei train_mindspore_resnet $version $type + bash -x $CURDIR/build.sh $x86_stubs train huawei train_mindspore_bert $version $type + bash -x $CURDIR/build.sh $arm_stubs train huawei train_mindspore_resnet $version $type + bash -x $CURDIR/build.sh $arm_stubs train huawei train_mindspore_bert $version $type +} + +main "$@" +exit $? diff --git "a/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" "b/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" new file mode 100644 index 0000000..b0ca1d9 --- /dev/null +++ "b/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" @@ -0,0 +1,429 @@ +# ais-bench_workload_train_modelarts训练说明文档 + + + +[TOC] + +## 简介 +ais-bench标准化性能测试软件，又称AI Server Benchmark软件，是根据AI标准（IEEE 2937及 T/CESA 1169-2021）对AI服务器进行性能测试的工具软件。 + +本文主要介绍基于ais-bench软件，在ModelArts平台（线上环境）对模型进行训练性能测试。主要实现集群训练业务性能测试场景。 + +[Modelarts](https://support.huaweicloud.com/productdesc-modelarts/modelarts_01_0001.html)是面向AI开发者的一站式开发平台，提供海量数据预处理及半自动化标注、大规模分布式训练、自动化模型生成及端-边-云模型按需部署能力，帮助用户快速创建和部署模型，管理全周期AI工作流。 + +## 流程介绍 +modelarts业务启动有如下三种方式运行: + +```mermaid +graph LR +UI页面 --> Modelarts服务 +modelarts-sdk --> Modelarts服务 +modelarts-api --> Modelarts服务 +``` + +ModelArts线上训练性能测试选择modelarts-sdk作为启动方式，通过ais-bench-stubs拉起modelarts-sdk向ModelArts平台下发训练作业指令。 + + +测试操作总体流程： +如下测试流程，本测试需要一台本地运行设备，用于给modelarts服务下发训练作业。 +```mermaid +graph LR + subgraph 本地运行设备 + ais-bench-stubs --本地拉起 --> modelarts-sdk + end + subgraph Modelarts服务 + modelarts-sdk --网络启动训练作业 --> 训练作业 + end +``` + +测试过程数据传输原理图： + +```sequence +本地运行设备->>OBS存储: 上传运行代码 +本地运行设备->>ModelArts侧: 传递训练参数，拉起训练 +ModelArts侧->>OBS存储: 请求下载训练代码 +OBS存储->>ModelArts侧: 下载代码 +ModelArts侧->>OBS存储: 请求下载数据集 +OBS存储->>ModelArts侧: 下载数据集 +ModelArts侧->>ModelArts侧: 执行训练 +ModelArts侧->>OBS存储: 上传throughput/accuracy数据 +ModelArts侧->>本地运行设备: 训练完成 +本地运行设备->>OBS存储: 请求下载throughput/accuracy数据 +OBS存储->>本地运行设备: 下载数据 +``` + +测试操作总体步骤： + +1. 准备本地运行设备环境。 +2. 填写配置信息。 +3. 本地运行设备启动ais-bench-stubs程序，执行完成后获取性能数据结果。 + +## 使用前准备 + +### 环境 + +#### 本地运行设备 + +- 安装Linux系统。 +- 处于稳定的联网状态且能够与云上计算节点联网。 + +建议选择以下三种作为本地运行设备： + +- ECS云主机，可以咨询计算中心运维同事搭建启动ECS云主机。 +- Modelarts的notebook开发环境。 请参考《modelarts_notebook使用入门指导》。 +- windows上开启WSL linux子系统。请参考[官方链接。](https://docs.microsoft.com/zh-cn/windows/wsl/install) + +#### 软件依赖 + +- 安装Python3 + +- 安装easydict python程序包。 + + 请使用如下命令进行安装（当前以pip3为例，请选择与Python版本对应的的pip命令）： + + ``` + pip3 install easydict + ``` + +- 安装modelarts-sdk程序包。请根据[modelarts-sdk官网教程](https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0004.html)下载对应版本并执行安装。 +- 当前适配ModelArts版本 >= 21.9.0 +### 数据集 + +下载相关模型数据集并上传至OBS存储中。 + +注意：数据集的数据量较大，需要在执行测试前下载并上传至OBS，OBS上传操作请查看具体OBS操作指导。 + +以resnet模型的imagenet数据集，bert模型的enwiki数据集为例。具体下载方式请至相关模型官网，本文不作详述。 + +由于当前OBS限定训练作业的数据输入为一个目录（示例名称：obs_url），故imagenet或enwiki数据集上传至OBS存储时的目录结构如下： + +imagenet + +```mermaid +graph LR +obs_url --> train目录 --> 训练数据集 +obs_url --> val目录 --> eval数据集 +``` + +enviki + +```mermaid +graph LR +obs_url --> train目录 --> 训练数据集 +obs_url --> val目录 --> eval数据集 +obs_url --> ms_bert_large.ckpt预训练文件 +``` + + + +### 软件包 + +请参见 《ais-bench_workload构建教程》，完成需要测试的模型对应的性能测试软件包构建。 + +#### 选择软件包 + +注意性能测试软件包会包含不同系统架构，请根据运行设备的系统架构进行选择。 + +- 比如运行设备的系统架构为x86_64架构，那么请选择xxxx_x86_64_xxx_modelarts.tar.gz软件包。 +- 比如运行设备的系统架构为aarch64架构，那么请选择xxxx_aarch64_xxx_modelarts.tar.gz软件包。 + +本文以mindspore框架r1.3版本的resnet模型运行设备aarch64环境进行举例，选择train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts.tar.gz软件包。 + +#### 解压软件包 + +登录本地运行设备，将性能测试软件包拷贝到任意目录下，执行解压操作。 + +``` +tar xvf train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts.tar.gz +``` + +软件包解压后目录结构如下： + +``` +. +├── ais-bench-stubs // 性能测试命令行工具，用于执行性能测试操作 +├── code // 性能测试代码包 +│ ├── config // 配置目录 +│ │ ├── config.sh // 离线训练性能测试配置文件 +│ │ ├── mindspore_env.sh // Mindspore框架模型测试时的环境变量，可根据实际需求补充环境变量 +│ │ ├── modelarts_config.py // 线上训练性能测试时配置 +│ │ └── tensorflow_env.sh // TensorFlow框架模型测试时的环境变量，可根据实际需求补充环境变量 +│ ├── config.json // tester服务器信息配置文件，配置后可自动将测试结果上报到tester服务器上。本地离线测试模式下不需要填写 +│ ├── doc // 指导文档存放目录 +│ └── system.json // 性能测试系统信息配置文件，仅当需要将测试结果上报到tester服务器时需要配置。本地离线测试模式下不需要填写 +├── log // 日志输出目录 +├── README.md // 测试指导文档 +├── result // 测试结果输出目录 +└── tmp // 数据缓存 +``` + + + +## 线上训练性能测试操作 + +### 文件配置 + +#### 配置modelarts_config.py + +modelarts_config.py是modelarts运行配置文件，位于性能测试软件包解压路径/code/config/modelarts_config.py，主要包括modelarts的鉴权和训练参数信息。 + +比较重要和必须要设置的参数如下： + +**access_config配置：必须填写，包含modelarts认证信息，请通过计算中心或者云服务的运维同事获取并确定。** + +**session_config配置：必须填写，包含训练作业参数信息。其中session_config是modelarts V1版本的配置，session_config_v2是modelarts V2版本的配置。** + +请根据配置文件的注释来编辑配置。 + +注意： + +1. 如果access_config.iam_endpoint access_config.obs_endpoint access_config.modelarts_endpoint三个参数需要填写，必须要设置对应的域名解析地址，该地址请咨询运维同事获取。 + + 如果本地运行设备是在ECS和Notebook中，且与modelarts服务同一网络，那么可以保证连通性，不需要设置。 + + 华为云服务不需要设置。只有计算中心才需要设置。 + +2. 当前选择的容器镜像版本是默认modelarts自带的，如果需要更新为指定的mindspore和cann版本。请参考“附录>CANN包和MindSpore软件更新”。 + +3. 训练运行参数v1版本的session_config.hyperparameters和V2版本的session_config_v2.parameters，请参考对应的模型训练启动文件的运行参数。 + +4. 注意节点配置不能跨资源池。要么使用专属资源池，要么使用公共资源池，不能一起使用。 + +5. 注意如果是多个节点，只能选择8卡设置。如果是非8卡，比如2节点1卡、2节点2卡、2节点4卡，当前modelarts不支持 + +6. modelarts配置项的详细配置方法，请参照配置文件中的注释说明 + +#### 配置config.sh + +config.sh通用负载配置文件，位于性能测试软件包解压路径/code/config/config.sh，主要包括离线训练性能测试操作的基本配置信息。 + +编辑文件： + +``` +# 必选，需设置为与当前环境匹配的Python版本 +export PYTHON_COMMAND=python3.7 +# 单服务器模式，取值为True/False，配置后各训练节点测试结果单独反馈，关闭时测试结果为各设备汇总性能结果。可选，默认关闭。 +export SINGLESERVER_MODE=True +``` +**modelarts训练版本配置** +本配置文件中增加如下配置，可将当前Modelarts运行版本配置为V2: +```bash +#modelarts version default "V1", Optional value ["V1", "V2"] +export MODELARTS_VERSION=V2 +``` +该环境变量默认是V1版本,不设置。需要执行modelarts V2版本时请显示声明该变量为"V2" + + +#### 配置config.json + +config.json tester服务器信息配置文件，位于性能测试软件包解压路径/code/config.json，主要填写ais-bench测试的tester服务器具体信息，用于在完成性能测试后将测试结果上报到tester服务器上。若无须上报测试结果，可不配置。 + +#### 配置训练配置yaml文件 +对于训练模型训练参数由yaml配置文件，用户有自定义的训练参数修改需求时，用户可以修改相关的模型训练yaml文件。 +比如B版芯片（Ascend910B）上，resnet50模型需要修改batch_size参数为240，用户可以直接修改： ++ 对于mindspore框架1.3版本，修改resnet50_imagenet2012_Acc_config.yaml ++ 对于mindspore框架1.3以上版本，修改resnet50_imagenet2012_Boost_config.yaml + +### 运行测试 + +完成配置文件配置后执行性能测试操作，本地离线测试模式(aisbench测试模式概念请参考[说明](../README.md))命令如下： + +``` +./ais-bench-stubs test +``` + +### 中断和停止训练 + ++ 云环境ModleArts界面操作。 + 在云环境ModleArts服务“训练管理 > 训练作业”界面，单击正在运行的job链接并进入。在执行job界面，单击“更多操作”按钮，激活下拉菜单，在上下文菜单中单击“停止”，即可终止运行的job。 + ++ 本地运行设备停止方法，操作如下： + + 对于modelarts V1版本： + +```bash +[root@node66 ]# ls +ais-bench-stubs code log result +[root@node66 code]# python3 ./code/common/train_modelarts.py --action stop +jobname:aisbench-debug jobid:3043 preversionid:13231 jobstatus:JOBSTAT_RUNNING stop status:{'is_success': True} +``` +该操作可以停止配置文件中job_name指示的最新一个作业版本。 +​ 对于modelarts V2版本： + +创建job成功后，本地运行设备屏幕会打印job相关信息，请搜索类似“create job sucess. job_id:c8e62b62-9529-4696-ba08-2969f4861a5d”，取"job_id:"后面部分，就是Job_id。 + +```bash +[root@node66 ]# python3 ./code/common/train_modelarts.py --action stop --modelarts_version V2 --job_id e7052953-3107-47d5-a5fa-725f9eced6e3 +stop jobid:e7052953-3107-47d5-a5fa-725f9eced6e3 sesion: +INFO:root:Successfully stop the job e7052953-3107-47d5-a5fa-725f9eced6e3 +job stop status: Terminated +``` + + + +### 结果呈现和展示 + +- 以2个节点（modelarts_config.py配置文件中train_instance_count参数配置为2）的bert r1.3 modelarts训练结果为例展示训练结果： + +```bash +report >> throughput_list:[450.77798444604605, 450.38567065252664] average:450.58182754928634 +report >> accuracy_list:[0.7138142585754395, 0.7139078378677368] average:0.7138610482215881 +2022-07-13T13:24:43 -Ais-Bench-Stubs- INFO run_eval(modelarts_run.sh:32) - run_eval called +2022-07-13T13:24:43 -Ais-Bench-Stubs- INFO get_result(modelarts_run.sh:37) - get_result called +[2022-7-13 11:27:19][INFO]get ConfigInfo testid:20210126-ljp0IY, Mode:training, Model:resnet50_v1.5, Divsion:close, Scenario:generic, test_object_type:single, tester_server_ip:127.0.0.1, tester_server_port:9527 +[2022-7-13 11:27:19][INFO]ais bench stubs begin run +[2022-7-13 11:27:19][INFO]workpath:/home/lhb/test6/train_huawei_train_mindspore_bert-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts-single-0711 go testcase. +[2022-7-13 11:27:19][INFO]Benchmanager::Init() enter +[2022-7-13 11:27:19][INFO]Transmit_server start listen 0.0.0.0 : 9990 +[2022-7-13 11:27:19][INFO]get ConfigInfo testid:20210126-ljp0IY, Mode:training, Model:resnet50_v1.5, Divsion:close, Scenario:generic, test_object_type:single, tester_server_ip:127.0.0.1, tester_server_port:9527 +[2022-7-13 11:27:19][INFO]ais bench stubs begin run +[2022-7-13 11:27:19][INFO]workpath:/home/lhb/test6/train_huawei_train_mindspore_bert-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts-single-0711 go testcase. +[2022-7-13 11:27:19][INFO]Benchmanager::Init() enter +[2022-7-13 11:27:19][INFO]Transmit_server start listen 0.0.0.0 : 9990 +[2022-7-13 13:24:48][INFO]train_result_info: { + "accuracy" : "0.7138610482215881", + "average_power" : 0, + "dataload_end_time" : "2020-01-30 14:16:00", + "dataload_start_time" : "2020-01-30 14:16:00", + "efficientcy" : 0, + "energy_consumption" : 0, + "max_power" : 0, + "prepare_end_time" : "2020-01-30 14:16:00", + "prepare_start_time" : "2020-01-30 14:16:00", + "proc_end_time" : "2020-01-30 14:16:00", + "proc_start_time" : "2020-01-30 14:16:00", + "resource_util_ratio" : 0, + "throughput_ratio" : "450.58182754928634", + "total_end_time" : "2022-07-13 13:24:43", + "total_start_time" : "2022-07-13 11:27:19" +} + +[2022-7-13 13:24:48][INFO]Transmit_server resource is released! +[2022-7-13 13:24:51][INFO]BenchManager stop done +``` + +- 2个节点的resnet r1.3 modelarts训练结果为例展示训练结果： + +```bash +report >> throughput_list:[14147.314993295107, 14155.048461692913] average:14151.181727494011 +report >> accuracy_list:[0.7705078125, 0.7707316080729166] average:0.7706197102864583 +2022-07-12T15:29:13 -Ais-Bench-Stubs- INFO run_eval(modelarts_run.sh:32) - run_eval called +2022-07-12T15:29:13 -Ais-Bench-Stubs- INFO get_result(modelarts_run.sh:37) - get_result called +[2022-7-12 12:19:43][INFO]get ConfigInfo testid:20210126-ljp0IY, Mode:training, Model:resnet50_v1.5, Divsion:close, Scenario:generic, test_object_type:single, tester_server_ip:127.0.0.1, tester_server_port:9527 +[2022-7-12 12:19:43][INFO]ais bench stubs begin run +[2022-7-12 12:19:43][INFO]workpath:/home/lhb/test6/train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts-single-0712 go testcase. +[2022-7-12 12:19:43][INFO]Benchmanager::Init() enter +[2022-7-12 12:19:43][INFO]Transmit_server start listen 0.0.0.0 : 9990 +[2022-7-12 12:19:43][INFO]get ConfigInfo testid:20210126-ljp0IY, Mode:training, Model:resnet50_v1.5, Divsion:close, Scenario:generic, test_object_type:single, tester_server_ip:127.0.0.1, tester_server_port:9527 +[2022-7-12 12:19:43][INFO]ais bench stubs begin run +[2022-7-12 12:19:43][INFO]workpath:/home/lhb/test6/train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts-single-0712 go testcase. +[2022-7-12 12:19:43][INFO]Benchmanager::Init() enter +[2022-7-12 12:19:43][INFO]Transmit_server start listen 0.0.0.0 : 9990 +[2022-7-12 15:29:18][INFO]train_result_info: { + "accuracy" : "0.7706197102864583", + "average_power" : 0, + "dataload_end_time" : "2020-01-30 14:16:00", + "dataload_start_time" : "2020-01-30 14:16:00", + "efficientcy" : 0, + "energy_consumption" : 0, + "max_power" : 0, + "prepare_end_time" : "2020-01-30 14:16:00", + "prepare_start_time" : "2020-01-30 14:16:00", + "proc_end_time" : "2020-01-30 14:16:00", + "proc_start_time" : "2020-01-30 14:16:00", + "resource_util_ratio" : 0, + "throughput_ratio" : "14151.181727494011", + "total_end_time" : "2022-07-12 15:29:13", + "total_start_time" : "2022-07-12 12:19:43" +} + +[2022-7-12 15:29:18][INFO]Transmit_server resource is released! +[2022-7-12 15:29:21][INFO]BenchManager stop done +``` + +### 训练作业日志说明 + +ModleArts训练作业日志可以通过以下方式查看： + +- ModleArts界面 +- OBS输出日志路径 +- ais-bench-stubs测试运行过程中，性能测试软件包解压路径/log目录中定时更新训练作业的日志信息文件 + +## 附录 + +### CANN包和MindSpore软件更新 + +如果当前测试需要更新CANN包，执行如下操作： + +1. 在性能测试软件包解压路径创建run目录，若已存在run目录，则删除run目录下的文件。 + + 示例文件如下： + + ``` + [root@node66 run]# tree -L 1 + . + ├── Ascend-cann-nnae_5.0.2.1_linux-aarch64.run + ├── mindspore_ascend-1.3.0-cp37-cp37m-linux_aarch64.whl + └── protobuf-3.20.1-cp37-cp37m-linux_aarch64.whl + ``` + + + +2. 在训练主程序.py文件同级目录（性能测试软件包解压路径/code/code）增加ma-pre-start.sh脚本。 + + ma-pre-start.sh文件内容如下： + + ``` + #!/bin/bash + set -x + echo "Start to intall the run package" + LOCAL_DIR=$(cd "$(dirname "$0")";pwd) + echo $LOCAL_DIR + + TRAIN_PY_PATH=$(readlink -f `find ./ -name train.py`) + BASE_PATH=`dirname $TRAIN_PY_PATH` + + pip install $BASE_PATH/run/protobuf*.whl + pip install $BASE_PATH/run/mindspore_ascend*.whl + echo "replace origin mindspore packet!!! done ret:$? !!!" + + sudo chmod +x $BASE_PATH/run/*.run + CANN_RUN_PACKET=`find $BASE_PATH/run/ -name Ascend-cann-nnae*.run` + sudo $CANN_RUN_PACKET --upgrade + echo "replace origin CANN_RUN_PACKET!!!: $CANN_RUN_PACKET done ret:$? !!!" + + # env set + export GLOG_v=3 + export ASCEND_GLOBAL_LOG_LEVEL=3 + export ASCEND_GLOBAL_EVENT_ENABLE=0 + export ASCEND_SLOG_PRINT_TO_STDOUT=0 + + set +x + ``` + + 其中 CANN_RUN_PACKET参数-name请根据实际需要安装的CANN包名称设置为：Ascend-cann-nnae、Ascend-cann-nnrt、Ascend-cann-toolkit。 + +### 日志级别设置 + +通过修改ma-pre-start.sh文件中“GLOG_v”或“ASCEND_GLOBAL_LOG_LEVEL”的变量值，可以更新日志的级别。 + ++ GLOG日志级别取值为：0（INFO）、1（WARNING）、2（ERROR）、3（FATAL） ++ ASCEND_GLOBAL_LOG_LEVEL日志级别取值为：0（DEBUG）、1（INFO）、2（WARNING）、3（ERROR）、4（NULL） + +### 域名解析地址增加 + +请咨询ModelArts所在云环境的运维，获取该云相关服务（obs、modelarts、swr）域名和IP的映射关系并写入/etc/hosts, + +比如武汉云相关服务obs、modelarts、swr域名映射关系如下： + +```bash +58.48.42.196 obs.cn-central-221.ovaijisuan.com +58.48.42.193 modelarts.cn-central-221.ovaijisuan.com +58.48.42.198 swr.cn-central-221.ovaijisuan.com +``` + +注意： + +- 如果在notebook中运行，无须设置该项。 +- 华为云无须设置。 + diff --git "a/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" "b/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" new file mode 100644 index 0000000..c1dba07 --- /dev/null +++ "b/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" @@ -0,0 +1,225 @@ +# ais-bench_workload_train_offline线下训练说明文档 + + + +[TOC] + +## 简介 + +ais-bench标准化性能测试软件，又称AI Server Benchmark软件，是根据AI标准（IEEE 2937及 T/CESA 1169-2021）对AI服务器进行性能测试的工具软件。 + +本文主要介绍基于ais-bench软件，在离线环境对模型进行训练性能测试。离线环境是指非modelarts云上训练场景，当前主要适配单卡、单机、线下集群、容器集群场景。 + +## 使用前准备 + +### 环境 + +1. Atals训练设备（搭载Ascend NPU以及Ascend 910芯片等昇腾硬件环境），可以搭建单卡、单机、线下集群、容器集群场景，相关硬件产品文档请参见[昇腾硬件产品文档](https://www.hiascend.com/document?data=hardware)。 +2. 根据需要测试的模型类型安装MindSpore或TensorFlow框架；参见《[CANN 软件安装指南](https://www.hiascend.com/document/detail/zh/canncommercial/51RC1/envdeployment/instg/instg_000002.html)》安装CANN软件包。MindSpore或TensorFlow框架需要根据《ais-bench_workload构建教程》所选择的模型版本来安装对应版本的框架。 +3. 集群测试时需要安装依赖软件--sshpass，版本无要求。 +4. 容器环境测试时，容器制作请参照《制作可ssh登录镜像ascend-mindspore-arm的方法》 + +### 数据集 + +下载相关模型数据集到运行设备任意目录下。例如resnet模型需要imagenet数据集，bert模型需要enwiki数据集。具体下载方式请至相关模型官网，本文不作详述。 + +### 软件包 + +请参见 《ais-bench_workload构建教程》，完成需要测试的模型对应的性能测试软件包构建。 + +#### 选择软件包 + +注意性能测试软件包会包含不同系统架构，请根据运行设备的系统架构进行选择。 + +- 比如运行设备的系统架构为x86_64架构，那么请选择xxxx_x86_64.tar.gz软件包。 +- 比如运行设备的系统架构为aarch64架构，那么请选择xxxx_aarch64_xxx.tar.gz软件包。 + +本文以mindspore框架r1.3版本的resnet模型运行设备aarch64环境进行举例，选择train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.3.tar.gz软件包。 + +#### 解压软件包 + +登录运行设备，将性能测试软件包拷贝到任意目录下，执行解压操作。 + +``` +tar xvf train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.3_modelarts.tar.gz +``` + +软件包解压后目录结构如下： + +``` +. +├── ais-bench-stubs // 性能测试命令行工具，用于执行性能测试操作 +├── code // 性能测试代码包 +│ ├── config // 配置目录 +│ │ ├── config.sh // 离线训练性能测试配置文件 +│ │ ├── mindspore_env.sh // Mindspore框架模型测试时的环境变量，可根据实际需求补充环境变量 +│ │ ├── modelarts_config.py // 线上训练性能测试时配置 +│ │ └── tensorflow_env.sh // TensorFlow框架模型测试时的环境变量，可根据实际需求补充环境变量 +│   ├── config.json // tester服务器信息配置文件，配置后可自动将测试结果上报到tester服务器上。本地离线测试模式下不需要填写 +│   ├── doc // 指导文档存放目录 +│   └── system.json // 性能测试系统信息配置文件，仅当需要将测试结果上报到tester服务器时需要配置。本地离线测试模式下不需要填写 +├── log // 日志输出目录 +├── README.md // 离线性能测试指导 +└── result // 测试结果输出目录 +``` + + + +## 离线训练性能测试操作 + +### 文件配置 + +#### 配置config.sh + +config.sh通用负载配置文件，位于性能测试软件包解压路径/code/config/config.sh，主要包括离线训练性能测试操作的基本配置信息。 + +请在配置文件中根据注释说明填写. + + +**注意** + +1. 非单卡环境下，必须要生成rank_table文件并配置RANK_TABLE_FILE变量。rank_table文件生成请参见“rank_table文件生成与实例”。 +2. 集群环境下，必须要生成节点ssh信息文件并配置NODEINFO_FILE变量。节点ssh信息文件生成请参见“节点ssh信息文件”。 +3. 集群环境下，直接使用ssh信息文件进行节点间的登录交互可能存在安全风险，可以设置集群节点的秘钥认证，提高安全性。请参见“集群节点免密设置”。 + +#### 配置config.json + +config.json tester服务器信息配置文件，位于性能测试软件包解压路径/code/config.json，主要填写ais-bench测试的tester服务器具体信息，用于在完成性能测试后将测试结果上报到tester服务器上。若无须上报测试结果，可不配置。 + +#### 配置system.json + +system.json 性能测试系统信息配置文件，位于性能测试软件包解压路径/code/system.json，主要填写ais-bench测试的运行环境系统信息，用于在完成性能测试后将运行环境系统信息作为测试结果的内容上报到tester服务器上。若无须上报测试结果，可不配置。 + +#### 配置训练配置yaml文件 +对于训练模型训练参数由yaml配置文件，用户有自定义的训练参数修改需求时，用户可以修改相关的模型训练yaml文件。 +比如B版芯片（Ascend910B）上，resnet50模型需要修改batch_size参数为240，用户可以直接修改： ++ 对于mindspore框架1.3版本，修改resnet50_imagenet2012_Acc_config.yaml ++ 对于mindspore框架1.3以上版本，修改resnet50_imagenet2012_Boost_config.yaml + +### 运行测试 + +完成配置文件配置后执行性能测试操作，本地测试命令如下： + +``` +./ais-bench-stubs test +``` + +连接tester服务器测试时，无需test参数。 + +## 附录 + +### **日志级别设置** + +性能测试启动后，默认在性能测试软件包解压路径/log目录下输出日志。 + +如果需要设置日志级别，请在性能测试软件包解压路径/config目录下的mindspore_env.sh或tensorflow_env.sh文件中添加如下环境变量。 + +``` +export GLOG_v=3 +``` + +GLOG日志级别取值为：0（INFO）、1（WARNING）、2（ERROR）、3（FATAL）。 + +### rank_table文件生成与实例 + +单机或集群rank_table文件生成方法，请单击[芯片资源信息配置文件参考](https://support.huawei.com/enterprise/zh/doc/EDOC1100192402/a1885ca4)访问相关文档。 + +生成双机16卡的rank_table文件示例：rank_table_16p_64_66.json + +```bash +{ + "version": "1.0", + "server_count": "2", + "server_list": [ + { + "server_id": "xx.xx.xx.xx", + "device": [ + {"device_id": "0", "device_ip": "xx.xx.xx.xx", "rank_id": "0"}, + {"device_id": "1", "device_ip": "xx.xx.xx.xx", "rank_id": "1"}, + {"device_id": "2", "device_ip": "xx.xx.xx.xx", "rank_id": "2"}, + {"device_id": "3", "device_ip": "xx.xx.xx.xx", "rank_id": "3"}, + {"device_id": "4", "device_ip": "xx.xx.xx.xx", "rank_id": "4"}, + {"device_id": "5", "device_ip": "xx.xx.xx.xx", "rank_id": "5"}, + {"device_id": "6", "device_ip": "xx.xx.xx.xx", "rank_id": "6"}, + {"device_id": "7", "device_ip": "xx.xx.xx.xx", "rank_id": "7"} + ], + "host_nic_ip": "reserve" + }, + { + "server_id": "xx.xx.xx.xx", + "device": [ + {"device_id": "0", "device_ip": "xx.xx.xx.xx", "rank_id": "8"}, + {"device_id": "1", "device_ip": "xx.xx.xx.xx", "rank_id": "9"}, + {"device_id": "2", "device_ip": "xx.xx.xx.xx", "rank_id": "10"}, + {"device_id": "3", "device_ip": "xx.xx.xx.xx", "rank_id": "11"}, + {"device_id": "4", "device_ip": "xx.xx.xx.xx", "rank_id": "12"}, + {"device_id": "5", "device_ip": "xx.xx.xx.xx", "rank_id": "13"}, + {"device_id": "6", "device_ip": "xx.xx.xx.xx", "rank_id": "14"}, + {"device_id": "7", "device_ip": "xx.xx.xx.xx", "rank_id": "15"} + ], + "host_nic_ip": "reserve" + } + ], + "status": "completed" +} +``` + +### 节点ssh信息文件 + +集群场景下执行性能测试时，需要设置节点ssh信息文件，用于在测试过程中节点之间的登录验证。 + +节点ssh信息文件由用户自行创建文件名格式类似于ssh64_66.json，按照如下示例格式进行配置。 + +示例：ssh64_66.json + +```bash +{ + "cluster": { + "xx.xx.xx.xx": { # 节点IP，必须与rank_table文件中的server_id一一对应 + "user": "xxxx", # 节点登录用户名，完成集群节点免密设置可不配置 + "pd": "xxxx", # 节点登录密码，完成集群节点免密设置可不配置 + "port": xx # 容器端口，默认22。可不设置。删除本参数或配置为22时，表示性能测试在默认22端口通信；设置具体端口号时，表示在容器或者设备中运行并提供指定端口访问能力 + + }, + "xx.xx.xx.xx": { + "user": "xxxx", + "pd": "xxxx", + "port": xx + } + } +} +``` + +**注意：该文件中的节点数目应与rank_table文件中的节点数目一致。** + +### 集群节点免密设置 + +集群节点免密设置的参考操作如下： + +1. 登录集群管理节点并生成SSH Key。 + + ``` + ssh-keygen -t rsa -b 2048 + ``` + + 安全起见，建议用户到“Enter passphrase”步骤时输入密钥密码，且符合密码复杂度要求。建议执行该命令前先将umask设置为0077，测试完成后再恢复原来umask值。 + +2. 将管理节点的公钥拷贝到所有节点的机器上。 + + ``` + ssh-copy-id -i ~/.ssh/id_rsa.pub @ + ``` + + @替换成要拷贝到的对应节点的用户名和IP。 + +3. 设置ssh代理管理ssh密钥。 + + ``` + ssh-agent bash # 开启ssh-agent的bash进程 + ssh-add # 向ssh-agent添加私钥 + ``` + + 避免工具批量安装操作过程中输入密钥密码和节点密码。 + + + diff --git "a/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" "b/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" new file mode 100644 index 0000000..01e4a70 --- /dev/null +++ "b/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" @@ -0,0 +1,160 @@ +# ais-bench_workload推理执行容器环境搭建指导 + +## 1. 简介 + +本文基于华为昇腾镜像仓库推理基准镜像algorithm增加相关命令，用于构建容器环境，用于Ais-Bench推理负载包运行。 + +## 2. 下载基础镜像 + +本文基于昇腾镜像仓库algorithm基础镜像制作，链接为 (https://ascendhub.huawei.com/#/detail/algorithm) 请进入点击 "获取镜像"按钮，下载基础镜像。 + +本文示例基于华为昇腾基础镜像algorithm 22.0.RC1, CANN 5.1.RC1。镜像名称，22.0.RC1-ubuntu18.04。该镜像已经安装nnrt和python3.7.5。该类镜像有多个发行平台都可以作为基础镜像。 + +拉取基础镜像的方法请参照**附录1**进行。 + +下载完毕后，请按该官网"如何使用镜像"来配置物理机环境。其步骤3，可以参考附录2。 + +## 3.基于基准镜像构建新镜像 + +### 3.1 准备依赖程序包 + +工作目录algorithm文件如下： + +目录文件树 + +``` +root@root:/home/lhb/tool/algorithm# tree +. +├── aclruntime-0.0.1-cp37-cp37m-linux_aarch64.whl +├── Dockerfile +└── loadgen-0.0.1-cp37-cp37m-linux_aarch64.whl +``` + +说明： + +- aclruntime-0.0.1-cp37-cp37m-linux_aarch64.whl， 来自于ais-bench_workload发行包aclruntime-aarch64.tar.gz +- loadgen-0.0.1-cp37-cp37m-linux_aarch64.whl， 来自于ais-bench_workload发行包Ais-Bench-LoadGen-aarch64-1.1.tar.gz + +### 3.2 创建Dockerfile + +```bash +FROM ascendhub.huawei.com/public-ascendhub/algorithm:22.0.RC1-ubuntu18.04 + +MAINTAINER liangchaoming + +USER root +RUN cp /etc/apt/sources.list /etc/apt/sources.list.bak +# apt源加速 +RUN sed -i "s@/archive.ubuntu.com/@/mirrors.163.com/@g" /etc/apt/sources.list && rm -rf /var/lib/apt/lists/* && apt-get update --fix-missing -o Acquire::http::No-Cache=True +# 恢复基础容器中python3.7.5在容器中的指向.容器支持python3、python3.7.5的python调用 +RUN mkdir -p /opt/package +RUN ln -s /usr/local/python37/bin/python3 /usr/bin/python3.7.5 && ln -s /usr/local/python37/bin/pip3 /usr/bin/pip3.7.5 + +# 安装系统依赖 +RUN apt-get install libglib2.0-dev libgl1-mesa-glx -y && apt install vim -y +# 安装pip3依赖。 +RUN pip3 install numpy tqdm pycocotools scikit-learn transformers tokenization opencv_python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com +WORKDIR /opt/package +# 安装推理运行环境aclruntime和loadgen +COPY aclruntime-0.0.1-cp37-cp37m-linux_aarch64.whl /opt/package/aclruntime-0.0.1-cp37-cp37m-linux_aarch64.whl +COPY loadgen-0.0.1-cp37-cp37m-linux_aarch64.whl /opt/package/loadgen-0.0.1-cp37-cp37m-linux_aarch64.whl +RUN pip3 install aclruntime-0.0.1-cp37-cp37m-linux_aarch64.whl +RUN pip3 install loadgen-0.0.1-cp37-cp37m-linux_aarch64.whl +# online install tensorflow1.15 +RUN pip3 install https://files.pythonhosted.org/packages/35/9a/985a1642bc493b96c340da6db366124b2609c7a42ca53f643585c01e4d33/tensorflow_ascend-1.15.0-cp37-cp37m-manylinux2014_aarch64.whl +CMD ["source", "/usr/local/Ascend/nnrt/set_env.sh"] +``` + +说明： + +如果需要网络代理，请在Dockerfile增加网络代理环境变量。在“USER root"下一行，增加类似如下环境变量： + +```bash +# 设置网络代理环境变量 +ENV http_proxy "http://xxxx:xxxx" +ENV https_proxy "http://xxxx:xxxx" +ENV ftp_proxy "http://xxxx:xxxx" +``` + + + +### 3.3 构建镜像 + +编译指令： + +``` +docker build -t ais-bench_workload-inference-arm:1.0 . +``` + +执行成功后，回显如下： + +``` +root@root:/home/lhb/tool/algorithm# docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +ais-bench_workload-inference-arm 1.0 d16debfbde9c About an hour ago 2.83GB +``` + +## 3. 运行镜像 + +``` +root@root:/home/lhb/tool/algorithm# docker run -itd -u root -v /home/:/home --name asend_inference_aarch64 -e ASCEND_VISIBLE_DEVICES=0 52dbef81d817 /bin/bash +a1ed8ed48256f7ece143c986fd337aefb29f50a307f342cee1e88881618eb29a +root@root:/home/lhb/tool/algorithm# docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +a1ed8ed48256 52dbef81d817 "/bin/bash" 4 seconds ago Up 3 seconds asend_inference_aarch64 +root@root:~# docker exec -it a1ed8ed48256 bash +root@a1ed8ed48256:/opt/package# +``` + +说明： + +- -d参数，后台执行。退出容器后容器还存在着 + +- 以root用户登录将物理机的/home目录映射到容器的/home目录 + +- 52dbef81d817是推理镜像ID + +- A500小站驱动等部署较为特殊，其容器启动命令有其特殊性 + + ```bash + docker run -itd -u root --device=/dev/davinci0 --device=/dev/davinci_manager --device=/dev/devmm_svm --device=/dev/hisi_hdc -v /usr/local/bin/npu-smi/:/usr/local/bin/npu-smi/ -v /home/data/miniD/driver/lib64/:/home/data/miniD/driver/lib64/ -v /home/:/home/ -e ASCEND_VISIBLE_DEVICES=0 52dbef81d817 /bin/bash + ``` + + 容器拉起后，进入容器中，执行以下语句声明下系统库最新路径 + + ``` + expert LD_LIBRARY_PATH=/home/data/miniD/driver/lib64/:$LD_LIBRARY_PATH + ``` + + A500环境资源有限，请及时清理docker资源保证推理测试顺利进行 + +## 4. 附录 + +### 4.1 修改linux用户HwHiAiUser的id值为1000 + +背景：物理机当前HwHiAiUser的id是1001，zjut-msadvisor用户的id是1000 + +步骤： + +```bash +root@root:~# usermod -u 1003 zjut-msadvisor +usermod: user zjut-msadvisor is currently used by process 91928 +root@root:~# ps -ef |grep 91928 +zjut-ms+ 91928 1 0 12:33 ? 00:00:00 /lib/systemd/systemd --user +zjut-ms+ 91929 91928 0 12:33 ? 00:00:00 (sd-pam) +root 94128 92493 0 12:49 pts/0 00:00:00 grep --color=auto 91928 +root@root:~# kill -9 91928 91929 +root@root:~# usermod -u 1003 zjut-msadvisor +usermod: user zjut-msadvisor is currently used by process 92042 +root@root:~# kill -9 92042 +root@root:~# usermod -u 1003 zjut-msadvisor +root@root:~# groupmod -g 1003 zjut-msadvisor +root@root:~# id zjut-msadvisor +uid=1003(zjut-msadvisor) gid=1003(zjut-msadvisor) groups=1003(zjut-msadvisor) +root@root:~# usermod -u 1000 HwHiAiUser +root@root:~# groupmod -g 1000 HwHiAiUser +root@root:~# id HwHiAiUser +uid=1000(HwHiAiUser) gid=1000(HwHiAiUser) groups=1000(HwHiAiUser) +``` + +说明：新id--1003不得与现有用户id重复 diff --git "a/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" "b/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" new file mode 100644 index 0000000..4c3a4e1 --- /dev/null +++ "b/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" @@ -0,0 +1,235 @@ +# ais-bench-workload构建教程 + +## 概述 +ais-bench标准化性能测试软件，又称AI Server Benchmark软件，是根据AI标准（IEEE 2937及T/CESA 1169-2021）对AI服务器进行性能测试的工具软件。 + +ais-bench_workload是ais-bench提供用于构建ais-bench性能测试软件包并进行测试的负载工具。 + +本文档主要介绍如何**搭建ais-bench_workload构建环境**并在该环境下**构建推理和训练场景的ais-bench性能测试软件包。** +ais-bench_workload支持快速构建和标准构建。其中快速构建仅支持mindspore框架的bert和resnet两个典型模型的性能测试软件包构建。通过构建脚本扩展，标准构建能支持当前提供的所有模型的性能测试软件包构建。 + +## 1. 搭建ais-bench_workload构建环境 +### 1.1 环境要求 + +ais-bench_workload构建支持在Windows和Linux系统下进行，要求如下： + +- **Windows系统**：Windows7及以上版本；安装git和winrar，版本不限。 + +- **Linux系统**：系统版本无限制；安装git和unrar，版本不限。 + + 其中git、winrar和unrar的下载与安装，请用户自行完成，本文不详细描述。 + +### 1.2 源码下载 +ais-bench_workload的工作目录保存在ais-bench源码包的tools目录下，可以通过以下两种方式下载ais-bench源码包： + ++ git clone下载[AISBench/training](https://gitee.com/aisbench/training)仓库代码 +``` + git clone https://gitee.com/aisbench/training.git +``` +​ 该方式直接下载码云training仓库master分支源码。 + ++ 在线下载源码压缩包 + 访问AISBench/training仓库网页：https://gitee.com/aisbench/training ， 点击“克隆/下载”按钮，在弹出的窗口中点击“下载ZIP”按钮进行下载。 + + 该方式默认下载的是master分支training源码包为压缩包training-master.zip，解压后training目录默认名字是training-master 。 + +### 1.3 工作目录 + +​ 获取源码包后，需要在ais-bench_workload工作目录下执行构建操作。ais-bench_workload工作目录为training目录下的子目录，目录结构如下： + +仅展示构建需要部分 + +```bash +ais-bench_workload +├── build +│   ├── build.sh # 标准构建脚本 +│   └── download_and_build.sh # 快速构建脚本 +├── doc +│   ├── ais_bench推理程序更新说明.md +│   ├── ais-bench_workload_inference推理负载说明文档.md +│   ├── ais-bench_workload_train_modelarts训练说明文档.md +│   ├── ais-bench_workload_train_offline线下训练说明文档.md +│   ├── ais-bench_workload构建教程.md +│   ├── ais-bench_workload推理执行容器环境搭建指导.md +│   ├── modelarts_notebook使用入门指导.docx +│   └── 制作可ssh登录镜像ascend-mindspore-arm的方法.md +├── README.md +├── src # 构建测试软件包的模型保存目录 +│   └── train # 训练场景 +│   ├── huawei # 华为模型 +│   │   ├── train_mindspore_bert +│   │   ├── train_mindspore_deeplabv3 +│   │   ├── train_mindspore_deepspeech2 +│   │   ├── train_mindspore_faster_rcnn +│   │   ├── train_mindspore_glm2 +│   │   ├── train_mindspore_gnmt_v2 +│   │   ├── train_mindspore_llama +│   │   ├── train_mindspore_pangu_alpha +│   │   ├── train_mindspore_resnet +│   │   ├── train_mindspore_ssd +│   │   ├── train_mindspore_widedeep +│   │   ├── train_tensorflow_bert_base +│   │   ├── train_tensorflow_densenet121 +│   │   ├── train_tensorflow_mobilenetv2 +│   │   ├── train_tensorflow_nezha_large +│   │   ├── train_tensorflow_resnet101 +│   │   ├── train_tensorflow_resnet50 +│   │   ├── train_tensorflow_resnext50 +│   │   ├── train_tensorflow_ssd_resnet34 +│   │   ├── train_tensorflow_vgg16 +│   │   └── train_tensorflow_yolov3 +│   └── nvidia # 英伟达模型 +│   ├── train_tensorflow_bert +│   └── train_tensorflow_resnet +``` + + + +## 2. 构建ais-bench性能测试软件包 +### 2.1 快速构建（仅支持AISBench 1.0版本） + +#### 2.1.1 简介 + +快速构建可以一键构建mindspore框架的bert&resnet典型模型分别在aarch64和x86_64平台训练场景的ais-bench性能测试软件包。对于其它模型的训练软件包的构建，需要通过标准构建获取。 + +快速构建也可以通过扩展快速构建脚本downlaod_and_build.sh，将其它模型加入快速构建中，实现对其它模型测试软件包的快速构建。这要求熟悉Python和Shell语言并对快速构建脚本有一定的了解。 + +#### 2.1.2 约束 + +支持构建环境： + +- **Windows系统**：git bash--Mircrosoft Windows git命令的模拟终端。 + +- **Linux系统**： + + 要求操作系统处于稳定的联网状态，主要保证能够顺利下载ais-bench stubs基础测试工具包，可以先执行如下命令测试网络是否畅通： + + ```bash + curl http://www.aipubservice.com + ``` + +不建议多用户同时执行快速构建操作，可能出现依赖下载失败。 + +#### 2.1.3 构建指令 + +指令格式：bash ./download_and_build.sh {version} {type} +参数说明： + +| 参数 | 说明 | +| --------- | ------------------------------------------------------------ | +| {version} | 框架版本号，必选。取值需通过ais-bench_workload\src目录的具体模型目录下的版本文件确认支持的版本号。快速构建仅支持bert和resnet模型。故可配置的版本号为：{type}参数为“modelarts”时，可配置为r1.3、r1.5、r1.7、r1.8、r1.9、r1.10、r2.0、r2.1、r2.2；未配置{type}参数时可配置为r1.5、r1.6、r1.7、r1.8、r1.9、r1.10、r2.0、r2.1、r2.2。 | +| {type} | 线上或离线环境，可选。取值为“modelarts”，表示构建线上环境的性能测试软件包；不配置本参数时，表示构建离线环境的性能测试软件包。 | + +#### 2.1.4 构建操作 + +示例，构建基于mindspsore 1.7版本线上执行bert和resnet模型的性能测试软件包，指令如下： + +``` +bash ./download_and_build.sh r1.7 modelarts +``` + +Windows系统需预先安装git软件。在ais-bench_workload工作目录下鼠标右键上下文菜单中点击"git bash here"，打开Microsoft Windows git命令行模拟终端，执行构建指令。鼠标右键没有git相关菜单命令时，请在windows右下角的搜索窗口输入"git" ，找到git bash，并点击进入，执行构建指令。 + +#### 2.1.5 构建结果 +构建指令成功执行后，在ais-bench_workload目录下生成output子目录，构建的性能测试软件包保存在该子目录中。 +构建结果示例： + +``` +. +├── train_huawei_train_mindspore_bert-Ais-Benchmark-Stubs-aarch64-1.0-r1.7_modelarts.tar.gz +├── train_huawei_train_mindspore_bert-Ais-Benchmark-Stubs-x86_64-1.0-r1.7_modelarts.tar.gz +├── train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-aarch64-1.0-r1.7_modelarts.tar.gz +└── train_huawei_train_mindspore_resnet-Ais-Benchmark-Stubs-x86_64-1.0-r1.7_modelarts.tar.gz +``` + +分别生成aarch64和x86_64平台下bert和resnet模型共4个软件包。 + +### 2.2 标准构建 + +#### 2.2.1 简介 + +标准构建是通过构建指令指定具体模型执行构建性能测试软件包的操作，相比快速构建，指令更丰富，适用模型更多且涵盖推理和训练场景。 + +#### 2.2 约束 + +- 仅支持在Linux系统下执行构建操作。 + +- 要求操作系统处于稳定的联网状态，主要是保证能够顺利下载ais-bench stubs基础测试工具包。可以先执行如下命令测试网络是否畅通： + + ```bash + curl http://www.aipubservice.com + ``` + +- 一次执行只能构建一个模型的性能测试软件包。 + +#### 2.2.3 构建准备 + +进行标准构建前，需要先下载ais-bench stubs基础测试工具包并将该工具解压到ais-bench_workload工作目录的build子目录下。 + +ais-bench stubs基础测试工具包用于选择构建测试软件包适用的aarch64和x86_64平台。 + +访问[面向人工智能基础技术及应用的检验检测基础服务平台](http://www.aipubservice.com/#/show/compliance/detail/127)， 通过“成果展示”->“标准符合性测试”->“人工智能服务器系统性能测试”， 进入“人工智能服务器系统性能测试”页面，在“测试工具”章节下载Stubs压缩包到本地，将Stubs压缩包解压到ais-bench_workload/build目录下。 + +执行操作后，ais-bench_workload/build目录结构如下: + +``` +ais-bench_workload +├── build +    ├── build.sh +    ├── download_and_build.sh + ├── Ais-Benchmark-Stubs-aarch64-.tar.gz + └── Ais-Benchmark-Stubs-x86_64-.tar.gz +``` + +为软件包版本号。 + +#### 2.2.4 构建指令 + +指令格式：./build.sh {$stubs_file} {mode} {secondary-folder-name} {third-folder-name} {version} {type} +输出路径：在ais-bench_workload\output目录会生成相应程序包。 + +| 参数 | 说明 | +| ----------------------- | ------------------------------------------------------------ | +| {stubs_file} | 选择stubs基础工具包，即选择构建测试软件包使用的aarch64和x86_64平台，必选。取值为Ais-Benchmark-Stubs-aarch64-.tar.gz、Ais-Benchmark-Stubs-x86_64-.tar.gz | +| {mode} | 选择构建测试软件包的适用场景，必选。取值为：train(训练场景)。对应ais-bench_workload/src目下以及子目录名称。 | +| {secondary-folder-name} | 二级子目录名称，对应ais-bench_workload/src目录下二级子目录名称，必选。{mode}配置为train时，表示选择模型品牌，取值为：huawei、nvidia | +| {third-folder-name} | 三级子目录名称，对应ais-bench_workload/src目录下三级子目录名称，必选。
{secondary-folder-name}配置为language时，取值为bert;
{secondary-folder-name}配置为vision时，取值为classification_and_detection
{secondary-folder-name}配置为huawei时，取值为：train_mindspore_bert、train_mindspore_deeplabv3、train_mindspore_deepspeech2、train_mindspore_faster_rcnn、train_mindspore_glm2、train_mindspore_gnmt_v2、train_mindspore_llama、train-mindspore_pangu_alpha、train_mindspore_resnet、train_mindspore_ssd、train_mindspore_widedeep、train_tensorflow_bert_base、train_tensorflow_densenet121、train_tensorflow_mobileneetv2、train_tensorflow_nezha_large、train_tensorflow_resnet50、train_tensorflow_resnet101、train_tensorflow_resnext50、train_tensorflow_ssd_resnet34、train_tensorflow_vgg16、train_tensorflow_yolov3;
{secondary-folder-name}配置为nvidia时，取值为train_tensorflow_bert、train_tensorflow_resnet | +| {version} | 模型框架版本号，仅{mode}配置为train时支持，可选。取值需通过ais-bench_worload工作目录具体模型目录下的版本文件确认支持的模型框架版本号 | +| {type} | 线上或离线环境，仅{mode}配置为train时支持，可选。取值为“modelarts"，表示构建线上环境的性能测试软件包；不配置本参数时，表示构建离线环境的性能测试软件包 | + + + +#### 2.2.5 构建操作 + +**训练场景示例如下：** + ++ 构建aarch64架构训练场景huawei mindspore框架r1.7版本resnet模型 离线运行的性能测试软件包 + ./build.sh ./Ais-Benchmark-Stubs-aarch64-1.0.tar.gz train huawei train_mindspore_resnet r1.7 ++ 构建aarch64架构训练场景huawei mindspore框架r1.7版本resnet模型modelarts线上运行的性能测试软件包 + ./build.sh ./Ais-Benchmark-Stubs-aarch64-1.0.tar.gz train huawei train_mindspore_resnet r1.7 modelarts + + +## FAQ + +### 1. Linux环境下执行构建报错：$'\r': command not found + +**故障现象** + +在Linux环境执行./build.sh构建操作时会出现如下报错： + +![faq1](../img/faq1.png) + +**故障原因** + +build.sh脚本是从Windows环境开发的，Windows的换行符格式与Linux不一致，导致在Linux环境执行报错。 + +**故障处理** + +在Ascend/tools/ais-bench-workload目录下执行如下命令将脚本进行格式化后重新执行构建操作： + +``` +find src/train/huawei/train_mindspore_glm2 -type f -name "*.sh" -exec dos2unix {} + +``` + + + diff --git "a/ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" "b/ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" new file mode 100644 index 0000000000000000000000000000000000000000..17a932eab792359f780d7b1d3cb942a3869b718f GIT binary patch literal 488919 zcma(21CS;`voH#eZQHhO+qP}nwr$(yj&0jBJKou`@9ca2?~8Lz{5RrubU$5Bc6L@) zW+5v(Q$ZRS1Pb8a5iWNq@bCVA4#=N^iM_Fclf8p8y}}O~$`1hIKWJ7P2TB%z003S< z000R89nH|efzHF$CO2tD?veo^^jYE=x9AG_yCpQBOsp4B4T6fbSC?oilWkSjW{NGM zj~7mq)MjIbvlz!W54pn^|LqFacYc+|S#6O=xY(w8#xT=9JG^MJ8KKnD)E<`Zo;d77 z?V~fGwH?lP8KVisaFFTN$b~%wWaWTBICGWE_GuNKiR>JH_~u#bAXl{>I7@<~G#zA* zI16oSFc1PYgw#SWdr<4NhQ|uI2v>R2AeW0}lXJ}@UNQT8-TyL3UvF*1r_CZ7Rr^m6-Ni_bTFPH9P?Ic>${pA$_4*^n zo-viO%RFv0{Qs#HvBU*UtskwJ{b&X8ziY+V-pTZzT4IwL<%Sp#!rV!A3IFD;OO~`X zCaC}|M@mV61g_J(Z;Fw~nP4yWff<<_8XFyjRt#(JmsnB+1ulGjzu)lZADws6`#ESKu8{5+Zu3Gj-bD0cmo}MXdV}5jm-w3)L9_U?tHL%GVwH~Ako4Gy8etk7% zugac!s^utPLF=&=?*uXv;6M{_zkZr@_3%oYQNT(daF|#pRTLw_ilZcKXWZ?6dqe9m zR;8lNaz%+LN@!_2Xw#J(Xvy5VrgcIf&wjYSvweT+@%4Z6@Ql5)_m$9mV*QRdf5(KZ z;4d0DCOE}QaI^LMig|$kpNdG}~(4qBL>ppDIM3B|d^r zdXL1jl2jzCc(5QF0U{TDk2!2h*5ho-6$rm@)fEt+*&1+m7BZQc&)-Dj_9PW&AGd`^2qA5%k%#r z|1QPug6UgTR9!=iMA#xhEec7afK0{~F%r_4s3N0evqz9QGXo2>D%JnIx!X1F&g^!= z8?_^k5y#)L%E_x`<|IZGP67K011xovQ2M|vdpZ0gpZ&+kS^L^ArhJ4bF1c*IA%iBW zDn=SpTx>HVDW^iH|BOY-5U~eO9UHPcC6D>E9ZguAnhkJ31Ydc&``h1E zm>V27q?6xe2Chc}(I-~sm|P?>b7q3%F?^%QlWhIDAg3=)7blucaAZd$ED;<%XUQl9 z)e;d6rjdhAC@+)Y5jDK)eq}Rp&U|SW@gkBflq*so=W`0~2sy{E0ZavbhJvzFH}_W; z960*q{pgqCdh?d0WE-i=qoY`Ih;^D<N)uSPN)9eHjYMm2Pc|Ytz8YMl$ zGbRH#Oa?fLjIN}p8vK?pPHQ-a9lYZr-k}SzNOc>+aNmnc@d8``J9I1X{*2W%fJd_l zH^45<@egRK`v+8X3*h0b{2L&nFwkW2$N~>4tWR5T)%aQRh{_187I2w3Dmzf78NF?p?gq zyRNvH%{nt;_azpf)}CA5pG-tHN8&k#jQM7F-PC4`c##ZY2yoHa0|$}|6EQ(B_{|4x zS}Zv%LV_TKxZ|v%1xGB6F+3P~Z!ul?nECz-oX!Z}H@eXQ3JTAfMmsCpT$z;zod3N4iR3~hf`Vh~Bo@?0jcOOxm z8rT>@A=I4eCfTD-sNH_&q#ip?TNC>q9aHE2Ukjj{=@eG@`qJ~N^fs_Zd$ zx(6scB6NT6wMsVM$YgW$I-il<2p`Ut#l7jw`$p|aeQ$J{W16SV+W?EYG(uCP6U=U* zc+QXqk$SL^Q!LIQHfhcRremjYu3v%IzUkr{u#|OOtvwoFz7$VI3^B( zAD5wTzC6p6AYi&%_r32;GNd>gv@iR;c+w=Y`ryZ9;)y;}W`|X>gT7vxDR$)JV7^nl z7s?Tm zwdDbGcMt+LIFKOBAjc#En|R>)(33_tLvO{?1oz;UfFLS4lxaYe(Ww}7OaXDpAO4g- z+=4<}01MLZmMrjXe4*&0x;ZQ7Jxc-m1`-NSJI1_f%pl8f5A-PT^dskjuzu-d5!&Y@#~G+hI2k!#D?{7^OCL5$ z!GPHMvEpBv%CiMu?Q|iHhN)Ob<;n8e?$UUi8%de`H@VE6Gu@Q)Q0S(aB_TpD;m8)b2E`(y&?4UE$NK&0WX zsRdG!4wsjGznJ>qNGUAATes*-@gFKL#Z!w|lUPlH-$k~$ZMP^qAf?4{Kb*R5FS|_v z$WwcB*jWvbibm70upT1yhf1=bekb`3wv)rBD%8+wW>J~0 zXl|ut$Bwj}Y;Ia_A=%n?+5XVoYZKAla?mcS{p=uJa2wrDE15u*@@9tHNvoc~ma2o3 z`Q0&rCsjn}GrrkVnHx}MrNwSfC9Movo6VkDTIqH%ojobaXit^i>Zu%AGm|Z4V!NwC zWPM+Tq?yg0I6m7`Ng`XP@U|W0soa^>Q+QmJ#hzO6k6@K2a%3C44$3X>A%yi5gqU_( z8PpGHE4?hM@}IYKQ0~i|en_PLt5#(7RF<@f&7LZ;_2;dt>1-*XpDNXp*;DegSna74 zlX+7_RPV`NqPj|1=R zN<2&nuV0+(=RK6zKM?{jD^Nt&+sDN`ruHe5upz4o}JH* z)ON9gRT<97x`~~Ut!3zr^Q!%VR#j75wCDyeSd}-0z_|yLUg+y!8?n98I-9E!w-tB0kCnv*eAI>0^iBi2IS_= z^3FJ@){s!8AX5;F1JiIVUzkPM_=d$ycQ3>1-JEpE${8J!|2+crxyJ;kQ(3SOIX=4nVyrpE+M<|TyV{Bo9#5KBdKw00@V%g4Z{FS< z0<@@M>{cXUuX@SrRhKPG8*3lWMftzr?mCfCSrx%q` zT9FwiXZn#MF$MjgkvJB{Iuar~6^RgL-PXujF@2ks)%Ehe+bkbUV;H`qhfJ~u+6Ze0 zkYRvmhf3^L!>SJf>7V5%*mXp8L;-&WyJxb6=GTy=W04Whyk} zyGO$7c;2+O5fYHeKxDgUEzhqHxBYymfndh-GVKf@^MLYXyOS9-zg=9KfaD$$rv(a} z2vNzw_*a^SBDKymxf6Ma!$wmSo39_bb`sS?6OrlFi7JFzIAv3ooeXQF3DBXl1RmAB zEA%HWdCZw6I3bkHt|$yNf>FFgnRUgF`mtfPbyft~@L3xVYTBqjZp716yF*pZXOGk; zo8dB(AT45n52stFAW^#+zV->`?W*G)s=4Xh$HIi%`?c$`CRw@v_?vIqMYXf2iF7g& z13XoYKX}%~9y1$_)9ERqh>MoRBPpUbva|N@3923)w|XGy(1)s!m_f`9;(L@w%&=2> zRzt9XOvMOhCt$(%Gb=#G;Ro{Z-EplVc)(b;tbln?gf5gajF#O=Odsd6DAuJ+i+X{A zn7U5a-TTckq%7iKl%f8)L8U^5=86FUk{1I)OF*uV$`}kfp5{s5U}CCXg7s?EN(2)3 zs-D!y#>vCxt={Df9JaT)`0fRj~+fg z0ek~bS*ITWl~EH(1bHs1`h!!gkhEe+3rZpKTQ6s_{uqoKC$?6QV&$k78`|*4U^k=! zGuH-Y&OBPazKAR{Q^hKEvuGD!a}Y~vH4~{XOY4#{oy9{bJW1mewDi5pA{-Ea8rUBIco}qcGy|eovd>I z$3M^JQXvnKhYg=32?AXNCj}cJlXj>O0(pTwBkBRnJ(7qvUE(DeijN8)Wao|EBbUd9 z86t2$qqWNf%w?^1Ys;~feF3#LWt=zND*YU&14+_iJ@o?S8&W?m-{Sn;Wdv`Yq%^Z08 zFkbuoD=?qp!gszV9u%0l_~74n7PwJk%7!)?{MD53*tA^7fW0yIs+Z*LOcyo};m*%O z#-fr|xAoRKczGgL-eRJ(@PU(Qb-Ulr7{ipc8kjL?!2V8gjUUMKRN6BDL5Y zsi7b1P&B{I(5H443h3@uCv@6?B1a1wExPF?tuYyQhG+OG=*86D|s|{b`Tbn0&@LI|Yfb)*B6pu~wLT zEYSv`G({Mw)+Li(mM72?88igFRiVhY0>^BHB$3vM4LqPA5P1x~v_!#kI^9DvM=uZh z|L=~TpNa3k96c@;rnaX4%iXh3cRT#@;|e0d1OPz&pNRiKXZ+{%xz?1m$7V<9MgQ%u zan;*sa!JfY1m4DXxax)^A{B*0N!vOV(XEq^g1ZxStWhAa&@x^K7^UY$u?Z#-fM)S# zol_>EOyFNAd}AdlrE-nIhaq|uJ?zWulY`kusIFeO8(--j(MdpK`F0{rx;HeD07?Rx z{dMj4Ts+lBP9g;)q8XI^g#!J3u^NuABbd!J$oTY&4>7yqTwJLv+uGU)Vn{;Ah*EBU zBjOH2@?~u|5Ll!E;mY+kc?=w(=Eb689XxRRYm-*^j@)qgvb1Y^xK^NR##IPn=>($`Z|`I z?q~Ye3D|&#?%c$XAfsieQtfgZ^f#8k2~(Z&?J`&EU_MyE0fJhVBcrdHI~Mn~RUE6p zhIsg`TW_yo+n=!SSkl=o)kpgSR*HLl%|Z8W+K8giwE`G@-9dG#$49l1L2HGudfJ1_ z(rdR;yC19~5={C+>6J|}>$XiGCL@30e>0z-n{Rh{-Zo>=$-d_D`TbwV z)a?jtia2xC_jo=pUzgP5`#nC7h{1ibivuBVlQE!gx%mFtHDaKZ6&o>+0e{({eL^5E z8F52`N+o2C9P|?tL;LzhQ8?|pHQ)|#7i)<0v`PhcQi|J zRoNNC@JufGJ1<9GW}7wyPS?3z!RVx&SyGTUa>|r?5N#$Kg%l~2V_;EvZXLsT?BWK= zXyh>fh7;L_q_=x`OK=!7i4Rj_g@t{Sl=0Z~f(YWZ&B)ndY|C)D^@5GBC9#1T!8PI5 z*edi6D@y{UTjxnpW+vd47_(F2sm#aXkV{CYOjss!4YC1&E*l$X)tbZ?9%MN8379L^ zmL%Qe>ThQ}26Lh!E@4e6SjmpkRR#(9q&wTtDE zr@0$?h!?7zVxD#k`0-)2Gq!9}HU`_UQ?wWT?Cr?L_g?gU)u+!_5|6qef`u!_blle4 zAKZE#?|6t(Ud~bv{ohCJ0lUa9h6)?!(d2YDMwTG>D{^6wG&%CheVW-U>jv#*u&fLzLFC(t^EI|A z1t#GXwogQzode?>p=$`&3Q?92bOYguW>|kTQ_0Q*TaqH0eJvLz)`-aA?P3zUH>+1s zEQ=ym5ohZ9U;-TtC)kA4Xfe1XJ95l;F%OEF7Hndw#SL|Z7a8gt`PSEsfypFUf}LIp zj=x|ogZ6|JjaI!iKo|&sx9*dgEWkkZB+!J}YkNY*lcsAv1 zlh9mm+K9kM+I?ma6^Kbah;TwH1~Ua(J1xJlFvC-IpG}BSIfYryc$bt=S6Fwb%w`y@ zfuS3Dw3FF2W`tm_7q9+gy3kA68JD?2pM?}j3;c?H;q1Dh#0nC$<;xV6Wzi66=xQrtuci6rSG~eh%`7i- zstOV~GFkvpjyXChN%|-aR91QxVO0uprFD6CX;ZvFa+B%6j7HjXj7H{S97)+F0*P1vWKnk(qf8DsDfuJC;ikBvlhC|G9i0Ut}Nk<>C@)gwp+e25K}Lj z4an*NT%cCOOSm8N72acuX4*Sp=XW3==v%LuqKpw#jS*3zmNRbeHH?CEmk>@RV@#3j zyl&F!u-$s@p;T`M!~d;>CarkopzUrNfOwX^NErrPYwxJq{-hxr{)M;gx1^+W2iDdd znHE5`tpr&CXErYom2b1h_7v3+3-LksS`tRvnhyInvCfReopm`rlK;~*1Mmb(6WggXNDt{QHJ$1&_qhFf5V0?WhF*I#t}kt;s&~%?MU&i)PMjt z+K;EAqH8&SaJ%WCO@M71uK*V53h??u{b=Yx9r@d*F4}v-gDD{vPO|dGK)NB*?vwU( zHzII;f@^Q3v-hj8Dgz9QBq?eGH-&VF$C8B9Zi#;Pmftj@f7G)*e-+NpE{9*VJ}UmM z_tZwkC8j^?9uRBr%ZCPAR&ug{8$i?Yt%*HC{|2Wk_zYRLe0E-8$I!IcY|i{)>e$=J z=AHdl(j1a-1XdQxkdRfE&G=ga&XwX3K|9yJem~T0?E3pP z{1tnM?%+gLu71zzxMCX66kb*{+S+%5kTDqyi2mG}ipQ)W%8eaHMDuiea!g6mI5A33 z0FuI5R-x`FzZ@wEE(?-j>Un4zUP67&Iq;+|L%%E)llnvgtd*+|EqBJ58nHvAOY#Vz zV{~yR>-0T{C9u3g-skl`PBUE$YMMSOnp#vbvFlF#E9(2u(D2YTW6O9h%2~c?e!m=T z%trjZGuTXR%~rCq>&$BZ8u<V6I9_`lWlwIk`P z*W$0#(yQ|N#-$F;4J`TEpKR?+6YRCOW%D%cP8*i5Hni=1aji#b+oPAB&Muo-{i)wy zNzV&|hshtCYkJ>LGwGe`^|gSSB%kn9CcezE()iR{8Eqj541!AE^pO?a-Wo{U6U~ z<^3-YN6h&kUq}8-Rj>IZCD;lL#tO_UDL6{uCC7V7Vq!`mZfAsO?PyJj&yl+0wyp9O z;2Bs=Wamkgw-9Rh?y&7lk5v?gYkX|YV}5_<{n!FwefA~DsPAwM6(#G}M{1^~<^bW< z|FV~{-F_W2QI+w@B>gINsuu|JYc{V}5%!HUC8>y8p;BIb!L_z+Wq68yJjC<}g7(KZ zOEVtwBjDyP_3tLo%G2#fJvY#=WGWfkvR)m9MJI^Kvj>rWJ@0~|T}swskEe#czW3p+ zI+nw{NMOBhW_@1n;s(MziGRHHUD{haxDxmk52S44Cl78Np37avlE^Ebt|Xf%@v@h7 z^cUf3APN$=xF&cW*Lkk#G~X&TzUshdU*q~{@Fc>PSUGji918kKK_9BVF0eqral+mQ zItYjt?1Jp3V>t}+N{#3!I4t1zN?oa=wiWag z&g)O;vvLoNWBRsOV|`cfbt1BDtszS^?2yjl?gHkG(NZ;9Dmd72hG*6mRI&{K87fO? zFxld6SoGJ8fi<}vnRTStO~u@!08^YjC~#Ct7>WIkXKFqMbVx2*BeO96c~Y9hDBtp3 z&J}@@OT+POxz$z9!ApKrnE6`fquCs|Hw6l+RwxriJ`0OXKp63Kg;vYr^YhzEQf0yl zPr^cbH~20OAT1^{GgnRZ?IBgvcdp^VWb=?ZJe&JszJtEH0l1gMnlfOQzyU)T z4Jb+~SYIoSV#n-xThUU<$c1v0J=h0H_u&Z7m_sCCG-W4;JcQB*_PX9$*)>J`u_G+# zGh)a(%!Nj>H&-&U8^gN+ezDV8j?vNpBJ*!XCv-PbYct3&7?!D8ndF%Mj*R}7MJ#RMqruC|)m;25 zZRpoXM;5aoeE&i>*lM+dJzZ!)c8J}z!YU1YJ>fFq=t50olZOGSf(Urq3T*TKN;4;+PyXUzVayE57sP-jU_k02Tj?Jlz&Cx*NQhdO(PSwztRt9Z0I*9U;3;xmu)3 zM-RE4)=@tYSoWUJS+eBQikA6PLIBtEI5VML<3)3Urqz-=`mQFq;X62O0crmiO}m$< z6qf8K1fc8#;mfs-RO;bXNeG2Cy}Sl<^3(Fj;OMCS-fgQM-H;_Ly@#D`W@3QbQm*$? zGPp$#C%ck|N8myPD91d(J7~gbp6^j5Z?6S!W-A>Z(cR4r&))Of@ov>XUa>ul`>N%2 z*5^Aida=sw5q(LzDVq<=r(=hSla%ZBFeKaW^rPy z$`#*itxj{^=f0yGBL6U+&$FH~&FbbrO^75-<3-Xw*&nSob4~U1nh9~k%X1hQ-PLhh z>xb9VT5=B?`WXVRSo|eo!$Tw3!8{ED!Tr_0r?q=khMTa};t$F-36w!0ES3!<8Kqu& zS42B`AX`rP_tHY#W94i#e-MdI3pl~qq8TQ1r9|vaQJ*v$+}rIA9#yaC(8_BU9CriP z&GbYnw;Ei$n?XSH@$H?OB>P{y{cK0Qv8f0`K56R}(yNtGO1jwM@i?yLuk$EIUu?~B zoZ}s`52tGHv_s$H*F(n#`I1SVX{}E^Z&rfJ0scJWj%X)j*#a_%X9b}oW(?h9;`L}{ zUp`-X8^eg8uXLgg2DklM$m;+rg>x`bzezh4s;q_>^KFT3cW-OuYP(~w2~VuT={lHg zYcR&Khji~abXC-{+ajWcF2|L5OVz0w8+AMuOxmm8rQ7)@a?4i`OKsQ0v?=0u#k90% zl8!6pDU&J)tI4~R;e0r@j?!g!pl(6&W7-!;K{8)Iq4p_br4Cx|?MQtGIS2Bl^kSU#G047r|&uhOOm7sj$&5gMIDPTFSp8e`l z`e_-|c3q}I%KEoRz~A(QV+)Hay6;B5x^Usz9|PmV`4}kZ4Mr7TqH@E5VjJ#mDa+G> zw?or|7n<45&z9$#`XdEh<7EG4-F<&&SYEEW`S$y2rvwV#k+ph$=$${Agy+>5uo#Jv z5Y<#?jI57+qyv#fv_G#+I<+@;bax4&%BlM19GYe9G$bocajldxSme8yTh^S?opT^;y~al2bmqK< zTncghh+bg$-W@IR5I8yM^T@{!j?(P@2G_qTdTy~ZUMxDUZWeCL0ueG?n>e=)vM4DW zo}QB}dK|LdcflkONraCx7up{un+C2lxc%E;ZDczF1h zV7v>dtCb@f3LwSqJx8KA9$9Ii)Oj4VCBN=iFYd>SGPBnK%Oxe|X&8;phiH8VetfBg zetJdDFBq_klig;|_b4;{V|?AGr@g9r*}H6LePm70!MsCk1op@)bw}(UBoI~r>4g7! zGfUxttX{jmtk2m53kp&v4q$H>t8}Tlx@#dP=fs-~oLZ)^RJnYvxhoG{YiA!T1JH~s zyO$C^7FC3xN&EcW_c&zg_gMgVW?W;+TB?-{{Y3J*V%dDB)wnrfc`dF9aJ2DeCXqU~ z>|CP!oK=wr>drJ%zw`(hArkkTV|#aHG3Z!YD0q@H>zbU(>w)5Y zi5wpv{Fmd*vM?onx_+$})OgOBHhQ_9rwe~t0=$%vUKd?ZfaLUMw`YA~@uxx~QRM?E zlaiX;(;fUA%3kjgMZM0ECi& z#lz3MPKiCaGTr(^4VhKG#f@M^Nd^5kjQC)~T~DTCYuGvJu_e?yTswW1>!5ZuG){OC zj#;0oTkFMSyU1(!+n-ve@!fTssoS2TS~Omzkva{@sd~N^H0q2#>LkNC2+&|TV@ zsQyQTz`-r^^g&wM{KSFF1JPr(0dZXoyqzB3)LP&Vpcu}!1U>_r7wdk*J^fMNmWR8k zi=eEFF1CP{;@oUra@ggrDE1K&QA;Hh$s23d1Rdz$Lvs)YUITnS?B z#<1i#+=tC@IBO1E!%9DEcU5%)>0jLoB9qjy$I-N=No^-xY)fM6>CjU2)#;Is-#{yA zdYFKFEAVGiL$Jr?~>6(6XmY_2ZWSh1CDjG+>dfkiNY4xkvui~~>V7TNb&?*7M z1;vd9QxAsgR!$XQiUqJ@o}c3PWpM)!{Ywz0pLAzalyKr-s8J<$HZ&k5AW(k6W9mBq z&Zss##?^4`Ht<%?j}M$y@z?H(`=*Qt#|;2m={*(!^avVGX!}S$4DWJCcR;GE(~iyW zFeK_eSW{VSWl@D@hn#Mme*uPD*qJrD=|rKmOX+B_GT*2AG&y(!y@&1=|Uin*&P)aJh`@yHDcDnz|xlmCtwAf0_J>f;Ud;4_O2Ha^=BbhL2;BQzm zD+&aweLB~kVPJ46Ovcq;cRyi-TjtZcu;8TLzXep_eIFgeU%FmX zsp{RqrS6epUj3n{K9`mW6NibhH~`48l3DIi#m>=iR3PCXy;QssO%@99(Gq($Ez1Qr zJJRzTj)%P(QfV$&{HVn@aCvCooi^#!KAF6|iwM3^a{aOsWmj`hoJC+Lo)GC_)BdMRLK zRPS@qrF>?!r)IkBu8wl{gfs53>AJOCS;M0=y1Fuir;6VjIa@}TT&<73gSb~Nd`ata z`kRA-^yC1#DYhd)^iPL$f=8+K<`}P9#cJMn9$Uo{x_va~`-imhzGA#nN;puM{7)!3 z#^Qr1)#IMGzRrHVl)`cLnId_9SfE*`Arxk8)K$f5Y-frP`4C@O-FvF3`9 z84k~YQXUCs|F)EgQT8elfFs#K_3w!o!7kz@MjP{Hw29EKaNx92X~&iVE{n~=aZW-H zQlc$WCQ_g!=16ext=5Jh6m&@CoC890ClK$aN<)MKzrgl_Fk%2k4J2#{(-w|%ua2^E z0d$ZML*OzE0)z^rN-xwA4W_jAv!Zn&ga9JAX`-;JA(jNpq}5P|^G6Chy-J|b6HvDrC9z!zMEd`;7FPB<8SHl+6oxsN=a)JC@sI2C!|DIG9S zM3oQl+q)#|H2pwU>lZtW0L})3G%BxzS zi;piiQhgP+bGH}BK=LA3FvXLnwIeh6X1-y7SR>gm>e_5$bzw^78(!Ig3;E@+!`!cu z&bQ>D)DCugBrd2U8V9h06Ky$0@ZOMcjb%&4V|gfPbOPFKcsfFLzJ`#Ih3K{C)GjU6 zTkFnA?*y8;p9bo`?%{OE-ofw(w>iY=u`y$L*a_i=;!BrhbaHxO7zNV7O>i5Zos!$_ zW_DHerM2~?rKQy=49k?q$Ys)Ew~tUZfyIUYSgkF$6^CEiA|Mu-Pmg<0s|z1YL5I|2VcGi|vmPN-a{}q^63ZWS>#lhRyh${g=z(dW#@pE$>&`XYmGn2Vn zYQ3tWY`RKfMK51%Ep)*5qr|+(Sy9$F<6BHNsXqytcfl(YjQxFhPZEbTe4b(g74!@n zKK<+@UgSQ#o6AA>zVg!GU|BFGg9UE<^zpxv5)NunHOYVh09N4u0FeIACptSa9B5yW3jWY)`G_!*}1qP~^1%>Brf#C%_h@7a@7uoAiKULkyJ`Z+}Vjh&dzJ++b zv*Y*ercCECt0oe72*A*xm%@A^WSrYvu&CK-%Val-cqX0Cx@>1JRSy z2)e6)x~;&fi6W>9KNhrW(g?U8^^|zxag4(e4_K55Lh!zq^a)?=T#;bZ7Fo%m4ie}T zRf_wKqYx~4S4mwRPup1Fa6ox$r{6ge3E zo!B_IcuO16xlrpYp>CmaMs_V;jj_a@pY;r4HS;jvvQbF9HzXH-b6A>{rluJ(S?}U* znTPrc9vJ!#r~_Za>*@=nq}pesO|pzu3=0PSp#FLz5#p-?P>wz z!)m|Qmki8~Z>ku*s&8ylv`cC`UWpzh#@a={j>bB9g8>CODNK72ZwgP9Sl=h*_T zJLkj1ywd}?i`GrcK?7j?DV`;nE?-c@{CPma@!&UQ;ME5c0OG%x362uhFoqigO#V9= za?S3OiLpBb(tk4-hQ)EI+7{LX)Hexm%6yL><{oTx%*+WKD6=@rjr7G>F7$ zvRpSSrama0aa0m)M3v(fW4a0_dHKD5u3Yx*i0y?(emOIrgX~^-%SvY7^Qz>Cr)$BKr!znvxLSF zDJ!Q{Gi~|g+c*=7ccMV5LOo3@XU8L_)h_NEP$Qa0qDaL=8?VQhs~?}MFyIp3N0-+P zB|mLRl96DR{j!S#ezt*}&1`2LB=nfzP@U!Ga^HoPV`3&C&s0De%CJsy(>Wg65rSP7 z0rc^CLG8$!O_7OIY0-ccVdEjoDUD{LlGS78GZ$r|Jz5iC>Sf>-Z-nH@s3g4T_#>sT z9xJpa<}emZP*iU^94@*6eQapPs)oo+#^^bXfHUR=BSJ9el%`p3PTC@Rd~-T=yJgQx zzz`q+0FdrZ8d0GN8Z-@Mc!Z53<|1Qe*%)isxjqUE#vf&byE6y?wHrwoAo;=ogD()* z1*1wDdFte7t)EJg@Et-j>093~(R~Xpm|{!?+DKRY3AzB9Tb7FAvkc4&QYPS%_~IHG zSWQvEsXWQIE{_0Qa1CFRSEvLaj89~^d}RxHXp&CQf;BIuJM3^?peL)VyZ!yq)lDZC zFe1Gi>KW}fm(n^iAMxh69^W0$xNlw&-IpOLR@yd=B^$C2Pc8<#NfR8*GB6S`c||FL zn9_HrfIUYL6%C3Xb#Z7RRR0SApcoSk58fS;;?_3(Pw*ds^m$#6B~ljw{cfGZAUP1%y4}@;-Y6(EcdgNvL`H8J17j?b5;bJy|jxA_C??!V@pC=6BdtMX163}u}^Wlr4R z9gFv9pv^#y@t20d4!&p~l4K50G@=Ec6})>`COzb2_p1&q3a`;FZe673L0Y;Vi%EOL zhCGTn(t1^LKSBRL1fvOaOqBpX>hyr2H46rLCd4DJz|WojCwN zz_)*-f}A)k6xP4&CRj-c5hVZs;0^!)pm~U&oG}2_XzBQ$8>EATrt?o4+3>#$FqIk( z8vuX+KvG0d#Utlx$6H6m^^5<{Gfc%VtBb(~m7plUwE#2(W%LQz5Lg$`jb%0wi=_){ zj{~NY(nb5|OeI+KnJG1`po+Z)T|j@14x%fpgpBPr!0q}&)@%$J=^O_-+g9nPkHn2zkI}o8DIiOxVX8uxVgAja;;c{g~5Ng`~%S3?h1N? z{k+M2v)diMkMRTBCkzzmw7$m0b-&$Jein_GW*|Y*^l$!o0x)k4k$QlPL!PM$=eJ%~0U#;_{@LA|nFvUqA+sNwUh7i#Bd|8{L0_KmZM}KM+DGu%Dtw z#`y_FdwY9+T^(uBo?Py>_=&HoYZOKn%09ovJOQ;`TW#(2@5aT8tWD|kdW6LA_X={d z4=$>|@N#l;A`f>ry!AAGjoR2fYrBPN#sUAx2ZX0bVBf~XCfV4@lciGdo zEo9{*G}+nEvm2>C-oB3J?m5_u>7ZBn@7XWBE+1{3b;11d(h`)er1^X(Nm(QLyTXam z&N(aYQP~8)ctS-)loV3l!#1?yU2fLYct+K<=EG}wAIQnWm)Kl>qV6dU zZfY!uTgm5Pi+Yb%zbz%g>G|^(-JhT3HeQjJ!o{H4e~vxwX|?WRi=)n8D)6f?VM=w^ zky%I5(5{g8+zCovGW$S^))c>_{5J7-5WK7hA>^9AC^BXVCb;V+!KgPUx5x+NP_`f)bTILq^HCUb5`ZG$%Vu3)QJ3 zO2W9Sw1GH8tEp2F$*XE|rf_x2f^S>9YJwf6Tt>CTI+DlANrhZKXVPlQ37eIm(r7L_ z3vN0(knc8;vy`m#Q)!cR>6FPZ^hqeuY+4h%lgbq$1E0Ilo~M?S>A91nO;*d)q(wTr zqisvdL`OY))D`PS6H{H4t`1}vig!$(V`8SV$$~BzKrbO#F~vrgBqA6XEEda;lp73% zf4aJBS65qaZ!f#HwYBw@KS6v%S`W(evK&$hni$6<*6xB490*2042>!7Nt z{us?ia`QcXZmvIO^B;MeB_kY;pZlB~i}!a+%H--#-)v^tW?-|RoJy270xiP)>6!X`7566AGq+Rtt8)gq#sOPDsTAoQ_mp}zD& z@2fUPQVmK(ONk=G2R0N;%(FQ}(RNG$=BW?AVv;n~Vp5~O9eb{yt_v*tbMa6t7%q=? zyk|eozP;KsS&Q3z4z}8FIk>-*KDRbr)A>Ifb`Djw?~gu@?k|6SKYjk4d)4Gtv=Psy z`eeuLf6-=bSI;mSyjjCQDX!@|ye`c-Hz8d7wt6S|KzvEb_^s{G}09@cMxm{7V_6ySY5v;aT ztBxF6nal<4Z+UxT2@JNZntx;Qw?*7uFs`O*dEfqW8q3^bYDe6mgPvK@U$CL<%4KrR zauE-1_uUbxtLn0K8X-MyZ(>F4`-P&9?Gx0V;@e<0bHQq@R9Ic zWfFbZBE`Y*CsG=D5{E$(=(VR~11r8zd$Q>S1rOT}p;?N|Gkdd*h`W4%o6Roi;32IE zhck2x)0e%WL1D?;Dz0)8gT8(9_d_LJr3BG|+md;bU`fq-vF=GQlkLa370T^hn*rqk zJv}Y$8)MmUFtm?sO#yyNmzVtE$YIxH{p?NC-O$R4MZJ!0WLF1LOk6yXw`-SiBTi$a-owqLU zWanjZHyLqZvSh8pU*uu;{X8vQwb72;wxYJWo_2$Q);+O*Z<_h;vDNjlm6H+ma?R2x z{f*^2!PVIL!us;x**sPUeIjnz*vXis*&HRjs_LL!Cw~v07@oe|jLnZf72m?5b??3a z=C*TG*+Q3sj(ReR2U-0S1hIv9e)#UC>+1%iIK8XY2HL{PPxvH6)qMi1LWmQDyi$%D zZheW9jBU`m4%Pr zmB-?(@^^T4U1wkik#5;t4JI|ebCTALB;6=&*7f}K=6=%C{`9TUdjoZ7*1a_?Q^^Uv z_KdDJ%Jnj~T_yi?E_bX8Rz+_P4{CS#sPYW`&e-&)FH+Kxml=J^`1j)?OR<=fv#0$( zl>KE~TTRqH3R9u91=<3|inYOtyIX!#^SnbK`NW?$6wXx^Zm_Spn@RuOQ9F@9qD+ zKrogcd9HjwOS4L$Fr7w|j&Drqz;i2N4`!^$a01nysI&G3%#(lrp{q-2Hur$4#{Ba9 zB6wg*4dz(vDjIGtYiS3$>N>fUW{mHu^fbMZwxGE+kbBc@75uBbx0jlcfl@~#>NsY| za;^*eGC9;hB&1+}+-XxT^*su6=lxcRHrVNGtEAG$+tMo2P&DAuh8UxEKkeFdMx()e zzRhb>8??sC$o+^|nmTokS@6t1`*>dPtp};Mwpj6x<|O3K1;AyDr&LpxkFkjsuZM#( z#mq~w8>;aR{^lmMEI7I^t9lyW;dyHbA)NIK1>sTPu%k`@CR;eYxl+L8fooeM+c#(C(~LCDjmkjxe%Z0B6$(! zZW!I1y1BN9k127DpOhTk){LiIy}txXx=NyUnOJ9;FUKqSI&j?1YkgwcCM%jyao_Rq z#T|OTM}#M8e|j_R?H$B!-c`5VwCu6UFNf|UTeY@O+UTD|V>;)rvXqPKxd^hRS*s23v^T#~>MS>bqvnZNxKfN6_VcD?X+haW}|MiMJN`Iv^4X@iW7ABem z#(#SAGeBlcfvfs?QzLfdUX=9YuU4($;WIU?#E|#eDzr)*f2A7TtLWxkOZJJ4YJYOz z#eM18f44PcR+gm6tSi4dMpz2IyOIS=4JJ?iIwusSSj7Q3wUwDgr%zdCrf$nc_ zz>MR}uPP5rK83YW5kXx@VXkc4;+Lg4nZTCUCP^fdlytYUr+LKgmLUq|bhq82xqW-4 z>i4ZPU7d0E}kK43A8nrwDL2by}@+mhhm#QVE%Og$e}Kt{JC_J{f7PK$ZZX**6*aYaMc?i}(f$T{C( z)!=oF=#)W*!biiG{yi#?))cy>jTdq_h5m`p)Rm#6J_ zb*gHvmVYI|ItgO^fn!R#8bsf#Nlb(-~< z8u*b{UES_weS#$)$)qg90RR{(W3G*f+DLucWA`ugFoRju0^hU$GIS%1Zp^5V^{HRf z>K7b;_r<5@T6UDA*t{iHpBz+81~zC9KS5|>a|ayzy>i%e!rqqM|9*eBGv(~h4tW7> zt3L`7v>&rel*M`H+1u@5KEOfOT&r2vv6sT(*%%&sf!4qB_@*Nn)h*G6ENG(pHh6NE zl62oSV{55^X^-ZgHjNkJ`9jGKME&0@cXksjo9e1AO-Ql$P@s^yTz2|n*4(&3h}k*^ zHBD165sDG|t*3{%GVx(Vv6>JFB;70NP|#|EY5hk*O-5O2W#Urf33E=e3u?&+&wcYM(hz6 z)!J9+_A1r)*f1(p6hG%Yj*73j8nmNK$LE4>K^xigzX#TfgF#s-97TP6wg3mJ@3Qff zXWC01>@fSGVPiMH;CtYCN8SO4^`5d%u3U_epDp9tyMNQa<%8%CWB$+#&wL znKpI>Byv9vpG#51B+NVORRidB7yFftuW_eBe!5!qpq(=ZcXLuVqPWkJnJ*_vL8lmz zlu}lcA44Wu7t<%n+IP&lK|&Z5@RO17=v$Es8ArCeg@4Jr&l;|f&+=Tq7=8T^$f1SI zcLW%t2HPC@7MSq&xn*(I!`Zt{gFpJ8ec_tX_HO2s(I`SQYzIFKsJAi*x-6(<@km$9AsL^9Pze)L`+13yf2w5a1f{TD(RA~)t;(}M_YNo8up8Wi88Yyv|JN-so zo~&^$+Fw2k!(sG;Xmh+d)#(Y%eXWaQy6<-eHf~ZaZ+JF_^CB$I=H{p2v+`>Zj#+%e z7(RvzZX)XB8E={W_YK*Cl>lM@>y7eqjb-?kY}fP_h6?YH;|3RsVwf2(>Gj*u)nIHPYp-bIP+k-PeX59V;L2|x|iT{ zcj;qk1G2*7H>t<{_Q~T;W!3AH)@Z3XjRP}0X0%LRN+|9070$iZRX=fiXJ6hTWhEp6 zPJ_$jW7I#42q^0Hx$NE=n{XvX2mKwk{fMlrlQHpEDR(ylEX{98lo9B1dSYNYM zHZAbOH7GP9b3v@R<~i>f)-<{tR2OGBecR)m71zASYIENwwK`Y_+(sY}9%0f)b3_Yoedv;`bjWqY*5Nulrbkt82;NrZ9ha zWTlK6Mz}6wlaNL?O7X5AOG6t$gUf#)$g!D>mtcLyPnAmR&hufE&mQrEz?2OS-mN* z0L!g%YO}Z-So3WGbV!9tLzP+(geeAdG>YdcGKYqYw zeh;7hXTXV;4G=S}^bdOj!O|eDQ>B!U5XU-BYbeyGtfe!GlFSE)tG@S&5W2Le3*L`{ z@68dPMqEsG3A1=wyIMSDk8j`$SM;(7gnNLp5)Ps;--EWNc;h@isWhp{$|Lt*tiGAI z4%0Q5^FNr#cs*W#-x|Uti>`^kS7)oWcXH?$FB^!sDEp@-UN1b3xJV)3MC03WtO7P*yQ0N%QJljZi`7H{*`v3tC}v?nnWu%Z%0o}Sq9iKBc zcu~EYuw{_2fGn*C`*U62mXL$8GA6KVF=~|;K|YrWn+IV98WT-QfuT%kjg#UymRfuE zINQ@d8(Jg zL7udpLwm$SQH?23^~p$%W?Ut=PBs~1rjpT(^qp{m!Tt$M3BAT0=fOZKw5 z2MR2NTqWv!YxQr!$k5+)z^EqX*fZutlRb~mg`^OjpaCWnCo(&`6Cg#V7XpR; zj&WYnOH+XY{`i)t3J@IcEw3Z>#J#n4ZcY=9TO;)LBLHhS^XxrkvQN|-l&Nn(%{~GI zoqXr7*lC5pSDo{yxp40$S&IlTQC*Qkiq~oa9Q>>K0qGk3ybxGgEj3 z`m_PeF^U0-XogLJ=OIAKE_SXy^2~0MAHN=@4G#}rh7Zd-=W7kE{Cli|zx~~3U~vi9 z?eB}Z;NvIClb11%Cs3Y${QAhfqP+X|1xUFB+KKYi^zrlm#s!yiLNm+TI4l_C*jSO0 ze2ON~R>Jz$4c)~ImJdD;y9Wm*T}Dc5CU%X7;2bkadOYOnv`ezci-tGe$7MgqCntSI zgmj$u_TjC{*rUb~4RWx(g6k2n>)Y$OMk}BJ2G2k>NmxNUJrR4D$uI&QrSq*qec1BY zwGdqA%-i8_+{W_le8V5#x%mk&Nor_n zLKg@Niv=UwOcLyrAAQgTlHhCP`)j#W@7c8Vrq((3_Fs!9!X!}}L2`9Byx7!rtLPdj zxv{eH4w;WSmV@hyCiGQeWv*}5Z0ikdW4g1M^|pi+fltgkun6zngOv$wx6Vi2mUg7v z)@?0b93K>Td!iYeQ_o$@Mx-BH+yDERbR9abo^K0Z^l|J8tX;msc=g)JfS_jy_603f zN5d<>MVSH{vjRT^4D!#s{nSyH*lEuDg5?l{{DX-EIZ0RD5i4Rs){~m&4K^kT_%L-4 zDsTX6?!43P;|_!gdWnP2zxV|a*Jv#+EuAdCqN6R6XR+cSq712CCM580aNHg?=V>qp zh^n℞Gz8nDB(SXQhX;P3m<7H1%+b%yb#r z+y4)#=QHeGM~=|-<3T`&n)W}5l1@tcO_M{@@b z+dStY16jzNXvak3@sb!QL|j1K3ee!nuOGbGyQJY0ph(G+!;-L5^i!XrK#TC@GBu(7 zqq8e!nQ3)A>FlaP+2KnqYn2geR5}NN+R}3_jY?aYj@*Pp(gu%osie(p)(`m<@K`aI zhEC);VICk_*hnY!mUi>6!_A*Yjv54<%6D?V32&Db&GYSIp^ef-A6UoAYFr+^;YhSh zt(%)I8%jy@O7P^Mvu~*f~0h(-A{vGnGyXHr>bijqNywH2L_Q<08@1pu#;BvG-t=I!cWs2{604`7Sy> z4qog}xVJComWK@{4{77M&06Lm)^L9ZgKD(;p8nh5^M6r^y>4*%{;cPgq>6gwdG$1U zda4tWqnXJw(|SPxwG!OXt6_a@H4L8cJy8#6gY@AoN{63pH=5|ocBjQKSsA}e&9ANpqF)t&Qqt$ za(}5otc z+#77VE5$8NDD|!yTS(9gvhKAOx%_WV*bu=#EA;pJ*-SToJs9{PaF(#-RrgY}00@`I z6}zqLmnMI|n<<07T`VV9N8=ls%(G&cEjbOV*r3|=k>j|r>X)ks%li3GF6<_{`Uy5X zT4nraKE5S^&yGc_HEO&?Tb)zI-7|U3^HIa0KGh`X>bP+meeT7QfEk=|n%u{8p-<{y zRq$`8_b-V|635f25q6QfIH7Z(H8%y&v^A zJ|r6=QmR_8|CIo8fXAe(za<5I;G8*SOl{%0g?%z-!&UCV^4718Sdz?Sc;&}ZhCN!q zuuD2u^<_8`>Q6!Z6-;$qFZNdtP!nI>EIfa*3OLbRY=q!pZIay0FJ5|rvi^v^aCLFv z<$f&XRWKe0icaR#)YQw%OA?Yp@Ohg%+7N1gzlf#N)J~`|H9))8QD**=dq{axPGwl0 zTNo-f&h|Wyi8*a2-6@M*DuXiiFSVFj)K@pTD^J0uc9ujyRi>zRTdNGC4;wxFeUuf2 z;Ye-Qv`yQ=MutDRpXSLw3+&Tl(V#EK3f=a&;Aq1=8%HiyI>CC@h&Hd0X7 zikFP*MyZ|2tbu`?Vga@uJ6TUwis)$0lh(QcX^`WzAt{>2rl%Zgs;m6ma-7uG=UG@b zv;?!^Bouo7cxGsvqj3@<^|`rL;D;ps_v5FpFuPmnrI+?am*bDonOnRb_LoPx;3J&F zuXbVWmP~k~<(XMyK8y16%qpX9Tqn+xfmm77YT?A3vN|_q^J&HhWw2@3mJmYcDKI|M z7+He`_iRp!pX#qhgLlKiz+k`TDe&u|{oQPVTFvvB#B{y1Enc-Xr!rqd0<+LA8eW5| z{8$Fp`AN|inn{X#cnuc-vd~}3<*JEQ#OrGibAs23k34D8wig{HUH51TvTbyZ4?3xN zsA_9TP3Q@Gj|%hJ4wxSd8Ap@zQqghFJ!?zpM#ICfv`~CF`@y?!!q>UKY&J-1{(w1G zUtJzq4|nr(oT3`gn#)Nod1mF;%@&kdcY!Z` zUvziVS65Xj7fpg7bzfb|4i66S81BaJABLJd5lg#!TI1S<^Vq2zTJWOC<^Dqpq8HJP zwu-ic!LO+bVo!trPb@$dk2Sg){p6C-c!MYS%+h?;l(|L8=kK!g(cZ1;C96Z7~mvS8$kmPQVeVkY=050TD-hTnW82_dj@z z|C8)mcBZvW(RHC9UcI`p<@406W~WiQ6HGt`<_}^5VTEbk?{~-eTbeW+*Y(#K)`_oA ze}pPgMqY;h1_bBzEL{A&WrZ^=4$bgV)c;=%*~AF{q1(w=OZnfTIyke7i^|2tVn=gt z9{=XAii?}^@8Nn17;0=*6@PX9S?DLm=XBz8U%2ShmRUkV;3~d zCn~@-L-TUYg?pAxo`%pW^jqF}WQJJ?&U|J{n4(hFV=bE%{XCtR%?)#tzS1)X!K~sz z0!|1w%|M;T0!1I^(!xh2X&!sNGe2X8HYzN(-Fovzh|3;U-AVrDyou%V(nw3wiOG2@ zBQ*GO^$q*@cfNMq}Ammh1kt1HYEE# z#^j!b$E73eQ6rNsNKQmQC*4FW7ZI9W9j^jz=Vz>wvyl1|J#m5itKYq3eAB>UV9|2o z!dHl8n)=$ORk}ohe5exb?N_7GbBwZ@+q$HE=HmER!(WF80fb#=?Fbr&)Ad@j3*X{m zm#3iH#!e0qvPnX80VywxF(S^!Jl;@&40ev@EO*_Hoel%k&G*yk1k-T z^GnzxVqgZ^000P!;U6zSg>*DFgV~Li zN}7C^bU80_v9sSE(x-Zw6j&sBh1T5W4+&M{E;ZDdbkRZ;A)bcqD>Zk3c?S@UqYpIa z;_krCG2IFIXjN8G%cyvo1o$Y|qqF?-=(e_48WpW(_V~X|#J=|^SzMlk{BCxNe5m|S zQgO_N{r;RH<&#aakg`teE%>`FVMM4R(%<)oxS9MQv2UdS8<%bWr?D%9i2dzU9smsG zR@HyTv+d^Y-e+15mkm`fHw_#f2>^i`z7NL>lgk2!AMrx^+ZuHr;7w(1nR8dQXA}9v zciw0Z4mqh>i?^R*TY9YJHP)&0V1@0D;vp;;Y$m_BH-c7gi+p6=@*tsIL4m_{OM*G| z9MG&Y15bOdo>l`gTdR&Ha|!d5@oI4bCQVNRWn*%&fHV#m)4IS=gIc+fa92vmy8ge` zWMTq{R|}_^m}T1c;aN>uJ05jYFYL;PnO9s&IRT7|>Ric(y>VW)hxf*FA>V#WkPL!B ziup+mi^B_tCIm*di`j)xzH~xke8D9~rxYa93*}w+i--FrJ6s9;mq^U(pNp<8=RiE6{B~(veNgc8I!7j<&*ja*r%4Pzdh+vVab9X zmtS{TTFyV2W}au;AApqr2+t-FPt|~>m|0D+4xqzEwQ|s2b_c;wvf`sTWcB$2nS#8U z@^-Rx)zOS8|4i#PJUnJhH|yGSzJfyoQ%Tljjh2fqo5wD`1sC^o*VUiTNROH3(2U_j zJ}~hap7p>EiNa0yYQg5JmdVlh;K4||J83*16y>&J7O=U6OH)5@UnN*8x~wmLzr%;>I1##y%h!V`%FcJ5ATFge+D}YT>5X`VtHu1PA|tJ|k!bt=C9?4ICCPE=Vh9~dQl>2 znA!=o=_G#OxwKNNIc8uM`2GCfj*xqQQFL)zTU-0IyFlhhF++s-T{W(I;r_ z)>3j+6?@Py=h&6GGn{JO7nM;0=5w}!+tc|?RtgrKy2j@C#@lK4!?oh%XklQy>4A|g zRtR~ei>nVJkVIL5!YDui537K%(arbzh|2MTI!{@SnyIMDnlB|49PtKs+19+v$gu$5 z#ILH1UV5kyR2WPJ>L&-$s?s-C^IbDjw%L^!bdQN>F)RDH!}{wvPmhGz1})wq9JiC8 z$U8>}1PRYWb9QkF?lxk)_#yXIP*>LxdE*Q^wR5er$hO6*9DB)m4mXlQ@;ChFAGlNi zQeSS~29`1z_%dsK=plz}$ahn5t}d*IM4-0Qfu(H0?>`xjgrCKj=bNOA(-VGTyRqA> z-NnZiZepV8TMl#LJL2S6258DuNWa!UXeib|O&#?N^ z&v9cQ8%JI-2SO^YU&O1WI`daekd|9!yzO|bp|EN>x&9nYBc#&j_=mWq$}vyN@xYFr zT`9v*cBgxDa}6qIYfab{8j~cWYR*-IDhUa4FL_;P&P|DsrDpH5U)!#%t#W4zCuM+M z^~KKT7p1BsC8r*jn4n8XMex
4Mk`OL0%_| z>BLrD0WWnuert!1wWLC3M#ctk2_JitMAx}d8NLU1v7x{%1qWvmtG2_UbW;1qtDCCq zraAwfzVm-_jc>8#X5!=1>f`EUr93RsN+Dmyztrc_D%L%pmPy^js&MnNusz&3AtS-m zgw}?n&2PLbE3QIb4S&WCnpufAOaS)P24mqIP}K(p%tUK+-Sx=bZ3kTnV%zWf4M(&c z*&XazE0vA3&L3O^QQq>Kpfg!Yv~a6qd)wOgq~!L6WPwch$a6frv78_Vb7(#S0=DicJWW>b6upislji&Nwt5}oW>-R$WkF1z;zyW>Dyr%I;)Kf zq2i$X_{$!~Xu1^gy0pLvPg+wk0WWJx&?$P5#ljF2Ya6Grsb=x&=ef7n!ni;UJ2aoH zlxv%2L}{&4x1>MB2 z|G))<&R~d!2Q8d~=7RIyiO!)dzAXE^ymHmChxn}NEaE7KgP-?Ai3yr9UrfIzLwKyi78lon?xVj~MPB&|nwfEeigj z9!GWk4)9T|on(CUV_HQ%5c3LfJ9!*u$%Ta_XU(>W=2JV}+VKlL@Q9bd>_^IH2jjr08{2rGrWJL@bwf5t!yq8Sft;L0b24uC7 zp-tcdTN#hkg4l~V6+FDeUD{bcz?U2YvW!!L*vSiCC0DuKPKmBcWMjX+{cKlv{Eud= zM&0Ig74V;v+I#_S(TZv7oEM3}IUV)!owFpc1@I;-ZeD-za3O7sL$2LJ7VC$zDAD|6 zo)5ij`Dg*a(0o7^UL;31IVNIZ&9{>B3bY(){ppcgH>GCnfk+>&@yF!^$iLOA4(hq+ zu2YxU*Vop_p#Xrwp+d@-+wjJ`FQ7qtigktgi9c}nUV|!T#2SQ>J&$j26JteTYP5(ZSHs_k{Aj5{Esd5 z!qcX9(NkP;>7oni;8s=XwJr%Fd^7>MC%~rP6LDK&&&x~#ZV;zQQ56;;0srDjJK1T_ zgHIHAW12z1MEmSNq=6W`mqfR7^P`A5i^N-|;pZE|9LxU`yBTS_>@B z1)QD0JPK;YjO>LJn!a`EAENQt71N<5j=RlM;Y_Kd^pr;Pop*tO3m_B1?L6!7&XiO$ zR{Lui4@%O#YXTD(OP`*ijjuI$O$);7|8e_o3Ezmn!`l;xNb)z6t89!LU0tEoB?ZP) z@s^K(zI>45<+H7Z`S<+5_|9*vr`ezNQ_Ng{(?wG=#?Ii}W$~}_T_`H3tj0!sb}$Er zCXQMO@k%#5SRKTNXQI-Rl9~$x!}xjwtdu^mr%H_A%FQ+^vAM-9_bUly<;1GVobA&* z^yRqsW}_1JddUjvo@h4+DSjZ~O$}_RL0kFqd{{XxH7b(hM)i_2omfOO8Kf77h1D>0 z$(`XzCdtAxDB`0;EF_D?dN1KS($PgTn*N?)fj9YF@cciDorlo)#{apW}IU=GakN<*rjp4VY|}#wASKV z6@)hTmd8}F<9pTmNEn%rt#S_!^3ji zBq{d3vUQSxt6&h<1D*WBu4OS z<8+H5wt3kz(;BTI=E08oZZ6xqDhUC zY;JyDfLX}U`*6cXxlPdH&?xe3XLgzqk4M4V4(2-%?BTOhO-}ovHTjGQDs~eb!j{e; zHC%{IEdSH9vv?>4#-H%eW3-T{5J}ot{cokH-Y{eEAWAhhn7K`%Q}j&9z4G*~M?b6wy6dm-mp#+JZaj7C$G14|(+7TIl~ z@MRj_8^>FL58IY-4#VTcCRrqxYg^0Q^??3d_nl=zM7+T0Gi+l1clCcVXB6c2H1QWH zuMzhyw#ONl=~5KfIL@p|e=a649M2e$M$EGzvruu_+YEhvuA`;W2H%n0ha-x;C=s`!qqmFEA^eIZAftq}UkdKp?Z@<8MdQ zV*>B(cIaK($ck=T?o)1UC61b2JKE?wcIwma($y}H--DEj2qXUvj!fGd<8*#AWS4}z ztzIyiB86Z1nDg@T5k>~qhkFBoQ1al^q-hg#>YL88Lry;o2a=l={pb1CRs;9$M(%e$ zZwXlw0*{tj5UZD|&p%!~ zMX~WWOt`#s(Vhktc)t(;%d_Q_{1Z=6zCA(z|9tfM(@i2STd>$;S**A3RKtB-6GkC; ziScX`i(EXEm>g)urSkH~uqh(+~p7fHH>9+|Qb-kRQ!ayN)BOxIXy1&Y< zs;VlX1FHD=WjxDkwZktbhr-`5Dk>_?eJioDk~MkiG1&>Fy{8MTqOF~rn3xz76O*2v z9uxB(7uR{`-zT~_#g@r&M{5qOg4gS_LN}^IoN}4{Kio#Ho|-Pl^&HZlYhBdBc=S*F z3R@Ynxo$SIYVhV4R)9La#qVa#(_>>}P0h?md7XB;@%4#WjJiSz**r$GifJ{D3aXmF zh`le*YH%0(hD%qHZ$?zcKLX*%i+2iH7;PpWlT(&*l+jY!Iem&emd2U9I2?Zcdusjx zhM}zDchdMyS@Rv7ryL}&YS^6rq*Hl{tY&5L&HP{rQHz_d=DQJw;wB5o_uHs~T-PyR znVi(=lFN2MQd+vveqCHmO-)V?6YKi+7Do{RVYrn*_?fDm!zDaV|J;Yk1yQ|_4iPL> z5#MvE%2SHFldgh1{jmFI%k&8tT=-*Cxo(o3DS2iD_xBPf!)_nbO`DD*1r}ctd7uDQ zeEdv~?c+5OKT~b)zlwlWl)q-=plJIw0|WBUdkr0*abju;7>&OpBhbMFmV*gGHZu*D zTal~X+RBehKjLah;~R$D)|31j_aJG?(ZQ0OWn6NhPrmncOUT<>7rl1r%TYl4bw9lK zfn(N;NAEVJkU5QR(WAsEs=d4%TKXHKu7Ez!e84E;TP^02v|r)txDjf-7@PX(Bx9xW zFvvKEZopninp#N$i|&3C3@g6ou(go_jZT%yX38XJYb2<#O!bCH9hV-Fk}ljGq%1H? z&c;xXU~)Fw&^WaTH0cDDW3I`6`riNqqAhp9d58-2!YdvbWGMEDi;Mt<89`%jX0~OX z?tK~GcDH$0x4e}nE+)3Qy^Sz!n4HfeNmLzO0LmTeP~9n;z1+^!<1bQjM~RPmx2Q}$x_MsS)IVr=f?zYhTgH!-HZb-J%CepXyr*R$kA4+l%p zer>be?C2Z%gs>`s5U@YeW1Z zK+^ONQU(MSL>p{#%2FEvLuk=7^!p}`XO-WA29oGlJ0(57X6&rx=*G+e9rOB{#F8!m zzAJnI%n-+tvtf~R0h`@Zys5O!t>{GaEpkI3qT@)y2y}`-w84-!e(3o11&WG_H8nK? zBytVhmRk5URfQc9rhR1=A7&oKa^_b>KI_(g?)NFLG;rdGwMgAJQ+-L#Do8YyV7in9 z?d(VRMC97W7o;0lwOocuH|tf`v$Qxo{d(DrBSCF7O;PYLpu4FWKqO z7%pdCJf|AxQdhar&i;mr#xZ@9ePr0uoX_2uJTN{!?r3Gi!Z-UvTf;L>3cKMaq6EmH zUb+87l{GjhTg#KU{F~qPXd%r^|EFo?q3o7+l@nR}nd$jZ>=HYKifjV#`IVP`Dyms# za>!s4h)cOo+C($s7a#X+S-RS%g%0Z|+J9G++vGC*9;qNR+h*EQQgZ%r-j+-I(w=MX zkRqGvyif(`C9k`k$e0SUOjO(xVo2h&T$V?L=ZICSYOlMWM}?bMGSX#A-5D->Q7!(^ z7Bs*PDQ#?iLw8Vco4|aSt`N?8Z?%Ef(8>G-PO~^P`6vLabh2Cnr`lJF2W=h-L}xf^@vY+aIg{IC6*$*r*%h5V@hgiyR3%q;be_%G8S6gXi3typ=n4M zJK5h)-XYh~%!Dyl{0umxqNC~UXC@c-FZ<~6>n0IrLjz(m$cru-TL`{w^Y0rjHQUJG zwgB(D8S4Sb3lQr*HHL-9g=N$eGjt}$P1%IoY_N^7o#R<)jBrSmPNrLv;c2I9uN2Q#MlAD)7=9NQ{zkzZ8nwg{thgX9py{-+ zZuDVOd;^a=B-~5qF2~TpabzDSG3jFHu4EESL3ojKX;@ddlA0sP_Z&S>+ws&>kh+l9nk`~a0%tjk53ShpJ zWt8t9cS#y#;~iBS{PWNJ55u6qGN8B>%yOZ=fKEnAH=B=;!g{(BSd3?6LjltpO2Cp; zTU)CT>>RRi(jBdv$EqD1FHWfc8}wR3E%JJZA2L_^CA^ZtA*k9lThF2vhX7Aahho7; z9=l?F@rU~eQk5JT}UsO|6MUZ*P zfQ0z3Ox9FJN+zi;8?BS#EDtRYe3=xV4AjdUr4A_UZGLW2x5ECE#(!m)^Ha#Ai=TJq zj5PhpNW^u5%#i@LOVnWsb$$L5bk-n}iywfuqc1fT?lEdg*j%D2STdxz<)573Vxrbv zDfmjUsnsvH%0eJH;Y4n#?+J&~M*#5nf%6<9qobns_V&a>2~KwN(<={$P?$4l(ASnT zC-g!$jfRqBo9?6?r&&CJAm|98`<;V>1+p3dPwY^!aJf`|9~>N9?F^u%qJmw{nj^Ky zQFdOb{^64n(fsmg@I)`!%m$e`BIK0irKR_JZ&1E)sX#AM(lfmyy~-Q5vjqY5_w5r_ zCv$D-GZg-uUmm~=d6qac1xOoE2m=3q;joz~Ns3D6$18r$k_o{U{x9^!yoaiR)9MOT z(f;gN0seng{O?Ha0HXWE?Q^r2S#U`a%G(+RDwWwr020|f2xnj*GkIV@su;>!j!^BF zC@2EoCjQHJ{{N1vJN~!%nTvxX2n;&f{P_yUMuQMES7VKj#h_W)MU`7uTdUXP5*C;O zU}PPgnIHfi0@==-1&{xQ+1_rr{>A;_9tnO=Lb52-4OmK`oaH}9MV+OUq|DN7a*2(P zukt}Sa?H)mxxklnb#;g2UcG#&(d>3|kG%E$@gvP`E$F@&1HY__N;bi6cPw|l4qC;u zxVVT_4-D^@hbsFFFW~4seQ7~KpMkWyI}d)LLmZPn>AQFTrejwn`VPpDcyx6&H6>a# z{iCDy*QcALq@;|DjFXd-w+Og?s}~OzDk|!Q0*6LPNlB4f$(q6}5NZcJd?baZvsI_Y zN>vE$h<@|lJtECJOcQ_RK@50q`X@*mRAfUZ$t3r#HNn`%>}96GoWpQ(>pFJpJ~CKQJfr z@~GB{su~+_f~eEl*?=_n9rT?+32?R5BrPo4Oj}z!9Zy|PZ)tAM>KCbiVaLy@sVUe? zRp7X_l~q=D_H~7y{fe!&O^i)vF#&-^Qc{xO-LdF;Sv(o+D)S^71iX%-a^a9upl3|_ zt%LVN_a{r9@QTNwyk*dz_5bnX$IQ8ZcL=`YV=EVha;q5XU-V(f#L*LSE7IUoqM@RC zUvI|aapKV;8rfg2{-0QY5l^Sp<-ow5)Yn@>UqFqOP`%fXTdzzatTL=dhwL5SirBN( zm!a20Sh;FDQFN_ypI>{0%y(#jK!pn*lWNm|8>P2~ma!|6og=P^%0B)#(5dma@Pq_@ z$fyD2$&)8Q6AZ~%%j@2{rk;yt0e=b2pJEg3@rN1X;q8&>!}@j`;1V?z6a_d6kaR5f(SCl<7=P+5KTh`h!h_UM#^oMPXUn2yeu-2q^19|DUPQ<9 zO5wc#GMGhCsDdw$=#0mtXQOE@Sm(EoSLuU#e@}7~aP5~qAZMPO_{=rXAHNiH3+&~dgMWbuFo&28-|Zy0S$y_R1(0n#smK3{N9 z$ReJ>ibFpG0kb!AolA)kBrd*T5w}9)qg2`4P4)G%DcmRR-=DqE%>bs|>Dd`D&unlh zfjNFr81^-uTR29`a8=s4S4aX3?mv5g*&fK`#S?LGeq%2gtcE372ym@8EYPvn&N~Bt zg!1jSy_57aupu6|L9_#8GN#p}FKSxf2MM6#@bGXm37W@=IWpqvIc&#S?cj;iz%VS1ZqY{E;#JUl2|4EWaG()D5u1^*Q?*k)$> zJk$1YseO<1df$4^iQ5z|8kL zh-;2R`p-b67WWy1|KbaaiZHWG81-jIN3oviKaSSBmh&h(qr4K>8_Xl?-Uu@M6hX%= zSvG6?lSUSNe0-emto%A2*S_R=rLutRvi-`&s12|nXx!M?pqQ})Fc`TvSp?C2+{Oc72+(-` zH&PG`Yly66mrpxejqwnzW5TkRZ@lmm%#O+0V0bT{b^VlqTG4=A zM#<2}jQP?~QX=Q(UwQg+=(W@S^4w#s?V`|(2oJ^7`16e@v!Ifq$7@mECIfKNM#S0F zpWQC2#QJ}ZrGH<{3bplmJa0h$tU-)$@#^t=yV(o7jQ?|-4``y$Nf(ZQb}=v0ZXadj zPS1aB;vC)P=?5aIYc{xw!k%vK^JktM{DTNGp`}Gs~+BEa#e+#$&qR^w*>UNSfx&Pk{%o*nt zUr7Z#s)65c{ecGkhd6-U*uMY&!x#T+VE;eY@0EoH4f*-oYa{ML@>#T@i1P)tMRPF# zsiB}~gdq3J+629>>|9-4p-^~VG_|(pt*Q0>sr$;&_F_qkZh0eM1BEHD!-#ptrf?ry zN${@As^;eAV*lde0__d4+jgp}d@8T$&`?{wjpjg->jP}oqDQy;&x0nh;N+w8=B6ay z34~;r3ODEHAHYrFjIf^Zv&3#P)dwd(q^H#&pXr?3UCvDO`GJbUNWln|w+b?vWB=x> za)AmVAt69KO_%A{IqN>Q_r>#;iIDy2GBR$v6}Zns(;$tw({ zR0uLiQt@(nuUx2i$c~5=%u3E5+D}WQGF796+KL?$%qzSq>!?#imMz+Y`6yt?ZOf#jN?w2eHzncX;dhZrzh zBvQfshzJ*#TK+x%_R@2w5g*Hbu2FZm~6c^O)`$7%k}Eu<~oIIM2~_T zdlPRjWFi{e={jNi#xrN%@6S`9sdFu!b^kL#ncoE^CnZtIrx1Mjprxw|3@g-;tyI6w zwmaNgG%e2TzCtNZaj^An_r?@=^w%fR9cnSl#Le~Y>*rLrUiXp38NwIi3PQT2eKhxV zHM9Wc26To5@%#7h-@Rk@INw=*xJLjc^UfU;6LX5rLUP3?zuStHMv`?swhf#Tja(91 zW18%I_LP2E@=M1hNi>ts)tHLU1F#i5%snt?-}$jV6pt?xb8{X5wFwFi4t8*Hfx%l4 zz{H+$CzlBK+ApcYQ2Sj_@VMiWk2=l~&VR-s-1<>4j`)nIsq@*)hhyU7`FMB$;~5eX z6|~b!=2LAr^k3CBf0qCNeYDU3-q_i3+ZrU)p8g+g_@#Wi&CcH55(3%OdqXDJyuAMm z1yMJAX1a&|P`&ozFfD-&HnAK`^c>ht=@;^Jb6|dMF7UIEP&yeUPGnP4Q(OW)x`oRg_Jbx4b+XFdD1~d7&xr7SdH@o>m?9>Wr{0%KFsfG!y z#RLoHK&*X+uy65|b@=0@wWd)1UBI)vUnhI8Fe9O+rUv&J$WmYfN{{(nP09X}-)i*} zC=&__^T!?3exN}Z=;-^b9J9m2rA$S@cmNIeyfmtC1A;V z=#W1O{G|bj=9uT-9(^Qe)#d?XcOHklg||WXc>&EQbHO=s%^LMVJmsb66O@eGuG&(d zW0QSb(2}Fk;K_M@fx1bJ22Q}?OBVsi{4s&dw)l=+5GF^fJblgA1AU_!QcTKP+uwPO-9uWYY(7 zu>pSQO-TE(9YBrpNrA1*PA0CkpCzjk?$l7}yE|gD15G@)`4^$;2xKu7Yv$m8SZ(wnz?&uhWuz1DgDk^H-C=Ynm4hVMXEpMG*;oMQxz6O^HiQHi4&J@qr{PbeFRNr+P!OeW&&Nk#N8 zdUJX^ii!2pE&pP-1NpE4jJEF3*wxnh=9y_98miWxCn)WBkN!VWZiCtDxI=U;()egg zydc)uTm2W~ll5Iv+fVcZg}#*9YlFDBs2Ku;Cv8ciA?*M_i<-%&53gik;jkO&Lm06GqY>H9R9jr&Hm)lN z4+e%7WvTI)&><`$*%rwEipr5l^t54UvpF-&d|K(EYHHwmdz-jFjS`t@fm%uq4$9HT zH9Ef|FhVkT$GM_?SVX3pB~l(T2kTP5Oy&b;vJC)JVNq_PY1V;bS7oSmXU@&Tvoa+u zttUM2aqyHw%wnG6fr3$@yy(=0A>_gnYJlj*#nnGf6&pX0H47+Rr3AZ4Y#3MU;z!9N zD=RCMaU@go4mk&m&0Bd($sX?2lzvSuhipI~@iQ|R=T(;ih7H9gpRE`(=70X&s4%PZ;{Z zrR$6qWMt?*{J{4*$x!?zw;ZS3%|h}RBaDi7EHSTBB!}p#EIM4JYQ<`sDjGzZXv{h! zhFo`$zjB-5L&=X9=pF8guuquYR4Pw}49zuO6M;e6<3-{}bq%>KL2h{b7% zBzsr4AEhrWsQ%qIyax^*^!eRiF7m&= zbV~-3>;y@IMj}99QU`y=qT)mWJR6TeTV8 z8S;(Cv_xr&gdmyYLg9dbkt~Lt5eq`g#q0%!U`CAtBx`X+ zJhw^8k-?2rPl-mAf$o+Q-0&Txni|vXXEL|KZ`` z=;-K*3Jy-)sF9J8&dyG0Y3W1HmbN#a)|0sy$!e*Xu%P5>wn?U^Ps=r#tArmy_g-#c z5D^mxABp2fCj0D6nJ5H&BQOa-qw;rsb<6?`mNuuRxvZ?L3JR%3Vsdh9&Tp?5c5fPh zQ6zNdKzzd$Yqp)K{e*Q({5cJ*{}nq|ZxvUJi4+>zN2}+rK;62HWFBiHo`=v2e`fr^ zs1WIZZ{owy$FAKU61dANE37)T%zS)qZSQYYP@wS7`-E7Cvq3i_k=xLV_a05|Rf^dtT`w2rd_gOt*Yh{7zPB z94{8!uDXsA(_<=T;4H)ln-@93?Dn2q$ z>9Q41AP%`wvk>dIngL2;IuV0i+)nDTlQBEVI&{HbC+2Nky`#Y;h67Ov{P0AhS_s#_ z+S)DCUiC3u4U2F@PVCNzIY-XIeteFBiArUdl07(y6T&iZLy#N_E4~;XFrXEjixbK- zm2`K9?(^IcyV_mFBhwrCVP2-Wo8c=m zfWsFtIq!M!`JDgBYN(dqULz}wmwMoBXjp3FmcL|zCEzAo~D6<~py_ zM44H>vJjZv6}8XtJ}m4$6!im$HvCVITUC92Q8aK|9OEt@m+VGxc`i)Sh}f-^8x3oNM#=% znm3Y@JPnxuP=dxm-`E zP#TJKzT~g|pxY#T(MPjL@c>V6{i3?YkIwUhmksy+w_6Y`33%`$kE>2#*#-gg3zlg* z!J$^OFRY@R`&VL-`d20AnytlCZxLD1l=Ki42dbaV)%)~Eq5ET9>x(qc;KM!#KT9$w z_lvbk`1-v}CKa~7V?nZyjn*7Xc^pYCM-3B2!I%A){RP09@hhD`zt+e7C@kI3=B@>AH1&&I%QmjNq04&gV$ zh|4pJNIWP?6HIK(M?(*--hIxaJ#s=y;+EZw8OkeNpW;5wK`#@Hv93FiEH~&wPdhu? z*1}dyOrdCs?_RWym>4(Q?8qlFo7w9vuk>9=_9^NnnwomfCp(Xi_P9eMQFA%7Gc(@R zSl#boeXTLPW@ZVR_5B%Wmay|0=7{f3ikq&tN&SHoB6*x4!|#)nGz2B1S|s-~u{Eu- z`jt;{!zm2;b2)n3cs+SMlP|mE>H$Z8%JMTX1UJACC=at}#x$;Pq|&C>engYVwzSQlDoWWBvU>9+T9m068q#<3T| zWWWmgs)_};6t>oj{cut~Nm?NxBq`!l8US&{GPj@w6^&{N(Q)ee4jE+)(b8j$u)`^7 zr0??dfV-d73m4`)7!b_<-0w8VPgY3@QEA$jz@h`(T;Jc_(eK{9(~QJ`%xFt4lw62I z$Get7BWK|}U>R}>E>5q0DLI#Fs6b8r)tZzEfI}mN_q%DpUvoRYC!OJ>dU~U*=C>e+ zq1`i{KsK9Klt==~J%cLkL{kNp%dAh~xKPyH!|A`r`h%La#ol-(4sv{`|C3^(`c*co z3#fa0j6AMT^5xqvQU?QFPy|niX<~F;LWgLf%dcO*o*k@?)3599gGL(((n2thvK5*4 zX(B(eOLU9W^>* z23nwA6khkVuGN+O~^>+8J(19J^d=ANFO zHbJ)_yl5a=z%YW`7}6Dy%23q0Yhynp`0WLd7Vy*4h?yB=xV`sMl6iXYRyjmPeK^fD z0>3CpDZA7-vh0mPs!*POn8LXSMLk6+;bXt~@$vZd^sAaO9~O1=8~)vLU}kjkv}ZrC z4zG<;%$Mqv(@xObfM{=j8%z@#ml3;MQ)1Jtr=niJMi+OK?a_;GT(|y~r+pZ@uS{7O zas+(Cd*42@a#B-+4n%If0wdr|JdJs^uJHxVw3Kqqg8I{)c69P^_Q(J!A=07C>u3G# zq6E8w;uf$yzW^I-!Wd{uXffjP8*(^^F=DSl*;7PxZnTN<6>m^n=}O~X=$R@Q|6(o_ zBp7D!*UawaLz?IZ~F4Vz%xSLKl4(|3yP85=7}7nEDQ;s2ELyC;bF7%~zVU5FY{ zMg;om^bv>Vypoi3WK4GMtU`Rq;p#-^#%)#{?5bK40$-o(7B_JuQVzr=yMs=Aub1>dSVENW%9l7lJF zPiBgnY$AAz42=C@FlQvmgW_6W1X_xXt@==cZwgN%IcCK#WTA+uGj~!l*?jo?;wi>P zNxwV8#mQcg8oT-7_L{{Z#rLwB&T%vH>j_9a?0y{D_arYSw}%rHY7)wBGDB_a$|WtE zO*+4crCtk?O3qE2&Zvz$m8-mdD8(V<=9O9F;8f+PTS)iYGWtOcOQpf}Ct8e^iYOEwv2dfv{jPcuHPDF=p0G;+dZN$aph@PC9bp}%e-Wd!fd_XceNq)De z&FAP_iwnF=htG_xe@NKVleZe9(%kOrC2jFqQQe+WK@78$!qXd`F!755$*NXixA%6TPfYuJ}aN-|@o(~5ji(W)Bk$M=a{x)Y=_Pc|#Pp3~ zJY9Wx?OxSPd{EqOBmpJ!9U*%C@jM(3y-7-1Z>Em3dz_U^2AzdO!IzZq+u@U8xL!GnIe%zws7Xv|6!NjJ}UU$8R zOj=83ZSgYzfsW zfW?hDQ&cB-9Us~qCaO$(&su657pk4yO@+NwcmTI55GozBcr-h1#-efk&DdCC^>J4B zM`iayl0o9G4IAd7Khs;Yd(|-6wMHXk(#7ShHjX&i4{R)~kB5N(^1~+Om9*rRbuMeY zYFu&kHH%4^*>e5sp?v%H?Zbx;eD?FYaBq*|J3KYMDMtd+eoyu79)VD?2B3P1i;JtP zt3fbp8gTU1ZXCZT$LLCW1Kg=(ixwc_Aa1blzz6o>Q}j>9K+NEFcwzxl>TB4)Ty84j z`k%#Inz2USYf)c6oR*BI{;&@156iObWfT7O1YtD!Ao01Xfpuyh&&{YPAU5&UTL%Og z!UH;zPU6g;tt{>d-3(1sNnZU|EC5Va(X?vOrLe8C;&GhN#1nLxtR9o~D3P|7F~nq5 z#WTGv?Id>tJdKnZIlZQzbvFFacM%|l;}SdFW_gKnSxCg57~#~M*FN?q>AN#`a<}`HgTiu9S#~9_#%xeL=3(+*k?MVN0=B zImhJ+x*b2(gkPtw>PgW01?1$h;b+{0#N$vqo(8#1;356`Va~xDCABOGxkA$KP20|H zGMDM(@kNX&Ab5nA*O@W>=y$XIn$2#p=~m?yqkWw_37U3J@1&n@Yt6Ufry(QSt2*(G zIxh#w&2prj#%(loa_mKz%sMwgb&SKqFiEa4T5gaPK);YARL#cY zt9oxeir0uCskZCw&8hRmVmHie?au3mC?G-P>uGE;Th{I~-ng{+KgD zKLy|0z3U!I{Mj<^irH16owOc4hhxv}I<3E}pdm!km6xEWw^d0kHp}*8`c%VAZcfHJ zr#*G4i7R;6CiG)p!)`zNx0&|l@A3^k`-L>rl)>*baN%?G?vy|6;(6N z*QXelM(sPQ*v>iRXfF|U_DYl}sx(;E!C;$~YP?E#2V9G=35;b z$>yhz25yz|TA$g~M7fmoeq3{1TDin->U~LvSHYcXHDPn}j=y^rDG|}#HEa+IL&|Fd z8?lJ`qwja=GO%^gPs>+_+K?bOASfv4&Ye5<^UV!_6HX%gF>W@;giN7d{kcsy`l4#z zSvjW|u$?)1dFH03IpE07{^KkNfShur?@YpZH&qOwACkaLg#|la5$%M^iatw-_(;f$ zf&x$ooKWY$fOa?z=u>ZTKb}bb1N{zjn~t-JPF>g=ON~3kXOMc;>Atu}3&+`HyZxeu2Px z{}zN>q_n=VVJt)f14zP?#qa#{Cttq|m#*iZmNeX1v;6b;&R5gv!I@woSc;ufk~qj45s~b<&z0?^e9=-jhQWVr(nfAE+Q-|JDQf4QrNceT4Thh z<#As7<9jvj5POp0bfqbr9{3MsEyvn+c6M4?T3B5-sqoI|1x9psy@y#o5d)Yj@E3rp z(6HdHn0T+*Hk;lqJlD`5)FKwo_nu6RmVJ)z-H`v*{VAdUE)lTnBgh0|Qc`9B@x0vo z2(Q6msZ+`CTm^J(_r6?aI0pLHO?DV?&A<_PtsdHepS6b)_YQ5l-ybNQZr`u-K^_CA z&BevVp$`?)*YDtYO%S@cxVX7#llwPRLBiGb7+CL8v?3xR6CeH}oB_`yF)%QUL{PRh z75tB}2)<6qk4|K24cBk-j?xs^xfCBxlhR{9}R zg`Ct*%nR&t`yZikR?@AX>{jQHvvZ{9>(4Yi>* z#pSh;JzIhJfh#36=q?U;=h&q|p55hdJ{s!%-O4yhAD(2b2BgJSa3gt2xXR9g#r{Tj z`4Mt{%7l|g;M^7-5uR&@(9x~cBe75VF~{MeQknI7!x^Y=F#7DISCMsgEf+I0)cf%l zajo#(TN`noPktZ-9N?-?F$Wr&-{)6j zMl6%#fbs>_EKm>)J+#V`g<~Jz8vg1G9F&NUCc}c>5TL|xqZ!MSP*_EAyJIFO-+CPC z5pWewKy9VuWT?;nY@)tF>$SGXCDF;s9bWB2x9W`Y-w2^Skytl?NjwC5_K)DOmrk{^ z<5^`~D}qz(m)~-qvYB&bYVXRD%aIO!Df@_h6|bOVwzJNMW%2P-`MH;BcbKtfaxx8jP}9jvMysgh)~}D{&!w*h0;u{;2nt z{QRc?f=-Y@r#j2W(+clHqey=kiaU7W{e&mP$c{5U#TCh`MWV+S(MQmeQSpW=g9pnmuvtYa3iCZ)JtP z7LVE@;CcrE0L2hHnRSY-L&e`rl|SyydYLZpBnOepO&v+0(A%= zoGM`waFR-qh>oo{D&ld2`ZBN{{)NpPfxL<@P(N0nXQPU`}kpsd6<@djAVtcfYxCIix9SVi$6wp z7brCC(l_m7k8mVVy4VlM?&vROc)-N*qWTR2=)Sxhcq^DTc=9H>?$tun$Jc<>LD5oZ z|CuZr-8h2hjnWDLWn2!vBNpfh$w0QR*f^*oB0RjqQ}Uj8*(9)qz)@hh9l_NL_^iij zL?HhVbt}n|9Re=G&ky1{Mt~Ee8Tk|q3m4a*(U}3LBpx2M_n6>*+Q0X|_2P%PmN(Fd zK++!wEg~hQ`F4weg~is!#>T|N#MxQ(f|1xCBZ#FN1Kpnv^tJm{(+$8A8Hq%F&oV;@ z-RA!%QG^CO80yX~*P;X0yFGvYSVR#=Uhf~Uk#Pf`7Qz9$4&;W6FRhIb1rRQa(U zVvD=~8*bzeGn{9&X*+o;<3-yOGJ9z2jk55$D zpJ1QWdO!A>-#~EBrNom&FSYJ3j2HR|@^P*l+X@`lEcAj4 zkIs6n1l{`xoZsLr#2u9O$&mj2YEq^%nJnWp&jk9aYc>U-#NsXx2D%@)<99yCX3CbF z$oXyQVkU?5%4r<(7Z|(`P_(ZKEna82dJV@ixXj5t&!jtNSz1YmNQgtn@?)e!^y`?J zT#_x1t?N2PTX+gZ(-Skf%!r&D8QcbCmtz&6dmq4HzFu?5oLsYuY>)vD@8HyqLl=ovlVeH0{M1X+ zuTEDaEj*?p0qOeAXxC2R@=a0Z8g-oG2*`(xeetyDPg6dSNa>Fs)4weSjeq*NEV8>kE`8a2m=@Nk<#ZDQ4n81%I9K4JoV6DEy>+1kUaN7-l!)swHzmZI zRA(I2T%NlhRfZvSNEAc4ASwn24od%zb+*fT19KS{$geE-9QL9&XpFJ1&nrC3!c<#I zkNZwcve(OIu^yWP8;J--7|yq76&N7juVM(hfScP`%1`VQ?V6Wb+By;wvdj8#4|w~5 zl>|(;D6DjC7q|KMsz()#r`_ZTPRkVG!hLDPaDC z1v+0!5p%;SjNH)?E^9}+LXavDS3TnNl~sq^O{-qHAICdna;Fh2j#GQVjmp6=?=4Ks zodLP<^QU0W1HhQScsLh(CF`4TehPj@^iZF;mT7XV%L7w61GMJfxS$u*YGqzlYRAoS zp9v^E7Fv|Nt<`lDFhN5GI`9AUMIOTE4gEYG=|UR$p^+Bbt`2X1=Uw}%`FKw@$K|P; zZg8KVL;bd>*3r3jd4x|4A>Q9NHRT*MuO!sJnwYK+|SC8m5!(MCaNmm)<(Ax7yvqoBFv0@DRsnNY+A0Wu~A^?iAxzMP_(e(aZ z5*es?gk2CxsmU5xs* zdOEz#-m25^u~zphvKaPZ=DY!YA^;|^qo=!8=Yx!sYAQeXO_{psn1WqrN>R9UW}cExshHZ7f>#=5a@SyjiJXxqxVqA#A;bM;0U`F z#&_JGrl1Dc4b`FQ8|u~SCHe__Uh*j7?ifbnLE`ME z)@W2o%O)i7CdQeQL$Jeq2Egw~DxlDp9<_Nt4X|GEhBtl>$!+U%vpx9-xNjY zwtvKeAB`j!Dq^oN;{os{GXjGunmILb|G=K`<9R8iu;&ggMGIedniFC6)!?~!e%`0P ztAn38*B)CVnvpjk>PR4x|NnQWP`Ep8qHQ8@e(p}KTO8VL)@i+b`Hsz+B7eZj)+wBF z)C7yzW-`Zfi3Pu`Tmxe`J+m8hBLgcDUbAu1CvV|!2h^a7@YnqN#i zIJ;E{3|VSq9HBS~RweU*ZwF^rtSR>9_SVl-g3Lov2olc&vBRA@Ew^bhTlLmX2oG=V zn1@c&_7aZHJN>gbo#m3(+F6LesO?0}tM`uuaIcz4TRQe0P3R=;DWC2vX%xK~{4+N? z{6_?4U>#mGAb3@R=1&CHB8=-qefw##NMT_1c0~Gz@f>ZhX}7v=u8>S;URYyQUu{w0 z>YF^kMHsnmA4^&tG?_WrKY#iF=@?w+FKBqUh)zLZ_SR=HeOu{kQ)MO!=mXW z8#AV@ncNy71Lvvd6jB1LYOy2*50u?v*Rr)PlLc9}yhI+H9jUE%0{?(5|5|Hgk;TRd zYJfG)Df5B`;9z07&*GB~Wl8JEeY}0E=$#vI!FMa2qN@EE{)ArUascTMWdX)*E zoia+{6aueueD5tcZ5>xB;^7ro`!uRbW`EapYhQcFvGQofEiuwBoG@*X0S=2(r;Iyh zttp+?4u)xx2*2=ro>ys>7H+z3ezz3IYcC=%qy;g6;S|A7(s2KQI5SE+ovJ9$?Y8Ud zxJh{}M68fShdyi0X)Tk<_FSI~PdVY6I&nSJP!AL-6e;N)ePC9{gv+o%$s($W*j;i(RZNlLZn0W^u-BurLkRolo(tJ$zZbkG#?TK{!X*6dY+0^^!J6i79RU_g>3crB_*5IP#BOlGTL4 zL727+Z!*iG@WkT{IVrh;fGw}QD_S(j>;X`+i_s1T5rSS)EwANw+shVqZvR zmo9wHk(sL>v{bGO26Yn|og~ktcnO3uxRMlMo`hA(cnk$=!WJwin`?7_*zzjR>PM?P zRg1Rq`0-Lc(Rq(6yXh7TGu9UkXqR3Eb)no{>F9eoyQ(~?ToIGgn(D(2z`y{Y(Y(=Ho3U>m zMR2vL5<%R6cKobXk5rGSEAuHZEbZXOpep_?S= zX1wI&v6H_KTAa-WU(yF{W{&!PmIKuo>qEYbdSvnsH3iCEVaO{P(2CxJ{GS3qA0~4= z$Fd)WLVDWL3#EhNRQX_Tb8FiX=ORLk8fv^5R^>J}e^iAxdOQ##)4#O(egfWg-foZk zMzvgg*>%0Ib&wVyIcf&$7+va;Q-%8F!dD8XOKTG-LOn`B3yQNlM{^yU>s_%gA3^x2 z_)g(Uve`1$Fq_%ttZnfi39wtL-5MmnJ5<8$(xj7*;DA%ke~_S<$Gv&N%B|N}@@$jb zO1V_1G}o|I8dIsA%gw^TYGu8gaA&tfYi_H>h=r4Uq$fYS(a>1Q;5yCiroTHXUWzCz z#qHsYknXVUyQd`r`Wk+YBMKINGG1Rigj0|!FPF^7H|rBKb`n$40OX&X)(_n`2iDeN z^qdfK!^*kyK-UoRA9;=yEg1&zh#L=^~@11`3dYv<;04Ud~DB?{322yv>ueNnKg- z4xogig*NHbZxAz+Mun9{ zt0Gg=!VKR8hJJ=l@(k@VC1dK^PVCQFJ>Jlczc;#tqGDa+QxvF;EcwTar|;rGd(wRD z?Dt1V8=IQ=Y^Lt#3{wraT%B(u&zvQ_u(LxE6_KD|E-o%E4$geDhZD$k<`Vc^G|;GV z%%?P1Gxst#)@*q#TTTlE=Ey544fXcQ$;$(LeRvJv0<3Q`U>fd<_j22z&%qOz5Z;63 z3yyihXz%yTK6{HD znz;0aVG$AQN9S*1092v0`{*{PUStY!<@`6KgC4-n0RsPD^A6wz(O(o@b8~aeOiW<1 zo}TA`?SMu7rcmYo{)eDHj1wR-|LP{*QqxM9B!0}VKB<6j5)IHt)PT0%eKIT^ zfcD0<{1+;5=zke7{U762{u5U8zZ-%5-@fo4;8U-TeaoXgW6|?JS$JY}|FFpK8-gtR z$X~<9+jFFCS!mFGVGvRdj{k;HcwGnbG(q78l(Fpp1BDo+4O@gEKF``$B^TFTY2t!V zbw9AjGm9DXf(2GDYbRmG?($LduG3vm;V_;+i zwF@-H@)DX-6ms`njJF$syF*KskddB2w_ zPbovx>v%KfiF~HBG@yJhI|DC$8K$#PGR{XnMZOWVqtG+@yo7Ev+tMt)W?=bbaBnMD zk9P>uO2?^gK+fO}M9U*ld*?<=E-m#v8rKL43{3Yqp3YAn_LyJ|5Tffa z!2|Y3$Yrq#@8r{vh91=?UGd5Iz#1mR^V&*YR7YDTCRvesEt%caQ|(e#QVil?B+1s( z<$__btYKBSydbU`iIn7K4V$Vnqnwj!D+MI3fP%lZSqG&0k19Q*Dq;%a|Ws?-`}enIU1O z|0&db(v|O3sLT`%K!iKMzRCO(&Nt8RbM6cxYCz^7$dYXJJX$9-@ZJOGVe`aW=4sR< z_a;CGD~&_=wDxz#K^=DMM0D5DclW&16_5mwI45e@OF>*arJ(auE-vu9TU(JL$5XLN zqQotqQ(N1qk3|#X6yqhVBGp~S01^Q%!D*RBx@azY32Yo;l90Q{%Z>e~Xt!p0^S zbe5ImX7;XjIYKAH8S*w^OAp1HL)~1)%zNno5whlKs!OS^gzK|fazug>#Ei^ z#Ss6Vw^CrRywus~YKnRy=9hJNIOl=YS&7jX{Ig{#$dQ}J zF%%Q}B!QBxEM@TXb&!;r$gH)WMS0onxAf!3tA|6{ne&sGv(+kl8(o(m1kFLqjx~i(bU7OJ7eSyX1Pt@YJGcQBw z!~OFcNbKe`0~>Mp$B!SwSu$W*w}3sjn&CHpodXERm-E-4AexBJ`JqiX4&=Xv+Ou=h zRp#_;JeeTywJ{K6+w7sBg7v__K)Rs^TuxXJ%xM;skNZ!IB8XRkm4hN2oqsWMRCG11 z!;mo~KJxj$x*P1)|HhNXjDTQafSd352j&jmXTl7!Lnb5uejoGiLuLN^m;FCepdTJj zpaaBZ7Es_pQEvd@EDOZ_tJez_S?$9FRsnbn;N^Hr$fYiWwqHQw*66lfJkAExemfBC zSuEKYG?hT0QA=F9&2G6ODen4V=H*_ssa#)ydl4Tj=br~iJP*hU$B@w>dL;cFg}3`( zBgE=vM0j~ukUD=_(J!vM3;fPNdIV53ezywVM&-Y)2H%d^)D8o|j{e=dmwTjSe-Bd9 z&?6YWzkV4o0l=YreShA87l7d*7&4@fB5I=;!(b?D66;9m*VarOh$O^(7@^W+K-ah_-5%lqYKDBiG7BcHSIa~Yl=MSLlr>2PfuHVNcB=CZRlao_W zkkL0DOFZBk=rF1yQIDkK{;~n*!RvtFFTNAwJ~sQdOUzf5-$s@HCusjIEkj8%Fk#>8 z=j>bt(I2ByHuiTYAK~CWir{4=vu)ELibh!%#1}N85^epR1B0?LkN$D#;oD=>AmLv} z1U=5q1i3P={?3H-uo1up!ID9xUciV}gxd+c{miRnVv7GUTv^T>fQSUw{{z1*uZa!; z@Mkj;k=pDq-(KDO%{td(ash!{D=8xB%i28Ks+GmYa)oFqdO+Jhb_cH;Juw(n=!qr2 z$~(DU57HZ^z-NDu7ny`?id-`b;dAkQwBBo0|QySTy!1RLy#CIfu^x@ zOMwU}Ho#=SX>YjBjE%V$1pHO;a*EX24~rtpy=rukcz$$!LA$trAl55%^!<1RWi-jT9I8l#Q*x{Mp8K3<# ztG1S-*o0vw)c4u?iC@)U+5ToGlIeRhvAG&<2&*wpU_(j_tO38JLDnIma+H!U_ z70LZx`rD>E#@FG=qdlKgfK2ljXS*yuDf69rG;$g9O(+!B-&P!^Mew#{UaujEI>rG0 zJ2AX<&sJ3@mT<*P(dHH1(;*iiEAQF@6X@C7I)S0*|Ka>;~tnh#b|>~XO7_AK~) z_w-w9M2%X|M!nX|TM(YjznF>~%~qZLmGm1N06wiVDebMp+In5!rxf6@Oug-f287`E zT2P2EIho{|Sd@d3KkWC0 zUoq|8nk8Eb)zMHmlAgSip?HKJgGyYvq0{2&A})6f%(7X~LJ(Awq=7Temb?8t;`Zut z!x6T2gX1ztV=xxMfWaOLrwhG&HF(k#V{d2|c%8;-r$>(0kIl?~gX&#@#!hhe(Z(dG zLN{GlSSWTmD&})Az&9QaTn{+;{bm>f^X%DZwVS~RZCw$u z-#QJRs8?>qj_WWR;{e1@_6?JI@hi&ay-reKf)Yf3`SL{~2$FR<`zbny;mWuj7(Q8%5_72{6KB&8n7g;(pi^h#wN z!WX%mXpp`fYX?HI-fP^aOVH7vuf`wHVShZgl1NH_XSoM+|JJOtmL!llg5!>~vUI40 zvzZ1=nazqW$?W|6KlWEE_8_1tllKjk=Y&Z8;SRixH(TmRl$7jh;)*Yem->xhA|m5$ z5XSYOagxysyb0oWxFUewngT7mw<#(3ow`Sd2*Q{J8=U0`*3YEFOp24(sDGn%=f5h{gE%I zP$CB6jG4-s@i&H_1eYa?<%ynN;WpKCve|%Wk0ITNlBzO`jS%F8X9VHOt$Ex*ATVv& zM0hWsf$ieu+r+YHEOC(~LoB`z%4G<89_Ei*9r3UtS3Yq#xtZpKbjyovJb6O%j)+n- zPQ&i-GBQWvK0er?k~P4p1hvh;0j$b1drA@#&!up_v94Je0W92P0;Pj>nZ(;PUX3oe ztl^tzXjICmm6{LolS)N3D~zPf%yO@jX=aa%=tS$dHzs@Ir-k$s@1FdSz41>~NbOSH z-rgS1q#?a-@i!QOws160DKS1iSQZ$>z-?G-vMbBa@1As6(CH&-n>jnWliQjmb(5uE z0jcD7kuxozk;)J1DcvL_1Cf82&xm2DglNg5UZygepVd7U+#%gyjzA4HpX1GG;0<28 zhH4o2@R%*hYZ_?9ACz=qntj3ZFf{uK#67`?UJ}|HUKC7O4lL;_B;p#L91qHwpc7R3 zg5B>k(0jKBF<$a0I;wKNy}w#*l$Y<$X>Xz zHXiuOVt^eN{IccAEN}`!Lr~7P8(2=DNUplmkbjD?0`Gv=YVHurzxC z=-3L1j#-qltWBf{l~w00)veyj>7px>eQyHHIY{zrnV*~Kr~#aJyr?6$W9Zsr+9u-8 zE@5=Bf?WCspi*K!_AR5Ml7sWtuJf-oZVD1JnGXA@$i-X`I^!%-cnKsc*^40UN0;sCKzR0aQ^RsMj zX=K07C+#ygYv1NG^SaXTn`Z>d`?>&-gg(BRD~!@e9W;nJ+v>v1?+e?5`9t{H+;pN$$gB1Vuiti1)`=*}r7xiz zmyau@#nr3V&6_ikgv~z}E;^bnsU5dRJ|FBy!G5##=)$=-zzu4*rZ?1CE_(}8HF0^j z9}*A%X=-|`B-m%JaMwzGVzcdZk+QU0cnBLCW%+(IReN?^_})gH_TD&9TpFdi#2c@_ zN^mj7#w?&XdUgt*Ltt&7iC5n1?#(ti{ToQ`b+R3PbRYP`2I{Qt!ijNMsL&#aeUg|` z^RUY{5~!B2Bk}tN^&7VU7!?)60kVq!v)_k$#>+&Qoj%6^ zoV2Gk50HjYt<9s?KV10+KA)-_Tode6Hb|0%ryFp{WEB($?bAzE6QcrF>$w03WLgE| zPgVed{khJ$T}&mflWEwCCCvn;rkkgqE~1hFdR46C=yt$Ae=-emYph5f_q&u_A=(B8 z_OAED7oG>`(rZBNc=Mggc_^e0v*z_*GuNCTc=6+(r@VuUKbThv-w+ z?3XI+U0MfvAoUDd|O5o}kGO-0c3zyYn$hJSHK|f#Y;~l5@ z7|rlpHIx?(SXo_M+cbed$|>u~u(~_^BG{z9l-`kkKQ`z77@SdwQcrF^c*{Nb<@}C! z1zc%82Vth>$6q(m_pVTgK&2?=T`SGKZku@(1jkkE^oVg!EEhZvjzg_vKK9Y?ruVt6 zx{`;_t%{%L7mE0&#+#K}`|3?iJ-@dQZ@E3Sb&G9rh8*7ikr`R9mwms}M+U-fqhMpW zWsd-%nxj_hd8P2vO=uuw@oe@$(l&Hc1MnB+wXJOzXON4%KGT4r3q;WHaWP$_- zzh{?K8roUz5yB3O|DJ)D(hx_8;^|h_<&3_HY zM)HwLfh?DfCJ`}%tfq=mwaXOkf25anpDK)>M#o0_xHX*f^zN{4x3b1ma48hy?%4mxN_9(ATgM{(G~QG6f&pP$m!an8Mt?;^ zwdM7a&ba-+ zLtA;TW&}Zat>cUIoxaqz4sQs5>^oDEB=ynb7Ql6a`0Y$?uNB{zxDzAM43MJ7BGdGw zwXEf|z-%dQC*@W`pbB2=dv-fZpNf+e9criDuZZaHCID=ka_*0&7r~yfy6_z)f0>(b zp%N04rN5JZH1U9%bZan_T7Y`dIX$YGYDmkdbmr&6R011;lVwshnNz)bHDOukn3_k= zo=*}JV3ea^`v-gzb&`4S#yuwNw4&LlM-aol)FkFGmCnkL^w?S)uL`Xt93pMx>Z3`d zDF1d{4Ou?sez+UI)fc@YLmCD!lo7c+gXDM?LcxnW-8M`dljQk@f?RtaBCg;Rdxy{Y z6(_7)9gI7!BuXmPY;Uph?Cd@IZtB%`kig&@CO_ciDHtJe&=&?0@c2h7`Tih6gHNyf|Dh)RaJvHbj%+Jm} zcoUgN>GTpT_HKokc+t7(x1BS~NxF4q={6&T8jIj3jGeuCA=cxyJNQm|_X!VP z;&&jTwKTMSekh-)E`P)obYyhf+yg(gC_Bf!Q}SBdv-5+1k;omzU#);L5Dm0 zt@qW7#`F1f4Gau)b!i((j_1b5)ee!qW;i;NE_#zSwlmxVmzS5GEux@!88Y)2RWcK(END+`Q41>6`e_^EmzjUk3Nj(IuQ1Acdu`1NT%3ZlgT#tkmAL{*h58IRYQD^xObL^ zxFNlj#T$Zsm4$^rYX>U(8BPr@1?n`J*BeVW8kzg99ELRzIlJP2dpYyS%rTIYJ$wKU zRfAQRRY_4Xox&HVeSXx5^&_Ye(_6Wj@d%re7{l+~-zKsKB&B}AQlR?y!$r?e3HR|s zso@L{MflJfZu5W)FQ0+$!VyMf=36U_Jl72ylKUGq_S4jA^&{ePZf(`p+pXFvxmkz*x?|+H@umu2Q@L|Z^5OGw+1Ybfr# z!k(WUn=?@}9hZ19!Ur}~VV}YYCmT}k(3P224L6x;f>P~GhO3NJG_5ho_BY}2a=49N zS<%$~!GYvoa=iM#{be<;PG2K1!UiQ?(P3cQ&%nhL(;nB}FcTAdhI9NC;*N3@HrUyS znET1w+S&v)VZp9ve4m4C4-Fp85_v~T3!c=6Pi%lA@{Od>+57^Y&-V%n0naA=A1B8H z&MQSY)t2Z@GKutUl=v-Z;vOR;UkrHH65gas7g23*r2V zOWSCgi#ckGA4;L7u!wxef9W-`6kz%_wenc_reOGh*E9yp7F#Lyiw^5e1)Z(-Xq@WGOksW#Q57cNr2YkD#h@j_Tv z!f}j-&I>;J%4?Q?WmO(Tz)0>a$N$f*KIsj%z3dj6^!`$Lv0}+>kJZ}=M>E19l_s(5 zv}P^PEQ8+Sh0{gkU1(a4xpIf!8i3IM5M; zMbPt^>HU1t^{TH0Oyn=RHSIr932!>U%x(I7FvqLijjp*FnZ!);-&fmps!k@92}b;7D8de4eR|+!myZu?uHX$ct3$Wc`z6gGf-1il|6Mp85c}=e#C<74`VP= zM3G(>lU9n0x5V>WoqMzjm_5ScD_3FqooihkM#rlRE!9cH$tl;ol$XjP5Z^4&DPU2& zH^Mb2uWXG7E-UE%S;_7!@$^FT@6-Kse0n%Psey^fz>W78680w<`XikdJUwvKo*!g&CKj5s~k=M(tI7wk$G$|jK7u4;c3X2=GZ2Mk8N z$v6?`K1qRzo8wTq$Hcmz!Cr47{z7%C)R@-yQ9YRS07Q9yZ&pvQQ5sn`+oSgnms+pIw_YFV21^QJ*J21lwfT?esblw zqqz9!33&Amk7Nzq;h69Fw#V24*Y!D-L#(}B17!g=;2^HQ(2LzXw00(41O@y`Z*|#$ zTEa)RY^d?PJD^lhoWcg%kzsYf;Y0u%Yf70c`7c@wTviw!Ya{k0RExpmLDH3YGAGO< zP3lsm!!EhM=vZ)z;7uTp0aHdHLIc@e=qHh-9$BW20a z@$gc>!0msGuLl}G7FublShUQ1cC|LQeXpz7Sh>ZmqKr<0HQj0|)d~_X_>qS7L<-L| z#=l<{6_Cw?@CHP%BXL4YkiS1UhD};3uAQjpUHd^Y=*Q#rmMIHdM{I#bibJtZ+CZm=NI8#+AriEqoO)T#3@5fR){Yg0|+kAKydi|$jW>M8GB|q{;Esu zRytT}reX&Iki4hSUlqo%`q^=S9&~s_&EJnezsIYYtr;OcwyuiJo-{5U@YzO93|BvMlzU%ehQxUoS%NbhJ^k+I#ed)trKfH$YaNw&8WPEPAt>aI&T*hAV zs8#@>pLv&g`j9lEigbqko1>tMhCJ@OS6>XY8W#{95 z;drg_#|0EaP{>JLXiYus;bOvY$*0LsJH-RY6FCMev^`%>YDKC z=br2)LjL<#=4bM}q*Kdmcp@{oFtj9ig$ zEL^9pf>c{_BP{rNMUVl-gd4k4-_&dpkJnEkFBIyj=RF%jBF_SB@Ahpb=yp30-oN0S zo}7H8`=pB2doZ0KUFZ0B)^5-vhUpvXl0~vVwfc#ZAsnpb4FfjfU#VCF0-+Y>S#i&h zu)ak5C5yx(%TI@8lj)VV7Pfw5h-~mbH>TZ5H)9QszG#4&)obk0B`cpM`SQ&N-fuXJ zsdhFQgkgYW2LC=4ee*NVLKJFZI!1wAlgyI7cIUR?|W$`syn&my+{sF^@wFI-0SgttC+}}|cptN;U zo0j`pH$tkGVhHmJkEssNTpg>k(fYgUq(^}*X}LO9R4<7R#x=^w(D z{su=-m5cc&D`;IOz&Z|bU@3+}s{Un_z0bWM8S?Xh8?4Wq%#0J+nVQfZLI3IF7h?T< zn0uKA!x;IgVmsxMHfw`cHCi$_fgB=B?hd+B6J}q_8=Np0WP4F#NKj5|_U4O`l7g+agC&p~IeZ%;S_+q;CjdO8k zjtd=01IUy|Dz9)g$wos1ybAld>bKE2fCe58!?mk)Rsh-UPDnO!ex-;@59&iX@JN;2 zBwVVl5>=_m%b0>-x6+>G8po{%73E$AsTGF5|brSXw|Ebf;2CKW6YFsUREH;KmkbuG5VxZhBJ`saW*P$ri=#FFi9ay z2YVRD0u2xb3KQYgg`)#roG2~RTj^DYQ0sQ5(U`1{y@MBA6an{zI0r;(QkEkmt5|h z5ZyLc&E+1L$hAe|=oTljmd4?{18cKS%vv%smJo7cEHnUs4E`KWDh?(yfYnmFM0E=8 zHPs^``dvr*PS!6%GNM#8_7~`Rz2kXq&+Ur$2z$QVtl@a_cUUj~<*g#&E9IO{=$Q$6 zhVvlT@*gKg67^YK$&@`y$abDE_LWF9WlAe++OL)Q9U+qAFU;3n8uBa{cpm z)l(aJ{BiM#hp+wG!gK8g_wcV-t+J&j$pZ+oauX=e@VEOiK+usRQqJ9Ut7-U0>(7=l zhu+JX5kFT6ffVR&3c>G3s3b5t8D8i?vyJW0U(lT!xe+Eb?#+r*Mbds?2PWlA`HeBX9NnL?AMc!zX?la}>a}j%@;053E&PxyI1?(X`R$9x z6bijf&fh`8At`>?MuC^hS9l-h-;j@!mFiHF^j?;5hTOeU1<8+2Uj)znRTm)P3BcyQP-#P-~fA^Q^o_ z-{;-|z1j&DkE|gn&ClGR7<_GsKC{MZyM2jAR!Km;7W5uR2^*~s5rSr({J_u3>-t>u zplpq~(Asx8O_w4Lb_dEq?W+rBzanWg$A#8!X{Cm>`mAj|U-DDtvK!X}&$~5`tV z(K3b2Z%YCuO}8EcL1=m2Jt3Kda`Oznt~+y9<=4s&Pcy&3x8$&%cKQ?(C%&V@%_@-3 z0yZ9rtRmTuw9MkAEGOTqADe}5f(%F6j}P!HOe(5Ogy6ja3lRo>R6c+ofLN5;R@4`P z-}wHUlBIGz4TN?C&ouGdc^Q|r&SaE`q#no8B=k}&T_>;#{Zk z>r?mF|W;Kkf4s)iqjsgWRP81 zgDo7u1^PP=vx^H8=?vcXM*{GnWuY7BA8|023CG^zSPgb1z05>ReZFKb!=GWM;oUpS z+~eICvuE8uA`e14YF}a#Ne+VTVD?l+?i5U*eL)%4KilygUQVjR%v9~Hk5FHY88>VQ zRK`|YuwM?u@en6<& zZN=P@?KoIv|Djy49u9^dRlW!v4GRg+C$8ZYanJ#_38SMOZ@OUd)V6qI%_KPpz3N25 ze0p3Q-z}a|z^7w!GWzd<3r74=%m>U@9aJwM*%bVJU*do_ued)4qGsTM-~y>eNtvRpkW8Y zJsm5aC!C|pm~=nnjuK7?mn7+L9pazCFR1~bmhhE+A!Q`*v=c}7nUw4};R)@R%tp1E zWZtgmIwLj4uJ5NVR4lHmV+B{xM*wdiD1PSe5924mFpa@)ypA->Ue2S9+hQ*EJ{y9c zFAq1IB9N;#st4DdoZeqxILFut1Tj2@XFhZkR9# z3B-w~CWZR5DSV|hVfFbq_D%#?fo^oB^af5a`1QQTEh=U^K4hlj{gJTx03t?NMorx$ zQd#_{7^DjCMc`2v&3@HR<(R_tlafC`?R=>-12012%$L@1=RyAi^A1Rc2(A^59w@{P6bKMl5!eL^AV75E|AsaF_dpc7jjqrX6>Zu1h0RIpCxMknR?~ix5bKSf zzL7sVSZNtmaFG}k*RVba?DK#Ru+1Z4*5Chm`o(t`M6wuFe1!!8LmGj8^MB|3q||*Q zojbdQc8gyM@EIIG2<$?jf%p_EBebLjSgw9_dTX#ML<$|QO!8R zKBOe1d3dZ9)P%0~-THq4cTV=QWR!8m4Y{Qa+o$uH6VlQxr@<`DgYcG^AD|eBm=nS5 z;8p%`-0b3Mv48fE!4;3ocH+?tzjz+w(`H<=!OrMtxUK@J>L1ld*OV0XQKRRX@$vn$ zJv2tvNa38Di<+>i{1ff>6+3J54?TS=B((gfDp%zdo<~0Oi9d)s_3VO{NE3`S8x+^Z zv3U%g;y05b#^?on3SP_vb#53^B7KE$wPb@?UmqXm|X9Jv}MB4^_D} z$F08h7ZMq{Q_U}|3UWVQ{{GHLO>>sW(9$u{)%9-5%9^I6p`~t_+Wcl+h2rY<`q99^ zkIhQ?Pp>a*Gvkv7I#{ywN6y+(eI3dRIKtC7ina%MHBNWVEU9FH2yGrkB>}^c<34K% z@gYV!4SCAACYrJ`iz1_`yjuJGQh)iP;WoRx+!{n+a7|r}hV%1Qo%XbeQC4jnE1U8i!OWO;n)~mMLao`wB&jWmxj}r+y`@IbHY@DYG_Z!sLjl%Tsc8;q^okxc8gbeHl4{Sf0l=-x-T;2--vD)GI6~*OQ6Z?8T zB4epMGk;g&1UgyPV8hfL$ef3E(cssGJ@2KuG|q10e@Z+gX{v{JC}_*1ug}^Z$+?{L zuVz?%>nqdZ`E3@=6ALR;6eE9zerVw|yBPu}elbu-=P~7Rk@DKqqajP;mWLbt$Q>J~ zMgffPjZRs}by}lheW;ZW_VI~=yu9tFllfT&cTh#FQt5#LK|3vVMqhl}-#&D5OO&>` zpRzkisB$4j64p}b5wXE$TP1lz@$eo?**wpUV4Q7a<_%BLyt$NN!QROEbD&kJ{lKmOTiJ0TZk zb>+Ofg#(KJ`Eh9=g4N2(!ffiZ0BN~!&Om=t!vQE5JoB+GEUdJM;=pMT_RKOcB+Wt= z$(zs;eR;6e#hrh7xw2D_XR(U=WNersMgY{}g!3*eEd6#VF&U*D-QR_f9>|fF3x$H~a2pkS$yw|e!d zZFM+%&Rchc*U7d^B zOG!lEFo;`U5oA=qp1lQzCB>c3sv#iE5*Uq!SYcJTye8FFwDs#VoUN4n+4)nXSbcWR zhBb-C=_rHj!8UszSeU)Jn1t4z7h(X9yJWF^{b1YWJ`YTqk;O#Qz3U)&dYf?xHaK9_ z%hM@mQy?AXijPfIp)pcx3JdcveT7oVKaHspO1=x&?+3-YOzXyc_a7$EmTL$O;19xv z_1e=~7YJeV)z548A0om`2~JOR6Fe~TusmO1=UH%EyA(>Wpl*x`Rg2rYGDlqE(@p2g z^Jo^k;T*F7)GC1)wl!3Qdis+pbzUhb;4yg&J2kbA2KN0W>z2k5R{PBh5rNHi2hGe# zTIDVhKnRLvU1OA>AKex5XhZ9V%$lH@nxqC1Ag`cYwS5Z|&hv+g_^?L%O2xUiOLLL? zo`^%`vV2$5^T5w&!JPGnaF(n_OYG9}WRs3ZXh8yJn0Oi^h23arA%W*B&0DV&Wt^J4 zi{rs2bPQ=|&Vk=2epJVy!K(_a#RcnA^aSy*le_Vlm?0#XSP2Pu9-Y**jVb9#pRuv& z4X??|ecbiub1k$_M_YcGOp9upKIN9}T*%O*5}28o@S{T6S+*=LEvTuh2YEcl_57r9 z`dMC|09$7%)njyFw!!8YEPaA53_~`2LqDvs5SGE#{`!FP7MGv@0~GsJdITt`z;(}LfQV^q3UT;}sKiVpL+CV1zLo4OG_Pg0UVAQ1rAo#bCO zs;}dI7r#7w#)DL5%wIJ1comJZ?3r%}dR6?sGy?x+HUT2jS4>gl+aIojZ3`obk(~{+ z`tikLE<02;^`e-+`0La}!()N~A9P%@%*q^#8_ zKfk@yVML-UDXXwDH!&$IE`PYbva-Xo;5d$-Res@`9!UZ>y1g0?Il!cqr}5hJ99_&W zG)aUWpP1ty5?8SL3E(gpP8aWGEWGBOAFZst7>#*qyYaj}?8S4;FCnd$_%?NJL-S%> ziEk8Z4rrgEpuD=)q`Z#KZYsDsrbjIdOU+*>FOxe+Y*fmuEvHgHAMX2BsFdyC0T*7EQf%te<(U=12MbR24PL8zjYG>h^^Fr2 zMTyIB__>oNs2J#_wJ?;n4BjL}p~LSXeF=<^YjY!A+5S)RhgKz%d4Rlu35okP`=aM5 zYUNa+9H6vSnCxb+N63On$Qe0UG@aT$mheiHd;chr{frM3euq`ehJaV#^MHS9~^S+QeyQA8qO`Umy)v&~OX8Ogsh+$hyB%RPpgb$#QQS zJmy^s_-xzyO*B9{M+y-MhskAnp+*uG_Nl$f)^IfOaPP?WYSy9(h!Zy$KLA62@{W=i zGleVwU(O^tE(uV`V(Lwwn#PwW59p{VHmv*YC@U}G^L5FGRJk%5aVwoG06nV8pAXR{5q@vCu9u!D_zz?CZ#&=Sd7I5Yl$YtkOFEpt>pToi za3ni^xpXQi&HHWB5QwN8+h@tBXw)DIk zfR1ZqPZyK>C{H6pjRu~IqGnCM$$=bj|L^l*gnD>dJ1Ii&S~@E4MH^NPyQ?flVn?8B z{36dvq^Sk8!|&2EZR0aG35xR+;g5n>NSTi=0wV;)$};(9)H;o_rc_GNPwvYBh;r%@ z`5g%8-EKM~u;52GuQJ!fU@kWy(LeV%$LI=+(C#=zdbM;!?eN5fX>% zxOYtKTx6V=yG3PuI^#o!?{O!G18EKa#Rce@p)fs+O>1sjx;(2e=Q#esln)MC2k;;R z!^&#RUwC0ez5u<=dvOsfYHFfrYJH*EsKH+_HDg*Up7%xOsDEp5 zX-PcKUZcGA49;rh{Y~``&(Y~-tBUHKKM;I56RDCsG3Td!d#}TYDU4ZWX#)&uVZu2R z#&%;reIw=;RkTx?b(bI8-*QAXN@d9ztfjo)JMf2SW8}6|bJf|tbw}hMIrJA(4MHSZ zmM`D#D9vjSu+eTSNf>A)s%Lb(hp6u3SctE%fTkD)s}zn z_tCVX_QkCIH9=PV2g!=rm$X5m^3*BOcN;+f>0iIX`<$itq{b|E_VKNRlJBq`lhF@{ zM<`9hLiXh_Tn7^+g##%&N7Y3PO1N|E@f(C@1%zPc_Z_2Q0jiR_(}4rLFR6-3noK70#%7Zr34p0A6yAz6gka8)qm}GlUPAd(p@9ezJPyg}B z&RBcfD*)4BY<-)@TKzb*6KoVulRagOPFZ_Cdoo?U$3&-LYGD=4E$3#$ zfNkyXs!@^|LNk+hKzlBE<~AE2TSIwb-^~ zckcMfV>kfSViKLOIo9*#foE6pKta0^Zc95Cg72CWNKqpD-Be@-S#Q-k$PM?SM-Ae$ z)T*`h^pI=Z44)fp&6suhg}<9WjT}F#2>S#YqP6gyyY(Ke zX?{zXYYW@qGW8_^$*C)`!p)705PZrSiDZzwzH*l4z&A92k0zl6!nniX_Ci;{S*S7f z2yL9>2~nlUm}0D19+951qi;qCv%}AftpYusmu0Z;Bj1Xm2XVzGLZ_Pc0vX3#*~AGEQ&zSBAB-S0)u(cDc{zlwqH^8qdQf-O=<;Nrx810Yb|R^Je>x zmRb}2eG_UGMF8+RuE@>rNIQeAb!T~N9Fs4*vTX0D)cz_NyQbIj=)x-#Q^Io~`Lry= zS%p^>Tw5gnIsiu38s)Ar&h}SUC(fFbSD@!my(obyG480-Z2?!^?B^J6a1DhRAalFX zud#+j8baY@{cw3-u)LxHcvAM%P`v z7}D^eW(UD9Q6NehetAbi941{r=k&$!GWSBorb4c0`eMs_1~*KS)YjG>0Dfm2hGoRgtaRHkcvs3^CqHBre$v3|ll-OkZvNK_5X=-K^D=-e zJCyjmo@i_-EI;{lR;kR3!4qH7ppfWT?J+$?S2xz%q@I|dLhchB`gP+}L&v{w-H`(po?T4Ak7a9_){DVBrRGtK*UP;R z1h;0F)17tg2WulLZVkeYB5C(Q-zGBM>Na1(G0QGP+9^6wJMoa`^7rq^(C!AAH?uuH zrPhns<79qIxqYA%zchRh+q)h5C8(OwOg?L+Z0QmJanjl;d|)VaV8>)DZqJdx=z)>c z;9Dn_R&xLKeX2&_87tL=cve7kBPnf|_{}fJnUy?Qa%pL8G%{#lFG5zUqK#BDD2z*TWl(B(pXv?Zlvgw2MtxE0&fQlPcpY?JrgR%k6J^ zH+rCy`*~<$J726dv+W&c5-^N&_IkF0#jC3MvU}R@@p`y8w#ekfi_!RXF|1ero!9-| z-CT_oO7PpSzi1HVK;n6qD?$J2(_U2f*j&{6K4k4yXTeZ{>H^Kj5}`?RZ>-ka*{}p& z?&d3ns4)lrdsJGlJnJjmsy_Tr=&-bbA~`*Imcj}ZUy*C1&c;h*2IeYA z>I>&8W;~k%sp8hZ5mM=Zz6Fuu$zGr4h=I}Jee2~-lh>G1sWw%Fa{xw|`w$A2_#8)5 zzMO{I#@5cFD~E5BoYs_Z&XlntC8*$f3UBDjnkQ}eOsAnNr?L8ui;bZCbyJ{ha?_?L zb=2YN*NLiscJBG%%}?khGM#I3f>wUDssc6{YP~0D*a!!MDPS7j#Cl_<>t|B(39G51 zS42r2tSTuDEma$Fc_hS=#dIOStKzU{W;$fGy@Mf<;26t(s`sG7drIVvn{S!WRr>(pYooocooNXokYE=4dcWxaMq^grpEh~1pMq(BL{{|>qPe~=pgC(z7CM_WeU`FGuq4=xLK zx~&vSU^h7YMH43puF1QF*K2 zm-D_nr*w39Jw%KdFyd?r*Q%wt+_ne{4yQk0hY3$vaHT!&WgK9wzCIl(E62Ngu5V7l ze+m&o$!^IH6E2u21Y7a9nyY(MfkL{_dTSZdjQ8-{yf@ohkJhe~JIf7`nHO1|?HTui zWyCd`Y=98*SPZpZ6c-*Yy?{xM$ZlN&&I`iKo4JS;h9Nx0=O*^MfKKNr=9gp!g; z&;-z%bTu5BifOw01eyN3BMFZI=&bZUZ?M&7Jq4Z34spvF5zXdBa;@H(Xh zyakm}XWK;s*0uTZ5vHgnTx0#`@V=a$a0KY65b+NfY=REE)H9XpMDQDRqIcG>_Z1DnUva2p(GRy2+wu{H?>;Km3o8d@mXYBZQ#Cmb*R+*XBwTp3W~XZ;?%hFb7VT;mQQ!uARF-bn{Y?YFhuVFE=Lahklu)s~(x&jU zCN!Y=XnvhxXe$Gk09P{a_RJ{o+(qNs3;gr1HL+U5;*rw2@S#LEIGwNy^j|+-n7P9t zLF&(b{0h>Z>D@jEkp2z|!RG)u4@MGrhiZhr_#7{IZbJQ~Q8eoIPqn*X4-rpn*q;zU z^@RZgEzDqiWg^F~!$biti^v zWWBc56RR^Z$qg4!s?o+y>XOX=FpQ8u7LL-U-g`t8_b)KU|Fbs;VTD;f>=tkB;~In) ziA)bhLTqcxt%dnY_LHo<%bA)-mPQu#Vem2;W(*EG`{2@$dm)wgIPqW$TD?(SojIej zptnc>R0Jv8H_i*>mp@zjmyKXe_4czp8soLGKk%%sjye=l*<>6Z;xRJP>1}>tRvhNE zR*)ZyS?PySrqz~QC8ub;t0?0%s3P@w2f9xLEib*9b0|o3Hhx`hTbKX{$c)tg4j-r2 zSq)lB(AvR~(;cJM%xurg;;(ZZ>*TNa1ooTUzECukITh*zDqj$6eY@b$clgVkPbemn1euLq%XRJUT& z*tInDl9Y!aO`73)brPXM?K!v*1Pj#F9fKD`dd20Dg!FvsG6Flt{)8?#>NjmHMR^dj zwmk(|x*CfTdd6ytp6@5_o4DQ)%xb1Toy@6k+w=@<+cr@-f;y~aDpDF2zTm_xdw5~7;o7i} z$cMoN%*GmWqE>}KaZeYa?e-&;DBb}}E-FqH!=FU2ZDl3G?u@OE7nuLx;v=tr(w22t>Gdqtc<97;rG9K^cfsjzF@22fy=MQc? z1m6}Lq7hrghfZ@dcoMY#P9coPXHVzsiL3Gb-Siv&K|ir@PLVNc1PeV_gHobRAh&5> z^2Z1>wD!=6qW_FRct-n~;Rl1QSbb8Kyh+o-6qf1D%cmV|9D?MpgOS}Lb}cIY+Ff6s ziNo+qB+P(TwkRx)C5c!~)CJ8s0e*QkL7iAf6LDcaH>ipiI{v5R^S$@?d~MYb&w-*I zPHeNMF9yKXjLh-)z9Q?Yx$`YwnppOx3#bTDxQ1|PhaQ}A zMi#%2^mPmM*U+A6sKx$be>`4l_w{U!L`ZN9|Hjn%%criL%udi{1E%m{XQZBp%T8Bg zLtx){OKbH(KFFj*Ei%?gsuy_^0{v?qe=Hv_212wi$8FO!odg>ERMV$3%)I7>)qAwz zqj{PERFZh7kpp%0ZdBf3iC866UFvGnNrgkffk@hqq!{O0O*~$=DNt;hUyqIBYI&iv z&K+WPkg0CIBXj$nxk)FW%6c+T)J74se~Dd*&#+@B8L@DVRqvyi5(>&&4cR1iT95n+ zDujLW{5G|3pP+E{vh}Vk(~+|0C2x~(E5JEOeo~}vPg%-E+Gj2m_Q2IT6cJj^1!+yR z?>+k0oq4+8h;4qaST|Wcp^p+Rm^~8FzClgGj@h@(8rgqxTX%|!#CcssEclOH8t9R- zp`zU20J)#hKAl*E`G=#8*Ao{}((cZrl#OaL2{z!ym!!#BnYJ6THH6!N|gSDdW%5^J?by!;~ADaxd>gh@p8 zHzI`cWAJ27OJ}B}c+gz76A#dh@TFGdV6g#tF?}RSLKr~*YX7zTSlPN&U6worUIznv zKPk?>9)J0GF~j_4n?Tp76>)T4Wx@@q`XsbRqiE)G-GmARCG^KH;{i9Ck! z{-VRx$#gCrBlOo3x_1?UaZK(dRWb6HW=a5cMG=DGqZzTf%fxVdQ|1~)1o zaz$L~o2LkJX5A+SP{}ECB1|p0_?jRh-i0PGg5J2)@h!Qp@8}d=e{V=XlDu^4?0bTK z;Kv(Ir?6*TyRHoVnxvA?EbCDy-|T5b6sygGed*eiDX zE4)0&>gLP*V_o19#SaUIAnq$V>2HtyO94_rOH<{JDte#xE?PPh_tA=joy_t_T~AET z^8vq@W3^!Uh%`DSK=uSO2TSq`Q}c=RVODM%sb$!Amv1*?V4MRWWc+z~oD{)vDY@9} z7x~b9D>lB+IHv{X{a@#Ki8!DeSv+_XOkf=A+%U5a2U5!OtiIIS)mU#y&6f;^ghm8- z=T54vkEwr$p&|Hs01!%HUy1ePr4y(0>@a}6z7kW&U++GNp}U9rhRo>afg_8?Pe+BbP1%$AGQ#k=iPo_QwDU!3b=T^d>E}a25!e=Z`9T^ie9bBaWdjFa5Bsw)`setnJW6YMnXllRAiO5?8)l7 z9W4eBmRN?RwpZPClfZ4>5AZs8KCS&j`C!k?0WS#-mATdj{^v=uy`JI2txybK+0nX} zb1rJ6!+rdeTh;p_CZ05SO`c|y?@!*}S3vTxnGor4KxEZ zO)wVRH<&Az^b3i6CeB|`gz`2TiH76ZPN&sX#lJ&I4E@GlJjFX_Bbw1tG{t3d_d)`<=#ezcBD*$ z)Juf`u(h^CsytdW;fY^vM&-2>OE1$7WFP@pPsflP$KcL2EWinZ2-JCy{!MJ|LEVbf zQoL0BVFK*6z-dYm_BGaiMgiyBG8HY(Yi-acGF_7;M%zEE9!qc;QSE{p zkUGJ%*lV}snqTLD(GT?fa**wG84|aWYoaIAUa%VxieOHsXn9RyKh7DJy)K={#M_Gu z7!$jvj%l=#q%gnpI$`M@Dn@A?{DDPlpl=xBaS!fMN?!hm#;s@Cy-oA>I@q~N%Vqwv z!iWzJf^Xk{jLX9r%|mpJ86U{pO<=oo@vM9AaLSpZV*G68H>X9sREv+md&C^h`895~ zj(Fhh{g)d@yWG%HJ<`|!^@60*;|`&VsE<|6^TgtWfj-_}vt+@fusbJV*bWIrLoW)y znp02AYTkZ)v7lE@e-0)rgKUWcBjyG?I^+6-z@poM~d;Bn2dp1VPV78 zujbWjer8x#ZSePm^8G%UrJi0&_TlyG(xM^5$PoR_$vi6|gt58upX0>Ocx49Brhk~~ zJ~RL$4+nD)=EgNos!>`0+&0ZZ!f=(C)eCAOF{c)-f`DqiiRdX-xskZ2ByE5Tdf7_-vFX=fE#2X?UK4(e) z_|36c%cm5&xC-T@U2?T{ot=u);WqjowjdHC-FmSo(bPb)Fb`&^It&590|#&!T345v zF5``QSbXe2>V#)G-TTM2a(=DATwQ9ee`otD=DHkFyB{Yp) zOLDIKKAlAU07rt|UZLL(=w*arig+E6>j#Tb0$HUBo{S6Yi8&!lHm&L1;o)S$H18Sz4Lkx5Sp1n!1BmgeWL?Hdj4#Vsjt2~rxicK z;D)@XCY36mF3Th{Gts+B^-?m_CoY6Yd}SEwQz3axwCUva5N|gu7n_txa4`wsz1era ziqw=q*D_l>ovV)FOdF@U^I;iM{Jb^G|A7-+~4Z+mkoTaR!mK7 zHA~&C>dLPV32y`K*0v*udzz_X&v=g34u~iz_t(|l9TN~oYfcMOf6L5pd}XmFGqjT4 zMl--PzD|%GJJL^|i1S0r|5}vRB&>LXly6(r}(5ltDXmS}V z*DB>4Gd(bP<=4dw)RXgbEigSx@NkNwx~P*6t{;3ZF^dqN6w1C{xX}0f?NFenUsy(e zlB^z^IR)L-MC1nEp&?>ByS}*gni-ym>3rq6hcJ!l24L%6lR$qLfgh-Bj;~u2BzXqvyh{%t z&IZPNH~-KIv~KXUQoWb}z=2Ff%a7}bF?cq>OR;&1<5cObKkuxEpv~bVCNrUVSjiu2 zkuwiA<12T;G7U3wX6uzKo*uNAf%%*BbARENjf;1LM)+}gP_*c9nXP%MvSgA&un~o~ zW70n5Etkx?;_CvxGq3tG97Jmo>NKBBa_t#k29)~zObkvAu>Uz?mZeh}CluYk=8OP}Ia@6!BJmrC6g9zt@3%O}CC)}NjHa8QOWnWJ95{O03K zG5a-}`ZGKfH^Y;#1OsSt6?+l%p(Eg~V#_$izcnAub!IZ?lRjR7l*9!%=^d;SF%m?E zxSHxC4fRKgy436MPT$s+@YX=e7ZO1uo|M#xdcFIL=NL6U%Vd2-z4SQ0u@%;z;QYH_e_VBGV?$m8uK!CHLWceq7eILw zSq=Rxa%KYV;E3ErU%{M(w%>t5733|sQEOoT1Y*gx%KA1fbEk*A8w*wIRA`*UmRJ z=mTC%k{Y#YtuIs?!#E0k5eTPqL0}3z32s}Dm7{mKWkY_wy*iuOu=?Ra|JIi-(z(rS zCpmg+!uoB@WxK&rl_lNnB&Fp zK>7{`I#5=%HCR>j9GtRSioV6SWTh2`qT=!(qN$Y0*AHMQ4!%4GYnz#+sIdf^#b6{T zN3yxKXt3Z)nf$;2WQ!E(P>k}#?JHVf08m}6IVDG1E2OSxB$2T9OE}IdZe|GBYRud1tdB@i8dSAwK- z;+D(4x}vvmYXX_umLveUPuNNZD^5+*p#m4!XHE`ijuFVBu80MIn1Nrm8@06*lqGq9 zj&7%&^(Je5&5Y;!i*VQT^1Eb-{@$sd>loSs>o#mq%m?6#;Ey6|e5MIqG z#6Hq=G$oV9%pg@^Lo!ePW-Ywab?SqF@jShx1_DAn7zWLDzhw&Pk-UF8YHFoxJuf}2 zdvyTofx$YfEJ><2UL5=n++!eSDnP~;uc_YQ!sl$(CVPD8K6v!+HAmqbV0+Q%@#B>yVf{6ls6u6mi8TKs{ShSa6ELkSTCwQ>IkZEpco$J6bL zZXiJL;10opyE}m(fdIkXEx5ZTxVu|$cW2}7uEE{i{Z0Pg{Z5^8?tAA|-Kv|~mDGe` zPfvGG&-%$)tsJHqIqX>JZNEbMw%lU5c^^ed6j@)pU&KD~XYO|CY!I7q&+B{)rrh`C z_bmEfmfgBiHf$>obIp%i0^NL5gPv{ZTDOEMS^<`VW1-}Z|#T=Lq1*y zb?M}a7gn^i0wpOv2;ZG}m4pAL34Hg~`=2Uk{J&>4bn8Kf#SZR)K7Mlh^MW}Bu-F$< zv)C#O^wh1+$9EW=2;+cJp-JFn$)6AWQ+*T5X(1~u@Y!vW+Okhu4uK{)%@fTY~nlNd750{b-y+a zSX`%%AVr#?h_{1(KN&nK5gE}RK5@Y(L)T*>t*vdG=(&WBg2PkK=SoOAbC5W8cUKAv z3t>Q5p-r~kW-GTbQ|=XIZ$h8UAwev~DhxWuhljO0)7Ar!)J1!P_y@^}4sY}PJ*zK) z08fqY`2`@43srX08=EWHD(xON%kf)Vm)l?5Ld#W>TOdx5^mFzm;~{*2JMy&MnF1(h z>xb)9BIM6N--fZA^_(lf|SJHlUxTe z1E;D1KNS%?=1TS$o1OiNJ%Y`smCmJC`gstbg_~`F&i%`jcYg{$YP!1znD*rs{OO!7 zz<~gha|v860QX8}S=&8+#}4ptr*irV798oDsD-hl65p*(q2bQ{F)0aOy!v^=x{x$)#e8(cUWww z@$9oq;}w;uEtj>p5kcSQ;*|#IwbGkiVRxoN4wY#sQ8W^Q&4W)PtMk4Z85tA-$%|R> zR>OKYSS9903-&mmA)w|K49s}%Vlm3wpU!3UhCS__q2iR%W7=&Ben4S)+UwT=bPK>e zaeS5&6`KaON*u-ny2xgVg=NBKYhPt8&7`N)_2nX3Io+8W2DbkUV$4zDCeZfmwTQM!`{kmkKy_!!$l9%<9V|p(KL;Pd~LU%dY`*K+p zRFbdSKs_ve7M?%-3CG4W13hYLiL{-wJIu6Fceo|5Sj!H3&Ks_)q1FO}t*_I2Z$>uh zS(nP%6h6&$wG!~?7Vfgk`R6x4yTM}2Z*(u30ELf&6X%|3Zd0xDI_xPKLUvOnQa~; z5~o<$Li9d?+t1ibGP6$fx|q!t)7i|PM+09x;}acoIs>|C+m&>;F1GE7up`nE-wD2c zd#B;?ytig9UZILgIHKoMUA#V3hSl8dPy9+-Zg4N1&aw(8Su>_XwRgTfKb~fdNF>-< z_k>K#Dx>K~M3aZRcJjE<+%rkGkWE*?vmKwOpfziR?e%7@P0=W$ar+pHZlO`}=0(>e zBu`szZuoqLP*p8SM_gm}!e_KO-)_I9lWAiz9zi#6OBVR(`f;nYrTaCr^(n&q(BcX7 zYC!%|NL3Fg-}*sqvwj_fnN}DnUm}mq;9mk{@AT-aAjvV&!x&uV`eAx?XD*qfSR*io9f_ABJ+sYj4`J_5a=z9epkP$lXQa`d4 z<<^pByFTt&1j_N%B35D6p70gQ8l-CScc5n3=UOlgl(2Tf zcLf#v#Ynr5G=6LuoCEzweJ1Fosl0Y%U*{XPD945_lU0{nL5)P~`cpDj?N>GqwyfIW z8Bt|6pTm`2regtnrNC{2cA_EY>&uHO!?-nS*&tbZ|E!WoS#fF5jw@WWJDKd8g6bI4 z1;L5GcTR(4VG_ah1xxq9L}hmgBSFLAtclS&tyoA@{U&DPdkxOsKVxc}YIs$~I#^4& zHCCgX(~XknS%MBF-Q0=T5U6iJk-vO*xDb!@n~(B>L$l9o`{2ir@sF;kM8fep z7Urtz2bW}nwGwafK7nH1^J3eX$fcA7#>b-f9Va^2=`bJ|@G1WGtRo#5(P35^+|Ae+ zPeM)SrU-!rC2ad@P{|wj=@SN4@}Hny{q-A-cv>17k?vP28XD~?i)fPh-(zyp(&4T( z1y~m$4QC6(x=gRU)<6Q@oCY5?i6rCJ^1%RFISIZ;Li%_IHdz7aH>u!(nFZ!|r4&?q zfm>=2VZ{M@>1l?*hZ63ol>kDxN@?)jq3un>{XRHjT9aC_E%g4N-y0=(Eb2!!35-v1 z$g{V#p{e@QHQJZ{F9o-CM+QB*OA&A>iWg7*8by&~UraRI)Ub^U1cRhc^mcRA;;=k< z{F!gtx0W8cC47Iqa*}X7+n5D1$GXRfpYmlYG5Yc}RJ{F258`!ZAzBNer(!=g3+Ck( z8kzWsH*eUFsM#!U@ojae`1nwN1hn)U{FcPP^CYwOwcU;Jb-_}a z)>dY}e9JR|W1(0x!|tZaeb?DxloM${r?ZpW~| z>JjCBXn_}z{(Z<@%(l4WEKh9eAo*(SSe~?K#ADWo}`YtDTuzT3EgJ)>GYF@wh=?iY_emm-w>*C$1p`gdb*=zbU zDKqGY<4rQxV0$8khBRF{6}?Hai<=Xt`^D1WMCJbJYD$CHv)1)2u1}ITCCjr*cFNHJ%CSTEV;F(yXyjtUi3NPcM+%O-Y^gIMvGsZZrG4Ybu61{u`q5QgDE*kNXv3K=tIUOY}vF?#Wz1VRDKI;%&Kf}JYl6KIqIH>mB-<44( zrk%zh7i|Yl5;FP%P4U38vK-o$%=-`Nu!{}{#@uDcHexwCi5TLV>`Lq~TSlXe_oiRP z`Unp=D(E`OJ z-adM|J3q>Xm0@w3^FKAGU^3)9x!*Kc4Y^Qv3%qIN)mh(I!-W<8)wBCPv+lqfn%&^@<)p=j(P{zaURKLyw3UJEF zNng-qI7V;-D0L2cspg0yIQM7OUxV(kL2chql8bo!qk8doXUi=&uVI$DR%Z-UQ0_$P>BvpFj2zr zrP`E}m5rU9ja}!KLGhnIJiLw&Q5m+dLawgXZ~%(dBhkVd3k~a`R3mEDAJnY$j{%JmxtfzE1owl)=(7hJuc<2)px2MY!=MU z2KGW}v^ReeR?V?uyS0sNTU^qdb%~dL+`J6Q{6vvg%sOLPTE<0i5*EgbQf;5~4JP#A zGQLVUT7n!+LqpB!K%eqMoEG?k5tP{p<$&*XV?l^vJE&pSfSOqA=e#k_Y%2Hn>wYUW zRcFgfg_ac+QDeYUpamOXyAu*dbn%q?H!V)b#K>?E@~TgUECC2w!T=*oslJjaR#7Q$ zt=__f`=9LumuWI?vxS2XMPbxH2=Nx~bV6&BsX#)60zbp23 zH(fYb@=Z`w{2$oa*}d+al07t+n;8QD^ zP+t=)41s8|)9o1t(9dbgjPNjKR@PJfHP`-fDEh}-paWJ>hVX31#rlrncICjD1B*#peY9-LL$TC@J+XW87N5H|nK%+C!r{KhK9^>uJzBjxvv|e))<8IW47)%f;9z&*tguX>T+oyJR@Jj)wTX;OS(A$xHnj&~PtwSlae;^$_kvv97vyF8xYHW8SSq{Foy+KCheV6?MK9W_nw4u3;<@;X&ko>+M9 zYC4FDQlys2sZrByBvvot;o*hMXL%&0yKi1UaI;d@sD3jYy7qzV_JE-o@!(*A@hL^& zdX#^DzxG+N-x7;8|8Z{zJAbu>Udq;2_xBLlBac-8(B(1@XH#4)x)n+}t82u~`9QI~A>PbJQ3gFH9*8W-k#)%mZ2z->QHdmTnDbcMF9Cb&-ArIk`F_6}y14GT*s1QAn;T0Fc+=KnU%>}0-4XJ_n4 zU&_xT9r~)2J%RqwV*(ZFF#zUi!#XhL*H6(!&ImPH34< zzi6wO)peeWDWX8FGb_lnga!If!yd1pkOuZfdN(*k3C-eh9Z6P}JFqZ9bvZBIYCb|j zGPbMjWkcBCBSrPQ&EZ~!0j#VB@q~hu^oFUaxi8T>D}1AOpq8MI?WK#h!_Ax?i9PhU zlfEp8T$~1-gS^6OWLDy|p%e%v-d8W5Oy5dKRERr(r!CQ&Ls0PTqK_U7JP>iB6YV+r z!01b`4wQFn{BXZMdKkhVR4*tZl}eGD(R&Z0A0`JCNMY@KGtd7*Z#{3t7xAb5GnN(q zr?a%<>$9Jjsat@sAiR&Vn@|#vLc>LWp`Q?WBV!V&RUJ26ed4vIbw9Y4YZNM-LT9IZs4f61=WkHAbOns zYNE>gW$&h(_bdz>^|E8{NcWA+yruLY+o>0uijV!t+~e(%$`t=uST0msr)h61=JuE{ zEfQ=C>-Io;$f8d>ISjpa8;yk_D7NuvA6INpgwhgj%ZrCl19HSE);~~eOdvac_lgEl z{Qml91GCGUt`dbCe_tDm5DNGW%3kR=JT-RJBguDYd;HQX81xWyg|Uyt=vdA_sh|`# zt1d+vt97vK=B<99^adt?Y@V#2J9>B=OCjrM+9ik1h+pECU9c)-ImgRt4z)J1o{HYf zA~=cvQQvCAk5EU?OezK~{bUQby$8DEGurFB`E!Vymb%~CSX*A7XX( zY}@TY_t|0b9;Fn(a?E zsJ%O4eBI7UGr>KdPQbmTTOXX1ZSQb++rAl6@2A*a!AS5YrZg~u6L>?XZe4#Q5X|`L zODw<%L!`fpiz5?Ig1w6k5;0Q;HYyJflUZ29Z-iEq3KEQFj1*0irnnMi@Z>ygM2pg*8mX_o_DvV1f`}0>WTQkmfc;+X-6WM zOMmQHR}0K-7*6xZnO6HK_?wdy*A+aAIxc4T{>u*hY$=(FI-6Ebo1e zix5O)Ud91Upc0#Xywy8F{}RN|ac%_y@wz0gH-kWKYMlW{pjLvYXau4U0Lxac3;EqY z5WRfOnh6y|Q+0u8!N=U!Xaz^elSPHi2Xf^qftS6)di>gAom4Dcm{~ZNL9KWXDk*pZ zNi6H19cP`d{pm@d2EvIhKl>-o7Z`hpjuWU%dzDRS(1i0)+N3fFNRlVCXCz+7TS{NX zi#h)SSqS0s;i`aYU2u#JWcCN@rLR9z$^{ZMfy-$siEUZDg*Nr;-8krzA^-C(&;1dc zE@h%`q^9l`$qeG*w2yguxv*9v1n81Rk4!^C8l_b-dP#`XN6^X#`d%tu~I& zr23g{xvSLI6e&1H*u6O!=@+%;Bf~YVLZ-~cxRZ8f&T`^a*=_LS)0xLG(aN?{LLE(V z%3W(xG8C~?N6_fc%ZM~mz<{0X^PY6(-Ac9{*TU>r76kISE68oIj&m0Nr)c9C2z^3N zi0UaFd%X>r4o_kPI^eEHlJH*Vmx*}eAN_XAGIuwJj(-9*X zHY|hAQRUFGim#g!3fglY-5=gCW{ST_AZ$AH28wbez=B-Dwv5h?--GBd&+n%Q&N+k@ zLV2hT!{SQ2E-(;lU|`P={VKgl3L$-t8gvbyte}EHTzEp1rEJ}Y`^|~()+gX_)wr}D zs=q#Xxwd}GWl+d$$~iU~?Q38-{)6pyI2WoFurt!sc9U|xlN1L3HFWq}=x|G=$9mWr z%k(D8^e+u7EO=K|Iw$Qh1rVqp@GK~nH*$AgG=ys5-(CQM4;*{}YCKJ26Tpl<_RME- z79)?71O;u%esQ`zArX9iWOH|vJ+dfgNaAN?e0CpSNA!-3UyjhP$wKTFJex}yAl+$V+Hr#} zE1craF8HHIwj`>pL+A>q`xoTh`65**`~+SRJeww`eLDh$?5eer9q*5lqdm^E4Pi(} z5!$qnmuKR;Yl)v&5l>9KIL1TsXKDCN-r{kNIE@MhJ1TC+FCN_N$M)>CIBa>2Aa1AY zOYh{a-O^y6Ogzv@-jQpzudIQg?)RV0MCK6h7IkySYY84^6DI4L(NfrW3v>P2U6&Ys zt}jk>zThJEJ+vDP(b6P(*LO6o9?D3I*Z2--xWSshQb!ARfvi0wy7O0ka@ja{BrTrX zZSN$*c^r?X8!`h+jG#)SA=X9T5!S9?9V7ofpE>R{*FU^IXp^w$8EFkKmlKWaFCnvX zb?qH*KgMD9u;EmpTwJSLN_K8Id9PLFP!&CiU;at*Uns%)n*)M^mEQb2n?KFuezmj} zr~Ul+xGO6wy(^XZioc=W+2`!{hz(p}u}?EK^gs1O5w7WdGxi{7V^cK+@dfs>UT6cTAmQB%YKJA2Xqs3oFa8JOPfgPgqTv+8Z7f#8y&B3maX8BR{hmVF}g zJkozqt;FXZ_KA5~>WwQG+3;{AB}K(Y_v@m<($dmGa7c}GKk}FsVeV31-Y)=liiugb zeS`tpcj=|@!FJI?S~hZZBT#lElUG_=nx2}Qeye_S77XihT~)~ha0G212#=1JMFVqI zy7GV^UN|lu0JCVpFCoc``pC7V&s5a526xl9)K3sAXDkcAC zJo>w|YzP2A=5v;oWhF*BG-Lh|yUi_Fu@M!{*UCV5o6(4OL zd$wn;@!_ZeWXj?Gcbc_$ukHnDbTNjvzgnl$6!AH87W8(q9+40An^LQ_a$I&%d{%1$ zwrmn|TD+305^3e9K5eVA_tt@2|0n4G+ocKe^)v|n+2I;a6clR~QBX}wh{ir*qT4R8Vq^m6 zpAFnp1aiHM)`>NFE^y)Kv(flsR{i0X(V~aUN>3^VhD9v2Pb;ffXtyVC^vp~X=1Zgi z!W)(w3Zf;$2yy9^)k&u0tkB)`lUo6E4?|u(>Ih#uI3Aje4UAMXamHK0e#2`pn1|@M zSmMda&dpM9rVqauJtsezs*kGN+(bL5XL~GH#apfyCH0F+$XP~hsFqjN)-Gal8v4?& zSp|GU#Q>3j+pezw|6wG~_pZ0W-(}r`>+=^d4YMK-868*dXg4Z|310 zP|wOMjCe8zK(|DztA+pt&_OHiWX9iatV{m3JyiZQ$bH8wn?yy2r$#v|aIxu4Q(Cp! zrXH98K@BG7lsa(ljHX`IA1pYh&%UL$zUu%c@vu%=PJAzbMS1W1urD$K0!kA(lo{Wr zczt|sR8Vm@b%FWdbWY{C(^ihF@Iu{)Cj1AJBBBv!Y>dM65ei4b#uYTLNHi_EQ*LD`SuCd$oQBcFSJiTi3p|U(I5NO5W$$s#p!Agzn7Z2DaGCa)Hcgr4Fp% z+7%TPe7ooY{-=-HyeHm&NlbWUBgaGx`APu8mWBEuzbQG`BAt5iyUBwZe{*x1O1VBs z`X};}eoSDw+hw?i>;Pzk(OccWM2dnWH*e4fV7UM&9Jx6j1`@wolK;TcxRzR2*=)e$ zEi5q|{Qrf1f3D7nOGs^K^=5njY5tIl`&BYlScUHS!Ke}XBNj&5Szc*)cMB4r?<&+ioMWW2CiznTvok_)tL z5q%?%3g8E@=7zB7Xhi>Zdb?LcM~-pDU#KYu9U?2$lCv1MCx{;@Q#_*&D*V7qPdER^bT{eNGUmrgW}h@V>Ln#z0!wE_;}7PlpVIq zt#LkwWA!)J)}x5QEm1$d^9P`s)F|F(6v&2fw~L6l+PX`ye}S=g_?Zq7h2*bk#vfo@ zDdO)(*tIKRMvo!K01IdODe_`RZ5j4srzJCov^X|MsnH8rP(YXg{pfjJ&SYiZT{S7< zsUr@I^9liJ5$z16lmGcWj=EktXVT7l-iR%IrjqjuriEOCBi41c*YCYkMK+e91s3fr zZ{e|#4Ysh5TV^I7*1ipWIpfKc-0Z|mEbEpAffDCkxGG#-3*Rm~0#57P%U@2hmsq`q zH8-Dkdp}u|T9#djuGQ+^rJV4mGd^!yLmG;E)$p-tQ&AZqJs$SUdeDIk1EgE`Q;n^! z?Vz|#+SID)6$biZNozWE+AkJ+g@(9Gm3?8+`PLu#J-Ej0Q&k>lWHT;z96e{|fWY<C1~OEc!v&RL5|S!oHBWHkpjVqBS*fr6`|gJ zMLjz!9JYPagj&D&d`7bR5{_tUK}2%1t!qG}EK{_2WQ2gi09r^QJ6oldY4wH`#^+)J z{Z6DX<`eRPJ1y995f7`%|BP)JR`kcOJ)yYH1RhhMEEJtE|90dRLZvcS^ zo>|K0NIF@U$NS>~*}()-Oj#lRlA?iN$^}qa%B|=rOk4Y#c7cDG^F&>B?X_o2n-Vy((eGToW7uYDBD@&C^b&wR z>~xgc4-Sg}wt)*3^w%c4m-W@MKY!Gx;`9WxGl;LXueKOr9^GCZZ#{!ZR{{3$KL#RSc`h3rfWID#`wUhDGoT>#->RZ?c8e}93pR5&V^Ygj+;T^NN$?k*B z5`PquDRJd)F%?3CJkY+*E2zj!p8mbWwiA0G!lLRQQ=~hMoIkRX z>wc+Dkq(rhY7RY`YRXXh5d4z{_~E8JhatAwho#)6im0K|*86u@@Gg ziGwrNo$cuEMUc;ODC$8hjH8?bbXFw>9ToyeG8R3?jM$12>KGf(7a1^h= zgm}0kyE>DK#SOS+6-?bTnyU9FvmVXO%)HtzLQir>kxi+rUF)H4(~u^BR2fU2GC%=U zKG83YYk-%>MF!aQ!6R$0!CiU^Oa#~M6wx)F`8WsW;PJHMqc|Gx@13cL#U&+-U;2r7 zFXjv05DtGBExO2B0u#RbI4#p_m1EBOUYBt^-}z`N6>&f#n-e#xo`iZ-x(N;c&3d_2 zK1XKd@%m(M!>lN8?z@%tmx<3ymYt^{c-c`R0Da43o`&{_P3#vtCq46Js{kQ1>)G;B z(_RO*6!yU(+kO5W&(KiDOXcf)B&BR+!_iES0@Y?r!xum*nd-|BgSi_=w6NDM zC=iyJ1uO&c5>jeC)V<*(A66)CwYj$(>RGJl6=B5a1A*Ja9)gj$$#@#+Qi3e(N3_t|x`l7XM{Q}zU11oW4Kp^{ z;RY+*vw**EZ`12)f5`nUsPF{>x~#w3qh(B|(Sf%nx=!h^Iom1;4u` z8T@6F_JUI;v(@J8Rm2}RmuJy|fZM{Ry^UX|0k#2mcdNMYyP#FL(?ibhhrg26lx~lu zDLtNTL7B(f*E-j`9Vh~5xXc(TMMty{`ZP2$J3AZSRjf>f z>Dtdfke~lvcp%lh05!2!k&t)cdL)lhrk+5=Gyj>Svu1Cpn>c@swxqo8KFgR47LS-K zJ<(0v=1i`MIkAbYlGQh>`DLOz?8)fRjE41fI;$a+2y}5(tR-$=p2ugzq9~ZUxVWlb z(u}8~AnEk>AVFTv!m97d(i)p3pg>jp$D4+eqe_*u==RVOXWb;L{MJr=_~gl`-H%@t zNB2ZmI}AT}%@ckzR`s>Sw~h|GF?PpFkC@XpJ&nU$5)Ulf&Bnt%ZX; zw4^_YZ>r_D#Qg>nD>ya^&J892UEq5i|#qJ!scJa+Vnm@akpVc_AH$xaO6TB9WSEdJf9AdwQ6S7FihoIl@2 zy~tv$SQ0=k08*vA`*DA~yj-VuK+0;1hSQT*GTnu3Rbdq6^|96OY)F3OoQBZVx*4>G znxUc0wIQ0TW4+(S#rjqw@+vB=nF`1qQ(wBPVmUx6u!PZ#JZH8$vomNW?h$Z_dD*|T z8QgX-W5AI^)<*HJHygzzMU_d#+MbPrxOr2zht(l_?L;p7lUi#z;Lk^|K;0t0+L&97 zerVmoaRTO!{m7aJ)_&)x{J3~P#xGVINns@2?JM=u@Obzq-|; zj4VzlWE))QTa?cx=UX&nSd4|Gq^u}uh&?~A_dbK45lENd!hZT9D%x8Xirnf1NQSE_ zCp?>w`$zfS+*$Ws@uUX~v03m!RK>WIy|k$K_)P5VGlmbdib@W3xIimxD(7M-hENO` zD&pecGgxtpEu`yA7=`O3G_{nJEDby4RTQED8{=^kgLryb7EDKf_dp4Xb9cs*6*7TF zNutUpfJ9jR(OxzrPnUlvPB!xP)lFTV21ZveThDADq^+-{xY$dXq$x#yYDrX9R8;i2 zC4|uhSx8)?)gJbCaX2#XCotV&GVv6(!uQSj_z4{qL=pnQpY6m9O6%9=t>RvWxd(4u|1-8E$#-=#)SN_08eX?%%CX+t8b=`6>VzcmQ15I&TZ%aQb^^ zQcIBGfEwh;9>4giOoF-y%tSqAMRE7!YsYC7xy~rBfYiG+6Z2wkmQmxOAEB0kL}RMa zrJeHZGbUavW$5Bkt=aOIYu3|>zx9oK)cIxudKDe(x55Z5E^jK5m1dbZ_30lM0?LYZ z1xGB_E|$4<@|b3n_}y{BE;ynY2ZLV4s2illHN8mjWq`bh?bgp{$Zm`iVBh_f&wtlT zcWWh}>x9)|MgJ?c@x*OcOn9>ntUPjPiv9o49Z>)Ml*LG?_ayX^n%XomAhQ&83o(sE ze=4NElKqs!X8%`I4X^eX>cT}6n{S>bqX;&8^`=EZcE6R4gjcy+N>s1MQ!_K{knvgZ z1T@1$VLRQgh8qlGrgA4R5xl){tssJ+p$oiT-L&_^O^_IZGU_?@XJS(2fEpW8rb8T+fTmbp@%LhlC{!BKHdwLz?$@_!r{4mZN zF0-Wo1Wa~bqFCqG#}&xPl-3x8EWt0BR04k`Ius`!_XS?y^HkOtrlpX#1fZx)F^qb# z!LQS~miR4XMHJQ(xym~+Xs^4L;CH`FL}2e=E3LZ!l45W4X$Av%^9AUapWbcOGeEd% z{VyQZWJJW{RQA@g>Zo$O``>^ROYv7(4cc}M7qi>>`_~=0DUN&pY^>JTf03RQOLt*- z%(bZ{u@0^GS-!BVV=Yf7DVZvCS{8;g`K{R3kj|G3i;YmN9;lAERq^{fzpS4?pv!Xu zjB-KoBA$fDjwKvm!^az38$9pA|4pC+{YKLZtgJws=stbx(8Meh9uh6}dau~I40F`d z`2{MNF^GkUf{0eoj~4tBZEvHWO49{7-R#{%=Lv_h63IHQmxmuIy#;ab6L!@VbBIHw zH{{5YE3x8kyY1x#$ElF6<_Y3v>euA`6LntZi}jGE{O0-7 zrdzO%#rQqFeehK(b}AbtquotPH&sY-Qioj{(dCU$&gF`ln_mQyG+Sp4ZKVW0!w-AzQtQJerdM}n?YdulMxwMGH_ z2>)n(Oe=z}FC}EBiv#%%?ww;|nE_gH74k4Jw)dl^38wjZs=9yz64cD|=g}=~vjAXcO+bj(jt*Sv zqAfAiCYT_^CbvK8DgLq{v^0L8Z()dR>}n7Gd|BV$b&U88>oeP5WdPj-tE*?#^Ic_p zjJmYA7*f~BG#em}-)w8I#VfvaRiAGemeO_Zd6VX0Yi4fsv)Y_@@AY+vjPOBq58B*x z{R4$@AUbZ^qbK2>+@YM;ol#o_-uWCwRxO}FSZE6RGa6KxgD&-}n$ez2eAN6K(eICG zIBw+9RJWJ~LPdRax;K>l!uNDfLhW~baCCiOHQNJG#7VobZTp$FaOT2={UA0YNa-*$ z!F65ZXDc_2l}=A*|C@gLs+4MKULb(8u!aX(Z=s{NQbV^EldDIv@l4Cptz3 zW#3C9SVeHzSiHj?na;tR)_3?G$Ydex4VQOG5J$iixJ|+u;pgx89>&^<48v6@7 zcVME2T+8~xSCfQXy4|OXPD&KNuwBRjp!E4*Ln{9B91U8qY&!RwTt|yvE(XAZ->_aE ztnf3o+jQkFdK2-IA+h>JeYCbwgn&v?*$_a}-UcODB_1mB0O|%!lwRw%ke$av{Ybzs~5>afzPh_l+zPgY?%z*fU z9uzRniA}IT>?c3HwzkG8Bw#J}!&2GuLhCo*M0R#Ck2owG?d%r7=B%+rF7?Uz0ey+X z*iz&%?Kun;k6zu_nT&=t?5QtBJA?Ciib_g5&bWt%mtZ<91uqqaSc>+Nvb>_Qo9aRy z0(#)#vry5A%&F*)KCwe`0A_i$7MY0WLj;z>mU{T%ZhpWeI038f_0QLmdU#@Sz?kBd zIA}O+Ku^0wVQ^?FH|sJs;C%+fNFsgfe-(`_;!n*w>c-|(v?PuaT;nVCS|w&gD&)60 zqTa7Z;_E`Znt^=4NtgQDS5rk%b#8Oi8f2;`E&RBk-~XAxSQ$2EFQu_QAm7eVOU$EkoXox2(do4@)Z1)TImLcvZT0>i`&uGuJ(?Q z2Qbo~3SRG(;>M=l>)Qzbf%}C1;j>>#hq59vEcg#MTL**yiV$N{p|bg*3~D)R$2$j{ z4?|vi(2W&Pp)hFnKR5WRldt-gcgfawn&!XhP`qua-3EyxqMGRD8n$w zjwr}gTA*(zS&`xyWf(LF z8#W&qRnvgkNk>LFP&^eRSN%CdhplN@_@lq}=#0rJmAhK(S{VZP)<2{9Mn6%H+rHeo zTl(H-urR8&aRm4v48xAy)3eeuuuA{Ykys`9uUhn6nDWoc4d#)l1E+x9KA@lXy>M3l;Za=;mgge~Ennb9KWgA@S7ve?h7qRw)f<(nPqr)qhf?djXZm=$Fud zp4Z+vn5Jh2U$R_i-Ow9!H|NZU_&h&FKiuUFB8ogEXO3R6m_8d`p4#8)1Xp<}IHe#y zo4pZaR&efl2+SLy7t|2x3R(`k5hwgoAUe*xfJpda3YPZ!5M}e?GZlX>o&BqX!dFRqh4P-hH+<)cnL>pI0naDbMs>3y_PA7RREG&C zjY?jhE`=J{ir}K@f+x}{RMxvb`lHn_6rn$1zj(@jwrPKb%j|a_+Brn5TDJ?8~_rd-_a!-cv;du)14wI2}KdO)cP%|VY!XJG0#d$4QdT8`!4NJ4el!b=e zQgLMq9(~YV@^g<-4Ld)-TOhFjv-c?*@R9+<=uCcE6{WJJEe$25yu4nb-L1UhVmA_= zxy7bXWO4B8EC3omKmuMf5INe{{Wd|nD}-GbFAIxh)lsGI%JtT7dLVlf&(i!k-EK+G z|EzQeA^dJ`epGfd9w>jAw|;)gmAD8M3qx#bb#p@aM{JdPk+j+RHy@R)i5J#LAO0W8 ztN&cgU$#j7zhMqvG;vJiyRqXFuy@3pqW<-mS0 z)fl`QN4(C&15PU|X)x}t1n&i30VT=7k-NIa>6mAo)9!~jErQw@AYo2al2uzw(V|@8 z?(c4@zKH?Ceu})TY_|k&N}o!f<4BJ(&&*-ABvh=%(H8olNS&_b=O5&{=xomzM~84EMq*`c}qt zRJ1xZ<-0qk|6%eC2HP22_AmpoCJ3@XmJc7-z}R@Xx9uuI3s!yWC!noTaT}FwK>|O;Iit!JsD%r$ zWrXdTlujGkSglJZ>bP7l?5HUmWCzGx<#F>qZLyRQMX@d%)y7Ka>F!a#6b|l}nvjp-L+)-nx#YTi8H3``3Bh!mtz3IGMO9oAwI!dIUtl zEzza3G5(F8)A$;89&R0edIC_GL@_XDz&GcSaR1AeyO*QC1GeKPuxyVOS=*)l%S*rx z5!&KcY(Yi^$g2&+g#dQcm8jcEbqzJI>gy#Q%dx3?{euHQ-xX=$H!2(Y2a#UF#PM!& zX!k=$rKNb0kxh#3R< z@*jUZY6!y0Ap-&bQuBMi4?sQ|>Rr6!^7C^)zxzzfa^g5Pc|krfxT07AMvDM59qqU| z^hGD6LXi9r7~_n}Zb!Shvm=T#3U#eF*pT&<3?Q$mpN4XZ(r=xjWxr!8gPik#EUzC#-M4>;$-+m?lsRZbMHB1hSd7U!S z|%`Tfxw2Maa_|5iC5!Givpt|9uSF1|}8jzFCQKKQjb4XCHP&roGU1QGS(?HNGi zltPR&4oZmVC{_^tO9gCGU%=z!rsrTXy$2NY;5Ea)sqa*%e__3_g!ym42muTi%>Q^8 z(5KuHB|#3brKijG`%b%B{SL`imbHeeK?a;a}_{v~Dr^w*7f`_L$DV|?_YelM^_I!8CQD3U|Wu+=Su zq=Mjz&Sn9N$B&<0z$*aA3e2ZB0dDUM_F^kHh`wEC5M$<;l`FrkW*55AeyLXR3-^So z5s+no_tjK>Kg3_K9kBt_^Xr#_pf^9`bsy_jk%GK3B%YgGV_<>buHvYg1e04;>156k z^iot5QQ6%N%NC2aX!m&DmbNH4x0Me)rTZ@j zgQN$2T|vt#hTh?KTiU_xdWMFoT3YF8cGFkBtI{5VcqH35HXjy<>AL%>-=A(ZlJ()x za$zumLcuM|D}t~f`7zxBA`Z`q(j5zH8f!n|AyiHezJHsIpNJ&1zYIWSR`|Y5m!e*O zv{|CcNxA<26OW~fw-ORn2>4$UgM*_jP0V>UQjxKCQN?90rJySWVxw*?b87!Q>OVEs z_DY7;P_DP*xA9(VAwiqGp!#Ymfd;?<$zWz9Mh|DwZ&u3Bo2&0bi;L!Clw%&O(>z+j zc|Y8*0S@hL`5w;L!$?wTzQ)b!x-vM>`XwHUiQ0+NX}Q>6;kcEKvW}_YXKme>MvS&q z`;U-;h-c%WxDE7YzI=9YP5k-_4$i4obEdD=`3`S+#nD?_1$ zU@1@d{}A`qQE{|iw_p*12M>_o3GTrif@>f^a3^SRcMBFmu;3C%2=4Cg4#8c6yF1*6 z_x;^()}6b)nfuSIVKtKRMk^wpS|}voO>#rKKGtA@Yf^S<~wNcT6g*G=yUVQ zNk^DSW>kd_0Zq>^ml3{T;)WH}##?7`Ww~yw4&nS*iRWa*J09sXk z%{FueRLpuK4DqlH>=|!<9>>$kT*JkMj>nVBE0jo{?OLNkHafAmRr$f3Nk^LNSl2t( zVVIO2tQ4g?zwUI^A!jtXDoCed2(iw;SO$V*=3e}n3$X42a@|Ywod3d(*shqtn!8!VA6z>78y*$<5?7Ka z*g7H82do0)PawSw>|xv1j&I!i$W6z+l$Q>+TS~RaYNWk_gAs zm-;yjxZ0ifGu}lzr>o!B)BeF$fe7U=!+;*XV@~|0!4Zasi3Gu_rXs#ny%3T6j}})$ z)=YtTDjCmFqc9_Mvz?##x!4@lq;QkEH!}hx%Kc_X>-hM6)N9710`9H05^GHzVMGYk z2J+3$yr-px9bFQuE?~_U_DTK7ObHQNKvUJf_hb46n-3ch-KRzkB zZEh>|6`A)Fp_dzQ(!o}ub5wL0olTI^{8++GjfV(>3q|wO7&;NP!X59dW4Gu3NZ9bU zym-su1I|_Vzvv?#TEZF;HSmhWPu?!DYsKAH+P;~4;oS}kz^2pskp(tu*=NwuIfmaqN`ZWw5VHK!d7%jVbwfS^$l>}%mfQ8O z4F^PeVIX>)E`&@F<=?6Z`e{Mb@8l$NJrCK95io^ywv0G)aph!NrWWFDiFwBlZAm<` zm7}`a5X8LOoT;9TFRgzY+OzQx^vZ632!)rp*MN_=4heAA&xWrmJS^nhdkV!nsd-=n zZ(wFU=o@K{rj_)7Im?W_sJzI+k80f-=7Ppn-e$0!Y$2>>JKYB9-m)b9pb1)K4qvUi=|8+|blaXxLZx z@BYi5^Fc{fN$C+USg!0IZf}1I7Jcn?KR5SY;qrAl4S7ZljjF-wLuj4qr?DK()W$nL zErtqw4}BV@*PB20D)pQpAi6;Z=R}tLg_r@Eslzxg{XdcxN={Wam9BAvGS?1yg79jP zM`1rR0aq>hm;?QsCspl@oQ0xQ;X~z8;=4!o{`WSqYP@80$7ubeo3qYGOy~IU$trWs z^))0NT%mwdeYH$CDMYuv)yf~~uw&)H#X)G1<&&dSO=`F(VA=0LCo>=e5qi z!5OZkM z_3S(8Mw$V>vDuSH!jLkvi`QU2Bk5AGBN2TN^kFE{w(dhqqDsR zAz#_g!(g^-s;o>{#AJaZ3&U>-3AT7QgS<{erdN*?N!lrh1E1K~+t$0%o#;FH6-i(r zi+M|Q$i3ivO~%?c&|$lfPgDA-gnkGF$Z}y$qfa_hL%*pL^05L(&)?ea9YDLZ z8@~pENC+O@O{ffZrWO>f8u<)XJ)zLbQ~)~drVehPG+ucr>3F>g`pw_!{2o~JrsHYM zD3m7*cK@1I8INxq-_;?shSSx@C58OHEk0>N&<%(xeHJ;qnf6MF9vyqs&jX0~c=YEG&Cz)UxbD}u&Lt6dPS}V_ z+6QlUXX1&mibm8+=16~d_mp z@zsc_2K0y}FPXIT(~tZTFjG|zUn*icR>Xu76Y+`W8pP98D=I443i9&$`g;+Y4NB-4 zzL|eAFSAF6$v)jPw=$;23bsGFf~vK`a!S0EW2l>oU-%^APfoors!z-PqKo`iNo3%y zZkDW?f|eru+mNhZS$WEew9f%+G(5g+W^Q&I0@r#h4PCp<@Vz^0P5b_}_Y)KdUyHw; z>k4_4 zgFLTKOWX$1RbGP+LBnwam2Wt)Ld@-szd|-#w8oV1^zml>cu-XojD z#y)n85|WoI3VlO88yl;xx$;=I3e4=h)?%dM%_`2U#Pz93_$9sAn4o?I+~yD%ICc8i z-J;)q-Y4L2V%KxoUUdLf_}n_Dd1BNmfoxyhLQ3$YwdF#sdq)!<5L!9fpg$hWl%)>m z9bQ7u*8Z`}y+34sXCAl+G<9_a^>2BL&O05d1V+TcACLesb6Ss9)3V<_tuZ4R6m+Fs z2kGyPk4$*apEG+G$D@kZHpLuH3YM+YU*CnFjEI9IlRtHOb!S0-{>}}*76puucdkHb2Z4X*Sxb>SyC90QXUk^`ai6G=SahkbNH3m z%Tn@GFV8GH&%sgm%gFnP>a(-6pzwPGE34jb!1JwEM>~m~GmBDQPDs zCsQxa&~cN_7SfLjygy27jfh8Ar_+r+pX00aJcwb_Z?nfq3BIU@^Ab6&D_0ferhcK! zNQ0AZVpzk|)WZiTjCH)Y4=O5mUhAK`sH-w}K3|`1oeaiD($HM=`mYf!=nA{R+%Kvf zPv#wX-OyLg?G!sRiCzlHhKshhsd}RrQP9-6>YY(73WjS!?5*`mi{3S$m+@EQp@IyL zvcnJhZ6EJT0`?c@Opmm!EUM;{M~LSsw6myOcI;xUaqxZ!+taos4YY?B)C`cRBbb-?MDj znn`%}-D=+=?ZoM0 zW?XXp-%J)`zKMYXD0F&1M}r&o1agbSNCrMFU&K0{dOv*AIE=l$LcTBI-_p^AmzOW; zY(4$?Ltv=UD8qy7hV5bOmZ@a_Dlc7gd!xcNuKEFGwbB(CYjSw_XLhz4DfLJAh-r)S z6)|51l7Jy5Oys`MZ_jiZ$Utz$AlN!iP!fl!i>iv?!T$`3@Cz-G%okJF^>=9*MZ91# zkbpbnz3&tLB1lMw>4@e11qSzX{gT;JC|i6?;r0XCrryKNn~yM#)t*SpFW(aNTNL+T z6S9q7e!C<#))m`)#+BO#_yj)e1p4?b;cgGSB9Rqti+-3O)UoDfGzd#+N#H3*ejc9Jlno7G+85i5%Bb<5V}pZ=W$V{+-4}G!m(fI+z3|AKkOdO&71WhG`Uqt?X0CHK2YEetQ{Z>u+|^ZRN)zkBCSevY z0&bA5%R=mrRD^Cc`Yeyt)P3e&eCW;PXtck)=9O`DDJVEIWa8O0e=XYD|1osn_s z*7DT4Pc&|VS5)f=*>RO);TUvdbq^a78VOISGsxUKW$wS_=K64Xuo@J2g5OFY$&K5q z?cmykP_=9rx^h+x8pHSX3@kQWQ0ZTxP>LY+^*QfxtAScc>=VDW1(;q*TmGVQp?`~R z?C8=O#D(*~-dra;{VCvO?w29cdcPNVCx?uz^0uUWGLh6zA=SRXl@UjbeIq~g_NS47c|>O%&q7KBT~_4es!Q9OIF*#yF>g3e(F4-NS;dg zB{kB|+r?`dym#NCv4h6Hny-}N7d`KlmvHgH^C6hEM5v>?T|_Zfh?HBu6&jwshN>$o z+kOZl;%A`y@DA&{pf%<9#L7^`Kk*4xf9Ts%Z~onvC3-K==2}XyMSyy1*_E;rfaQbA z=1d$nV9a#wGN`Ls#GsCw7!#G(u}U)sG!CDZAc7|JEL5VEW`jSLUl1dy@*%Dv3Acgr zLy1>!-W#L9l=?DAl=!*kB1_aL1wypcH5Lcg^HI}3(lh7*p${YPPM$gQjV)tc~K=h?odTpg}H;!8LJ*5`3R2^=_a3O>Y0v zP^=HjXORe>B2caRh95OMC{(#^ngSNhW7e*0xq(LgZCz;(Sq`%p#pYZaq>*p&-PCrP{+XT~J` zlUnaRA2BhrxiC*Qv-p^M(#cH~mb%ncl|mo*x}{`$V@ zuRpsHybn{YWUx-~=r9r9pYC~gR{sue37A_5Ew%W|2UV^jkZOx>VNZHbAk>``f2T|v z!=Sp!Xz?8%Pt$fsXNc$I>ra<)1mQj z>X2UNa=i*bQY{=f{?WQi{Cvrsl5%itY}0Y}UQnDch0m6d!fFGc5r1mD3_|7D0$%Jw ztp|&aG`JIE&H`JQ52_&=0%J$mS$li%gGV<@va)j{OU)s+ss@Z&Xm@O2AnxpM1@wBg zxHk+|jrz2RF#M^Bx6nbGM=`{^GJuopir z@<)_2>z3SDBsSw9G9yRVnvHNQQ$)&6L)^Kf(+72@KB&oTwc)vLR+&+>u5sTX=+!)Ir6Jm1mEB+%7_f+T^Ax(j8 zmupybo-OYx?1d=G#!Ugztd^D*GANv5vh2}RxQ_`i8>!K|xs#G3kwwDfm>;|??$20r z312#DUFU+KDbJCVmV+ZzZ9;x3Y@7>@xV!RrAWHDmEUFesr6NZ>y2L~7p=UyKtiYNb zcL*MQ`C`|$c08yNrqAmWaB_Eg>bAjPhWM%>ppqutbM4E9J3{QozC!K>iwF}yG3%OSMx(M-)OT; z`0k8HD{nuic}>KE2#~i>-oHEmnc$4@gUWC7!WVL&WPnR>`}oKTNDFCbHW(Wi{Ii}t zE>DHLCkpRWFZ?is4a+%fcXoE7tQJuZF9v!BWEw7-+ovK1GDnTlZxpv=K!RaQN5j2w z2iVSIixSMt%m89VCDe;bwkr~;6O$y!DgGtQdIUB}Di2GQl)8_Jnvm-!$;nvnNLp!W zX>r72BoSG9!S%>3jj*Ki@k)~vK#jqZQOQ9(y$h%A-55dXf9tT2v~7JHbR8D_Fh&Xh zU`{wPOv%i&_??(b8_E!66$Y+($*SYeO;UENJ*B1=2@duSJw{;Bh~f$@p?s{^$G%6F zmySKDnu6lu-b;Q~orx-rsmqK7FoX zV`cY)me$Deea2^&S>HPn0XGBF?8OC7z`5>cW-hIl?70`EBN3$J9m>{kFdPyHdqX`T z?3$|@>Ke)X&SodE+L)0N{%dRRIDza|fY9@`$6c)8YSH%AmV+S`(w6q6_s1UHqQ;xO zvZ>?TDE!&W3I62t-F4sbRF4NfE0{6F6FJX^wa>7Vw2Ax;vBv|gN&%?hS6#TZ-uAyc zoNG$vo$RIE9pW5*q{Uwid2bCp{@0Dh0+Wl*VT-5jJ5qhBUT2w6cR{R~Upqjtg9>0s z`qSmrkbLN@*Blh|x@xo|3(-}*r0dmb^Yb=i9jn!`FO|EDWlVx_=7pb6;48ySO@sNv zVmeU6=YJGCsu3(mBH^^}-|h=5w_#tX-xyCX^3=O&K7pcjxQbi)Ut1&xf1bC}a{Fu4 zU|sC)U%pm)S~5F>vM9&7KV=o=bh6Z4S9YbVg+@H+`6m{uq;(xnD<-JJ0M(voWqTJ~ zZf0kAJ0^oJL;dY8uHA9=8y+u~4*CrK9HR*91QgGv-mmJ4ydd$qJJd}O*GH{{Z~ifA z)UYi*KA9G7Tyf|w!+YK2EzaN~qg z-~FC;EB4yea)Dz1p|#plC$G@2>#vm{LEmgo4oKTLjI-FYis`N)Zp~NTBC1-q>xg@> zYY`o?D?M4)`$&&vt}uc3O6QQR^)6&e0p>RBp%h~Uy-Qm3M>~muPXH;C3kxT5&i^S0 z<=i(#k-v0>qJjC4UGuy59fWq(r)3wh#Mz+Hw)#9xgUZ~d@EA>C8?Y_UK| zh$d}XFEsEY$;?G&zw*=n@3`|0OEra^ZJ31LPOU&REtB?vp)k+T(AfH}=?ldJa(O8m zLNOD|wU!0niK5S9IWw`9W~IcOZ0&CwT~?)4CKU#d#F5f``eeDVAqclWiD9-IBe?xP zQOAk6iNQ+zd3W|IjMr$VyKDK41erAL|JOM7-HOUL1{3|g&@$tj8mS@^>(=Zr>=)^u3Uz$=IzQ6R3^ zcqD)E!Uz?5acd^s3Vd;_($j@@cN=Ami~CoWVVlR!jP`hV7WW+2RYqwtJ4CMwT>@%t z39MZ`Mr!%HHb}Q5hP0VSovvRP-xCtj-d-Guw7H@#qoxz0tTEQsFZ#{eG_J1|o)rRt z2oTL%r52zLwzrFk6PXYPp+O>$*)aml>x^2{R9>_3V#diPlPfazU9?3!rMv6o7pH2$2j}7 z0BjhnkrT$M0r3J6I`jf@C`9skE0RrxMK|lDkeVH;2dMzjXaZI&oU?m=HQN1~j!@`* zC_|P$E0$8?oQ6{JS8DBBBY5aNjL)URrR(R(ulAxc8yvgePpC3~bY=8x^5ZZ8+}jkJ z-|0Kw6r!%Y&6dzfWOo5O|)2ugAdz zhE{3uv^%E7hMx2?rea-rGuJCLWLmQwRuQ@LEvRi;bi?a#*w%%E=)8j>14K9= z6d##6y$l|OSXjzP_}o}J+fd(PHvL`g$cxubUt>CH?k}$x#SwjdbTxGeqWw2Q`+xNi zepaVDsCQ3t*IYr_{S*V3Dp2x9iD71dATrE!I}Rzql-+aQ_^e!;XkO)PR4VAL9J@vO zVu|CDM&K^MaJLuu)y)pZimM|1UL0J6E9fVk+TR83`cmM&CJ<@KFA*f535Qc;etfMN*m3Nu8C!!J(sh!@C%ETNsbSg*pWZIoZTRxznL6 zW4TmvB4oL0J4+CZYcV3c?LLN+eFQlmuJwE3p}QQqpTTcu>-6-!Hbbq`-8sNMKTY2! z)<@qVg#X}w$kO_KG4;p(F3VZUn+Gun_RA}AkyadKgW={*&Fs$z`KNw)vcc1vpIO8t zKfoXiEBZ>F<|}-m(s*N?$xPrf3HM7Q{+>!)qs10NLv``udxel~J|~6y{^2`GKJm%| zhxJJ7fXQ2n_O^A-7jI_f(-PtW;SZPv*}V_1tHldQ#H+lL6CAv?!m_uX8>Q@>4$qHn z-12$PSaUWAD)zSHfan{$T(%|AX00|Mr=%oO$jofIf}jW3nq;ec(F9ltSfzSCw7rAn zYWgXuferg&#juHu>kuHN5u%C;R{)n}y_DuEJ3u1|aCm#1_hmE*cM;BttKpj6VOgz~ zy}9G{;h!%}+C5c=fvV8$p8C9M_5;BE{_~C`lU=_g!;oDXaMb%;9OBZ_b zg@91h(iF&pB^aYSCFHyzj*E5ej|)BQm` zLI%3k_%M!=^?|Qd`rhU`Za?;Q2YR9Ha+jD2CKe3| ziJdxBb1;AW2oC-@SjMv2FoJ&pkX%=YDsz^55^>8^_fNAm{O3!3$*{)T#^RVwqCB~L zgZ6*(l!qGTTCcp!TRHIX;+_nTj;eDl7Uj8*!HktS5LULg*5Sf?VzR-65kiL^*!E1z z;Vw_#HD4lia|dI(>$kzXOtGQzdD;}|x#7P@C!q7#djfI%#&AO$J#X_twU@ELpRo8H zz53!C_qPW0V#&(l388L8|GPjUdg#&vf39ogUh3O43hcY56&U!rk^61`^(d3*EKRxN)nRIZwckcfhQ1`M{r zklqc?6@>*u!S{dfb^;;K?Hyewt{l&FlXVTv#Kzg8Kx`vdJ$5d91E9cg#c_Q9aQRzOE!^WX z!X|Wp9LpT>_WE(p&=Vv9zZ3W0aMy}&^V z-7m*h^Fh2{wBYvhYf7z*lgjvN2ApUjzS|QC-!j9Y<43c&T{M@i<_qQ6whs-Z>FkBO zM!7Fa?PFlRoe(1F)yD-z`Du$ApL?qmz!p=Ia~WQJPXGJCcom@h=T(oQ>&$%CEKo{0 zky{LLOTcaRwG&uly7_<0rt%}(cUS=E8c1y`1uBv$owS8@7w*FW4Mqr7-09P;1odCK zo)?qjJKFO=r)?ICr3>SS(Fu~N2Xpq>nVIVyNuaFG5{+wGu{Ba{CK|= zjMC=7efO{@4Fb`4;XZ(s_0V{==vL^9eG6mb`kgKjuyUenalir-xp@dn4o%FoGhoLG zdL=8+ViZqXq+}X5d8ugd4WRA4d{tm#X@+^XV4`!=rN|%mA@lBN+j&f!6$0-s!g*`f8eq6XWAX#%r*8C2Fy_o$M7A z)7$~jhwF28PS)qa?SRdMv>3a_)S7aoqjlAuy?qyIZjr63qoCZ*1ot!Nm+{XcCCYqi zMjF_k9izjeL`>PXL<=dGcGPWV-1qNQoJ-Lx5pLbsVpM;^kQJ z!Y4SgF_)H#sz{Q)5Ju0~pm{ml&ooUy&G0WV+H^1@>IZG&y1pU zYuF_ioxhf(`dN2|DO?+Fyg_RsFc22fv@PiFW^aq)GhmExd6T)|9&yE1_pugqE%QIc z_$mMIgmVEt6CDoKnMepegnOTM(Y&zY`JhFgKX+J%ax#+yC zBsaGZ5Va19ao~#>6zN>HC&$P-?gQW2%+7P41j1jRIXd=#1o~uF)~NdW&uCkukKzV0 zYGP%^qU`K~0?V0uax$eg>BTH{?$;$QGapFI+$}9FUB$#C&LSTRRCqBt56}?M^;+RUSbpx%%3z*32wi)BCwXMR#}e&mXm-lAJM-2|w(p-jU-vJ4$E-^esQDtg0ww zJJ=38Ws9?pmgtH`rXw$yOUY_EKO1?;9?du+ktn65gbiwztBZ+c1P1azXc8Um?Gabj zqoJU61n40l{zy^>$<#z~v|45thB>J^rz=ZK!+C4h2nEmX>(E-kAkvZlVa|xIemXFa z3@RR!l~l^sd*wAYHUiCsh=^yz9<<(6ngr@=)YjIv?^IdjGbA{Q`k(5(E>Tlo!SyK& zt1OLP?On@QIgfFE#jjYV+WdmuZb~qi69whUyYu0P-G$dE>USkYrY>CuT3WeU&kG<4 z;30NmcNPc|BY5`9o1;sA(OLq5k7b{r} zO2hGR{?U;1Y1ES4fplQ%t@%RL;z9r5;1BLqesFTU_Of-7E6m9WJ~*YBp01g7-qoys zGYq;uj@CZPOQ>`3^3hm0S*NS51{NRYF6FybAq#|a*ogC`5=r`$RZgtI)jvNp~2AzJ^FpvqVxt z(|pl)b1k)ha$+6nY3sjvZB5N2sycNr;ZHn-s2vwID+*16_%xB&oyioH#W zLZ23clJbfQol>UG@8(TOcf2 zl&w8ET2U@ddQ=}p`OSl^wE~D)Vc$CRgz)9_B@~cFPtM5!Z~buS11Nd^wvD(cL5Yd# zR>awCuYvqF&%F_OVGyb2aL-{ikmJt}vX<18-rVP(8X7!KXwgHAt-HoO<6S)W@t2$M z5H_aCm?&k1=3X0FU&29I0!M-rqubAWEzFko{3Ix#dfE4`w^(0#`1+3iQw zSZh0RyjuCEasFm-n8w}2O4fk5^ODKR^i=m*^Z7#o8iC13@c`3RrOmNtN;^1f%va|< z;LL{;wNeZ`aI}@YaM#F&&sZRjK6+#HA3QbT5v|6u6R%KHsEKz--^@H z7k%r^`fVYdQB6LDg7w1e;fzq-%<_d7d$d=qM=$|a=o$IJc?APER&DqYh{Q(p^#e3} zC#pfXR#O9KM{&^Dz4_X!r++IdpMD^L6k)rFasgRqsmVnjN>Of;HKTKuHGZ2k-8t9q zZ$`foCy8Q2g-{3U&H;#sKZB{M4ocTLx?U)U;bMZ;v^N`y{q7_d>o7~tQu%Uh-0gl9 z<*dLY;O+^!?CxF(uHiRyg{ABWHZbt3?EY|l*X4yWSTH}xN6T|_#Z+{6c*RFXE_TUu z@4i;~HmWR@iPxyXPIGE0Ff_EYK!69$d2sLUMq2Tihu7sC0T`u`h*Bxc``|VWoT|6` z0lKt{NVcOUDQ4vdZ<1u^cs46vEu`RjHxm2YUsqBxHjmi8*h5C1Cze*ncGfjKt(kJ! zQmXl-2ZUV$OAA%71CBSNDtExW8{+|DLp-X?aF=5c16! zS$7LF-H0(fi`#Xj`X{u=%4|F9B(9>O0)$6<3zcZiAh|Rbh`N_e0;7NxPOOE$z*M~) zTeLcO1Hb2ch`PBQfWy#D}ey73(}#hfLXk&-`JGmO3R5Iy?QgfyykbL*=px>+N(C3lQRU?2 zx^1N{^`mjLC(XG^-q^4+PpfEnMgo1 zz4x&O4=};?EDm`-=L+dtE3fBQijg+gGL~59dg5yXG7tTbqnuSETm67+sI#?ZS2~l@ z5+y}eFEJ_$JEIRsZmcf9!U|rao!S1RfZ3EzB*?44lIB!*IQqYVwT3YIgVu!vyyt9y z@bl@B+{!!X=XJ9o9_6xMJ?VU(+?rA<9NDY0|-*I9m!y88M!MX>{^B87q*tR4~K_}w#EnOwQ4I;}I zBvxN!L38Fe_*Fc+)0CmTT99Po)o57|ICeZ;TUYRXxDrd#d^s%wzD@b4K||=Fr8N}d z9306>R~f~VFh2OUj*GZVvnM09nwyGq=N7#`s{4^N%*6U+ZIJ;d=Bf;2ufyYizN6u@ zrmvZe28c{^yN*OvD2|W621Pm8m908QcS};p5&|mw!$7{4nBVS2nYiR~qaujHo>w(Q zElVst&{PR}h>E)>@pJ4-u9Z7WUq4%L z>6n%U*7SmX$?P;*exD8wVRTfH<111_P zU%zo$D_yB#H~R)1UT;0KO-g}MmpX4=l3R)lDzIm!V9#Wa+$brnG9L`sq=wzi!XW?v zq!2Q;BLnwi?&xO@ycSS|gMF<|Dx-|LfSFwa&yc}N zspCfw%+Kx#Aw9Ka-%(h??i7cn%2FiVZps@MJFj z)Z;3`*LkWX9r2-Kxj%0*>PaM*{m1Wo=Ew;Q9IGH}5-~QR#NZV-* zk5FY*6qo*I$gy53K(qoD?aq!qsOe+>#0RCVZLbKlkt`5|$}Dj4vrf*G@~cvO6v`Y=u8Gtw#`~%N}-&ST*~Rn3p+X=Gk&YM>>>%X04dans|oaR>oxsSLu8XBDP z2U}iosT`laUvOr7;aNu`arJHi+LGbvb9sJ<3X)nVw(o98oQfOvbXBdswC_2MccOKy z*}%`vIyo}Ak?J!2*o;@y7cBbX!VsJ;Q0CWx0biMtU(}OdlpSL{Eo>zgV}wCr1#3-3HN2K*O!_^U5CcOqg}=kc z=xR~*wdX2jH>K0`xFH?eqgnE*&OsQ$IzEG|kNJ`RV@R#I4t0w5on}gH@vf z;ceY1%%i60Tp5TE^c0d375dO11JIxvy^5253^@%eWD(^Yow5tMEf(eg>c_BMnHQ9K7ZamNBtMdiCQQ zNS}yfChKGi;D(|3-TnUV8K?p($X7ASFfJ)eUS;a8HEoCBeN*e%wg znE(*Te8Ttomq_Z~`yQR_ws&m_J*Im1(KfB`4i#z_gOrS%1qtuyPX6yS$v-3s(1^j? z+_rC_-8ld#;|9Mbrxht#Z{+Up+h_%H=Mb34Q#~6M3P(c0Mi=)JZE$0~{Z}&ty@}Ze z(M9h0DJ-ezrvz_HT_i#sD)%h{(%DncSE_3B(g7= z-n|PzqBT$6kOXt?4W~tsc2hxC)~Ew4V_X|K4^Xc>qJ-0Tk3%`}exiLI@1?f7GJAVZ zO3r(ugamxNxM-`XdUG=y(!?U$1p9J`HZK=E()X0d#|bzNzNd5nACTFy;Tu10in()i z;K@lxP?~6uk*Nj)2R)xCMK%X}dHbFrOSKnfCY z>Sh7gKuQAcow>Pr8JdH~KVEx!P@T0TU-cysFLq3qkktX(Q+m+*mG+oE1!$y1*GyYe zOYzgiwU44uLOBH)>&@ST4Z{l_JOZY=x82T(;in~UNe!hnp>rqL$)ExYjn&GNtj7Ts zp|M>7w_PIs>QL8wn-A)w7y<*mM#$>ZWi@tXAx1;3z2l%60m~MRPVS?1qC5bLY9A%T z#Q5dr&L{H|!?J|P$OJH~8M(ayY4}y8xw)VqNx=XePIOLQj%`iy_9x8oM-V-eCa*gG zn(c9R9XXA0w&qEvr)sv=u$9WGO9}yirJNaT@(${3K&u_pU;27o@kJ>+b5fUg_|L`uT{xI_Z>|8cMXm#yQ!&0&HK z`hRhR{6BuR?E!&fcLmmhlHex ziQHm}{;Khz+q~Cx@G6BBw&<>33GJ#+Hx>RdLje*|!lr1iK{>@TZPLW$4e}%5LbovnfWOZcwuk8xlUn?trvlS&{ z7iX9d_~Tb7QIvMnFFn~`$n1yjsgLN+v^ai;HeTxT*=atrXZX)1{z^EqVl7blTbL*J zFyz?5|M7ltkC=~ZH;U%>GpPA>1QvsLg?fG@VwR$GLPEr1^{OTrTcAkffA-L{vOuSS z=fyK(0S}i-6mIW>PrJ-RkxAyP#9G>yMV*m?#4l{0t|}G5(w9hA&9a#h?w)q09Ol5< z_Ci*i$mnWt#L<&&Q<@?F8QL;gjHf9ZCUsw(*R+jhGF&HynEKos%y#2Y&$C%@l$1pG z7wGSBD35%ouI*Lx`Ne;CxfzjM^fvp;CtX@4IQC4b-UzhYCX-8jCrA4H7BOMZ&iGqy z?$0mprN6mum38;;beW_ zWMhbgkdTm=xN*OBhmw-gf@8Kblpy5mSD;LShlj^y(vd}sBsc4eWe^b&Df1Lo8WbGG zr??+QDXk4`Xjv^@6I4GO z<@nZMqn67Q*49zF-X#$ri0l2?&dXKgo7Ht26_{1l{($%jKUBPEG9-;_wxXX;2!jnj zR4yx|!{FC@N#*hB!hg?~U-sC~;8Bo~xx&iS=(o1E@Ixu_Ls>dK1Quf%G_C57I=_8; z=GQ_hh4rf4l&!L=DlRs5#-g&dRXDth)%*Uova&KPEKJk7!Ld@_DDh;dToq>;`Lxx! z3YA;fZUqNzi8jO(_IkYWFkTjhREG3HAIAamM$Zlo)p~x9o4&|wfA)I%r6Bj1-C1Ab z-7sE(yyrTBX4{{f6}H-;hfVa5t%bU#a{{pph*WYW`t%!<_W>1!F$5i6@SW-k5)nBo z$eHU@p68`rs|AQX@hNOz;=i3j`BQxyn@eG@(8Hj6ITKnV4ncZ1IuA`$I#X*X9YbEF z*UFrpd#gD7dlCrZf8po^UCdo|7Xy0AA1{auduxkIoubK|vxBV0>8cejxxDdXJ`%*I z&({hT8UMXx7DidlQ*PQHBQt31N5317_-(Hz+yJ?% zXN|sm7I@+O^PS|kiZ@9EIz`Tm_opCi24~;`f-EKQr0Pq5%9YEC!s%E`CVYu+$3r!g z!lBI=C;po`X@G#kLaovwL8pj>h-jd{pU;&*I)_{e-cKlRoTc;n`WjqOyGJgaot@Mf zU}4s%8CxQXn-egO4-5<(c-)-ymrLRQ>ZNWRTb1K$U{<%!6Z>GaMv_wPsc}Ge+P!=A z(=8xYD+@;lF-P;`GsvlH6BR%0m5m$~?4cf;C>NgV zd8zw%y^{|tHeQnk=j3sWTI$+{GVdmH29d0jSu_78>U+#_k4#RjeG`3w@RLFF1D@He z;B$EBC-ar?ZF=iTy>&8usiy$9wJdpUF+M(nf2j7sa`I02sZ}-m6Q1KkK#|SMS5AifH&pSN5 zZ!MjO^Mt-`ksjhh0ok>>fnGTI&rYE%oz}8qovQ|3!8aw#lZzh9W;Z3qR_+_byihbW zELl`F$vi$e0JC5Wl=Yo_xccLabfLHwxJz9)pOU?e5Q0r+kW<#?*8Te?Qq!+ z$?5$~S6$hDgS8nY$pz4e=n5Q9o&1^V(XL+8b^H6LF4ygTr*eqX$m6&s^x>t)N!RR% zfaAk4y}b%S1WYcZJ8aE}xReS%d=+-94v;SgH{^Ey?BgxDX z{!qyl+jVprpZ&vrCK+VAcsXnsVKmTWqFpgWLTM+fsu=h5C+3)El1?Tb>FB45nJ5*1 ztncL+9qX>_Y12#>9B$~-t0|)w#&&7rpOE37@c3~PpLfW;q-NYyDi}#0bw3wD`QDip zSKsK<>oZDeq`|$BPUp0|QHv$)z@tr$)l#VOIKTV~iS04eC^H_&e3$e(vYW=bK(k6N zc5S}SIo0f29=TLPLc;3mYDPvzoVL^S;16EEmZk0SJSr-xo9pW*zRgopQ@t^CBEQ~$ z{0Q8FV8)+q@)m@?q{XMC?Avgy991POxDbE;ZgJs#BsigC8;L*QJ zDc*vU4toy^r%Mlsx^COQIL}iX1L<<@TJPkozQJWS@m-4RuA2CpqsD%MNJaZ~y;7YY zelOnP&bd5r3VIA}xDjyPo1L^v&0;^H>YT!Qz0P=}b$74pb)k9ad(-H(K_xKj?J&B! zBjJc*b${h7Afe;E%*!xeZrTwic)G|rf9Thh;&Rlly2(d@GnDs+J5VdKEQ*Y&uZ3WB?rc`d9F7wbEADACvY8CrCs8K! z@b;I&f84|V7t*k;rFyGKu{?0X;M7#G4^WwP=jCH$j7d*V$J8&LKdJ@4YHEx%q6W5G z-OgNFS1HbnyT8I{R?O{PB_=0->)8aS--z1IoitZah}Es)cl`ZbEN^IdI5z&dpE(`l1#< z6PHmaJz#kkdO-(5(psw0B4s3fY@1CWC-8f_c zE9NMM05T9AR_m+Thr0Ku`r;WClkub<0se|&m;h;Tu`7$3VNm@*i`V;sR5qn}q3H@w z#GEJzLeG}WGw++ts&gdo!#uT@jf-|YThB{=0^jyluj>RoqiYm~r+zFR7mMsyeG!-_ z43xlMKA0~8beLoLX?6j*7W3q)w(pIs8=s~^sZk^wHjn*qVPU0aVZC^8C^3H@ z%z2M0$oWdgyrS-SHk^=P|8x82e6nrD;JEhPg@l&)8L0apFv}y;Vkd29EebEE39>CA zPB%Gyv>umFUwTy6XOUVVQocLJG|{$&>N7xu-tnhF-YVNH`#!eS%ARD9s!v=+r2vm% zQNgT9on1}QiN1&_TG?F57et?riQ}c(TKe7`MdSxc)JN3ub>N(hMyIh+Q5~V2FaW}! zksBP3f94|X3>Un1jF`8_i%)$)aEBXLp`je5((TuMYlM5!cNuC)twMMcBf#gF*B zmBK;CMG82ZEY(@O^qUkYBCZNXJ=*-GBahUR+}91bgMcy zLvT2wmfOAD2$A8w^}v3_ZQ4z<^7QiZ>A=D%0>U+)hNM@vzmi97iw)qlyS5V3U;|NM z0=7A`_Cf~wWyp;LZDmDGZdS8jkMsE7fQ0rR0}bWMT3fyJ^$F3#$=*@8y16Z%xB|Wp zY=_1UVT6K#6jhuX+1qK1rgsCCGjbR8t?m$W8uA>#3w5W3(-(sOv>>jFAdLut;nGj= zW_6Yb05D>P4t2esI@q&OJY0F6x@&B|JRIQjdTF6Kd)T&{e>p8%e>j>vjpvz!WYWk3 z?Qwq6UxJmlVT1cY?MMs>GCI)`QUF(6@=9lMNAgHt$c=E3wmEaKOGj84Z~!oNlCq3z z#ypPdRd98z8u5&EWM7FU`p@$mq;r#W7n938c?gaG8%4b6kL$+clR`A*vl9B1);n$q zzU)?;Ep|MYiE`|bGGM^2LGR@2VLI=Y27d@JieS`%9p|}gTP&v822+TPT}$<)S9LXy|o`jo&`5EOHKd>k4YDnS~zzXUE|m!6BisEQ3x zKxb7^G04iuGGkqPW~y#H1j?KU_v9^N9mJ8+Y1WjtG`OCf>YQEvxNv{|{b{C#R!>>s zwh4>^rbHRBgk89H$?+olpF$v;$TpwKicLW%LI`6(Jk?dLlRCLuzTZNHWtC2={!6-7 zt9DUmWLPQd>jOtcRlECdq_@Y-lRH|^9j(=f=*dAG|o)WrVZPMGAC#kAb@Ukar0?A$NJ`Mqkc7Tmga95b5_9*mT4n<@Hn zl{R&xi9JIh0GvyWA#5dmDh1q)ji(LiQeLi&Y$1?l%d;MN3HV3V*Dxkn*SSEWTKH4a z$Pifcs8-5!U##ue*F0EW7b}dUQ5H|~e!R+|ZYDA|Hv^Z8iOr9?Q>i253RR!UDu#?9 zh3X*RLsqfuLq`=m8ao&B=WmXb)MI(>Z3GDq96fop3MsJdBbk>|DXRvOwD&DqyrUVa zarBWP+JMfKrLiXzCsx>5>3?D2TJhMI*Hv}2fM0jBH@o{?7u(O7ivGuQ>~De1+-CsK zjj>w3+r z%S%in%?ejXM`kXrMSVt8@L+S|8Y!kEFVD!ynUc~9E>xjf7MxnEKN(kd?F{?oJ( z34c^3>5+xuE=vLg$m?kw-ac;EWHbQ>hE4 zO<)a=rTwXFi5;`f zi;LwdszcAsi=I)1L9Qw*2kaL_VUqa@ni6eHpObOdVM2?C#D};oXKu4Yz`!S6N!f9e zMxFG4A==cnuc*!qcZC1|cIwO_Kr< z-#iu54V6}_I-k>;e*?a^OP{8cQfjJt=(O@Ql!aE^#ne&Xe*RRfLwaeJ+uc#Ce?gRu z+iYl-jT&MHOy7>9{`i)*IfZnNWACZ)HoiOLlkH`Gi%KKDY=wI zv?o@Km|)B}b7xK;D1$TV+<-|3U5}kasbqRKOa#w%-MqPk3+W=GS(#p(FfG%MJ-xVV@~W0}CSc3%AP z;3H)2acP_%6WB_2Z0EYezK2P6Ggo)m4>a}GiUtS(Oe2J-FOd5BWHB-uKjCkJQs0-5 zp0~m8wIDA@bv|jB4*U6CuZ_Q5hM5F@1OTj5JLWoU^qqR}vVDtx?PKF!LHN@(HpOIn zdut_oJvN%x9re9Es_e$=bslZ%@y>CvqC*jqTD`{Ag?{OL>Hb_51(~0@Ue6ghiRgFV zy;_$2w1DBOR68b$wf_OWf&V3(WVD^dHOMH#hDVp@@Pk-U2SG-Dt+`-q>iRZ3&*MiK z_s=DuRX1l)@la_ep_@23z{{m17b7G{^|pNx=^IT=)fh6>uX8hh!Ku7e;x{+Cd=3tG zqC*h~HJ50RWJ;!xfaucBnCWOWnD*2o6d45%95pt}t^hy;aL953XR*<&)CC^^m{_p) z+M5hGZ!5SO(NuDDV2VJ>Pg~Gm7=MpmL0`fs=VmXU0w*xI>S=D5TR~6NBj97e2gGoq zEJRr2Ip#ZgK0bI36mR70wRhM2Dl<~(u7>gPAB^Y#!gA~C>Hu9s&2?pEG}ZlkVQ4g9 zCox~8#{itVnb%tf7hhUgp#@!D@U$}blT%QD<%aKxj@9!wImayP$&_v*k-{+=?7@aPpuO;X-{B zFhw>aWFdzdL@<8m;IA;O8w2zgq6-kx*M)yG)^=WkR1mQnWW^!BwOL|wwDzamY??V< z@qgJb%p6T%62)I_e@HbunY1he^DAO`lS6YHXK!P6A&FTwM|$k&BhxkJtk^f4-1tv} zW^Z%s&WC3J%a!GZs{+4{+1A^iz8rV}3IHs>l} zmxRK+wjSt~^ID`oD1avc1G{E^7rmgf{i}_KUIwhQb0zuh$tFkQVrG@le$-NGqD}_n zu&fOI5)>ryEezsm@>9@=11?dJ;-0@B<*Iyk-n_d`y6@z*If0jynJF)`?b!i18aO`2 z4Na4dT^mhzt9NmAb-ndMBg2lz0+`#7X|$}pDgeHVlMGjn$885^Or_;I$LCtshE08N zoI=$Y2F<6(kBr5T&7)jBR?w&>hiHAYU06;~ZgKRj>?c47&Q;oEX|Ll>kCrTXFx5>` zyc(U^LCEmMePWFmzN{O?G}^Sk7b+i*`iGD*g&)5H4?5o7yi9HMU%vRPwzz;N(qIk> z69)$eAOCcJe{ghE2V5o1@b>s31V4JfwARGfm@`v0uzO;1vf5}MibOoBuAu=uHBzsb z1HaU1$w7|7!hy>ts;GV27uVO7%MTx3RI&>}n(t2Mz}`@@OIr+O2$1e>WD4muDbyC2PfH&OqIQ|V(8lXYHS2wv~+m9j|QWs<_T zQJ!N1!$4si$WgtrBN0#YQVhJe;n`zi1r(T({UakIYinyCR?(bajiMn0VD?oojgDpM zhBowtV9v0Fq?vy5yr+Y{rQT)-gc|+uSyw8@nfkWuin6e4dVGxDHrT zDTMLo6exnkA4dQ18y*6l699mhhSdlB1k4~e1slyn%|h}HG*t-XVw$fPp^5@WXnrIx z_eZ_)%$Jl>gB)40!6EphdBRqV(Le3KUW?n|EU5EcgA_!@G;+{5Jv<@6#H}Vv^*ZUv zYix~A7 zVa<5OmJnaM_RK9|N}ZOyHXin$ZbokDW$I_Qb7x^;c}hZBBLWSdFSvL53|ceaKMN)g zm=WL?M4;{LVIzhl73Rsk?Z|CTuL31XX*c3W0e}Pufc@{80cw~iD9A1VCX^5gL{x}T zhAMY8;y9Tlv}?=Pw`80W=Ua5ID%gDIYi)p!off02CA8 zW!gW_eB$9f^t0UC15iWZQgXPxL!TC3PL|m$pWMp_LPg=-H}Nm{cC7n87tKW!{R6VC zu(O*v-HZSL6yRMbn(k-ez|R_y_@L@gM1AUMW(}gR_J%l784Ex|92a(R<}vj`7y$eR z#5p6giwo@rC#-5;E$ zl8hwVUN!jC-Ku$q`433I_vekWWob6a*01()-(S$esjU)dR`Ul2irG{eGQ)5}bptm$Mk0 zdK=fpIQDO2*Pub>>G8qf9x<3M>t$kJR?&7g zw@7SppS4yC_~%Ia_tVTa27;%B9IqcEGV9-Oa()d!CjxBrf2+Uq^=v3>8oY1YL`*kp z1!uhnW5~HACCTDF5#;!|Diz$QV9&>66hkv@qyfST0?&#wF z_2v-A*YBJbkrn&k5|pHtkUa8uZq%<%iF)hlV_{JeN1FAFUNEzL)3@#DsR{qBUF^)x z5!W5!TuJPo^X$j5`mBO81x0c+MnubH6$DHcGYpQ?G0!x zMqi&;qC1nn6(!iNOr6i0>E8c($$lIu;v06rH7d``+uMA55Vf?Fv;a^m70%9jr4*M z)MJye#ri{%jpsptJfdt6o}opmMViwwd_l*nS(X*0uGhv-vD2sN2oJZ&Bnz=Dhg)GYd0QOQ~sbZH|40f}9Sz5!JV2S=~9GG*oX-`FZ_x zBPqj1=}umtgwex?_&9%ks<9b3)u&wHXKZqO$lXnf%GsQ(t4m-jQ(K5DPu3h(F*=>c ze>)`?@8clw{Yfn!KUwbQeQDY zU#X+uYRfj*eeoq-kAt*=TdOe9q%tq~z6}7Fo1KcZ7{;+vGy_~>d?;vZu{c}AIP2KQ z)w0e&L`2Edy}w@tboGlRs=W@i>Bc3w8cKEQSbVsYDhntPzyjyKRPxv%Y<{-1^)6c4 z42kE5BaZV>yS(|@!tkq#k;=~(2G!+|&I!%W3C$+$H!Tr-vv({arT}kw#{)-_S49)$ zTRW%KKXw?jR=+r27Zh_{hx)W}-YqOXSE0ei%39+z@2eGu2B0iRZ#Z^8Zj>`M&8I1S z{1sm1dy+G*rkddW2Fp)?g~dQSa^>C*1fq*7yk3mYXFncYnUIP0kQgDh-o6ZA@H=_G zO`2OT->%-c%$gL4E;pQY+!R_lIn`<{x4u5bX7B8nGU#$fGh>JMgV_~fVUlsgXxsH_ zFEAwWGQaE;^o26hQ$2@*5oJyNWE3x0<0#G&Hv83BZf z+-l7d%B$68js8rDAarFwoQ{z7yYCN+`O&25=FxbHPMn!SumeC_CK2+~>9;lAB^JYo z>I)`&ZlDZZHJKAyqP8S8V8R~L9begGX>Qj$*8^xFO%{MyMd(5i+_mflZ(Xf z%8u~y+Kh$TDw#zLNb<>004y=);lnZpSbOvFfgtmTSSo0s3nWo`xy7JC7;0X>av1%g z+4Jv^Vjczdt`rnHa?b}?(F1Nt@3<>=3vsG67^_?h=XAH=O3Es>xNMhizl5WY@T@G* ze*d+TOm}pA8s4{+&VEKoL-%@i<}H)7bLRU-fx~Wt6f6`doTv8^Jzk++eV=Nkx6YdQi|6U{hRlwMsGaSCXMQss4}}zR&~bT0 z)tBq8v{{n+o$=?Jr!Ta z(a@}^rm`a>{>RzVfKW>FY0Ek9)6;ZvjyJEj4R`DPi?fr8=Wznw;!~fC0}eNtVQovX z*%!SBM>+^NZg&2wqmyd?(rVhy8>j3}7lVAiJ2}5w<-MS9lDnlU^M1FVImic&$N3-j zZeV$cl}D;HZYRA?E2l_54t_N{k$XM83#3x>JJ?Iv)tM&~W<{Z~S%rqu;JD|Fdn@%4>S|>t$%N zQ>WMTO?YD?jhfHP5r^8N*4xYZm6p}%!|+YPb#wO1ZPnY#QzVW%VM{59|G7(sP5kX) znd0?!uTQ0Lj{n7g*Wv`J(~0C@^*KKK?dIUR9A=F}*CyfZYM`@hf)oE`&hN{G^rIV} zHvErAvFO)bHJ*P(VrZ1~+k zYTt7yq{<%ZoPV_)zM}tX>ZR!0;&nTrBjqixyA4R=I8krsNuWTY5IdTd;6nmLv_yPe_dwuFj_Mh)G;o!qnN z#={N>9nZ!R1|W#+%ndG|Rgj=4`SR+&Xg+b|e4!3zESU5Ym;o`YaYN=!h9{*+tG@B1 zz#x*+v=r9C75uZxFR=7l&Ca?$hyxBLCVj1UxckS4ISHRqB!Qtb!~ayShTq(`o6amKKkT>#vhP=cFOrxfnf)J&27ab!LMP#6-1K zRi2ZRqBbI}pria%%~mHW}vipa?0+@Ey|JnUn(KUHZ(Xu4*O55ysE17;c4LIt}Omde4h;gg#6r zEr5SnIdQP4{c$q3A}mnKRPJe?4n z-ESb0^})!D+5mpcAF0ynUEle&T_&69#+dBb8(j_XdwLlNu662ay!}=hPf9{AUc^qyhbC?uw#MbeVREt@ z&^_0B@c38UIg!BxN=>rYHZjrQM4gkEYGI>k*Il1g)3Kd63WvW^tI{-OO1Ev@f#S}z z-8+53CCa{Ez?;`?S49nm<#W3$q(eM8(UHO!my?xcPM0BQK2imuXle5J$6QVsW~zFf z%v!zO3_n&^haOews43{irMy_xYVf-s-9X*uj6~GUCeyKCUI)d@%=|`LcRAXv;6Y+z zNBh0U-dH!VeL1UEp;7C}HGWm=MSy$G}>e<5d#i5jOBC)whG3pCiD?t-S(K=aItPBlC=*)Q95>=;5U9_KU=_Ho1rhR zfIqD}UaxsIU#tJMkwwy;14B?*mUg^6`_o2TNhf%%)kw}xcM(T#Kui;fWY(j zog6<$)ovcmm5bRcrFxUpK`0{t(_D$Ye2)~PwsBZXl&>>uJZd@$aKxW$O=7Jd6daOh zHKBeW5evt3@?`hj$;inmJwMfntke){o>SXyvfBqPUc?2JTBp2*2gO~hCy5B z$LYElMnzC@I=&?#I?IuzDFE6Z-xi|s)?JBxQn2x!5+Ly1gKcCFB0fxUCBZfCfKIb3nh_z;&{ubCkKZkt595gPQl^QFavhw|_KfdOL>Y@=W!Lk2Mc zW`IX$PQGwOwK}evgylNoHk#hR15qFZQ2Wwtn@m^JRB{Q;SrsX9b4o2)u|*UzMx|J^ z;9~1MBL|@X62Rp3+eu+OqfzNCHlP*Xb!hwB)%N<^y#Z9X6xIj)CWqKX1P}%W>+{or zcpB}z&*2Af<@4(yKfbk&NkBwW)yvG@oaHF;!t1e-IBTgA(~A2Yo(ha8R`#>Rcr?M= zL628+Q--z|=p=c*KrBBXK=1Wbe4XQcO{w1bf!6$zvk`WtF3mxi+WLKHKtpH8d2;Cr z11*ij*4WGurnTzw=jhxz3-K-6zH*PEDR0BiF4G|;u<0CR_>2I?wA70T4gp_=Pr$Fe zGUi-STzGY@pWOqZBERNJua>!nNy5GZmzNn^6m~llB|NgW} zVzezNIy?vhvRiUNcB%6o<6+}MIH}r($ZSCXQn2yx_U}qVetp$$7bz~i=f&4GW5W?a zN0Mm~#*XhRpU+Tiy8os0JPEbuBa^aNO5ZN6;f*EzA#zX5^U6GVex2S{~RdLNd zvzZ*iA2|vY?Sq=`7qCGiF;-Eu`Q|xJ>(uJN{X<~1km?tuaC0(c zt&OmrFkhEJWXZCC@E>CC%>@90pem9^>LJem^JZQaw-ld=oqg41AwX0qC)B3+pG2Ej z2f&Hf|KU$g?U^wD&ft)RoX|}F(YpN!6`c|{ZeZ44bVURiF`dtR`Qb*H!phqo&KBvn zh(AY;xx)MGZg{{R*Q2dtxG&DN0w!4+mHXW`cJsc zlFFW)#I)<^!EUp7G2+CKn1rrLw`{mkkU74+OziN1Ra~puzfP5hY3+T>5S#i?M8)Ptay=~IQI zL;*%)shTzp3jekD$qbe9#OJ(3zoiLq|Gn=~uIWb$9b2Tl@{WcuWEA+|r>Ae1SB{cW z@Gk)y3r$b}MWr|pDD3G8mdeCzf#~0(h_3O$JRV7G@sKg<9|ao+TjS9Uw*rcXE@OHm zcy9eF002mn=ammeY@Yw6vcce~wbm?T!@kzd=0^_?&z0BFv1Qy3?3U-nzw>;)aZ%x} zh+^8kdSHv|b#OP*`VW2JgIMbq%RSp|@7BNygftnkNHN0%vKC`H(|E%3+wRTRfbOX& z;{T;BO;_vKaxsm%^^2S9w>NU~d|qw^2nR8@J@`;Uu_Hs;R>}`cFUi6&jPr}KOfreI zNe+px$L7CtKRo^QAIL%&T+G{8?^sSm_cM)`dO0Be!=U-cPo}K^=T8O^7GxWrw}+XQ z|9vv_o+%cS@2Lxj$AQGMNq-4JxKYFu-3iow%$}!@e~l_=7j>j=9~$9bvNfj}=D!oe z4+v-#oks0_0I$y7{Jb{NBujLwq@-jzHd|awYb_e2 z{=HEm)+bZ`X?zRIrva%q_S6XR^kuBPFv)4fIDY2=#85XC6)x}mg7UhU?WTtt6)=JM zwQJnA`@14GTu^TH!f}!y98-FvW~8Yr=wHTpG#5mio6X3Y#OYyc>xWMEH^sMTWOMv7qS_qPwd7^Y=1O=0Q>Z|(xrySWTB z%}k--aOCBsCA!x&wsN%poU*Z#qAqQaRH}+^dK%2x4eh!pNKZ8$)c#qz=2*Tobld*o z{rWXdIsghV%QQ2$#78d2PXb^xR#7ViDbV+ckPis=Z?&-oKyz!6Us;B^03wErbVhqD zAGe0B0q>BHZSh!&-ml8666KEn+~>9Z2EvG`BUiGt^#QNKxLGr@5&d3J;l2>s`pM*` zWRza*tfFvvk-R`fl#Vf%;Q<<{E*^CGq$sX%2ho~Nc?xd=e$e#ofq zUERC3ovUjAg0lF!04vqdw5L6dzDU+cik7hUAUN>1-b8_(TE)26o$In=AMU4Qo5UD<=Mxc*N5`uZW2mRnQb zdwH$BVoB#D=OeuAr$@BDmQ*fV+65Q=ZxkN7mTzmjoBpo$(Ejh*ZM9Zm=XS;0nX22` zHWmseyXn=y8W}_nJfGEeqcuXK(R44Q*38rEN1DpwDyi`@9*_04F&(@*XAP4}dq1h~ z9A-siU~Q&e8l48lU5k$fpiABU*V@^qxcjjNgYZ5V{a=c?EiYJX;YoD0xO^++Qe$=G zsX;jgN{V9ojthKACK76`TG7tYR%)wl+`*-JtKv(rI`E$_snL~xlRH2wlda57;iEgh z?)p5EU9ntaL+YCnlCz7P~E+A(`%( zrI0@L2F<5bAU%0!NJ0oidOdoI)4ub(-yhsc7^=t%ab)_DZ#tsYQu?$nn9W8eBRz$4 z+)=zZjCZ|&(&lw;Yq z6(-HJYrZh6P~OrBn8i>mkz{PwiOoK7`C`p74+~A;gM~_T1pu@HVw*%0+_|vJPzTjO~aUgUWyi^ zn-A3pVMjZaq9CTgw6VgFz8rE znDA;ds+cMRv4!ZV;ljkcLjhTHAFY5g@X(oSO}WL6yp_e4+NzjvLom(pwnzZVi;XUc zAKsIiO2tnb8|J_y6_l-ZnQwTKOK~23o11fm!gPgbSCPD683l}vPfr$1AC=NT;bzuu zl0v8TD18~|x0kqf-lVYj{oxer(&`!x(PQqgW;(Rv#Hy<~hRNLJa2P2NAM7?9#HF$N zc0^N!cqis(`98_VJ4s2^qL7fNcX3UyK)576@VY%lISZ>=BYTC$Xtu~Jf#$mbj=f18?D0w=iGRM1G4;JZMgReUgDz;6v3$8 z`gzxS&4p(1)rl838BH`4q^tmq74*9%xC}w8#Alv7nj*pUr@!B^y-~P&O+LH$vX8Nz zQIV^K3ifzFS7_FduDGk>ZFN=3F#*rg5{^O3uS0676{b@xr_(PYQXuQz=}9h1skxd% zZ2ghF{nfp5DX_BCWRdjP-0tKSWa#T*qg%uN8}D=YtM3mc(YFw2iPfAFQ7{Ym) zL}l0Afeid2u*$1rD|pf=Q0dX{o`fCZ73AP3YaGo7a!hW}q%eL5L9Xq#GjNCpXTW?Y zbz-UKh47EDQcmtH#r z%F=QL>9|%b)@tXS#Hz`a+96m#bw)>-J>ayom3#c7-HDgw6FQpLbdE!{`H+y!(`!$~ zQetz-96(Jg(e7}mQD;y?y&tK*;cjj|%wc#Bb*|@rG6~-+)YO|XO;b;gd&0R`DUXRK zx`nOpswvpxloUHCq52Di@5a!wn-Su z7_~om2zSdhQ^;C*((nv^W-Y1Ln9`LLbP*G6IKf>+O3TVF;q{E!#)<~=#V?9{_8AG= zy7S10$_d=2;->oT=w&3yXv?0gx~p-WrcTPpWt<&>Lqpp#l+zKF75nRf#`!hp#RdpE zOh#yEIOf|=Q?o+7p8JZcqqZk+83|TtEx6p#z%%x zu&e1$DRY@W6#!N$u%fEY1N;ds>NR76Sjx-$-xI?UMH4u)Ln_M3L{e-3vpbhCu`CYm z$XUMG9kvx@q?tTq-9kjqSyzi}g{>LS9?8R(_Yk)jGUkXM;DG50IQV2rHnInLDh1C{diw=h4}47mO2GUr5p0 zfc?6GB=*~#w@<$z#YQZK#lp_`*LFSXoQ8Bug0jz^G2`ibGPNS?3hh#?ew}J#wv|

VHvEea(kw(3a-ZQ%ZHL}|Ztyc|Y zlCpF$F=jup?uA?5HqEDEdN|~aMY_S5uZd0AY=Nv|+U4UofsL3^Y#TpPm8gO+OlVpNFoP;8v74L z_tE+9+}hHe*<=FD{!)|@Ajfj<1ua(bfP&Of7cq7-q7Jc5M46PsYT-P0_}z1z`cXjK z-WD^?*`rgJ%UaC6N0>+MTq6aIQx_55-D%^~NwE(^CJ(ysE_$VrWQ(LarP>^zgD85K zql4M1>#Wp;=}6xN$|JvXCPV#2>CBAToS;sR+H-&ZEm#l}SWG*MmVbKNAPtNXls0IC zz{@bLw?cZNh~sE>4cG6O)LgWBK76d0_T1^^n&z*H5mOzX>viLMPC3`dWc!jHv{AcH z89FRKWYBjXiEt>-V3+C*VCUUb3{#TRfFr&v{oZHFV_jo9eJwn$y< z&A!HsClw2C%gQU>*ycE;*?LQPT)#(Oy}lrA1{gY|lI*^h-sX zI;$hdXl8>7HVxf=D17{K;Pvd%&U~UTaYDgiYZ35?o$ImZf3%r01hvkd5+g}&L;i7|}06PqN9d0z8 zbkX;UW^TyGB}vrtzs&pS!iWsB01mcS5d%Y zsg@#A4d23D!rUnPlVP<;qm3~-%bcw~H;*q~+N(Oa0naL(6+lK6yVHCXeL|*LB!g2V zzLS=W%Pn+o^SusDO9l(Zdc>pNYdp^nR&-5ot(Y_45Yy35T1R6hY?^)zJo_dwV1zZEX9`n#ED+_o1@FNRL1< zSxbVZEpa+=1uTx5i1AiNyOwX~vx5_7ItGJ*=c{Wjez`}%s`T9Bj^B%FAF1%iWVM~I zE|rkv)}^k!{v5=bL8;PGuo=5!MCZ=lA8EGZZhZH}}p08&0#$ zA9KMucndpBw{f6$0t5oAI?Q3-W!r?-&})RRoGSTe&3HVSyA8mpZi+#XEK`5M5)KIW zV%f8=Z8qH8(T=kIgz_X>3SS&KB<}EFHfgB3rg0?8Wa1g0_IW^$m564SdM7|Ku)lm1 z3H#A~5yn(6p$i@{CNv=9OsD8bGXsOpZG!5x_p?_DpB-4sXtgbJSn%*%kjV~hYWJUn zdn7AKZ^Vt}4TyV7NqZTMwT?*}0LHpQ@+ds5b+*;5wn*W_``!~cf^6Lf?6H+beDKI{ zwx~GHA#|E6@{>|G9_x$Dp$o1qpeBv6uSvi#^**6o5C);Dii?&>}I z)F=*kvI2A+-1+FNB=;ksJO}%Rq|)?)z7pM87_n4y2qO__;>FM{kn{s4m6F3k%)76o zx1MO%rn&>ZwRPj458{66O);TuIs-PKXwsdmE!?J^;$w?e`BS3fPo!dr>YHy?8lqaV zjjV|>r`&IJKhg7LGPu9AP9hDTxvKV#0<>jS(sMTSlq;}8@Ot*OaP|FR zn0Rpu=?|i_Lfl=I1B)Hcdwa_-2JFOMh#CR00fwF^@9WcBz^M}ZVBX* zHrQ(J3Og$`1`MX|*j?w*!KKj!TVJ;BZ9^KWwe4Hd+m!Q{7)A(_c?B_?HXzv-T{FAy z`4@kjt19MN8Wq>^yvoT~*1uS*vdUOJC!rHf*3Q&ryo)MQIvzPTVRbK@ zdI*)6CcGdmb&F@M_3aCH{n*(i9*#HdnQ0d$0B{b$hU#Y$WujHQPVu#+kgG@9@@t-x zg0#8pHUuzTLLGLGnU?6{c6j294dV z%3?fy4)T8d9?mcQ5SZD{6Cr{Jm2e%K%3z;l2RcloiiCG0wY=Qx4Bf4BzYk@QjLNyN zH|z6B)0KmGHpD%oRC>1y61Y7o=a}n<(5F6y> z8H=rg?GWNo3tm5Wv^l3qNul04dqz-SICi+5)r`E#tX!KF9fGuGK!o8Ig*@X|rCK9I zWfyOTCEHf}$|{F*3)`174zgR!&_k5vQg7pL*0QsQ4=bFS6D8Vcl1YlLjI{HtEK4k3mPU1GlCl$4m?&zyuonqNT z*@mLtGj9(!M;c~w4kyQYQ&MOT7aXof6k`h^EFeYP&g$v9?lndh2L~@H?_w?deP2RY zvzE&7RZZBLf?lYada6FLc zN(uXw&$FD4|3`?{tA+KprpLE^O-BO(8g*mSe#Ov!GYL>2zI&;(F>SaSInGU9XCNWE zdvMKP#*!1*z&R9eDDUSweI#>%acxv~B?@T|aIm!v*T>FQ`sORa9OV8J zzx4n^M-GrEJlB~G;Kom#v-ZyF(ZLR#cF5hptqH0dCu$uO~k1T{{h1`IB z6%;^vE~KYN?(aElo>!IpeebJ3c~VZ|V60ovzL64z;?KRQYHEWsM-@krVXY_L4xdhgAGvnwX z2u(7O4h3D<(7qoU)m3=_CR7h+_D%=v(GDzBx*^GEnSU--TMgkqAg{7$E59%qba0g_)#{#WXZ zpP-M6s=?|~A&w*eNtrRO|6u3ke}!R^-2dPADIjtRid!HV8JRkpku}4e9TY-2g$pMD z(s9;3IoX)%8unkUd-UGGz`zOB+STR*oWbq@%pEpcWl*#0YGhoTn!J3-qH|G21-A_O zIAeRF=zm(w9Q-rD?Qm^~7kQcBd3K7#qNb-?UvKZ1|vF#3I7wiDppZYu(~#sIXJ*?(S2lqQH_d| zJb!<7uv$`5B6a;Yuc*idXirfQn|B1KZsm`j9x0!{e9xW%p+vHajM%?D^L=!(8#M)* zB@7Z?&mDeO|KHwS1iU+p(eLkK7^9MsQvbjJ1l4-HxA-0S6tL8Qy*wNtB_*9Jo1m#U zv+Oxt@W)gwR=n98Lu z+b~_<&~UYp7nc^t$-$wirDb7mPD(FENkKtDNtu+GXp!xD3)DGXLBZxEK@93|dyS4W zfjZH`iysTSnp;+WvC*=)6VVkt&#`!TKz1P+9CS&{|r0^jYhw4oH@EW zU5KL-JxJEQIjvq72sc4uuP;_E;m?bondBLtA)R~ETWlmP$Khi(^iPykdpz;vX7x~K$FIm%4%a8? zAwmR>re=N|>SHsT|KV^iE)6-4nVFeN!zQQwr7rDeH-IHDlNEIh4eg$XYn)YW0q}Z%8ZfCg0EqxhDxd&;FDWkW?d#j$+e_nnhubg+ zoQgH{I|q(HRzDUV4c37etGZ=gcW=IOvuRSvVn`$#ooXGMgY-xL4tioWIMY9e((F^# zl15Z#;D>5ci0T-MTA8pUww@UbPLJ%JBAty(5GwLcEXmYwZ=p}Bt(COX5eHDSE-|RG z%9P^Hi&EdG{tUxT@b{C^n`8CC6RJxYtt5qov(4!O%TkaGiK}%>6=x&w<^H*33RAVf zpp8w6l8p3+TA@>@DlpCSa(?L=rn_ zsxm;z2!U@#)~|J&Z&$7^>3XZ_7vFCFqKcv^b`hbR$`c68H*49QdZ)(k?J04#j$o-5#_pqzG{%MkKvm&kA+%b9irbdsNpi8`L479zBTF((}eSHo>>r>E9fT z5BRrQ%EhUutaMuGjUOH!_8}GlgAEN0rzR%a|1L#!+MzbFS=Q-6z~Nz?`Jaw??JGQY zjk?*^P4#k;D7)L_p|){pmz4JDLJxV%vRP09s4;pw)P&lIiZbw_{0Hl$})9wuDcPqil9=U#~qpFdGk)@F=^P2<-M$I#{wCX@CBbk5+R9uLY8dD)^>Ir6Q2(A;v~%t+kPbR6pJvN;>_+FI&6W1&qBiILE?Hcq}@@2~sn=QA(a56%5b z?-vg4ol3bH6FH-Uu(X`rijS)}aa0v^a=r?x2SyavfRkG|i~N&ML;D;zwt6JUP|@LM zMR33M{-aI9o*IlPa=dUgm&tnV$Ah)R->}wHAyq(*W!wz2~7XL3@KQH!e;KtF(m zsi>&Nd$R{Gw~X(=GJZ%^uz{OQ+A4$D$0w7gctL|35ihrjh7RK-p)dZP-%CsFy;rt+ z=R!4;04oBWWL#6JBX~fDP=g;MnLJCpf2l$}aHZZz2prDV;h=~i(9xu~MOwP5InBvL zkuFIy^`A@q%!b=;NshWv8=nWAg9F{tF*H{_1hLJ3)MM zgqi~(m_fDMcP49wgY}2RHG@O1*#~F+4ksIe)@>Yi1I7Qdd#X596buF+*6CJ2p=xs- z0L>;V3~_L9#+BKS`7BZ(jvf8m>dE|e5?7?16JM^7wL0|tqFlBDpm6}6?G}@~OH|;4 zPw#3Z(wqs5wG!{lx6@Kk$PIm0ZKtB3NK8q=2LbG;qOML(MRt626de(f%W6k<=Z?U# zgS;3(p^qEEMj`u85#|SX75xUA){ALukTE=bZ6BtHfVs$fg=SRf^ba)=>GKPf`f(`- z;b2*8Ryy)NM6zMESTXA4IWeFq}KQPe+X?xuPj==y#cMCQ(uK5vNy5QQfRb9^1jcwOa5`X@%Ru z{(lzKPqfIl^o}8IFnc4g1?qPc2P@*FKkY#Qf2fJ!tIjX%dB5e&yn&tMr1==4)SvagIpp)} zsqB_nqFdRa7_kHX~O}8cugDN-d2kUI{!^;dC zKQ-3fsm+>1n~ssw3s1kHPih@f)ax9nLRFO8HLw_Wp3GLm?4yHX@NXK;)7Nn2%#eBk z1o>mge!oy|^7NC&+~OfEh75$3$$j79Uy+G@o7=Q!_ngYfG4!Ni-!-6Kvn4-x#esyG zKpA}pqy<=AU9IXkFKs(%9j)@e!swd%Hng=F@gvu}uttbRtLk%LCg~=IkS|s8I!+(J z^^q4&?_;3AKE68E1+ILnX@CAzu;Fu2b>@71F!HcvY{B^AFiMtiw!waCB_zU<_Jzam zM#ov-Q&iDxR9$ZF;*jt{+iHrT$%n<;+Z`Z*VKaqo)*60Mxv9L^mnKH@<7_!T2Y?kd z514f&;dlONCrfXf|K&0Oth{$x*0a5LTtw((!w=U-uxrA|v*RlNZD*0|&75mR9?$)C zd&8Hg6yAlWDnn;qO(AER?>5@Wys_KfjzuBl-;8OpmoAZz%URedocghdmW`g88qkwQ zX{OI4_TNKEG^<9|)~r#@$b;mn=g*(Nrts=^{{zzOUeJ&l4SPQH8U|z@1wx9qO?;0h zOs~+o{Z>&fD)i#eOY7CGfMyO=U70Tb+Yci?^xzKbQJJ?ri_M4B(P!{ny7q(kcJ-#b`CyO$Ei%i8}w}9df=x z38`iOA~4X5fSP|E?)AIXkfR7Ad@vSEuME3so(2BLY2cGL#gb_BUtRxSUu^*gCZTtC zo2DHO;smV+Ut93?Is=;b*Z)aMLSX^8LxEz2vH60xt)z!}ZN$e(7O+foQI!p*PbaYa7 zKnhL7G(2f-U?_S#>E+^CwqxubUUsW+hHr0+E?*_1@7DF5^(5rCOB80N<>Iq^lIvI`Zup7^~g>G5~$_(4p*Z z*Rg~eF8U+6>|HW4&I_G^Hj~}GJ!kat&5++4g-f4T7$ZEkXBrOTE+?O-Kw&V`cCWWW zC79@kf|VxiN(??+2(M#12m}I!Hn;|#5H}YP;~^3skhAt9kb*H6+T z5~p*F*Lz{evrOb!w{XWjLz0h<$xTuRwdAr6;t&7ED&r|+AUAwgenFK@eNGj=RQ}oS z_nlW@^wbXzGdZHqI^`_6l;b@IHrDe7@A-_J%-Q%7F!rT-i0ga4AI{??Vg~vozItCX zP!t3Bp2X`%EI|lN$u6Ghs&CCfuEM;ZVfbKQSFDL@QC(r-E3`eYue!4FuDzXIS`d(b zfGO`MLNEdsf0Uj^pm(Oh{&Z_vOZ%qO+l(++F{**2IaJ_c(18?gsf+0mC&YH6-iiN} zcN1f&kMIrg*t+OE9WDmq+9Y_%3F$tts1Y(x(o=E=O8-ImF8#tagkPRx#OV1<{Z>Tx z^MGLxX>mbi?Ro{1FCANaqu6@O!B8>_o0ip1UGVc<7T4wKxkW#U`%$yUPIRhoumvE0 zdgwg7@5>gl#q-cw2FtZEjr2#NoB(wmRxowfg5a}qMtL0)X*wJ6roUzDca5CrFg^e8 zTB`xK5eXQ2ykyG6oBO<-;PmP}s<4V6Up1-nEYrkK=y$XFWlk2wr0)i(5P*2l>fr~G@B^zV}q4_I?<2{K+ zfWVQqe#Cn;8ERcG^yv{WA7iKU+rE0u17`MXup1q$->EwR^E^78Wi^=o^3fIBU0 zpufKt+HDIoPS9o;a83CF8lg}XZr7oXh==mbik*cO@RCO29hVMbx8%O+e!NIQU2iCl zV!})=O0`%e}M_?#My+x^?x7ac&N;K|w?&qD^vCV$_Bwm)?6zP*!K zUWp_95{30u3j)*o$*`qyL+WIMo`lWH$_d1mE#BzXbG4I zIQtx_UxprNLA)n!Q$w_hJqlcL}5@SGq}%dSH? zd264$B(m*|w?{;)5Kd_^6jgMd6qB287OE)65S#d2tfo%3(?tSXm7?WMs&tbvN$dN7 z+4wjL&{-f$(pf`DdxObEwNDmzVM<5w_NfVTHzlL771nf8if z-0s=!llP|)?{1Wk*8jJnPj?e$~ z2uNV!nMpe8Li2lhM&O)#*#D4#NiI5Q9**CZT@h26;I8oRUoE0%=HXnL|MVS@!gJ5I zQGJi)|L0BPuWVWq<$)aS($qE64x6Mm5B1*Pd;_DVqUzEa{O1#0|2&-V)w-_hlYf0j zRB8T^Mb^WS-{yc&2i2;Ht?w8C)(W5-A5)LzgWGRD|5ou107j#sfznJ?C32ZVtU_#9 z;YKgbhl0Q(pZdBP!5&-Za3XpvD{moMh)-g02p{9>(6aTTX-9X*N;c_Pd5{8TE;?sGZXj<`-pme3uFh9=WgI| z6bU^^i7w6kOXExc)Tm{+L35pYMS~yur8tVVqSM~h>%B1?Ehc*V`js}jfk5s}V}r)v zG`uEn972E_#eyNSjKM#)HTMri#TpFjZIw`u+`U5+7H$SE>3-Hu8>SL*-;fzNBza@0 zE6Qipst`cYMc`n)_ZQGnwHp)~PVx_g6d=^UL8?p)TyOt9=>H1>gFtxfx}qbop+L2O zn$R0G8$cd85$N#3P;)GvOYY_|7P6Z@2Jo#s26!UiSJX|V*Y)$%VR?-f>V*6&j{Y5a zyRG$SUJLV~1WNR=GqK7Y%h=W_H_@4yglAR~JexEBkm{|$DEvCsE9`sCQm1@Z;jlqi zURP%9dtm+-m|ve9(R+GzB^vu})3$TpSX?tJEetxBop-{%FTUT5VeUR1uHU{sHOV2D z4Hq1dIyv%$9hL}Z&@8xKH^4`b9!}3fA89FqOHtHc&iTz16VI3DjD@vPb}@}AbUwh~ z{VOlP1_L|LAwH!T-K)8=v9YPC^}{pC%RM>fAAosfq^_=RWF+o=vh`Bf_tJ2L&Uss3 z=hKE44v266D*@wgheja9P67Cn`$5ivx6cM~H>8P9vSW00nCSZVRKsa|x?op&T1H04 z<)4z(uTxVYrYmYW*_nbe;VmOs=kjl;SVNrRlvlO0OKA=;(o9V4>esqW`A%|^u&K4j ztCPuZHyhvI53Q%6R|UxYN#&In?BwXkVW!>|Q02(0ZAg_d;$*WZ$M1ZzXsXiqVjJR* z0CX;}z6AJESZY`O%gelX4Hu+V#@bx2-dv1vGCYHd&=rt9-%1Wel6Z`Z(W!*R`ulhJ zlJ@v=`YqA7a<%jLx{vLvQ_0y}H@uT6`L5CWd68}Y%`Oiv=rsXy0SU!S0?7*Prz#7; zDJ=Wmjs9h5KVUSPAzi0&%6VP9-Ud3nH{Uqb6{2T6aDr(&QZ2of1Ku-Rvc0r=%m{XVJ3c&t0k1hB!Qg1ca85M;2KZ98A@dKwUVwsX+YpYVz~{zfxNzaT0E zlhKRS`(ynQ7p^Le5sd&{fEcQ$<*R+CQ;=fB+}|tG<2GYCxs!{Wxr>1&Rph8h`|@Uk z_v$bUfofZmrJMIUEGkaQ5R8pj7@Yon7KKs{JrQ^5bTcM&u^x$(r@yVn;D@QeemH21 zEe-W>d`tyQc@k1mV14$uKbcc9hu2#$o6!X6J3aXw(teytLxViMjP#e3WVxgtkvxZp z#%vb>>#b+WO%a&k4B?aUJBxlN$-?u$+n96>yUH(81|<&0rt(E?D!ol-h1||Qx~;&% zt`9d$sj8@a%C2IBRkw}K_jjx~vv(-~_V=DF7fI4OWh^p5AzZ&rV+$&ukF=k~NI+$q zk=TA(h|fgH2Zd8waaqT_z^?K9do8E$#Vg7LGF^ZZt^rwmXE~TW3|IwqSqe@bWmC<1 zZPxAfP2yP1Z)kSULE(eBz*K|t&e0qZ#n#v^3?aV&^*pJzni#NNTN}RA3^Du*C&PZ9 zyUYsWmEhC;J7P?Ke`#>v|D+XrAvHLZ$;S-ax#i?CH&!J0^dZQs+-n^UYji@1i99gc zWd*1-wyq;e?8^EkZKZXA`$t1AW@$MtEabtp^Tv5F(b(L;`q~|D)!`V~Ip^geQBhHf zBlO>`TmHIGAIQ{0<{v-zzmMo%p>E|QFC2QuZXNqq)c85csH*M^hUr#voiO-aMIXs; z8H_cTiwLv9)@spK8A@y`D=Tx)g8xQkem4Y|DNfJu@l)?-O*_s9!D0Cs@mgNF;5#O7 z0f+?Z65S?TV}#;v-1&(p^f!Mrnv1aka77R_0hSAAEoThF)upxlRf2vARJ3Y z4#P0tcCk`^i&+dA@fV@A72RbQBs)kt3rC{IRr61Zh?51ON>(fTYcsx3<&5W+9SFW5 zzy7Ch>zwTjr$`hf#STSoB)*2FkFIX&Q7NE;g@pk#UFEDX>#Dq}Ds>bt8(vvi8C+px z2@`t3eTL~Sj75?;HSgG-&nk>eU=`ee15Pnj<0+LdP{t{#Qp(|(u=f3OFq`W;Q0Ifh zoE)UhNVdB|{GE=I(A!u6-;p?&gs1m#Iy|`x zD*97{O6x-Vp}|{Hw&;yRG7^$AuTzgJ9t|8^0P9FHBpB1kT?>vjZ5+emkRC=H&epq= z%N>gtbJ=0X#E%zm{zj#LSwDJ+{j{$ho?aQ|WPkQ9vDmYd>ct-kU`~4OsAYK2odPc% z*;;6yMgC=6qbB^nn0w2pDxttN~cISNJ~pM2-4k1Y>;k{ zmOg9a|2^Y=xOcqgo^d{$HQ2-LUh7#o*IYAxb3IJA2=__zQ&(Q%nZ8@cNBN~MXnV@9 zkqDh^6s0v=qdFmANMG<)BNi#Csi|pboVe57pe{2vmnUH3fAgv~bWL|E}Lget&*%7+q*>@OC4^Jm0{%93TU3I{*kn(?KH*R8g=8m;s&Z zPUK<4U=T5S2zB2c&jT;NX@dHIcg`Lh*fv2I!EIlg+<_TetyIzBVARr**=@g)> z>`WDdizoV_Vd3G*ctBkggsaN9VWvv{wdjXvtd2|P0PBt1Durx{^Zp#$_6w@j%T9D) zpMi&mU-Lzs+do$=Rx8!05V~>zCUgLUl9R3nqkL5$^Uf+bS$r_M&zq%`HJ945I>%Px z=z?;RFj}X;@mEu=rOiK)&h1g_ngnxB;FhNRA@_rDXExIZlrII`o1^@=@pf8=g&#KXfQ8>%>KEq`f;Gu37; zbM>mbRw&vQXYR?X8*|k()vk#;m3bG|B7=ysXhn&>zjfbi(;swA=Ftzto&K-+DFYPD zcq8Ky6Qu(qfmVJu1XPQ0$cC3r8uWC{wbZ#$CH{@*W#!+hyvEh_94lRy#&ORbjnbE3 zHCUdjoB94}e{phh($v_v=FY1M{84MJO(b%&?mv+@NI3<;nw!M1SWM4|saNu6%v~us zL;sXWyTY>t>Das%&w@g?@RGr+ENrH)XrDET>2~13HUY?bTX3T*+niEft=t%Gne)jTvb-MpElrA@- zWUL7dEQ%{uE4fsLiPCl{MpTMNY<3dKGOSBu%~>H$sS2t_mF4k;azo0+bGjvy$y`r& zKs{gc8VbE=?HXXo% z*7f)ITUl8ty#WPb>7$~J4&FQ=At9EKin@BaDzKxso=t1R)O&4h^!c>5s%gcg<*1?? zS34tP#&sJ9PAw0X0z%Qqd4g&Az_zN|+KDU6VJm-v8lWzI|J0Ihe}&_* zBA_euTI(R&udkG`3-2p<*Dt`1TF~jM4{KR)E{_FDMMX(TsRTYr%%)PZvXSu-<|?n! z7et%jTxo|Rd4A_QOd}yqZX>Tm(Ic7Y$d{xQ!@;R>p=+*gtsZ`hO2YV4%h_Sq+uQ}p z(+yWCwbgYoK@cimFYX_YJ#z|?ly1xa(L1H6i+AW6d`YBulU{@DPg~>_md!;6)V{kV zLhy@YL=Gv`Vm(>P6ODHj=oj5*4XE=Mfi>2EG9Nv%jJb0zptof7WWM_FL~`YD!P`SN zXY05#W!(Bb5e_VI&U!UEI&Zo)&l9LZKCnKepy_PdMy8}+#OgWli*3(&C5?#Wy0>W6 znq%DZ&0^AXWJwdBTXerlEm5oQ%_WaD0kV8Md)+}QVTDrIcW+r`%v+isL|^%>4x;z( z@zyi(N9iq)K}X)QCnp`8UV3TQzTZALsGZ*j;v6^z8oRi62voI2XLghdIby+T0W@vS zBGa+)2mOL*#RMg{tVJvu-QqPujdrh2;A)dFr}q{2{_&mm($;7s!|c!h38Z+ zhgcM`Zuf6qIERMi4N!L~MtPxP5==mP7zPG1$>j9a7I?YM^EsStUF26KzO;^Z!5@@; zkjXXj>DnD_A0S{6##eov_>!j3SD!5*xbf*Gou1Ch41Xz!InQ1)Uc)U$a6aFwu>Amy}_8LiVOd165n6cTNff~UZEc_Bk>l#6dF{`8ubcjEs3fsT`4wZbKm*m^-+&K$s6R{~hj(d%Wh?q9FVQa-kP&v0+%vrc+6uavfp2O!u9DvhI zi`6Kve~v&bQeI#gIeh*8nJxOw zP~Ae8S8N{0&93$)orI%^cQ9eRZ1g#%YB%mqevVPjg+vh@qLqS9hPUibgf# zR{%3ev(lr)dyR}U9L%`IMNO^a-aubx@j=pQ?lhHsbk|a=%F!XimNbiHf|?x}jI)}< zwlv0;(ARvwmkR#uURkH4Ec%;YR9~Uln8J<*Lz^CvRHCQQrfh}crTufT8HZBK_W}YJ}rakjde#|PL8K3jVZ8DM2<%PUP;CUyZCFdwjZnH`Ix=4y#{gSV3)Q5ZC9k<6YS~JJ$5qSbRGp!UIILXwdL)*JMV=`(NM-bq40m z0Q}1~P=4RL8vrt-)vU}d0k3^Y_m_kaiyW+|wpYi! z(=X_D-FExVtKQW?pE+hQ&$d7QYieN@^MHgu^Bol^mfdJDkwwqma9P#Z$m>IV!zUit zB!suR*XGq&u|O4Y@x!XYPQ7e^Kq^RU9eJNdL{it@*(RgdC`)h520!md+InOV6=p*R zn!52X>xQT-&A_{1sO)j_h;}m*=daqYf}5Q|a!tLuIupO9){hW;#jPhK7Kvdn2iXk0 zDZ?ustyDZ~F?x%c=Q+}2^~_1A(>y=xu(xzYhSEnpzrJH-z~?IFrZCK%*P>ir@h*H< zuiG$QS~-<-m##BWttUU2Pi-l^zHP7;6kVS>BWeC~TZ=?a|NMm~LK#AOwrD%{mAb%( z3P)Yrx?`8E67F5K$h5lX5OG3!fEVC{k#0?JOxw;+szQ8DO$TQ=M1sm{HwoK2MAW;F z-^|A+`Msl~j7FVQSnUuO>4v-7vFFftPY7P!GJ(D6Ek3 z`nYEvVL^=Tn_N?mfgfz3c#)>D<%gagQJu%bH=yg9V}Hy5(!A8DP)onYyE zv;aKSvD^Al=D!9r8D#!R!qMe+Nn;tmUVWaRIOR;U$GxE<`&*dA)6`MwReECht>vEX zxxwX6wQMI;q*f7`Rsv>~L0z8sUNe|=|7f7z*}S7y*hu^evfV$FX_!^pe4~jf)k~FB zWZ>Dpbbp&srdcLh22b3A^}J5n&W&yG)qz4|;{c~Gb~NwXI>6a2+ZCk93^6bp3$QE} zhjp8CbX7jDUl_Dk;+x_n-4Ufg8lT80AG7Hgd{tXoZ^@l(QWY;+<0DSlV}sAM>XWV^~XZO|Fk zB#&8<%Ob$#r#btSmzc_pejwp%YYw?L+;&Az7AG7IZb>Q@<ab5$0mJX>|HwdFv0L26%Lw{lkp>3^-= z68`Oi3_|O}LmDRgt_j+IfW;C&7t=5I0EJjd%mca(mbmphds67T*uFC~(S{_+riHe_ zn*6?r-IF1EE>YXN2y*%I5ejOZ_O!UP`7pL7Kz)E4D(MgP{uy*u%(wQ&UYiU$_59HE zZStDTi*yMiq3c9){;1nv zaw`*Qyw?i(j@+gwl0AXI26s0+H;&L=BQLjI9!3&i&q4H_ztmGx^%$(wuWvj|rY0nA z5(g^;PrE1hw|Ko1K6faMH-8m8(P&v}XtKypzi zvOyO7?~&-U+Y>|L?SebMD1DjBS~+%gx0Q_-dH%LWdK~5-kOm@N@I|bTqnXXMj!q3qwJt{}s zNGC5%N1Y~{-OR_=hRf{jpmYqgxthgVTrxG5b z^B5m2PN>mOEA)0A zyL)Kc8y{rr7OYiq^wcRaIK?$ts9z(xYt-enk8zu85WeHuwCIwIy@{(jFgdYw-}7SJ zjA^RRSM)VKe>XBlQl#IJPQN}b*kLr0h3rpaF2jMAvr6zqUU4e$m!}|?dd-Lyoodx0 z$IohIJ<0Uyzo8qBYxMLNeb~iP)zU339U0Rxrb9wut}Wjz)mU>{gvF-i%hC3muq;2z z9BxOo z|MlolDMzj2XfBTC} zP0i2q*e&wzuER7&!=FJRT>!wRprL`|snLr9)V#iXZ_Gz3KyTXZ>K4f>ihjSikE*s1 zw)LyRxN%&Zy@hq1|xY*4Q=30Xt&8M9Le_1Njv|0&xfl*{O`{ z=9Oc1PirZ$xKM>y2_tshN+TjWfqGQ?zPDo|iN`NbefrJIFK?1hk`rwG33k=P!(zR7 zBhXstRJHNv)$_7U8^ZG(@npH4Q_+&*(qT;`Oxg%dT`03P6u8sFKvg)aq7YTfqoJ;cM?KhJR;cYZzCs25-F0r(yCQS%2*f&oLQz4eS3e?Y>06p_4F5(Ra@ z(ifEfMW=i(JA10EhoNbU3$(-GLD1@^g;i@%QZ@S?%P9~`8kng?a=YNyXa1AT3$6=Ea0iNk3o>KXPrrT*4vYmDlF*c$Jl?pd?P!1CH^T z=3#DOT0cw1dt((kYVo1e_b|b*`=LW0E9g;Pelr`1s|)!<#Wo}zrf@fAqDzvKUs_$3 zI!ZbKbAwVoIGacNB=FWLOXDJn+6ubJW;Byw*oyCd|F0NBA-<&Bw!7UnxEHX*pw|qH zEWiVN{E+xhl1Fk$U~X(!`c6#?ot>IG>!&HFtozWSx83FU9AUxMZD$2WNFaTrj6^hB zYH%u`o+&&9rS=vYSjdQhSp=2@Y5lf`ySuxAH#rrX&-c#D*x2}37(6Mc1e~7$-Sbvk z;ywO)oe_;?rDB~>j#VkEjS7Fzhr#(U^9zgBsGq6AU0rPqckT9ZO-Pv7_pSARFZ6kw z`KS~vyKv0SSTR)7?chdij$WhT3aO3pWgbEMNbaS7(=p%pKKuR?%4MU* zl10lfr(_M4z6hC8_!Eee*XlUbz*z zoR)6kT`LX3O4DCLLx^a{(J@A*qE4vfW|ykR&2IW)DL}}P%Pvpz-PNw!qWdO@aEif( zZ5A~8PnwLKeI90`B36w`OKo&q&WOSTCG3s3uNTrMW)cr@ zV%CPTR(9kQ>!8Q`O@SzoJx02G?8D)0>nb&7IV}ACm2mKmEF9lIzu&EoxGj0RX=)}H zjV80018A*&uK*$Mhd)l4*9)>7(73QiV40QV`cr#fVN4;cM`5|iYpyvckwwHs57;HY z`F#9AwWq8^vq`b0YX$c+Fm-;3&{Cf&^h_crCi6e$XSGw*9=FKpat&-J8x|Zlq^|2k z-&b&O*h<`N@ugk77O%J&xo)+2eVZXrRw}C^-KNXXmUlS6+;$%vuk!xYbm79q2uW`2 zXCIxLN%5?h%xA({cbnb_0#xkQx9Fn6kjVwQ$I|fCXWrgetAjt#vn}{GxLurBiy{Yl zLR&nLUd57B+V3@H&*=&bO9U4-+lc%PtQy|3pCpEkU+hy_v9(yrGAtZxUtz%$BW8S# zHd55%R81!r$B-$bnrLs74{m?jq@Xx>!x%Yis-@TmY|cZHBJWhpJ9#>q9E($03RTZVCPbXNvV zGE$=0eY$=KTmH#89Wb=-Tq&43{wl%c{S}X-kj0tFOCKSJNRY zUQE?xG8jAOos%~7XAa%79`BKoH=$2b=({(XzAcW^B~N;7efyHe@^$Y!No2H_(>K6@ z6!)$-6pWh5l6RTkGiakCy$c>xP}9t?0QC<)cGudfRG)U4#lWP-$@t zbGe?YOAHTjjD17u3$*-AP!hWgH7JKp%ua}*!%W+NCcKEw1 z@1f1Z20~o%Rfe5O2f1$GEc%fIF%L1?6)Q@elNK8vGD>UIWn+o84QMQs&sod2m5kM` z)Gk8SyxW%}tVxkltjS2xe>dSNRDv+zO=?@z&naK&dhu-~qQS*924cidd8b?u#u6k_ zg*mnmMj@Abcrtc%Imwr;!tdaDuhGnL3Ms_>sW`M^jI$XSe!>({a29Q1UDT=u330&v z=CI0abH)fxjWu{m@&i&~6I1Mc>m0?!SGb`J*`X{eAKZ88$Xytq&AE3>b`(vJp;Z{Mzf)?v5A*9O!X3L=q0@kCDa$Ps5-3{uh^Rg3{f3T)?_Ox+{@(G4 zx)oL^PhUk>;k-v0apA6HXJHkg5@Nf!C(UQX1$Mn3+$-=ibDSh)`^L9kJA{hw<#iA@ zl~ZN~xtO`>K)RKC0UyBY=D@3N{}zzxddcA#SG4%{OH0|TmE9-f|?L#c@olz>&1aDuCO4$dv^ zLycJR=8PIyhTA74vr~vBa%BYVM9EccdyUA1Vpk>!2Fp@ENy($P|H@vKxVlK+@o=KA=YpPQC0KKFC|2Z)nqPRQ==5o2Pm?; zMF<5yf0s>g^`$(Qgv{U`F_scV+x9sKX=r3fUxY;LuzH|G(5H>^^v%6sFKUuYTt2Z~ zKF2yrb8k^C%GOfq+ai})Bte%JhRb9Zqaa;bXS~m?@mUV2iPDdyfd09Qm~1>d!QID} zf^;Pv0Mu|gzjtF14h#+UMK4s!4YDix<|(0hx~3o(ogg~_HcH;uI2-1paS6VQt^%sT z_d7}>Q1^{NT$?PX&Edk@we@u}8VK0eZ~LC4tp8pmWq6h;o7_Ie++vw9=Io1JxQCoW z(?Xor7|^8+@AIpSCT4?HCCL zj|Uv6@d9Vvd?`VTtr0*?H8y~5J|u~92ceK2BHx88WJOFEQW2z&akvz>`pU`^EfyC%re0Vu*rYX0HRo%PlkCw6gR5Sf;&LKEI9O}14&>)mLrc!2-dn% z*=+jzaiPWj`}}-e@bp`407>%#`5>+W7Ft0Dd~6@bC^yQ|rT$?~|3-%E073I@@hnF5 zeCeJolX4+la(_lHc=m_J*8`aR%a~88{H1>|r&*WKvTTk?F`X%|AGZ(Y1R;HhSOF-- z!a)$SBg1LcWVmxcC_f`-gY)29`^d?Z1O(6s7o}{Ljd$zENf#k(7ShZyFrif9pCm35 zi~hO7l=msI(AZNIw{dxB49I(g_5cRN*eJgm%H#RsANi`l>Ni4gyPMd`rW0kDk!7U28ehT{A`fXM%e zNcz9I+6WHcYY?jo!%qdEOaR!o`u+nkD<+XMmt7fFX<}m(>Eh1|_?=csCGR@9))28^8PUTJ>{;e@{K??$!q( zt)OJTEFjoml$FG`n>>`2qaLEf<5ZX&3!?OauNSBjT2BCk;2}A!^56Y6MwN1q0%6Ml zWCgC;AeYaey;zS6s?Z0}bAosOQ9J}q3~8zW*WLfjbMVD6C6+?|?zjKzGMxrU3t6PA zx|!&Ibda|>0d%UTfJ0{r_}+nTq6pDyeE;p%b`1r$?ahvCa~}s{#V_k z_e4Ps2XwBcbK|gHNs7velnEL5DM&wk219YYd=xCVBIsk^=hR~k?ur8+n3n5i=V-!U zV`^e)$Y5z>Xl}|yZ*Obe>smOi#6+zOpC5#Bf1J$)^&P!TTiEKqaw&*kJMf zdTNOXb{&E`+#8(EnUm$!<-ug9VXxuA$NYyUcg+Iky)B*>z|{`YS6?KWl%98-d z!xw{RFaIg_KVJUNiV*iR>niH1&eK?oud4g}LWj>?r$wA|$`oDmj)lh7o2xeMCfUyA zS^ib<5vSvZh8u5mk6gvCAqwg2n(yn;z|=1frU z44qgKy5d+@%tEu1fPSHs_3m2;0>ViF1-HBVa2mS{8fs0An-|7KS4GJ}tHnR&j>zhb z6GnS{GIzD81)*-#s60HWMi5P8D`WSbTVpFuWP03J+veqG@i5}whJRk3WVWo5{yazw z7iNyywDsDwbD1lCx3hy6Wx75$KmU@P+kGgRs!2&#RoD7g_d^eV2u@IBg>=0<@$D8( zO-?Z*9Pt*|K=)?v9LtO7x1?B5Z*){)WV=HgN zxcQ?;{_*VxG)7DvURjwGPMavOK|-38rY6#R%TzFRX_MJql#bimSlkHu2K#@Kw*7l1 z>ap2qC5~Y$@v+NYy6}#VrOwI5^dhu=baB{YsbGFDqk1FZcNxjQ!XvjqkQ<)hwRFBr z&+~v26#hBKBh-2^jPLt4_=lqF+g zJajTef?LwhuC=F^|EpS=L-SGa{7E@ev`FvCWBYd}65sdSATmey_Fv@~@sx3yah=JqoWMEUVuPGT0U9WeM?7R>OQW_GbMkJX{l zm_y$<-NHr%XZM+(?vO^^`A9V-m95uSIy#GDBdT z*<8tH-ZCn5WVDZ}Xc95L1Kt0(6|1*~=#X1B21&CAR~s&HgHfXQq^7zBZf(FoNRzfE z_seW2*#Mob-d-7RKQV6HDSS#T$)OuMLvppnw$duhPTYV?VDyOsGXg<@)Q3*{)x) z-|@~(@nh50?@Lz*1c62zGIz}L0+Q`&`KGFr^0!3IblrIUKe`^x;&9WhtEwW;&NNkR zRG*+gHnbm;s3WO?HS%>np-Tt>VwKqKKvLN{x539>W}o#m(%bcr#*?H~X|`j`>KUry zy1Zv7)~q%fs-lx{%*}d|I;6={{*I9H!w7ct#8BbYAnNfEk#v0W^#=we`#dzZ$k1GT`$XRL_{{!Rxt-? z^UG=Da>uKy5{2vbANsN7+ww}ZkqvMq!Y*P& za=h8obLx%7{fl5(g>P3yS&}nq8!XcW1>)$aUCoa9F`Det1qv}`@`BlwFQ#8o&lCKn zhkqHePG%+Y_#pdZJpzG<3Da#LowLl1(+gCtdwRc+w9>)CmYm4mm2J!RLFFAv8cwm* z%-?ng^kEUv6uxUDox>_g}+ewtr~IIozLJ zVK7e_W(m7(FCY-Ba*QgH_Fp2hru00t)P-LwBSS3WZfW9FW`smcEYW39ZH)`}H~YyU zct-&&qVd)}fI@_Jf9&@3Hs%}++$<**F^%h?7LvVJW&zLsLc$c~Qs5uC^ctsC-BbIMpsA&S1S><-hznqGkI>$|z>K zeF7_I1XCf{Or+rJFMxu!MU`BVUIx;xFg%sU!iA|92m1v5Hgr8^lEO+Z%>WcLH z+K{~PCO==_e+OM$u_D^6Slg>%<8;+VQ%CQ}mrYV1(rr$o`*tov+a)q{73K2XV{TSR z?}fa`=@k26c}PgDSNws%^n&ShZ>6gvVb-=8c{$2Mj|Hbw6-Yr}kfO^0M%&QNy!4_J z4(Z3@@EkY|#_b3V!y+T9t|5+%T#;u~aepZjPBteY5Lf33sRPUTKs)1*^BDg&QmDRV zREiVT-1%4j9jXod!V!+#hTDz%UVIT5!L1L9OhC*LH(!B5eofx81Epjk-yw#8^?8LUq|q`6_ASTaLQ$m`V6hVg z;0XCR9k%X+57D=?YiqISOSRc2fR_Ek3qI9kpg6IyKKuZr1jro9rN{4MY@|6-q$i1O zy}2TbQJ2pg9>lI6Z%41M!`3qR>cHWMGS=Oz$|G=-(?aMbc=+6tXE&Yb&cS$%8ZvjD zD1S<&@xVkrek$iM-AyyrY_PGZTUx4|wjP!#=VEah$ru>oVE|G=4iCN+I6=VQ+B%ni zGIMdu?fErRh%juBkKm&8ro;YdMlde@J>NypKV_B)pEMyHfr8D$NdKT17%(ykQg(a? zLS71s6Bwzh>om7*XBow8d{JX#_d?YsJ%;YV$fe)Is(kguq5Y3+IjVm+F#Ydr^FaFl zB~<<&lTQCPD-vFReNb@^Wl@v<$`OR9$jfTr zUX)H!*hKz$^y5+KyOjf|=f#(C7!&>Y;)PH&_@SjqhSl%mgRDvT@}($TXcM?cj^r8f z!$Us=DVZhS;yy6l*H{4%|IQZAfHBQ~6~6ml5zbqLqI=ULqj~VJ8>wKAJ^J%UQ0+gm zyur5${6~y}F3!%ZY;1KkHLfl$h6yIHLjJ;(Prkl$IjaGe_wn5Y#Hah8XOzJZv9lUVUpineAI{+4 z*(klIJFaZJ_xB4{b2(#+6(_9mc8sXNhs_~lZ$+Y~U9Iz$MrblWfK}8Sc;C)lwN>w3 zQ4kV&NO{G!rM)5!*{Z6Z6J1nWeG`nDx=_!4;Xbe(ORM6F=JsdKI($EAYEqZdvT1Cz zp?6?M!#1`ha`sdbYtWxU44E%s;n za)^9c>6-3#YIttg8eHJ>O3&HNL^?-&;`H0RFQvrS`U?n$IiDfs6&79|QIwzr!xr)M z>C*tn1cjzf>$skx)8@?dJ>Sva985O1$tv{i*|VR6grOh0qWeDmnQ54CzxW-^ZC1mR z?uI~LZJ&(Z9T-#FqCZ8eZ*bk&n+2Z9RiC%U=Z;FVOe20{X=mIWiA0huaM0XgVu12R z@DSYjF~`M;-vrZ}AH9D5bMR|T@4Sb;782@^_o8Fk>URg@pj&ar*vuM@Z96yGI?F?K zwwuz$u-jyJ#;YUo;#jer{n-$Kz)w zeh75o%_T|ETDfVYkGIkRqF-EgC;|#o#6Jf6845*%MM%GHHr)yJAWuE=9NQWhx1zC% z6VqTa6b3}<`=*(Fa1Yg1WIBDgsm*24l$o%g)tb=kytg8)y9!(D?>zInfq;S_S@*e> z&>l>ijc#w$v1pda#_)|SbMSJ;%bQcKnTAyJd0hMQn#)3>8;kqRnkoXs`1(d|HweAU z3t0gS%a7iRO>VDqs$es572NKz&w1R>w<`?+uE=8)2Yf>_Jfs1 zr(;h`A=|<;&7XuU$$Z|Em6x^8v9*8zS$v1YMHCjHU}~?YWZiDQQP9G`@b%_xKpJeB>p1EGH=kl z(?jTS-eQ5ND3=mh$C+Uf?S-fei;M7ZXs3vkd3a?!CN%0@*ar(BCtu=Zi{jxlHb#@!fmh zWE;}^4mB7B71u1&BbtC`l2(hjl9~_I_dMuW8ml8Qv7vVyS$cjlaio$wd&5t7vqX*7 zUeTvPGR-wc@8$9Ly$RRXGNziZu*K+nIHQ22@i(#gM};*&#kaeujWbyDTs+9BV6ERN zhI`Dy#rFRN^kT+4g-$|rdz~L@IeRwXWs?j+kEilJb2z(#F(@;6qV+@L$dZuuKPxEM z7Y+K)WZBu)7KfmE+Lv9U{&pCPZmijFf*G%v@Q5Lhndi%DF>4d7zJa|`gmQzGG`5*Q*qbcEhyJ8Zlr!FTnYV|HY!@CwFDbM{d%&!?dIzMG0Ec@ zx`~-@L58ni&Q^E}@V9rzJ{@@K&J5N7CHkFEG+;f6UM0!Y(SSDcc8I2o*_{@&a`wJ$Q zPGR#i?ao0xSeU&rtlk@j1J8i$fPU#yJJ|Kr;pNW-I*WDeC_)e2@v-aPPuRhOjyReM z^ZmS>>hp6mV-lKbLbgb#W6R!T$WB7o4Pky8QZW}~)u@^;{YF zT*rPZRli^dUh4V^cFb%k!@U%prgc)#WFB_-)pN*ST8Uhj;_)QF(=sW!Z(~YsC-sj5 z6}FiGU}9c(pUMO#zcTo|VIJaf{p4+~ZC!9>B^<1cIB{LwFfo^JIhg*_FObO1tc+?$ zm9-;Em>2u-jqh?pry9gDqwQw@c=AVcYowUYjgW5ruEUs+TqO+~ma#ei*PQPu14Z+1 z@{xNFTBl=|H#nlhjaxl!2NEinm!meO=O(VnpJR%8ew3yQ3Ez zE$5o&eV((g+?q5C_pbsm9iqHimfdeX3ez{+3|3*p%!AF7eCPb`E~epRWj)iSSKQZ| z>Dak9j=y(DL9+|Vs>$Jle)!BAcJy&lx!4H3+TQs7O0-|II8q~$M41GwLV_ZunwGL^ zBC^y?6=^jP=Y$){s7i<_CdZq6dWF|2DVC-`Mv=pwVd~H(8)x?{XG74z;891cQ1<%xx$5WbGg+w@UHG8|b1GoNUDk*= zvlHKyc>hg#jwirmL*c)0!Njy=?Hzpr4K8jqCL;Hau3;WNe2j$q_|flA{tC|sY0b{% zBWwR|=TEWV?P?4{{nUKU+x*M#B^YF?x5GIDBwW`M0t35}E|s-OF(=i2<}stKjG&=% zMUVh47H-o~m>?^nM%v_P9nq`BhN0*zj~Qnrnq*rXbYT~#^Q|KDpkNu3Cdzb~P~>>! z)7#h5klBiIX|dr~NRDLStM}!agb5vv^^uJ-NiIxZi4g>;cqSrV||y#%k1j8vBoo6HF`?3&K#_@2r?W6h7XB4gIVa}vrl zfAHU31x%$R%N8`e78&VQo^3$GYtpcGYun&%d4p#wD^h8!UnirFz9S?;%2Ufw`on6r zBpT0LMoXr4 z?O4D}jMQ6QRXPA>Wxfj5%aT~gfAOC7<>|tfW-<2R+UrrRJzk~NFyY``zMmfl=GuOh zPx4Y}iO+H*TZeIn^coGG3P*e7i-d%{@o77*{`PZnL1$oh!?rxoyY2K|X6SlNE>L!b z$a05;gn>PF*H6?L;vSb@BC*Z2y=okQ-NxmW5Yj93Eem^h1-od=K74{SCo?O22;zQu zy}ZLDU86z_cwZeCe@^jA_Gu^VC-s*hNn<>*36%tzpCtaSE@MF|opGfpOoysUQp*?c zSbVJpmf>+5#^dp>LyOY2@(j&HM6iZrNmI6xGOt}a%DEAC<5~NQ=JyKRsvMm(T~dFh znHB|aaHhIrCpw%r)QL4{prdL=)_&bNSgMtDZ7D6?|I>2E?p_R=ybuF9I?0S7{Sp^5s1rQ#&iASZeyJI#WHqL#aHe6D|vUOIrtBywsI~H0mv4hKB=~7qQC!yoDl1~mq zhxB&YeL`STe3xop=H)%}yeM%QpRDg1jeemLhifiIli2GsUv!tor}NRg>gH*qi12A! zCYCqhZf8)dI8p7;+&FL54%HY9?a%Lj8!e(?<9;nQ;mHfY z6B0jaLvFn@rr=9*2)Rq?v<%M6B{5b=i2vegJwDEs1rC?*c_c9LF0kAZwsK4Pmw4{r z01B^QsWIj4udJswhZvo|g-@A48=gtZ}8gd`pR#hb^=L;r^4-w*Z_T zV$?q^`p!vfrA_V&8k5T(5mxKdhQHpIVDrT$Txk{&mP_%R;c0cF(t0#-U0QEU)S|RD z4~B*4Jn!_%5&@^bZZ$z%O;A4b@~k19>29MuoQv(i($U{zVJ6g;#*0UyC09y^bbySm z22nQJQt1e~ZC)%;uqel72@|<@!jehv7vhU1p03VTGrKm-qa99R*65I;D)r9zcwBbm z%k8BJv~u*ZV&Z7K`vV_0H5C=sWy`t)e!(v2bgbR5iM6POOmTH?WW8nFHozA134*phjeUdZ92qBgJt>Sf-F_PP;?^i&fKErQ!^)F^-a4FNm$<16D!c4$uJxpadB~FWhZTIYjt(( z><7<~|4)q$y}VgIxo|N5aY!ggufUK%!ugAsfJsFazrGf#9{emhIfYqTNr@%D zUN&Qm`uDV@jrBbl4)T8m|;{vYy2y)>WeX=j$7Uk>+tZ{F8_O#(su1y8)X zZ-Q?6GJkvs^XE6_gnyU}BTgNYuWXk*SmLb51GA-Lb)M_o7~9ym=alhN$PbEg!Upv1x@y7C*x1=Kxzq8hS6-*5>uUDMw>5W` zH#bXZs|94jEi-H?LD<+hSlNAM#}f%llAb-ShSmAI-KbX!m0Y}?lu4zV2h3q;@BD;+ z+h;K@cOz|lYaeb=Q%*IxjK7Kccux`;#&DiE5I;Mi7C?%LrQlCxFpOazY>Iee2}WK) z{@&Eu8vo>@fW%(A-#_;^A?+=zKNdRS+FGc1qz7-?@9V1POT{IO=F?6xTE;5@4q!Hq zg;8xeu~2JjYchutUy^@5+}pXj3q}}AB_SZ#o3s{nl*}o2{Vy!QF<_ukXs7~e$t=5E zW(GJ_==mD!r5wWG{T*`Y!?(V>YQTFuJvq%pg&j`|Gjjm)P`j7i&{oz~|FDb}>;Ag~ zj$XT?)Xc`_a<{zB%4)ZMufM!)v9h+-M6X4`*)k&Mr*Tv=m`H)6&egd}8nB8B{K`r! zAzSNYK44Y{W1_K0YFq0<-L1vNuYLDE-{-X+-2@Uf33%49-5JhZu0bB!dbV#OY$@C+`AH0L!7H>q_^(F`5Y=eL zXQ$%5ww6vdJS4o`dIt-}!#3C5=+vsJ;)(p;Xstx?Q9i8&4 zZ+9bT&zKAl%`z9>GMU{T9~WH(b#G7P|0%fe{{Ho!-II_O4*%=auU|4E3lh|r(l`3= zf#H3dAH-*9Y0pb2pBu*%8yxl!F-J{Vivki^&M1oWbq1zUS63#Q$;s)wTr+1^S6%8> z-Yy&FI<5nz2jlGM3^1sGTpa;AZ@?N~x^Jlvq zU_cNKTjFf4s*I{iyq2>TS+L^|ZJ^7mY`_WWG0=oRI4C0^F9SG>!JJ2(5;rA(`yl_ZNn(GA zu<*0&sd_WT4_yrJ+)r&XcogvVhqXgUv%8{?cUji8^`z{vsaiap%ucTcgEj^&%w~La%Qi)N_GZs# z>9~N#XjZvuy3l9Z39rte8JB_|5LBY%2KptJA4@JAe=QB%RaqPx(-e`cXo?z)IID~9 z4m3x?kFX7GnMj{%kBXglK!_j z@pmGf^SN9s9zG>9mgwt&gkza|dcD@opU+R3@>PrWmzCep&NpuiZe3PXuS=A)&0!6% zK8s8H^KPRWieM9V@m?AN|9P`~+TP^n98T!@bgwV#-^>PXU5X5~J#FiGwfVZ7-Jaw% zfiWxIY=K$6jii3*Mt)#_*}@Efyw3R_ti1(LoI%$nI*4euhyLgpL=)@c{mH{;Gfp&Oi4Qr~F55~B4jGifnscj_m-`nKDif8KZNW7= zzQ-RbZP?){XO&GhrDyyHbG!~!jvahT3XvL?NMe+ibUgw;_8$kd`b3Uo>R_lgNcWXz zR!pG8cHh7&_7vQ2vbJ!*RMZ!9ovmG-8!|t^FdwcTqRQ3rm{y*y;=5Bsq?ORjVF_pX zXn%QwXV`^k?>@#cJ>jy+S4GOLsE@RVbIGV^qLsRBeUbP2iIOf-t0{wTzGW=G>FNT! zeISe4TkaI4q(N(HvGz&tywQ{ST^%$kEM!=^&vRyHh@B!K_!NZ`tz~*g62U6+vR>{^ z_trxK0p*K4a^Frsf8WgQBHKBF_W}l*DJndm2FB9*Slb2^U?Ck z++ynzE=51biwpd&u4|MJt=#)#%jn$AF`wYs(nyUTG7QNB;zW%9ZT4#lV!hN}8?OFW z@qvezPm+?5M;2v~8hUHzUo!FGeX}N7rkkIf0|VE2VMzM@=tcD(skzGq1;`X48L$sO zYz=n8P9GeIbHSUX`S>LTyWyfSkY+Qo7Rh%U`bS!-;;~v=$TYdMpK4HR3SHN5In>{M z^w`;zLBLw*i;BJ#^m38#bkh(Xjf#KM8}thdq;&E#$L?vNUPLy!A}u#3t{Pv?;3Ex> z%N32P$sSWfEU@B=B~TD<;up{CHWr386BQ4KZrhc&>pWQ%Yu%hMAZJUk z@_GA|5z$g5!5?rZ?63KIxW)T{)&JSddR z-MNR#IePD=z(n?IQejD6PS_W#aLni8TpK_KKl4$L)tJ*ya*a*ntn0wXKLf`uL4kNO zm+D8J#$Luo)(&ETw)%-)P&S!hGFwpGZ3cA>_-#z;w2X|t+x_^Z$6l|H|9-5;4L>?} zajP-_F$A}2)Un>A)0{7nkpdzHxHb7)Xp_t-%0Kn2h8lq3Z#h*i#EMHp4*eWyjzgxm z89gTc{F)t$#NO7lPrL2gQ)!Gv)oFxh8M|1AlWnpz_H~i?26WN=0}cWy`Cdhp4##2} z;|dz8;AZLb) z@BUo7NK!&9W*2`xFJ3-8EABhE^0}!@f>Ec-4r7?1$#i~QxT%AtAC7^0E-o4ZH|WS{2wmIn+@j%1m|38LbzPHp;BU{h4lcZ`IJC)eV5+AUuFfY&Hr2j05{($Sy z!G@luClYC8pp!s0w7OA`<=+n%y)aVT5!XI_v#2i^;fJS(mzx8RH)mE5+oCMhvC zCL~E9TioQZ;OjAr`W?BvBYav*{?*a$(^V7Y8r&z`EY!Ck-7z7mGgSMu+Oaj8yjtkQ zHSDt@MFqwl1j3@jD^cq(mX;g?ZMNu4%|(pmC7X#Mh)|H57qBk&*~W`}piOnn z;R|IH+sdh*N8FSYT^nkIdrk&ao+OiWugS`_lLRvmlh96$WDZZF4FV`UZBoaUQgy&_rHm}Tbf18Ze&At>6?!K8z$_9mAzbI-y! zD7fm|^ge_R3l6Tgsxoi!mrR{vX+#LGa;Q&=I4gK-nw(+5S*54(L4%=!FE7c@d}!!(+F9J& zAYOxHhg71eej$sfU72tV*55>-J^6>&p_mNzIf0~>0V9X}y-}$|+Mijs_@&fCe}0@z zy9keX7Tp{1fEA>fBT~(yxqUHv??yt&Xi`s^!jD@C%2XsTAkSX~f#_AfYsBbo*z=5< z1Tl5=l_`aJEY!}7^-C99!MZBb@fIa>w7Zt0-l1a=!N~`cohB*d;CaqZgS0Sl@>DMi z0WM|n;4kA2PN|UGz(a9J*`vz%p&ENG3#T}%vlL{%6;s)xB4w%=wuYVq4zG7?o;@Dp z9RMk;xqqrwmRv?2(-$Z3tHT6zYrHP^ukQe{P98>5T1d0T_H&9&I2kvIgB+C&il$1f ztbV7b<3Ow_SPPOOn9KHF6qj-#A@@|io=mUR-%Zq*^2KhGn#@v=tB^fNycp2yXA+%H zn4~0IQxVx4eV0?ItT{Wq#zBW}2^M5%b=1-}TqefmjCl?k(ZhJ4$8lh_QA56vC}H<= zj_7rF(ZS&cp5*-5i8L2&w+O*m*kd#2cQHdV8CXYtfw16A8DhwKTdhM>q!Rx^D*?f3 zt2Xq(S3KdO;!q`m*@F;RNd?@8^2A97l;n6SdA#a0e)L+d-e}ecCy<_fB;ULKC~Pcr<(~grAV!;N z9=FsCV=$m{$)lgrc$t8d1Ytp)O0GZ1RNIYmu#prL2cSDao}`pK zpEh&5c7Z*c`5Pbf1OC9^;4?IIj%2-8_G;n$9-}z*0b#EEdq59QAY$l+vl*kQA?Bah6R{d=4c`w&O7LomwMfqG;=%W@JKnx;Kx~!=0%~<68V?sQ=`<7l5lg*)Ljo^ije4*l8nY9Wvu(b?5w@fvz3W<9g5{#u+RaExp=Ay$0 zSX|GJP<{;x?)tFWMf(2T)4MZyeB+Dsic!nD{ohv0lo~Y7Dv@S@k+8~>p#n1}ez!QU z4x#GdPs#4>{*v@n2r8LGLy|W*bQf24tr{TyBqg^Yr$S4S{lnyT_Gun1!v&V9{R8)5 zgiqe;m-YPF*lCH~`;yyi`18@k#*~^LM8x$U3+7DsRo($B0Lq*< zfGctQ^llySC1Ni5hG<{rI!vbY>h-{E*3bq&uleVkxjt$|>a1K=bXuAK(; z^{cmXE;lEUd+%d_F%yW_gBQLvH2*O8SNytaXq4iXtDpOA9-6kc)9<0SiyyHeW>=-m zxd|A0ln-7?Zkm(FPJ+cTa`t>JnERirSa)!f_PQ1MnK4gHL^D)r6-Ps<jCra6}aJgbj}71tHWN{tVteA6rn zwAn1NER3&Yn<h65a{ENz59A;tD{viUY9IU$4Gc-irb=8FnOy*GK&FgC>oFP z;g+~Rgc)Ub?dGLk*<;YmhwrS_GTxgOMe3?~anZ7EEu?MF%S*)cb0lr#5IZsK?!*3n zzSZ2=Bf&c}UvfMk?)sM(OtJ+@IM^CJ7k!c8GMy82@{@1(f(8BS=ZgBQVC}dQ7cucl z9ha>=z}fXetc9R(->U%`A+2A5E^wx~@h0NN1aH64wAerahTa(EcMKdd_jHVZfFct@ zT-iUS_5Vd=@xOD{n5Ja8X{h4?L;}CT^W8ZcrHmvLfO#IA05k{y1GNc0Z0_ocLaB>z zOb}a5wAJe_$ZcezX6mj=HwtvL*HyH&F9H1R!AkTWo)SKfdaQ6+tG;KXW}=c0O^sjOD#Q`PU8W#6LW?( z<_PSdWNC*iK={=acUH%k5@JdjOACWx9UtJKyTfMDWaDSrCtYLnf!FN8LP)~yYFEvs z3urln9L_?=gBA4kt+D8PyYfhpoot~&o7LTV6S`a?kB?UxJM~(eBkx3-kPHZ7f2sm?d1fV3n%^}oaYM^Wr>h|%}{gBSwH z7ju-)eVGG*u|o7k4Bb530T>DZnCMzGMWB)^8vsm~SyNkARnV7V2EH7JKumM96O@utI*734pIoqe<&q>!~Sa(qbty_^bwobu^46eT?^6 zJE}4(G5~qfGV1LX3d8IMbmH_%43wWGO4=tkIQGLD)WQ>*_R4JIOFpUhB1jt$xO6q= z7yzfke1jrj1fwlxEgrPf7U$>1pq1R~fTBzdO~tuU+up{+$j{H>sChGvJoX;n=>yTYBqF{cG405%^$%3U; zFVV#k@;nvAb!|XaOafHv+@nFj4zqt*M6GSr;LWM@g9>FM0!1T&`r<-aDc!0z(Fm|x zhtD;lQ&3xb1c;Fk2ss?G(W|7SWrE3aN%AoL9fLU`W`fq^n(`z2GFkTUiPw$#7!l^s zHH&i&Qs*9Z5Pw@!hR3h#>fGiBqT-6__e6Y>MpC0{qwXW#bv_y#b3mgC050uZ*)Hvj za!w+Ho7`eakl4pJFZ^D()S>$PI)|HV8v9i0RoqZVO$3(VC*&tt}isQGDj+&D`vG8VuOryB{wvdi;rT)~G8qZ^ZPC4DJ zm0Av09W?4$ckaNGFv+(fI;FDWW+Mt1q2+^J{KV-r!c%#-B<-E%?a6#o8C9nrGnp0M z*}W?4+~Sv>3_u%_KiH2Y@t*((!yE(O35$yx!N2#;yT|mGk{Ui~#_QxE1YqjGdqqgf zI(u>xEd)sbP?NMa*=CmAEvy|6q7hDOAhlT+UOSLGDc=O&#`2e$)yF%&>(v`AhT*X6 za4A3f56F`Xn9tKQOyqZwTHysooe^itZ9WV@XvYs(!IJ0eT6Pl z>oOB|^;S9mXB3Jn6m7iNs|Z+26!XpS&l zCa!{WC!VdLYTP|3^F zNIYMU)1O%X+1{D{(hH(r1}etqyqSJ`z6K1pmeo2zlw=g5t2G_=61s~ttHLbt;l={f zmHUL5I&LEgxV}p*FEz%HP~LbJy^~)>UHJrp1#;!)rtQw-xIo7%x=Phz949i6t!y2I zHL+`vLt>6m%g3j1yu^M6Tw7u|BHU;`TZhT zzHA_i@-oL{b9&rTgWXu5b1R9!Lm%_%&pwpUU?K- zQaE4h?CA6xo*od6Q-f1Eu^N4Vbd^f2xB)R%dml&A(RD zTKNO0WILJgBJXa_T03dq%ax;ze;G%)Lj3?HM^G?TQlt~h*7-xwQS4Rqr}$@l9F3N# zRE-*q@drt9HL!M)-eWTs`84=c2)x%{qMa4W7RII*=5Oo)9`DXQLU&N?ao(OX7Y@C8 zd&O;bb5TQ2G&gW<^I5*XMKw$tnE*0LB-dE>544gtQa zoX>LM<3>uI^p&@9lI>)(O`|>R?bgmygiw;%53Hf{bu7iWIBl+n;gH~;x?Y13+rl9` zR@biW8*%crpEPU);vl+2uk#5Vv&|vSvYMH*p-cSJ0_mNG!lCjB=e-Md{WVH*m5*Z$ z8GC%R9p7iw-sFqVasfuiwLM4kr1+;2?JMi4>1W4V6YKsH3jPSr*TCjyfb8q=ocs=I zr<~dTSB*oHJty{P^_DV<`?1Ku?r`~^d=8opl=82)F27OBHKZq1IoYS^N+?#S4!ieD z&wqFstJ^V0g7NC$j|nh((tg*hEpw2sLoW54E!11<5!)r`m$SO=*Itkh)+>uzlp0UY zXIF*OnAeEFpg3O6L8*I=r7hImHcZ`Q`RvtWXw?%{Ta!_zPzU`bbQALj9ZyPXnM#^G zN~V`!nV(%a8oOvTH|VM(cDaDruZQC^P3vfuG)ioGULg|pQBa+wUZOtv9>PH(7WE_ovKxQyCo zEul#)vnWY8MoWcsMrICirP=AfmFqp?CO1+^WKjE&Go``3W?aO{W)R!UKshf$GHv4R z?t8XN4g*fbP9`8;9myJ`*a0fRbQExbPgV?4_G^21ivo~ZEWHh zNOm=-iyesAEpUftSk2_}(PoGUTkoJ>zQ#!YB{zhWgTh5M<6eTwu<<#!NEu3fZ)Z1~ zAG^uM=bAp#a*6yzeX;I&0+{Y=?*zw+z3u!SssTDv)w;?{dbqrK6N29(A+y zTuf@`hCG4jDf~0a)N@yoD*Fw>6p9@SGR8?cs196WGuh_=CWS9E;42MZd0*uX(BNlCpPQ0zdQnkj4giKI%iTNt@M z8gb3N&(|I%@h?VKdYJo{&$~QPOYvDdye_^U_6S_OTQ|Vm>ax_@8D;ZMuXy(VLz+_O@Wg=9E-q&ou5ginu3jr+ z2Fo@KtRRlbAp6@-p^u`3g7Ep?2N!#o1sr*2Se97FtAidx0b|WWgxrQDl-tkVkubPG zYK)#h+1E9|I=N`$*m4LV(1T;pw`JY;Zs;c#Sq8J5Nakw)!hn6dCnMxjnq9|$c}>K$ z=~no6+iFM3lOm(vD}@)qN9QXlC(>%tNdj@1RhnOl%vnvT4NiuBI}NhiSV~9zMcur% zX95;?go7x-fWbbgCx;-+;dQ_EcvP!fDJVc~(+o;ah``xAKN@#Cv5~1*-BZ%SSmz~P zM@8O05^Nk27|+;(5B~CDIDD1+CxvBNf(=g)PANv8)Yq~+noiE7{q!wHRPff(1kUS$ zjqgwRRDk>a3RMt1W_j~>&Fz+dbL;ts$H!|kYZ{5TPr-DxTz)mnNA1pQ-g_NkO*|be zcSYaiy~?h>9X%`R5P~s>G)MS>uWE5soq~Y3dF`lCYr_i0$V*GXs@3_p^P7U)P=2r{ zT#8U|W!Tx3BmO%4cJYSZM|_H@q-gzp2y7PgM8ohWvO-H3KGxwo-n#%sT&I4c$^|x@P1?InHE)ub)!Oy_X7YM|Y7RdCx{9otJ z0J;zo!CfCI#Y+LsYmK*@&5BC59`d*O;La2{5`C!;a^u`^72CszU=(gXrBV3()4Fp8 zaDbQ$+!>n&QOfThEQ!#s-I7k4>vQhZISg`p__qE6`g&dE1(&@c05_~s)E4yjI}8Yp z&QSQzTCHP$SWRxK)0JmUna%<%3RzC?OQs|gHZ%hc;6G3ZMgDtOi2n)1IAdaA{rFhB zl~Ac+*XC{22)N3{1_AAMNg}7;J?l6nap`&s+pODwTqnCSNuLk&$kOO;q~)jPr;QE+ zm#|nB6xIyCMqfv?^oz1T^nf=7idHVG@0R#HMzt#%*28}wV{m0PIb;SPhpQt4N zr@<`#r&j-~5hTJi^|R>!@YZ>l=NBgd-iW~HWh~00$BHH+K#F=g127iG*hw8^1Z*>A zL9Y_PrTq^R3sp|aYWaTvAm-nNT(N7GpZEZY`e<#24ZrZS7+V#5&sFhntIB9P({9r=s=Fy&$2ZJv;ysZOVe_SNVaB@`q95Q7Lx5#80IrZ!z?*wGg*=Br6(<&Y=y4vq z3TxL&l6n{5O!sWFb|(OQ06-IF11rp}EJiRMiV^k~JMs5@Ze@wTea z4s%{>NY1zPoNrntDk@PceJ&X?QdR`#t^hr!?8ItaTU=~d7xOCn@drQe#&_*fV5SGg z8%F2S`p@R(eE<{E*s9$d<=<(&-atwAvFKD`j>iK?W+Kg(?>X{L9%-rLg_|#>#fnO+ zOAY{(M~2VLA=BY?=eOM3i^#uyKjZYDs`As0^wy&Fh%&4vTCB|UuS=NkQDo?H?$S^$ z=L)RT$@do99*WMronb(K|IqNE6S@_~1<$SiKHS)zUW$x*{AV%$4Ay1GUg$gzn1>Kt*B)zIfhYQo1*lmNQJaU} zAGiJfj0+<7qjs)aWYkbY0{)}=KiB_pO54`>Icw1Ke54BC+`<;Dh;w!jrA$qe9brt!eIn@IpH$IH>!5x!PL_ti5O0V8sjb-XFw+L!in{H%X*FQqC@6txA2X0AZ&GhGv*S+s}VL`gDFS&@U`E$=w*=4Ngx3LV& z^xj8b+qYR1sRc=6EXqL~#HdVM3?ohYM8xUbDXar%qw~sKbkVgIy>~V3CM6Ev4@euu z$1e8VpeA~AQh=}v0c-I0mGAE@iHmS}#qH$@AB~85=Fh15bPt@+$2~9ef&PIk_q&IS z6XnT|*{7_J%C14By4nWo@T1=!zgin{EzToPJ!@Aa*2bD&%%~#+R`8-$* zf^PDNaBcpqb)YC7T1U~x_e2f-?YsR?t&)VGBi`qh(molYTW%qfY!nyq7W{DxWziFv z^0X&~k1|ylq{XjNLV^p68h)mk@|^qQTsJR81qTCT$Yg65mwh>XmY6 zaL=e?_4@un++2BZ#GRI%@_Vq{BL|U4f=X3q%>WQChq5B+3twP@OLQ=g&=tQzA7-qq zzN%}ky)cmi#NKDt_hZM2HdE`}C*!4!+B*tJJZm%1d-~q%ahQwlZJW-fE_NeP-L#tU zR=uyX-XhsrpC>GCGu=uJHm1(%pe$-VNJ&2$=Mjk+a{_@NQ>(=TyP^=a6sz*Y-Q=B1 z))k5_yWm)@S2DVx){{%S>NMa+sq>+Mep)(2$Id^-AnY1(Gz^8nhQK}t+)zP>e58s) ztZkQ~qxI?IRjA2$tOG~yVpUsEJj1*#sm!`&aAD2&ww;%b91n3m#8x*BT5=+71ipqEa=o7tl_{!PPM6RFA_AyEzz za1Y<5K)zK(Vae!06Y_5O9ptfP85ieQLj%$9^x`VcbDwy(6U!DuZyGSpR8qX3Xw_xLB2n zst{$AKmAd%uaCoEb_OR(*$!mM-6~MBmP5FTLb$Bu+EUH9z4M!tzO6?kOXw~=2{=o; z8}nhVBa0HY@fZfw(}UIC^ckmwylYtinMmEg-e~+ONmf0s4&uDq_m` z!yjYzovE}*BWW)ZS$k~5`qhV}YK$1JG&LM!$o@z$7W;fpS^w|d4*XFjgWt7|;MdtT zs7QK;*u25|?2s;NbaIU)$Lm|!>1in))ZDDk&uv>lb-(t@BwRjTPTlj^fWdAZ;TUeL z7rhQ)UoG+)PuR2%@5|Bc{mE=bJngR9Ip`XZK*nVi?{qO+fZ>Xkx??@UFEYcC%uSQf6r@a`iLDgn+e>h9) zv*<~@ow$o;{Q8H@mUZUOuB_wZ`#=$5Vs`t!thYWMW{O<(CoSvFbehy!p+lOt2LU)l zM|1e5Vq>nJ+XJOhT8!4IWNtDBe+Nfp{Y5Gr?)i$5T#rx~`Q8(+wa91=k@LUXY;h6K z15S&2*_@aLXn%a6?oH%|l{nRO!mDl)>q51S*`ncFr3nR@eU;dcjn9RYwZ~ z#j};pTp}PL08NkkZqIxNx5BEt`G|`{^hl92yf?=+yTI&!+|GQxWkMzRZG%^Ox6y*| zW=KeF_8K`tkJ?$7pl);}hEV+Gu{}JB=9Tun&H^w7wfDbNuKP-B4t+8ju7CT$-NCNBJ8M;MXSAFg2VG<3Z8#K&!O7ONLO3vKdBuWjWi(x4** z2%;W?K}&c2uB)^Y?9M*k4?P0e;1j!!5=Oq_$cx4kConjAQOXs{71st^*9vG{wAEzE}EPf?x;)chdC{x{1jXza+S}hfE9sN_J2*p1ClMv|rQ@<)4fnTu)e6Jw48iv3%!8`8mucx=u#YfyADmze7WReL>IL+k^Bf z-XK(5T>OiRshJWhECx21Bo;2)y9+b-maNm0Pav9joy}@6kQT1i$pmzBbJI1Hyu7%m z@u-9hVq~mnc;f)0=%X~bxQIw>-L45-S@^{Y+zis+9=bX`HG621uHXO#d$J4Woi*8J z6WZ>fBF0HJbE;pWoU|}CJlX1Myjc1c>THi1M4F=ln zvLg+gNyhwrbGqU%%^V5;HTb36^}a5He~nfX9BfLQq$K!HB!2JIp)X|<2B0*&l)hKk zp{QV>M8p3D2ceKYUcQ&HfdxH0fPrkSA;RKNDCq#FSw}=_Jx>b;)PPWiyk2+y?v z!Amu4bxfbb{?)_doAvgk8n7Ek2b{k#P@L|vLm+_dtfB8IQjt-^0|8X#L~5on%SkIgC0!aL1H;pG>9L31_RSmfO0VRKiqR&ERp)8We{GB) zwDwqV7v2&B&6Yd$t$oG++PC?qD*>P1arsw^S8wm^f=}Ek!e5{4`+`jC)r2}L<*x90 z<){5#qDvp}m|EJ& zRD2m-ZUR1qjIxE(&hI2NPDM9F(vpDgDJ<(8AOYc+wYe)DUZ7IKqJ9gSsM?tH0blJ` zCDPK!wAP-f6qxT97hAB*w6?fBXXpA;hGQo2+FY?88SV9ZcY<8U=bL(96NGrjip^@Rc&3y*i(N!L$lGwrn zWbr0w;urII^yy>nafk*tJzcJLk3lwO=hwSdceY{XE-Rkjn=I3vB2GECK&L>4x}ou`sFS$(LmeEGvW-le3gV2QfmPTs$qn(xi{vh3XYGkEMoq zn{w>C<1th>>HuW3r>6H$t_8hprL@27FYh)f2{WHAVs{v7nSmjNOHrJsagg{axtzbs z%WY|hOz553%lox+$;*0WW? zU!}hz&)+-ksGuk|S5PP_=hX6veTbd+U{jbu9cX=8Qt9#2Z?UMLZ;>gC zotS3Xkd@*U#fi#~HLAZTXhW&td~5vfoWg}nOZTX>+6*Q7GK$?7ZM=n^R{q4?sq+W7 z|BiVWTlg5Qk8jU>u4+|5cxG+MKHQbWik1E7)=EK*moi; zv*EAo1*!#VCilI)vRaY1=XXPfK;BYv9NZ>kYzk|S@-NTLy%8Gc(w}@Ynp~`cmW~=d zfTON?j_Y0T&e9@VAhx^MTzQY`NYV(-R-fB=nb;&feah#)Z}YEcRJQySiWJ%8%+*?L zI;jX;JH~J`?C27uKq_X1Qqjs;rZR;x%J*WnKn@CR7tY!u7;$0`E0odS2n$LyP`gBG z)n&*kXU|crw*LJ{G?P?1i&Zxax8SW>uI}N>MvX?t(d>L3qD1KDRUp?T6RVQ<6;Kot z5^3v-D~#HFK9oDs)Ct6HSrS+f?paO=*1S%;W85*lq#KE=!UfGe^WzqG)7@k|B^ZgA z#b|9sKiyah5dXiC~G~-8yUpDsb`(OkD)T^ch1aVjiemC7wN~H>h1T?yMqD5 z=wG6t52Ok#JB(c84GbM_&iX>$cilq<2M@#mcX4F)Ni@Kfd zX+Hzzv1$|3N`Esl$VCi(TVB3r7R(CXup2-_Tqi{}*}0aT3|FDRX@w=MD%Rd(KRbG( zR?c3YK{4D|D~-xnh3qt7v{QLXZcvV}?t_>DL`!VZsNUk}Rkb5(+OU%v@UPzPERv2a z$dcxK-Q}*Ex=S)+oBdXuZr-04u}!nKyMK(${?EAF-Dl~sC*IF{CCZ{2XqBPevoqYK zMAq--74ky-@@2{s0qO5c%+o?(e#>f=3(w(!{$w_@GVlW-yEqlbX#@_9af-sh5RUP9 ztYV#;?A4X#$iYNk)ZV?Pc9KFzTt!e|j7*rqYUzj3H%f;cw_Nel1pT-^yHTFX{y!e2 zPl#VvxY9~h&!#+5vCI2~jnqHLa4k&nYTPe#U)!2c*_RwRANSrLhK|5=G0KL$x*Yg; zpf3r*Y+!s2O5ExV|1Ip708GGXI(&>SWxqd6b0+kKMSZS0P$a2unl!jG7>|06yMwy^ zy#b6HH_F4!oqU@dK@cd=7)C+TOe?g*m_)3*Y4Df{hQV8BSPa)7CW*}{iehM=dTxds zWW0-^F?zkHb((6R$=fi~!;p1I$KJygc5o3|R<<4SRdBk`2A#JH*GFKV=VQlr1p-YS zTCe*w6V+SFlhQZ1XZL#;<3#<#TzMnjZ6|&Nc?XG>U&);a2|ns%z;;MmMyYA}$350s z>|4x&OqXhlxVE&@S)n-!y;~A*#hX)zTA*~BKjWNLam4wS>{-fm3;2D5yV!Dl)bHH2 zs}S^MVeS$28MM&eorX)e(V`4{?NF;5 z6>W>t7M@V1Pj~?ncb^RV!_o7diOP9!zhZWQCBo9t!d}4}jdJxLU<15}hhw5wy-{c) z4*0NCp`)ZvBTnUF*y>Zy7uSxbsTEeK!%u6v6fY7^lDmATObxkp);Zq!fJUb`VY1c0d_7#dr;^L{)jd4_Ds{4H>XsaR-K1*Ns zslmpVYf=-M49*J3$&$SeFj#cka}Cwvb(T;|jj0na^y6y%oID0M=Z%UwP#o|>t>*`3 z-bPlnmYjN`AYFVkE#MsCRIagjo%ioQBg$7*@$<~RT||#=&Czv=E7!5VEqfAe z*r$3V?&TxWI;M<%l?vDm5;JPbN>ibu(C^{jCZLu6^?z4pRiQabcJ5AJ$;%^UM&H)$ z%}-hZ-OI+PAk7!9sS=Z(1wJU2StG9Y#+6I7qo*?}pp<7uHdef=QGGtoqR zD)i7Ti({Ap?{A>hDI zTUFJeab^?3n|0NkY3Aq^EMcjCc;K-ARt0`~vmTB?W4GE(JUq|9{eXTu%&a^VjgoTi zc4zmcgeUjSVHjj-^vS@UFVXB$V&XWbT^Ju$`0HHU96u>V{732%zi^BYRDNZyYa5M3 zwgT)R&f}a%WhaCJ8&cDRN{HMY@ql227|reghLVD0u zLbIby}>oQNv;(##)W`$*vB|vuB;CrkM2u zj8pg?+NL?DBtUq3c-fb?N+EB!I$4p2JS=Mo-~vBwp&#Ya@b*v^~9_b zxyx^6PT9b;UxU7c=!-KMUUC^)3XwpG*%Gh{|8V_ftWLjiL9EoMO$p>VOg))PA025@ zpnpVvzSi{=S!^WyX!kI7()gBlQ@!%AYB7i@7*mpGS-d^HZd(D~g;lT}}$4m;6&+_#}aHx2vJ9Q!=7Uv8oy zfypKAD2vP$n!9K_+|q5Ue+5coo+rx(vTHc@?`^u2z+p|y=vi66>;{u+&-WuVL}YA& zE!>6m@Ich5Z$ZS%zazh}DGLA#9}3{`FnP|x-g=}N{at&hmeGJz?X68gCz8~_Yx;ze z2#gSY^I=_Fc-_U%Fh4Ri@TI8`Lj8rluBSTzhLc!|FiVoqK|ZZfd|$y&r;zKa@W(5W zek!gj(P?>k9g@7!_K9mFzBKa-G>4XB52L7Lg5rsQs_NfXddUxYRtjV_ES;euf}0bD z2Ph$0%AY{c=8OmwX*{;dk2)PdESshPhGlgWAu%`)AkItJ7s<(d387u}0YTp${YJj7 zfXE3HI+Xp-Lf#j}n1dm@g$%PJGDB6#)4*~jJ=UNIWY#AGoOr%gp(2V-%9mp7;+2@d z3#(5{OXQNIY@qM{S@D$+Hs{lO@ck#><)R*hy-c$l#^RJW0&>cfplz}U;Fq6m0E2u1mJPWDCNPP?Gkut1Ah~kwmjWfxBMR8a?v0Zs>Z;h>!qogPBb5X2+ zaFF!SbFXG&=PvnhM{4EEZW1yVIUGA1+!NX7q;yY`@jfN(C#;1{)+K&ORAh*bPZvnQ z$1s-7NNfvnBxu6A$q|7>;U0<$xMWdA^|=}&ty8?$*xMv5=D3r6OcU$Wq!9PVH*6XfVE3}pHIa8Hr9m1I9X z;GZ*Kh3WyPTLk$Y8qU#b=;&7R*=RtC7LAS^6|a;2o%+=sTLbs>Cj9x?`NZ%^#G3v4 zu3*{Z3D|gxD4Keh%C+q+RQ!6Nt2k&kYZy#WFmR~axVE=;-q`W1217U2KIJW;pyF8% zIQi&(qbApElzd2n9mewtz*M#Hhxi7oeHo+qhA*xI$sVc{&5Qgv7<5odec!@*d>nH{ z#l94GU|%vMhU*{^4O!y>(W#)^xrPRfv#vtizYALyE}NetZO1(xfIv_E+~**&-uaPK zWu_kmS^ZG~8HsK~dq^)P`h$fDGWpgaMiw)pefmY*sCRzwA)!81h`NFf?K7pmP+9CL zdOD-2k%AxW?L%#=(c(>@cM1;SM11yI;A3F}Tb*l^f~y;KJOD3dq56Bg2Gx82dX`4A zGPvI9wLv1oJ^de8$-Op~F9EzbJj-HJ1v!cl{R$0!QDGndftUXSt&Puh zfB_KhEL0di8s7kQM+TLDM?Z7gtxX!5U1ngM%e?$C55C9KK0h z)n1;SxXtMZG;qH$1^0r;Q-W@>HaF?3@fgPg^{4oE4eE)ku zo&D+jLzmnq_apw~txoV%bRgIsNa;|?9+t{fEzc&IuLB4NTa6lFf=NtsUyNWqya4jA!Svgp%@lBXV zChVKbYU8Il6|hWRC$qTuMMO+9YRP0=UGR7<(8l&lYGt?Mrg}8=TMR>jnXwZg{-}a) zaR!E_-Uz$}(yZf-GpP%{JeYWsL;ms-i58utmamDXj*gCujZS8%3oSB<}yi z*;_`%5jBgVgG2Bj32s4xgy8P(?(XivHIU%$?(XguoWb2)f)gOP-6r2T_n!6ckN4K= z#Tusf%v5iws$IKuwz(iPGn2*{8T3a>kBBTmY>kbjfxnyEj$TSiii~1qZqB)~Jh!ql zr{b9v#Lgv?!E?NTZqNnfPlxgA9gM7s(wV7(&bs*vHom*V13LW|3HR>2FD7TYS1gJ` z`s3@$z0gNG9x;gI0+ zU1}q{N7#8Hkw%R7+>xJ;eKaN&ooiko+ut7tzH>8q7xVil%>u|!yb{&zh$|ol!?y!& z-{q_OxpGMir1#;v(oP^UgD(KG>g-dza&!wkuW$!%mB>|W(j;w__|Sl5?p@Y1SG+%KJ1hhEK{aaN9RBvEE%_HJEP z%U!hr*O#yetr+Z*7l1_{C0z}eYgxUVj#n|5hsv^Bt6YHJl0c6^^>ybV{k4r}F!!^O z`YUe%_~Pv(hS;uaSJ7M}qZ0A9DlX{9`yC3ZTu)6nLUjB7vI_{n=HlXl;Mdafv{wrt@CHO3>|9do3YjEF*=kUvo*nT*!`bf z|G*$|s`%D^Xm&sG1?fSiAKfytsP=x5;3y`>$hrhI;(M0BF%=zQA9wME_F9@sW813( zj@`?j-7@PK2Oxq3FCWp$fXmR_X2nw4Nd=N0$vmF;e(kmwG=KHV$39*>^StYT!$k@J6(~=rI7I46GcowQyuv@C9)PN<*H+2!uU6MFf)Uf!} zPT{Ui%9$B^>ren#RFj#`fo10Tjl-kSnASD9)XO5X3<4tPSzL!F&snYXa_0FoLGm)p;>iJ zO(4#Z#h~XdD2h<0#OmPi5J%clM@NTJHWSv***8>!s4WmF0o1*;WZ`|1s?qgQ^__fW zWyS1nl!awOwBtgItL^vcNoJJCPLwej1}@1-FRC_y1~Mz579QR%vL~UPo((s@55D^z zKJ!!pg?(DlIqQQYM)S|;B&;AmG&b{AK|($PtfzRp-=|v+bM$Bg3GRX}%)R6?@ojXd zhBv}(47=UP@xQyB0}Ac-#EK8xBngGjLlm??_>e}JC2bQvSD$KQqc8SGs?BF;&(Mb`W#RlG8SR5=*5|6(MoFK z)MEU7RjAqeFrg4-UxNVBUAcS*?c{`#9p7VY<7((fgZ`_r~=cH@1 zDeUQawv*HAu(kddBEwGI691dmL%hePVQVj-EudTpm z{Jh9?@R!|F%g|I+qin$9p25SL!BH)2l*XIsz|@epilU`fx-NN8JWi2JkrdW2DYR4h zFq;xqtGLV>=V&uIttOivS}Gj}{?b_;M~iTULh+z_%Dk>{?;r_ojE}eZ{CTiMs&v3v zfZ-QO^x&f86bUf-ot({s`Ap8AEXE*K7fu*NtAxc&D$$y!v>UpA@=VBRE$6PbQ#2hU zyC2PPQD6P&j+lZ+LmpN3lyRO?{Wx~tXkpC@V8EhvY3kY4Y!CUALq92z8nr(fTtpZz z9gGlzSTzAb>37(blu(~~+HBku_U5rX7-Mjc-N>9L+{tFP0u3A-h{w}^-~oNlb_7jQ zfO=GXDN%4wvXNDU$%Zr{PnygmqKC;Qo}tutYWh!*E-Y3FNX5E_2Hy3NCu(O}sZ zZ>G&{9NtbYZ+aq7;YVuiw@DQEdCyXu} znXXl^GdAJRjNRJbGn*a6%LNR4t)K2J0gwR(H*d5y8UY#nko(;l@sKt=%2tLWK!yh`*xG>a;g-^08iL+7z82QN>*0wXVQ0< zMwr*9wBd-7FzyjxGosLOvvL%tLP8P+dFN9U7A*;0VH^0SbF*uU`xBjX&K*R@Uz=$U ziPJGsFGtFj3$1^3OHCz#q0PFqw_=py4yy>HxdEwcEj_xS@5}a<_C$0fbx!u&?NwX6 zj(5docBE?ONQB*S?EsDlu+Zt-6NxAnEAWn5G^J&b-^a@OAKX^1%8BAF3?8E6;w$;q zX8P~i>+Ru#_a%Q~UE=>xw5{Jlj}@i+Xj(Fa;qQ@3K%l5!S1_7Hu3&46F_)PWtMP_V zf#LXuqqEtmF3RuHx=&50oos$8P;&a1=(axWD@DmFOZ>!9)1~+zvG4y?sHR|bLAdUx z;3cX;NVEi!N3LE=DbrLxr_ru?aMF?- zvo>#7(_#aIlbTO4-SK5h4EZt}hhEWSliaSkkrJZ(hXz%ION z_B%%HNz2$27_T9=$(Z-a;i5~@e}EMMYl_8tT6?BF3`O`KByvrj=j zzFq=Zma zK4!-<>fPQ7o%+q4X0YFOYxT`=Z5BdXT;TjNyN(DUZw$6>KoMC5swL8^ElSP1*~F?B ztKE>6ui8nC#C*=8>0ZZoP9OZL6P7(Wh1K#)n`W&|@HfZzYMCmTouN6|L4Nfz)HdKn ziceimagYZ$BT2+BpbEW zq)DIrO!_>5!^Hfh*tJr}znITZVsge@UoI1yz5YPR36Mff5zv8%>Pa z$dZqos=+*d1JngZPo;H|Zo^q^N+~hV;fK%Q*#JD4De?#{9`5XzP7zpT??)Kz*V);A z-4cZswI&?KXbGz?t*j(M6%Zhx5(~6%C7(_1b2yIiND7ZoDxxVv)nOz>#7uL$5hTdm z282S8@U%*Cv_u35QSGbRF`d6-pVWtg&U5xj@=UC-0j*U|+$ zJVV4C8GTd!*>H-7BUrK>(;6wstCy^{MNtL0lQ~HO6Rk)>reR?bqzNy{%0^-3nXLS8 zoo(<$Zd2V%ujw^&Gv%4i$eu9?>`1?YmE2^N^enrcy|uL3Ot0RT0L6lb%vxlXiaK#S zSN|Amlz;CEHuw}WeI@KsL>zhY_X2dE27@HMxN}+-VaHbJDMEiEN$ymG%C-2hkTdQ{ z;l2#+^!v$Q9CdP0s5e%lv)nueC3#6gy#S?;HrIs8w}`g>IamvSCj}MJ<;Z8DH&}ak z*SijaKzDa>(YQbT%qn-SaS!L(HVI!TU7E#8A&I8fk+`a}2aIwHg0t7FGehM$3~i?X zWv2px&zFjg-?c^w=>q(Qey9FW7QzL=-+vao>G?Sc4WeEcUyaAK50i~j&`v9n{;A0* z<1(+Sbh%veAl%y6V;}d5pPl#_$ABu6g-mP^>MQd66rZZ(w*rLtXGxUrQ>!#MYQh8% zC@7X?YP^pC?oedNt+fpbw-xMqmo4W1PMLlspY9p~3tIk-OEYyX76Cv1}co-(K zG#XNGu?wzb{$48-huwFqiG_4NB45_-g;6lcsxZKs%80N^A_FHrY?FquBwU+y4T&q)`v1yR@P_9`SAWDJdUYY%U%S~(l`UzI3Z}Z^Qip2 zF61V4qNK_->a0kZ?`z);T0=`~!X5_5CE# zO_cKYMNbG^?)aOq9U*3wO-QqdnRC>lk0PWhwQ##X1eEv zt33Nqj5kO9#p?+c8VT@iS{xW?J(_0wP+H2;)giAU6M5~!Qci>{1=6?WjrHLl;HCX? zqM@qp>&U z$0j9dSNhF}K~*nH_LpwI+sVJ3^_wTLQG-{GR|dkmXH@6-Lz0x4LHpA^OB7$h{coDV z7dD^zCIutIqRo!mFCzViZc8N;Q&ka!IH- z6)fcRWr2J>l|PkBi;?H_U{N5n*-EN~YZxSz%Oq?l9&qh{J1)t%&u$uC*Q|UqZX=$l zzKulLBPLX^&2-GKqf$$8v<-}|Fz-{;;cwwsZP9j#dm@2COMox1)u!gM2y(pooMf`j z_V$62ZKY+!Dn9>X=G%5%(4)P#zkO@h&2pt6YCz2mGGC%Hn&Tg!LpDQPiSQ)_;AzbO zOev8)md*;7F|r`0I40V~l(c8*8n^724_ZaLG+L5q# zoW%@160U`W$J6=_)F-rYRY(@Q`y&|KRhk}RGHO*1K_9f{v7xW*u zZZmO7#M@*ZhL+EO2Iv8f?&ofye1>6Q=ZF>=x@JeRE-^3kx7qs7^d@-TszSP&6|s)> zT9Sk*wOZc*IhXKyEYHlLKhM#Tmd`7fyH3=D-#Nsk6wMJo&FJp*h1p8QxQbCPd|c8>BB`9eQ? zz9Ju<_wjg{?|#6Zonyo2b$Pllx+eP7P*dXr;7DqfA!A=&j82Cnr!W#Q~`uG2|!~?}&0X{Kb|M;Lkv=qFy22bLMMuP<2gB=JoOG zJJe?$zUSWWKP5r>JhtgIp+TbU10D5AG^~Yi1*#$rLA*aKXorJ}H_fP&r`Kxy(HLR5 z!Qs10Zpl`{-TMU$KVD#=Uzs!bq^y68?oK?Rx{Cr@&166^%FLK@qOf43lmrQAR|Q2` zV+r?h_eMuYt;;L^hoW-;C}?kBWy7OfVnjv^s3(ozEn=8?F{YFsUtlcBm#h1O=0CnH zeE=W~VSd%brIz#bXv^a5p9=$2wGl2K>5wZrJ3BMiJ9#y-q(wJN-ySYX7M}Q~$o=u{ zm%=*m4^6ODzy3|JN|KK#D)s-nK^^Ivz$Nk~tv|6kElB?}#kO51ZyF~d`g#2A)4A`% z!vj1MQ#5*ZW6PKZFh`6wjnJu_^`&bVTA*4z zZiChv=c2m?>HYg@+L=e1xGVJPuI{3VU; zU{+tzTJ{J@lc@ZumFAUu+tKiqad2zMv%Z4HIj`d>UO_D;=ZpPoF1f#-Ln1Bct_x4D zNSfjNFvj~;NY2bGE2G0*kEpDo<9FAGWe{2`8LEQlWFTqA(vM5}PoAaHF_;&m6LbvI zTVTT);SIhA`9w94d=t3>g3!Nt&l0{zx^U*%JO`GGNSKv$?NgFVkBssnf8CgCC~Uz2shTF* zIM8l;ACAi1%+?Mk5-IAO27YB?+A%ZLtL(VE)UFEtGZ|Mj@wDN5Aw|z#>*xG9o{^G{ zqYzW}($xGnE!D2>=C}2ulOkUnC5}y6D5HO5%g5Re$Wb|qYC+boNx<;!`fhwtai~kV zxYxj=wRE~5duKi3b7oNYr4UxtMyQ_7__HCG7ZA(uJA9vwynyiz<;Su_=*`@nwZa;a zFpj`p*oe6qrGx|bIdk_Rzq`woj*e9}4rzc(oNpTiCO&)v)HPI`k2#*p8Ll98k#@*7 zH-C;KiYONoQbwEq82ZA7G9L}2;#qw!?j~+86_q_-E_sTysp-Ij<{xp<>K8O#UGemA!8bBbLa{ z&5$!Jb0Fi73#5nJ`(C$yEhf^!3N@1j&e-@=pMDD*CT=u${7KKDlb*d7%*bbd>3vBI z6&4l0{tQm#(@t4-<^ctIEW*?Xv0tV=qR zIi}x`$L(%%0JxT0OYWEVmTBDYDH9!%UQ8p;q>E|r4>k&sHNto^BsMM{9B@VQh>YMJ z0kByQy(~GKeiM6lag+9ru1x$_M?lu&>=_Vf=^i;pt6eZj8+Cc1+rCdh7^iHcMNgak zPhPJ3;Zf&Z<0;nTsd?Lm&aZuu(71%B?r$<~gCc}YYgpFfuB@jba*i78PQwuQM7i{J zMRZ@3?$;UO^|spS?VT<{o}WVH--s=D@BGu+Vg^F$ZL4`Xx(U_UmtqN^m$BnFWSWdj zOl@w57X!QFMRFnZk(}{Q6H<+hrsJ}SZ+{}#N_w?-D^poB)8`#0CUoql=5}WK;8?AN zBzg(cmq_j#VVpF(4w8+42IlF%a+*`lG*V%{|< z1W|1<_bD%us5bmmZGYjQMY)k5?0_HDqrD?Ses^thfNJPhc$ra}+= zFVcQQVO>8?hpXv3sgCGC#cN4h<%{q1Kir(f-o@os3%d@=zi4TMZHED*B+2N{+RdM9 z17i2y_igpetOX5dL>wO<*EPOf6kQ%ZU`&5vV4tF;rSGc3sv{xyLIs(U`fKL4r;qk(POSmJ=ucv})?^a&9EFw`I$? z$QXZq|u5|4l}O8s)3%(S*Wj>@y}ay7dy zrv`SN6O7>ZND94Nwm+U33rkId0EL>LzC-)wkxkcO3?;GD0B85bKKWOp_p%fkG((0B z-XYIn-BIKjVM;1j8lA5^oGvA_Ec61&J2+XWng>2f2{?`2XA+HhtF2zcmk7Z$F76lKJqHy6KMSG#_*5R`Tw znwQ{_kmoZHDNqvD+uc^;$j2sU@oP2WiXI4jRY65oobJ*d_Qw40im{`(6_kV;KJ{Nv z26fDu_n8;76FiuH2Wrz@sWrKk;6pwKC&K$Dzt%P+V68q&+<*nsv~1)8de69h1e4Q* z#+uediuBT7Yn}IBJQ1I*waud9&>5K$D~E(uVqVpjvu4(pC)iw8Vx>jVpr@pGA*$T` zC(`?4uU+GtB)gKIGVM(EKYT%|ll+4Ci#*{o`u!5JEL;zlggs}73sfKTO9&~auM6)j zA{WTRwTg=DAQ{m&B)X|h!`Y#X||%(5VhUoTrE^c zZg|~Z=zhn wDcmtW{-?^x;IS;Kx5_pRI=Br<_!u$azI()EG>9juMb&SWd;c~~4t z(XXR5`S|{Zt3Y5;PIr-^(qE4owt4uHA2SvQ5MOJjlVMk-7_Tzt;jAd=Nu0&$$gUk9 z7R+E=&*io^^=ZRySav|Dn~cjifxHM=z$m%c+)IT6TN~I|*?}YV-ytTIn;U8uD1dh| zB>k{RSt`=Ocoz>x+d`Jo*&*P&5}QkDtGW1scvTWuQ1-~EFI7wFm(R{C-YK}=iY`C| zBf`=lZ-f-;7M48lp>h;DA<*ekm6Fk%jaCom9Y}2?ELE;|&ZO{!AW<88Tsu~?FD&S^ z#Qwyi=f+sfd}zC&1E{j%`R|r^q4z1D{>}hE^?+K0h^OkO=tD_VX-^98+)N{bzU+oD zsn8v@B&U)IB0*~JzT`@pwdy!bUk{f(tq_+;so!RVpM8W1^q(UWJK2vGpX2W$9-B8_ zx5jsgtMRJ^a6FE-Zf?2>$%d&h50tmva0Cgd2BhGAci$kjf$3#7Tu43vLm=DIC< zrc`WWLJ~)_)X-9_1N)#_JE-o>D5l?>tX9+}Hkam7Rh; z5fKoYR@?R3cH7tGnbl-~-YWV9M|@+Ra*!!gT~qhU7SpSlO33Lha;Fx`WaLc>=+bn_ zmPWT6ikd^jf}&Nmv!jRj(_B{ecsS#*Ghoq4A3z*Txwgi))+UDry+l+*WCVhU96^bf z$-C{7IN*#{JcN;jn#9PK5HvDNH{zJd}}X=8r&G2lpfxVIGi~d ztR9Wkk z#dk^$ro=^&)r|##Jv>|57J5^HrmEq+`Gcva}zo~gqLmgW9Qmwvgm8cG4gtT3w^l=kk zb?5&(yy@6R+my2~o_O8kRyWA=BPAs>$qm`#SSoa;)(T`v3IURZ0x1Wtn{y?8!;aPM zqFHRufmi*anvumDE{*PkycuYnS~nAcw=LH7G(9c~_6!XdKw9*#f}9B1EH^K6zU{GOipO@325B$em3 zs_7rwmexuyt1iP`JwxTOeUS^P2JW;TVYQX_#k?6Ho;k^T^;0R8RK> zHNsYPCrDuB8p{o)I?b$mEbPso(n|??Ta829>Tucr-D^FQeb;JoWJWo+TEU^V)B24I zJV{=i}zLae^OU!G_s*Q@qiP2u4+TZ*tVnD%_U9oJ=2=lXPARImKlMeuxo z>)Y(KANFA&YJu-TOiwR-ciy>T;s~{QE(Ynxs;*n5ah}#gOQXqW-)Dt6=Az1!A3c zL+RM*W)ia5Hu9PoT_xSl@NzlYQv?Zl_Z!v#0`1HBLj>xrSD#@GJaPMP%MF_>d+ z?=KEgrQWG&c2mFe>@;>5>~)r6$Rx={>do}YlYWj(mao_3ewDV7g0-`ncss)ZoUJ<> zcj1RW)N+9Am{=XT=;so`Ks93ts;c%a=MP(>9tto&_Jhz+V zT$H)FTkVP|4dTGncI`m;|!H)YV-%3^mRa-8pu6Aeqh?Ik9Yl~rQB()Wkty_8xjPg_DGSzN2j+fw+5l z8VN+f=W6BJKM|D`y7BmoB%~{LR#q`a_JV&s7`t0bSE|=WnbaZ`>bBdWG3_$auTfK_ zM@O&O`3YyW<8JSJ*+^XH=5c4Rq~%DOPSQvFoCc>%1m)MV`~&qVe$Q=8qR}yRrE(2Z z+Turn`5PtywCF!tBdII3;YXPWJL(U@DhBC9EG|G1I}@|S&6dDxqNIdH$ft4l~R8D&Be^mSnZ$=9z2T_S2n zp15r3xs@3zXvS2HpQ_8wud(EGe}qvrGQYoz107bsX1B}B(8K^11P#IH zzlh6yDiPIOEzrti9Qd=P^AmGW6je)S-X0>0tJV1Tk+w@dTx!`MSX+_}#NS7pd`{QA zW`*`zDOJD7cG}+~#@WbnFQGN&-cU=yZ31Zq2iBRgIUgBdip3QV>GRgnhiF zlzT<;E4BM5VR%&iAx~z<`dR)3#$+WnvOSWPqb7^K%Mq#3#-cLG?3?oLFS)2pI`->n zDS97D%S>+f_xA}A6=O@=??KhIwMU1iI&}}o6B|#WVu~)i^4iPA<-n1xSf^w41AUbQ zqkeqTzb!X#D0>Zb7584%pV2rgi@A>bt2q>?Wg_{Z#eBh4n;IMD5;VkRBnJWw%KPj` zh+JL+!g1sj$6pRWR)>8_ZZ%VmwSr#y`#$wgWUgyujLzbV&_X~lFF~CA58flYKT6)P z+wji3Eh3&2DBaf?80S6LDKpAkC#t55;XJu}qV+yPw?^~mTM**>o8-g3P1$x-Cc~=f=l`6B#H|%N5B~uG%f8X@cW_FO@ z{od~>9FB%*WVNBO&cl`?raKHt6PllJnZgT20k7>I5Y$cOJj1w0y%YIw&L-LJL7>Qy zbUEq?sNqNuUutbOaMuxZ!uSN#!@8Y@HA`z2RVYJ zvx7@tMzFc^*_jw|m6B}q`b9ro60hpPH8hN&w*t@kBh5135YwA>Zr3>p@5>h^SF?5b zKK|J4ySX-hQZ9E$wJliXLwZpmJ{{!fy5~OKx|r zyCMkb3etBB&~pw=vMuII65Hqy0kl+fx|*5sI=y$*B+fn7Lz>@sqkRmg@b~Bd`jpRi z;VUGhoASl|Njp$apR&QcLNB^9_9jOM?`5tKE8|p%bh>?A+gp8RyPATK-6os*>LkG%? zq3L7#KHo4E-9Uu^dqZ5mt>)|tfjxqh9TnZ>D}I zot@oskf&@WCs`c&b%P4VKop!GAT4h5Y;3Hqa&uB&p6)^ePVGt`W|F_Xn3E&=ePkPM z1Kh|4khGlx@lKB|>yxq=kEj`uV9s3Yg#lB^q1zVAH3}KCp?UkLDd^1>lYaiZVb$U7 z4IKtE8*GOiW-{*2d%vM?JO1!oHReiR^X=p7$vL#z5gzrcyMfZ`D^Q0P4g|RG^Ksal zPxZiDdF!$|PXVSPoEgyae~d^2XeW$9Z$_j46Cm(^MGX9ZZ|L8CK&~kn@q4pIF*|P2 zXA%DpjQ}iIm6eqU7!=8^#PB&aFrM3Jf_bn4XgoYTgN~Lun%^1x zKTTQY5sB~L3Ljz6(9k&3dcD3p{|j#ke!M-$>w2>+p$U@t&6}4rG&m20;~*m=b7FV; zzP{`Z#}!GZ_)onBCj7`$(cB;g^706<(qds^LPJBFkJ|%)_?{2tu&}WBUZ1WyJZ~ug zBTM|AFg(e@>FM+R@2u046KNf^ACrr@4K^jUwTv`0R+qcO6%`d(zAr7yO~4zVia4~u zCkl$rOE66<*U3MloGAUic}aG*KJZnmg-UX*Z~$z(5pc#^2@DJ@FE0m**7x-E0I$F( z{QV)x5=(6Pg6e=u%=sb?yFmsjS!5H+q`^CV2%m(~@KiLI&7C=VYdf|>^M%p|^^rzo zt!N85y3qxTUz76+z+iB2Fg!?HTAJJEiJc?f^K`l4_Iv{v&cv{3k-xt`i`BezCN~E$ zu|ArOl;SUn$Z3l2nBAUr>g`LO*$78USwbUGaY|L*vV+wD+W`iVmx4#BGtts)2+CD8 zmr{?Ds)+&_lFvaGn&fn$N&`6W1@O&`e-cdRvPa@x%dp}XWD}rpa5hFrPEw%vDKOEv zYAR))F=HyTastLnP3&+b#CPw8)|(ov7J&4~fRfh3-QC2*1O^&9TIp<~D+?%yt*3_y z($&=kO-@eo`y6y_mP_?|`p9uIwL`q?*5=c_*s_#Ju1&q3d&<8yoY9_hGwZ|afB@zD z%XD&FZ$~%OtY5LV7MaTc zgR6@;v;*v_6Ds=z{#f5$(}^<&{zjkSW*2Mn0IXeC{&cLSrA5gmRaVm^o-$3D^*XWL zq=oAdFi+I3J8M;S&;9{jZFDMfN~FPLSELJmxh&)2DUvX0svtUL#i|Cc40y+(d4f}u z;*|*o%U{f&SGJVv%+=~apC8-9m6JEoh#H@Ev`wrlQk#{lL+VH|Br&`=c2hN?)!#i~ z?KXj33S04I)3bFQW+zjP zY~Wp$-aB~sAu#v>z!tu{O{fR)LI5QJ+{~dh(;4@Vjg~U3QKx-*r~0uaOxWicI}A99 z=v`ZR&DnLZh0>eRL9MG9UQilX?_>VG(jB6qt4REyrKqDgdZlF5Q9N-$hulI7{AQb} znzT~`1;OuKgS# zh>l0f@5p#Ra&F4>-XOv<3UEfG*7owEXG!>Po{Sh?h@Q9L^(WQ(e|HTeIiBvzi>3fN zXy%qeGo&I+U%Jtcb?Keah!t4Aq;gJDz6jnP(O6K`e7`4loPUWJL#f4wUz?qQy2OPp^5tNYK5+@m|w1FywyY)vxj?G|GX2vsscq z%2h6pn{EfH+yW%D%38{5v-4Birxz0ADeg;jk1{ z&T|Wbf~GN9D=Q>@cJc(wULwzB&wryFwlYZQvm$*+MkH5!KT3DXr&g1q*G|WOa30Wv z0zsQqi&_{%b{J0D6><4wD#IeF6(MWgr7zB{BgTrNphnC2Viii= zIS^#kR4_YN)A`V*-&OsS`;y7Gx?nGZfj-;@ z)@md$@d{f`{JXmFP+X_eH1+a2ZYMTflrI;1AC2_yRwFLwKbWv(p<5m?^Qd1qj4O}b zEi#;Bd&INKg9D@hLNLi73cS^Y5y*wCJUAp%@gP1pUWwJ0gsr&=gmfXy3! zRo2j0T3&8-0obqAf~c@?G+8HvzP;SQ;fHkZ6qom`H$YAx5lxRrt~`p0z8eID@$$+z z<)p7{C{wwjB9xdbl+co8$F@SjsCc^e{?kXTD;qITnXr24)m_E{JtYQ&B-CPV5={XihAVNyCZ6pBG-S++Szo$4`(d>@`Qm7QUIp6|Kh=*lpCaHZ>~>v*Jt7SyP2X9(m5MU?f!Qk=!hm~ z82)bG4Pv08y(6wN&Mc$O#KYolw|CYrD|T~ax7cPN)uU<@ z@-l@TWRIkkmG4F&Ye!f=Tl#@|fKCB5kB$nVrQSh&8}m8nD?PO0uutoGEqzpRlN9lf zXz+R8>0-@J04lGzxcFj?QPBQ*C(uB~#r2q=rfXm=Bq=T3?s0tt$WDBZCrnRc>_0>y zLWpfYO*-)r7al-gHxF; z6a@wS_oKjouM#1!z@$n=UjZr18Eig0G7^SgRdJfIx|?kKFapDDChw_+0WjDcU76vn zJBo2mrr;(aAZqku?leXaAlTvD^pX6K_PccZdHa5=MW)F8l%lK4rcMkN%Bi&m3v+ zADGg=c4NTsA&a6=Cn5hO7_a;jm6=65+-*8Z>i#34<}gU&v#m!jRRePYz!m@Szn!uG zVfJ4Ge$$WW!-o3$Nncu*`Em^kS|r@9NJ5@>pyT-pCTD@kAt2Yj`|X(ysJgDMuBnNV z(#6>sU?ZvZ+7s~MiPHOhAs8Su92|=ye_{+8KnF-mn=f~9dpl@FV)*Uv!BlA}^(E8U zr{Gy2LiS{_rnQyRG8Bj+x3)eB!JvG4em0xPwCSmhu>328lt&(CVP(bSj6NAWIX#`r zT2NXV+DfX(YO}n1ddj2yQ;-auN12pQzxzjPYipzJnx&CZoy81=nVFfV=OfUBK`Cp! zT#o<(pvvfUPgOKDBs}@;B%CZ1`8oHR?FIxER6-M}U8T zO`nX6>@XmX8On-T{QLLsix_O>vgzzo>+7;U6nrOZZAniv!@~%JUrBy^ynlR*)1j`Z zshRDwS!qnE6gC$=I5~j^H6-RcAI}?ns6h9Ax)|(wx&&O~dL!hY!b$JHf4#4w&W$kx zmSOhr93TTaIyzuh-V+9Wn&doIvL&r;?u7K9cf}U^2Q@FQ1 zJc;Noe?ej*_Lvy|kbzB>*Cj%yfPVDN+1lg9Adyu>B6Ak_S0(9>K)Cz%PdJ|}co~8Q z7`;Ax$OYCoi_7(FIF1rn8kwk2U}E1PB0fJqGbrQc14&BT+S&&D`x(TfY;0^&qcrR5 z>%TcW8`x0^_U-@#S@;DK*P)sPw?qS&#TC1*4x_=yp&@ZfN=l$_fY(D$k02f%p7rL( z_WAjF0Gcnb1fx^2jfuVk3*DgZlcl;t)N^7Xb}p{8R(Nw^ASUJp*sPd~$~s~Yi4X`! zre|hwT;GF|S*)-F<&9D*Do(|Syx;Kb`SDf>1L)-(VZ0F3&CBCC-f2p6;@gGIopy^T z`W%c&NN6j5`;Ji0NW<7TzcEd{os<;l6u=Q&U0h{J8R~-H@ zP-aS;-qX|b==WB1jdT`o#*^QFPHNGd4L{-bEX z_3^){?EfOWeRB&CePfTS!5bAGr1U*Wr`Oe8De*#&{I@SatMl>E(Gk-9g_D++mang` zmpkqobIFtd4vCxRZx>&>2yj>cgqrX6tN#v=(O*8&y}|n*?BjnobiHvZ_#B3#r~9kG z(ZK(VV}Oc?C@V9QP;es(KdR5Xq@dtuzoCNfcq+@tkDqU_C+ooT^Qv~m&}3gVss9gk z>owWx_)Zs(Y$-p6uzBD$y~WU@c{C=crQtEI1cxPRYHH3FDyzI*B{Hk^7I95YiT1^v zsaZhg%Fv#-E}EayL{SDZN>c8g+OjC8!7fqythXT+|V}A zZQa?vLBQEy!u;>|$luXF(2ZsCc)9l)J&>$`FaPij6g)Bv zc9@9IM+Y8P6iQiQ2tPPyvq<>V*5Tnb8TSxvJ)m)NUe}$w>cPcV z3D1#?QZx1)s{?!bqEbDZhGf__dwM$OOv2kxaGhbjuC3{P_*^_ol&{|41@&I+skxSo z`cNkVLb>A+ocsXFR7kIHl^sy4bSj3g!c?S3E zNwBi8G&D86DGNYA;^R*MCt+h_BO)R)`cTfe7og-bsLFxEr31{1cucb*#^kIK7+mk+3l8N6rqG@I zrYgZ$ZQ=AN{Ri+f6iR!Tu1kS6pYc#>zinY)eJaK4zJIx-mqz_C9XKg3hE`oUvVjI#RCZs8VbS4 zt{|qRP1w4km5X!d;RLXv&+AN9eQ#V!?I zP>H^r@|o~@%6V#9RJp64?YT4Z;5NwrHr>ixTn{`iZ8%P|1#HH4tLkkQ@YvvVH0W+}O9~PMJe6cM;1}m*fZ@2uSsv zMhknh2PnYojDUm^SZG`N`fD46qDWyQHP0LAsIdkZuq}X{5Wm8)@zw-tYH)-y7@x?q7G-dd_;zIkWf7vuF06UC)fn zyaBtG^+T2UZI=E{;{Z)9#^wg3>AfW}+&80rXtjj}`u9bUO1{}I2$?`^HpPG%RIV+q zbN_cJ4fUb8Fl6sb>0WyPpS6L^tXO{FVTQ#%8zKiZvEAO?^~$NLw#7nd$!Cngzp68s z_w#lmwRP0nG$lBW`yQ(lxhtLE$rIw4J{O8Z-aNT?Se-dB0ehuTT2T=k91K`QFR>7) zM|pY35c+rZ9CjNDFHm(#4<^Hp^|eTw4CzDJ6k)C#d!s&SyP2c;#do_?v(yr3R^ylz zpQ5Nm+=!XGQ2KzGC*`RfqDQ?r2xL^SUX{hH6cn1oX#-a+*9Xf=OGk<|`2cD&ln?>5 zqL}1Pn!M`k*6j@^4?W0SIra9~@z8I<)qM>CcDzXm4cd@1gpTr znfQ0t6d~`MHdQt@uxSze>mQtd%(aY z3oS+S!)Co10{iv;7oHUBUXxLm-SYQiQudKw&kfxVB!L+l8xOGk1_~lj@){Z%vY`nl zb+cE!Aa}YM@uIJD_a;+>=6>AQyCd#ToV;$hi+`bQP<(LSRF@$~-#xvH;eIL zHc$lOb0t3FCGxo^Th)FfK^C0EI71eP0u5ax^eN!d2K#N8qGA-#miyC1a%IGRdfsX! z=`sAf2U$a-qs3}f^gKL75K~jrB8@sWmdsndgMbPzEmJIlqF0wN3* zoZH*mV0WB#l*h7Ew+63oMDlMsPEJnQemPgytAR|3-I)e(etXS_!FF?IP~fdAW~9NWmqe{NfPd}1CdnnK_~X&a1ALH-D@88 z4nzj*-UFbdZEP=$KzpT1dpeT zMrD@s0@nIwre2cblIq*HZ%&_R;t~?j z>OF3WL6Nz-9SQbr0|g8i6bc12UYe|>gvKv{va;gaS_>!jkz^VyLxnaP3><|{Rx0hS z(){azz%!wtCh+?GQ_w_wQ|`jn7LUV@dPF1=oNhYfKqCiK1yiZ(G^(lT23QupH9YWU z2^Qa~mkTx0@Vt5ze$7n{+Fb2TdNI4rf^j$NoTyirZ(rX+K1Rnh?7;#~c`>fVQ8c)T z^j7D>TQS@qW%`WMc2-&Rv*8ZdgllNoQmI*aA$TlFuyv1q_j7YchY?$FL_|bG!xd-{ ztP#MgU@VFZ!VN&tFwa5ZViN^=V_?rJ4hnz7T9Q>hJ}Q_aDG10NUbPJ2q8eFw>ogcqzQ#DK+Xh628kuKnTo!#Rv+j_P z1fUm4rvob>SP#Lcv5`?ftAi^mUx&q_z|$MjtC0pNE0JfEVPWzh&=VkbKO-xj0kmxyhOoYd6i&<)y^1JRQkfNK z+viSy7|p;8jLU#ptt)1(L6!EJ%rvhk3T!S1cd<9t(IG%`CYu)Owz|sSQQMELFtUa9 z2rmTRf9)=&g!yK&Hg8foU*GjHXM;H(s}~5-)R2gbuVvQLrL(cSD*Iz%Ex*iWtiY2M zqY@{@Ch}n>K&Mnq8X5?r?fC*CLW@fe0v6hZ@Mtn!8H67ML*>2|g@>6wZDlW)NB%Gz zHt_nc$K?_Qf?2kbk!)h8o?gTo?k;>kEyr}my3-K+@9L<4m@>b(#WVW;DdlBqs z1_DDXHCS?`9uxYP*tl2iBj}={*rG#nhxu+$wLpj<&Tt{gK~uyjO%5-GjqtGmJZ6)K z>R*>bV{5b^D~bt^_SAP{!_+w9Vuop%MNK9J>4Dks^_1r64XbeC*Dpd$=t_nbWy?+a zqJ}y0SGhtD_HFKY)NXWx5Le;+s0UtU5u~0yua7Uu)Mpkrb7{8d>m4HO=Xpug_AURbyI`sxk6B6Y2O-wt-<;TWE#xIc4HHjc#oz z=1shc^dv_sjtHRaK-y6F9Lu)D*8};^Wm8f0X0goIy94$2@EnbgXI?fYRy=al?6`Oa z0UF?=LuM}b_+=Q- z@re7w!78xp_PcHmUMzuMf!9Xj-Y4b{ZZ-(blMK1#8PQYcTe)R(H&qVyU(IxtllhWo z5T>3wFieQ`3{;Y2YHqC@8{5W1+}S&Y)}vRl8T z!FBoCHgihMAIo4AIy6LZrVW$UzmwQVHijD*huRvx<~OpMc_HQ|tEQwQ@P50f!ML(> z!MdI&2gjgDncXBq^#+-Jxe+PwnO(_-aqE}~5o8aV5qlEIpPl>VBU=XJ6@RbO_Ln>H z&tQe3D3}>&p^_ep;t9w}CtPjC3t#Z_haxC>Skps(1Tb1Wu3UN;x-%M`(E6UP!n?GQ zl;V!D?D<=1S?WOE%H_=EYEAjX>(f$=WkwC&^UN#N0i1Bj-%&ak zKb(ToMLGwEHzTQbKS0e0eFIzH)2tfMy-Mg3LuWQ?jeq|_?A0S7gV$*JT?P#*+;8 zjuf0}a|zT)>Nw|!6|g>roMw?^e$t4(aCp0zT7k^;x|!f^ZnZ3hG)I{zdKD`&Wlww3 z?}@t@il~({hw6iDQz2{B><_l^GM@*O)0Re0n>Dmq7KHl6?j;gY2;&`Q*Dc|p^G+zV zD-UO2)WB%ILvz)V71<<&KKg$1>n3gta+*F(Xsh#!)GwP+R6WTVMLecwFB=I+rjLo~ za85t|&O4tQhkZj##j_({=)F~oF~fTCMK}Ou|7mhcEY>-3$dD1`&2o?vuUJM=PIPhC zL824yI##aA45{%NBlmViF#8Q{H>FV=E!{|;`Pc*r1j6z$ypKYRY^0p?(v(x#^^30N z95KdrV25>3fRVkVO4)=BeZ1_{OH8BGPD0Y86gdkuq{#!{G49y$t{_7q&RIDt5-F;y z=Fg3K8OFs`jbx*^b1}ynRAdi7_?+K4uBwkcLOfkQ^~8~bG4ein{H;LjWdeCC75Vh+ z?4fj2Px#ksMcN!A=BsZMC)J`xHa41UoLG+>3%CoCUS@6oREqnhu<==`zVR~_^fcdo zySvh=y8>b6@a?dKkxe6M<|m2V&tt>*0kc-^{2v5|C|Sh$6-+AC-;EeBN3{faS@=J+ zOl-ssimac5y-I%bYYasZy>w8$3T;2&o7A;h~G>u$IZVW7(>0jzz9IJ)ve< z*^L5to6{|PaioAAC5x{es_XNasy4GYmCV}Ok3?JBn!e|J{2?i+n=FO4c7E;}q$^MQ zaps76Qrv3^(brY&5oIy$$`L}<7SyaOqfaF`l9a}{dB9lbMB96=9w)@Lz2a{EG3>rw z#2U5kw{n)FOo95oW%PqR$A(6aH}8DVlwRZ`^cYyrR&Bcf!{CDg>Ka**07e|Do`N{HVyGwL@6@7{EcJU#Y*CWJbbzm{(cYjMnoBAQioveiaC4gMFl@o zjIV~{rQ+^DJX|w&jRP*-*rdK^;>ahSh(ELQHi!_+t*9xnaOsh;pHQia8c`mv9y0!s zXA~P2pQOL@%PhuNc>## z<%_x^QesG>mFmpzr{LJ@Igi{2{Rlep3fl|Q*IM*uQitp!aeJd!a_;{cTjh^0744;_>N@0);s1PjS`vL*V*|nD;s5eOzdumG!P)L0 zpkr*F%yH`Qj(x61nBnPOoBNPu8{ zK5@kU_1UM|r9?JIxfNN$Q;SBpv(>bryEn>@xsiqOOgOScjVf#TodCfwPgpRUBK4sm z&2!{E1U3DVVZ3ECCkSQs#(8h!^)~yyMY=@VF|`Dd8fU6xux#?!$sWiG{t(nIWEsqU zJmtGXMlCRuN=)6NLWtV&h1a!TeVP~s`Y5;W)PPIp*f3+^ela;(tZ6Zzb?1wq%)z*v zC*{{)MZ(Rix~kE7A6OL$UQccC$LqbwSbqBsbzp=arTj$(OTuxHaZT<34K@k4#tpxH zxM%&Kh4||{T-hRrmf4^M(1mC^M+1(Z&rn$M#A7AMUhM4<31@IG%i8$;lC&h}!{22h z7$8ebuM_wLlL%37bg4=GMi%t-vk?a6<%YU=hsd}pKb}sHd%giID^Mk=0Bf*kZ1KmX zG+{4znnZExme|)O1r_IXe2O((ndS7MX8P-;Ph=#^?pl2gN2kk()K3?9-4lOP`{PAV zX@7l2AI}t434SVt3zJ5aUZ8R_tH6s39f5H-c~n)6t&y{@nWY6j`aN0XK?xK&4`kWs zmePHfPHLRZ$cpH`y;M|uO4W0|*cC}sv?wZrOo}K{bl)gbOCe1UHl^ZrS6Z1Nij6M# zXR2<1(>NDSI+d9x?ialCmMPl1h5AOy z$5dr!s-^l2-o}Y5Q_!UrBe_PsO5LkmS~@PS2iWr?;+8-9apd;fbU86Mtk1=${2~ct z1KSvM^-yEJV?cMhg*(Pv&@TY!t08 zRxTOmHDhVf9>0U8SO?|VAK4I&5fd0hbx&-(k3#yr$1%_1vdB~z!uxS93@mA+)E0hD znumxO6Hv^qoE)Vb?HUTwz(&ibDv7mut&U@kB^7g9NEI*9JHm6?BpO1GkF`V3mKoe@ zrja$>&X&>KPg@z9E=$1{7dXzl4Zml=hYGDen%43+r0LxUeZ!lpE!@-7yR>FTH*~Hg zN9t&USKsc^H6x)_R)G>dv%XWQPAILoukHu}jLuR6#X1sw!+T70E5IU^VA>vEZArxO zXBnO5{jKGEQL`69F*!1zF4+{lV(c^S)`%r9|ABFpwBuxCfP2~o?MSi@zXhbZO4#Ii zM+e33^PGOEn{+bcQPc>~_a0Dp7_NxUk?o~Nhmp;~oU+wWO}j!W#;Q^n+G=}VFy_X<+Od5x1IOM1JvGg}o);9lD5b7gmgpDaSEk7y~d4(cU%Y*b;6Pv@D8cCvIlBJ7%LRvUDZOSvDdg z>DSE_)lXf1O`DksN1JS;ul5=i-ZA-Et_$t+6z2IvUGhLHv@)*0PfyFiaRjP=QTz$L z>`DQo}u*PP-Q&ZK4hVjM-c>ao5kQ)amQI zNl7WN2E%tya@LtN8q!o`{34IB7d$ua1edBVAh`++w0JX>>E0nyo-j2!_8m+i-v~64 zxSRaOvYl$JJCZuqA}m|Pykiah1o1S_q_tI+L4Z63NKmEAC{ z;wJ&dZ0_@NF_0xd-<$*0tCitKDka}uLK1&F#0_=?wpJFa9zv~Part;JiVeF**-DTh zEzEvi{Q{=Pgb-5pZZmTuFgRgZU)!0u+QnWw5j2lRCVg9a)Q{Kk%irY}w%0JjCP~h;17Y~ec9)G1&B>a? z{5R6&_kYf}gmg8VGmzuP*>zAZ^43gGPt}CJY*ER`zV9E=U~a6GymeKUSL(Ym*E!~5JHG#s?|MHpN<338_wuBw zi9C&c)@5*U-;pOdf>L@XDrgw!4GY^Cd=%QE#mrmWZ$>g6RG92T_e zevT@wY~dv%7azQ~zRxN5DO4aX68e>$5qB>DMNf`R_+Un+)o&kx&73C*&+MfqN&w?Z zC`0}zu+yC1xC+g%(xO|hHdLm>?WxoEl+vO%t+BWygxbsuu9)q9ps&jfqjRy2g%aF6 z%EUTgohY%OnU~=x%Mpu=(5WPu9KeWPbn8mLv`gDyYGhyGmT6YwC9-*BFYMQLafylg8GG|w!3k^VlFl%>AAT+9j11F0~d)$dAllM+wwf3Rx&;$4g5OGP>Q6#r`lvV-xDh{96QncQgYlELh#0G6)O!`uos z>2eOcAM|O-^s8f8FA8d2@P5_=$}~7vm|wvrP(rbt>j_JL*dqc|-1|L`w{~7Bdy6qu zvpH~t7AknJHH%v+_8KmFkOrM8SF@`2<}ALAl)q?~`dWUYZ)jBlSb>sUK&U}u#qX!> zv7b-j1e2w5nBMrgg^r0BRvQRw!gfW&+1hHXmyv&NmsU;-0*hs)j-o5b$<@7haPap> zr=iU|=;&v6{6$moz9hei#oG|W2|!4jQ!LEdHp>ZSRRtDV%pXQlO__Q*2y{~n)Ea&e z%sAX=&oO4D>UrxkELpvRFR$URCxP_R!mq&#Kd%B>6M0yk`&g<$VfbYm3pd4(Jf*+7 zi2)?ghsnBnO|Ws?PqS4O0EZk1`qxj1B0W=iczfyaEnH!>lydf%|@VZ!8QMUCkVX9_f2 z6lZg~FFI*!_dNDE(=cjDJv1Tj-o160y(MK?@05ec)T6I6#}`a zImK2VojPM9Er)1Tbo0V{7P?%gmgU^oBu;@@x`BI9EuF9UxUy%oT1L-KUHBhv-}LV) z-B*hdx0NLwc@Kpf$08>zF9=YT7J!pkLx)-w9PPNXJz3?TmeC+EPK%F;vhG&OzZSh` zdrWWA`-Yq^m(z|K7Tae|KOS8E1bDs5*Zgc|u9SyJe%5T^TTXjYXvm-SrfRj3MR zcR3m>wa&e!Y)8hn&AG{SkH1ou{ZKlZC*xa_r}N6Nx>rD|n|=1CkPFD*Hj9eAqJsP6 z>U|QDL%4IIza%#vSw&2QFTVNDOsk(gmo z4E0bamC0n%Zc0viOQuMse5cRZz)Bj-3U||L0s75?L;2y;XrCY)rVw<6d;AuZz%fH~ zz|}iQI6r`Yap%(7g`Y3G^h;e}?Dg}SXD!B+B*STFxNYTELc*AL>)cAGh4sn#YFeNb z9J?D&6u65gFb?i{3jP>D4Bw&D;5FV}m{CN95tgudj^nKOeoSroUAC*l z#YYjv#5_EfJO<9q1c^(UsrVhVIWLwDAg14nGfzM%B2}5iExzzf>!R^C<0 z4HbMy_^I%|l4ri%Py$0`JT>ZOsIed$W@9gRihn+qQnU#zkLGid4e$b4yd-S4N;f(- zoy;7(B}W=yTF$3^4h&UHW0r{J*G zY*IFISRu7cveINBL_c!Zj6irFC7NO1^M13RY>nIJ_%9lJQ>ILhSB5WQFy@sUI zHm$lYOY@}uiN`1iI%r`Pi)9sc74bQDOj4|g=*ty?`irAJPK&JsFmjRvV6MEe&guOg z)zy`zIsyC0JYPe!u4r*oG5x(JUn%$#9cpE7)LMa1o-eEY%%pv*d6W(ab6H-_1B(@& zqaE>8z1G%9_xm`m5-rZG(`Mlcq7yLuE_R#TK|-_5YA@$~M_>+vbg-j{l*=VxDEEk< za76|@v19UuK(uy&Z6_8#K_`K`#X4Z>)ryP?k{Oa0uPC5ctH8OHpWl-pE_4Lmr%B;; zw%5}7awakbmbrWAu1EbMJ@+Pc3hIRX33ga`%EMKDYha+GRx}VEd2qhMN6@lZOJv&q zlKUV6GlRGVk|fPU9JgX)_RL`nmB>W`=*7>>XX=d_4jXn|cMq1kSid!%E(C((n7=!% zuJj}nH=MwrB}%RmLPtv-nE$fMTY3)fJ;vVy{`?bvYMuQA6RM{;)B?86V_3sK;*w$Z zvsGZ;-QWuN4;re(UgSrpEttLS!GZ{o=8&&V`gd#@H&8^l?tjvl z=}YD{VJND6b`NeGuc;6fkk?t{icPpZfq(y^V^tD;y}&SWNhCgr226Sg#FqB)Nl$`B z*avs zLrai-DAv<;XG#xm|A7#_Bd8}a($&>w}s zpFdCI<2m#wyh-hfrela)Vau- z5*RgL>J7V$PY2$UEPsp|B4N_Vh@Y%;s$$=KBjS|Z7y-C zejghFL~1_Rr~!Z!B}8 zdG8c%QjwD0uC>op3G=B;*1w!)-xM?uftL*mR~j8pSkYZqyZ7%#tiQoKa6*H@2m6z6 z>d=IGETAArrOPu@3N%xKT^@78Ww1g2G9sW&s z>3W(ydXGd4UKdy5g4$b~AKhJsD%kZ8xy?k5P_1*p{MD13dqGD|BRvHG~Eyv{D%J#b=XKqPwQG&QPmk~QF%-0=ycYcmV65?B_#|L z#aY#n21%bC{P~RZ<$vPw@r26|?p~l$yXaeB6j9zV#jPg`N3AncXDsDHyHcjn= zx}B{~ir;Ta6=+|_KmO3w=xgkW^pSC`4QsXb;EJ;lB~0SJhQ;NwJN_lxje~p7&9vqX zi|skz^){iVsq3`8=~Q>5KF*N_6qOuBv4WmXQf9+r zx|*lh?}*Fil>v7l&FEWH$I*D6t6N6>dM7jYKjlA^yC_ zQB){+OSU)+Hmh9eKU%=zuK--UQsa3${zJyMh&n*1piILse}R^oet z#ooCD^hK(LawBiQoY)Zov;-8osSiKt>J+KRoRdu^d?(S9Oz7au1m~BIOQi$cCJDeG z;}xjQf5eDwhpy(8fOm%tW>W& z`%s6z+>bDerZXHpAK8AW&{+V?0aQ&)%j4Om`RnvilY&xg;G;c6veM?SFo?R;P4xUo zo!^2h>bDYZX2p*xGxNl5Qi;mXSP|H;61^rQQ7)A;f!*0<@hIi7Awy19E+4(|n$lIr z29g||>EhSX2EOYCrw`YLPI8VWzuA)|ac$D2MFgt>2Z3l=?O_&BA8WL3=m{|;m5UqC z6w%E&sTnkzr`&W~E6K7=qeW+<>8D2{!S)JEkBDLDrSWVAcZgz$D)lW>ltj7qQSV$% z-t6$Xtj=QeyWcEm4si#%_vPV!{IRR14fnstLNR1XV6t#eZg@@f5_Piv3s$;~>->cpP@ z7?sm;izX*sz_q!w{YK*J>(~PK`+w<#Mz#-cCEIkcV$O+MF;)9sO@qZ{Nc64B@QUH0 zo&@xl1FNtbA0g#}Tp*SCyM}z4S-9{V%bR0isvJ0{Rx1?H6_C$jlU!>g&$lNOU)HZK zz~Lx+E6Z!VsORn6`xTer-ImdvHtqewsu|kR;)*kpb{3!oa{lR}6#y+EA`M-|06rEoKn&fXqEuG^Q&oh8Mz{Jm3DspP_u$ zr;F?}j*Fd9G(uzn3!k33oi0ARIR#)BB(!jZ&5etubiA(TM~k6+l`}w@c!chHzMd^~ zv{Sd6x6(geW?(Vf*wEP6Yc(|!hzy6YgmCO0T!XY)v8phIcdo~Zq)ulC%enH|kEFwE z@BbcF*4(a%g;#_i2`w^P;nJ&f*sKH@V50!S;w(wGEBhoh z>42$)&{~qHs|QKkH0tzam}G^kvzL2yNm0X|^(%v37n^n`8(S~O)JU^E!if&cisXKP02|;D!^qcCVnqsr1j3gj31E!djQ%pU4@mH%TW8 z<~c~6gTNX3zbfil=-8R*ncLDknwj3xw;{UAxRvpV@Nb`Mul~m9<$8%kQ*n~X1vd}v zP-1zBSK)FSO}le!41Q6;zkV>jaIwSNi7KDD_G3H0Z$?1Q+_K;;^c+Qzt+cn@;QG3h z?DJRk=z2wrR|8%olRbxn=XYV|`;<0@T$rf=Ecl!%lMQml0e2smYY%_?9wlz{jV`)2 zi)G4{^UZ#6rP)Ps~qKThagL+Bq8Z$wb8_atZ? zqWpPg7w5mLa0Nrr1++(S2nj?@(8Amn&|g#GgQcF027DMQBD>^T81a05_ntmZGEHKe zYq26iMq!Z9`-t`=yop@LCXP(l(fJ0e(~zG!EuojCgLAdZD864eGf($nuSB}dtM2<` z2tC1JwcQ zv3;eJz9ui6Z&i08NXlb{-*p;7mgssOP4*@OhwdBVL++LI@hJ`t-umsAdD2n^u8bQ; z-MzjabPikh92SMPoCk+%G(^}7;=F#J3g%Exa?()PIV(IKp0UJobTcr0ca`>A3d#97 z_vX6?8u^?Y-}x>CxA?aToJn}RJsw=|;=@+myi1Da_#gzcGp8QPFMh3X^3?hK$O_Kr zQRKOX$VedE!Giop68&|&z>q-@AkV=kjTzEEjou#!{+UVP;_iT6Mm?nf6(0j|I7h(C zAC12Smh`{-tYc?mYhm^u1piE7z*YY$@aISZ3*W7*`@hi%E;;`wa1aL*r2iI=e|^@! zN>VuBAD#X3XTZP7)xTmwJNS_+1{U(D;J;FB0*L6_UjH|azk+v5+Ai<``m6&#$!*TX zzdMq{`Y&*OAXyaNYn%QT)&H4jf2IH~=fB|1^uX~Df14{j82BPzzC$3<;JVdA{@t)P z*opkM-+uranY}mAV}hTS{MWw!^^E2J5V7m@{@+pDp1Ay193kZI6sjK}KytzL=Wb*{ z?)?qNtAFZ7N4Lg4v;_hY1lONYe13@dHx$hOgkq<_|5RfI0U`#jKcircMfe*EmVZK_ zu8i<(Hx2?>2G^fay!rD$+|rZvpHLW9#Lyvf0V;y)&nVEl|NbCs|AeAYF|DlVBLtEG zu0Nw_bG!Su2Vwsw6feoLs1ySr5D9So8HL#WzmF=%KcR55?6CQ10D-iC>(40q!~Tu} z1Qh;0x8@76RVe^pjDVb?Kl6p$tA9;Tb2~F_J!>O#gZ~JV|Hylb$hwz5f!NRnOz~%7 z76!iFPH!7MTU!8a^H1RFD8t{<0ZRrtLm`hug# zZ`-yt)HBoj8&CEZ?S4N6qK^#!uXHQ^GvF;a(;pp2yj_imfI_$i?#9jGHsAjNwNk)! literal 0 HcmV?d00001 diff --git "a/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" "b/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" new file mode 100644 index 0000000..ebb9bf0 --- /dev/null +++ "b/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" @@ -0,0 +1,163 @@ + + + + + + +# 制作可ssh登录镜像ascend-mindspore-arm的方法 + + +本文均以Dockerfile实现镜像说明。 +## 1.获取基础镜像 +### 1.1 华为开源arm镜像 + +华为开源镜像库网址：https://ascendhub.huawei.com/#/index +以下以基础镜像ascend-mindspore-arm为例，集成mindspore r1.5来说明： + +- 基础镜像ascend-mindspore-arm网址：https://ascendhub.huawei.com/#/detail/ascend-mindspore-arm。 ubuntu18.04系统 +- 登录基础镜像网址，点击“获取镜像” +- 在随后出现的Uniportal帐号登录界面，选择账号/邮箱登录、手机号码登录、短信登录三种方式之一，登入。如果网页出现“禁止”字样，请更换登录方式。建议“短信登录”方法登录。 +- 版本界面，选择版本“21.0.1.spc001”， 点击下载列表对应的“立即下载”，进入下载界面 +- 下载界面会显示下载步骤，请按步骤执行。 + +​示例： + +​获取登录访问权限并复制到工作节点执行： + +``` +docker login -u WX926930 -p 4u9xchG5IzMuGgVFxvvMVH895SwE0tIXAQrBwl0C46uHzhMwYEq5eWV0EvYbG7CdO ascendhub.huawei.com +``` + +​下载镜像： + +``` +docker pull ascendhub.huawei.com/public-ascendhub/ascend-mindspore-arm:21.0.1.spc001 +``` + +- 在工作节点查询镜像： + +``` +(base) root@node62:/home/lhb/code/ascend-mindspore-arm_ssh# docker images |grep ascend-mindspore-arm +ascendhub.huawei.com/public-ascendhub/ascend-mindspore-arm 21.0.1.spc001 67bcd3733d57 5 weeks ago 6.67GB +(base) root@node62:/home/lhb/code/ascend-mindspore-arm_ssh# +``` +### 1.2 自定义欧拉镜像 +以下以EulerOS 2.0(SP8)平台ascend-mindspore-arm-base:1.0基础镜像为例说明： ++ 镜像功能内部集成训练通用的第三方库（系统包、pip3、openssh_server）, 未集成MindSpore框架、ascend toolkit工具包 ++ 以下仅以实现ssh登录说明,请自行部署业务驱动和工具包 +## 2.目标镜像制作 +工作目录ascend-mindspore-arm_ssh + +### 2.1 镜像相关文件准备 +#### 2.1.1 华为开源arm镜像 + +目录文件如下： + +- ​Ascend-cann-toolkit_5.0.3_linux-aarch64.run 请自行下载 +- ​ Dockerfile。内容如2.2.1 小节所示 +- ​ 容器启动run_container.sh脚本 + +​内容： + +``` +docker run -it --ipc=host --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 --device=/dev/davinci_manager --device=/dev/devmm_svm --device=/dev/hisi_hdc -v /usr/local/Ascend/driver:/usr/local/Ascend/driver -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ -v /var/log/npu/:/usr/slog -v /home/:/home -p 8000:22 ascend-mindspore-arm:ms1.5 bash -c "/etc/init.d/ssh start && /bin/bash" +``` + +说明：-p 8000:22 表示外部端口8000映射容器22端口，提供外部ssh访问能力 +#### 2.1.2 自定义欧拉镜像 +目录文件如下： +- Dockerfile 内容如2.2.2小节所示。 +- sshpass-1.06.tar.gz 点[这里](https://nchc.dl.sourceforge.net/project/sshpass/sshpass/1.06/sshpass-1.06.tar.gz)下载 +- run_container.sh容器拉起脚本 +内容： +``` +docker run -it --ipc=host --device=/dev/davinci0 --device=/dev/davinci1 --device=/dev/davinci2 --device=/dev/davinci3 --device=/dev/davinci4 --device=/dev/davinci5 --device=/dev/davinci6 --device=/dev/davinci7 --device=/dev/davinci_manager --device=/dev/devmm_svm --device=/dev/hisi_hdc -v /usr/local/Ascend/driver:/usr/local/Ascend/driver -v /usr/local/Ascend/add-ons/:/usr/local/Ascend/add-ons/ -v /var/log/npu/:/usr/slog -v /home/:/home -p 8000:22 ascend-mindspore-arm-base:ms1.5 bash -c "/usr/sbin/sshd -D && /bin/bash" +``` + +### 2.2 创建Dockerfile +#### 2.2.1 华为开源arm镜像 +工作目录创建名字为Dockerfile文件，内容如下： + +```bash +FROM ascendhub.huawei.com/public-ascendhub/ascend-mindspore-arm:21.0.1.spc001 +MAINTAINER liangchaoming +RUN apt-get update \ + && /usr/bin/python3.7 -m pip install --upgrade pip \ + && apt-get install libnuma-dev openssh-server apt-utils sshpass -y \ + && /usr/local/Ascend/nnae/latest/script/uninstall.sh +ADD Ascend-cann-toolkit_5.0.3_linux-aarch64.run /opt/packet/Ascend-cann-toolkit_5.0.3_linux-aarch64.run +RUN /opt/packet/Ascend-cann-toolkit_5.0.3_linux-aarch64.run --full \ + && pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/1.5.0/MindSpore/ascend/aarch64/mindspore_ascend-1.5.0-cp37-cp37m-linux_aarch64.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple \ + && echo 'root:root'|chpasswd \ + && mkdir -p /var/run/sshd \ + && sed -i 's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] +``` +#### 2.2.2 自定义欧拉镜像 + +```bash +FROM ascend-mindspore-base:1.0 +MAINTAINER liangchaoming +COPY sshpass-1.0.6.tar.gz /opt/packet/sshpass-1.0.6.tar.gz +RUN cd /opt/packet/ \ + && tar -xvzf sshpass-1.0.6.tar.gz \ + && cd sshpass-1.0.6 \ + && ./configure --prefix=/usr/local/ \ + && make && make install +RUN echo 'root:root'|chpasswd \ + && mkdir -p /var/run/sshd \ + && sed -i 's/.*PermitRootLogin.*/PermitRootLogin yes/g' /etc/ssh/sshd_config +EXPOSE 22 +``` + +### 2.3 编译镜像 + +- 创建指令： + +``` +docker build -t ascend-mindspore-arm-base:ms1.5 . +``` + +注意：指令末尾的"."，表示使用当前目录的Dockfile。tag--ms1.5, 请自己决定，这里仅仅示意 + +- 查询当前节点镜像列表： + +``` +(base) root@node64:/home/lhb/test2# docker images |grep ascend-mindspore-arm +ascend-mindspore-arm-base ms1.5 2454f44b88ee 5 hours ago 12.1GB +``` + +- 执行./run_container.sh创建工作容器，并查询容器状态 + +``` +(base) root@node64:/home/lhb/test2# docker ps +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +0f6f9971a646 ascend-mindspore-arm-base:ms1.5 "bash -c '/etc/init.…" 3 hours ago Up 3 hours 0.0.0.0:8000->22/tcp compassionate_cerf +``` +## 3. 拉起容器 +执行命令：bash run_container.sh +## 4. ssh登录验证 + +在其它节点执行 ssh {user}@{IP} -p 8000 + +示例：ssh {user}@{IP} -p 8000 {password} + +执行结果： + +``` +[root@node66 ~]# ssh {user}@{IP} -p 8000 +{usr}@{IP}'s password: +Welcome to Ubuntu 18.04.5 LTS (GNU/Linux 4.15.0-29-generic aarch64) + + * Documentation: https://help.ubuntu.com + * Management: https://landscape.canonical.com + * Support: https://ubuntu.com/advantage +This system has been minimized by removing packages and content that are +not required on a system that users do not log into. + +To restore this content, you can run the 'unminimize' command. +Last login: Thu Nov 18 09:10:30 2021 from {IP} +root@0f6f9971a646:~# +``` + diff --git "a/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" "b/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" new file mode 100644 index 0000000..edc35f4 --- /dev/null +++ "b/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" @@ -0,0 +1,10 @@ +# 安全声明 + +## 通信矩阵 + +ais-bench_workload通信矩阵 + +| 序号 | 功能 | 源设备 | 源IP | 源端口 | 目的设备 | 目的IP | 目的端口
（侦听） | 协议 | 端口说明 | 端口配置 | 侦听端口是否可更改 | 认证方式 | 加密方式 | 所属平面 | 版本 | 特殊场景 | 备注 | +|:----|:-----------|:------------------|:---------------------|:------|:-------------------|:---------------------|:--------------|:-----------|:-------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------|:-----|:-----|:-------|:-----------------------|:-----|:---| +| 1 | ais-bench_workload集群构建的多节点之间进行文件传输 | 集群中各个节点 | 集群中服务器的ip | 由用户配置（默认使用22） | 集群中各个节点 | 集群中服务器的ip | 由用户配置（默认使用22） | ssh | 传输节点间的文件 | 不涉及 | 不涉及 | ssh协议 | ssh协议 | 业务面 | 所有版本 | 无 | | + diff --git a/ais-bench_workload/img/faq1.png b/ais-bench_workload/img/faq1.png new file mode 100644 index 0000000000000000000000000000000000000000..e473985c0bdef03c836e34e27fdade56728223d6 GIT binary patch literal 30108 zcmb5W2~?6_pzd9ND=SkoD>DbmvC(i+Wr%js z#aqi2x0a0+8d8ro-?iPU<9;(x;&=4YW3>yrTcua^isW;nk)4GXHp$5yvb;P@3N(rZ z3fEvevv46saB=6Qu}<`SToaw(jOl84PgKt<$ae{`F-@T{B8;DULe?&0Fo^oM&^YYR z^@aTh;9Z}i5@a&-l^llJ5mWa>%9JG#J0+J|n z#QHkp40D~e#!^`JvudQ~zM&1N!wQgflAC6eOf@gun=>@|mH@Ei-UzY_&F59Fd=$sY4QBmUnVu_ReCJZQ4SpT>o&Vr!xoxVzacoljPZPIfTZC9NuLbc?R=p_tEMQT2YJ(-XHZh8?r)()002nZaHQegC;Lb4 zOY>*ujVcY-n{$Kiz5=MF0@}FCL$)8CbU#E+nOwn2$_gD9-Dc|d6<=X2bVA9tT@By) zuV)bMYIOF<1KjpsWO@J`sY98z2H4OA{XA-Iw4 z`J?}M%Hxsj@VSLH6odo=J@a(YJwWL>h|+~m9uv&`J*i64?+WU-pG@Pt{*pj#{w)D& zq97eG+urY4@6u1=nC1T(&^TYc9MU!9T%+goyUi8b)!ridoqe?2_M*K-lwV>UrBm8( zuOwM7!Bq4Xs+Z%ah#trtP}|mRJ0)9^Sfg)9zUIc>TK*cmw<#LtlC3=Id1+Hy-tdu! zitTKg>JR{G5iV6CQSXhF7-jH_2)qP93C5?yF4 zgeF!RgBamA*DE$iX^jWIj{kB4Es@OSLx0wu&f*fhI|?~MerhQdAg?yMLO11^}k z+o?F7yETX$8p-esbuVAe*czVPjSKk}(LMAIk-9?myDqS5a4@}Ypef8W30)j)lh((k zvp*qcjmpv;Z`{pm@6%q_hJ7rM2Q(sXaT6BEeBiQ2!^RKg^2g%f_I5+i z_xYQ!^0H1(7Low>4Qq&sYZJCEtRoJ`W^vY-{m>yUG~A~bHSc4FMZ_W*&~W6m@adW% z2SDKNn}h>eTeQR{5a23y0{^>}S_7(#`2*_FFgJ-D?B*fryL6asR-Z=qT6Wzjhr%G_M%!-sN>Ki;|?c-n!Lnkm~! zU60tIdHa=*c3aQ>3%ywnd)_!HXFSi(s*_8s1RT_Wxc}%mG)r}`dJ>vaqE{v1&+;~Q zF}(rm$#lHAI@F$c9I)~}ctvyn_dC)%!mofDr2MC95{knu-ZaiX4e#iwekx06VS}s9Qo8bdPECgIEUWyD=&5+#{K~kNBC}M?p>j36S2b z8C>g`WPsr^M1eA}qzz(*LU%aH0C3oDguWn;(@S^eY6hcfpe!FYN*)mIA&<6GoLSX9 zUtFYyV%3qqg-fySO`+~c=w35AmL9?B2!1D*2(G@d0`AQu^xSpj+%iq`0D#K{X=DRt z-?@U_s)5P3Q3;Sv;vl-Es~d8p%M2vuHrUqMp3G&-+LPK7pU98C=}30m#y+k90A+!c^Uj3g+@kxxwG+QE&Rr);0`cTt1|rMq86)2 zFxlR>k9oy?v$1K)MbO4%z3!!SDLL7qysIHy7f`nh>EY8i#NQYF;>fWK=qD#yUOFgL zQB=#U7$zq_0_m#q?-e=uD7T;O($)nv384||6Raz75z%3xV?z$ZI7N;ZE3&_+Uqw(L zY-BBJhEAo@K`*C=uX^%wc~1*y*o=ic{CnNB;3)(iD+RKdzheH2cQ)?_PY0{C5#p?G zklk1geN@=LjdK+lo_}>x6X2nytRTd_Ck|Vfz6R|&;s^i0rnIYkM-Kb*oYPUOk5B4? zt~BTX=!dKb90$IuEPxCnh4{@KeU`ZjIeQkzMBd+W^!U=!+%UsDmycjgcJdN7i&jsWU5J zf=Z2Sp9!<&p@!9a>@$Gck{zMo@;idl_--bX5k@Jh1v3q+G;$BeP*cf7b(%& zY5s**7d=BYA-{cbPIGdT#ke^OS+ZNxEbx_)Vvzg0;kfBRM!R5dL0^OIyGmYc_+CIY zK-}QO0{T~Rf|BWQ9W{&Hla?5qGN%@r^y}vlq}fY$@8!$~N=Fk%bHv0gggWcV(W;GB z(iq2STFKE@06WDC4=)QlgGr9DJFwZn^5AyKlmghJij}X!P8`?XeH+2KgUL6)0lkFk zP0!UU-j(5Z$O6q-$AxRdNP`3XVD1^Ga+$o>uGSc;xR3rpmnEvzKH+tIcfdJ0UF-$k z6?hhJC)_Ox{c8w2T*vWghYk?CrI`miK;?)v!R^L#O%m`|9>z--U^m;P37}pozlz*f zwv^%dD%D(nX@yy&1iWizI$aj=_@`$51zsoq%}3~I?Y_$-O+NryJ@{Ocm+{XQW&-Qd z1z+*`M=a}g6-TLwTcyk@t6L;N+lhPn}SNFsr>SUEIfcz)Oai%if?i9^ICr|_cBf6UeZ3mbt_Z< z5zy>|n6cre15x=FKRkz zE8T`&tZ;G)0svDL>ihXBq%8(R%k4UqX!ALD+i#CJm$`+)JpSX*;)m`xG(n>&A?e%} zzExm))HT0(9VdF3-*Ke$J8VpJ*|Pqkcs|3i=q+7g)rjb-fOo$-FQIkpqCDr?5xfSd zn=43=^8#RIV$cU`v^@C<`>e2?nY|-V(1yCCF(e)8L+(ghnp?_r z12S(+U)t6O#V>7c-FChGk}EvN+9`{8QR&{$W@_~2{wbQ~wg)WL-cF~)TVv|wDM$K@ z^Ldpp#^&nF#6C+io8WIo@k*wXqw+ipQxo_rvC9y7#IR=u;#G2oK{#OtI_P!sl)~*p z)>%bxDK6H&Xy9)4JXXVzCGM5@>007CM3m98lNOecVk@!&oMxdv|X@ z$yQa!MrF%&=>kNM~rE?@kuw7XaR;6AR;+7RX$`i>Deuz6GNJJ*2F=$8+#QL z(2U$hxz7(08Za|fDcl+=)+>!Dyc+qAE6s}Om!5kBPr3EL8oE2j_R!;kSXyY|ODbZA z&_7(EWS^Z?NyR+W5+fkYKpu=&sx~qsmb*SpEA8U-ZCE*HN zLE8wvf_QSJ14TH$aY_|YNd=Vu(HgWbNq7D{CcQ*!F-|fPxLWr1rRsyN3LV|+c1HF= zb(A#1n+>}Z7u~iBAqZqcMrUMc# z|G`(vwzm{pf?Z9Ja{>v;L&up5wGay32t3`I)ukq151Mn8dmo8knoL-p<(PjFto0+8 zDv6VPg0tYJ5d1|@umPO_;RLRI71mE9l~#@!X5m|d7|UNMI3F&ZMCfozzSf}>_1=ij z4ph88y-r#pEww~jsI@dn`~G&tMncXZQ;0KZZlU1Lx&sz^o;7%XT!K>d_T75=pC|U#NA+ymjXaiQ9LC^Y^^_9OM2(F z&A;Z;3{?Mgw;XzuIfBkh`FcA0ji8uk(&oomxEL zqO5q}uW|i*^oV4V!WuA&0^;bVew%9kzm1(zD>34^xCvkGV)nPuyJr{H`iQ#){KnG7 zBg;hWZZZ+Y@aABgu_(k@&iWp>f?vhF2*Aj!< z*lk~e*39Q>6LoY!voqh0dj^yaO){Bxsl;&Ubm!pMzULsV+`*j+gKnk>|55A(zVjVo zrhxbZ^xKq!NQv45a>$iN_*)3Sx2%zZzCJ8RvPY`8Q~hwtyXcKA>peU2>z#g2Jqx+V zA!})jmSi!QxPtJxnQnVo-~{ioFu8s(M5<%Xls_4G`fGgRT6|Cvay>hodouesown4y z+{EHnQ2Sqgek-JPySA-6_uMF=RLuL&frlfeOgqnXKMhiJ;Sdy?b2q9FE1KP}xknG1 zFbAZX3m1RmANH^9#GpaULgv~vF)2Gi{F}X=VUKZ@+*N|v_2&UeNYT#*@k02k1i(~l z*f`YNvLFB~2Rp35C37mw4Ls`i1$#36{z+(Z6vMQPeICSq(Z#{t!a{kO0*gq%R_Jh@ z@gQyt+R9zwH+BA&-0!E8lVNe*G{%q^BX&S7N(}qX0;_+z%3|zx>F)|UzmnD2-WQto zJ074woM1*Ln_(GS+8kh%$9xIOp5-X0aQ*JKs^+JFs}t?&OQr0-PLEDhw~s$XEy9n&W4E?RG!cHMwX7! zONrw8Pn`GAWYKUYVX#Ytc!qKxBI6})UQ-DlSZ!$Hr6xO*5m z5*jUw%|^G+i9Z@hF^=JuFnF|p$v7&M!&wgbgy>!X7zy`G3dwMFVe2}ieQoz5tAtVe zX`}7%?Jr)ZCD1cdAT1bDE^jxNs|lOZM+jr_n$>jAaN(L^V}?-r_9wZ%A;>+{N8!%i zmrwc!fo{UIDS=^Q)3;xg4wCc(a*%#2cW-|duJ!ZD`eefluGzHu?EZ#p3x}-0Op~Oe zX~L|)6#lc4qa})(w7}K_#s{Dy3~EeDra%Td30J<;ev%p;S`3L51uy<;AN^~E4?Zx7 z>4_}sjbABqfIh4_=d`ar_wGAb^@xdzbi0r8Vc`)0G%=tY8WQ?#`Xj3<5I_xv?-3rV zya`JP*K^L}d|U_r%shf-$C2A4Z3H*hPh#%DZyXn1V9ltXE+2yTwgbm$({Z89`wXiU zxwZShBPpD_-q6M!=fF(%BTYJ@e@0Wiz!XsYGEK_rlOB?*YZ)t9d9%e zjy${?v3k+Ae&yb-)KlLM0i5H0m=x2)ufR;=hGeqVTI>d^;Y#HfWY4_+3X8Rt9erTn z_mj7GQXr5303pe6=E3a8oUwwgGfy2StEJQSN)Vv>%v5HcZFV!)=yj++7P;&@-V0q1%M;{muogMP#{@l`?_(}&qdh3d^_Q2z4W@fzY3U^g5$1v} z)&q+CI1WAT2r7x*-)@uh?Lw8=*mmbM$b;b(Bs1JNBX@#EIhuE|5fXITZAykBkJ_HE zLrrk0#m-8{Xx}W*Y`PdxaEr>elz)zR#Lv}0QTK(u6Uc*YK;&D>E#Q@B37oAII%XN! zCKiLxz$6J7=d#$xAK$-%T5d}OKKVQc_~n(hW)pZtJ|d27Ask^N+`eXiZ4x*o;cD2r zFlMM*b9&dP12-8F40Eb9<=kDr$dBZnIRaE1U|A>wv$;c434YKS%W99_iBs=BEl2jD zOs@IhViezeYl95h^jH=OW#mfpU4A~5uGKSUGmGpsj#9dk?534EeP{z8r8f?d)0lND zJ`57q=Sf^Su)8z-DVt$9|LoUwcSI3U4W7?WEUP6tpe6d%`HR*XQf7w-(xyK)e!jJO;-yt;$0y!7y@IqoS-OF!7mgp0k6NEU z?U@>Dz5S(-a>#dbr=e9S>o%dl4P&=1yQ^N>M1a?HBkVgayM9zzN%lw@@3pg)`DLsH z0REXF!%>yKcN3f_25fhwpt}$UP0yh}K$Now_idtnaj(+=tSm!PQdpcULhTq^Lk`rKah0;R41x<~mD*MOdAbj8$BV9dD(FWx8L znq1Z;ods2b^*romO7cuM03EkG0xpa0JNJzwEz;|D+Fh2C&>dTz3~*U*WTzq2Blmc& z&oA(QmqUWyDZZvD@z02Vv{1sn;8{YY!N&lrsf5exaf)CE)2b$W*d;b~`)V?WwU}Ft z2P|#1@Uo~u*M{YD-qQ_vm%g4hJb51-(o}Su;LoRX1I}>Iod;k#;L)fr-wWV2Ck}`1 z1a^x7e*`RAjhSWZVA5dVe#vr-ocM)5=5!m?Lzgr zalXN*YTzxjDMEIf!`%bw*19fZBWUonmKt*Pb^f3^6{olxsl*>JWgnCvH%s{aF!4YN ztMl3|2Q)MFez&ED^&{*+^CgnyrLf?N_@b*iBbpE6#R%370niqU!8dJK5s0kdShn0$ zdw!mpcx3{=5)I5WYbSU0Nqa|l^53sEC=rAQg=$~bh0wt=ujy4#y6l1ipF!w)Xp-U$ z`?>-BE;A9NI;+|X^#N2s0<$=c7eIahKb2q=<^6xA0A_vZi)Y3-JMAMRulAzen}Q$q z$}KBaTQ11Ds6`aqW_aQ^rOYX{DlAf*mOdt~Q_* zWG&wE(?&|GB6Q`4_?yw2Ydmfym<~P4H@5zi%L8UA@N8DqbITV@cmX5#AYkh(W13pi zQBVlsAU=Q$ucZU5Vly&6(0ywv`_{)$(l7kF>+#j`%=H5)5=j|GDlH+62d!<32mB7XR2o+EC@YJ$>Ho$+Xo zye&r!4@9vY&9Wz&xMw=L2EgMLeKekzP$JlU9I7VN?l5G5msM+9>i2B;Fi@S+!Rah% zT&SYcDnf}KS*7;R`~8yBXp3p~QHdKjlLucCsy?&Zz_8ULQ+?kDFvRbE=mK$#1XJi9 zeOiL63p8?Y>)PC&MFyitV5N|tJPa-rR;3KvYcI5mj|Zik1ZPh?7Yt{PVI9}KL2}HO z^ZL9BD%2O6&B@k_@N)mn#SKB<1AEc}pX1L&joKcp+tcj=q{PI6ky-Ol2zDNO)3d*! zG)hHUEw`&-DKO34SJ8~7n4NZO@Fc$HYZ2qrADKHm12vUAHk+$o3jZ)%eOUe1JC9zI z&IQhEh2CIto4ICe*rg%;RhvyOibtluSQ4L0+8#?cKPMH(Sf|;9Be-La=j@WMPus0` zDg%F(+4@Y9I;1?mx7)y?j#O?Vr(KP3-(h(0+S;w?1%hP4WuVz}Ud1HkZ5(*#n!M9D zDW^XdtDE-ZCsL@N=wgHoc?O807E+>$slY5D#<6x%G zu`L>>kAl7Uhpem>^-bIxg7;F2z6wJam+v@ioZ`xXf=+aSo=cZ%1_hPEda!q=pT$9k zfvfnNd~3R)M^au4$GE+=6vrJRWxc}~2{sW+n8-$VEr?<~hh}bbfOa{|?1@xiln&e) z?cg*b+5?ltn2I_aoGWDAPKT^X>3r2CyvED z5^}p?BN_e&I4Agw@cQo*z$qBXY_TG1L$%v#mqeTl4YsmkQ(w{H2^-nYHB52Sk)I~! zsp@S%6ne1{5^E?m@GnB zc;7f7@fzSj9=?`T=0KQO$8{Vk}J-Y=j+jNWnGX?d2 z=>yEqQ4do)GK*i)y&}|f{iJwUxXLKHVSO&#g+4rv#$Q71#O|;k0f_^83}rWdvd|+n zojT^&PPlgw6$}yHZ!m)IWPCNOQ=JED3RANO;vjvJ_iJa7i- z3<|f^`_C0c2KX#82ZJ5>zc}`J_9TnF$S@_zR+KDD<3Fci0c~!Kh`k2)put5q21r>lPun7W_|tUrAc4 zpq39XuZ7mF0OrjYO}|I~2vY8fUY=^6RBS%uOV}-id3XCeVd+aGNG(!ubpFiirXAJ! z{Qm;e?IhD?PdxvTpR(wQ)!F5Q64mPn?6*4y14%OMtg(|C*Qi`i)~g8|PGYQEt}B3E zCEr%@!Za};gz7sdG!DjdmahfhTgV2-xLk8=fj@V0yHWF2_+#>GM_uyqqZ&Dmro`ju zH^R=KuaR$}r=7a?q+KL7qvw7E*rmJ`TXSA8rXH8b(Pe(xRp*9D=2S5!lmMgM%My?i zqRWx$#-7m8g|G9V_kRdq&>l+=>D%tkB~LFO&BJvu_<3F4RZ0tK%Zm#yZKm05t?-F| zh8w|^aIbpItP@olCW(cWZPz2jeIe%x-kKA zXY}-;Cs(H5hH-GD3#fUKE6}~|U>Z>G2T;u%&^8B4fZAV#LQI}|l4JiHQ*Vy|rXs=M zwtzvwgpr!9oTL|NMDdsBG!AF*IOr^uhGInH z`5Urss2{Z`(gu;pxF!I{_PA+7z^|f!vV5r{aioI*+S(iBRq&4Ah_>-Q)NaAflf|{9 zyf(m(!VG@_6^Q0 zb3I`>3HqrI6@J45O9Mj?ZaPVT-6dq;9-nK%e6(Sg=TFZ2Zs!=t&x|g+`?;eh`)k(Z z#E{WlYm`kIvFQ&Qosy~jn&8ro+#lIgd(wsGyj>a|RvB3qow4ik^S3sou6T0V6|VTk z{cH*0hMVQL&2jQ}>@u_-;)Q-xvDb&Aoe6Vb~-XSZl%1b*7#|A(ydaP4!e){<8b9&s%Kqpjc7!hO4P z!mC#L(%d@0qV`@E*u;;%JDK5mrP4OMU^Rt(pTEih^bI9@Hq5mryuLy<>$J%!zqHLg z2Ub}04&f^dK9=?d*LB4xBOFD3+}P|<1lU!mgL<=T@8^x(OP3LD1%Ah;Bc4DM>oAQ6ID9_{4~nf;E5LCxD%P!Cw(@86rTxTg|~b| zZG-r(2fLgzefR5au9X!mL@Jd*b9u%3z^?hN8>fJ67yq?mf zFa7ScW*eyeX1gXOU=Bakwv68*UErC9>l51-zVA3v;u8A7H~i5k+ekmHqJe-tSbISS ze=!-b3l_;+dQbVlaZ7=;*eB#V{N{b?h#FQ1-z5dJmil(;qHW536-uy5-|dJexBsMS zhkvO$`%kLA@Zeng@>*UCKgqErLhu8d<3^<&rOe4Lt&a@ z+i02OT#9N%_ild|b9gxO?VMI#7Da)xxuIoN7+dSo7;93VI1TIw zy5`0zXR!>uUiKN(@rsvOplN)1NUpz(BIkgeT_9Oa7r8l@dJA8057)?!i-_VbB;^t@ z9*DzOM`qx*78UqhUH6W}Cx}cNB!+Qm%Qb_Hk$UsmSNL#&^Cg|0h32|ZAmd76Rtz3| z;fuDp=B(obH^R(&yxEj;cz>U`82W3w&pU^k!Le&EfLZ(xk55qw?xhVSYuOAt3Sm8J5^8>jP!egRx@y-QB#> zRr3wc^h8+&SKdgjVO25;NQ!BEvagDY{=+&r`?veJgZXB#-sVt-)zYZXt^_e)^_hS8 zp>S5V@2OLQkx-xe9%b9~uJ;JP&V+86mm7j(jQVk^Z0lbV!XCw5j9&{zYPCSCBh^hv) zxRRGuxYbeB3Ec_d*EtcZO)TvzpTzG{w2VSWnoGX$e92gQu6u#lgjdv)#Ns6xw=imi zlJ0jcbt>nUZ00$8Rls=Rbh!Nj;A0*knl^PL--&>_l}IlqiPu)KkBqetT$BOl1@0%W zCUu7vDi=4FBnKzoF4_zTTG@es1+fZ}OEGVki_VXB4kUbtF2F({+ z-WF@VLus;WpXP-VM@ce0>gq0VNtd#kT&GWYeP!o+WrfJIcR6kE%#8J+5sOHh94jJyp~XV2hysrvb>!5JJ){wNWYi3;#`Eg z93EfrK)2IM8GaQsZ@KdNVWObRM3gs`%L076jC(~g;gyw=+HZmif)fSWZuJ2_TbGG& z1QFEcNmI{5ORR8+0>RlRYq=dI;Y!1C`^@Sy9-CokOb0q+C70~9(68;oH~Cx{+hepg z63`qMMNFeCr^aOen!3M5`*Z&$dy~#Td@sIcTrG{bYjLsO;g3W-^NisvO4jtZ@!`gp zoXXNZV2e~}ga1ICyI9)JyV5V)4)X=vJ(Ey^omW$WjOc+w9Hn$T%9#5S+4wO*^FDb? zf5*t_#hb8w+yAC&1>5Y`rS;2Fm+}tWZ0fR+MzCfo$X|yXp&`vhe}^7HXFO!ME*>X3 z?!=%pPz={0aP3l4ksaYam25gmP}aGO-iwTiY*hS1yX4t_0=D642l(x!j|!0&PDOlg zE*e3ztUkZ#kq*m6d{8Qk+9o_G2!`D-r>X=i-O*nq^_Wdtm1Dmrzd8Vz*g(ZV#G!vd z^=Ve@ti-WIvcY*0s{1Px{+sJZvAMio7zfQ{#J7D% zRYXm=Y3N%Ua)vi5QIu~{bsQ0N$E(yDcK{Mn6LlAvtOH+ZuN#Cjr3r78Lf2nd<+gXZ z2y|hC=iHPQ3jKXLz{MIQW#&@F=ZKMZ} zJ59<&ZFMS6VuNsT9B$hTdvT=3X4|3R!^~1?xcsxlq1NfPdoB;Gbsp}jIexsSj`#Cl zj&@M;aoNCZ9w;QZf@S>$eZgKSu&xog+p>JCqLHTHvsowqBuf12FpcSitD)*sE zDp30mX|^fwmx(JmT&O;aBgUTnzfm-5&PlA-spUd>FLS6!SRM6<(zjv0@t^3t5nQRT zQCPkoX^;}h&xL7oG|8T`C}s_{DCM`z$K3%RMI}mgwoR=AV80-ea~}RUFcAD8YUDz` z5o*TpDB@zCJuN=!gy1#>>41cdy<_0pz?l%}^pVwQ!)nQX)vee?OAK#g>_Ty02^BceqbAOE?{2G);&Yu z2B52<4K`m*iW#?)H-^vmh;GB7;2{1XvV<&rH8JiATthgzv2;~%3ga{-Ch&Fl=q!8Q z0gm;g{Guusnw}SbDE4)ob{g@qt-1E0!N#grb3A0rZ$R-GDygU?+9&Ar!E861pJLS# zV%mztOxxF*9AYb0D_zrFzJ$AgOJJJ~V}dG0-GZcFq( zWAjwA-!>%UoW^hZSvTn~8H}- zbCrU{BxJdwvi%a5RDZve4eit{AAANg+*MgNRcDB0XD)Gp^Hzpdeje^}@L%E5&#Yek z+DK)Rm}%+I4)~zprt)G1@g56%nQ!@xy(lzT1gHstp8{-z0bZ!RsF1}p#!G@CjiB4y z?BQYNhIqU4egDW@ z8Si$$SJ6c<56zcA8q+xqZg3lPv0kAmuhO&pD948hS|Y> z{VnQOUEE~Ecr7dbzw7y09&<0n*|ZmQu?X_rtXGjDGvDdlx50o1*|K|{bG|ntEPOp3As&}W* z*XZ|g)V#_@3eSA-&zU$-MKe(+9yjUwb0+<@FUR6m0i9rp6 zZOJPkZ096R5soLJ>YihbgGEt@VuMe8q?WP7gCtqMtPrDnDP{l&>A|m){YVnb8qggN z=!ySSpvfQkKMDD-+S}Tg_s;y5H_++*d{+H$A>Vrsqd{$r7H9$`XC*QgEOE)Q$s!|v zq}UD9{5op${X*zLuk=UEDDU8+I3)CNh1d4%M;^X&%}8_r-P{`<-TnSJBD`z zH4Af#REf*NXemYDX4uF-Nt}=&<-os^WzgyBAHqlbPe_z`iQGOAHa|u zFEO$u{9yTj5y<_BX%4Q&1<$()kwgM-X1i5+(;+kgg+^HsJa)cX|W;|n7^{$ zxBlte*X5-(%|gIPhR|Nim5kA;hJUt7)7lZjGYbJ1+N~??+6JrvJfEfEwn->q`^X^gTQDUw_*EZR6`tr{&o^0iDhPkc z8R*1bXbMPJXHfzC{(K}m>6l+*(Br*{+^Y!c^lDArvFb1Uq*Nk#rcAcEsICe zpj7NZguQ4VM+IND0Y?CG_1AGp(v4L~cnzs#-T~FY%c&5qbI^X)_%42$@8Qm!R!>h# zW3TtGxHIDn5(QRc4yJn}XHYvpqy5y;F6lC1as6uF-75D{&%e8Hjt5mUeMs@Mo94-? z5aTp)y{))^ReYQKpF6fM{vdZA6N^;Yb`gFql5UVm2;2U#ZoBXK#58#<6cKC4>b22E z;L?ttux6cMtMtffo#Y0T0+#QI`06b)>sAtI-eOqqfkzl#^Gm5q8_M@Tv|M2n27jetP-q4e9 zJ!*(C1G@kfN2#xtD*Hzrw)cw4BBlr$Dh?{xy8Q&2U-lG(eXwe@Kem*H)IFKJB)@7o zc{=LAB?A8{J+7qb#Oq@2OM)FDk)>B~sWU{Ow5-ol@Yv+JqAqBLVdIT219Dxx<3KXbgHwI?pGN+L|1k26+k;gx*XmFl;882bM*gVG zYA@&9SAuw)dwv7E%R0aMn0@Z`py^G23i%bjZ=5-+E7$vW-@ZdJ|0(33 z|DQsB+y7h0H}fG|{vUPxQ>1A0axe zf1ZE4Vf}$)#+*pVzb$5RcWV36KumxmmwW%>jFlee#bTi#dAhYsMHa3Ly*?eUplHrr zxj7eBie`LG*WyhZj3_8n3sc=XQm+kln5p&5)!yvSxvFQr@c$3Qds*fCJsus6w6jHCo zi%U;4$mO4zw-#CXNac-k4r@MgdhA+HK~5NtGmw&XvLK(OkS9k5K(pH5(X}!NE&j^N za&Lm-fbhCbjo976h*mwBE55QwA5OGix%tJ19}O&F#gB#s*WqFUUDvOHo}tZ{KCYsU z-00Me9?f$5j3R~}6x~@um~`Rxt_~)fW*p5JQV6?g-;c`)kJyj_G4y&lUnDW!7UNb; z@p-3oJ<2)FCWN3e_L&kN+s5*d0KX{7x*)hMMw_xdK(r*QI@&uxU*LyZEH0~3p|cOo3>CN|0(4^ zbQ0EZ{=Dm)*WYQm1xL)tH(F`4E}0U2!1liTurnh1bPZCO4WMQ9 z?n?%I*hqy{zfx8;*apc9cE&uFuCmL?PUoNe`w!L`CXJAENsy-UbC^MlHO}X+i1_^M z1@cQWgnw*Ki)^t_v}@4*=0K}E<|vvk7Nx=5H(oa%DazkOw_xC7UKHOe(8BpGCv!0^ z@QJ}}gadNH2lC~B?B_q@3BtLR*P_+;?|1?SzM8V?JR`o0D#VUdI*T^OaJ9&)BR(tR zPY{M50k8!0L4BUny6djwM|n&B&fQ-}tZrLc4!yfnX$O0;ENaDmfMDOnq59rV{Tx}X z6nOISC+Nn6gvovS_?&PiHIf%O?Y6EAFM`=Nf1~|RQ~%IErvAfF*d3(IZRz+U3ee+E z?q+>7DJ<{p7L$Pe(Y)}L)gu1m{D#A3a*rvX%A+e9r2+PMPsuXgy_(Lnin_+l8oTZr zQ*wiG(CTN%$&bOSVGK>hJzTPpR5r5Ye>ZnqN7+YsBB=aQqk9P#$%+!%@Tb#qGX)ErSF3owRn(>6i{JaUia!%=T=}`i zY^km0)H4*W+}`9z^29^nKyEq>m%`tFca4?R1xWB5fXhL_h(9pLKuT;s62R9ZIH|Fh z8^duNbZU?LX{Bv#>SYQ5q;r;80ltXy@bCPiuz3bx^yZ2lN4F)V+I1enNvtZT8a@ZK zRFF|?#aEO&U)!%fO1FxzUd)}MbU^(eD`1|>0B~?$}bQv%o3nk-HDNy$oTGq>T`g(m5Z2Ed;USHFKU|+XWB6S1dyz5r!j~ZUMf@SSgFG6-2aVu{z}P1qRL`UEi+ufGyBCE! z1!tfatmXmatX`TVXW*7uGPob7f%ybC!`8~yxe4!3OX7P@VXwhQ8}vFfT4qx*hfd{R zzo4~EV*?e3&@O4T#pWic6Sl2YJps(5ho7P#M}H=rndcN$TN3WCgz$;Cj{NECi>v^f zaF+)bO0nz({S>t2x*|-CXuC507-1)DoSyme1N;_m6vW>Xa?15gud(L_xXS#t+j25R ztcJcAXDM)9cP!GE1%5mVTz8k*ICi_S|!+>uG-FB1XsQO$hcQk=6+H;|@I-B*T zcc!`cSBhQ~xg!IIy`;Sc!AuEI!KR~SAZ z08T%Ss1O2N5jX+bALS{moE}VGXp(-Ik=y4(GU7x`FljX+J)cf4J6}ddg03pUBaNiW zSsRQs#@bZ#;nb-~af7u=Vv}Z}RoBwMst6-tVH@lj{3Hv{6ArAdEehw^5$q*b!FCP+ zatB$uLTz$o26eRjr>cMZPgQ?R4fbDEeW~rwmX{{DB2^zJQuRx-=>`tA0rs>f5Q;!FOud9ljZL+|xAl>o(&>%d2RjuCp(A#9^ppetIEiel6h<*IOW@aw5+?bvD;%B?q zid0x?VI27^fSv@I!BbhH$29@k7t^#lZYwZ+j*Y)@<9@w`uN+g6^tQT|g z{`Loq@F|r+@|rLA<4 zJ2Joa-;Xa4PI8UgbdqqLqAmWX1!qhjn*MPu5wv~`>GbfkSQIhDMMfM))$2r|Y@Pm^ zN!Jb9bkagl&Q|$VfwK;1o-j6{M-vq{I&0D*2{$5q5q!0?ODhd^zoQpyHw|+Y@%h#BO|4igo@ompMY)0Og$qd?Svz7bA zijc+~?LzBBbOmBwB;@xQE22=EbRSt&0=I!OS3!8CwMMEG9msIYw$YYMB83lSNF*s5 z&#wia-gJ3xug9P%Drbv}sqoL7@1|U)c~Gf^ z<0_h$sn*O~XeBZL583a8JEYDA_CG4@T8(3ytt;{G_0V{p!UB6*U=JvjgBWJaO?8o) zu{qIvwfcknV;I^OfC*O;f1gJ)_~!@%=3~P>?`6r4g@#sE7@AN_$;j;w&c!CCEN7%m zdPLwAN5XdYTNnj$W#mL)^1mHlKuyl*Oi&qwq)1xG;}4kDB80ewrZMY%Ypv?v7>0{E zl!z#7j;Lo76*8|-7IynS)id9sWO*XuRQ+WREGzzbiBg+6>vE^wE>NLMD?x%=f+`QA zRehVL@saFx1QC&r6GPaH4+#@_RAE*7L5spqT%|#a5%hHDlC~6A)Q-g4b+EP>kM^&} zxA8vQwBTOd05LeSZ~%P-ALXgk;v0G0UBDwQYmcCv<}8!whg!*QWEjna5pMXnRRX43f{0J z!LvdL9u@mtYEQ@k$)x2O`HZ+eK}hO^(=NH`PM6B~(9l1U1&oYe$$|!|k(BXSFh)zb z@cutYiBbOfuA9H21;d3tucQ?_+!TT@HKbG9{k-^u>Jta^jCQ*Qb;(TUsyu$>%Y zopQbmRMNv9v_IHRq-!vGl9$KBYWGY!{*K?usbeOR8n5|;Ikl({**c)?4T%PB-8^8X zDcyrGuRn?WUGOqxGrKYkIV^@opy`A3=D`p}VA;_#4naY{^=imMwrm+ub1$xEg&^5E zyU={g4x}0%mk945MZJ$F{Y$#wON>x{W}%g+Ud#5b9zBFL2i)RL7PVCFYbqxGAw2v* zCQaxo*m5F)w6F-y_<$1K_Wm`$_=bht5}Iu8xhu?esbp(ni8c$C}M?4Q>v}wsz9&)1xxUcg_1#F z)FsV+;vnJEo%e{P<_l%pN{$l#9hm%9pIv)@OV{}9U{kz!fp6uRL%MU*aeJ|Gn=k!= zOag*yPXYl0Gi6WN2XLqL9d-uWX-NlO_>BA8-8TP)Os1Z%*G%)7VkfphyK*d#%_j+L z<8Rk{Bl=lXbz-Q+RfiWieI4>=!@x$JP3i^VLbH7`b-XeTVmlg0MM!CQYQMo3FW-e- z?C0y7m6HB1*o4&VHOUiSPY=4o-ye}4oQ{3NSt%9Wo_rEwUNh%uh8W=MXr>!aSHnxg zr1on%z&}o4`irb!)SCXdCn7dAn~|ZA|7XYG4{Ea960rvxi;(k>W81KNLZVK%M`VG0 z5PDd*JelZS}772?p-HO&M+gZwl|HMMHa_)3b#;jt#Dt z%G-!oE#z2w-&c^63<)Fi^P2vD9; zL)fnG97yD76svmlHpfZ3RZPj&58T5HJ?T~Txk(w`sTUVwuKilFmLNFvPhb)&sbqyf zJWFKs2i5L45m)DpS+o5aESUaBuz+$=c3^qZpi1?uwgw^~Yb4K>Kuiis ztf%;CE~;O-rg?0pFIz$@ItQ$V#~a20DcHVm$JKDR?)d2h5%f7iw^TI6Pn=XI4sl3Ko& zdNoD9+#XN254c9qO2RcCu}b6l_q2TFI`&JyXQVI4VuRQYPX&CB%9j^{8UF|v1hOCb zM23J7b5SB^k=ppHMR$M-5hg1#`_r0OiE02jd4zfl9_ilu(3aXM#ja)ARY{U`*{8hZ z_2O4uTM3P%N#b|z!3U|M+hz`Pn{kQthxAJIIu0!5>_2w&TROD^qRMNfy=B@VE&(|M zIjQFe5JY0M`JT~G*OsA7o3@sfv8V(Scp*7E5mT|(a=7kGB! zWWP5dmV{|aIdwj`5Fxyoo`U_i`_H$(#Lx?U4u?(S z7B)8B;ydaPuMGK=Q_$&|Yx;U_6SZNJ%L;h;`Apsy`n|nJYaV|VpWYc}K}r$Qdw6=j zc`txT41Qvg7m(8If5jxUb+Y750wNy#t7M=dPP{PcyaF-(E|=~aavB&9r@{G?yekhb z`Tkim@Hs#2pX%AHUW%heQ;zr;W>~v9KAd-3C88j3WX2p3tbU9$Na>@1*3Y&po>CE& z5{sLzy-6bQkMxpdJp@eyiS(W8$JmR^_GNjjDHVr9CV%aO$sn7RfANxMbl-2Dch2{# zP#rF?`YUhG|1eo}zmb;a=?kVNFZ z^JfF9RsZJ}Vl@89`8B)?mW(Cy%pvH>%1m4K>;%3SkpNpthJPg7={>xm zZ^h*_PYrjOlweisCAHe|RPlS(*A)qiF#!m)R$m*u6ufk%s(${$w2H$w84OGP`@whj z{*{sN0gMDZj}C5FVFJ|fWF-^%5Lm5#nRNo(xw*BUxL9R`w@eu_L>9tH7z`pwF{ zmiqC;yY~=jvHKh_>y`#{mHC$UZBji!#oiL0;x`G4Jv?3FaYHd2_Nuz!TVv&x^Y3CQ zBHNGlt6u?u-{_M}=SK`llm1f2(}0y~?c8HKD*rmN@$w&igI~ymx|yQs2_r{fK$g5v zDFZ$7c>tL>%G5?bZIz@kTi)J9WDj-AFaM$@hJRBNAqA&>A)md0sG21StRo@PxQDG3>MbD8GN-!r)=KaJbazQ5l2~NXaH%?Htq*LFi{N95T_A{1huu8LUBCXwi4v?>xTt^Y@l8~hW(;OL&+SM;o zPm@QyUqNdtsSHu92MSgZ^1Hu5+NW5R2&mefklq^Q<3DIe(zhN;urFsh6GXDb*D=Z} z*kxR#F9*;Pszb6kHK|7GMH_lUmLAHu@X7HRKLU1;I?_}Y4YU`cAj~V?^rZ=n;*O0b zQWG#ikGS+G7&J6mgV@N5l##|83<3|El&|X}>9F8V*aFXBko2BcFf#K6@6V7ON9rW# zzUocdc+f)F&z}zLF~+1Ms4Z4g&ZMm#YLu6?YMSuUSoHQdK0;$?@Jz#g{;f@njqmIi zB+Sg6pz5osJJgL@+A%ftI)bvqPclwIrtnZUeH$4$hzhr3(+dvy5#R0gm)-c=auE=8 z5)?Ra@%Z1y$iMx`2E|9(H1ty#8C*B2=C z&yKV80yz7zj5NMfYt8f^_BdtZ>!Ng|3N5q51C>!l|v ze)uDff5s@s0LMjGdU7Vq+ma%0V@ha$>Y1v7nJq5wYQv&u-k0g|F0bEuCpSfm!j-wU zH+S~ZQGstA;q?Ow6^LhWP0+O7v2Q9a_dO{K2+42mB_R$ND^s6hrzulx@Pa}Y07|xI zZQ5usG;hR7B2g!#XHKt%y@c$1n9jU=qDJd@vfJp*`Wz8m(e;H(ixs7Ol|^H~&-kBl z$77u8%gbYQN7{>z5tJY4%G$0Nyi&&n*pj!61V1s)OApkvdcRe^d{umH^O4CJ^Q!x* z&tzSUc*F$lL4_dB+o>yQws?kBX)jL;^8IA}MJ{|iI7)l^sukNU2dGa}9YO-0Nh4~S z=v9@`L}3ITf~M2EyNGqrwP8vJN+;C!a-Nu^E>>dGJs+fC-m3igpoQ}BD|aq=CKCkT zK8kXOXQfN@-XMgv;j0y4 zguuYaJhN!DAVNxzT_D~DeN@omS#+O)Pa^S?fPJsMO`J_cihF=;=szQm%^^F-mHfyT z#;t@PS6SK8Pk0jAnAVwtf5#o8Cucqre4?6iVpAA_!jMAbJAjQeE~7m1gf~n;&D6E8 z^oS7=3jXo^k5G)4-V5Cy6^62vv%EBR@gy+@H(HN@Em0ywKr#>w=OmtmjLA=hq8V1jcDX7qN1G#}Lw!kVctWdJ7YTo+(giN{Bw;`lF+28Xo+#)R5ox ztcm^T4WXlu>p>2zHHq(d>+(^A2y$f!jNKtO`CC8qJtS3o5}C*IArSXTQZ_6CIIHT9 zkvuYqfW=Oz*L+_GQa_X#`C6XIU_;q5M9E<3XSE^(gORBJvN2B=>5zt>E&qa4EtIfD z@^*mW@;hvMLZ@-~4+a4{H*@t6;e(|8BTW;?iFFSC>hE?R2rP?9EXk7xSZCQ;(e7N) zvFEX)+>x8^*ujlqZ0h~%*FF0|am!FS(*Es@4)>QaeO_@#%=cd|M*urMuf1(1HXryMFG@xXU>DR~*u^U5K^?7wW26l?Ex^03O{RnZ7Aq++TiB`n259wtg6L@a$E0aVNZWJLE z5UNtkK9XERNno+oiVu5RllPGu(7W;P*!N(qHsrukeuZl03e+cDOi5~ghYnk=&YJzA z?j_Hxyx&xL;RF9jniPwEYaP3<0tHi`J{Ajg6<`^|)wrpM-Fg=h=QKZ=^%||g$gOYy zUQRTAnG^PmiLxuj%@;!kvi+Vy3)yHj*ZM6X4E=a=-%)-AlfM@~qCu1J9h7HCV4Ovy z>2y?dq|>kn<$IuM<2Y%05PQDGs)m<7tIvrLQCb^yB8Qv4YLL62?H$U?YcJ-7SD=B8 z4h^DK;vMPNTa~vW1I)%;X@SYu;plZF#AH5ri9khzZmZ4yU^`EivTa+VFhw+PHN}NN zUCJZsTlx!*OEls7t`MjA=W#NI*e4o=wAN0R;`hcL!J5g$JQ!xk%XUe zi-kCSWE~eqOk^IGjQcmc1;Zva3H|OWBlL`_$8nTWU+<3711eht@FQbn6ohlEgBWy1 zoamlY^dFf?^mL4K>JL?iGa#Dsy(P&NcGy67 z&MY%v@4U)8p{1*F_t+X-BD;q$6SCepZ{46jj_wqRlIKZV-OshGIz?2}SgEGtqoDU( zRPiO8K!Z0rY1f4%%=SAPs&6q4-2g(9d@+pDy~FXsS2ULe7Jr5-J6u40zlD{4;LEyn z_D@|VZKU54Yj5&XmPx`Q4E6VGKX~oguCg@3we*y#d}}5XXxXG>!07eMIqHxO6VZq{z-$${qtYH?Sjw`x<&?GQXjgyjaD@j(~{Q z0rpcooN1vj(KfGwM%S}XE}c*V%j#9d8U?*|<97TWPO{GqM>ZeuR*8)sihcGteF~|O zXHsj;(WabjCW%3!>AH1~(d?+b_4oFnZ<^FL(|C`qUe4TVY>s;!Of#v59OP!xFS4Z( ze&b5hvXN%``Vnp`Q?& z8O$Q~Vi^4E&Q$k9`Dqp{ak|7c?3|K)EVqi&Lid=0?c{d`$@v7Jp^Fd8R_kcNe1yX2 zBe3pM4|IBz7f^!Z70zV+20%5QIKsdV+(pdLOb|C|@`!v?G+891ZTOt6oVG)nJ<<`B ze11`zQc%>k@MjTIk@w=$+G}bJ6aqLmP!CwNP`k=*ZrsmiY3#BLfB+5P2WG4YdWFk$ zPk-L?EcPV|7?HBJpYme8#QksOMr0M~MtZY69noL!1g-Y|Vcc9AtYAJE0Z(iV=vmz= z`Xh{)NU%wi28^5fS^PcREV57*DA6+6vW_=zJ2K>Wmwk4|S$nRVC5DyHWJjW};(9!f z@wUd$3n0eCQ`W{Sgz_E*9dh6xw-C~wL|^#Evw?J$v=ZNaEj|=(67&4~)N9-wL+U#9 zW?{fBMgb+1pmKPFz^?;pw>GVc3@$t=0`+xh%nW$AU)Igx2|af&?G&IbCy~^cjA)Gmf?x{N+p?#y3z%LY9p<})E z7PO{R!yq4#F&|l0pq6mnf6PO{{3ypk9Cq1Y$KlVnrAa3khj!NrZ%#Mq3AOM&+12V2 z*dOT35oBCiyeu=^J|mcnWyn_0if`$X^rewCjq>`+~3#aEkkIDRcP{|)mFX1{806MFEcC)9X_bS7j`|AL#s z`clp~MZ5kTe=gHQ&pt7pPNS@02_z4r9OmCt+I`I~2ILr{Rr7sZR_Js(X# z+q{$lbfHs}>2Lf$Od*Ez#opLeqw4LHq>g|PvcDqIn>|AB61n(PC|d_HsTzwFM($SM z%ECWFc8m5VFeB)>)IdJONipaonpJ_q6@rL36ZRRF5%6YnGdUtt+rmvarqD_kz+JqV z2`nQfgkQ!e1daWVvS*U?bN;(|JwysJ?8Wit-zN-K!*~5PftsY>C?z%mMb}Bsm!uoq z1l*IxttV|cD77B`U5n}zm8BU;osUfc^_|?uH3N8Wh7NCJ(uI&`2IX32V^%J2z^|7P zcC{_D-|QXfu879i(udj9m2JBBklkV6nkUGTNJLqlBl!aJ%^uGDUQ?T^wQc&w9>_7P z*!hdq`6gV;mvgSBUeX+@`FrBWsRH9Z+O4K>`UP-+WfVEgyWhK?Opt;Ya04dqhe2%F zj_}#yT&8dXXFTi_<3_|O-Z-2g1BFC+(r>Lm!+GmtA#)p z@pQfHXP<+4zsT8Wz(2_uUk!sV^NVAg(D$b3k=m&XEqm>W>V}b&C zVuHq}TlJp}iD%ThBP;ih(ckpEq0PR05?eKZv*Ps)d=>s6D_F~v_Tp^%Lv|v6=)NuC z8NBN1$#B)A%rb*%NI#YiYeN~im3?atas77uwKJtkKzXVmcq^RttvepFv~ij+rR;IX z!+vJJW8i(PfNhJe^Y$*ZI5-^;KXCn|R1w+Z{^?h$-)4hObZ@_?#7W(L{x-_){nd;` zV?8fB)M{I{EPCu8ZW1iYcH^j|w>R2b)ps}%Q%wG?RSZLIM1?M=(&Ngnt=ugP5O*A5 z)!?t6on@VH-s47h`fJYh=dA1R&*NDdE4vl!9)viQAFD^x1>J`$_p86%cY3Pk*15{x z5I?`wEZ$3m?`qL$g$EOa*bV!DWbn2B1#ZLW^25$lPnBd9qi~OjQgGa(krc-JvaaIl zNW>7|NJOlqCoO35Nr=QeByOoPV4RXmq4&1tiWl)46~a!B*S`VL%7MRkJFwJxAK30U zlILAQDj(pRE!!cC@Cg4xPz!fi3>naW${& z4WYl6+VWKZXI1WRtnpti+$B}{Jj@cfs)C<3uINf05R5*X#q%#mo-WV0a((+<3RxeXcY6aeRf3t@c*4gqzt3aYmDy>_{3vs2k0|%>eQB zzyyI_b@jWA+aJN1t}>JzQc3fw^Pnep|JxLi1GZ|pqs8Xm?dad-{ZDu88Kh~vN%PA5 zT0hn2{oQE8FidvWcdR-JRle7iET5X$uAPA`UHysLzPrWMq(jtAPjxtwo4y43^gpezPtXZOpT{#9YWUOq%LvPTl|d1Ym8_kry*jyS=Yq zOA@?FrqmG{IVl%Ju5a~+gX$6-=_=*RDI!vpdJ*TJuIqb}k)6mi(tnw6E!z7dyO06= zqMF0W8!M}SxkZL7bvt4*uFbwM27DRf@92ff!46!NQa4kF1X4FAVKcG@waAch;C=t#RTlk)756K zVQ+NBEe6OhX}IxS*b=f1d70xc1nFm+`?l%5bi|qu#D!f5`TL}b3!}XkSeFfY@mk)c z(E4cUq-0hO_NB&6y8X>WY3uT{@HZR)Ag~?RvAXNG*pGurk&|daUUtSKRKu4n*Ft8t zaaTScCbY9$>Q30;eD@21)T*ps*3c&fx%^I-_dQ8gGTPNoGwMFgm~VNXdZHsqY|g@8 zBkd8>o)1@+U0M3l%Y&hH{J)Tfgc1grS=P)@8E z)E+|o7+C!lsKAS~d2BknFD`bhOrWx5eYp(Q>KO;`50lEaMXA`*ok06cr`tRpi$|2gr0?5Fi?c(b9&_tjUz)1q3snEtttUZ zH*h}(Gv5gv&1Lrz(1*{zgE;0h|H-!0sgl!=#X#@r^0UjpwF|-W#L${to*S1+m!(J3 zlBl_iRRp`QHDE2I8Q%)7_c$_fggnU*We}+b>o^fuRfrPr62#AmTG)UnehfW4mFfCDk9yysH+Hehn zP+X%?Yd2oAiU77y9h83XwAbik$bCv9`@I9(X&s_F$!f)K;idb0((B3l$S*8vgD*MV zy6%{G%WHgCYF0?=k|6weQsTYuVw5>CjkkuvKgUl2!7(S7eycWeZ+6nZhBjWa7;X%O zlxn6u4mFJ_Gi<-SzW8hR^fBgrUhYs}nlwfMKcnMth`h`oDiP^#GS8K>^iwS?GUq&_ zlyr$mr9mJa8&8~W;QbPK@`5VYw;}qNzmAaHL$on-!SmGS%TxY;Y@USCu{@W4&-sHM zS3o}4PtebZfZ<&tYJk1N{{P!MuHRfXBXdJ~5GKmHLiiV-f;WT)#kqNOAY7@3q6ctO z=+G%ChpN4>f{_h!HZZ@8Q?!0Ckf(vEUR~j5jEnxUcv}9oc#8JS(5Ae2dJ_MFD}jiw z9W2#et$i%wx~xT+m`hW*IzOZXEf%Y~^Fh*?@Qy0y&q862(H+-E^;o9~+7< zysNKrL^2k}XVpkSQb6r>urX9zYL?$fBJ#pn^Wr|ai4;fSP%USd3AqP~oA0$p&hrmS zLGOOxgS`)xHLcypxm9&krvwr{{k!#MK{c$s-<~PI=p9l&XO+Pzgkfy`^ zwNJYK$_XlsU!yd2VYk9OK(hzt+W)X`b}p*#>GZiuS8V@zF(mLVaw2LQRPQdVS+rwt zB|DFk{{+~H?`|$fMv^j4x_qn&Kw-_6gm~x znSgdb+WMDV^GNF($4+17HPe6C)1-E-4Lo=<3^S`v(+_7ts75#1Z;h+v6NmGv*Qn6| z1aaXt%A7iaPh(%iJ4O7Lvmk+NDtq}e5fND}j_3&bfZ!8`Sh8}#+&d?9k1YLOq&Zy< zZ;QQ*IHdLCJjkxu51yM&aR9f5p}_dEp)`JU7)`%J^GjX+4(d+=DgLF^l$#72D&4UD zs6MU9_4#TDZvn)b5X!MHNe;W_Hz?W7hla(`A=H+gg0G3DAx!w|?9M`K&9JK^?kgY=87@g42sm=uVg| zVkBLxGfVs~S|Z4C@9iI)Vy9>!T(LmS=Uo5%uc=X{#yK6u)|H#zuShe*fsF9S&3u1f z19`R$=k25m=2Nob>9H)?TGOH-JHeAxks|`!Fc2D88DafuKQ$axQBfzEl`j-jTew*w z2B(*v@m_?az+$WtjM%Sq=X+W=eg5(dyNt}&i}i%^{Mf!fcSce181cA$APb(Df3dn~ zE8`^o)=dWl$|nTzgW<*F||oBep{nh78xi>=x^5RLK_f_%#YK z7=h9h8KF^e;aE|m<&#<7qg)42pl%cm)WXX}$Pm1>Yv{M8Mf+jE!ica?n(YCip8IhX z1c-~ust>L#Gy1>twx=J4TBR11!J4)a^JsZKB{q_HhpITwKBP)4v1Jn6-8l=HVOq5b z)zLm@JbYSafRhT;iaX}duRw4KyFT^I#W9F8STf z-G^A8+LF#cnFUI8kHM}y+rPfMpMW~jQ<21ckO5r!zy)ej3V!hb_f?Gs>wk4&?jhX> zE1+a;YJyQRpF3Pa-R}66Q>&6KPrmhJz~sLExcRXMk{>e&MqpMwKS}y%u1Q{n0YIPy z-wbQo-N?m_gVw@*bGQc)`S7uapxv!~^hvfGS!sc=>e`O%QNZ)^X_uF!Q+w8!hPZbaP^RtiUDcxD_JjZxHpkjOTxZ2K+GiahkwZVthQ9V(kwck*i~ zd$5PST=bB4NP{t01E zT%Ckkmym~BdnjRtR&e*4*lAz??^Xxs*^?Wy%;h=O!4aa+9}myezaF0bk0F~SquOK3 zuweuCI`lwW_#oZ6U%Ecc%l-o5p*7ajqcIG#>v}lkIMyB!&U$muamDQ*zs#me1F3+o z7V(f@B*6RDtXgK-*iYM~6ZKzh7a?3+s^#{%c4Q5CGHSBWpuxpca5tk5;c4%qZA|+p z)I_XUU2O=m|4Il|65PSuCl)86v0Kn4mTxZQ6ujlGPk-uLI}>bz=kHzdXfg2&n}&Z} z_}Kp%xbP7G(nNZV-;kP(CAzjRyZcW??B%Po5g!cUD-r-B`M@e-#X@!hcK@cl{Q8<( zLn!|y?W?81Nfuf2jtZBv6L*p0;fCa1&w=@~Zq8~9-w8NgjC^RLtDy~i9RH7x1K#7Cx35Z? z);?7S8l5pHQIzq-&v(tKa;a%U8(U}kjOcWA1-omLLt*Qc-?XG;Dg|x+dp5ORnx8A= Za~5*;wX>WOaCLQ);RRFuqI33-{ttQB{(t}g literal 0 HcmV?d00001 diff --git a/ais-bench_workload/src/ais_utils_adapter.py b/ais-bench_workload/src/ais_utils_adapter.py new file mode 100644 index 0000000..23e8858 --- /dev/null +++ b/ais-bench_workload/src/ais_utils_adapter.py @@ -0,0 +1,28 @@ +import sys +import time +import set_result as old_set_result + +def calc_throughput_rate(nums, start_time, end_time): + if start_time == end_time: + return 0 + else: + return nums/(end_time - start_time) + +def calc_lantency(elapsedtime, count): + latency = 0 if count == 0 else elapsedtime/count + return latency + +def get_datatime(): + return time.time() + +def set_result(mode, key, value): + old_set_result.set_result(mode, key, value) + +if __name__ == '__main__': + fun_name = sys.argv[1] + + if fun_name == "set_result": + mode = sys.argv[2] + key = sys.argv[3] + value = sys.argv[4] + set_result(mode, key, value) diff --git a/ais-bench_workload/src/common/calc_glm2_result.py b/ais-bench_workload/src/common/calc_glm2_result.py new file mode 100644 index 0000000..47db5f4 --- /dev/null +++ b/ais-bench_workload/src/common/calc_glm2_result.py @@ -0,0 +1,46 @@ +import json +import os +import sys +import ais_utils + +RESULT_PATH = sys.argv[1] +RANK_SIZE = sys.argv[2] +RUN_MODE = sys.argv[3] + +total_throughput = 0 +accuracy = 0 +merged_json_data = {} +# get all json data +for rank_id in range(int(RANK_SIZE)): + json_path = os.path.join(RESULT_PATH, f"result_rank_{rank_id}.json") + if not os.path.exists(json_path): + raise FileExistsError("{} file not exist".format(json_path)) + else: + with open(json_path, "r") as file: + json_data = json.load(file) + if not merged_json_data: + merged_json_data = json_data + for mode_key, mode_value in json_data.items(): + for param_key, param_value in mode_value.items(): + merged_json_data[mode_key][param_key].extend(param_value) + +# sort all json data +for mode_key, mode_value in merged_json_data.items(): + for param_key, param_value in mode_value.items(): + merged_json_data[mode_key][param_key].sort() + +def set_result_single(mode:str): + for key, value in merged_json_data[mode].items(): + if key == "throughput_ratio": + ais_utils.set_result("training", key, sum(value)) + elif "start" in key: + ais_utils.set_result("training", key, value[0]) + elif "end" in key: + ais_utils.set_result("training", key, value[-1]) + + +if RUN_MODE == "only_finetune": + set_result_single("train") +else: + raise RuntimeError(f"not supported run mode :{RUN_MODE}") +ais_utils.set_result("training", "result", "OK") \ No newline at end of file diff --git a/ais-bench_workload/src/common/calc_llm_result.py b/ais-bench_workload/src/common/calc_llm_result.py new file mode 100644 index 0000000..db4695e --- /dev/null +++ b/ais-bench_workload/src/common/calc_llm_result.py @@ -0,0 +1,55 @@ +import json +import os +import sys +import ais_utils + +RESULT_PATH = sys.argv[1] +RANK_SIZE = sys.argv[2] +RUN_MODE = sys.argv[3] + +total_throughput = 0 +accuracy = 0 +merged_json_data = {} +# get all json data +for rank_id in range(int(RANK_SIZE)): + json_path = os.path.join(RESULT_PATH, f"result_rank_{rank_id}.json") + if not os.path.exists(json_path): + raise FileExistsError("{} file not exist".format(json_path)) + else: + with open(json_path, "r") as file: + json_data = json.load(file) + if not merged_json_data: + merged_json_data = json_data + continue + for mode_key, mode_value in json_data.items(): + for param_key, param_value in mode_value.items(): + merged_json_data[mode_key][param_key].extend(param_value) + +# sort all json data +for mode_key, mode_value in merged_json_data.items(): + for param_key, param_value in mode_value.items(): + merged_json_data[mode_key][param_key].sort() + +def set_result_single(mode:str): + for key, value in merged_json_data[mode].items(): + if key == "throughput_ratio": + ais_utils.set_result("training", key, sum(value)) + elif "start" in key: + ais_utils.set_result("training", key, value[0]) + elif "end" in key: + ais_utils.set_result("training", key, value[-1]) + + +def set_result_full(): + pass + + +if RUN_MODE == "only_pretrain": + set_result_single("train") +elif RUN_MODE == "only_finetune": + set_result_single("finetune") +elif RUN_MODE == "full": + set_result_full() +else: + raise RuntimeError(f"not supported run mode :{RUN_MODE}") +ais_utils.set_result("training", "result", "OK") \ No newline at end of file diff --git a/ais-bench_workload/src/common/calc_power.sh b/ais-bench_workload/src/common/calc_power.sh new file mode 100644 index 0000000..d0988a8 --- /dev/null +++ b/ais-bench_workload/src/common/calc_power.sh @@ -0,0 +1,149 @@ +#!/bin/bash +# need to specify ip user password + +SUCCESS=0 +FAIL=1 + +function get_power() +{ + result=`ipmitool -H ${BMC_IP} -I lanplus -U ${BMC_USER} -P ${BMC_PASSWORD} raw 0x30 0x93 0xdb 0x07 0x00 0x11 0x00` + first=`echo ${result} | awk '{print $1}'` + except='db' + if [ "${except}" != "${first}" ];then + echo "ERROR:ipmitool run fail,result=${result}" + BMC_POWER=0 + return ${FAIL} + fi + high=`echo ${result} | awk '{print $5}'` + low=`echo ${result} | awk '{print $4}'` + power_str="${high}${low}" + power=$((16#${power_str})) + BMC_POWER=${power} + #echo `date`" power:${BMC_POWER}" + return ${SUCCESS} +} + +function calculate_emptyload_power() +{ + max_num=$1 + num_i=0 + + # clear + BMC_POWER=0 + # MAX_POWER=0 + SUM_POWER=0 + + while [[ num_i -lt max_num ]]; do + get_power + if [ $? -eq ${FAIL} ];then + echo "ERROR:get power fail, get_power exit" + return ${FAIL} + fi + # if [ "${BMC_POWER}" -gt "${MAX_POWER}" ];then + # MAX_POWER=${BMC_POWER} + # fi + SUM_POWER=$((10#${SUM_POWER}+${BMC_POWER})) + let num_i=num_i+1 + sleep 6 + done + export EMPTYLOAD_AVERAGE_POWER=$((10#${SUM_POWER}/${max_num})) +} + +exit_trap() +{ + trap - USR2 + RUNING_AVERAGE_POWER=$((10#${RUNING_SUM_POWER}/${loop_count})) + + AVERAGE_POWER=$(python -c "import ais_utils; print(ais_utils.calc_single_avg_power( $EMPTYLOAD_AVERAGE_POWER, $RUNING_AVERAGE_POWER))") + MAX_POWER=$(python -c "import ais_utils; print(ais_utils.calc_single_max_power(${EMPTYLOAD_AVERAGE_POWER}, ${RUNING_MAX_POWER}))") + + python $CUR_PATH/ais_utils.py set_result "training" "average_power" ${AVERAGE_POWER} + python $CUR_PATH/ais_utils.py set_result "training" "max_power" ${MAX_POWER} + + echo "RUNING_AVERAGE_POWER:$RUNING_AVERAGE_POWER EMPTYLOAD_AVERAGE_POWER:$EMPTYLOAD_AVERAGE_POWER AVERAGE_POWER:$AVERAGE_POWER MAX_POWER:$MAX_POWER" + + echo "exit end $$ $loopflag loop end" +} + +function calculate_runing_power() +{ + trap exit_trap USR2 + loop_count=0 + + # clear + BMC_POWER=0 + RUNING_MAX_POWER=0 + RUNING_SUM_POWER=0 + + while [[ true ]]; do + get_power + if [ $? -eq ${FAIL} ];then + echo "ERROR:get power fail, get_power exit" + return ${FAIL} + fi + if [ "${BMC_POWER}" -gt "${RUNING_MAX_POWER}" ];then + RUNING_MAX_POWER=${BMC_POWER} + fi + RUNING_SUM_POWER=$((10#${RUNING_SUM_POWER}+${BMC_POWER})) + let loop_count=loop_count+1 + sleep 6 + done + echo "calc end" +} + +check_command_exist() +{ + command=$1 + if type $command >/dev/null 2>&1;then + return 0 + else + return 1 + fi +} + +function calc_powerinfo_backgroud() +{ + check_command_exist "ipmitool" + if [[ $? -ne 0 || -z "$BMC_IP" || -z "$BMC_USER" || -z "$BMC_PASSWORD" ]];then + echo "not valid power env ret" + return 0 + fi + timeout 4 ping -c3 -i1 $BMC_IP >> /dev/null 2>&1 + if [ $? -ne 0 ];then + echo "not valid bmc_ip" + return 0 + fi + calculate_emptyload_power 3 + + calculate_runing_power & + export power_monitor_pid=$! + echo "power monitor pid:$power_monitor_pid" +} + +function set_powerinfo() +{ + if [ ! -z "$power_monitor_pid" ];then + kill -12 $power_monitor_pid + sleep 10 + echo "send signel sleep done. power_monitor_pid: $power_monitor_pid" + kill -9 $power_monitor_pid + fi +} + +# main() +# { +# export BMC_IP="xx.xx.xx.xx" +# export BMC_USER="xxxx" +# export BMC_PASSWORD="xxxx" +# export BMC_PASSWORD="" + +# calc_powerinfo_backgroud +# echo "get power ret:$?" + +# sleep 10 +# echo "sleep done now calc and kill " +# set_powerinfo +# echo "set power power ret:$?" +# } + +# main $@ diff --git a/ais-bench_workload/src/common/calc_resourceinfo.sh b/ais-bench_workload/src/common/calc_resourceinfo.sh new file mode 100644 index 0000000..4b03412 --- /dev/null +++ b/ais-bench_workload/src/common/calc_resourceinfo.sh @@ -0,0 +1,88 @@ +#!/bin/bash +export npu_info=/var/log/npu_info.log +export gpu_info=/var/log/gpu_info.log +npu_monitor_pid=0 +gpu_monitor_pid=0 + +function calc_resourceinfo_npu() +{ + device_group_xp="$*" + npuandchip=($(npu-smi info -m | egrep -v "ID" | awk '{print $1" "$2" "$3}'| xargs)) + num_i=0 + one_average_usage=0 + set_result_file=$2 + while [[ num_i -lt ${#npuandchip[@]} ]]; do + for i in ${device_group_xp[*]}; do + if [ ${i} == ${npuandchip[num_i+2]} ];then + echo $i | grep -q '[^0-9]' + nl=$? + if [ $nl -eq 0 ];then + continue + fi + one_date_usage=$(cat ${npu_info} | awk -v a=${npuandchip[num_i]} -v b=${npuandchip[num_i+1]} '$1==a && $2==b {print $5}') + onesub_usage=$(echo $one_date_usage |xargs |sed 's/[[:space:]]/\+/g'|bc) + onesub_num=$(echo $one_date_usage |awk '{print NF}') + if [ ${onesub_num} -eq 0 ];then + one_average_usage=0 + else + one_average_usage=$(awk -v x=${onesub_usage} -v y=${onesub_num} 'BEGIN{print x/y}') + fi + echo "resource_util_ratio: $one_average_usage" + python3 ${set_result_file} "training" "resource_util_ratio" ${one_average_usage} + fi + done + let num_i=num_i+3 + done +} + +function calc_runing_resourceinfo_npu() { + if [ "$npu_monitor_pid" != "" ];then + kill $npu_monitor_pid > /dev/null 2>&1 + fi + calc_resourceinfo_npu "average_usage" $* + rm -rf ${npu_info} +} + +function run_resourceinfo_monitor_backgroud_npu() { + stdbuf -oL npu-smi info watch -d 5 >> ${npu_info} & + export npu_monitor_pid=$! +} + +function calc_resourceinfo_gpu() +{ + device_group_xp="$*" + one_average_usage=0 + set_result_file=$2 + echo "device_group_xp : " ${device_group_xp} + for i in ${device_group_xp[*]}; do + echo $i | grep -q '[^0-9]' + nl=$? + if [ $nl -eq 0 ];then + continue + fi + one_date_usage=$(cat ${gpu_info} | awk -v a="${i}," '$1==a {print $2}') + onesub_usage=$(echo $one_date_usage |xargs |sed 's/[[:space:]]/\+/g'|bc) + onesub_num=$(echo $one_date_usage |awk '{print NF}') + if [ ${onesub_num} -eq 0 ];then + one_average_usage=0 + else + one_average_usage=$(awk -v x=${onesub_usage} -v y=${onesub_num} 'BEGIN{print x/y}') + fi + echo "resource_util_ratio: $one_average_usage" + python3 ${set_result_file} "training" "resource_util_ratio" ${one_average_usage} + done +} + +function calc_runing_resourceinfo_gpu() +{ + if [ "$gpu_monitor_pid" != "" ];then + kill $gpu_monitor_pid > /dev/null 2>&1 + fi + calc_resourceinfo_gpu "average_usage" $* + rm -rf ${gpu_info} +} + +function run_resourceinfo_monitor_backgroud_gpu() { + stdbuf -oL nvidia-smi --query-gpu=index,utilization.gpu --format=csv -l 5 >> ${gpu_info} & + export gpu_monitor_pid=$! +} diff --git a/ais-bench_workload/src/common/calc_result.py b/ais-bench_workload/src/common/calc_result.py new file mode 100644 index 0000000..60a9838 --- /dev/null +++ b/ais-bench_workload/src/common/calc_result.py @@ -0,0 +1,43 @@ +import json +import os +import sys + +RESULT_PATH = sys.argv[1] +RANK_SIZE = sys.argv[2] + +total_throughput = 0 +accuracy = 0 +for rank_id in range(int(RANK_SIZE)): + file_name = "throughput_rank_{}".format(str(rank_id)) + log_path = os.path.join(RESULT_PATH, file_name) + if not os.path.exists(log_path): + print("{} file not exist".format(log_path)) + else: + f = open(log_path, 'r') + cur_throughput = float(f.read()) + print("{} file throught: {}".format(log_path, cur_throughput)) + total_throughput += cur_throughput + f.close() + +accuracy_file = os.path.join(RESULT_PATH, "eval_acc.log") +if not os.path.exists(accuracy_file): + print("{} file not exist".format(accuracy_file)) +else: + with open(accuracy_file, 'rb') as fd: + accuracy = float(fd.read()) + +print("throughput_ratio:{}".format(total_throughput)) +print("accuracy:{}".format(accuracy)) + +result = {'throughput_ratio': total_throughput, 'accuracy': accuracy} +result_file = os.path.join(RESULT_PATH, "result.log") +with open(result_file, 'w') as f: + json.dump(result, f) + +try: + import ais_utils + ais_utils.set_result("training", "throughput_ratio", total_throughput) + ais_utils.set_result("training", "accuracy", float(accuracy)) + ais_utils.set_result("training", "result", "OK") +except: + sys.exit() diff --git a/ais-bench_workload/src/common/cluster_common.sh b/ais-bench_workload/src/common/cluster_common.sh new file mode 100644 index 0000000..30946c3 --- /dev/null +++ b/ais-bench_workload/src/common/cluster_common.sh @@ -0,0 +1,267 @@ +#!/bin/bash + +SSH="ssh -o StrictHostKeyChecking=no" +SCP="scp -o StrictHostKeyChecking=no" + +ssh_pass() +{ + local node="$1" + local user="$2" + local pd="$3" + local port="$4" + shift 4 + local cmd="$*" + + run_cmd="$node $cmd" + [ "$user" != "" ] && run_cmd="${user}@$run_cmd" + [ "$port" != "" ] && run_cmd="-p $port $run_cmd" + run_cmd="$SSH $run_cmd" + [ "$pd" != "" ] && run_cmd="sshpass -p ${pd} $run_cmd" + # echo "run_cmd:$run_cmd" + $run_cmd || { echo "run sshrun failed node:$node"; return 1; } +} + +scp_pass() +{ + local node="$1" + local user="$2" + local pd="$3" + local port="$4" + local src="$5" + local target="$6" + + run_cmd="${node}:${target}" + [ "$user" != "" ] && run_cmd="${user}@$run_cmd" + run_cmd="-r $src/* ${run_cmd}" + [ "$port" != "" ] && run_cmd="-P $port $run_cmd" + run_cmd="${SCP} ${run_cmd}" + [ "$pd" != "" ] && run_cmd="sshpass -p ${pd} $run_cmd" + # echo "run_cmd:$run_cmd" + $run_cmd || { echo "run scp failed node:$node"; return 1; } +} + +rscp_pass() +{ + local node="$1" + local user="$2" + local pd="$3" + local port="$4" + local src="$5" + local target="$6" + + run_cmd="${node}:${src}/* ${target}" + [ "$user" != "" ] && run_cmd="${user}@$run_cmd" + [ "$port" != "" ] && run_cmd="-P $port $run_cmd" + run_cmd="${SCP} -r ${run_cmd}" + [ "$pd" != "" ] && run_cmd="sshpass -p ${pd} $run_cmd" + # echo "run_cmd:$run_cmd" + $run_cmd || { echo "run rscp failed node:$node"; return 1; } +} + +get_cluster_list() +{ + local cluster_config=$1 + cat ${cluster_config} | python3 -c 'import sys,json;[print(node) for node in json.load(sys.stdin)["cluster"].keys()]' +} + +get_node_user() +{ + local cluster_config=$1 + local node=$2 + cat ${cluster_config} | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster"]['\"${node}\"']["user"])' 2>/dev/null +} + +get_node_pd() +{ + local cluster_config=$1 + local node=$2 + cat ${cluster_config} | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster"]['\"${node}\"']["pd"])' 2>/dev/null +} + +get_node_port() +{ + local cluster_config=$1 + local node=$2 + cat ${cluster_config} | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster"]['\"${node}\"']["port"])' 2>/dev/null +} + +local_run_cmd() +{ + local cmd="$*" + (eval $cmd) || { echo "Warn local run '${cmd}'"; return 1; } +} + +local_scp_cmd() +{ + local src_path="$2" + local dst_path="$3" + + [ -d $src_path ] || { echo "Warn src_path:$src_path not exist return";return 1; } + + # rm and mkdir dst path + # [ -d $dst_path ] && rm -rf $dst_path + mkdir -p $dst_path + + cp -rf $src_path/* $dst_path 2>/dev/null + return 0 +} + +# nodeinfo.json + +# { +# "cluster": { +# "xx.xx.xx.xx": { +# "user": "xxxx", +# "pd": "xx", +# "port": xx, +# }, +# "xx.xx.xx.xx": { +# } +# } +# } + +# interface function + +# 根据参数1串行调用命令 如果失败就返回 +# 参数1: 节点信息json文件，包含节点ip和用户名密码信息 如果为空即是本地调用 +# 参数其他: 运行的命令 +# 样例 cluster_run_cmd_serial "$NODEINFO_FILE" "ifconfig" +cluster_run_cmd_serial() +{ + local node_info_file=$1 + shift 1 + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_run_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + local cmd=$* + local node_arr=($(get_cluster_list ${node_info_file})) + local node_count=${#node_arr[@]} + + for ((i=0; i<$node_count; i++)); do { + local node="${node_arr[$i]}" + local user=$(get_node_user ${node_info_file} ${node}) + local pd=$(get_node_pd ${node_info_file} ${node}) + local port=$(get_node_port ${node_info_file} ${node}) + local cur_cmd="export SERVER_ID=${i}; ${cmd}" + ssh_pass "${node}" "${user}" "${pd}" "$port" "${cur_cmd}" || { echo "node:${node} ERROR when executing '${cur_cmd}'"; return 1; } + } + done + return 0 +} + +# 根据参数1项调用命令，只调用一个节点的命令 +# 参数1: 节点信息json文件，包含节点ip和用户名密码信息 如果为空即是本地调用 +# 参数其他: 运行命令 +# 样例 cluster_run_cmd_single "$NODEINFO_FILE" "ifconfig" +cluster_run_cmd_single() +{ + local node_info_file=$1 + shift 1 + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_run_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + local cmd=$* + local node_arr=($(get_cluster_list ${node_info_file})) + + local node="${node_arr[0]}" + local user=$(get_node_user ${node_info_file} ${node}) + local pd=$(get_node_pd ${node_info_file} ${node}) + local port=$(get_node_port ${node_info_file} ${node}) + local cur_cmd="export SERVER_ID=0; ${cmd}" + ssh_pass "${node}" "${user}" "${pd}" "$port" "${cur_cmd}" || { echo "node:${node} ERROR when executing '${cur_cmd}'"; return 1; } + return 0 +} + +# 根据参数1调用命令，并行运行，等待所有命令执行完 +# 参数1: 节点信息json文件，包含节点ip和用户名密码信息 如果为空即是本地调用 +# 参数其他: 运行命令 +# 样例 cluster_run_cmd_parallel "$NODEINFO_FILE" "ifconfig" +cluster_run_cmd_parallel() +{ + local node_info_file=$1 + shift 1 + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_run_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + local cmd=$* + local node_arr=($(get_cluster_list ${node_info_file})) + local node_count=${#node_arr[@]} + + local retvalfile=$(mktemp) + for ((i=0; i<$node_count; i++)); do { + local node="${node_arr[$i]}" + local user=$(get_node_user ${node_info_file} ${node}) + local pd=$(get_node_pd ${node_info_file} ${node}) + local port=$(get_node_port ${node_info_file} ${node}) + local cur_cmd="export SERVER_ID=${i}; $cmd" + ssh_pass "${node}" "${user}" "${pd}" "$port" ${cur_cmd} || { echo "node:${node} ERROR when executing '${cur_cmd}'"; rm -rf $retvalfile;} + } & + done + wait + [ -f $retvalfile ] || { echo "run train failed";return 1; } + rm -rf $retvalfile +} + +# 根据参数1拷贝主节点文件夹内容到各运行节点中 +# 注意 实际拷贝命令为 cp src_path/* dst_path 且dst_path会执行删除然后重建 +# 参数1: 节点信息json文件，包含节点ip和用户名密码信息 如果为空即是本地调用 +# 参数2: 主节点的源路径 src_path +# 参数3: 运行节点的源路径 dst_path +# 样例 cluster_scp "${NODEINFO_FILE}" "/home/src" "/home/dst" +cluster_scp() +{ + local node_info_file=$1 + local src_path=$2 + local dst_path=$3 + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_scp_cmd "$@";return $?; } + + local node_arr=($(get_cluster_list ${node_info_file})) + local node_count=${#node_arr[@]} + + for ((i=0; i<$node_count; i++)); do { + local node="${node_arr[$i]}" + local user=$(get_node_user ${node_info_file} ${node}) + local pd=$(get_node_pd ${node_info_file} ${node}) + local port=$(get_node_port ${node_info_file} ${node}) + scp_pass "${node}" "${user}" "${pd}" "$port" "${src_path}" "${dst_path}" || { echo "scp_pass failed node:$node"; return 1; } + echo "------------scp done------${user}@${node}---------------------" + } done +} + +# 根据参数1拷贝各运行节点文件内容到主节点文件夹中 +# 注意 实际拷贝命令为 cp src_path/* dst_path 且dst_path会创建 +# 参数1: 节点信息json文件，包含节点ip和用户名密码信息 如果为空即是本地调用 +# 参数2: 运行节点的源路径 src_path +# 参数3: 主节点的源路径 dst_path +# 样例 cluster_rscp "${NODEINFO_FILE}" "/home/src" "/home/dst" +cluster_rscp() +{ + local node_info_file=$1 + local src_path="$2" + local dst_path="$3" + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_scp_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + + local node_arr=($(get_cluster_list ${node_info_file})) + local node_count=${#node_arr[@]} + + for ((i=0; i<$node_count; i++)); do { + local node="${node_arr[$i]}" + local user=$(get_node_user ${node_info_file} ${node}) + local pd=$(get_node_pd ${node_info_file} ${node}) + local port=$(get_node_port ${node_info_file} ${node}) + echo "------------------${user}@${node}---------------------" + rscp_pass "${node}" "${user}" "${pd}" "$port" "${src_path}" "${dst_path}" || { echo "sshpass_rscp failed node:$node"; return 1; } + } done +} + diff --git a/ais-bench_workload/src/common/cluster_common_2.0.sh b/ais-bench_workload/src/common/cluster_common_2.0.sh new file mode 100644 index 0000000..71f2ff2 --- /dev/null +++ b/ais-bench_workload/src/common/cluster_common_2.0.sh @@ -0,0 +1,128 @@ +#!bin/bash + +declare -i ret_ok=0 +declare -i ret_failed=1 + +local_run_cmd() +{ + local cmd="$1" + (eval "$cmd") || { echo "Warn local run '${cmd}'"; return $ret_failed; } +} + +local_cp_cmd() +{ + local src_path="$1" + local dst_path="$2" + if [ -f $src_path ]; then + cp -f $src_path $dst_path || { echo "cp: $src_path to $dst_path failed!";return $ret_failed; } + elif [ -d $src_path ]; then + cp -rf $src_path $dst_path || { echo "cp: $src_path to $dst_path failed!";return $ret_failed; } + else + echo "Warn src_path:$src_path not exist return" + return $ret_failed + fi + return $ret_ok +} + +local_put_cmd() +{ + local src_path="$1" + local dst_path=$WORK_PATH/"$2" + local_cp_cmd $src_path $dst_path || { return $ret_failed; } + return $ret_ok +} + +local_get_cmd() +{ + local src_path=$WORK_PATH/"$1" + local dst_path="$2" + local_cp_cmd $src_path $dst_path || { return $ret_failed; } + return $ret_ok +} + +cluster_init() +{ + [ "$NODEINFO_FILE" == "" ] && { echo "NODEINFO_FILE not set, will not use cluster";return $ret_ok; } + if [ -n "$CLUSTER_SSH_KEY_PATH" ];then + $PYTHON_COMMAND -m ais_bench.cluster init -n $NODEINFO_FILE -s $CLUSTER_SSH_KEY_PATH || { return $ret_failed; } + elif [ $CLUSTER_AUTO_SET_KEY == 'on' ];then + $PYTHON_COMMAND -m ais_bench.cluster init -n $NODEINFO_FILE -a || { return $ret_failed; } + else + return $ret_failed + fi + return $ret_ok +} + +cluster_multi_exec() +{ + [ "$NODEINFO_FILE" == "" ] && { local_run_cmd "$@";return $?; } + local cmd="$1" + local mode="$2" + local device_num="$3" + _cmd="$PYTHON_COMMAND -m ais_bench.cluster multi_exec -c '$cmd' " + [ "$mode" != "" ] && _cmd="$_cmd -m $mode " + [ "$device_num" != "" ] && _cmd="$_cmd -d $device_num " + (eval "$_cmd") || { return $ret_failed; } + return $ret_ok +} + +cluster_single_exec() +{ + [ "$NODEINFO_FILE" == "" ] && { local_run_cmd "$@";return $?; } + local cmd="$1" + local node_id="$2" + local device_num="$3" + _cmd="$PYTHON_COMMAND -m ais_bench.cluster single_exec -c '$cmd' " + [ "$node_id" != "" ] && _cmd="$_cmd -m $node_id " + [ "$device_num" != "" ] && _cmd="$_cmd -d $device_num " + (eval "$_cmd") || { return $ret_failed; } + return $ret_ok +} + +cluster_multi_put() +{ + [ "$NODEINFO_FILE" == "" ] && { local_put_cmd "$@";return $?; } + local src="$1" + local dst="$2" + local mode="$3" + _cmd="$PYTHON_COMMAND -m ais_bench.cluster multi_put -s $src -d $dst " + [ "$mode" != "" ] && _cmd="$_cmd -m $mode " + (eval "$_cmd") || { return $ret_failed; } + return $ret_ok +} + +cluster_single_put() +{ + [ "$NODEINFO_FILE" == "" ] && { local_put_cmd "$@";return $?; } + local src="$1" + local dst="$2" + local mode="$3" + _cmd="$PYTHON_COMMAND -m ais_bench.cluster single_put -s $src -d $dst " + [ "$node_id" != "" ] && _cmd="$_cmd -m $node_id " + (eval "$_cmd") || { return $ret_failed; } + return $ret_ok +} + +cluster_multi_get() +{ + [ "$NODEINFO_FILE" == "" ] && { local_get_cmd "$@";return $?; } + local src="$1" + local dst="$2" + local mode="$3" + _cmd="$PYTHON_COMMAND -m ais_bench.cluster multi_get -s $src -d $dst " + [ "$mode" != "" ] && _cmd="$_cmd -m $mode " + (eval "$_cmd") || { return $ret_failed; } + return $ret_ok +} + +cluster_single_get() +{ + [ "$NODEINFO_FILE" == "" ] && { local_get_cmd "$@";return $?; } + local src="$1" + local dst="$2" + local mode="$3" + _cmd="$PYTHON_COMMAND -m ais_bench.cluster single_get -s $src -d $dst " + [ "$node_id" != "" ] && _cmd="$_cmd -m $node_id " + (eval "$_cmd") || { return $ret_failed; } + return $ret_ok +} \ No newline at end of file diff --git a/ais-bench_workload/src/common/common.sh b/ais-bench_workload/src/common/common.sh new file mode 100644 index 0000000..4676c73 --- /dev/null +++ b/ais-bench_workload/src/common/common.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +check_file_valid() +{ + if [ ! -f "$1" ]; then + return 1 + fi + return 0 +} + +check_path_valid() +{ + if [ ! -d "$1" ]; then + return 1 + fi + return 0 +} + +function check_command_exist() +{ + command=$1 + if type ${command} > /dev/null 2>&1; then + return 0 + else + return 1 + fi +} + +check_python_package_is_install() +{ + local PYTHON_COMMAND=$1 + ${PYTHON_COMMAND} -c "import $2" >> /dev/null 2>&1 + ret=$? + if [ $ret != 0 ]; then + echo "python package:$1 not install" + return 1 + fi + return 0 +} + +check_mindspore_run_ok() +{ + local PYTHON_COMMAND=$1 + ${PYTHON_COMMAND} -c "import mindspore;mindspore.run_check()" >> /dev/null 2>&1 + ret=$? + if [ $ret != 0 ]; then + echo "mindspore run not ok" + return 1 + fi +} + +check_mindspore_run_ok_Ascend() +{ + local PYTHON_COMMAND=$1 + ${PYTHON_COMMAND} -c "import mindspore;mindspore.set_context(device_target='Ascend');mindspore.run_check()" >> /dev/null 2>&1 + ret=$? + if [ $ret != 0 ]; then + echo "mindspore run not ok" + return 1 + fi +} \ No newline at end of file diff --git a/ais-bench_workload/src/common/log_util.sh b/ais-bench_workload/src/common/log_util.sh new file mode 100644 index 0000000..7f58b68 --- /dev/null +++ b/ais-bench_workload/src/common/log_util.sh @@ -0,0 +1,60 @@ +#!/bin/bash +MODE_NAME="Ais-Bench-Stubs" + +date_format="+%Y-%m-%dT%T" + +# Some useful colors. +# check if stdout is a terminal and support colors... +if [ -t 1 ] && [ "1$(tput colors 2>/dev/null)" -ge 18 ]; then + readonly color_red="$(tput setaf 1)" + readonly color_yellow="$(tput setaf 3)" + readonly color_green="$(tput setaf 2)" + readonly color_norm="$(tput sgr0)" +else + readonly color_red="" + readonly color_yellow="" + readonly color_green="" + readonly color_norm="" +fi + +if command -v caller >/dev/null 2>&1; then + # return func(lineno:filename) + # NOTE: skip 2-level inner frame + _caller() { caller 2| awk '{sub(/.*\//,e,$3);print $2"("$3":"$1") "}'; } +else + _caller() { :; } +fi + +_log() +{ + level=$1 + shift 1 + echo "$(date ${date_format}) -${MODE_NAME}- ${level} $(_caller)- $*" +} + + +logger_Debug() +{ + echo "Debug $(_caller): $@" +} + +logger_Info() +{ + _log INFO "$@" +} + +logger_Warn() +{ + _log WARN "${color_yellow}$*${color_norm}" +} + +logger_Error() +{ + _log ERROR "${color_red}$*${color_norm}" +} + +die() +{ + _log ERROR "${color_red}$*${color_norm}" + exit 1 +} \ No newline at end of file diff --git a/ais-bench_workload/src/common/modelarts_handler.py b/ais-bench_workload/src/common/modelarts_handler.py new file mode 100644 index 0000000..e2be624 --- /dev/null +++ b/ais-bench_workload/src/common/modelarts_handler.py @@ -0,0 +1,269 @@ +from re import S +from urllib.parse import urlparse +from obs import ObsClient, model +from modelarts.session import Session +from modelarts.estimator import JOB_STATE, Estimator +import time +import os + +import logging +logging.basicConfig(level = logging.DEBUG,format = '[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + + +def get_config_value(config, key): + return None if config.get(key) == "" else config.get(key) + +def continue_waiting(job_info): + print("waiting for task, status %s, total time: %d(s)" % (JOB_STATE[job_info['status']], job_info['duration'] / 1000)) + +def exit_by_failure(job_info): + print("task failed, status %s, please check log on obs, exit" % (JOB_STATE[job_info['status']])) + raise RuntimeError('failed') + +func_table = { + 0: continue_waiting, + 1: continue_waiting, + 2: continue_waiting, + 3: exit_by_failure, + 4: continue_waiting, + 5: exit_by_failure, + 6: exit_by_failure, + 7: continue_waiting, + 8: continue_waiting, + 9: exit_by_failure, + 11: exit_by_failure, + 12: exit_by_failure, + 13: exit_by_failure, + 14: exit_by_failure, + 15: continue_waiting, + 16: exit_by_failure, + 17: exit_by_failure, + 18: continue_waiting, + 19: continue_waiting, + 20: continue_waiting, + 21: exit_by_failure, + 22: exit_by_failure +} + +# 调试需要 超时后停止 +def wait_for_job_timeout(job_instance): + count = 0 + while True: + time.sleep(10) + job_info = job_instance.get_job_info() + if job_info['status'] == 10: + print("task succeeded, total time %d(s)" % (job_info['duration'] / 1000)) + break + func_table[job_info['status']](job_info) + count = count + 1 + print("modelarts run time count:{}".format(count)) + if count == 6: + print("modelarts run match:{} 10 so exit >>>>>>>".format(count)) + status = job_instance.stop_job_version() + #status = job_instance.delete_job() + raise RuntimeError('failed') + break + +try: + import moxing as mox + moxing_import_flag = True +except: + moxing_import_flag = False + +class modelarts_handler(): + def __init__(self): + self.output_url = None + self.job_log_prefix = None + + def sync_job_log(self, session_config): + dstpath = os.path.join(os.getenv("BASE_PATH", "./"), "log") + if not os.path.exists(dstpath): + print("dstpath:{} not exist no get log") + return + for id in range(session_config.train_instance_count): + logurl = self.job_log_prefix + '-' + str(id) + '.log' + logname = os.path.basename(logurl) + logpath = os.path.join(dstpath, logname) + if self.session.obs.is_obs_path_exists(logurl): + self.session.obs.download_file(logurl, logpath) + #print("logurl:{} sync log to dstpath:{}".format(logurl, logpath)) + + def wait_for_job(self, job_instance, session_config): + count = 0 + while True: + time.sleep(10) + count = count + 1 + if count > 10: + count = 10 + self.sync_job_log(session_config) + job_info = job_instance.get_job_info() + if job_info['status'] == 10: + self.sync_job_log(session_config) + print("task succeeded, total time %d(s)" % (job_info['duration'] / 1000)) + break + func_table[job_info['status']](job_info) + + def create_obs_output_dirs(self, output_url): + if moxing_import_flag == True: + dstpath = output_url.replace("s3:", "obs:", 1) + logger.info("create obs outdir mox mkdir:{}".format(dstpath)) + mox.file.make_dirs(dstpath) + else: + bucket_name = output_url[5:].split('/')[0] + sub_dir = output_url.replace(f"s3://{bucket_name}/", "", 1) + logger.debug('create obs output{} subdir:{} bucket:{}'.format(output_url, sub_dir, bucket_name)) + resp = self.obsClient.putContent(bucket_name, sub_dir, content=None) + if resp.status < 300: + logger.debug('obs put content request ok') + else: + logger.warn('errorCode:{} msg:{}'.format(resp.errorCode, resp.errorMessage)) + raise RuntimeError('failed') + + def create_obs_handler(self, access_config): + if moxing_import_flag == False: + # 创建 obs登录句柄 + self.obsClient = ObsClient(access_key_id=access_config.access_key, + secret_access_key=access_config.secret_access_key, server=access_config.server) + + def create_session(self, access_config): + # 如下配置针对计算中心等专有云 通用云不需要设置 + if access_config.get("iam_endpoint") != "" and access_config.get("iam_endpoint") != None \ + and access_config.get("obs_endpoint") != "" and access_config.get("obs_endpoint") != None \ + and access_config.get("modelarts_endpoint") != "" and access_config.get("modelarts_endpoint") != None: + Session.set_endpoint(iam_endpoint=access_config.iam_endpoint, obs_endpoint=access_config.obs_endpoint, \ + modelarts_endpoint=access_config.modelarts_endpoint, region_name=access_config.region_name) + # 创建modelarts句柄 + self.session = Session(access_key=access_config.access_key, + secret_key=access_config.secret_access_key, + project_id=access_config.project_id, + region_name=access_config.region_name) + + def print_train_instance_types(self): + algo_info = Estimator.get_train_instance_types(modelarts_session=self.session) + print("get valid train_instance_types:{}".format(algo_info)) + + def stop_new_versions(self, session_config): + base_job_list_info = Estimator.get_job_list(modelarts_session=self.session, per_page=10, page=1, order="asc", search_content=session_config.job_name) + if base_job_list_info == None or base_job_list_info.get("job_total_count", 0) == 0: + print("find no match version return") + return + else: + pre_version_id = base_job_list_info["jobs"][0].get("version_id") + job_id = base_job_list_info["jobs"][0].get("job_id") + job_status = base_job_list_info["jobs"][0].get("status") + estimator = Estimator(modelarts_session=self.session, job_id=job_id, version_id=pre_version_id) + if JOB_STATE[job_status] == "JOBSTAT_INIT" \ + or JOB_STATE[job_status] == "JOBSTAT_IMAGE_CREATING" \ + or JOB_STATE[job_status] == "JOBSTAT_SUBMIT_TRYING" \ + or JOB_STATE[job_status] == "JOBSTAT_DEPLOYING" \ + or JOB_STATE[job_status] == "JOBSTAT_WAITING" \ + or JOB_STATE[job_status] == "JOBSTAT_RUNNING": + status = estimator.stop_job_version() + print("jobname:{} jobid:{} preversionid:{} jobstatus:{} stop status:{}".format( + session_config.job_name, job_id, pre_version_id, JOB_STATE[job_status], status)) + else: + print("jobname:{} jobid:{} preversionid:{} jobstatus:{} no need stop".format( + session_config.job_name, job_id, pre_version_id, JOB_STATE[job_status])) + return + + def get_job_name_next_new_version(self, session_config): + base_job_list_info = Estimator.get_job_list(modelarts_session=self.session, per_page=10, page=1, order="asc", search_content=session_config.job_name) + if base_job_list_info == None or base_job_list_info.get("job_total_count", 0) == 0: + return 1 + else: + pre_version_id = base_job_list_info["jobs"][0].get("version_id") + job_id = base_job_list_info["jobs"][0].get("job_id") + estimator = Estimator(modelarts_session=self.session, job_id=job_id, version_id=pre_version_id) + job_info = estimator.get_job_info() + pre_version_id = job_info.get("version_name", "V0")[1:] + return int(pre_version_id)+1 + + def get_obs_url_content(self, obs_url): + if moxing_import_flag == True: + dsturl = obs_url.replace("s3:", "obs:", 1) + with mox.file.File(dsturl, 'r') as f: + file_str = f.read() + return file_str + else: + bucket_name = obs_url[5:].split('/')[0] + obs_sub_path = obs_url.replace(f"s3://{bucket_name}/", "", 1) + resp = self.obsClient.getObject(bucket_name, obs_sub_path, loadStreamInMemory=True) + if resp.status < 300: + logger.debug('request ok') + return resp.body.buffer.decode("utf-8") + else: + raise RuntimeError('obs get object ret:{} url:{} bucket:{} path:{}'.format(resp.status, obs_url, bucket_name, obs_sub_path)) + + + def update_code_to_obs(self, session_config, localpath): + # 待完善 验证 + if moxing_import_flag == True: + dstpath = "obs:/" + session_config.code_dir + logger.info("mox update loaclpath:{} dstpath:{}".format(localpath, dstpath)) + mox.file.copy_parallel(localpath, dstpath) + else: + bucket_name = session_config.code_dir.split('/')[1] + sub_dir = "/".join(session_config.code_dir.strip("/").split('/')[1:]) + logger.info("update code localpath:{} codepath:{} bucket:{} subdir:{}".format( + localpath, session_config.code_dir, bucket_name, sub_dir)) + resp = self.obsClient.putFile(bucket_name, sub_dir, localpath) + + def create_modelarts_job(self, session_config, output_url): + jobdesc = session_config.job_description_prefix + "_jobname_" + session_config.job_name + "_" + str(session_config.train_instance_type) + "_" + str(session_config.train_instance_count) + estimator = Estimator(modelarts_session=self.session, + framework_type=session_config.framework_type, + framework_version=session_config.framework_version, + code_dir=session_config.code_dir, + boot_file=session_config.boot_file, + log_url=output_url[4:], + hyperparameters=session_config.hyperparameters, + output_path=output_url[4:], + pool_id = get_config_value(session_config, "pool_id"), + train_instance_type = get_config_value(session_config, "train_instance_type"), + train_instance_count=session_config.train_instance_count, + nas_type = get_config_value(session_config, "nas_type"), + nas_share_addr = get_config_value(session_config, "nas_share_addr"), + nas_mount_path = get_config_value(session_config, "nas_mount_path"), + job_description=jobdesc, + user_command = None) + + base_job_list_info = Estimator.get_job_list(modelarts_session=self.session, per_page=10, page=1, order="asc", search_content=session_config.job_name) + if base_job_list_info == None or base_job_list_info.get("job_total_count", 0) == 0: + logger.debug("new create inputs:{} job_name:{}".format(session_config.inputs, session_config.job_name)) + job_instance = estimator.fit(inputs=session_config.inputs, wait=False, job_name=session_config.job_name) + else: + job_id = base_job_list_info["jobs"][0].get("job_id") + pre_version_id = base_job_list_info["jobs"][0].get("version_id") + logger.debug("new versions job_id:{} pre_version_id:{}".format(job_id, pre_version_id)) + job_instance = estimator.create_job_version(job_id=job_id, pre_version_id=pre_version_id, inputs=session_config.inputs, wait=False, job_desc=jobdesc) + + print("inputs:{} job_name:{} ret instance:{}".format(session_config.inputs, session_config.job_name, job_instance)) + job_info = job_instance.get_job_info() + if not job_info['is_success']: + logger.error("failed to run job on modelarts, msg %s" % (job_info['error_msg'])) + raise RuntimeError('failed') + + self.job_log_prefix = "obs:/" + output_url[4:] + job_info["resource_id"] + "-job-" + session_config.job_name + + print("create sucess job_id:{} resource_id:{} version_name:{} create_time:{}".format( + job_info["job_id"], job_info["resource_id"], job_info["version_name"], job_info["create_time"])) + return job_instance + + def run_job(self, session_config, localpath): + logger.debug("session config:{}".format(session_config)) + + self.print_train_instance_types() + + # 获取job_name的next 版本号 + next_version_id = self.get_job_name_next_new_version(session_config) + # 生成输出路径 + self.output_url = os.path.join("s3:/{}".format(session_config.out_base_url), "V{}".format(next_version_id), "") + logger.debug("output_url:{}".format(self.output_url)) + self.create_obs_output_dirs(self.output_url) + + # 更新代码到obs上 + self.update_code_to_obs(session_config, localpath) + + job_instance = self.create_modelarts_job(session_config, self.output_url) + self.wait_for_job(job_instance, session_config) \ No newline at end of file diff --git a/ais-bench_workload/src/common/modelarts_handler_v2.py b/ais-bench_workload/src/common/modelarts_handler_v2.py new file mode 100644 index 0000000..d37b605 --- /dev/null +++ b/ais-bench_workload/src/common/modelarts_handler_v2.py @@ -0,0 +1,201 @@ +import logging +import os +import time + +from modelarts.estimatorV2 import JOB_STATE, Estimator +from modelarts.session import Session +from modelarts.train_params import InputData, OutputData, TrainingFiles +from obs import ObsClient + +logging.basicConfig(level=logging.DEBUG, format='[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + + +def get_config_value(config, key): + return None if config.get(key) == "" else config.get(key) + + +try: + import moxing as mox + moxing_import_flag = True +except Exception: + moxing_import_flag = False + + +class modelarts_handler(): + RESP_OK = 300 + OBS_PATH_HEAD = "obs:/" + + def __init__(self): + self.output_url = None + self.job_log_prefix = None + self.job_name = None + self.job_instance = None + self.session_config = None + self.bucket_name = None + + def sync_job_log(self, session_config): + dstpath = os.path.join(os.getenv("BASE_PATH", "./"), "log") + if not os.path.exists(dstpath): + os.makedirs(dstpath) + for id in range(session_config.train_instance_count): + logurl = self.job_log_prefix + '-' + str(id) + '.log' + logname = os.path.basename(logurl) + logpath = os.path.join(dstpath, logname) + if self.session.obs.is_obs_path_exists(logurl): + self.session.obs.download_file(logurl, logpath) + + def wait_for_job(self): + count = 0 + while True: + time.sleep(10) + count = count + 1 + if count % 10 == 0: + self.sync_job_log(self.session_config) + job_info = self.job_instance.get_job_info() + + phase = job_info['status']['phase'] + if phase == "Completed": + self.sync_job_log(self.session_config) + logger.info("task succeeded, total time %d(s)" % (job_info['status']['duration'] / 1000)) + break + elif phase in ['Failed', 'Abnormal', 'Terminated']: + print("task failed, phase %s, please check log on obs, exit" % (job_info['status']['phase'])) + raise RuntimeError('job failed') + else: + print("waiting for task, phase %s, total time: %d(s), actual training time: %d(s) " + % (job_info['status']['phase'], 10 * count, job_info['status']['duration'] / 1000)) + + def create_obs_output_dirs(self, output_url): + if moxing_import_flag: + dstpath = self.OBS_PATH_HEAD + output_url + logger.info("create obs outdir mox mkdir:{}".format(dstpath)) + mox.file.make_dirs(dstpath) + else: + sub_dir = output_url.replace(f"/{self.bucket_name}/", "", 1) + logger.debug('create obs output{} subdir:{} bucket:{}'.format(output_url, sub_dir, self.bucket_name)) + resp = self.obsClient.putContent(self.bucket_name, sub_dir, content=None) + if resp.status < self.RESP_OK: + logger.debug('obs put content request ok') + else: + logger.warn('create obs folder failed. errorCode:{} msg:{}'.format(resp.errorCode, resp.errorMessage)) + raise RuntimeError('create obs folder failed') + + def create_obs_handler(self, access_config): + if not moxing_import_flag: + # Create OBS login handle + self.obsClient = ObsClient(access_key_id=access_config.access_key, + secret_access_key=access_config.secret_access_key, server=access_config.server) + + def create_session(self, access_config): + # 如下配置针对计算中心等专有云 通用云不需要设置 + if access_config.get("iam_endpoint") != "" and access_config.get("iam_endpoint") is not None \ + and access_config.get("obs_endpoint") != "" and access_config.get("obs_endpoint") is not None \ + and access_config.get("modelarts_endpoint") != "" and access_config.get("modelarts_endpoint") is not None: + Session.set_endpoint(iam_endpoint=access_config.iam_endpoint, obs_endpoint=access_config.obs_endpoint, + modelarts_endpoint=access_config.modelarts_endpoint, + region_name=access_config.region_name) + # Create modelars handle + self.session = Session(access_key=access_config.access_key, + secret_key=access_config.secret_access_key, + project_id=access_config.project_id, + region_name=access_config.region_name) + + def print_train_instance_types(self): + algo_info = Estimator.get_train_instance_types(self.session) + print("get valid train_instance_types:{}".format(algo_info)) + + def stop_job(self, job_id): + job_info = Estimator.control_job_by_id(session=self.session, job_id=job_id) + print("job stop status: {}".format(job_info["status"]["phase"])) + + def get_obs_url_content(self, obs_url): + if moxing_import_flag: + dsturl = self.OBS_PATH_HEAD + obs_url + with mox.file.File(dsturl, 'r') as f: + file_str = f.read() + return file_str + else: + obs_sub_path = obs_url.replace(f"/{self.bucket_name}/", "", 1) + resp = self.obsClient.getObject(self.bucket_name, obs_sub_path, loadStreamInMemory=True) + if resp.status < self.RESP_OK: + logger.debug('request ok') + return resp.body.buffer.decode("utf-8") + else: + raise RuntimeError('obs get object ret:{} url:{} bucket:{} \ + path:{}'.format(resp.status, obs_url, self.bucket_name, obs_sub_path)) + + def update_code_to_obs(self, localpath): + if moxing_import_flag: + dstpath = self.OBS_PATH_HEAD + self.session_config.code_dir + logger.info("mox update loaclpath:{} dstpath:{}".format(localpath, dstpath)) + mox.file.copy_parallel(localpath, dstpath) + else: + sub_dir = "/".join(self.session_config.code_dir.strip("/").split('/')[1:]) + logger.info("update code localpath:{} codepath:{} bucket:{} subdir:{}".format( + localpath, self.session_config.code_dir, self.bucket_name, sub_dir)) + print("bucket_name:{} sub_dir: {} localpath:{}".format(self.bucket_name, sub_dir, localpath)) + self.obsClient.putFile(self.bucket_name, sub_dir, localpath) + + def create_modelarts_job(self, output_url): + jobdesc = self.session_config.job_description_prefix + "_jobname_" + self.job_name + "_" +\ + str(self.session_config.train_instance_type) + "_" + str(self.session_config.train_instance_count) + + output_list = [OutputData(obs_path=self.OBS_PATH_HEAD + self.session_config.out_base_url + self.job_name + "/", + name="train_url")] + + estimator = Estimator(session=self.session, + framework_type=self.session_config.framework_type, + framework_version=self.session_config.framework_version, + training_files=TrainingFiles(code_dir=self.OBS_PATH_HEAD + self.session_config.code_dir, + boot_file=self.OBS_PATH_HEAD + self.session_config.boot_file), + log_url=self.OBS_PATH_HEAD + output_url, + parameters=self.session_config.parameters, + outputs=output_list, + pool_id=get_config_value(self.session_config, "pool_id"), + train_instance_type=get_config_value(self.session_config, "train_instance_type"), + train_instance_count=self.session_config.train_instance_count, + job_description=jobdesc, + user_command=None) + + logger.debug("new create inputs:{} job_name:{}".format(self.session_config.inputs, self.job_name)) + inut_list = [InputData(obs_path=self.OBS_PATH_HEAD + self.session_config.inputs, name="data_url")] + try: + job_instance = estimator.fit(inputs=inut_list, wait=False, job_name=self.job_name) + except Exception as e: + logger.error("failed to create job on modelarts, msg %s" % (e)) + raise RuntimeError('creat job failed') + + logger.debug("inputs:{} job_name:{} ret instance:{}".format(inut_list, self.job_name, job_instance)) + job_info = job_instance.get_job_info() + print("\njob_info: {}\n".format(job_info)) + + if 'error_msg' in job_info.keys(): + logger.error("failed to run job on modelarts, error_msg: %s error_code:\ + %s error_solution: %s" % (job_info['error_msg'], job_info['error_code'], job_info['error_solution'])) + raise RuntimeError('creat job failed') + + self.job_log_prefix = self.OBS_PATH_HEAD + output_url + "modelarts-job-" + job_info['metadata']['id'] + '-worker' + print("create job sucess. job_id:{} job name:{} create_time:{} job_log_prefix:{}".format( + job_info["metadata"]["id"], job_info["metadata"]["name"], job_info["metadata"]["create_time"], + self.job_log_prefix)) + + return job_instance + + def run_job(self, session_config, localpath): + logger.debug("session config:{}".format(self.session_config)) + timestr = time.strftime("%Y_%m_%d-%H_%M_%S") + self.session_config = session_config + self.job_name = self.session_config.job_name + timestr + self.print_train_instance_types() + # modelarts path end with '/'，or report error ModelArts.2791 + self.output_url = os.path.join(self.session_config.out_base_url, self.job_name, "") + self.bucket_name = self.session_config.out_base_url.split('/')[1] + logger.debug("output_url:{}".format(self.output_url)) + self.create_obs_output_dirs(self.output_url) + + # update code to obs + self.update_code_to_obs(localpath) + + self.job_instance = self.create_modelarts_job(self.output_url) + self.wait_for_job() diff --git a/ais-bench_workload/src/common/node_common.sh b/ais-bench_workload/src/common/node_common.sh new file mode 100644 index 0000000..d00be79 --- /dev/null +++ b/ais-bench_workload/src/common/node_common.sh @@ -0,0 +1,113 @@ + +get_node_podname() +{ + local rank_table_file=$1 + local server_id=$2 + cat ${rank_table_file} | python3 -c 'import sys,json;print(json.load(sys.stdin)["group_list"][0]["instance_list"]['${server_id}']["pod_name"])' #2>/dev/null +} + +# 通用检测 在主节点上检测环境是否正常 +check_env_common() +{ + : "${RANK_SIZE?RANK_SIZE not set}" + : "${DEVICE_NUM?DEVICE_NUM not set}" + + # check ranktable set + [[ $RANK_SIZE -eq 1 ]] || : "${RANK_TABLE_FILE?RANK_TABLE_FILE not set}" + [[ $RANK_SIZE -eq 1 ]] && [[ -n "$RANK_TABLE_FILE" ]] && { echo "ranksize=1 should not set RANK_TABLE_FILE";return 1; } + + : "${PYTHON_COMMAND?PYTHON_COMMAND not set}" + + # check nodeinfofile exist + # [[ $RANK_SIZE -le 8 ]] || check_file_valid "${NODEINFO_FILE}" || { echo "nodeinfofile:${NODEINFO_FILE} not valid" ; return 1; } + + # check basic command of the main node + if [ -f "$NODEINFO_FILE" ];then + check_command_exist ssh || { echo "ssh running failed" ; return 1; } + echo "ssh running successfully" + check_command_exist sshpass || { echo "sshpass running failed" ; return 1; } + echo "sshpass running successfully" + fi + return 0 +} + +# 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE +node_common_check() +{ + local pythoncmd="$1" + local ranksize="$2" + local ranktable="$3" + check_command_exist ${pythoncmd} || { logger_Warn "python:$pythoncmd running failed" ; return 1; } + echo "${pythoncmd} running successfully" + + if [ ${ranksize} != 1 ]; then + check_file_valid "$ranktable" || { logger_Warn "RANK_TABLE_FILE:${ranktable} not valid path" ; return 1; } + echo "RANK_TABLE_FILE path valid" + fi + return 0 +} + +# 通用训练函数调用 +# 必须依赖变量包括 WORK_PATH get_train_cmd 会做检查 +# 参数1 是否绑核 如果需要 传入 "true" +# 参数2 是否老的ranktable 如果需要 传入 "true" + +function node_common_train() +{ + [ -d "$WORK_PATH" ] || { echo "not exit WORK_PATH return";return 1; } + [ "$(type -t get_train_cmd)" == 'function' ] || { echo "not exist get_train_cmd func return";return 1; } + + bindcore=$([ "$1" == "true" ] && echo "true" || echo "false") + oldranktable=$([ "$2" == "true" ] && echo "true" || echo "false") + + # get server node id default is 0 + : "${SERVER_ID:=0}" + # get rank start index + if [[ $DEVICE_NUM == 1 && $RANK_SIZE == 1 ]];then + : "${SINGLE_CARD_INDEX:=0}" + RANK_START=$SINGLE_CARD_INDEX + else + # get rank start index + RANK_START=`expr ${SERVER_ID} \* $DEVICE_NUM` + fi + # set bind core + [ $bindcore == "true" ] && { cpus=`cat /proc/cpuinfo| grep "processor"| wc -l`; avg=`expr $cpus \/ $DEVICE_NUM`; gap=`expr $avg \- 1`; } + if [ $oldranktable == "true" ];then + podname=$(get_node_podname ${RANK_TABLE_FILE} ${SERVER_ID}) + fi + + retvalfile=$(mktemp) + for((i=0;i<${DEVICE_NUM};i++));do + { + index=$[i+RANK_START] + export DEVICE_ID=${i} + # old ranktable should set DEVICE_INDEX and new ranktable should set RANK_ID + if [ $oldranktable == "true" ];then + export DEVICE_INDEX=$[i+RANK_START] + export RANK_ID=$podname + else + export RANK_ID=$[i+RANK_START] + # 应该是device_id吧 + export ASCEND_DEVICE_ID=$DEVICE_ID + export DEVICE_INDEX=$DEVICE_ID + export RANK_INDEX=$SERVER_ID + fi + # clear and create path. + RUN_PATH="$WORK_PATH/train_parallel$index" + mkdir -p $RUN_PATH; cd $RUN_PATH; + # if bindcore should get cmdopt for cores + [ $bindcore == "true" ] && { start=`expr $i \* $avg`; end=`expr $start \+ $gap`; cmdopt=$start"-"$end; } + # call out func get run cmd + get_train_cmd + logger_Info "start training for SERVER_ID:$SERVER_ID rank $index, device $DEVICE_ID begin cmd:$train_run_cmd" + # if bindcore add taskset + [ $bindcore == "true" ] && train_run_cmd="taskset -c $cmdopt $train_run_cmd" + # call cmd + eval $train_run_cmd | tee -a $RUN_PATH/train.log 2>&1 || { logger_Warn "train failed rank $index, device $DEVICE_ID failed:$?"; rm -rf $retvalfile; } + } & + done + logger_Info "Waiting for the training process of SERVER_ID:${SERVER_ID} to finish" + wait + [ -f $retvalfile ] || { logger_Warn "run train failed";return 1; } + logger_Info "SERVER_ID:${SERVER_ID} training finished" +} diff --git a/ais-bench_workload/src/common/patch_common.sh b/ais-bench_workload/src/common/patch_common.sh new file mode 100644 index 0000000..107df9a --- /dev/null +++ b/ais-bench_workload/src/common/patch_common.sh @@ -0,0 +1,67 @@ + +get_modelzoo_base_code_by_git(){ + [ -z $git_url ] && { echo "args git_url not exist";return 1; } + [ -z $branch ] && { echo "args branch not exist";return 1; } + [ -z $modelzoo_sub_dir ] && { echo "args modelzoo_sub_dir not exist";return 1; } + [ -z $commitid ] && { echo "args commitid not exist";return 1; } + + git clone $git_url -b $branch || { echo "warn git clone failed"; return 1; } + code_dir=${modelzoo_sub_dir%%/*} + + cd ${code_dir} + git reset --hard $commitid || { echo "warn git reset failed"; return 1; } + cd - +} + +make_patch(){ + [ -z $BUILD_TMP_PATH ] && { echo "args BUILD_TMP_PATH not exist";return 1; } + [ -z $modelzoo_sub_dir ] && { echo "args modelzoo_sub_dir not exist";return 1; } + [ -z $target_dir ] && { echo "args target_dir not exist";return 1; } + [ -z $patch_file_name ] && { echo "args patch_file_name not exist";return 1; } + [ -z $CUR_PATH ] && { echo "args patch_file_name not exist";return 1; } + + cd $BUILD_TMP_PATH + get_modelzoo_base_code_by_git || { echo "warn git getcode failed"; return 1; } + cp $modelzoo_sub_dir -rf $BUILD_TMP_PATH/origin + cp $target_dir -rf $BUILD_TMP_PATH/code + + diff -Nur origin code > $BUILD_TMP_PATH/$patch_file_name.patch + + cp $BUILD_TMP_PATH/$patch_file_name.patch $CUR_PATH/ +} + +load_code(){ + [ -z $BUILD_TMP_PATH ] && { echo "args BUILD_TMP_PATH not exist";return 1; } + [ -z $modelzoo_sub_dir ] && { echo "args modelzoo_sub_dir not exist";return 1; } + [ -z $patch_file_name ] && { echo "args patch_file_name not exist";return 1; } + [ -z $target_patchcode_dir ] && { echo "args target_patchcode_dir not exist";return 1; } + [ -z $CUR_PATH ] && { echo "args CUR_PATH not exist";return 1; } + + cd $BUILD_TMP_PATH + get_modelzoo_base_code_by_git || { echo "warn git getcode failed"; return 1; } + cp $modelzoo_sub_dir -rf $BUILD_TMP_PATH/origin + cp $modelzoo_sub_dir -rf $BUILD_TMP_PATH/code + + if [ -f $CUR_PATH/$patch_file_name.patch ];then + patch -p0 < $CUR_PATH/$patch_file_name.patch || { echo "warn patch pfile failed"; return 1; } + else + echo "no patch file" + fi + [ ! -d $target_patchcode_dir ] || rm -rf $target_patchcode_dir + mkdir $target_patchcode_dir + cp $BUILD_TMP_PATH/code/* -rf $target_patchcode_dir/ +} + +mk_version_file() +{ + version_files=$1 + echo "git_url: $git_url" > $version_files + echo "branch: $branch" >> $version_files + echo "commitid: $commitid" >> $version_files + if [ -f $CUR_PATH/$patch_file_name.patch ];then + echo "patch_file_name: $patch_file_name.patch" >> $version_files + else + echo "patch_file_name: None" >> $version_files + fi + +} diff --git a/ais-bench_workload/src/common/sshpass_common.sh b/ais-bench_workload/src/common/sshpass_common.sh new file mode 100644 index 0000000..3dfd4ba --- /dev/null +++ b/ais-bench_workload/src/common/sshpass_common.sh @@ -0,0 +1,212 @@ +#!/bin/bash + +SSH="ssh -o StrictHostKeyChecking=no" +SCP="scp -o StrictHostKeyChecking=no" + +ssh_pass() +{ + local node="$1" + local user="$2" + local pd="$3" + local port="$4" + shift 4 + local cmd="$*" + + run_cmd="$node $cmd" + [ "$user" != "" ] && run_cmd="${user}@$run_cmd" + [ "$port" != "" ] && run_cmd="-p $port $run_cmd" + run_cmd="$SSH $run_cmd" + [ "$pd" != "" ] && run_cmd="sshpass -p ${pd} $run_cmd" + echo "run_cmd:$run_cmd" + $run_cmd || { echo "run sshrun failed node:$node"; return 1; } +} + +scp_pass() +{ + local node="$1" + local user="$2" + local pd="$3" + local port="$4" + local src="$5" + local target="$6" + + run_cmd="${node}:${target}" + [ "$user" != "" ] && run_cmd="${user}@$run_cmd" + run_cmd="-r $src ${run_cmd}" + [ "$port" != "" ] && run_cmd="-P $port $run_cmd" + run_cmd="${SCP} ${run_cmd}" + [ "$pd" != "" ] && run_cmd="sshpass -p ${pd} $run_cmd" + echo "run_cmd:$run_cmd" + $run_cmd || { echo "run scp failed node:$node"; return 1; } +} + +rscp_pass() +{ + local node="$1" + local user="$2" + local pd="$3" + local port="$4" + local src="$5" + local target="$6" + + run_cmd="${node}:${src} ${target}" + [ "$user" != "" ] && run_cmd="${user}@$run_cmd" + [ "$port" != "" ] && run_cmd="-P $port $run_cmd" + run_cmd="${SCP} -r ${run_cmd}" + [ "$pd" != "" ] && run_cmd="sshpass -p ${pd} $run_cmd" + echo "run_cmd:$run_cmd" + $run_cmd || { echo "run rscp failed node:$node"; return 1; } +} + +get_cluster_list() +{ + local cluster_config=$1 + cat ${cluster_config} | python3 -c 'import sys,json;[print(node) for node in json.load(sys.stdin)["cluster"].keys()]' +} + +get_node_user() +{ + local cluster_config=$1 + local node=$2 + cat ${cluster_config} | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster"]['\"${node}\"']["user"])' 2>/dev/null +} + +get_node_pd() +{ + local cluster_config=$1 + local node=$2 + cat ${cluster_config} | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster"]['\"${node}\"']["pd"])' 2>/dev/null +} + +get_node_port() +{ + local cluster_config=$1 + local node=$2 + cat ${cluster_config} | python3 -c 'import sys,json;print(json.load(sys.stdin)["cluster"]['\"${node}\"']["port"])' 2>/dev/null +} + +local_run_cmd() +{ + local cmd="$*" + (eval $cmd) || { echo "Warn local run '${cmd}'"; return 1; } +} + +local_scp_cmd() +{ + local src_path="$2" + local dst_path="$3" + + [ -d $src_path ] || { echo "Warn src_path:$src_path not exist return";return 1; } + + # rm and mkdir dst path + [ -d $dst_path ] && rm -rf $dst_path + mkdir -p $dst_path + + cp -rf "$src_path" $dst_path || { echo "Warn local cp failed";return 1; } +} + +# interface function + +cluster_run_cmd_serial() +{ + local node_info_file=$1 + shift 1 + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_run_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + local cmd=$* + local node_arr=($(get_cluster_list ${node_info_file})) + local node_count=${#node_arr[@]} + + for ((i=0; i<$node_count; i++)); do { + local node="${node_arr[$i]}" + local user=$(get_node_user ${node_info_file} ${node}) + local pd=$(get_node_pd ${node_info_file} ${node}) + local port=$(get_node_port ${node_info_file} ${node}) + local cur_cmd="export SERVER_ID=${i}; ${cmd}" + ssh_pass "${node}" "${user}" "${pd}" "$port" "${cur_cmd}" || { echo "EROOR when executing '${cur_cmd}'"; return 1; } + } + done + return 0 +} + +cluster_run_cmd_parallel() +{ + node_info_file=$1 + shift 1 + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_run_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + cmd=$* + node_arr=($(get_cluster_list ${node_info_file})) + node_count=${#node_arr[@]} + + retvalfile=$(mktemp) + for ((i=0; i<$node_count; i++)); do { + node="${node_arr[$i]}" + user=$(get_node_user ${node_info_file} ${node}) + pd=$(get_node_pd ${node_info_file} ${node}) + port=$(get_node_port ${node_info_file} ${node}) + cur_cmd="export SERVER_ID=${i}; $cmd" + echo "正在登陆${user}@${node}，SERVER_ID为${i}, 进入后执行的命令为${cur_cmd}" + ssh_pass "${node}" "${user}" "${pd}" "$port" ${cur_cmd} || { echo "run scp failed node:$node"; rm -rf $retvalfile; } + }& + done + logger_Info "now wait run cmd done" + wait + [ -f $retvalfile ] || { echo "run train failed";return 1; } + rm -rf $retvalfile + logger_Info "now run cmd done finish" +} + +sshpass_scp_cmd() +{ + node_info_file=$1 + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_scp_cmd "$@";return $?; } + + src_path=$2 + dst_path=$3 + node_arr=($(get_cluster_list ${node_info_file})) + node_count=${#node_arr[@]} + + for ((i=0; i<$node_count; i++)); do { + node="${node_arr[$i]}" + user=$(get_node_user ${node_info_file} ${node}) + pd=$(get_node_pd ${node_info_file} ${node}) + port=$(get_node_port ${node_info_file} ${node}) + echo "------------scp ------${user}@${node}---------------------" + ssh_pass "${node}" "${user}" "${pd}" "$port" "rm -rf ${dst_path};mkdir -p ${dst_path}" || { echo "sshpass_scp failed node:$node"; return 1; } + scp_pass "${node}" "${user}" "${pd}" "$port" "${src_path}" "${dst_path}" || { echo "sshpass_scp failed node:$node"; return 1; } + echo "------------scp done------${user}@${node}---------------------" + } done +} + +sshpass_rscp_cmd() +{ + node_info_file=$1 + + # clusterconfig file not set as local mode + [ "$node_info_file" == "" ] && { local_scp_cmd "$@";return $?; } + + [ -f $node_info_file ] || { echo "$node_info_file not exist ret";return 1; } + + src_path="$2" + dst_path="$3" + node_arr=($(get_cluster_list ${node_info_file})) + node_count=${#node_arr[@]} + + for ((i=0; i<$node_count; i++)); do { + node="${node_arr[$i]}" + user=$(get_node_user ${node_info_file} ${node}) + pd=$(get_node_pd ${node_info_file} ${node}) + port=$(get_node_port ${node_info_file} ${node}) + echo "------------------${user}@${node}---------------------" + # ssh_pass ${node} ${user} ${pd} "rm -rf ${dst_path}" + rscp_pass "${node}" "${user}" "${pd}" "$port" "${src_path}" "${dst_path}" || { echo "sshpass_rscp failed node:$node"; return 1; } + } done +} \ No newline at end of file diff --git a/ais-bench_workload/src/common/train_modelarts.py b/ais-bench_workload/src/common/train_modelarts.py new file mode 100644 index 0000000..dc2c66f --- /dev/null +++ b/ais-bench_workload/src/common/train_modelarts.py @@ -0,0 +1,106 @@ +import argparse +import logging +import os +import sys +from statistics import mean + +sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')) + +import ais_utils +from config.modelarts_config import access_config +from config.modelarts_config import session_config as session_config_v1 +from config.modelarts_config import session_config_v2 +from modelarts_handler_v2 import modelarts_handler as modelarts_handler_v2 + +from modelarts_handler import logger, modelarts_handler + + +def report_result(handler): + ranksize_file_url = os.path.join(handler.output_url, 'ranksize.json') + ranksize = int(handler.get_obs_url_content(ranksize_file_url)) + print("url:{} read ranksize:{}".format(ranksize_file_url, ranksize)) + + total_throughput = 0.0 + for rankid in range(0, ranksize): + throughput_url = os.path.join(handler.output_url, 'throughput_' + str(rankid) + '.json') + single_throughput_rate = float(handler.get_obs_url_content(throughput_url)) + print("rankid:{} url:{} read throughput:{}".format(rankid, throughput_url, single_throughput_rate)) + total_throughput = total_throughput + single_throughput_rate + print("report result total_throughput : {}".format(total_throughput)) + ais_utils.set_result("training", "throughput_ratio", total_throughput) + + accuracy_file_url = os.path.join(handler.output_url, 'accuracy.json') + accuracy = float(handler.get_obs_url_content(accuracy_file_url)) + print("url:{} read accuracy:{}".format(accuracy_file_url, accuracy)) + + print("report result accuracy:{}".format(accuracy)) + ais_utils.set_result("training", "accuracy", accuracy) + +# 单设备运行模式 +def report_result_singlesever_mode(handler, server_count): + # 单设备运行模式下默认都是8卡 + cards_per_server = 8 + print("server_count:{} cards_per_server:{}".format(server_count, cards_per_server)) + + throughput_list = [] + accuracy_list = [] + for server_id in range(server_count): + single_server_throughput = 0.0 + for rankid in range(cards_per_server): + throughput_url = os.path.join(handler.output_url, str(server_id), 'throughput_' + str(rankid) + '.json') + single_card_throughput = float(handler.get_obs_url_content(throughput_url)) + print("rankid:{} url:{} read throughput:{}".format(rankid, throughput_url, single_card_throughput)) + single_server_throughput = single_server_throughput + single_card_throughput + print("serverid:{} count:{} service_throughput:{}".format(server_id, server_count, single_server_throughput)) + throughput_list.append(single_server_throughput) + + accuracy_file_url = os.path.join(handler.output_url, 'accuracy_{}.json'.format(server_id)) + single_server_accuracy = float(handler.get_obs_url_content(accuracy_file_url)) + print("serverid:{} url:{} read accuracy:{}".format(server_id, accuracy_file_url, single_server_accuracy)) + accuracy_list.append(single_server_accuracy) + + print("report >> throughput_list:{} average:{}".format(throughput_list, mean(throughput_list))) + print("report >> accuracy_list:{} average:{}".format(accuracy_list, mean(accuracy_list))) + + ais_utils.set_result("training", "throughput_ratio", mean(throughput_list)) + ais_utils.set_result("training", "accuracy", mean(accuracy_list)) + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--local_code_path", help="the local path of run code") + parser.add_argument("--single_server_mode", action="store_true", help="the local path of run code") + parser.add_argument("--action", default="run", choices=["run", "stop"], help="action (run or stop)") + parser.add_argument("--modelarts_version", default="V1", choices=["V1", "V2"], help="modelarts version (V1 or V2)") + parser.add_argument("--job_id", default="None", help="job id used to stop given job") + args = parser.parse_args() + return args + +if __name__ == '__main__': + args = get_args() + + logger.setLevel(logging.DEBUG) + session_config = session_config_v1 if args.modelarts_version == 'V1' else session_config_v2 + + handler = modelarts_handler() if args.modelarts_version == 'V1' else modelarts_handler_v2() + handler.create_session(access_config) + + if args.action == "stop": + if args.modelarts_version == 'V1': + handler.stop_new_versions(session_config) + else: + handler.stop_job(args.job_id) + sys.exit() + + handler.create_obs_handler(access_config) + + # default run mode + handler.run_job(session_config, args.local_code_path) + + # handler.output_url = "s3://0923/00lcm/result_dump/res/V212/" + try: + if args.single_server_mode: + report_result_singlesever_mode(handler, session_config.train_instance_count) + else: + report_result(handler) + except FileNotFoundError as e: + print("error resport result failed. Exception:", e) diff --git a/ais-bench_workload/src/train/huawei/common/mindspore_env.sh b/ais-bench_workload/src/train/huawei/common/mindspore_env.sh new file mode 100644 index 0000000..e0ccc28 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/common/mindspore_env.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +export GLOG_v=3 +export HCCL_CONNECT_TIMEOUT=600 + +if [ -f /usr/local/Ascend/nnae/set_env.sh ];then + source /usr/local/Ascend/nnae/set_env.sh +elif [ -f /usr/local/Ascend/ascend-toolkit/set_env.sh ]; then + source /usr/local/Ascend/ascend-toolkit/set_env.sh +elif [ -f ~/Ascend/nnae/set_env.sh ]; then + source ~/Ascend/nnae/set_env.sh +elif [ -f ~/Ascend/ascend-toolkit/set_env.sh ]; then + source ~/Ascend/ascend-toolkit/set_env.sh +else + echo "warning find no env so not set" +fi diff --git a/ais-bench_workload/src/train/huawei/common/tensorflow_env.sh b/ais-bench_workload/src/train/huawei/common/tensorflow_env.sh new file mode 100644 index 0000000..62f348c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/common/tensorflow_env.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Please change to the actual installation path +export install_path=/usr/local/Ascend + +# driver +export LD_LIBRARY_PATH=${install_path}/driver/lib64/common/:${install_path}/driver/lib64/driver/:$LD_LIBRARY_PATH + +if [ -d /usr/local/Ascend/nnae/latest ];then + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/nnae/latest/compiler/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/Ascend/driver/tools/hccn_tool/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/fwkplugin/latest/fwkplugin/python/site-packages:/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/nnae/latest/compiler/python/site-packages/:/usr/local/Ascend/fwkplugin/latest/fwkplugin/python/site-packages + export PATH=$PATH:/usr/local/Ascend/nnae/latest/compiler/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/nnae/latest/opp + export TBE_IMPL_PATH=/usr/local/Ascend/nnae/latest/opp/op_impl/built-in/ai_core +else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/compiler/lib64:/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:/usr/local/Ascend/add-ons/:/usr/local/mpirun4.0/lib + export PYTHONPATH=$PYTHONPATH:/usr/local/Ascend/fwkplugin/latest/fwkplugin/python/site-packages:/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core/tbe:/usr/local/Ascend/ascend-toolkit/latest/compiler/python/site-packages/:/usr/local/Ascend/ascend-toolkit/latest/fwkplugin/python/site-packages:$projectDir + export PATH=$PATH:/usr/local/Ascend/ascend-toolkit/latest/compiler/ccec_compiler/bin:/usr/local/mpirun4.0/bin + export ASCEND_OPP_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/ + export TBE_IMPL_PATH=/usr/local/Ascend/ascend-toolkit/latest/opp/op_impl/built-in/ai_core +fi + +# Ascend-dmi +export LD_LIBRARY_PATH=/usr/local/dcmi:${install_path}/toolbox/latest/Ascend-DMI/lib64:${LD_LIBRARY_PATH} +export PATH=${install_path}/toolbox/latest/Ascend-DMI/bin:${PATH} + +export JOB_ID=123456789 + +export HCCL_CONNECT_TIMEOUT=600 +export HCCL_WHITELIST_DISABLE=1 + +# log +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +export ASCEND_GLOBAL_LOG_LEVEL=3 +/usr/local/Ascend/driver/tools/msnpureport -d 0 -g error +/usr/local/Ascend/driver/tools/msnpureport -d 4 -g error +export SLOG_PRINT_TO_STDOUT=0 + +#system env +ulimit -c unlimited diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh new file mode 100644 index 0000000..4505785 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + CONFIG_FILE=${CURDIR}//output/code/pretrain_config_Ascend_Boost.yaml + # update dataset_format to tfrecord, since 1.9 version + sed -i "s|dataset_format:.*|dataset_format: 'tfrecord'|g" "$CONFIG_FILE" + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + + mkdir -p ${CURDIR}/output/config + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/config.sh -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/modelarts_config.py -r ${CURDIR}//output/config/ + [ "$1" == "r1.3" ] && { cp ${CURDIR}/config/modelarts_config.py.r1.3 -r ${CURDIR}//output/config/modelarts_config.py; } + [ -d ${CURDIR}/doc ] && cp ${CURDIR}/doc -r ${CURDIR}/output/ + + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh new file mode 100644 index 0000000..5a835db --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh @@ -0,0 +1,18 @@ +export PYTHON_COMMAND=python3.7 +export TRAIN_DATA_PATH=/home/datasets/Bert-Dataset/ +export EVAL_DATA_PATH=/home/datasets/Bert-TestData/ + +export PRETRAIN_MODEL_PATH=/home/models/ms_bert_large.ckpt + +export EPOCH_SIZE=5 +export TRAIN_STEPS=12000 + +# 8p +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# options needed only if rank_size > 1 +export RANK_TABLE_FILE=/home/lcm/tool/rank_table_8p.json + +# needed only in cluster mode +# export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py b/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py new file mode 100644 index 0000000..dfc8b22 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py @@ -0,0 +1,159 @@ +from easydict import EasyDict as ed + +# 该部分为认证信息，请向相关运维同事咨询并填写 +access_config = ed({ + # 登录需要的ak sk信息 + 'access_key': '', + 'secret_access_key': '', + # 连接OBS的服务地址。可包含协议类型、域名、端口号。（出于安全性考虑，建议使用https协议） + # 如果是计算中心，需要联系运维同事获取 + 'server': '', + # project_id/region_name: + # 项目ID/区域ID，获取方式参考链接 + # https://support.huaweicloud.com/api-iam/iam_17_0002.html + # 如果是计算中心,请咨询相关维护同事 + 'region_name': '', + 'project_id': '', + + # 如下配置针对计算中心等专有云 通用云不需要设置 设置为空 请咨询相关维护同事 + # 设置该信息后 需要设置相关的域名解析地址 + 'iam_endpoint': '', + 'obs_endpoint': '', + 'modelarts_endpoint': '', +}) + +session_config = ed({ + # 运行模型的传入超参 + 'hyperparameters': [ + # 模型配置文件，默认boost模式，不需要修改 + {'label': 'config_path', 'value': '../../pretrain_config_Ascend_Boost.yaml'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'label': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'label': 'distribute', 'value': 'true'}, + # epoch次数 必须关注 当前默认设置为5 训练的epoch数 + # 优先级低于train_steps，如果存在train_steps以此为准，否则以epoch_size为准 + {'label': 'epoch_size', 'value': '5'}, + # 训练step数 必须填写并审视 该值优先级高于train_steps数 + {'label': 'train_steps', 'value': '12000'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'label': 'enable_save_ckpt', 'value': 'true'}, + # 不需要修改 + {'label': 'enable_lossscale', 'value': 'true'}, + # 不需要修改 + {'label': 'do_shuffle', 'value': 'true'}, + # 不需要修改 + {'label': 'enable_data_sink', 'value': 'true'}, + # 不需要修改 + {'label': 'data_sink_steps', 'value': '100'}, + # 不需要修改 + {'label': 'accumulation_steps', 'value': '1'}, + # 保存ckpt的step数 注意 该值必须要跟step数保存一致 这样提高性能 + {'label': 'save_checkpoint_steps', 'value': '12000'}, + # 保存ckpt的个数 默认为1 不需要修改 + {'label': 'save_checkpoint_num', 'value': '1'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/enwiki_small/', + # obs代码路径 程序会自动拷贝到该路径 + 'code_dir': '/zgwtest/lcm_test/bert/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/bert/run_pretrain.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'MindSpore-1.3-cann_5.0.2-python3.7-euleros2.8-aarch64', + + # 资源参数类型主要包括如下2个值 train_instance_type和pool_id + # 不设置pool_id 默认是公共池 设置了就是专属资源池 + # 只设置pool_id 不设置train_instance_type 默认为专属资源池的默认类型 + # train_instance_type 在程序打印中有提示的 一般为如下四个值 分别对应 1卡 2卡 4卡 8卡 + # ['modelarts.kat1.xlarge', 'modelarts.kat1.2xlarge', 'modelarts.kat1.4xlarge', 'modelarts.kat1.8xlarge'] + # https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0191.html 该链接指示获取方法 + + # 专属资源池id 不是则为None + 'pool_id': None, + # 训练类型 如下为8卡 如果是专属资源池id设置，那么该类型需要设置为None + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练结点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type' : None, + # 'nas_share_addr' : None, + # 'nas_mount_path' : None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) + +session_config_v2 = ed({ + # 运行模型的传入超参 + 'parameters': [ + # 模型配置文件，默认boost模式，不需要修改 + {'name': 'config_path', 'value': '../../pretrain_config_Ascend_Boost.yaml'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'name': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'name': 'distribute', 'value': 'true'}, + # epoch次数 必须关注 当前默认设置为5 训练的epoch数 + # 优先级低于train_steps，如果存在train_steps以此为准，否则以epoch_size为准 + {'name': 'epoch_size', 'value': '5'}, + # 训练step数 必须填写并审视 该值优先级高于train_steps数 + {'name': 'train_steps', 'value': '12000'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'name': 'enable_save_ckpt', 'value': 'true'}, + # 不需要修改 + {'name': 'enable_lossscale', 'value': 'true'}, + # 不需要修改 + {'name': 'do_shuffle', 'value': 'true'}, + # 不需要修改 + {'name': 'enable_data_sink', 'value': 'true'}, + # 不需要修改 + {'name': 'data_sink_steps', 'value': '100'}, + # 不需要修改 + {'name': 'accumulation_steps', 'value': '1'}, + # 保存ckpt的step数 注意 该值必须要跟step数保存一致 这样提高性能 + {'name': 'save_checkpoint_steps', 'value': '12000'}, + # 保存ckpt的个数 默认为1 不需要修改 + {'name': 'save_checkpoint_num', 'value': '1'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/enwiki_small/', + # obs代码路径 程序会自动拷贝到该路径. 和boot_files一起用于复合参数 training_files + 'code_dir': '/zgwtest/lcm_test/bert/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/bert/run_pretrain.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'mindspore_1.3.0-cann_5.0.2-py_3.7-euler_2.8.3-aarch64', + + # pool_id不设置或者设置为None， 默认是公共资源池。 设置了就表示是专属资源池。在ModelArts管理控制台，单击左侧“专属资源池”，在专属资源池列表中可以查看专属资源池ID，类似poolc90f063b + 'pool_id': None, + # 训练类型，默认8卡。 train_instance_type 在程序打印中有提示的,请注意紧随“get valid train_instance_types：”之后的打印输出. 由modelarts.estimatorV2 类Estimator的接口get_train_instance_types（）查询而来。 + # 请参见https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0431.html 该链接指示获取方法。注意不同云环境查询的结果不同 + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练节点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type' : None, + # 'nas_share_addr' : None, + # 'nas_mount_path' : None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 b/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 new file mode 100644 index 0000000..6e7725c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 @@ -0,0 +1,159 @@ +from easydict import EasyDict as ed + +# 该部分为认证信息，请向相关运维同事咨询并填写 +access_config = ed({ + # 登录需要的ak sk信息 + 'access_key': '', + 'secret_access_key': '', + # 连接OBS的服务地址。可包含协议类型、域名、端口号。（出于安全性考虑，建议使用https协议） + # 如果是计算中心，需要联系运维同事获取 + 'server': '', + # project_id/region_name: + # 项目ID/区域ID，获取方式参考链接 + # https://support.huaweicloud.com/api-iam/iam_17_0002.html + # 如果是计算中心,请咨询相关维护同事 + 'region_name': '', + 'project_id': '', + + # 如下配置针对计算中心等专有云 通用云不需要设置 设置为空 请咨询相关维护同事 + # 设置该信息后 需要设置相关的域名解析地址 + 'iam_endpoint': '', + 'obs_endpoint': '', + 'modelarts_endpoint' : '', +}) + +session_config = ed({ + # 运行模型的传入超参 + 'hyperparameters': [ + # bert模型类型 默认large_acc模式 不需要修改 + {'label': 'bert_network', 'value': 'large_acc'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'label': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'label': 'distribute', 'value': 'true'}, + # epoch次数 必须关注 当前默认设置为5 训练的epoch数 + # 优先级低于train_steps，如果存在train_steps以此为准，否则以epoch_size为准 + {'label': 'epoch_size', 'value': '5'}, + # 训练step数 必须填写并审视 该值优先级高于train_steps数 + {'label': 'train_steps', 'value': '12000'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'label': 'enable_save_ckpt', 'value': 'true'}, + # 不需要修改 + {'label': 'enable_lossscale', 'value': 'true'}, + # 不需要修改 + {'label': 'do_shuffle', 'value': 'true'}, + # 不需要修改 + {'label': 'enable_data_sink', 'value': 'true'}, + # 不需要修改 + {'label': 'data_sink_steps', 'value': '100'}, + # 不需要修改 + {'label': 'accumulation_steps', 'value': '1'}, + # 保存ckpt的step数 注意 该值必须要跟step数保存一致 这样提高性能 + {'label': 'save_checkpoint_steps', 'value': '12000'}, + # 保存ckpt的个数 默认为1 不需要修改 + {'label': 'save_checkpoint_num', 'value': '1'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/enwiki_small/', + # obs代码路径 程序会自动拷贝到该路径 + 'code_dir': '/zgwtest/lcm_test/bert/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/bert/run_pretrain.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'MindSpore-1.3-cann_5.0.2-python3.7-euleros2.8-aarch64', + + # 资源参数类型主要包括如下2个值 train_instance_type和pool_id + # 不设置pool_id 默认是公共池 设置了就是专属资源池 + # 只设置pool_id 不设置train_instance_type 默认为专属资源池的默认类型 + # train_instance_type 在程序打印中有提示的 一般为如下四个值 分别对应 1卡 2卡 4卡 8卡 + # ['modelarts.kat1.xlarge', 'modelarts.kat1.2xlarge', 'modelarts.kat1.4xlarge', 'modelarts.kat1.8xlarge'] + # https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0191.html 该链接指示获取方法 + + # 专属资源池id 不是则为None + 'pool_id' : None, + # 训练类型 如下为8卡 如果是专属资源池id设置，那么该类型需要设置为None + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练结点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type' : None, + # 'nas_share_addr' : None, + # 'nas_mount_path' : None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) + +session_config_v2 = ed({ + # 运行模型的传入超参 + 'parameters': [ + # bert模型类型 默认large_acc模式 不需要修改 + {'name': 'bert_network', 'value': 'large_acc'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'name': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'name': 'distribute', 'value': 'true'}, + # epoch次数 必须关注 当前默认设置为5 训练的epoch数 + # 优先级低于train_steps，如果存在train_steps以此为准，否则以epoch_size为准 + {'name': 'epoch_size', 'value': '5'}, + # 训练step数 必须填写并审视 该值优先级高于train_steps数 + {'name': 'train_steps', 'value': '12000'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'name': 'enable_save_ckpt', 'value': 'true'}, + # 不需要修改 + {'name': 'enable_lossscale', 'value': 'true'}, + # 不需要修改 + {'name': 'do_shuffle', 'value': 'true'}, + # 不需要修改 + {'name': 'enable_data_sink', 'value': 'true'}, + # 不需要修改 + {'name': 'data_sink_steps', 'value': '100'}, + # 不需要修改 + {'name': 'accumulation_steps', 'value': '1'}, + # 保存ckpt的step数 注意 该值必须要跟step数保存一致 这样提高性能 + {'name': 'save_checkpoint_steps', 'value': '12000'}, + # 保存ckpt的个数 默认为1 不需要修改 + {'name': 'save_checkpoint_num', 'value': '1'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/enwiki_small/', + # obs代码路径 程序会自动拷贝到该路径. 和boot_files一起用于复合参数 training_files + 'code_dir': '/zgwtest/lcm_test/bert/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/bert/run_pretrain.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'mindspore_1.3.0-cann_5.0.2-py_3.7-euler_2.8.3-aarch64', + + # pool_id不设置或者设置为None， 默认是公共资源池。 设置了就表示是专属资源池。在ModelArts管理控制台，单击左侧“专属资源池”，在专属资源池列表中可以查看专属资源池ID，类似poolc90f063b + 'pool_id' : None, + # 训练类型，默认8卡。train_instance_type 在程序打印中有提示的,请注意紧随“get valid train_instance_types：”之后的打印输出. 由modelarts.estimatorV2 类Estimator的接口get_train_instance_types（）查询而来。 + # 请参见https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0431.html 该链接指示获取方法。注意不同云环境查询的结果不同 + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练节点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type' : None, + # 'nas_share_addr' : None, + # 'nas_mount_path' : None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch new file mode 100644 index 0000000..f64e7a0 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch @@ -0,0 +1,182 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-04-10 14:42:15.600000000 +0800 ++++ code/pretrain_eval.py 2023-04-10 14:42:15.610000000 +0800 +@@ -32,7 +32,11 @@ + Predict function + ''' + devid = int(os.getenv('DEVICE_ID')) +- context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) ++ + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir, dataset_format=cfg.dataset_format) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,19 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-04-10 14:42:15.590000000 +0800 ++++ code/run_pretrain.py 2023-04-10 14:42:15.610000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -42,7 +43,7 @@ + from src.utils import LossCallBack, BertLearningRate, EvalCallBack, BertMetric + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper +-from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -192,8 +193,18 @@ + net_with_grads = BertNetworkMatchBucket(net_with_grads, bert_net_cfg.seq_length, cfg.bucket_list) + return net_with_grads + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -268,10 +279,85 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-10 14:42:15.590000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-10 14:42:15.610000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch new file mode 100644 index 0000000..adad995 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch @@ -0,0 +1,199 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-07-11 14:30:06.510000000 +0800 ++++ code/pretrain_eval.py 2022-07-11 14:30:06.530000000 +0800 +@@ -151,7 +151,19 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ") +- print(v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("BATCH_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-07-11 14:30:06.510000000 +0800 ++++ code/run_pretrain.py 2022-07-11 14:30:06.520000000 +0800 +@@ -39,9 +39,37 @@ + from src.utils import LossCallBack, BertLearningRate + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper +-from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + ++import time ++from mindspore.train.callback._callback import Callback ++ ++skip_pre_step_size = 1 ++skip_pre_step_time_sum = 0 ++real_start_time = 0.0 ++ ++class ThroughputRate(Callback): ++ def __init__(self): ++ super(ThroughputRate, self).__init__() ++ self.count = 0 ++ ++ def step_begin(self, run_context): ++ self.step_time = time.time() ++ if self.count == skip_pre_step_size: ++ global real_start_time ++ real_start_time = self.step_time ++ ++ def step_end(self, run_context): ++ step_seconds = (time.time() - self.step_time) ++ ++ global skip_pre_epoch_size ++ global skip_pre_step_time_sum ++ ++ if self.count < skip_pre_step_size: ++ skip_pre_step_time_sum += step_seconds ++ print("skip:{} of {} step_seconds:{} time_sum:{}".format(self.count, skip_pre_step_size, step_seconds, skip_pre_step_time_sum)) ++ self.count = self.count + 1 + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -157,8 +185,18 @@ + cfg.data_dir = cfg.data_path + cfg.save_checkpoint_path = os.path.join(cfg.output_path, cfg.save_checkpoint_path) + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.finetune_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -209,6 +247,7 @@ + + optimizer = _get_optimizer(cfg, net_with_loss) + callback = [TimeMonitor(cfg.data_sink_steps), LossCallBack(ds.get_dataset_size())] ++ callback.append(ThroughputRate()) + if cfg.enable_save_ckpt == "true" and cfg.device_id % min(8, device_num) == 0: + config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, + keep_checkpoint_max=cfg.save_checkpoint_num) +@@ -250,10 +289,83 @@ + + model = Model(net_with_grads) + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ end_time = time.time() ++ data_sum = ((new_repeat_count - skip_pre_step_size) * cfg.data_sink_steps * cfg.batch_size) ++ throughput_rate = data_sum / (end_time - real_start_time) ++ print("train done starttime:{} real:{} endtime:{} train_step:{} skiptime:{} ds getdataset:{} new_repeat_count:{} data_sum:{} throughput_rate:{}".format( ++ start_time, real_start_time, end_time, cfg.train_steps, skip_pre_step_time_sum, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("BATCH_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-07-11 14:30:06.510000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-07-11 14:30:06.530000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val/eval_10k.tfrecord")): ++ config.data_file = os.path.join(base_path, "val/eval_10k.tfrecord") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch new file mode 100644 index 0000000..2e18495 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch @@ -0,0 +1,192 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-07-11 17:53:51.470000000 +0800 ++++ code/pretrain_eval.py 2022-07-11 17:53:51.480000000 +0800 +@@ -33,6 +33,10 @@ + ''' + devid = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,18 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("BATCH_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-07-11 17:53:51.460000000 +0800 ++++ code/run_pretrain.py 2022-07-11 17:53:51.480000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -42,7 +43,7 @@ + from src.utils import LossCallBack, BertLearningRate, EvalCallBack, BertMetric + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper +-from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -158,8 +159,19 @@ + cfg.data_dir = cfg.data_path + cfg.save_checkpoint_path = os.path.join(cfg.output_path, cfg.save_checkpoint_path) + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) ++ + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -262,10 +274,85 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("BATCH_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2022-07-11 17:53:51.460000000 +0800 ++++ code/src/model_utils/config.py 2022-07-11 17:53:51.480000000 +0800 +@@ -196,6 +196,8 @@ + parser.add_argument("--config_path", type=get_abs_path, default="../../pretrain_config.yaml", + help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../..', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-07-11 17:53:51.460000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-07-11 17:53:51.480000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch new file mode 100644 index 0000000..0178e64 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch @@ -0,0 +1,182 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-05-22 21:56:20.710000000 +0800 ++++ code/pretrain_eval.py 2022-05-22 21:56:20.740000000 +0800 +@@ -32,7 +32,10 @@ + Predict function + ''' + devid = int(os.getenv('DEVICE_ID')) +- context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +56,18 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-05-22 21:56:20.690000000 +0800 ++++ code/run_pretrain.py 2022-05-22 21:56:20.720000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -42,7 +43,7 @@ + from src.utils import LossCallBack, BertLearningRate, EvalCallBack, BertMetric + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper +-from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -158,8 +159,19 @@ + cfg.data_dir = cfg.data_path + cfg.save_checkpoint_path = os.path.join(cfg.output_path, cfg.save_checkpoint_path) + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) ++ + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -262,10 +274,86 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() ++ +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-05-22 21:56:20.700000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-05-22 21:56:20.730000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch new file mode 100644 index 0000000..4932a62 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch @@ -0,0 +1,184 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-04-10 11:15:24.640000000 +0800 ++++ code/pretrain_eval.py 2023-04-10 11:15:24.660000000 +0800 +@@ -32,7 +32,11 @@ + Predict function + ''' + devid = int(os.getenv('DEVICE_ID')) +- context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) ++ + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,19 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-04-10 11:15:24.640000000 +0800 ++++ code/run_pretrain.py 2023-04-10 11:15:24.650000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -42,7 +43,7 @@ + from src.utils import LossCallBack, BertLearningRate, EvalCallBack, BertMetric + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper +-from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -158,8 +159,18 @@ + cfg.data_dir = cfg.data_path + cfg.save_checkpoint_path = os.path.join(cfg.output_path, cfg.save_checkpoint_path) + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -262,10 +273,88 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) + + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ ++ + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() ++ +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-10 11:15:24.640000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-10 11:15:24.660000000 +0800 +@@ -99,6 +99,15 @@ + config.device_id = get_device_id() + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") + + if pre_process: + pre_process() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch new file mode 100644 index 0000000..76c8d69 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch @@ -0,0 +1,182 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-10-18 11:56:29.660000000 +0800 ++++ code/pretrain_eval.py 2022-10-18 11:56:29.676000000 +0800 +@@ -32,7 +32,11 @@ + Predict function + ''' + devid = int(os.getenv('DEVICE_ID')) +- context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) ++ + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir, dataset_format=cfg.dataset_format) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,18 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-10-18 11:56:29.660000000 +0800 ++++ code/run_pretrain.py 2022-10-18 11:56:29.676000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -42,7 +43,7 @@ + from src.utils import LossCallBack, BertLearningRate, EvalCallBack, BertMetric + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper +-from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -192,8 +193,18 @@ + net_with_grads = BertNetworkMatchBucket(net_with_grads, bert_net_cfg.seq_length, cfg.bucket_list) + return net_with_grads + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -268,10 +279,86 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-10-18 11:56:29.668000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-10-18 11:56:29.688000000 +0800 +@@ -99,6 +99,16 @@ + config.device_id = get_device_id() + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + + if pre_process: + pre_process() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch new file mode 100644 index 0000000..932d168 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch @@ -0,0 +1,180 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-04-07 20:15:40.860000000 +0800 ++++ code/pretrain_eval.py 2023-04-07 20:15:40.880000000 +0800 +@@ -33,6 +33,10 @@ + ''' + devid = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir, dataset_format=cfg.dataset_format) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,18 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-04-07 20:15:40.850000000 +0800 ++++ code/run_pretrain.py 2023-04-07 20:15:40.870000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -43,6 +44,7 @@ + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -192,8 +194,18 @@ + net_with_grads = BertNetworkMatchBucket(net_with_grads, bert_net_cfg.seq_length, cfg.bucket_list) + return net_with_grads + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -268,10 +280,87 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() ++ +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-07 20:15:40.860000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-07 20:15:40.870000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch new file mode 100644 index 0000000..932d168 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch @@ -0,0 +1,180 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-04-07 20:15:40.860000000 +0800 ++++ code/pretrain_eval.py 2023-04-07 20:15:40.880000000 +0800 +@@ -33,6 +33,10 @@ + ''' + devid = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir, dataset_format=cfg.dataset_format) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,18 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-04-07 20:15:40.850000000 +0800 ++++ code/run_pretrain.py 2023-04-07 20:15:40.870000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -43,6 +44,7 @@ + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -192,8 +194,18 @@ + net_with_grads = BertNetworkMatchBucket(net_with_grads, bert_net_cfg.seq_length, cfg.bucket_list) + return net_with_grads + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -268,10 +280,87 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() ++ +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-07 20:15:40.860000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-07 20:15:40.870000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch new file mode 100644 index 0000000..932d168 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch @@ -0,0 +1,180 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-04-07 20:15:40.860000000 +0800 ++++ code/pretrain_eval.py 2023-04-07 20:15:40.880000000 +0800 +@@ -33,6 +33,10 @@ + ''' + devid = int(os.getenv('DEVICE_ID')) + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ # for modelarts mode eval after traing, should set alone and no need set device_id ++ #context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", device_id=devid) ++ from mindspore.context import ParallelMode ++ context.set_auto_parallel_context(parallel_mode=ParallelMode.STAND_ALONE) + dataset = create_eval_dataset(cfg.batch_size, 1, data_dir=cfg.eval_data_dir, dataset_format=cfg.dataset_format) + net_for_pretraining = BertPretrainEval(bert_net_cfg) + net_for_pretraining.set_train(False) +@@ -53,6 +57,18 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ v = v.item() ++ import moxing as mox ++ from src.model_utils.device_adapter import get_device_num ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(cfg.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(v)) ++ accuracy_file1 = os.path.join(cfg.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(v)) ++ ranksize_file = os.path.join(cfg.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-04-07 20:15:40.850000000 +0800 ++++ code/run_pretrain.py 2023-04-07 20:15:40.870000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -43,6 +44,7 @@ + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num ++from src.model_utils.device_adapter import get_device_id, get_device_num, get_rank_id + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -192,8 +194,18 @@ + net_with_grads = BertNetworkMatchBucket(net_with_grads, bert_net_cfg.seq_length, cfg.bucket_list) + return net_with_grads + ++def modelarts_post_process(): ++ def is_ckpt(name): ++ if name.endswith('ckpt'): ++ return True ++ return False ++ ckpt_save_dir = os.path.join(cfg.save_checkpoint_path, 'ckpt_' + str(get_rank())) ++ if os.path.exists(ckpt_save_dir): ++ ckpts = list(filter(is_ckpt, os.listdir(ckpt_save_dir))) ++ ckpts.sort(key=lambda x: int(''.join(filter(str.isdigit, x)))) ++ cfg.eval_ckpt = os.path.join(ckpt_save_dir, ckpts[-1]) + +-@moxing_wrapper(pre_process=modelarts_pre_process) ++@moxing_wrapper(pre_process=modelarts_pre_process, post_process=modelarts_post_process) + def run_pretrain(): + """pre-train bert_clue""" + context.set_context(mode=context.GRAPH_MODE, device_target=cfg.device_target, device_id=cfg.device_id) +@@ -268,10 +280,87 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ end_time = time.time() ++ data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ throughput_rate = data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} train_step:{} ds getdataset:{} new_repeat_count:{} data_sum:{} single_throughput_rate:{}".format( ++ start_time, end_time, cfg.train_steps, ds.get_dataset_size(), new_repeat_count, data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = cfg.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(cfg.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set OK") ++ else: ++ print("singleserver_mode not set") ++ + set_seed(0) + run_pretrain() ++ ++ if get_rank() == 0: ++ from pretrain_eval import MLM_eval ++ MLM_eval() ++ +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-07 20:15:40.860000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-07 20:15:40.870000000 +0800 +@@ -100,6 +100,16 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ # modelarts sdk fit ++ config.schema_file = None ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "ms_bert_large.ckpt")): ++ config.load_checkpoint_path = os.path.join(base_path, "ms_bert_large.ckpt") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_data_dir = os.path.join(base_path, "val") ++ + if pre_process: + pre_process() + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh new file mode 100644 index 0000000..3b91c09 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r2.3"; } + + modelzoo_sub_dir="mindspore/model_zoo/official/nlp/bert" + if [ "$branch_args" == "r1.1" ];then + branch="r1.1" + patch_file_name="r1.1" + commitid="9c133b6f709e12ed7085c31f028e7c925ee57828" + git_url="https://gitee.com/mindspore/mindspore.git" + elif [ "$branch_args" == "r1.2" ];then + branch="r1.2" + patch_file_name="r1.2" + commitid="cd002779dc5e2bc2da85b9a33e8950aa3bb50ed2" + git_url="https://gitee.com/mindspore/mindspore.git" + elif [ "$branch_args" == "r1.3" ];then + branch="r1.3" + patch_file_name="r1.3" + commitid="d9d4960262617d964d669ef8e3287daf347d5a7c" + git_url="https://gitee.com/mindspore/mindspore.git" + elif [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="a6cbc7bc9e23fd04b53a406e72ba87e88d7980d0" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r1.6" ];then + branch="r1.6" + patch_file_name="r1.6" + commitid="6496c699bd404076b12a6edcc40889dafaeb5285" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r1.7" ];then + branch="master" + patch_file_name="r1.7" + commitid="3406fdabaee92f1b22ce0703fa25befa3c40d18e" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r1.8" ];then + branch="r1.8" + patch_file_name="r1.8" + commitid="e68e09de9e97eeccc0804bc5f43f764a7a2bdbee" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r1.8" ];then + branch="master" + patch_file_name="r1.8" + commitid="b68b6bfa919465567d89bc7fdcf6d0e63967d5aa" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r1.9" ];then + branch="r1.9" + patch_file_name="r1.9" + commitid="5318681496ef9a37d337737325ad1b238ef75917" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r1.10" ];then + branch="r1.10" + patch_file_name="r1.10" + commitid="8f7331e6a846e7c306dc8ac30313d9f07cf6ee98" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/bert" + elif [ "$branch_args" == "r2.0" ];then + branch="r2.0" + patch_file_name="r2.0" + commitid="f211f336e8bee3cf531bcad5f611f408069c6f9f" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/Bert" + elif [ "$branch_args" == "r2.1" ];then + branch="r2.1" + patch_file_name="r2.1" + commitid="44f2dc18e9bd52c6bcadd18f6567817ad798f641" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/Bert" + elif [ "$branch_args" == "r2.2" ];then + branch="master" + patch_file_name="r2.2" + commitid="bb9ab4fdfb2fc205ffeb4dd671be77312908ef88" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/Bert" + elif [ "$branch_args" == "r2.3" ];then + branch="master" + patch_file_name="r2.3" + commitid="c94da0701a9ede6c93df4cd5fec88df7942a1dcc" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/Bert" + else + echo "bad parameters : $1" + return $ret_error + fi + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch new file mode 100644 index 0000000..f9b4505 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch @@ -0,0 +1,76 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-12-16 14:59:50.550000000 +0800 ++++ code/pretrain_eval.py 2022-12-16 14:59:50.580000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-12-16 14:59:50.530000000 +0800 ++++ code/run_pretrain.py 2022-12-16 14:59:50.560000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -44,6 +45,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + + def _set_bert_all_reduce_split(): +@@ -268,9 +275,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch new file mode 100644 index 0000000..154dab2 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch @@ -0,0 +1,75 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2021-12-10 14:27:15.670000000 +0800 ++++ code/pretrain_eval.py 2021-12-10 14:27:15.680000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2021-12-10 14:27:15.670000000 +0800 ++++ code/run_pretrain.py 2021-12-10 14:27:15.670000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -45,6 +46,12 @@ + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) + ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -262,8 +269,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + if __name__ == '__main__': diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch new file mode 100644 index 0000000..8874b51 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch @@ -0,0 +1,77 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-01-25 15:31:09.227309487 +0800 ++++ code/pretrain_eval.py 2022-01-25 15:31:09.235309621 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-01-25 15:31:09.227309487 +0800 ++++ code/run_pretrain.py 2022-01-25 15:31:09.235309621 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -44,7 +45,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -262,8 +268,32 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() ++ + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + if __name__ == '__main__': diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch new file mode 100644 index 0000000..6ae1e5a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch @@ -0,0 +1,77 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-05-30 13:10:29.271296733 +0800 ++++ code/pretrain_eval.py 2022-05-30 13:10:29.283296877 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-05-30 13:10:29.271296733 +0800 ++++ code/run_pretrain.py 2022-05-30 13:10:29.283296877 +0800 +@@ -17,6 +17,8 @@ + python run_pretrain.py + """ + import os ++import time ++ + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -43,6 +45,13 @@ + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True ++ + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -262,8 +271,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + if __name__ == '__main__': diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch new file mode 100644 index 0000000..ed4e853 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch @@ -0,0 +1,78 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-07-07 09:44:09.800000000 +0800 ++++ code/pretrain_eval.py 2022-07-07 09:44:09.816000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-07-07 09:44:09.800000000 +0800 ++++ code/run_pretrain.py 2022-07-07 09:44:09.816000000 +0800 +@@ -17,6 +17,8 @@ + python run_pretrain.py + """ + import os ++import time ++ + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -43,6 +45,13 @@ + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True ++ + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -262,9 +271,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch new file mode 100644 index 0000000..627181e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch @@ -0,0 +1,80 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2022-10-18 11:29:34.308000000 +0800 ++++ code/pretrain_eval.py 2022-10-18 11:29:34.324000000 +0800 +@@ -53,9 +53,17 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + +- + if __name__ == "__main__": + DEVICE_ID = 0 + os.environ['DEVICE_ID'] = str(DEVICE_ID) +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2022-10-18 11:29:34.312000000 +0800 ++++ code/run_pretrain.py 2022-10-18 11:29:34.324000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -43,6 +44,13 @@ + from src.model_utils.config import config as cfg, bert_net_cfg + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True ++ + _current_dir = os.path.dirname(os.path.realpath(__file__)) + + +@@ -268,9 +276,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch new file mode 100644 index 0000000..2f8dd46 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch @@ -0,0 +1,77 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/pretrain_eval.py 2023-03-27 15:45:24.796000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/run_pretrain.py 2023-03-27 15:45:24.796000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -44,7 +45,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -268,9 +274,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch new file mode 100644 index 0000000..2f8dd46 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch @@ -0,0 +1,77 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/pretrain_eval.py 2023-03-27 15:45:24.796000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/run_pretrain.py 2023-03-27 15:45:24.796000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -44,7 +45,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -268,9 +274,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch new file mode 100644 index 0000000..2f8dd46 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch @@ -0,0 +1,77 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/pretrain_eval.py 2023-03-27 15:45:24.796000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/run_pretrain.py 2023-03-27 15:45:24.796000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -44,7 +45,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -268,9 +274,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch new file mode 100644 index 0000000..417525e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch @@ -0,0 +1,77 @@ +diff -Nur origin/pretrain_eval.py code/pretrain_eval.py +--- origin/pretrain_eval.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/pretrain_eval.py 2023-03-27 15:45:24.796000000 +0800 +@@ -53,6 +53,15 @@ + print("==============================================================") + for _, v in res.items(): + print("Accuracy is: ", v) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(v)) + print("==============================================================") + + +diff -Nur origin/run_pretrain.py code/run_pretrain.py +--- origin/run_pretrain.py 2023-03-27 15:45:24.776000000 +0800 ++++ code/run_pretrain.py 2023-03-27 15:45:24.796000000 +0800 +@@ -17,6 +17,7 @@ + python run_pretrain.py + """ + import os ++import time + import mindspore.communication.management as D + from mindspore.communication.management import get_rank + import mindspore.common.dtype as mstype +@@ -44,7 +45,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id, get_device_num + _current_dir = os.path.dirname(os.path.realpath(__file__)) +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _set_bert_all_reduce_split(): + """set bert all_reduce fusion split, support num_hidden_layers is 12 and 24.""" +@@ -268,9 +274,31 @@ + callback.append(eval_callback) + + model = ConvertModelUtils().convert_to_thor_model(model, network=net_with_grads, optimizer=optimizer) ++ model.build(ds, sink_size=cfg.data_sink_steps, epoch=new_repeat_count) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(new_repeat_count, ds, callbacks=callback, + dataset_sink_mode=(cfg.enable_data_sink == "true"), sink_size=cfg.data_sink_steps) +- ++ all_data_sum = new_repeat_count * cfg.data_sink_steps * cfg.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(0) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..d72557e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + : "${EVAL_DATA_PATH?EVAL_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + + [ -d $BASE_PATH/result ] && cp ${RESULT_PATH}/* -rf $BASE_PATH/result/ + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh new file mode 100644 index 0000000..9d93256 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh + +init() +{ + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + logger_Info "init called" +} + +run_train() +{ + logger_Info "run_train called" + [ ! -f $CODE_PATH/code/ma-pre-start.sh ] && touch $CODE_PATH/code/ma-pre-start.sh + sed -i '/SINGLESERVER_MODE=/d' $CODE_PATH/code/ma-pre-start.sh + [[ $MODELARTS_VERSION ]]&&[[ $MODELARTS_VERSION == "V2" ]] && modelarts_version="V2" || modelarts_version="V1" + if [ "$SINGLESERVER_MODE" == "True" ];then + echo "now set singleserver_mode OK" + echo -e "\nexport SINGLESERVER_MODE=True" >> $CODE_PATH/code/ma-pre-start.sh + + ${PYTHON_COMMAND} -u ${CODE_PATH}/common/train_modelarts.py --local_code_path $CODE_PATH/code --single_server_mode --modelarts_version $modelarts_version || { logger_Warn "run train modelarts failed ret:$?";return 1; } + else + echo "now not set singleserver_mode" + ${PYTHON_COMMAND} -u ${CODE_PATH}/common/train_modelarts.py --local_code_path $CODE_PATH/code --modelarts_version $modelarts_version || { logger_Warn "run train modelarts failed ret:$?";return 1; } + fi + ${PYTHON_COMMAND} $CODE_PATH/ais_utils.py set_result "training" "result" "OK" +} + +run_eval() +{ + logger_Info "run_eval called" +} + +get_result() +{ + logger_Info "get_result called" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh new file mode 100644 index 0000000..3b03d7e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh @@ -0,0 +1,113 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="true" || DISTRUTE_ENABLE="false" + + CONFIG_FILE=$WORK_PATH/code/pretrain_config_Ascend_Boost.yaml + + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/run_pretrain.py \ + --distribute=$DISTRUTE_ENABLE \ + --epoch_size=$EPOCH_SIZE \ + --enable_save_ckpt=true \ + --enable_lossscale=true \ + --do_shuffle=true \ + --enable_data_sink=true \ + --data_sink_steps=100 \ + --accumulation_steps=1 \ + --save_checkpoint_path=$RUN_PATH \ + --save_checkpoint_steps=$TRAIN_STEPS \ + --save_checkpoint_num=1 \ + --load_checkpoint_path=$PRETRAIN_MODEL_PATH \ + --data_dir=${TRAIN_DATA_PATH} \ + --device_id=${DEVICE_ID} \ + --device_num=${DEVICE_NUM} \ + --train_steps=${TRAIN_STEPS} \ + --config_path=$CONFIG_FILE + " + + export MS_DISABLE_REF_MODE=0 + export MS_ENABLE_FORMAT_MODE=0 + return 0 +} + +function get_eval_cmd() +{ + chipname=`npu-smi info -t board -i 0 -c 0 | grep 'Chip Name' | awk {'print $4'}` + CONFIG_FILE=$WORK_PATH/code/pretrain_config_Ascend_Boost.yaml + sed -i "s|eval_data_dir:.*|eval_data_dir: '$EVAL_DATA_PATH'|g" "$CONFIG_FILE" + sed -i "s|schema_file:.*|schema_file: null|g" "$CONFIG_FILE" + sed -i "s|eval_ckpt:.*|eval_ckpt: '$CHECKPOINT_PATH'|g" "$CONFIG_FILE" + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/pretrain_eval.py --config_path=$CONFIG_FILE" + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_file_valid ${PRETRAIN_MODEL_PATH} || { logger_Warn "PRETRAIN_MODEL_PATH:${PRETRAIN_MODEL_PATH} not valid" ; return 1; } + logger_Debug "PRETRAIN_MODEL_PATH path valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + + check_path_valid "${EVAL_DATA_PATH}" || { logger_Warn "EVAL_DATA_PATH:${EVAL_DATA_PATH} not valid path" ; return 1; } + logger_Debug "EVAL_DATA_PATH is valid" +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + cp $CHECKPOINT_PATH $RESULT_PATH/ + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh new file mode 100644 index 0000000..065998c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh new file mode 100644 index 0000000..8091e85 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh @@ -0,0 +1,16 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7 +export TRAIN_DATA_PATH=/home/datasets/vocaug +export TRAIN_DATA_FILE=/home/datasets/vocaug/vocaug_mindrecord/vocaug_mindrecord0 +export PRETRAIN_MODEL_PATH=/home/datasets/pretrain_model/deeplabv3/resnet101_ascend_v120_imagenet2012_official_cv_bs32_acc78.ckpt +export EVAL_DATA_FILE_PATH=/home/datasets/vocaug/voc_val_lst.txt +export EPOCH_SIZE=200 + +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# need if rank_size > 1 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_64.json +# cluster need for node info +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..a98a0fc --- /dev/null +++ "b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,75 @@ +# Ais-Bench+Mindspore+deeplabv3使用说明 + +## 简介 + +AI Server Benchmark 是按《信息技术 人工智能 服务器系统性能测试规范》对人工智能服务器系统的性能进行性能评估的测试系统（测试套件），简称Ais-Bench软件。 + +## 使用前提 + +本程序包运行需要基于以下前提 + +1. Atlas 800-9000设备 +2. 安装好CANN包和Mindspore对应版本。并可以运行正常mindspore测试程序。 +3. 保存数据集和相关预处理文件等到设备中。 + +## 集群节点配置 + +如果运行设备大于1个设备，那么需要运行设置ssh节点文件。说明节点信息 +{ +"cluster": { +"xx.xx.xx.xx": { # 节点ip 必须与ranktable中对应 +"user": "xxxx", # 用户名 免密可以不用设置 +"pd": "xx", # 密码 免密不用设置 +"port": xx # 端口 默认22 可以不用设置 +}, +"xx.xx.xx.xx": { +"user": "xxxx", +"pd": "xx", +"port": xx +} +} +} + +## 集群节点免密设置 + +设置密钥认证的参考操作如下： + ++ ssh-keygen -t rsa -b 2048 # 登录管理节点并生成SSH Key。安全起见，建议用户到"Enter passphrase"步骤时输入密钥密码，且符合密码复杂度要求。建议执行这条命令前先将umask设置为0077，执行完后再恢复原来umask值。 + ++ ssh-copy-id -i ~/.ssh/id_rsa.pub ``@`` # 将管理节点的公钥拷贝到所有节点的机器上，``@``替换成要拷贝到的对应节点的账户和ip。 + ++ 设置ssh代理管理ssh密钥，避免工具批量安装操作过程中输入密钥密码和节点密码 + +``` +ssh-agent bash # 开启ssh-agent的bash进程 +``` + + +``` +ssh-add # 向ssh-agent添加私钥 +``` + + +## 配置文件信息 + +> ``` +> export PYTHON_COMMAND=python3.7 +> export TRAIN_DATA_PATH=/home/datasets/VOCdevkit/VOC2012 +> export TRAIN_DATA_FILE=/home/datasets/VOCdevkit/VOC2012/dataset/mindrecored_deeplabv3.mindrecord0 +> export PRETRAIN_MODEL_PATH=/home/datasets/pretrain_model/deeplabv3/resnet101_ascend_v120_imagenet2012_official_cv_bs32_acc78.ckpt +> export EVAL_DATA_FILE_PATH=/home/datasets/VOCdevkit/VOC2012/voc_val_lst.txt +> export EPOCH_SIZE=200 +> +> export RANK_SIZE=8 +> export DEVICE_NUM=8 +> +> # need if rank_size > 1 +> export RANK_TABLE_FILE=/home/tools/rank_table_8p_64.json +> # cluster need for node info +> #export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json +> ``` + +说明： +配置文件默认是8卡训练。 +单卡训练时，需要设置RANK_SIZE=1，DEVICE_NUM=1，且不能使用RANK_TABLE_FILE环境变量. +同时还请按需增加指定执行卡序号变量声明export SINGLE_CARD_INDEX。默认 SINGLE_CARD_INDEX=0，可以不显式声明。其它卡时需要显式声明，比如export SINGLE_CARD_INDEX=6 diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh new file mode 100644 index 0000000..de43867 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="abc34438588942642e45e7cf1e516134952a2f86" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/deeplabv3" + elif [ "$branch_args" == "r1.6" ];then + branch="master" + patch_file_name="r1.6" + commitid="a58deaa4745a71fef902b73ed220054b6c072f24" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/deeplabv3" + elif [ "$branch_args" == "r1.9" ];then + branch="master" + patch_file_name="r1.9" + commitid="adccb235dc4d0f00a8d9abd6cfcc2fc43c83570b" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/deeplabv3" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch new file mode 100644 index 0000000..4072566 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch @@ -0,0 +1,56 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-09-05 17:24:42.232000000 +0800 ++++ code/eval.py 2022-09-05 17:24:42.244000000 +0800 +@@ -247,6 +247,16 @@ + iu = np.diag(hist) / (hist.sum(1) + hist.sum(0) - np.diag(hist)) + print('per-class IoU', iu) + print('mean IoU', np.nanmean(iu)) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ print("ACC_DIR:", ACC_DIR) ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(np.nanmean(iu))) + + + if __name__ == '__main__': +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-09-05 17:24:42.240000000 +0800 ++++ code/train.py 2022-09-05 17:24:42.248000000 +0800 +@@ -201,8 +201,31 @@ + keep_checkpoint_max=args.keep_checkpoint_max) + ckpoint_cb = ModelCheckpoint(prefix=args.model, directory=args.train_dir, config=config_ck) + cbs.append(ckpoint_cb) ++ model.build(dataset, sink_size=dataset.get_dataset_size(), epoch=args.train_epochs) + ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(args.train_epochs, dataset, callbacks=cbs, dataset_sink_mode=(args.device_target != "CPU")) ++ all_data_sum = args.train_epochs * dataset.get_dataset_size() * args.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum / (end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + if __name__ == '__main__': + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..2bdca78 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_FILE?TRAIN_DATA_FILE not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh new file mode 100644 index 0000000..a696118 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh @@ -0,0 +1,106 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# ȡѵ +function get_train_cmd() +{ + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py --data_file=$TRAIN_DATA_FILE \ + --train_dir=$RUN_PATH \ + --train_epochs=$EPOCH_SIZE \ + --batch_size=32 \ + --crop_size=513 \ + --base_lr=0.015 \ + --lr_type=cos \ + --min_scale=0.5 \ + --max_scale=2.0 \ + --ignore_label=255 \ + --num_classes=21 \ + --model=deeplab_v3_s16 \ + --ckpt_pre_trained=$PRETRAIN_MODEL_PATH \ + --save_steps=1500 \ + --keep_checkpoint_max=200 + " + return 0 +} + +function get_eval_cmd() +{ + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --data_root=$TRAIN_DATA_PATH \ + --data_lst=$EVAL_DATA_FILE_PATH \ + --batch_size=32 \ + --crop_size=513 \ + --ignore_label=255 \ + --num_classes=21 \ + --model=deeplab_v3_s16 \ + --scales_type=0 \ + --freeze_bn=True \ + --ckpt_path=$CHECKPOINT_PATH \ + " + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # ͨü Ҫ PYTHON_COMMAND RANK_SIZERANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # ǷװӦ + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_file_valid "${TRAIN_DATA_FILE}" || { logger_Warn "TRAIN_DATA_FILE:${TRAIN_DATA_FILE} not valid file" ; return 1; } + logger_Debug "TRAIN_DATA_FILE is valid" +} + + +function node_train() +{ + # ͨѵӿ + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh new file mode 100644 index 0000000..065998c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh new file mode 100644 index 0000000..419eba6 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh @@ -0,0 +1,6 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7 +export DEVICE_TARGET='CPU' +export RANK_SIZE=1 +export DEVICE_NUM=1 + diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..6f346a3 --- /dev/null +++ "b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,109 @@ +# Ais-Bench+Mindspore+deepspeechv2使用说明 + +## 1.简介 + +AI Server Benchmark 是按《信息技术 人工智能 服务器系统性能测试规范》对人工智能服务器系统的性能进行性能评估的测试系统（测试套件），简称Ais-Bench软件。 + +## 2.使用前提 + +本程序包运行需要基于以下前提 + +1. Atlas 800-9000设备 +2. 安装好CANN包和Mindspore对应版本。并可以运行正常mindspore测试程序。 +3. 保存数据集和相关预处理文件等到设备中。 + +## 3.集群节点配置 + +### 3.1 rank_table文件 + +单机或集群rank_table文件生成方法，请参照[这里](https://gitee.com/mindspore/models/tree/master/utils/hccl_tools#merge_hccl)。 + +示例：rank_table_16p_64_66.json + +```bash +{ + "version": "1.0", + "server_count": "2", + "server_list": [ + { + "server_id": "xx.xx.xx.xx", + "device": [ + {"device_id": "0", "device_ip": "xx.xx.xx.xx", "rank_id": "0"}, + {"device_id": "1", "device_ip": "xx.xx.xx.xx", "rank_id": "1"}, + {"device_id": "2", "device_ip": "xx.xx.xx.xx", "rank_id": "2"}, + {"device_id": "3", "device_ip": "xx.xx.xx.xx", "rank_id": "3"}, + {"device_id": "4", "device_ip": "xx.xx.xx.xx", "rank_id": "4"}, + {"device_id": "5", "device_ip": "xx.xx.xx.xx", "rank_id": "5"}, + {"device_id": "6", "device_ip": "xx.xx.xx.xx", "rank_id": "6"}, + {"device_id": "7", "device_ip": "xx.xx.xx.xx", "rank_id": "7"} + ], + "host_nic_ip": "reserve" + }, + { + "server_id": "xx.xx.xx.xx", + "device": [ + {"device_id": "0", "device_ip": "xx.xx.xx.xx", "rank_id": "8"}, + {"device_id": "1", "device_ip": "xx.xx.xx.xx", "rank_id": "9"}, + {"device_id": "2", "device_ip": "xx.xx.xx.xx", "rank_id": "10"}, + {"device_id": "3", "device_ip": "xx.xx.xx.xx", "rank_id": "11"}, + {"device_id": "4", "device_ip": "xx.xx.xx.xx", "rank_id": "12"}, + {"device_id": "5", "device_ip": "xx.xx.xx.xx", "rank_id": "13"}, + {"device_id": "6", "device_ip": "xx.xx.xx.xx", "rank_id": "14"}, + {"device_id": "7", "device_ip": "xx.xx.xx.xx", "rank_id": "15"} + ], + "host_nic_ip": "reserve" + } + ], + "status": "completed" +} +``` + +### 3.2 ssh节点文件 + +运行设备大于1个设备，则需要运行设置ssh节点文件。说明节点信息 +示例：ssh64_66.json + +```bash +{ + "cluster": { + "xx.xx.xx.xx": { # 节点ip 必须与ranktable中的server_id一一对应 + "user": "xxxx", # 用户名 免密可以不用设置 + "pd": "xxxx", # 密码 免密不用设置 + "port": xx # 容器端口，默认22。可以不设置。本行缺失时，表示测试在该节点本地（非容器）运行，设置时表示在容器中运行并提供指定端口访问能力 + }, + "xx.xx.xx.xx": { + "user": "xxxx", + "pd": "xxxx", + "port": xx + } + } +} +``` + +注意：该文件中的节点数目应与rank_table中的的节点数目一致。 + +## 4.集群节点免密设置 + +设置密钥认证的参考操作如下： + ++ ssh-keygen -t rsa -b 2048 # 登录管理节点并生成SSH Key。安全起见，建议用户到"Enter passphrase"步骤时输入密钥密码，且符合密码复杂度要求。建议执行这条命令前先将umask设置为0077，执行完后再恢复原来umask值。 ++ ssh-copy-id -i ~/.ssh/id_rsa.pub ``@`` # 将管理节点的公钥拷贝到所有节点的机器上，``@``替换成要拷贝到的对应节点的账户和ip。 ++ 设置ssh代理管理ssh密钥，避免工具批量安装操作过程中输入密钥密码和节点密码 + +``` + ssh-agent bash # 开启ssh-agent的bash进程 + ssh-add # 向ssh-agent添加私钥 +``` + +## 5.配置文件信息 + +> #!/bin/bash +> export PYTHON_COMMAND=python3.7 +> export DEVICE_TARGET='CPU' +> export RANK_SIZE=1 +> export DEVICE_NUM=1 + +说明： +配置文件默认是8卡训练。 +单卡训练时，需要设置RANK_SIZE=1，DEVICE_NUM=1，且不能使用RANK_TABLE_FILE环境变量. +同时还请按需增加指定执行卡序号变量声明export SINGLE_CARD_INDEX。默认 SINGLE_CARD_INDEX=0，可以不显式声明。其它卡时需要显式声明，比如export SINGLE_CARD_INDEX=6 diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh new file mode 100644 index 0000000..ea28dc0 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="abc34438588942642e45e7cf1e516134952a2f86" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/research/audio/deepspeech2" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..2fc1ff6 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh @@ -0,0 +1,77 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh new file mode 100644 index 0000000..79af49a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh @@ -0,0 +1,81 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + cp $WORK_PATH/code/labels.json $RUN_PATH/ + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py --device_target=$DEVICE_TARGET " + return 0 +} + +function get_eval_cmd() +{ + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --pretrain_ckpt $CHECKPOINT_PATH \ + --device_target=$DEVICE_TARGET \ + " + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" +} + + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh new file mode 100644 index 0000000..065998c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh new file mode 100644 index 0000000..16c8008 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh @@ -0,0 +1,16 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7 +export TRAIN_DATA_PATH=/home/datasets/coco +export MINDRECORD_PATH=/home/datasets/coco/mindrecord_coco_train +export PRETRAIN_MODEL_PATH=/home/datasets/pretrain_model/faster_rcnn/pretrained_model.ckpt +export VALIDATION_JSON_FILE=/home/datasets/coco/annotations/instances_val2017.json +export EPOCH_SIZE=20 + +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# need if rank_size > 1 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_66.json +# cluster need for node info +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..efa2897 --- /dev/null +++ "b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,72 @@ +# Ais-Bench+Mindspore+deeplabv3使用说明 + +## 简介 + +AI Server Benchmark 是按《信息技术 人工智能 服务器系统性能测试规范》对人工智能服务器系统的性能进行性能评估的测试系统（测试套件），简称Ais-Bench软件。 + +## 使用前提 + +本程序包运行需要基于以下前提 + +1. Atlas 800-9000设备 +2. 安装好CANN包和Mindspore对应版本。并可以运行正常mindspore测试程序。 +3. 保存数据集和相关预处理文件等到设备中。 +4. 支持Resnet50预训练的ckpt文件作为backbone.ckpt + +## 集群节点配置 + +如果运行设备大于1个设备，那么需要运行设置ssh节点文件。说明节点信息 +{ +"cluster": { +"xx.xx.xx.xx": { # 节点ip 必须与ranktable中对应 +"user": "xxxx", # 用户名 免密可以不用设置 +"pd": "xx", # 密码 免密不用设置 +"port": xx # 端口 默认22 可以不用设置 +}, +"xx.xx.xx.xx": { +"user": "xxxx", +"pd": "xx", +"port": xx +} +} +} + +## 集群节点免密设置 + +设置密钥认证的参考操作如下： + ++ ssh-keygen -t rsa -b 2048 # 登录管理节点并生成SSH Key。安全起见，建议用户到"Enter passphrase"步骤时输入密钥密码，且符合密码复杂度要求。建议执行这条命令前先将umask设置为0077，执行完后再恢复原来umask值。 ++ ssh-copy-id -i ~/.ssh/id_rsa.pub ``@`` # 将管理节点的公钥拷贝到所有节点的机器上，``@``替换成要拷贝到的对应节点的账户和ip。 ++ 设置ssh代理管理ssh密钥，避免工具批量安装操作过程中输入密钥密码和节点密码 + +``` +ssh-agent bash # 开启ssh-agent的bash进程 +``` + +``` +ssh-add # 向ssh-agent添加私钥 +``` + +## 配置文件信息 + +> ``` +> export PYTHON_COMMAND=python3.7 +> export TRAIN_DATA_PATH=/home/datasets/VOCdevkit/VOC2012 +> export TRAIN_DATA_FILE=/home/datasets/VOCdevkit/VOC2012/dataset/mindrecored_deeplabv3.mindrecord0 +> export PRETRAIN_MODEL_PATH=/home/datasets/pretrain_model/deeplabv3/resnet101_ascend_v120_imagenet2012_official_cv_bs32_acc78.ckpt +> export EVAL_DATA_FILE_PATH=/home/datasets/VOCdevkit/VOC2012/voc_val_lst.txt +> export EPOCH_SIZE=200 +> +> export RANK_SIZE=8 +> export DEVICE_NUM=8 +> +> # need if rank_size > 1 +> export RANK_TABLE_FILE=/home/tools/rank_table_8p_64.json +> # cluster need for node info +> #export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json +> ``` + +说明： +配置文件默认是8卡训练。 +单卡训练时，需要设置RANK_SIZE=1，DEVICE_NUM=1，且不能使用RANK_TABLE_FILE环境变量. +同时还请按需增加指定执行卡序号变量声明export SINGLE_CARD_INDEX。默认 SINGLE_CARD_INDEX=0，可以不显式声明。其它卡时需要显式声明，比如export SINGLE_CARD_INDEX=6 diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh new file mode 100644 index 0000000..4123cfa --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="abc34438588942642e45e7cf1e516134952a2f86" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/faster_rcnn" + elif [ "$branch_args" == "r1.9" ];then + branch="master" + patch_file_name="r1.9" + commitid="85ecbf257f70f7a5ff45640229c529a1c3690e97" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/faster_rcnn" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch new file mode 100644 index 0000000..5fe1eff --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch @@ -0,0 +1,71 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-09-21 10:05:25.304000000 +0800 ++++ code/eval.py 2022-09-21 10:05:25.312000000 +0800 +@@ -123,8 +123,18 @@ + eval_types = ["bbox"] + result_files = results2json(dataset_coco, outputs, "./results.pkl") + +- coco_eval(config, result_files, eval_types, dataset_coco, ++ result = coco_eval(config, result_files, eval_types, dataset_coco, + single_result=False, plot_detect_result=True) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ print("ACC_DIR:", ACC_DIR) ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(result)) + print("\nEvaluation done!") + + +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-09-21 10:05:25.308000000 +0800 ++++ code/train.py 2022-09-21 10:05:25.312000000 +0800 +@@ -35,6 +35,12 @@ + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_id ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + + def train_fasterrcnn_(): +@@ -225,8 +231,31 @@ + cb += [eval_cb] + + model = Model(net) ++ model.build(dataset, sink_size=dataset_size, epoch=config.epoch_size) ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size, dataset, callbacks=cb) ++ all_data_sum = config.epoch_size * dataset.get_dataset_size() * config.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum / (end_time - start_time) + ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + set_seed(1) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..ed6c8cd --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + + [ -d $BASE_PATH/result ] && cp ${RESULT_PATH}/* -rf $BASE_PATH/result/ + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh new file mode 100644 index 0000000..b851bf4 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh @@ -0,0 +1,104 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + COCO_CONFIG_FILE=$WORK_PATH/code/default_config.yaml + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py --config_path=$COCO_CONFIG_FILE \ + --coco_root=$TRAIN_DATA_PATH \ + --pre_trained=$PRETRAIN_MODEL_PATH \ + --backbone="resnet_v1.5_50" \ + --mindrecord_dir=$MINDRECORD_PATH \ + " + + # for mindspore1.5 + export ENV_FUSION_CLEAR=1 + export ENV_SINGLE_EVAL=1 + export SKT_ENABLE=1 + export DATASET_ENABLE_NUMA=True +} + +function get_eval_cmd() +{ + CONFIG_FILE=$WORK_PATH/code/default_config.yaml + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --config_path=$CONFIG_FILE \ + --device_id=0 \ + --anno_path=/home/datasets/coco/annotations/instances_val2017.json \ + --checkpoint_path=$WORK_PATH/code/scripts/train_parallel0/ckpt_0/faster_rcnn-20_7393.ckpt \ + --backbone=resnet_v1.5_50 \ + --coco_root=/home/datasets/coco \ + --mindrecord_dir=/home/datasets/coco/mindrecord_coco_train" + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + + # 检测是否安装对应框架软件 + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid file" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + + check_file_valid "${PRETRAIN_MODEL_PATH}" || { logger_Warn "PRETRAIN_MODEL_PATH:${PRETRAIN_MODEL_PATH} not valid file" ; return 1; } + logger_Debug "PRETRAIN_MODEL_PATH is valid" + + check_file_valid "${VALIDATION_JSON_FILE}" || { logger_Warn "VALIDATION_JSON_FILE:${VALIDATION_JSON_FILE} not valid file" ; return 1; } + logger_Debug "VALIDATION_JSON_FILE is valid" +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md new file mode 100644 index 0000000..846beb4 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md @@ -0,0 +1,201 @@ +# 基于Mindspore/mindformers框架的glm2大模型训练负载使用指南 +本文主要介绍使用基于glm2 大模型训练业务代码构建的AISBench的负载包，进行服务器性能测试的流程。 +## 运行环境前置条件 +``` +python >= 3.7 +mindspore >= 2.2 +``` +MindSpore安装参考[MindSpore官网](https://www.mindspore.cn/)MindSpore需要能成功在npu上运行，验证命令： +```bash +python -c "import mindspore;mindspore.set_context(device_target='Ascend');mindspore.run_check()" +``` +如果正常输出： +```bash +MindSpore version: 版本号 +The result of multiplication calculation is correct, MindSpore has been installed on platform [Ascend] successfully! +``` +说明成功。 + +## 负载包中文件夹主要目录结构 + +``` +├── ais-bench-stubs # Stubs主程序，负责流程控制、通信与数据管理等 +├── code # 业务代码目录 +│ ├── benchmark.sh # 入口脚本，会被ais-bench-stubs调用，调用业务代码，被测试者需要通过编写该脚本，对接运行的训练和推理脚本 +│ ├── config +│ │ ├── config.sh # 训练相关的配置文件，包括数据集、权重路径等信息 +│ ├── code # mindformers全部代码，嵌入了AISBench的打点上报接口 +│ │ └──mindformers +│ ├── cluster_offline_run.sh +│ ├── run_node.sh +│ ├── run_glm2_6b_finetune.yaml +│ └── run_glm2_6b_finetune_eval.yaml +├── config +│ ├── config.json # 测试配置文件，包含tester服务器信息、testerId等信息 +│ └── system.json # 被测试环境系统基本信息json文件，被测试者自行上传，比如硬件信息等 +├── dependencies # stubs的依赖组件 +│ ├── cluster # 分布式运行组件 +│ │ ├── ais_bench_cluster--py3-none-linux_.whl +│ │ ├── README.md +│ └── logging # 测试结果传输模块 +│ ├── ais_utils.py # 打点入口脚本，设置相关业务运行参数并反馈测试结果 +│ └── libais_utils.so # 测试结果传输模块lib，负责将测试结果传输到stubs模块相关文件部署 +├── log # 测试log日志。建议无需上传的日志文件，另建目录存放 +├── result # 测试结果文件。建议无需上传的结果文件，另建目录存放 +└── STUBS_PACKAGE_INTRO.md # Stubs被测试者接入使用文档 +``` +- **后续对于相对路径的描述都是相对于负载包中的一级目录，例如 ./ais-bench-stubs表示Stubs主程序** +- 运行环境安装mindformer需在code/code目录下执行`pip3 install .` +## 资源准备 +### 前置声明 +- 运行glm2训练的Mindspore/mindformers的代码全部在`./code/code`文件夹中，资源的准备参考[glm2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/glm2.md),具体资源的参考详见本章其他小节。 +- **注意**：需要确认环境中是否原来已经安装了mindformers，如果安装了，请使用`pip uninstall mindformers`卸载，确保负载代码的mindformers能正常安装。 +### rank_table_file准备 +- 确保`/etc/hccn.conf`文件已经配好（如果没配好，参考[数据中心解决方案/配置训练节点](https://www.hiascend.com/document/detail/zh/Ascend%20Data%20Center%20Solution/22.0.0/install/800_9000/install_800_9000_0029.html)配置）。 + +- 参考[glm2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/glm2.md)的“生成RANK_TABLE_FILE”(单机多卡情况)章节。 + +### 模型权重下载与转换 +- 参考[glm2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/glm2.md)的“模型权重下载与转换”章节； +- 资源链接： + - [glm2_6b.ckpt](https://ascend-repo-modelzoo.obs.cn-east-2.myhuaweicloud.com/XFormer_for_mindspore/glm2/glm2_6b.ckpt)(点击直接下载) + - [tokenizer](https://ascend-repo-modelzoo.obs.cn-east-2.myhuaweicloud.com/XFormer_for_mindspore/glm2/tokenizer.model)(点击直接下载) +- 下载后建议放至code/code/mindformers/checkpoint_download/glm2目录下(需手动创建checkpoint_download/glm2 如`mkdir -p code/code/mindformers/checkpoint_download/glm2`) +### 数据集准备 +- 参考[glm2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/glm2.md)的“微调--数据集准备”章节； +- 资源链接： + - [ADGEN数据集](https://cloud.tsinghua.edu.cn/f/b3f119a008264b1cabd1/?dl=1)(下载后需解压) +- 下载解压后目录结构为： + ``` + AdvertiseGen + ├── train.json + └── dev.json + ``` +- 建议该目录放到code/code/mindformers/dataset_files/目录下(dataset_files需手动创建，如`mkdir -p code/code/mindformers/dataset_files/`) + +## 2 负载启动前配置项 +### 2.0 和tester连接的配置（仅在线测试需要） +`./config/config.json`和`./config/system.json`请参考《Stubs被测试者接入使用文档》中的“配置与Tester相关的配置文件”章节以及测试机构的要求进行配置。 +### 2.1 ./code/config/config.sh配置 +`./code/config/config.sh`内容如下： +```bash +#!/bin/bash +echo "set env of glm2 train" + +export PYTHON_COMMAND=python3 +# 以下cluster配置二选一，仅多机场景需要，目前glm2不支持多机，不涉及 +export CLUSTER_SSH_KEY_PATH=~/.ssh/id_rsa # 用户指定的ssh私钥，确保通过此私钥管理节点能免密访问所有计算节点(单机场景注释此行) +export CLUSTER_AUTO_SET_KEY='on' # 'off' or 'on'， 若为'on' 不需要配置CLUSTER_SSH_KEY_PATH(单机场景注释此行) + +export GLM_RUN_MODE='only_finetune' + +# FINETUNE_CKPT_PATH, FINETUNE_DATA_PATH, EVAL_DATASET_PATH 这三个路径是相对mindformers源码的路径, 必须以./mindformers/开头 +# 可以在下载对应数据集完成后，将其复制到code/code/mindformers目录下，比如新建一个dataset_files存放解压后的AdvertiseGen +export FINETUNE_DATA_PATH=./mindformers/dataset_files/AdvertiseGen/train.json # 微调数据集实际路径 +export EVAL_DATASET_TYPE='ADGEN' # 'ADGEN' +export EVAL_DATASET_PATH=./mindformers/dataset_files/AdvertiseGen/dev.json # 评测用的数据集路径，必须以./mindformers/开头 +export FINETUNE_CKPT_PATH=./mindformers/checkpoint_download/glm2/glm2_6b.ckpt # 微调使用的预训练权重，必须以./mindformers/开头 +export EVAL_DEVICE_ID=0 # 评测用的npu 的device id + +export EPOCH_SIZE=1 +export GLM_LAYER_NUM=4 + +export RANK_SIZE=8 # 集群总加速卡数 +export DEVICE_NUM=8 # 集群每个节点的加速卡数 + +# parallel run params, parallel strategy config, DATA_PARALLEL * MODEL_PARALLEL * PIPELINE_STAGE should equal to RANK_SIZE +export DATA_PARALLEL=2 +export MODEL_PARALLEL=1 +export PIPELINE_STAGE=4 + +# need if rank_size > 1 +export RANK_TABLE_FILE=./hccl_xxxx_8p.json # 配置为生成的rank table路径，是相对于负载仓的code目录的路径，如果不在code目录下需要拷贝到code目录下 + +# 多机多卡需要配置，单机不需要配置，glm2不涉及 +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json +``` + +- 请参考`./code/config/config.sh`的注释将"资源准备"章节准备的资源的路径在`config.sh`中配置好， + +### 2.2 yaml配置 +- 前置声明：所有修改路径均为绝对路径 +- 需要修改code/run_glm2_6b_finetune_eval.yaml: +``` +train_dataset: &train_dataset + data_loader: + type: ADGenDataLoader + dataset_dir: "/path/to/AdvertiseGen/train.json" # 需要修改为实际AdvertiseGen/train.json路径 + shuffle: True + phase: "train" + version: 2 + origin_columns: ["content", "summary"] + tokenizer: + type: ChatGLM2Tokenizer + vocab_file: "/path/to/tokenizer.model" # 需要修改为实际tokenizer.model路径 + input_columns: ["input_ids", "labels"] + max_source_length: 64 + max_target_length: 128 + ignore_pad_token_for_loss: True + num_parallel_workers: 8 + python_multiprocessing: False + drop_remainder: True + batch_size: 1 + repeat: 1 + numa_enable: False + prefetch_size: 1 + seed: 0 + +train_dataset_task: + type: KeyWordGenDataset + dataset_config: *train_dataset + +eval_dataset: &eval_dataset + data_loader: + type: ADGenDataLoader + dataset_dir: "/path/to/AdvertiseGen/dev.json" # 需要修改为实际AdvertiseGen/dev.json路径 + shuffle: False + phase: "eval" + version: 2 + origin_columns: ["content", "summary"] + tokenizer: + type: ChatGLM2Tokenizer + vocab_file: "/path/to/tokenizer.model" # 需要修改为实际tokenizer.model路径 + max_source_length: 256 + max_target_length: 256 + ignore_pad_token_for_loss: True + input_columns: ["input_ids", "labels"] + num_parallel_workers: 8 + python_multiprocessing: False + drop_remainder: True + batch_size: 1 + repeat: 1 + numa_enable: False + prefetch_size: 1 + seed: 0 + +eval_dataset_task: + type: KeyWordGenDataset + dataset_config: *eval_dataset +``` + +- 修改code/run_glm2_6b_finetune.yaml**同样需要修改上述的部分**，另外把文件开头的load_checkpoint设置为glm2的实际ckpt路径： +``` +seed: 0 +run_mode: 'train' +output_dir: './output' # 当前不支持自定义修改，请勿修改该默认值 +load_checkpoint: 'glm2_6b.ckpt' # 修改为实际下载的glm2_6b.ckpt路径 +auto_trans_ckpt: False # If true, auto transform load_checkpoint to load in distributed model +only_save_strategy: False +resume_training: False +``` +## 3 负载启动 +### 3.1 在线测试 +执行命令 +```bash +./ais-bench-stubs +``` +### 3.2 轻量化离线测试 +执行命令 +```bash +./ais-bench-stubs test +``` \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh new file mode 100644 index 0000000..7331278 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +echo "start to build glm2 workload" + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + return $ret_ok +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + + mkdir -p ${CURDIR}/output/config + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/config.sh -r ${CURDIR}//output/config/ + [ -d ${CURDIR}/doc ] && cp ${CURDIR}/doc -r ${CURDIR}/output/ + + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh new file mode 100644 index 0000000..4d398c2 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh @@ -0,0 +1,33 @@ +#!/bin/bash +echo "set env of glm train" + +export PYTHON_COMMAND=python3 +# 以下cluster配置二选一，仅多机场景需要 +export CLUSTER_SSH_KEY_PATH=~/.ssh/id_rsa # 用户指定的ssh私钥，确保通过此私钥管理节点能免密访问所有计算节点(单机场景注释此行) +export CLUSTER_AUTO_SET_KEY='on' # 'off' or 'on'， 若为'on' 不需要配置CLUSTER_SSH_KEY_PATH(单机场景注释此行) + +export GLM_RUN_MODE='only_finetune' + +# FINETUNE_CKPT_PATH, FINETUNE_DATA_PATH, EVAL_DATASET_PATH 这三个路径是相对mindformers源码的路径, 必须以./mindformers/开头 +export FINETUNE_DATA_PATH=./mindformers/dataset_files/AdvertiseGen/train.json # 微调数据集 +export EVAL_DATASET_TYPE='ADGEN' # 'ADGEN' +export EVAL_DATASET_PATH=./mindformers/dataset_files/AdvertiseGen/dev.json # 评测用的数据集路径，必须以./mindformers/开头 +export FINETUNE_CKPT_PATH=./mindformers/checkpoint_download/glm2/glm2_6b.ckpt # 微调使用的预训练权重，必须以./mindformers/开头 +export EVAL_DEVICE_ID=0 # 评测用的npu 的device id + +export EPOCH_SIZE=1 +export GLM_LAYER_NUM=4 + +export RANK_SIZE=8 # 集群总加速卡数 +export DEVICE_NUM=8 # 集群每个节点的加速卡数 + +# parallel run params, parallel strategy config, DATA_PARALLEL * MODEL_PARALLEL * PIPELINE_STAGE should equal to RANK_SIZE +export DATA_PARALLEL=2 +export MODEL_PARALLEL=1 +export PIPELINE_STAGE=4 + +# need if rank_size > 1 +export RANK_TABLE_FILE=/home/hccl/hccl_xxxx_8p.json + +# 多机多卡需要配置，单机不需要配置 +# export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh new file mode 100644 index 0000000..8b4e8eb --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r2.2"; } + + modelzoo_sub_dir="mindformers" + if [ "$branch_args" == "r2.2" ];then + branch="r0.8" + patch_file_name="r2.2" + commitid="c0f478fc517b1daec896f5c72bcea10b2ab83bd4" + git_url="https://gitee.com/mindspore/mindformers.git" + else + echo "bad parameters : $1" + return $ret_error + fi + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + local changed_code_path="$4" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + target_dir=$changed_code_path + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch new file mode 100644 index 0000000..553076d --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch @@ -0,0 +1,231 @@ +diff -Nur origin/mindformers/core/callback/callback.py code/mindformers/core/callback/callback.py +--- origin/mindformers/core/callback/callback.py 2023-12-06 11:50:58.448000000 +0800 ++++ code/mindformers/core/callback/callback.py 2023-12-06 11:50:58.540000000 +0800 +@@ -33,7 +33,11 @@ + from mindformers.tools.register import MindFormerRegister, MindFormerModuleType + from mindformers.tools.cloud_adapter.cloud_adapter import Local2ObsMonitor + from mindformers.tools.logger import logger +-from mindformers.tools.utils import get_output_root_path, get_output_subpath, get_remote_save_url, check_in_modelarts ++from mindformers.tools.utils import get_output_root_path, get_output_subpath, get_remote_save_url, check_in_modelarts, save_aisbench_result ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + __all__ = ['ObsMonitor', 'MFLossMonitor', 'CheckpointMointor', 'SummaryMonitor', 'ProfileMonitor', 'EvalCallBack'] + +@@ -551,6 +555,7 @@ + + if save_ckpt: + logger.info('......Saving ckpt......') ++ save_aisbench_result("model_persistence_start_time", ais_utils.get_datatime().decode('utf-8')) + cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \ + + str(step_num_in_epoch) + ".ckpt" + # update checkpoint file list. +@@ -629,7 +634,7 @@ + self._config.async_save, {}, self._config.enc_key, self._config.enc_mode) + + save_only_network_params() +- ++ save_aisbench_result("model_persistence_end_time", ais_utils.get_datatime().decode('utf-8')) + self._latest_ckpt_file_name = cur_file + + +diff -Nur origin/mindformers/core/metric/metric.py code/mindformers/core/metric/metric.py +--- origin/mindformers/core/metric/metric.py 2023-12-06 11:50:58.448000000 +0800 ++++ code/mindformers/core/metric/metric.py 2023-12-06 11:50:58.540000000 +0800 +@@ -39,6 +39,11 @@ + + from .utils import PerplexityCell + from ...dataset.labels import cluener_labels ++from mindformers.tools.utils import save_aisbench_result ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + __all__ = ['EntityScore', 'SQuADMetric', 'PerplexityMetric', 'ADGENMetric', 'PromptAccMetric', 'EmF1Metric'] + +@@ -541,6 +546,12 @@ + return None + avg_loss = float(self.total_loss / self.num_data) + result = {"loss": avg_loss, "PPL": math.exp(avg_loss)} ++ result_log="loss: {}, Perplexity: {}".format(avg_loss, math.exp(avg_loss)) ++ try: ++ import ais_utils ++ ais_utils.set_result("training", "accuracy", result_log) ++ except Exception: ++ print("ais_utils not find") + if self.pipeline_parallel: + print("Average Loss and PPL Metric:", result) + return result +diff -Nur origin/mindformers/tools/transform_ckpt.py code/mindformers/tools/transform_ckpt.py +--- origin/mindformers/tools/transform_ckpt.py 2023-12-06 11:50:58.456000000 +0800 ++++ code/mindformers/tools/transform_ckpt.py 2023-12-06 11:50:58.548000000 +0800 +@@ -17,6 +17,11 @@ + import argparse + + import mindspore as ms ++from mindformers.tools.utils import save_aisbench_result ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + def get_strategy(startegy_path, rank_id=None): + """Merge strategy if strategy path is dir +@@ -84,6 +89,8 @@ + print(f"dst_ckpt_dir: {dst_ckpt_dir}") + print(f"prefix: {prefix}") + +- print("......Start transform......") ++ print("......Start transform......") # model_format_start_time ++ ais_utils.set_result("training", "model_format_start_time", ais_utils.get_datatime().decode('utf-8')) + ms.transform_checkpoints(src_ckpt_dir, dst_ckpt_dir, prefix, src_ckpt_strategy, dst_ckpt_strategy) +- print("......Transform succeed!......") ++ print("......Transform succeed!......") # model_format_end_time ++ ais_utils.set_result("training", "model_format_end_time", ais_utils.get_datatime().decode('utf-8')) +diff -Nur origin/mindformers/tools/utils.py code/mindformers/tools/utils.py +--- origin/mindformers/tools/utils.py 2023-12-06 11:50:58.456000000 +0800 ++++ code/mindformers/tools/utils.py 2023-12-06 11:50:58.548000000 +0800 +@@ -55,6 +55,28 @@ + _PROTOCOL = 'obs' + _PROTOCOL_S3 = 's3' + ++AISBENCH_RESULT_PATH=os.getenv('RESULT_PATH') ++ ++ ++def save_aisbench_result(rt_key:str, rt_value): ++ cur_rank_id = os.getenv("RANK_ID", "0") ++ cur_run_mode = os.getenv("LLAMA_CUR_RUN_MODE", "train") ++ result_file_path = os.path.join(AISBENCH_RESULT_PATH, f"result_rank_{cur_rank_id}.json") ++ if not os.path.exists(result_file_path): ++ with open(result_file_path, "w") as f: ++ init_data = {} ++ json.dump(init_data, f) ++ with open(result_file_path, "r") as result_file: ++ result_log = json.load(result_file) ++ if not cur_run_mode in result_log: ++ result_log[cur_run_mode] = {} ++ if rt_key in result_log[cur_run_mode]: ++ result_log[cur_run_mode][rt_key].append(rt_value) ++ else: ++ result_log[cur_run_mode][rt_key] = [rt_value] ++ with open(result_file_path, "w") as result_file: ++ json.dump(result_log, result_file) ++ + + def check_in_modelarts(): + """Check if the training is on modelarts. +diff -Nur origin/mindformers/trainer/base_trainer.py code/mindformers/trainer/base_trainer.py +--- origin/mindformers/trainer/base_trainer.py 2023-12-06 11:50:58.456000000 +0800 ++++ code/mindformers/trainer/base_trainer.py 2023-12-06 11:50:58.552000000 +0800 +@@ -42,7 +42,7 @@ + from mindformers.wrapper import build_wrapper + from mindformers.tools.register import MindFormerConfig + from mindformers.tools.logger import logger +-from mindformers.tools.utils import count_params ++from mindformers.tools.utils import count_params, save_aisbench_result + from mindformers.auto_class import AutoModel + from mindformers.pet import get_pet_model + from .config_args import ConfigArguments +@@ -51,6 +51,10 @@ + from .optimizer_grouped_parameters import get_optimizer_grouped_parameters + from .utils import set_seed, check_train_data_loader_type, \ + check_eval_data_loader_type, check_optimizer_and_lr_type, check_wrapper_config ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + SUPPORT_TASKS = MindFormerBook().get_trainer_support_task_list() + SUPPORT_MODEL_NAMES = MindFormerBook().get_model_name_support_list() +@@ -543,7 +547,8 @@ + load_resume_context_from_checkpoint(config) + + # build dataset +- logger.info(".........Build Dataset For Train..........") ++ logger.info(".........Build Dataset For Train..........") # dataload_start_time ++ save_aisbench_result("dataload_start_time", ais_utils.get_datatime().decode('utf-8')) + if dataset is None: + dataset = self.create_train_dataset() + self.set_train_dataset(dataset) +@@ -553,10 +558,12 @@ + * config.runner_config.sink_size / dataset.get_dataset_size()) + # pylint: disable=W0212 + dataset._dataset_helper = DatasetHelper(dataset, config.runner_config.sink_mode, +- config.runner_config.sink_size, epoch_num) ++ config.runner_config.sink_size, epoch_num) # dataload_end_time ++ save_aisbench_result("dataload_end_time", ais_utils.get_datatime().decode('utf-8')) + + # build network +- logger.info(".........Build Net For Train..........") ++ logger.info(".........Build Net For Train..........") # train_launch_start_time ++ save_aisbench_result("train_launch_start_time", ais_utils.get_datatime().decode('utf-8')) + eval_network = None + if network is None and wrapper is None and \ + self.model_wrapper is None and self.network is None: +@@ -651,18 +658,30 @@ + step_interval=config.eval_step_interval if config.eval_step_interval else 100, + epoch_interval=config.eval_epoch_interval if config.eval_epoch_interval else -1, + ) +- callbacks.append(eval_callback) ++ callbacks.append(eval_callback) # train_launch_end_time + + logger.info(".........Starting Training Model..........") + if int(os.getenv("RANK_ID", '0')) % 8 == 0: + pprint(config) + logger.info(".........Model Compiling, Please Wait a Moment...........") ++ model.build(dataset, None, sink_size=config.runner_config.sink_size, epoch=config.runner_config.epochs) ++ save_aisbench_result("train_launch_end_time", ais_utils.get_datatime().decode('utf-8')) ++ logger.info(".........Model Build Done, Traininf Start...........") ++ train_start_time = ais_utils.get_datatime() ++ save_aisbench_result("train_start_time", train_start_time.decode('utf-8')) # train_start_time + model.train(config.runner_config.epochs, dataset, + callbacks=callbacks, + dataset_sink_mode=config.runner_config.sink_mode, + sink_size=config.runner_config.sink_size, + initial_epoch=config.runner_config.initial_epoch) +- logger.info(".........Training Over!.............") ++ logger.info(".........Training Over!.............") # train_end_time ++ train_end_time = ais_utils.get_datatime() ++ all_data_sum = int(dataset.get_dataset_size() * config.train_dataset.batch_size / int(os.getenv("RANK_SIZE", '8'))) * \ ++ config.runner_config.origin_epochs * config.model.model_config.seq_length ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, train_start_time, train_end_time) ++ save_aisbench_result("train_end_time", train_end_time.decode('utf-8')) ++ save_aisbench_result("throughput_ratio", throughput_rate) ++ + + def evaluate_process( + self, +diff -Nur origin/requirements.txt code/requirements.txt +--- origin/requirements.txt 2023-12-06 11:50:58.396000000 +0800 ++++ code/requirements.txt 2023-12-06 11:50:58.492000000 +0800 +@@ -11,4 +11,5 @@ + pydantic==1.10.11 + mdtex2html + gradio +-opencv-python-headless +\ No newline at end of file ++opencv-python-headless ++pyyaml +\ No newline at end of file +diff -Nur origin/scripts/run_distribute.sh code/scripts/run_distribute.sh +--- origin/scripts/run_distribute.sh 2023-12-06 11:50:58.460000000 +0800 ++++ code/scripts/run_distribute.sh 2023-12-06 11:50:58.556000000 +0800 +@@ -179,7 +179,14 @@ + fi + fi + shopt -u extglob +- ++wait ++if [ $? -eq 0 ];then ++ echo "all train processes completed successfully!" ++ exit 0 ++else ++ echo "one or more train processes exited with a error!" ++ exit 1 ++fi + + #cd ./pretrain_parallel${START_DEVICE} || exit + #tail -f mindformer.log diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh new file mode 100644 index 0000000..fd370f6 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 +declare -i ret_mode_failed=5 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) +export DEPEND_PATH=$BASE_PATH/dependencies/ + +function get_node_train_data() +{ + if [ "$GLM_RUN_MODE" = "only_finetune" ];then + [ ! -f $FINETUNE_CKPT_PATH ] || { echo "finetune base ckpt:$FINETUNE_CKPT_PATH";return $ret_failed; } + fi + return $ret_ok +} + +# 配置训练相关的环境变量 +source ${CODE_PATH}/config/config.sh || { logger_Warn "source file failed:$?";return $ret_init_failed; } + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh +if [ -d $FINETUNE_DATA_PATH ];then + cp -r $FINETUNE_DATA_PATH $CUR_PATH || { logger_Warn "ERROR: cp $FINETUNE_DATA_PATH failed!";return $ret_init_failed; } +fi + +. $CODE_PATH/cluster_offline_run.sh + +main(){ + get_node_train_data || { logger_Warn "download open glm cpkt failed:$?";return $ret_init_failed; } + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..07ff4db --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh @@ -0,0 +1,134 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common_2.0.sh +. $CODE_PATH/common/node_common.sh + +# env check +export RELAT_WORK_PATH=work +export RELAT_RESULT_PATH=$RELAT_WORK_PATH/result +CONFIG_FILE="config.sh" +# set nodes work path. 仅仅是管理节点的work/ +export WORK_PATH=${BASE_PATH}/work +# set nodes result path +export RESULT_PATH=${WORK_PATH}/result +local_env_cmd="source /etc/profile; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + export PYTHONPATH=$WORK_PATH:$PYTHONPATH; + export PYTHONPATH=$WORK_PATH/logging:$PYTHONPATH; + source $WORK_PATH/config/$CONFIG_FILE" +env_cmd="source /etc/profile; + export WORK_PATH=\$PWD/$RELAT_WORK_PATH; + export RESULT_PATH=\$PWD/$RELAT_RESULT_PATH; + export PYTHONPATH=\$WORK_PATH:\$PYTHONPATH; + export PYTHONPATH=\$WORK_PATH/logging:\$PYTHONPATH; + source \$WORK_PATH/config/$CONFIG_FILE" + +check_env() +{ + # check ranktable set + : "${RANK_SIZE?RANK_SIZE not set}" + : "${DEVICE_NUM?DEVICE_NUM not set}" + [[ $RANK_SIZE -eq 1 ]] || : "${RANK_TABLE_FILE?RANK_TABLE_FILE not set}" + [[ $RANK_SIZE -eq 1 ]] && [[ -n "$RANK_TABLE_FILE" ]] && { echo "ranksize=1 should not set RANK_TABLE_FILE";return 1; } + + # check python + : "${PYTHON_COMMAND?PYTHON_COMMAND not set}" + [ "$NODEINFO_FILE" == "" ] && { echo "NODEINFO_FILE not set, will not check cluster";return 0; } + if pip show ais_bench_cluster >/dev/null 2>&1;then + logger_Info "ais_bench cluster module exist, won't be installed again" + else + cluster_whl_path="${DEPEND_PATH}/cluster/ais_bench_cluster-*.whl" + if [ -f $cluster_whl_path ];then + pip install $cluster_whl_path --force-reinstall || { logger_Error "install cluster failed!";return 1; } + else + logger_Error "can't find ais_bench cluster wheel package" + fi + fi + + # check nodeinfofile exist + [[ $RANK_SIZE -le 8 ]] || check_file_valid "${NODEINFO_FILE}" || { echo "nodeinfofile:${NODEINFO_FILE} not valid" ; return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Error "source file failed:$?";return 1; } + if [ -d ${DEPEND_PATH}/logging ];then + cp -r ${DEPEND_PATH}/logging ${CODE_PATH} + fi + check_env || { logger_Error "env check failed'" ; return 1; } + + # init ais_bench.cluster + cluster_init || { logger_Error "ais_bench_cluster init failed!";return 1; } + + # refresh result path + rm -rf ${BASE_PATH}/result;mkdir -p ${BASE_PATH}/result + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + rm -rf $WORK_PATH;mkdir -p $WORK_PATH + + if [ "$NODEINFO_FILE" != "" ];then + cmd="rm -rf ${RELAT_WORK_PATH};mkdir -p ${RELAT_WORK_PATH}" + cluster_multi_exec "$cmd" serial || { logger_Error "renew workpath failed"; return 1; } + fi + + # copy code to node work path + cp -r $CODE_PATH/* $WORK_PATH # CPU可以执行的都在host节点执行 + + if [ "$NODEINFO_FILE" != "" ];then + # sync data if work_path not exist so new one.节点的work/ 路径是相对于在node_file中指定的work_path + cluster_multi_put "$WORK_PATH" "./" || { logger_Error "deploy code to work place failed"; return 1; } + fi + cmd="source /etc/profile; + export WORK_PATH=\$PWD/$RELAT_WORK_PATH; + source \$WORK_PATH/config/$CONFIG_FILE; + bash \$WORK_PATH/run_node.sh check" + cluster_multi_exec "$cmd" serial|| { return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + if [ "$GLM_RUN_MODE" == "only_finetune" ];then + if [ "$NODEINFO_FILE" == "" ];then + cmd="$local_env_cmd; + rm -rf $RESULT_PATH/*.json; + bash $WORK_PATH/run_node.sh train finetune " + else + cmd="$env_cmd; + rm -rf \$RESULT_PATH/*.json; + bash \$WORK_PATH/run_node.sh train finetune " + fi + cluster_multi_exec "$cmd" || { logger_Error "run train(finetune) failed"; return 1; } + if [ "$NODEINFO_FILE" != "" ];then + cluster_multi_get "$RELAT_RESULT_PATH" "$BASE_PATH" || { logger_Error "cp result between nodes failed"; return 1; } + else + cp -r $WORK_PATH/result $BASE_PATH + fi + export PYTHONPATH=$WORK_PATH/logging:$PYTHONPATH + bash $WORK_PATH/run_node.sh merge || { logger_Error "ckpt merge failed"; return 1; } + fi + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="$local_env_cmd; + bash $WORK_PATH/run_node.sh eval" + eval "$cmd" || { logger_Error "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + source ${CODE_PATH}/config/$CONFIG_FILE + export PYTHONPATH=${CODE_PATH}/logging:$PYTHONPATH + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_glm2_result.py ${BASE_PATH}/result ${RANK_SIZE} ${GLM_RUN_MODE} + find $BASE_PATH/result/ -name "*.ckpt" -exec rm {} \; + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml new file mode 100644 index 0000000..3a5ec8f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml @@ -0,0 +1,250 @@ +seed: 0 +run_mode: 'train' +output_dir: './output' # path to save checkpoint/strategy +load_checkpoint: '../../mindformers/checkpoint_download/glm2_6b.ckpt' +src_strategy_path_or_dir: '' +auto_trans_ckpt: False # If true, auto transform load_checkpoint to load in distributed model +only_save_strategy: False +resume_training: False + +# ==== context config ==== +context: + mode: 0 #0--Graph Mode; 1--Pynative Mode + device_target: "Ascend" + enable_graph_kernel: False + graph_kernel_flags: "--disable_expand_ops=Softmax,Dropout --enable_parallel_fusion=true --reduce_fuse_depth=8 --enable_auto_tensor_inplace=true" + max_call_depth: 10000 + max_device_memory: "30GB" # 59GB for Atlas 800T A2 + save_graphs: False + device_id: 0 + +# aicc +remote_save_url: "Please input obs url on AICC platform." + +# ==== model config ==== +model: + model_config: + type: ChatGLM2Config + batch_size: 1 # only for incremental infer + num_layers: 28 + padded_vocab_size: 65024 + hidden_size: 4096 + ffn_hidden_size: 13696 + kv_channels: 128 + num_attention_heads: 32 + seq_length: 193 + hidden_dropout: 0.0 + attention_dropout: 0.0 + layernorm_epsilon: 1e-5 + rmsnorm: True + apply_residual_connection_post_layernorm: False + post_layer_norm: True + add_bias_linear: False + add_qkv_bias: True + bias_dropout_fusion: True + multi_query_attention: True + multi_query_group_num: 2 + apply_query_key_layer_scaling: True + attention_softmax_in_fp32: True + fp32_residual_connection: False + quantization_bit: 0 + pre_seq_len: None + prefix_projection: False + param_init_type: "float16" + compute_dtype: "float16" + layernorm_compute_type: "float32" + use_past: False + use_flash_attention: False # when use FlashAttention, seq_length should be multiple of 16 + eos_token_id: 2 + pad_token_id: 0 + repetition_penalty: 1.0 + max_decode_length: 256 + checkpoint_name_or_path: "glm2_6b" + top_k: 1 + top_p: 1 + do_sample: True + arch: + type: ChatGLM2ForConditionalGeneration + +trainer: + type: CausalLanguageModelingTrainer + model_name: 'glm2_6b' +# if True do, evaluate during the training process. if false, do nothing. +# note that the task trainer should support _evaluate_in_training function. +do_eval: False +eval_step_interval: 1788 +eval_epoch_interval: -1 + +metric: + type: PerplexityMetric + +processor: + return_tensors: ms + tokenizer: + type: ChatGLM2Tokenizer + bos_token: '' + eos_token: '' + end_token: '' + mask_token: '[MASK]' + gmask_token: '[gMASK]' + pad_token: '' + unk_token: '' + # vocab_file: "/path/to/tokenizer.model" + type: GLMProcessor + +# ==== dataset config ==== +train_dataset: &train_dataset + data_loader: + type: ADGenDataLoader + dataset_dir: "../../mindformers/dataset_files/AdvertiseGen/train.json" + shuffle: True + phase: "train" + version: 2 + origin_columns: ["content", "summary"] + tokenizer: + type: ChatGLM2Tokenizer + vocab_file: "../../mindformers/checkpoint_download/tokenizer.model" + input_columns: ["input_ids", "labels"] + max_source_length: 64 + max_target_length: 128 + ignore_pad_token_for_loss: True + num_parallel_workers: 8 + python_multiprocessing: False + drop_remainder: True + batch_size: 8 + repeat: 1 + numa_enable: False + prefetch_size: 1 + seed: 0 + +train_dataset_task: + type: KeyWordGenDataset + dataset_config: *train_dataset + +eval_dataset: &eval_dataset + data_loader: + type: ADGenDataLoader + dataset_dir: "./code/mindformers/dataset_files/AdvertiseGen/dev.json" + shuffle: False + phase: "train" + version: 2 + origin_columns: ["content", "summary"] + tokenizer: + type: ChatGLM2Tokenizer + vocab_file: "./code/mindformers/checkpoint_download/tokenizer.model" + max_source_length: 64 + max_target_length: 127 + ignore_pad_token_for_loss: True + input_columns: ["input_ids", "labels"] + num_parallel_workers: 8 + python_multiprocessing: False + drop_remainder: True + batch_size: 8 + repeat: 1 + numa_enable: False + prefetch_size: 1 + seed: 0 + +eval_dataset_task: + type: KeyWordGenDataset + dataset_config: *eval_dataset + +# ==== runner config ==== +runner_config: + epochs: 1 + batch_size: 8 + sink_mode: True + sink_size: 4 + +runner_wrapper: + type: MFTrainOneStepCell + scale_sense: + type: DynamicLossScaleUpdateCell + loss_scale_value: 65536 + scale_factor: 2 + scale_window: 1000 + use_clip_grad: True + +# lr sechdule +lr_schedule: + type: polynomial + learning_rate: 5.e-5 + lr_end: 1.e-6 + warmup_steps: 0 + total_steps: -1 # -1 means it will load the total steps of the dataset +layer_scale: False +layer_decay: 0.65 + +# optimizer +optimizer: + type: FP32StateAdamWeightDecay + beta1: 0.9 + beta2: 0.95 + eps: 1.e-8 + weight_decay: 0.1 +lr_scale: False +lr_scale_factor: 256 + +# parallel config +use_parallel: True +parallel: + parallel_mode: 1 # 0-dataset, 1-semi, 2-auto, 3-hybrid + gradients_mean: False + loss_repeated_mean: True + enable_alltoall: False + full_batch: True + search_mode: "sharding_propagation" + enable_parallel_optimizer: True # optimizer shard + strategy_ckpt_config: + save_file: "./ckpt_strategy.ckpt" +parallel_config: + data_parallel: 8 + model_parallel: 1 + pipeline_stage: 1 + expert_parallel: 1 + micro_batch_num: 1 + vocab_emb_dp: True + gradient_aggregation_group: 4 +micro_batch_interleave_num: 1 + +# moe +moe_config: + expert_num: 1 + capacity_factor: 1.05 + aux_loss_factor: 0.05 + num_experts_chosen: 1 + +# recompute +recompute_config: + recompute: True + parallel_optimizer_comm_recompute: False + mp_comm_recompute: True + recompute_slice_activation: True + +# autotune +auto_tune: False +filepath_prefix: './autotune' +autotune_per_step: 10 + +# profile +profile: False +profile_start_step: 1 +profile_stop_step: 10 +init_start_profile: True +profile_communication: True +profile_memory: True + +# callbacks +callbacks: + - type: MFLossMonitor + - type: CheckpointMointor + prefix: "glm2-6b" + save_checkpoint_steps: 1000 + keep_checkpoint_max: 1 + integrated_save: False + async_save: False + - type: ObsMonitor + keep_last: False +eval_callbacks: + - type: ObsMonitor + keep_last: False diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml new file mode 100644 index 0000000..1955ecf --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml @@ -0,0 +1,250 @@ +seed: 0 +run_mode: 'eval' +output_dir: './output' # path to save checkpoint/strategy +load_checkpoint: 'glm2_6b' +src_strategy_path_or_dir: '' +auto_trans_ckpt: False # If true, auto transform load_checkpoint to load in distributed model +only_save_strategy: False +resume_training: False + +# ==== context config ==== +context: + mode: 0 #0--Graph Mode; 1--Pynative Mode + device_target: "Ascend" + enable_graph_kernel: False + graph_kernel_flags: "--disable_expand_ops=Softmax,Dropout --enable_parallel_fusion=true --reduce_fuse_depth=8 --enable_auto_tensor_inplace=true" + max_call_depth: 10000 + max_device_memory: "30GB" # 59GB for Atlas 800T A2 + save_graphs: False + device_id: 0 + +# aicc +remote_save_url: "Please input obs url on AICC platform." + +# ==== model config ==== +model: + model_config: + type: ChatGLM2Config + batch_size: 8 # only for incremental infer + num_layers: 28 + padded_vocab_size: 65024 + hidden_size: 4096 + ffn_hidden_size: 13696 + kv_channels: 128 + num_attention_heads: 32 + seq_length: 256 + hidden_dropout: 0.0 + attention_dropout: 0.0 + layernorm_epsilon: 1e-5 + rmsnorm: True + apply_residual_connection_post_layernorm: False + post_layer_norm: True + add_bias_linear: False + add_qkv_bias: True + bias_dropout_fusion: True + multi_query_attention: True + multi_query_group_num: 2 + apply_query_key_layer_scaling: True + attention_softmax_in_fp32: True + fp32_residual_connection: False + quantization_bit: 0 + pre_seq_len: None + prefix_projection: False + param_init_type: "float16" + compute_dtype: "float16" + layernorm_compute_type: "float32" + use_past: True + eos_token_id: 2 + pad_token_id: 0 + repetition_penalty: 1.0 + max_decode_length: 256 + checkpoint_name_or_path: "glm2_6b" + top_k: 1 + top_p: 1 + do_sample: True + arch: + type: ChatGLM2ForConditionalGeneration + +trainer: + type: CausalLanguageModelingTrainer + model_name: 'glm2_6b' +# if True do, evaluate during the training process. if false, do nothing. +# note that the task trainer should support _evaluate_in_training function. +do_eval: False +eval_step_interval: 500 +eval_epoch_interval: -1 + +metric: + type: ADGENMetric + +processor: + return_tensors: ms + tokenizer: + type: ChatGLM2Tokenizer + bos_token: '' + eos_token: '' + end_token: '' + mask_token: '[MASK]' + gmask_token: '[gMASK]' + pad_token: '' + unk_token: '' + # vocab_file: "/path/to/tokenizer.model" + type: GLMProcessor + +# ==== dataset config ==== +train_dataset: &train_dataset + data_loader: + type: ADGenDataLoader + dataset_dir: "/path/to/AdvertiseGen/train.json" + shuffle: True + phase: "train" + version: 2 + origin_columns: ["content", "summary"] + tokenizer: + type: ChatGLM2Tokenizer + vocab_file: "/path/to/tokenizer.model" + input_columns: ["input_ids", "labels"] + max_source_length: 64 + max_target_length: 128 + ignore_pad_token_for_loss: True + num_parallel_workers: 8 + python_multiprocessing: False + drop_remainder: True + batch_size: 8 + repeat: 1 + numa_enable: False + prefetch_size: 1 + seed: 0 + +train_dataset_task: + type: KeyWordGenDataset + dataset_config: *train_dataset + +eval_dataset: &eval_dataset + data_loader: + type: ADGenDataLoader + dataset_dir: "/path/to/AdvertiseGen/dev.json" + shuffle: False + phase: "eval" + version: 2 + origin_columns: ["content", "summary"] + tokenizer: + type: ChatGLM2Tokenizer + vocab_file: "./work/code/mindformers/checkpoint_download/tokenizer.model" + max_source_length: 256 + max_target_length: 256 + ignore_pad_token_for_loss: True + input_columns: ["input_ids", "labels"] + num_parallel_workers: 8 + python_multiprocessing: False + drop_remainder: True + batch_size: 8 + repeat: 1 + numa_enable: False + prefetch_size: 1 + seed: 0 + +eval_dataset_task: + type: KeyWordGenDataset + dataset_config: *eval_dataset + +# ==== runner config ==== +runner_config: + epochs: 1 + batch_size: 8 + sink_mode: True + sink_size: 4 + +runner_wrapper: + type: MFTrainOneStepCell + scale_sense: + type: DynamicLossScaleUpdateCell + loss_scale_value: 65536 + scale_factor: 2 + scale_window: 1000 + use_clip_grad: True + +# lr sechdule +lr_schedule: + type: polynomial + learning_rate: 5.e-5 + lr_end: 1.e-6 + warmup_steps: 0 + total_steps: -1 # -1 means it will load the total steps of the dataset +layer_scale: False +layer_decay: 0.65 + +# optimizer +optimizer: + type: FP32StateAdamWeightDecay + beta1: 0.9 + beta2: 0.95 + eps: 1.e-8 + weight_decay: 0.1 +lr_scale: False +lr_scale_factor: 256 + +# parallel config +use_parallel: True +parallel: + parallel_mode: 0 # 0-dataset, 1-semi, 2-auto, 3-hybrid + gradients_mean: False + loss_repeated_mean: True + enable_alltoall: False + full_batch: False + search_mode: "sharding_propagation" + enable_parallel_optimizer: True # optimizer shard + strategy_ckpt_config: + save_file: "./ckpt_strategy.ckpt" + only_trainable_params: False # 设置成 False，才能在策略文件中保存所有参数 +parallel_config: + data_parallel: 8 + model_parallel: 1 + pipeline_stage: 1 + expert_parallel: 1 + micro_batch_num: 1 + vocab_emb_dp: True + gradient_aggregation_group: 4 +micro_batch_interleave_num: 1 + +# moe +moe_config: + expert_num: 1 + capacity_factor: 1.05 + aux_loss_factor: 0.05 + num_experts_chosen: 1 + +# recompute +recompute_config: + recompute: True + parallel_optimizer_comm_recompute: False + mp_comm_recompute: True + recompute_slice_activation: True + +# autotune +auto_tune: False +filepath_prefix: './autotune' +autotune_per_step: 10 + +# profile +profile: False +profile_start_step: 1 +profile_stop_step: 10 +init_start_profile: True +profile_communication: True +profile_memory: True + +# callbacks +callbacks: + - type: MFLossMonitor + - type: CheckpointMointor + prefix: "glm2-6b" + save_checkpoint_steps: 1000 + keep_checkpoint_max: 1 + integrated_save: False + async_save: False + - type: ObsMonitor + keep_last: False +eval_callbacks: + - type: ObsMonitor + keep_last: False diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh new file mode 100644 index 0000000..713c278 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh @@ -0,0 +1,167 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +declare -i ret_ok=0 +declare -i ret_failed=1 + +GLM_RUN_YAML_NAME="run_glm2_6b_finetune.yaml" +GLM_EVAL_YAML_NAME="run_glm2_6b_finetune_eval.yaml" + +function get_node_rank_id_range() +{ + RANK_ID_RANGE="[0,8]" + # get server node id default is 0 + : "${NODE_ID:=0}" + # get rank start index + if [[ $DEVICE_NUM == 1 && $RANK_SIZE == 1 ]];then + : "${SINGLE_CARD_INDEX:=0}" + RANK_START=$SINGLE_CARD_INDEX + else + # get rank start index + RANK_START=`expr ${NODE_ID} \* $DEVICE_NUM` + fi + RANK_ID_MAX=$[DEVICE_NUM+RANK_START] + RANK_ID_RANGE="[$RANK_START,$RANK_ID_MAX]" +} + +function node_init() +{ + export PYTHONPATH=$WORK_PATH:$WORK_PATH/logging:$PYTHONPATH + + if [ $1 == "check" ];then + # install pyyaml + if pip show pyyaml >/dev/null 2>&1;then + logger_Info "pyyaml exist, won't be installed again" + else + pip_cmd="pip install pyyaml" + $pip_cmd || { logger_Warn "pyyaml install failed:$?";return $ret_failed; } + fi + # install mindformers + if pip show mindformers >/dev/null 2>&1;then + logger_Info "mindformers exist, won't be installed again" + else + cd $WORK_PATH/code + pip install . || { logger_Warn "mindformers install failed:$?";return $ret_failed; } + cd $WORK_PATH + fi + fi + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + get_node_rank_id_range + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + rank_table_path=${WORK_PATH}/${RANK_TABLE_FILE} + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$rank_table_path" || { logger_Warn "node common check failed" ; return $ret_failed; } + + # check_mindspore_run_ok_Ascend ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return $ret_failed; } + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return $ret_failed; } + logger_Debug "mindspore running successfully" + + if [ "$GLM_RUN_MODE" == "only_finetune" ];then + check_file_valid "${WORK_PATH}/code/${FINETUNE_DATA_PATH}" || { logger_Warn "FINETUNE_DATA_PATH:${FINETUNE_DATA_PATH} not valid path" ; return 1; } + logger_Debug "FINETUNE_DATA_PATH is valid" + check_file_valid "${WORK_PATH}/code/${EVAL_DATASET_PATH}" || { logger_Warn "EVAL_DATASET_PATH:${EVAL_DATASET_PATH} not valid path" ; return 1; } + logger_Debug "EVAL_DATASET_PATH is valid" + fi + +} + +function ckpt_merge() +{ + transform_ckpt_path=$WORK_PATH/code/mindformers/tools/transform_ckpt.py + result_output_path=$WORK_PATH/../result/output + cd $WORK_PATH + # ckpt merge + $PYTHON_COMMAND $transform_ckpt_path \ + --src_ckpt_strategy $result_output_path/strategy/ \ + --src_ckpt_dir $result_output_path/checkpoint/ \ + --dst_ckpt_dir $result_output_path/target_ckpt/ \ + --prefix "glm2_6b" || { logger_Warn "ckpt merge failed, rank id range: $RANK_ID_RANGE" ; return $ret_failed; } + rm -rf $result_output_path/checkpoint/ +} + +function node_train() +{ + logger_Info "node_train running" + export GLM_CUR_RUN_MODE=$1 + source $WORK_PATH/config/config.sh + run_script_path=$WORK_PATH/code/scripts/ + run_yaml_path=$WORK_PATH/code/configs/glm2/$GLM_RUN_YAML_NAME + rank_table_path=${WORK_PATH}/$RANK_TABLE_FILE + run_processed_yaml=$WORK_PATH/$GLM_RUN_YAML_NAME + cp $run_processed_yaml $run_yaml_path + # train run + cd $run_script_path + cmd="bash run_distribute.sh $rank_table_path $run_yaml_path $RANK_ID_RANGE $1" + [ "$NODEINFO_FILE" != "" ] && cmd="$cmd $RANK_SIZE" + echo "$cmd" + $cmd || { logger_Warn "node_run failed, rank id range: $RANK_ID_RANGE" ; return $ret_failed; } + mv $WORK_PATH/code/output/ $WORK_PATH/../result/ || { logger_Warn "move output failed!" ; return $ret_failed; } # store in $WORK_PATH/../result/output + return $ret_ok +} + +function eval_run() +{ + logger_Info "eval_run running" + eval_yaml_path=$WORK_PATH/code/configs/glm2/$GLM_EVAL_YAML_NAME + eval_processed_yaml=$WORK_PATH/$GLM_EVAL_YAML_NAME + cp $eval_processed_yaml $eval_yaml_path + eval_dataset_path=$WORK_PATH/code/$EVAL_DATASET_PATH + load_checkpoint_path=$WORK_PATH/../result/output/target_ckpt/rank_0/glm2_6b0.ckpt + if [ "$EVAL_DATASET_TYPE" = "ADGEN" ];then + echo "run eval using ADGEN" + eval_script_path=$WORK_PATH/code/run_mindformer.py + $PYTHON_COMMAND $eval_script_path \ + --config $eval_yaml_path \ + --eval_dataset_dir $eval_dataset_path \ + --run_mode eval \ + --load_checkpoint $load_checkpoint_path \ + --use_parallel False \ + --device_id $EVAL_DEVICE_ID || { logger_Warn "run eval failed" ; return $ret_failed; } + else + echo "invalid eval mode" + rm -rf $load_checkpoint_path + return $ret_failed + fi + rm -rf $load_checkpoint_path + return $ret_ok +} + +function node_eval() +{ + logger_Info "node_eval running" + if [ "$GLM_RUN_MODE" == "only_finetune" ];then + eval_run + else + echo "glm2 run mode not supported" + return $ret_failed + fi + return $ret_ok +} + +main() +{ + type="$1" + mode="$2" + shift + node_init $type || { logger_Warn "init failed"; return $ret_failed; } + if [ "$type" == "train" ];then + node_train $mode || { logger_Warn "run_node_train failed"; return $ret_failed; } + elif [ "$type" == "merge" ];then + ckpt_merge || { logger_Warn "ckpt_merge failed"; return $ret_failed; } + elif [ "$type" == "eval" ];then + node_eval || { logger_Warn "run_node_eval failed"; return $ret_failed; } + elif [ "$type" == "check" ];then + node_check || { logger_Warn "run_node_check failed"; return $ret_failed; } + else + { logger_Warn "invalid argument '${type}'"; return $ret_failed; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh new file mode 100644 index 0000000..abe3be7 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + + mkdir -p ${CURDIR}/output/config + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/config.sh -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/modelarts_config.py -r ${CURDIR}//output/config/ + [ "$1" == "r1.3" ] && { cp ${CURDIR}/config/modelarts_config.py.r1.3 -r ${CURDIR}//output/config/modelarts_config.py; } + [ -d ${CURDIR}/doc ] && cp ${CURDIR}/doc -r ${CURDIR}/output/ + + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh new file mode 100644 index 0000000..3322bac --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh @@ -0,0 +1,18 @@ +export PYTHON_COMMAND=python3.7 +export TRAIN_DATASET=/home/datasets/wmt16/mindrecord/train.tok.clean.bpe.32000.en.mindrecord +export TEST_DATASET=/home/datasets/wmt16/mindrecord/newstest2014.en.mindrecord +export EXISTED_CKPT_PATH=/home/datasets/pretrain_model/gnmt_v2/gnmtv2_ascend_v180_wmtende_official_nlp_acc24.ckpt +export VOCAB_ADDR=/home/datasets/wmt16/vocab.bpe.32000 +export BPE_CODE_ADDR=/home/datasets/wmt16/bpe.32000 +export TEST_TARGET=/home/datasets/wmt16/newstest2014.de + +# 8p +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# options needed only if rank_size > 1 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json + +# needed only in cluster mode +# export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh new file mode 100644 index 0000000..e9540bd --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.9" ];then + branch="r1.8" + patch_file_name="r1.9" + commitid="f4eed0f958b40992ee96dd6cfefd76ae989c872f" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/gnmt_v2" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch new file mode 100644 index 0000000..024184f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch @@ -0,0 +1,72 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-11-02 17:55:22.768000000 +0800 ++++ code/eval.py 2022-11-02 17:55:22.784000000 +0800 +@@ -101,6 +101,16 @@ + tokenizer = Tokenizer(vocab, bpe_codes, 'en', 'de') + scores = bleu_calculate(tokenizer, result_npy_addr, test_tgt) + print(f"BLEU scores is :{scores}") ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ print("ACC_DIR:", ACC_DIR) ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(scores)) + + if __name__ == '__main__': + run_eval() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-11-02 17:55:22.776000000 +0800 ++++ code/train.py 2022-11-02 17:55:22.784000000 +0800 +@@ -41,6 +41,12 @@ + from model_utils.config import config as default_config + from model_utils.moxing_adapter import moxing_wrapper + from model_utils.device_adapter import get_device_id, get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def _train(model, config, + pre_training_dataset=None, fine_tune_dataset=None, test_dataset=None, +@@ -222,11 +228,35 @@ + callbacks.append(summary_callback) + + print(f" | ALL SET, PREPARE TO TRAIN.") ++ model.build(dataset, sink_size=dataset.get_dataset_size(), epoch=config.epochs) ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + _train(model=model, config=config, + pre_training_dataset=pre_training_dataset, + fine_tune_dataset=fine_tune_dataset, + test_dataset=test_dataset, + callbacks=callbacks) ++ all_data_sum = config.epochs * dataset.get_dataset_size() * config.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum / (end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + def _setup_parallel_env(): diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..d72557e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + : "${EVAL_DATA_PATH?EVAL_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + + [ -d $BASE_PATH/result ] && cp ${RESULT_PATH}/* -rf $BASE_PATH/result/ + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh new file mode 100644 index 0000000..d84c638 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh @@ -0,0 +1,87 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +function get_train_cmd() +{ + CONFIG_FILE=$WORK_PATH/code/default_config.yaml + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py \ + --config_path=$CONFIG_FILE \ + --pre_train_dataset=$TRAIN_DATA_PATH \ + --device_id=${DEVICE_ID} + " + return 0 +} + +function get_eval_cmd() +{ + CONFIG_FILE=$WORK_PATH/code/default_test_config.yaml + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --config_path=$CONFIG_FILE \ + --test_dataset=$EVAL_DATA_PATH \ + --existed_ckpt=$EXISTED_CKPT_PATH \ + --vocab=$VOCAB_ADDR \ + --bpe_codes=$BPE_CODE_ADDR \ + --test_tgt=$TEST_TARGET" + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + cp $CHECKPOINT_PATH $RESULT_PATH/ + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md b/ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md new file mode 100644 index 0000000..0bb01a7 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md @@ -0,0 +1,156 @@ +# 基于MindSpore/mindformers框架的llama大模型训练负载使用指南 +本文主要介绍使用基于LLaMA 或LLaMA2 大模型训练业务代码构建的AISBench的负载包"train_huawei_train_mindspore_llama-Ais-Benchmark-Stubs-{arch}-2.0-r2.2.tar.gz"，进行服务器性能测试的流程。 +## 名词定义 +|名词|定义| +| --- | --- | +|管理节点|运行大模型训练负载的环境，只有一个| +|计算节点|执行训练任务的环境，可以有多个| +## 运行环境前置条件 +### 管理节点 +``` +python >= 3.7 +``` +### 计算节点 +``` +mindspore >= 2.2 +``` +MindSpore安装参考[MindSpore官网](https://www.mindspore.cn/)MindSpore需要能成功在npu上运行，验证命令： +```bash +python -c "import mindspore;mindspore.set_context(device_target='Ascend');mindspore.run_check()" +``` +如果正常输出： +```bash +MindSpore version: 版本号 +The result of multiplication calculation is correct, MindSpore has been installed on platform [Ascend] successfully! +``` +说明成功。 +### 单机多卡与多机多卡的区别 +单机多卡执行负载时，只在单机环境上部署和运行即可；多机多卡执行负载时，多机就是多个计算节点，管理节点必须是其中一个计算节点。
+**多机多卡需注意** +1. 为确保能操作计算节点的数据，管理节点需要是root用户 +## 负载包中文件夹主要目录结构 + +``` +├── ais-bench-stubs # Stubs主程序，负责流程控制、通信与数据管理等 +├── code # 业务代码目录 +│ ├── benchmark.sh # 入口脚本，会被ais-bench-stubs调用，调用业务代码，被测试者需要通过编写该脚本，对接运行的训练和推理脚本 +│ ├── config +│ │ ├── config.sh # 训练相关的配置文件，包括数据集、权重路径等信息 +│ └── code # mindformers全部代码，嵌入了AISBench的打点上报接口 +│ └──mindformers +├── config +│ ├── config.json # 测试配置文件，包含tester服务器信息、testerId等信息 +│ └── system.json # 被测试环境系统基本信息json文件，被测试者自行上传，比如硬件信息等 +├── dependencies # stubs的依赖组件 +│ ├── cluster # 分布式运行组件 +│ │ ├── ais_bench_cluster--py3-none-linux_.whl +│ │ ├── README.md +│ └── logging # 测试结果传输模块 +│ ├── ais_utils.py # 打点入口脚本，设置相关业务运行参数并反馈测试结果 +│ └── libais_utils.so # 测试结果传输模块lib，负责将测试结果传输到stubs模块相关文件部署 +├── log # 测试log日志。建议无需上传的日志文件，另建目录存放 +├── result # 测试结果文件。建议无需上传的结果文件，另建目录存放 +└── STUBS_PACKAGE_INTRO.md # Stubs被测试者接入使用文档 +``` +**后续对于相对路径的描述都是相对于负载包中的一级目录，例如 ./ais-bench-stubs表示Stubs主程序** +## 资源准备 +### 前置声明 +运行LLaMA（LLaMA2）训练的MindSpore/mindformers的代码全部在`./code/code`文件夹中，资源的准备参考[LLaMA资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama.md)或[LLaMA2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama2.md)，具体资源的参考详见本章其他小节。
+**注意**：需要确认计算节点中是否原来已经安装了MindFormers，如果安装了，请使用`pip uninstall mindformers`卸载，确保负载代码的MindFormers能正常在计算节点中安装。 +### rank_table_file准备(llama和llama2通用) +确保`/etc/hccn.conf`文件已经配好（如果没配好，参考[数据中心解决方案/配置训练节点](https://www.hiascend.com/document/detail/zh/Ascend%20Data%20Center%20Solution/22.0.0/install/800_9000/install_800_9000_0029.html)配置）。 + +参考[LLaMA资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama.md)的“生成RANK_TABLE_FILE(多卡运行必须环节)”和“多机RANK_TABLE_FILE合并(多机多卡必备环节)”章节。 + +### 模型权重下载与转换 +LLaMA参考[LLaMA资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama.md)的“模型权重下载与转换”章节； +LLaMA2参考[LLaMA2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama2.md)的“模型权重下载与转换”章节。 + +### 数据集准备 +#### 预训练数据集准备 +LLaMA参考[LLaMA资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama.md)的“预训练/数据集准备-预训练”章节； +LLaMA2参考[LLaMA2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama2.md)“预训练/数据集准备”章节。 +#### 微调数据集准备 +LLaMA参考[LLaMA资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama.md)的“微调/数据集准备-微调”章节； +LLaMA2参考[LLaMA2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama2.md)“微调/数据集准备”章节。 +#### 评测数据集准备 +**wikitext** +LLaMA参考[LLaMA资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama.md)的“评测/文本生成/获取数据集”章节； +LLaMA2参考[LLaMA2资源准备](https://gitee.com/mindspore/mindformers/blob/ac5bb9ec8d1ea85fd2021ca5c6f13b6ae821c270/docs/model_cards/llama2.md)“评测/文本生成/获取数据集”章节。 + +### nodeinfo_file准备（多机多卡训练需要） +nodeinfo_file为json文件，需要用户自行创建（如nodeinfo_file.json）并按照如下格式配置计算节点信息(不要把注释加进去)： +```json +{ + "0": { // 计算节点编号，为用户自定义，非设备实际编号，配置要求：不能重复、必须是0开始的连续整数，例如共有4个节点，节点编号只能取0，1，2，3。若不同节点配置了相同编号，那么只会读取其中一个节点的信息，另一个节点信息则被覆盖，实际运行测试时被覆盖的节点不会被测试。 + "ip": "xx.xx.xx.xx", // 计算节点的ip地址 ipv4 + "user": "user0", // 计算节点的用户名 + "port": 12345, // 访问计算节点的端口 + "work_path": "/xx/xx/xx/xx" // 计算节点的工作路径，管理节点进入节点后处于的路径 + }, + "1":{ + ... + } + ... +} +``` +**注意**：作为多机多卡时的管理节点的计算节点，work_path必须填写`train_huawei_train_mindspore_llama-Ais-Benchmark-Stubs-{arch}-2.0-r2.2/`目录的绝对路径 + +## 负载启动前配置项 +### 和tester连接的配置（仅在线测试需要） +`./config/config.json`和`./config/system.json`请参考《Stubs被测试者接入使用文档》中的“配置与Tester相关的配置文件”章节以及测试机构的要求进行配置。 +### ./code/config/config.sh配置 +`./code/config/config.sh`内容如下： +```bash +#!/bin/bash +echo "set env of llama train" +# 后续对于相对路径的描述都是相对于负载包中的一级目录，例如 ./ais-bench-stubs表示Stubs主程序 + +export PYTHON_COMMAND=python3 +# 以下cluster配置二选一，仅多机场景需要 +export CLUSTER_SSH_KEY_PATH=~/.ssh/id_rsa # 用户指定的ssh私钥，需要确保管理节点通过此私钥能免密访问所有计算节点 +export CLUSTER_AUTO_SET_KEY='on' # 'off' or 'on'，若为'on' 不需要配置CLUSTER_SSH_KEY_PATH + + +export LLAMA_MODEL_SCALE='7b' # '7b' 、'13b' 、 '70b'(仅llama2支持) +export LLAMA_MODEL_TYPE='' # llama : '' ; llama2 : '2' +export LLAMA_RUN_MODE='only_pretrain' # 'only_pretrain', 'only_finetune'，决定负载时执行预训练还是微调 + +# PRETRAIN_DATA_PATH, FINETUNE_DATA_PATH, EVAL_DATASET_PATH 这三个路径是相对./code/code/mindformers源码的路径, 必须以./mindformers/开头 +export PRETRAIN_DATA_PATH=./mindformers/dataset_files/wikitext-2/wiki2048.mindrecord # 预训练数据集 +export FINETUNE_DATA_PATH=./mindformers/dataset_files/alpaca-fastchat2048.mindrecord # 微调数据集 +export EVAL_DATASET_TYPE='wikitext' # 评估数据集名，可选 'wikitext' +export EVAL_DATASET_PATH=./mindformers/dataset_files/wikitext-2/wiki2048valid.mindrecord # 评测用的数据集路径，必须以./mindformers/开头 +export FINETUNE_CKPT_PATH=../../open_llama_7b.ckpt # only for 'only_finetune',相对于train_huawei_train_mindspore_llama-Ais-Benchmark-Stubs-{arch}-2.0-r2.2/code/目录的路径，权重放在code/内将自动分发到计算节点 +export EVAL_DEVICE_ID=0 # 评测用的npu 的device id + +export EPOCH_SIZE=1 # 全量遍历数据集的迭代次数 +export LLAMA_LAYER_NUM=32 # 7b:32 13b:40 70b: 80 + +export RANK_SIZE=8 # 集群总加速卡数 +export DEVICE_NUM=8 # 集群每个节点的加速卡数 + +# parallel run params, parallel strategy config, DATA_PARALLEL * MODEL_PARALLEL * PIPELINE_STAGE should equal to RANK_SIZE +export DATA_PARALLEL=2 +export MODEL_PARALLEL=1 +export PIPELINE_STAGE=4 + +# need if rank_size > 1 +export RANK_TABLE_FILE=hccl_xxxx_8p.json # rank_table_file的路径，相对于train_huawei_train_mindspore_llama-Ais-Benchmark-Stubs-{arch}-2.0-r2.2/code/目录的路径，rank_table_file需要放在code/内 + +# 多机多卡需要配置，单机不需要配置 +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json +``` +请参考`./code/config/config.sh`的注释将第一章准备的资源的路径在`config.sh`中配置好，并且确定好训练相关的参数 + +## 负载启动 +### 在线测试 +在管理节点上执行命令 +```bash +./ais-bench-stubs +``` +### 轻量化离线测试 +在管理节点上执行命令 +```bash +./ais-bench-stubs test +``` \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh new file mode 100644 index 0000000..568350c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh @@ -0,0 +1,34 @@ +#!/bin/bash +echo "start to build llama workload" + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + return $ret_ok +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp -f ${CURDIR}/README.md ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + + mkdir -p ${CURDIR}/output/config + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/config.sh -r ${CURDIR}//output/config/ + [ -d ${CURDIR}/doc ] && cp ${CURDIR}/doc -r ${CURDIR}/output/ + + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh new file mode 100644 index 0000000..ce63835 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh @@ -0,0 +1,38 @@ +#!/bin/bash +echo "set env of llama train" +# 后续对于相对路径的描述都是相对于负载包中的一级目录，例如 ./ais-bench-stubs表示Stubs主程序 + +export PYTHON_COMMAND=python3 + +# 以下cluster配置二选一，仅多机场景需要 +export CLUSTER_SSH_KEY_PATH=~/.ssh/id_rsa # 用户指定的ssh私钥，需要确保管理节点通过此私钥能免密访问所有计算节点 +export CLUSTER_AUTO_SET_KEY='on' # 'off' or 'on'，若为'on' 不需要配置CLUSTER_SSH_KEY_PATH + +export LLAMA_MODEL_SCALE='7b' # '7b' 、'13b' 、 '70b'(仅llama2支持) +export LLAMA_MODEL_TYPE='' # llama : '' ; llama2 : '2' +export LLAMA_RUN_MODE='only_pretrain' # 'only_pretrain', 'only_finetune'，决定负载时执行预训练还是微调 + +# PRETRAIN_DATA_PATH, FINETUNE_DATA_PATH, EVAL_DATASET_PATH 这三个路径是相对./code/code/mindformers源码的路径, 必须以./mindformers/开头 +export PRETRAIN_DATA_PATH=./mindformers/dataset_files/wikitext-2/wiki2048.mindrecord # 预训练数据集 +export FINETUNE_DATA_PATH=./mindformers/dataset_files/alpaca-fastchat2048.mindrecord # 微调数据集 +export EVAL_DATASET_TYPE='wikitext' # 评估数据集名，可选 'wikitext' +export EVAL_DATASET_PATH=./mindformers/dataset_files/wikitext-2/wiki2048valid.mindrecord # 评测用的数据集路径，必须以./mindformers/开头 +export FINETUNE_CKPT_PATH=open_llama_7b.ckpt # only for 'only_finetune',相对于train_huawei_train_mindspore_llama-Ais-Benchmark-Stubs--2.0-r2.2/code/目录的路径，权重放在code/内将自动分发到计算节点 +export EVAL_DEVICE_ID=0 # 评测用的npu 的device id + +export EPOCH_SIZE=1 # 全量遍历数据集的迭代次数 +export LLAMA_LAYER_NUM=32 # 7b:32 13b:40 70b: 80 + +export RANK_SIZE=8 # 集群总加速卡数 +export DEVICE_NUM=8 # 集群每个节点的加速卡数 + +# parallel run params, parallel strategy config, DATA_PARALLEL * MODEL_PARALLEL * PIPELINE_STAGE should equal to RANK_SIZE +export DATA_PARALLEL=2 +export MODEL_PARALLEL=1 +export PIPELINE_STAGE=4 + +# need if rank_size > 1 +export RANK_TABLE_FILE=hccl_xxxx_8p.json # rank_table_file的路径，相对于train_huawei_train_mindspore_llama-Ais-Benchmark-Stubs--2.0-r2.2/code/目录的路径，rank_table_file需要放在code/内 + +# 多机多卡需要配置，单机不能配置 +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh new file mode 100644 index 0000000..8b4e8eb --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r2.2"; } + + modelzoo_sub_dir="mindformers" + if [ "$branch_args" == "r2.2" ];then + branch="r0.8" + patch_file_name="r2.2" + commitid="c0f478fc517b1daec896f5c72bcea10b2ab83bd4" + git_url="https://gitee.com/mindspore/mindformers.git" + else + echo "bad parameters : $1" + return $ret_error + fi + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + local changed_code_path="$4" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + target_dir=$changed_code_path + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch b/ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch new file mode 100644 index 0000000..73b95b3 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch @@ -0,0 +1,231 @@ +diff -Nur origin/mindformers/core/callback/callback.py code/mindformers/core/callback/callback.py +--- origin/mindformers/core/callback/callback.py 2023-12-06 11:50:58.448000000 +0800 ++++ code/mindformers/core/callback/callback.py 2023-12-06 11:50:58.540000000 +0800 +@@ -33,7 +33,11 @@ + from mindformers.tools.register import MindFormerRegister, MindFormerModuleType + from mindformers.tools.cloud_adapter.cloud_adapter import Local2ObsMonitor + from mindformers.tools.logger import logger +-from mindformers.tools.utils import get_output_root_path, get_output_subpath, get_remote_save_url, check_in_modelarts ++from mindformers.tools.utils import get_output_root_path, get_output_subpath, get_remote_save_url, check_in_modelarts, save_aisbench_result ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + __all__ = ['ObsMonitor', 'MFLossMonitor', 'CheckpointMointor', 'SummaryMonitor', 'ProfileMonitor', 'EvalCallBack'] + +@@ -551,6 +555,7 @@ + + if save_ckpt: + logger.info('......Saving ckpt......') ++ save_aisbench_result("model_persistence_start_time", ais_utils.get_datatime().decode('utf-8')) + cur_ckpoint_file = self._prefix + "-" + str(cb_params.cur_epoch_num) + "_" \ + + str(step_num_in_epoch) + ".ckpt" + # update checkpoint file list. +@@ -629,7 +634,7 @@ + self._config.async_save, {}, self._config.enc_key, self._config.enc_mode) + + save_only_network_params() +- ++ save_aisbench_result("model_persistence_end_time", ais_utils.get_datatime().decode('utf-8')) + self._latest_ckpt_file_name = cur_file + + +diff -Nur origin/mindformers/core/metric/metric.py code/mindformers/core/metric/metric.py +--- origin/mindformers/core/metric/metric.py 2023-12-06 11:50:58.448000000 +0800 ++++ code/mindformers/core/metric/metric.py 2023-12-06 11:50:58.540000000 +0800 +@@ -39,6 +39,11 @@ + + from .utils import PerplexityCell + from ...dataset.labels import cluener_labels ++from mindformers.tools.utils import save_aisbench_result ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + __all__ = ['EntityScore', 'SQuADMetric', 'PerplexityMetric', 'ADGENMetric', 'PromptAccMetric', 'EmF1Metric'] + +@@ -541,6 +546,12 @@ + return None + avg_loss = float(self.total_loss / self.num_data) + result = {"loss": avg_loss, "PPL": math.exp(avg_loss)} ++ result_log="loss: {}, Perplexity: {}".format(avg_loss, math.exp(avg_loss)) ++ try: ++ import ais_utils ++ ais_utils.set_result("training", "accuracy", result_log) ++ except Exception: ++ print("ais_utils not find") + if self.pipeline_parallel: + print("Average Loss and PPL Metric:", result) + return result +diff -Nur origin/mindformers/tools/transform_ckpt.py code/mindformers/tools/transform_ckpt.py +--- origin/mindformers/tools/transform_ckpt.py 2023-12-06 11:50:58.456000000 +0800 ++++ code/mindformers/tools/transform_ckpt.py 2023-12-06 11:50:58.548000000 +0800 +@@ -17,6 +17,11 @@ + import argparse + + import mindspore as ms ++from mindformers.tools.utils import save_aisbench_result ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + def get_strategy(startegy_path, rank_id=None): + """Merge strategy if strategy path is dir +@@ -84,6 +89,8 @@ + print(f"dst_ckpt_dir: {dst_ckpt_dir}") + print(f"prefix: {prefix}") + +- print("......Start transform......") ++ print("......Start transform......") # model_format_start_time ++ ais_utils.set_result("training", "model_format_start_time", ais_utils.get_datatime().decode('utf-8')) + ms.transform_checkpoints(src_ckpt_dir, dst_ckpt_dir, prefix, src_ckpt_strategy, dst_ckpt_strategy) +- print("......Transform succeed!......") ++ print("......Transform succeed!......") # model_format_end_time ++ ais_utils.set_result("training", "model_format_end_time", ais_utils.get_datatime().decode('utf-8')) +diff -Nur origin/mindformers/tools/utils.py code/mindformers/tools/utils.py +--- origin/mindformers/tools/utils.py 2023-12-06 11:50:58.456000000 +0800 ++++ code/mindformers/tools/utils.py 2023-12-06 11:50:58.548000000 +0800 +@@ -55,6 +55,28 @@ + _PROTOCOL = 'obs' + _PROTOCOL_S3 = 's3' + ++AISBENCH_RESULT_PATH=os.getenv('RESULT_PATH') ++ ++ ++def save_aisbench_result(rt_key:str, rt_value): ++ cur_rank_id = os.getenv("RANK_ID", "0") ++ cur_run_mode = os.getenv("LLAMA_CUR_RUN_MODE", "train") ++ result_file_path = os.path.join(AISBENCH_RESULT_PATH, f"result_rank_{cur_rank_id}.json") ++ if not os.path.exists(result_file_path): ++ with open(result_file_path, "w") as f: ++ init_data = {} ++ json.dump(init_data, f) ++ with open(result_file_path, "r") as result_file: ++ result_log = json.load(result_file) ++ if not cur_run_mode in result_log: ++ result_log[cur_run_mode] = {} ++ if rt_key in result_log[cur_run_mode]: ++ result_log[cur_run_mode][rt_key].append(rt_value) ++ else: ++ result_log[cur_run_mode][rt_key] = [rt_value] ++ with open(result_file_path, "w") as result_file: ++ json.dump(result_log, result_file) ++ + + def check_in_modelarts(): + """Check if the training is on modelarts. +diff -Nur origin/mindformers/trainer/base_trainer.py code/mindformers/trainer/base_trainer.py +--- origin/mindformers/trainer/base_trainer.py 2023-12-06 11:50:58.456000000 +0800 ++++ code/mindformers/trainer/base_trainer.py 2023-12-06 11:50:58.552000000 +0800 +@@ -42,7 +42,7 @@ + from mindformers.wrapper import build_wrapper + from mindformers.tools.register import MindFormerConfig + from mindformers.tools.logger import logger +-from mindformers.tools.utils import count_params ++from mindformers.tools.utils import count_params, save_aisbench_result + from mindformers.auto_class import AutoModel + from mindformers.pet import get_pet_model + from .config_args import ConfigArguments +@@ -51,6 +51,10 @@ + from .optimizer_grouped_parameters import get_optimizer_grouped_parameters + from .utils import set_seed, check_train_data_loader_type, \ + check_eval_data_loader_type, check_optimizer_and_lr_type, check_wrapper_config ++try: ++ import ais_utils ++except Exception: ++ print("ais_utils not find") + + SUPPORT_TASKS = MindFormerBook().get_trainer_support_task_list() + SUPPORT_MODEL_NAMES = MindFormerBook().get_model_name_support_list() +@@ -543,7 +547,8 @@ + load_resume_context_from_checkpoint(config) + + # build dataset +- logger.info(".........Build Dataset For Train..........") ++ logger.info(".........Build Dataset For Train..........") # dataload_start_time ++ save_aisbench_result("dataload_start_time", ais_utils.get_datatime().decode('utf-8')) + if dataset is None: + dataset = self.create_train_dataset() + self.set_train_dataset(dataset) +@@ -553,10 +558,12 @@ + * config.runner_config.sink_size / dataset.get_dataset_size()) + # pylint: disable=W0212 + dataset._dataset_helper = DatasetHelper(dataset, config.runner_config.sink_mode, +- config.runner_config.sink_size, epoch_num) ++ config.runner_config.sink_size, epoch_num) # dataload_end_time ++ save_aisbench_result("dataload_end_time", ais_utils.get_datatime().decode('utf-8')) + + # build network +- logger.info(".........Build Net For Train..........") ++ logger.info(".........Build Net For Train..........") # train_launch_start_time ++ save_aisbench_result("train_launch_start_time", ais_utils.get_datatime().decode('utf-8')) + eval_network = None + if network is None and wrapper is None and \ + self.model_wrapper is None and self.network is None: +@@ -651,18 +658,30 @@ + step_interval=config.eval_step_interval if config.eval_step_interval else 100, + epoch_interval=config.eval_epoch_interval if config.eval_epoch_interval else -1, + ) +- callbacks.append(eval_callback) ++ callbacks.append(eval_callback) # train_launch_end_time + + logger.info(".........Starting Training Model..........") + if int(os.getenv("RANK_ID", '0')) % 8 == 0: + pprint(config) + logger.info(".........Model Compiling, Please Wait a Moment...........") ++ model.build(dataset, None, sink_size=config.runner_config.sink_size, epoch=config.runner_config.epochs) ++ save_aisbench_result("train_launch_end_time", ais_utils.get_datatime().decode('utf-8')) ++ import time ++ train_start_time = time.time() ++ save_aisbench_result("train_start_time", ais_utils.get_datatime().decode('utf-8')) # train_start_time + model.train(config.runner_config.epochs, dataset, + callbacks=callbacks, + dataset_sink_mode=config.runner_config.sink_mode, + sink_size=config.runner_config.sink_size, + initial_epoch=config.runner_config.initial_epoch) +- logger.info(".........Training Over!.............") ++ logger.info(".........Training Over!.............") # train_end_time ++ train_end_time = time.time() ++ all_data_sum = int(dataset.get_dataset_size() * config.train_dataset.batch_size / int(os.getenv("RANK_SIZE", '8'))) * \ ++ config.runner_config.origin_epochs * config.model.model_config.seq_length ++ throughput_rate = all_data_sum / (train_end_time - train_start_time) ++ save_aisbench_result("train_end_time", ais_utils.get_datatime().decode('utf-8')) ++ save_aisbench_result("throughput_ratio", throughput_rate) ++ + + def evaluate_process( + self, +diff -Nur origin/requirements.txt code/requirements.txt +--- origin/requirements.txt 2023-12-06 11:50:58.396000000 +0800 ++++ code/requirements.txt 2023-12-06 11:50:58.492000000 +0800 +@@ -11,4 +11,5 @@ + pydantic==1.10.11 + mdtex2html + gradio +-opencv-python-headless +\ No newline at end of file ++opencv-python-headless ++pyyaml +\ No newline at end of file +diff -Nur origin/scripts/run_distribute.sh code/scripts/run_distribute.sh +--- origin/scripts/run_distribute.sh 2023-12-06 11:50:58.460000000 +0800 ++++ code/scripts/run_distribute.sh 2023-12-06 11:50:58.556000000 +0800 +@@ -179,7 +179,14 @@ + fi + fi + shopt -u extglob +- ++wait ++if [ $? -eq 0 ];then ++ echo "all train processes completed successfully!" ++ exit 0 ++else ++ echo "one or more train processes exited with a error!" ++ exit 1 ++fi + + #cd ./pretrain_parallel${START_DEVICE} || exit + #tail -f mindformer.log diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh new file mode 100644 index 0000000..a028257 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 +declare -i ret_mode_failed=5 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) +export DEPEND_PATH=$BASE_PATH/dependencies/ + +function get_node_train_data() +{ + if [ "$LLAMA_RUN_MODE" = "only_finetune" ];then + [ ! -f $FINETUNE_CKPT_PATH ] || { echo "finetune base ckpt:$FINETUNE_CKPT_PATH";return $ret_failed; } + fi + return $ret_ok +} + +# 配置训练相关的环境变量 +source ${CODE_PATH}/config/config.sh || { logger_Warn "source file failed:$?";return $ret_init_failed; } + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh +if [ -d $PRETRAIN_DATA_PATH ];then + cp -r $PRETRAIN_DATA_PATH $CUR_PATH || { logger_Warn "ERROR: cp $PRETRAIN_DATA_PATH failed!";return $ret_init_failed; } +fi +if [ -d $FINETUNE_DATA_PATH ];then + cp -r $FINETUNE_DATA_PATH $CUR_PATH || { logger_Warn "ERROR: cp $FINETUNE_DATA_PATH failed!";return $ret_init_failed; } +fi + +. $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..48f1f14 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh @@ -0,0 +1,146 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common_2.0.sh +. $CODE_PATH/common/node_common.sh + +# env check +export RELAT_WORK_PATH=work +export RELAT_RESULT_PATH=$RELAT_WORK_PATH/result +CONFIG_FILE="config.sh" +# set nodes work path. 仅仅是管理节点的work/ +export WORK_PATH=${BASE_PATH}/work +# set nodes result path +export RESULT_PATH=${WORK_PATH}/result +local_env_cmd="source /etc/profile; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + export PYTHONPATH=$WORK_PATH:$PYTHONPATH; + export PYTHONPATH=$WORK_PATH/logging:$PYTHONPATH; + source $WORK_PATH/config/$CONFIG_FILE" +env_cmd="source /etc/profile; + export WORK_PATH=\$PWD/$RELAT_WORK_PATH; + export RESULT_PATH=\$PWD/$RELAT_RESULT_PATH; + export PYTHONPATH=\$WORK_PATH:\$PYTHONPATH; + export PYTHONPATH=\$WORK_PATH/logging:\$PYTHONPATH; + source \$WORK_PATH/config/$CONFIG_FILE" + +check_env() +{ + # check ranktable set + : "${RANK_SIZE?RANK_SIZE not set}" + : "${DEVICE_NUM?DEVICE_NUM not set}" + [[ $RANK_SIZE -eq 1 ]] || : "${RANK_TABLE_FILE?RANK_TABLE_FILE not set}" + [[ $RANK_SIZE -eq 1 ]] && [[ -n "$RANK_TABLE_FILE" ]] && { echo "ranksize=1 should not set RANK_TABLE_FILE";return 1; } + + # check python + : "${PYTHON_COMMAND?PYTHON_COMMAND not set}" + [ "$NODEINFO_FILE" == "" ] && { echo "NODEINFO_FILE not set, will not check cluster";return 0; } + if pip show ais_bench_cluster >/dev/null 2>&1;then + logger_Info "ais_bench cluster module exist, won't be installed again" + else + cluster_whl_path="${DEPEND_PATH}/cluster/ais_bench_cluster-*.whl" + if [ -f $cluster_whl_path ];then + pip install $cluster_whl_path --force-reinstall || { logger_Error "install cluster failed!";return 1; } + else + logger_Error "can't find ais_bench cluster wheel package" + fi + fi + + # check nodeinfofile exist + [[ $RANK_SIZE -le 8 ]] || check_file_valid "${NODEINFO_FILE}" || { echo "nodeinfofile:${NODEINFO_FILE} not valid" ; return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Error "source file failed:$?";return 1; } + if [ -d ${DEPEND_PATH}/logging ];then + cp -r ${DEPEND_PATH}/logging ${CODE_PATH} + fi + check_env || { logger_Error "env check failed'" ; return 1; } + + # init ais_bench.cluster + cluster_init || { logger_Error "ais_bench_cluster init failed!";return 1; } + + # refresh result path + rm -rf ${BASE_PATH}/result;mkdir -p ${BASE_PATH}/result + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + rm -rf $WORK_PATH;mkdir -p $WORK_PATH + if [ "$NODEINFO_FILE" != "" ];then + cmd="rm -rf ${RELAT_WORK_PATH};mkdir -p ${RELAT_WORK_PATH}" + cluster_multi_exec "$cmd" serial || { logger_Error "renew workpath failed"; return 1; } + # copy code to node work path + fi + cp -r $CODE_PATH/* $WORK_PATH # CPU可以执行的都在host节点执行 + if [ "$NODEINFO_FILE" != "" ];then + # sync data if work_path not exist so new one.节点的work/ 路径是相对于在node_file中指定的work_path + cluster_multi_put "$WORK_PATH" "./" || { logger_Error "deploy code to work place failed"; return 1; } + fi + cmd="source /etc/profile; + export WORK_PATH=\$PWD/$RELAT_WORK_PATH; + source \$WORK_PATH/config/$CONFIG_FILE; + bash \$WORK_PATH/run_node.sh check" + cluster_multi_exec "$cmd" serial|| { return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + if [ "$LLAMA_RUN_MODE" == "full" ] || [ "$LLAMA_RUN_MODE" == "only_pretrain" ];then + if [ "$NODEINFO_FILE" == "" ];then + cmd="$local_env_cmd; + rm -rf $RESULT_PATH/*.json; + bash $WORK_PATH/run_node.sh train train " + else + cmd="$env_cmd; + rm -rf \$RESULT_PATH/*.json; + bash \$WORK_PATH/run_node.sh train train " + fi + cluster_multi_exec "$cmd" || { logger_Error "run train(pretrain) failed"; return 1; } + if [ "$NODEINFO_FILE" != "" ];then + cluster_multi_get "$RELAT_RESULT_PATH" "$BASE_PATH" || { logger_Error "cp result between nodes failed"; return 1; } + fi + export PYTHONPATH=$WORK_PATH/logging:$PYTHONPATH + bash $WORK_PATH/run_node.sh merge || { logger_Error "ckpt merge failed"; return 1; } + fi + if [ "$LLAMA_RUN_MODE" == "full" ] || [ "$LLAMA_RUN_MODE" == "only_finetune" ];then + if [ "$NODEINFO_FILE" == "" ];then + cmd="$local_env_cmd; + rm -rf $RESULT_PATH/*.json; + bash $WORK_PATH/run_node.sh train finetune " + else + cmd="$env_cmd; + rm -rf \$RESULT_PATH/*.json; + bash \$WORK_PATH/run_node.sh train finetune " + fi + cluster_multi_exec "$cmd" || { logger_Error "run train(finetune) failed"; return 1; } + if [ "$NODEINFO_FILE" != "" ];then + cluster_multi_get "$RELAT_RESULT_PATH" "$BASE_PATH" || { logger_Error "cp result between nodes failed"; return 1; } + fi + export PYTHONPATH=$WORK_PATH/logging:$PYTHONPATH + bash $WORK_PATH/run_node.sh merge || { logger_Error "ckpt merge failed"; return 1; } + fi + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="$local_env_cmd; + bash $WORK_PATH/run_node.sh eval" + eval "$cmd" || { logger_Error "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + source ${CODE_PATH}/config/$CONFIG_FILE + export PYTHONPATH=${CODE_PATH}/logging:$PYTHONPATH + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_llm_result.py ${BASE_PATH}/result ${RANK_SIZE} ${LLAMA_RUN_MODE} + find $BASE_PATH/result/ -name "*.ckpt" -exec rm {} \; + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py new file mode 100644 index 0000000..bf150fa --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py @@ -0,0 +1,121 @@ + +# should be execute after distribut +import sys +import os +import yaml +import acl + +run_mode = sys.argv[1] +cur_path = os.path.dirname(os.path.abspath(__file__)) +try: + soc_version = acl.get_soc_name() +except Exception as err: + raise RuntimeError("get soc versiob failed!") from err + +pretrain_dataset = os.getenv('PRETRAIN_DATA_PATH') +finetune_dataset = os.getenv('FINETUNE_DATA_PATH') + +rank_size = int(os.getenv('RANK_SIZE')) +data_parallel = int(os.getenv('DATA_PARALLEL')) +model_parallel = int(os.getenv('MODEL_PARALLEL')) +pipeline_stage = int(os.getenv('PIPELINE_STAGE')) + +if not rank_size == data_parallel * model_parallel * pipeline_stage and run_mode != "finetune_eval": + raise RuntimeError("DATA_PARALLEL * MODEL_PARALLEL * PIPELINE_STAGE should equal to RANK_SIZE !") + +model_scale = os.getenv('LLAMA_MODEL_SCALE') +model_type = os.getenv('LLAMA_MODEL_TYPE') +config_path = os.path.join(cur_path, f'code/configs/llama{model_type}/') +epoch_size = os.getenv("EPOCH_SIZE") +sink_size = 2 +layer_num = os.getenv("LLAMA_LAYER_NUM") +eval_data_path = os.getenv('EVAL_DATASET_PATH') + +if '910B'in soc_version: + target_yaml = os.path.join(config_path, f'run_llama{model_type}_{model_scale}_910b.yaml') +else: + target_yaml = os.path.join(config_path, f'run_llama{model_type}_{model_scale}.yaml') +if os.getenv('LLAMA_RUN_MODE') == 'only_finetune': + ckpt_path = os.path.join(cur_path, os.getenv('FINETUNE_CKPT_PATH')) + if not os.path.exists(ckpt_path): + raise FileExistsError(f"ckpt_path: {ckpt_path} not find!") +else: + ckpt_path = os.path.join(cur_path, f'result/output/target_ckpt/rank_0/llama{model_type}_{model_scale}0.ckpt') + +if not os.path.exists(target_yaml): + raise FileExistsError(f"yaml file: {target_yaml} not find!") +if os.path.islink(target_yaml): + raise RuntimeError(f"yaml file: {target_yaml} is softlink!") + +if model_type == "2" and run_mode == "finetune_eval": + seq_length = 4096 +else: + seq_length = 2048 + +def change_parallel_params(data): + data_parallel = int(data['parallel_config']['data_parallel']) + model_parallel = int(data['parallel_config']['model_parallel']) + pipeline_stage = int(data['parallel_config']['pipeline_stage']) + + return int(rank_size / (data_parallel * model_parallel * pipeline_stage)) + +def write_pretrain_yaml(data): + data['load_checkpoint'] = '' + data['run_mode'] = 'train' + data['runner_config']['epochs'] = int(epoch_size) + data['runner_config']['sink_size'] = sink_size + data['parallel_config']['data_parallel'] = data_parallel + data['parallel_config']['model_parallel'] = model_parallel + data['parallel_config']['pipeline_stage'] = pipeline_stage + data['optimizer']['beta2'] = 0.95 + data['optimizer']['learning_rate'] = 3.e-4 + data['lr_schedule']['learning_rate'] = 3.e-4 + data['lr_schedule']['lr_end'] = 3.e-5 + data['train_dataset']['input_columns'] = ["input_ids"] + data['train_dataset']['data_loader']['dataset_dir'] = pretrain_dataset + data['eval_dataset']['data_loader']['dataset_dir'] = eval_data_path + data['model']['model_config']['num_layers'] = int(layer_num) + data['callbacks'][1]['save_checkpoint_steps'] = 100000 + return data + + +def write_finetune_yaml(data): + data['load_checkpoint'] = ckpt_path + data['run_mode'] = 'finetune' + data['runner_config']['epochs'] = int(epoch_size) + data['runner_config']['sink_size'] = sink_size + data['parallel_config']['data_parallel'] = data_parallel + data['parallel_config']['model_parallel'] = model_parallel + data['parallel_config']['pipeline_stage'] = pipeline_stage + data['optimizer']['beta2'] = 0.999 + data['optimizer']['learning_rate'] = 1.e-5 + data['lr_schedule']['learning_rate'] = 1.e-5 + data['lr_schedule']['lr_end'] = 1.e-5 + data['train_dataset']['input_columns'] = ["input_ids", "labels"] + data['train_dataset']['data_loader']['dataset_dir'] = finetune_dataset + data['eval_dataset']['data_loader']['dataset_dir'] = eval_data_path + data['model']['model_config']['num_layers'] = int(layer_num) + data['model']['model_config']['seq_length'] = seq_length + data['callbacks'][1]['save_checkpoint_steps'] = 100000 + return data + +data = {} +with open(target_yaml, 'r', encoding='utf-8') as file: + try: + data = yaml.safe_load(file) + except Exception as err: + raise RuntimeError(f"load target_yaml: {target_yaml} failed") from err + +if run_mode == 'train': + data = write_pretrain_yaml(data) +elif run_mode == "finetune": + data = write_finetune_yaml(data) +elif run_mode == "finetune_eval": + data = write_finetune_yaml(data) + +with open(target_yaml, 'w', encoding='utf-8') as file: + try: + yaml.safe_dump(data, file) + except Exception as err: + raise RuntimeError(f"dump target_yaml: {target_yaml} failed") from err + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh new file mode 100644 index 0000000..87a2dbe --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh @@ -0,0 +1,181 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +declare -i ret_ok=0 +declare -i ret_failed=1 + +SOC_VERSION=`python3 -c 'import acl;print(acl.get_soc_name())'` +if [[ "$SOC_VERSION" =~ "910B1" || "$SOC_VERSION" =~ "910B2" || "$SOC_VERSION" =~ "910B3" || "$SOC_VERSION" =~ "910B4" ]];then + LLAMA_RUN_YAML_NAME="run_llama${LLAMA_MODEL_TYPE}_${LLAMA_MODEL_SCALE}_910b.yaml" +else + LLAMA_RUN_YAML_NAME="run_llama${LLAMA_MODEL_TYPE}_${LLAMA_MODEL_SCALE}.yaml" +fi + +function get_node_rank_id_range() +{ + RANK_ID_RANGE="[0,8]" + # get server node id default is 0 + : "${NODE_ID:=0}" + # get rank start index + if [[ $DEVICE_NUM == 1 && $RANK_SIZE == 1 ]];then + : "${SINGLE_CARD_INDEX:=0}" + RANK_START=$SINGLE_CARD_INDEX + else + # get rank start index + RANK_START=`expr ${NODE_ID} \* $DEVICE_NUM` + fi + RANK_ID_MAX=$[DEVICE_NUM+RANK_START] + RANK_ID_RANGE="[$RANK_START,$RANK_ID_MAX]" +} + +function node_init() +{ + export PYTHONPATH=$WORK_PATH:$PYTHONPATH + + if [ $1 == "check" ];then + # install pyyaml + if pip show pyyaml >/dev/null 2>&1;then + logger_Info "pyyaml exist, won't be installed again" + else + pip_cmd="pip install pyyaml" + $pip_cmd || { logger_Warn "pyyaml install failed:$?";return $ret_failed; } + fi + # install mindformers + if pip show mindformers >/dev/null 2>&1;then + logger_Info "mindformers exist, won't be installed again" + else + cd $WORK_PATH/code + pip install . || { logger_Warn "mindformers install failed:$?";return $ret_failed; } + cd $WORK_PATH + fi + fi + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + get_node_rank_id_range + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + rank_table_path=${WORK_PATH}/${RANK_TABLE_FILE} + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$rank_table_path" || { logger_Warn "node common check failed" ; return $ret_failed; } + + # check_mindspore_run_ok_Ascend ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return $ret_failed; } + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return $ret_failed; } + logger_Debug "mindspore running successfully" + + if [ "$LLAMA_RUN_MODE" == "full" ] || [ "$LLAMA_RUN_MODE" == "train" ];then + check_file_valid "${WORK_PATH}/code/${PRETRAIN_DATA_PATH}" || { logger_Warn "PRETRAIN_DATA_PATH:${PRETRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "PRETRAIN_DATA_PATH is valid" + fi + + if [ "$LLAMA_RUN_MODE" == "full" ] || [ "$LLAMA_RUN_MODE" == "finetune" ];then + check_file_valid "${WORK_PATH}/code/${FINETUNE_DATA_PATH}" || { logger_Warn "FINETUNE_DATA_PATH:${FINETUNE_DATA_PATH} not valid path" ; return 1; } + logger_Debug "FINETUNE_DATA_PATH is valid" + check_file_valid "${WORK_PATH}/code/${EVAL_DATASET_PATH}" || { logger_Warn "EVAL_DATASET_PATH:${EVAL_DATASET_PATH} not valid path" ; return 1; } + logger_Debug "EVAL_DATASET_PATH is valid" + fi + +} + +function ckpt_merge() +{ + transform_ckpt_path=$WORK_PATH/code/mindformers/tools/transform_ckpt.py + result_output_path=$WORK_PATH/../result/output + cd $WORK_PATH + # ckpt merge + $PYTHON_COMMAND $transform_ckpt_path \ + --src_ckpt_strategy $result_output_path/strategy/ \ + --src_ckpt_dir $result_output_path/checkpoint/ \ + --dst_ckpt_dir $result_output_path/target_ckpt/ \ + --prefix "llama${LLAMA_MODEL_TYPE}_${LLAMA_MODEL_SCALE}" || { logger_Warn "ckpt merge failed, rank id range: $RANK_ID_RANGE" ; return $ret_failed; } + rm -rf $result_output_path/checkpoint/ +} + +function node_train() +{ + logger_Info "node_train running" + export LLAMA_CUR_RUN_MODE=$1 + source $WORK_PATH/config/config.sh + $PYTHON_COMMAND $WORK_PATH/pre_conf_yaml.py $1 # change yaml params + run_script_path=$WORK_PATH/code/scripts/ + run_yaml_path=$WORK_PATH/code/configs/llama${LLAMA_MODEL_TYPE}/$LLAMA_RUN_YAML_NAME + rank_table_path=${WORK_PATH}/$RANK_TABLE_FILE + # train run + cd $run_script_path + cmd="bash run_distribute.sh $rank_table_path $run_yaml_path $RANK_ID_RANGE $1" + [ "$NODEINFO_FILE" != "" ] && cmd="$cmd $RANK_SIZE" + echo "$cmd" + $cmd || { logger_Warn "node_run failed, rank id range: $RANK_ID_RANGE" ; return $ret_failed; } + mv $WORK_PATH/code/output/ $WORK_PATH/result/ || { logger_Warn "move output failed!" ; return $ret_failed; } + return $ret_ok +} + +function eval_run() +{ + logger_Info "eval_run running" + $PYTHON_COMMAND $WORK_PATH/pre_conf_yaml.py $1 # change yaml params + run_yaml_path=$WORK_PATH/code/configs/llama${LLAMA_MODEL_TYPE}/$LLAMA_RUN_YAML_NAME + eval_dataset_path=$WORK_PATH/code/$EVAL_DATASET_PATH + load_checkpoint_path=$WORK_PATH/../result/output/target_ckpt/rank_0/llama${LLAMA_MODEL_TYPE}_${LLAMA_MODEL_SCALE}0.ckpt + if [ "$EVAL_DATASET_TYPE" = "wikitext" ];then + echo "run eval using wiki" + eval_script_path=$WORK_PATH/code/run_mindformer.py + $PYTHON_COMMAND $eval_script_path \ + --config $run_yaml_path \ + --eval_dataset_dir $eval_dataset_path \ + --run_mode eval \ + --load_checkpoint $load_checkpoint_path \ + --epochs 1 \ + --use_parallel False \ + --device_id $EVAL_DEVICE_ID || { logger_Warn "run eval failed" ; return $ret_failed; } + elif [ "$EVAL_DATASET_TYPE" == "squad" ];then + echo "eval not supported yet" + else + echo "invalid eval mode" + rm -rf $load_checkpoint_path + return $ret_failed + fi + rm -rf $load_checkpoint_path + return $ret_ok +} + +function node_eval() +{ + logger_Info "node_eval running" + if [ "$LLAMA_RUN_MODE" == "full" ];then + eval_run "finetune_eval" + elif [ "$LLAMA_RUN_MODE" == "only_pretrain" ];then + echo "eval not supported yet" + elif [ "$LLAMA_RUN_MODE" == "only_finetune" ];then + eval_run "finetune_eval" + else + echo "llama run mode not supported" + return $ret_failed + fi + return $ret_ok +} + +main() +{ + type="$1" + mode="$2" + shift + node_init $type || { logger_Warn "init failed"; return $ret_failed; } + if [ "$type" == "train" ];then + node_train $mode || { logger_Warn "run_node_train failed"; return $ret_failed; } + elif [ "$type" == "merge" ];then + ckpt_merge || { logger_Warn "ckpt_merge failed"; return $ret_failed; } + elif [ "$type" == "eval" ];then + node_eval || { logger_Warn "run_node_eval failed"; return $ret_failed; } + elif [ "$type" == "check" ];then + node_check || { logger_Warn "run_node_check failed"; return $ret_failed; } + else + { logger_Warn "invalid argument '${type}'"; return $ret_failed; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh new file mode 100644 index 0000000..065998c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh new file mode 100644 index 0000000..8a8bb37 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh @@ -0,0 +1,17 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7 +export TRAIN_DATA_PATH=/home/datasets/pangu_30_step_ba64 +export PARAM_INIT_TYPE=fp32 +export MODE=2.6B +export STAGE_NUM=1 +export MICRO_SIZE=1 +export PER_BATCH=8 + +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# need if rank_size > 1 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_64.json +# cluster need for node info +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh new file mode 100644 index 0000000..5122d7c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="abc34438588942642e45e7cf1e516134952a2f86" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/nlp/pangu_alpha" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..2bdca78 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_FILE?TRAIN_DATA_FILE not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh new file mode 100644 index 0000000..fffd35b --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh @@ -0,0 +1,83 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="True" || DISTRUTE_ENABLE="False" + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py --distribute=$DISTRUTE_ENABLE \ + --device_num=$DEVICE_NUM \ + --data_url=$TRAIN_DATA_PATH \ + --run_type=train \ + --param_init_type=$PARAM_INIT_TYPE \ + --mode=$MODE \ + --stage_num=$STAGE_NUM \ + --micro_size=$MICRO_SIZE + --per_batch_size=$PER_BATCH + " + return 0 +} + +function get_eval_cmd() +{ + + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_file_valid "${TRAIN_DATA_FILE}" || { logger_Warn "TRAIN_DATA_FILE:${TRAIN_DATA_FILE} not valid file" ; return 1; } + logger_Debug "TRAIN_DATA_FILE is valid" +} + + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh new file mode 100644 index 0000000..abe3be7 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + + mkdir -p ${CURDIR}/output/config + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/config.sh -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/modelarts_config.py -r ${CURDIR}//output/config/ + [ "$1" == "r1.3" ] && { cp ${CURDIR}/config/modelarts_config.py.r1.3 -r ${CURDIR}//output/config/modelarts_config.py; } + [ -d ${CURDIR}/doc ] && cp ${CURDIR}/doc -r ${CURDIR}/output/ + + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh new file mode 100644 index 0000000..333b071 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh @@ -0,0 +1,14 @@ +export PYTHON_COMMAND=python3.7 + +export TRAIN_DATA_PATH=/home/datasets/imagenet/train/ +export EVAL_DATA_PATH=/home/datasets/imagenet/val/ +export EPOCH_SIZE=90 + +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# need if rank_size > 1 +export RANK_TABLE_FILE=/home/lcm/tool/rank_table_8p.json + +# cluster need for node info +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py new file mode 100644 index 0000000..6219139 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py @@ -0,0 +1,134 @@ +from easydict import EasyDict as ed + +# 该部分为认证信息，请向相关运维同事咨询并填写 +access_config = ed({ + # 登录需要的ak sk信息 + 'access_key': '', + 'secret_access_key': '', + # 连接OBS的服务地址。可包含协议类型、域名、端口号。（出于安全性考虑，建议使用https协议） + # 如果是计算中心，需要联系运维同事获取 + 'server': '', + # project_id/region_name: + # 项目ID/区域ID，获取方式参考链接 + # https://support.huaweicloud.com/api-iam/iam_17_0002.html + # 如果是计算中心,请咨询相关维护同事 + 'region_name': '', + 'project_id': '', + + # 如下配置针对计算中心等专有云 通用云不需要设置 设置为空 请咨询相关维护同事 + # 设置该信息后 需要设置相关的域名解析地址 + 'iam_endpoint': '', + 'obs_endpoint': '', + 'modelarts_endpoint' : '', +}) + +session_config = ed({ + # 运行模型的传入超参 + 'hyperparameters': [ + # 模型配置文件，默认boost模式，不需要更改 + {'label': 'config_path', 'value': 'resnet50_imagenet2012_Boost_config.yaml'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'label': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'label': 'run_distribute', 'value': 'True'}, + # epoch次数 必须关注 当前默认设置为90 + {'label': 'epoch_size', 'value': '90'}, + # device数量 云上场景一般不需要修改 + {'label': 'device_num', 'value': '8'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'label': 'save_checkpoint', 'value': 'False'}, + # 保存ckpt的epoch数 必须修改并注意 该值必须要跟epoch数一致 这样提高性能 + {'label': 'save_checkpoint_epochs', 'value': '90'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/imagenet_small/', + # obs代码路径 程序会自动拷贝到该路径 + 'code_dir': '/zgwtest/lcm_test/resnet/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/resnet/train.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'MindSpore-1.3-cann_5.0.2-python3.7-euleros2.8-aarch64', + + # 资源参数类型主要包括如下2个值 train_instance_type和pool_id + # 不设置pool_id 默认是公共池 设置了就是专属资源池 + # 只设置pool_id 不设置train_instance_type 默认为专属资源池的默认类型 + # train_instance_type 在程序打印中有提示的 一般为如下四个值 分别对应 1卡 2卡 4卡 8卡 + # ['modelarts.kat1.xlarge', 'modelarts.kat1.2xlarge', 'modelarts.kat1.4xlarge', 'modelarts.kat1.8xlarge'] + # https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0191.html 该链接指示获取方法 + + # 专属资源池id 不是则为None + 'pool_id': None, + # 训练类型 如下为8卡 如果是专属资源池id设置，那么该类型需要设置为None + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练结点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type' : None, + # 'nas_share_addr' : None, + # 'nas_mount_path' : None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) + +session_config_v2 = ed({ + # 运行模型的传入超参 + 'parameters': [ + # 模型配置文件，默认boost模式，不需要更改 + {'name': 'config_path', 'value': 'resnet50_imagenet2012_Boost_config.yaml'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'name': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'name': 'run_distribute', 'value': 'True'}, + # epoch次数 必须关注 当前默认设置为90 + {'name': 'epoch_size', 'value': '90'}, + # device数量 云上场景一般不需要修改 + {'name': 'device_num', 'value': '8'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'name': 'save_checkpoint', 'value': 'False'}, + # 保存ckpt的epoch数 必须修改并注意 该值必须要跟epoch数一致 这样提高性能 + {'name': 'save_checkpoint_epochs', 'value': '90'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/imagenet_small/', + + # obs代码路径 程序会自动拷贝到该路径. 和boot_files一起用于复合参数 training_files + 'code_dir': '/zgwtest/lcm_test/resnet/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/resnet/train.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'mindspore_1.3.0-cann_5.0.2-py_3.7-euler_2.8.3-aarch64', + + # pool_id不设置或者设置为None， 默认是公共资源池。 设置了就表示是专属资源池。在ModelArts管理控制台，单击左侧“专属资源池”，在专属资源池列表中可以查看专属资源池ID，类似poolc90f063b + 'pool_id': None, + # 训练类型，默认8卡。 train_instance_type 在程序打印中有提示的,请注意紧随“get valid train_instance_types：”之后的打印输出. 由modelarts.estimatorV2 类Estimator的接口get_train_instance_types（）查询而来。 + # 请参见https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0431.html 该链接指示获取方法。注意不同云环境查询的结果不同 + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练结点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type': None, + # 'nas_share_addr': None, + # 'nas_mount_path': None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 new file mode 100644 index 0000000..e7f207c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 @@ -0,0 +1,134 @@ +from easydict import EasyDict as ed + +# 该部分为认证信息，请向相关运维同事咨询并填写 +access_config = ed({ + # 登录需要的ak sk信息 + 'access_key': '', + 'secret_access_key': '', + # 连接OBS的服务地址。可包含协议类型、域名、端口号。（出于安全性考虑，建议使用https协议） + # 如果是计算中心，需要联系运维同事获取 + 'server': '', + # project_id/region_name: + # 项目ID/区域ID，获取方式参考链接 + # https://support.huaweicloud.com/api-iam/iam_17_0002.html + # 如果是计算中心,请咨询相关维护同事 + 'region_name': '', + 'project_id': '', + + # 如下配置针对计算中心等专有云 通用云不需要设置 设置为空 请咨询相关维护同事 + # 设置该信息后 需要设置相关的域名解析地址 + 'iam_endpoint': '', + 'obs_endpoint': '', + 'modelarts_endpoint' : '', +}) + +session_config = ed({ + # 运行模型的传入超参 + 'hyperparameters': [ + # 模型配置文件，默认acc模式，不需要更改 + {'label': 'config_path', 'value': 'resnet50_imagenet2012_Acc_config.yaml'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'label': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'label': 'run_distribute', 'value': 'True'}, + # epoch次数 必须关注 当前默认设置为90 + {'label': 'epoch_size', 'value': '90'}, + # device数量 云上场景一般不需要修改 + {'label': 'device_num', 'value': '8'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'label': 'save_checkpoint', 'value': 'False'}, + # 保存ckpt的epoch数 必须修改并注意 该值必须要跟epoch数一致 这样提高性能 + {'label': 'save_checkpoint_epochs', 'value': '90'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/imagenet_small/', + # obs代码路径 程序会自动拷贝到该路径 + 'code_dir': '/zgwtest/lcm_test/resnet/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/resnet/train.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'MindSpore-1.3-cann_5.0.2-python3.7-euleros2.8-aarch64', + + # 资源参数类型主要包括如下2个值 train_instance_type和pool_id + # 不设置pool_id 默认是公共池 设置了就是专属资源池 + # 只设置pool_id 不设置train_instance_type 默认为专属资源池的默认类型 + # train_instance_type 在程序打印中有提示的 一般为如下四个值 分别对应 1卡 2卡 4卡 8卡 + # ['modelarts.kat1.xlarge', 'modelarts.kat1.2xlarge', 'modelarts.kat1.4xlarge', 'modelarts.kat1.8xlarge'] + # https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0431.html 该链接指示获取方法 + + # 专属资源池id 不是则为None + 'pool_id': None, + # 训练类型 如下为8卡 如果是专属资源池id设置，那么该类型需要设置为None + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练结点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type': None, + # 'nas_share_addr': None, + # 'nas_mount_path': None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) + +session_config_v2 = ed({ + # 运行模型的传入超参 + 'parameters': [ + # 模型配置文件，默认boost模式，不需要更改 + {'name': 'config_path', 'value': 'resnet50_imagenet2012_Acc_config.yaml'}, + # 是否使能modelarts 必须设置为True，不需要修改 + {'name': 'enable_modelarts', 'value': 'True'}, + # 是否开启分布式，如果1卡以上的话都是True 一般不需要修改 + {'name': 'run_distribute', 'value': 'True'}, + # epoch次数 必须关注 当前默认设置为90 + {'name': 'epoch_size', 'value': '90'}, + # device数量 云上场景一般不需要修改 + {'name': 'device_num', 'value': '8'}, + # 是否保存ckpt文件 默认为True 保存ckpt + {'name': 'save_checkpoint', 'value': 'False'}, + # 保存ckpt的epoch数 必须修改并注意 该值必须要跟epoch数一致 这样提高性能 + {'name': 'save_checkpoint_epochs', 'value': '90'}, + ], + # 输入数据集obs目录,请按样例格式填写 + 'inputs': '/zgwtest/lcm_test/dataset/imagenet_small/', + + # obs代码路径 程序会自动拷贝到该路径. 和boot_files一起用于复合参数 training_files + 'code_dir': '/zgwtest/lcm_test/resnet/', + # 启动文件 必须要在code_dir路径下，请按样例格式填写 + 'boot_file': '/zgwtest/lcm_test/resnet/train.py', + + # 如下为运行相关参数 + # job名称 如果云环境Modelarts服务训练作业job队列中没有，则会新建一个job；若和已有job同名，则会在该job中，新建测试实例. + 'job_name': "aisbench-debug", + + # 使用容器类型与镜像版本 + 'framework_type': 'Ascend-Powered-Engine', + 'framework_version': 'mindspore_1.3.0-cann_5.0.2-py_3.7-euler_2.8.3-aarch64', + + # pool_id不设置或者设置为None， 默认是公共资源池。 设置了就表示是专属资源池。在ModelArts管理控制台，单击左侧“专属资源池”，在专属资源池列表中可以查看专属资源池ID，类似poolc90f063b + 'pool_id': None, + # 训练类型，默认8卡。 train_instance_type 在程序打印中有提示的,请注意紧随“get valid train_instance_types：”之后的打印输出. 由modelarts.estimatorV2 类Estimator的接口get_train_instance_types（）查询而来。 + # 请参见https://support.huaweicloud.com/sdkreference-modelarts/modelarts_04_0431.html 该链接指示获取方法。注意不同云环境查询的结果不同 + 'train_instance_type': 'modelarts.kat1.8xlarge', + # 训练节点数 + 'train_instance_count': 1, + + # 云存储路径 默认为空 + # 'nas_type': None, + # 'nas_share_addr': None, + # 'nas_mount_path': None, + + # 输出信息基准路径 整体路径为 train_url = out_base_url/version_name + "out_base_url": "/zgwtest/lcm_test/result/", + # job 描述前缀 + "job_description_prefix": 'lcm-debug desc', +}) diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch new file mode 100644 index 0000000..7d6905c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch @@ -0,0 +1,162 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2023-04-10 14:53:38.020000000 +0800 ++++ code/src/model_utils/config.py 2023-04-10 14:53:38.040000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-10 14:53:38.020000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-10 14:53:38.040000000 +0800 +@@ -98,6 +98,12 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ + if pre_process: + pre_process() + +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-04-10 14:53:38.030000000 +0800 ++++ code/train.py 2023-04-10 14:53:38.050000000 +0800 +@@ -17,7 +17,7 @@ + import glob + import os + import numpy as np +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -285,6 +285,17 @@ + metrics_name="acc") + cb += [eval_cb] + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + def set_save_ckpt_dir(): + """set save ckpt dir""" +@@ -348,6 +359,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -366,12 +378,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch new file mode 100644 index 0000000..3a036dc --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch @@ -0,0 +1,230 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2022-07-11 15:10:57.060000000 +0800 ++++ code/src/model_utils/config.py 2022-07-11 15:10:57.070000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '..', '..', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + pprint(default) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-07-11 15:10:57.060000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-07-11 15:10:57.070000000 +0800 +@@ -98,6 +98,12 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ + if pre_process: + pre_process() + +@@ -111,5 +117,6 @@ + if config.train_url: + print("Start to copy output directory") + sync_data(config.output_path, config.train_url) ++ + return wrapped_func + return wrapper +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-07-11 15:10:57.060000000 +0800 ++++ code/train.py 2022-07-11 15:10:57.070000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + from mindspore import context + from mindspore import Tensor, Parameter +@@ -47,6 +48,61 @@ + + set_seed(1) + ++import json ++import time ++from mindspore.train.callback._callback import Callback ++ ++skip_sumary_count = 1 ++skip_data_sum = 0 ++skip_time_sum = 0 ++real_start_time = 0.0 ++all_data_sum = 0 ++ ++class ThroughputRate(Callback): ++ """ ++ Monitor the time in training. ++ ++ Args: ++ data_size (int): Dataset size. Default: None. ++ """ ++ ++ def __init__(self, data_size=None): ++ super(ThroughputRate, self).__init__() ++ self.data_size = data_size ++ self.count = 0 ++ ++ def epoch_begin(self, run_context): ++ self.epoch_time = time.time() ++ global real_start_time ++ if self.count == 1: ++ real_start_time = self.epoch_time ++ self.count += 1 ++ ++ def epoch_end(self, run_context): ++ epoch_seconds = (time.time() - self.epoch_time) * 1000 ++ step_size = self.data_size ++ cb_params = run_context.original_args() ++ if hasattr(cb_params, "batch_num"): ++ batch_num = cb_params.batch_num ++ if isinstance(batch_num, int) and batch_num > 0: ++ step_size = cb_params.batch_num ++ ++ if not isinstance(step_size, int) or step_size < 1: ++ logger.error("data_size must be positive int.") ++ return ++ ++ global skip_sumary_count ++ global skip_data_sum ++ global skip_time_sum ++ global all_data_sum ++ if skip_sumary_count > 0: ++ skip_sumary_count = skip_sumary_count -1 ++ skip_data_sum += self.data_size * config.batch_size ++ skip_time_sum += (time.time() - self.epoch_time) ++ all_data_sum += self.data_size * config.batch_size ++ print("data_size:{} batch_size:{} datasum:{} skipcount:{} skip_data_sum:{} skip_time_sum:{}".format( ++ self.data_size, config.batch_size, all_data_sum, skip_sumary_count, skip_data_sum, skip_time_sum)) ++ + + class LossCallBack(LossMonitor): + """ +@@ -311,6 +367,17 @@ + cb += [eval_cb] + + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res ++ ++ + def set_save_ckpt_dir(): + """set save ckpt dir""" + ckpt_save_dir = os.path.join(config.output_path, config.checkpoint_path) +@@ -373,7 +440,7 @@ + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +- cb = [time_cb, loss_cb] ++ cb = [time_cb, loss_cb, ThroughputRate(data_size=step_size)] + ckpt_save_dir = set_save_ckpt_dir() + if config.save_checkpoint: + ckpt_append_info = [{"epoch_num": config.has_trained_epoch, "step_num": config.has_trained_step}] +@@ -388,12 +455,94 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ throughput_rate = (all_data_sum - skip_data_sum ) / (end_time - real_start_time) ++ print("train done starttime:{} real:{} endtime:{} alldatasum:{} skipcount:{} skiptime:{}".format( ++ start_time, real_start_time, end_time, all_data_sum, skip_data_sum, skip_time_sum)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("BATCH_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("BATCH_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch new file mode 100644 index 0000000..f9aa3cc --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch @@ -0,0 +1,162 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2022-07-11 17:36:33.370000000 +0800 ++++ code/src/model_utils/config.py 2022-07-11 17:36:33.380000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-07-11 17:36:33.370000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-07-11 17:36:33.380000000 +0800 +@@ -97,6 +97,12 @@ + config.device_id = get_device_id() + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) ++ ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) + + if pre_process: + pre_process() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-07-11 17:36:33.370000000 +0800 ++++ code/train.py 2022-07-11 17:36:33.390000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + from mindspore import context + from mindspore import Tensor +@@ -296,6 +297,17 @@ + metrics_name="acc") + cb += [eval_cb] + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + def set_save_ckpt_dir(): + """set save ckpt dir""" +@@ -358,6 +370,8 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) ++ + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -376,12 +390,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("BATCH_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("BATCH_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch new file mode 100644 index 0000000..5938210 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch @@ -0,0 +1,162 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2022-05-22 20:29:23.680000000 +0800 ++++ code/src/model_utils/config.py 2022-05-22 20:29:23.710000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-05-22 20:29:23.680000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-05-22 20:29:23.710000000 +0800 +@@ -97,6 +97,12 @@ + config.device_id = get_device_id() + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) ++ ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) + + if pre_process: + pre_process() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-05-22 20:29:23.710000000 +0800 ++++ code/train.py 2022-05-22 20:29:23.740000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + import mindspore as ms + from mindspore import Tensor +@@ -294,6 +295,17 @@ + metrics_name="acc") + cb += [eval_cb] + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + def set_save_ckpt_dir(): + """set save ckpt dir""" +@@ -356,6 +368,8 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) ++ + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -374,12 +388,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch new file mode 100644 index 0000000..73ab6b9 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch @@ -0,0 +1,166 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2023-04-10 13:02:19.630000000 +0800 ++++ code/src/model_utils/config.py 2023-04-10 13:02:19.650000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-10 13:02:19.630000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-10 13:02:19.650000000 +0800 +@@ -98,6 +98,13 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ + if pre_process: + pre_process() + +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-04-10 13:02:19.640000000 +0800 ++++ code/train.py 2023-04-10 13:02:19.660000000 +0800 +@@ -17,7 +17,7 @@ + import glob + import os + import numpy as np +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -288,6 +288,17 @@ + metrics_name="acc") + cb += [eval_cb] + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + def set_save_ckpt_dir(): + """set save ckpt dir""" +@@ -351,6 +362,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -369,12 +381,98 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ ++ + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch new file mode 100644 index 0000000..3c7a8e0 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch @@ -0,0 +1,163 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2022-10-17 21:17:23.696000000 +0800 ++++ code/src/model_utils/config.py 2022-10-17 21:17:23.716000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2022-10-17 21:17:23.696000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2022-10-17 21:17:23.716000000 +0800 +@@ -98,6 +98,13 @@ + if not os.path.exists(config.output_path): + os.makedirs(config.output_path) + ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ ++ + if pre_process: + pre_process() + +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-10-17 21:17:23.684000000 +0800 ++++ code/train.py 2022-10-17 21:17:23.700000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + import mindspore as ms + import mindspore.nn as nn +@@ -286,6 +287,18 @@ + cb += [eval_cb] + + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res ++ + def set_save_ckpt_dir(): + """set save ckpt dir""" + ckpt_save_dir = os.path.join(config.output_path, config.checkpoint_path) +@@ -348,6 +361,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -366,12 +380,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) + ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os + ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch new file mode 100644 index 0000000..4956e0f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch @@ -0,0 +1,164 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2023-04-07 20:52:22.470000000 +0800 ++++ code/src/model_utils/config.py 2023-04-07 20:52:22.490000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-07 20:52:22.470000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-07 20:52:22.490000000 +0800 +@@ -101,6 +101,13 @@ + if pre_process: + pre_process() + ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ ++ + run_func(*args, **kwargs) + + # Upload data to train_url +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-04-07 20:52:22.480000000 +0800 ++++ code/train.py 2023-04-07 20:52:22.500000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -132,6 +132,17 @@ + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + return loss + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + @moxing_wrapper() + def train_net(): +@@ -197,6 +208,8 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -223,11 +236,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch new file mode 100644 index 0000000..4956e0f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch @@ -0,0 +1,164 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2023-04-07 20:52:22.470000000 +0800 ++++ code/src/model_utils/config.py 2023-04-07 20:52:22.490000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-07 20:52:22.470000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-07 20:52:22.490000000 +0800 +@@ -101,6 +101,13 @@ + if pre_process: + pre_process() + ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ ++ + run_func(*args, **kwargs) + + # Upload data to train_url +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-04-07 20:52:22.480000000 +0800 ++++ code/train.py 2023-04-07 20:52:22.500000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -132,6 +132,17 @@ + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + return loss + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + @moxing_wrapper() + def train_net(): +@@ -197,6 +208,8 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -223,11 +236,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch new file mode 100644 index 0000000..4956e0f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch @@ -0,0 +1,164 @@ +diff -Nur origin/src/model_utils/config.py code/src/model_utils/config.py +--- origin/src/model_utils/config.py 2023-04-07 20:52:22.470000000 +0800 ++++ code/src/model_utils/config.py 2023-04-07 20:52:22.490000000 +0800 +@@ -120,6 +120,8 @@ + parser.add_argument("--config_path", type=str, default=os.path.join(current_dir, \ + "../../config/resnet50_cifar10_config.yaml"), help="Config file path") + path_args, _ = parser.parse_known_args() ++ if not os.path.exists(path_args.config_path): ++ path_args.config_path = os.path.join(current_dir, '../../config', path_args.config_path) + default, helper, choices = parse_yaml(path_args.config_path) + args = parse_cli_to_yaml(parser=parser, cfg=default, helper=helper, choices=choices, cfg_path=path_args.config_path) + final_config = merge(args, default) +diff -Nur origin/src/model_utils/moxing_adapter.py code/src/model_utils/moxing_adapter.py +--- origin/src/model_utils/moxing_adapter.py 2023-04-07 20:52:22.470000000 +0800 ++++ code/src/model_utils/moxing_adapter.py 2023-04-07 20:52:22.490000000 +0800 +@@ -101,6 +101,13 @@ + if pre_process: + pre_process() + ++ base_path = config.data_path ++ if os.path.exists(os.path.join(base_path, "train")): ++ config.data_path = os.path.join(base_path, "train") ++ if os.path.exists(os.path.join(base_path, "val")): ++ config.eval_dataset_path = os.path.join(os.path.join(base_path, "val")) ++ ++ + run_func(*args, **kwargs) + + # Upload data to train_url +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-04-07 20:52:22.480000000 +0800 ++++ code/train.py 2023-04-07 20:52:22.500000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -132,6 +132,17 @@ + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + return loss + ++def run_eval_ckpt(target, model): ++ eval_dataset = create_dataset(dataset_path=config.eval_dataset_path, do_train=False, ++ batch_size=config.batch_size, train_image_size=config.train_image_size, ++ eval_image_size=config.eval_image_size, ++ target=target, enable_cache=config.enable_cache, ++ cache_session_id=config.cache_session_id) ++ eval_param_dict = {"model": model, "dataset": eval_dataset, "metrics_name": "acc"} ++ print("eval ckpt begin") ++ res = apply_eval(eval_param_dict) ++ print("eval ckpt result:{}".format(res)) ++ return res + + @moxing_wrapper() + def train_net(): +@@ -197,6 +208,8 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -223,11 +236,95 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ end_time = time.time() ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ throughput_rate = all_data_sum / (int)(end_time - start_time) ++ print("train done starttime:{} endtime:{} stepsize:{} batchsize:{} epochsize:{} alldatasum:{} single_throughput_rate:{}".format( ++ start_time, end_time, step_size, config.batch_size, config.epoch_size, all_data_sum, throughput_rate)) ++ import moxing as mox ++ result_url = config.train_url ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ result_url = os.path.join(config.train_url, os.getenv("VC_TASK_INDEX", "0")) ++ mox.file.make_dirs(result_url) ++ throughput_file = os.path.join(result_url, "throughput_" + str(get_rank_id()) + ".json") ++ mox.file.write(throughput_file, str(throughput_rate)) ++ ++ best_res = run_eval_ckpt(target, model) ++ if get_rank_id() == 0: ++ import moxing as mox ++ server_id = os.getenv("VC_TASK_INDEX", 0) ++ accuracy_file = os.path.join(config.train_url, "accuracy_{}.json".format(server_id)) ++ mox.file.write(accuracy_file, str(best_res)) ++ accuracy_file1 = os.path.join(config.train_url, "accuracy.json") ++ with mox.file.File(accuracy_file1, 'w') as f: ++ f.write(str(best_res)) ++ ranksize_file = os.path.join(config.train_url, "ranksize.json") ++ if mox.file.exists(ranksize_file) == False: ++ mox.file.write(ranksize_file, str(get_device_num())) ++ + + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + ++import json ++import time ++import os ++ ++class FileOperator: ++ @staticmethod ++ def create_empty_file(file_path): ++ f = open(file_path, "w") ++ f.close() ++ ++ @staticmethod ++ def read_json_to_dict(file_path): ++ f = open(file_path, 'r') ++ dic = json.load(f) ++ f.close() ++ return dic ++ ++ @staticmethod ++ def write_json_from_dict(file_path, source_dict): ++ json_str = json.dumps(source_dict) ++ with open(file_path, 'w') as json_file: ++ json_file.write(json_str) ++ ++def generate_single_node_rank_table(rank_table, old_rank_id): ++ import copy ++ server_list = copy.deepcopy(rank_table['server_list']) ++ server_index = old_rank_id // 8 ++ cur_server = server_list[server_index] ++ for device in cur_server['device']: ++ device['rank_id'] = str(int(device['rank_id']) % 8) ++ rank_table['server_list'] = [cur_server] ++ rank_table['server_count'] = "1" ++ FileOperator.write_json_from_dict(os.getenv("RANK_TABLE_FILE"), rank_table) ++ ++def set_singleserver_mode(): ++ old_rank_id = int(os.environ['RANK_ID']) ++ new_rank_id = old_rank_id % 8 ++ os.environ['RANK_ID'] = str(new_rank_id) ++ os.environ['DEVICE_ID'] = str(new_rank_id) ++ os.environ["RANK_SIZE"] = str(8) ++ #os.environ["SERVER_ID"] = str(old_rank_id // 8) ++ ++ empty_file_path = "/tmp/tmp.txt" ++ if new_rank_id != 0: ++ while not os.path.exists(empty_file_path): ++ time.sleep(1) ++ else: ++ rank_table = FileOperator.read_json_to_dict(os.getenv("RANK_TABLE_FILE")) ++ generate_single_node_rank_table(rank_table, old_rank_id) ++ FileOperator.create_empty_file(empty_file_path) ++ return new_rank_id ++ + + if __name__ == '__main__': ++ if os.getenv("SINGLESERVER_MODE", "") == "True": ++ set_singleserver_mode() ++ print("singleserver_mode set") ++ else: ++ print("singleserver_mode not set") + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh new file mode 100644 index 0000000..c79dc74 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r2.3"; } + + modelzoo_sub_dir="mindspore/model_zoo/official/cv/resnet" + if [ "$branch_args" == "r1.1" ];then + branch="r1.1" + patch_file_name="r1.1" + commitid="9c133b6f709e12ed7085c31f028e7c925ee57828" + git_url="https://gitee.com/mindspore/mindspore.git" + elif [ "$branch_args" == "r1.2" ];then + branch="r1.2" + patch_file_name="r1.2" + commitid="cd002779dc5e2bc2da85b9a33e8950aa3bb50ed2" + git_url="https://gitee.com/mindspore/mindspore.git" + elif [ "$branch_args" == "r1.3" ];then + branch="r1.3" + patch_file_name="r1.3" + commitid="d9d4960262617d964d669ef8e3287daf347d5a7c" + git_url="https://gitee.com/mindspore/mindspore.git" + elif [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="f6537762d0aea541adfed8644da452a476c28321" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/resnet" + elif [ "$branch_args" == "r1.6" ];then + branch="r1.6" + patch_file_name="r1.6" + commitid="6496c699bd404076b12a6edcc40889dafaeb5285" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/resnet" + elif [ "$branch_args" == "r1.7" ];then + branch="master" + patch_file_name="r1.7" + commitid="3406fdabaee92f1b22ce0703fa25befa3c40d18e" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/resnet" + elif [ "$branch_args" == "r1.8" ];then + branch="master" + patch_file_name="r1.8" + commitid="b68b6bfa919465567d89bc7fdcf6d0e63967d5aa" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/resnet" + elif [ "$branch_args" == "r1.9" ];then + branch="r1.9" + patch_file_name="r1.9" + commitid="5318681496ef9a37d337737325ad1b238ef75917" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/resnet" + elif [ "$branch_args" == "r1.10" ];then + branch="r1.10" + patch_file_name="r1.10" + commitid="8f7331e6a846e7c306dc8ac30313d9f07cf6ee98" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/resnet" + elif [ "$branch_args" == "r2.0" ];then + branch="r2.0" + patch_file_name="r2.0" + commitid="f211f336e8bee3cf531bcad5f611f408069c6f9f" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/ResNet" + elif [ "$branch_args" == "r2.1" ];then + branch="r2.1" + patch_file_name="r2.1" + commitid="44f2dc18e9bd52c6bcadd18f6567817ad798f641" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/ResNet" + elif [ "$branch_args" == "r2.2" ];then + branch="master" + patch_file_name="r2.2" + commitid="bb9ab4fdfb2fc205ffeb4dd671be77312908ef88" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/ResNet" + elif [ "$branch_args" == "r2.3" ];then + branch="master" + patch_file_name="r2.3" + commitid="c63fb183e748427c2d59d96e5a79f9543f56844d" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/ResNet" + else + echo "bad parameters : $1" + return $ret_error + fi + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch new file mode 100644 index 0000000..731f70a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch @@ -0,0 +1,85 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-12-16 14:40:00.810000000 +0800 ++++ code/eval.py 2022-12-16 14:40:00.850000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-12-16 14:40:00.830000000 +0800 ++++ code/train.py 2022-12-16 14:40:00.870000000 +0800 +@@ -17,7 +17,7 @@ + import glob + import os + import numpy as np +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -35,7 +35,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_rank_id, get_device_num + from src.resnet import conv_variance_scaling_initializer +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + ms.set_seed(1) + + +@@ -347,7 +352,7 @@ + amp_level="O3") + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") +- ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -366,8 +371,30 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch new file mode 100644 index 0000000..9b88725 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch @@ -0,0 +1,87 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2021-12-10 14:54:43.810000000 +0800 ++++ code/eval.py 2021-12-10 14:54:43.820000000 +0800 +@@ -84,6 +84,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2021-12-10 14:54:43.820000000 +0800 ++++ code/train.py 2021-12-10 14:54:43.820000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + from mindspore import context + from mindspore import Tensor +@@ -43,7 +44,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_rank_id, get_device_num + from src.resnet import conv_variance_scaling_initializer +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + set_seed(1) + +@@ -358,6 +364,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -376,9 +383,34 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() ++ + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) + ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) ++ + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch new file mode 100644 index 0000000..10bcb6e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch @@ -0,0 +1,88 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2021-12-10 11:02:30.430000000 +0800 ++++ code/eval.py 2021-12-10 11:02:30.440000000 +0800 +@@ -84,6 +84,16 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) ++ + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2021-12-10 11:02:30.430000000 +0800 ++++ code/train.py 2021-12-10 11:02:30.440000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + from mindspore import context + from mindspore import Tensor +@@ -43,7 +44,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_rank_id, get_device_num + from src.resnet import conv_variance_scaling_initializer +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + set_seed(1) + +@@ -358,6 +364,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -376,9 +383,34 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() ++ + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) + ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) ++ + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch new file mode 100644 index 0000000..408f74f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch @@ -0,0 +1,83 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-05-30 13:48:49.358918532 +0800 ++++ code/eval.py 2022-05-30 13:48:49.374918725 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-05-30 13:48:49.362918580 +0800 ++++ code/train.py 2022-05-30 13:48:49.374918725 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + import mindspore as ms + import mindspore.nn as nn +@@ -35,6 +36,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_rank_id, get_device_num + from src.resnet import conv_variance_scaling_initializer ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -351,6 +358,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -369,9 +377,31 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) + ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch new file mode 100644 index 0000000..b4917a8 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch @@ -0,0 +1,83 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-07-07 09:59:36.316000000 +0800 ++++ code/eval.py 2022-07-07 09:59:36.336000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-07-07 09:59:36.316000000 +0800 ++++ code/train.py 2022-07-07 09:59:36.336000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + import mindspore as ms + import mindspore.nn as nn +@@ -35,6 +36,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_rank_id, get_device_num + from src.resnet import conv_variance_scaling_initializer ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -350,7 +357,7 @@ + amp_level="O2", keep_batchnorm_fp32=False) + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") +- ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -369,8 +376,30 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch new file mode 100644 index 0000000..1710c00 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch @@ -0,0 +1,82 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-10-17 20:39:19.284000000 +0800 ++++ code/eval.py 2022-10-17 20:39:19.292000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-10-17 20:39:19.284000000 +0800 ++++ code/train.py 2022-10-17 20:39:19.292000000 +0800 +@@ -17,6 +17,7 @@ + import glob + import os + import numpy as np ++import time + + import mindspore as ms + import mindspore.nn as nn +@@ -35,6 +36,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_rank_id, get_device_num + from src.resnet import conv_variance_scaling_initializer ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -348,6 +355,7 @@ + config.run_eval = False + logger.warning("Thor optimizer not support evaluation while training.") + ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size - config.pretrain_epoch_size) + # define callbacks + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossCallBack(config.has_trained_epoch) +@@ -366,8 +374,30 @@ + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.pretrain_epoch_size = config.has_trained_epoch ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.pretrain_epoch_size, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if config.run_eval and config.enable_cache: + print("Remember to shut down the cache server via \"cache_admin --stop\"") diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch new file mode 100644 index 0000000..9fd65ab --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch @@ -0,0 +1,91 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2023-03-27 15:53:34.360000000 +0800 ++++ code/eval.py 2023-03-27 15:53:34.388000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-03-27 15:53:34.364000000 +0800 ++++ code/train.py 2023-03-27 15:53:34.388000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -31,6 +31,12 @@ + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -197,6 +203,7 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -217,15 +224,36 @@ + cache_session_id=config.cache_session_id) + eval_cb = eval_callback(model, config, eval_dataset) + cb.append(eval_cb) +- ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) + # train model + if config.net_name == "se-resnet50": + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) +- ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch new file mode 100644 index 0000000..9fd65ab --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch @@ -0,0 +1,91 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2023-03-27 15:53:34.360000000 +0800 ++++ code/eval.py 2023-03-27 15:53:34.388000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-03-27 15:53:34.364000000 +0800 ++++ code/train.py 2023-03-27 15:53:34.388000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -31,6 +31,12 @@ + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -197,6 +203,7 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -217,15 +224,36 @@ + cache_session_id=config.cache_session_id) + eval_cb = eval_callback(model, config, eval_dataset) + cb.append(eval_cb) +- ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) + # train model + if config.net_name == "se-resnet50": + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) +- ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch new file mode 100644 index 0000000..9fd65ab --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch @@ -0,0 +1,91 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2023-03-27 15:53:34.360000000 +0800 ++++ code/eval.py 2023-03-27 15:53:34.388000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-03-27 15:53:34.364000000 +0800 ++++ code/train.py 2023-03-27 15:53:34.388000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -31,6 +31,12 @@ + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -197,6 +203,7 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -217,15 +224,36 @@ + cache_session_id=config.cache_session_id) + eval_cb = eval_callback(model, config, eval_dataset) + cb.append(eval_cb) +- ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) + # train model + if config.net_name == "se-resnet50": + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) +- ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch new file mode 100644 index 0000000..9fd65ab --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch @@ -0,0 +1,91 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2023-03-27 15:53:34.360000000 +0800 ++++ code/eval.py 2023-03-27 15:53:34.388000000 +0800 +@@ -82,6 +82,15 @@ + # eval model + res = model.eval(dataset) + print("result:", res, "ckpt=", config.checkpoint_file_path) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res['top_1_accuracy'])) + + if __name__ == '__main__': + eval_net() +diff -Nur origin/train.py code/train.py +--- origin/train.py 2023-03-27 15:53:34.364000000 +0800 ++++ code/train.py 2023-03-27 15:53:34.388000000 +0800 +@@ -14,7 +14,7 @@ + # ============================================================================ + """train resnet.""" + import os +- ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore.train.train_thor import ConvertModelUtils +@@ -31,6 +31,12 @@ + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper + from src.model_utils.device_adapter import get_device_num ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + ms.set_seed(1) + +@@ -197,6 +203,7 @@ + ms.load_param_into_net(opt, resume_param) + config.logger.info('resume train from epoch: %s', config.start_epoch) + ++ + # define callbacks + loss_cb = LossCallBack(config.epoch_size, config.logger, lr, per_print_time=10) + resume_cb = ResumeCallback(config.start_epoch) +@@ -217,15 +224,36 @@ + cache_session_id=config.cache_session_id) + eval_cb = eval_callback(model, config, eval_dataset) + cb.append(eval_cb) +- ++ model.build(dataset, None, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) + # train model + if config.net_name == "se-resnet50": + config.epoch_size = config.train_epoch_size + dataset_sink_mode = (not config.parameter_server) and target != "CPU" + config.logger.save_args(config) ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size - config.start_epoch, dataset, callbacks=cb, + sink_size=dataset.get_dataset_size(), dataset_sink_mode=dataset_sink_mode) +- ++ all_data_sum = step_size * config.batch_size * config.epoch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + config.logger.info("If run eval and enable_cache Remember to shut down the cache server via \"cache_admin --stop\"") + + diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..1bb080d --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh @@ -0,0 +1,81 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + : "${EVAL_DATA_PATH?EVAL_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + + [ -d $BASE_PATH/result ] && cp ${RESULT_PATH}/* -rf $BASE_PATH/result/ + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh new file mode 100644 index 0000000..2c0ca52 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh + +init() +{ + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + logger_Info "init called" +} + +run_train() +{ + logger_Info "run_train called" + [ ! -f $CODE_PATH/code/ma-pre-start.sh ] && touch $CODE_PATH/code/ma-pre-start.sh + sed -i '/SINGLESERVER_MODE=/d' $CODE_PATH/code/ma-pre-start.sh + + [[ $MODELARTS_VERSION ]]&&[[ $MODELARTS_VERSION == "V2" ]] && modelarts_version="V2" || modelarts_version="V1" + + + if [ "$SINGLESERVER_MODE" == "True" ];then + echo "now set singleserver_mode OK" + echo -e "\nexport SINGLESERVER_MODE=True" >> $CODE_PATH/code/ma-pre-start.sh + + ${PYTHON_COMMAND} -u ${CODE_PATH}/common/train_modelarts.py --local_code_path $CODE_PATH/code --single_server_mode --modelarts_version $modelarts_version || { logger_Warn "run train modelarts failed ret:$?";return 1; } + else + echo "now not set singleserver_mode" + ${PYTHON_COMMAND} -u ${CODE_PATH}/common/train_modelarts.py --local_code_path $CODE_PATH/code --modelarts_version $modelarts_version || { logger_Warn "run train modelarts failed ret:$?";return 1; } + fi + ${PYTHON_COMMAND} $CODE_PATH/ais_utils.py "training" "result" "OK" +} + +run_eval() +{ + logger_Info "run_eval called" +} + +get_result() +{ + logger_Info "get_result called" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh new file mode 100644 index 0000000..9cd1b0f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh @@ -0,0 +1,110 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# ��ȡѵ������ +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="True" || DISTRUTE_ENABLE="False" + # 基准代码r2.0.0版本中训练配置文件resnet50_imagenet2012_Boost_config.yaml中，将训练参数output_path改为output_dir + CONFIG_PATH=$WORK_PATH/code/config/resnet50_imagenet2012_Boost_config.yaml + isexisted=`cat $CONFIG_PATH |grep "output_dir" |grep -v grep |awk -F= 'NR==1{print $NF}'` + if [ ! -n "$isexisted" ]; then + OUTPUT_PARA_NAME="output_path" + else + OUTPUT_PARA_NAME="output_dir" + fi + + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py \ + --run_distribute=$DISTRUTE_ENABLE \ + --data_path=${TRAIN_DATA_PATH} \ + --device_num=${DEVICE_NUM} \ + --epoch_size=${EPOCH_SIZE} \ + --$OUTPUT_PARA_NAME="$RUN_PATH" \ + --save_checkpoint=True \ + --save_checkpoint_epochs=${EPOCH_SIZE} \ + --config_path=$CONFIG_PATH + " + # # for mindspore1.5 + # export ENV_FUSION_CLEAR=1 + # export ENV_SINGLE_EVAL=1 + # export SKT_ENABLE=1 + chipname=`npu-smi info -t board -i 0 -c 0 | grep 'Chip Name' | awk {'print $4'}` + export MS_DISABLE_REF_MODE=1 + export MS_ENABLE_FORMAT_MODE=0 + export MS_ASCEND_CHECK_OVERFLOW_MODE="SATURATION_MODE" +} + +function get_eval_cmd() +{ + chipname=`npu-smi info -t board -i 0 -c 0 | grep 'Chip Name' | awk {'print $4'}` + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --data_path=${EVAL_DATA_PATH} \ + --config_path=$WORK_PATH/code/config/resnet50_imagenet2012_Boost_config.yaml \ + --checkpoint_file_path=${CHECKPOINT_PATH}" + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + + check_path_valid "${EVAL_DATA_PATH}" || { logger_Warn "EVAL_DATA_PATH:${EVAL_DATA_PATH} not valid path" ; return 1; } + logger_Debug "EVAL_DATA_PATH is valid" +} + +function node_train() +{ + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + cp $CHECKPOINT_PATH $RESULT_PATH/ + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh new file mode 100644 index 0000000..abe3be7 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + + mkdir -p ${CURDIR}/output/config + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/config.sh -r ${CURDIR}//output/config/ + cp ${CURDIR}/config/modelarts_config.py -r ${CURDIR}//output/config/ + [ "$1" == "r1.3" ] && { cp ${CURDIR}/config/modelarts_config.py.r1.3 -r ${CURDIR}//output/config/modelarts_config.py; } + [ -d ${CURDIR}/doc ] && cp ${CURDIR}/doc -r ${CURDIR}/output/ + + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh new file mode 100644 index 0000000..6520d4f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh @@ -0,0 +1,20 @@ +export PYTHON_COMMAND=python3.7 +export TRAIN_DATA_PATH=/home/datasets/coco/ +export EVAL_DATA_PATH=/home/datasets/coco/ +export CONFIG_PATH=./config/ssd_vgg16_config.yaml + +#export PRETRAIN_MODEL_PATH=/home/models/ms_bert_large.ckpt + +export LR=0.05 +export EPOCH_SIZE=500 +export TRAIN_STEPS=12000 + +# 8p +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# options needed only if rank_size > 1 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json + +# needed only in cluster mode +# export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh new file mode 100644 index 0000000..76749c1 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.9" ];then + branch="master" + patch_file_name="r1.9" + commitid="f4eed0f958b40992ee96dd6cfefd76ae989c872f" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/cv/ssd" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch new file mode 100644 index 0000000..a9d32e6 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch @@ -0,0 +1,76 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-09-26 09:51:49.660000000 +0800 ++++ code/eval.py 2022-09-26 09:51:49.668000000 +0800 +@@ -58,6 +58,16 @@ + mAP = apply_eval(eval_param_dict) + print("\n========================================\n") + print(f"mAP: {mAP}") ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ print("ACC_DIR:", ACC_DIR) ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(mAP)) + + @moxing_wrapper() + def eval_net(): +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-09-26 09:51:49.664000000 +0800 ++++ code/train.py 2022-09-26 09:51:49.668000000 +0800 +@@ -16,6 +16,7 @@ + """Train SSD and get checkpoint files.""" + + import os ++import time + import mindspore as ms + import mindspore.nn as nn + from mindspore import Tensor +@@ -36,6 +37,12 @@ + from src.model_utils.moxing_adapter import moxing_wrapper + + set_seed(1) ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def ssd_model_build(): + if config.model_name == "ssd300": +@@ -198,7 +205,31 @@ + print("In sink mode, one epoch return a loss.") + dataset_sink_mode = True + print("Start train SSD, the first epoch will be slower because of the graph compilation.") ++ model.build(dataset, sink_size=dataset.get_dataset_size(), epoch=config.epoch_size) ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(config.epoch_size, dataset, callbacks=callback, dataset_sink_mode=dataset_sink_mode) ++ all_data_sum = config.epoch_size * dataset.get_dataset_size() * config.batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum / (end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + if __name__ == '__main__': + train_net() diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..d72557e --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + : "${EVAL_DATA_PATH?EVAL_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + + [ -d $BASE_PATH/result ] && cp ${RESULT_PATH}/* -rf $BASE_PATH/result/ + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh new file mode 100644 index 0000000..d4f3814 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh @@ -0,0 +1,107 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="true" || DISTRUTE_ENABLE="false" + + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/run_pretrain.py \ + --distribute=$DISTRUTE_ENABLE \ + --epoch_size=$EPOCH_SIZE \ + --enable_save_ckpt=true \ + --enable_lossscale=true \ + --do_shuffle=true \ + --enable_data_sink=true \ + --data_sink_steps=100 \ + --accumulation_steps=1 \ + --save_checkpoint_path=$RUN_PATH \ + --save_checkpoint_steps=$TRAIN_STEPS \ + --save_checkpoint_num=1 \ + --load_checkpoint_path=$PRETRAIN_MODEL_PATH \ + --data_dir=${TRAIN_DATA_PATH} \ + --device_id=${DEVICE_ID} \ + --device_num=${DEVICE_NUM} \ + --train_steps=${TRAIN_STEPS} \ + --config_path=$WORK_PATH/code/pretrain_config_Ascend_Boost.yaml + " + return 0 +} + +function get_eval_cmd() +{ + CONFIG_FILE=$WORK_PATH/code/pretrain_config_Ascend_Boost.yaml + sed -i "s|eval_data_dir:.*|eval_data_dir: '$EVAL_DATA_PATH'|g" "$CONFIG_FILE" + sed -i "s|schema_file:.*|schema_file: null|g" "$CONFIG_FILE" + sed -i "s|eval_ckpt:.*|eval_ckpt: '$CHECKPOINT_PATH'|g" "$CONFIG_FILE" + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/pretrain_eval.py --config_path=$CONFIG_FILE" + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_file_valid ${PRETRAIN_MODEL_PATH} || { logger_Warn "PRETRAIN_MODEL_PATH:${PRETRAIN_MODEL_PATH} not valid" ; return 1; } + logger_Debug "PRETRAIN_MODEL_PATH path valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + + check_path_valid "${EVAL_DATA_PATH}" || { logger_Warn "EVAL_DATA_PATH:${EVAL_DATA_PATH} not valid path" ; return 1; } + logger_Debug "EVAL_DATA_PATH is valid" +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + cp $CHECKPOINT_PATH $RESULT_PATH/ + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh new file mode 100644 index 0000000..065998c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh new file mode 100644 index 0000000..7ec5482 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh @@ -0,0 +1,12 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7 +export TRAIN_DATA_FILE=/home/datasets/criteo/mini_demo.txt + +export RANK_SIZE=8 +export DEVICE_NUM=8 + +# need if rank_size > 1 +export RANK_TABLE_FILE=/home/lcm/tool/rank_table_16p_62_64.json +# cluster need for node info +#export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..aa00154 --- /dev/null +++ "b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,63 @@ +# Ais-Bench+Mindspore+widedeep使用说明 + +## 简介 + +AI Server Benchmark 是按《信息技术 人工智能 服务器系统性能测试规范》对人工智能服务器系统的性能进行性能评估的测试系统（测试套件），简称Ais-Bench软件。 + +## 使用前提 + +本程序包运行需要基于以下前提 + +1. Atlas 800-9000设备 +2. 安装好CANN包和Mindspore对应版本。并可以运行正常mindspore测试程序。 +3. 保存数据集和相关预处理文件等到设备中。 + +## 集群节点配置 + +如果运行设备大于1个设备，那么需要运行设置ssh节点文件。说明节点信息 +{ +"cluster": { +"xx.xx.xx.xx": { # 节点ip 必须与ranktable中对应 +"user": "xxxx", # 用户名 免密可以不用设置 +"pd": "xx", # 密码 免密不用设置 +"port": xx # 端口 默认22 可以不用设置 +}, +"xx.xx.xx.xx": { +"user": "xxxx", +"pd": "xx", +"port": xx +} +} +} + +## 集群节点免密设置 + +设置密钥认证的参考操作如下： +ssh-keygen -t rsa -b 2048 # 登录管理节点并生成SSH Key。安全起见，建议用户到"Enter passphrase"步骤时输入密钥密码，且符合密码复杂度要求。建议执行这条命令前先将umask设置为0077，执行完后再恢复原来umask值。 +ssh-copy-id -i ~/.ssh/id_rsa.pub ``@`` # 将管理节点的公钥拷贝到所有节点的机器上，``@``替换成要拷贝到的对应节点的账户和ip。 + +## 配置文件信息 + +> #python版本设置 +> export PYTHON_COMMAND=python3.7 +> #训练数据文件路径 +> export TRAIN_DATA_FILE=/home/data/criteo/origin_data/mini_demo.txt + +> 节点信息 +> export RANK_SIZE=8 +> export DEVICE_NUM=8 + +> #need if rank_size > 1 +> export RANK_TABLE_FILE=/home/lcm/tool/rank_table_16p_62_64.json + +> #cluster need for node info +> export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + +说明： +配置文件默认是8卡训练。 +单卡训练时，需要设置RANK_SIZE=1，DEVICE_NUM=1，且不能使用RANK_TABLE_FILE环境变量. +同时还请按需增加指定执行卡序号变量声明export SINGLE_CARD_INDEX。默认 SINGLE_CARD_INDEX=0，可以不显式声明。其它卡时需要显式声明，比如export SINGLE_CARD_INDEX=6 +## FAQ + +如果程序运行遇到错误--OSError: /lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block，请执行以下命令可解决问题： +export LD_PRELOAD=$LD_PRELOAD:/usr/local/python3.7.5/lib/python3.7/site-packages/scikit_learn.libs/libgomp-d22c30c5.so.1.0.0 diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh new file mode 100644 index 0000000..88da82a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="r1.5"; } + + if [ "$branch_args" == "r1.5" ];then + branch="master" + patch_file_name="r1.5" + commitid="5a4ff4e3dc9bcb46dbb71b6b16fbadbb68c5e8dc" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/recommend/wide_and_deep" + elif [ "$branch_args" == "r1.9" ];then + branch="master" + patch_file_name="r1.9" + commitid="948fe927641651e7a36103b96e96d8e86b4a5255" + git_url="https://gitee.com/mindspore/models.git" + modelzoo_sub_dir="models/official/recommend/wide_and_deep" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch new file mode 100644 index 0000000..f8e1e85 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch @@ -0,0 +1,73 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2021-12-10 16:23:44.520000000 +0800 ++++ code/eval.py 2021-12-10 16:23:44.530000000 +0800 +@@ -112,7 +112,14 @@ + + eval_callback = EvalCallBack(model, ds_eval, auc_metric, config) + +- model.eval(ds_eval, callbacks=eval_callback) ++ res = model.eval(ds_eval, callbacks=eval_callback) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ print("ACC_DIR:", ACC_DIR) ++ if not os.path.exists(ACC_DIR): ++ os.mkdir(ACC_DIR) ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(res["auc"])) + + + def modelarts_pre_process(): +diff -Nur origin/train.py code/train.py +--- origin/train.py 2021-12-10 16:23:44.520000000 +0800 ++++ code/train.py 2021-12-10 16:23:44.530000000 +0800 +@@ -13,6 +13,7 @@ + # limitations under the License. + """ test_training """ + import os ++import time + from mindspore import Model, context + from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMonitor + from src.wide_and_deep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel +@@ -20,7 +21,12 @@ + from src.datasets import create_dataset, DataType + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper +- ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + def get_WideDeep_net(configure): + """ +@@ -83,7 +89,28 @@ + ckptconfig = CheckpointConfig(save_checkpoint_steps=ds_train.get_dataset_size(), + keep_checkpoint_max=5) + ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', directory=configure.ckpt_path, config=ckptconfig) ++ model.build(ds_train, None, sink_size=ds_train.get_dataset_size(), epoch=epochs) ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(epochs, ds_train, callbacks=[TimeMonitor(ds_train.get_dataset_size()), callback, ckpoint_cb]) ++ all_data_sum = ds_train.get_dataset_size() * batch_size * epochs ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum/(end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if not os.path.exists(THROUGHPUT_DIR): ++ os.mkdir(THROUGHPUT_DIR) ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + def modelarts_pre_process(): diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch new file mode 100644 index 0000000..a1a1125 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch @@ -0,0 +1,78 @@ +diff -Nur origin/eval.py code/eval.py +--- origin/eval.py 2022-09-08 14:41:47.140000000 +0800 ++++ code/eval.py 2022-09-08 14:41:47.156000000 +0800 +@@ -112,7 +112,17 @@ + + eval_callback = EvalCallBack(model, ds_eval, auc_metric, config) + +- model.eval(ds_eval, callbacks=eval_callback) ++ out = model.eval(ds_eval, callbacks=eval_callback) ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ print("ACC_DIR:", ACC_DIR) ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(out['auc'])) + + + def modelarts_pre_process(): +diff -Nur origin/train.py code/train.py +--- origin/train.py 2022-09-08 14:41:47.144000000 +0800 ++++ code/train.py 2022-09-08 14:41:47.156000000 +0800 +@@ -13,6 +13,7 @@ + # limitations under the License. + """ test_training """ + import os ++import time + from mindspore import Model, context + from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, TimeMonitor + from src.wide_and_deep import PredictWithSigmoid, TrainStepWrap, NetWithLossClass, WideDeepModel +@@ -20,6 +21,12 @@ + from src.datasets import create_dataset, DataType + from src.model_utils.config import config + from src.model_utils.moxing_adapter import moxing_wrapper ++try: ++ import ais_utils ++except ImportError: ++ ais_utils_is_existed = False ++else: ++ ais_utils_is_existed = True + + + def get_WideDeep_net(configure): +@@ -83,7 +90,31 @@ + ckptconfig = CheckpointConfig(save_checkpoint_steps=ds_train.get_dataset_size(), + keep_checkpoint_max=5) + ckpoint_cb = ModelCheckpoint(prefix='widedeep_train', directory=configure.ckpt_path, config=ckptconfig) ++ model.build(ds_train, None, sink_size=ds_train.get_dataset_size(), epoch=epochs) ++ ++ if ais_utils_is_existed: ++ start_time = ais_utils.get_datatime() ++ else: ++ start_time = time.time() + model.train(epochs, ds_train, callbacks=[TimeMonitor(ds_train.get_dataset_size()), callback, ckpoint_cb]) ++ all_data_sum = epochs * ds_train.get_dataset_size() * batch_size ++ if ais_utils_is_existed: ++ end_time = ais_utils.get_datatime() ++ throughput_rate = ais_utils.calc_throughput_rate(all_data_sum, start_time, end_time) ++ else: ++ end_time = time.time() ++ throughput_rate = all_data_sum / (end_time - start_time) ++ ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) + + + def modelarts_pre_process(): diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..2bdca78 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_FILE?TRAIN_DATA_FILE not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh new file mode 100644 index 0000000..6d2e209 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +function get_data_preprocess_cmd(){ + LINE_COUNT=`cat ${TRAIN_DATA_FILE} | wc -l` + data_preprocess_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/src/preprocess_data.py \ + --data_path=$WORK_PATH/data/ \ + --dense_dim=13 --slot_dim=26 --threshold=100 --train_line_count=$LINE_COUNT --skip_id_convert=0 + " + return 0 +} + +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="True" || DISTRUTE_ENABLE="False" + + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py --data_path=$DATASET_PATH --dataset_type=mindrecord" + return 0 +} + +function get_eval_cmd() +{ + DATASET_PATH=$WORK_PATH/data/mindrecord + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --data_path=$DATASET_PATH --dataset_type=mindrecord --ckpt_path=$CHECKPOINT_PATH" + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/mindspore_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # ͨü Ҫ PYTHON_COMMAND RANK_SIZERANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # ǷװӦ + check_mindspore_run_ok ${PYTHON_COMMAND} || { logger_Warn "mindspore running failed" ; return 1; } + logger_Debug "mindspore running successfully" + + check_file_valid "${TRAIN_DATA_FILE}" || { logger_Warn "TRAIN_DATA_FILE:${TRAIN_DATA_FILE} not valid file" ; return 1; } + logger_Debug "TRAIN_DATA_FILE is valid" +} + +function node_data_preprocess() +{ + # data + rm -rf $WORK_PATH/data/* + mkdir -p $WORK_PATH/data/origin_data + ln -sf ${TRAIN_DATA_FILE} $WORK_PATH/data/origin_data/train.txt + + get_data_preprocess_cmd + $data_preprocess_cmd || { logger_Warn "preprocess run failed"; return 1; } + + DATASET_PATH=$WORK_PATH/data/mindrecord + check_path_valid $DATASET_PATH || { logger_Warn "mindpath:${DATASET_PATH} not valid path" ; return 1; } +} + +function node_train() +{ + node_data_preprocess + + # ͨѵӿ + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + CHECKPOINT_PATH=`find ${WORK_PATH}/train_parallel$RANK_ID/ -name "*.ckpt" | xargs ls -t | awk 'NR==1{print}'` + [ -f $CHECKPOINT_PATH ] || { logger_Warn "CHECKPOINT_PATH:${CHECKPOINT_PATH} not valid path" ; return 1; } + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + cd $RUN_PATH + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd || { echo "run eval node error ret:$?"; return 1; } + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh new file mode 100644 index 0000000..2c92824 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh @@ -0,0 +1,12 @@ +export PYTHON_COMMAND=python3.7.5 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/bertData/cn-wiki-128/ +export EVAL_DATA_PATH=/home/datasets/bertData/cn-wiki-128/ + +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh new file mode 100644 index 0000000..ea18ddd --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/nlp/Bert-base_ID0060_for_TensorFlow" + + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh new file mode 100644 index 0000000..0a30145 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + #run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..498ba4c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh @@ -0,0 +1,79 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + : "${EVAL_DATA_PATH?EVAL_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh new file mode 100644 index 0000000..95ad39a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh @@ -0,0 +1,96 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="True" || DISTRUTE_ENABLE="False" + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/src/pretrain/run_pretraining.py \ + --bert_config_file=$WORK_PATH/code/configs/bert_base_config.json \ + --max_seq_length=128 \ + --max_predictions_per_seq=20 \ + --train_batch_size=128 \ + --learning_rate=6.25e-6 \ + --num_warmup_steps=100 \ + --num_train_steps=500 \ + --optimizer_type=adam \ + --manual_fp16=True \ + --use_fp16_cls=True \ + --input_files_dir=${TRAIN_DATA_PATH} \ + --eval_files_dir=$EVAL_DATA_PATH \ + --npu_bert_debug=False \ + --npu_bert_use_tdt=True \ + --do_train=True \ + --num_accumulation_steps=1 \ + --npu_bert_job_start_file= \ + --iterations_per_loop=100 \ + --save_checkpoints_steps=10000 \ + --npu_bert_clip_by_global_norm=False \ + --distributed=$DISTRUTE_ENABLE \ + --npu_bert_loss_scale=0 \ + --output_dir=$RUN_PATH/ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh new file mode 100644 index 0000000..0f62ce6 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh @@ -0,0 +1,11 @@ +export PYTHON_COMMAND=python3.7 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/imagenet_TF +export MAX_TRAIN_STEPS=200 +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh new file mode 100644 index 0000000..a32ac5d --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/image_classification/DenseNet121_ID0067_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..87ed5ec --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh @@ -0,0 +1,79 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${MAX_TRAIN_STEPS?MAX_TRAIN_STEPS not set}" + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh new file mode 100644 index 0000000..6e9c9de --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh @@ -0,0 +1,78 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py \ + --data_dir=${TRAIN_DATA_PATH} \ + --rank_size=$RANK_SIZE \ + --iterations_per_loop=10 \ + --mode=train \ + --max_train_steps=$MAX_TRAIN_STEPS \ + --lr=0.04 \ + --display_every=10 \ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh new file mode 100644 index 0000000..9293b11 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh @@ -0,0 +1,14 @@ +export PYTHON_COMMAND=python3.7.5 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/imagenet_TF/ +export MAX_TRAIN_STEPS=500 +export BATCH_SIZE=256 + + +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh new file mode 100644 index 0000000..d9e49c5 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/image_classification/MobileNetV2_ID0074_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..b993115 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${MAX_TRAIN_STEPS?MAX_TRAIN_STEPS not set}" + : "${BATCH_SIZE?BATCH_SIZE not set}" + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh new file mode 100644 index 0000000..e12a2d3 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh @@ -0,0 +1,86 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py \ + --dataset_dir=$TRAIN_DATA_PATH \ + --max_train_steps=$MAX_TRAIN_STEPS \ + --iterations_per_loop=10 \ + --model_name=\"mobilenet_v2\" \ + --moving_average_decay=0.9999 \ + --label_smoothing=0.1 \ + --preprocessing_name=\"inception_v2\" \ + --weight_decay='0.000004' \ + --batch_size=$BATCH_SIZE \ + --learning_rate_decay_type='cosine_annealing' \ + --learning_rate=0.4 \ + --optimizer='momentum' \ + --momentum='0.9' \ + --warmup_epochs=5 \ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh new file mode 100644 index 0000000..410eb52 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh @@ -0,0 +1,12 @@ +export PYTHON_COMMAND=python3.7.5 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/Bert-small-Dataset/ +export EVAL_DATA_PATH=/home/datasets/Bert-TestData/ + +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch new file mode 100644 index 0000000..99e7cd5 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch @@ -0,0 +1,28 @@ +diff -Nur origin/src/pretrain/run_pretraining.py code/src/pretrain/run_pretraining.py +--- origin/src/pretrain/run_pretraining.py 2021-12-01 14:52:58.809717535 +0800 ++++ code/src/pretrain/run_pretraining.py 2021-12-01 14:52:58.817717584 +0800 +@@ -237,6 +237,24 @@ + else: + print('Step = %6i Throughput = %11.1f MLM Loss = %10.4e NSP Loss = %10.4e Loss = %9.6f Average Loss = %9.6f LR = %6.4e' % + (print_step, sent_per_sec, mlm_loss, nsp_loss, total_loss, avg_loss_step, lr), flush=True) ++ ++ try: ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("ais-bench Warning: 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("ais-bench Warning: 'RESULT_PATH' is not a valid directory. ") ++ else: ++ if rank_id == 0: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ throughput = sent_per_sec ++ print("ais-bench {} set throughput:{}".format(rank_id, throughput)) ++ f.write("{}".format(throughput)) ++ except Exception as msg: ++ print("write throught failed {}".format(msg)) ++ + self.elapsed_secs = 0. + self.count = 0 + self.avg_loss = 0.0 diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh new file mode 100644 index 0000000..7a11c1c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/nlp/Nezha-large_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh new file mode 100644 index 0000000..0a30145 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + #run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..498ba4c --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh @@ -0,0 +1,79 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + : "${EVAL_DATA_PATH?EVAL_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh new file mode 100644 index 0000000..0ef0cb4 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh @@ -0,0 +1,99 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && DISTRUTE_ENABLE="True" || DISTRUTE_ENABLE="False" + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/src/pretrain/run_pretraining.py \ + --bert_config_file=$WORK_PATH/code/configs/nezha_large_config.json \ + --max_seq_length=128 \ + --max_predictions_per_seq=20 \ + --train_batch_size=64 \ + --learning_rate=1e-4 \ + --num_warmup_steps=100 \ + --num_train_steps=1000 \ + --optimizer_type=lamb \ + --manual_fp16=True \ + --use_fp16_cls=True \ + --input_files_dir=${TRAIN_DATA_PATH} \ + --eval_files_dir=$EVAL_DATA_PATH \ + --npu_bert_debug=False \ + --npu_bert_use_tdt=True \ + --do_train=True \ + --num_accumulation_steps=1 \ + --npu_bert_job_start_file= \ + --iterations_per_loop=100 \ + --save_checkpoints_steps=1000 \ + --npu_bert_clip_by_global_norm=False \ + --distributed=$DISTRUTE_ENABLE \ + --npu_bert_loss_scale=0 \ + --output_dir=$RUN_PATH/ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + + check_path_valid "${EVAL_DATA_PATH}" || { logger_Warn "EVAL_DATA_PATH:${EVAL_DATA_PATH} not valid path" ; return 1; } + logger_Debug "EVAL_DATA_PATH is valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh new file mode 100644 index 0000000..6b09294 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh @@ -0,0 +1,13 @@ +export PYTHON_COMMAND=python3.7.5 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/imagenet_TF +export RESNET_SIZE=101 +export EPOCH_SIZE=1 +export BATCH_SIZE=90 +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh new file mode 100644 index 0000000..f65755a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/image_classification/ResNet101_ID0063_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh new file mode 100644 index 0000000..0a30145 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + #run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..e27eaec --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh @@ -0,0 +1,82 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${EPOCH_SIZE?EPOCH_SIZE not set}" + : "${RESNET_SIZE?RESNET_SIZE not set}" + : "${BATCH_SIZE?BATCH_SIZE not set}" + : "${RANK_SIZE?RANK_SIZE not set}" + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh new file mode 100644 index 0000000..71e55c5 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh @@ -0,0 +1,84 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/r1/resnet/imagenet_main.py \ + --resnet_size=$RESNET_SIZE \ + --batch_size=$BATCH_SIZE \ + --num_gpus=1 \ + --dtype=fp16 \ + --label_smoothing=0.1 \ + --loss_scale=512 \ + --max_train_steps=2000 \ + --train_epochs=$EPOCH_SIZE \ + --eval_only=False \ + --epochs_between_evals=10 \ + --hooks=ExamplesPerSecondHook,loggingtensorhook,loggingmetrichook \ + --data_dir=${TRAIN_DATA_PATH} \ + --model_dir=$RUN_PATH \ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "false" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt new file mode 100644 index 0000000..443ba89 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt @@ -0,0 +1,7 @@ +resnet50模型代码基准来源： + git_url="https://gitee.com/ascend/modelzoo.git" + modelzoo_sub_dir="modelzoo/built-in/TensorFlow/Official/cv/image_classification/Resnet50v1.5_for_TensorFlow/" + + +origin: 原始目录 +code: 修改目录 diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh new file mode 100644 index 0000000..b9673b9 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh @@ -0,0 +1,15 @@ +export PYTHON_COMMAND=python3.7.5 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/imagenet_TF +export RESNET_SIZE=50 +export EPOCH_SIZE=2 +export BATCH_SIZE=128 + + +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.old.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git "a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000..d9bf811 --- /dev/null +++ "b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,71 @@ +# Ais-Bench+Mindspore+Resnet使用说明 + +## 简介 + +AI Server Benchmark 是按《信息技术 人工智能 服务器系统性能测试规范》对人工智能服务器系统的性能进行性能评估的测试系统（测试套件），简称Ais-Bench软件。 + +## 使用前提 + +本程序包运行需要基于以下前提 + +1. Atlas 800-9000设备 +2. 安装好CANN包和Mindspore对应版本。并可以运行正常mindspore测试程序。 +3. 保存数据集和相关预处理文件等到设备中。 + +## 集群节点配置 + +如果运行设备大于1个设备，那么需要运行设置ssh节点文件。说明节点信息 +{ +"cluster": { +"xx.xx.xx.xx": { # 节点ip 必须与ranktable中对应 +"user": "xxxx", # 用户名 免密可以不用设置 +"pd": "xx", # 密码 免密不用设置 +"port": xx # 端口 默认22 可以不用设置 +}, +"xx.xx.xx.xx": { +"user": "xxxx", +"pd": "xx", +"port": xx +} +} +} + +## 集群节点免密设置 + +* 设置密钥认证的参考操作如下： + ssh-keygen -t rsa -b 2048 # 登录管理节点并生成SSH Key。安全起见，建议用户到"Enter passphrase"步骤时输入密钥密码，且符合密码复杂度要求。建议执行这条命令前先将umask设置为0077，执行完后再恢复原来umask值。 + +* ssh-copy-id -i ~/.ssh/id_rsa.pub ``@`` # 将管理节点的公钥拷贝到所有节点的机器上，``@``替换成要拷贝到的对应节点的账户和ip。 + +* 设置ssh代理管理ssh密钥，避免工具批量安装操作过程中输入密钥密码 + + ssh-agent bash # 开启ssh-agent的bash进程 + + ssh-add # 向ssh-agent添加私钥 + + +## 配置文件信息 + +> #python版本设置 +> export PYTHON_COMMAND=python3.7 +> #训练数据集路径 +> export TRAIN_DATA_PATH=/home/datasets/imagenet/train/ +> #验证数据集路径 +> export EVAL_DATA_PATH=/home/datasets/imagenet/val/ +> #epoch数 +> export EPOCH_SIZE=90 + +> 节点信息 +> export RANK_SIZE=8 +> export DEVICE_NUM=8 + +> #need if rank_size > 1 +> export RANK_TABLE_FILE=/home/lcm/tool/rank_table_16p_62_64.json + +> #cluster need for node info +> export NODEINFO_FILE=/home/lcm/tool/ssh64_66.json + +说明： +配置文件默认是8卡训练。 +单卡训练时，需要设置RANK_SIZE=1，DEVICE_NUM=1，且不能使用RANK_TABLE_FILE环境变量. +同时还请按需增加指定执行卡序号变量声明export SINGLE_CARD_INDEX。默认 SINGLE_CARD_INDEX=0，可以不显式声明。其它卡时需要显式声明，比如export SINGLE_CARD_INDEX=6 diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch new file mode 100644 index 0000000..acda739 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch @@ -0,0 +1,48 @@ +diff -Nur origin/official/r1/resnet/imagenet_main.py code/official/r1/resnet/imagenet_main.py +--- origin/official/r1/resnet/imagenet_main.py 2021-12-01 14:49:02.440337230 +0800 ++++ code/official/r1/resnet/imagenet_main.py 2021-12-01 14:49:02.556337859 +0800 +@@ -405,6 +405,21 @@ + flags_obj, imagenet_model_fn, input_function, DATASET_NAME,NUM_IMAGES, + shape=[DEFAULT_IMAGE_SIZE, DEFAULT_IMAGE_SIZE, NUM_CHANNELS],) + ++ try: ++ if 'accuracy' in result.get("eval_results",None): ++ ACC_DIR = os.getenv("RESULT_PATH") ++ if ACC_DIR is None: ++ print("ais-bench Warning: 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(ACC_DIR): ++ print("ais-bench Warning: 'RESULT_PATH' is not a valid directory. ") ++ else: ++ ACC_LOG = os.path.join(ACC_DIR, "eval_acc.log") ++ with open(ACC_LOG, 'w') as f: ++ f.write("{}".format(result["eval_results"]['accuracy'])) ++ print("ais-bench set accuracy:{}".format(result["eval_results"]['accuracy'])) ++ except Exception as msg: ++ print("ais-bench accuracy set failed {}".format(msg)) ++ + return result + def main(_): + ############## npu modify begin ############# +diff -Nur origin/official/utils/logs/hooks.py code/official/utils/logs/hooks.py +--- origin/official/utils/logs/hooks.py 2021-12-01 14:49:02.444337252 +0800 ++++ code/official/utils/logs/hooks.py 2021-12-01 14:49:02.560337881 +0800 +@@ -159,3 +159,19 @@ + "steps: %s,elapsed_steps:%d,batch:%d,FPS:%f,ips:%f,batch_time:%f", int(self._total_steps), + int(elapsed_steps),int(self._batch_size),float(current_examples_per_sec),float(ips), + float(batch_time)) ++ try: ++ rank_id = int(os.getenv('DEVICE_INDEX')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("ais-bench Warning: 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("ais-bench Warning: 'RESULT_PATH' is not a valid directory. ") ++ else: ++ if rank_id == 0: ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ throughput = current_examples_per_sec ++ print("ais-bench {} set throughput:{}".format(rank_id, throughput)) ++ f.write("{}".format(throughput)) ++ except Exception as msg: ++ print("write throught failed {}".format(msg)) diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh new file mode 100644 index 0000000..efc23b5 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/image_classification/Resnet50v1.5_ID1721_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..b1f687a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${EPOCH_SIZE?EPOCH_SIZE not set}" + : "${RESNET_SIZE?RESNET_SIZE not set}" + : "${BATCH_SIZE?BATCH_SIZE not set}" + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh new file mode 100644 index 0000000..0750cab --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh @@ -0,0 +1,74 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/official/r1/resnet/imagenet_main.py \ + --resnet_size=$RESNET_SIZE --resnet_version=1 \ + --epochs_between_evals=$EPOCH_SIZE --hooks=ExamplesPerSecondHook \ + --train_epochs=$EPOCH_SIZE \ + --batch_size=$BATCH_SIZE --data_dir=${TRAIN_DATA_PATH} --model_dir=$RUN_PATH/" + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh new file mode 100644 index 0000000..a379b00 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh @@ -0,0 +1,11 @@ +export PYTHON_COMMAND=python3.7 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/imagenet_TF +export MAX_TRAIN_STEPS=1000 +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh new file mode 100644 index 0000000..8e68d15 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/image_classification/ResNext50_ID0070_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..35df41f --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh @@ -0,0 +1,79 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${MAX_TRAIN_STEPS?MAX_TRAIN_STEPS not set}" + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh new file mode 100644 index 0000000..c0191f9 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh @@ -0,0 +1,87 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + [[ $RANK_SIZE -eq 1 ]] && CONFIG_FILE_32bs_1p_host || CONFIG_FILE=res50_32bs_8p_host + ln -sf $WORK_PATH/code/code/resnext50_train/configs ../ + sed -i "s|'num_epochs':.* |'num_epochs': 3, |g" $WORK_PATH/code/code/resnext50_train/configs/$CONFIG_FILE.py + sed -i "s|'rank_size':.* |'rank_size': $RANK_SIZE, |g" $WORK_PATH/code/code/resnext50_train/configs/$CONFIG_FILE.py + mkdir -p /data/ + ln -sf $TRAIN_DATA_PATH /data/imagenet_TF + + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/code/resnext50_train/mains/res50.py \ + --config_file=$CONFIG_FILE \ + --max_train_steps=$MAX_TRAIN_STEPS \ + --iterations_per_loop=100 \ + --debug=True \ + --eval=False \ + --over_dump=False \ + --data_path=$TRAIN_DATA_PATH \ + --model_dir=$RUN_PATH \ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code/code/resnext50_train + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh new file mode 100644 index 0000000..8220cdd --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh @@ -0,0 +1,16 @@ +export PYTHON_COMMAND=python3.7 + +# 参数信息 +export MODE=train +export BATCH_SIZE=32 +export TRAINING_FILE_PATTERN=/home/datasets/raw_data_tfrecord/train2017* +export RESNET_CHECKPOINT=/home/datasets/raw_data/resneet32_pretrain_model/model.ckpt-28152 +export VALIDATION_FILE_PATTERN=/home/datasets/raw_data/tfrecord/val2017* +export VAL_JSON_FILE=/home/datasets/raw_data/annotations/instances_val2017.json + +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=1 +export DEVICE_NUM=1 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh new file mode 100644 index 0000000..f6b87bb --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/detection/SSD-Resnet34_ID0048_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..cf8c92a --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh @@ -0,0 +1,83 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${MODE?MODE not set}" + : "${BATCH_SIZE?BATCH_SIZE not set}" + : "${TRAINING_FILE_PATTERN?TRAINING_FILE_PATTERN not set}" + : "${RESNET_CHECKPOINT?RESNET_CHECKPOINT not set}" + : "${VALIDATION_FILE_PATTERN?VALIDATION_FILE_PATTERN not set}" + : "${VAL_JSON_FILE?VAL_JSON_FILE not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh new file mode 100644 index 0000000..411c1f1 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh @@ -0,0 +1,78 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + [[ $RANK_SIZE -gt 1 ]] && NUM_EPOCHS=8 || NUM_EPOCHS=1 + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/ssd_main.py \ + --mode=$MODE \ + --train_batch_size=$BATCH_SIZE \ + --training_file_pattern=$TRAINING_FILE_PATTERN \ + --resnet_checkpoint=$RESNET_CHECKPOINT \ + --validation_file_pattern=$VALIDATION_FILE_PATTERN \ + --val_json_file=$VAL_JSON_FILE \ + --num_epochs=$NUM_EPOCHS \ + --num_examples_per_epoch=64000 \ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh new file mode 100644 index 0000000..0d87cb4 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh @@ -0,0 +1,13 @@ +export PYTHON_COMMAND=python3.7 + +# 参数信息 +export TRAIN_DATA_PATH=/home/datasets/imagenet_TF +export BATCH_SIZE=32 +export MODE=train +export MAX_TRAIN_STEPS=1000 +export ITERATIONS_PER_LOOP=10 +# 网络信息 + +export RANK_SIZE=1 +export DEVICE_NUM=1 + diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh new file mode 100644 index 0000000..acf00f4 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/image_classification/VGG16_ID0068_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..444493b --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${BATCH_SIZE?BATCH_SIZE not set}" + : "${MAX_TRAIN_STEPS?MAX_TRAIN_STEPS not set}" + : "${TRAIN_DATA_PATH?TRAIN_DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh new file mode 100644 index 0000000..4ac0171 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh @@ -0,0 +1,83 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + [[ $RUNK_SIZE -ne 8 ]] && DISPLAY_EVERY=10 || DISPLAY_EVERY=1 + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py \ + --batch_size=$BATCH_SIZE \ + --rank_size=$RANK_SIZE \ + --mode=$MODE \ + --max_train_steps=$MAX_TRAIN_STEPS \ + --iterations_per_loop=$ITERATIONS_PER_LOOP \ + --epochs_between_evals=1 \ + --data_dir=$TRAIN_DATA_PATH \ + --max_epochs=1 \ + --display_ever=$DISPLAY_EVERY \ + --lr=0.01 \ + " + return 0 +} + +function get_eval_cmd() +{ + return 0 +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${TRAIN_DATA_PATH}" || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return 1; } + logger_Debug "TRAIN_DATA_PATH is valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh new file mode 100644 index 0000000..3eb8977 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +file_change() +{ + local run_type=$1 + [ "$run_type" == "modelarts" ] && { sed -i 's|RUNTYPE=.*|RUNTYPE="modelarts"|g' ${CURDIR}//output/benchmark.sh; } + return 0 +} + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp -rf ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ + cp ${CURDIR}/../common/* -r ${CURDIR}//output/config/ + cp ${CURDIR}/doc -r ${CURDIR}/output/ + file_change "$2" || { echo "file change failed"; return 1; } +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh new file mode 100644 index 0000000..f7b0ae0 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh @@ -0,0 +1,11 @@ +export PYTHON_COMMAND=python3.7 + +# 参数信息 +export DATA_PATH=/home/datasets/yolov3_tf2/data +export MODE="multi" +# 网络信息 +export RANK_TABLE_FILE=/home/tools/rank_table_8p_62.json +export RANK_SIZE=8 +export DEVICE_NUM=8 + +#export NODEINFO_FILE=/home/tools/2node_6264.json diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch new file mode 100644 index 0000000..dc26fe0 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch @@ -0,0 +1,43 @@ +diff -Nur origin/train.py code/train.py +--- origin/train.py 2021-12-17 19:18:34.127267733 +0800 ++++ code/train.py 2021-12-17 19:18:34.131267604 +0800 +@@ -309,6 +309,8 @@ + best_mAP = -np.Inf + train_op = util.set_iteration_per_loop(sess, train_op, args.iterations_per_loop) + sess.run(train_init_op) ++ fps_sum = 0 ++ fps_num = 0 + for epoch in range(args.total_epoches): + loss_total, loss_xy, loss_wh, loss_conf, loss_class = AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter() + for i in trange(args.train_batch_num // args.iterations_per_loop): +@@ -317,7 +319,8 @@ + [train_op, merged, y_true, loss, global_step, learning_rate] + ) + fps = 1 / (time.time() - t) * args.iterations_per_loop * args.num_gpus * args.batch_size +- ++ fps_num += 1 ++ fps_sum += fps + writer.add_summary(summary, global_step=__global_step) + + loss_total.update(__loss[0], len(__y_true[0])) +@@ -351,6 +354,20 @@ + if __global_step >= 500: + break + ++ # output throughtout ++ throughput_rate = fps_sum/fps_num ++ rank_id = int(os.getenv('RANK_ID')) ++ THROUGHPUT_DIR = os.getenv("RESULT_PATH") ++ if THROUGHPUT_DIR is None: ++ print("Warning: The environment variable 'RESULT_PATH' is not set. ") ++ elif not os.path.isdir(THROUGHPUT_DIR): ++ print("Warning: The environment variable 'RESULT_PATH' is not a valid directory. ") ++ else: ++ print("THROUGHPUT_DIR:", THROUGHPUT_DIR) ++ THROUGHPUT_LOG = os.path.join(THROUGHPUT_DIR, "throughput_rank_{}".format(rank_id)) ++ with open(THROUGHPUT_LOG, 'w') as f: ++ f.write("{}".format(throughput_rate)) ++ + saver_to_save.save(sess, args.save_dir + 'model-final_step_{}_loss_{:.4f}_lr_{:.5g}'.format( \ + int(__global_step), + loss_total.average, diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh new file mode 100644 index 0000000..3b23d70 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +SRC_PATH=$CUR_PATH/../../../ +. $SRC_PATH/common/patch_common.sh + +get_git_info(){ + local branch_args="$1" + local run_type="$2" + + # set default branch + [[ -z "$branch_args" ]] && { branch_args="master"; } + + if [ "$branch_args" == "master" ];then + branch="master" + patch_file_name="master" + commitid="ca293a6071e44f6286e9ef3c1415c9818c1dd7af" + git_url="https://gitee.com/ascend/ModelZoo-TensorFlow.git" + modelzoo_sub_dir="ModelZoo-TensorFlow/TensorFlow/built-in/cv/detection/YoloV3_ID0076_for_TensorFlow" + else + echo "bad parameters : $1" + return $ret_error + fi + + [ "$run_type" == "modelarts" ] && { patch_file_name="modelarts_"$patch_file_name; } + return $ret_ok +} + +main(){ + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ];then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + local patch_type="$1" + local branch_args="$2" + local run_type="$3" + + get_git_info "$branch_args" "$run_type" || { echo "warn get git info failed"; return $ret_error; } + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$patch_type" == "mkpatch" ];then + make_patch || { echo "warn make patch failed"; return $ret_error; } + elif [ "$patch_type" == "loadcode" ];then + load_code || { echo "warn make patch failed"; return $ret_error; } + mkdir -p $CUR_PATH/doc + mk_version_file $CUR_PATH/doc/version.txt + else + echo "null op" + return $ret_error + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh new file mode 100644 index 0000000..766ba21 --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_init_failed=1 +declare -i ret_run_train_failed=2 +declare -i ret_run_eval_failed=3 +declare -i ret_get_result_failed=4 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export CODE_PATH=$CUR_PATH +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/common.sh + +RUNTYPE="cluster_offline" + +[ $RUNTYPE == "modelarts" ] && . $CODE_PATH/modelarts_run.sh +[ $RUNTYPE == "cluster_offline" ] && . $CODE_PATH/cluster_offline_run.sh + +main(){ + init || { logger_Warn "init failed:$?";return $ret_init_failed; } + run_train || { logger_Warn "run_train failed ret:$?";return $ret_run_train_failed; } + run_eval || { logger_Warn "run_eval failed ret:$?";return $ret_run_eval_failed; } + get_result || { logger_Warn "get_result failed ret:$?";return $ret_get_result_failed; } + return $ret_ok +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh new file mode 100644 index 0000000..ea3a88d --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +. $CODE_PATH/common/common.sh +. $CODE_PATH/common/log_util.sh +. $CODE_PATH/common/cluster_common.sh +. $CODE_PATH/common/node_common.sh + +# env check +check_env() +{ + # base check + check_env_common || { logger_Warn "check env common failed:$?";return 1; } + + # model info check + : "${DATA_PATH?DATA_PATH not set}" + + # check env of each node + cmd="export WORK_PATH=$WORK_PATH; + bash -x $WORK_PATH/run_node.sh check ${WORK_PATH}/config/$CONFIG_FILE" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { return 1; } +} + +init() +{ + logger_Info "-------------------------------- init start --------------------------------" + # set nodes work path + export WORK_PATH=${BASE_PATH}/work + # set nodes result path + export RESULT_PATH=${WORK_PATH}/result + export PYTHONPATH=$PYTHONPATH:$CODE_PATH + CONFIG_FILE="config.sh" + source ${CODE_PATH}/config/$CONFIG_FILE || { logger_Warn "source file failed:$?";return 1; } + + rm -rf $RESULT_PATH;mkdir -p $RESULT_PATH + # sync data if work_path not exist so new one + + cmd="rm -rf ${WORK_PATH};mkdir -p ${WORK_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "renew workpath failed"; return 1; } + + cluster_scp "${NODEINFO_FILE}" ${CODE_PATH} ${WORK_PATH} || { logger_Warn "run scp failed"; return 1; } + + check_env || { logger_Warn "env check failed'" ; return 1; } + logger_Info "-------------------------------- init end --------------------------------" +} + +run_train() +{ + logger_Info "-------------------------------- train start --------------------------------" + cmd="export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + source $WORK_PATH/config/$CONFIG_FILE; + bash -x $WORK_PATH/run_node.sh train" + + cluster_run_cmd_parallel "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run train failed"; return 1; } + logger_Info "-------------------------------- train end --------------------------------" +} + +run_eval() +{ + logger_Info "-------------------------------- eval start --------------------------------" + cmd="source $WORK_PATH/config/$CONFIG_FILE; + export WORK_PATH=$WORK_PATH; + export RESULT_PATH=$RESULT_PATH; + bash $WORK_PATH/run_node.sh eval" + cluster_run_cmd_single "${NODEINFO_FILE}" ${cmd} || { logger_Warn "run eval failed"; return 1; } + logger_Info "-------------------------------- eval end --------------------------------" +} + +get_result() +{ + logger_Info "-------------------------------- get_result start --------------------------------" + + cmd="mkdir -p ${RESULT_PATH}" + cluster_run_cmd_serial "$NODEINFO_FILE" ${cmd} || { logger_Warn "mkdir resultpath failed"; return 1; } + + cluster_rscp "${NODEINFO_FILE}" ${RESULT_PATH} ${RESULT_PATH} + ${PYTHON_COMMAND} ${CODE_PATH}/common/calc_result.py ${RESULT_PATH} ${RANK_SIZE} + logger_Info "-------------------------------- get_result end --------------------------------" +} diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh new file mode 100644 index 0000000..3112b4d --- /dev/null +++ b/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh @@ -0,0 +1,105 @@ +#!/bin/bash +. $WORK_PATH/common/common.sh +. $WORK_PATH/common/log_util.sh +. $WORK_PATH/common/node_common.sh + +# 获取训练命令 +function get_train_cmd() +{ + rm -rf $WORK_PATH/code/data + ln -sf $DATA_PATH $RUN_PATH + ln -sf $DATA_PATH $WORK_PATH/code + train_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/train.py \ + --mode=$MODE \ + " + return 0 +} + +function get_eval_cmd() +{ + rm -rf $WORK_PATH/code/data + ln -sf $DATA_PATH $RUN_PATH + ln -sf $DATA_PATH $WORK_PATH/code + cp $WORK_PATH/code/eval_coco.py $RUN_PATH/ + eval_run_cmd="${PYTHON_COMMAND} -u $WORK_PATH/code/eval.py \ + --save_json=True \ + --score_thresh=0.0001 \ + --nms_thresh=0.55 \ + --max_boxes=100 \ + --restore_path=$RESTORE_PATH \ + --max_test=10000 \ + --save_json_path=eval_res_D$RANK_ID.json \ + " + return +} + +function node_init() +{ + export PYTHONPATH=$PYTHONPATH:$WORK_PATH:$WORK_PATH/code + source $WORK_PATH/config/tensorflow_env.sh + # for eval env set + [ $1 == "eval" ] && { export RANK_SIZE=1; export DEVICE_ID=0; : "${SINGLE_CARD_INDEX:=0}";export RANK_ID=$SINGLE_CARD_INDEX; unset RANK_TABLE_FILE; } + [[ -z "$RESULT_PATH" ]] || { mkdir -p $RESULT_PATH; } +} + +function node_check() +{ + CONFIG_FILE_PATH=$1 + source $CONFIG_FILE_PATH + + # 通用检测 主要检测 PYTHON_COMMAND RANK_SIZE和RANK_TABLE + node_common_check "${PYTHON_COMMAND}" "${RANK_SIZE}" "$RANK_TABLE_FILE" || { logger_Warn "node common check failed" ; return 1; } + # 检测是否安装对应框架软件 + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" + + check_path_valid "${DATA_PATH}" || { logger_Warn "DATA_PATH:${DATA_PATH} not valid path" ; return 1; } + logger_Debug "DATA_PATH is valid" + + logger_Debug "EVAL_DATA_PATH is valid" + +} + +function node_train() +{ + # 调用通用训练接口 + node_common_train "true" "false" || { logger_Warn "run train failed" ; return 1; } +} + +function node_eval() +{ + RUN_PATH=$WORK_PATH/train_parallel$RANK_ID + RESTORE_PATH=$RUN_PATH/training/ + cd $RUN_PATH + + get_eval_cmd + echo "start eval RUN_PATH:${RUN_PATH} SERVER_ID:$SERVER_ID rank $RANK_ID device $DEVICE_ID begin cmd:${eval_run_cmd}" + $eval_run_cmd > $RUN_PATH/eval.log || { echo "run eval node error ret:$?"; return 1; } + + if [ -f "$RUN_PATH/eval.log" ];then + accuracy=`cat $PATH/eval.log |grep "Average Precision" |grep -v grep |awk -F= 'NR==1{print $NF}'` + logger_Info "accuracy: $accuracy" + echo "$accuracy" > $RESULT_PATH/eval_acc.log + fi + + return 0 +} + +main() +{ + type="$1" + shift + node_init $type || { logger_Warn "init failed"; return 1; } + if [ "$type" == "train" ];then + node_train "$@" || { logger_Warn "run_node_train failed"; return 1; } + elif [ "$type" == "eval" ];then + node_eval "$@" || { logger_Warn "run_node_eval failed"; return 1; } + elif [ "$type" == "check" ];then + node_check "$@" || { logger_Warn "run_node_check failed"; return 1; } + else + { logger_Warn "invalid argument '${type}'"; return 1; } + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD new file mode 100644 index 0000000..4e0972d --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD @@ -0,0 +1,80 @@ +## 1. 背景 + +### 1.1 bert模型基准来源： + +https://github.com/google-research/bert/tree/master + +### 1.2 训练指标计算说明 + +- accuracy 训练结果评估中masked_lm_accuracy的值 +- throughput_ratio 计算公式--train_batch_size * num_train_steps / （end - real_step_start_time）。 + 公式说明： + + train_batch_size 训练批数 + + num_train_steps 训练步数 + + real_step_start_time 训练开始时间。该时间是单纯训练时间，不包括训练数据加载内存、加载device、训练热身等辅助时间，因此忽略了训练前2步耗费的时间 + + end 训练结束时间 + + +## 2.训练过程 + +### 2.1 训练准备 + +#### 2.1.1 环境要求 ++ python3.7.5、tensorflow 1.13(gpu版)、anconda3。建议在conda环境训练 ++ 可以在线拉取github.com的代码 ++ 执行nvidia-smi，检查当前设备是否使用。若使用请终止相关进程 +#### 2.1.2 bert模型下载 +到bert[官网](https://github.com/google-research/bert) "Pre-trained models"小节选择合适的bert模型下载到本地并解压使用。 + +bert large配置包信息： ++ 配置包规格：BERT-Large, Uncased: 24-layer, 1024-hidden, 16-heads, 340M ++ 压缩包名称：uncased-L-24_H-1024_A-16.zip ++ 下载地址：https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-24_H-1024_A-16.zip + +说明：本训练用到该配置包中的bert_config.json、vocab.txt文档。其它暂不涉及。 + + +### 2.2 修改预训练执行配置 +#### 2.2.1 修改配置文件 +执行`vim Ais-Bench-Stubs-aarch64/code/config/config_bert.sh`,修改配置。 + ++ BERT_CONFIG_DIR 修改为bert 配置目录 ++ TRAIN_STEPS 训练步数 ++ CUDA_VISIBLE_DEVICES 修改为指定设备ID。只有一位数字时为单卡训练。多卡同时使用时，逗号间隔，比如“0,1" + +#### 2.2.2 修改run.sh +执行`vim Ais-Bench-Stubs-aarch64/code/run.sh`, 适当修改train_run_cmd变量中预训练参数。 +### 2.3. 执行训练测试 +进入工作目录Ais-Bench-Stubs-aarch64, 执行以下指令进行本地训练： +``` +cd Ais-Bench-Stubs-aarch64 +rm -rf output + +./ais-bench-stubs test +``` + + +### 2.4 本地训练结果 + +训练过程，屏幕会有日志输出。训练结束会打印如下2个信息： +#### 2.4.1 train_result_info信息 +该信息包括了精度accuracy和吞吐率throughput_retio。 + +``` +[2021-8-4 12:35:39][INFO]train_result_info: { +"accruacy" : "0.05533597", +"throughput_ratio" : "2.456748253218857", +... +} +``` +#### 2.4.1 tensorflow回调函数打印的吞吐率 +``` +actual callback_throught_rate: 2.7236 +``` + + +### 2.5.训练执行注意事项 ++ 训练环境需要能联网，方便在线下载代码和bert模型 ++ 预训练参数max_seq_length越大，越容易造成预训练过程的资源耗尽问题。请选择合适参数 ++ 中断训练时，需要执行nvidia-smi，查到当前执行的进程ID，强行杀死该进程，避免影响下一次训练 ++ 每次训练前清空run_pretrain.py执行的output目录（ais-bench-stubs同级目录），会确保预训练时tensorflow日志sample/sec打印出来 diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh new file mode 100644 index 0000000..4af8c96 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh new file mode 100644 index 0000000..eb5006c --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh @@ -0,0 +1,13 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7.5 +export BERT_CONFIG_DIR=/path_to_bert_model_folder +export TRAIN_DATA_PATH=/path_to_bert_data_file +export TRAIN_STEPS=200 +export CUDA_VISIBLE_DEVICES="0" +export BATCH_SIZE=1 +export MAX_SEQ_LENGTH=512 + +# bmc info get power info from bmc +export BMC_IP="" +export BMC_USER="Administrator" +export BMC_PASSWORD="" diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch new file mode 100644 index 0000000..8918fe6 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch @@ -0,0 +1,35 @@ +diff -Nur -x '.git*' origin/run_pretraining.py code/run_pretraining.py +--- origin/run_pretraining.py 2021-09-02 13:52:16.480000000 +0800 ++++ code/run_pretraining.py 2021-09-02 13:52:16.488000000 +0800 +@@ -22,6 +22,8 @@ + import modeling + import optimization + import tensorflow as tf ++import time ++import ais_utils + + flags = tf.flags + +@@ -463,7 +465,13 @@ + max_seq_length=FLAGS.max_seq_length, + max_predictions_per_seq=FLAGS.max_predictions_per_seq, + is_training=True) ++ start = time.process_time() + estimator.train(input_fn=train_input_fn, max_steps=FLAGS.num_train_steps) ++ end = time.process_time() ++ data_sum = FLAGS.train_batch_size * FLAGS.num_train_steps ++ throught_rate = ais_utils.calc_throughput_rate(data_sum, (int)(end - start)) ++ ais_utils.set_result("training", "throughput_ratio", throught_rate) ++ print("start:{} end:{} batchsize:{} train_steps:{} datasum:{} start:{} end:{} duration:{} throught_rate:{}".format(start, end, FLAGS.train_batch_size, FLAGS.num_train_steps, data_sum, start, end, (end-start), throught_rate)) + + if FLAGS.do_eval: + tf.logging.info("***** Running evaluation *****") +@@ -485,6 +493,8 @@ + tf.logging.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + ++ if "masked_lm_accuracy" in result: ++ ais_utils.set_result("training", "accuracy", result["masked_lm_accuracy"]) + + if __name__ == "__main__": + flags.mark_flag_as_required("input_file") diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh new file mode 100644 index 0000000..1b0418a --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +git_url="https://github.com/google-research/bert.git" +git_url="https://gitee.com/lanhaibo4/bert.git" +branch="master" + +master_commitid="eedf5716ce1268e56f0a50264a88cafad334ac61" +bert_sub_dir="bert" + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +get_bert_base_code_by_git() { + git clone $git_url -b $branch + cd $bert_sub_dir + git reset --hard $commitid + cd - +} + +make_patch() { + cd $BUILD_TMP_PATH + get_bert_base_code_by_git + cp $bert_sub_dir -rf $BUILD_TMP_PATH/origin + cp $target_dir -rf $BUILD_TMP_PATH/code + + diff -Nur -x ".git*" origin code >$BUILD_TMP_PATH/$branch.patch + + cp $BUILD_TMP_PATH/$branch.patch $CUR_PATH/ +} + +load_code() { + cd $BUILD_TMP_PATH + get_bert_base_code_by_git + cp $bert_sub_dir -rf $BUILD_TMP_PATH/origin + cp $bert_sub_dir -rf $BUILD_TMP_PATH/code + + patch -p0 <$CUR_PATH/$branch.patch + + [ ! -d $target_patchcode_dir ] || rm -rf $target_patchcode_dir + mkdir $target_patchcode_dir + cp $BUILD_TMP_PATH/code/* -rf $target_patchcode_dir/ +} + +main() { + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ]; then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + commitid=$master_commitid + + echo "patch.sh run type:$1 branch:$branch" + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$1" == "mkpatch" ]; then + make_patch + elif [ "$1" == "loadcode" ]; then + load_code + else + echo "null op" + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh new file mode 100644 index 0000000..caf5531 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_invalid_args=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CUR_PATH/common/log_util.sh +. $CUR_PATH/common/common.sh +. $CUR_PATH/common/calc_power.sh +. $CUR_PATH/common/calc_resourceinfo.sh + +# 设置配置文件中的环境变量 +set_config_env() { + export PYTHONPATH=$PYTHONPATH:${CUR_PATH}:${CUR_PATH}/code + source $CUR_PATH/config/config_pretrain.sh +} + +# 环境变量检查 +check_env() { + return +} + +# 文件路径和环境依赖检查 +check_sys() { + check_path_valid $BERT_CONFIG_DIR || { logger_Warn "BERT_CONFIG_DIR:${BERT_CONFIG_DIR} not valid path"; return $ret_invalid_args; } + logger_Debug "train path valid" + + check_file_valid $TRAIN_DATA_PATH || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path"; return $ret_invalid_args; } + logger_Debug "train data file path valid" + + #check_python_version || { logger_Warn "python version not match"; return $ret_invalid_args; } + #logger_Debug "python version valid" + + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install"; return $ret_invalid_args; } + logger_Debug "python packet tensorflow valid" +} + +main() { + set_config_env + + check_env + + check_sys || exit $ret_invalid_args + + calc_powerinfo_backgroud + + device_group=(0) + run_resourceinfo_monitor_backgroud_gpu + + bash $CUR_PATH/run.sh + + calc_runing_resourceinfo_gpu $CUR_PATH/ais_utils.py $device_group + + set_powerinfo +} + +main "$@" |& tee "${BASE_PATH}/log/detail.log" +exit $? diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh new file mode 100644 index 0000000..78da140 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh @@ -0,0 +1,48 @@ +#!/bin/bash +declare -i ret_invalid_args=1 +declare -i ret_train_failed=2 +CUR_PATH=$(dirname $(readlink -f "$0")) +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CUR_PATH/common/log_util.sh +. $CUR_PATH/common/common.sh + +ulimit -u unlimited + +run_train() { + # create work path + rm -rf $BASE_PATH/work + mkdir -p $BASE_PATH/work + cp $CUR_PATH/code/* $BASE_PATH/work -rf + + train_run_cmd="python3 -u $BASE_PATH/work/run_pretraining.py \ + --input_file=$TRAIN_DATA_PATH \ + --output_dir=$BASE_PATH/work \ + --do_train=True \ + --do_eval=True \ + --bert_config_file=$BERT_CONFIG_DIR/bert_config.json \ + --max_seq_length=$MAX_SEQ_LENGTH \ + --max_predictions_per_seq=76 \ + --num_train_steps=$TRAIN_STEPS \ + --num_warmup_steps=0 \ + --train_batch_size=$BATCH_SIZE + " + + logger_Info "train run cmd:$train_run_cmd" + $train_run_cmd || { ret=$ret_train_failed;logger_Warn "run train failed ret:$?"; } + logger_Info "train run done cmd:$train_run_cmd" + return $ret +} + +main() { + python3 $CUR_PATH/ais_utils.py set_result "training" "proc_start_time" $(date "+%Y-%m-%d %H:%M:%S") + run_train || { logger_Warn "train failed ret:$?"; return $ret_train_failed; } + python3 $CUR_PATH/ais_utils.py set_result "training" "proc_end_time" $(date "+%Y-%m-%d %H:%M:%S") + callback_throught_rate=`cat ${BASE_PATH}/log/detail.log | grep "examples/sec:" | tail -n 1 | awk -F ' ' '{print $NF}'` + echo "actual callback_throught_rate:$callback_throught_rate" + + python3 $CUR_PATH/ais_utils.py set_result "training" "result" "OK" +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD new file mode 100644 index 0000000..57ac828 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD @@ -0,0 +1,55 @@ +## 1. 背景 + +### 1.1 resnet模型基准来源： + +https://github.com/tensorflow/models/tree/r1.13.0/official/resnet + +### 1.2 训练指标计算说明 + +- accuracy 训练结果中accuracy的值 +- throughput_ratio 计算公式--imagenet数据集图片数目 * epoch_size / 训练时间（包括训练结果评估时间） + +-- imagenet数据集图片数目 全部数据集 1280000个 + +## 2.训练过程 + +### 2.1 训练准备 + +#### 2.1.1 环境要求 ++ python3.7.5、tensorflow 1.13(gpu版)、anconda3 ++ 执行nvidia-smi，检查当前设备是否使用。若使用请终止相关进程 +#### 2.1.2 imagenet tensorflow数据集下载 +到resnete官网下载imagenet tensorflow类型的数据集 + +### 2.2 修改预训练执行配置 +执行`vim Ais-Bench-Stubs-aarch64/code/config/config_imagenet2012.sh`, 修改配置。 ++ TRAIN_DATA_PATH 修改为resnet训练数据集目录 ++ EPOCH_SIZE epoch 数目 ++ RESNET_SIZE resnet层数 ++ CUDA_VISIBLE_DEVICES 指定设备进行训练。"1"表示仅使用设备1训练。"0,1"，表示同时设备0和1进行训练。 + +### 2.3. 执行训练测试 +进入工作目录, 执行以下指令进行本地训练： +``` +cd Ais-Bench-Stubs-aarch64 + +./ais-bench-stubs test +``` + +### 2.4 本地训练结果 + +训练过程，屏幕会有日志输出。训练结束会打印train_result_info信息，包括了精度accuracy和吞吐率throughput_ratio。 + +``` +[2021-8-3 14:35:39][INFO]train_result_info: { +"accruacy" : "0.11714", +"throughput_ratio" : "76.54648206089946", +... +} +``` +说明：当前结果是基于 P100 单卡的 1个epoch的训练结果 + + +### 4.5.训练执行注意事项 ++ 训练环境需要能联网，方便在线下载代码和resnet数据集 ++ 中断训练时，需要执行nvidia-smi，查到当前执行的进程ID，强行杀死该进程，避免影响下一次训练 diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh new file mode 100644 index 0000000..4af8c96 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +declare -i ret_ok=0 +declare -i ret_error=1 + +CURDIR=$(dirname $(readlink -f $0)) + +function main() +{ + rm -rf ${CURDIR}/output/* + mkdir -p ${CURDIR}/output + echo "build call args:$@" + bash $CURDIR/patch.sh loadcode "$@" || { echo "warn run patch failed"; return 1; } + cp -rf ${CURDIR}/patchcode ${CURDIR}//output/code + cp ${CURDIR}/scripts/* ${CURDIR}//output/ + cp ${CURDIR}/../../../common -r ${CURDIR}//output/ + cp ${CURDIR}/config -r ${CURDIR}//output/ +} + +main "$@" +exit $? \ No newline at end of file diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh new file mode 100644 index 0000000..fe8b058 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh @@ -0,0 +1,13 @@ +#!/bin/bash +export PYTHON_COMMAND=python3.7.5 +export TRAIN_DATA_PATH=/path_to_train_data + +export EPOCH_SIZE=1 + +export RESNET_SIZE=50 +export CUDA_VISIBLE_DEVICES="0" +# bmc info get power info from bmc +export BMC_IP="" +export BMC_USER="Administrator" +export BMC_PASSWORD="" + diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh new file mode 100644 index 0000000..edfe17c --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +declare -i ret_error=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) + +git_url="https://github.com/tensorflow/models.git" +git_url="https://gitee.com/lanhaibo4/models.git" + +branch="r1.13.0" + +r1_13_0_commitid="57e075203f8fba8d85e6b74f17f63d0a07da233a" + +modelzoo_sub_dir="models/official" + +target_dir=$CUR_PATH/code +target_patchcode_dir=$CUR_PATH/patchcode + +get_modelzoo_base_code_by_git() { + git clone $git_url -b $branch + cd models + git reset --hard $commitid + cd - +} + +make_patch() { + cd $BUILD_TMP_PATH + get_modelzoo_base_code_by_git + mkdir $BUILD_TMP_PATH/origin + mkdir $BUILD_TMP_PATH/code + cp $modelzoo_sub_dir -rf $BUILD_TMP_PATH/origin/ + cp $target_dir/* -rf $BUILD_TMP_PATH/code/ + + diff -Nur -x ".git*" origin code >$BUILD_TMP_PATH/$branch.patch + + cp $BUILD_TMP_PATH/$branch.patch $CUR_PATH/ +} + +load_code() { + cd $BUILD_TMP_PATH + get_modelzoo_base_code_by_git + mkdir $BUILD_TMP_PATH/origin + mkdir $BUILD_TMP_PATH/code + cp $modelzoo_sub_dir -rf $BUILD_TMP_PATH/origin/ + cp $modelzoo_sub_dir -rf $BUILD_TMP_PATH/code/ + + patch -p0 <$CUR_PATH/$branch.patch + + [ ! -d $target_patchcode_dir ] || rm -rf $target_patchcode_dir + mkdir $target_patchcode_dir + rm -rf $BUILD_TMP_PATH/code/research + cp $BUILD_TMP_PATH/code/* -rf $target_patchcode_dir/ +} + +main() { + if [ "$1" != "mkpatch" -a "$1" != "loadcode" ]; then + echo "target not valid in:[$1] not match [mkpatch loadcode]" + return $ret_error + fi + + commitid=$r1_13_0_commitid + + echo "patch.sh run type:$1 branch:$branch" + + BUILD_TMP_PATH=$CUR_PATH/buildtmp + [ ! -d $BUILD_TMP_PATH ] || rm -rf $BUILD_TMP_PATH + mkdir -p $BUILD_TMP_PATH + + if [ "$1" == "mkpatch" ]; then + make_patch + elif [ "$1" == "loadcode" ]; then + load_code + else + echo "null op" + fi +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch new file mode 100644 index 0000000..2e8b20a --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch @@ -0,0 +1,59 @@ +diff -Nur -x '.git*' origin/official/resnet/resnet_run_loop.py code/official/resnet/resnet_run_loop.py +--- origin/official/resnet/resnet_run_loop.py 2021-09-03 18:05:29.200000000 +0800 ++++ code/official/resnet/resnet_run_loop.py 2021-09-03 18:05:29.220000000 +0800 +@@ -27,6 +27,7 @@ + import math + import multiprocessing + import os ++import time + + # pylint: disable=g-bad-import-order + from absl import flags +@@ -41,7 +42,12 @@ + from official.resnet import imagenet_preprocessing + from official.utils.misc import distribution_utils + from official.utils.misc import model_helpers ++import ais_utils + ++NUM_IMAGES = { ++ 'train': 1281167, ++ 'validation': 50000, ++} + + ################################################################################ + # Functions for input processing. +@@ -478,7 +484,7 @@ + run_config = tf.estimator.RunConfig( + train_distribute=distribution_strategy, + session_config=session_config, +- save_checkpoints_secs=60*60*24) ++ save_checkpoints_secs=60*60*24, keep_checkpoint_max=1) + + # Initializes model with all but the dense layer from pretrained ResNet. + if flags_obj.pretrained_model_checkpoint_path is not None: +@@ -560,8 +566,15 @@ + tf.logging.info('Starting cycle: %d/%d', cycle_index, int(n_loops)) + + if num_train_epochs: ++ start = time.process_time() + classifier.train(input_fn=lambda: input_fn_train(num_train_epochs), + hooks=train_hooks, max_steps=flags_obj.max_train_steps) ++ end = time.process_time() ++ data_sum = NUM_IMAGES['train'] * flags.FLAGS.train_epochs ++ throughput_rate = ais_utils.calc_throughput_rate(data_sum, (int)(end - start)) ++ ais_utils.set_result("training", "throughput_ratio", throughput_rate) ++ print("starttime: {} endtime：{} image_number: {} epoch_size: {} throughput_ratio: {}".format(start, \ ++ end, NUM_IMAGES['train'], flags.FLAGS.train_epochs, throughput_rate)) + + tf.logging.info('Starting to evaluate.') + +@@ -573,6 +586,9 @@ + # global_step count. + eval_results = classifier.evaluate(input_fn=input_fn_eval, + steps=flags_obj.max_train_steps) ++ print("eval_results: {}".format(eval_results)) ++ if 'accuracy' in eval_results: ++ ais_utils.set_result('training', 'accuracy', eval_results['accuracy']) + + benchmark_logger.log_evaluation_result(eval_results) + diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh new file mode 100644 index 0000000..9bd2357 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# 返回码 +declare -i ret_ok=0 +declare -i ret_invalid_args=1 + +CUR_PATH=$(dirname $(readlink -f "$0")) +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CUR_PATH/common/log_util.sh +. $CUR_PATH/common/common.sh +. $CUR_PATH/common/calc_power.sh +. $CUR_PATH/common/calc_resourceinfo.sh + +# 设置配置文件中的环境变量 +set_config_env() +{ + export PYTHONPATH=$PYTHONPATH:${CUR_PATH}:${CUR_PATH}/code + + source $CUR_PATH/config/config_imagenet2012.sh +} + +# 环境变量检查 +check_env() +{ + : "${EPOCH_SIZE?EPOCH_SIZE not set}" +} + +# 文件路径和环境依赖检查 +check_sys() +{ + check_path_valid $TRAIN_DATA_PATH || { logger_Warn "TRAIN_DATA_PATH:${TRAIN_DATA_PATH} not valid path" ; return $ret_invalid_args; } + logger_Debug "train path valid" + #check_path_valid $TEST_DATA_PATH || { logger_Warn "TEST_DATA_PATH:${TEST_DATA_PATH} not valid path" ; return $ret_invalid_args; } + #logger_Debug "test path valid" + + #check_python_version || { logger_Warn "python version not match" ; return $ret_invalid_args; } + #logger_Debug "python version valid" + + check_python_package_is_install ${PYTHON_COMMAND} "tensorflow" || { logger_Warn "tensorflow package not install" ; return $ret_invalid_args;} + logger_Debug "python packet tensorflow valid" +} + +main() +{ + set_config_env + + check_env + + check_sys || exit $ret_invalid_args + + calc_powerinfo_backgroud + + device_group=(0) + run_resourceinfo_monitor_backgroud_gpu + + bash $CUR_PATH/run.sh + + calc_runing_resourceinfo_gpu $CUR_PATH/ais_utils.py $device_group + + set_powerinfo +} + +main "$@" +exit $? diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh new file mode 100644 index 0000000..1ccc4f3 --- /dev/null +++ b/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh @@ -0,0 +1,39 @@ +#!/bin/bash +declare -i ret_invalid_args=1 +CUR_PATH=$(dirname $(readlink -f "$0")) +export BASE_PATH=$(cd "$CUR_PATH/../";pwd) + +. $CUR_PATH/common/log_util.sh +. $CUR_PATH/common/common.sh + +ulimit -u unlimited + +run_train(){ + # create work path + rm -rf $BASE_PATH/work + mkdir -p $BASE_PATH/work + cp $CUR_PATH/code/* $BASE_PATH/work/ -rf + + train_run_cmd="python -u $BASE_PATH/work/official/resnet/imagenet_main.py \ + --data_dir=${TRAIN_DATA_PATH} \ + --train_epochs=$EPOCH_SIZE \ + --resnet_size=$RESNET_SIZE \ + --model_dir=$BASE_PATH/work \ + --epochs_between_evals=$EPOCH_SIZE \ + " + $train_run_cmd + cp -f $BASE_PATH/work/model.ckpt-* $BASE_PATH/result + logger_Info "train run done" +} + +main() +{ + python $CUR_PATH/ais_utils.py set_result "training" "proc_start_time" `date "+%Y-%m-%d %H:%M:%S"` + + run_train + python $CUR_PATH/ais_utils.py set_result "training" "proc_end_time" `date "+%Y-%m-%d %H:%M:%S"` + python3 $CUR_PATH/ais_utils.py set_result "training" "result" "OK" +} + +main "$@" +exit $? -- Gitee From 57582e74f40444591adc1b1cc5085886d4f66de8 Mon Sep 17 00:00:00 2001 From: zjk <15778488+zjks@user.noreply.gitee.com> Date: Tue, 6 May 2025 16:41:30 +0800 Subject: [PATCH 2/3] ais-bench_workload -> huawei --- .../ais-bench_workload}/README.md | 0 .../ais-bench_workload}/build/build.sh | 0 .../ais-bench_workload}/build/download_and_build.sh | 0 ...\257\264\346\230\216\346\226\207\346\241\243.md" | 0 ...\257\264\346\230\216\346\226\207\346\241\243.md" | 0 ...\220\255\345\273\272\346\214\207\345\257\274.md" | 0 ...\236\204\345\273\272\346\225\231\347\250\213.md" | 0 ...05\245\351\227\250\346\214\207\345\257\274.docx" | Bin ...pore-arm\347\232\204\346\226\271\346\263\225.md" | 0 ...\256\211\345\205\250\345\243\260\346\230\216.md" | 0 .../ais-bench_workload}/img/faq1.png | Bin .../ais-bench_workload}/src/ais_utils_adapter.py | 0 .../src/common/calc_glm2_result.py | 0 .../src/common/calc_llm_result.py | 0 .../ais-bench_workload}/src/common/calc_power.sh | 0 .../src/common/calc_resourceinfo.sh | 0 .../ais-bench_workload}/src/common/calc_result.py | 0 .../src/common/cluster_common.sh | 0 .../src/common/cluster_common_2.0.sh | 0 .../ais-bench_workload}/src/common/common.sh | 0 .../ais-bench_workload}/src/common/log_util.sh | 0 .../src/common/modelarts_handler.py | 0 .../src/common/modelarts_handler_v2.py | 0 .../ais-bench_workload}/src/common/node_common.sh | 0 .../ais-bench_workload}/src/common/patch_common.sh | 0 .../src/common/sshpass_common.sh | 0 .../src/common/train_modelarts.py | 0 .../src/train/huawei/common/mindspore_env.sh | 0 .../src/train/huawei/common/tensorflow_env.sh | 0 .../src/train/huawei/train_mindspore_bert/build.sh | 0 .../huawei/train_mindspore_bert/config/config.sh | 0 .../train_mindspore_bert/config/modelarts_config.py | 0 .../config/modelarts_config.py.r1.3 | 0 .../train_mindspore_bert/modelarts_r1.10.patch | 0 .../train_mindspore_bert/modelarts_r1.3.patch | 0 .../train_mindspore_bert/modelarts_r1.5.patch | 0 .../train_mindspore_bert/modelarts_r1.7.patch | 0 .../train_mindspore_bert/modelarts_r1.8.patch | 0 .../train_mindspore_bert/modelarts_r1.9.patch | 0 .../train_mindspore_bert/modelarts_r2.0.patch | 0 .../train_mindspore_bert/modelarts_r2.1.patch | 0 .../train_mindspore_bert/modelarts_r2.2.patch | 0 .../src/train/huawei/train_mindspore_bert/patch.sh | 0 .../train/huawei/train_mindspore_bert/r1.10.patch | 0 .../train/huawei/train_mindspore_bert/r1.5.patch | 0 .../train/huawei/train_mindspore_bert/r1.6.patch | 0 .../train/huawei/train_mindspore_bert/r1.7.patch | 0 .../train/huawei/train_mindspore_bert/r1.8.patch | 0 .../train/huawei/train_mindspore_bert/r1.9.patch | 0 .../train/huawei/train_mindspore_bert/r2.0.patch | 0 .../train/huawei/train_mindspore_bert/r2.1.patch | 0 .../train/huawei/train_mindspore_bert/r2.2.patch | 0 .../train/huawei/train_mindspore_bert/r2.3.patch | 0 .../train_mindspore_bert/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_bert/scripts/modelarts_run.sh | 0 .../huawei/train_mindspore_bert/scripts/run_node.sh | 0 .../train/huawei/train_mindspore_deeplabv3/build.sh | 0 .../train_mindspore_deeplabv3/config/config.sh | 0 ...\275\277\347\224\250\350\257\264\346\230\216.md" | 0 .../train/huawei/train_mindspore_deeplabv3/patch.sh | 0 .../huawei/train_mindspore_deeplabv3/r1.9.patch | 0 .../train_mindspore_deeplabv3/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_deeplabv3/scripts/run_node.sh | 0 .../huawei/train_mindspore_deepspeech2/build.sh | 0 .../train_mindspore_deepspeech2/config/config.sh | 0 ...\275\277\347\224\250\350\257\264\346\230\216.md" | 0 .../huawei/train_mindspore_deepspeech2/patch.sh | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_deepspeech2/scripts/run_node.sh | 0 .../huawei/train_mindspore_faster_rcnn/build.sh | 0 .../train_mindspore_faster_rcnn/config/config.sh | 0 ...\275\277\347\224\250\350\257\264\346\230\216.md" | 0 .../huawei/train_mindspore_faster_rcnn/patch.sh | 0 .../huawei/train_mindspore_faster_rcnn/r1.9.patch | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_faster_rcnn/scripts/run_node.sh | 0 .../src/train/huawei/train_mindspore_glm2/README.md | 0 .../src/train/huawei/train_mindspore_glm2/build.sh | 0 .../huawei/train_mindspore_glm2/config/config.sh | 0 .../src/train/huawei/train_mindspore_glm2/patch.sh | 0 .../train/huawei/train_mindspore_glm2/r2.2.patch | 0 .../train_mindspore_glm2/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../scripts/run_glm2_6b_finetune.yaml | 0 .../scripts/run_glm2_6b_finetune_eval.yaml | 0 .../huawei/train_mindspore_glm2/scripts/run_node.sh | 0 .../train/huawei/train_mindspore_gnmt_v2/build.sh | 0 .../huawei/train_mindspore_gnmt_v2/config/config.sh | 0 .../train/huawei/train_mindspore_gnmt_v2/patch.sh | 0 .../train/huawei/train_mindspore_gnmt_v2/r1.9.patch | 0 .../train_mindspore_gnmt_v2/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_gnmt_v2/scripts/run_node.sh | 0 .../train/huawei/train_mindspore_llama/README.md | 0 .../src/train/huawei/train_mindspore_llama/build.sh | 0 .../huawei/train_mindspore_llama/config/config.sh | 0 .../src/train/huawei/train_mindspore_llama/patch.sh | 0 .../train/huawei/train_mindspore_llama/r2.2.patch | 0 .../train_mindspore_llama/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_llama/scripts/pre_conf_yaml.py | 0 .../train_mindspore_llama/scripts/run_node.sh | 0 .../huawei/train_mindspore_pangu_alpha/build.sh | 0 .../train_mindspore_pangu_alpha/config/config.sh | 0 .../huawei/train_mindspore_pangu_alpha/patch.sh | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_pangu_alpha/scripts/run_node.sh | 0 .../train/huawei/train_mindspore_resnet/build.sh | 0 .../huawei/train_mindspore_resnet/config/config.sh | 0 .../config/modelarts_config.py | 0 .../config/modelarts_config.py.r1.3 | 0 .../train_mindspore_resnet/modelarts_r1.10.patch | 0 .../train_mindspore_resnet/modelarts_r1.3.patch | 0 .../train_mindspore_resnet/modelarts_r1.5.patch | 0 .../train_mindspore_resnet/modelarts_r1.7.patch | 0 .../train_mindspore_resnet/modelarts_r1.8.patch | 0 .../train_mindspore_resnet/modelarts_r1.9.patch | 0 .../train_mindspore_resnet/modelarts_r2.0.patch | 0 .../train_mindspore_resnet/modelarts_r2.1.patch | 0 .../train_mindspore_resnet/modelarts_r2.2.patch | 0 .../train/huawei/train_mindspore_resnet/patch.sh | 0 .../train/huawei/train_mindspore_resnet/r1.10.patch | 0 .../train/huawei/train_mindspore_resnet/r1.5.patch | 0 .../train/huawei/train_mindspore_resnet/r1.6.patch | 0 .../train/huawei/train_mindspore_resnet/r1.7.patch | 0 .../train/huawei/train_mindspore_resnet/r1.8.patch | 0 .../train/huawei/train_mindspore_resnet/r1.9.patch | 0 .../train/huawei/train_mindspore_resnet/r2.0.patch | 0 .../train/huawei/train_mindspore_resnet/r2.1.patch | 0 .../train/huawei/train_mindspore_resnet/r2.2.patch | 0 .../train/huawei/train_mindspore_resnet/r2.3.patch | 0 .../train_mindspore_resnet/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_resnet/scripts/modelarts_run.sh | 0 .../train_mindspore_resnet/scripts/run_node.sh | 0 .../src/train/huawei/train_mindspore_ssd/build.sh | 0 .../huawei/train_mindspore_ssd/config/config.sh | 0 .../src/train/huawei/train_mindspore_ssd/patch.sh | 0 .../src/train/huawei/train_mindspore_ssd/r1.9.patch | 0 .../huawei/train_mindspore_ssd/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../huawei/train_mindspore_ssd/scripts/run_node.sh | 0 .../train/huawei/train_mindspore_widedeep/build.sh | 0 .../train_mindspore_widedeep/config/config.sh | 0 ...\275\277\347\224\250\350\257\264\346\230\216.md" | 0 .../train/huawei/train_mindspore_widedeep/patch.sh | 0 .../huawei/train_mindspore_widedeep/r1.5.patch | 0 .../huawei/train_mindspore_widedeep/r1.9.patch | 0 .../train_mindspore_widedeep/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_mindspore_widedeep/scripts/run_node.sh | 0 .../huawei/train_tensorflow_bert_base/build.sh | 0 .../train_tensorflow_bert_base/config/config.sh | 0 .../huawei/train_tensorflow_bert_base/patch.sh | 0 .../train_tensorflow_bert_base/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_tensorflow_bert_base/scripts/run_node.sh | 0 .../huawei/train_tensorflow_densenet121/build.sh | 0 .../train_tensorflow_densenet121/config/config.sh | 0 .../huawei/train_tensorflow_densenet121/patch.sh | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../scripts/run_node.sh | 0 .../huawei/train_tensorflow_mobilenetv2/build.sh | 0 .../train_tensorflow_mobilenetv2/config/config.sh | 0 .../huawei/train_tensorflow_mobilenetv2/patch.sh | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../scripts/run_node.sh | 0 .../huawei/train_tensorflow_nezha_large/build.sh | 0 .../train_tensorflow_nezha_large/config/config.sh | 0 .../train_tensorflow_nezha_large/master.patch | 0 .../huawei/train_tensorflow_nezha_large/patch.sh | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../scripts/run_node.sh | 0 .../huawei/train_tensorflow_resnet101/build.sh | 0 .../train_tensorflow_resnet101/config/config.sh | 0 .../huawei/train_tensorflow_resnet101/patch.sh | 0 .../train_tensorflow_resnet101/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_tensorflow_resnet101/scripts/run_node.sh | 0 .../huawei/train_tensorflow_resnet50/README.txt | 0 .../train/huawei/train_tensorflow_resnet50/build.sh | 0 .../train_tensorflow_resnet50/config/config.sh | 0 ...\275\277\347\224\250\350\257\264\346\230\216.md" | 0 .../huawei/train_tensorflow_resnet50/master.patch | 0 .../train/huawei/train_tensorflow_resnet50/patch.sh | 0 .../train_tensorflow_resnet50/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_tensorflow_resnet50/scripts/run_node.sh | 0 .../huawei/train_tensorflow_resnext50/build.sh | 0 .../train_tensorflow_resnext50/config/config.sh | 0 .../huawei/train_tensorflow_resnext50/patch.sh | 0 .../train_tensorflow_resnext50/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_tensorflow_resnext50/scripts/run_node.sh | 0 .../huawei/train_tensorflow_ssd_resnet34/build.sh | 0 .../train_tensorflow_ssd_resnet34/config/config.sh | 0 .../huawei/train_tensorflow_ssd_resnet34/patch.sh | 0 .../scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../scripts/run_node.sh | 0 .../train/huawei/train_tensorflow_vgg16/build.sh | 0 .../huawei/train_tensorflow_vgg16/config/config.sh | 0 .../train/huawei/train_tensorflow_vgg16/patch.sh | 0 .../train_tensorflow_vgg16/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_tensorflow_vgg16/scripts/run_node.sh | 0 .../train/huawei/train_tensorflow_yolov3/build.sh | 0 .../huawei/train_tensorflow_yolov3/config/config.sh | 0 .../huawei/train_tensorflow_yolov3/master.patch | 0 .../train/huawei/train_tensorflow_yolov3/patch.sh | 0 .../train_tensorflow_yolov3/scripts/benchmark.sh | 0 .../scripts/cluster_offline_run.sh | 0 .../train_tensorflow_yolov3/scripts/run_node.sh | 0 .../train/nvidia/train_tensorflow_bert/README.MD | 0 .../src/train/nvidia/train_tensorflow_bert/build.sh | 0 .../train_tensorflow_bert/config/config_pretrain.sh | 0 .../train/nvidia/train_tensorflow_bert/master.patch | 0 .../src/train/nvidia/train_tensorflow_bert/patch.sh | 0 .../train_tensorflow_bert/scripts/benchmark.sh | 0 .../nvidia/train_tensorflow_bert/scripts/run.sh | 0 .../train/nvidia/train_tensorflow_resnet/README.MD | 0 .../train/nvidia/train_tensorflow_resnet/build.sh | 0 .../config/config_imagenet2012.sh | 0 .../train/nvidia/train_tensorflow_resnet/patch.sh | 0 .../nvidia/train_tensorflow_resnet/r1.13.0.patch | 0 .../train_tensorflow_resnet/scripts/benchmark.sh | 0 .../nvidia/train_tensorflow_resnet/scripts/run.sh | 0 235 files changed, 0 insertions(+), 0 deletions(-) rename {ais-bench_workload => huawei/ais-bench_workload}/README.md (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/build/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/build/download_and_build.sh (100%) rename "ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" => "huawei/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" (100%) rename "ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" => "huawei/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" (100%) rename "ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" => "huawei/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" (100%) rename "ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" => "huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" (100%) rename "ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" => "huawei/ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" (100%) rename "ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" => "huawei/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" (100%) rename "ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" => "huawei/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/img/faq1.png (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/ais_utils_adapter.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/calc_glm2_result.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/calc_llm_result.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/calc_power.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/calc_resourceinfo.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/calc_result.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/cluster_common.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/cluster_common_2.0.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/common.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/log_util.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/modelarts_handler.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/modelarts_handler_v2.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/node_common.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/patch_common.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/sshpass_common.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/common/train_modelarts.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/common/mindspore_env.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/common/tensorflow_env.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/config/modelarts_config.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r1.10.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r1.5.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r1.6.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r1.7.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r1.8.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r2.0.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r2.1.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r2.2.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/r2.3.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_bert/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/config/config.sh (100%) rename "ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" => "huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deepspeech2/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deepspeech2/config/config.sh (100%) rename "ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" => "huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deepspeech2/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh (100%) rename "ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" => "huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/README.md (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/r2.2.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/README.md (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/r2.2.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_llama/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_pangu_alpha/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_pangu_alpha/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r1.10.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r1.5.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r1.6.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r1.7.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r1.8.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r2.0.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r2.1.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r2.2.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/r2.3.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/config/config.sh (100%) rename "ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" => "huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/r1.5.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/r1.9.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_bert_base/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_bert_base/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_bert_base/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_densenet121/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_densenet121/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_densenet121/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_mobilenetv2/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/master.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet101/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet101/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet101/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/README.txt (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/config/config.sh (100%) rename "ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" => "huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/master.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnext50/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnext50/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnext50/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_vgg16/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_vgg16/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_vgg16/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/config/config.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/master.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/README.MD (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/master.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_bert/scripts/run.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/README.MD (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/build.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/patch.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh (100%) rename {ais-bench_workload => huawei/ais-bench_workload}/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh (100%) diff --git a/ais-bench_workload/README.md b/huawei/ais-bench_workload/README.md similarity index 100% rename from ais-bench_workload/README.md rename to huawei/ais-bench_workload/README.md diff --git a/ais-bench_workload/build/build.sh b/huawei/ais-bench_workload/build/build.sh similarity index 100% rename from ais-bench_workload/build/build.sh rename to huawei/ais-bench_workload/build/build.sh diff --git a/ais-bench_workload/build/download_and_build.sh b/huawei/ais-bench_workload/build/download_and_build.sh similarity index 100% rename from ais-bench_workload/build/download_and_build.sh rename to huawei/ais-bench_workload/build/download_and_build.sh diff --git "a/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" "b/huawei/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" similarity index 100% rename from "ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" rename to "huawei/ais-bench_workload/doc/ais-bench_workload_train_modelarts\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" diff --git "a/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" "b/huawei/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" similarity index 100% rename from "ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" rename to "huawei/ais-bench_workload/doc/ais-bench_workload_train_offline\347\272\277\344\270\213\350\256\255\347\273\203\350\257\264\346\230\216\346\226\207\346\241\243.md" diff --git "a/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" "b/huawei/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" similarity index 100% rename from "ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" rename to "huawei/ais-bench_workload/doc/ais-bench_workload\346\216\250\347\220\206\346\211\247\350\241\214\345\256\271\345\231\250\347\216\257\345\242\203\346\220\255\345\273\272\346\214\207\345\257\274.md" diff --git "a/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" "b/huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" similarity index 100% rename from "ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" rename to "huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" diff --git "a/ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" "b/huawei/ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" similarity index 100% rename from "ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" rename to "huawei/ais-bench_workload/doc/modelarts_notebook\344\275\277\347\224\250\345\205\245\351\227\250\346\214\207\345\257\274.docx" diff --git "a/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" "b/huawei/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" similarity index 100% rename from "ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" rename to "huawei/ais-bench_workload/doc/\345\210\266\344\275\234\345\217\257ssh\347\231\273\345\275\225\351\225\234\345\203\217ascend-mindspore-arm\347\232\204\346\226\271\346\263\225.md" diff --git "a/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" "b/huawei/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" similarity index 100% rename from "ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" rename to "huawei/ais-bench_workload/doc/\345\256\211\345\205\250\345\243\260\346\230\216.md" diff --git a/ais-bench_workload/img/faq1.png b/huawei/ais-bench_workload/img/faq1.png similarity index 100% rename from ais-bench_workload/img/faq1.png rename to huawei/ais-bench_workload/img/faq1.png diff --git a/ais-bench_workload/src/ais_utils_adapter.py b/huawei/ais-bench_workload/src/ais_utils_adapter.py similarity index 100% rename from ais-bench_workload/src/ais_utils_adapter.py rename to huawei/ais-bench_workload/src/ais_utils_adapter.py diff --git a/ais-bench_workload/src/common/calc_glm2_result.py b/huawei/ais-bench_workload/src/common/calc_glm2_result.py similarity index 100% rename from ais-bench_workload/src/common/calc_glm2_result.py rename to huawei/ais-bench_workload/src/common/calc_glm2_result.py diff --git a/ais-bench_workload/src/common/calc_llm_result.py b/huawei/ais-bench_workload/src/common/calc_llm_result.py similarity index 100% rename from ais-bench_workload/src/common/calc_llm_result.py rename to huawei/ais-bench_workload/src/common/calc_llm_result.py diff --git a/ais-bench_workload/src/common/calc_power.sh b/huawei/ais-bench_workload/src/common/calc_power.sh similarity index 100% rename from ais-bench_workload/src/common/calc_power.sh rename to huawei/ais-bench_workload/src/common/calc_power.sh diff --git a/ais-bench_workload/src/common/calc_resourceinfo.sh b/huawei/ais-bench_workload/src/common/calc_resourceinfo.sh similarity index 100% rename from ais-bench_workload/src/common/calc_resourceinfo.sh rename to huawei/ais-bench_workload/src/common/calc_resourceinfo.sh diff --git a/ais-bench_workload/src/common/calc_result.py b/huawei/ais-bench_workload/src/common/calc_result.py similarity index 100% rename from ais-bench_workload/src/common/calc_result.py rename to huawei/ais-bench_workload/src/common/calc_result.py diff --git a/ais-bench_workload/src/common/cluster_common.sh b/huawei/ais-bench_workload/src/common/cluster_common.sh similarity index 100% rename from ais-bench_workload/src/common/cluster_common.sh rename to huawei/ais-bench_workload/src/common/cluster_common.sh diff --git a/ais-bench_workload/src/common/cluster_common_2.0.sh b/huawei/ais-bench_workload/src/common/cluster_common_2.0.sh similarity index 100% rename from ais-bench_workload/src/common/cluster_common_2.0.sh rename to huawei/ais-bench_workload/src/common/cluster_common_2.0.sh diff --git a/ais-bench_workload/src/common/common.sh b/huawei/ais-bench_workload/src/common/common.sh similarity index 100% rename from ais-bench_workload/src/common/common.sh rename to huawei/ais-bench_workload/src/common/common.sh diff --git a/ais-bench_workload/src/common/log_util.sh b/huawei/ais-bench_workload/src/common/log_util.sh similarity index 100% rename from ais-bench_workload/src/common/log_util.sh rename to huawei/ais-bench_workload/src/common/log_util.sh diff --git a/ais-bench_workload/src/common/modelarts_handler.py b/huawei/ais-bench_workload/src/common/modelarts_handler.py similarity index 100% rename from ais-bench_workload/src/common/modelarts_handler.py rename to huawei/ais-bench_workload/src/common/modelarts_handler.py diff --git a/ais-bench_workload/src/common/modelarts_handler_v2.py b/huawei/ais-bench_workload/src/common/modelarts_handler_v2.py similarity index 100% rename from ais-bench_workload/src/common/modelarts_handler_v2.py rename to huawei/ais-bench_workload/src/common/modelarts_handler_v2.py diff --git a/ais-bench_workload/src/common/node_common.sh b/huawei/ais-bench_workload/src/common/node_common.sh similarity index 100% rename from ais-bench_workload/src/common/node_common.sh rename to huawei/ais-bench_workload/src/common/node_common.sh diff --git a/ais-bench_workload/src/common/patch_common.sh b/huawei/ais-bench_workload/src/common/patch_common.sh similarity index 100% rename from ais-bench_workload/src/common/patch_common.sh rename to huawei/ais-bench_workload/src/common/patch_common.sh diff --git a/ais-bench_workload/src/common/sshpass_common.sh b/huawei/ais-bench_workload/src/common/sshpass_common.sh similarity index 100% rename from ais-bench_workload/src/common/sshpass_common.sh rename to huawei/ais-bench_workload/src/common/sshpass_common.sh diff --git a/ais-bench_workload/src/common/train_modelarts.py b/huawei/ais-bench_workload/src/common/train_modelarts.py similarity index 100% rename from ais-bench_workload/src/common/train_modelarts.py rename to huawei/ais-bench_workload/src/common/train_modelarts.py diff --git a/ais-bench_workload/src/train/huawei/common/mindspore_env.sh b/huawei/ais-bench_workload/src/train/huawei/common/mindspore_env.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/common/mindspore_env.sh rename to huawei/ais-bench_workload/src/train/huawei/common/mindspore_env.sh diff --git a/ais-bench_workload/src/train/huawei/common/tensorflow_env.sh b/huawei/ais-bench_workload/src/train/huawei/common/tensorflow_env.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/common/tensorflow_env.sh rename to huawei/ais-bench_workload/src/train/huawei/common/tensorflow_env.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/config/modelarts_config.py.r1.3 diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.10.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.3.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.5.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.7.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.8.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.0.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.1.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/modelarts_r2.2.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.10.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.5.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.6.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.7.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.8.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.0.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.1.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.2.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/r2.3.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/modelarts_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_bert/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/config/config.sh diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" similarity index 100% rename from "ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" rename to "huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deeplabv3/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/config/config.sh diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" similarity index 100% rename from "ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" rename to "huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/doc/ais-bench+mindspore-deepspeechv2\344\275\277\347\224\250\350\257\264\346\230\216.md" diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_deepspeech2/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/config/config.sh diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" similarity index 100% rename from "ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" rename to "huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/doc/ais-bench+mindspore-deeplabv3\344\275\277\347\224\250\350\257\264\346\230\216.md" diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_faster_rcnn/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/README.md diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/r2.2.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune.yaml diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_glm2_6b_finetune_eval.yaml diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_glm2/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_gnmt_v2/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/README.md diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/r2.2.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/pre_conf_yaml.py diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_llama/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_pangu_alpha/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/config/modelarts_config.py.r1.3 diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.10.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.3.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.5.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.7.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.8.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.0.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.1.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/modelarts_r2.2.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.10.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.5.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.6.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.7.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.8.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.0.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.1.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.2.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/r2.3.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/modelarts_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_resnet/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_ssd/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/config/config.sh diff --git "a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" similarity index 100% rename from "ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" rename to "huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/doc/ais-bench+mindspore-widedeep\344\275\277\347\224\250\350\257\264\346\230\216.md" diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.5.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/r1.9.patch diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_mindspore_widedeep/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_bert_base/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_densenet121/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_mobilenetv2/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/master.patch diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_nezha_large/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet101/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/README.txt diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/config/config.sh diff --git "a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" similarity index 100% rename from "ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" rename to "huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/doc/ais-bench+tensorflow-resnet\344\275\277\347\224\250\350\257\264\346\230\216.md" diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/master.patch diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnet50/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_resnext50/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_ssd_resnet34/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_vgg16/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/build.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/config/config.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/master.patch diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/patch.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/cluster_offline_run.sh diff --git a/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh b/huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh similarity index 100% rename from ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh rename to huawei/ais-bench_workload/src/train/huawei/train_tensorflow_yolov3/scripts/run_node.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/README.MD diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/build.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/config/config_pretrain.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/master.patch diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/patch.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_bert/scripts/run.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/README.MD diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/build.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/config/config_imagenet2012.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/patch.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/r1.13.0.patch diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/benchmark.sh diff --git a/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh b/huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh similarity index 100% rename from ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh rename to huawei/ais-bench_workload/src/train/nvidia/train_tensorflow_resnet/scripts/run.sh -- Gitee From dec50c4a8019a4e6152ae8873821b476cf159028 Mon Sep 17 00:00:00 2001 From: zjk <15778488+zjks@user.noreply.gitee.com> Date: Tue, 6 May 2025 17:03:27 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=8E=A8=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...rkload\346\236\204\345\273\272\346\225\231\347\250\213.md" | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git "a/huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" "b/huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" index 4c3a4e1..f36cb61 100644 --- "a/huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" +++ "b/huawei/ais-bench_workload/doc/ais-bench_workload\346\236\204\345\273\272\346\225\231\347\250\213.md" @@ -5,7 +5,7 @@ ais-bench标准化性能测试软件，又称AI Server Benchmark软件，是根 ais-bench_workload是ais-bench提供用于构建ais-bench性能测试软件包并进行测试的负载工具。 -本文档主要介绍如何**搭建ais-bench_workload构建环境**并在该环境下**构建推理和训练场景的ais-bench性能测试软件包。** +本文档主要介绍如何**搭建ais-bench_workload构建环境**并在该环境下**构建训练场景的ais-bench性能测试软件包。** ais-bench_workload支持快速构建和标准构建。其中快速构建仅支持mindspore框架的bert和resnet两个典型模型的性能测试软件包构建。通过构建脚本扩展，标准构建能支持当前提供的所有模型的性能测试软件包构建。 ## 1. 搭建ais-bench_workload构建环境 @@ -148,7 +148,7 @@ Windows系统需预先安装git软件。在ais-bench_workload工作目录下鼠 #### 2.2.1 简介 -标准构建是通过构建指令指定具体模型执行构建性能测试软件包的操作，相比快速构建，指令更丰富，适用模型更多且涵盖推理和训练场景。 +标准构建是通过构建指令指定具体模型执行构建性能测试软件包的操作，相比快速构建，指令更丰富。 #### 2.2 约束 -- Gitee

_4?kCDGzKmW|SM(#w8*$j-j)mk`k6WR`%rzHfT=48O!>%T?8m$IEt_g z6gXI_sy%ldSA`NQPAVT4m9VKAyP{E3iwh`#JmKz0m`ZQ}iFlwG6hZQ~x3*cmCMUXz zV3@eKtei;RrqKmqXh~vtQbCGHqgtZM@YUn57P8%LTR(7pq@vx#>S`!FqXz^+Z1|^c zc`NpPw$!<#jv)aoId@>$Ct{NAN_*ov5ZzN?7`rL4s#c4J-SGVzcptwv?z|7>hX|Ye$ULpQSLvy%saNSvySQab7+Dd8 z$~PWj75S_zB$Hk0+RQ48Hz5(ls9tdzSiuJsNg4EewS3w~n)CQc@XbC9fQAXZCx zP6V9SZoG&>C_Rsq*0p*87el6y`AY9>=&s@Spmo!ueds)6JUEVrW;Y`*sWRQStX zPwY6jNIhQ6V_2+xW>6~xu|UnBN&FhtxOVT1dO1#JO2nkJDs&R0|HzDh7b`N;@5IR< zA;HfUfOk!?4$)$}qrrXDL|L0Po@Fvs(=nuk&}uAVVT_bRFuE~V0Y!<}HnyQVh1!}G zAiCLZYd8R$k#FMIG9OOlAucJ9n2&}iawMF&Odd>OcZs8xO)Y5K7A57I!$>jEq7k1{ zeKj@@LlqGelLRJckcq|xQHDwsl7-4sN)Ya^ERm1#a1e~RY0buUT`*Uuu4Vj?6s($0 zP}4q2z@Q-cF@ZkUs3C%a-ZptK78oE%`o5odxjvdCblfyIr#Wjp<`D3f-sG>wdLP(pX(yR5G z23Xvxsq1mog;6(T9cJqsr=#nS9^>WQ`2_Xk9Z9BI~CSe zQzsY0L=FFhDg!0tt?x^WjefOqitB0o z^V_&ZF=~6J@M|G8tbb{-X+fY2qVr9B0b+f>^#E#&XDwn24@QnM*XJSAylPGchWst- zkSWZW`6UaixfG+Xq22VFlFOIW7Kc8}#9^}uNjurROhtKhqGVh;P*6~&{`l9 zK92VK5*mk&7G^~hqji4$(LJgXE`e)A8veh?_GwV3trBm>B;HYcRFE}^6J1z<5&R%}DBmvVBK zFJpCtSG#KPRZ$b6G^@J=cl%mfK%&6VBC9AsDGiQ{A;Nu&8U_mq*id?Rc2PFb9G0Qv z1Vxp&rfr>ZazrqZlVI$`Q!gn<5$%Q)Z|fBhcpfR^@Gh}Ct<3Ns#jY+}Jlw)YMx`Fu z#5uW$rzoXiQrV}Vz0bokH(QolTTUfcW@r(-q&`E6kb#@ZxDOea$JF3?Gt3OK!(+q# zBW`y4_T)K5{!-TKsMqhZI6WhblgN$4wKMyowAX)?rxCOOUCjvUUAIF&ga zq0t*d3q1#uVHjFQq4&gWIMzf1Su{RMmGLZ|FZkD_KBMX%w-aa`VY zDsW#*)?@fN4ZmKq&lY6AU39uud2JoAQ?Rgc_bW79qSW#=7U!@g7w}Es#XDhe&zDSvaSO+7oq(jv5YM+nlyo}bk+1csu4kSI>;_*`r`HhvlCm#luR6tvPX zYTD~k78^y$%^$TJ5?qbXLj7%R$^22q;UmdOD*iYzbD|^*JR3;9yx!ZWHk~xcoHDNg zQ$5I?9xIaUQvq0mJO5GA^i}#vyQdJX#cg{N;`6J2_?pZnGR2cI zt+^a}Q^u>JRj$ihp)psxCry?1o?d<7E~4rDDN&Wc8<;EQCFX=lDmanYanT~5QHWKz^Fp?ICKI54 zOr;@klWk4MN@xq@&za9rQ4*y`h$t}wj}XbpB2hvUB#Xos-4zM9d-mv`4(1}uR{QU= zVd4HoDRSjbT`pzE_2KezqwUY-@=5iTy>tc!n#jJbN%itU9Y&TT11xfe@l;u}wM94g zr#VFQ+Jhvrt&CeGa}TGbW{}t0Lt;2UpW4FvG$3sL?Zl+2_CYSI6c0Dk*227_=`L8f zJ-{lPUgssytTAntz+B{XJ#E?$GsNJ}+x4zBi3{#)imX|kCLLg8)Z^}y|1#6A1#C2r z??QAQJc`B}a1cF{*0dpqQ(&brV>`#13&j}+>XJw*GAAn3l#As> z^vA(<{g^iPAj;gFp)UApVt4`0P~(|Np1I4JHE%IXD%cxu5mH!(?LwT2E;{*RUNSC0 z+5{C|wL-BU1xQMgCpcj8K(j-cqE7oB+JOBKWym=R!&#F}G;Y#8u8=ZM0VAADRAK6f zl{xP7P~BG(@QG-%5 z+D;s#F0ZZbzd;`Oa}C462V4()b?@e((Qdg|-!8qz_3nDy!~4!|eWa!_O9b`q%PXqp zTE{>(?#dnJzfC`N;NY!S*HNj)L}-8Q&kA`=)AC(9O4maO!e6OQG~k=qj#^m#vyVV? zt-30CX(*Wq@K00;zF^WSVo8Zmx9xgO_nBVX%xt=3;o6mgYIjm1s@fIX;?#VLC4)v8 znel4c?U8CX?Zd3p8{Ibo8S@>;skgN zF__AgZ*Df(saN=1u1VXRWqm7iQ;e*@^q2j^VX#hB{e^v*Pn?c6_&$_x^ zWQ?}1YWb`sX4;(j3^as>?*0LubkP)N(Th8fd8_8qQQRR}A8fILNy$#C`K|9oKS6_c zbG?N6U{k3xgSO-M+JgB-8fLt`r>xp}TDqX@>C3Mh^dlUY&T6`s zUkTxug;dZAz;8f|bfK=7)7wY}2WAu_tB!`kltamYA-m50KBRxnrUSs;Y<-df--mF( z;Oy{J&WtlpaD73(LVxih^k8SXrWFJ7Iq;>45E_oCjh~;MR0s~PjlWus%KNhI4r~?? z(NAAsiM&a+vjUL^o5UKullY6sfK#bPjhZr23|#cBKao*kE;crmbiP1A3&xN-tT!NX zebjO?vN3b>b5IcU&APfDy6Tq4BqD_~6!y6p!%M&6Mih-*Zmf(J0WXdP6gCkg3YU_Qfq{b>ON^EYB3n)Pwzh&W`XD1b*62^jjf* z0F+Q@5HqT1l-vlVx@y^^JZg#j&t&eWci+Sy1dv6l?1<(M+`YyPLJS&gbhS2yy|z!F zE<|ctYFHWOV~^^GZgTOj(&}g%bbK zr0a9U7y8#d;*OXQl4s8@l)fduu;f`l9Bl}Yqy{`dm|5!ZigCp9fA9Y*_3PaA@|xL>7P)h}K~3=IHK zwsLdQKcAF(AO2f7m2Is~g$^Hb%r$U@9vmz?NpwiyMwTmWR#X&r16?$nXEs>2r&LwZ zUx3_|<5~5$x6&w;OZ(zi;9#Y_7ryM3ZZ;b~x2+3S37<-1!)950U_j~mzu9_pNu29J zQDvjW3O4nM_`Z_24v>V_mr9O;rbA5^=>z~XS_TqX*g`)8fk80K-0PfJclqYj_Sx?V z%bi})^3noT649d|>V_>kN1WR&b55y@B`X*BkRF;`QptbEC6v7fEOV!ZbYx@zV$G&L zJLWiI2w&FD`Qc(;YW9-eYRX%(M55)pV#p>O{ zCJI%ybtNgzvUNnS>OXXHrmj)y2h9`u>h39n5IpTF{5sDE4}S}nHQ8@B>r;vr;1HUm zU2(Cex!CEcl;uH3EGJ=f*lVeJb=#vxSZ6;!ob32GoGhizEZf*|>* z;;>(DnncB3ZLKJ&U!QTiQB*p#I!7Vs@V6AK1}AD?#7Moo?QmcEJDE=mkwN)tX$yg+ zqK~ulHJ2rwaXZsIe+k1P^t^rOlh&}T*=g7E;>O)`?A%iU3GYf)5y*gnZ;yZ zy0b4zgh8Pf(c2!f8)$@S)_wLhvlJ!ab~{+9TOWShF6+g%V5Ft+SY41Mww}s_`L(wQ zl-OM+>R9`tsX(gT?1sqNws9fLl7^~D3zM_)Twqz1N6?4-w1yjty!6ediwn&{u-0*M zfhFPgdr7;7agYDe;NZ1G(yult7X=wiJ9K{Wql0iuEw4lDksl}Pf#RV^Ox@?Mb-;vr zVaGRys%Hm>->sHoul8)vgtNHn`;)Szv}kn9*M7c9M{#D)W3CuVdkbC&8QhdDeWyqF zLLB%LS6f);m6>I9qlVK>@G{oYspzY#>Z=DMuRr#xr9pRf zIRdf1JFWY+#*_Wg@$3W>QMPk;@bs4DP40|$nfWJSaB?&b)KuJDWeH&@Cm_e>6*fRd z?&*0Z6m%O*#l}vE8HI>Yw{Y@!@gQPdn6_IlVhbdY_~ULVaRKG)I5N>lzsuKFK4O8> zaFB=VCbMBH73%Y1Q=u~Bf!B0h)dmhhKVt^h0dG<-?|oFK#LH}S%tTHthK>w$jCcY( zfjXqgR&F8QnP6l-kvtbW<4%SlI|SpCkZ3pSsbPlI!Ovggrq7%|_e|h%{+zXtxXPvu zFvNX#8Tl*q?N`P;zHf>QoE9?^2k*b=bFTAUo7!UT#>Q%!P(3rPhn0{i=R=*3Ze&vR1;;dePzLJ2zP`z@`dI|FK<&cp{ivM2HyjW z4DO4$d_D9f(IaT79bPJBE;1@z14fhr>GF&+Rh1g?N}o+Et){jN+woYx-(f z24sCE<0{4($rMp33LIZ8xh}c(_OwRssRurl#D&#O?Jvm@Sz&4-A=XKk z??A&V5eQ5j@oL?M25#mJ$G6e6rdoD`_`*wiypj#&h#<(PW$^(hku}!+`l?y!*oVgP;gWxtHH&$8k|^9= z12Ehl2Ce`Z`9#}ucW)uPL3lC>**N4OqEyqEb!@*Js>c?67s|dP4dV^(bi5Jh*h*zN`AH0<;1TIwN8t(FD+|M zwjUzmyklSZea0E6oMqe-OUt31UL;E(P$k_dFedzToqp*Q$k+wfmnWCXk%mO$RxWGW zpts5vx1Hapl&=!>>c8y=&E#dScCw_5xy)2wud;Dt7QfhBdePx7C6humO;^{dI6rTz zkb$9CG3&+ZP*AlzA7=Mgt;H05L^*zXA=3O zCAGx_{gsF%U?|n`{GMJm4%4PcZ!Zq*%kP8H+R@AuSl^%=8b~~bs$dsLsCc(U3RQx2 z-Wpe1k-HaDgrC@C-Qm|Z-Ei57k$ofXvKpCYxs4txlhi--nC^6C1LV< zuQF(Ac)8Qm+E}7i`*`qpkjQ<6Gn?v>he`68Ke|f zmhp*BSu}o*Zg(?R=mYDnU}$sOdpu4qgyIVl0*0U$tEk5HuHx!qg;Z25T{0G}Puou8 zD0+mipMe-We`{PEHZ}p$qDFfZ4fP=0t2}pMxeYzV2xInUmpMB_1@4eXR;iwbg1f>b z$6|`yWa9{kWJdkLD(q631?iIzXx`81<)xH!Z6}xvwM{&N-IQoLlYj9yvlfDwQWZJ( zbl^BKEOX&|SJSWY5ZcL^1{;k9eV$I``;!5m*>q*D9RAnoRTBh{<}=>T+FB{*$DN;G z1K>iu&moj48M3sr(jD#Z&xD+345RiorCr?Dh@pDP@}e_DUiZc|loc#M{nEw1mjDh4 zGDoASZj(NA{(EFA)+2O@2YENweF*U z8GD2LHjZxJ^gq{kA(3D?C(_X|7{j2v*|2ZMsEL1nsf5ewt0ildU|=?&%r@Jtm#q-y zO7ZTw`6W|7CF&pjK3qlVNFBK}3)S?TOV@KVSGSB<*;z zbi~lT(t-}xrYCI?eKD1bC15PxGFNghzJUKebb~R9UT|of!l*J;5FCV!81=Vm#y!o$ zc3t(_Uwd6$wxyb?^dAxDt=l5@lq?254u?(>(Y~r}ZDz?F3^K%^w)16ZegJ?VYU=2^ zOhlWQKQFGF7z-(7M8u{~q;Qg$p3?q4o=gnU6;(OSMdvE(v4gGwF=(6#vY6maK()5z z_t5vtSl`RiN@2O@#z3x)PITDS$3c|4UIskcVbkH7r$1)+sYRMeGHj8yeItfl={Y`$ z&Prwdgnl5iotK47lZf2>^87UC_mZV8|PnFjMc`!&U+42MJ*Yl ziVob87zank>rt%=ux=&9%y=5J=@VNGu4-ICl;%emKzh|W+*3elOOcD^U#X26B3T{_<;b^e)`b=lWBKVk`@V z_410{g6?<}t_<mwiWb^{?N;8xVimjOLua@rHvw=RIzek%jR{%V^jM$}e)!bvpJ z*Q?IT*3xvy=Q;c~JzSqg77ZYo75m3C%5|Z7#D>7!7hPv=*3ht?0LS4Ju6esBw5KfI z&1iQ}0cmO}^&El@%Kh+x7xmfIu?o^YlfM%#drD~uIP!ZS=Hp!B;_T9XJJ>n9IN}y=qsZ@ zr@qh*D~D3;Az8%yYy3s=jDPZp|Na!&mZ$3!LYZKo%)7{%?YXrVr8rE6k(PK#{Sjv+ zGXoP!o4a4DdxHxprot`ezqdqq5P+`X-R4VS9#d)?oO0i*Au0W|Hz=&HU5;Oy@DRT^ z`4r94Q%W;MA8>+VZ1R1_9p51#4mC^?&xpr33SFj=Gdo(OYVo~?#Q^cK5&sk7{oYnu zm}(k-%o0CTJ6g*o1qu+Nxu><>GV=j>FT|q( z19xOpgG>?QMv0{0R8$Jrq^e>+N>|BBdW32VL`87j>MKxfes||xN(#%ReT}n|r9ozb zpJHMYLe^zJzkf3&4;z@Hn?VCt(v16$01~FtA#=Xg8(RoK+60*x&0MO=jS3KC*P{1o z8b7H`o1&Q0-#(Khl!A*thMwpVJusDrGrC zyQ2Q8m<7E_hbY2)$LF}Rrw~76WZhuXS*w*>9DS(T^8ty-%3<@N=Smq|XS2SK&g5fH zb}M7*XYonk_q?p`T5Gl7&&!|GY-SSGS^~HUOHBx_wh&bijyN-Kky|9-hn2co`}K^$ zx=j@oEw+{Sk6DCwA#m0iP7mYMiBckomQt{c9u#GoUd-f7&4)4A@7R+FZlRJoi36k= zwpxF?j79i~`zR@w{!r2^`^%5S#yrtYD+g*bzP~X-N@f?;5)0V3{T=3~iL7WgKGU_= z2%}0wm^ySO@E#Txy^M&6voGGu54+d0o8Cvdg>9i;Sn$J?EYCALoHn=8mH+JIFjT$U zBDf4_)LiMDE-Y+>WUXqc(N+HyGxzxpq7_T^@O-T2d#!X6^(+ZuO6VSS<@I#9 zku#HCbQHg&!r$GtVj+ZN$yLk}_vCFspT2~d^^;&S3ZMg9w`#;vOohfE{e>8rVq2B-@v+z{%lw8A$_ytT8z>0Emc-lS`q13)j>AAq5RAu!!iC zgJTy?NW;J8Y&+Sk9WEL*sp-KJn}sb(Xxq3#o<#>B7q~3QLPAT`1Mf`=Fv3BNqk?8Z zW|Je6iVG;97Dfd}L7|&aB)7{M;zyxLnY_9FY$6L5l`5u|PVs<=hnE{mpc-S5Vn&An zok?UPyU!A3ffpl{+BH?KQr$E)@9)5#KrZ-mkU1?yIs@~^1V}%35Pp)^ z@xBs^6hHSnhF}vVWD{`oDMfM;7+zAG3{n_S;AT$UF3*#vK1hRuE zS9k7L{#M+=+h^oH+5%>U)L->?AHQJaiFyFBH0g*b+}-^H{fmj=KI!X^M8(8>?K3B3IRGtjjLOiZd~+k?j3saky}G{sSC#L!wAnGF5I@E`;EBK zbYouGh?0S(*nl|la1{r1Ickgu2TtP_EyoGY{{^_4FYHDwaY8|YMU^HeTq6oza$*z6 zr4|`cQIbb`%X%wEqng#9li}rqrho{s$}{ zWpD?$);=h-L z`Cme*W8U_dn+!V8bggqUiw6hce)`Qa_g^r%c1Mg3(%ISFLXQsa&|^D&2lBY_ z|MPmGMm>?A?|Va7`Nc=_mFJ@Hpau{LrER8kcCLNF~IXrkiR3;!MdS zK*f~O;VCCH2;WSxwet-jgovdgj4Xl`*(Vz4M3952r13{cz|rNT62N7?g8<3OeN2;1 zw#5G(nR|XoPM~>i7aA=tb}O3oE1k|!R?2=oSkT`Ba;XB*ghKHOc?uP4jTTTr()7;R zIW=wF*6ZEdqC&RJLj+;u5#d}gzlREx1peO&|($^d`*B86*`o5JR&S~ zWXUQy%0#f%^?n_r3Spun{eNF|mC`8PdEJ7}@8D(4fqeX4Sj#egE-y1JgKuYBYj<3x zgjIxWe}Of}ghg(FO$aapr(IRt&|y%d%m69k*~W#%%^eY``)gn2e7O;$L$ByZNTNyu zgNB{`w<0$fG)Z9VW$*O=JGm&+7#gtau-g6FbUZ#0k;239|2{uIwPZ@? zXNjM)gW;h_Yz>Wjy~i0J@C-onDQo%s@%;!ZVv_-&2@ftKy%}yIop0Er{Sz6;CCe1g z>+kdr$?$D^_6*oHY|vFrx$<1^@ba>d{2>MY#9A6Ed*klo^}O+tMGNUR?3sJ@W2)K- zCQ7K+<8Ims6ul!jhN}LVyIL+h2RYFg-V~n!MLMyC0@v z<#@jSybT}WP(7Y+$#EhaV7l0@pX932uUWEK`~)LIcw8D1gD9XP;Pa#4;GoPr-0?a_ z7B1~gu27|s6(Q=A__K99q%`HRt@JW{ z(#zu8DHr~B~SEEJ{)aG0)K-f0NN;j9^RUL zk=Imd`~I*;Z?hOlpBrvMa7GKd6p569Q%ljUqF0bGf1k~B1X%=MU-*@RFo*5^`RPW_YuucGY(QWno#rj1{#_|R@J<^Vrfuthl~Ra45Dy>1pkJMt-_ z)b(pEsl>fzq~qzeONI)KR)WvuP3kn(q;K9*xB#IVbNcaGc+hxFrD3%akn zb8q$LpA2-6yxX3E#H}{$`^OjT=R*@+dk(5eh4aQ703ZcnpmXY^S9XCID?5w+lZD&! zX0)uSLI>YXM@?*OaIunn?9kLM)F=fqQhWOHV-OwVh|wLINOi_kQBAn*C^Q5`AS#1j zQJ&U`=^sePWzbD(;fZ1UU)k*7JEP5(nSsmfX;V&rBWB zC$N{T$Z?hDi9)V!iLu|tnDlNx)bnM<>dPz31pOTp1KSo)P8H1?G!X9#v6G2n6gJcW>2nZvUjL@_#A2l#@Rqqll8a_WycV z!J-uIP4eA`GO(z%1mw#add*~pL4pZSOov_DPnt^#qp}cq8*t5XO|`E0?d>6mVLkC4Qv?(J`-tJ{r9f?d<_x`Bt~w~Ofa$x3%!v1|*c&zZTvX3ra=Uu~5tqiH{- z!RulCbXg$1fsd8>#J|^E^NMcmzp;<_-tQqh__~I=NWPEO6-q0yUzdu@!A1|L&8BcIe}TOzUcwxf|z zdS0$OuS^HLZ-dj)4Gvb%=Of~vDdkfJKDE&>2%S%FbLr@F&HHoN^mZwv%J~5csY!S% z)b8K^mIwvfp5jZ`*Z96~7Fv$UvgjOune_LC4&GD}o9mTJubStZv$ zCg;m?0KY-G+4-?sHJ!2&IDl9y$|KDGaXeu2-S2x$TV+@PXvrc`%vJ7l`3X!|3Z+0v zxayeNj+<6M z&yhwbgLEV8mv5;x_U2Te0230{W!I;st`c67Hoy03$}1fd{_l!h#Eh>A4Rmt-8mh*`6Pf&`C)kcKbgFGVxd*7xSCmz_uDDiM z73z})O~Dh=%`C~o3%m1*6q&jC(T+_4^9iYx^e@=LeWNt$eJntgN&&f})?Ry-I$~_F zw|UXPyJw7O8Joq-^gXHXx5#u8#5g%+Suz`u)RRYPrt`Y$TWA(Lp(F~7lLvQ&+@CV0 zl`?6|giS3WF>o%uc>SnCh@8uFE!`W_8EY5XNP#|`tBI_IP~p8H!5gIRk0kA?Ga_ti zT51|%S681#7HhlIRN(&_V6#(5&!kGjagrkj5>^yrMk_T-nPc<*X4xz^{30OP|1OWiz z;8h3(y{NjqDrn)hoS8;MOc`1&S`UKQ1dGt{J#V_i(@q0cO7?i#ZepOiYZ2+-NAH|2 zb9Ur-yP5v{@d;^)P)m?3W#RZj*?LYa^=Cl~{{s}F1BiK9Jqq}gHR-#bBA=)gzVj~h z_3|JTmu&S-@A2?tUVll<;IuhBY_U8p->G~196IlKm-kHKZJAbC>1%+(0ufUiMGsMOxW8K(5If&{E0R5&N~rT{fw!P{5X$AvGvi!FY# z9V)Q$@Ps@YD3(lDno10>lpP~JR^FDsY>ixsd4?Aj68U93RsL=;ZMi#6&K#d1GPxd|6*cM`-z?&cN6*CLp1V|Ahh14YmVW_-U}&M} zaIli!RMzQbjCwGsYOXanXJB?KsquR(l(nG~R!_G2r-cI;0=p1k*>?1~*M@dXQl z!Qfy`V%E^eWaSHTe#%Hj@Eq^|z|T&|Cg4mLJStlY`B-n}PG#83?cC%bxPu4(d1%b- z)K+3mGNH~^!xXYksciPWSXgD|YMrho4b{Qpb+lkJP>x~e?6!MiBi@xqqiZxX>7qb) z<~)H{ahDJY!aQ>LATeZJ7uVL5`Q>xYx(~H;!p);}XYmtke)Tq$%ys5-=6KNH^IT@& z%6>(!3zDplb#Z|iml;Eiji;!(`Zh@rlf6;xpf=~0Ax0z0!x&@A_J^0>Fwex${65KI zV||;Tbn(RU=YESu#SXy(5x)3mxdFb>}x!uJgmS=jCP@ z^9tEdjcWk(+}i%JxR~Bm=u-0T^<#YzhMTR!g=}xFa2;DY=y;6588)KiRp`+=uiM37 z)Jaqri+W4h9iD%;N?OOwJWm*0&sU)cOs@jL2Ccxe!p@AY*Ska5Gg31agPy>kOJPmL z1{%g=+ac|_?pp7|Va^!%tm)%avOM3bQn{>C`@UdoyC~s_flG}G^jzEhu~$K3ovm0! zGQ0t&H#0d^xkmMj1S{oC^ytxiRp!~t*ua91kxM15nN_R5-j4w>h8+GwnbCdL4>L&w ze1R(a?io{J5P;$&Uc$H_WjKq82w$prD9~zTgUCR`3Q3Q5v4(-=;jtOBniMq$?zn?2 zsjjJNsq6@fF^d2J24acU`hM;(@i=I-)oj8{r1RNHQ*7$VWAxvpPG(cwserQ$@1Oz# zqxv$X!sprL9phK5z+&{*Maj(Z*DcZD5$WL%ZV)NJE)k~_``n$u*H`S*_pw1X4vo_g z78eKwW2cM5=BUvBCIV zNZIP^Iw$aP+hTqFKK}f5aEt!~y6=kUO8=$j`zh>V*Z%8pXXE<=$yLDL{3&wh+aK;{ zVeiE0sjqUkYhim*FE}7xJ9BpNq_9GzJ;F6 z9Tn#tm5#Hjob861i)(9-i#3Fzl-P&lffSg(j=Y}|J1gW8-@j(+Sl)dvk9jowzO1KT zRXS$$oGvZOHXmM!DU(NX{VRSE`^(im&eizo%qV<+;t@n2w%Pq9-Q3A`6yUr&n|Vbr z_^e1cnfZG9Ig{&q&iHK(_nXc8p`@nnr#LP#-lblg%v2;LcTlYVXXco5$4oO)il9Ng z;dw4!nb%{;uYuR)Vo+e0etzaFH0g1tPD>cYr@H_`+kt7KMJfN!amC7ICgy+68m*vT z9*&lN2EqUU%IX_|!j3uV^QVdXiibXosE17VC^d(`{g&K0Y>NEm&iio>7`Yq48jpBK zX%Qn8YKc6NkLjP!x7_a{IkCHR_^||j>YuAb>DqsEw#2pmJZGKVwc=MVXZ!bft*jaN z`FFqe2q<|UBC1p1K!vCejV_eLKAx|Ct@uxQUfTO@w^6uX9rIOme?GFaz3e<8){N6z zUdJx43Nyx{`oA6fvpp>{BEhRyE9jjmv47PN9rWCCd>`MvtuLhIXQ$aZ` z(7NAEED{&T=08F^cebhm+WAIs(EWjEqe&q}<&JPI*Snq~tr0@4X+~$$;HhSiVw`+> z5mQf9*=Sr89Gr=gl^s!8ytoh!ZO~9p-gx9lB`a2jVsy;p&t37KAF1<7PB=)cYn^VK zt_tGf8CdxADs|}S5C1CwW>>*nwIJZ5HS;G4V}cW_^D!A`bpb7G1~J2i8SPuGF|HB z*HG4|%6=Z_#H!@KX9Lz?68RjYVbo~z?rmS*(TEcTQGO#YR))Jh{i_f>+c!D`+j zqelmg{!M&8*O_^23<~DBWrHNX^e}~28*=8-FyzHBs|(nl}pB4Ro^P( zw+FcF7+~Lo(|95K*|kdTaWFvK?c?{^>dW}s{Os%=$07a0YH8z?l>zo@i+6SNk<6TP z*vLeVwYAN>yc?rSNsYLOb(OVtm2W@{H2}bsnI_>>A5kmp?Aw<;}yz?)CJ# z-OpI%+Zac7(D?T=FZW)@94=XWJK{j_w9c&UL=Gt-ob z>3I|V>>3M)VEf7)`ry{xWZ?G}psaHIh2@fBPp!chHYLjd@WsS#`d|r?&G*3zF_yt&gNq3*{Z3(SvUjXf` ztkyzD)*MD|?FO)Dh&TQ3uG?qU%h`EbfYqLbCh>#Ef4??@L~i<<83Xy~FNSAQklIH)t-j?T5*b5EM068-?j_|6;VneP2=#2iha4OwZ;Mxu$+8I`8F9n{wDw%|V6105T}z zKI}HZE%lyz-%if$fvY!c2s9}ro5h|hHJ|go;e)mveELdF6tCOpTj1&ram)2X zkaM-l@;xU)4O6KNY8bRX=A96^xR?~EuJHc|&VThk!5$tvij9T5O9I5<^84q%)e-5i zu>q=#sEJh_x!_@%Y@5HkO0HSXRTkd;UP|^|UOBp7 zz2`m|G5PL@mbkP*1*_Bnk^o{EVN&e0DUO!2nk}MduTK@$oC-!b-8NwL1=2e~E8n22 zy9>95-&HqmMa}i5Oa~)}z-y7KDSOtepEyo71R!npxK?d-yHJa)y`p1;=<`t%0@|eF0Wkoqm0gOh|pD^8bwC8+u=CFus%QL$M;&t`AzFI ztK;fSfRWiev^bNoOufxSf>iNp_<&;;r>-n)hk^eFy-i2Q*5#F3%T{xTWxL!4o9!ye zuZuKe@iE~@OSZ{L=aYAl;X$bdNPN!c!--TEm!sLqtb2Y&-Wnd^RZ!`HHTd96U_guod=m^uYauRUx#8jjSn7Pji_+r zM{XWd;Ao-n+pr;@#${0D#}${>d7bF0fbhHA;Tu(M`Frb>Wq<6A9dDlp*0j zZx$nKRKImdU2tKBwDI2oNU-#!mM1Vme_@~*zyl4%^OwP^%p}2kbAG20VOIZ!MU?e6 zVnggkcVQ2T54}p<{Xv`)Ea1|I%3clpn8W}nQ6begQ^UU9Njku9HIR3aT1S$kaFSdo z4fqre;dtOB$$?v2$f9{cNMrb;i#NTz*aU%}tPTcSCB$Iy!iCHdpz@?glyTZdT*-Lf za)W3Ph3PelK$j%Gx^-RqCAr-sbB`?V^ZE!Mx`WO82KxEyEz9-W3wwp5GYuMqpQdI2 zFS_)EgJ^n$p|yK3iSLI@2KfB4q!Fnc+^vv=bYaVp&?1e?Y;q@PJ;5W`^mXM~P-G0I zJ61{c-`(hEXF@3&a@F{tji7y$x|+o&10e4rNQK4qFfc`&$?WUOa;11UYwKLWw5=jiWh?P(G}!TU zU0|bdz_Qa;W#3j2TJGZjDU2=@52j`+HP7{j8Y;ZC?q+wMm~dOWO4B`-rE&F1xaw7Z zFJ6)SFIyc&v2clJd|2*bu$VVFB(^>i6EAK?^qc!`BXdCw^#D?6tMW?ySsEDG$0xZb z3d9q3Ugn-Zvhv*egijPMp>G(p>-Tycwtv&cg@1Ff{lv4Rj__kmDg}!vIw(HKZE0#82=3GIFk70_K!{VJ6HM3@Sf32(vS>~s@0;uS$l|{Z`CWmj<1)%vM>E^k$B#skmwl9up#fL5ko-(9h}HL z#WVtaT$(p9bX;^ymj#^NCS9e#;Fx)pw4$6HLC)^Y!&)A1J3NhD>xS-4$`f4NRNcXwOfMLrP%@mR(< zujyjcCEm5XUSB0ep1(cKVI?syhH@srzOd8XG@&2}3v2pubmwGZ@-Q{8)?-tXB4~H_ zxMK#tLzxJKpH5|WCuVCeLW^vNe0J~{*GJ;B(XziE4%z7s%(o>2x}TaC>-aiExVKw* zjEk$!P!q>iA#@o4hTy`clDlAnbliodZ}Z7x#L(9lyL-N4LRAMb;#*Pswqes%ntuCg za2BLYE_oY07kg9vry?~YMJj2BbxYl~AYWa0Q2dI9>fbDbn-A5n- zusI*b{bN#Kqs3|5Jr6SVYaClU^$h@k2ks`Jn%BtfCt(VN@ z1)1$HW@$NujoTx2lifv~u-ND(pq|xqsl@05{+v&-?cRKP?7@~>iX!*lU#dLllpkqr zp4|Nne-m}D-f;N<02S<$vnrOQGxE;Pt_`Cm8^J#|hmuV?uzcZ-9TRO=>EGA2Oid*-sxvcO#hfp;Vu9zCr7@W!v$pYE6j*wcbouqg0n zg+4R{SEocUO8ue3kbueA#fYazprF8Jgs(zFBby*vuZxHR?=f1AtHT>Iyp*tF$^0$} zUIauDKg=g*nS(uRyX)CAipIK0nN;uC`0?fnCUh47$~i1`3>(5b*4FCB@cjf*LDb;G z=IX}F$C~lxi>cM4Yu&~P9Z{0Z=MftPJJbwc%)!9n#C-6vjdO(>`N#+Akf3f6N#|u3 z+WNuoBt{dGwE^DT7ni5X7=LT%LsvFjd(dGU3=*E8^DA(hgK|1Fq#o`wKHY{+W7Enj z@7BqNO6l%c&ERkPGpbU$or=zSVs96|vK&(z`e-!#0fWnZxs~#f`APkwGzFe>2ag~a#0aVVlf0P3C@|FX zMwx@Woa6+o4TK3G*;hulSp-b4gLG|Ww81YVtb32ahuVQcR|#khn;9(7}LpKeDv9~F~<&wOrUW@l*j>c zB1$reb@L}YzC%MOc=kQtSKBHQ?l$?6cTs5|071trB*5>RTX#Pz?1aLyHuE2$Faavc zRi67<_GO$VTJxRX4hnsw3l2LYrBJ{Gn>HnlR5U41lub+t{xe2v6dAnM-6T9s1OrwJ zTmmJ{d$4RGlVh8V+vDUQ6W?}V$W<}vh(z|Q9}xxq5QttFa>8n56ay@9sr7hwbbT+K z+6EQ7UMH5eyTR-cT~?B+u$;kRRX7tNkin)83WX7~3#cOsa61KdbQ9ka=iEv2QNREfct)4+#?nb*QW5f)j2}2{13OWQdD0&Ds=2uUGvJG&ZLl(^t z-`>%^(A3%}Z|lnOo+wc}19&{b9Ze# zlEw5*j!JjintOCk_e;`AhutUWglGz^Mi{1@R$K?6Ttm$u*LoqfDGC@gzqBYjA zggKdQ>M?~OcBpvBuv`m!$|w$qfv`S4tuPtsG|x1{74D|{eD~4z`_HoQ>(8^0IFp%} z#)j0=o$8YZ7%cy1p6H5|&7<|_N&fe<%&S$Vuad5{BAF`Znj|g4_0GG++RT9=4{@1r zp%^7A5ZK`HJQs?niP68m0`Q`QGbU&71%#4Sr=TqKop*NZbTK&Z1O#{wU$zLa-jedE zwDnA-iWbYI?d^Vj+E&b8lYQ{%z*#b=k05pd9Hz&Qc{2mMt# z#0g;0L-*hSR1 zwyF%3nEe>;ug_$O#z5=$;4qN*_A`dzks(ittRL7}#@M)}ZOZQwM_qP(@B=2!ToHxf zpdg|7xl3#^m9d}_AT8lW*XWuKwfDQ2KUQ0r_CV!OCKH7M0B!K-ypl%9G5y4l)#xsx zg2f>V3vo$;xIqvsqv5+z2-1qXl0q{eP3V|Qy|;cRO8qQw0Dy>a2e|x{pehVEwZ)6W zFr+#JwqqJJm;tCG5|i?5a^M3xexh(cR0irxVd0UF0ya(eTx%Q=?nX9z*eHI8(}d@p zj_N%(d4Y_O^62vCM?edx>PKiM&$W%Q14HhJP_mUMO;S?nX#frYK!Zn=n?c|f0zOm( z=#9rp?zC0FS4tOpXf_444JSEwB8P3Vg1<1|3lZvNb9gqJy){b4`g-Zf%Y?$qAh8M) z3G3a2MGh1q0;eFcZ=Idh9v#rcN;~fR>)v;eDcCGr1^WlQ(+zx?KHx9w%}Xp}Ct4Y? z%{DZ^kO$%WuzgcLGrV_RVzkhIejWT_zY)rDFUh@B|;iIM52Uu$K1XFt!Iid^OzW@OocHreVCUti|`GCA(6#~B7qf}ZpR>RYR^yIgpR zpr0SoSjXb>(-pg_btw`pjx|7mu*=(LPNt$pC2F_hQDNYI_5)Ug^Qi}UGzQL+d_n=v zx^?D-1Pgnt)n)tZ+hIv8;YEc?VL;LnZlSyb*zfYnc||Aq0Ws+^QIgKH_jrYn5z5XX zwc*nom^fHr6%_swmLvS^g2wyh#ZIS}*3v9_lML>NMeWF8SOEN#HAauqS5%A!trE^W zj6ktAm4=~zb8RlXXNj#o>>ZeILbDFrLJ--XWlj6?tMjv?Q|K76l7V_(Fjg_~p{Ei9 z$;=FT=J!!LH63FuJ31%;1!JogCq87n*l_a1P`z-`=S45z?7WFt5rns{pX6d*aBFf06me!Y&;Lmw;$ZVBbJ3)obbe;2%#x=qMQ_jAbpy_g(=({DQZC!b0vA z29-1|lJHzgY$=vG2qxrplQPpFpbV)t+EZFf{3TLdfoBbF8R&;KFT{m zsd&JJ3ARk)I?EnaAaJ~MV@%HhU$FZS+`W=9+Q^Bo4spB7+HAdQW)XOk^6JtEl)atD z3bSXl*XweNL_{z_&avC?@@rRqoV48(+1jui4DP#)?6-N#WamW&w2C~i_`h~%resmg zg_wU{Fm~{Uvmq+=SmV%X@2aO%KLoEYa{PYSY(uGB$vhIA4q&P5IF!FDMadKA!rdd6)!c zRkHBpQy>D<>e6KnRza z@f^e}My^db`~G^w?&38bUG`%&bAas|ygh4MCQ)-c{zgjn3e_VhB%zLhozgPv zwqK=U-h&`tXP?cF_ji@~KTQqRcE}97Qoh9#Bk{^@Hyyq9 zlEK6(1H<=$Uhz6HCgq=u3$`Hqx?cqcVNd4)E6ev)EY~KdInp2vqQhpwRybkR*#)yb ziGQQ`2+z+pPjtq^q{J5drO4(*=BA*e1AK`7stx7IoE~o0|;q~YZFLOyv$_#FSaj#v)G!DhMgEFCh{WT8O+K6CmPB z^4;uDL4}MR|BnLDc=9K z$o{=Rg_eZR#bx0f5mp=qMqI62g5J;g2h0qoAuVR!#!tD@+`Sq>Xp;A#*<~ zry`2izqOv&Zs@omF=!wO7;K}dQW;_b>kD$z2+74@XKTUk!R`~le%!$v4H5zH(G}NY zPKk@dY0!z31sV|b!8YjR>mXCZeaB7^%kL!7@ZGu+sRaDMk%4uC?6I(LMI=eGPF5n! zp?ajSKQO0444^IWN|OR3O#R>xy;VT4)~FVIHavZ0RH~rh{FWIY_r&4&Sm^QKpFjHZ zDOv@XojYgaNp5$A`im^7F}cJT1K6`NAROfIe%op_>jk+d=vu9k0oTNh_ksu!uI{RP zvB6_F+B6n5+ZsV(reG9#u2AuC|Bo^p$P(>KS3o#!Yp-Hg~l>{S-n>->?5T11&GCywmy1Es5f8z{?O?lNh&|7Woeibh*lS!|S{|M3kB#os?EuTQUc;;w; zAlMALwl9C4UgP)+V?mXzVSdqzv|)?$#`BXEIias zYtW@qq$WFPmNEp|zDa4rV#CLcu&{E~tj%@Ix{8?Ka$kV(Z-_~2_az4zRw|QqGH7re_b&en$_5exi+L+<}Xg5pbL>-4@(_Qs*i7+jq26iX}!2YFV8~f)Z z<;M=B5~Uya%F2A5`L)$LTJZyKrgCaV>Z53re0~3RvQje!@P+jdf%8O~Znqsw6394A z^O~RHIl^s;4mDGbIN2j5O4yLV=^_>Ec8$PVg{nU;ye&GV3~GpL9O`rL7OE#ZO~{&m ziD@G=7hGUIxx*oD_eoaSk!>S~Spt;?^5Ch7kwaY#yhCNVw9!SJoV^>zxH*X;%HZ_W zb#2R81?ErTFU0tv{>5G?dGMU6%*!;{=!EI&1B~yud8-u8MHRrmWk+z zioO2MLtAN}!V;x8{cJ1iJX~jYG>Gh0E~jJxQ^2XNYHSX{lo1yD$EV>D5mhow(yx&# zmRn@&P*ABO2SbUOYjsS=)Bljgjruh&g@ha!fvX496d75j%j!|J!nZRKs_vPEUB7|ruE5VqI3A_$VD8LIHU9NyNZIY`>ND~5CIAhGp7E$}c$izce!q?d)6GRC>EM;n> zRsG1wl@*(F`8c$ikOGMb1teE8R@K>j7a)sg!DNVW=R-<~ zTM^71iwbkKRvz9ty1z3{8V-klc)G39G$RQdYomDyvnoo8D&b+F9rBiw5AEURM3y{j zcGEzDU1{^yJJk7j{AH}wXrgM~szEzeoD`%2&BfSiFAH}VRY(&|Ayyy)!ts|#d3XEc zvo)FH=$=DPL`DV{{>#I<{Izizdl#VH-oqXQv9M@j;FRkRon_hDM2!jOf+WMXh*e3o*P84a6fX!}MD z!l>%CaOBxwlXi!v7!H~r8L z-&%j`)F^pp=MsE1Wbc*>YF+yjsl;?(hB)>~UUSgJh|*=GJ4mlU3mCQXTcoDuI1_X;`~@$Z2YsgRJs!R4@m<|GP(J5WfwS_QvcGt(us zaRP|v6sp!JTPgMlcK1@VO)cfJFGtS2Vn!BB8;qzSWjn=xTy$8aqhUGomq>yGyGbRi z+2kx3sN}!$gL_d!E3*9Nhv-*7b^B1QU)*CiL7!MOyM0wtzMLI6Ev|}V z^bN_O-IQA-4G*a=h&EPC$arM`dfAOWdK8TwT?MYXZG@5DtG*C9r^nJ0b}s+Vk%TWQ zKqd+(8#UeCU~A^VSW=}XQ=gGD+!x2WV3NWBON$jLSTD@^W!wuUbi4d-I>X_;#mg{O z9AY^S;uAs7>NObx(7NBIFVSTPu^ct)YUUf=)j~P=pJW@j@iNYgA)HypdAM~1XR@7ws);Wr*-oAzJ6l8cI!@N&vp-0+gnIx;%W*m{GZRU&-UpgXVU+=b0)fhltP5%!*o zyT}q)+tMkb$#;=ESwcLR%zE`AAGwa64mKxjzIRb}M`eZlE7pCfOd_gIn|>Fac#@dg z1cRSkaDXvW*u5Z0stC2IeM&c==oM&99{c2fthB#;nXrJ9X&Hh6Bh~p}2+<19m0vhD;UH<{}Q$ z(i4U( zOP?Qd@xxwY_(-RbxJ z&f`qKT@QzAifmWHN2=LNYq*Di_@Uc>o5Qe6ajG^U>J%5^~m*2Y|Lfo%ogRy;= zrR_#Wyy!cmGJAuvmmB+99dZV7#lt%xSiJvOLA%1n22`556Knzo&Qgwb&Q@?UdHTi3?<$r6* zGq34{-`yo2j`Qt`H)qMSa?H!S4>3P1dxJGQEc@?uLY%X-OK0SBT`dMbtKQP@?~Voz zn63UBYJ2fbOJi}~9MJ0{O>bI8ceewM02qP*{+n8V9oaaCU%xmJU z{^BsE0Ep1;ZF=0G!HVv=JSCZ7ORBzwP{DUD*vuG)*Z=>0{6B|<;T^uih(cSwJQf9| z=>wOi=aq^t+@9A8%7=B_L4m^DY$3o1rAjigoPRoNnUe=fIY+j}Yh%4vT|uUiIvT2~ za;k317v4EqUzyo2z1A{xt?4s{SZ_I0c;l{VSK@lv2HYZ;n&NA&rwj2$!m?!W~a#{PN}JIXTH^S(j=Q zwm5UXsuk{&wI)~csosn3Ll2VSAsXx8f|1M=c^nsAb3aAC98uMVl2)S{NZ7Ag?&ZmO zlQ9;k{R-1yvX47*;*&}q$2Lt6meB>(e;+#eo)gdiVgYo1#joDQ*nWbw{n@;*sEHP@ zgoi94uWm^FL(rqp{%-6uI1UN`@dxqoE!W`7P)G#ryLop#m%L!*M}j40El-srL>6&02r zeaQegxP*_s8#9++2vfWDFn{zXMksLBCcD%&C{Q^Gr=RtLJK|P7s`Lo5Otr0Tv>rep zFG~7jQI+)L)ep}9ofnY7cl;b(feh=egIjE6!90!Vjj;apvnhS!BN8 zTQt@)>tj9rma`=dB{_@0y?NDhz+e3k^nYjE!kt@=xn2#A)AN^$KlgA^l{X-3_T|Ld zas%d%7vV08ss@4k{TG|ejetvtyzWgFG0{n-c9o5C3C9itd}3zIw;!q(4+x$Yf?ulY z{^~JENzvr_9>dv}>~rAF)ct2(Kl{=T2oAT?5T`QEUoXodLr$Iqp9rL3^s~B>h8#Y_ z8$Y|BAGUs|;&hr>ip*A86%`P|L$x$~Hzmj{Jv4H$wNGohY#p51zd0I&WYDUCCImxW zEdR6Gx2B>%u(d~@^@*gNcGof1zRb(bO#dCBhb*bj43K%X(j2Oz{r)^PbCTIh@a;|v zR+PDM%ztQTrcNl}pPPx<^m~^~Dr}5(we9JbDy-jJ=>G|-fjE8s?QLG?>bP51ccc3y zg$Ls{^TLS!pbi5L5*hkbTKIW3yj?Q06a`L;itlY^kzMKTxo>K42Hp{2ffX?mb2UH_oJvCwbzhj z7eabcv=*4}I?UxBud_`60#Dowaphv}G>BQ}M_ov7!rADKQcg?t^=M!`R zqcQ!?HZ6eFLg?R}cL>`S(jy6~XkfvS#K?);%MP3d?cVN1P2`yWWf@IE6a8Q#274YV ze@?Cm<1jW-E547og&9;CKjt)>SizS0F=b->@)zeIv6z70#Fw@B-Jj;xf8rY&^|2}l zn0U7M(qZAFJZT%(cT8Iq!69!Tte#}7$H$wZ;eAE`5M(OBW}}DmM7=wSnW+4mZz!_<;x7^O1AdQ4qg1idvVa4uDV zuPZ_I$R$R=t%RqUIN^?w6J!e+1L|$-{(IXp9vsv|%r1qIHwyfl^2T$}(q-!HZBg+%IKBV3t$sT96a;HqsMG@>t25gClM3AxbDXLA87F2&I(P z0JO#alsvqD6}r=diwFh>10=1?RD)?m#F)bw5=7vD%qiKq&R%}KBCOeC=D>1*c)Bkz z7Yfw{DQz5oaz-qX0@qDp*)Pb17E{3tn3&+REh~dfn6|3l6AEO8*Pz_HX$xix>(5&y z1Lb+IO3D;T5>H}6acmkw|D2@>w^zxU zLfs}DJX`#>Nlc)HSjZpvi@a-PA}9f(7bi6Aid4Kkq6ilCN3E&Ha#S1(qe=i54v%>vn|YcAsFgqUQ=iHKlIj~%J=ud+Ef8k;~2$VPn{9I%*O@t zXYqE;za7|=mL_4KZ(c~LDRf^Pg%nVd8)^b4mO_-t(X{mUI^ye?QXp^G_(Di*Ul$2w z%0;K!?M#=@$jCItKVHWp8DpOEbujuopM+R(6!-&cpx>;`N;iD_y7nJxZrkc(*1V?0 zA8^ZjrrTXUHAidd*~um2JKiQgoR(7xS>LAcJa^L78W7%RZ{Dp$)Rq1z+>XJRX7#O& zK!g_n++LT`rcAiQ7W^+Rw_i~DZGu>?SK7m5e$Mv_{ z^CHOgZK-4MpfwyRBhJk;XT9E4f2ComqBmz9j^l0siT`Wv-(8%yzHsa8EuO2%^+*`L87h$`-ET?%|@KMyp6Jx(G3wSEz zAR{C`5|5456ZlH*iP%6-0_Z#Jdc*jL#bv2tj}#i=ReHFXv|FwGYXBIL9x{G9i=*qV zceF4t_=0(9~Yy< z2g7Tg_?l6ccUlMMOSK-tn~2N%al#Wn>1eaND`R7_NuGPlQ^nF<=arZ)*nF~o_+F6r zdYwcR*mspiCd_tD%r8tJt{^4r^igeYJ_oToRlnZRBNg7{IAARYm3h3s4xW`^WP@8C zBtE`RQbObeldb`E{XU*&ymLHkv+ZYtekJ1zlyeA8WMqhszP1TRC0B=eUdx2>F>&{< zQAB&K*sU;di`}*EF69G0UO29NO>Re&9M8t2uEKK$iZGkby!`A(T3!np zCoyw2-e0w>Hex;9xP52nF&BoXglGL)?0xNgz1Y(kI8NY|$-#Ukd3?eoFs(*n9^}Go zoL^5!FGpgi`M}?H{$B3w>2&A)tWEa7^Ln3a98 zcnBips7OJH_Lk{#yigU67hZSIcLcJo35t%!Zk_m@Oe~%LU9xw~RbQN9&_)3SP7V7^ zdz|^Y#w+S9+U>kdd7cTmY&hhB4}ka@Szy|}gx0I~>Shne)vV^~>IAHU@lL{U82ZYUY&d2GC zD_Wo1sx^fr26!AE;HgH%f@DB*M{Z;0o=+~Ia0c!)ytFZ|D`-c5SNF(e8PHDKN-5Eg z+v`Z{I%uH_H3fMoKjp9SmPqty2F!{t!QR6Xd&zkQ8Uv&RvC3_(8zy64{#eme zl5ccXi6&zzLNYBYzHsBaUY|0hMk1Cr)DHz3V8I$}Y!;Tryjn)8C|AbI^1GB^|nJ4LTIFgA!2&oTKk!MGja}iJFVF*buK&}4UegPw;GO*{?L^5b--R2Y3oj0d^aVhd9!xp z^pwBhX2gr}RKiZH1V+8aVg0PIn89JQnAqo9rFezkO=;8rM9`iX>0+zttMpgr^9ufR z9W_OVx7kR8a_x4LA~vT3kvx5)uFEF(Dy$UStIYlPBJ8v{*k0a7(vyR$$D$X{xK*)R z0W|ohT64#gn%AjG%fI%@!fa$ZEiE?HT~P(swe>!p-Lu%7ET62-JWC)kj8m^bO~#}%SkQaEyY)%2#_c6D-4Dr3a+-37xyhmFKGRJ(A-TUA+-5^O85t+;JMN{l6C@6* zTK5;82A2?Q5KJO^X3a<1R^`t8uoZd!f@r1%+AWPG)Lb4)+fiF@raOMqD2!`u-Z**R z?#=N}&mOnc`1)b{J?CvkxmzGSEL+M$2#fp@+c(p1N18stvZ1304zcGi^FK0UH*IIl z0f6lSp@wa1j4JZ=n}MyrG9%Vm>F0|>=bLM$gn}L&&ta=rO}QF#XvLi;txP`sg*&lX z+*+4njshRAtrY8Z#~V}EBMbp6M`bC^{!dv8|C%;qM;n`{{GKWt-@C&V8x9-G@`T7~ zK}Az{H_X}5);621QH}AA6=yzY%0Vty2gAS{WB_Z|~JM z>)!wru0j3b=<{oL{gzt1Dx5he!N0uj!hEzYhj@g!eC?N4D+N%d=Hk1JytW)uryJW* z8gcC%b?*K^XlgPsBmwFlm#*=M*702)Zk}!LL0K08Is@U_c~4InN4@GFFBjdJE*BLq zE1Ll|;_DC5T*mLtTslZsf3COrT4qtA|bZsd05OxO)) zD{WjRI`@}olU_q_&nWz^3sXLP#2|9R<%GLLRsGD1CJ&0*Wf~7M&*zvQ>gOrM-4$rL zPv#eoe-2Vec{aulXK6kf%+2^ZOjY~OEUUa$uE<$(?#lFw4XS;5bQc!>-V7(UNs}2? zO*y>He@p;{J1%;WO|Au5Uh8Z`D`8>y2y0!xyN6!>6mQ#>=dphAAk8MT-*o!n&X#nG z8jXRzkiqO*hH0RpwOZGJDS*+FkWpV0SJ;$NNk3Bd{`T+m&rVnWA%2yi-7gy*MTAly z01zrBWu({jWmYXif^nC_+T(08mD^pTX+xbMzaRVKM#>SiYWDkYO#zH}5Q9?s)cxrN z&}|Zokz{K}-`n!4mz5STwuW-1* zTtkNj1CT~32C0(oT(_A2ef8rs*D&E9I+t*hvk>1?v|+CX%w3saE^!tTi%nODck9Pe z{VH{~ja!Vp?dqB--`z`|TkFE6#y7zqgAl_urHnYN;^i-x$Z%je@Gzw{J3b0H^h?|b zzD&}Hl`Cx&CP;H+zmB$V`)Z@afoL{0LYrC5;92nGyP!XSY(u520uGR`o~Xc`qp2BD zcmA7&zUk-2tm}7Em)Z!y^cp=KBKbk+QBW`tRkHH-+YbrAQTjA-a%Bo20ckRq??e(X zo2rp^QZtK|#GDYxm@_u38>RF^p$m{LV4ye6OR7l=ay(UtAdVhqOyE6Q@yqUTt`&Qn z)fnp3Rv?b1fq+47o;pf?Nfc?C=e;P%|9VK33sHtLHiTy({hk3!@-LJ)RSXVBIR~(= z_D>_E5SIw{a*S7J7X2rN(zJ*mnlG1cEzt%T>JU1CI4tQ~i#+VKXNVr+ZqaWWp=8y| zET|bCzzi!Go(z4hpzF6Z62cl=ghektAP1Jh({TDjJOwoz*;RPdjmN=15d3Hy08W82 z*%`H9Rt}6n0Whyh&<7_3MY5?i{I1{Kk+)X5A5Wb(=-PDFbM(IMEg^qE@~v}{!)MX8 z{Iq_XLhL=&3q!T+L;iNX_rAUk<50ay$f$cAD}9o?;ktGJEGv8R{8*<-!Ls=FF!y7H z_~Xby{NkR*=$O8%7PH|q>gL;++*(=Ah|NX2PvXajH-MVc*QHGHKK|pjYUb!Lz;A6m zF%}$L>qa|9OqM>nON+8(wUs{3-?>lpDRZc4OwJEVd%P@d^%X%;M?K! zAQ3$1dSfvD@e+5DJ@sN$hP75YOr{4sG&@xGF{@DAd|AX@eZM#?vu-Tc#?T{wetz4T zsh|F;U7EssScb%g|@OqbpFq7yjbz@O-~12cPS`^YhTulr60=;b4Y{1%r9ef{}Miv8gqzTWYPvESQK+U%FY zw+;6+vL6QJ+fCtJ--uJzuG(SD&n>bV&+HePIy|?8H;DA+bU@qh^On@qLLB>56-#S{ zUi*700_uFOztZ$O4-h=vFJ~qggjZEPrymxzMlxg|e|_Qv?u!C;vmZxy9lgn^Pt9u^ zg&*IQmEJbr%1TqWrvLXHqOVJ?~ems-2JU5CyD}J zEEs20j0=mf>S;Q0A2v7wrcY=(Yjt+KSd3ba5j1P|GUb{PtPo~l;2%^>&JdT@J{i8$&#mxuO4x~R=@qPh86Ph$@}8UX z@Y86}->;g*LytX-KZ?(*hZh!Hs!t7+%Gy2Lm^#i7>P|;Yj5iEgPi~V9zZ!NPEx<9= ze!S1sgwN^rmTFEcZ!Y7%n%76b0D5}a09b&qsnY;c3ii$~z5QufIwnrQ?fb*j2HMYs z-EBxgr$WEy3+w{E7;=zitUUi=u5(^5#r-1kVmpW%0iZ~VkaKFqR4j$K86o0Zmc&U! zz$?)9lpBEha8UW^;#pt=TNvnZ!?X5M?c9Ou5;{nlY|I3FmvH|k_Z66#J3f+_%d{_U6=f{5SxQG&X0+n$FK#FMZi-> z-ot{@rr)CKs}#y~qn^#7#%E-^afSu3k{|$jK$Fpff`Uy`-@X=KfM&avH1fCfhNejw zIfaV34s+1u^jV-4el8xO!}2y)9SS|2F5JAzIIpg&rF^?#3fo$y!WxP(Z;`7aIY-fw zlvNBRU=1=d{&!WJf9q#i{T7ig5Tr36J*q;RG_zxHHCRn;L2c{#Tn8cCh$<-Hss!o? zL{3kEyw?h&kmnouJJ`=7ADi6X$0VO!Z2w{l`U3xuiu|$)8k(9pO<3S^4f3PDtcfknUfOQWZ) zaLgRk3b7H&1U~$de{0DrM*b3JV)@}0!^YIlsyo5cW@MH;+6Pox=kr`A0yhNxu84dV3 zKvZ=(261wgpiDu2#P4cqs|YJO3r%*Mkk!S@Y;pCh{$gTg;bNmYe<%>)r=5||`Ct<- zcr*^jK7T?yZW(o~CYXmkp+ZZ~&dGiEw%zUDd7s!GDO7hXiW{{xgmrSo5ho_-P+qBd zT|1yjT=aThtYkNcsbF0t?cwFd^Qk{qB&ZN?Qu_`K+;1~WJ($NdVs?;vkD$L&Uc9>~ zEXc>d`A?v>d03LGs-=}sl4*e*34tRyRenN`l4l54PS%cD3?@Ual+C_HBcjmtD1j!6 zNn4J$0RzyZqpB~^a8}lx?sv)XK#5qWBE$Prr8pJtbxu_JN=z_MeoPX1jZn@`120-M zNx=4k|IcpBM5*BNK@!}efLwCFw(aD?R)hWD4U3BXLPU2GtH;zJ1Jl`O{@Dy) z)o93~9h&av^m1;V>*Nb!$X!(Wd3)^AnT>sbB@t&uAYMMoh)?OSM4h%j5`ErrVgzt8 zd-#DME#MwT4|!d;-`lq);Nx@NF(~}EfhOhn(QA#48f~fQS+$P}7lso5!IFelB1aTq zf7>`ciRu+*R`cq^lN%8}z8hE=cAy8@6if(!?5&43CcUNmSR{ZzS>EiYi==#|PZo%#>C=wTu2s(yAuT4&2HCMq9X?AUXJ z2v^xt@-D?tQ*0*Kxc{`cP{pwRHn52yGl5K~Y}Hz^*QX-ma4u*G*y_t;jWI>cv|?ro z+`4rAxG}blk|XbW;wHO=WJ4z~0E)nT-;#Snc%I10RsN>;hvttd6|1S44`H7NQi4Hu z8uV9wKP#Wv>Teftn21QT3Xa?+PL9r}V8K>{(UP?bW>4Aksa6ADC70S9`JJUWltF?y zpHH30AWG0*6uZj0PKCUO7jbA<_`K=QA_95cTKXFu8HSTt7ab2>$^)z-dd0Sj`r8wh zb83oA?H{C z8^^`FcfT*_pP?lQf3>nKFKX;+YjOuz%fQM+adtlrJj7Qn3wz&eLiOxKOixXh**)Z( zrt=Hi{|c*%;Xc2^IZPKOm;)AQ1gaK#}Uqfz=_Ji&nTvm!c>oho?WLDL7 zw8o!6of)E0DqfSW2Qd2$g`ih!x4QrbSple+fB5W_P}5xxELNZ>i^Jl`tC+J}`ldbw zg()@BuA?FjO+_O@hGkg7AI&*Z9yV9Ez2L>g8eSS_ul1B>16>7)JFZ+*JYTxlHc8H) z*c4hnm8M!9d&o zerm3%a>UG57QAV|4U}h^{JFl$ny{=k&X{h$u8!TZ&Fal>$9)0TQ`e@+i)_-;;4?9J z+nX5_65ns&%=+Wn(YCCeNODLK&NVDcuKk-GQp!MVbsEYs?Nm*tF`rsK?gHmX%Qyi0+Sjt;mX+UlGWbu>1Wp z2W1Wvz@`QcFvCfJ9K`hr74&GHPe55HiUk(}p!}4LSO{8jmWd1T!8Wz#*wgIsJdFQ3 zl7!f88)C)*%E(uS*;hBpJZ`GB(41fqs6SDY(fs~NxrSsG5XcEGQ(}m2XCM5|vBC)7 zXVe6|nh0{&Kk?}>N@`Rt_j+t)QRW&A!eDy8K{QP&goe08YESw7%NpOYK?TG)2zlM1 z!J20@J}-~ymId7HJVko1^f>7#T8oR4XXz`ztbNHxwsKJk+Q<)1U=@J2Jp+}j)6~9s zVn0s(7j#Hrad{#li%K@t*Fo*~wtssstMLt+9M9-ncxd!jC09vjJ2E0lCUBCyYg+vj zVcP_(rH@I3D3C2&~vIKe>r9qjMq*Y`>;bVT&@v;wmdUQJ9hURbe>S|TYK)360(F(EMh zZH*oW`@=U`kEbg0swdX8xJB!n{S2-z4(h^R6((KTyXMv--f)rFB|nV)xC^L zXysBYlZGUroTZ+rjHGQ{^5dJkS~@F6f_y5C-^>-Kv*&;0y*O+aY<4@}$D}D8hL}QS z=={M+({8wI-}ISKf)Sgk{rbuh$h8^{O$mo;+D+aHD7Hb=8UI&PQ}e87_6McjnqR%4 zC$S#pw84cKptgUUd&y(G#@##;^i&b>YS$^?IbXlY#CILeGnISJ{rhNJL`IT2xX$7b z5C47gp^Vo?gx2c-H=)KM#R^LzFAstE@vWdz6mNIp?Y#RV3{>)FA3!cQS~B#8uw_vP z$&p3#Y;d7N;j?mzUZ7}QJq%_qDo#?XXq5*vWYxHwa09lNhw%=g28ay65?+N~wrX4$FjbK9E#$#!Q5 zu5ROXSlcCM>yCy$Otf~hP!O1)?tg*pDzETd zT-#1aP6BQ+@3uV;LAfR|qqUm+hw=S$-^H|Nx-Ftz{ zUB>{tT|*ak7Aqw>*iKC8ccb=H97Zf`lKc)Y7q?4?Cq=;fzfq%qj$|X90kv$&HXxQz zbGZSA&%>Y139c8laun&>r@e2CF+}UjV`dLg7(B{1@r0M-2EBqz)32ULEi$w0a2(C!iyQ3&MVW`&MKAIEbUvS|&ne_Dsk>z%vLm!qKQqZb3pu!L zJ+$ceA%;?Y8Xy!l+dDh^kwjvp+WX<53zXjmV4jii?P+TYxtK+yXEJxYat`b?gb|Zc z*M;}hOqYAYg|SJ&D2j@=&*iroo(?_d#-*b3%MhRANf0XDiIk1$+BS14x z28aNQt0&F@MdboY7 zAh;Gy7rR#lK@C~Q#)NKZ6}d#|@k^b-o9=A3&rv^vpQEGTELTUsE<6*xS%dAp-ve`Y zvhS`iQ?H`#)j;ygKoN~{=5AT?!M~4Op8emSn>((JFNB{YB(=ou9F4q|9h5WM&9)ys zWfFrXi80({UG9TZ=BTZzmoML0;!ILo_DUXT67d~q(c9qGjY+>HH6oFHp)xFd1{~P_K|%V^P8)g?39qZ2<^{`Yq-Rc`%7;I+^hXkDiTCJoNEnS^O^-_pwZSI6(8 zMqx6F+S{ZtXnbu90Kj?Ko9^!7?HXUv!+LDz1s~^g9C7&G$x?ewxr?mi-B9dZSA-YjAO$eA*swjRtWUiuJzHQpefwye6TgYjn<)K6mE^X&z4R zdz!-U;2O`xRKdwE0;>_fFR8l|=OK3f%YTD|WBE62El|rX>zS^%N<$hg2kB~FU&zDH zMg*bH0uZ&b1D1(FdZYQ#9)Kv}RBA(Hw zqVIkdo@zXV>n?F8REq?VEy;1|=}FpzZcrgauC{~6l8vZ0`D`rQy=rCrcu8d}TmG~h z?{CA(ZzcHJIB_NU8%aJ_(fi5x!_;vFR3C9o;q|Xkn*_r*z>XaXAe*uotZnT74pz({ zan+h2f1}nIoYTk8Y3+%E3M~3bV%&aji299KJj!*p!^?B0uzB(AxXkBE2?}}6)&Aeb z_uV`X=hW9_jLGivu^INW=sEwVRq||iMJE-xjTCFij}jjLc7G95wW#rnoNUf_0+a9V zJAY+QjJ$KVNmAssR&_}LGDBnOB(u{n~LVRY`jeTBo zliJqjWz=Ji<_lZrn(Fy5N z{O`mYzDG|*oVFg5#u@5nuff*q2ZsTVL%(H)b=kXmOg{y}5F4E@4c84lmUaE>(xVLS zu~KxN;gLsmh#H3o_FYSp<68;1k8g$xUIgvSl%gWhMG{6L<=HN%tmpM6-hDiciBg`3 zEt2uQUDd2>C*XEpVL*7VtG9#EFjd3vH-{&`|0dtNw$~k0j}b53MjOHmW^_vbZb8V^ z#j;9KXeSKJe%5&k=zcrV8_LvcVIM}`o@#PpWb<@;wfWfzlgHi<3(Cn&{PxN zWBj`VkR~pz{Cvog6EF@Rwr8cJVBCDRZp{9x)5WUebhG8nV6e`c_ugKewy2+Um0QbQ z3T{mG)vKw z`1hWYa-4fdSJgwgG>z5#*MRL%?by;{O=S>`RkZcS>XUCyQ2q2%)XP*Stp-&{POgoZ z(Y3U^*=IM8f)3u(Ncv63_TpJvY&nVFxUTQHm!Z#2j(-#6dd@j&b)$FDz){yLv7Ki; z`K`BoJ8o6$k*QN;HVA<4u3*C0`6evO z&#-`>+e=;QL2wLeNkUn;Bdna~8?BqrlJn2qRiuUyRLgch)dM%Q%L&*Nj7g0 zkcS4PdvH6=@-s;!-)Xb!UfVhRMt-pkfh;C@(u2+%r>^R$Rnk+-xC1U6A%WAg`fy$1eFAf9H|^^r&>=)0c%bhhoO=~}CLS+WIa>EFo$uBsl! z!j`4vvVzlD&8C1dDBvz**kzYC!oBl#PB`~L(m4sl%0yi>JMh-D_lw1E_r#l=a{Sg9 zq`N0cr~?yH&dx#~xQ#k)Qh^GqXd;u0n>2tcssXSDr}l|p!O6}dW}4b!(G{X{ikr!( zgLI2*kmqoWG;C4_HR!16!LrV%q2?6P9Z~hDk+Ow>iKvo01c9aWQK&fb6q27b24-=> zOe+}ez>3@X^jIuofuA`0RR$Uh!dFsa>FXk#G6xbF8DM4YqQ0yDqP`&*{gf|PC#Kp^ z8WCKK1&YK5M+VW#AyVnawKi<~NdM@1&?t{I)rAx)MGP(m#+d42Nn_Qh!6Qq{Bua{^ zK~WK;mLigfV4Krh=8LFb z#S}3Vx^ZgWp{z5oHojo$_@)q8TN7J|@_X34g{qoHk8El|&AkSNA}*{dOiYh+aIq5z zB`J6t6lTg9Dqkn3-U74*DUM(j>K;>3uSisYXuTiE5)`bGTCBllIz;N0MwnbZ z1Pda;B+(GEknNNR&}fi1v&WJx7?CQJu9l!?n^Tb<)=~wI%o??56Gco0%Pc9_DQQQ? zszD2}9IHlUZ7DB`;0Gs3xjFG*BK390FUq8C0<7;k3X%k}uBZia5oUA-iC%<7y3mh0 z8c<0Etl1XB+-0M_3M!wZT;ac+mc~&_TJ#NI26Ad1cz-eljn~$7f6OGlSw8sBN|q|1%L#CXgPWyZJ4DE zXjsg=e#&XcWbgOL#@r+v-B4`zv--XHrVcAx2K3WRqG!peCc`xeD_?{f4q_&871tje zA?lxj^(4|eGCSU~lCYBXsIf3=7K!*0xKNU{B+~Y3F=?Ct3feGpVLSY{4>=Scbkb@& z!q*;72oD4Jf4KSzptzcC>%js92m}Z^xVyvPmf#ZH-6eP!+}$O(dyqhIcMl9UxCRd% z+~J@5-n-wu^;dPBsWYcf>)PF?ckkY-)qH5H+;jY7+_&ga6zBdYh<7pYcMNl&y5Toi zbqR>nGxi6kFNu49;wB-&Ovv#`9E6Q(AV&P5{cs6GmmN_~VHOPVP<0(u_Apuwh0YHD zwa}QkF<10}gbJCUVq-)IC-plS+z&yeA}trmg))o@=#f;4B*Ggr=#rb{a-6bI>Np#i zz#vL*W>&zQ$XSccpv%9HqP78{${jDCrfa_)PGm$Qa(;$)lu5BLOyCB5JQWZqJdH<2rqB`|r(Nb~54$rvv(Y6MW4#gwTuRakCQz zWj*Aq>|LU?7@Gm&pDW{t;s8KMpZ7{}sHeS#yqjFLnHzN+2-hALgUf|LWriJF%Uil4 z=*ovrS8q3Eymanz8%Kcv08_4@Btg7*=tMT=c>jzj)PIzkXue^v*H>|fitO!K0M`u` zeJ2u!Vd;YjVjvvA%zbaKSkD>|&}(AUVRx)aj{ffK2yQYoz;wvKz0eFXFButm@-339 zfo!MB%2kN@+h{5H72Z3_rKXz6cpRG8;&bIB)pJSxXq)#E$Q5xJkkX4fSu)O^ma^v8 ztEEf}`8|V^!OlQZ=z8Fb5XVVgUKzf71Hwtju>2M7y1<&E{_5Z0`lZ*a<8LcM7jk8f z_A;9|=(bJ|s5mZsBfv{?O9{A?sL&Pf>I>1pN3 zR#Y93Ow=gnG_L=Wvj20Q&sJIC_lk-*qr4ll=pn?SO(p%>Yh4AVsF}>KXQFpKF7=~} zGt16@{+_c5-8;$aN!!^yJ~O4Bu72&Y=Y?bms->o|thjG*d3#AbLGUeS&6WkHjBFK{ z-m+5$K~Tsk3DW+ z+HCSz%jcWs|49mI!BjV@aC_UJ(--01b15?N92Nixxv5FHsqg@`vX~!br08L?N!(c? z^$OzxIsbae?SFoJxnZvYhZ%p#!H$%ST>6Onog#-jP=$m^chD*2K*mNni5x{TH2;Zz z;MOLZbyP|rkaN@OYRSvZ3!?|;6>97G4MHC)E}>4t#oybvJgA&K{NLpNmCI>Uy8{au zKn#TfF|!mdAG3>i_#9J~llw4vKf()K(_I?*x&%%VaYt<1$UyhK3!oo~>ag{dKyIJnpMbdZDDa0a+eDvx$=YHQB!#%{;{7nbQzKIUn9{pG8 z>s_}Ow*^xx3e5I>HM#EZI%T;Pv%7JB$#Tu$;zgE671p8$nm9Fy$Y2vF5~2VLJKh4M z5F!B#jIaawNqveyH~@!u%7=j|(D3hysKSmYECSHhA-budI$S`>Ic6UJQ)?)eszxWD z-al=;<_A_(Km#ZdL#8|o)V})!`Cetj4x+J!Szc`t5^(7#o3gA&&J zjk0o7%B20bO*;_E{Vms=o~a2S_D$;+NIWsg zovQqEZGNrOm7#*@?I%Yt%YW6lI?9Ftn~A9@x9Z%gj1W#Nsgdit_aOP$lW^#VMsQdD zw;bDaKFC4CG_sOc+PKSTlzXWNJHp(=m+$LzK7E$&$9u+E}E8mg$GNI9-pYE_ zp`Z=fLucAcSi*}EIe?mQnmaMkT$>YmJtsYbrwFd)eg{CwuV=I5#FAxga+JZffZ%^q z+v1Pv3-?t8Ram1i({lC>iZoKTw&$H4Z;Q%RnE#istx16)i)KKC=HvpDn3D4)H4+s= z3wobX=7qR^$vJ$4#@3ub1?j&6TrsdBT;tCmi>u-hD9uD&%yiN`}~Kx^L3qLkp5 zynR&EO&;)N+gte4Qt1qQ^olr{(8(OuD%ae5tFkSU2B;Jy)6}sY2$AE#diZ1ZpCYse zo8);p7Be$#%FLR)-d(aRoZRF&Mm_IuQ2R$UK4f3Mq50G=6V$_gta%y2qnP|*(1Dil z@bsv2;XhKT>~&T6p9XAepUVN=w2&ur5MCMo2g3($uhD`Wr7RG>Z+jc3a)168UTY=^ zrN8RrW^k)(K;L0$X#H6$_`hBU!xfjW@jM!NoTnh*dwn`_Uyy+e{w@&k2`6KA9OAyp-wq5^np0fP?*nRQs-#$90(`Qgp6td7*-gG;ef ze}rnb`K#;vvr=(L`@`HUH?ea&Qz6v1w4?L7*e)pLzTRP$Z?PbA#$ZVBaYG)qa}xZn zJ-7r_@xz;TYgfFh7FEhsqHyv9`;3gX{@ubjJ?;G*5#|~8* z3TE=G(DoM_{vFKAi&!dFrzqE@*rIqE*;4w8VENRKKGRrAbj#nf#>J+%Ym|-0V{BGO z#i869P$9c$)vfx|WyjSJF2t|Q#S;OoLA z4!V3V!?AWzA1urc`2Do{>cjmB)NHj@nKg0GE96c6g?bbj5`LaR% zr3^cZs1n-M*-F!wjU@*=mrNwgGq0nRMnNUbX0I(SS0G>ez^NnXX1;bJI|ldkP(W|` zCDze#QSVH4Lay1+rUB7$IFkjLf|aWGiNd?QUCooN>s;5@#b%xk9X-DK(Vh7@amRUI zJLL0oFPOKZyP3*zTt+^>rrq}Z)OVYQ!SjL+U3&7^t2~)$GP(#3P_Ak=H}v(W89l*< zu$)&l`H@w<^S*4pN{9cASnXyU%UM_bceNy>FMp2;*g?ZMuz_y#Mb%f{T_Q$AC|0dYz3zp?R4MLC1 z$|!iD!?!ASa%93%-GAvaw*FJ_%V{49gh9=~%ivTnXcr^cxBVf^}hv;;Qv zzyG1Akw&#w=mUNo`mO-{|gg*zYn6kIz0NFt2}$ ztpmj1Gx#G8+-tphN%d_^EBSWy5uw~`WwNWik-v+5Oz8Bj#M>K(^|~=DIpunefSvVa zm}VBEY?mh9F+!ceEB`RxUMbG(+dP*Bu{+60+K+?Co7y|+4{f%Xjv@pjndu|nU9{E` zzphkHG=?QXlOWsh6=JucJa)YHE3Uta2vxBSs4$S%1^JSa6UVUtGVS_$jh8-CeSF;y zheN*7N~9t#7duPF>kazQyW|hpnQ^B&^_|Po0~pOBJ+a4IFHDL1O+1hcIZw8OhGGi7Bp+jal;dVuxLK_B0j>mCJ%(e~Q=Dy7% zh8J_lv%T;6xpI!a02jfg14(%Jw>nadeDScggQO%?R@3QQL8>dN6O z$v+$?B!4E~Lv68Q`iv1fIC`822mmv%6))!-AgbAldOWVqHv ziL<7X>)PBPp&fjk;6TJVzGFflsx7s&41(VLaeq?1zB|I#-9y#Lh8q8}F4OC`wHB{~ z)}|&=vKY5tmfG4+C|1u60wB6s#rMY*1vM_o}KTe4DvfvilciXsBmDV z#?rolQonv?JrKb9w=Do*u_TSl} zTg8cqhKxo<)<1vLjwU%1M*+BaT? zM4!4}1!fk#y~gZn$ZVsSSSP_QfMPDHnktuvs;Jr0{*#hu%daP%olt(e-Hxc3kH{oK z&yTku;oUy|-yIByg{OI2Ugt1)xU@uDJ#HErK7zAjZC4&=jQ6&c#}$Px*t}1eEL%k8 za9C7}U)r7>j~#QGo@(!rnr_q{H`X>d&kn2@xq!tYfyWLwL>B~Ha?v;q{dASdNlEWW zR;?Ivvx>d~9V5+Ybv)S96eq|)&O^hk%;vM%+8~b5Rj)z{KDw20to4WIIghwyiw==?z2Jp zJq~&>eg*3%JV@TM>(j@cOk-#W0i}d61piUk*E<+#3=Ja+7jHH@N!DIjZ5t&c??bp5 zK!&*O^YzS4H4@WVjsXXS@Ps`lk@t~%$Il{1DbL6i1v^InKPPG6@GUJ!mZrLDvHKUA zu{>{czg*}5HUWLoceuc9>wK?C&G_f@d!0GQGb<(+yH_hVKF8kGkSRMlNiMW11W6Uj z*_)M$+?26W`TdH_C2qZqNX5?5Vf>w3qx$$<&An2`{SPI74k{c@F+wC%LKS4>6=UjU z0?DaqEf-ICv*t#P{c`C?lwAevYXdfNGt%}t<|BqZOd715uatRqghdrLM2C4SR9CAa zoBEj5-q6gVGwmX2f;%|apz7(L8cA-BtKxaxKVoWl^EXq>Nx=ma!j`3mu{Jz3tIGm^ zMii4e;KG^sX?ktM3L-2aa!oiA-#2G0qSvBK3j!1DIk1B0Z3$!z_}4y~J>8|xi(+kQ zhtV~9;F!EoKoXCkHz3HtgEHT@1RF>Gyj91nw+}2oNXV1r(f%{Z*trH7CPfb3OfH*b z(E%--#PzdRRoTy+3M2o(lQ(pjFRRng(JbW1B}Yk4jQ(g^5f&q*sm5c7#SpclO^dzx zK47}%R~@AscK;NvtdyBhoXHymridhFCg89s0bVpJwX72}&Nh;UMy%AS96#>Z=jrqS zZgWmTYFSdIaLS_hr0ncju|5h3A40dafy0F=q{Ghjqgas_6%~XSRbg0@Xt=U2K%B@6 z6pEt24X~IhBqoRfo(Gj-i9)6<7ZF3lsqhy;c)jkk_6rmpCh4e`lf2?B`xFPnO zG8fkT+=^l3!OdfFq6|hj&_tc()sQvap3-n3@jXze0+7JFsD09{+P9o z$pu{K!jugj;{7Jd$WzQy)C%C9hj%iKk~)m&9}6@B(a{#s^NyC*3#v=s!NCDWBpkub zL^CkT(^=>OPE3U+6C5D0dN0TOF%8mOf=6Q{GUo;}WF`kYghB_5IVXn^bv}s&?Afe< zdyoO6^_rF-k^Sk3k%?Cj|7!`d{bn54U#I)9EZTNsd8?KP*APOVBg!+l;ABnR8T5U{ zncvsMxw!t^tx}X7*C}nInf}K!U$;$;zRi&>S!T=CCPUa5F?b%T8N^OI$r>{MB66>< zrxXSi)92p?&o`ASKdb6ym$N-PNyf7e&bxK3bDF5Cmcjo%`+=#flb%za_{d5^^bY-2 z-x^bIr%JY-iNsy`aqwMkmFU!E49qeK6lI*8X_Iht<0-tL!V<`XJqXqOWU0*1CA=eSG$^G<&zV%$2$F z>*k#;5;-r)vW4Ew+}5-oHoU2Bda*S%v<1)35#-%anled0mR&_ExOzLg%_OKx{9Ic# zHC=In?|q!Xe>+|q{-ETzt|LD*{6Qgi8_j-0LmK2Hn)!E@(cw~*lzTNTibz*r-{(Xc zhYH!RyL0V?+nW`+RDUV0pvjelCVscLm|jPx>}VNXc~bv8pXYBG0wBL!G8b)={}#XW z{9QjVW~$I^s5@_S}{Pp3-z4WOic z9YUYZ3Obou>*ZGgg=t!!v)I_N4x~RYL33^rSu}jVm3Np2p>d1k_Js$lZVmK(1VGqi zRkajU;ACSP55I-aso5D1bz8KxHgpVJldL1}m$ZwY=X}9U^Jj?e6T9#(vJCpr*&)iq z#H16GG2iT#fsMx^Be#{c5L<)l5TWP85jwdSqXomwt7SzwhCZCm0 z^+D*`#<|9D0LCq50mT=|Gn}O#NY|$491qasWkWyQC#UWS2p8jAcJB# z6U;=F`F(cW&b}eZ(^?K0tu!t1FP(_4?J+2gr-rf4A|>Ix^@!qvIWQ~BNX@PexvNcV zKKUv=*Wewc?*eUgGPKALoHNk5dboEV{S;cnn1I`Yq8f{?kps*=B#orJ0yryx!WjroJhf8MFvEK02ohCH#S27VkOZK#u^3=NjyIj85_Hn5jpTe#Q!43 zai7BlM*P-|55py_ddmS3zOhx7Uo=-$2F4{UIS*Hs^d@xg3VU8l+Su$(?!dpcrQ=ah zN6id|mG(<3Ph{$By6{UsUZDf|yft?P?KWzO8>6X_bOly3NZeoS%3*`av=UkLoC@Oa#>DbWFoZ4;b%k7jzA5O(#4xdCxIZXi29)BZ_B|X;4WTp#$!P%29fhAr} zqNJ9P0|!O_d#U^TLyra)Pd)>EbTBE*O1-Kr%UHbh&hzJvU7xqSP0O z#rZNV=`8@`!7wUqvEIGtEelte^uIJb-dfHhwSCM*t)qi@7%Np=-c}w7tn8+R?{!@T z%Lz7Kg3l1rONB4cKWkNsR?%<)1r6`k`Ic5gH(Eq>uCbGu_Dci^dZFr=)sMii(}~`= z&3p!Tez8T;bS~lPiPMcHCco(o(a(EBDY;sCq}dnsj(FP`HP4p?oBQMx;KPeK9B9+i zc53Ro-MfOMVgEYHaKQ%$@XoJw&}v4i*^jjFDpKVQnc0Ha_IwFl_GLV2H8GiDEvdo^ zxPYy*%ZDi7{CP0RV1j>fbx9jHS3lwthIF)BiqKPw)W%iR2zFkAQ$xG!;>qpRqiJ0s zoUv)X=T9iRzh?ndfbDraus+=XdhVGDE`!tI!2816Ulf9yq)z8+u`NTi*sNjr5^T@t z)wQC`3Sq8Y%!+q3-1o7)R=>jCHo8(!i3 z&S;9pnj%Qg?hv8+x>bn`IVvN$HO%y%e2?t7Z-8Nm6#yu{Zum;Ale{LSOVT#=dnDr% z>Z8QB3?x7jc?vxn0%gBJEDNey>Jl6vEX)_3`loGPF`*2+nk*$F3Ylg`Rxd^zO?26| z?T%Bw@o9tXNvt$n50N7t@tg3lt?DzxS*cWu#pszS`b=v9Q-l^s>zSGxR}F#Cw72+5$A>UK=`rRU*t>#yMCNtW859yG za|-VPi2V}uQmlH}k7J@8?%9Dhm_lQV*kqk-<!cP^pm5?8*;@&#H8JaETUkJ<3j@dyh%}l zQP!^pf~`O2?+W`c+bPSK^&NzSacQJv=1Q%x<8@8|?7Wg7hLwo0bc;%$DYvf1whoA^nA(k-xjgz{>iP?==muKiB&PwOjP3-DEG)rKMobjWYJU z3v+U&kb6HZ#XA2(0MfEOQI>zkH!Qw~fKrC6ulCenmqNR%K&Z*rps!c8o4?5SJ<_&= zcc~2-hAnxFnCNVd-$-O%E@S+zyl&Nf2S|_6`qio`8&>c|6N-@nG|Dq~s7gA`N_E|( z(yScyD#`xFbWLb9u_hDS6X%{Tj8R3MfS68C?gT*RCS=tQ3q#qVm7GrD3-9|(21fxf z4sOu*Xu@0q-Ex9{k~VVR6hDH2?1Q#aWH+lS~1Hf0dYOPSxo66zB~o zsUe}}hu+3T`!FK^MHm>Gf0dw)mAlf6d6p1&V< zX}J-3+f({bvp?_1hg z;ZKGPB?Y`US3rF?k4?Do+u<%!Ku&WVe9#CsbZj7`&zci@(GM|U-bNo`{GMq_!$C9* z7x0rN9RnF*GkjHxs|CrsqR9bWIrIDk+3K@oI2>Qx`UV0zJ5`(sV*k474O@omYnsc3 z-Xr6YyIzb`Sf6Qs?8?T5!QUM%G?L{+lCPsoue)0D%#C(G+HZO5Tuf|C_lHihEZH6n zhqU`EJ&7mhE$~I`2PuXGzMShIAw{ZO-bb_XkUo9YT2C{)^Y@%P>uT_iAJ!oHLl~1v z_=fN4>OKcw!oTtSD^pKhS@-R@+if}cTF`5XX4ZODjnltt8Dl- zQGGNdGIwIA7a+&R{7b%ey~S~}X~@vjh0wpkc5>9UlZ({X)-r5eX&sUvu~ zB4L20M#^sj>~e|g`rKYs+L>1_b&}d-SNaF`eGlyXSHSKIVabi5n2=I-a_b=$n|1OU z_I^FSQ=yqngNvQ!OSp)Zt=7IrZ>8nV{)5W5Dq0({pF)LC9^Yx?)i|VjgNI6$c&GD6 z7z1c!dl-U`;t+EmM=@1r<2Jh7zPDeMGruX)F8=gUjC`zcZRF>BteTDu^A-rC_?XQ8 zh)@+HG9{TyI*-7Yw9Gm>CGMAylqqjkXiu$47^ilX)`jF+2sbkof?uiS=f2?w&Ts6F z^fIG)gY+Ce;u)M8Kh< zhznXsptB2zQYN$)n+NxrDw;5m-F?VaH>XQ!^5D%UoPl1GgQrjcsI{X?Z-4gQygJ$j zArNcwl8e!*xyfCdnKq6D4W9#D62((U(jkpYP_U@66T10x?dE>IP4MkZeiz}Nk0Tri zZGLNn*% z7)uRhR?HQq!wR(Ldz@-oJtm50!nZtCqAKv-aS>xf zX>D^l{g8KSgT_kl9wn9dn7+nc=`roenfS|ar5j|-_ig~2tGZr?xJ2J#Vo=&$R86(Y z!|~YtT;gkFCyc+wWQ;sdojmUzTT@JYcnrUIIcPS5aGUy!?fDDOQYvKgm}~}Ddd*|p z60zVstgv}=UE0S8+Qqekm+-rD?A~bfvz-C85cfxzt?eD6EnAV7m$~utr^GPv(rioF z0iO%s=hBF4=pe|yHS1PU)M)d5L;59DlsN6D!fXg%%}q+;$CkI7WPZ~Vk$po zLBW-0iA^`z)U#!!xuXLk@yikAbC?a_v3Za&Tlri9XZNK&+rz>dMT(Q_^mSf{_F|pP z&x{39G8EbY%GhBIk255rLoZc3ct5lgs>h<;bMT1Jri?CEAbP@7;99n<@;V z>(95zzARm=G;X;PMnCk5=!2QT&= zIrul(CplgAI$@|3y%utKZ=@ttH=L?ja=$&bgUQP3826N1wq?-V7QP6rbJzVIsK+Hi-eY9GC?5yktH zl4QXDx^N#YlOpZ&ej>;FOpfcj>PEnlHQ)QDdDp39eA#uVdiCAjL*=)>V9Wn*KhWff|E9--C8_h|ni1FAN0(qV;WaQ@jZr(y- zg5YdqvGoU)YM#dpgi^@2T=9U;<`%J3%j)UIMT+Zf+X0RSUe?q0O10;eYC-;6g8g9e>J zIhF-5%dV4!-3X$;=#ho#-w4Q}J__d+z#Jj#+gl%=k-07Yh>e9ffPrC~z_GX(4>h$U zhZHus$@gE8AGmfRR6RkPfiL9QM`7%5P|>xIjf;$BLiFkMJ1ZyB3o4P~kd6IBLLN6e z@&2F$tA+5{77+I0B$JP#NbBWuy&-(pa{s4p5;@zr9_?~lXX)$`rjvi@$u$6NmhY2R z%8-z?ea{VJ6=GsxS|=hQI*g>>h3+d&fU#VD?)}Nw-X{+3V4qoV-ZAYqh;9PA-C`h* zJEJxOr!LZB8$*}hx!LgV(j2bTe8GXzrIrw3%QMO!uO0E(pGDBDzxD_vbFZr6jAiBE zURv8g<(PFL?Pp}$IfR1@O~u^AY(%b{Coc5gU7Sl&)QY18Hh#T=8kc!4v&C5F2mr zvmJZuo}-WrtMWzh>0$qzh?z;L9y)#GeJGMF9sr<78uPt%6!nen_q8^QMUC3xnY(p) z$82;^uAfI3|7@4Lqw-6!`Wtc}D$y|mgQhR@#o9;04!1zo41+%daP(zp70O08*Ix{q zdyg3P+uWo~;#|)%o85r|{U2C#cyzq3(_TA|WO!0_GvzQV*;{d<}{}}Dy z7ys*fqbHlR``3DK-lntuenCKY#?sQdHtn3$wVg`sqGGi&)HHgu6sr5ae}h*^ zvW4s0)A=_THW)k(l}TB9N_E!-*fUftyLrVFq{E7WD%_~VwifL~y(|`L156IoR?_Y= zZ=Z(k%5*15HtP4?v@=_aKQHjGrzGkXru0s|+^$5H(AaNO;+$vCsy9X;0~9#9IhV9L zJbuW9AS5v8%$I_VM3%9aFZ$3dESQq;4ZSa)li|MJo9W>%0q@-p23}tly1P4?Gsg_u zTW3%M%eTua;>kpD>W}aC5SFxR>IlLPc;I^>4wi@vGnp~8MTXSmb3{Luiq#@SyKvKr zwsYxgeu5lTbW|KNKYjdMVBABN{)#%JBSF1WBs+4O+7|rkTj*K1xw=$D7am&@YC-#w zc>4}x;2P6jClhG*VEo#8BM%vl&m%Sdy7;Fx`Pvs_f09&qo}uieE?g z?De7Z;rC$~c&B1C!h3{dvYMXrFMCA1ev4RYmCG-gM1@e#+hf5)HQX;cDQpxR0Ny{S z6*)Fh#4CW>h`f~aY$6JP6OiQU6UKqNNuT%K$9hh|Zf+(>t|={&xusGZqe#*u5<59j$^ z`y0ZJ(F9HDc6Cw5=1ay5e6!*T;#U1;L`!E1CzWo|{2qjNu6P;ysVUypMu|;E1O<~I(1At0%~I$$*A*jH&L7!ZI%Yhu37EhZ)4lJXzzt6 zfq%XGaU;13K2v8J%LW>Y$j@T%7YgNaK|`1`{tzizpA@#*%;(Z(5}+$yBJE+d->GEL zU}e*iU7`HjAOKrtFi=JrtBGxzd)|A{J!5M^fp!3Nx@?*@TAp!pQ~g0BC0MekoFUab z3I6wArm>Gy=N~1Q@sNWlZmX=8l0G3iUy5ZF{`m|l)LiR#wLTMF7fZ6_AQfMwEw#`| zEwo9Ap(F1eDA>VAA6D(kh;(kwOR)W7S)~7Q{qCT4ebZ*VSPkdA=@OzQw7XR&5t|;c zvl^Nl%M}@aixP!<_(?wXPuGr4p#wy8=I1`-ZhOOs;q}VQv2Sf-Mt}(R0q*Q>V_5Mp zpB#>b2p8c=XE%|{ymF!Oytbs67Ky3g(coe@ykzM+_{%;S_g@^}roUG8*XV9?9!-tm zagaqpQ@o^IOUWOXw=m*=Fh3=^tND#8+|JL%9yV2j*WG7XoNX2!H|JK;2&TTz;G>bC zDL}$Jv14_piu+y;T2aCib&I5eRO)l{KmSPc#kF~}=OdIM=dw2NE`Fg#^;h+P{`o*a z-q(uAP5$Ll{BtLCz@N|d0;Bad)lcts<722j-6>WCur>$!I=IWfiN02jdI))by{883 z=0!(*dn?Y#d4TFvtX`w?*Ap<^^L?KdA1FaSZ>w<`*lg#1x-0;3GN(s*DF@N^4Zx2R z_mkK<7zgq4m(+==8YeT#X3KqKYy3o*b$_1s&Rsg!1giN_A=$k&Ff(eZ9VE>ZEayw`irkwwu z|6x~=_E54!GZ<#Ae^RY8oD>^y}gmZb--FDKh1B%31?`l$^YLqpsrS zXY>BH55fIrWfg39Pl4LZezoPoY=Q{vo4E$5n3e!_6KlvW7}wqFC6` zpLnzGN6s@xZTN<33Bm-SCum+#|IUkiB&V*2gC%Yo6gXy%-6+TZDkGHmJ6R~zM0$u; zJJmo}Cp9mNAH*%d%c0Te_C9|2==d}%=em?%RXa|vsTc@|d9zkhZHV5_`qj|ART$}B z$23uf@_jAyu=DEv1{C$5uDmzZ{hf{na6)g^f!M@-yMGO%O7vjv#ZMRvZG|}?I_(S} z8lvlT^$Q{=$`Vj#988U^Zr;QY{EjMNQsb^qT}QOQpbNAD?#r! zCZJ7|#lL#`2QUeO7Mo_eab|1vr0=SaLy6;Q_KH5AWjh)>?VL-K71JLEV=8~xu`ngb z-E4q?ftR;r!Db&bddL((vglJNKVH2GMVf4hX{0tVb}&qOm_HSI#QmnzokID{?;}(J zgl+Ig_y0f+qseqr5Cn01{ho?!% zX0hI*ild$E5U#HgL+#9Hg_}ivP|98MjsCD zby6{T7svLb)NzXe97ckFkywELr}x<7wqJhIsN#~mm7ZlBv=Z{OUH+ufFlS1Q5@ewh zzfh)935r8|BMB1?;Uni(VV8{AT-!lsAz)m@$3ii7ZVrA(-LVfUC{FO0H&p)(fEyB{ ztZ@GnSh+i9KI10+D@~~M`b`pd^f?NA|4%&{i0MDG)a==Ze@2yr(0H4~cEDi6zrnkj z_WA1HPf>zaEB`e>{`2hryK_joP2X-2BPVz(-4dU~4hJw!*QQow+(i^$wH2rRuT&KI zB5e##m#Bu7CL#AYkppwc96-LT7$i>}nn5IRt=Io%F!I4|7VPOnOCu;Bhb;gTU7sCV zjxSaVySQ02JcDh8wy5#2{go?^{W#K^Q!)Zp#&J3nZl!2)@aWw5juj{cBW_kmv{gdzEN>5)ycEF+(wtHm7lR-3Q^k1b5!JfB33`WJcq5m`Jz z?>BZq#@<8p9L(j<1djnxrYz@i*()}z5-y^JRO`zKHQ8@LFBdhh+GlNd6XA*+Zfat% zxi}yz4hjzlGaHbmYS1F>We);h2EX6A-6IO+;9#EdeMpQ%aZ?aaCE3~e8qYZeLSf)# zk2ux-fd6(FsjO{%!@s4?MrZ=oZu(Y+FTb1Uw02fOYvWqps|IZp(Gi{#T6vR(t!vb+ zwb{S_I4+wS872Di9r=kC^+C$wW))RG9q7tswbO32O?_w+H!p2p`ouEIMj5FrdNqHl}0I;Yjy*fKfC#r;jC;o~m^ufm&d z+GBqW8^b?xzJ>~v`u;m>IEI&K%2l2_{!Rn;*1h_^dJCWY}Rxm4B3m_)G9nRaF>Ur$=uU5fJ{ELe}M zpPq@%7f3myR@a|dN8Sy_-O@DLRQyO`ngz~zp8LET zG$2koP~d7k5Y_&^xyyimW3z|0NODcl@k~Jfrt$SwtjVeBr9ZFJm$L6Ii#*xG(QsXa3 z{NaGQr}ibW+V^oX=FQebiXN$6A{8~e03m<(*$H$lxvI%}zn}Q6ebOZ787Jqw0IP5D zjK`%F^2FJS+S*CkH@oH&Z3A0c4gjm)X)Ir*G=k9A!%}HdW{$UKea92l`=_F>n~SfW z2ZHy%CO>$`-;&Mk`cXZ%NrPCI{p`=!UK?-kZ(pia*-Vno1K6m!{>T6#-2GlxUfYh@ z{2LjJ&OIllg_lV+tMja%|2EzdTOtJU2)=r3V?(vq9=ji=r&C~VZ03O$oa~-|n`;b# z8Pne!ye#*n&NA|bqF&oR3iw*Swvz$Bq~k1-dYpzvE#AWrlr)}D`Igkf;jz#c*+04Q zaFSP010dPG-sF9swE;;nxJ~LU*m!!_ou`QBC_8N3&;2Qb!bJz>BYd0QhA} z(5m0Y|7n1T3Ejdqq}eIrfo{XB6J4H1ev6mah|n^fY>13{1aFldxT>Ua(C&dLQJ9s*nU zUr+Q}$$0vc0yDV6Z*mULBOC80McNHGnb)uS+Jr>noJXzoATQ6cII=^3 zN1%H0U(BtZ@2T*;l0v+;lUJDRqnMCt&y%~Kztk8nUB;J@cs^^9akn8!T+Z+3F01>L zkR`@>98Jt-SJ=7Uw>&jwk*&0t?_Ev4?iLEiI{YrDzgw0JUeY{?iEDzb?p%QH-KP!S-VAFWL48U9UjegEr9`wh{Q@{ z{65Yx38jA5c6uzuk0X!a#K?Vh-v&2){9191M))5TxOZG2N)w!vl$$QC$;@dH1KNQy zj*a{|8BGbzX52}6cKy!wKa1qQ1p*s3vdz&ARU@ZUkTQyuaqtt{A5^uB&3oRhPWQqDybd5$-ZjMvy_TosngDUj^ zyS=+Dxg6@rUh3Sn_g%1LkG#75Ga56Z7~e0~|}t^KajO`Jq-Yv0Rz{ecQlofkfz`&1e#REZ?1 z+|-=irOwx=^)S1~XM4_|+u>ys*Zvqor{oSs&-^uC^=rW`PjH?SyOwR~C+rB7 z@9D@rYfY;gBsir!v3I#mC(#aMp61>(hsijg(|WndFzn$)d^39${i9V-v$z=}X5V~w z)U4>y>wqrRVsJQf;-gxWM?&j&U`;n~ejkUrgn4y^45~ArM=JP&W%6Zi+2wJVk%XwK z@>Z5=4Zpv1FlFd(#>4uz5X)itst^n#0ne@#E-tJ( zX1Or^eY*H2>0-e1Uk`-EnVj<+GZW;3gb zO-_>;HymGemnsI81V)LXL4Lng$2@AO@R8eIpWvpWMDNpt z9&*wQPneY0=KbCBwy>Wutl}D4V()kDNXE+#Q^kVPnq4G4E56s9xKUHs9^R(bEPXDH zyuG^pD7-mu;{e%f)bY^&-0@OIoEC9^HwP~7s+DYiN-R4rFzVRYWc^x7e@8GergJ?~ zms4>{^|*Y9T+b3pM)D5zb{CAvt9E!rf~nav(h6sy(yGY6ROc_zd@m>chS*Nfv%xw0 zmqDvThT2qH8QsBOrEnJc>=gg=HImWMA+X4I))C)jYWRqFUPGAf0a`Px_b$gFkAZu6 z-_cKf#P`Kbal>9U*|5pe)6%E#XdpvgB+jVLMKM(JvNcn|{^!fA+d5TKX-llu`bCA$sjj<$JkrmE~-H z-p{Drb~^%Y`P)QaYGrwzCET@{Rklz`;b-=Kws7>r;j6h_a<8WjD8=bzM}1_Nx!1>r z=D^^$Xs@{*znm)oIBT?Cq5F=>jz{W$E5d%4u}T=T_p{DZhfC{$z4tXE4NODuwX(FL z{`knIN5ki3Ff&Ks8?KT+cU?yDM_h0?(f!${I=?R}JRAO0YZq%9fksNX-jDBRM{5_E zpN>arTd_shDT9{|!vo)@nT4oRtBTEc`j!K0vR+0dPT0@B^Sn)IEDsU_lAK#!yTYHh z5{+phFZ=|XEo?`yG@gtw!|*&i5plDuM`wH#U{@UfCxJFuET$SZn`P+{-M!?G>cE3B9|U z@@6ahd-pd9QoAlNzptJL`0id*mHC!ciI1~c%=Zdf@x7#GW(z^IkMTK6_sS)CGk)4d zY(-zm%S#E`cX{zTHdx6@0RX%SX1vg6-~_XWmtIFBTm-^TPi5LB4)8J&>zxqH1}FS`}N+sNRBm zM*fAc0HU7DRs>#x%4>1{-5-N{bM#BXGJN z4=wYinprgbBCj~tuR?e^_-bbDS*(k-iZT^@gH#f8XbIsa7ARc7v_2?4Jliaa1B=c~3&9()a2yrifRZxw^B3FW5K0lv zwoq|IEh63(7%`Nf&`Y)7n+P=zL-^KP+g3jY%zQC$ZB4!5vy1A-P~yJHX|jAAzOb3-x7*t7WygP- zSzZRXK?Np%jGtsCg)gDs23ncx%go0{3ipXILI3`U`S6!1ZzX@Pt;W3v#e+dIoxs`U zYp$YC3fE8UdBkMIwpFoOUv)bfw7wYh-iNVUt%8W9LVfSO5X3D=W!O4KGx04;p|>@@ zs*F+vsKUHY6jXKFQ?ts0CpKHHo<@+5MAK(3hQO5PC{IaHrsg4S3XXtr%TZ| zVQR|rOx8=hw`gu=(cRqM`s$$pbz*crM;rIZ61cp?zGLBzL`wIrh)8u9b=Lmv`PmwvZ0XqCEE|NU+E zI&%YwmnnUje#+h{_5)Xb6?1Si0s_5`y*<{{n+Jeit+$+@ts7@h;M;TH%OMZA6c!m^ zN;pbR#z~|l-aZ4w&WWdm3ED(Plm3jk!;p=m5T*l?W#f1~gD6dS5Hp9Jo^yYf`1q~+ z3<==<#O~e+4cj$ckB?NA(r9xWKKrY|l^=JnRd5!ipPBaSh=q;=(>7*5)&9*-LEN-q zg?%yK!Tu!Iy_C4Nl?|z|$bI?vG$&)Aqtmwxq#o*ax*wk-e@joSzkPF{w=gV4|17|* zmsrD4kYkTFxU(K<54!pn$)ly0-gW$1r!k?G=(v$Wo%8)fXWR!^X4mX%I&u>!^ldHUAPkMKi<~>8^PBt88YMC8e6}UOwGR;r+aIV zy1B2Dti&B-SFj=D2#^q@Ka=gYMzEzEo3um_h?A&L9+o1(7bA-mYP=gA~Y3x!^DyB4N_b#uj26+)+Ew&QoX|0R;3N`39EM&>PXRSMVVG|~b^ei)o z!<-7qRdO&$!AmsZ^Z*!~zN*`(B5Ucd%L3z)sq&-~ToiFcnf4)C{3U6>7-O)I!r?JS zj$&W;T`hjK7-^@m!4~#g=Kga!t&VSsKYw}Y+<2E$3Jlw#;1vMlFL=O5$mvB(yG_!Ln}^69(67i!J8#Y`z5}I<|9)a66cD`4Ar zWN7hm`touED!=c2-)Tb>MDAX<)`;D1qLz8CPmkps1hAOSDZa1vK3$5u-*neTZ~BEO zu;B>`5V9H=m*Z>`L?nw-Vj^NO;Ad8{xyNz!g|nFusIa$13k$z{8tbeV-eu8DUG@vF zn5n{4j5$Abf-hgBF0bd~4^H@N?4*-*vkL6c5*}}1yMyTKmp#To8X$koN@Mh2RI`Nt z$S^j-RqnocqtvYb!j7e3%8Ei8rVrlqm^ghk+^*&zi1wL;~H&G6^pk}01Ku)`IFk@W*t<5bN z;_{;0v}!h~l`YySe(98AMgkw5;B#M;=$Fl&DXPpfI~uE7F12(D`~6~_n!**H#;ZMw zopU#NW`J`WsYsT6$6Z|zd<-o}%)guR$G3iZ^=R%Z_wLf?l zZz79YAuq`$Z2?JOO>jotO}7E(aQNs=0p(Wo2%c+VZksby2I zgE=!N%tE?aM6vv({`>0+cP3Ba0+Rh7RFo1pKat#-OWe`F(q@@xsR*IQsnD#}jy3`S z#w^d9jZ@7yh*&?BbQ)(HH_~wMpT^MnFgO0(ySm9;ZQPpGA-HN>6uD_TAusX^W$)1x z+1G^5vJtzhseAv3WBFJOV#KnDuyf_+WmR7D++%+2qZMGL)kpHX{Hb5M$2A(;SV> z48ARh+N<@9l*0u@*;E(rrPw&uT2QIuDG_}w{r8W%+l%Q7a0u9%R^1%Zj2o9ic5-{^ zVHNmz5ct8c$p8GE>~z?#cC_DxL>1Y*=RVV{jt+nrbm)FBgOTDjW@$G zQYGAaWS{>J{!u=e!TG~3;lOwM$Um@mEDBFltIriuDAT;K; zZXWHerQleP^)h}qp_C^xv4D9ZhlaD82)LKA_R`{~{tn`nKBTZreQ6x_^n=glpd2;ww(&b87cBqb6it=f7G2yV$W$ zVt;4SeLOWZ+4+vh)JZN_-p$&z5JS}i2yVg!hCxm*9jiZy^U$sS{+{4U&duA9hOyvC z$PQ#9%E2a-TI3Pm4$ai}>=rs!;W!^&yv6XVJ`STi7)mw&SbQ7F)^t(;`O|@+nHp{| z>qBpb>kK30Z~a{o=zKQ7fekYq%l_tl!z}Ws9boZ#v2A7N2S{cNB>wkSXVUw%ZM#W~ z4m&TIsMY^9%jt3T?q&P-`okadV=z(ZJwBlvvRg$rrmUyJ}RZe#Rz_6>mmT^|CBK@j3)W6@bii{u|0TxQB z1pT%YjETNoHGUY~d0Q}f*;eeoZ;cS-qEOCEA^<#q`Pm2I4Psc^vZS_;znyO%kt=JV zd$}$7Y4K(Zh94cF0AweWTH4wo8B*lw%5}rI@n0@yX+M^22rl#8mYp>5W!Uk}TTfQc z3?N7Ykdlw5>t5%J`w|ucoHzZTY(gzTv>gIEX;U0Zxkq{^epEzxM5o5+Lr@i`!axmc z=-XU>#Kn_5q+R(v{&riOOX5~(UCD!V?uVe^Vcc{z7U(~CqzL&bl*l5dB7E~|eyp9c z11tH?r{H3jJ^vn0;&Vhp>D-rM_m z3`njY36e-?^e8xfawJ_+p!B%_2xP;O?BLNc)Um^!ppRe}+jHr1aJXyZomV8z1u+`061vRF+Y_UylK34BznfcG_&}*!){ZT30s8| z03cu-8JkcKOv4r}QEDiUT-v@(q&dbcILA=MjkxA)&)L znTdzTeDl;ZyO>Wm*fFjm%(de4c5zZ;N25Rk)gusOU=N4Eo4(|*^z;E?Q#i7Bcq58r zOIUiM&9*z0amA@d9Tti<2 z!O+Jf_=GKZj387^X#+8_;PQG&t1h#7Mz9MDWSe(LXX(pV-yx5qo8-TDjebo(=)YKd z`@ta4aEsBpyN0v!81y?crBjHSa)ST>XXl1}qc%?swS*3jyP4Cb<$4kb%q9V&mN*5T z4Iv%~Nzwj7X6Gv>AyD4ZFT{)hYVfS>F_J;l{^k!d|i161l)I6RAIDIZY6i^m5OPKdX5*|dq zRdww?3%L6G--6li_NyUmDfEvhd&7fIcVd5wtd<*ED=ZnD9~7zW*qEkJ6EKxkS)lbN zSdc)>ceDgLw5PTU*e!|P3b;&T}P1tl+ zuT)}XW23&jnkO>oy6d1KF!h$p*aH;XK~XeTwtYT4RWN1~Ch!0fg~gUJt=jdh)XI>U z3IA`PL_h!6YyN{R9*o7$CC+^Z1Ti}RIB}MFt52D{vVM;!HSXp-h7L&1nm87A2)sWT zh=N#_t`269b*Pf#zUO>JVFb9{s)qM`9on(sbJ5JkHQ3W70JxV9el=Ls4PO|_=?#Lb z;NXK~RyX`$A)>6oWi)Sq2Y_ilA{Xk*{`FQT5m?tSm?hrmack!UUO?LJ5cMR5U**WYqlQyIJ_KLms%(|EH<9d_koBXw;;5?&?9OfFS$sI_ph=@VJ`$ zBpQPfqYoeQON{PY{MSH>cZ4QZx529{5IA!EjVfp?CO+_KN&g%w0`<7|E}{17`5{>L z^@IrY*J73h5wm(ObN9v3=-q5`&!3$<2M>vSt)r&w&i0(9%@#EDd2m0`#CAWz*PmqC zZGVwGfoAV0QMeJ~L-Rd%M2vd@zn}%fdgIa%ss>J|K)&hDWx15{l`~RYOTZL?(fgEr zcbI9#lIdxGfX{LQt(_msG?(aJS(TWquBMtwkU}QdoFlqqt}+kVlmOAZfdRWzZ%E*a zWwl|1+gIC4UzPEjY{V}syKTCC!yFts=UZ+)pE2t=7;I@(2%_nJ(~#~m=W)iF>7v0I zIxaWfayT7~EPj_E`*QLNI|I&;p}!~?9#YS;!~ZslD0KCmW&0XYGo_^Bgj16@0lDOb zfjH|tav;12Ery8fIxWrqIwr=b<#Qfa0}B9)C^G})uZXjL^|-eo3IBcc3bJMP*qY#m zC4PEV-kz$8d%qzK%-28U6u0~B7R8r!0aj%R&gXAOmzTOO5rAF&xEOfd40ME0!cup1 z9zN0pxXhbJp9Q~5`k3A-q-4b^v`1ZI$00xCk&}IQXG3l=jsY8`dVE(}DRHr09^#{!3=9-T|aGH4Pfg+f!-o6FDe`|N}^TBiNynUScK6tY}bA6U5v zGnWwh*&m!=JN`qG^r&EMPzcTy=|_)k<@WJv85n%mKjx_-BWLn7Jg^-^uKfC z;%6BZLcDL7S^62-9NGO*C{y}RoGd-QvvFzL(VxPj2k)f@)~cn}ipznX)GvA=6s$gU z%g}_V!j~+s#Lm-mRX%Su^jQvO39{$Hwb2FsYA2%q-_8egFVvqvE({ zYQaW3s@g}|M!tVwE1Mc7$00lHYf?F@KRPL`*79vjw@{jLDJPM7t#wr_$@^^`=HxIl z$F}IOftEdarGpYg?qdfVsmM{OzniN<*0kjwl%p;2`h$Iwh&Yl!`r3y8nd5#p%uxDL zSN4Hk1=9y5FCv+&tx0rs0k>`~iS_X-Tw7Y*>&$mOSq zKK{`P&v0f1T*xm9&t9(N)ZGxu7WU=#QpHoJ=^I4MGH$^*-#lraAVS3)3oIoTP5-Vj zOQOs(G5+na`g5v8!in1ME{XV^+dvaiQlIY=GHd*gxM86iO<%O;z=Cfz%kpGP`o-V~K@oZsD0N7& zZz_1rVP1Z|f|51%T*UobXR+>@e3*GyM#nSOXn06m+3kcMMC#^-GSjXx$*vf%PlB)dVh6wuc?6rbYnG3%3 zcAdi|X5?fDbu|K09Rd%={lQAPGQR`b^GSt2=d?*o#&&cBz7)41*_f>^YZOluc}Ou9 zAkqBp(_Kw`_elKZss?V?pw^>ho{6KMSR^?PZXcX*XYnhwreNiX* z1g+Y8sG)ED<+o~eT8y{wQ>seCW#2=ZjFkddM5A0LKB0bju(9@Tx#4KZ?|1f%Ix;Ih z0lKF$XKlTWdC=99MTL|$A7+T!-CZVak9umwybWDZ=<7%OPKnPV$a`KWb{~T)fLrzA zld|G@)RbL{mpGk%-Y*Xn0N^cYFbg#~Ip(VCAd^q-_49LSn02wLhmE!}mdHatrtqEW zdtd#CWsDC!AbR5PS)|yL=L=C`FJs zqVBJsTEQWRew4VSLCR=!pOIX!6be~B>{6*h#8^M%5dDHxoe1o8o*D_>6>I&#?YD`f zD%V#HTZ1;>r#d1^4I@l`gQB|bmh);FChs&p);!X15$B+7KEB#)1(G;nj^p`7Z)V*g zjTAW_sY7mf$6l#x358OSKJJOUEcP^^qvI1RYc6_lw{Yi+1Rh>QU!R6De%Q-q zKPo64-c;SB;9mWfRbid+p+U2Bx&j@c*ti%c7M`xVzdD7kn8UD$-6Y*%YLx^fcVAX! zYa6x-I+2LiZxN6B=-1G1J6)0HTBApccXXOX`$dYTpO7<#iOU0?@=M8~I%h6I#c+}% zL{>>FN){~$jBi&%RH6@00>OdpR*x|IAzL`_FsBZqylI$XZuOK_DfNMAy2YdVm!`v1 zWxBXw(bUmrrs$+P?GtzV5@V<7a&G-Rsj7wOF!9iSdro}~4JCrj{jrduf12SzQA4)j ze9)XoqJMk~DjB%Cx3SvV`g^}7j?-|sO)42{8Def>l2D{vbL7DFKP2g9kLS#V zVfq%Anu$WwHf=L@D!7S+#W;8?R3KFh9!S6pEXcTLq};GMtkGi(Rs6-V5=?6OI3tr< zF7+pS4@b}ZABddsHAUR?T4fvrHALysG%49}c-<2}XS1|W!bOXkv&X)d#1%4Ru#ZZ9AlU?|GXCNWUoh8{AD{}CwpkdiMf3Ssl*WRZ5p_L? z6K`w-)MFJlEg5Qz;p`N%Xy)6av6)~RS-7y>l>5Zgtz4`!Z;$SZLlE&%Zx)R+TDC}4 z&mDyAb)Jr z&$$HJkFX7fPG5)>m5V2s@Nqb6X07EXhuKq{IeJFhEgz|xCc>I>6>U;3>K1}je;s>h zr(fv9MQW8def{z|8ZJmSvlRQ1Mv#Q-01t0h|29K2HMdVTS>9=ciABpCS)4-cNa?#m zrIOQ>tZej5u34+4b^5V4vZ8&hZpWD89O`e~G}so_HhaC=FaSWYh>Hk~AGv^nmcL03 z={G~KYn}wX`+Z)Sl9qA~qrmEb8n;R%=jzv1+*rQ#$_4p-dNigQj4ERSIh?1lK}_ramZ>b|Ab|F;*Sg1UkWn(|I;NFZ!dh~?Ta^{{j? zbunL{CPF>MXywJQisBjxB(Z7yHT)mRx0y0WiNUxZT5|=VRQng49Zmf_ZPwH%Dft?) zx)g(Px(Z1aY>1^e9k&=CDycY$WCu4oHe7VOt51SH11F!x2Ss(uVM8rPm7Cl%E=?9r zRK18~re^7GaVk7U?(a?dF=k=nb&GbmA&IpxXyZ)r-)rQVuw!@mH5@%F4TG>zc4oG} zh9~{xQ!LSnM$SN(*&H@e&!tOdoE;;I*(!k^)iQfv!pGq~6b^F0gL%b7ks6Z}fx^7FOGlRXOV(2FK|p6pfFc6HY_Y>s@bszD1=jru7=eVKuxde)z>&9Ddjxfu`)m{DR#~zTq;lzI z6;K+-=i! zeeQm2dA(>|2T`g-zdGrtEo%5J%kUe==Yxbz+0tHC2qe{|Ntp#3TO6iefsPth&9PFI zX`3)Fdr!Yoi0$uJu$MpYXDk}X)to&C27nth#Z8TC%*xMqf+BmXuQoh8BXp>Q5+_}g zG+^r3SnIHKg8BWxZ??%TSk!reOF)?K6oak=r6dRdh?2BWACalCzc+LPke{M>Z^tEt z)yipuWB>7EtNwzx2Xs8Z81%%@q<>wfya$^6c59Ifvf!jOakd$bw2 z^JNWqG%zCqg-(fx$b6k49joAGT&c)E*e&udITlHsW5lvEEJM^$s{_p}5OwaFluP*D z`${0ZSdS_H{mb}Zb8ld+;@wds)$saEz~P2@$wv2mTU__fkNW4AWjFuY!*RN^j|(p` zEhWk@D{)T>rk>3^t~4db8m-Pz_au&~HH=Mv``6WzI#~%~WNE0jTay=U#MInViX3}+ z@3^iI#pMU_8`NAA<(M@W)P_Tg!vSf$ zc91;(6aBx|0Sd1T@lx0wUN$n7BpSV>Oww3zT_97IYzG`=i#*2x>KN3yAnNRG`I1FX z{^hVGew>u8Es`Q?WP~u=y33ecXOzMTKp{sYG2?YUKRB0QXkqWx+&doJfmDE7Uk@AzHVo}`|YxNLFfM`{Vb}$qq32||X)%t3T zxG1=s%bORrb}j)fX0_lQ3*8~}FISKEHml%xgC;ojTpdsrmo)m*@dd z1(i_&+){1L9z^K;vCG3iorNBC+)vTcO1G_Q=&J*u*BNZZd%lz4;2MT@2=BscgB%Z`dTr7}$UjLppCtKlljs%Me#sw(o?$X=j z*k}ZrLifK0m`pt3QmHej*Y3X~%hg2xp&dLd6j^l07wBB*0PWWQ@x7Sq?|R9k2!<+m zPqjduBL8paz;Huw|JzOQ?K=P0uZQ*j>)$%(IR^-ss=?*vCY)WGE+0N`j)M{&SgvEG zn^9_u29iyYQR4=?(2PK@H>HFmU>`v$xoLpt6;j3?K?EfdESMw48VDdu1Ow3^je=y* zq&@EV9jGdcTf*%h`W|#=XRpIkSs=#Dkwq||2b#pvy0lO;0*@A9?(qha#NB?6+d_uy z&O%|{gdLs=%!p~+_$}TM>V3%Ll8?W8qs~d_4!jsGBVhA-bcU$xQE#}A`XM^mVTYI4EsFg~~)+qB_;|DeR`V;Fjwo zZAzLZmFxHuY=4w0Llk{V6PXC49v0@vbrKqbn5>GGFwo@&(-+j|viQ%CB_9|0@y&GU z05h2;(M+!mocW6(IOyg(+?(lxJ|7AA;ChYBgn=YoUb4`6P5{3TNP+ZcFF#gthcj#G z1oh#CGKGd0!}{(2gRZx-@m;LEM}b9#DY|#8@Bf{{TkG=&@r@F;sxK8L|WY&I5aQFaR+u&oX+Pcsbg%Npg2zI2t($ zGQM20Y-<{G`z}HA241}4JT`XlB`$5FfY6($EM+FV1a{dfB7iETdC9O}RktL~w=1BH zM-=O~q~dFA@4;tDbCN`8nj{Ar2>7pZl_JbLKQqQ9<@XZ;Y_KQ=lr;$bac=6z63=`i zr^Jn&_J0jE8u|;x;M&eHp@DQWYAPI3qAFfqNJ_+AYJS?1fa-%yWy`bz|9lagY+{Kt zgHBgjAe3RUDWXWcj(&chZ1gZ-lHmKP^AEgp`&`g$`0(K3XrRkMKKb-eUs{kA9~xM8 zd*xZoR`6+$FQ3904WyUx0ZsB?NN0NR)v#NTfTPDvlt%0V$w-|H1CByAq_62RbSGWmo4P?1m6}L`S=x-rdWVAdnGtoU#%?{BPA}~@&B}phcql{ zW6h_&$am#Rs~sVH>H{mZy*q6GfqCt7I4C%PqwEyRv{YCbspS4RdSxgV;1nN8L_&BI z*x8s^h9@7xU{)t_W>I=9#T9)|#04iC&^D=3a0Bq69rvJ*n!UQQ?3tA8@XF>l&aOv6e+iwFEaA3<}x;kx-3ILL$=g)w?$I2y1K3+`4V+H zdRo6LTWF~VLoD&A>p1Rj>w2eg7X)HfidhYgYSH>k9Z>Bo3+N*iEY@TLiE_*ZbFIR_ zEJ@LD-7DI+0WKCQDe-HSi%;MN4vGPjtLO>n9@4m2ow}upMG0kEhZj0%{ovsUQ(c;A zzXid~RCo#T)3=`v-+iRdLG5YKWqe!FIfw06!lcZXfyN(GMR_M2Os5lD5O`9Wx%nMl z?r`8wUJ9DRyR?|1Q6|U`By44N`J}&65$PzlvO1fV^{8J+Z8elwTWq%%fh=c35w7R~q@_9GI&K*Z^K~2MWLl|g& z&v<)zRpnplRb5xQL0C6yj5cD?Gqf_q)@1UrEHl4X4k%`06pM}fJzdgv=WmF(#k`ia zl78;P_4-X>?_)jB-e26oK^-h{3uu{AM|z1P52mX|6s|cfqd~hF+=6orX0NwiWoUkV z0xV#l3C}{q-@giT@wkOE+qbB zS5cQ|AKBcIR%jl&(y37>HJhFx$)_gIEEfKj{Jw^Ahq)!v1sX;+px;<6;%mMx;niSMIKIl5dJ|KY6@8 zN8O~6i+Hvcc6YDaHddf9y0Xly>TfK}pIg!J$FG@Jf3rXHY{2POmV?E2MqAWMUu-5= z=a5qmnGsO+Dy?X$3!F-1Ol`iJytXoqb^2{}eFxgM^Ac&`rT@x7#=nvx?C|!>-7NPG z@8jjNezrk;sJ3CDhMXMTrmdabd0SW-Q9QvRwuP^SOhGL6$($?O;a8}#l8gjoX`deV z2WV>XcY3;9JRXyqhntu+j9;0R764fK+0a9dhCm1dV6gq%DF!d>;HHo{GH!os&$54F z_wDqM6ggEsPu;cfbwWxIIv6M$loee3zj@f?#6hul-Tsc>z8B>V093~u9PG~s8f8A~ zw-8vIN(iBo`U@*pYCw?C0EM=F(=&6^0;ULA+O++-pmp?`b!*JB>573&c%WHkdu?q) zivuu-Azq%anIK0x1R*bC00zr7@vb?CN`1xiNm0XVL)2-bug{rRn9JDjs6C#*!NkVs z*_j{{Tt(}*G_Sh2z_b6UiaW+_ouA<=d8IvYMI5V63yZ+v{JioTF^YMkcgt=>B1kIF z;#TOdWZuA$v$vmEugKwjq`BXmJtz7NrpH~jAX(kK_4&~msmD{dSsHE`$-6}6Dm}Ia z{*1+=?eB)qK+|Z=0wTdV(jB`A%V=+1psJrvT`w4yut8Fav%HQVdU)@0G?qG(BO5uq z2xnyQ_B+9WsZkyAfw?ExRG};nq{>TREtPiTr$POd(4Huvl#Zc}7q4@+I7xy1rOngn z0OCs`QMh?zv=2J%&sD@0u=QyW3Y$0JDY$cwNlt|9T_T3qRgCE5BH~2Dpcaw)UORvK z2NaL`@UyTQoeCh~ulnr+d=M4|JPD@9x2f~UL@!5wD@p%Q#QXV`P$b-^L9aCqo*Y0~768Ap&Yr|0+-VJa%@XUrBIknT z)a5}p|5|tb=6!1}CRfOPbi(is2h2MvENF&&J#K>m)k8tULSH{2dY-N94`kx+z8NT8=Bp{G=v#=o;tW)1_w7e<&hDI0EKP& zv_0>ZvPPe#r)=0eHuudco-gL1ZiY6Ff}SIK^T)(IK=2vnU%r!rhl$Z|^PocAzWIRG zBg1*qn!vN$aU1=bg8PICldhhkggc>Cdw(<|A7=%SY@f{ulYpn`204TT5Ov@%>Xk?4 zyb|{r5o5upJAc0eMP!^vd*GQx#C}6MShS|y~FTies)QZCD}`ZsrgSH~1%qds2s{dJ-A%;g2$l=L{kd zaGRE64Ocjv#rV0?n&2VEHOt9uI1C3Iy@}BL=`N`FxnR{H#(%=;nyDyb;o?f;x6tdr%{c*lKpx4WG zKzg>Y!=g+;k*w2W_&6af57eo$@Ki(sfRAucxs5ceLA-16ecZgr_nPeB0}L2+W`F6u z3pjt^n~*V>Y>to9Q_klXK90W071n^hYkeTm_66cOlkjUfBIj0FK15YD7L{%*`skSe zh7k9>^KavO0*e^`TmkxIUcnIN@1*@(5nS}d5#1I-RyIU`DBrUrKCLfoJ}7r7SIx>q z{__VjF&VyH3~xIBFne~WWXb`vHBV7loDy${SYtE_PRfLcMMNf-hQ7C7*3YZJ0k)S* zL!EkD%j2!FC$HB(3JZH4kyo$!v}PvqV)Xj|`4f@jm4E0et!J>Le>X z#auo{0b$`1_wScuGmLf!-yEmY>^`Q{wJ@_C^37t43jV^S86Qu!5S0AZlX_|_3Oy&u zu*hqK_&uBXUbcFsd%uw$)f-m{$7I)tX?1zeaU=X@6Im6F_(qK<0tnHywKYw%7Ve(s zR<1~`WzZf$`f8r+*Fi>n*m|o!B$8&>ApV(EEKG1B)oAPr$K_l5GwQdWh8e!q)8!rU z-N~|=RM?Yw8Zl-|pRTR3#qna5O4M=wfUMyQ%Bv8+n(L;)>}dg(fif^MIX`Jpe(=F)RANgM-Z+(@V$N%0NVEQpS?23{(zFQ5lSN1 zU;BDkg5OHNBfOZwMI8D{4HhA-M#K> zmt#b;Z;8q`@iQ}HtLI(RI>*G-E(eo3H+il~y6AaU=jRnIVdgU>m;r7X9oK_O)X|p- zwQ5(jGQD~XEvGLRo3c?5Mk8Srhc%BUcMs9m?!aClnSfeEPJUgTGB$rtspuocU&sGM zxqTbkx2i>i*jJGN0TwkbcWnWmVIsbeRyKdP^n>21SQ$AOC77vKAordTG;rUh!TH8z zsP~TkogRZ;P(oThk>HB|76I@5#RA7N`@4UBOmyI-&UgkbUj@ABRoGrzP0+{AL*nZ} z6k&W}PS=0ccf^>bQ00+|fxx(y$=%&q%n!6P=p3f-lNA8$R4~ZrLCXW2AE)YiYP(-7DAg}KjZ@1)^#QBU$Q04$ENXYVv z8WrE2{FsR31Xa7Ojc9e&yfG60*CJVsUW-|1;O_|mA<7j`*V)=DTf;kl%(cuR48Mno zyT1gV{g*?Vb~-jT^hseyx*`Mk>AP<4J&wgq+(HeW*G+`-Ansj{Ly;EPNKmL`S*}3F z^a6Ff#@YARgsz0w^EOR)m35%UxO-YK^lK-^SH;|Cg`=eHpz)tB?^3c?j?{ZReF9b$bW7R^HY0Y z1s(sz{KuEbHbO!ko`lEJ-VLCbd2?)fzcdeZSbe|Q6K*tyxQtXFr`6RI$opGvg=D^4(liiW`qf<$Wjvd$S1JT$%-_IIgW$*g?RW?C3CX@CVE zF`QmsZvXltpYvbBP08fr2NHFz4Z+2n~JKm?M|dr@!$C zS^p$Z`QN7kLfuc(I#vp{TT>}$l%qfRG2`vuj8XutvH@yJ|vn zkyRgPK+|z`aKKv&S#ogW&1BVc1|eR4;Nc^K0mDJ#s{oio!cla6Ts~9l0E5V_$moMF zJG`OUh!fh)iFTao+yDU9iExL+;5rxm?Vno7k`rTp z%l^zjBp%vy7*cAB|-eF=}+lZ){TX(gy zzN7rHXO?!xt;hZ68ENOXb{x@`HcQA3fnN-IY{1Irzd9GkFO}cVjFvO!?!+Q#adc>g zf^cHZg?l%w&VU*kHiXLLO#tup&}bAw<*BsHem(VcEUw3u^}I}oBz zX6M%0bW4oi40PH+32|?Xv6J|x|7hIsUyd8J)Jc>bm+6fzPDNNpX?(zUWykyPPhM^# z*&;#)+V#QFIG0``w`;pby}VN#9nH6Jd&dRuE*)<-cN-rj$>S6&e<}75n4+D!!3zo| zna7ZbaA2R`xc_4M#XG1VZDAO-2MD^v=!`6-1IJDlbRB$Y$$3gaw_gVLu&9m3(HB^L zIsozIVyW#uqDoSwj^=JL&{^r`h~QXYiJ1zX-uzhFwvq~ee~ukBd7)>lcr7USO7d%m z-q!?@9*be=!}$&6@BL zu-Ujg&uNsK1{-Js&0s6%_8X#PR5ydJRHu~FW04(J;@eEflP>f01h#>wW#nV-r1jEu z!$Ku@kr4pJkq!%WrDF(i4ge>b{Flv|9}e{5rY1 zVbQdlsgsn-G`db{2i{@2v5Bko_Q+Vc6OjhWI;2&{<3D=US!tmxADP6J&=I*=FjsE7 zY&mKn-_Z($&cS>Jxry_8b_7)k<3uUK-nKgK@9#0AvO27+**^+@=3y8!mH<97YITjP ztOrKEJud#ONs+vA3{)hL`WA-3I5ewR0w(t&&Gq50a6jABG`KG@iHczyZW=Wdw%IWfCjd_7_7S!R^WuBj8=~4wgaMRqn-!AY5pId2*t#&SWxx!4{$A7bU9;kdzi80k z>KHNE7RX?Aup-U)Q67Jy=(SqJWqXW-mZygmg_oo{Poje2fgAh!C(pN9X?RM1*r5Up zsadujFxdY7l;5uT1DtUNQ~ZU#k!y^!az~J)*b2Y4f$(O-7c4QmH5&JH0S@ITEXXw= zN`^OhCE@Ino(*&56r*?cXna66`v3-uiUSMs5pO*+IH>N`*}){%=lJx|HRwg}@tH_K zw$1JTaP`(vaRlAE_uvo+0RjYqySuyl-~3G45gUU_5dzJlhpsoe zaq=I>^965Y6d1WIV{9o?g@#*#4uK` z_@H|-&L-?qor_3A75x-F@HyD*xWD9*qG_Wj=^kPd9}@P45S$)iFU4BVz<{`*o?hrX z?7(|rZRw=R-I5t`a)lndj&3GGpD@PI4FEC<3H^ktV@hB%^U}(}%|Ka03HizzVW$LM z-<>Fu-5<0ZPYer?SJ;VfxPgO-Y`9mwD=H|dRA(lopXbX~?NATeX;vTzi|KFKy=hPG zXOaqSb94OMc_~y1$rv9XVZ;OgVh`iTOA2y{;%%_EmUhh9C0{TZ3$zGn6oyZU!P7-MnE-Rn3Zdh8NQyuY&%}q^oOock? z4l%6#oguxQabX7?+T%37j(I}Pw;%l+5@Q`CD0?OG{n`1u@6wZrVX<$xe|WRwb{-dm z@ctR+oS$YwC)g`2A`2dX>~7IFjsd1*mV~!jSaQb3We99ze1q0MNhw)EtYfDmlOc6y z-3q?aqyv+!Q&SjtcYoO5k0usEzF`~)*$tIfARzI;(Db+p($IEBapV-7h5T(C?*~YA9yG^>XIiVT7J?WJJN4T;1`geru*>cCj_&d-in~7rgYt znXQ#3)Jd(>!rk;hLi8k5SxSn`dl8T9?K7W#d{ijbZ^(7pQisNDWTs3lt|WP3-CIhv&B@ zoY?`{@qub~g(g!+=E3A4s{Ul?v=Ub)+*(U(M!0$VaEe70noYX7;WwVUmtB$1NE6@$ zv2GALNNPjvkikC{8W#L4IfEWtVC7TaXA2StKMusIL9dG|=b;*!wmfSWl_G*XXV1yg zp+ut?ZvedGjn3um3Fql}NZ_f~kGPXtPSln-`nt(QEf@O3Q`~~PK*K_Gx54ur=iUl3 z)Zf&*5%Q*?4nZHgsF+ya2apP+X_WBRY?a!tyLPM0TSeb_r-@}A9T5WwCYa7?hGo%* zxOuh8I+c?uw+)9FXr<_1UjBNGE^T)vK7FpR*SiXxtde==dg|5H@Q_YCEoB|WAa>IN}c|xq`YB6BfCukm2Kr7ivG|!t)moFH7&y}dU7v4YR1QzUydCu=%{%&V3)y3^ z>JUPUsL}80r^d(~Zp#`JMCKO1j0Yi<{w-X_eb_dTD|zTYSGRd;BIUep$=FlU|b3k#lMHFVs93Bbo4jZFowvaOM zFvO-S8Ijf+O0zeA?WdC^eb`Pdu6Vg*9{PaGouMeq#r}mhN!Wm~rY)S1OxS3nl%Q>s zM34-^kxp>1L}qdNe9w6#je|fNPv+pc(%UIr;@G}%eYC0|>?e`BRM$);f~;?<&(=HM zuc=dO+F+p)5{0=hwND!R1y#iLuL$jCXYh1XOUe4GR6>DY=SXg4q%=R^X?#i^9P7Lq z#btmqI=B${=!7VGr^ussSw2pf;MROG-6Nnl<=t87VtBgFphte?S+Sl)oxuF!7ruoq z6*3kI_0cY#5D%WJsz~o%fl0YOyps1>;zTB4v#sVuPAYFqA(Tdk@qLYZAcQl9(_kZx zPkk(4jCYl4R3_M85QE^5P)jpMN=nMNv{Rg!ad}C3lcx2nN96U+gFRW8g9(vKX^!LK z(}nNzzzdh}zLRVa3^ZHJRfF=OOO*Gimryvb$1 zcnPMik1at>nH%M>7dDF$C(a$hftGI~8!SC6=YowG7>4)*>JT;F&}7^Q8&^AjaOar< z-s)N|ot?pslfaa*#&%^0eOknQ`f3Q(^x51*nU*RJy(p9-r07y6HrN^f`+Zf`1qln5 z6-rpdW~WKR?;9ls$^LK5nJ7J#f0rnI;*!GK(aCW?>(u^E7B5{S8Q0su!Ns9~Jb=IE zz4J2=i4`D*@PUzWVB$bU3QetRaDd7Zs<`yb7MAQT<+*T&msTcFUoNbxnl=!N?|cBH z1ARk?JYs_lVC`IImOz&M@g0Fa?mF?XdNL zc-Y}8`$Q6onXSj}4&-IG+%dHpE4pg&94SqQtBx=-j7syV?OnLL#3JcT&(<6Y0y{^E zNEBz*?T4UVovx{)LSK9KeMdJi0GULK;0?=!YFts7P!(R;SaLJK?O>$~nDH+;g$liB zQY)+GqTuL@P|=`5wM~-bH#DFdHYB~5#H0l(Nb7YDz6I(8ZiF!%NrIRdnqNUT#D6%W zv(;`)tgFIj|9sZ%Dm@EZw(FsbZtKf$s3Jjiy zuhzy=e-wF9BriJ@ zia4@$m#zLA0-4B$)>nO~rZSLikZf{BwZ$4bHS4@6)$L-H4AXwa+U+9zmkqzV>)Rf@ z7yVJg*T;*e98mr9I#oCX=nnyq7(pd`ShuS9TI0)n4XM@FEmd~n^{>$rXy@yBj zs`m<4ciCx|*>-u6^>HIz@Y(&?HMI44QIw?`=`J4tk>?)q|N6sChg>!{Ynld1@y^u7^DhMvozz4Qh(z<&x!j1M*QM?366%U zu4|`1z3~oC=+8lBn`;h>B6k!dQoo8!X=#2Cphyj7;UYIGH<~Vfl7|4c1t)phP7NzF zK?@sin|%M`hYTuP%(Rwgnmo?Hq=e%cEP7V$4eoA!e%70)<<#NDW9D%ubZG|!;Ua{dbjfpbNWcgQsSC%j1)vGs_`kl zGl(g|;b72!d|`sld>1y8$=@;(VA~w9aH)p02gZT$#wM_AkIZmH9gh5hvMpYoI zz4lhK{hJ}f@$=QEx2-PhEQS*w0|^_Ky40qN=ZCD4`Vgt4GplP6U*|+ac1R4sHTGCF z!q^Q40H601_rl5cNsF#~vk`{0IxQ{NxqBxSW9L5duSgQ0^pmRK#YE>!$SH|Q zu3p6Wg&B*x|3gsBZX6YC3_Asn5FX-ppM~gzia*iaH4Ldc?3sm_nFr315d+46ZtHa7 zkuucTD;%iO;N1(qndgVPmIv9(aI`+-$P%bNfH#`_q z;*VSas*U=apafW!dUzFUQk0=fIB~G7bK;MwJA^UQg#2f--}md`NO)KmmL`{VaB*VcMbEvnW{|#B{=sAJrMu7fEK!4A982GR z7Mo&?h8H24ucth1JE6-?L-22;M7c|hB^jCVK2%x3PFnHW?pqHn|5^G*!dOV7#?~a~ zY`?VR)6unKUgW%h6UUdB=1EcLwAHvn@kA^8E3?qWPRBhl66{yO6gf~uG#!1fIINFY z4GNQ!o0DE&CI6s5OSZl?MYQBu>4M}>?%ctG=_K6*7Q&yDl@=4F3<~wwGEA>Sneme= zM0rKV$&ce11}Bf?)SO+%KQ7VgEw+tXbWnic!DHk2#bP>OVnk%h`su!~%kmtr@j=0B zd$=$K)OG_aA}1d%rh=`7p>n7KBtYA)`6t3rQByNFizD+Qa`!^BdSuk|HKxfikm@hH zEFMS&GHfq4Uzqurr+!K5s2DHV;ULv(X(=KcM4eq^@W3EuSi7||NlAgoH)~K4b)tdM z1i6HS%zDSwv9Al0>1k>bZK{S#VWYKR=E7yRNnw-M@WqV`O{RTgBLx@LgsIbHmSSo~ z0tjaA-WFF^zcs_!!owRFpf{f=C8;xv*WA);KY(SZ;J>CIJe=_INXmTOCNiu2Y-o>+ z9@D*vcp?hkjYar<^6Lea)n|M0$zZ6y2|p7qDk8E~NW}vct}t9mGHMBy>9h;6*E1+h zzVMWfL)GA~pCcuWJca(5dqBVOVJkIY#D;t7j&R0dcnWma)I1xS@2vmFnIICo%$oi0 zfXL6Ut*=h*&sNET_nxLMHN~xUi}R&3X3HoxDkkYX<%tZI4O1uag4w@rKd8_`Z@3R` z8tO0tTaGFh9NXrYO+oChsl^#%{|S3$5hf`enc3_7$kG#r6?^8<)<*4Je+gxv0T@03 z?HwH{d>8G>ag85{2tT5BKMri)l6$vHC9`N31?PW#&`*` z(O@C7WP%N$eCd*`2jD=Y<+f`}kD2gygp_+u3qw&;5W5`2g^2JyLLpw9G>$OSiG8C^ z67+-?2g+0TeUC~`Q|COKqFFFqfRrH~#6p1P0YHKa%pEWLzhgJ3OKkhASEMT_8Bd=L zQnN-L%_+no*tE*h3|5F&&GFXjbj%%4P-L<@J>JWHLHsZ3iT^y>(!*e3YD*ZXV^}w^ zV|}D=lsr^USs+-VluXk&em}n&2$C!|t>cn53}JrHlR8Ng%V44$P~_$f$VRjL83` zTK{umU#!DHzOkVb+oBkRH?z~@USY=9~B=ASzuEGqx~tNkCHFtv?0bK} zCfKA49d>t}o<;%?fIkSB=yuS$R{wv3V4qbliaK@2(*aRLS!yCG&ioMyBBHIM<@iG^ z?dAXXNIu(kZ}~RrsfLKyoXUzz%r*FI1U=UU2@=;LeGJjT-bX$=|G(Xz_mdMRCnw`a z^k-BN2>f-HY*Gk%z6b8{hxjKe|8E1 zups_zVRn@$Ng`RWJ1df7jV&7-b%uv=`}+RR5)R6@il(1IzNwFa)fBgm-J^$`?w%YJ zt)3;cslh)4}296+qL9LX^ZpDW9j6g`(;D6*dv`$%_!w`Ls+oSbL2y7+T-lu0l z?(gqX-lRhoM4mEUC?Q%ytbfJ?go}@-)mwg?io_{|j0G+Uby$3k=Msk_-rGMA%O+PM z|Le>OzMH4-a2C8xQhTg|O;=X3nGtda(;825BwJc;+kTt0;)o+p3n!4 zhs1zlJeyMt-2l_f<(yk13-qFuv#}|P3<9(AV~QTr9eGW(4GGfv`Lg&lvakg7!2#V6 zHhL0YuT~8MfxIE9+0YkG$Y4Kez@CcM;Uz;AMA9KiXq7G7QH#7H|L>ydF}lWNjz~QY zE#O~;gBdV4K-AJ0=IYVCuU1tdtrZp%K^K)I@#%reIWKm|XnrjG7t0g7hK0LiD!YDY z0P1{i$?s7-<7*uW`PPk5%I~)&Ew+p(CiRR=cf3M&Uv~JJ9*-+hUv!1!HOadipFoF- zDLf=a-Ke1=yLSd1&^0$BE1X{A(KY8nn?%c%M{9>CE%$D*QJ#aeo26Z%a zJ>^3~IkN40IfGX>;c^wKrw|yv+*i94?ah-AL5$^8!R4iLSfO7;- zdYbao$UYRH&}JaA;>v>W_|MT1>FA4U_G5P6uY>TGub0D{RG`Ip|o4F{o__jM)A00_ZIpFp-P zSF5iQR!Rmc5wZaW1On8%RHv>7)24c3UEk-_EF{d;Mz>;TWpsxL8rs{)N9LvG?;NjV zS31E)L=i!}>uOk|S;>?=F-=v6AkXNk<7(mq{gZrR(6`8}M(5Sl6*dk6)wenX;z$D8 z)%q(rMgGhrTMx_G_Ji}REUq*btK86c%`NAL%B-1`ANl08oxVsa4SN9*R#&IjQg^m< z#O(N)K{uo*RxjILnR!e{WFDQe7L;qo%HyL^c>r%C!iF~1&ezpL0cs{bE}8xT=u)ST z2d`E5C14^EDt(iM>%+j0vGBljICWU&g{&AdgE#`SgUDZ5FB@S)>NG7bC1Q;o?KgfI z1_v9b@8AtxCT}OJ>*H{!J}%*Z+W6{?ro=n%n+|n-T05%Vb7<{{;>`ifi?;g>Nnb9_ z6Iom}a)TUC?YHk#SMJ&bbGA$<{pnBBDT)34N-4ocx;uSuPKXL7Ev^(Gt1=xx=Ev0M z(A4MaDGXkFS6w@Y!!0!g`=h0+%UETfpN_PpvgU*sow>VF5<~g;m<%!m&#dik!I9Tf zRMXSr>%v@7J_7E`8{XEk&HnI>4y}#4Wv_$RJ;CnFPOFE8X*<bZbvN7SA+~y*U!E>J@Z1vOyt(-IV|L8S{H54DUQxf0OwE+f z^&q~$SuoMRA8FT(%BfvtXHiT&%gu66><#gPM@d=vl2V;f%|W6$uvvT0=Yq~l&JwI| zAK|;+mFvX;P0;D9!HC+3DFd`Ol93t5%{!J`>QHlUJL6hm=)hx^Anh#x5(l=Ow>dN5* z7LLa?Q9VOW6QfzoD?iLQuVWh-5gwU56e#T&G{?Ys8a`^k3CF@a#e70lYu9 ziOGfe#j?l1g?Nz^@prSgz)nK8@Av`y9G-t}xAk;v!1jsQ*}<6<@s9&Uv69cVdvY+e z=K$QTfJ8%|biYO|<0EZXM&p7d)vGr{6#73Mw}uHx5ZU=BCL{G7Q~Nu{U3}eRVWdV9 zLvORm+c`976jVtEBd~uIE^|rj!bnZ&XTpbY(+PDv-1xYAOyYLot~N&<8TU`l@a)6v zOH!-O0!Ex$UDvh5yu`XRD0^NxSH-xq=gI)aDGFqLA9;G2F<^VuVGAeo?7wi2&reVB zj8AP2(GA50UAf%Eh~mNT96hepzmKvSQ*0VU4nLrL6|el}H^mHpaBO-&Y+MQ-l%^Ap z6*q8=(29(BbOdRxLd5`!atRsSXVXcfY(rG2UR8i212*A~Kdd+wtM4;|zMVw&eVm8; z>I1SrKI>DODl${tPmg&GUQzwtHLEBtnpoMK9>1&U#Y@ulnR^TtX2y_<*W1o=n46#f zz7`R0NX51^yZl{^e9Df7%lOCvvANTTSVv4NAfO|oC~GF>W#v^B1!Q`T%+NpNez-`; zfMMlS8c?umTEHoRExw|bImF!BtauB(`r1Soxh_%H=-Ed&eP8Ge#C~9*fk=$jq^Wsu zuz!qLeitwMBa(sGHo>xNgnc`0qh7G3Z}-N-gOia*Kqx_XNy8@Xxu3Sx;dSB64-Y`L zJ1axeljZjGTDLY3;a7>xf#TnhGoO85fWHgzATz7I`<9@K&0bhn0cUgRS!%5ON+Psn zUfeWQGD`(FxVEvYbbP$ZnWlbuac*N0i;~}7WJF)SIHGHLR!u<}7x)IV5m~dDITNj3 z^}B?}rQSUml%o*s6@XS8uC8g!sTRaNHo-L{9=;eMr;HX8({jx_xXki0t7(5aaGaTa zX9sMqI-9;#-=50fb1mqa8>6A+q%}$83D>q7WRuP^Wu|p zoKwpxAWcUa>*8FgK2!i0KsEBiAn9~z>Ld8rwTUyH)&?OSj~i8L7^ooO{(#ppJ4k8c zBA~^HSKGwQ{1EAqZ(4yG^}U|89wUe_TS-19vQ>SrS_&r%`1;CePEb^n(+3&v(Y9#;{2K7`7;-s{$P`zOonO_d75VX8hnHEoljp|ktk!i} z(q=387RhRKl#lpW|C`a4a?7O$wd>Pq>fQQbT(`iVSB|Ut9&5`hoBnzIauRCger7R8 zP0>(Fe4KAt)zLSFQ+9UcaE_#8D9`$e-=d1ccDl-0UlegV(do0@=!Nc!UU9yq$eo-` zg=S3saXOHTGpd6Ze^Prg@uJESp*8#P*vj)^?%%>%aF5WJd&zn{C@)wP2QGg1t7@fM zr!TO9q%UH#rD?)=<59i%{szx;FLIZf$B^Um7Wt#<>t3kw;X+g?|M$wvq0SnS?AMZ0 zm&;T9I&lC77PS|A*4NlS&$|5Al8o2~_@8`Izs8IP=XCTACMO(v+SuIlm=nyo+x46j zw|%a7v{32Lz*jY7h>=FK+}3JGMuiNDP*CDVBk{7T4`*CPXZoZc-^Ybb+%H?#j^3uy zaR=?;)ETW-XA?+XFNV%)@9q{*dbPar*W~<@IxMI?&ySo6v0Y873>|YIL20F@)DioI zFb}vf+dtjTqh>$~3L8H^2ts|lgan{;q3GIV{EyH~=iN^z5{}<+Av+j=?BKT65n(xk zF!DC$KOnl@>oKA7bXj(Y+3^?p>>+jhoSr~)%e zi1WDf=|kS)tVvw#yly{qq=oBUIr&FVL)lXK-x51KY1OB;BD_>q-049=0>qy&?0lM? zR~Q=4XJUp8R;PnQXf*ky;YED&fx zy)9iX=9Wuscp`-pGBNAJQUNtl%nzv^oq!0bIx^t!M6jQ=YI+(IgBDqmBpUk>f`FrP ztOCK0X3f$0a^i)7-^ADL#{Q%voqu*~-lQ@?gr0R&!rJA0Nc+|`_wV$U;&I_wrholxpWI7{Mrem zr{o#elPC|34HUb$5O{dyV(t*Ds9{%If4HjTJ4ywk4Ahw`LN_ofoI|KsIw3y|lbqgR z?<}ZLq>V%0KLRc7$S9{DuzOyfpGXFjzuQ|d|87%N_9f5I;&vD%_U%R%B%Q;UsODHX z`Q`f0IMu}Dj5?Lu$6(o~C&rrH6ol^5rI00gwV0^>Q_0|&jufbYrgX!1!9-~6jmEHt zeO%Sv5Z*@8#r-5pbpjlek~85pAVRX03dJ(1oS*IbxZB|=x!rj z!!~Q^pICk!M_agOq6X^Zi`!R{>Rixj;E$h@qX4iy{w5S%oLwgFqk~8@Y57!Wxe4_| zkAa3zXI9ZUK(~9w(g?zj&ZpCr7bjVp#&mY&td45_nZ@a`vZXO#2N8Zwy7Uz`99^>B zG4rE#C>L|$sx47Q9Dyze@BVK~j@6R1lsp~eU1vlagt3=F=-)E8O-IsroEmFD^z!jr z+=|9R;Cv1DkM4+u`!`$ilClrsI4TBd%}y_bE~|t7eqsLhim9Z0g@`3pfvSveWxFij zBn)$J>fPFtOXIQ80DDRDy1pIu87_%B7=e$;j_2PzuQvU0jMU+48PyV>QX_^U>q~DL z_YtlxI2-Fi(qm9Lo(Wv9c}Ie(63{AX(4rbWQRh94c$+-TJYMS&0`~yjI;ZseTS7at z>GA8Gyk_sIDI?!3HK3pYn#TQ>lb|(17s@avrBbF zBsv(tQ8JBIxr*e$%1hwHUs`YTZfhZRR)DaeZF>LEQG^~bi@~D;9_y-$h&uWOw|n(P z%)7pWv-8y8r`VUKcvK3|^5M(qs4isR}4XM;;{Kkp%@113)Pa0zOt4KYL6YGd?sXC-DqBeW2jvzOiA z&?WOoH4?|_G(t6Og3A7a4o`vm=6V&%!Sw0br$h??5HyK@8zS@b4U$zQr!DQibTTDZFvn#yQz(S{7xpErm4pN%Z1mTnsEJ*GoBOcnwJMXD6Rx^Fjp zEG|r==7v5tcwUxOP4yU8p1k=Vj`CM?e0ZHNkCGxwGesrgc=8utbylUtqmnn^fZ!c9 zn5?#C?k;=Ds%+V|!*yP~O!d-iqd?AcOQB#Wscj{@-I}%~M~5gF-Ga|Pqt$m!8JkS} zUg5QnrnL1HgB0!|MBJ|r%ztXUn=eD&k3I1ae4ZMQ)3xO>k9@?ytqQmc*BUiV&FIm8 zHs#QTL4O0_S=hDW6{ndwb2K-rFAW>cQt|-!*LI)0?XO^Y1eO_nVt&jJ0BRnNnF`ge z4r~!5rHWoXWB`C4VbET-obM&lg5;ac66#+~y);={WAgGXVcjklE<^wlW>#ZTG6H0R zqVEG<4k%~6l1zU6O#yRnS$LX_{fJsK+IFZ>ETY9sxqNNW^H5mr1Ls90=whS7iq!N< zc8Js|Blu_kd%K(EsmGdP$3Fuos%5my_J=u z`(9lp_0z(tYZiCKe(3KXljG@-4j7nNtjo4G{=1PZL82B!Kbjwa*zj*;)2ibW%?YeO z4ftmZVMuKXNHwB-_^BAUheJzDH~`&m2H@cB_`VpI7k_Bo>0M5n=3}`U@KufsA~+BA z&F;vx^(@)avZ|oHh4IOYd$bu3G_VY1BY@xl?+uRK8!E4-@u~%=u`6|zl0g7KF2UxY zv^8=r&hCyx$`w@Bs9dfNCSll^f+7e8xwr?XOXw#ukIo}vs3v;h!vRyKqKZc8;7QQ6 zc0jR@jJ{-akWh4Za_(gc0q{BcAj+t+k1SX%z3Ob31cIUfya)c%UU~g~0s{c@|?S^_nqY+bNQC&tD3RN8wHLgP!nsf7I=%Ed2-VlGIJt<{`oyc6eNEfqau;3qhpU9sBnUEZHtMu=eei;4u`-*V?hY@5~5){hwunFMHJpGC}26kCx&63L%BklA=y z=3ElN(R8Rxqlw7-3Bdg7Tf_?hr8bZLxn~%tp^CU@RU4`%(pJilVg?AJe-o=XNfc(J zfXx-84HMQg4ZU~%NJ625@zvps8J?(__$7-fa^-I47NY;{ zKa4?vD8CNB`455G|Uvn=Vb86++j>IR`3{#)l9Jp&0;RRP@o;QK3SZsji1D z`bzaAH$i$uYLixJcJ#DsXba`ymN`>_@N?LXmeC}!a$l?lViFka} zW;0BFFJLLBjb?z^`#cVi@&o>9&MwyZgU;Ex#&VF=*mStyW#T}(mt{$6O+4BbuPYLo z$gH6D61ZI}60j?W)jc5D4!e=(cPpZ?xKMnM5d+_#=3|A0L-NZ~t%=_#f@h9#mqKEO z6+M!$1-U9=IDL{BA1%wYa%KS&VLgHFHvv!;ccaxUQ7I%EFQ` zo7aEYdBSBL!cQJuLAe<7cz4kmyYR%aEC-AC>UB{V=mr#Z&AtfcsSX&Y)|#tUVx8dm zSgVo$<5g^WR>-n=!uxv9u6mnzd1Z3Z%|tqAn7f!uW%WmwFmWH19hy9}2K(0yutk@- zkWsz)(-8PT$5s`e6O3Oi9-0y(*0G`J*w*!%`M?0`f$f0*X)>gx84B0{KZbS6{OLy* z0hRFZ&)Gf%e>yS|uWSDK>Yth{IK7Pka8w@BS`%GHb4D92dGnFCu4PpG@6n(z%+a8s zFGl4=jn-RYZ|c@`wV*^1&sWybdRSyJM3f`#wjS%`F0o~o$8UmXiJ7M&aqsZ z_JoB?3LMnvK0lq>LH?MEq=4+)o&*dPP=GyS(jJdK(#1^33A$C=l2fdb8FWXHz<`DaJD(l$#MH_Npwv}(KQ zhjas}`R-H49z>1Nbp)4XnOt`IhYDacRN9GPIKa#|*htx6X6o~e?fN}o?Yn7#Y@N-R zAx?lWc1DGb>&iOt*j)Ch_n>8UdtR>He1uRkGV5&9NaL$-!hq8s^X(;xXk}}PnaKTZ zN@mIfF*I?#-q9nIV6$1zJ(zE~X4j0bl>&zg%{){~6U$E7Vf3N5|KhguM zSlNkzioE;y_l^1K9sg%kR+dck(^flA1n%m~>($)VnsTD&`O?-$V7c*ytKAR;YT*_%=Hhc1deLv*noJrb-oI&7C2vUfScaC#B@=@EJ@0?y9dpT_l zNK){e-$=HSV=FhYCJu*)MS~G4!6|e$(Rjs-hn~47gXkRVz?h)iEEy)!l z%p_G+xH6^a@;B*LHi;TMki=_|;PPTkB<9pOQANX&hnA8j)Rpn{4;%dQ3Tu!Zp8b|s z`1jL7uwbgbpkt_%f+-w}s;>^&J#rhNJUQHlRT}cL7ioI^8_tpNw=4P^`*Y_X9VI=1 z6xZetVh%U2Ob{V_r(nk?CWjSG7TOr3BlP6H&_X(aIwqX*$_hv!{qCQDzFIxK$l6&! zg#t*(+q>FL4p~<%=HE_k?IompCrmVz98qg~dK`#D^Z|mPd2!`if-Ic?s%*M>W@g89 z3n@`>D+x<}$CVepX;8kiUWDO#^v^Y_lUV06F%wq0&T*u{M|U^`|6>!Ib#4kqj_ z1|(pYt7C-p1U|)Rp2i8KX{1UuhHKgh2pXuVt-OZCebS_$k)-gpTF(e+)D8}&-^KW% z>fQJzaQM&reIddI=8^kmWBBNhf5QmWs9yqWjcKe`Kl zipg{^1Xy@oud0#U?S{b0Y^(gG77=G;_Q~wJn4=eY( z2M?}~9NS<$R3Ab>w}@Z6(;~xf-}M8kIaF_L_DBM6{9t_2T&)06$CK3*b(^_B{=4U5 zrkp&bLMp7E-sBQ!UjhIuNVwu{>zu&OVGg$`-H_jB z7i6lr8%78i{|dp!4<$sBgaYGVWu)#q@&FKfYX$(2e+p}3a3)B0H1C$z9jNkU)Orf- zIFP|I>TDe+D;8E8sFs**3d$h}AwvonU_3@v5t2tvY_SM~ZBFqAJOiJ6Y(OieUxn*i zk7GuwNy_HT*})0{cL)%<(ZFDT`nscYoz&mE>v6m(417--35P%IJ43j+c8sIM!v7N4 z2rYX~BD)wreEcw#X@Bi4yKzgL za@3?J;!rKx+4Ytzj@!)FnWfsdC!VExr~Q~I^AQ+8j1?a%SdF#pQ}s9!udU}H-&L1Y z+v|QAUFYwPvmW8us)QX_@NNcqPEH<%ae%^GzwUQiY@*;7O!f9;SPr<3^VJc`)OQxA zrAs^vY)C=9{Ckzoi0t^;Ly6k%n{tUja(Yw%y+@!ZD5c*{wcuPc4%Bz;wWP9I?rg@k z%Yas*n!MsfmAiPCc&BCg$*gy`S0-zJ&xPh(I+z_z5Hr&FH>g#G`#v~AG7}@2YV*Q#0jliWi0~C-H+3vsO##l-Z#G*>E4zC;Tdkz zPJw!HH)rUqxasVjN99q`RmR5EPHbE}x+AfzA_0YFKMJRpt6|DQf#(}6%nppen+s?{W!VIW7em~zLfBd3TGn*UZ-{-$HRn6Qnw(FJVPB(Md9zUp_ZY5D`36K0TnV9@9ocdoe#eyxptW;{Z*CL^aza( zM2!nnI&(0gv=%|RqE=$ub$z>NGBjv$US=6XYV3_INcpBIgQNs{wCKf=q9n!H29rSv zXk!QnQPM^Pc0dZqw8@Bk~tWv^-f0rMf>mWM|=YfdMkeu3eiuD|+j zevoMx^FGn?tUW{$*4#|*`+E%H=%XPPHt|B1<=ip}`r z-@*zYu?G^8hw8Emw7*egTGxS-GCPX0w8B4m4&q^k?!J^T4%Ij33eRwC|I^Hm4`Xcj zc83zY+ggGmr({+4`(0)>T1IpQl5eXed>h(tFdy9C<*O`EySUpILCl&%667aKFly}` zma!18`BR@;hAonG#W+39flPo%x&5o)E5K5pt5)9?9>E0NE6$M_P`$k|@E;{S#y!Xp)`rpMFgJWdB!g3y6PFD9T4%2W#` z!jIJ+B=`dp2EU`qcAuhchY9eG2D~u`qgo4yBSnk$rchYF>P&tQ$F`l@?gk(fc#nNA zvjuZ!bIp4vQ7YWkU|ndhZW{Y}!;0%1XUFKRotR$xy{xPXPFFrLb~N0#^z=3{5^+{Z z`H#7jbOAl9v){MRk|6xzp6IQuC96odkGe~6kO-c|LhkNf5Fs`OQh=NJdK^t(?C|h# zwZXVa)Gsue=%R*ugveq9Att5(04g9`HHBkOQ?@IExk)yeYN@PfP;@sJyU6Inv?V<3 zN4WDG5Iw-r*xFbL8-D&k`MD4Wd3x{jt_l}NquCYhC?iZ2xKPHX1OdQWXToJJ*Ho~8 z{UMJSBPGi2ve2uLMPyQ=|Jcy-b-(fU`N@Y|*j?FH+VoRBI zh3)jus`sz)fP)jw=kbw~+u3?5DpK~8q`$7^MMIU2Z4ZT|=d-oBAfHHH?^vdywL#Je zNwlj6QOLg8QH6D0hNn%4;N<82Wz?wMm>Zc^+Ob#!2Sgb7+S(MK%a|H@CnwuK4K9nk zJ&L5R&)1ia$E7S|yOy%VA{``Z2P+aBviEA zxj3eq9#L7B3VA9OuC%xF#p!Ni*wQKrL|UQTk(H4Wq?A#|DbB$l@I85}n*}F#1eI@u zOk6Bw2tfpjVb0hc^_)$!?u{PHCch>*Y_Tp&8>uO?)p~3yZNzoP2=+QWKG&%);@NZ= zPTTEprIaH>pW$2mR-TWSl};4D+Ijx^RL6;Zc^IW}Sp|$Ls3u&1zP0KLKqcR_gj>Z- zXjXs1G2zYZSlA8d14XM;{YNz$blXq*y{UOP=?U`zYzqs4M|6$Ruz+DSlbDhc37FsF zjHL*sog>87HgDB;iDaK9Ie&bt({L|9Umq@Jj>NAC;9>dtY7R@PZUhr8>%mQ&Q*{-C8eywHTPZXU4D z;bHK$pFL}fSP1pMtd3dqw+>}iYXVOxYH#j%B5!Le0I=k@V~L&i=c;cg9YNAcAV)9t z^|&}h*jnrq%TkK2gMbV=2TKh#`!Q115ej;Mx3|&YrFbi0bfVkK?}~zJ3#>VW*0b@d zgt$WzGnE?H(m6Bx1}RBZe3sO+0{Q_FVwX>QGZ~W`j~_G*srfcWS(cBB5LIk0iB*-d z&gt)j^&XDj0t zWeOX)pI@v7Rc#8J8mI)(kDGIHLX`4S7#|Q2B9IWTcFSk)&09xUm|074f3qo70#h1f zVU;9BM-TFwfrRJSb~^dy^%q!5lXG4L3SLgHDd6t{0pOmjafSivjqane_7R z*>!a4a?w9=@G$G5R@(*~sO%&edANoGu?Mx|>TV0ytgO?}mkRc)f!4RH?}Gq*Tez&D zGGI+-7k(~1*94lXLmiW@Lx>}9XKM1Ut8nm?FtlE?!}5lGrUa>8E=c`etWRR*I+Mb7 zKG7b2h!M`!wzpuS8na$9Ae~>=OCSyXIhe~EW1Rj{@8^jvHGbk!VOf!(EIXL@hKF@! zEif&tdQ{;bf zKJb}=r5+D*=UKL=-}NH)jRYX;ar_WpBSZ8ZWwf=mAceyE&PkR|og&|1e8aQGnp8+x zwan9AW(7V`vob&JR@mY}s#VuqiSH*Ecj?FVh|95bk)DEidd0QIB044A=yZ?3aTwmn z{ScJezON{Rs)}D{&MCZYQ@~4;04Xq!0aQm(F$ouq09MYXNmIuBYB)(8-Nl~LKCAvk zK}$W!Ou$o=3bQWW*9aXZNK|Av)zn%?CB@QSW?)_EY-3SLQ&7&#$FZ@&i!(7nF-ccl zjK*%jik^6|>yz_;i2BCBN`kH16WcZ?wvCBx+vbUF+s?$cCQc?3+Y=`fO>FC(`@Q$w z_wS&)c6U|R?y9}_T1(9z*dFd4d#m8`^jyUbQemGuWwg6h*SG3WD=gHi9o5zI(N_CB z{U%{D`p^9Pc=v{sezeWo^FZTHQfhj6G6cZ*){S6&ZT)O79V=W@uB-a%y)&3syT{E& zCLn>$(dm*fDkwnIpNP%x(Qh^@QO&|Kheh0GlEavs*)*r8tgT)9Yn&^xtg@J9+LY2S z5~YyXZ9%SMQ#Xga5XF@3XZi19QRQFe>JRO05$WDFni+QHILO@URrD?6ISF~hSu)tG z8T(C5eJ-9pDmDQL-!y8}|GCjtm5#;W>*YjK(!|dHNWVigWssNB4~?94n}Ivx=VF|{ z*=xyf;pHrYYJQGB67_Z*o`AxuPgWr*Pf#fE{E2~4QUWn#B3Ey+zBgBhB0hi(r1)TLbzb&ez-7URTP2~(*NQUyN#qZ(U zb>Vv%+DMPju#S@Nhnm@2i0PQOdXmyL)Xj95>mb1+ii0V8+v>*1+R>5jkLLIgnz$|= z4L2wCjz(&xZX$nEy+~?%Gx!YGb1DIqJ#P^ z-U{LHAQ)npviXd*c77e`^6Ml)_-i!6)AQZBZ%7Y^I%_)P^DS1sl05F)&727N6+4bs zBo*m+I(WA74%4i3kOdm}cLW)XH+dVtnc$@*4NLQb=?aehVl}C!D+hwSnx2>7GH+c> zfXG2=SQ(u`xapDl$G^@IJg$fG=^8iGT<5q797ehteNqIs%WePGS*%$Gy5-c{TKO84 zZx8_Saz?$RdC$}74~5VZPS2edI+XXv@_K1am-WNORvJndcZ~?NpbNgvP3A=ZAS)S9xJTGd;R;xJ9A6`@LEa1X28(_CnoCl4e zLAzAt2SD4$amIe*Bv{AqqSrGcjY*wvhNBIoctwIhE2F&p__+L=67jlkxxcqD0S-CX$Xd0br9TnCUtWT`G{Q^eI z(VrLu;3A?q8}_x=CJ^WI?l!wAFXkNKLO%JJ#la^6dKG^flij_`ldr<^>8*j~-?7d{F=ykq*Axp+G6C=lTLL4-U zLzgHNsVicEW**(~T694MyMw>J&*4Y}n`_@i4C(3prmO$o`+2qFH){D#PI}o|czI=8 zHQ+R@dPHq&qd0|_Nd2@TyBQ6+bM3mSb>G|zbXE4SjV@jcX&SHC(JB8L4xLU*GuxzU zvTUB6J(0^zTkaX~`g+Y!$R9%q$bsI~gD&Dd6c59MLc$7EB`V-GJ;d#>X=eMJ6>ikzKXH2TSAlurR4bTbuR zxrE7CnC0RvCbN`cqDk*Uc}n6jWwScyR|K#=)#rb>w-X!CNfQM};!t z{deC{GV{CqG^{53!U|gSNqPoL|G~_Zz7QT2*52M6Z0L?oA)ifR$_d?RiT5ikc?^l` zI&Q`x`Fxh>S6_iit~905T4zO9wY(&1sF;~+=ai)!H@@5S@p>%6ovF+E6-tggGyv7> z)6duQqgg^CywkN)A9MJ!K$I_8TVId2zrp}mknePE)b3UkH=u%OcXeK8sEmym-onOi zzw(5S@OhFh&*r-=q??p!prUy{BT9M&GxYu36dqX=RXbs^#)bE0l|lAa1yw?b`1Wmx zgRbJPvEbPF$Bj^&V|IOVHC0~kzy=51Xz^P( zvA1%iJicb<#i1$^B~%IHtzeom5$Z^yoXuYxb>M&=AVwUW;76*D&cV>)+vE8~PK@D| z0)yxLdL7c?)p7PkiciO)&AXAwJnTaTFjizNy z^DP0KrIk6w%W4vJ$Rh3aJ9qdffi$auyke@f4>~GJGi>k}Y!q8mG^D4NoRgC$D(%ILhb#K*Co8;5lq!|n8Zn_;-Ai^nQjHuU({_dzW8wvzpXzE6|z@1iNP zS%C?fa~)+4lsDXFvSJ38=E+aBo24kK*4c zIj-mC7!d~6&G5w_a!RX>nRYjT2n!~fS6@>nmMAKoC??iXH0jZyB~s#_H)zQic-J)9 z;gj;a&b9XQT2IP}G-;4cao7DZnvan1Nhdej|9rje*o7Xokx;>uxLFwT0suslHpIld9pNRp)EI$qOS$Q zV?M@D6-=<7Ty-CX`N#gl;CuN-ng2ZseS8OzmYf=NJjm>SPD@duuTpq>8sp`?I~Jz= z|GyV{aSKVM9qgycv(;S)28&!fU1j(FzjHvHc9n!#@xhM{FX7SCe*E=$?aJe`6}ctz zzpbK=SHyu1@ZCd!^z3QxXzEPgei#ttFGC?2E758c~zbhz+Dd!8^%buvJSY3SZZ zK_Rh4nM7{VRJ0@YP`jB?SHEf*kiHHns!A0mA0y{6To5c(Do8j;oSZd2LY)qa*a;w} zQldhDJo@)0Ote8%WeEp%+#<}d;sy#7hAPW)Bba@kC7m|CeG}uz=lI9Tlh7#$mN&py1>M0tz?L^&dFVYz0!yFn+N-$#2n`1D zq4U4B0Fa{6tfzmY%_#}hYBcE57tRohVNE%#hmy4g3g*r@a1&lXOuk1DxZwwqrpY(c z()uLqhYm)m(yrRlc2E!8`Q@x(#xCwj%PUmTs2=9rfr&`^#0aA%jAHQR>y}UnKiBs% zsbG2+T#?}FD64=25D}UV!aE>Ejd>LtMnK5slO9wF6|YefivJxSX-bm*v)?vW2AAy# zm&0w#pnS~BQO#Ojb0t?mdDHOz1GAp)8`(Quf=wG{L=iwlg-#UJnNUhuy`~J7P)bEz ztV$e!D3%V77enrf7i>}~sZb0XQdB8hMGC7=MS%)AUrmcBrGi-|jn%LVM<}MEUQ+}T zj!{;xqmob&S5bEvc5TH+2cSZ#RHd=uk)@*1#MriB#(>&TE|pb=1*xrPRW7UYB$Ut; zm#Y`knBtbFO{y@G=Q5*|fkO_msF$gtLNcXd(1ciXP(~EfM5L6;7VWzH$4&{^@nnfg zY6$5Mn(CxUG|1#}SWGca{8LkJOvvC#i{CIizpkJ~Q9(j31g#|IWPf}g?fyHH(8ovj zt9j~ZVv3v(4ywx2ktR88V~#+Al&Moq7*=#RX%?k=+Z5_=0e@Rll%o2X z6zTtew-%ym4ckbdoqI?*DrP9w6rF@E@=8#rFpL({*s@kwmg;4!GTQQOfIjc47Lfh- zxhGHFwAPY1@G6L1gi9TV1%#0P{>zbIHXQz9gAJr1ga*`UhMH(|VE40z|9^)8n#J&1 z;d)S_pgW-P@z!?{P)yt)y5TX<&Vm8JhFl92%2PEM#Ql$#_Q7239W}pNnX2I@V$gZ0Ae`iX=P$^upbTp`=z@>7@x`wmcVwx-g3pdM8skc$LIR?S zL4jlzTuD}W1r_ef*8lnxkRg2!_dIM-HV3?l**pbKJdE+u>70aO9I8#9o<1)IAxqDU zbnHDI4~iJN|J!A^p7HTNXFZiLmsju)7sTK}-s2#|ZV~6KHD?0Kf?A3D_z;AMnL4tx zDx%?IKQqf^A?pbZ>gHg(0wsP>)yi8KaPu;5kt2q*n5LOfD!G(*SzS~2-m35~g<#>Mnz)b9 z1$C~-8^HOrC5nu+e3I^1bXk>(|8n z)Mh<5%3SpCvRT-Wtjt^;y%yuCOa1mzOLI4J>_nWSV~`%y57>nLQAG|=?17WB!@DOS z;aC(C4t%!N;=o+TTePx@P73w+#m)586k{#zD2ol`!)S~Q`1b)$?wu+V+%jX0*TOF! za`0&kWevzz;mo4A_yID04orw}(J-()02Z*X|V4^ok6JkorA{UPV$E7)DLgh=yqEURRqZo*g#L+`w zU1e*ik7o^&l@UWuuE=qsT|8DCT*Xi%WIuNE=_UaJ-rD;V5K@9*E5tY>p?L zX0|-8$d|hV{<0NV;9_Iat+m;GgOB#&7{st`Ll+Y<#u#+g{|y_T)71zi9$aL$$0J3f zfuD{jii$cc&M>p@s(M2vSeGyN99lb4IU@XrO1R^U|MZn4Er+R}JjRq9KW?6|yp$vD z_S9fP;$8H+nYv%2k%RE-oA#&wgN`f!Re}r;N$P}B^mMrt5ley#4M{>IvwB1nlZT2h zp^?0P-qt)S%RD9Qdw7gKIX=8(N~M0)YBk1TnkYiC$nQmv zFz~81t2CWnEVx#p;^w82QWL^sBYxcDNoTv4`5(=v)Q$$SaDCFGvIn%nUAox^6u(-c zohfHxqh#w;&@0DS_pI}zVB2yiQG*jC%w!Ujm2Fj_2{GbFaKXrE+f*YcK*{=C=^)kU z;a_Aq64G^&;S?akzi29P{i0PBTyOw+rnK`<`3Tr~XGjY7QbZ^fbMzbOJg~UAN)1Sl zJwkMEF3f1qMpovBz^FI{tEPC`(r|Yo&F_*u={zhjwKEPnyotFcnrRcZ@EmnxtvFQU zzfGmm%B6^qOxV%auJMnjZEER#DHzH%YFHpeFtf!OMocLOVTM`( z000f$YA(m`bg2#+O4E7kkj&&XNtq}C4ESLl$PyY^mm*8rTNyr71!K0d4Amc`RQvO= z)066@x-y&B?BDR5QD$zipsElj4dJPZdDwZ`PLSsrn9+HlVXqltjl=X)yDF#nzOs!$ zFG%V0Tc&+6v;vr{^ISaY)`xIP_@GJFc*N-6ycsY6(sWa$+zn;p8M#h#Da4<1`ZY|V-RFx*`DQAnu%N<;*zSzSskjJptQW0KMF3!c1eV}FM=sEVn0QT) zCQUPWaHeucTFAu;pZEA>kwF(Lu3vDb?BHKV9Tz$A*NoxoXWI(OS%kO2su5QRDSY^* z<@7UbAc(z^Cjk@^y?y})Nv84gPP+)ikcP#9JeL)&WRWsxwQcz7%%HgF_RDev$YFlaWavY{ZDZMJsJhi(75`z0}=#;3KZUko0l~{LLnEC~6ON z>a|hq_Vs=orsKW3ICrj|s}J;FD_cBUzD>p{Bt8+le2$ECGRhOOC|oWtBhX$yN!<={xe4k2RPbN67jRb;)Xt?WNK zO^+CZ@&Z2m8{(4`%M+#l{uEF02b|peMegbZ1ccO6PaB4V*TeW>@t9}6Pl*eu;%H;| z8CP8`S774^vhe+5!ZF{mw7#bEVm=DfEEFIZnONBMr!RSyJPp?-P|rOBTO<#Ur@O<2 z2ft*oT643rT2g|vcwo&pcPHS3(__oCU)9NL)oAyxqvwIYJ}v;zW41LN-}rg6UPCbO z-P!fub8^Yn=Tt?XpB){(!xr%R-aZO7E05E2TNRY18~+TeBRz!Bt!E>h| zyleJ+r*j|g-YB;(gE7Wmp?k-c*y^vcVc^*-R9E>31p>Fw*khs3+n*{?E9G>Rn*o12 z!;gNWncHJ_rJ_5M!bj&d|Ta~qX@!sOg1l> z-GD#924UbxvUEt7iA1?vg<0paN%>uji?j@Z|E<#QW=@a;;6bJJYR%!Y`pxQr0??j};h% z>#ER({xyD1PMqXJhRxgG{%!uuffuV{D}Ib56*c{@z1{I79=-8MiX>k9%O0yRR3dMf^ zA3H=c=xcf%^($e%D1;X`!~5zkTcKxm%WNEM-X2a$BS-M7B%V*V3}QNw{z< z#9}aI$HDe&?Ie|`c^3+iLK-6nWaRetL;2$)iGZfueDF!CRARihgt}d?#hq)7CK!Mm zCbKxh-+Y72q)gG?7?VLS*e$ocj=5ET3TIO4>Ep2@b$z+e|K}#>T!8&fL2wN9mv8FD0v&sB)d!4X{ESbr z3aiu2Z9F}WB8ktV5x#r>$K#MDBU8+KC(+l#`}wcHKAn%(?UkmzBd>|#zxNZP{q5tf z!)-~Ah`oIs;{>KD)`J*E_SYoB&D~C#5pXe3XT)}6l3>qSSG$8l5Nc>tRXq=bb)C-# z^}X?zq`5(P)O|gMfnr12`eWRzpl#}Pi8dQ?Vl1h6s7Zl(MMZ^GX2xM;FA+u))L}Mm z45H^l6*K4u?a{{f|P#>d#v()lkqRd zVo7S@7zS{)QvSiCLWvi?MsVF~iWUThBB3XNfp5G2s__Ame#_+Nrf%tK<{(T+NKjZ}HgdbY?@UJPp0AYS#FM&6fQxjs_Ub9F z9ov7n+6G3IW7u-qaQqt`$zRjq0!2Q^PVs_hkS`Q}OkFGEMWH>b5y(e5%Y8d=b{<2D zwv=WFJEuG+0WS#C!v;j{x(&=qk}bn~Vh5u}eDtym4#b#h+XqMCY!d<+V9zEC*j=@35vfTV*WKA2O4Ady|Q z`8uM)Q~UNs008YOd8^yY2_00tQ`^w(dGU*(1sx2)dgSk4*z!ETf%RF1llU|}&m;1y z-RCXCf)|qD-1cI+fg9I+puO+Ge3BSI^Ybp&$SB`iOzUhUI{xwxe+By648Mk_eLV=G zPg-mAIwuH{!%&0^8{OV!#m#T)fEkC{xH2r;--`6Ox+?HMc^&I{1rY-D`K%~r*7euL z3AOZhcwg(>f2Qjckh$H|L9tuSBowW1YMcUL9!UuDb5D1#)_!W|swYD-E%cRtyrCre zAY|J2nMw3*YqnpobM9WgjF2Ug0wTaJ3eO9XcbY!XOb-U=hs^FAbQOGVJ`$NysP(^| z*8AYT$i-g6k1Hke9RmF>TInOSoc{Ruww0Uh6!eA$-&)vpFvybdbXsXe(pQ)-JDfGo z_S~ChW@`KIJa(z?5cpPbOf)SzB=Z+M>!;&g2tTZRQgt&EOB>w>mFzQ?-tIBivJm=T zyxa}-aq^aTS+qsIrU3?8E(aiBpd#DTdif3dKEr(kvfFZcjenp*_sbKF@BeGhSM2%Z^f9$f>4A<+rqzvK9?g^ z^G8432Y_-q@KNvTTK!rEFNbfkGeTcyk5yQB61qL)72Y#uWsP$A36g0hOHPr{;K6-SOvN=8S<~* zugI?-ISKSh;<;QtZ(Y6h`Ynw*^fNj?2Yzx}oUPUz)r0}-tW4Hs`#S@SznEtpo@QPx zsm>i<51)4QGP&p7ezlqcy?Q7NJ2msp0&c=PEY0yeow~cV3Ha~-0xPIfm+h{)JM#iB z)`WQ>Cm)&pgob*ZM#D7cTlxvAJ|;4+Klq~{3i{}&9=iP<`de3W#RXODe2X5qxWN{5z9wx!Ox{kH4t{1$L_E{6Vk&Ij(+L( zWxd{VIkT??UL;TObG9EvQ}<<2nAgbrDl@}^1`TrW+^G$N{_`!D+x_xI=geH(t#^gl z=rug>vsp3Wn~q_`HDJ#9&5Fcu>g(!aR_XU8S3qS;!=I?Pxl5vtn4DRAooWe?Qri8i zy6wdyQd@+Q-*N3hK4LZkv5y}wcDwM0Wyzw&34urL#$(vp*0ZY9*Fs9fVaF!01)lfr z{jv{jb|k!PgSaMrG+)@KY#2-L2i3u4A1hEp;%P>(Q?GC{g?Q`c&Pqybho)A~$iIcn z>+_HV33A0Em7IuBYWy1?XAzxtM^7ab06-`v7~nc+){(vst4~}O_4oDu*J~MjD5xHP zM%U-bY`L=Gs}OmHQViHYa#C+#NG?b3_jjeuTaq_|yl-#@kK-_Tlq>#x{mjib8r#K@!g^7 zbTMYVUN^qqLYevuPXyNLY(0-Q%&z#o)ZZf1btQRfQcm+haj*PgJg9r0aJ`$a2wdkf z;PhPDx^8>3HGYn4P?D0%!a<&Ltlx{k-9IkuvOE0;l#;YeQbx&b`%8~vK^S;cQCIJN z#^z|gj6w!mmN($6?!;kvY&pr|;W}0@)_9l-uL63h3Aw~X1%ZP5@SsZWkRPikj1iBXZ2YsW%%GVSj421?0;@0(Or_#0qrOD+a;U3?pS&cglNXXxQeH7AFsuEMH2< zjL64bqsC%JTJP-{;hJ_!ff-A-0Mg5RYd-E+KuV5Ao|TGWH(?etGc0)&C^<_L6XiUd zw}KaG)UZe>{7);*l8H`~FHxTOtg*JH^g59WX#mL)L_4w=nV6rQQ=mbIbv)y#@8j&TAP)Zv(|fjJ@Qb4 z2yI~W1q+ySEA}+jE>o8*p3h^@^VTu3^6Uq?mOIKQ_~dy1V%5-8om&%h-d}lhN)-l_ zVEQ>i^|)bkgNIWZGSZn>V#rS7e^c}8&D7;J#EH~VYLU{(_-)~!b9m%tN-ToSJ+!Q1^E&S5a| zKpNZ%9pwqY0P7!qJF6J-MHAq5-P4i5yKSwLc{$ykt3hsM?*li=a^NQYwaies?<|tg z^M#abm!Y@M&hL&wzvi1cy*jgDn`C2D+#F#0<9OfoZ!;^;3mcLGjkrFdY2BwfhoyS0 zT)YN2Oz3&{KN6XW|X0c8|iDm_G7h3 z(}YGv7&(ZV5HajSQNhzF{IW$79}RfjZ&8$6nZa9I?Yr;9gmQEeb{k2(i*MMGGC9A# zv?`}3etittb^5exxVXa#wc=k1e3*B_D)>B^X|J6Q*5X4ndONz9AEc-l4OXJ`MoJ+4 zNj!$uEoG=?8?Sa(_BS+@sgO}362nA+gSWsKG9D2Ez{T@^cHDjSI_hLGp#o^SFBn|~ z1HPw!)cLn`4mPP7{K5I!OCGU$bm#jxaNEJ-dZIT9bUW^Ny02qa!}~eJEaVv^=DM=! zVPt>|zgq~GY+s#1YMzlcb0-RdAFEWi0e0g@5)L?-Z2n-o?sQW1W)RY%#T?n&rY{Q`-c$f%9)|Z_8#c-lEll_b9poj0FZN=C6=2L z0MsnB*IE__D<=Z%e|0%Fxjqk_U}911iw$`XqpbYf=J6F0*OPoIOi#Ceo^gnUrMy{J zGHTpM-qQz_e^=h*(OytMS@>jk>{{u`R{kPxO+a&L~4@D$Gbo?**g+u|iCHr38FQZ@A zC3yx$fn^Rar}Gt&u?6p+sqG_L{hl@)QDcK;6Q!$Pt!=5Fas$psdi2p^OAe1enuOvu(niDsDe?qR&)Q; z3_;#t^RD-+Ix#@xSR%PddcoY~TR4XI@PQ|5R~|Z>Zgj_EM&U=pO5vxaPimXh>d4+- zPPwg4venQpw`2XI1sQmb&wqCOIhUW&a{mO{3q5&B{CoWyA6JC+`fWYb4se)-7D-5` zR<pHTN5Xq^&z}u%BxWknQQ%X|#^13_;m-*g)Rb6fY z@A%ATZa0qd+MrgUFRyO;r?2w!?LprhFDu?j|-HsCV-Cx-veTTKHvYK)m(Eq+(%JJ=d;cxr z(miDuaT+pYd!JfT4#U+snutaYRKcefSYGB#ZuL?m&(*-rjw(0STuVpS4?n_TVo}5`bp-z5eXH7GYrg>6W+~!?^IP)8>X|Er;Kd7e52p;dO9k%TQHZt zdx!C6W@w>%-+^x4Ha8cs$BA(f)0QPP(S)N_)e|lZqDcVpySOT;#C$Y~n{Jd^7=S1$ zlxWx43|3iLxYarz1Q`7+LNSdCgB^!!F%3GT30HQl<{gOsxn)fN5cVs1l?z#2GtKb4 zAVG+J+G4=jn4AgrQNHSu_ zrqEz06o?^#1J3L0f$C6&;3#-8l|N~bPTR;w90Kde0B0!(w;1O_q##I%$UJxnO~5A| z6vFc^zmPJN*C!xvawsmhkIUGj6AADu9R-6cHhiDS&p~jx4Z#VE-M=mp8e)17A@a9W zmr|sfdw8s)xeYaxm4#SSuEKMy>23BYeGi>4{X{<=gLZ*dI*Kz}TwBpk+|sm!F)U z`4Mj#b9(lH%vIK+`+ct7L?Aw67Lg#N&5~k#ZM|D6V~G!!NMx01jnw7e`PdW`!dk+g z(f-Z0xuwe$LEnPyz^UmrsgDN)Ee?ikpZi1ZPk-lsgHB74x{Qn{oigCJ6oUMw6>CzT z5UOTHF=*pRz++B+Tnel42#RBJ7`vIi7p!xp;pygUUDq%fzO*{;Ce zP*$;@Z)5d>fq#I%IoHv`PraWQa{goF>tb&YroYpNaueVF1t9`*42fh5?>HW%Ff0wb zPF7Bt+t+apaR^&3(6Z64o9))H&zdfLF<^x#i;%(u6e~tT8|Mb1t zDtiR-@5E5^a^{wO#9G0J<`SR0ezw%CKYdKFeO%EI;uwlVyE`VVd?>_&Efm|s)xNDf zy}tmJgg*R?o=l7EEeA#p*iSBtR1Df!Fr_;SM*F{*rHcNJ_CK=%q{$ngWOGz2lu+8I z==+?%6UWN*``YXlewCLW3{AW@r71fJw_Vq;^YHfEglmny120EK_}6HwLc0^5E4EI! z5~@j}T*a*PU=+gd==A{r-Fo+(E)>AHF{#}_6z^hhsR*?@V9DO2`$QO;7^~CzdHG@-4{QGavB6%p<4&V z8v5PdG#Ngab9y{JP593xo_-8yE6kuQkbuf+($1lE{0N`mZHB4!Vx* zo?HRtf6ITR`gRv%8H|o+gwNzxq-Y5+vO0U5C9Ld^CQe$Jc~eWNx}cKx7L5^Gv3 zmDwVKsCpR=V8-Y^r`G7dFBRM?z1}@3NUpV4#?0+jL!phK*PQ{A*X8!S`Y<*$IK-I} zJCWL82BBZqFwu9Www&F82qKiiqD+FsK;V8niQ^Ba^u#V1S&di~8af~c8WI8o?Xn3W zMc{xFvjPA(kkENqUW3a>DS~B#e~1M|po$nmOTa{f(}Nkpyt{LrpGCY{{tUzh0O&)b zAq~Oif&p{xNgB$Ss122@XFtY%EQ)|DLvD&6ccNI;WVJOGyK5O4s+GA3&>`>vl%YrO zvlWOJ!}0rvkAQkM&ED~P+aC)F>_u)lri}6IS1*rRR@r32MZ<-VLbktPLv|%(G!ANM zRH>9E{dWqMyT1~Zh-zMBvXNt|!6ZT?rK^kYUh+&KWLMP<7cjY);qRYy(n;kh)5ov7 zEEO&Pg}|54poS~3$DAR|e3&a82&E@;rcVF*>3bVOPNk;y?je!kPAV!xveC|H&|<#D zAgu5Yp}#qx9-4u%_hItmvTafex_LJ5PwCu@M9oz-MpY@t3M_@Pi6s!H1|0kR4vuK zxh*+<7VOws3RSr=1XEvu@C-GIB1 zif41B@#m^edkP-JI90~o*}PtjvHhcpI~7s^7XcVdzIXpf#O$~}G+L~)a>V><(}Kbi zk%kptOrxPgneU^Y;ue^EgLn+}G;TWb4$5*upt$>p>56Z)* zDU!Hy^T?!nE?q)sOU$6A^lrNXo{gq;;bjHunBtRNhJLdX1GyT6IA}$Dxj8?cae9x0 ziGvjR_{0?r6=j`$7^+urKqJ%F;8%~v%69{pZ6TsY7umNB0*|ulFWV0Rh*XUG@{Hw0 zt?Amt0{@H7x&)F>OC6sl;it^Nz>i@`cmqc}!BfYqzm#C--`vByi*1ILz~TPx?v0M& zeYHJuc^>OA6Y_tz_RuTvWI1iiKq9axE$uJEZcT6_wKMTiBwjb)R!ZQaM6+Nv|d z30K=`l{z2k*#Y>^?S{QggGiEH?I3KPDxF}b`H0?L@wct*9(xtB=P?Qt&tvQtF~QKo z$xQv5;5(R@l9N_=O@-A;6b4Ulzz@4(*x-?qaQDx_z)hk~Jyu&0m9T2SqzZlz_`0)F zqP?TNjwVSsJ_b!bb5I)kqP zPtQxg2fy#NP=J*dW6?njCC7WE{LZY{fJX60fpUfP&m$y~Tw!M{$vIifDC0-bHi zt5qcDz_bynj@)A7*)as706)UNcg@=w7+RXx*xc>i-&*2Wur?QN{5guAZSAVBSja9a zniq>yn&S{Gv5{49?_{7KS0Ax#JNG_THkl()qNJFA?!MR}?y%d+$uMpd$vAM}e}oUg zg_rU-OkB1dKlWBwEN%+hxHvyQEwjlXjVjo_gA@&;P^npF6FSMBJT$hRus*<)=>5YM zCj@LI>NA8TNC*msEWxR^XE6@vRtnKaSlntng!Gc1Q%3ZF=JO#V0{>xLfMIIUc*l{q zH((r&Pn{(6#}%Cr6B2%42rw#Q!n#DYH$Ylj0}TK`XJ8G~t2OSxwoDW)z&Q~@FvGdL zOx%h`nE2v);Yri02S|$T3Tfr>{(QBo!I*XAUx5lHx~T+hPW=sf!!HI_N`a~gmQ z6kTQS543ZTYa+k&{yW|%HI@vPrO9@&++p};kjT%Slf(W=N6Fdkvc9sVM|{PBA^Q-6 ze%6U`buP$RTB%xHH8?%W<>>1vcgs0L74E&A0)*WH-0H5M5O(g?9JP+h!pVOuauWQUM>? zR&i-%0hi>taUV+)CX+!lRz%4eUCthBQ@ zb&fZs5rx}cm-XGJ5Ebo$50S^T%RkNQuBJBLng@OM2AIE`am$e`6niyVUoXr1S{xRg zbPWB*X-H%CY7t=oKRR>#$E5<^He1=?Iq+#HMm+`IA{+A*(zk&Plai3P8zkQ3x|U-L z&K7!M};(@@K0O9d5$-pR|5cehZsPr+KnpT0SE zi9JncP1GBd-s@Ucv+xWBC_g-|wq4ehg=UGpO~h7=;2)pw6@6yX>gwyvKWo_q}uGMN3x{9m6Eg)bX< zbF^yv?aRE(ejE~H2!GxSqok|D%k&bm!4&!~1dJMSzt94%XT3!HE~}*Y8LI2KnG86H zqttoXoawlZVGfz7q3Sz<&w<|F?33ZX#RB{tV}~srJ=g2|C?xk>4BGCKp1pyuTV~XR zkGDsMK&CfsUdzCi%MPV?J*!4ZB`fBcv~LzZMu8r3>Sgf-yUjY^G8J3omFsC+D(ro3 zS2Z(gvRi&xlmdc|dSWx`1McFbp1xR>Iz#AJ>%mi4a~QEx!2odu)fP|{pU?jSOCgsn zfXuI_>QsKOiB1QXpO4>1HRvQ4isEMr;H&tovRyWE-q*A~FAGT!mS!%6{j3MqAM7%w zY<7IqCn8eChjZ3HL8{Eg2UuEA{-EwBFP+UJvDWrLL4`yOK3cBV3VgZ-gV?X>vAbMa zT0>G2T(?wlGd*g)#anSg{BW6U_`Exe`wmR;&ksfgAUk?Ur4&TE;l)Hu3=LA9lWWVn?b^sRR6*y&UDn=gx$E=>@vU?qYlEu{;^0 znkG0hiC}#$Un6$|%O9jz0UO@!i^-mu-RV4N~^HS3hs>G&Kx%+FhQ^ zJ}l$!8tD7i(|mI_18u)Hs3Xi3CvcbhzTF3l! zB}D=R-IW6kET+8yU65aVwAq~{1&(~>!?k_DQQ`nJcCOpohJZu#)= zf2^poms#DdYcPKri09N|sS(is_H%kls;S+y>HNGyC?rH%<73V{#TI0*RTry5Qo z+rXZ|g4~nWv+}l4@wIXiNVfIu0Q$YV88}$%r>#$Kco4*o@Ly@$vf)Xn*k1iR5nSze z9z3_^P(%&ea$M%YkZ!a?$R<^)Jd?0&>=`@B2$4R%sEUR zM3H1tgn?9JGm#8Zx*%|4?vV7CF|MxMNUDncB6cZcB1e+Hei|*gxu zKqKpFRHZZI3KcA%cXYdswooR-x`=HEIH8__H=*=-?bK)$v-vDYhEaS5OF9B2 z84=!ILkuxmN!Ixe%y?Upr@|IX7bOo;=P{M$pc7YZ0KAwu z0z$bt3gM~XcFjMDsD$!0by5q6ViTWazajPoDNG&Kc~DEG(iO^{U}v5*KNnwD6-m#W zH?ny*KF6(j2@KIgnlW7S;-NQ3D|6IrTwu!<$!iLxQeSw4rpd(eS#hRDYs)M17Otkn z{id8SE`nXRsz_Tt~=Z z5W2|ChmwEF<_HLJmr6aIZ_WFScJ#ES#Y9px5jIi<4p85Emke75T$%P!9gdDjs)bxS zzaea5qp}zMJi6J~X}}zRDatcEV1clXZra0aaOKKewNPU(>f5^s_K#~J)iy>m41BX! z3%^@762#N*`^>;w))ee*bC2~t)w-&60Sl@huQH!9rpGl}lTN>zS5{xb^Art^o=@GI zZ~f6^**$7u1*p-5SF$(==e6N>oEN+Wc|HCe0(@>#INy} z9QG0ybDr+ZURPMEESmu8%6hfYl$wBN>PN`(8QFu@ymK#TOd7CdPRK~ID-6n5_C4>i_gR%NB3f$(a0M| zX>H4em3WNRvV78t&-B3p_rjr+l?SalR#p;SE3530xiN#*B)($KE?4ChlZltQb=JBN zmZLzW+VdSq)iFeFxj<*vo~r{$f-*#9s%H?QY)6sa`4F9s+Pek zVByu4E?<<41&NTB{@``MP9eRmN;17(-`49_SmnSiWR%fbY?A&?T8msU`HSW|6QP_P zqyw<4wv|v;nz%s&#Pj-pXnV)_NSZ!ubfQf*-q_k~Y}>YN+xEt`xv_0$quC@I+qUiG z%>CTYbAIRJ`S89UW~QdQrmA|ny6fLn7bvFb_>_b;|TSEx((SA{Pb&gy*Z!8|o^J56{Ay8Z9OKDS_$ii1x8`r?F<(=;l$d`3@$4 z{6jDv?;%Qdp*=TNQQkG=J1*}AFR9W*`Ms$c1laXfgnwd9IU<-0Id2nZ@VE6D_A>N>BTwiFy``oSf|Cbvclp9KP-X&+sRe>cQUFbFX z5K8zz1;OM;{LFZeCQeaBi}^(vNkrxrL-*fGUTC&GKyGCJtq|Sp|C4^;6JypN^<`>npXbx~aY4klRD##Koa}Vv`xSCmHJ9QCbjqUGZ$t6gt zUGAeQ9`fx$cI^9tEq&u_f0X_`9GBZRO(t7;`_I)^t#!J?Ox0H0m@j~Ek=@IdM-}xH z-L(>bgYH!{s1Bv zC_W^!l}R-^AQD1MhYn(5*Yx?@vQ6q?$Y^9DZr5f+!@Gjn*hf=|@6i|1ueUhk{+_jm zUoc?*)1LgSKZE8gDpa#xcSRIZo4y9VEExXxF}BX;^7OQ{-~svD(+V6;j?%Hl<}!JS z5+#op+%_yAJ&-MqhttzVN1Ck3TSzdqsOLZRvzaybIfN`Y2pKjDc8&EkY=*wWCZdcu z2t%SjuQBd%DbvOt?;bX{a^!yfW<355iia(5;G-qkD>(i=lj|;I9J7dsv3OigJ9o3y?`b+4!+v-EkD@lR| zSr#=dr4-q$UK|`iMbwq+Tr3C-z}5Nk2DK5v)kj~4OK1Unf6)$6>6IdZR688Mv?1ldaZ>m1F90H^l1zAXn`+x)h$G3aP%!(+f_1j7SGOOnFi}{iZ%E)>#(Q zR_#u&38)~b%nC-DT{q5iX4?pn8MDR`rHqZ9pVwp}AWnFn5u5B8W5HKqv)zPr4Fwzn zqRqvX%Uo@H|4aGr=hcNa`*%`Eu8!7}E11UFPEt)PcQ%8Aing4Gflh~qdQ!S$#XH#R zC+<#pz%gnmHk1@y0%&pLSGNDO`^6~p>AJ}+%T>#`yXN@0OF>ohzdKFSIJ$hK4 z15Yc(NvP-YYDCd6lbM+(-n=R!+lGm;XgDs%^W=G4ai)2|ZVFv8z8mTNX(M2Ql*>IE zewD&a0cnFvn}0!`w!*!HV!?pXQJwbj&-M6Z4=)Ut1PV)Shw*ch(T$gJVa@(*KC}7D z_`Y%t-jO4@xViW z)IlIm8YT|k&VvLH7380UyKh!SxSy|7U*}`OIXOG|M{%N@ZqfDK=QExtyo*TdFPZFY z%jHVi_<;}K<91=Gj-0oxERhSiLTGU)L8#u;KQqX|a2lu~a^hjs|X8$}U;MSxCVm^{(cfTH52I;YDHQ z_%u_)ZqWAnS+ca9O}uKC=&;&-G2gg;WEM}<7h@zz=7olsYIF0Y?0g@zxRX9XK$ zZ+-u4L9$-ZzW^<$fA@f6B3DNh(L?b2Tz@C2^*i- zGB~`4{$HC`-7S3qd=NV8)lv6rsdNAUj4K|?gx8dy@^1l4ZD2Udr1?|ZA$_Os_SDtV z?}c4>;act4o39X{$MLpO+NX&dO3aHRuhWS`umF~JOgalkhjLkZ8xQLwmi?C678c9| zfR5m?ihtO=gEVRynE21XdukiKFL$8>_s-Ou0UHbKuXwjwR<#X!vDx_Hq}{XK2d4;8 zN_>l+4|w=h<@AoubIrV2X^&^pV{kmme15Z!$VEe%Q%f*FIhVT(Tx(youBM>>40AmO zWE^XiVKx8&l(?jbpvo)`hnJ(a_-STkq!MGa9Nd ziV>UP0~0_1T=Hs0oKDjD=eSk-=VVW!l&O`~D5joM%q}Vn;qvosY!K$|Ns>*&8)rge0Kr2!y<**Qt# zIo)Yr*j<1+T|0Bdj5D+7OgW7kx($(=_~YYuUYc29QotC|veZ85NdMm}BRPUlKB}jT z2Jusa4QWe3V0sPXWH%2Cu;s)B0u*``PWL%3=AqF$sra7$4jm>jzRXV=KE4IdCcr)} z#;Y>@sF3)m&c$-M@N;&KETeJIpi$s(-?q7(qPFjHvm?Dc$FI(7u4}5OI=ntNg-fkt z`3gXHR#VrpCD-!W>Z>!oXF8q#cwz(v*J|3eO}*YPXYD1!u%+A1&txYRC|rx{Y&7y2 z(_5{$472}3IviC!c<;G&-5-@gS z&|UW0$F1e7x_F#qQ|_oH`-2Kc>+;FV`Kn?4$Vp#2t-a*;qhzme@^7VihLn>N9IV|O(>ziv ze;UnLgrWSNa=w6{82=foeFZ_`O&;xxy0Uy$Hb6iNccCVl!20xRT!XoFFY}P~WllxX zY}AVCY;K}ZyhWrC@2jo7jc%Kpt({BNrORPCajs(UKt_=chdd?<#5?c$`a%Z@^R9~U zA?iWJvCwOcEmh|m{e`qE{+ipI18_}Be?OYC1j$5B*L)2fjwdAVpp9ZeWqPVi7YM2B zT3*3%R)qu8umy9O}(U8XQ`)I6p! zQ3a$-`9CDNVb z5ul?+elXUx&#y_nj~(*gzdgc`o3_IaUKp;6fq_FPk)2C!|2A;mrsp?zj=&GFmTGkc zejj|&#Ne5}(sUx6f5R$CL}t)$%#!NcDEi=bV(crLmucu0)5?S$#(1nX$vXf?W`f9&l8F&bRTYsnr>MGk|BK)~(k-W7B@!V)3I+_*|9wgzg92SJzL0_n zXso$5|29?V=pYOfzzZM`qO<=u-ajG>qjmcCFD4EqqyFy^gEstMPgi~-sUC*7yo)A4 zgBaHUNl`hG8X1D1`yttB$N>-^I1QZ6S8dDfs#JQ`-x-p zau;c1`-yAx<0G6SoQ@IaZKEw-m_sD>%4g_R0BVW59yo*5OQZ_T0#dCO8{qKc0DpMK zdIr1pK{xxY8MNgYlVC8+HxiNXt7WM7BB*d=Vj<9XJNj&D|0*h}m*BzwZNbdWBD~QI zGu4bNS%&kOnGpf)=s!dP%X8Nr`+TeDfNI7dql_#_*PeBPD<%3PPOJ4#xjsGNWgV;F zV-2LDA^j$UaZ&%2Kgp9Pt_p0Rk(?$B$rUErnHK%0tkT>`V@R@bhJ1&o?^lN;UUdI7 z9h6O7+O#2hy7aL%b7UH9HZ$VjnpAW9}g3o4I|0$xYu~qiOj>{ z;bADB9tqMeC|h%Pj*1GaxEu~uRp*xa@?!rf1GY&x(f-s0w#WiwV;6QlUKu4cQGNO} zQ_ZaPjrnF;VnbITDrjLj4dEuB}scQDL(i2LS+ zpkjX4_PUS#nW=pKd+^TU!Z!H~9PH5I{2EbEhlF$8rC|U)t<2dar-slBHF2I$_`DUs zw>r!Bt%#SbP#n1sO)q1J;iG5eL~bD6F*aJd%jKRT%yCZWCdKF@WD^Wzma1&N`KODy zXvzW-j(wPWo$DZO@ecr?vBI{zxHiXXlu0^NKbQa|b? z@sv-K15f`*%iOu$j|&#aeE|rBnZZ}boI{jQGK=RFwRtE_G0m-+qFC+@rd8aePc$R^ zlIqUxwWv>ab*gin!p1D@SAYz>MQE0V82HZ2K~Jm|TM+JDRe0eu`R7l1P6DuE@MtKO z_$=3we{;NIcfTtON$~k_c%rohF?u4DlPN_A%V`o3s{{(RIpDR&IcK2SeDX0SW+LCJ zS;|9|1u}};oC`PWC3N@zPq1B#x=&+`vpb%N&FW@~Cq-#OP?$M1ccCRWff84|N=eEC z^K$xX=?H``oi0n^LBwf@V`pteK$Hz>GfLT6VaEqhtXkMhYcF0U2hl5z41o}CmnOQq z&w>NttTa<)dw6Jo0ES1kdH2|u*zE$29GhwK^;zB3CG!V~WJMIdF9KSj9i6g*ve`E? zw6I$6hvHo@mIOJ4g{n$t@7?ws2KYy~115?EKXgvw@c1qtLW|eBBx%MHNiDGmuAko} zz23uWl`11Z5?F=1yJ}f6hUvw5AdvCE@mlMr!~Jqc)!*n#-_uk6=V_#c+G;F(LjU7^ ze-*WPNQbxK9}Muq({l)jeU4o{`4NVi=RVre44DP3L;j7eJ2@goMuV z?9_luzsnHziZFf@!~UnE)6aY8Al!grGVjSe!n(C1Cfy+5oZ6}htRHE?TJ$gTlH!sf zj9#Vfj!EmZVQ$rszu7FCf(4nqPi!59mB%O{14x`Sh8!o%oEuMy2dQ@!F9v$<4 z!Ru81EIB>hyUW9CLcl>Ys+9GXn}sTw@9&rV0KzgFXs$3$J6~0Iv*ygg0fWCPyPC3> zNXti+;7qE>dT6AYe(gN|`_URZ|L}w{N<7fw*+}w(oi<@nQI*$9`^>H0W{Pco#i3-` zK$ByHRb`U_MP_ttd_+oJS7nxF_t@Ib%Id^1l)f0*_VXwrxIwu7IRq#q?A?{^{$=T^XF(LFFAt_&aM+UxSKG!Z2Zmp}32(Qs~ zo;I0@krL;aLz~|kD9ksuwN=LMu`fNm)5uF+@ulF#X&3V1#CUw;G48Sso|$z9Tcu?G zz)TtSet!Jj4+Z9NEtwZDFE=YIEvZb|>DvM?@6wF@La~`u@IyxjqIf`o5Zz+F`dFlnURr;^xZjDs3q?vAA^O*+cfW(v@`|^=w?Wl_G+UGA*;Wr-dO;dYG}ve zf(--NzwPimx=#`L^6pnTUovk|h; zUo&zGq0BNMwbKx(RG|rrgZk`o7j)Nq92i)_Q{M*lT%Ck zfIgOt%El&v=i%2wm3C|?I)>!MIsR(7X~f5-R(IDB+?5~umH86SGz5l{MW zXe5_R(}$FmcSEpXpL`Z6QgpHC0X{|b*108uISpLgh{6NT@4|wzIu1@#Vgfc3kxgZf z?WDKD2;bGdkX}(B_OSoyhCp$G~cYzUG_o#Nu&2C~?DrrKM1IBk$C z79C&=(xIVc5}ZDYY^Hr@k_Q`0bT$?yKtXnNDbYjN!oqDKwew`qMT(#X>RDjB*M^Hm z(~7Z_a-jvq&Q5Y**^)VAfW4-CsnAe6K zy-qe}zE&@~tEgZABFCU1kFL!oI!-u1SJFUE=+|6{HVtS&Z+wiXd1+&wW;QyliB{ZI zotZgRC{EZFL-w4bEaw_E006nAJl_+RK~Y91S|@y|v^0FMfT^26rO9yX%mv`|BMnYe zfyR2jh6Vso>rc+M$hg|Z#!H3#NkS$^_(RKrj&pr6XgImlyaeVAeAQ+wcs6gq*h1u2 zR7{j^9gK#RQNt$PL>n`+_RqHLS=1oZ3zcGuqs+Ye4$9ENspEnalw2k@d2Bd8wlopo(=|Og? zzsn%q1kAKdz}-8GJHRk;OjmG;$+qsRx^2R&FQ)QaO*~?7 z+~5$!P}ii4Kz~eE2NW+m6Bj`r#Ia?$qSZa^WQ{IV+Ss^+eA1Zdu2|BLm2QDOCVc41 z3is`;^R1H;hEo6b)yQbY!NRTdG>B8zFS{`5>gFmhD~HeTcyd&a>I)IX%iFzu<=PFk z#&|Q#{OP-8b9iL%*rWb&b-H4sF%6H!>jfW!v zvU#CD9!hhS+Xg%-4vfma=4+t6W>bP+a6SrRqM@-Jv9}jz3 z<-QQJxNc>zhdF{A1X=1o<0j*{uG%%D$ipXPC)&QE^sP_NFR(ML%{uJV`fsxDe??iC z{!ahV-QhfC!ZLT^`+oF{4VZDch}r!3Xtpb~oH)+0Tzxd!x4;Udt^K^oNZUNdVZ6`r zzHR;FFdF_jXiL87v#p`KxCA1Qx8}W1-vZBm!kK*+^}yNqSp8jbYdl=PFdFnK=K=+u z+Yw}+camKUc9rqa+wJSIa>CZ!9E~Llq04ID)*!wgg132i6?Y)z82N+as zpR=EkNjT*AK0mM6C5>2M;UEPczUF%0Ht~vpe9*J}AAwfn0gbI=419J>4}{OFxAOyf z0Q+72&#S-A;@&Z21Iz4rp7mBAk)7v9%b)$7^8=tWu*7OsY9oA4-*a;jFl~av*Lyj1 z*m(vY(0Q@E$@glsUN=4Gx3#R#_qwy)k;}NdwQ=~lVsHZnP)tNuT|8YBNCF+KSy_g^4V5|&R~GF zJOxLf6v(D0qjFp>8G0?3A!afuvB{$vYxipuRZCGB<~4xgGdoZ94|>0KBi`=0m{21P zx#4dO*a^915yc6RK3*JQttElJ14Pirh+PnCX_3mGUIU6Sh=q%u^5p(RijN)PKZ8F0 zV-(3%f7R*sqFkRXNp5)%4Iga;6ypoKeJqxEcVQ?BK+uM*6Uz&Q1C_xF(Zy8khu0D@ zc(clRjup29n^x4)H#to8k}YR2NbvT+-6qhW*ChV{FRxMUEPNhAjvEHXp-4GGl{fZv zt8tYPH08Iq(f}d@n!Z19X3}q8PcdK{Y0|lc5R_tLh$=z?+IYZl>%4_b!odM|GlKBK z9KJs7=|HHhuZ8<~S?dsjX^#CI%=S4r7nLEF2`DPCx{T+ zIp%1d5mqz1O#v_&7F>RBPuCT4(1PIc3;D4XFM5^pTq}D1E#_Q!@K4tYe=eQcd<}8l zlHR{mI%rywd?wNl1e=qYPo?j*N$&=uG4iw8WH)D)MkAP4#IW(h33U907#QXpd$VvK zJMI9x(OSIBOU?3d4c-3Mx*9}ez#@kwo1|`Lh?QBkXkB1`aX8JgzQ~UZuUpK#C30Y&oDQ@5I8r#ohoz$iJ zywG*OuHg@lzD*vYzFbTAUgrP&Xdf%x`}v0;qWRdH-$k}1bVmi9yVj^}YD zn+aSyjjb<;25D{a@j;3PiE`dy`-cjM+U)Zpe8=>h{F>)g(Q z4htpu1sQ(#Iz&fPYPzKh9Y&Unv)8+BwZ-d}bv*v4b+g@YZ*FiH`eH&I${L_LWNPDS zP}5h+eqMSKRMOXa3|S46LtKZqxD7*9d-CUWXz=u>Eb$ z_3=V)c*)6k@;t|7jTYYZKC9sJ^Rd>1GJcz?p!=YK>QRh5tA<>`0%fh!!kb}dR|h#s^gxv(ldBJ+sn?FJ^Z9SZO=xujzL8u+`{|EZ*)au zQVs*nJ!D6riQy)p;Cv@x`QACpjZAzfz)97w<`_+kebeM6F2cz z`*8)A0UQ1B>Jw{BUg3M}?rZFhorZuN$9o$lth+2f1QAeTG2h*J_c!?_Ym+wjF!%W=e6gp>N9N1Og09SmV*Q zZq&0#oA1P7X;njW2v^>=hib9JO^D4ps~)2IgjQ!<7aMIbQH2&6GVP0=va_*m^SrNM zY=_Hhi3oZgXtD!-8Xgenz@bz3kZqSczS_fW^@jBX0O+u>8#Znaxnq#6P7;*yk;mIm zBD<(8gzc(f4kkcg+Htaaw9HV_c6^uR^ga+m!6vaxg%aT4A=qJ5Rdqd4+sGM(cLzrV z(2ynGgrkOS;$Si$b{<{acSZep7VpHFoGf#*Rj-#ZXlrqj(YP~^n>)^pdNzzS!U90< zIx+TeaMseK00(Gyft6m_xm`q(jze#iHN+88us2dsi6+|_v1v$m<94I-5h6{-SJj8W zgn%fiRT9IWh7^ef8W_lEyH1=GfpzfcO-R`m+49=0I={uuzZ&^9xnEMhlcd<~591$O zspOPdUuHR&3^}Z!qM~BRB1%ttEb;hkRdcEo(P=m5b7%^M4U|O(=^e{Dh`vMzk|qyy zkjLU%<{%(58adT{=$vjskauaGbjn*JmQ>)QU41);M-FvC-aH`EPjcaam(Sz3ks z(A#mgoC=eEDWF?~7zlFr+f&LY-{OJFD z`oqrn@goSD>>aOmnMO)uRG^E7lT43l@2Vf1U^Jh?X#-EGMAgDX2N3_G^v5gG_)7qE zd{&Z^>JfYzaBNHl(=h;fbYCacOyg9cl-yoBGBOVQ$Y>X&@%06jkAGLOT3Q~t06-$E z!v(Z=D?E*2()V`u*hpRHHIt^;7=}1}PI|5A49))7f~3@R3%WLUe2Ki-a_0Yx+n7IP zv)PDm`}2($_;>5K?`3ow@)}R%Wt#(~(rsc0R)7edVbI>ttGo;nd$_V+4hcvBAqmDJ z3Z2ySc?1sajD?R%y&-fy3Z?eX?c(B`ivh z>+fr|kXs||@rf!R%XXHfROA0vKd&_0^jXZ@9dvAcYh2@r&B187k5rlx20k2AOX!?x zPdyp`sCJI)0zQfkvirGbVGk}yyUi*28kOZR87pU@8HjPBuCg}_F=(&P>t8YJY5Fc1$q~9TdAI!o z*E4DOD<(mojIo)F;qk>$mTO&~JoTmm3R2K`_4meDWb90GQkt=wHuVRyC80bzrSf^$ zZ-KvjRGVPWn>f1~X=Pi|Q#LzlX*f3Bx4KVK+uzmaq#OJiO7}FU0?JjTM?%(-$ zeUU}T)_Z5fMzO8>#z1xU>75h4@UNbBd{yer&f@Z?lfjZk;ywu;vukN-I|TqHrGvq- zKC`a!T&7txF#|Q1Nsb$akkKw%GAhatf7nwixzF5tabwom5l?o&7txHRdj+|1FpUq51w?~P_Zxi3b*4o~}#&7GxdqzpwPN5&z zx`(^}2P&Nz>k9?|tdEyXg(@-9buFGAh6n&Kva>>T0vn7xd&;(dyE%kP18&np(j)Jt z*tg!S?hZt0z614tsb`kPJsTyPD^cM6;c#~ZK5H-FF^2Qe*-o%l~RDI}Kr=>jUSu#7Z4n&+o!2l0Y6T_U8cq zaqjlg{t{@y2(XBP8>)_>Bh)E8c^b8-XLQ&NC+s)-;Uem3#Rou7$|AwK^^|B+o7bgL zL{?9)Cb;+1@UvOgl?S;s1JE6!4^m*?i)N4Sz6iNcY&}33p*hW=^j!6U0~pGfy1S;N zIdVj%2umdLR(F$zQ)*QhV07HfnwwsqQ+EsepCa`Sg8Ac!-W)|+jI-Lls2ASA~ zfUHvo2>`C^q{d>tcA~GV;^RIV5~!JqZO=2yL7?Rbyl5DF9FGBuEWd|pnBH)pfLGGo zIN`oggV@jER9!CbX^1YKYe2d z2^|tABF@5nLQ$&J9g!aMld~)7!UV!V21Ylj6bte3*=1&BeF}gjEgBm}|_U4-knD3UY^-Dd38J6LT_5R0stb_TA216&w2x0 zi`}>d=wdL~T#1M>JG^wy!)OVNjBc4&|$;rDGJUi^%2d{D0 zS8I@4U@@Jiig2!8_(M+lX{O_Zd>(i6BANg)*^>9p^XIxZMwE69d4-(%BS%;4MyOaE z8KXKaxZqmoypHgyr`}Fxv<29{`>O{Qc7N%YiEO5|x{|)BW4B5k#$Fa}L8CH({1LNP z?NB_u)uZi49AQJ8Krn7NJeISZbFWTAHXr=Yh0Vst<4=x*>HYOIzUM`M7Oif*OOJg` z`MCn`KyhM+blQrKN$2|;54kPY`_@#=-$1boL)FzsyQz3_*Xge)U?VX zX^jRiO4iBAapveow6Ze^L88^QRg7WYUJqZ=+&1L7|Dt%MyeHzQ;bc9PmV`F zAhy6SJNi@FJF|`Aaj|Dj8fgHky)xUfD~$GwhcGj*DA;RSa|y+}0<~Wi2$5YPS@4z#u^} z?7VuT`~vh4u6Ca8GE!bW$Ko3^aKA4QRCo>1&9UtuaKJSuXqC=x!ZIr>8}6%eGZo;M z?Z9}bCZ7AlBSMIgb0Yc;X}3|4EhNx!jfd}Or)03lUhvzPhZsDwqv&a4AqPk7u1I)6 zMm%tT*pX)|5C2@O@DAncq8V!}qV`pQeOG194B1e*==Lw*>9FG(KPRu%p%uyxCRrV( zoy^P@&hy#XT|Y&;s`R2OlnJSG`OkR@0$=f zy;M|k#ZnsTQKg&eAXF10vC$Mav~frW9EeO8j$vqo*btFXODkp>kN4bd;vn^uOp@m_ zAGuSq(-UOKZ3hmk=F!vJjKc#)w3IRaGn5+vA_fDj0!l9C@9z*H9UwmqrAi09+Nz3N z=1FMb-=0zXHo61TyW}ZSLj&G!n?C(Wf)=6eg2-k>$ng?`TGeLT^sGOA(6eU^XukO| zlU{&ib&ffuhyy!wl4*#UHG>!jt5PN?$7V{=(5=Pt{4M=mWuqKU(|YS^lcH{t)T(}_ zrs?r2%>GIB_4fuZ82uv^E;TZiVdHVycilMqFi6`8H`Ld5a5#@WrtEa{Y;1qY>*7Y` zH9sLjO6BZHv&ny+nnJq>2LO2BNA5Yo@rMQ*V)Zml~z)l};7D*U_xSD8<-hAV&$BDDa@B$9AMV;5Lm z;A66krCQbZ#PmJNCDy+6p)lW(01&B&@bOg-xk6)Z9g9~-8w+b5cM`ymen&~kD}a|s zjKs&OEV>}djY_%OjVseAp*TKdFLP`o;Yp z4!9EwkB>-`5l)2dg#aP2THh4|EGBpy_@K~ZPp!0X?)g}$cOHQ!MJ6d*zHaRT1gNwxomob#fsv$fjP9}18fxz=c({9@njKo3dKLnPF7!Z`uB2hrHa6NxEK}p z3Ha8R3>i9@*)}vEKHAVQBwqLS?wTzB7%+h95=&!}3)>p**fsA&81OS39gd+^1y_k? zLJb}6SOAl1^tnBe`4$*nddM;A?`+bsT^7MnUsW{z!683ghneV3P2 z;^eB}E??sw#jK)Z3x2iC<@(O3_~jt>A(xYyy_rFc0KGKDz)+>iFsIs9SB=zdY&bg#8pYTNU1 zTH9IXMnIKZ<=DuCj%trk9&&H;tYjjIP(Bi!Yp^!yQO+^<;Yd4m8sVMF$9Y;wu&(@u zF7XQm2C%i~+8($>{&8vi97V7BVRD^;*zGD)|E|Aq${#Z$`h@p0YZ8wGhpxxo{6f4= zCtfoiOo7Iaga-e0G6F!VwqSYU0T8Sp!FG63h@!H|f&yh+? z9OkLcP|u7tQ+21>;i$Lmo*Sq0%!YH${Mi|RVg2gWw&q|O7#tK0-=iKc(JeY&W;fhY zO>^iLGb%E7$>os{aU22_%9D%7BHh>OZP!;|e{K!GX89Q@v?bic>19oP8+I;gC+y{E zO^GkeExxWq`IZb!+0XCFIsL@^t-E*;V`M~X2Tx}))B=H z^j*3<-ee;7Mi`4%|3QM4M@=ozULh}16o)(!<1K}u5&E-aVGGIM$o~jT(v0hp-*5YA zr|^cJmku3P5$zrFWf|&zxQ=F^4i60skopeq*SdEG>1&p4rnzMZGtgj&gA5>m_Mo5$ z4j+P6-JLt}pwWFVCkQmtT-d$;*!KU>(NsNm(_hkEw`$Je@hxfWG*M zuNUfA#g@kpqwc24akwp-$@3>6S?3FEw1*WLtwf=mfrc4XuYGMRs$=b8%zZtQf(kIV ztE)ww+|;7_40o%BnRwHsknW~RfNxjm`I6#uRuMZa+*k`+B&l8;7U(9E9qQ{;5u5HQ zOA*ly%H)kK^H=%pwz30vubyUXN&s@X=VPiXg)LJe)sR+gg#oXH8#@PM+YzkBv4V{H z^DrZ67VZ$0kDN;F7BjRwQSI}p{@iJ0`zshja(QvXCT^#tb3!;oUy{W<6MuG-KYnY{Uq(xXY*F42?1Q!Lw-)f?>jD>v)jYOrk-fM5u@C^%>UqB=%( zilf0v&ZoE6@~nKZbWQ3u#?=>Mkr^F$Kb7C7jZ;0yolnh4OEXV9k<0+U#)rS`eg@2x*5 z$$=DAr0==2@)e%Fc)iHT&Sc9vG*K#F-KC`YQ)>BA-FzcE6TlQWCF=wf-?weC%y~pjl zX-LCcp3^bws@G53=?*1P&p^2?6|TTE+Ry*rv$ZMKh*VjZHX%NlPEZ( zRG3biAUB5QwBA9>LNz0*e_hpCv7T_b$-xdn+OE~w1T`$IWs}mcO)|hLL z|0Rr&b+%Z?*`I@tK9c`sLLYI3)bneGsQiZJH(K4qpc%&f_c-IEEN0Uc4=U=w4tAA3B++~- z8aj+6lSHr(AJ1OnT&`F9__rTPBdNA86G573F`MNji~eqZa`Os|eAnjV_`A47(ekq1 z6wehV!J~-#2I9XD6rnWzQbl3>T?}K1Rbzdv4)1p``b#=i{facwEF#LConQ`re@;%lP57 z&^ANb7_Z~aUHNE*`{Ox1)!#*OQXN;4Z7g2rI;T0Q8J}3Kq~85lre+1~yx|+hIz-mz zLTTl>u8ao=`^nS~19?6|B9_^YPN~CQe!o(kryktC8MJR#ig&>45%x@Yl#d^Gs*Bvb zc!Q2l3a=CmMVm4Dt+yW*`9X3cR5Wzo*Upaij#x>>s@~p{aQy@{ za=FA#8xxt>BOBH3n_TY1c0iwkVE;e3`C!wPTKeT?!8t|=vs&eQ_0AUi2>u0&_Zy@S zc|N9(7w1pgEmF(i?OtAAHpA70xifV6+Sh&yS=A~pNv{hGBHg5z^TWwJmZ5L+3dw7p zTfFK7mdxM|#n_&r`cDTF3j;wFe;fKYUdk2AHTEJ#=qM?}U(?yPO|ij|e$CzIp@rUg zJK&`Mnw;$ZZAb`hlin9Gb&pUImw@Z~7Y3Z18sd`R$`I?dk+2dpY6l7D>%Xb!YT|Ra zeqo=SL9YzXJgSs7uxx0(xrAQDr@?Q+#&^1{eF7NC&g8gn=H&y3QAB*G@2r*E&s(+e#_` zC2Kx~p}6aHy9AlBvSk-SwFL?V*j`2!^%I>4R;yX}v%lZtY3g0JDf(*bUS--$}ikfdY?RPDVyfxA5fb@Z(zUm&K!UocV}H)9QhV zf=H2+ZZk7FKr3q_gKk__aWnE1ZToxh0BX2MS_;DGp%>?5qB!k*#1hdr_^<)U z*zX(zxa{1_aF<$?eYMS!KO)B1MkYQ>6uZ4TF#ro2DFDDD=HFP(^u+8lUj1h&pl2Q( zg4j;N%zQ;GMrU<3S$Y@cJ(u2i7_=bgUl~m07ENjtj@8Fy6!(2=Q5x9?wli^;cI@kV z^v)mm{_DVlHSb=(g)=mB)9{#k-euE>yqSzc7KZ)T8qkiy?E0wx7RbdtL=#$%^HavvUY1shjQzcx=I znvmL2Xin@y0LZ!6N)2YkTsj80xb)k6L~Am?H87IA2=yCTSszOh)zNx` z^t>t#%H6B7eOzl7I^ ze!&}8CD}O=Fuv9mLMrJi)J4YiGI|Ip7~hnfyo3cZ6WA)|>>B&)k_U_aZljwDAGm;x zD`y-@@t1d1?KziG?$W=|tAF;5``{WFWw~lzMm4IX=$__t6_h+gb~ydMZcSu{hjvASVhOR3i4jxv{)xR-~HBRvB;TAj|1-2u5tQa zzAu4_?!zrUUjn|iPZIjxn&iEl@^2Ql%OXGlYDgLMyiJZ8kVX;*JJ4YO0U*Kfd#bSC z`Sd0aFG{`jg43(}^xEn)E=?a7$Ho}@3$n+`#CYwV$M1og{CXhl zUExO-Qf7bKgdPhZC7bMi+4YyHuecM>drPws{k8kYJ1jG_pX53=%)jmcB2IskB=@JX zJ>qSzllrsDK+kuwb9MrKW?gk|YU!AJ)*i24o0HQ%`=$Dy=^~oo>;!hk?(39ve-Dx1 zLvsB`oDcB}MseN-Ep>>+KcBLWqFHhU>OArUcXEGEm|4993Qx}Lt>J;#KLL@!*NR$D z^xVxo_iHqOFw90uy<&}@4*w5TZy8oc6Ksuc zoCKHP?hxGFEx5Z&(BSS8Ab5b_79h9X3uzRMvs@Gao zU6nL|tvhKRzx-^`70~d2@{3D|O@NZ*PdKx5RzgGGgF2Hm&6iwW)<*klI9-(-=0e`$ zx)L+XzXOA+!xzM>fo&EIm6X`jt%8NJ2#W3Q5zx=j+^uwbWOSd;Hn5Qt2CQmo<3*nT zo-|km>T$FM;NJYkxbzHoc&+U{Q6GEpqpJHs4hKsn8O+s%PDPk+GBC;*FOk}3vVyM9 zHOi?lF_*buq3dD0=11(ju!I_;Z}JY5-RY(29W=kE*HDx>o=1mlIZQBYi_dkXrv~a5 z7$jBHjS9ZrH23y%&U%Na#iHsrzl%Rz$cpa3rF#~yxp^9&?ka7ca^NiYO4h^5#PYn1 ze1jSR1951KAPd2ayi05_*PT5Ey@Q#trGDD!i>p)p*)@YS)!)yr|1q*V_m8)iqkSWi zS^GS!+~Qj-ALtbryriMew_qH|${PeV-qFlghp0m%yC%vGVSGmwf(+Ny|B0mUruyZE za3ZvS)!~##c2VnhyJ1-zrvTrr&~tlWLd$KtkB^CKX!d3Om}m&vFq z+3$xV(A&iK^+7{qFD~nP3>Cdsy=^U5^_jBSGKZ^C3+`meDkZp-iW@AK2qu@$$J^z8 z>JyEA`^$tT$Os9l!`pI4Ln0ey)*VIptH1QS&aW_V!voO;W7T9t_OE~Zv0lBxSa}>5cBhO^S-AZJ8G=)Q#%jh6k<(^Ei4=f?VPIOY_d@ zeMdhd#4CMEJ7~f52(TP$F(_o6zd8R>3+qsEfQ}s&SEXVna9&wW4-sXM%G3){|23Uw z9-RB~C9r*;4R-qrHK>2EG0&yg!h{F;8MA8Vnu*#e%x>1g>Ik1xR&XkJ;!9@pLA@9e zQt?6E&s%fvH1?zM*Qk_>ZD_IWue2gY6zj%~Oe^~*+OK{Ve+U}%K6(!=WE8dhu3d_$ zqk}jpxxe_Inutv_kXC&_$p|kiU(L{{VxEDJ(J6PiH{6cl(jit+@XSwfmTc(>;lpJN z6LZKUb~S zc0v9tQ)vI@uEha0_GoHn3oSjW&Zq1AeS8~bXReH_B{N%ABQ+Y#GvfrQzXUYvda^JI(23oF;jt$67 zG4!S>Gog>EV^hXVTKu4mZUTo?RtO8XC`x3Du3${94~{Nrw+^{LGjMqbq?d76CryGT z^f_6x?Ap^&O;%XrjH2f&l_^|NSTUmv--aEUsJD3f7YlH z$hFy#Ym?3Vl;-JTD&x=iUHUz#X3ojddv1rq>^V+xef@-6ga+e6CD+`csa7jPCxxJ} zRm~_tlEFl%-cKHjY7A<@4%r5V+elw=|7F_T(mCMyo=UusxV zepPxu-ePorw_B;2ks914@>aF2QR+kTqf@O4%9F4dR}wgEj>3d(|2QO?(WL70bK)ye zAI*v?+w8cx;<{L!R(a>+o4pgi`#{tRzoW<892@vi6s?e7i~>RuCjxtq2ijYYsQ2%O z;NZO5CfFE0jQO5h-j06C-?oIdsF`9T5#i?-reFX4gfAcQX@oxnE_gTz^x{BhczN(f7n^N0 z^}2ZeM9v;s#C1HF4h}uh#js2d9u=3-8ghn)|2lcSmpC8iNlMbRF7-u)1m#)56UJ+M zdg%HI=Pu%k$4nKZ@dn(&lyQZ%*uga=y4x|J67eUdC%nwAc$V9azO1#b#>K|s<8eMc zG>%=NA&2*@Ct6CSJic3I8)!|p38%knV ztN!UENa|)!^?_?GpH#TgYxM7*4qB=_NN^od`Q}7SfThH!gsJxUF=VDtFO@O zZ#)BSo|(AOg7OeijXyl_u-&Sx5+;Y zzL(Aj8>%PB$vra*iC1c8bwlj?jO<{7?}DL;5b}*;gyWPXzzgU88( z8We)mH;R+rm=|rl=gN(Jbp%3l#61=R%{6hZS*7pcukly2gc<~qAS$TJ6c|hv_q6|P zTmBR;A2A5^3vvFS(}CJ#UfVxJ35z$_HwqOEDIs0#jBT?L zj}jGJT^Fnk?&0`9**%Dw@O*`k!YmgiF0JAe46CR)MGB${J`snWXsnM%+H$b2H|Mpe zN>hB9e7hT$Q~F*O*EFL}tyx6Yxd`31q|$jbYF#OCzH*riwt9y49nGi5mq6ja<~#gj z?jvS)Cpzy9Gse(KNpJi5N&OvPyGICl#*rzLh9Qu?@wh676ian=^&Gz^)v%3!d(e!o zyV|73f}TYeUJy~t%nl|NvA^OT_{M*Ox3&cilzDFd$hw;UlJ7eR^X*UQs~S~5gPm-f zd0$k1P_FkeN#3KkSH?N4ZM(IBS7DQuLCBe}oCk--jMVljQA}m@n&LOLDt>FD6T%~* zkF(LjLo0_lRho-Dxsk5#K@)f0MwRck0WZ}s4OGY%x(4fH1uDhBJLB2c?e% zZ^@3{hl?G-34sW@uRr{5`te&Rb2%cDkn4>uU!(AsK_kCb-1Cqy)rSqu%sOs8`e>#F z4T{~6UyXUVxK)iAt3L;(C*x;;E!?cb&;oBvrou zS3D;QjZP*rfgZgKJEN{rpv`KOnDiUXa{aCni2`obNMaCKO$DN+-~3M%*ye;!bR#QG z#U+%K+PANZt*1YWMxUY99B-(|$xS0LkR`Yo@_D25ohQpGJ*?{sD>obdr7Y?Pwl^9r zzOS5JDLd}lu>MrPLBb2^4yI#3m6W;(|B9aYH9N~SAW%~LH|9II^JhWfcs^RIHrhA4 zzAK6*zlvg5IN;z}ojgW8HvEg_Ay)q;RXUK*Y5zGtv=o4B!ZjBMO&uKR&;JA|&G1<9 z#;+7cCN%0{QoDB7q2RIQ>hdt&5GZ&D))0q@L%#b5b*uc)HHe${KG$DdHm9S&N-gj> zxeJ$`Cec_fpGN-@`dhk*wJzBk?wu{MNi!#LxZ+>p!-6tyt=`mh2PvVSJb0+Y!Wk*fvvD?!=dF%|7B0HpeOJ2GQIh$}+LbXR7-2o@52m;-pA4h&&goWe zKFpRl(x?#)#QiEA<#=E)9SeE7@rBJ&T%|{rKrbLrUSGWT52bBHOZ!q(dM4%^J;>&k zHtuJ~LzjV#m@~XiCRZvxr>4z7&hK-N>Ow{S@Gt6jq8_yby5RI8Ny(w(FkT)s=v^{V zPKC)fHP~-3T~^xa*%-Wrd(=+>&ZW7VCk2Z)T5&ju8XJ=}AURoKo@M-Fe>VlCIj9r2 za@9}&J=ri*F@OFlK}HoOoWPk%Sis9}_t8X0UOm*HDiN=wI~0xz!irtW)KV#Z%zILW zTu^c55H{rF%P=gnbvo+pW-S)$(s$e^Sv!J760!#gU$b9nMo=b zm9CP6qr6iC?|9H-YTzT~nTML@n^4eY(#wjBj>t;{I-Hu~ktIMSjFX5UNZUMvmrgTa^i36nVCG@w!v5a133^hZ!4g+z3jD;IUNa zq6oz_s2(FW9(@uPXcS6@81nt3TLl693dM=_e?EaBLU#Y}1_*)#F{~ImXV?FJ%uW3D z@ZjK)f%v~q<%egk(_@xCnPMFn7?802AcqxeM-)SEH*yp@B!m?k3sEfbop|h#7)JvC z`_2c3y{J*b*=gm|pp0SmemoHSa=|5J%{$y?4uXn)H zKtUtezCotRLvTGoCQH+zN#?>YQTR7y+*s9IrzO!)e7n*%AXK2uIXhcsbAxO~>hJzV z+DAFxz0Oy1N)B5pR+=*GqVCd49DQ0SQzbRo!+zi^IT#uf6H`@H)&2cF=%hYZZ4AoH z=vB`4^q3#b*Vx$DB+)1y4CbjOU2MGMq7@o9yIlJSb&n)#B)tFgm`Ig5$?WZOcaE>j zxtZEPzd4;yK+92O&`cqgmQ$sKCXu1)zoX`Zzr#Q=c!EpkC9ZV6E%8>1tk{Yp049mM z^Sfva>z&8W{c48e>G0I00^Ie8xK(O@)|Yx;McvG#_^mC2W|zW5dSn-~n^|Wz zn_fPF@v~SoW=o?=4>!+(gdDPM+3UV5B;2hPiS|;Vfv(JN_4~A_L?W&yL&@*8s|*Gm zix8r-n2vqyTsv0Px4HGMzGdsgxbrBV2s^P06hrsByRT0y8fx}cezRR7qoq;RHS{_R zAL?FQAuFY$pEwgFrXR~+)|5Z)epGx>y4gudsz<9m4CaV;xHPqq9}CD*<#8OKBzbFE z68e3&XGcjzMTOO3;Hy+L&i2+8ECRyg!^6v=@vFEhCp$Yk3yb5wIva@yR3;Xd)Ikd& zK2F=oY2mM4XpwOwd=%r^mMnN&JWJKbpO7U~fhEB9@#kK3sQ)Ya3!lx($}o5Tb zNK&-00)03+-AAPkzHPj4;n8=TsYQg(`2__9JzM)C*s?VZ%OlmS$WkiPeyVh{JzJTc z)$}a}6dV+PYOiu{HtusYD5|cpq)ACgq*QzTt{T(@ms;J2`|sI7bt7fdNazU3aNh&l zOM}pS%I7TSN|{=!zYo3*dI#(2U&94x(TuB63E0{B2R_(Mrl|~zp=0;3V$-)azbb{I zN~ucCgf6+hBfzeAMZ;rrU`_hs-ZmK_PZve8!}91n1ryYA;uH{OdD)UL-=`>@DgVnp z(v&Vr?|Gc`vL}qJBz69(+?*8;3JMA@9EFhEzCVr_I(V&q&ZxYya=FG_MnmIUL*56WTsk(U+@) zx-ssfH{I*}29=azO9j*$R!#6+OBIqiwM|nES?5iFaN=eyTxW9kHd^;bQhe$Q98W$~ zMt1VMN;r4xB(JC&_DpUO6_)aMSdILU)H%MgczG%GJL1ltzxd?ge7&h$*4a|;%d*<6y8c+LV|;6LA!suQmHG;y?caBcbTf- zxSwUu=|F8^+~6sRs1=*zkDLP3jK}4SIlg$+yNELC+qmLq`X`q|M4q6!=})T-Y=>Hu zn1X&1PY8D*D1@$OA#!2;OWIKZf2l`4E(!a03tLRIC|j{zZ4@BPkqmxym0Xu!@t>>O zrGYeJcbhcx_v6ik)}j?rVQs-djo;|{n6?9X)LaDl=E-_3%3`6ZuGgW zTGrVf#`SzWG|-{CLX6PgIeKZkc+7eAsn~AlZ!Y#G+&dl?J8>L)y>kY2<4d~>E561r z>saE7r8!71X==I6x1E8On|{(20jI~TR)01Pd;~Je7hfA6j}*h4&q7FrV?c*bS!Jph z&2`xpkNy0As&^}`qzWi*zESF;=vSZZ#g}^{u|`_(FS)5!!MqjI(?(O?miDUZoxlbLJe=|Qlqop!9eBG(U zU){8sSvZkc<}7}n?BnZqf1mQRaS87`$OXBI8airw z--oX>SgIqtTAIe@`xZu#O7K+vm{h*QZL8vo=buP~q1Xb&QDFO$p4qd`RkI`s*{jjn3V-L9DQnA9pU?EoZ~o zp>H%xi*1=F zvp(~)>+{Ta>J&x&gLJl5&raHr#aqqUku=dpl9W;j^y%n-5&YOvuCA^q@S%;3je3=T zSSu*~IBn|uxr2kT?G)l-={)wOv^hM35z9|M&XYg4*^QGh+D_kg&Uig{$g zo%#97STdVP#4Y7lH!NK*Gnz%{F2?rZ>2C{xR`yuUMEJ&pIJY1kUSE`JHlf+VMhVY2 z6zR?pCTACiGm?Yt-(;nLjo`VBSmNH03cALI^KhvNrNWK3O9xN3qG2s?#pMSo4b%N^K0Zo2rd0h(Cky>1Gt{;Ek{hf zxst3JxZ76Sekq+u-}&J_{n5x~aUwjhc^Un3*G(x!1knkN;GDB{-^fR}=j+EkDj>_C z+4jmU4T$Geon!qbDcoqfn*FPp7|y%b}m{C~#YyB~(M0%{$zB(*4q2d@-8H7Mex-z^S*an4C}C^L}740Ws(n z@l~z8?SV#M2ZaO@#9=K7dEqwk4&-dT+q~75QPykmGG)bD-87ZYlB$kD<)rQ8Jw=3w z8ms&9Kbmdli$IE~H;abcQ=pz<&7TCJ$j z!$grZNUN0PDe;)76Xy?`c)A}_2hBTX- zmx?Y#t~XPHILG=_Ls*^2EhHDk!u=`7Nr}1r|6*Ssz;C@&0+MxR0kA6gV&Ty;o zuL%wvSxm2~89dejkVqOwwRDJ#rZGLjY@t3Z#rAmy8m5g$Pf_P##_Oh)@0s1kpP$6i zZLKHIbI`{l3Eywj^df{1sn}5=n6X%_o6&uaaTlQ6h6#iG;SY1mG0xoOvsllk93iC* ziPsfNBkJmJHc({r?@#2VW1Wib9~tl5_ohYSpPcK01VnxySPnx}>r0%yqz8V|DGS_> zhP;JYLm1O}STEBKt7OU?HT1etm8AmZJHpNTvddDOP^HuZ9=W61#J@MkWY@E}Y@LWG zWw(#y;-5@TDKCkkV_=v%WJJ$eet{xmKQcD1bGUC4g~xzEro;M~az7Lpzy*CU(xdU% zuJQHqLk#gr9ueZ_;+k1q6=7vXhp>X06)*^>1|5Fs3N!#+P3~X4!J@ISvRZ9+MUa{p z8X5`;3ZfN$A16w(#mD=27Ppu#SzFNm*@h#$>8gpz%-P}MIn(}aH$CUa*`HtcQj7y% zZkV$V7-a^uj-Tk1PK-KF7JCBx^*%@VD~Hmn;}NMo^&3`4CN-xRTHE8|;(i)xaI}Uk z$~5k+8^pr5M@Z%#`J6kr{YK@8B627ou`1jHitX1Z@%?6<$tdfxapo-U-J{dNblTx2 zX$i61K-T9b5v(MS0#_NS6``xKM64Pniu(D{h9_bfoyqVaGx$qe@#&HQK+Jc;#`&Lf)NieUkW`0gUfpz9Zzp+1WIC zUTDN=*mYr`j}71(XlomF4Xy@0rAS+N!^LLu(|_x~6}Z=r+V%YDy~gr=t3oDF1jSRp=x(aN{O(7C3dxLAr@6F_7{A_0CA`BJBCmvP>O zgBC->!;jmk#&p==v;krI-;)ijaDwevc7oNxrH{+wYVX-<4Djys!osaiMSJjV?aHWK zxW2P53ku#}=l?2HoQpe6&9ST zrMGv;*VsrYU4Nu#ah2>tL&e7+NXKgA_whM5l~k41mPr@FxcTNpv&0GLl3n0omSd5);J^Jry}fY4np~jyob~fTFRk2=-hAjCCy^5E z`-ySCCLz1(P}4NM6_b3vf9mHxg>`!Qt{6=vhe!t*P^b~NXGo@ahg8x2Q6I?vp<6cHohMUWi{GwI@ zld|@>p7|{;6QYu~eJ))$sOa-hhMMll24O#&NTAU?&<^`m{N*h*lf^$#EjkOrLet@)R6Q_m5wkgSY zY)sbtsr7Z2`=*#g#3$-?d%3RoPE)Qvvd6_&3thL0aq&+P1o#%gWfz0 zjmfe^0(?l&{`#rnR7(Atr2S@1@6>!rca8z{%*w>@a=M(oh;^uRfoV-k0BSH<3D5lJ z7K}XZO5QjUAvrSidHb8aiGs>GQa%?+Mx4j{`}T{W_hZxv=+gqz8&8L~g^0FvEzVH_ z>#sXuLQ<#=v{#Adk;Na>`<01jy#t8D`4N4DTgoj6gnZlr0y>%nr~ZAxpp1%+?&;~t zwfs$1a*MBtVriL71B)KsCO~^PkvyUtJ%A@j4$TsjfE@CGcT%fa#mrFG|LKw((Z`lm z8z=_|S=Fc^WQYW7CMExmf{^d-dP>OSaAhb*wrBY{-x>{`#K#3Oq^pJ}?}de({UV)Y zwMbV^QnHsr2p?z>3xIlZ5-1uMLxca%*!B5;R0zC-|HK9;R50KWGV37U80RQ*Q1UqL zgJJT3xso1iJ8!6 zT-=ip4JB`kU=qC;x5fg*3@}=wBZF+M^1jV;?6-y0Sd*DoKF^^DXn(6qHOf*`anlt} z7O8{;Xp-n@X^TrsZ2%L6Ir4>tnBaY6#iPQERG=wPp#y?|0*!j{)PyzX?<576y9#iI zPlb-(A1S{7*MgI{?q1lfSwst{o+QA}kNUIuGqXX{PtQ}MOFM#CZrf=(78aJ_*5Ke^ zn2&BI1ruH`Pxs(fyLxcwl$4Or&dO?TF7kj2FMQCVcB)u;ZhD$ET_I`k$K2ub(-SBU zL8F*IVeOyY#SIb!00f^sye3|~I6J$0c|7&q7yMU-wBYl~M$s=LI5oW_Qs7Cj+Y|Q5 z>bsG|(S+pqbFu;Rsdi(G(Zuw$g*Q0WKQ%Q~x$1vd*=S~FmgAeB6PcBjg+;4WS62st zfKfvrX&hFgqC<<>e6G?36O1}Py9NgbtvN{qz0SaLJpLGXM0@4auEB8Jw6eUse9?0L z(nl=d`L6LrP=@8+XF~QiuGI}iB1Rh4|BB9%XY{Gn`v_6KL|ugUd>>5)rAX{j#7K~Q z{@k+MO$_}0#!QTD}P&a9$fo`iRWoB z2>8Mz3@*4U^ ze{ynS!QNa{RMgRtGrY~7{`bW@B7bThNDYNkt@c@i=WqS(?nPEI2}U(ypUy~M({Khf zxo5Psv}WnS`zVn9_v0(iY?MByt!{f;X=z7otym!tRS(|3C!ChXhD50kAfh3#=WBB} z(rv}gBqyW1jm3Z|F`_Rouc)Z1RL37jElxIKUa!#0{%IThSS^4$L-^I|9{)fKAQHRpM=&dluE(K*}7Dh%`$TaZmT3X-^OvKN7dwUp^ z(pT=2KKR*6Xo!es*Vk|mWPJ7?KYu!PAuk`EoOD=E@XM|Zk*c6U?1ejNu3ZnV6*Lyijb-Y-B;k08ViQ&pBTB@+S3_^`L8!1v(} zeYmP(^~P5-6rPmmYFpsFGp^Q?5G7fNSZu%u3-wFdGGQFo?y$k1Sa*rCs@(UuO!|-8 zZnJ}EFY@<0!+Ar7b~iZTKT%SvY_g2>pO!=N?)w{Fb9}TTr;xMH1Lb$&7jsg0$WuVA zX~E=S(z$D)0c=!OMjX3?=~4t#B2jcIDyprcg*?t=Ek#Ua zo8x&L|KoxVMkgV8QLy6hVh0N+6duLa!Qo_KC!-Ckvq@?7rw6vk>)xv8atb4S&_3pM z<7=kH+&BNUKiTzLho-LTxE%?^sSml9)!AMm>t${au;t6T>RJ{4)qKzOt0bKJ5lFNw zPaaj}H!tZh{g)ZB-Pu_qKZfMn-|Me00!~B)@Mt8?Yc+ zO~PFJO{wnW7f|rGB_vp6elGUIkEGA4y&thYChi_Tnl9FC_}Gtsk@_W&^=5+Gib%dg zEcV_^3GvP9sc_3~&**5Jc;1)3jh{b%2G5DE^Zk6|4s=zZqR!fVle1J&`D%#~(bL@A zobIae9DINJ3wZkBzuvq^NrT@K5=;Xo&X;ku-QC?SEvcjO$;rugxNw;b*3P|v4?kJ_ ze2pVWB4RgN0Zf{Ule4L{wS4BFwzjsUq@=5>D**4HPOlR17BsX@_d}fAF;6Ea6^3MR z8J>#a!2Z=~KALKzxZBB;oyhAKd5V##5eO;f=0!}Z}@<=p4bpBZ&) zf8=t$jgO4Pmqoexo1dQ_BWsk7_C>lNL9X!l;4{1$nm7?EEjl6&qmI+- zi+`zxT=A5-xjAtE51R4(c8w0pWfc|n4h|lko-*jLQUVR=HPUD~g8bL1jZ@!HZv<2C z0{vcl!$go%T0f*e*gR$02M^@Vkh`+5EPJe(4WBP1c@aTO20J}_z0k5qKPZ=IlqJzB zLqmW!9W|@5=KOcQ*&ivnk;Z2Jr5CprzD#+IdDX$!Nu3+}<$BI|`E+f2`+7o#^xUh> zD(h!zMsulVF}#AITQQHiAjzsDWjB_DDz+H9AW3}xW^Q0W)#h{sc*J8{>MfUh@G=Bq zKM9hYUXF=08inq0*`maC=dG(I)i&ozA)Uceq@F5p4o!E(DN;gdR9myN-67|1!{uua zC?E^Z%>RI96ksn>#dr2_<0s*$FNxuV*IlI0E^&h;n2Hn6XTyW9hou%^9q_um|Dia- z-_uj<6jwkPx^#NcJN9kDn(_U6S9fW{TZdxYDD1dT6C5!L6| z+_`3#A!lcB6WwiZl0J>5t;>2aVnVt`a{%A+cuSDv=r5d8aEWCw9VAAP@Rt%T4HN~y z#Z47f7(*&EeuWeNFLE;#3Yho*7ftCrIT5PWTK#DRxDoH^ z=?O$kjEnPEYu>m6OR!+qrAcz1DGZmSfP;s>NmBYRUZf)M^Os?Q4BPRj%pAE#92FAu38HX%tfBt06REgj;6gI?Yw0?V3F)2}n{lB>YsNi{o3IkM4 zfsg+eJC@HlQsEJIW_26A_1}ZC)V<)#YlP8ic66^ycPJtJ7eRwu^{7PP6O@g%wtBVx z6YrS#%iT7M)DswZ2E;#q$wx%Tr#JP22`f*C%&a8@nVV5o1+;NszFeD_|9E_q0475o_^ z&hXQ}AE^{GQ}BHg4AnFCM`@cm`HM@YjvvnZ)RmIUH z$Toahq``?EiFS5&jzA^;y?JgoSM@F$1l#Wly!|aKw!YD;cRZf7j*qJ}8wmT|oq_k{ z55TX7hX(+a5Xd2k9QqZfy?AzX^e`4tet8=Kn77ZM9l`Np<5-T6J|hmPu%Fk>(E(1ZPj>pm3^A7(GTv2sT*^CIXxG@DT-lzUJrg@bIjSvP1$q zx=rpjqFHuEvk5UU$P;Am{9XWt1@a}3NOH$S0FKnvO)n_e%ogwze%LGMe0@Ge!exeq zSXfvHMhr(7!9V|mqXE3-UG7+XY^-6c2UB$4zj`}dNY}!GH;};D*x16t!qkft2J3$B zUV<$byjeR?js4(&QJ2RK=x7-No(|^bn;<+2x)i~|0R{lZet#PHT0VCuMWG6UVhRe# zvC>JhRHmk;Da3GqkN3Z4Vv3FY?+Pwx5p9S2kH;<0C(D7K&%53hY7kj#;jdPy<#X?a^ayTuu==GIny z0>R7=REbSZO@Jv(&&{#u{4{yIIR;Y>4FL)hm?XT29w`|aeaF*{4R(NRzyqgz`=*e= z87WE8e*TBBKqemb?OS$7R+lZ=W`mr>MDgOO&5aEZHxPb(x&#cQFB&%dn%Lu4KS+h-Q8acM|lzv!i!^wK5y2l zwzTzd{FYmf(8f_NDaqse58E}wRA?1wuuvmH!}Q_{o0{fLSaqr+qoVSsa&P^}0sI5; z?BP@vM;n{_$JN)--OJ9Wi(wB8+>o5?mGp(6K?{C9zU!epvJx;SOrXt0x5<^exDgfj z!((y7+_Re-qYghlh-mptOG^th1bh>+`?xENmGx(`y?bXjx2FqvULM@5w(gJgz_5PI z*jDPN3bvPPSKn+*knKhO7tSA-&~J38tf*M(415I-pZj7R_;SAsmVfRGFZ$Khf3&tE zoWiFas+$#C3LP5`uu|LULw6ra;JBjPn<>M|r3<>hm>4=vy85&W3kmr@T%;K@GD9Ct^5r-zXVhcMD$o|u$NtYw zl>YX&HN;dIrBrJT9!<`@_Rq{eW|^%;X*<1lJ0cJ8vG?S9U?5*p!Y5m)-B;GDdEKYd ziiB?~(c&xr)z6-<3=W3}g>Cv}IAlMa-l%tdI-&~{+~9W6`{BDQK=E+3Mar%~+3`RY zem%`3DExHRV;pb~2O-0LOOGuF98^F+fPz9ybMr?aHfxmwqu+Mo-5t&5IHf>G1nvj| z0VV*9>BjOrKOEpJx2Qb`^UykZA1{Q z$qodk_uX19E?jDKD%489NanY(xw-k}tM54+Jl_u%aVngi6M_g=dK%9=2TN}&DqQDb2yI2k;QUl*fksu2YHV_*D z~YGre&;d{p95a(D-bdbTHH!oT6{MD#HKPE{_F5>RiNovcLEL`xJQs_ z(Cl|#+l=-Ita;Rez0rQ*w*@;TC8aJS4zjySF^DI@iUD>qIXMZW7OirwTMH0-xVgCj zjAmkDLWn4N>4pk61J`HsZ$ci)=^1I&Z;e>$Vs($)7NZja?#=8*t5(}8s%NJV!mWfP zbxfgGCbaGC_Tsk@V}`vZyu01eC_Su|TA2w6=4hY3A+acj$)Pt|!KWx}z@wIVp;|C_2FNbCBTi*QH}<;bYR z`#XNlAmtaWjU)ZRiGmjNoKe%L@9hbgNL^_UfO_Op7|$;*q9P+d$jb7%A56En9|{Tz zuJ{}>0VwNzVZo5BXKZ|WbmRv(1%GxK9I2!2-*eJNUxIPrBxlX@9L%Y4a z%;a$-Nsw&|eC<#wzP!GE_crYO{2c6VU?QqJ9?AwS*mLIw2S1RZKV3~JLxR9wFp|jw z1Njq!|I6?G0@xMU1=8dSjoN)csj9HyZEjA^7y>R^P)Hj%|Bt^GNd2!=MPA6o{E;2G z*xrMqvsPACAPjyA01R-hJFBGOratXPj5t;n7Ej>Uii(QyftdNLO<|@F86u9RGD0I1K-*4EKVduvS%#$j?xkrgzYM+hHjmGSHOG$DTH zu2w4X(lwUD60Jz!b8y5j>zx=~l7gb@=Ub8RW?ftlb|ZZDx^1}z9m}#d-*|<%3nvSh zprKQu7e9$VP0E!O zwuX)9cc$MP@>M}Axt6=0+yx2TYJH(}cUH^((3XGxB97`(ZO_}q)f36@lmip=DjRBQ zCV+ff8x9h%yuGiV5IH{n9T*ytlb6?T^J0ULiHNKk{QM6r)B_SKa(0g@CM>R8xXL=0NFLnbU=j9R?K{bo zHK$BG2@@kDr}B)xy1MJd*5Lg7JS`JR;OrL#0*^zzhhtxFcP1KGaun3K`M{g%Pm^q? zZRG4jvarx5<8)Muz7?)2B~51fs%|tT29nc zJJQkFBjgh4oq;kjRnT%jT;YWd7AH?UvtS5Q09iZ83Ex&qJ6&Iwa(>AFI}jMLciuE~ z7P544`;0|;jfPz2eNZe?s^N1uTM0UqX5!wSEbFVO)mn{_Ktw-$7zE4`mr>_zN?C*) zabtJaiai~OEVrWqr)fO@kMA0hX5AV73wy6M^UdK!uq**3W7WYz#m# ze%Br3=svwRuNrTnl*75|)y}}c?V)7lM_o~vZ;B=I%E|!eoL^s~ynS2i_9aiP6sW6X zyAfew*&=~2U~oVRBLEoPTv%dVC&zrV_+6#E@Jjx7N%DvCh?^`Mr>JaBIhO(z%N1zI zCHbHvbh0EY1RVmQs)b3w;Qr`i55Yp%Va*OxW>=t{cKkZH1tbOKEZzwp%Ro>|3dIJ*m!%jKjkfG1Wd`hh&4||mI^Zhh49a| zi@u&7pe=@mhBLO;vz7Xw1REZ42_bi$5=8<80_GU(!eFO`1ofFoNlT*wzY4b8oig)w zquktFQgrmQ`9@knUDlW5NNwDJ%t08))Pbd&-ex91=ZLyqFypQ8j;F#q~FZ?fh zy*vr@Ba(A*O3cU|=e&nsI13{jGO`8`*p3dG_I7Qlt zV}xeTA0!_-d71eCB`@3*f2RByA`MGZeaH5c>)Tek105LUboUn{y)|&fe;j7o;13NW zDUF4bbV+03%Y!zOvU72%?u@(3*(iB|LG^2b@8PToEluCTT+advS%&jd{P7TCrj?)uy&vpv1rL`NA07Bu`xUm!aL1!;pKA|jf%0R{EB9O2PrOcBpRL_+G> zJO|*CE-FKzXl>187;R4QmsqhnSd);h1nV0Il9v@C1`=hL>$iAY9CW7u z76{Ng(rRN@jJy=dW)W*{U`spQ;$fUrHnCS-@C^%#@QD_D)-C#@7rDY`|LMcat@pgRy4*b{3YCD(_~S#o>@TQojW|5)x9eV&=zZU;>RfKMZ+b zh2ln|VfcvRWU0(*bf~GR0UZG0Fdl+fFnc>Y%0COSU{Op!vZB1AqW{)MfEqwv?&-%+ zz~Bzu_%9Cudk+M}wz9NzKbuN?-28xQ8hA(+mg zoHzf}=rCRqTsj#L4{l#{J|8<85|3E0AFs48vIppZ7{u9nPiSapDvPmFp|DFA7vh0G zcYQYr2?^jlph#Pj6Opc;fnVGTD<{+@)@87zMlWo2c*ti}M>k!HkkZ8nf6?oa2is{Qrr6re}jnKGbsnS)KXy8T8Eh&dlVyzx^D z{_n+5q9f+zfn#&8V6ACtYQ7Bvf(Hg7EPUwt#s((1aByaAR0T-lBTzKSN=Oer`0hVE zJcJ?P2Ap*xU|?W8-kzepd4mz%$H&J9fz;>#guM+Se>#qxCScS^I8RTPW8fa#w_yN2 zgE!!NbL71l%k!5IP}=&ElD*2t6%g-XfS8Pb7Xu8(_iRnf$%!=#x?dpVL6laA2kbpS ze(5!Hc6I$lJ`^!gk@! z;#u{>l2QdLacgU9U_kcJ69d-!_4#&0`431d|1aK%4yy}1fOqd3M4wToW-Y@3sT1YN zDohXZi0=XAxo+eyNCqiCRGk(tQ9zeuoSxXF1r$! zKxc1XGTKwou8Mv~pET6EI`H$KPo|4ug=_==WkLk^f49B~@8<>S+Ced&W%@H~Ei64{ znCJSUH1LT%_kUtSsBGdC$P(X3GJNf=WZo%arnaC;n-hGp%EY6xQ97(oJv5Wz))#)V z`jGaCeJC+Ggs-Kx|AVWvIBl-0Zf$K+Qo~~jhbG$C%z~*{k=%G+>dMJKfycRX4j>U( zvaSgbS`es(4D1FoYyB8>G)yIBk+)e4Y5SGNm-pmac5(Y_sqHa%~G=3MxaFs@>rn_O;g z?m1_ly+3>J&)LU38omn@W1jSh@ zSufX~e{pIc@S76@Uclzjw`;WHj2N;GZE_)6CvYl=I8v@TdG6^(5k zK*GB}{P~~j9AE>yU&6}xU2;+KT%p>!(We9S?DP><_d^34lj?hfc)YI0F%h<@D^#iD zW>ShCx^OK!U;Ky#(|LP-u27U$_{YPh@CRG9RW^4h4}GWfy#Q5cEdNcbe70&6Jq+;`0_Aj_IXo48AQgp0VB_Rl9>QJ-2f#l$++b0?UR7)R{E^5-Oq%Jz zE8BD=U&ZtZWzhm9=*EImvn(^}0t&rNSrMhv&JasNcUh55&&&a+-=X+NGk+#(8#8DgN z=PWE$*FP%?R*}61+A&n0G$?xlTCRb{UpFszFLr@iIV1{!m@f}j?I{f1Q#Fb$_JA#) zb-_4qsxo!CU=hlt`Zx=lRc9 z0wiALJzxzDoE2r)Mon4OYS}uBkxjwHZ1txTHlQWLwNJWZk>TJei;woj3Ky7dE{bt5 zZQPBm1Xf`xgeB%)9PF>&2z}J6aSOS^U&LA2HDEFaq`uW4NA?>uK$<)YA*Dk1PqRZz;Q(8#{TSV6xRou-l>8ip#tqv^L>wvmD1q~w%G5~ij$5>)j#VLXcY}}$;drD{GV{j%33H&*pUV7h^qoORJ4g4h5qJBAIsm-!+S5W4) z=}kY%0mt5pcgTe{n5Vc6W4B7_tl!!cE3hTybYA&r`9{N}XXefoQD-g%yHtYQ-XT-JZzV~mx|6BKOvP0|NPKtb;G4Y;gNc&>Scw`fEKes_FRfbvLrTr9(d4Obt%aS zn}8fkD!AEV6u^Nr`uM0+7=AK=2Dh**x7ik6jI+Ym*#Ud)K7DX?t>+CTjLGw{D5tV+ zlxj=R&;u!R=RILBs_{EeePc*9lJUaEJ3BxNx{;hOI1%ARVlL`yYm;jj(R+vHwYhRr zk&VU-Vx3O~B2(9#^wY0?H;BJav(B}w2KtC~qI~ulY_{CycE9tcx@baIR5p@a(@{yD zjSH>V_5tF&XBD3raQ}r;1Sn}*vKItWCbI$L>C)>;`qzbTYd`zktEd}qFd&^?w(d;# z7o7p*6AiW4DJ4_Z1K32Z(eP`)GQ99awvprusjRb=Mvk^Gz<#~=TIV72DV01^KdPN4 zMx!Y`^_ULEVKCa8wb-x?LsL)PlF^hRBg{L%TUbcODpMB8m1`-gfCh%#?~6n2EA3a+ zW+$&SFMs$DR@MS5Xe-bCK;W%KqoTw&KXFwR&$Qm0ryaOl0fLP@5{m)o>?ZseTNUq92qCc5aFZNjvr9%!4iiMwd*5Amn(R9O_)(UunrZ@b$Te*OHB(y1MqgKM8% zeaE~c*M)`MmbeHSFpv@%II+9Gkc4f$|4@c4$+8ob9IXPFKP16*v+8cQ5}Wt5n|!en zwvOZbGiu1q*zak{g@$?{=^G|?9%Z`ZoB(36I$RdFsNK?k%Mld2uK!PcT@r|2=bzXA z#98gc8g^?ZYM#i2x4vl@&eMZI8G@_m!Helgy=M<%#K>;zM%>~6Lb{^^{88e(rdU)t z5Z9Sc!p_-C0n)R0@7 zyzX!pl$0pVtFm7C)|y0?LuPdPj|!;!=>vco+Kth&Hn>sChxl~^a8y9QR#zvWVef!d zYCwiUF1yS}>2FqU)x?Qf=E9J&&Kq(dg^&64UumW_YHMo%fy2?U7>NX2do&siEIDm$ zZ3RMUy%v`rkw73aHbsbxc5kH8#$gLD01QIe4!8*bl?wzy8=C0q(%-&) zHTBy$)aAWsOh~Vax$^M+H0ddjM3lL)%k}+*!e657zr!B+Z#>$E^UN5^^LAjqzefFw zXL8EYHu7z{phip_z!4xtIW-9C#p`#klrLNa!ndHSpb)ir=_poH?V77(paS3zD3k_* zySQ8NI$2xUoAcT`ncG_P^SCZn@HD_4HEbtUgum}HWx)Xd z33z?E|DQZQwZl_|X3K6?QY0E`W4i*5(l|_oKDOeb15Xj}IfNbPl50QhFwGC?cswBd z${0SRoZWb8(>6?Na=oO*X|6^9l}QI|Qw6-5Y~dX0cm31?EWVh}@0ZN;8zEJ9G_-7w z-QGnMC}d>&5)CpVJH#)s0Q^MFd2|F$oAN&8@8TT6>?ho^IIZ5sa566$WZ|^*_ z7{^XmMudK`&SP5+4~uyl8U_n`=>c=|QZj6uZ0r0{c$7idKbe`9Kte8$7wrsb3AL|k zCKtrV<>%L@Tj7}aXzU&C_ciuk?)y{8sdT*lo|>2jO{W3kgdl0Lu{zS$ZK|}Ro(O@D zB6eCt$LT-6-|mjuqlw=b%#wD-L6(D7X9pW&Q%#4CdmrC(v}O?`)}x1~Re>EZ9e7y+ z#Qp^1q(W7nJf@Aw>NZ$2Rf`HJ-taXfmho#uC@^@`XYAA4d<1{&ix#gAoSDy`sneY) z<|n2-wJyxxa@y}7&q@qnVccQ1VPh*?hW>pd+(7tF+{X*-Qnb%)zr4>OOf>v%P8!j* z*PiM3`#~YV%=nsT^0;Q^>~JyFDzYOufN|7I(D+$<_q=?WT`zua{`p$8$8m$>b-~1- zPom)Lw%Xg+qOXM~KTFkO;z$R2I>b?@N!^jM}jpQ;@D=2;Yw_G3P+*DKAB_$y~fAqiSz%jXw2=By)AtX&LAmoLmBq;fH2|W!mw5y@L=1zp~H$DFwCa{Vy zOuntE*sQuyY6l5gZCfjK0;EFyW2%ggjhTl?3?sgbnq*ex-Pk39d zO9gX;nSvD(kO%>RxUr&mlMVSk^0b8w$D{yHk}cGTS0%h7_+i;pQ-pG@F&Lc1!O_*- z-`&mogol=9@D_t<)6!+MJ&O?|I`mu5m8hNM)5XeyhaujxoBRX4)|uJ0h7gtF^cmwc zMhBr$p*3cn|{R@^RDmt-=l^(4-rxd!I;&JR2^Sd2Z{XZkm zHqZ8Rp^!)HggCjN-SnfMAq*r{X8V$aa*CHiXHK(biE^wAVY{x3LVXo_Iv$g;Uu4a= z0#29c<{shf`*kQtA0%eXc*`%1gYo!+-o zD(O-SgMn?DX?5)G&G{rl`!b;BHVE`A3$|@#<6vQISS9Z?dlm>6PSv$9pNt%lU~CLk z)O zGA^NnCT4#9OU~k#%Jqh{W3YzNNUgBMlQk=)ve0;3b~NevwEyk$qKJAZi$DzHD*Jv5 zbB(swZ|Qa6c1=OBgAs!;LwB9T)6F16QF>Ttl}a%tgM!Mx3#(rVc3{K>Cg`X;&?BVE zh4&Y8ceCf1RJ)$g@>SvS``=&ZGd~#wwo3>T{jB*pfS+w&S}81aTtw>N*Zas@dvJKw zFxt)zx4M&*T05;Eb=117ZX`JLS0TJNw3o{0c0-h$S*UB4rQ^OZytna2JLeSlPxp0A zLs;*sZXvHYuNdtIb#xEcA|ge3i&U&1YqCk0FeV{9D+%V-8(xTrbe;4m0qXTnqOpZE zWsIWE4pOi^YXLD%4=tM741^D>83a-;P_oDXmAj5jn5iGNf1$$Vmy{G9+BI(erdwJ# z@$H3mSdkUhq#j;iq@(o(t|Q#);NWa+4+f{aEAC0-r-?~h@8R^hOS9_WFqj>a#nUEH z$ndMM_mdnXF79rkcY>De`Jhf;K<*#sux+g5vxGG~#qsrX@HXBr-;anBL1Mk=Xc?M_ z4-t=_@MxvpDU#bp<=8r;tziplQhbk$Pv4u4I#bsrrbOJbO%LntuN35q6~zUwjCRxK z+1wgMbSR%W>#ct$W%!H?PC|>rFq-uRFsGp81XjK{5gy?C8hGg^W z5}5@9?@p^Fg~ zN9-PFd_F=XJ#AH_$N5ctsNFUtiP~!|?5ZTzaByBzNy zIbKx%ZK#^HJ+9T3pNt5?d88b1zbT`HO;j{wbzc zF6S}(!m;22$yZqaOW*@DJR4Zk>-P$S#$*@;+xQgaUNS&`bZUB}^65`bs{D%UOfwRs zJK;i7OStAaGY%LW`t30f4@dv0hDoZS>U0#-oQUMkw}{&OP1+!pUOV&-kT3&*G9Mt* z*mHGr{KrRUzvi18|3FfUJb?+E>PWpvPV@yk?7UUZ@=6t|uhVw2n9VNYA-^j|4O2Q4 z+^TP>Y35^(WaBmEZ=YEl8bp6Z+^aDzGDzguYZ?Gln}*noamJmodIv^*)uF!rDt#mI z1%c8K4DPnyOlw2VICrXL|E({{Z6~qMMq9hv#wVa(Dilno$ENo03Hn)lsR%YlbWJ| z=iSGYWr-V+QcR&kQO<{5om+<-7kbl})A=@k^`m zqPFY$Cl`J9lWVf))|HA4v>ZfVkBo$wW=KRv003r|(eGCf>)LFF7Q1gPv7%~#w_7|& zh_LuWB7aBjI+Og7IRAJ-Q?{xwN1r&t`=EZU4La%5(p^#Q=BDc~9Wy?8VqeqIW1Y5n zgF&nIXf9i$eMrzk+_k6X_en2ya$a4CV0Vc4>sw5tsAAY#M|y4Fo_!hY8_g5!SDx)? zauwoP`_@Lh!-88g(Ta$r4sY{;i)h(9K=!h?hLSJxG*!nf+YYK}w(hCy%!)8yo;`%MAWpbAU2? z0dE-H;2yD^!=pIb6$b$-<$^#fk9KzXP!nc($4WH`prL;&F7{=Q2ZKK-{k@`Q z*ZCA+x~MZe+}qpTeZt9Ubd$qKRLVd@!`vy5F~aiiLo%s(1=cn;2I70vb#)7Xl|pur zjeq|!SrQ(8O+sQ2*m=sa&~fk-RVVF(lQ_4nph{ahIy&w--uz&nSSIiDGnqR@v(NJ7 z3}e|j<(>qK1JO7TDBaZBTIaIk+vS}cl;F^#^HZL_I#+xvxyY*QFr4R~L3@?!hn@8^ z6zA`yNdw!Uj^n=inynZ zcTiL5czN?VK-70+yZlnE=aUIgS_7P1#v+HHGmzx{mWSf;nnlho(2rf4+94x|T$-^7 zx=X4{RtlUtn10tQ+hp|X$z%mSW5S}+x7nEi9eQgKjXOP;N1`^VC+<>;EV=Ldk9YKM zhifA{ksT6d8LzVLE}#lgU(mvyzss(SIqvpGU5!Lj?3dl7iTSWs(YRK@)-om!c_iRv zVCNkk8meur5S`12tJkake4EBIi`{&?kG1%=r|aUFT3L7XJ#n+=7F+E_dHc6$YHyI1 zi)h-FDJVWQn3>gfw425Xoi3bc6>QH7r0PhPxTH(uEIJ;{S2y+2^W8oEIThi*E(gi^ zqwW1GA*emSt3$84abwtsHH-VmdV#Sd0H&ja+CLrkgCg7nW94~tIZu2xD#wBJRk=z?Lx}qzac(c8-C+A;#dsge{#3irZShtldW67?kvpxg=o*Q?UhC|}c ziOXhwkh=yRmAdz84(kFGfCUY_t?U*z@U7){j`nI_cCTpkrDu)530u&&uztHD5;?|5 zKo#~;3emoQ$Q~X#en3i3Vz|}#-0`3wbBSMyza_iwff(ZRQD zUSV{00KJW~mYuK8JG@lBkA^``NF9Y}Mxl`9*2`+Mr`?O+A1vk1rqSZ*YY$<5F^k*+ zXWL3IxlAAHyM z@LysvJ97Usnk>N^a5x05l@ZIje8)bbF$n7WdIX11Lp%Pl&@mgzzHpXu_r{^=$y0-r zQVW^lo&32*!u>2d4aNO2mxeDAe(?2I0`Z|$y1fk*W`u*eIozf24i|E%>wIn|3^h;K zxEErRH;p)pJa~Z~`%^#0LtNmy`?}KAV2n5`oF=R=MDge(Wws$#eiJii{aaf$`?AZQ zKdDfLdj6G`fl`H_=%ZkAsWzqKZJ)1YBFvKL#45u>o}!etmh{vkqV4760!Xp%9*?4G zXUtICoXdXlX%dfb?ZI)UTlbH%QB3x(c8|?mdz(>ssJHkct;f3e*U*m8t=W@EIf%s1 zG4|D81&)d*r<}crjC)p(z)yzDqE)4shsWh;XgCz~w-UKeq`dJ3Mw(`;o#TH7htGc9 z2Lb>zijk2GN4WftDFqIwP=yx*^(68CH`QnaV;S;XG`H`z6Wtm3YDVUlYnhMPDN%!R0<% zP8<&~f2#|ic>tASvD(>c+4cskF_kbArgnUV0tP^(Q}k0CP(c*!13%;Pu(pg|pH$7sXWkSVR~VVuC~;6K z77Eun;Dj3J1jYyuA`^;SC{%5n*6;&+t-M?m`OxDPN zlRO?tFGc?&>g~ggV;vf0nwX)Xe=IW)=lG_uvU?8&29I&>Q10TmU%}~C*X*CU6fc?~ zzZo^pNO8Ga>i_D>j;mCt;zcNOtpP0SO`f8E9w{DYpPCv zG7uiRd{A9>5)fi2?5_dINli8+q@E;f5=#P7Ub!?5_ds*{6OF2J@skWRV`b`AA71L} z4Tl>m9!Qoc5K+JW5Y2pdE9v=mwwcQhX=h9lsoGA3b{H#V6kYbrHAFjr>hp~0qa;_YbT^24=oJr+hC z2YV;+gWw?UyB3eY36f-zfIb?pu6l2J@C#Ys^1;OLc={K0-JURm+n!l{S9_0J>7Wq> zyOXYRLEc%Md6B=WT+Uf>_3Y8^hN5H<((jv}x=+A{c6S?(t`@bypTd5Y+0&KZltILA zg6ZSUxIJ{DSt^dySQZmXG(zy+72CwT(Q%3Pt?4i{v7;}wDRQwNVwWj7@z=>2S$``P zc=Tg;7CdCRZgkdlHI1lW<0dRnGJ$A(_U*T=H_-Msb1azYpIvO>oWSeJlS)G;AD>!` z>dDLA$@hi5hK%}~YkC2EfI+Ja=Fj^}jfa0T^e{B-3d9|6E?;-{SNC-Jj3%9+?>B`p+4ky{ojd?eS3{ zyO;lJOcd3=(+t3o8A#}evy*=@KM^7?IndnT2eemOEH*J{Bg9fF4#0v3>6RGI6pg~c z8*F4F!_5t1-9Xi0kk2`S0%czO4;~>``s2v)vimlp4H?`+AP1X_0L@bEYkhv zn-g2DmRde=MReAMNC2h1KyTFAGLd2fPv)C($Y?s6Fa)}3ZK(JJ+!qLavHTJhMV?$8 zxZHL~KyQ|Ao9r(h8PIISoH7#|fEAlSgYJ0#5>LKOQ|aFn+R`t&Ty0~iFa7h#ApfJ> zk)Hy9?$qk71!V@i;1lxa@JhFBY?@Jw*AN1kZw~o*>3SV8;BO#=N}ad5v^;DcAj<*{ z22$6^)YRF<(!akBs|mD(hs2#~H?}Qy^WnRM{6i%97P{6Cvu50Q#J1eo9xzqM00_mY zJJ3T~O8rmxo<4n5pbd}(eSLjZ^$6d0;&k}4WWh(D)FP93R>elw6;yT4}%Ol0{`X=U9X4|j;#Ro$r8f5uYg%Me+Z7f)A;zYv0k|u z@h8t~CFGh2--8IAzUk+4^7C>nz80=l{d)k4H#2lWFrW9Rbhqd9p&|gQCN)Zb9CqH- zvyeKn-(9O4bQ?JCXe}29o1R|U zyAz~4-|RP0t9Lus$j9P~P4im}9_c3BY&E%snMF&GGYBFyGyhW_Xxg zMF3QcogeT8>4=q|IqknlLp(XvFf(8f>ZO6Wz_SuIm5d`1nOsdY4F2CwW_u;AkM;kk zMOYdUfw<-Q4?b&FK=ySvjgDeK12%vRmsl-{KbwEM?e3(S)YIK$lcQS=8v|~@ID8%w z$f^O#Z)9b?D8eQrCQeP=o~v;FdZO_EsW!TXDwM8Yzpf1|Mg;2gWS9=eQxa4W18yRs z<2x~cUIBea>TjMWnnADL{maq*_m3wgWSJhEA{F>l37k^SP>0P*iwO0^ESI9#?jT1h zgMTD(Lwpw>uUT+|HlJoVzs_haB5j5gB-WpZFI*KnIN2L~3kI{|sMcrVlC)5~h8O%c z4Wg;|pb=!}*5f_!^XU^qkpn+i+>mB`{oGuDFVP0W6556JdTSFUh8SottI2gN zEVb)^PX;gK9GXiwW3-v6+mQkY9D&3n#Kkvi>q@%L!pfTI(TL`fMvb~9T_G@E<$E|F2dzAmKpR*tQ)Fd=@o_vMs_Qg=&>^?F)o z9~;s8888deAE^H$&8}-|I_N5+J9LQ_w+fqIj9&=hout~SzDbihC}7H*e-i-c({+5l z(X9w(9cWiB2%x6Tf&?SJ47bqBgqe7t~8Aw!TAZeD2;1J?i<}H>`R3f47 z32>uRTED%ws{#^rI&(VSLZ`=QAhod_@iIt7m<70W2>yC5){N>@Z){{WV&W0Ho$w=EdokjS6AErB8~#mg`DsD%}YG>%+=og#KfHOY^K>5vSYaz z#V9GjTy=v+WT%&X5a)m$ga0AnZ)acdBR(?bpyL=5V0k{+@AAg&p(-GveQf_0;3NYw z+a!>iFMO8oI*)ic@yrY$b{rF?bu+AhkkoPGms8)ynmN!3ZF{-4QSQi z|CNvjvHt(^9N!xR4&cR<)X;|%;{0e~^9 zxKxb!`kNflQ39{M17C#%Rb3c<3v z>1MmIve=iO$aJ>Xb~f}()%2HxkxeZm!ia#4`g1)WdDHP>=y?tcmC7+^&&>5`8wBixx&#!-Ng)E z^~V~KZhBbc-Fp7VRbSbRaYur#*mHC>2Mh#sssFzQ@;Cw1LDFf!D>PkZ+%=fiXBTK@ zxD>31siM$lTDQbzmBIL-)yu@ETA0Iz+``eJRnyVSm}1H{zdJA6ZlPvUSe_3zY>0a2 ztDPJ}6M5t<<^6zgGuHT-pdg>1pqu-8x+mM=NPiPmz-w2={Xc4+ghBEP>jxJAL%U&I z-xHUc$hoj@2Y2KV#uwRYXmV%>N3rVChh_|WV}28!WQ)R5lMyQCZiqkTXz!iP=xh*6 zOd-ds)UY2hv_|9Ib1ol?%dFMSsRgiUij@7*g}c^$InJqY2ImPa?~sZwdfD&TbNUWh zJCN4cI9)%c4A<>i(&nP_+FjAO>D-8z`c&xqTmb$NoKazlwsC!r<|epa5u%!UpA&Z* zx-g~y{Z)Oqn!EfrrTr{fTqljvw{gyB6A2=PJSA^8k` zP`RmQ=RiaK6#Z^JZJI9|lY&|n8dw^Li8~y-b~iv5x%e}xlsL-VcPsLG+Tj|bI(3=0 zpm7;$nkguCYni%ktl6w7eN#oba&pCaD9vB$mk1VGOoR37`;WA{ zI3CGmCulB&Bo_O&6vpJZCBUQ^ic3ru>2ciUS-^cT;_ zxO6gZs+U(L@3sy763%jO&NB>MBpggt1#ZT1lYv&!Cap6@26;=KSC+Vcza9^R^x@Q2@9)ub6&ghvr z>nq%$J~RI@;NntE?IAT$-Jyk&ztj{{$K3%n+UT!OZK&#;7KHPMCC0e5epN4-?7qsF zlb??p$k&r^i2@`esK`EXuXSX5MIdSgDiiOGCN*W0$NR+K zXXpn5o1=gl?x)&7`C;%1ZmLNm9GoT7u$%+lpcNkQ`=cVM!!^SNZi7!j+}e-e?fXZU z@|)S#{c85xr`r-$MJ4FGwrdTshUOCH&d!bbFC!!gMn(oLt?Yt^>qi|bGZgW`W3Px~ zftWva)ke)HpGe*vwp5gX4x7RPyNQ{GSb{eOS`?}FGH!RZNfp<$(s&kP$N;q|D&&1u z#^d>7pixdnxM=3kcWjjL^)jlrbw~GlNs11^#!x_bLNQQM4q`Q2K`SKa8L&H)4*{|d z!I-%f4Dm{^ZglJX@2%~&uxWcOZU|Q%<}58#Vfy#Z0>a^7>@5K87NFHW+rHL~Z%-R8 zt)am4Kpf4{ej0nN3_2+KLzLgLDfah+*uM^O$HBqxo7@lI!w|%B0sdUu^hv_0dCVyo zaJIg#Y6WgazZBwTH!GUz)q!|+8{-RV?V$FVVlNl>tu{_~s}pCh76x=W4YE_o&Q6}V z!XO2QH^hAlj3ci_txtng|B%W1jlms$NM>j7e9Pp6lj^w1AZTAvL>j8{YpRR{kO%U3{hrMGx9* zjW+4+;)pXNFw`=^8{cs-_JT3Vao3;6f^hMyG$g2iMNm#0arbPZYFG7 zac@z1K%ODQ+;Xw`i!l%}=nRxhHQ0Ot9vb4%?Jn>*@?8yN2jpskqU2?D(;R^N-;55I z?d0^f{Cabl;&QK<7qsIRfW>~TF_=q6j+7W|8ydZ52*O={UI;*|mF-2qJOY-|LF$FQ zU8b;)CzgHY!`!nA_pp@JL#5M92*nmsr2lnva^-8nlQX0uDb1N<+n=XL3&QV$g9>*2 z{3$NRUd@UatMG8==1LK_)JmJbo!Lv~HQVm5uXXstSS z0!nD^WEnjPaxxaLLn!nol6vJJ&vI+Cx;7o95Zn{dFWsp(69Y?hnx-mU6;wgUCY|W1 ztd&8TK-$?`$)wK4(V0YGql8LfCNcFa%E;T(OLJn^0H9X1+xnyLS8lDuU2$HB(Sr~* z6Y8YVb$eR0&gSA^DCzzF3D{k*uy4m6iPGbpIvBQ1yXPm23AO<^1i=w^n2FSml^=yX z+UJG)PPQ;|vPIP}bn-1P{Q9)2bA%d!wsS@Uci5>r>X5zTPe@Ejw{dW~HKK1wPg4e5 zE-X}eWN)FVRY|8$DI zA{|!h+Ro1OOm=}+RopFct@uajid9wRcJog^4&yJ(iWkQ;4;SWne8NHQ`R7?v4`KrY z%j40bD+Gmf9J`SLC)J6CeiT*JdE6n*)Y;scwiQ@q4iFrG5;F}A)3bT4?WnJ` z96YtNkkv`#p+t{%g@64jRaUX8(F34CK9?rHC-)Ykq1s5T5u@#xUC-V-AK_l*W={Em zr)Xt{ht+R+s{8%{$`-SwD>k}id#+q~A1`PModQaRIvNf+lmzQ6Ky#^A;4n{3l6ub8 zFojFqVSB(!cJ)y&_?2;kKuN}cr~HJ4%n0>hRrI$2{;t&{_Tcc)j%h)f@`0e?-bhnu zQC;qyJSC_`S>;oG6^vnkC&xA(DEs$K`$<5+$uf04n|hYLDo&ARqCEfUmT+&*hMYd9t$knN-rW}FJ36WZ=0AB+mgB3M9fYsI6*NJiEYo6@- z>6EE#1pgEDfB68@0*i_(pVMMHo zX1kkoKgYPS3JaUTqY!RmJoNO-7~hFmWN;=?7ss6bR2m=!eiSEX+J0oHrTmxtAu784 z$kUlqeRg}`FM|rieIBR-UXNw|gGcj$Q6_1>c16hfx9+z5pQe`lKPfFKdiuqHpAJ$5 z@RTEP1VRza|H_kWNNLpYiB=>12H(nY!EePBPnTB_6xhdl^|5G*ruzWxp(g*}AUN+9 zJye}7p%TXXW;s4Vr}D4+9|#9B4@$zc*r5J7J0RQBJN%_=5eReqGW{RtZ~pJ;mX&cI zem*cjK^Y|hG4RlX%WJH~)Dx+|%3nWOTW%n&@Dd{%Az@YT>{9}wq3NzJG99tpS8_$@ zF&73BAtj-~3FwFm^rKI(g8J1jdL!}w^gygMr4ga8rxcNc#QlfxvDifTl8_OnZy9lN zSXH-k_Ha81Mot<3sMzRcx=HnNUCZUQbklsiA_2vk&9LcMr4IDbO6GaIqj$i6KTDV1 zGhnjfZFyP}k|fYipuPs*{}XdC{-v6n205 z41&Q6xR*zM)9)W-8jw+Y#q9X}27~qDDwu#3xxpa}(`_fz6I2zW!WoOV`A}T4*Zp7x zb@$4qwIPtMMza(e6zN5Pc7YUK2Q9{C56iI153OC`qUq$>HuFH!g?L0S5e&D-!xyld z8TT;rHiI3kW*D)F7v=Cq@sk-#^ri@b$!DiGlyyq)z#zhE)Bn)K2ex#J0?$#xW3auC znufC*g)YkT&aPE%AIgAZF#l0T{9*B^HzvGL@|`du!i>BHd_ta*2F;Vf13 zRaL~zbxT%YG{?FSiK@kooa{Ex$@gz>(w1u`IA8Z*p!rMpa0T`v+Jcmn%{W6)1=l|R zhK3s_q+V8#@rK#AT==LX$Sxv3ou>J0q_y=EVmOxcO$(ln*^$+e`TkRKRLaC3cP07O z0FJMA-(U4vK!=yUaBsCmJ%Q{n(B7~nG<0dIJ2Fa18uh~MidE8h-T=Lj6N;PcrUZ#AN2laf;4$q#?;ye9m+yVBm4(IkUqHhwbtvg&#r@^rV>UcHYdweG9>H0ZTVSbV?1SnSwbWeKJ z|BKddJ22fVl=gTm<1+B{0)?zo&$he4Kta1ohZ2?fR%F zY5Jwq!R#}Y+WPHF0%r{07e2J8%?`C%N|=cs#rLa^Y6wdCh^b-~KW0(n3e2TD0!|9X zdpLZdUCwGTl%_pOc$dyL$a~R;IBlXln^0>>-8lX_|87(xvZd{3>EStf&`#4=Hw>uI zs+2o4Ha0dq46SpoPeX8FjPF#xthe;2H87Kih*b)!bK@#2m)=cl!&Acb?yNrXIr!l1 zO`wb<+`0Eu0Z8GgT}Q@W_po>&87mz`9dob()L9)HWq+ifAU74#10PdWa|T@fShVo7 zy2y`LX=uv){b~yi4;%D?zWE*EKXot%*j$=YN>`XjD%$*Dntp4*R%NzTuPNb8a1;*% zl(jj;aV#>-l% zN}ls5)02?LN`>@YY$zs=4${ERdk7$1hYi@vz#5b{RbxCmIfJ0=;O?BPVsTi|VY>!+ zY&XNG(5rmJ#eFApJYJ>@JtsyA1K3h4+PIXZ@~v*%B^@19;O~l>cv4gXg}$9bpk#@! zghP5**UWDQ*3ACm;DmaYAU=nZEB78Uf7?54qohf`_1ZBoJL@R+_xMtjB`m(k%xaN? z`K6yNpHFK{CwkTv&?L(Fa7N<*2Z<$}O;(r9y9R=CKEC+yMvrKH;b}*!*Bc>E<>VW< zz_s;4xskUu0_^c-ZH~x7;}eTf!`200;HB$_fwn{i`m}xxh`(U-?U9n-iiLHP?-Z;p zq~C~z6MXX)ex2c$K78pfpAQV4*#6ZMBjrR#Q&s`LrAR~Lmd354GVv&wo6o{YyrzgVWcsC#WKu!Z^UAli>)7auY%uq z{F)TL1^w1uxoWJXuLW&s`so(UR@A+%?bt>byIcjl@fct~eH35c40!$m((<#Qq{Gcb zxG-_X2DdB}s|KLI0UGq2xB>IdTOpq}YY0p1&`H}Fsgld4ps~aRiLGZQV{ATlM!_Sg zT9gFRt?-GMu4A)sD+Pd;X{je9N40jG?B0d~lUt%E&fy>P zA7Wp*Cs)w>zIxRkm)V<}OH{6{JhV7aD$VxMOgU4p!n48n2vaBVAgyYk(rv7{Q7MTa zF6Bf0tcfLh@znKInkg!(M>LowyJSka zysaxz&{NO$v1C1SkZTqcuzYASBLNt!_klJNZg(DSa8;t| zI}d6f0~Jy`a^>!5?w=4CS%HE@)(lGLqTbe7t-KTKyi%X+5a&GI+Uq?TF>IuFPb$N= z_Gn|BI{Kcd{$4+l)M(o1s*>`V5#wxI*inpT>ASnNT^%Bs#wu`zK3R%^t?hcQamKuz z&7O>R11Kgo$8g!UVpdmIArim;orzavOz@!l&OqUmX~sGZ;Kl6 zRE|bsn1$+!bQ`;0tY-l%3LQNUB6Y;{?R)nJDR3u78M_{IfX7<;JO7=HT+7_v*Gxq+ z3cnGvS~hx+68@RK(!(U*Z^64kQ4Y=n$@sIX+kkeu1eCad&vd&CxxYM?D}Sg z9JX?97I${T1q7S?F%AzAEGTgnLrJ#rDw}l2gOq1yxju?R)}z~mSkG)o*BrU3GC_-0 zS1hMPdUMdy`H1a~=t$EwN%_W&IC&(S9|eVNxMZ)v4I+#xU0bmjUHIs@yPQo`hv;&wK=eQ`S zGa_gL{4miUwm8r`(IwN*BEH@5sCb8&Ngqb9AfLiD5je29T{mc1)o1AUXNDvPNt4&! ztQR0POVgO-05>~H>E5ZQLphu40tgmp@&!pe=;lU;iDXduPW=>4 zJ?k>=JcT%2J0%)aK7)dvb8tYF&~uhJU^Z_4`^RUwrTTtmb36buD$~cI*+z&3K>X50 z57o0cW>mCI32C(NslIo7Zt4Z@LlEryOZ)?A|33bEMj+mZ) z*15BDFtch?JkQGls%fvgQ#4sevEmIR=g)iOKkV4a{1@8JGOEq4YtX@IDNx+CNO6ig z1qu`>P~6?!U0VtiFYZz(?oc#1#oZ-9aQEPWFzNF?-^@2_&7WCohF=Nyo#V34-q+bX z=Yt&+3sbA%d{2gt)(G>ds~5oC(E>$~B-IrjVe8kXuabK6bZnXrek$DPDyvp;1V(-dJ?jqehA zWeMWzEfa13ZN^ST6q&I*+3y8=Fa@Ant}Z}w;dX1cQ^_`f!U~V z6$YD0$*7d6)6JNYV~h$=U*q;KaW=fS_-6513o|m(B5;<|pv&aT$V>awMJh{)(UiC4 zh4`$mex5Mg3WY_Xd!TW#$1kH3vPBZ-^J*{Wr9lUVoO-aQ<{O~_o!&MHkR!nfK}JFF z9&|h6{knL#&0MSS3TSQ3hoA$ZRO8sG6(6;4dTH{ZQSE4~N|AT&i0_}ZHqwe72!d;2acj?URpsX~k7H-^q2u>;&?lLlmY+xM z{IBZ4f$g_9#?|kvL8lE(U})3r(oiSH(-ru(2`WamFVRg$W8P{B{q`g^AGd60LFM!3 zNosQ<=b|qNJK*Zz#Ohh;e02Te!2Orp4=-{u;TO>szh*yQtqk7dojvVjZHx~#z!Xr| zPj}dxj_SWGZ)~i3G&b01GtV0+=t{fIFZ|ME<&^`)k4ixnJL$j4rfTp%JyrsE$;vpc-C#mPjh93Y zU1iYIJC@c05tPSsdwoZHp|uFoL*f(*NL_lNRPPew@Xu6{owmRN`XDc|wXdVce=5a& zOnxcrTs#z#1>g3R{Mydt@IE#<1<-tZeHkFJb@7Ag6CYhzX7O2F%^(4Sp%*4nDwN~e zkaUvd6jiLrOluiqWN*x}11JpgTg$(1cCM>j@HR_=jQkCS4-JdZMzcotr ze`bdXKAh6g{F%bvx>$K~U4IP9wm&&heO@pUf5JV^wO@m6PV8RgawFS+^t-TfdjPt- z(dGng1hKXImFn$5Aaf&u-o>{Z;TDxftsAGS8{Ok-8Nk4mV4oZwa%`G!iv<_n=pp@E zG+_KKq8y1;F${$+)>bW)fP9!xfdk5 z$Gv!RamOv5Ti|QtbJFrz0XH7<7V)4gW(-Lz}_P!{^Ib%<`wwdQo_Tx{ANe zp}XB^+{oGt*x9qE!d@{)c}dH*bphx~MLLXAT0=P{9p`<{P2>l7%R0zu&q81zMk+s= zJ0@)U^>^aiRzImUe&_D{I6{;cv7PwQX~1l8_b~gXGr}x!mDLT%r^)>Clcr2%!c)tP z+J{kWft02sU=x~{udM%Jo9wm|U&ZB-P}EWmO#E5nJ9hc;*kR<78yCO0f|JM0p#6S3>L`$6HzaTb@p$JbC>NS25l4^z30D%G<~DRw&NWE8 z-h~L@2%a@+bq@UVKmh_8VZB;K4IT(VmK#blv%af71sRdbXhU)^FfOtKiirT6P&5pw zd`Uk)%he9d&mwZ@E!g$hlq*xY!q*FH@f-m<5}8{){@TG)j;FHiL{E!k@FqZS5yw|f zK-;?MOWY}B%r*+Z^}c#%w5FUV{$eyo)T<9eSW90>=c&m2e45c{+-KvRh!BYJifUSl zN+#k34OruETNmGOtQv$ZA4u|%lf8_nh4tK~=?sEXA+Q7$-tEos@cbunP+cP~v} z6?=KxUs)X3VR?<=GNJP@3hpSqcXL?Cx@QS&^y%;#RDOKD;HBi|xlRaRHL`3C zzB2gfZ+HFYfjI+w%KZCVF1#yfqWQ~*pLW%xMw%frPb{;zn|DFCCJ!_Kz|ZO!i(Y( zbD6l^EimidUSav2+iU)e&-s^+vOFZ`aTOKI>a?OR20+r_hI*s}6Y(h!GxRbSB z>%%A=2{XU6vk#8GAu_*c{im?q6#_7*>~P z=Ob;DNqNihq%WS*;Wf#^#jDJ^T?c?y`~&+9C5K{`Rnyf{Z|6s$w4 z;A~%is3HGsuwx(2TM@BnriU%>EzFDnNNzJNeu`KpF!S)V6LU?w1ibeyd)=;3S9p<< zE=0eA=U+M4q^Y~QqB=H)sZ5ZsYPm#MopCSTFq{BAQNMnfAUK&+0WUztSg2f8EjWHC z)ZChvWM0uK!dr(HkdbiJQRxY-s<&6y%Dtc8oUVAS_fi*LlUlvpjgYL|j*G1@Y(BIW zos`On=;j(}mQ9yUg@%hpL67TnC&5%1cyl%t!z1jJ^F0HF{LZEiEp>SLi2&526T;a> z&hsX@NaZs@ONiKP~XKtGPrTv23Ss@!t$aHkIT07q8_Pmh?ju z>F=R&IW3_ru&;_s(l#Pz>8HpvU@iD64fQCwc)$~PR|uH8c8CEL#Qo|Rta_}6ecrle zb5Zv1wQ!TJ9-Uy_2QVtpWe2^L1j5TIdQ2;7EaYQ3o^5|fam#kGDnt&hoJRcoT+9`G zR%#6-0M0{9Qp`Tj3E)zm2kLo|E|9(ee0=-EA{1a2kevCgx>BI~+q*yv=%G4NgxtLF z^3+W~*7!@n`;TFb`tYjP7Nh)-=IkagPsuy_&Mj^SiMo z-s=dlZW%tntD9@~>!yD3(P0E*!)SH95O}x0X)HY6eD~rO8A8fYE}yJfow|%7qU3(v z*KSi7F`1NxWH~TM2>1aXWTZsVf}usKiqTbu`MSyR0@(2BoOPjhSfTlnv|(r0O@L(> zS4B9y`9aqvmzt{KK=|`q879K0XiPE1r=GTtI?CSMca=LLdZT8T5o`nbu8BL=J)YVu z2-N;(GDpK#FYm3*^Xl$bcJnN5RnfUUO?~WW^9U8Fhl8~dRb-rtzX6$AWD;@gu^N54 zz=eSQB}iG0h$ujW)sC4uO~WOJ*; zg~ka#F8&Hbg?rpvL3W*w3#%Au#Q|C}&;FZ5MYjrow%`H7=!~t-E{d+*MMhZjnLK=T zjObJ-)#{+_U}AP{ZTCJG(?||rEwTNGGk$ z^Z9pPyUpgky(|4_lQTMDmyCXTt=6WMQm51et*-v2^yDy0!Vgde{{cD)xK*sr^`4*- z%QgeoSmh0NCyyOO^s?jUX2rRV$A}`_XrETjuZ6tsh+d#U(+4Os?UbCnn$_A?^v{R8 zJ$q|RTwA`Bgp*RuJGf?Qe@))8S9eC6Cb{&hFnr0rTx|D4^B(eILV(Jdf$7_*oomtwl zJ+Uzg`KxrLE)Ia9u5}>{5TL9KqO=Qpy*`{}iS9KH`9w=6MT%&g5BrkQ#BM(n8D7=F zAw(tE_T4gV%UtiH@ti98MpCYR_DM5y;QXI262iZ-&3h3r~^avFOy zKYn4^+a7;3;Dc2fnvH?ERLlD-f2#GYnh$+K{6VAfJ%c%E?#RoaL@8$2xgFaKX*mC} zj=KOLT$T6MBV8EJ?+o{F#89+|@@n;=A9nTHOW$z~GTdG7wbK1^;-Trp#E>(ql<=uk zF5D#hnk`9(^Jh7=UZ(aNyibN+sYxhH;h>maR6_Byxz(?s%a{9y!L1@+h&YRwUWnsU z!}pm15zVlsG* zIXeiiu3fl-H@@VIU~Hz9ca3FlZDDZ&1<9N0wkqEkv}-4OPz%#fX8V*6`6$QXw1fLq zBMQYOu^Q=eaH7OwZAx?|RFCoGfYE@xPQ=5sz3IJODt=4NS8e;}8FX>My>xoy=Z*m8~FYv5gNOT zlHbV=x#w>V8f#qby3XP5Z=XuzCn8pj+{9)UA0+e+Xs1UBR#=`Q`cv7GLp(F`{CmS= zuD0EdRj}#Zm|ag*Gj?qML)T&*hvf29hGumzgbs zYw|r_WVo^Ez+;JgdtAR$Jz!QCwNw~>chzvvH#mWgIx8EQ8E=4ZwES{UtVBt!Ms9J= zjhvsTG<3IlK|go?67;a2AsBYLyBp%z=O^)kYn*InzKK4;Fv8v@S_3(fiLGP}}0#TO^sUd8| z2jfGvDf`2tk%}h@pOud3uxxnu4&%R@_Vx@iP>MBy-{B}}%A|*$5Ok<`bFyvK(1{D$ z|6FDCRtR_WQVE${_13-g?-04%_nnP!t$J>h7&nVqwPn}Mr3um_9YLM+Q?G*VJsBwb zZBEzHy%4U32aIliOr}V-I-0gh2xZs?eTVBdZg;jYj6|VZ18MNnv&#(WCoUPMOql;B z1LSUrO8n*`w3%-R=(n;DNwCjH)a!#@n-?zP z2JHoHZukIp{xdl_om4vgrUPRrCxCcdy|}NaA}S_ou;i4oR`1;VqBXgvfQm`l$JL$u zlLD@0j?3F%$k#or=gnkGB<@PJx)mYP55?v_CYd<)5`i{Cxko#PVi^srItiU(-an=G zOUMq*M8$MCOUoxdmCyX98uSnJm{p!B13JAL#PAa76gn)XwGemHTXC(aFk1{9_pr+k z%5c>lJKjy}czVAJsqOKQ>*0xLp=mTy$--yRPo@5p;7*)|Ql6Ad((`>z1*e+)h?m1# zfo;TJO!MwBqnRf?B?5nT^LqlJgM}_nQ_YwFUmoj9ExSZBJ5Bv1D;`9-5sKh1@!E~- z_n`RxWsSi;La$A-Yc};XMzEi@ennN`NFOnoS<`IY=yzVGBU7~mbJ91c+e^YZaRkd$ zYN@>Im(n3h6BlXJuSLL`M?_>~=zzef+OH3>j=%_s+M|$k>uyG#=!IFIwK>xM?82Mg znEgZUF{zxz1Fd5Pd3a~MZml{jMq#4K0j&39>9w@oI0ZD)JLw-#=&bJ{0V?QM@UGtu&w?SiR98nYu6XNe`mb5iH0Iq%XE~WN;|KOmHR|+!Q?Eb^8=SC{>P=ieQg2> zk-2@|Q`cFJ5k0Q{OHJ;<0!`b8r>g2?M+}*dT4CRMI$P> zh~aDoWg|XmFA1-ea^dmZYCUN1%5qR^e~s6VNLU#Otg!Ad$y7~c)Nf)qEZ&?-wIJzk zL~hpN6xXdfKcr&}~hUK~LK>7C1^dSEdS4 z)fkEojhRoLMen_(w4ww_$Au_b)HRCQ>Aza+jo_HG)wTOJnmVuBu=6f(>m~=!=`{dDOi7~wv2YAZMW#)Dl@VDE` z*?9TH&KN-^O-3ozL|A;u*WdI;Mxz7G$xvfKjj*y>_#M*%&h_@e z{wBetFF~NtbiZ>d=CaN3kw83(c$dYtIKVm;`94-~Ljs4u%sk)XKgBhIXaiP6-fYb` zu?VGM3!L3;Cl!mb@e-13qc+#}=0K`N3sGpmMDPRqt+dzHxIYy{=a0LahiQOI#{}!H z*9ZNV%jyr|ji%e&O#%fcPLDUo%S~kSbuPQ>95+n8e-NES(srkl5G(Z$IoRSh0Ie7) zV{u03F`vK4m-5NT)J?8V5t&bxhnpcA?w@{sq3M_|!Pjq1{DbJkn*Zk7Pxjd?P{uSb zYO(H#Vy-`k<;i)^+RemWHN(&~Z?ku0YImiiN$DaR7fv5*gck9oHk3;f4%N}w2Ji9( zq-duczFUSJhLPWwWzd0PFEZ3K%a`k5IzO|+VHp(TiVHJCr?IPtRf%3!_JehPmA7ro z_7R1>bNZga@*wQJz|Qd{y7xbfa7GY=9n)*xk6Wo`Y}s|~MD@pS+zgNYDtK5U8(-D* z*aJSKI=*vSjT$4Fx_L$G?Z43R?rAdg93Ij3cCt@LNIF|zh{1JEz@gGM+?V(`O5c=d z^sYi)pM_nR`88wBoW>>NR^Jw$#2QONA7-YGW0x}|hj;Gean*$Nu5zeK=Lie1SGGVf zj3T=L?LLicE?|mKV^8MWWkYnsIT+pB$fP>`;FQc+sRT{sDbX0&6A2Kg{5U7>-KY1% z?4YC~wX-ZTu<*N^yjWjskRm%%?AwgFNOT;?EYeBOs!7G9$o=VM&*=967zfo`MFY>H z`ubx4FlRLEC(CE~1Rk{XVu=Y>{#YaK;EC%e=UdN?S>9y1O_?rl6rtprRiwM*5{W?v zCy`%bzCT#rP2g9Me{Y~$!+UA2qs$w%x!wL|NWznitRnfH9t4}oE3Ds8h4MwTz3~Qj zIHqC-+?J{MLYbhl_7y7kVp3z#ghi{WSM9_OOmNgqp5SvDMesQ*s74gWVt%GtO*L%$ z;}VvmvZtNTs5%STzC*XF7T)Zpz;1a?eCQGan?rJAVB^d; z?e=q^pL%N!n2W9mE}|`ALH}5q|11yO z9ld?!V2Yd0VxF^O@A9`D11`m1+Ko9K8@g0+ytt$PZnuI>NcVwcVOzoV4W?E@7X7^2 z@&d>2lVkcn?8rIBg4=u99pW}V#>B0N*mjw8ptI7cehlAv4k?rbWnXR)koY;a&~WdU zqSM-))|uW%cRpwA;!X91g%4R^_R$rQkaI4iZ4SRiq0k*l&^J-c}Sy|wGpJnY#I($pYNBUkCjt5 zUXeLpqfZ1r*Vs5nj^9r%5>zQk@_P}gIT^NDNdbMU0RZikl<&o<;n^*VBkH+3k(@c2 znqp02U+%en?as~U@mpC1xarvQaM;`XrtQ2QiMjPRizK^s1% zPY%O*mw5B)%2LZreFrLlG@=Z48~cD{zhC-=A<(SQf}4SiosH1NRZLzkh*99LBlZl( zQ$VztQuU>(ojE3NUA_lgS;ad$Ncv5L7>J({ja^An+(_Gbt0Z_;kx7=`KqiO=B4_vI zceF3-P?=j;D~XOG1AF+CJ#HHtRsV2mFV&!8!YUa?4=eh%_gGH+M6`@Zs7y+V$+z?N zUT@1Rp94QLs5xIPVK&jg97~;tUiCRY2KHLhdgg()zi@@3QG=tD0$CFZ{78<;mpt@C zcdl&ixBQk_9jkDk`OWcVWCpI`2VCaS-o0P^2VF|6)m;pF9Xu?ykINM2A1ar(mg0L} zRu=KNTp-^M{Yd*0`YD~eh4D^Tacgbvy@HEUfZ7X=uOm@W0$fWutt<8^Y&i??iKdXv z5%c0Q(XzmHj#0<;h%VIJL=94mHKLbL+gRr2Vvd$MUGx}FY<22J$o||*#&ske-)j2R zz#Z}`5^PZ53tkL?a}odX3n1*m*{O+?(p2sE=b@^Ys`Yh;{S-1rYtQ9*&-VtT5w_Jl zze!xaHg6N<-yXSBB{_za$j`=sTF~^WM()S;VrS)lCKX*FY4&+R_ZatE6G*?#a?S*W z5fwM0p_|Cc9_wsethQm>&$Au3Y(c&y8tqbACJm1_j0aA0&Z3Y6s>R* zOFhClBIS_?|0WL-N7oA3&EX`)_{u@HQ|=M;=i8;(Yf^|i3IBFCAAG{uO57T-+*jK? z7~Y&(`yUSQPUIhpvgOP*1>tnqS@KWLe|N+&dQDm^1QZ^v^fkblNnsl-m&++R!WwZ9 zGHP*ODm%<{&FgvcxPWXFOodAO6&T#CxUK$r*)=Wf)vCB~^EI>%m4>qfpWdAGP@WM} z@M`#Rv@Q<8SjYBYZQn(h3$A&KR8V>a=o%!ryN^5D6C|-~@C%i};IW{1NRqz&;mMgJ zn&q#?#ZoOak@oRgT@^D2AA0YTt&EXhxooQc*iv4x@1A_5yst$zF z>>oRk&Yy5T_3(ciKn3tmtS3M6C{p!Yqg?LjIEl2G7!Ulu97fsjXe`6fr2w&NkZ5AG z4FLGPp*8IY!DJ*s(pRwzN`K4{h(m7W1CPbE7GMlXB&vhfzj?I<&UMC(R4S~OaQkon z80!_hta#Bw+Wlzi0S!Dyr%2L{-zXB|f7g6z&UWL8&vlVOrC;)AY&FE z0^ni!aWnY0CRP`Y#qBRyXFc-Hj>S?*T~sth!0UvcnCafc^(mXb3JI3`m`_y$=kH2W zex(cEHkYwE#K=3hB~2uK8G0|-v#L&^v#9Mx6Kl7GWNDl;*b~`4Orf81?-w@K;-bIE z<%c&lr=Sk_M!+jDTVLyb5D&LAnFY_9{G#gYgAJ~BCx@Pit}XjAbJI2{edbwN#qIhjV$veWW}-G}xv(%+gHM@WPJpm+?Up69Pw_lVKOD zcG(Gu<3>~-5oGCkMC~&;PvM`P_`3vb?Y_D8@F=#{KKVOiqCZ%0*;%Ri>C2=MMHeT> zeEG^uWlY*xxFq}wZE>v}g{fuWkT1NL6_LH%R$OWAng`%H==QKsrsRX+s{Q`+{w3nU zyFX6YJ?&YM0EX6%qAtw1Gv5<$qfLgOS9Zw*!ZoN}l`!R)ENS}`GUw8QlgqD_;>gF6 z!?6Ct)`nVOTQfrD%8aeBsr1h^$AXiX;oyCKLC4q{mb4Am){wffhb#yJ-Yi?+4lWZrtp zq&?|Xjwmw~weYatTla_Qjzw6zxs|r0CbhK83ihrej?8^s!;lgT)oqZ~n~c;J*%?2x zS9?~hhFN^-`h8=9Tfo^`zYD)V*0b7R5fr&3V(QC`p_{H?+Z zz`;SN&tG~2+BM6l0)KL!+HhlrAV=6?sHd95Fg`VJoDiXnUfZt`3JZV(U%hWv;~P2; zqR(U>@SOWr-?g&1>i3Q{+6>FdJBwa_{(?p5TmTSuErjj(TW!U!ltIR@3FANc#dNlg_aOiH3SH%+wlR--TD498nIv7k1H--yb_`0c%`XX)bMf+`}l}BIN`1BuGz@?VYJrFu68! zvUOTnKg~((S{$7bA_A@&2OHB$Qy^Tmjrn1%x!dYj4=tn^CVsb-^^Xnt%8kR2&rXg0o) z)c#3uad(}LBOiDqvg;6&t!-FFXf>~kHu7R<(v}Op{@~cr(qgdORB5usaLjqiH;WM9 z0T0=4q%(r@-GtyOJ{P|qt7S|(?()rTQl3cLt)#3Amn=kKd>USlDbzrwHJ08G*UaTm zAmiPisoSe>X!h=ba~HQ}#B*3?&&{Mwd~X}BcJd#MzG2^uLAb`_MnO%YqjL;?;3b+e zDwHmtpOSTmUA}S~8(AJMC~iv?p@>g8W2or63oo-i|e>SGTT>b7X3?PCWn z4bVK%>H*Z$NnnGOWLP94fE5GNRM8k+BxT-FZ;>4ADh${-@ z2&LwuN=@ZWq|dOHOyZDvD>JxJEilGsr_3Kz_ms_$cz^|2DS5yP0)sylVR!^_EP^dL z(?1nei!DEwr{XVU5%Q#eqJSc?DOa~MzGOY&N@viXygG>QoCvh>>gX}%O8=By2x>Z6 zab)`0GJ1B3-|jbec^t)(O@+P0;Cakny<*OlUge(CdO1h&H~af3m{A|Okl%pMGd#iN zo3%;0xB>AR5Q>=|&hr^|kS#pHFy+{`Kl~G2_B&B(g5u@-zrkQ(t+fET8mM@W5PQp6SGj|%~QF~M|Zq9}2k#qM) zq<%vkHql7cV$HQmA_Q{TVM!cvH@HYJ4gwMFI=n-1c5U>3s+I?iG7eMBG{~qJPZ=q~ zn`hkF>fUpAv84@FB8&pk;x^$S>VX4Z52k&Z#1ByX=%v&LH>F=)`JBQ4<0yJ43 z=DL;1u{Nde8?SfOOcB|~r>kMP0_V!b9e?Xq)ztW+0wu~+6L#^_s84&|XJ9Z3`$of| z)oo`rwGlsP`!n0_wCC~ONd+`|DK+q(ulr7|#11?W=(kN4B-YpmC4#6qy!^_V+7H98#9SAmJrR;8;^@Xl8N zR<$#l&Kv8AyP>e@5e&AC3E5B8(qWys;$^*{fHt-Bo5L1d;t*3SU%IoJTb25F>BE*% zCwY%r9M<%;F6uPCc+xjj*aj7QSh_ikjPcq$o4o7j?>|~}0!76>ZM|HAXa8(u56a$m znpSLHTq%1GeBu(Mz%RZ(v4o1Ht(D#>;`jP`yuK8j6Tj=zXji$tr+~Yt*`C_GHo@Cr z9175Zqb!Uo16U)~#T|pF>z!Rsw@mEIZ?Q+JpT9vD>BI>UYfwtR{>pZVfp`Pg-bRjr z!Oz37J5JsxkFVDqQ@*3HXA+;)wLU zI+dRRpFb{7aAjrhM?-x+OK+(PSdA0LYLLw8xOh1;SCPjHic&RI5CVkX+%b_B&+U-i zl*s&y@dC|4I-paFf|NuNQlBh-USz$!?L~cb~n?5l-$C0`#z4G@DDduQxOSkU;%cQW8++= z6cx5-9G~spMJ>M&LMXS-c!IqlK9t`n10(2H8u%D z@MYn26=-H==5@NFb60Q?Vu*tXC=9O_y^v}et5xO{|6t>ETy;?M_;5Wjz{UT(|1&8@ zTktyX4IJwl6eY8lSC~ArT@umJL!tq@q*lj_X7G-Tjh#%%QUz?|?1B6kc{*5p0Dpft zB};j~)xmPsM9gjk6!l!-@mz4fUp~3jdlj_ewF7_p)A%2}4U7gCnc-ZcY4`&$0nJQd z1y3}WM7gZf8{$9#5l;LU`ZACcOZ0>X!G3!-K;`-3AM$MQ7X8|b{$Je8ROZRxzbG{S z6S(2_-~V&r{eLFa{#)`9I&k`K8&q{R-Sw9O{9A(i<$I3HfJJF)uqYhRVVV@i_zx-f zhyL(P{{IN{{kQb~e}HxW_sRPhR~bF)2Ndb~Jemg-)ODzoiUg_#6hYwwgvzr8jzfM? ziFzY1M3GZ8cSL3AGRXfeR+#47r|tN^q5wnYry`)&S!_(qYKxoIMx01Tt!exj`E~kf zx5R2fG2vM5#Ko0Ac~_;Lk$!s_@cf|A2O`kT@)eH%y`V7>@JpiU8zp=FFV%zS3MayY$hA z>M!QUs4gXA0%$SF(met}u|Qq&c)Cny-qKyc>%U^7uKuX;YBk!RZ_En+m>rA8CD?My&wWV{$oXg`F z`>((S|7w$I^5~nE-N2GC<9X{fo~JaH$rl5bC~8fM(WQzmck`1vJI%_6RYvJ)S-Zh^ z^i~fzNW*tkDSV2T3jVV5R=5__dbDYSC7q7_P_wE07*x{Ebpcz_X4zw@6bTBUc3vv$ z+T&|;uQ@7$%P=0vPx+?HjB;A<++;}91;wE!z^54l`{}POZK^k3%j1s`&uNg2GXKmI zciwdUtdIRUw||13SIXdGg``>Ch)@_~*~>k*e$U}03Jy@zcxu?rc?)|Bq(yNsV4v!7 zvuv#d`WU~(2<0bfq%`EfTB&bZ;Dd4LisF<3vMuM}_bcvkK$bkzQM<Cu*vU zK^?`(m|>3Nmxd{4ZL0gASi+pL2&aF7;;Y)e>u@N8rX5;nNwd>_6p{&$i|!b=S4vo(`e8 z_ofNeJE8v}1b1uyA>pvWE-Y!^8o8tz)XOH>5&LC4_+{D=0Q_jOXD!Eu8wY6qI*>CT zK{U9HQ*=q1ma>?+Wr3!$m}#RF@E5_$Up-c{qM*P%9v%9jUoeC&rj=c!uU}{jzZDbh zyIQjMLF1i^3Mh$f&d+=9y=zQZF7rt_!P(*)B-T_J^9_?Pu8CO(`T$h8`BgW@WOkqN&LRFS!}^8?W0PhooPW!bw0It@ z;WwfikGOI}0=%-b>+5?4vj3JhYq29I`%Jjg>G_ai^2e(j!XCu-D)MV?WYR9^2J`C3?Yo}h zJ5E@T+T4HyQsR=akDp&s`h)pK9vGWt$q(OhtSoqUQxIc*WJ~@7S}p5nHA&QiiE7-L zM^nTwcaLy^>%c4z3stvB@Jj?bMf>e0gcyODQFOH(#K6L;;Ryk*t_#4?!jgt0<+!HR zHj@Xyu8(k6L@VkI{!dvX^}b}bfUwUhinU!6n8XL=1Vmanlomd4^H{vfd?i-^5d|h+ zA%%<12SqutNS?}DO4lPl-{qUdzK{BQuo%x(F}=t)X~v_r9*qGcYpq=qk-5$ahfx88 zF5r?q4!r4oou z`}&3{mAP;;mmOHepDYeKtDoGWmf$&c!}>2CSR?q|r?>N`Q2qt><*9Lz=fdgd4-KgI zpP-~9l>04h)UixCnZg4D6q9gd?1*zVYqs0hGKweniBOrqfj#*zTD|UVvnfJ6!JoXX z)dY62GWkulF$KsIt&=0D+;9cxs}h=1Ir3wXX+O4Ud0`MLx!qnj-1oiaaRsnmh5uG0 zauS>o(nyvJoWGhkb!b^1?zyFe0|Q)~?3Xzp2grf%Bo<)+h>&S`y2BZlXh>@IbwnR@D9{5f47nu>S6Olwg*GzX>I2of!OXmpGVkB|8y22St;vg-gQy7J`sGq;!4=w>6wv zr!83DegPM0JOrlohpOn^7n@a#4ZZNtJ&ImUPIqEkgPK%{v-21BDSXxSRAwCPq=K8A z|3R(|{3$1z6Jo5&-E3Ez9AhizVvUX#tcMdn7r!09PwM35o%PHVD)LcWB)ILK(xxKT zyOI7VtXnTtsUaqZdBuO4O4e)y=_axIq{m)id6F1FypPr8+}qgMKOQ1GUURYT==-Xy z=rjft;=f4dBYEF)eV~jaRki@*R;4wf0vNvnxX0Yn8eE1C!}Yz!uQ)yNk7xVmMoa_p zQs|Q2O48ejBFs`aB%YSvacXe{(QgcrNN5gPL&+rRZ%?q@0=j=cf%uMu_K-~ac%n}3 zEJXCSQMKz0H~ivKb-5XBYqvyqyl-eoM(^c-V-#gc4I0A$6X_#d9`2uS0G1s83s@Up z)(PQf=u5?URT78K{|5NB<2S*su#KL)c~6aPLO|TsE^Wgk`EADSlrGH0=!5&K@&voG zALh>nve3JERyapD{YS%pyXj!lte@aR!4G=bC;*(EADRz94kk>t-K=}02^4h}2k3@Yuo=Lw-Qsybf^pF*HCrxZ6BDwan3w$uXz(RL)7<@WtvuK}69Vkp$d)<^4e&5<%ld8v} z!6t;?RqWUgt3s>r_Y`s)&z-Ole`<;-wv+cRJNFH;*pTQG^mI3w5+B!!cv(J|y2yma zY>4GD@EK&MTC|t!F>X6o+-p*;{@pA9L8~$|HS@O3_57H^tEQ(O<^c2xfCFm*-g4fv zia)dxUjfHy;L!X(S6o0AGCj-hqiPX+R05X{RC>Pyd2#T&h|+G2XPr$5O#Z9GqQEtXVI;n zs>we)pnH4?@!m}y)lFg5L3NksG!ZFfa+!<8%9E|F{*b>9TM1eqgU(2;Fs9A)b+~IO z2H?8p*B1s?!O|+J`KzvCS-;;P0OoNWKgX;Yx3FYXprR(zXHV(8xLnXbU8yO0RPKu* zTG4sqn?g^q`YiFKMBnZz*l{q!XFlm%eZ%9pX_^i}n(Jke z`v+RD=|`%}-`)qlTzC(;xXiH=be2)nX<4WOb7-}aQk%;-^0LH`J#S-Br%~zVO&5ue zh5n_*dXuk5p5MhDouK9^-^T{p$(8UY@;irIs6$k~P>-MizG_`0pScsy%OHOIsEl=k z*ma}Fx$LbHXQA6SwHQOePL{whq9%yFg4pHRJ7BCz4#*0hccu`31vefqXIQ{3{$pM- z?59i)wBCL1@7A{TeBD6sgGOOgc?)BOPiI6J2m!#rvG>`yh*)$ZeqMBWdlt>3)r3F| zz|e#jKwEG1u77|njp)tXpXMq|ptDx_P|Y`BBC^2-B7QllS&C4*eQJ(s)RymzZv8Go z@TsZMw(YS=Yt3Ky>paTd4mMAI>&NL4hhuB*4(c4TA%H0y>p?F5> zf(?_BPTu@HH(C+oxTV_qem1#kV@8i)8> zou*c50p9AjY)M;uDjEXYxd`tE(H0}}RbjXzHL8C>_dMZ3+mr^Eu~HSivNl0}_R5-g zxy0pDLv){#f57f24w~Y6Fqgeu`)pYuT4b1s<3AvMRMaX44k*qyX%OxQ@EfZgP%YbgQQ8ur zq|v>Bn_r*i*kanGsCFMrE2>@9cI_Y*9Mv@|EOGyWQZ>+D2g2nqF<+3HrlgzHrQPd6@)rIyB2 zHNJ~dOY1x2^*)7wYil74xHLE#7jX2OmSbhrNzrpm*9#z^)n4_PU10c%+QCmDnMSk0 zbwy^*bO7RjzAsSpk-0L#ssH}agJhtUAoHgn!J|Nvh*U+hcOh7~b#O({L7h$Xd-QZ( zZOF{ZL8*Z%xe8A?0nTpw?R%iJ7cmQuatE-DliM`+c?3;fEp_$bm=)BKTW_-eGAV&B zriJ)_r?LXf8R1jVeHK~&RfnytSFO{uC^BG`N#nhVC!nX^ena-d60Q*deaP8E`C#%{ zO)Gba&x^2D7+0~Qach*x2MMOwl7U%0u@Cz$qFVNWQONTf#_$kFuJat`-%R*>l z@NbzfzneX4l7i$3erVcyt#|MaJW+mnBmSaBv)_J9{MQNzsh_kD9cSH_(21h@(}*e@ z{)xN>5{)Un)}ua>o{&wOZCp$#t@RDwDg9Wrr9kBvzS!qU7kU9P)NI@%HKoTCfT7Dt znf*3VLCqA+D3%=I$MU(0ltsT9CdP$MnO=RA6y%~P$MHl z-C5+`QV^}uQIx05nBGpo^A(wv;<)1}Yti<9u}X&aoyy?yiUra%>aoSCw|W*VHDtbe z8PS~)uUwH<7ELDZ7jjJH1=_tQF?-}*g-Bh{+?!nQMB z5PQ?EbP-VXUeJ1nZFW&=*Rk1;V}pKRAp!w0b&lpw8fLim=KIsIVZ!%x7!21r3sUYw z)k(3lqa3_cN}ii2fGVaRx6{KtV_HXC^A7te`n>(KH%NeNP~-U`;B}MW(RtOI_k>8d zHdWYbQCtJK-co+1nNxrbEvUsphsAHd`w}bfRQ95Kf0gN>d#=3nZ)v26KCamZF@1BY z&5}dP{7vKPyyojFFlvwJ5jc=A`G!?w%8Ai0RRFM^G;B7=l@Px47-1sWk_J52Zkl*o zEm;1x0tqdKwmBTCzVo?1L1)ze--vHV->wG;~Ze4H_=#J zw$7O&qe$6ZP~e+d_G}yt3leO()b?^XGoMlK^Or4Az<%ZUpNtI)u5fo%1m@p-P+97> zmn^I-TR%&^!eXtthZFdx{}PX8euje3u#{Cr96T`IPIWHW1*Nuz4yTp|XrURhA`%q^*G zS0KQ{<5z~5Xq(4ays*7MlU``EOK33R7x^fJitn&mg|M4KLeTJ~BcI3-Q-i%WDmYPj zr_)4sB_khq+n2@pdxy8WTP(&Gphf&f6ZDTEZGt9d9SqLt1|Kdc1RqlAyhv1!+Wam1 zMKiM2v*9kmOc{|?W_n2jpHzLJh>J%IlH6Ed(3IRv^?RU(SQwT#P*IFF7wCYAf_C0i zgFcB06ZJV^$MaMdWLRaF_gi4$+xE8C0LPH6 zI%#HrV6Aux*@0(A@t;`(a|FP?@NC~2Aj5v7{hAkV;lEKI6;E`VBObJPkVMj%Zj8og zyy~w6*Q;=Tke&LO^=$&ilqwm&Poep?r5JYFfp}UL63r@!otfc7uYD(YXBSuXr=Kt~ z45;O$6_crq8CY2)6ZoCH8MsY+F*Gn2ve>Pzx>$Zs>S5x`$;9{v{YFpt#i7+Qsr61b z0~)v;9lVH{s-DqCc*^(#K(Pu_`N@}->5JD6ayXC&=J~k zxUR$51_Ic$Toq5V6~wEuysWH6EP@ePQu7&l%zs?rZy?!_HShpzDYq+su1)p^yD$L^ zPIU$&ANCHH*8OQ%-~qXlSgHbN|1M%#4d0aSVU!>|n5rudG({@kF5Zk7O{mxCNuY z>Wh$q+_)21gs`%InxjSWL-L@SN0BQeTI#UCue`jFoznj9MZjiL)$fFsgTLjb)5K7q zPFNi=tnyX5PC4X8Lm_NNg?xHIo}GZmA{|CmNd1~i@AO${G&DLvjG3jusZ5)gk~!!? z+DA#>p8C4{!pHSqNl8_sU*(~JMCb_(WsOkL@z!b^&!1G!V&u1@on15#P4e`%M%s&a+O+?E6nbllxbOKXK24~^dC z$v>k=i9Yf`S%i5Mq8mTUh#R%#9a>cuOJiMTiLZUXENva&U}v`S2Eh)jmnAOdR36;&aI4@XG*sfF0zt5%gXGoQ4Qxo>MjLUJhVZ*Cu8#V z+k3D?Tpe&Y5f(U4*jX6to^slHO=4?IEV0yyoEN6te#*~C^k_~JG7{p85LmJq54JYT z007Da!-01~CXg09j|74h0MCa&*MwA1I_@6uyo+)FAE zGP1r$W+L-+jh!K_!t~~0*H`Sy)4Y7k02*L!eT!&ypT+JrNcQx7#~d>(&gx5=KBSe+ zaB>;6t|7D+tkxP$Ueg?Fi&4HU6Mv_JIwAA5KyXn@5#eCJ3KxKaWo#T^3SLx;de5PhMl)18ia_<|QNetY7J&@WrfwNe*iueR3&9F;J2 zzQ&GL zWd{t;v9tCOdZKi&zCqYSL=lVOV567Er|hzz`m3~g;LAqCs}#IY6BCx(1@iq-lA7kq zy8vbs&vktWYITkHb_0UXMVf!pm5Lmuw%Im6rJKZS7W}rz`1KSLuGIN4G}Eg3MmNX~ zrp_!zRyjR0bXrhzr@}uLUPhQn!DZ7ir&ZNwE!}<6O^%%FYR4Pd-!-3Lj4P^L_#Kkf zwx_oG<(;?8Fu(S+&!e#Zu!t_-*th)m4UAw|Q23?S6Tjy1cdo8Mt*ukL-=ox9Ap`2= z7^)%9N<@Fur&mUfoE<6qs$5W<-)qocHP+8O}=YXedE?B z$Q`>4a?VXifPD+E>>Le;KUT;48I?^w)fG)i^|a&gl>Ge5CDiAs1Pza5nKFLefmt`w zyL;5ptj!2qh>Kj5w9T5S2qkSJo;uc{KDUgBc+4JxmVO0y4_p^B!)x12Ao=6>F# zXZ6?p>v^ZIqh{oZ^5w+r}f&`>>d(J8C7lifu5vSVoak7!0$FS_IAGWtP6un zh^?m`5@Wd_3*L4fbv{b{nN`ih*5Skne0RM05UWClx6S;?2#NAW+FlTO=k#N^yC3C= z_Via~Hm4H>7wBAb-I7xt)FLPF#6E$6{Awt_TUxpm5ltI$k2`E6)a74Z-jLmZ2~Ng{ zLPCzp9P~r`!=O`rq;;*HMa6=Y9n$a9zJp2QZ^!{hXB(-pJ=w{EsPsU&_uXTwLG4Pj~rpuxA z1T`;}=20jHm>8JO9f9+u5KS-8B&zwOL#oz(LPN&y^k}Ve0StKq8q#18<8o9At2di00)9bkLLJ=}^f{hxGka^hYYp8r1S@xvEuf#E zJ4J31*vnN(`gTj)|hJuM7fGZ;_f>+p%?9703FXFm+>s?xRN50@a)?G+7ynZJ!3ti2xxVvoMGr5*|D?%MzF3;M! zx>=a3@&xJWuN*l|Wb!cA?>3=Emk=4VShv*W=j+vF@6nBk(l)Ww3(2aG6G3WqUhkeCQ`}EZjx9aG zKYtd{78Ttdn9KT|ODM`}%j}s|tK@@~OTPhQxM80|Mn=ZL!9h`Rdg9mWRhQG@qWy&& z6f}QjcfxjD8pmu71qPUSKi)X|tT$O75WRW6oK|-%pzZEg+&nxy92_90#=Q{^`Kbqm zuPzNG(tO{WYIAejPhUi9HhQ(VDZ6$Y<@ouvM_wg?!SDV)zDjaI-molUP(?sNaYl_VXliJrq^72(q`-D)bu(Sf$;kcJ=pj{foo;*H z>_lQ2nc5&0*vdxc{khv>)z8hpKqF=ijTXLKEd&AS5AQIa z@AP)2Zu9(hFFg`thtS$;1N(GBxS(gf0{A>eir_8Zk)37s)s3L{;APTDJORrfoi1!J z6;bGS5g~0sU5gC*>7{!Z3VeXZR=)RwX6{D(d2;^*aqqr6i`vKl_xA`&0iV^mQ-E1y z{bQ$a8-@Q*FOwu%DO-8hm~CyCCZi=oLDQ?+?&M}UQ>6}yjP~Ix zK)~Q8?NioGvuvsjZ3iB1g!rbtVO~7pcuXI-{Y#{g?-Y2KG zD@|{7+$~PGEak#!Vr?{rQ3^X3NT42lQu0t*C1pCj6ofGcJ5&y4C7SZ%vEj?QW_>kY zp=3Pl<1CIn?#ZBT5!etz;v#E&%1(D~FT_-A!XYTDr=)Z3gpqqB?~M+|UfKZWG5Sr= zf`exS99N?blZZ((p=+ovA_n{XeFH(v=fNFxtr=(lE)fRpK1mH)+WFO<^ZwGDjVK&i z2}^1f2p+-~Z{oZ5oA|=vd2nfnuh>+beq|z;n5)b{&NM|3qE2U>v#!OkSicp|Z*+GO zp0obPq3iW;VKd)$Nq*QSf zpkrbGx1L~3I?=z=&_xm+8J7Tv}@SA81PR!eChoM$| zpJA4pqg$#|Uou8|`oH?eDYKz43v=~3V2VKyGY(m~0HWS~AWRnG^uwJ5ja*Y~4y$)R zb@2GrDi?hfw;O+QsCc+3+vxXy_N)0bTZS&0!Sn?~F54={q0)2J8ULNO-_a$&X!*E z3DpX#&nM;X?{IZA+Ws*?0x;Jb$%eRHe2{l&cZSJpdx*N4)!luG4<2SnWJb+=EoAB$ zZEm1IrzNC`IBG=jc)w*5L*8_$I+5_(r6Vj5&D%b+&@}MR#pIAObOKBd2k&`Ho^UP^ z_o5oElWgN-Gdt<99$fMl;L@;#pw=)TB86(xi(qTAhpX|yeh^?;*6!hVqvm0MizUA{ zDQ=~ekLGRYOrmz=l+wp?-AICWbp+lrJk|114FhI=pf}YnLIqCoc_Ox zTHs10@MmaYA}e`mp;)0rh1jXoaOkJ(rE}@W$u>Jb&&tfp#&(z9SdX4XFVmoD0y{>s zF@T$%r6sskXX-8ALzEX2GC^s0cntNkl93im?hXbr9-chT-rq#ffHvEaW=rihoOPmG zFxs^WA-8^o&G`lKQ3yZ@UeU8jUsY6?!pxLMFR7!hAr&yK5>#_>8{PQE5Ez0a?n0ymNXy)POuFPUve^Ah0=hw|8y8bwZPlb6M z8TpET(_Tt$^DOq z(XLbWbvq&Grp?oio1dqHkbP3yb;9ax>$y1E2+B=*=g{fCU__2jeKBQbKE(V64O?iz z_SwpNpX=#~)wNN~CIHt(WK|lijGK?|mdO*zNudg9G-JkEvJRnJSt z=Kzda$%_LWrQ{S9W@c&Qb<5l&Ycqgfxx7^&oVOH;rnp#Az2_eLZtwH z1vbJ+054O`@HTsLvWiYai@#$fhmG}{-rP*gMZO!v!-3dztT~TV-=Zx!)iCtYDaOZ^ zAP*d{8zSAMeFQLoHBPX@Q_7$`tO$+bIUFX`RSlzvyy>T>;!coTuwh9(GKfSLCmgVvdEE}!n-pgM#N<}^^ zJBU+n?6JS-JB;;VS-Cx^-0A+s#cDMc1o6(5U3+%I6mCs!))|&V_wE;wxuZ|ShCkpM zpG=n1N7QP+_RCcuwL9|)zhO1Ch~!sgO%&i43eRCh(e7u~nnR)Z4=w=75pJzR35!re z!Gs0kdiM!p&)1M_#rkj_t#E51^@EOcU=m-Rowo;x-h)Dlkt}39b zZa|;Ij-%w5zhxRM2UwO$E5-{Q z`{}o8a+RmhudIXANLikwZ|<-O`bf&6UcaVBkW8-2K%w702e3Ua-=OY+tD+qSPKnb)ZkZOjd2SugO_J4X&1Z2O{u%+V<+A)Cd&B0%DJ>n!LSxB!!9RYLVweZkGsO6sB#3yI zac$4)(4nGwmC>=icgZ=WbyHp2UBs?ubwWKYt;%gT8W?y~>&2zwwkLvf3qS9wVAf2I-8_*vn8d;R|kx*|85Bwf(V`NLa?!Jo*|JvkB1&qh*H^Bz@zDex@G5Mn16O)GNT$ZChWQ zQe8QZ(h+bSfLt9w^;3&&V%PFU*T*cnMF#HesV;$wR@qt(=j(sd+*ZIcxiAO6fj|VF^zrV*w zc%HG{%1B}7{d|B;ERCpeFM-au5fYSeS?~EgT!~-TxRw5qj&EJ~s2)6!C}zj|m}KhcpO5zKPhZAM#bWvA17*2@8ndjX<lFxeS1IvqcRRja7ENH6U*+s3D>-Go_40<60 za&q9ZV2XN8t@rW!Wp{|ZNFsUsj#}-Z8L?I@NtNs7AQv{zp4e@t-}}!s=2Im?;*Tu4 zds3O+$N;1pQ=P+`yfJM1 zk~H|YA(X?BW|?r(o;znC>rXIds4%ek<6rD%6?<3eaAZLLEw<{W`feBQ19f^0qs$^ElRmh4c-H=cn>!K?h+ zmR3tE>hpT%6DA5Mb|(2m#!L90rQ@?}&B^%b8pD54)0Xu4=jT;t3+`-YY4}n_%3D6mU#AH+K&>^6vSiZc zZljs}^ZL8@%=Qbun@cGkiZVMlns=Q|8LD@-+_BHPwBJ}zxF=Oh9>L5UkGrqSf`Qwt7oiA|z& z-dHqJ>2*9IFRkW+d|6)!kbrNrISH%epD|9C-CdhLY6~qzos2kF=}jO6uoxf*$k%&q z^3~olb4jH9mEPnkWJK^kmLFGH!alo%;|WVOG~(^9S%u9F3JOE#O_YJdByw*P_bz_yeqD+<4q?9EJC0b2%d7}*HM<0FNg5-~ z;~1}Z9+N*S*zM|mv}?^~$Ze}Pmv=|*pqoqON2ob*Q@Uzig&6BRef#5vnRT(*4neh( z$%{G!ErV``?d=GFVgfj9A~aO7F;K~*!kugqB#I0Q2afX3qNj@Yp?Q2ay;$@v5+=F) zJV93WU@L&hYrpF}p#%a_j$#j3w_uG-t8p~(2TtMka4_UpYG z*ip!1q%Dxd?UcbGvH8eI;`=|fb9{qNB%M#knc%F*eAj;TXC3@mBqqczk1~>?Vc($? znPC%wFv^!WLurX_A`n=%Oqg?}q5g`NtjqdfOsTy%Byy|Dac_oORqriLkarwnmu$@r zoKr?i-U%@;BfhK>-uT?f19@sXHJ<_>s#UL0SjouRj3hewnwnJwIt5`c?P*8l z($&Y~;~~w+rijDH`wC1~#E??VIDT#E4QlE&{BD{Fi*l_ z-4zC8tv(EVW$%mO*7o0%Bu*U`R&7=auEnEiYX&qJVQHq>g5~!cD-Ub$pRFTjJqQ+_ z{P)oxaN_IVSY?xDvBvs94~Hg*HPR1Qrc*Mr)_@GCAc+{q$HG3C{Z+ERNl44hs#*l0 zxpyqF*UM`SUsoQcEkH|}CITFmdMzb;VRtQh>)bU>I*+L>qjxm;oN5iF>WV(yN9VYcn%582)W; zPRPG-nC(=GTDPqoK7;R+D2X&Kb_xErJ#10+$(83#_0H;QfNpghZz32B|LmeBg_=Cd zJAfx^ySux_#>VY#(8IQ~N*?(sv2MW|nSb&mla<3lJ5se{CjU$cZwP+>+bNa5bKE8V zT4SsGUrYWZ^ahsT*;Tmm`aAtU%b0@QclK5CwJ`ZP%qkL!0Oq%vnpzqrQQZm=nR570;=n9rfbPE_53gg_bMv{t$Pi8uCf(N+ zPm)O(S?^uNcI)RZ&(vpEr&U`pOKKCujp$li@wBt9LH@nIU~un)QhYVj$Ly;XW8jQM zYiUclz_Tx&!G`=35M(xb&(}9ZM4wo#CNtct6MnA?rl4$wn4gy(bI?B6H)52bCEb4< z^z)3}Z54bWpPYy#Os}`tks`M=L*mgR#JOX6;d!hkfDY|m58u0N$Ky7btfS{h^!zrJ z1roE97+Dc%0nyg&GOF9L%B5D*=?z&7igz0`aD%!X9AqD9Z31LbY_CvUzW0SRV(6Fw z;_h@DGVAsin>onRAzMS*-9xYpKic;U`*jPXlkcBhDX880MttbA<(b@Ut|9~PF75l+ zE9J0qkK2qY*gygUmry+C#?`m0z0~7lYB-lyicS;YOuyhs4$;+i_!Zg_G|g0%EO!6_>!f3#@thzkphPM)y|ywh-Br^F4-=(MjbM(T(`p zq7m3&q_q%rXhhZ~4zcq!ADGMy8^pfu=Dm?>L9Mi}o@$NPOm?D{%%D#55Z z&+D-S+Z^NGb(1LB9b5E{2PeP1dU-40CqXN8>T2=Nys&ey2o5M@m|)nLI^yP5SyyIt zq|djr+@+L_pX*N6-61QxU@P!5$HiiOW+g=QBA#Rtl}Es&;{Yh1-s;p~IgojIL}nHo zm=kxZdKDq`j)Z>1tfS_xy^LQz^l})7>V?1O)Gd3L1vY@`rphp171WT|k+9PN{d)9T z$)r53p5Edql^nsws63KSz;ifTO%vcWZ=;U79A9OqzH>A)Gt&D5FkyQpogVMZyln`h zBVB&@xV5rRvfZJv>m|P|0pq$k0e2JmtY`v-37V)pbQ&Q+Faz;wcQf{^sbJf?PG1r* z-;5^!)XgMt?cy9H#v#dvFD4UWAHeG5TAcRygJYwiys#;_ zZo`~H@E$l>N03@m!in2#vd7};T+tAs3^N^u+>K8|HO(BENs#YTW}_u>wBHg&)gYD& zF($gvu)VquBP`tp~aZ%S5>) z0*=~+{}e4oF4_1n0zH&?{C*Pj(7iPK@{2V-{%q1T9u#bl1z zb2pdkkO5y$PmU?YeA7qZ9F_^*prT?BtHLK_O!Yz?JOU33>tk}=14q!E5PhFiuWnLBSnLf}*x;8CscPUifVS8Hx8ph}jeZ0g7HyNM*;NBuu%&W|1V7I5d9=Iz(Yk`{8v*MH}2s2+wK_PZ`h#*G?9h+aZ+1AgHV6=zO<3 z7Y^09T*>62nweVzyf=-KovGPtX@w=)KS4@oC_%9axy-7T`&Z!gTYvXWdlHuQBV1C8 z8`!TENt3Y6?cV0Y<}}hwEe(9U4v&x5{rzE;rSEg6$a`gLsGTg-jP89^Sp)K|SCd;f z*+n&rzR-K{F2jmj;u0U~dgrlhFEd^FF7}-ORgJ*)S}S)iTs;f`I&Do1?N~Ah?L(6N zrm5SuLpB(o$HQXU`y*Eb9?&;>b8;XC=u!|o2chD2HCJ*Fc2jpy^Z@G!x3=}&=e@Yn z*4(*y$tML)46V2LV8G6uNhu5>B6>j9P0-iBv8)zTX8sD1*C)V@n z(K?rOe$@yM#Fhk(Ot2JjO)*mGDtwzq0}Lq^jycav%2lG4^JeY|IDR%+R(FDITcNWG zE}HEhT1IlcJ^E~>Ce+^#+aRW__h@3)N)%Qtra~(GY=d2^jPH)Hse+K3+!329`z8E_ z^17UkAc`hW4yb?>3lzCTfN7CxHcH0R!{#3HrT5*hrV-WYG49+g51k3Rc?%c7?r5ErjnEBvsJS-c&u70yM137zj;LF@W_PPs{{!FB-TIg!-q z?fwg+**#~sUVbi~Kwm6vm*%J@s*~#~5?yhQwhu{NMYUTx4=B1Zm!E@*tirrlPTv}A z%w4C+2FL=N2S2=wK63>c@TpwV0NQS80UvsT!>&-T9%nTxl8@8X**qg+*GqNVqtE*o zIUdP}U*g-6x zh-KP)?desfO|L*mJ|QFijUoBL3Sxj6BbuX2osQ98SwHO83z&G!qBKt%<{(82NAV;K zTI)%#qi*_FxD1>|?}F|-p{}KSg3<4s?O*Bq;p(k<1=wJ73p-_tAjCs+Rp7dZ?Xm$> zH~zZvXjN&=urq3|fR-P@Gwp|8~O{9x>Xoi%c6KpUlEI~>mDt6)6#DJN7@ z>plbk-Iq06D7n>$jb_hPa>%||2w#e3b5TTvmfOA7LR$3hL;r}I)BKl%i2Z<8+$XPJ z(*DK|oJMSV`?*gKSIifIhq4@QJPIwD`CpzD`Yn6k4tm&Xxc z<(rCnnCsNr39W&<4juF+@S6AC+MgsPtvbk%vipRVQ7_;f0!-P*wx7MlU zb^+$>BIEqGP6>Bz$4B&VCb5?hd}M8Bd~Mn7qY58IKP=ErX@{7SWxXTcczAQy7|x}2 zFpNPcHiHO&AqfNV(wj5gQ3%H*NViW`1kJ5|%x=dH zEn(}+UKPAG63s+=W|KVIgJvZ02?{`@)CR(hs%t!nxqsDU z?sPxj%-23eAD>KX^ny+VenYoMu&hMxmIXq*q(qL}A3QFJg-YN7M$U5z`sO|5{Nx5D z(#}+wA!&RSejx95QA*aM@eX)EBCM>##IDHeGbHZbZ00iFW>5KLskwDo)5>H6<{4`J zLRyppIi!K%WIHa!HFgDV+r13@>AOowDfbuns`SJ-&79YdN0FI-@Vu$6Mx`PyRHQk3 zbW7P2ToDZ)*&08+InI4hSCpHX&T5&RKm$1ac3&Tl9ZS>uq2P5P{cTNT^<~!Pr^_F9 zzdD?K_R_f4mQ_2_3uo3$ODonE!0j#!Cp9kk{RYoC2UJj~L3puJe!-#zuU%sNWZCP3 ziqmcd<(htRgpYw?iP&7=wSgBK(NR>s#m3eG%a^we1Sbs5B~LUjf7!bA=bgDN zn%jj?a%DA76(n>``Ch{1)OkVhR&<6pTULAR@|0?R{OYqLp=NJUOWRRx?c2!7)bB#S zY`$3bsZv(&^qjz{xv|Qr8KPdWK&JOkHTU>l*`1W9k+w|rIY~gm-f01RN{)DRqVd#1 z%80a#`#SDuPcUQx47z!4%e~m%i5pr8v7EP?_I%4b4(mLrUQehx8k`@~Mn-B0^siiP z8oB-*CFg+sExL-`Me&!D8YZS2xwA~1B(i%MzlD+ElDb~a6^*{=M8U(dv7?)E$usR5 zo{rnSJqhiA<#$lB4jecT6BRD&-hR90Q$T)vk^ZNFRM?q3OmIz-p6}c(pesK7(e*L| zeZ1sZmzaxn{rp^P{C~oIOC-=7q3QULv@fj=@`H&1$Kp=;_NM#ZCqFOSm&2R>0zsRg z|0H9va^s0l){ql-JuKhvzwk5(ji>FwkR!AZXuaC7It zOYp=|47U3q(En#kP<}u_Vxc?m$`>XnH;P@U7F4{w?7xTt-t&qahAG&#F+RWh&3{f* z{r54AoSsVUZ$ zk&KX@8^1j3y&XOtArl#dBU0la+4-}AkR?lTCu@ycCo=tu)^0H@HHJi!4~;!2NMi|v zx=q(vZI1qDT&iU9!(>$PE?KuO8ZDyvA~JXCS=V|Rqb_DX?FA3|U*d1qx4}F$M7-}-M>z5>7oBex_Sz{7R)L9#P4Md)DWQE_4f*9#hUVjI0@d9Y75#ko4Y+}{%U z;UqB>h7utgWaP21IU9x@ZkyedGyC`Cdf@Eg6$8ZG_yGfquw-HDfF z=Bhex%M!yjzpo^!@jG6wb)U@&foQtAx3eOV3G=zXr^eh5Iplfw*YXapO(aKY$H{Ee z-HvP@Bj<@G!X#IGjpdwc;SX7Ny^=QObiL0Eogb5jh(?)xn&Ver zt`~bdV!x8iY-DC~vt9G%4=86GtB7m1?f;P1-(F!PL%HkDcVZ{}wr)inX_a`L`Ho09F z(f`;AIG9>F5g$I+hFF)~ z!r$1XXD0Aej4(ZAZw0-wwF_*U8MS5PB$;?Js+{OZe_TX{sgAl2$YZ<4Q3n&_!-T4HXeM!lfRQc&Cq3q5iV{b(8=#XIQ-N}D~UpIl#q_#f4mrUIwZaj+DDJ|g9HgY_;*`%>3c$@s2 zLwuK+M-F^)_ECeQ7yadjQic6!7)(mQer;tcazi}d)H-s z%apDS$b#ubJ7PYZqSIM%_ePf{YfVUa0(sc1CwyKF;Nh&|*h0$r@mq2y_ZT_Pv0>Ji zT?LP?E_?3G{vikm3SU7Whi%)@mwTazRwm0gGDeTL`ZOOH%Ex453rm-j7Z}D9l3S(v z)~@f5k|i4FvWh1A7k?q;qlfz-0(f~b%NbbO&K6vV03T$`CS9Wk3QBZL%kDR~d_H8) zsCc|D?|WMW$6)`*jEf|!_$~i*g>W(6S{*J!^2u2pgh*JTpT)UPjN*cBENU2fsAz@K zEozA6Iu>IF3=R4ei00bTrp3bo$*DCiH0$qh14NBf$i@!KK!uw^gLYE77Qi4XjJO-; zTY8XZP{2?&JC7`jYQ1i+@COD|({S@=GZ4@DkZkF8=j5!BdBo7JaIvT0*#>A?C>fCz zC|>U2n>hNSk5@NGJ8fc8&hbgoM=K%5w)Ojz*s>9upF=J#-Yn+1rw8 z{5G08u*28)gnkqiMJAq6%eCj`-{NZFc}~Gu=mORs5QLRMMD_|wY1?1j<~g^OmUk*~ zORI{|kMX`7bi@SA_Cv9P2duMP&Em1FC0MFUOPa|EO-rpzjq*#HQ5FFC`FP?z51(#j zQ$a)L-e39d!c1#Jh#C*Yxt%{aE$+A>+l)KEn$jbgE%YvGAO)9#r1qdv9~R!m&npFY zYJUJV{epRSxqQW}%Y}(P1An?>Ei(%tFexbZO3$a9WI|(2`O#J(|C?E0>e4IZ_3yNH zM&d?hY=YpOiTZ-Mcg!GNB${*c^v9lGb!2@ql21tq6+U}R$s--G2||8a;F?9iT?e9z zYPC`#`G<^ANr-#xltU?fTu71eGPx`?xkpe^#nY+4-XjbDgMq@NQT5(+K?w)IXDe2m z;pKYsVM;26rhArzW_QdmdPx)RIR&k}{5DjIa{h%1ijm;)%PE!sn^AL^%?ImPO`e|1 z8Cj>aeWu0=hPdTWV`bB6drdkdC@~PFr_zj{MmML+cY-NnUH2t|q>cR8{~(77anjGb zmXyWbQoW1E??h7j!>36&p|OX9A0c^cOL>J`42b`A^LiEB+<$9$qu(q#b5OxN$_l)+ z5+PPq&xKdwGiz>83}-DQ*kdx>IGj7HxHZv+2v=eVWZiFm7gvaGdkLFCF)>FLSJ{rc-3Qb<&IYM#iPH7>Pxdlr; z$0NmxKwMU%$3f|!{@rD`-}A~jKSGaD?U~YXtga(g z=-1-#FW=>f6NbfM-%L`*>|i!KBbFIZo$9b%{j`EF@=M|Z$#uDLkDL3_DV1M_U5gbG zx|(*TY)0Mto%3yCv4r5J@#Cs$_uIHNoKW&VTZbTo6^)*#0{{2~0U3sP#C>fT0I%j* z168aL&Pb?ujW|ZAHlaa&#Euy}lFP5Em^c6Ql8aXNSz{zs*@^sR=-|3?SFhxs82P&9 z2Ih;cTsX?wN#pjEehOwD-8ZnLpb8+18ZJ9yx_mYyoh_fI-d&ddznJ?9ptic`Px{qM zp#_Rt30AyCi>58Xt++#RhvFJ&OL3P#ahKrk(o)i+LIj{h~tjvAC$Fvi9yruL?=- z4~Uwu0LY`u&DFug0ylf{#`ES3Jbd4+5?k4#;}5HY@eDvDYR?bG&7J9a$5E*|yqv{3 zyy|7$j)#sapr}cSx(23V)28;5E^hb418a+S!+t(g@@FK?Y?#@UVk}2T+TDo+ku{HB zBQ_3oLjLn24&h`~*8P-Si~FAzeoCkmkbn7H2^m=QMw`oGjyxyM zE^dpa-Fy$T&2l=ywcxv7uTKQWQY?0vOo`)_wm{ke@iM40z}EP`smUsA6FOuk84GnS zt}iKg)%)srIQBiP{{H8K_wRAAo=GjZyNACIC3GL$I$sdLC|)=v1kBSLe6Y(&oGE-8 zBKGI#S=j*}e!}-98zV(UO{nUWgaZv;+m`I=y@8`L!`2S`u9!3oQ*lGs;<=d2o>3Ot zF%j%RFy=`JI+-z8@g${8Us{#DMcMvsD~U_>eUg#fS;V!@7hd@)i%eG|@+hQrLO<@8 z0PCvP=VPCjcY}U?ICquY+K4N__}6<+Ru^R#D6foGqaTW6>~hTHZB?6ow!s~ZGS!iSw(F+8{0(Qog0_D&sJF@ znssua`shSSg%8&SR$57!`Q6y>oOj}hQ_QNHjVf%1LhjDo+X6IQ$}Yjrp0R^m?@76M z7A+0~VJ}fSHNu>}Qddk;W@}^n*Gebf&mmkZIkwaLx#p3SwzE)OU=b|*QK#dydYKwAb1ncbhf;}Vl&8*>K>ux+CmgTA39-6Mlq&Y}eWWE7MV zPYGDA&05%6&rHtRZkq?RRapMS{h87`d(}newI(U_-7+5dovORNCX;x*Ma=hdL5<^b zPwVWtgxsXPu<1KWk+3TVf6uhDS6VEkID8UCN2wMhokR3x(+dRBKOiW>hDaq z&o0tyQ^tw%NR-=9i+$8}B!Ka^73*XxsjTP1+vP!3-r}FBcE83Hm%qD7Z#i8M)Q_vOBAxle_mCI;N1u58F*1}^ap(_S6F_-*%c$5JE}%*o$dKIsQHvG zF(~SS>ZT0aKqEj5^7%cbpXBSrbhp>!;sdi)(Wx7U5DdLPg4e@itdp+G+^#y+I*-5B z(n#hhqxsAm;h>t$c9Nb<3Pm!xr$qe_d(n5XcYZ0pLtJ#(x%hXy35sk=jS!h$sW?9jn~~IXK^}NJh-|OW1GDC?t_;23VdG%bew4(0d~T@aY-K> z&LKVQV9@oy=&%lZ;G5mZ(ivPqS0Oq$)LoXk^rtoretIQi(UAQF>UIoOnmqZUYGl8t zDMi9~3~vg*Kic5o=oKx_9Gra}Qi(4jzoJqqdN=!QS!TeU@%{v)qgF^#jst>(VNso|Bxp>UL79YLe2of zCxu%IYT3lSg-NFD1-iGdykVxsb9C?UOaEg58r4(2M5M~ZP>;%2#5Y9m{J7jqy0|&l z{Bg`%9T8Yn>+BL_rCHVI)v%@XQ8LhBce?)xFbD^9Y*_~DIo$NdmQ@0H%O%%fK1UlH z?TwZBnhVF&%qho^M;G0rO--Nxl8;L4JyZ3hxw`J>cc(O!G!lNm=>6zH^5oBseO%?q z@sHbR`s>CS=MXN*h&L72qq7>8MG*-^c6QmMgoo=!gb8~FHKJRGha=kH%Rd^?F44k@ z=L#g02PeO&9~3a$R5s^eLO(Mx*W%E*4C%CRl?sV)@$n%Yz``~AP4pIrND47- z>(O~Y3Md6XI`Rz%mwfh;sOXtV)YW;@mGs4T&$vOUCrQVjU4{dwG2U079nInc^&e{{mW)~^$si-|nOHTf5o|b*0funZD-#;K?pnE#L znq;J{&Q+VH;P&pqZ(N_V>!iFZ*!lS~dzPsl7))pD7;mFVI~F9Znkdh9oXqi%S&YHl zQO;iodgY}n9$TI*+qNDO=wu* z=NI*#V_}itcgJi5{C>)Utd~0G1q!TTo6wK+lOBhi?gv};y6!}t_Uqs&(hs?3*B1C4 z>@Ll}8Kd$tPP8Jw=~3pEIEfq2HtiIn8*-nlF3i4goZG#jjPe>16)r7w;krNItVn_8 z2&^go<3WT^*@+|Itm=Fi!aE?`Q(~~;Pa?xiQ0a^i6UJD5h2(Gh8hIv<&$y}gaa2I< zixmpG=)2|7X837v)l}hm0geG(DCc%~0JZZbkM;C#88HaNtaU0~YFYvac^iB7VRT!< z2K7SDv_7c(g+?&zltSZ3D5M@be{Z7(YGB)prrQJk3oFrq=>G(|ZA9 z$5|BdQ$tTg!w=3?dJa-T;-vTFLj6-o6;385^;l>6qisP$Pr7xSaXR8}Q)h^w77^QS6=qUGDoCutH8Y253r zUR%LOdC?ARN0Qa67A}ZD=Le>C7PAA_F>$LtD!nk)-}aQ(ogYIsLl{(zn`w3>>sHgi zv4r}rgh)PEd{W1C05;Z__x**xX6L$Q=H}*fb#+rxQYa}YX+3ObA9^*jQ=B@M-)wKg zg}dJ2GUrBveJ0(Jzez{nq&tT#*94!1~hRyehR@ zx}JJ_frWbgA9oqMWB%Syh-AETdy@K6;h)>1(EC(ilUpn-w{Jp^Z;yZOx#Qm+7as4^ z-5!4u{6BxeqGzQYmhW7uZ#+uy6EBa;>^3=R!4m|1aCe9QaT1gkB|C!W4H})Db*Qwv zN3{?99UL4Tt=@zJ;t`#%kV*Yr3QIp_XJ_Zw<=P*}tCJ7;-_zPAqNx3Et7wtWBFlOKH-oV4B1u8ho6yN@g`?QunA3}sC0%vU6nkeL$kyY3{UfIyCp;--2P zO;l~+wy9^Q#!6@Og?@Tr%SzHm19>^S5ZWU`eV6|3o_K2i}oKXe*5sjC45 zQPC_{*pjmVmK{e?vo?$nFAwRkzns~S-+wq#bHFTf%oCB66XZ0rxjE#bj+?Z0J=Zx3 zHT!0!*+6K)I%gIed#GYHc3u%Jx;_JeTxR9vbdM--=sCXS|8%<2sNA;AmbAP!8?`Xy zdlw7q>F}qa7@N|YY-K9!%U?FaD2JY}Xd(_*8-pNs;sWCW7pWoxj3=>?$ryqEG6>Xn zLTABeoHpa6*XYvIiIK@}`gDe}(`Od6td28wyRq(5MW|~x$mljZTU3niXj+k!n_5)F zu5pMsts`=JloDHOj{CfK?7|vYWO%zB3%x=RwX%AImf&O*UEzncMEf(pyV0!~XZ6nZ z{0}{3f}$%y_G+dR|+Df)zFU;d|=J$g&cRha5l=A^Ss7!d3&l{8-}y3dm4 zPRD=xfcn7nM+^*Z41i@)l4+xtiUvI}CNieV04c_!&WK*){i5{L-N~>Po9T&YQEK&u z$DT*kIA8JL(qKm4ofg{c^(h0Xgv~4#fSMtJ-@_|h9nLEwN~Z0WH_mvYxEisC-lkaF zpe9t#dPaKB5$t$&t=gH&rlAp00IelpMa-ZjO5V-+9z~u`T78|SS(>F)N|HCsHquQ= z4Acsz^1Yy~@CsO?9G&BO;vT3j27Ufwdj=Vs>iaKh%adC-JBABm-U@tj9% zjtJ?+e4@IzNDG5nZw;$gxe8>(mkm^5R&H>ascKc}p;t9drCH2z{E8#GYMR`=+paQy z;Ts`_2b56Qw(RdrS=kCu@2JK}{ieV1O_Dlf%`Tr953jY%Kxutg;@ZFR3|KCwLG$sF^l zt)Eui(g^M=Tn**laBox7j7}kj$9Z4;&gSx*6}Z_{#EWXnB)A@_;WRQ5ld_QXxI7aN z8i(|*5`^caUR%404W+T?l@Z6qCWvUsr#6w0z?eWs>*RN{xItleTU*ZyixKLb5h6BE zpB}}(3@g@wQ-qQfdZm?+^fAvY=k`;Pudq?F;S|@%$Ig@$@E^JxEfjN^LN27$wX^Dt zHhc)~VEv4@`@GiEXCr2!$28_edkJB66lx@sca7z_FlM-@t6~$ZTAzpWm$K?_2#}~U zF-o5?=HTZMMr{%Bk?#NWGilC&B%R%{ceV+0WhCl%cF%&M zwi3OflB2vG#2_nkmyWyB^G_qDggVxTw-Q}flZp(GHI=T#RpTJoNHOBvcxtWjxS=ZC zVXDk78sFY4eSiU3m?*3Q7!#*UdPe5f#UQwZdN0tLj^ZJW z@5|V;v{}%Jmv_5@C5=>}+RJB|bG;~wG@_K+V*-Xz4aKU)5A7)4mX?a_KcTA(m2D7a z84(+~ZfR;3C4IFDPyNX%&BV@#5cw*ILZ5W4-ueE7+U`&7Z%twlue-D!Jwd?FFW7;inTu75p=$2rZCS+9MK@F;7#sBjBwK%xhY)3Ca7 zt36$Gy3h7Tr?xI^3>4segFx~%O^EDn^WImyx`l2XxNPsPvUT62YspCa`KVh-rzcIGd7@ekHPl_YC%coEs`)YkH z)N8n5Yv?)Qo87ak1J+V<7vZagc^?No%`S(_v*^P-l!y0tk}C$Lw-pl;kKxc#t2rxa zCiSF_AK;=U4&aZm3-uV;*Mii5ZE=ztP&@i@tdBRRyBTp)HMqT#lbNOn&xU<)iShOn zdaf}35j9uGZ!jFnL|02aVzrI4W6wwMNme2s&rL8Y@Nqv7hXhpxhBniV4TMTRG|KYF zh-fB|S@d|;j1+@FO72USWo5%^c>J130k7?!h!G2iR2otNOL-Mv8*3X4rL|Mms6<95zi5|?;5l+ym4FJgB(k;|dRTaN7%clFnI)bL&q=)BH>+7pAj?6V zDAeT=<~rHt9E2^FU41luh7eW1%$q_zako@G%{Bnx>y@$3@-n!*V!0VR*gw=RF;L20 z{5jw5Cjjg)3T$;2NHKSrHKx$gEDMRl_@Gl5)msi^Bg~oE4;aND{-~o?d>^-k_GtU- zv=+Y0lK3KDM(DRLRP8pOD~CV}`leWGZ<=GDT8=c>Sz6{3HfcPM^elo=0>x`qm3(;` z8ptUz(y~xc{n|Xuh$+jCTwPtoL4XEYH69NGf(RY0V+hencVld^OIYeboX6pZB0Jp{ zO;+ZIf8x|k+O46QJ&KQ`MjjuFP?N)h>{l420efoN&*U2*FCPDc^~J94R}!P({Ds_l&|{II!TtDQgnsk% z^}OZog;6sFS!}DL+ghKDgs{Ci1W|y|jPn&XNnyXl+l+ubk*L%89&+)9rUvwd{BiLo^dn4sRoIB(<8a;c0V$?n;A3uTCbpoMl zA2yGTfdR8bI{MzLP2Z_G&4gaWOOOG)l2chs$K{E=;gEIn4lzKQ9`$1&dG}|%L{$ig z2Oi~6A>VG2qFO$X7F(9xZ(p*k>$^lt9udbAE?4gbNaFXR*HKh`HyT`Jk`W=nb}JeK zfk4JDJtF(oTHx1jZVx+9iJupLbIGa078ij9ri(AHLX8vOxd(@LKlmfW7F$CIJ#BWR zi;u1&QP_`G?NRa&=E<3yvQm?pin}C+EoBF0m#AGM4UU(^*3YwlSPo6Rl z5p+gA7z7bnZ+NOeEug32^Zr&86@z0yqb<8K(5h1HIl&>!Ri2q%l?s7!90qEN98=iQ z+AO1#>^XQgM-Rg0qBEHa*JOd!$9lp#nLwatO;vCQQ~Nb4Al7g^gB`$ZqG&;vP)GszD%<6>ZCJ#U zpd_^C{Q7=HZN1aqY*3mR-X`Dh_K9I39(48V_d zYX5PZA=V?HqT*fjZxpi9s5~A11~Z()&E5qj(UviEg<_UNlnTkFa498VxSNwu$@R?d z`cuT#y@uPwEe^{P6^7PAB)uKi1p2OH9DhS&kK7#8=mw)%WMU)T7useLqp;OG z0>|3>zAE0g(Ifu6p===n-6U5`V?`Yf-}HN+&mQ5y0S*q(6ENpby~>^}h}~3ZHJS0+ zzf8c#i^4?Mwe)dH?F&qo{JWb`{yNYj(L|yg@WFPfTL1*;Nloj1YL+&vS`CPj8 z*-{7l>Qdgz%N{ECQffvrUYL^NpaJ#2mMl5*UEBl+A6G^2_(A3t9sW4TBn3(u>E?vi zAgL6Xtr`X^sLSnMf#T8fX#)EZM!IvL`Rk}GWhJ9o-GNyR=Z1=^qAVGRmu~>00-1z8 zfS4S=kZkmikvUZ(uW$a&JKDN2V6ZJ>!|nsL0+?B2 zlt9nsj+z1iaKL3mPg}9V`e4kzVZB;0x?=n6MLFA2!-A`rCp9r16Drmn83CcY>grpC zh0z=8kEz-{4x3>U)!Us;yFZP5%5)u@C~48Uqu8n(aECB~Q-djP`$U`1gjUE!YU zld^Hm1o?+5d$mJ>yEpZ~s;4$d-M79mq87-{(b&)+ASfs(ARrEDq<-ONS3_Sobc}k` z%%Tzh3PemkZFd~z#U&p_M$aX0{j_!7C}U)A~e2TML)U|CvSU0v@&y#~+`)ot)! zu&wOV&0oy!r+WDRDf;#oJljZuYv{i^+2qWmz@;3Hhip_OUdsbeH+As%fbujcjys`L zz1Gp4t8mUrD;ZUX8g@~P!_GzYs>85vBaq+8p7&R^V-`(mzPLMSd_tj6puM_Yl=0f2 zqX+XcJM@bCqB040!<9R?zN)-A$BThI@2zfsaq3V>a!#oNQY>zTFz5(ZmwZK)t{&Qz ziyRt07ls*L4LYpb`2+Z_!$C93r3sw}EO)*n`3VJiCp>G0gGWD|eSn>p?F_!gvnVI%1Gz*FtySh456U2h|_k$l?7J)L$ zb^2X=8uytypKi$Fh4HFCb$^g0Zq=G6VrZ;8 z9efnQaz*EnG#QPDs@Weix^R_Ems-Dmo)2ziI zu>qqBDm_lERmRbIuSKP#UJvovLuzv#5DvYMw<9X!@iPVG`g$dWTmqPBD4<_QY{dKrQ7*IeTd58cntt#$piLj6IG`=;!WacNi)b|e6WO!p=X?gZ6rsoAJyg5P}_rt$WE!VD& zskLyE?eAMcATDdFTA&8FMr)Q%bfsgpe61zBYfmQ8Uc!|y;NZY&VOWtm?3OU;!+CX9SE0C+_>8&*aNU}O<0_Gz-S2=rtal;8BcjWj(Gq(Q%NRy) zQqZrXn^w(jVr=TKC3_@SASoyA->J3lK(d?oW%8Y|2kh)zhtqj%6R_l{TO zD{kTBR?fqVD*D`9{b?gINABSvecYgpD1h5$iC9GgZXV(JH-u;zkZ#uta+zKV>}(|? z&sCk)iT*yA?t?6Vl7`&d=CXY$W>*y4(>h_lNlXEm>rNH|^VOO?PCN(T_MhnzsgTwZ zU8~lQ`vJ%h?U{cvW4LzvBp*d%AB7dXMwgx!q^I}gYQ33I=j+O5IFXDD7m!QyHh4ms zGT(KoD$yBC2Ai^e%vP}SLe)KiZI4MpwdcTjn%npKixl5)Cn>yur2T5Ae5Nz(K4dx+`)>Qpks z4Gd#%A)%+K|6YuXqY_m5id~rngb#~?<#PzVe$O|!U(!>NTPPY@!SD)cObE;9OZixY zvS(sLa%ymuzA9BLaeS``s4XL<;FyM|&cGH9kYnpl^4)5#&hjaMevJI2DcMl|s z2Q20o!UwcO)ltj37Y1o3=KACar+3U-l>6D@Nrm<$Sps3M)%xRlbyUxd#FQNx#M?C# z2@Cp^X=Gc zb}se8Hvow(Rrc&bQCs$p*F?pSNK^-yvLFa*3+e7)2R(}JU++$&>kYOZsfHaLR5|D{ zyI@`T9d_IXDR9yY7MDwoI!u5%m3i!qsz8MJWN@{%LUbo1qAyk2uoci=U*3Nc3c4OA z>4SFBNnFUp*I)h#3RkR*h8TZPnv^$kcT{rt=y+CcsXFTV8`Aba3fzd0vC~}}Q;q-j z;RCBa;6IjVd#GaN0A5fwVD@g+fh8ol^jcVj=v_={CcM_qgY&(RGWKU5jm#^p9Y+8O zan#*n<1>@};mk1Jcvee+^dT$RzJZD0kw{l*I%XwRRfFXC7wRgC6sBr9o8wD05VM$g zb-GOH7RK^@(ST-dfS-@j)hx24f>{b)2aAvG#;QEbeNf4qIb@f+ud|m>FA!$ZCl?kt z@(H>)Ilj$ZI?5ojc}@k`ax8sPAfkb*=wS9@CYU26Lh{|=N+W+9*L6Y8fAV`&BVpK^km(%9Znrq*Z6!eVF42(-hLdm;~F{hVRk zOu*>a)D@ZPm2CI2SV|8f#Nsy4W6n|Zw~qf6>)=}_tFQn6U?zGvdvT}k%}y70Lyi3$ zx{XP>o)pW1y2kSH=Nj{MjYrP>BZxH-CY1+80>vg;DzrdjHn#+-lgn&~H;-M4oyGVn zg~R-X`tachrj&w}{_nk@yz0;Gi=A^$Gkw8DHAC&IwAFg~&kD+oR#&7dXHsFsOsart z3yhve=frsfUThd<^DY^owUN}RR{8c<_;-W8u}+gUhLm2G0*4>3;ZGS&uWv1odhUNW z1Md0Pa~y*1jhWXU0)n1yt+v7czy9R^V~_Tq<|)1}8SB<2&KSQ2nfOF4tx{)KSI=Qz zU?H$j_T|r?;mIHQ?ceeA&na(x=cil~;;V`O(13!efT+q;N&nY1Dal06^^cgf<_83?|hV zj=NXN?XyRN7ZsZ`QyUx@OI%dy{nI65^s>5IOB+N8FN{if)*c)Y4AouvSJiYEfX9EBCr zfZ{_AZoH1Z-ZWySo<|+KD9dR3Kp|$r0*)9f&ry#s#}UoK8Z%ADWgMofiz&EPbzpbL z0kJqvL^YMdz*M5g?-U=uK{YkL+A49#_xh)fn1KyNT03L*)u~=f&|kixd%7s(e)It| zkv}pb0s<`&ldds>s`C^s#3cM)1ze7MdtbQ{2zm|_01*k;;Liv1r()n__dh4SSV-ZJ z_z3+fr`*^Q?WWlbsjZZ|O}fgMgeSj)J075seHUbUfFte^&D@ph zDgMTRkl_1&PHm#c%un6umYpkjD_kCoo6CHr0ADoHfIZ!p*}!_7f=t=1xRyY~Lz+I# zGI6n1qw(@Fea%S0;Ic|3@1R<)#C)TZjaq@RP4b)!n24Hn$koj*0{>-96**$I)dhV8 z8g-=De(AcY6cY%6(?0O-b>?yLgf3JJ2iM(`1nIkp!2WPU+|ox#K2eDzE6k$ ztEPM$i~(hm`M3@iG~S%mEuLtnUMnwjiuHIe5vLpvDda#;mZH89dsn~LbF-|`f*3mL zAeT3%)NPKYe&B^yzGCY;maLYgq+b^|mb>u$YK>}8X*bAGLsFeLFHaBQcCN4nM^)t< zPu5R9r=B^{yzTz$4L8&_`6*|9rxkS~LtY{K)Uz|OyepaGm4brlug8jdWg=kv#J(2F za~2{I~QiqKEPHb91N-o2=Ob$q?M$k z@Dy*sPS~|d8dv)>B_19um3;mE4`gL)V(n|1gKpTI9xl*=p9_4k=omRwGdfe1{4`0= zqxG)R&RJ@93~?d*_h%6&2Y7Gd<0&I3_Wl5Q(EtHVVV!G%Xqxdb=+zWySU(4FF#ejI z?(Ug!B@+wtDYbSR=)Pfw!BDeSZBn?#kNKG@Hq&1Sji||FYGjFK(`6l}4x)OHzwUcL z0B>y-B7qE0W~2*D&bj{N^~qi9f0eEP4j>5CYYXHvxN)l0wIQg$&684X&IyGA`Aba1 zOt=b9O>i3X@ijF&5_h)}+a!aMiKl3`PPos7+vS)nK+LsbhuA)aYig{c?T7_eey%Jx zA_XjckoH#SU{$pz4(Y#)FDL>(KCNkSVIh5LXp~uC#SUm-uAf&QAS>JTiT_ng+EESoONuXt6G0} zpTx4W0-COe$Lu&$xesgvFW240qPRyPd{H#H9 z+x&~If{ekzE$Xzu&`=_D7N!yLP`?~7YDQgjfs%W_6y($Rk=m7GW`lE*KPUtlzP`Ri zs-Dt_x?R(?$ir*pUqk`AJR+7#(6fZW-dHDA;KP*JKEO2s%D<-^d2n#Ts7}Bxvfmv3 z0$if^fyTf9YgINJITS_+e9MPVseA7}Zc@HfCgC(FqR7 zpO&ku{(+t6^6Dz#Th^CCLfpIz!E}EgPy`;}qgS2ZZoD0V&oSjx&u<F9Ow2K%AJ! z{Eo|U^&HXds}|6%x_5?a0iy1BWD0>$2eusry7Lh^kC>mBSY2LT#$e{`prawGF1KG`QGL2yc>vY@e-zSp zI=@Z+w@sy@b3Rv#{lRIjdi6lKo^j;liKX$-V9rB$;>#6oGhKefcN0U^zVH`z)6X zKYjIOnL{sd-P5s~=c+F?eQvoxH<0~oUTU|6a}W9(d6>NOZ&>9|$92SI!Dd$G2)WPP zL2-?vY8Nj!wV_MdzBq%dOJ&<$J~K#ad8c87jRizFpaS8j{`Da`RfR3^Dm)N zRa#`(z}B9%-0qe^>Yv9RdTNJu^>OX6>^&*bxYX!-vs~^;U3aSWOZ~}+j? zm9*GF%&LDX6@;nC365@fS6L)Hq-Er;H<$VE+_~5%&#Py44H!mSw|Z@PhBz(#is*xy3GQonSl>`Z=1Dl?Ykn6)veHigp z_aBtq<%z-Xe_{h)fm&zwyQtZ&i1FMV*V*xyuDaHTnh%39?Z7Pd6-R8@)(?K4Kiwy5 z{i*aIGnT@TXH?hM-*yALtF3H&hIgAX0swa!))#UDhoPeKQd`cey)Z{BZxg|{4w%X} zeN=)6?PIw%=U<&TFY~~gEiP;p=t3!NebsQMalWVIf}NWmDoxKL0%ZXHYIAIDa;Z_` zq1ELt8D~rqnY_pm3O6gERhV{2$G?06p0(@q43|BM_pZA{#Ug7wavkW{cr+nOb}4mG zw0Jqg@7;m4brKO+|6h&cI|UH5I#q;xZ`hUcz_?u$)vR^?O#8>SYZDr5o5sHf6&;fh zldg7Ly70F$8w?J#JWr|Nm1a|cOQmWz@9YO$SReHyadu9O91W!PH<+1zS}0qQT7Kr^ zzzqTwnfZRfy2l693>Bf~N&n^ir8_|JZ|Xj5Aoe4vc%~?5gsYq^MbyQ{$?xAEJQ~#` zpjT&RsayGlm6Y`+YyBH(!c#MQ=<`>%t@G+jsM(Ydx~lwy#v0G3zZvx|A1v2_f2=t? z?ZR+PggDt=?A5QvF{yT4NwhmBmg;z4BWWs41zJ2;-vyoP6ID&<)p=yAFd5-aoErnS zc2p@QpbG-?@y=Yk^t+E5W3HqZ6;fR30tIV^QqSR){whUu@cWFbQW0SC5WT-&-RY+g_f4KjU7{m zsGpIJ6Zg?^4+x*Ez~ju%Ol*b2ZnxeOaX9=DW&<)Y&$m3G@@#X_sgdviz|?s4ckhNN zU2!kp-nLOE9uSwD7j%p*b{m?>RRd{Fr&?bDUi|HIPOximXBjoQRTyqvSYk1K0cLML ziwHzpHX{z|0#Eb5Wl`(yceoNiT|A5Nquz0y z;bA^!yRC43AA^e|AIEK-qKoiANoTp@;*UM&CAV#O5HY_0yT&~<+*EeO5oD#)W3|m)dPLFbHaMyNxz>yT5JzRm8SGPv4csol3&f2MLzhyo817h+_ zD(5A?*)gj&289$P!>wRo=*fJ4KBn@aL-M+Iy4Z7DEIy2Hx` zsKfv;|CUCYiryrV)76;*^~FNg-I}m>vw#&CydF#@zBxECJs`vj0;D{|=K=|6R%^k) zR}%BH^)i}G68tixfMW)_RjM~>%&RZ;ujOEOC%moX{Noa#SEZf7qcWT+%0TqL5Yua? zD8`3N4g1+A>aNtcrM}p`{+~|{yjV`}RZ=N{OHOD(e{KG$8VN8535Dxz{yZ3H2-Xp94`K5O~ z{knT{ZCF^Mz~^=(T=sACk&=Wc zVPV|`MCVT2qua@opjo*gaCrP#QqvI&3%C9Da|gzP3(U>_%}(T&yglct6SF=K{$uB0 zR)#O$7Dtl~xr4{}dr7TKg{7LgIIp@`4s#91lsl!4QCSI7)=bp!s#;xa1Zm|=g) zkAES@%MXOyiK0}~NI2Pz5d8kUEtUy&f_6_~tlx1q8TYC~$k9sSKmYKE=KJulDE0P} zm)~zM4qoTe+@9blh6+t#ewEz4E%2PvcWrr@Yts4M?ZpSvPe{hU&pr2hk8^v&Jg_a@ z|MqEah`~qE+cIRC|K&X-<9{phfA!Sz@6)0$jJ}le4USCvi+u@KdBwx8DtFh*`L@ms z1LK$9riH$aIVsGziyYg=Jyn9!sA{E1ixzNaODn64dBKZl>Wda->HT^CmghY`8N63c zV_P~vfidl%QzZk+PfU~32RDcl!V%GA7&_b-6$^N~i$cbEOWk4wne&b6gTD=S+bm7z zorhJ@^M0ln3RbZQW+soeRinGqjiR;Vo*D>~>#s2CnB=_Lpo1fb3VXtGdUaqW;DXy) zX}FsXWj-c>=Pml8_b3F4^DC~=txvqK8fUj{DZUZ?Hd3fam(UuP3X7xrN_PkAS~)`i zk&Zdgx~=yS4hIx0jwzSg(WWAmC>ReYebMR+LT-!LT7THErTlFY$t-8@dmceH0b|fD zQ93EZ*?SwX#~ZQPmX0ZzICP);cPOJoY!#EhgPSv5|471Ilt+uxMrEdVe;N$)xj7nI zCuB<>c#0idTdI~+$Vse&td`h3A7FXuxYBeC-jr67 zAZ1bMPDCBLHfipxz9MJ$f|uFJ_yQWvV=je1Ut!i{($%e8crb2y{N6mD=22T)+qWKL zPLihrDe_q(4;sZq$)uwHp65?&0U7Lzi2?*(evfqj-rNk;Pq9MNE{kh(5pV{B6&h!} z(X{e$NlpB+Fn^VT6z7J#Z4&l-n7H5~JEWz!U*%)jX%!;OYuf5Qy$n=;hJ?As<)%~)~~@XK}Zbz%CInvYfI z!>pngVay}V{?+%xI0*UdV1=~Bfx;!*&#}3jC$omv;8JGaYKCVKQHFg|39<|_v$OkN z2e)T}_4SoT!(S?@s#f;gh=_=Mb)yysfF5@(YieoXx`+WB-@RE$a=1r9Z4=`$HYNB3 zmY^4UGFNPi;N}>FU%W4An9gpVE)x7$7{q4Du83}NF%CCKwD@|OD_=MkrJ&_9`nLQm z8>kZ1*g0Fnb1`bdsd+9^Np9enSrpdpvKa70DrI{ZFXpab$z8IYqT=Fu9rl{EqsL@n zovRMT#l;UBK>#0C9m;(q@2Yh$8r|(7_wFmSY<2cc%3*<9fnOEtJuHnae4*vmf$0BO z|CEPmEOOB4k=fDeGaAB$xQiOLBVF4Ur-*I2_@214gq>k$wHYrZ#Lhb`Tk|H_xR#Tt zi~K6R%Mbec`Y6A>6QA4*F4vveUnOk|l~q+$<>jQ?T0-X5*4AaZfc%FZ4H$ppr_!;B z*@`WalXf&BU^&}PQ2UMzzsgY3%zVRzm819YAz@;kO<>kwzvDo|ZybSy5nIM2rw=k( z&+ovYN-|#tg4ponT!vlWTP`+1XtQS97AnhPE5AFQ`S+KnC+HjEFEdQ+9l zDpT)N#wfnMd`mI-N%!hk=i+A-W@)}p3D_JyWtl+hDco1V-h4I$D zXDb=o-gQ&twtTB~FNxPzPXA}_O|5*=WKr6?$O2GWZMq+)1&Z&eD@2Ob1M2SI+K-KYIY*7h&Rlgv)RG%_N`RSYy|o7DbZv- z9u~kQ5zm$pIW|u;?|Y7+9xp3hB=op<=6CXo4sZ8JrSeSQdAS?8+ZHD5{uR1){*?`> zoyI?;FW6|^b&bZHVC&Vt=`B2Bf-PcUrT-kpEQlh0u(-6&jE?GdS= zp)rm`8fB4vc=TTXK2L?F9N7TPsZ(rD=KJP?^qy^L3raZSqO#wyGW*^)dg1C->jW~+ zNGxiB*_I&_fz%YjpZD|oM8?GanQ^ZLiG-qSh3p-@%o5;XJdStku$@eHjrTS+5dE86z6GpJ`RBDcoG=g~Ubn$oNC+Me%MzHwlAS(&-HImx!x%kRSrXy5d;Ap?sfm3p3m z@v(C2h7kG2YUzorSaG1s>*8~sh^p5NY;tn{e36&fGXFFo#X-Qws>guloZhG$N0~MB zGn^whP^+9%`?fVgX95m)(wp)t9lP?@*-nm@1;~aB7yQ|<`vh0WEAW0k&-2PU79mH) zWLWE_tV}g?1USq%ohlVuWpLC6h29<9ep!hts`00{Iy~>8{l~;N3^5j35bZO?(*uH{ zP#J6SF#XyI|LUF2>k?)RUh^=osMBoKM-m)HPtxq%QXS%*>GP_^&Wxu}`_F8IgHOUk zYCIs=$l@XvduL}J;TAe)tD(wID9!^0+V8`GbMg6kz#n3^gc9^1@VCy${C?0Rpcz_DVRHDpiQ{ePYwe7> zNae$sAbg@E$%f+&bI(zfhlZ4`vVGd(FPcsAIOTSdEnd2Hm;i6i_YZ10GjXyyO%;qM zG-khrnB>+B&z~(0j=!2Fu>QMh;a`LMp-3>il`@UP6zHh-NJGWnq_6}ssRQS&a@#j@ zhavJ3cr#S9`+`Vc;hmTA)RVwd_`}Fg*H1ot)lO>t^4+5H9VX24FD^8i1YG9*@G@f$ zT#gpwtTVPyfTb$8uQhI-1{2TYxmavw5-llVBok?ffOGRe4*Qsb!xuQfoP`x(9Qb@#^9jWCjyoVpTHyYAGPnT^fcGkE{z*IJabxua8Yc-OO1 zh=K~*&@{^1d10{*jp7CZ>!KKo5ppi+lDw^koTY7|xH7mhn^taK+l4_BS6rR^++r-c zXsY!ZhZ*tZD58nqAv6=u88ZbwC}|mo%`U2CR{dOL3)%$ax(&?6htDXA>L}>&d~3&g z01z!W*39!XU!>(Ff6=V8fARyg!<<_S*6!&h>OE%QZZ#*^*wrJ1!RbhRXs zu@Uv3TxVSx$s{O+aaF_d0DpAU+q?={q;aWmZMRZQB_!V^qG$l#KfP4g&<-5Q9ELNs zRZ~mVJg{M3F4hSA`&-kw`7U#=5|g34E33|5S}|Y8U6ykatNmmU)f%0Zug*3DBJp#J za$PSuTQ8~8YeshOr7hqtT+mOM52a(&14X@lM>Pn@+JJ#fISeV1Ha)wbu&mB-n8>4pLn zh`D1(N2g}g*iGwmx=x%BugkvyibgmSt=S9qo!#RMzQ2F5SDNKoFLY=%RF&O5WKz;V zh~q1J&y4>mUrJ_KzwM#S;*r}p6Lz53lYj>!;@DdgE_%H!y6_weT$>-H2^^N5Z zEwkMco7zcmroLxL8lZsKv@$IRo zpX4sR<-M4LRps$~O^;Nnq zIRd-!XU#&sm8^$oJ}|x38QPD2l^C);FBYj-1T;xB$#HC@IBr6VJap8-y3-0YelsEF zS8&eASuxw(3*@QILR&Dr~(_?D(U(eg=B@N;c6q8$fitB%RmP5Y*C z57MYvktpwi`@to$*MLhP_YSQw?5$Ie3H@@44R#OPGOa~l;&ZWj19+EbL(99$9S-Z;HKd0g}m6qLyisx&EtC3 zyr#RxnPuAn8~uri%`!N__v}yZLP#vH{ytK)aD4rm^q#28m~~!@d=2ltwaoP)6q$r-Ang< zjaHl1=99rh+Vsr+@kH^@2jPEN?kg1hgVWNuh5poSK6K4AACKz&B3jpWG!p>aMNyJo zm0exP^5uL#oO!wXO9y$Dn)4HLk$UnI{%4<20{=1*wQuV&woPsqqt{QW{$P2n^l-w+ zO6L6hKK1648s4T%;jXPh4f&fqUal7G>E59q8TSoeN77NQW71DteMUy53;5{HtKYKd z)69VPG|ghHr*YRqi~zA+O73)@F0?|!}30a=I}X-liSQV`|{5{J~q;Jb)+ zJ&oOn77=S|Ydba_B?&qBKu>&x_J3qTlYVg_x`=>@LsV2+T&BUjW8SB~M@J6_x@z21 z&`vsbz5qGK>437`!y+yCR?)o&k&dYkxW6XH6~*LazE*46gdaMUJaZP75XaN-0SAYN z`1trpEm)wBe7`hNg!gaJu2Gc<{%m0+TU9XO$olnW*H#OjS8UcJhCB9qH`x>C+T@nG zq)O>DcY0LM7sw&J7@v4-o^W<&JP6rP+;bMe`OCsHqjCjFp?n$tlw601X6t5LH=v4l zhnS5Ia<)8@FI4dGuvqcroSazF<@>x;R-(aE(hCdP3DRtHm_o?eB!!H9MYtCvYcFNA zKf;l{%Gc&B`NRS-Jx17;3p`_W{2xf{g@nm_ykAwviQWAJS?;%r{EQmCcjiQq^QVS;5V*i0rmR27FQa;<^X~Mq( zP=UKS-~X^GX!SQDxO9aVhMTzmz$`_MwJquYP$iPrf#5C#gvI{3KpIn^CA!rPrix-Z02}KfDkGwCE2UFlTh%BTi_S}L!tgxP#YrK|Gr5$ zP*`b21!nuBc<)t*m$Uup_p=;3{@R9yqaR`afme~v5nUC4Fmkfl+3`@I-b<+Xgai(g1K+%^6tZga6>$zSlr%d5s(?$|?T;jTAQWBAK(MJO<=E z2DG5R=;Pu#<5>VxD8x9iM3W5+kO;|AIDNDV8Gp)E8c2=&dRnQc7mHRK}TnPoQqh^4O+~alm6cVM1d@Q-tUxyaZJg;_u8n_-LIR1M)K(} zcWg^-y0Wd6g2_<#CUD{D=fF9K&$*hA(S+C8;f6ZzprJF4w~X|V-^xRHEsI~zrVJfD z9b%TPITPE!g_F|>YqNav&d7eQT}}?w&CUqg^=;Kh4fUJooLy6vSXdS|=A|6DIB>2X zbFf&MT`X5bULbGwryQ@0hUB|=jS-i4c~%wG(QizMwhfhqrKLPSX@Z8@yRU5jdX9GX z_3LopqnKxVNl8ImIxX#XpL|-gPvQIiZzaUc`GX2Fyq=7t|3=6r_DCp!i}`w+`ED_g#>| z?k_3ns=xyWJD$8=X0JRs3}UzrhuZRyF8gVxtH?T=iGioWBk*ltsqlFq?@`I<8>FD;HR;kC7{FjBE6ma=<*r z^1iis6s}BkE$-czTD9Rvi1uNzsnOZBQ9CaaX?eTq9HrG_$!zyg+xi4l+{t&uS*f1p z^(3v=J)(~oJXgTHjApE`P=wH75L3ki4zd_EN5GIy_bb&z=jWf>;9n_$fywa9ZZwth zCIW;@wj#lnPx5HGdv~|GN?s^$Z#U|DClKy>#75x0r!jzZV(b2O!aJ7C@8Ri>L~1=d8wdW|>;P~j zA;af)SMCNT%+heQw#K-+iZzxZZfzl<@XU~6lZ4xF$1ZZFN)dmdt8) z@@V3=162yMLl5@q>i&*Y3t5GfR+ca~R0mvI6pJyP^{_v6RG9!>R>MJDK)<4ovZZ!G zv~Bt^ybUAv)!PS)q!lPJh_#}2Wc6u}tEwPaPfE3=PsMWG384U!i0$ zJ+Ecw0)nwyK{ESDzFcS@*slWJ*xlV5)1M9>y6zK3O<(r;Z>3R#v4PvmKZAP+i2p>f80ZL)cyAUQyGTUAv`G z*Ym|C*Zk2V+ZgZJfy`L-T$$I6v~?2!w~ywv=sSt?%|$m^6i<$YM{`CsshR!F4jGfz zm|YI3oK|O#Om=Vmc@MvnoiPKCO}heel=F$UwHhecp4AvNb-A&Bh8@X=CT&1`981B~ z1lQLPe>>SP?48~)-Hc-^C8K7mVNELktD-HA$*5;vkQ+eLv=w5(rt>wo9#X8AXUEl0 z^1VA%kf_B-8;CB~Lz{~;vUfaUqxgIyC0=7ei}&FHi)Anpmu_0*87P%4l`)O?VNbqf!xg^faL zH&+H2%gW+8SlMglt&{0G8s{#@=D}HQvQ7q6-)#0pZgb5^e3yTEEG4wSu81Jq6tPS0 ztD-rs&JWL}r>B>U{a!C;a_w3_Go!u{*ik+Impq#8@|+;Z#sa>_Yo6AKqaj?u7F!IV zhmpVpZyJN+t)1sl_*7WpZfLBJ-|LRM)BTC2P@X@aJhfyA%F@)S#q_}S)8Nixb0|j^ zsH**zQcFo5${}!jKVok8RE;fmVqncTw%Q;BVB~5^a zHJ#D-8n2IV$n={NQ~2W*OcLUDUK=V#r_;M=5U0f>%E=u(J!BP!^3z}PkD?9Vf5HJ+ z6-yqr)bh^~>uro}QkSX<(MX_>_1PRm;4SY1sJ2Lxl8P(Pzr3_p;3sw2x-k8&Ff@>$ z#KX!#3r?NZ)@A90cg}dIfF@l5DE1W3EiNgUF|U)(9~^SR2sdda(3X?aGd5Py^5`iv z@V26=+jhiDO^Gm$H^LM0@#&UGZD7WI^AsQ8v`ZY7VWzp+y)v3Ozi?g=R0n!#JE#&$8c84XCwsET{CU1riN`* z92H`+qii;`;j$!7{qnk;;$SBaYv*UaOD&uwz72-#m^e6G_I0%J#7O{Dj((4l@vl}C zY@a-Ba1=Z?Pahb)9aPgAnPfS?apP0|g~;I(W&SORmtt?Lzq@4glILZcM&44FsXFkB) zpz2nUWcTzw?|z52=9UOB|Kr8B(MF-O-|nl&R<++32NA8Pm(xw=Nzl++K-`}pWEE&D zIovDTMjP>zN~KUJ{FR7utZE7X=KOHYrBV!UDB3ky=wNaZ$S(gDP)3~}L#Ro*Ko8-I zi``i>{O6i!^DN#zt#&KTaz}!NQ^{`-i>EB=%HximwF!au&cgX%u4DB=a;Yh}7H4i$ zz*Ic=Ri>VaN&ZQ}7bawE?A^oU@{vd=>kD&SCsV4i{{GR%3_iC}5xsT<@cO4h4f@PG z#kmtdvgg%ykyY5KA1~4arAF_Dax?t5o>$tRbW`ttza6+H?haT;) zx=pJlc;V0DdVSs(MG?jflOFNo9d9|ZP$k#M!v$O`e){c*a_LM{A-xqmPDmxE@lj0W zwYnD7z=ES&t?IeZQ-H-LBODYF-QLF;cUiy;j0j{US27?qN$@ zG5jsoTRjb2vD@1zAKt7D1-Uq+Av1<5J#)4El))8!w-A!j63N`FNl)@$S3T@D8x-__ zAhp)Un^|^hnJ;q+NbvC`FsfVuD_31@4Q||AQ`Q-|izyRrrp`P!O`&_ADlMXHq_L{U z!*KE3rbceD(O>OPEyNtK8!H_Kv*W+GVB`uWoU^V^N|LZi-!QG9b6y~$h|@Ay{S}vL zHni>T<)x~w{=wPP_gODrxyO*HPP<$@H(UO&e|LUyad3N@H=}LLoU!{=Bx(8?Yk=FT zKKKyeFJH(Ldjdy}{ zq>znZRVRb}yo~TZN>1+LjqIIyqETQ@YaWjO_RgS{mRxf8h3lq}X_M}RZM7zN3k#Be zN}Z2EJ}&0$5Wbq>w5vwI$mRtgjS)uUq0;`kb7TsVKNQ=8yM2Uck$`Ek?Z&e}rf~q< zP&COHdI6k-#yzEba$@G7Hth(~RDX`;5{EUQ7AuRWsbBbAK1>zx+Y_pe93kx4aScGpv-*v!F6q5L`x}B=V+Q?OC)D+4h zOx`0z%8#|V@Z1eI)uCvlq@qPKsI_Z3M8Z`p^)M=QezPh?E&0S@9(o``#rQ93sI{S*kA1#Ju zCwJi*rKMK!()osEcJ|}$n@z|S45RNxke}{T({~y5*Xw1im`=~}*u2#Lz>at>$8MvM zxqbF?!+m)?11tyN{rhpVKTljNL???pg;asj(VOB5LR^_PVohz#6HRFI>Ml8%V;f~j zR}0AM9~)5xtj;w#-U5Lyfh;u|2!LRp*4Cz_sHW+pyS_X@VHfp$xuVcc1S6AVZ>QtFAE^I$;zxgd z%LA*#sth~2K%($S-Po4C!(2K;PLU|r>grTSl2w=0oLvHqfQQ<&YcsI9?-oI1 zcj%xubR1yCB!~-4`*Q@XZwd4Ah}{jyFR&~IA9n13pcQH%tWL{1@w7>;bc4vr%?S>O zio9yp2pZ!j+}$~leUBg&d$Gfdfkt3%HDtn!%Zg{SlJ=uOKKWO&OKy$w@$N3m%-9cU z;WvoJ1Efk*1@g^_F=B;Np!p^boL>k19+`d#wDL;$T-`}DoM0?9CL@vU@9u8$EDTLf zz5wcduh+L}aka~5rh7vBoU%$LwE3`{)O7zoHodFXqyuL<>bO9{zejJ}YBuKPTAQBp z!L9XjlM1J2T_H@pMfvvtGzMM!TGHHQszAd#6O89rWHoyb!2RW z?q0&vl+0n(%m1u9gb>uyl$_l3no24m#PuscOJ`aijjF)r8F)lA9GfuKi+IJ~3- z!95|?v;aJz!e~b~Hs1URb9s61ZVVaA@5!VcBe#~|>?eL(cU=gIm24|U!JlBXlEb*) zUzwl!?(d?nSh&J=4Lj_g%j7N1&CIN3C+l}NzT7JmyiDb@_%fasdf!o5-d5h)#>dTt zj|a4dMz>u%FANDl=vK!zMtqM0IjmNnS(RG%bABzDtP8*(*CZNPfctkw{ZI0IvGtqj zte{%M4R&+vL|jJiXWZA*ZD2+?li77x9Rj)~G8PK(^~b7TQ$8=#aW`u8trK8)UVGw` z<$kjI1FcE!(b|h{_W-PwL;H|PLNTcdn!VD>TZ@^ zi{Chb`^TXv6FsD%#Fhk%)v2UZrIKg(kcJhb_)OIbzXY0z3cu;w%}aJOc9~69zLK_OLW*^O-b5`@&CaNqp*U8;(knOIAENOOZCHWG zgQ($nYR+~cp-o*b1b=n%v9Y#{)XH(krQWEk8`o(!6`HL(u{*b~r4hJn- zu(Z!2hljnnhS=-awO8Eag_|!jC*MdG1U3o&P;3o@Av8Rsm&}MTMtH5Dq%} zecvY>!4JYk52rh^|F?|te}V0OK1Y$c--kA$AqnsCL&`pdzZWLOsK^D8W?hN@ftK|$ z|MzeE-)KAk-z)XSxxEQQJ9g-k=b6ZDqeuF`=#Ll zQ_(aOYWrZ@AUwh3cPt`HaZ!3Cz{<|n*Kt=8hwk_B(CiI7NCtt(CdP$Od3E)EDxudG ztdUGk5`RmvR7_l77LJomnD5nSFKl}o4 zH8(uh4@T;cb?`r!dJg23B~601Bb~)|2}EPEaxCQ6jPJ?*6a{8dJbkzq|9XQlO@e0@ z4G6@8{0rKIWNaQ08#}wteNO|puMa2I@Bi|-Sh!q0)q5vRUSc%$;=R!O;NL0(eoy3W zr`G}owKX+=pL;-~*r(w(6Zc?_AOwB{IgoVzMim|KmDzCP3puWz&YTVX*QzCm+2t)L z<3{D*yiLPEyX-p?fN0aJ$-n>e`>u9p#KU~`89`+9KYl~lM}`=BfN|xw+Mmv`OCm!1 zGGVen{*~+21;Jfw)%ZZK7AjC6pP{L*ucfELgA5V>XL3BaOQCXU{gKUirzV1fVh8D5 z>7p`M=KV5~QY{^wGxzK1X*PHRi~s0@7(rqvj%=`c>D0b6bSkxpIa1u|aONn->z>Ut z{5WB$*`=kaiEG!CNKH64&jB9)pKabB=-j(N3H}~gT3T+OTUzoho^AF<5p&cRH)&A} z5pN&A2AH*;zh|AT8>yg5%uG|4lv)uIBH29Y{8uDWYP;CT;Gm=UpE~#-QLeQy5o7}b z3YjQxFzmV5pS1Y?r(UHlGrJ-mrpn9d1fEZ3x3?40iF1o8Ve*$M7Fu_t;X5}@SN|;a zU7R81>>iIy>e+|XEt7No05ZJj_Ij9zX>i#^c>Gq)AzJSBtDLEk)wwLv#wfgI1@ISW zjSoTWKT)#5QGc`V8-Q1F_XDI~=05wWxPB)hQxzqp1^_ZcQ_upe1fX}52FMp~E}{kM zp@3B@GV^5bf0J$W!%||ZzU)H?SP4{8knstOCX*uk&qcc5BeGqtkvdR-137#B70C9y zV#@z#mPDqa><@s`zw-Zm{IB-^Pec?5M;54-xE^sS>Xc?>%*@c#d$)Jr?pN?3#Y?M+ z??j@9n!&A!ZfG6+=P2W+%#X*iou;*k^m=v1yX_FGjJhgA;`^()1vZ6+1;^oIU7b?N z$BW91*cv{scS__JyTk6bS7C5vjQ=X2$cS2-l^Ek;d{MZn9Btr!Wpf~;e9i@EwpnUH z1Cl8o3iTSfO_ax%KlxG`z5T}(nVaL83ZZhFqp>VDdC0gT0f_=waR!Q)zf0|mb?^>? z0)BgG%{NIWSXR+a%2;F3G8vX76+RehAb^G0z3IrB;nLB0o6s8&*!vPGGWT_Q`t5_? z3g{Vs!3HvwW)lqzj-tLx{c2~%Gl#jKe~_{Ott_6%J^*CNLMs%&J$9@!3ptkn%N}s6V-; zN|;0cZ#Wwna=wJLZ5Z_3MzOyMH?7se=O{yci`QT^_KiA@66j1Ea|8JiMui(nbvmvX z_v^?80bkE)<578Upb8X3E>&RxWpwoO)-xa5U;p<)+kZSuaJGf9w}hSd4Z4Vjq`m)t zEC0r#mn?2aT!xzJM?dV?X%kAo;etwA0VnvF%7|(Y`5-K!6r}1MirpZ|Ge*BFF#%#D20?$(n4Oa9w0a;*h;EohO93>%>vV>?u#5a2z z_IfbUqtYVuYO1i2Rx-;J^oRY9TX6c?IW-vdg-1UtWkfTH16+N7&WjyB933i=V8T+$ znVgDr@|b#AAGRb#sl)v9DRu{9LX^XJ$`L7<*cn=Dwv#v|J`G-2@p(KL>`51Mk}R?- z&rx_OwcM>@Q0|rT&@iJZD~C0gIX9?D_H>v!9q+zbj&=KD7Il-dqMvurdYk9!s{k(t zw(s>$pV%qSZ5ih`Ctj4;5#M*(kzp7N8!|n+9uA*h6vcZ;I8rcC$CnI@7YhCvS{mdqr%eEAhjm?F-DGbF4utuuW*h=<#nDe&uV zVuCpF9I>V(p?Cu$1miW}NJcI8nhMPX-^ONyHCOi22StG*3bP@LJBN{H$Q5r|=Uk(` zP()RrK{i3?UcJH40IiPn%g-?$8&GK>qz>ZRN4apOCUaNGRe9D%r50P#JBpGWx|35kz1#_xs(hZhg%yHroDbNaJ`}3= z1~vGq7tHf5G5-Er9sCo?@RH=5bjKr%aa+_G`ZJkSsu=ZOS4v++qwF?#qhFqc8pbFB zP|8C_zNBCMs^M@;|m+e@>RXSTmT@_8GZeg5ui zHQ`^D50T*KHQ{G%%yO&*SN+WY+$56?5j#4#-bF#IFX4*)PWO`X=ewVzQr{%A9exeQ zgiIcxW>-^*kx5&)wd&-p8_B4|*()9H45~T^f7^XrVPioS=IhyY`aL3+{@OxW4msiI zTr`$INcI6`51vYa1`ART2Wdr!*>laa=jXX|=O)!$^PYK6m$g$v)+GP_#u>SgOULme zb&ri&p3Wh4{>FFVO3=dmoRXqns5_3Lxx0$mQ__yG(GAfshbN0Xu?y~#KSEW~$Oxxm zhiUTmAsH7Zh556eKOy{?eEluiRJ%VWZ?O;b-!21rJ?Ij{tCtzo4hd*V0okF9LNH3f zfTE8dU4tZY#rk8-#GUpQet&gPJ|OJy$c=o5oiCWjBn&)%l>f>3Ys)loXH;vyu@SlO zxD_o@xpaRee?yLFC~iY}};zJT_{i{;mFj#2r7U(0mq20~F1RF>7wTQ`(I)$d-bm>eveYO64e zoJ)0AJxVk_2RQ<>7ZQvs;)S1Ix(Vfsglah=~^OVJ%EbQ_C z;P4yyNhuhM4aw(bF>N~+&XKHKNX~(_qnDoPy9iR%YvfZ-zh_(2#(7BV+;{6mNfI?& zAzAALPN6rwgqG^QDtssU(oJj>ma>blWDhoE`r;~w*s_fGboOm2Wo)TbKhDP?WvrL( z-p1?2l9^~&0CO2Ei8y#cTS?J>N9e4e+8(1Jk!YH5{u}BwTT6uG{xgns*;oD6*%=kG zo~M%8yRnV-aRpsbs~3%&-EnC$^RHi$g_J~kdD5!%yy|WHY1)osgItheqdMhgb5R^V zdVv9>Sr-e~=cj&89kB*q$9jg_P!yzWK=jONY4$s)Z(>EIzDYRjlOgq$*t*0YF=^3H zuW-!Rd{0^XCHYS}n4w!bbI0`MQZPB~Hw!hHH)*>Fyv@!fddB+a6HY-iK(kM00J%zW z(X2P6N5__Y+v~p+fBpn>PmTCojtd!u_Yw?rUA%AR6(3J;jV2OJe+dbbzAN79$ScN% z`}oOwp_S_$*Ev!h3S#y;<6&GU3relN0|Kzd_gtwpHnx0x z24l?zV%1?B@ds=quGrr+@9Dzw&yxX~eBfhJmQegFrOnBwBx#_WOGHy#2q`pW$>aDTP&QC^A@eqDDEMhcZAvn7m_aEtnN+)jDr zU2O-t+#9dytIb&6=h$5=p1@|NyUsK8dIizQWONnnM1*@1xgKh%d8}-h=8=SIx`4ZA z$}Y=l^jjbyVOTtY%^(#{s9h(onx{3Iw{y_#@(vG}*Nyv6Cc;5W-32|p?}205H= zyQ4s?Kgf5=)soIFY+Bk;nIBGXge-c$lXr<%_b=(-otyvRQ_5IXfu?3P`_eQ`n%x~( zxGv~WUZ*v&n%=1bit!v`bozXKC7*RZ$#d^E(Zt#v=OQ@;(GRridTynC8Q$QOwIEHt z`zNNUM)y0uU5Cx^!=m4<%mV13(;R0Vlwwj*Lh zso>QxzkiUt{Bebl>ew(so8V1wnz0m<9e)~HC%f3?~!2Je4(J5%g|J ziU#WN;!Dtngg#MfiX-eB5q(rwr~;SdiE_aMOnj7) zM#Qap$i|NmqsnPXm1oJD5`ceTW=l^D&)wQCJm3v87`h@Zy3@+^7!b&8$(^iOrWEt2eZvP0 zuC}G9oDfi>LmM*QEaJ80aM?jPBOaJH{_;ZGm4^KxksBvnJi)0Ds`cIk3HXC!w>vd! zM_mwwxaJ;pQ}+TW1^nFhw|Npuq*;jDG>m_&{@5WHsMblt9cr1y3@6gu30$~C(NLaZ z2=#lO>FrIS3Dsuu0&%So*waGS3Im<Q54>+;JKHPZ-~8$7;d6~; zz3Q`IUMN+$W!feT$%)jy*Gm0t8K`XGQ;K;d@Eds(wR~N4f}6`QUMH37Tbln|V~BK# zf|F{VA`UzkT8t!^wy=j@4!q(vPSA^D*Y$H>pEadLe(TKAvuaJeMxb7*kmdb>^R&*a zcwgPs+kmCFU@fL z{*X~%G!bPCp8&X^($o`{LdLa~wBEYzy#9tX{*(R*zccuwXSJtkkm{qAO9)f20f~)I z!p9GULm!s^DcUK6xP15;XFJwWHH5EleAafnfl}Cy8wEG!%o6#Rs@a9U z=1CuNJ4^bd-qdYKtfhl0Hgd z0hfka7bx;Gncr>%4+Zw#+uc*eo6NBcdK?ixwg>(OqpnY#^v-Is{z}LdtSi|y%`yAK z^#JHCVlEP^NYC+&nib9p(vBG#@x%;M`~e_t1ZqDzZVw**;&T}2KSYGrCTAjIoAp(V zQ%4I>zroqoy9B9TC&9+R-KJ1($+KZJ9DJ!FKv?^LVCPihi7?JWICtDnr` z7!hX}3zBv*U+*jYvk>0jSTv1E;!~_Y6Jiv3SL@GK_BOrXRn_;?hzN5OlFINHFRw4I zoL>GnlOTs+zP{s9<(lNMwCRgN_v->IC3 zb)1%0zzp2Zr_juwc<`&|PSnV#k_wXF{56?)4r_C<_<6oXFR!Q0M;gBg?g=k$$U zfUFkFxT@^P_Q&Zm!kZBT^8TIFd=K6Zf_eDav@O1+6HZ~2028u#SvlMqe8lwyr`NUP z_4gv+EumokWR2$ciFrIHhGk{1(aYe8vd@!wrSox1FF40<-LCnDCO*II#QsJ7{WUzVX$P(94^1@nD{lv9xp(JRlH8gX==ih_^4VWiYs=kNL<0#Mal z43_;Zbk%pM-ODu-UwT|^A}9|60P$2GnQvWL95hAQKhr+v5>8eYVxgEeegL!+N+6+c z0B4iQ!yZ>(7EXf1I zx2k!QleAkDB@PNj>&QQ&U<)t$n7rT&{f)f|(l@)M! zT&uW15YqG*)g&1F@%A+;fyge>vefNG>*}B)zv`eBSB47D^Tcc)Nv_~ubBfxtZk~Pz z`pvN{;R>d(-%vWGryn#S38`8X@BmF|7Yp>!fToA~%Y$aNqCi(yo>;lGj_ciN##efO ztj3HF;h(%_(>n3cxv0HP?kIr0J^Cw@SIwSV;}zDyUVCv$D%c}R1bppQlvT)pyx?Vx zA}vv-Bu@=(5}-VEmV(aG^A;HPxumOIMG_HvG#%@ISZ&(UVNY`w>r^H=iM=%yL-4Z~ z7t89(?qCr*E?;c9empsftXy>D(jyWnuC$vwzvF{A%)iWwa3bq}JZ0a?@~z!W zC9^Ifb?_2pC0KE2{n>nJh^ya?g2|RX*Lm)%G6|+Z$qoi4wn}%aBUw)T7Z8|8>%Na1?AIXVe=bm$CfmX6`(irOs^+8QEMt|79$h$hyo&>j{Hc?F zZ;t_kkbWWuw)4wNW@MQ!C-8yT{)O>o?-JCY*<_7$`q`}cqomo@=Z9u)9S`@t-&l~C zF!84dnc^gvJN(`l>Zfx@wJFFDEgqO)G$F^U@-PDHkU8*h~_-L=WYgij>A zM|VxBV*&ihm_9^jN zoInhdI-!KIeE|+QG%y%4t@WL2tE^FOf4+hMhPrQo+%{e-%R@Z8)JB3-A+MLns1Em1 z>Ahck`@=r-IxKyB`sSIHdHxT_-~S86 zmkQSzV`pb$qNL4WXR8k9AC5luJ3viWyI8pxu5%=4?8|2mZMUk^25`2I|9lJyXvtUI ztw{oJsKr>Ke+l44?rg8;Ojv^7fd>LViZ}5Ts~&7jN&|+~zib&EcTCkRng{?oI!y-o z*5KZqut7_ps=mg4YmxQ{O#?;kgi!vu8x4mdKjE3{1Z9X74qfZe4SMek<%OrFj)5&)`} z36T)Y@E!v2nEgK>Gm`Fp0O8@9a&F!cROZd*2EsUfOA6|b2NJ56mx%j{vL9IK`ifBN zk=Dq;^VA{vC&v-mOg#Hz@u^K7%L3=7w)&Hq)JiC(Hwr!A+dw*i`-$5U6nb&YwB4S_ zWhx|NJoa~^(9v^Sd6`?+%&m_#d)oqi!YHFuvuG!@yRa$M0^) zMqsI3WqjZ>e{LI5RLaA!3ax`Pp66P%R@p&{winN0SA`^_+AOq2%l!ryg!L;fjiHE$ zQA#2=7HfS)m>4TVgQIFCs(L5fWZ)y6{?+;!$ANkuz}B`I?G*e)CV^*EaFgW%D~l*& zHzWT)JP&Br|5rlbe?iVc?e$KtK~4Y&(PpZ3<1zcX(&)W??^LRRfw97KFA;k>PYI4@ z<>GtChhq>R4h-y_o}qbuI@9POH!)S<^c#)XRj=F&pZ)f)6QS8gt|-MIZhZ5X@2l(7 zf!a_G)^Of$bx(12H5@d8Q<%QT_2oRPdD4Mw&4P*o@AU=jEu*iyZ_>~>{@d0}>;O-* zH9L}gbLjO36)T;68SrQ=6r#8AVWU zdy;TPOoed;2$Hu>mnJqfbK5*$>RA5B+j4@!TP$F9!|)iYLBJgR+qCi@??M7y1;y?x zU+K_GHR!Dj_O|z*yE#%(^)jLG<`%vzUPRse!0FNf=2*G@zvo6WeHRBv{*6)LC!^b! zkWn2dGqc?e0~~yK%vwV0Z7}P3B5s#2yLxGqXk(^dSAJpQ9vtsX5+_Lf zE0dZ1F+e*>FH<_dq_}B*dfs7ZmuIHJWAb;vGMG}#Z0Kn3JJ=)gWEsoPJstDbiSvYS zSOhgHwd)-3klPdC#y=>Kpd4Ula`9aQXpPV)Xh<6o2Hi&gd79lJyEB6W9QTPN)Lb4^HNl>^XcxT z(8vry*^eu9DS)EVaOwEO3VA~B0BH{?kGO;>DxGU!7VRpkxmw@Y#MnZ0Ty^nJ)i$%7 zTdD1r1*c@-pgAX2n_{(Rz62izA$5j<<~(dYtgS2Cc+9tgrr-@FKKorC^Dz`W!r*U9 znOX2gzEev{iP{Ejp_$OSQs$lJ+YEf#sz0jv-E1vTrZ>M_Q#MmhUZz`L7GGAQN*ks| zPaZ1ysaS|5IR$s>@%dLuK}xI)!{L+nW#90wH~+5pD3{8`u@g`3Y}w*h=&0df-r1NJ z#?&Ofnn6ai_^_rKALq>m zLOO!`CAv@Lq_cFYb^R=2$u0`1iB+jmmgLi;;JQM#vp3Cbc4d^1-S?p>VJVPn|M zu9MTP9OvpE@{V8e4&JVAXu*2`{^yUk1_5YIaoC>k*i~B^q8WZ55z*b*ckz}6oJJyz zIUSis*0Pxo7}`fhERIJ~j2tebK=DrQRo_*Sp;?$YskqHX1Kz}Sa`T~F4v*$))YE?y zB4xaw^BpvTkLHc=^rhm^f#b9#^w@aDz~}8;9lN4fovOumI2?nlHjBOeivYk-`J*ZH zuhPVwLKL=NeVz#u78ey z(K&onYf`aXWD?0*>PDw%RYWb*v>~1@iWPo$*SbGAi|55OP3g-n&y{@1MI*g}T{DUA z;N_Y&Y(5=1#xbjL9i$fKsG>Z1+}-%ZR9_K#Q5G~~((uKV40l+UH!2CUeVqew4Ed(g zYzb*=`;G&lYTC~=OWl>@#`T*Ju7BAnD$>-SE2laay~goxH-ieNZ4~XewJsMXEbsV5 zAdNoxvUG{xjJMXf>LnfmrI?dtKMjvHW$0XE6g6ivel{XEM$;Z;B~_UO9%waR?lJ$Dwz*gJoaWY#0vW0J%>>_hhT z@9HmF+QEd&6o`4<-)Ms`?s4r^ZC8go7@NM!E0-ScN9NN15;Uj36Y!ZR(@lKP*c;Pu zQ3{>p5w7OkpZ!^CZQ~ykJL5CSNvs11daU2A`nHomPe$q6UWLEozp5%<{4Bk~?vP}} zXIp2*A#U`4(e~Csd3(|SXYk@}DQ?AzyA`LD0>$0ko#O5Vio*wYDDLiFTp!%s-CZ~B zZ@#<#?5xbrKEpuBljQnM@_wCj?jK)1_aUe?k73Ae96sYIVhh3U!@M8A+IJPhUM*P{ zR6*qV=a!mcxJKweTN7(DWBEG+`{#7eFtZLkGmdaLb1+b`Xtsyt7v-x`6 znOqHY!gO1AlS7U$HXu&lws3gP`g42zxIMvt(?bULoJ1pzLOC3P(bs~P{bPrXKy$up zWV#h@-0!15enM7tYO~h8C0Jv6_|+Of6k>VC>*e0d9W-t^q~&|r(bTFJ8scy}ms8b{ zZgo@FL`G+ao}SOw%T%<&zImj_*E@O`U56u%hR71dyp@qJZ%EH`EpF5sJBp?UxBhkoS^e#^9$}Tko9WHa`MjB~29(ZwoR7C6T;30+<5g3o$fD%#Oaej>#>pIPlLV9+T(tTZ{=TI(L$8_7r1nvjLxZ*~QLd z+>OVo2?UV`I&*2V5wWkCPWfvqfl`aj8tO+&bzi1`-gGiG`@%dN&)+^q7YJa}Cr*yn zbRCvblGtP~4I$$0C;T^xUC{e6!T8g2s#>eKjwr$_BV}@&N!qLE8^CX>)OQw@T<95{ z041bquG9@}M&?(rgxwzZ4VOMUVz``pso|f)^B^=zL|wUS4#r!4;LF0c#%P*bCDbqE ziLy?&B}z2D#cJFprng)9XTxD(!`$9YCHc$u?Hm7{Or?d6ea6ea;ilJls|vlgz=*65 z_;kXXa|+k2CExrw%Fb|Yz4B77Ybn^0=2ArwGYmlQIH8i1prx4jB3UM?zZ z*nE~e{Q7QYvEb@pT<7aX^>&kGKag3~a<&@U*tGNH`}?l@4yD=J9~Jkg%wSiF=Ji@p zxi@X|{624V=i9x?<$o-e;qHg=hP{27#$k_^e^U6-Co)}iq$RmOM337Z8EV)rjm9)y zv(&83yEfQ%mFqyfqT(EMOvi>k%p!|7G&Mxm?81+Y>&^QGV@**>Co4#0Dr?JC+FnLP zGju&(@0aH0PRl~UXfJ~G@Z^3!xI@o_?ZvE7PZ6$8cFH&#Akf6})65%2`N$Xz9IERqP>(GB~LO?TwzUe&NIhaE=(9(9+~dHz)J1@5L^mXy6)? z&{0d8_KW3B!xk*Mb$ym6=WPskYtq`cHw%xxm=EuhZt7LvJE2Rkl99ePEgAeSZHr)f zT`gG((VjWR(XBr|Wv)ZGtxr5ufvidfx8zxTqMfTg+*>QhJ9_xAIy5+YvXt#PqOQp% za_q>$%E`IIWOv&Wk!ZOCcMIe6)Hdf%3lpW5&^BNtc2Qs4$wsWoh$*}I(Lj&3*I4O`lW%zM=B5k#o?BENL#?@FYsvBq| z%PP){*L-z#y>E5DU92-h6_k)DwpyHdU$}KW9UE1jc;qxO)6s+;^-WYzV|K0x9riom zKb}{9xYTz49l;BYVX$4f?6e5kD$w^h|I!kF#pYI~{T!3d7eyO1Bg$=}3@W6zSu=DH&AcE~V$x#;c1I9STg*srPjWf@ZS-M$EM%Av*)CoFkq5$rG~CORr@SiByvf~0Xy#}jHv z@@i^o>NFn?4*jEU3BRu9TL$tJ~Ybo&=gLQd44@Lzq#hgeeaWNGn1PifEWqIbcJ_}WpI0;+BT!KHk0yH9|l!el8y>B%kb>uT`4qF>T}DiNBV;= z{(g{(SNH(Ukv23zItCjXLsNbIrEdT2ZS3R}c6L^hYfvQ_Gt0`(27@`4F87OxJ0b-L zfw3;r`HywaNH?j!>EBngvJ0FaXSbg)?lL~)An*wtOs`isuKf2HD*k2Rh4A|hYugW( z^l#Vw)h*xZ6Qi(|Fbfcg_{C))rk$qa^Gjh}<3pou=?OX=hAgU*c{IhsG;!iaJH~l@ zZVnj1gAAHa*noF8!o!2qv2~VUcuUSLhb-BMZ(9%eD1sI9>Qht(3AqSSE}d&?YI+|m z%#d(vcUH=T%wfnnfkv^{Mx+w6EunxGON zkM!*~OIATijbg#WyO2TQ-&GUxS(@62A6Xu{148$0YA*hYJ0!10>@kRml7SI*b@B1> zu9TBiEVg8X$!>0zfJ_Ges@>whqqZ8SV9h`E9!I{qii0eArYM+96N%M`yhKtvj<`F|GxbCa9Se0W)E$Mb5vatz)4h2s&8AnCK0!^J(D z!KM?ykf)u_wXDFmIKLGd!`FAR@pn!_wYafybpFT zI4g_x=xdwdQ?i+4)RM!xH*NTu_Wa|sn~wMP;=pi@8YJOw-cB2xsbvohjqTVUd)pJL zf$r4L%XLzR;CV9L6lfL4+05-_)jN`RM4Kjuuu+OvL{d+0U6)mgikU zgS*a8w-E5+4GJbFm{`=S3s=X>d9^S5-S%K%jc{sN?+y;H9i3P@>SVg3H(TAEQaFtM z1UGaCw6Shr#B55Zo$Ks0GkvIWPxIRINeL+i@Y85|S5D=)86IX{SVDBn8`UXO-`e9;Rvg=a`YCnNjdVaa>1 z;5iEFnum(fNK=VD-7M#bNa=VNlopImi`+t!n-4{&uJ2ofZUg&XuAS3_6VL_D(htVF z`C1|*GSPDrjo`c=d9-`q%U=u*ghTDX`@)*^GV$>G2zF$$`1MA92 zvoTM+@?^PnUk(qf(2F+L1nAnXXm78L^_i7^rpv+F^kw?dyc}s9PMU3@hfVqQO?f@e z(9xpLbT6l|Ja7S)P7>E;km`ZL0wQl?YvZQ9VjCsTJn=zl=s}>Y+!2Bv#8aR?#gBI) zXfUN6?8>ju_VQ^D9D=|?ofRL%WH6=d9f=fqLh!c?S6|ul-~jA3XyZsj4z?@uopB)z zT*u2-QFfyGaz~kKUkf^{%S+eElvB3X#n{ey=T!ul#9h{0@OE$0B=VCtrlf%BK3^2Q zOv(9}rA9r%n$@()wwY6%7Mw!XwUDM1m!l7?48UF&n-&NDR;Alf@v@II7y^^ix9tH3 zP?1P2Sonh>bInFK&*Vs*w6EQ-Y4YC7;SVu5vV0MxuF_Wm($mXk%gv7|WymBDQtZ5) zw1+yOvh+_1iUgi_&c7R+D_kMU+>R1C2=r~t6k}a)#E|P!-7-g&`)cJ#HeCm2dqlCy zJ1!jGRNTr~Q_0j0Utfa4H=)5DlJveBQTH0c$1%)t37?)HtVOlgt5#w}A+F~3$Jq&) zn~{67d;IlH>z@JLE9crl8cx(;FA-az6@ydx%$xNeL5t2O51#9jRTL79N4TLqbo7{* zd{oTlCbE>(c2mh^))Lph#9K#7yh8B~*o%5z(|)5Yqn#|H2+m)BKc646zd!Tm)C#DN zSmSMrHkcd(<34a<<@4z2MXIP%Lc5MHgDXXcPN*N&{A}4^2G(qQ;%c22OE@ib^25Sc zujX9)Lh2)!k4{r!_7~UMLL*9l*{j#bnf&TVJuh!N*#B~qK9{~}5@69>aVMrkGl?0r z3oUrSU@7jS8RB(W(f^KpeaC26vSXAOs^eIp0Z#ucQr3oBa$J1FUca&Z2D0uNL2{Pi_MnaSf?I!)HmLf0mwA^E6dM0qB*7j8FHl0Jn1= zIwPKFTQT|k<2Ou}X^qmEwzO1t-j~JOEJ562EexYPv4zFyv>wCSVam?kUzys)iOIhW z_{lDIOddEUgp@rDZPViGw3MNWW|P&I&Eu9QC}>2JvNVy(>F;t3Ms`FgE3;5EZv7r% zvm`2J->eMcys|7Q>svWE8qOf_q%qm!|1Ku0MqvJ&u7#<>jwOBFF}kp>HF+t_Xx1_Z z3h<8IT+IsAR>}8Fjo!y?8!wj`3L{Wo?uMS~LhSzdL$?K2ET`;iw2$@-xIe|8=+eu3 zatv<7H{B^CbVoqRkyxX8e`2Q%y>wojb$(q5mQtS| z#^|f-a@fj2DbqyB7^au^2c#air`%$d#+IW43`l&d$z3-g+~DkySrEh2gU|Oef-^0r z5<$8R?t8e>A8bk%ag1#hCL;@7`})B-twSoso_tNr=hkrMJkG@8_q#gaS`Zie(2Jbq zle4>5wI^nlOV$Qw4BOo1CuPLYKb#2OXCyAkh9>ux$rzF69|I;)%<-+DvkR8va+j7RU=U2&ydsR4c^SS+~Q)wv(VKk?}RZukf9na@_Szp|&qH>YJb}Efr0mQ|g*H;@p0Ow_Yr|q)zEptW@DJp0OrZ;CxhR zz-5m&u_DXAIf%nz_jZALbGnP;wBB;0P0rQa#y_0QVWuxE{T$v?2}NtoHIk~=EK@32 zXeuOjKq@*yOUf3Io((ZpaJm%bkQ%?+7uXnH!ouu^DkDC-;?u?EDOI`tq-WVN<%gJ0 z7W9Ln@UX8O?XYoqs3fVF2Fo@fXxk;pV(yB{wrA-Nbzc_&*F-Gwr^j!qL>G!$D^(>L zy*eVoY84+Ft4t1lcRgoQsbL`9fB4fJUGA)bb-`f0T~iY-|{=06|~un z%5B8A{;Rd;(~4K`40Jw#lJY|0K;MsTP93-#!fqQQL5rU+=!7-C;+oPO3yXpj+z!2w zU7f-(!P7bOXX3uEeS2MHTI23_>ul0|a4Np8Cpatc2^bYySSU4}2S2U4^aKiMvJug5 zPoRCSki0B{?xdCW39K)*dg>LUOQ3axC<% z&qD;3T)*ErANngxpWr-`lf9dCnkn5RQop7Esm%-Zo$5#-HU#zlL@{#`d6tKs#Q8WU z`E_k*@NL`sXrqF?U*BkT7xR=KX=mTiN3MT)cLOy(j(0ONia8EbF^4)33Z*EjQZ_eV zY2KkWjZ;fXhgQOHSyoekxG@7SoJl^;8aHk4)^9HQ)poQj7fP-N7;^LKAZ6UxynQ$e^Sg=pDwH>0A@>%44Lg-g&v$zZf5T!ZE+O|+t91h%svCNe+P8BhzN851 z*CZbU+`SfIwUzo#eKeIW1Ceu7`@!4xox5u!asAn_p$vzGzW0LX(LG%@(vY)i`L>&P zk0NerqjNO`{hH{g(IHtw7jHA~`t`0c=o$x(7EELFxI5YP3VZmBPoF!fIbE*f8&WsE z43T-Sr5t>u^^h*PoY#gIL$8XgMXeTlGw1k8#`Msc%EzV%=GP@SYdL8}Rn0{$`De2N zX3Bf|?E=DdSd#gr$WE-%;5o-N((h}@xSPBG*gKy6AqnL6X<>3(KA^|W5n`@xilen} zRr(Y4W<@$fH`MXG*z)pV3AqA~#}hgK0NKKL<fVv>f8k8p0P|tTA&DiMm@%;Z})w+g>kfz9KRN#m;IbSM6WYX5WfN?5s~j ztmAJS`P~8XXd=xvZ^BYTuhxP!^!t8>F{0d>j0j*H^cUgT$-Z5$-&o&`9#^JwRoiGn zY#V=!;|pul8}C`Q9^K1AkW_VMx0Xogh&tyLlzpBbQ~VKC+EW&$Ni2aJo0}i(7MEk( zS%f_}5p$_&oG9>$C@yi@Tbh(i)94sE^y-neii2DiC`pZgk@xieZdcQ6LmgX&_{&+{ zvG;=n#&eo;y1hDkiL!>Ju&P~uAh;9xcY^eib8ve}JDX9AbL#esz;ifM4#IpNG}ZUJ z{htOuu3(X;PzHWA5AuG%t3wJ{Z;Pqd1fvqeQVU2%RxLc|w=-|KHs>+C6ngF@ctVk% zN}i#Lf3_$flR_+TXH32-|J_1IOUgaud{LF0(?W|_hc*3d5(vK&%oQM*CMcEMc791U zG-C1`o>fwZX48wVcd6>rLT6pOIEq|Z9LU731r?qE_r8+KXjUJ>^yP$XC<$t47qy5> z^W++~nPIv;-O2?Ekqd6k0tfnhPaA4+sRsaRQtUQZYRKZhqDg=;kcV?p}0yTQ8_^(Um zOz!}iiu2FhJi3;xKg8WI9~X2r5x~WI0O{q9Sg$qH*&JZ59-F9r6BfZ>1WJ&!rxhtI z&4Bj-h`13}FW_V_ZgC#H zJa=dT(_uQy?e!;wOzD<}uYZO0Y94creR^d={GQchHVJSqKB&L`6vH7h%{)X11o4P5 z3OUQ5G=|CWcMjh-AZ0!jcN~ifiyD;Dig*X@?GwV<_H7QVWnS5*v59EE?3FbtG(r&gM z*AS8q-5#_^Nw^Dg7JC`*ft(m;{p4h-*K zHm6Ip!}XWvQhc6wboW)LQg@=Jv9sCBFwys?!|1h=w=pTw+hrdZe*;t}#FeJal8eFs zlA&#Na1@T-o0Ov|hFwq&E_sAGjrDL*IlY06Qn)J32C-eNt=D~JWL!e7BPNhPgA=j{ zW6Vw)i-BoFa@q&f+2K%veM`p}n_4EZ;f8grtwA^kY%Yv=2}B2c)=RJ%n>&|%8ikx> z;@uD$>hIx5gzEOB+(3u?iI6BHuiffl#67uO5GFYA#20rwD~R+rgNQWEwa&Zva<7s+ zhgylrt{bs<>56fvmC`a9p%uj6lt6VJPZbQdHx|r+x1oeB_MFRGrAtlq-!f_I77u&K zK!b|!Xi+<7DYNXaW>j+MyJu2WpP&oIO;ne(;=zk)#l)#H`DH$p6c$Vcpwm^7H|&=T zkm>)<1qE1B)xCpTf_|8yzsl&X#?kO_7L^jsa>6ftY_`$iD-s`PCYqUBI~q>o%xA>7cpnFiv}CQ`^M88dwHdAVNlZ+w5iDD>GQ%O8;mP zW}$mjYsF)4Fjhb!pHO?)W>4L0AD}WRKi%f9B_s`XmnZl&O=p@7yXKyW@u?^_a?8(a z@<`qJ_E}|(Vj#@E`nhUFtp8IXgVDj3VLbLrl^R0BhJTF$8ttOeFseKP?W zjTA}>CKg>t`u;P9GDztrC_n<5EOK+g%gWa+ z@cmO4fU(B%QO;B8*N;%-82I<6(An@E8OV|C>tl$XO{66jBQL^|nN`Y|4Jetul=c@7 z22&JEaNPA>r9sn$EWcLzZf>2j7g8uzM#{Yol8*Meip@1NMy+iY`<=glQ$e#(jB@~V zSA50@uf>~8@Zr8q61|<7#EXi5;>UNuXZ^0j2-WKF6o2B3^i7^t;MNLOX8T3xI->k( zrVjcZ04Ey71}H27GG_&t3%~{G5s#nkYGGVk7$H^xZn5&uC8ET!j%c2=5#cAV9v8Y2svxS^ZW_B#L8pja+giVIVoht#Np^o@w4pyyHT~^aSp%4UF zHzdRq#Evl|*Bl9{3*_4hn88fq_qFlWG$M7Ee%USm`S)2#{XpI+wu$b~&C~oeN5GIH zgxCQM=vDemFBU4_X5_h#D~fnTj$6DxDN+X5PL5;UHu-g7fgTFii1=Q>rZ;O0w}_M! zt^Ow1Fn}KI@B_}iO+=Ai9z-YH{Jm=_2eZNu33Yw(qc*#z`cG^>&+Jlv&hWxy)CK2> z$GiN5DC3^}fJlkAg_hLbKG{Y;_Fw@z`ZwZnTud$WH%j#`S<+u!C?d3b#q56oy9{%; zjKXoinG6=UGkm$PFpwHAh<%4|VCAu_W!I#C=SkoRWSR>nw<6wXm5hFZ39+qvV2s{F zBAsUfK5LCrwiivc(k&hM+}bA9&G^sYJJ%K8Cj6H*@;p3eq-cSvL-tp>$Qqy(g~bj- zA0FUMiE-r+oqli3mKV*HULLY8OW92^>jAaO6Ih6`#tucxpg*Et1NBp7{S`59)bAG0 zLBn@^dVI$pr=hf4{d%W{RMQ_`BLOI@O({kOBTk&u;Z$}PiKDR0V=j+ zj80HwFbAQ;6n<~eX zZrdbpDGnOE({Hc&>J#Uyv>*lh)+(%aRhkR}nN(afS=?G^fF_LgvYgSzu%|mLQ5Rh& z?#2mjD`VALU@M?9nu|41ZDw$O%Uv!N1X1uG#bo;5(VT`-aozk<5`_`eOiZT-O-i$a zg!}TufnN&Qzf1TC4%n@2Ez>m7PQ!}$QwiP&`xGBN#$xA@@3J4;6~r286_iQ7RHlu4 zpd%1%`IXcNA?ogRIEP7#z5D7qX|)uHSF|j=95h!=iClPOqMOntHoAdze_h5;lvcgWAJ6 zKKC0Hc;$R|gO+%Kt?3oui}hKnatLphVIyS{=-AeuGz9}8xoZZuO zNjt5roys+Nta#NC?Jgz@iAK**HGYfwQ)&qQJU}PHVSI?TA0gH>h5%!im66iJ-l@5x z?8-pTIZkcu%t1rgQe&curyJw=^#EZduK_HeYkl+$@X`3;(PiP)!bFSi9cF*KV7Apj zW69xtz2lc_U&3iirw$W8>F0l!yij@Zr)Bo11i%I1b4Ww=k(nt)M#Sk^)tw78J7}W0 z!?lPzPhBEt*I8R;Aa@At^`ko^NyB=?l|q9Dw7n5J2Xhb)7uag*tGc5g-Usg=^6MJ| z!!5r*sN1Zvy+dZ&ehi?inTp!Y#c_YGa2uIGgn2@xyQ`)|`^o^wUErM(?sE^$C&!@p zLVu*V3$@i7-Vl|1ndork1DFV(g@|EG>xfC%ExT&Y5jCglM|H4UA8v&dc#O2K{)5>v``3zLR@q*KI3L&eIs+6-Zak3;`U#_(C(1D3Ocb|uMvLpBjc_`L!z7Y z5Dc<11o!Q*9+b!HadB^Xoho-9f2zq`di!vX@+8}&2$Py~Rkus_9& zi{UnEzRd2knvACFR9+!-Xw>f`WHXy@QH_)-`RZiCzuu#vj$iN{io4zcNt|;WcraFDcLO{a>)g7E~lCMMX2zlze&r zgsZNuE?-E;$7e8HfbQsEKElt)Sni@pBgvJ)y)@uEGDB#(czV?6O~l7wwU~FEQo!|( zK6-0{+Dee0D6cj?Ljp{u5UE+GsT`#^%ny$HjOBJh7Hb>wXfdc7^PZja1zxwSJNDVc zzXWZIPj346jNiBrLwQL_$<+Ax`d)2ONy*dgM4Ys7ZG1v??fzt&XYe5w%Z;A zzkuamdXHH&vKC&XsE*aa8>h{3dEMatKmg({a(Y{ zw-{Q64_2$cI^;o->Vuk!{U5Za=o_2f5si9wEGBm56N6<7aNs{%mDJX<3(RGbd?<)c z5cXa8pYz~}2UiF{bjlbE{l7PEGV!2vzkB%1k53@8a>^78k6I{D18u_lH8$_zlR%0J z_Pg0Y^{F>zlGVXkqbhb-BrF658JBUISE{q+asP9^B!nVPc)yjX7n`Pv>dvPlZW-5nlSf( zZ`Zf>)SKpd>;52YGtxIJq}tn!N}A;5NlIcH+W(iGsT=9C?7G)6Igl2_LMecUZ1kT~ z$MjaK4s&Zvla+GoDL>0Q{$aWPaNQvt+%^9AK&ulnr`aD|oJao{(({q&bB@~a^g@U%vs~-qYF_yk%+aG&5X?aq9AASBITL}1@F+oZc;fg7^1aIy7U8R zG#p^TNOx1-4%Ogs*O^lGm%CmlyPGueQ@r2rUDMi6C}aCi6!#~!JL{pdby4asiKSLX zp?kN7*kH2z2oNTV5OAS&EDbASdKEVDgB%hMzy8o-Ss`hqKmEIcp=RUF@@*%e?6qwH zA~O59T<`@AWXW;PdEdf&^u4w@RqmV4na#xqb|*nBTQpgdmrB9(-|1(=AKpsT!NOqS z>>wvv?zdGfiEj78n!g{!*mJpKa4a8gysk~|yj8Kb6ePBaIe=Lnr2Ag)F<@@=-?YuR_EzA#l$G6Y3cn0`^YQW#5Bt24@vMg|Ajx zctf7b96upZk@0R@7@6K7jc&Epup6#+1x3b{$9-8GEcaNojQiX3>Svm<3zQVax;K!* zNfN|8^sB(D9E=#h6OBM8wAAoUH#RHW6V}{VQqu;D1yra?f@a&l^`%GCnQ=Q#Jl_RH zrtcX{YfUQ^3DcW1esZvJ>5a2EpvF}x3phc&O+Pr#D-s6NX&G5*6{NrSNQNp8xBJEx zZVh%1XFR^@eVKVrllp{O;+;nh-X{o!751mCD9UVTl-QY6*nP2 zKFSwlEPXjOyBixH!twkbDbwG!xl2^$>PT)ntI_^8i&mw)T)lG5w(!jI0(>{kYvN|* zc0S>U{o51nZ)^)?Rq83e{fj|nrVP@^ypRdJ;3(1v0%t<5CtVkZhg@31uy92gGFd4+ zB^~9jIvJP4-z?$|9jN*WzLU!X_WU|fx?mS8KkEM5A%)mnP2zVy!h&^IL~X8Wk*EY-*IjGyQH- z$^a5r0Cu->Z7CGHogg%PzWnsZ>AOZIPskvnS5fc;&@X%&`CQ|6k{&9vg^D?(DR8Q4 zJ8c2pQ6I0(!igy8=UAvNl@t?NN<-eONIMX-kGmgIe5pCD=0`uxyVBi#-EwWU1uZo$ zl2ao(#y0{vIq%|D*&rHE6zI1rEp^TpSGJJ$Dpkdf9>w$&zo-Ro~XLr*us zc7IgdH8kKXO6+Yf(hIweJR{x|VSuczh~MRge;isW08a&L_mr3Ycg1&CiizFTS9|lj ztAGTwN7u&1=0}PX71oxADh;%vN2nN0uC>h_ zcJrUjy9?cZ3&epN$nGA?OUI%OC}#_Ay;=|GzzH{MR+!ve@#De$W$Xr0YtcwA^g40L(XpKWDXFluQ60q$8d;2|DZVecj^#A8}27)JR#Eqi(;+ zf(QQbL{+I|)X3!qy0W+i+7Ip6imh!o*v3!KMdg3Il}6i5rn_pYA5FaK>@L-6Kg`En zJ?P+?Att>WSqsgb6NBA8k| zH0z$4f7?)qzi|Tn7sc?uOQK%Hio{=M>9CM#!U2%>Z|ZJ%J#OxjF{8y0wRuM`1xtJ5+~5K#EdqlSJ##>OR9E&w{()GH zmw^tQZU{Fe-u?PCllg?jL5+yTS*(2KU`Pc~+?u5^T*RyhgMfY5)d%8RA{sDmu^WcB z!`s0qJwF$N$xlf`3{3bR%~Ea!l&7XS(FacjVzC6gXx=LGf1pmjAOrupkT+{+u1G39m^JI4PzT7C$-em@A@m_(3n@VO z-18S{73i!@^}!YCRJMmASSU+Y_(VmRzktSZy+D84H2~bPMYV(mI^BO8Q0s|;2}zOp zfE2JHRP%>N4m|JV6V0byDsu2Gp-hbX_Z?|&XSW1%NJbgWhi@T_ib2XTo>k^S;PGQb zJz32m|5b-Dj&e*=+@Df~JkDGytR`nHzk;E7hL_t8f?sR{YLSg#uWnMAqiZzx+;o4`%Jox%u+}^6WvUM$ zH6s|t&vL(I4J~nSVrBo9EhHbzpY872QM4~ECYYH*s&)LAk^=Aw=WFN!yNHK3TA0+% zX;_~fW{4@kL$j#vJx|7brr?oQBGa#JyHUYiZ+pD>9yc4C%c*o2lPb20dbww>*U;M! zV5sez!k--;#1LNkR)tqxesi@?{~STQ1=As4$EPetOu-2Gw|$sZ{2EVXQC4E54)s;2 zrFBhqQSEW^CsNPke+A#9p$Z3Sci0Zs_@=qwb7(&FR8Hs1I(kk(2XTAbpL8k&Z0xtL z?sAv%Q-V^rn)i~z-$KjVo$zZ-ME$k%}IxfI=Nvu$5t%6wC~p?@`IfZhKe zg8&x`kpT4V)wC;$llzv3^=P3%W2 za1_oYAd07~Ss%+vD(m(}RcJ=^|5{_gI4u_&R6BVOO!!%-_nW_t`5e~AlCyIR_jtmh z0}p81eV_7w0Ue_jbt$+&Vsawo3LUJ=anm;&#P9U+FxXwII2x&|9+;?^BPOT05xK=$k)aHgaA; zSS!E8BW`l!inAG2T_$gt!ure)4bF^O{Fe;{u8XR(==WYOEfr%|s%jnJ=AA^FSXzV@ zlQ-yE$_p!AW*%HmT6r*-jV92WzJ>yjX(hfA3iizf+u%Dfh`JW$hjG9Ig!c6^Va{oC zbz}>pS48>+li)I%nr;QR4d8O75h)O2o7|OyPwzNYbGSv#75B!HF~*^Qcsp!vH;!kk zNt=~P4>&wAHV5?KUub+E-Z?Vc$rrwcRCrt)u;&(QB@@+&Ryp{Eqm7N>u!BS8sk_jM zqHcL)uMFzwl7)2*RXqr!ICUM=PQP z@phmP0uSZI)auUlmO*|9;YRBV4&|+4LF0)=n{ey4z*++bjiJXf1ur;l8|YZk4-{Xh z%KFS*E3kUWw;fx}>lkRl7?fKrCgJ;wphh>9Xx*ks%%{=vB0F98-Vh$Ww?pdSgu^>{ zD#~lj90_t=SIPAt$G%E2o^5JAZHH$3Zk(EOoWLD^T>ND^S28X2Xy%g%eLZ_P>e)Q# zD$kc6RN`b=Kp>Hi2odw3qY`W_Q;I(>+WV$4%TTwf`?6ivf!oS{obUAID~A2Rd5eDH z*SLanZC;D8jBsbu98TE}yPUR_Or&EY1<@`4muTW9%wRUFNsT}Q+iNFY(Klhclk431E9e{&^|NXj{jQoLIB~$8Z>dvmBbxzcEaJNS_{emeGvG%tvfEK6Qgbs zV-UqizA2R`lZxDOzarJcBB*l1!Az^ySbUSFxC?^%)L|s}`=RLVIu`aPRPq%#5mX-m zY$ETjEQ|+-Z>qEEeKn`Q(&{@JgQCCcEEm4)CZXX@USE~rqnhkJ__mtCq&z$ZCK6m$ zB8ip6aqr9S%o~-L`E4ob+c*5ss9l|vrYE1^l}j<$UvFPFSkTJ|9^Yq>e)t4kSTwQ| z{c(1nDhvIam1H;^D!AYtWV=D4jO#hFkELR}Y(x^9at~rea-P750J56@?V6gQoEZ}!mAaZY{OT(8BvsQ3?c#W)`zWG;V1s8+R3f?h9j}RBLLxvk zvgctmqcCJ5_UcOgel)}NO4ZV=@T{j;vfrC9ADf@yOHNhp6I4WeXBGxfQS*^XX{Y0? zZZ2QFx8(mRPk`QAftzpqtS&cs>Ux;83`{e_4rDrr8u~L z22Z#EhM0)#7+_@fk^WDoDUZRjfZkh-a$yc^{->q40Elj@1Z^3r7$qo5p|ultD#mP_}=z zUnTPGW0Lq0*x)X58cGKXxawINj)v*Q*V=L1Ykx2?!TJUDW&*?R=ino$cm$vR^f>Sq zJI@3eNZ;He8LupS4{S^oP+6b{xrJ*GXiz3}@njdFQw9l>uF*X5ZwF8hxJoq>&={#< z7tzLeTbi(~cKISSexxmxg~s7NBh1E-@0MM0z^~5=H0P9aA#pj-zD%Y<6-+CG@W9=2 zST1;NAJjs2-0C?e&c0jN7=G@y_6eFjTH$MERA8`7x9OdNN;q+`^$=%2*aw!(A?Q( zG%|(YdTGNSjX}de-|a98`!xJR4U$vX|8FAtTofWjdDEB#l4J{(CZb)Y(+Hvcb&uIlu!Clc^>Ui0r7YjEguZZ ziIvOMxb=M6ec`-sy7l_uK8kB^wvuTf=&C``6Aj;d0H`~4!jKKH1oSTw;71RkeD?RS zA3mDaKEEl{!_NzsSu1LDbQ8NiSVfjPj&&WE;D>18Jf~rh&o}3nNzucDs&`W7VaxDG z-dgtqk<0$Q?^p~Oj1pRW9>TZ@R!1l8spaeX2aWXTbgFzNf=oehXV3NJc}S5gBKNQ?QY&&!^xmr}p&Mc}%39!n>S)&?nTJt2`#jY%_ZdG&Vhc z^6SW`0~KrbKl54pUuX8RjzAgt>%30y=_3ei6JJ{0pzPL{hX%e=zoxfRvmH&?R4l)J zEF+_&#xuJpQ$vMGA z(6yGOu>LyhkRwfLC$b@HtIe^4kqK=P zNw$=J3m#LL!9)#h%k}B=@%3`>iNj>(&ze8Y$$>kz?c8usNo*?d&*l*8+2{zQK|B7f zYlA9nCz@{eQaB&VjSJC8B4?K3gV#8*9DEKdmYlo*s6ZrN*#POTUtjR9v&!`>UO>E+)yu&l;N%)W6H04oZwYAz?w(_96^+o>>hw))vL`=}9OtBl%Wk%TLuef0q7fkXuS) z@R`Mn6H6_dJyQ}8jXV7V3JHl&&?N~n`p>aLC|yCxc~`9MazMdbXUNPD?xd+~d+D?5 z!D+OIdceQ1Ejen6^e#gdi^k^(mopyNY?Rpv&{+rdg;Wc+A;5R7pBx(bIRvQ>`IrBiK2iN*pfYIN(j2?fWQ7pkm|nb z_7*dOR>`kcfcuP-dERqOJ=8y^n-yBdDsoV1`l}?P{`2YqHzef#xdOethzA=CZhPgO zcm1=7YY#Gi+{9+k(EtXzgc10!yDH6O zV!Mz$Zq4J)^`Z>kf}YW~gw&s?A)QqMH1O1)E!isxf~YuTRn=zbDgUWu$tsG` zq%{9q;Fu)b^gNID?4crYVN;|=;KkYYw2Z)?L-JbcH>FHHVUy>zagvZPA~9|=cikef zJ>zE6-VUKH57A|D5*A9}O`U&zwM|^@*f{7k*!N|o6skX6`Gi~k$NB}?r?2htgYJ%(<|F}kVesGS#;$}Y-*KAh0xom4; z%u94WExBzfe`Ds?;&G7mSF`VJ5%tCMN!Pjx5hw3^^*SqTE~JrTP_g8RwXy!AePku4 z{z-%Qzafi}#G{3Eui&lC=-y@STIdSK2WwibwBzDZ8_d{K-qdnQy7ywsY2o}SQ`G0g zohM|@%A?2e-sQZcb!RDzbc?k{X}l^eTy!?`bfI08rvmSWoJ;n}5@a+v^uNW+FI@!gId!p&tgH8j;R782!k8&;IJ!h6ylCf zGit(HSG9$64z41OW4f%a$R$ zo-X-)8F%YKXSE^&=9ny@D=^?aRfPPE} z2*@68);O$uJ^ytx=apu#n`y0bn8*{Si!;cz8L`wriRCCMAizc%9WSsrY8(_?9GY6} zA2J(D?#(T?#L>uk3KcX6F?`?y51nt~mMJ3REtHf~D`c`9GSAA+U1;1JQ1FySc9I|j zJlwzTcDA54+gu-y?wc!DoBU1lZSHOHp+a!a)Qh1*ko|rn5wZ~=8Kz#xsoSITR#BNI zJ$4A9x?ok6}+Q^|1M;InVRGaj{HS(TTXaUgY*s!II^b7bEM^+2k zgIFu{MZlSjQz3&yeu^5cg~aH9J;i&oZH2}S$%C&J64ny&8~|GLHxrDy$&V{Ga?Uy~ zd>leA?XAs`q0FnV2roqgDATc$`%VL-oR}#kadW?ws?0By7WGo!$Vta9mpE2?sWw-c zEE5ryC=XCq8p+KRX%*&{8YqQ!2gXU?vQJ1(NJ`C~Mi!bCB^l&!2nQOj3tc5Qviy$)R32-wz)C|fEu5iF&j0O_(YMhvUR24oMQdiwIw|n zZ>4fl;O(`ny-%+wcvo4LL{IbBYTg&|j%aEldd&V-ZePomJ}v%w?plX86^8oR!Xwaj z?m(HWkw`jF<27xuqodoW`r)*T4WXjN825EobNzyz^U5rJ(w(fF`2MfrE46e}0}tG$C8=->Wl<9}oBEr9B1y0y{42@({Peg#{6eWvOf4-ykfl+6E#MEz%KII zJ^CCW{SnjXtHqs;wS5eHhO~m6=dsfB&wUw1zHAPNar^CA&x}t3aG#oZXhp`-rRVHq zj1u3R5%i^PoAj%w%teYoNujvQq1$u~xcE(z?y%LFi?~~&@U&$`ULWSjfIdXly-x5* zwo7gZF864-O{EO zQXK|GM!${}*QyVuG@Qt;ImN8RChmIIx`_xIxX)gD9Z4xU*R#{q*sy>gyfay?@}U)r zxs&>1QN_ppSK9k~J$24%6?B<6fNW~L^ix6AbujiPXUgAAI~=ak%1^1EFB#J&BAZ!b zzopBhDG+kLBS+^=-DO2>SBAVUEGqK4>9F%y&Y&kCBJui){!=@8QemWnu$xyWy8&ia(ehTl<6QRcu~(z+W_a#`D5y5 zSK*S}?YTBI;$#-ccD%Ml((WeXL_dkky)bd>&Q1qpa4?-X7GqpFW^H0)l^B7a03-ap z;?;dER2zBwKE5PZ*d&EzxF_X zadK}+;2z|=;_F4g(d4j#7;&#&sFx7urnk$_PAICFU(l2~brt^2=pvV9n-m2KQ1&@2 z*;Eb1)IGGIasTn@_H$z3QAA4>7f)=_i!nrjaX|tyL`6uK|+b4e^fsICbE{%Mw8KQJJDc@uy;Rf z4GHmhKDj&z8xh369@iUMToi`(P7F>yPYg-vfr~0GCvEf`w8RUZ0zIGbuGCEELM7E6 zrTJ{)Lkq~BBBGTX1wDmYu3X0956?Um_>hVgiH#mKu$HfOWw+7|w8Hx24$HpSGWsw9l zH9dYN>S}s;=*n?1)u2io%F`6aOcBi_*dO1h?kJW1pv6eH8Pan(KMySOQd9AHQi!Mt20{7LykGBeR# z(pztlXBw!C++6kxwk)@T>Wp#}Z}?zCBW{%s1vJzJK)ca33)6U6No-8aE8yaIlpSvf zqV`Q)OYRk@fZD#drsX#ZpVT7(%IeV^qIKdbnzvtI1(Ze9t>0hU=EzdIA?ZY+A`h9DD>>s~r`wM(l|$*HUGFg|G?NkRhX&KAFtPX82Ox(xtDmvaUCJoQW~W%jUmkH2eOv}FJS8y3pN@Hac~B_>RAtMmRh{Iuh?S~a95BQ{chXX*H?2H5fG zP6|1rl(w+VE2{U<`i#xLYy(lmM z&6XYw&^(+Np4}s?r`KxPD&ZDCFZZZm@g9oUMo}H~8f_@|zQ9>Gr;-~vUzCCZNK>(2 zhf22={i4dFN@tySI|~&-Goav@9Z_dl0NsH|`fK;o07M#G3{T0ODp>97QfX#H=Y!|u z$osL1;`W__Cz%O_RBarHBZ8ea~o^?8e+4Skp~t0LMQ?gvqr3F3O`q5lzMUTw@gmUCi4dakv>Va1NsM_vJ@PM=3sc8F#W-wL5klBY z?*+cux6=@&>un2bYHC)@$_NQn^sSX+2E)MHybA`ATV0bBP}n&&zGl`R>;}^SZf75? zBj~LoN}EkG%JLEh8Sxg^liV>#J-UA-?&@^^qK(XTaAsAZ?f#IwBvUcl<`um3XZ)Zn zM|RW99PGPWUo{CW!Ohh+B5mY7FQKbkI)^N{X|b}J*v4g6=msw_v6)8J;p$DVOJI~*!Ra`QzYp|4;ye;z|rPoa-f87=!m-K$g zaFsH+gObv`EHG?1KaOLUF{R40^ga1ofS|Vb$B3l|a(MC5)vHt<@v{L$nxKq+%b)-k zqt1`EUWklZVuNSqZx@^>umSIEpr>;o?fkvEuGo<;^1`QeykPDstp|Dd_Y@Y+u`5tH zfi2T2q0h-&+|tWwaaA3)riJhH^BQ;&hu@5(MtLf3f@?Mv*CxZ?=exTGmEy9VNA=mg zA=ku4MoShZ_KA6oNl7_dCWA~^^elv21WuFv$8opxvfsY#kCR}GNB=C{ka({(cWlMA z{N;<31wD1zcE7Iv10_Myd}RyfRd{z9w#^CFRohfz78yg&kk!b~?DG3U{1BG~Ek$}g zL7n_vSN@{a7_6Fe8Ao^K7zd6pa%{E_OzlwKduQ@jv(7F8w);-MVp1i7noEE1m_YX} zn>tEvAvuOvLOT&$E1-nT^BgX$IwFtc{IXNQ1>U?GV58GOsoJbEo&X`?(S(kZsBz@TF!uaQap!ZbvHt?uYvh8 zRU?BX7;his)7>N>gJn38*o&K;7?4dMus3eqFflZL6Z~Vg`$#nK976FZ0eEG)})> zIG;*1#}qj2>{<_~-1EG{lHj{amv;TJJTf-2WcEFr;@;)JeN^V4`Ia65RfKM-ImH@+ z02Z6IpPk(Yne=jtU+sCu(_u7Oo`ZIG1AsewnHMM?EqV^!KYef+;0M7jM_iLCq|`vj+g95s|m zMrXM@3)Dm=W<=ClCfMA$6>Ge16$7boLY>Mh(oM$3Hi z^-%D$%akwMsWvASu0P}9%RAdur{@7hM>5<$k`9u(e15N2*$v|MHHU+e3zB4N>=9Q_BoGf&4 zh0{I>-hbZ)KwAPaY1j5h(x#5BYTpcb;s|6k`7<$`y&ZlXq$>5(#}JTd<}hWki(0Qn zQz$}zCJWD_z>@fg9(d?yTaq-Az;`)Ur3O$bkNdarRTqk@$eqr%3$$HQgA^o#e~!M{ z#=7Ok`cmffr!t}Mp$w>C_Zc;9(J&?3rK1i;N?Z9m`o<`Y?=E=VV4bo|AYex$TU^7J;8N>x~=JjX#5D3d;&UsTu1As{5=8yB}tyR!?HIG$d+rV_8DbIPhp z`h_Jc%fDz&pMW~uF4O+XEoJ<>8I3Abd|o!1zvCNqtnkCm!teDcOo?Yc7aNExvz+6l z*DpP&VwEdrED6J_HoBy{j|_pie4vr*1(jJ)xSBBz`Hr-eiOOWzj2BIrtS4K*cfRA{ zKx^6*eruml$AuX6`+u33zEFQ2;=uxkidp|+W5I{u3#tphf4nnC5C#2*_nrUTDU63R zQ&jxL;7nXvh~6LkY<-_5a{FV6hx&!oh)~Yt(3E_SC2{q`&v2=pt_+v!_rvnzC)Ynl z%{dbe_ZWF2mUH!L%JDSEjvo*F?weQwUYmNd_lBxFfA}CXnA|`8X>d-rpi>QuNckQR z`|??#`7)svKZ#3!FI-F+Q{?xSnb)0I#yXPNOh)jle?-Mw;`PRUgX>|`e2MdH_|UoX z<(6Rs<|nWH+JubK(y)gO6O*CTD*EdHRGm$m^QC7b&Bi`;#rKMeWI;baX)0(m>P-o& zh6bMM$T>=#C?gntO+Jk?hmMbliOb8?)LwmD24w?iH4i+Pn7U_OPE3N=M1D&0mg*Zumg>ms#>PcnZBHc3Z|NP{gfYWL^k>LXQ^gC_-;~1HnTWj;S#srHQb8oq zEWU;_i}#WazkkZBy~O9!3e0^qkkub5Mo5t!C zm0+re;o?5banD^MFjGjUerxPLoz{P6EXYN0Rp{N#&_eveHap z{yBvnc}bcGwEkcR{lBQDpGD13y|g+*76+VJ<=jv$M_ynhyetYzYGvU zT#xO;0&84cJokqC@`Ixq@*e-7>)(oBJv-IKlVAAoZ2Zey0o3+C`g!ACk0q&)grx?^ zid`|BH-6nwAN^E}=7$rp2!3x68Es2Lwh32Q3X&dn4f9t{7+eIk#k>kD4N^{Wf<=A= z|FBCGdxjj%WF7=dC!j+w5IciJ1JW-%*0}ROToJe<^dCI^|F@l**0mA^>doPgsrJ?;x}CMdcx z#hsN4eljyFxW~ZBc>hptWM;=vsyb4zxZTmVQ~wEAb$R(kZ^g3#bMLs^vsVaU!~=%` z;(^jV)eUe`1p|L^JWGlNs_(MQQd%1F&E#qgIsFFWnLfkBXF+_ey0D-(;}m0x>3$c8 zKF@lW3|_7G!>|WUMQ*41#bN%dqh+mV(MVMv*geYYxWr*(DX6)?XEd*_%RG6-rAlY` z6b5T|x}UGgR}$xrl$eERos6A(Y{X;~lPdcsmZm1I$521Jub%UvwR}_RyJ*|=bZpu| zFauRT;@2sZPKOQ?#9L&Epza2{sbIyY{^NwJrnb$2`|`?cAqYLMIqfV1=iM=b(i}%z znnutKbHNwax}i;k5l%BhYh7^LYTZTp?QT$ZNLM`F6XVp~C)R5~Y{wmSxyNz7nx|j4 z^n58~;nMCh**Y|uMaf}QR92(-bDSwY@9T|Z)in?4v%Jy^QPu$xWzQJe(}fruGS`80 zN3!Y_o82?dJ_^UJCq29GTbo*v@4w{L=+>!+eu=_pot_E>xfwlm!QQJavZc_3KNq#? zl4g%k7%O2{_8U#y4G?P?DwA7X$~LUGf48lUH<=^bD7LjWv?$Z>Vk9KmtaLI)8HYlX z>++-%vxTeD{2o+)H69T`xysed$fVMm?9^|ieS&}sPT5Z-g>X!nt|RI&EtSJ>BkC^3 z2z-0)4pPC$H|L(}6gW68r~PcrA-wfZHERjq1gor_JsulmVY zuP}T(khaNEYbM?rAEY#~nGMZ6kd(RPvwcWKm1}UfCkv+j#(sy*(EMhrJvRH<{9TYun&L8)SI$>ST!yjFKbna6sQDmMT2aXSrM>U;r`zNy%ifH(G3IE>0 zY5pDg(R&rUxHU){WAzd>S7gJPgubT(r2n34O;J2lkE{!Vbk+6(bDn1%-YaJ29yuv3 z@Z%=_NiIzYXM_bbUe(5XRn)4zogS(Et=z{#%!)d%2phQItN*s`&%%tpxj2`1(qg%Q zHk!3IS~_pw4?xz_s6^Ys>29rtNSYqw_N^)#ARF=K;Bb3?ZFiqAiUjt3$p{ii*>3Oe zZ60B#(FlL*Wtj_9_b6$R$B#- z&;MjY8+WkMp04wY^@!PK?#-Sxhf~Hn)wWL%|19^Ru<040Ln^z+FKq{DL)(WdOwa+Q z42pOR8N^OcscQ>HVA|?FwcnLVUAdmSVYbX%BPgjk;gCjS$(^^^gCi}rL_P}R>3dOf zSwNn(j=QaX?`(`E-$P)uMzeEX>>#&z@J3ykjpMmupPk`xHIi5Z{vpe-=K`kKg8rYU zBQ$3RBAQ^ofP`?mcY(?A3VGV%SMD>?0dOYsjS!?LYsx{|e-bEnK67;3uoizK5aVIO zwlXz5BSdnXb3rJpMn#K_V?rH1_+ustgExh+9+-_@S{f)@lut1E<8cf88=(Zyk^;t1hV>|;df z6vYv)T|(S8g98(pVhPcNwO4FsM6ti**&bZsem>mO6vyDdeuC(nx9PVrp+tMo%2wFf z>)8%U*ZI)-#@!P4xgX6-tjFK>a(I4VejT#tSaj~}{b{8{N8+{vJ@3%w>F+%fDDg^b z+LN3hW@n5CwgH>9$Ra6tYzXE&&O^C_vF@t5fu|u(7stQA7{~v1rGKl+X{pcu%69dE2wbA5(c!;5U)PIG~hB%PDWT^^e_DI}0yg7{S0sAGK__|6y#)lxQp$WxfBo&^b#GYool8|MDegO43l_?(#c58Vum{Ie|P8BjR%h2 zrD5dOh^BNp@b!D`?-N*8Gl3|c8ztgWUnoH6)YD7DW3tj0HeO=&4j$0?X!NP0>;vFp zkLYg--JYCfQ}+gX)r_dQ=R2wZ;Z@wtW#)i5kl)>kjKhw3jH<+?&`G0bTjci2pr=6|yI#l}&)yON; z-48&*koFNfEJ&6Qe|w^YckvbJAx0UdjEANLT&Y&Xu%6N}5devW4i7CiL4R72Uo6gi zUe&dGhgAC}CR;zNQ;-DQX+m3Ewti|?5cSvbgwv$->CHn-xbIz6S-1#mlPm zB6jPL0p&#{k=%JQRk^q8j^`FC0z5nUp;bhXBX=KJO7!4vFjUml`6^DqA8W|^n7S310lebzusV#zq@yFJ3^mso;(uU+tXL`Y1P)#iYS$|~AG2H6 z8%+q3M)&tOz}c|FC`fs-$u0lcPHj%$K{w9EY_2dlYnS`p_UdN+;0wnB+VkQ!{A9Pp z|C`sGrX`tWpv4AyXkRbwWfT$zOnqO$kb#9WVW}CYSVDjPlX+-X#@25f!`UAQ!^I{ z4n|9mUzJ}0*~oa7raP_$tFNBr^|kaPa|kT34~yW zL!GZ&yHrHJ6$IGvBLsYUT=IXxXQ_E$9-+~fTahd)Ao{DNu*!YczW+*$;x2r9Wo2%UZBfug*h8?QAF6Fhw`H zlEph7SNPpIrz%)Kd@VRN4C3qqiS#_n{2-NTv;77;WZk zrgIt^Q)&LBhj+ujSL#b)W3EO;4BW{3;X)wpglwy%@gYup$YFMDb5G#II~W4sO&l_3 zTo_$xp_~&f_rTbk`Bm+vAS;|%Ph1%8wFCghK`iUDFK2{zk&ynu5b6BZ)=}+9RC;ng zKeK7XiXGu)zVb-Sz!yTUXBwQ6koZIME=$iFf%3T;vFrzIk9xpCx;{Dn(9P{dNn(U_ z4`J{%jg3+q6+>r(nIs_+K){ArXh6rP|K`J%vy^!(qIG*ia)RM&dd5e`FHi7tFu+5S zuWo1iH@R2d3&bvj$)OdGcKeFZ0(8{C5#kVK2bXHtPiX@eKGGdbd)iI=WUlz%PZ+j* zc?GjCTrKw0`H=To%Gkl=BDhK56S%jH3`-9Wzpi%8*&t&G%TiU4eFdz}Hx7mIyXEd) zp&m-g^-BlLaNwc9@;yM zPbN?CKIxzj@%s!bvd!UQ%LeHv7L0c_=bFV)lqnXSn$wL+%*rr*KGgd@@e#~{uwY8M zwU8+m&O~EkJNr#A9Cu}djH#`gnpW@f>xR7)b1C0` zelOPLIdf_>`Onfu6*!Cudk1gQQ?y{_KG)eM>t)!y`7t91zDvL%VhW@|Ag=}ckSDs5 z5@+LK+D*D=)?RwUz-&Ai&OFzzEcIj)^_z7grPvyY5;t7g-&n3066TZ$zFcO zoENH$gg#}<@~TuER@}Mi?Ht$T@=T$|fHK_;aj!11J2PB*_eKUc@W3w+gM{1!$&4RZ4uAMjeJAZ4AMat;4Zv~aLQM`Cp-V|PlhM9`{n z$ivC%9ylIGVBVBz?u&x$2LRT;6*_W@&Yk`isHr=O|1E4O3wT~NFEz-_xc^#4fr`;drQ=>2SuX6DFL} z$(3xub*a?*4>0@s^O|8(-8_#N%Ec_ZXH#v6HJec3(v=G3;J58=ujajuq#FhTl2*u& zF+`Z_{dBb^UZpQT>~D!)_p8M8&~*Mq!>PNF5|)%gN7m3K4r?2wGb6Jm%UQ*8$L_8- z^awV~50`Nb>H%fyy2PC19hy1Eoq5!o;>_3YMPi`i)-2Vx4u1-e#rEWDA^_M`W=SF8Ph`?t zu_FKc2Cw<4M?L!Rpf}O9ASLE)7wngC&UYp-86a!>%|^WFd@6~0p<>fz@9tG>kd> zEV$y9x)JJiedjuaoxF5*Iq932~gF1`NNh;oph1X;koU{wyQLA=ovrpw;! zq9D>WyBhw-kFsC@;C~q6Y!1!Kf}+7x6CQfzyP@&!VTvFPNl7Ti!G_yX53-!C=Fe`z z{_xsIe6(n_@)OJk4D9aBxfW^0Xe2=#wwxoC?Zx z2aYI}8ZSOD?G`=|RQATo92}D4;o^M}PM5Qo9uYUNFAxb&jrwW_vcKtv>!P?d(rf!0 ziV7NT`&Y+jc;1)_NxN zy6iJ;3k~tzZJz&Hm+y8{b)i6B?NJYRHSSTk~KC%$o`V}k1MM*0D!qkif&_u z(XG_FCJ5pS2pPr(p=*Bsib5qWBgiiT%@pRq^#n`&zEUnqx}w=iTgQV$WPtD08Fnxk z0MhGpnwu^6+L@n;Z)(~wZ+{@iu)mP6q8##Phy}`d--t-2_BvqfDk(6sb2a06(*>4Q zu9 z9Unn~O$PF?TW)ip>7heK^TYZP(Tf+6)?jc2XkIA#63n>q?LT@1Tp*JDkH`!Et2?6& zYnFab!vDp^!y7?wC+HdqnZnluNJYiPZpJ;VfJkT7a6nRtT7!bR zdJ%HK%mYAtPSS&lih2u|1Aw-e0SUO^sv$O3EX?1$OkFcQaP=uj0uJCLsI4G5o@22! zz10Umn+_CIctw5>B@!2C0d;e4K5$w1;=(rPjrVJ1^N2L*EeyHXay;*4fzr~j+Ev@w z{6=^HH1<$duWJ5vW-3Joza+3S(Y-jV{kvV71Mdu~NQTSh^X)I|f%yQrieKUMkJs}k;Fy*@d6j{SQfA>fy&tmPv^yeVPwT&)dBMA6~GDsaPZGK z37mB0Ts1v-fo~YTpm7>lePAU5{t39CMI;STn6mrki;tXNA7n8;l`h_n!XRV2gb{Nn zDQ)aAZ_I?OI2}LVZJ@!hvLQIEA--HGct^>m+L{*&_Sys-eC}4>9K}bpS7IogLup$V zJM3}s+LDPlXUstZ=BjwKabWP3M$qs>O{5W|OJc?M_x7bO?%Vd(UOIDXQFk{iy4cb! zGCydV6wH(E!n1Ro9#t9>tJwE6(vK%ulOA(7SOXWVZBXGdl^uo+lFe`|oVJ=8!v zsk%pGRmI)Pgf!VF@k=1oA)v9cI3dD%-0o}WmZt`3yx|kFrOOgGHQ3$8I4@Oiv#F2e zI+AOx`+{H`9pS=h%t7J3PNOL6F!d@v-sTm&z(EfpdMoh|>QT>)kXUs)c3M z-F3tKv!wUq$dFTA6s}$cPZNv3gqG~rU)tx9AF?UTkS3|qPh8y8)RHPc1ax}dRc{Oa z&T%&8Do9$KoSt5{t06|FZZgUCyAa*^RdIZjey9;PC1Qi&jkwU((ZC#wdQI376%$98 zMPh>2-bE_)+}be=eG*PNNzl&>YD|GM4pXD8yEy)MTynba2T z=G6wFx68b3{q~J#$0&sXrzv5ZIaF=D0SeT^E!P3DTa=9$2qc={gL5X@|5%p={v{a+ z?SS~k(yHCCPls613)3NZCu7?Qq{e^e$3Sk8Pu+hFkc;?gy`*7xI@5>hbCed%li;6sz^14o;o00Ca zYuw9%JEBnbn=?<*A#vW&Un%7w5gcfs6m~(qzY34$_xzmF4U+4bFEhpx3B=gmi5imk z=`=CDK6KRi(xGnWSj<8)ZABUbr4?Tppm?A9Eu0URsKrk1mouU@P+kU7&G>3)w+ zc&t}r1~B7iRrvVvE~m8;rL}slHR!$HxtaIC#l3%8+h03d!ixd`sFD|30K(KC>_<7> z2@MUQumB4-#|{fFS19sFRb3H>$G8PntfC19GDxd=UMDoGXiyRjUaOm_g6|CUg&Z*i;JdfS8 zsfObQr%i%Ge}8{s^k@r;kgOPd62jAcw42YwOh*EQA7s`t=#|10md7_y{iL9Q&F@~A zEW+)uKs#ZuHvvJ*cgKw;voQ}XRntLSsyssq?9evC3bXs1?Z)5Z=mJayQ7sb`qq<34 zK)acT)85@{tD)=t#rH+=c=Wpt>mim8T@@5rwF%JIzgix{ak}X1Bz@sX8)fqe)5M+5?- z*9A;!;_2YH!N1@a6Z~M17)Q}P?7MEz>84D zM8C*`F0DyHKnNm2gGVg=W50kuuWUqB?E&C**UJ|wmIi#3{txpJmz2XL?GVjgkJ;0P zDdOOFuShFRB~;Mk4)a-4Fvat2@rCKp8O9L!F=JEOqo6iQr?5%Kf)#P#EGBJep}uEc z2FKP>LulT|L@4Zj)60OvgpC;DG=hU)_f^4{M+qvY9zan`Z1Trhzgk=5;o*56IUX9i zb-up;u3P+iXY}}CZJF43dn7T17or3O02y%4#mAtyfD%?B761sSTPcb^21q;?9y8ix zJjO?R%{U{1Zo$`;awhS$dB>e`ILEZy3XZ{JCJ0plFU2L9084Fx{I9$Fs=-6^cIFS^ z#E^rEdGa)mt2x0A@K|w7lB5LHCED`|S|oqV^)f`_2fE`?LBDm8{kipnQZ)BU;-|-b zey`KWCiP&jV&CUJ9IPd-q^l5lJ{=w?$$056(+&r5Ur!n^-;OlM59=Bm?@TFcJBGL2 z?G=H=TeN=7}7?0-jvKh3%GrH?W^?+Wep6OZhi-@40YiCgnOpBS8t_l|y= zbbe2>bvdK$wVi0J<+_D=zDiTpd^W@~+Wd64@9`B55IFvlI6Wc5|Jc-a-vs_R*m1nv zt>s4(BUr01&nXnG#*L@PkJ^7OnI{%YDOWc+mV zXCrue@_N}_J+<||uJtr@Z#2yM(*uNo9St2$!Bz26Q(>GI0AdUz_m?5ZkD)6!p=F+z zMf>~v{VS{w+q+wf%e&TvGs{ETByBbi8-tfh)(krs$G79@@n(b|WP(lN170d@Vdd%S za(`xnhtH4qA}?OMSU9tqBN`9M<$maR<$QZ)`<%bX^T{{LQb}^*?Axc`Yp^-q99nxdao4xB&`C5OE#F-4rIn@5sTB8`fv<)<-3k#ur^$;K)M>e$ zj!t#1m={s_)tgs|)&jsALp5=#7Qub>l{=*gz9Y}gZsI2gyU|UFDKEY`8yNZT6Dyq8 z?VDYj65<6bkhA_3ng!0i77)WqNjWnXgxn54%ZQ_nmm2L~^> zv=YADD*~Nfe@o|fzp-h)lVE6>R5Jm9ivscqXl{XAI6y^lYo!MU0KHlLdv_EB&j4ds z%yi(1+6Ysy*H?dSw5J*)y26D5im7}4>9haCML#5wK?Ol)p!U+u8x*N7$0Zmp3E=^t z9jOugmpJOa-4lv`e%pMk174vtP$FK?8GsgT1ZMEgxk!F5PTgvk2L;R&vn$BuKlA=C z$^H-8gV{&YoH_A=5L7DyK|P7n2Fzlj9-E5AQBz(<pp+cvg1c1 zO#C;f}lWIm0mO+xvpy3jPX9mghi#DOzw@TEEc=)BuAMnm|)L=6CpA4qXt)GrRzzZ#ddubNw( zu$vhat{g8rG4A#*Rf&bhB&+{EBzH1 z&n0J4T5kur_BUu9PI}7#01@xwOkJZB`a;zI@?S!q%l5m7)J%~U3x9CjFb0Q^i1+M} zS=GYN(jZ`z(5(|d;QrTP3ItlPB{m3WQMQrl!b5a)vz%DgIyFWZo8TNc@anO&ohgKc({@ZAV+4n3&2`7VEe)6u0C%9XUUkhHFfjEqkn#ygU&zI>Bbx68gT44YOVZX0j(F z;ac3yy5dapS2cnpo2x|uC+) zOCgi?tjU15Ptx{LlvTY$SnE>5@e0TVlK36hXUgQ~U(EFBJ)KOUlvzAN=Z|fFzHJv5 z*M7?%w-!opC%F+Uu}C^SknKV`O&i4nWYD-_N>EJYnT*t+v=cd$=(lDmlVea;! zEK8E3Ct0!z|K$tHWI>WjLn;zkXI((!Wj_+j;nlGcxf^K04E`jsymzF?T$a6g4FhBw zs6}eWMDi1o_BSN7#gtH=N$9en7wi*ZUqun;ITu8KlIHLiZV8BKkv;wN{z9nIk0Sff zH?aj09>3Rj_pLuVWzI2wA~zIC>TF5DTDMmYLR z<{6^yHZK&ZjNTZX4%zexxteRO)= zDZL95IYV)Y^dvOe%D_h8EGHRpKeNq>UMQFIejtPIG;L9Zz&yK zLgRS4wRh9%drufugUeN4+gA2XeIi=C#PWzHD@J~BxrEcP;;SY&#a)tEX;3($S-Aha zuY|_7r3509STf>bts1+%3l}y@d1l9oOg-|bF`T8!nd>$*fhViv&LvV5eoA`KG)^h> zl`2-J@(zAoZ7rALLRHbAZSJ>ZaVkecFDP?VtWG(gCx~}l|BNr?6GN+I7Xf|c>o~Fy zmM9MAPCU)vm;;*M!p!NV!gL1LhGg8z?uJ;m6SwJFTx~c$+_+Zd139G@qopH8@qNF5 z;&MHj^_`v%bk0~*bW?rBY5m-hDSKzsN28e&431Jso@zT3O#QBTJkGe*{A*Z4sjgF^ z_caLWWL|tM=?0y#hjTwDjN+NqW@kpsgZn#{5|@k$!se|ofIX>^G~T9)inlaxRhCPi zCumKL)Q(W3vjPicFGuPrh^UOnlLnk~Be7!aD^G(h2qo*&5^crZu9w+vP03+b4=J@c zmV?V1ZBiJ}^RnHtKNtPD;SGvMIVt<8e2-PQ-+dgViFpuvJowv58DjUWTw z@Y#t<+6w_!`Jqy%vT@f$Kde&fyd~s&s8tznMAy;^k1e;4+=Jf$vf*<=A!3o$Mj>MM zMnhJYOj@u2PSzb0-Eo_LtaYBX`LpeC6X~tS6?#i9Qr}O$#Yn zi3rO=J5Q09zN#MUh?hrnRQw6T%yZe#EBNjoKVJX#A6Z(+y>_d8n}@V$Ykh154px&= zs>TXIQyF1}rXraT1@Soy>vy`2Udib(%Mp*0)62ogmmT{1;sr?@k}&O`BU?q9H|y*T zGQOVe8Y-dXIE20?7bPNL3=lvUv>Vmz#n>&q=VXDzN3Z;J?SWu6y$h<^zBO4h?i;LQVpwn9 z?qvuk2bQU7l_)iv;h&mfqoy_9$Z8O^R+nQ9@s01#oRF>(WFIzOXN1*mJ7Id=xD|p3bQ_^noGq^POS=7;%CN(`<;1vBUq~rDoST|E@R1KX zCW00K6+LRqFh6|4FbZR~TmgFw0&swP=Zpr1XBmOtANAtUJ#Tico@4 ztW+)hjrR{hH4=Ahbb;qR7P@O^Tazk_f1$R)k#XGGvj56Ork76t$#{adzD2pmlB<%; zZDKzW>z{ehGB%8#7Sa1-UKTDTOnUtP8Q(S6qFaV6iR6~84R-c!7McZ@ zqkGRQpu(rBspaCWVdJemifO*FV^hbbF_qj)^fDk^1W&@v$0%_$Hl1|Y;Z^+Eg%N1I zm=o$RJLnUT_&Y(PwWKcZSsv*ZPR8NA$%J;kSR!*R>b%sOIM^g}*g4vy?iUEFj4p?-V-j=djkiJN;0)d&y3 z!ja9kkKUsSH!+1!8}Q(?sX=_)uA3h}ge<=aMWb#vbWtA1<_8aWP{)}LhkjK%qLnww z9q#~q2PI9=`EtOc1_$V)i~g|Z`Q!`<*)7Z?Yzyf*d?%Z!DQ_@D5J22AAj>8fCkk=-fJ!K zm|jH$5m7=iu#NNe-NuxiR38>QMUu|@)zD#m5!vy^kF+-HP!_)ZQwuWLm-VYK7*Hh^Q?tiN~1{+uT%L-Z>jrNp(q~7E1`g+k)$I66)4I zlOC<~Uq;B{vqXfDAs|H$e#qy*wP-K=iSbOL+}#=`p+KAx^jfqLB1@Qz@Ho@W20Ki9 zq$WZ8J8_Wd5`(N~N_^iCJno#e#V87~yew|Zudf;0(x?HMZ(NMe`5&97DAw>cH4?6K z)g3M@0)u6$y1cH)j8Zi*Z^%5p|rd z-lM^#+QgNjy+RiTLd}{9K5wWH?@tqvlwxBpM$oL05cF` zuz$x!?9203O8(%J1)HoZoJhl6pXoYZIeC&=c+7I!noI>JJjzrO^m}4$ceaXvfS}^} zm-W3%{=sN)-uh4|YjVL8Wq5RsIg70ML^sqST2yIKq!yD>Yj}W@0t#VFyZ*nnngjQ~ z?nh+#Tf`0ov8r`Sbih2rEwcuCAXX&nN%`c*v^jN4b)1Vys$Mk^xX!@Jv|HF`^U#0D zPZ((qRNYIP*oCA08oXnodp75 z{@>^mJ!rua2U7<;0`Hs2U4l z9gMZ_;LYD+73yx!J9tNtB?~q0);f5F=fd?F&l!un+yrUZE`DHzd#(MHuw=X2tEelS zSqX~~3-`Q1&$u>GfVYSD)oCk$Qz4pP9_skec+nWs6g!LBLE0E$I&)L2yqs(X8X`_b zgyrw|6Q3YTLtfg$#~;~yP0XeJw!d}qTfum_U*RT9;!=)elW3*EDTfqxxO)ueXDgrisUoy}$O1WV)F<8leoW?=e)&X4r_UiivT) zI~i$K+cO1s;XR?5XhL*??okjbg%}YV^1@|S%)e5ExZjkRwIv5rMG8=p=k|=Di&~?wGV|SdPe{a1+PiOr^ z<52e7n;pdO>V|f%&^24 z?_9_BOcb>i&tFer;#o^;0L9h%uxbgg@hgjmXfW!YLM>@1WD$Bfh(^dL=g^Oa@;ST@ zrk_ul0#{gRiPL4_=f$@4cmvTb_JwP>Zie4-m$r$3>UDtZXq(Fgejf8T5yESw@cY&n z0PwRqpXIf+;>`|d_J!qA&qr{!aWxYdy*iKcYtT3OmDUaWKb7IgQRl|GwX0639^w9q zUjjbDWBXAMxo=ycs>|Vp17G2hogr!4+=fG`MkR|T@my;9QC#!$g0fm;)r3{je0GFn zfY3HrR>n?{XVa=6Y#g$`hc|`i4{w?tvN(frqv>(Q@vdkZVj@#0Av~Yd9tKf|RxJ%U zQV=E5(Ommv600@dc%4%ntvHm`9>?|bN1RbJ7EZW(zB|fChHfA}Wazt53(29gXYntV z16VMW>GDSH?Vn(dnan;X6kEK+sZeolo{HzUn zt*qwJb5BuakHVh|Rf7A9;N{A?9P??j^J1Nd*zH}P7rTqiJ^KyiFucy9N0KU-G6|iH zmq%cJO5@@5#hIB$x3+pWF^(T`YhQtAmdt~$N@0OHWi@L(+Pr&V>|=p;m2~i3;1q@8 zs(*4Si@%lDc=ebi3sAgYdrQ!YmgPf6Eai0;6&M^ZvnphFP;x!pSNv>5+`r{Coh`Id z-gHx8YD!I~`KxELli_I=3V{U3U zBu=$s+EBxLs%!gHkRes!i}Ia7ajh|K;cAO z5R+L+px6VW;|L@$%U_RopJDwE<;{+sm3r^xZzQ`qk8;n*UEMdXp~ncTK#qBb41ZF~ z4Qt3~2TZji8;)9d;7<>tljX{D1Svj+elm_=loCw+17K~aG@;z)QI;AwEA%k%nvY>d zl4^~qe)EsQQXsLq`2r4hE-==6%HG~DA6&+7_z40ou;TPZQ9yKgKxuPhfo>UZxyN7T z>P%jTDSBDTF#G%hyppLAR-33gv=D!vBH!&D{RPj6JNfWIKd>ZpIjW4QvFqcM$S(QOnxcm|i)mUr(%MBP@qsl|L!G z^#T&nfYjN2Rtu5b^BN9)ho|0bl(3LmB#ikh3k1T;ySI-3C@+i@P|}hW zR<39rJFc1HfRZKuuteNPjGo8h1vGVybvf61K5G$P9Bk584&`GnSknIm8$|0L$X>b} z79_(ImANu4*am6y@!f*~99gD}{vDjoU#iX7DjpY~t(DLIQzV;0m`M2-l7PYPGm@B# zUxt`2bZzTB2O#U2FgYeNNmDG zZ>>(1j;3)1$!p8<$bs#omE)AKqWWrjfBHE=5GK{lk z{q(X3h6nGfl7G)Ks?@>~R{bqCCQR3GN3Z>KAXIUz2~pN7$MP0rEq2&=7~VX&*r9qJ zD7x|@;Dy=g6P$H9|?waZ-d}W`0TfX z1v18od$-Uq#FO*j3AN%F1><^V*Bq*=KkU&FMf51(U~ zz(S6$aR4gcAU?dxzup}jA7+k_#yl58B13b>J$gW_^4C zpT9QS$pdq(G9coZxn<5RSRf7wg>g@ZE%Hb69dFT`>|lkE@-)Kh2M8HcY^9^*?O2L@Pr9r1f42INUeTY}*F+ZqEmrr}Ve``=Hr)Pn3)+w8TvS zAl?}%Qn_XHqjG^;0=agItwuKAer^nF*7mzfN2XtH;Cl`XvbC0s&7t$m3z2zRcS`G` zz4(Oh1r>-@KEo6#PJymo*b*6#6aE!<+jWtLFT1^;4hprdwEdtipCJ$%1P)359CwdN_Q^cHWDQFu7xJ zvkdNTacI9n^N8(k{rF~enz%vxW)0dTOVWC_zb9FEch%QKa_4+k?rs7hinDIYaaz^V zGCYkz>~ndrHMnUa3RE0zX@l~yEoaRXomGiAAi5m~UD}%-XmVeLYvy7RWyqFuXfcdm za>3HYJ+g7Dll|d^CP4>Q$yeD^`yVro>x6-03uaTM8Agm8D^iBFz2Memcia3T=pGkC zKl%HAlU2FZYPPsar*M8oht+M6W@8Op5S`AokiR!)pD+hsxov zAbNIueYM?^4G|9IcZCDLt~g)Xx@LoO3$l^bo{@}CpPlq2F zyESvYF1R|^iCZriJNni;4+g^-BzvocxIHwlBFte#y;ns#T5AtVMqeFI4XgCmaC=Xu zgTbkpfG`Bn#aJ#~c}f$5kmxa$LD?S##ihZ^^0qyly7f zJy(x^MSE43tCbA8oNk@!GI+JZ&bXan&p01cHNL}p+_ItPQudu4Fg01W&FRZTGfZEW z1IJeWloZ(z z=eHYheN`8F3Q&SMGbWC(tqmxjyMWrrUEp~uZ+4zT4sT?vf5keHi zR#L&J+qZ8;iyHnHC&Gqx?QkpgA+NdoJ2p=wZ~m(wM|~O7Sp@l z@Fi)^wHnJw{^@oXdV6;+9lRLTe0gXN_iVpi`e1;%cUb3xBMN{fu@H?q{-V3z4&*~d z73V0UY47lx@pqx5Nq)gft>+Pt#k4aWiVNWF2(LR!lXcgTe%`DgiL!!mwTbXI#-dPk zoOIv6Rd&8*Ux#WI8E>^@TZzq%0sqlDLiT{+Mg{Ku;o)cEDAnmU&0dw9zeMJzMre+6 zEY4jYvRdlF=yh_SC>a=Uo)Cost66ZFV{QUvO_40 za)sx-lI!ZBchGW;z18GHaYi)HXj%ecOc~{IfhB&=y+t{CEU4r3R}Tu|D)H^}8nJj` zNfJ-JZbF->ik!^a$_G9!+_X(%)36ei z^WtzA`q;Wh#8S9QPzxHD&Dc%#f=FApYKP%TO=wvAGh4T~s)N4KCoK*>kK<$y zB=>YB9Ss`17mKXk+3A+t+@*0taXajjBmA-*)^nEoOV(Skc!0bf!nTZ{ahyv;RdCZE z*^}J6D{5`caeic6@dldb<~SlXEWL=hP1;=YI*FQ}^pI4x7e?&CwOxhBj2pd9-7Zlo zvVsxs!pH4x8bq&vXUrZ(EKJVKeRC0$EW8TWZ5npwZk`^Ow!~G1hc%HR7Wa97G?PvK zo?dN7e{S;q(`gB8(YICQx1y=t+3svuSQx3Leoe!wztKD)GRqx-lAgzh88|OJ|K(&C z1nX{?|28W{J#lp!M%3vrthTZ{(w1CAS14n!97ITpXdgTP(N9E}y52FAxx>5Jbb}1y>(1Jp%sa0vtK*yy zq%on z=%4PYw~O@EeBkZeeG7V&J4`I;X(YxMJV0hIR`55(8~${YX$5|L6nWB6d^2rUxwiK& ziF8D9np*q{eSEEAC_7{ zJ+#)u&c%=`$6{ycqlU6n>j(A5Maiia*QA#M137KBFA>B z`((*5-5Fx>_y2e@-ZKN@Y1(&(Efk408uJIJi7MxCCw%Sf}rYu{Mu9)>Xq%^QE5d+2>g zIv)JtK$nA{aij|eMliifoi5Mz`CE=i#}$rbdP}uW z7ynZDcmG4-Z-=o^SogqAh#UN{1jvqPFL2rIw&oIbjQfax&;}W4WKOKgBw?^m56-2b9)c2>JC&+KB=3 zCrt0R(E8e|UTD9%vt(WTNbG#xEM`mqNJWl*yy#{voMVq+1m5;8N%+miGgrqn0hjKK z=0!&mV=!DqJUXu0wxkj;x`wjXq=A(e4EsIo9K=3A4FRXL@Xz4$S4qHdyh-TekAEQ1 z+~dBtQvKp)RU8*6Th8&N(2M$WHLzz2h{@?MwgFA6-SI#++a>>4slf}u0mzf}zo~Q) zz9dt;&vmfjS~i~K?d(#g=@-=OmjrF!UwOwxwUBx?|0J-xlm@htG~Nmo%1(lcDTW3z z>8UL0oo|46A6ixGY%OtN+ULrx&_aMqYxSgY7r<$9V{hNefE_`v>47wjsb}}W9pLzCLJGgDwP8TK zBgR3($ljBR{E*jkBd#r8f4jk69jHXyo56IWKD@ceTbugdQ}kav1H|OPNi{nTKq(pC zb&e|f#R&sWsv$wkdud~+CAzlrW9I0xgWJ)qo})D!IBtL~DIUQ};}41Say2#6MDI91)($`v|{eyW)FhgHrrfz)|t1C`01^N-Su|R>dm(C0qj9qx|-W zgPWNmO>UWAKR~9yp;d%=e>etF4I+>VXs22u=d1l#k*2WVMq4TvNc8@`#(RZssy>q% zM&)u+N4Q^VGQi4!YcIbxuX}0|$o5+`F0;BYcPgE4>ZfL%H~SXW7#JYYJhi-aS*gMR z-BmjymEqV~)43OMSN6Y8q_ku5GNvx@q&3#So_G;$+_SXs+*-HdI^lPGcj+o)n|)`y z?Ko9Z<$Dyo30dn{GX5Wu3L4XSd1^$GQS?&(X5<6o9?@G*-NsJAp0*j0D`oHIGLn8tof686l&?K6$G`;Shr4O3!z%2YDYPp^3CQS zM<8yB1&T^JWR9m*YjjZAD@k$R!{Ya!0cfs zu^8Pb*(!R_k)f0hF(}|~6Qw%>FdhHX>+n}oLWo0z%P-YgRScuORxl><5#KT^L z)T&Jp#V`$<*8EwcQ`V49_&;GKRE2A0(aG{|2q29S+^HI+7wvwjAY;c_Pr$Q zPBpHeZS*p{(NoFbba-DE4|r9WWUjOOLs|H~zWgAw^Slr+2@jpu|Bo)of3pDei~dfz z<*vSS^t&_)oD6w3*_nWVDb<(Qxip$=j>NK}|3D(seg==e7K*^A7UzR5Y{R0Dd>==P zJsNl$37`!7IsiETh1S@}MQbb--OZkItMmEq$je8Ht%>O*rI6IwZO z>^IIPjj4w2rS5(pCKEtwp6b5tP>a^$Zr1mVq&8kAZvaR#l42`7YoFIaOUWX3Vu=I2 ziU>Wx-9%z+2?y?NJoGzpz+CuBTfeJs9r0-1AUi&p!`&;tp&kg@`lJyMs%8);z}53D zR->$C%zO=SO`uP<9ru%MdTB6(-CK?-NQr-ZX(z~gyB@9*Z)yK%;M!XL3qU{z1GP^d z;VZjSP7SY?1BOP>+--}*TDsxtRN@kDx-0T5U{*OTMC5Q=BV}{86NM#sr#O~*tS>WPT z1%Aly7QgyuyuE7kKAO>m!ays48IZ^?(wqF6jaza;Z5iRnjMl(Qdw(+jPO8H=aCGa{ zj3V*AE=Lxg$OZg?21>26bao#TZyw<~`Ztmd0?$gu=2LeV3A7-2N8tdB=uvL8( zKJFs0{?k$9{o*e>=l{@m^dAKNUr=ZL?!oJ_ZCpgADlj?Yf9o5iCeNc$4hDgwfPeo} z-zevO-zabscV7R!ZxrLmZCOQqg#{a8tba&BJ3&$D8c`*D?{`*|kBN^Tw zPv+lsAe(h6_pA6xI-qRTI)o5vm^b;LiD`6dxL%G}D$m)Nh?q~RU^_}7)E|1irhn9= zQu^kJ$+z0&FRJA7H5&5a1Mr9>Y06&d=WkLyVT`&h;>;{d2H^wlN4$@s$&no8%ryZ9 zQl&%!n6B_=90u^C(f3VYrf>j8Mdj}0E8t)kBFnwK2;!O|w=7JF=a1U2Q$+r`5?d(3 z_Y0xcbL@_hOyLsxrspReHpbI6HFb47?h|Awlt`%CqxYj(?q@duKQ6x_5r4)9TUwf$ zj?UIR71;RQXQmqj z{_83eCUe(w@mvK*vzHjoPAFYm_k$h&zKw04jXz(q^h>>%Q+eTt*{LZN&st>AEvtt{ z{&S|qh+HFq_6eGF6%`fmf`z^P@hEV_z$Wc>xi%W8`*8L4{Mugdix(cd#cPmzjWL{C z3fI}px72cv_9dnZFwW$k7ecMtnYIS~sJAs3Y??*iWCVFnJ22uTs1_;#*X9jRT8-H$ zl^oytqLGPWaM(6!+uybM9#y>rhq@5++iw48+uewNwUKsdKD|?}efO)QFp|U9tMU9e zD2CsOec_^Daw5>q-oC12`RaIc9dyu+p4HmY_+9aFlIm(EwQf?q31r#iLCD9;-VXnH z+#|*@IVF#cXQ)Za@Fr&ig0Qby?i@VxgJFrv%F&RH^$7GjM#87c8LocXvOMuEHV*s) z`FN}^q+yP(F5nkkQ;rI3DHkcx9pr&mMM|lMr+nj<+=2-Oo3H)=;?<8hYRv<9meTA zg)3OqQ5%y~%X3?sC4!D(m0CFXVul`x$zm9FJxmTWchVK^-RxagJT2C+erbcgWby4= z2~lQE_AGqIoqSlWc$Qs}rQ|`t5KRonlvn^uc0%=5)BJMOx3BMf&MM~Qk6bG;PUNt0 zhMAcf761NlBC6uQGU&o#I5nUBUCGi>P?$mOCpJ_d9y3cerz+8l0nh)RqS{F|A8S&EvGP7BL}~HaP{ex>Pz7dlC+- z_cUJ*UI#_i+-2NdIuDtbqdLAFy)C8fwj(SqmrxP8^4RBRL(Z~`@C`kztu z3lP@1);?g3bx%)ip4Pz;%JThq_IXeF_xTX`W+avC#X5Ujnoxie>Po!~ALfWWX&j|7 zr-_izwtT|Wlc$}`fiu;;NCiJ)s%KpM3NqDHN{{1t$oL+#b6Bqg@%0)pQ+Pvf<0Zp? zTlKDebd{tX?(sX2NfFuB>NNh1+Ih}wkm_#jo)>M5I0S1ZtIMFhT8FH%@H38L{VGft zpwGas>+SxcL8nfOq9PDaXEg)7t$EQ!NH-qcFB`0=<0<=he{x8Lf<>V`!xf|Qq((jy zz+%8KeH^1qCg$_*LI+y^;q{CNWt!37!C@F17Ir!facZqOv9uTA@hU96Ii2R7An~%X zeQ(PUO%4QiDZNav3qD?xBGr^B%xZ(Hq@*NI_KfiTXU@s4s?yR@ZcTG5D?S@00uaoT zZY3=@q)?3t#K{%zWxTbrnQ0h?z<{As(-V6>|#ilJ5jEHz;C%a~Oem-eG z>ARgY9P^M-Mp>+$y1D5qa@ZceO&GR{Nl^8jS2B1y+gIFXN&FJmE|GWmX;FXSaek&6 zz883#GDwZ1eOx}8rV48_P~v_)3w7k#aD>0;;B$ELnWKPDI3x22dYFXWLL+31Vsj8A zZ>`Nb`84)alBpz7>75D4zqbLZzZhlh})#6%rEv6}$j2*0W{J>O=nULx2t5sQuOkiit>z*pAHV%@NS zXbil-pPgJ$VpoM$?(^}+hAK_|V$zcyt@mqV9{|PMn_g z@C!v&WhxlWQnAkZ@pH!)qk@cP-E%&L^}-`hpUy=ILBuHCd#ik$Oh(%*Y3Z_xdavg& zvibEtaQ=?#YeV|c6ZK7%|0uMkmzjdvsQ1u(HA$)PEd7^OlQ^7ye7;g=qrvIPU)$F2 z3Ty(Leoq>1Qto=M)~c%GuLOM35;E!yW}U!3=Bx8ma;dho|Jg9GvjKB^fUj3QhDNzY zUY-~0{H@n^>ZY?^x5v8M9*VUK`bXEh%`yTI{waw}()306r01Cy3h4Z5aMbm#m=_|K zdR@^!%iRoB6diC zbx&FiuMO*|vx*5=xse+rpd8(#OFrUjUBvIUL{!lBPOhE45pF#R9zLK85>ejDE__Q* zW`EjsV!=~Qz*koE`Y9-^A4Qp`0)vuCT={^l$I$$hW^&#^hLnag?%Nxc`jXm?G%SB zuxgYxcLs6N6g>SO4XP|a?HVQOYq)4NZ#?ce0Eb?4MtA%%lOD1LJflFn1hnjv(cB29_L_s3oCyg z(JrR__MXEo-1xb<%a@YxJ_AL;Y!i+4JUC#}GudgFTbvi{Nmykoh|M>$54?*0-k{318As()Rpw zOt!gO?y_1AC$K0vNzm4Hrc$0te$OacAjl};E})QSI9(uR5Bl~oFGei@qrIS4d@7QWrEv`c?EbJ9f~8q752g1PLPe75cCH?u&a$3dC))>SrtT+PzcfnMnEP~dn1{!EwKr$5 zMf@^?5_*lr0zqrcNgkZaw6|oLnQO;}NxZy0-tN{s%KXn#RvpAI{Ui_8sMpifYDef9nQn4ks&|cX z?Vg=A8bgl({b-n=*j%~;!$`Qn1De*YgRF!TV#$QDt52QdGm7i)mXSaUt#{wnTKNwc z*gu~ilnqYa8khnT46!g6g=-FX=g!KtTaiH0Bu-uT1*4M{Df`%36u0$4bMD>7DFMgn z(%Q>Lt;IA~*WUp#3H+WR!LLnyOu`!GtKK+}jE~QH47SKed@nD4)dI&J#sqcWnJag0l(K9(9$9~H>jzs(ewdrKN83*E0-aX0wD z^WlfHFE@UhMkdhJ5Y)_u{7lmM@wvjo&ehR3!uJ7)dMB=Ld#%QSx;u}3Dsf{nzf6C^csKJ{R^y1cpbbFVz>? zC-s%T&K@rq;;q|Z2YJtZ1x{I8Egaq@A`mdNP<%=;7HVdrNFp@x=}j(r5nhKPu;Viu zDgD63mt;1SbgPy!Lvb8TWWI@|7E)ZsU3}w9!?aW}`b|m5QIw}xX<2VsJhIkiHxYz#@P6O(l*z|W$n_HoY z;Gs?~q#r|8N)vrAw{$I)HQcHxXu(mw@7WR1iOQBY6}xHe+@Orz2HxVzCAgN@%C&KvIPN%=CVSv?ZSz9kV|*S3^e`cqu4 zcHvaq7Pv(?2gu)&AA;W}jS_|t;_259qx*k<6j72iQL+M3V)@}0v+Xd!63TI({y5vn zhnoaFwPu;*-)1A76vc)eXLsRL##D8Xm_NtA+4Ym9*nWrUe?X>ezOa}X3>9lsx*i2q z?epgwE|h*?6MQNB_LSgE?Ve9&z*6r|(2sX{wKF(6%!f|J8l|vjIpm(+lt)l>y29kJ zqyy5cYd5Ek0xPkw$oC_2#}R}$ z1(C8_BY%sJ7*-PNo;gmL4_PJWlPoeNM@+CM*GE&6wlnZ$YhjW?ij?y_w;j;=_K_sm zOeM*yh}PVL@M&n+;5fx;rVn(%oX=4fdq*6*x!1oZMLe5bqv_QCg~6o zCdN}kWpDgsz}bSlKXyB4kw7v>K^*raLeXoV{SDl^NmiJ8nSS}M|N8d|!2mFtfYib-p%e%WP>ud~ERQdhKRagYkVPkXf!VoZt1Zc9ykY z>tMAz4C-_v>T$z%gPI--plWSh<4_Wg{Yigx=_PtW>^zQx-R{KiWD#8pJs%zq?-v&+ zIsax~$W;D$UvML~_6L+`vmex{qL?u%#jO$BUaYqF?QP2b*++Y&Zd(Ig~Czii|WGg!{GZ<0X58 z1wpnei2=FN&Artx3J#J&w=EbonoV&a)b>{uY+kwDUSW0OO^-EHv zVHa7M!-qfE^l}dmh1S<6mtU1(vT>dJh4jbtuRpCKB6KK9&;hU&m~E4M*DNd0rtW-z zWm~&1+>bq0yv?&{U%#ga4JL$BvJOMoAhDc9MCD0Cc zWCjMJPmC=nXlOJtnY^70u4-s#D5%aNe$;zIE&3?p^@nYMv1Tvau|2L2t}n?7B( zL`}`pHSNf|qjxrIAIHwlo;FwF{f6sOu3oVTIHY27>|nNM6!h)*)9T|{2~dSw=|e{> zbE_~aLn>nm&xOgYAB)o}N@b;zb`i_6{$`s6Bt-a-z=8vym-`X{ESlly{QDPPPAwh{&Bh#hsOL%N4dI zkWN52YRyhH_Jj<0({tAqw`yb1)mpno6bI~4&(CozsXw&Gc({sIN*Ch3XN|`{vy0V{KuXJ>E5AGQ{5M4t)Z za<<1Bh7DAayG0#~f5YdjDhh|J2g;$QVG^ZJkCF1ySrjrJoKyLstBm761lwtnzG8GOu2X z-YMU3bu!wP&e5;2y2U0nyWAOsZKE5WU88W7fwsr`-Qe@Ajzk zKCjABT727_u;PjRwl%@S#l-xgvR%!*BL57WTJNIhrA?f`$YLK=%rOsqPSz3NDSYJ& ze-@$*{ek1EH#>oDeTPq_aa|UbT<{7a%WM%?{v|f2)f>twY|yJhiLscXP|Evc-Y@p# zo%!j*P5PGiVGR=7>J!$t4$mp2Uq4m)&L$-~t-c@dbeC&Kixyy^eydL`E=i`wy&NKZ zshmNfyak<2Y|{VPJ0cGIrQXofU);7Qvd9^#5HB(OU=aY_ij9LQd!XfIGAG48=8!G@ z#{|Jf3&+w`oPl|Bs>WkgGF9xIb?3XTR>?U)Nf0_A9sx;~RbvLh3RFSt4_J$g1Xac;r5|j#zvkQwNt@i55Y9z|MgkpRcyb9MXCt zdSb0UQ+_b=1tUzS*7?{!0>zA@Xn<4U0-^VvICfCynWTCE*jWOBpLUxTc13SsV?+#c z`4<*@kMsO_rO7C%Q^|POT$fgbEU4b~rYrHi0oE>m3q>xaPH4G`0K0B=?Aa+uKI$){9)mhtZP`ezV{7pLo!d*%cf ztI;HDIW;y`SyQ?gOVJ(fD(Y36Rbi06rDP}CP+r>eaM;F~EBU3Q-278eHm((;pE$oR z;vvX;Bs43cO|`2;Inkw44hjIW$tx~FIx|^fU?2^NG5HWz?^4pD$5uCs2gkXqMPiZ* zy~nTlVX;O?x{`4~lWpeJRZ^yswKMQM10(c~{5zxQ{IaowK?glqD#MR>+Xjj=(Q4O56|l(%P?Ro1T0+g*D?9sz~H9DIJJ+t zf--n7HY6t1H(Eww^cSU21n!m#wTcmx1JXQQRbH;FFZzr2I&IwqU(`Lag~smt44Roz z2hR|uSE#Tw$<|kV;cnBf{fMj(E^~-2ViDXEm|&*B#AmbF9O{y%+dwluXJh53{BBQY zjkilcqL^iMx)PTyGrgvcaQ{_?gyk#)&q66z%!Q$C>iMyu*(z6OHSSk#;-Gvtg(^O~ z#2X#L0#*I6r+qSolX04hNDG2Ankgk5jnn{hqcaewnC!0Hn%7l;3Ga3ILbg!EGBLmTft}Eu-1dzOctdxLyv>=MbHJ0a+0&9Tt@@ssmEFZ| zsT$DRD0JX-{s-6}uCOgiZKm0-cCmACWbo^rwxcgTDF+Dv^mbSZ2LMA%4GdP{a4gW> z4ctppL!%%9Lr&piMJ2Q26Ve#bEDLjUgDwEY@NDIM+c4vhDv7Z@)9O@(oYhwIOt>y} zb-9hESi1}-kNR8knOJ=)RNCkv!neW461mh4EdgqB`-Q}!VV?FMJ?BLar2isx--r6h z54Tle{FMkNURJ!Xpedwr&$w04)U?U*P8NZ34`r0kRbA~!#89F@FN**Q31nh+$p*l% z7Mou}K&g9A7dAHvym@V9SLYnlo+V8nhr=-kIL#fAe*hcV*x>L>cdy;uhG6q)>zNrf z?r@U3?#+y2FQaf8eFNDk;Q&_tiLi_;!6MpTs6YuOw~FPN6z8+#2I!{ZDAe7V059pcnqDsB%=ckS_TmmzEaRK`Jvkc5SmYZ(X4oG&dXO2AQ4T99CU4y?hy( ztH)~?xMlh@92jjuz<%H!3ea3bpCU-P=qgMg`J%|Ua=^wP2vFB_ zrk#5HdViOhPO4vjYKphU(ZV+5zpiOC_5PX|FslZ9#tu#ly8|!=+PI>I^~@rw?MU)3 zZw6Y`|7Hb&mptKo-P1;aB#2S9<<*$+=pG!&q>*)-i_`G?D+zx#8wU4!DH5picuE+UDmbt$}@nxld z(~lly3L^hZ`$tXz?%@CbBXMAkop?0)BhZ@(>*u@)3=?1Ri6eJ{=WdsViHL}D z1VFUU9h~Y=kZ7MjL|$PWR<}5*v3t#T9^rRXWqjxH+s1S@Nwe++hzT5OGZ(_xay$J- z4hzvT^yf@tY$1=w)^X3=+zA^!e#1BKFhRO42S(nx&Z}QD*h#oyPPxRO5=u_Yl+~Mw zrjW0)o|)xULD<;j5=S1Q(E`{ui1h8%u-0%wi4t9wZ71RGyX)*;kFk-LUUd`_(Qi-% zQl)Z9ANvf^NFYl@Be#KioJicRPEp6 zP}k>^XAd#f?V%6k7z?f&Iwy)tVC*~$n}ko$VySLh@(nrD@z|(lmz}#8JQBi z?Hyf#2_H54R8xBLyGMaur#dqMF;ZH31OHSk8y6o%`_l1&SOHlpxdfm4`9aD+NeHgj zEwe{)TwJaIlS@Frcl#CS8k_bYd@vP*^Fir0ZQc2ct5@BDtc=bLgfm>aGX$ns)s16p zBgez9ZBX!R@W2=gsm@cYiVF6wsph-3@3;-TRoYkGiwBJL_WHg=&)=pF-=@~-2!RB# zbDSW|&wC)cMzuFu9PD>*M0wEs+T6&G1Gpug2dF&*9Y|^K^hCX~v@9a1o5LsI-WAn! zaKJvDp1Zw{ehM<>bfA_y(@ZyJx&dt+a$=P_zi^F?%x(d@!6+XId9JomRnk=+OM=vv z4^QSARz(cN5*u4c<(@~Y^j%f#a6#SK*US3s78>UijeGkTKAmQ6g{x~sp}1Xqi^VUW z@fZ`xM%m95_X~cABzS!${*u13&)xF(0+iwBS8QvUibi*_$tff`4IH?1O^JNuEgr)j zEz^j?lm(=fbdvjw>9~)MeU65Lf#65~WKDG*D#%Cii7NUr`4v&FEiq4%MH}Xx8Hc~X zo5;MJw0Cq~ww1;s?cTn#OE(G-(_I!JgRP{A()pi# zip^;z1Ok^tw`h33t;*h;QREOVYaV0ZmG zEfcdBt#vv)bq2fYc8f7)FmmmOw{t_<)c8`-!Y|o4NeXQ2yCIKnq(QCk2;b3NFP=$+ zY%$JZ;E~Q;oLXCz3G*y|^=#POSCZSo&BstK6mKEf`Dp9ky~@gXN?gkZ-|6U-Oxmxm6kCErN#6g0&S!5MC%7@;~o(ZXIBfl;}^}ZF8?DCqC@wJ(sigVr8{&WI`+=ohRb z9i$oYeV4HKGL4+-!9o#L$vhZ7O<;m;Z+|5bAQ<{P3BeKNx%dd4D7o!EE>kShT!a6a zm5UJ6$x91j;x2n^)?gC=saQz46Dt_YtVYUte;1tXXNc;ob5+*9BIq*Q)(bP$+Tnm4 zcSTC_8|O{6{0y7|TB;xGd*NFt!tHjVQe@hDS|mEeK7MZWfQj2%KNr~8-&fN%~=S7roe7f?O_o(mcj4t!H3vjMFyK;2metFfezuztBjk{Ck z3(FZ8=Ifx9vV6SbTqMo(chPbnvblRw$92Ie>$jf0ajZ@~XcDDyM-F zAGF)?d=k7WyR_K)akn{BayCI0^DtnIiu3rbf2D6wpx6emFIXXeAHj1xJr9iXGY(!h ziB3{+9U6$x)9&d#VKwn7 z_eVH$p_<+VynT1kFq7ld6~7sdWRwj5$gSa-yNfG5ononQYyv`b9(QOEtOoKweiv|r z(%vm@%Q^yuT&q;~)gmvD3XTp1PRpQw-ptkfHT)K`2tGB0Gr zMkc+pS;dy!9&|!cfmRKkpWr>Mm!rBGXf6vUD^kcC9MJCvOHKPVQJYMS{a?g&d3Dk2 zIFqxjNEhI3@gU!0ofWoqA_K}K3`qf*n&jG}KdRwtknun=XVihrqgGuQ2D^;>wyIM<Oe-StaWB>xd? z+K-p#Kn1Bd=RP!KYIu^(B2n7a4)>;S^mj6|M|Xxlmg3kNo3lk%vBmn;18XH-lt#vN zu(c4og@X5AI3vX);uygU=v9LzHr=n+lZm@(w{&$UXXb2=?=L<#ovQz$+2z9;7Fqix zLy$m1PaBu^Is%4 z1)JV{+l0sd`s6RcyCL9Ack>X*Et9?sSrb%CKLSviOmP#nWS6yw8T|yiFM1#Lo2&%s zvl|#EXH*kdM8Ep!^(GZ{NKWh(s|bzvk1tvT2O7o^j9Q&hPL>Jg^``Piz0XcR_R{fq zvD~^v2S*?xRK51X??Ri5mHhngrGm%fi6$Bajn&nJhDS}eu!DX_#SNDZx3MW4O&izv zt2JWu{wXqT(M>z$io^#v0k<)(y4<=s=JSK1vQjl^9%(KT^VMVbyDE^6k~!4;5ADWz zQg0c`gIs~^(7RIT%<2X*M*3~srnP`IFHW@+PILI~fhAIcXRN+X!MHemQOUb;h#0-g z5m|RDWwm+hj+jo%Od|G;#<(do<~pFmi+H`~_L1hv8&IW zlAn(83=l#|7|hnje6~c~-9O=e0~EGq!3_g$Y=DCPI1oK0I zi*Vms2)0sXMyPV)Hs4D62HM2=6rA5bvHy^5f3A!j5FHr{5m5)!f4e8CTq;mB!DC&O zv{qIsf#~6q0tMw;DT)^&x*zdi`3WqFRWuc0q#=c-Z+VfHZVeq1QhU||a--jHl}5z- z2N;%*zd-vZ!MsD&E?3LsR3-PxLuJE!;&yUsO~l#}Z|!y_;T%+e>_J54E4}o+Fdz>u zt<$2iJ$A}7G`?N5<+iA&?P7q>Lu5po`<>#`_AQJ#GdN`eukX(;5qhCGP~t%_pt$0q z#&ubz_43-q65md3KzVHCjUzLptW@ZW(-Kp^#eRM7UbyO2nc4$bku2+!d z;NAad?M?k(88_3&fT#X%$S*-MhMU9g^t6I2vuQ$X06ms}udVrRV7l?41=7Q*irs za0J_cb{oPO7|;ZU5lkb1>VS!6ZbLFAe6>@O3}$@nXOa<=@Og|_~G zE)`(c$B`&D61;%#141!+B~sK|8;H4EQ}KO%W4$duA=Q4lZ&wsTs3OvlQh|6U#U;My z11&uA*Tdhx?soqsTK6j0>Now7!=Ia{6_d}{gjd4V?`90^{B2uYjO_Q**T|x;-fGS* zBHCW{hPTdkoP#xbTd8f0yN{iMDd~DL{nY$RbAurcsP3eUKP10&r3z_Vyt{T1y=nx% zq5rHYkwul;zSD?D(-VW!ka1^6w%FQ!Zb^iZ<{EWZEz)CiEvB5a&r^5G!n*;jgGsfW z1d$?)%;Ca?bcH)hdIJE4r<0=x)6tZ5EH2W52Gak^%eypU=IEF~52QSy{ZDa+XMWOB zClY0$QKLuvJG@5>mxY3}6#@Z)0|IXOo?WlN1$1v0PSNn}&%Sm=l$9srMq=(c3iSrc zE{>(U?@_da?1iy4nmON(Ce&a`A0_*reuOq5c^aMabg)Lp`eVGU3mKQ~3#lR5z+}6Au+{+N+m?9 z{AuZtA8FqK5tQHAKU<&qUH6cewzC`xrsYiz6wVpzT1gHQ8`I!jkns}D0U z$*$V;oo};%>4ltKFM4AG-Iuf!1352fI2)yVF5_PUO5=2!TGH;RDLKj6fGKdO#Z&vh z0FiwQ<%u(eUH{LQD^C!cyawK_WDkW#WeWt_6$BY|Y0e|H>f(i*o+yl{B*wFdjw}7S z=qV!2YNt7NJ;b|s2Xbzyd8%;zr=ib7j_Wo&EeDR~Q6kt}6Dt`1#r^RpwHyG!h4`|U zcE@Ge$H&tEb`mlB`ZiT~VkDRVe#F}>WGe$Cwz&w%vDiO6g4Y8EX09&Z^zSV@(NE4&T zf@FUv6G_ax?PCk?oYp6zd`-pfzR!G?;N?=m1wmsFN5FBusen`uCb~k1_RM$_dx}n6y_EHgkHf zfZW!Epd&GI+sbzB%WoFYe%xI>P>*(ilFD2y~6UveqHQ4-`fJKqD+Te=DrQGl$hyQ!9wso zYfaT_m-m}(J1P~bIHXzd&T7PXryimA75%oH_vpr|W+j4`?S-FDCfoMBj4`X`+WYvh zyw$Pan2IGcQ>*RmRr}za__#y4arfL6eu2pDv-y5HtV*Y64O;Mv{4Wz`cYlww8bG{J z18MvphVGt!r|Rz3AC>W3+APoNlmQ@Cf!|&#gNQ9@}GO-4h}uy8BBkir}+gLROgA{+u3(oRj#G~>0d3kb7!cu& zSVqX`s}P*gB@jn|y`|#z#qdAAhF6njq(pYRH0*ahZg03$V(xET3SK?VJ$i4eLc$r1 zL6?D;L3poA+AY?FrZ@{?O$4M%X&%38+ywNOH=61XJD$qd5CFyfkyTTE>h5y8v|HRl zr%a(3FTf1i@S&@t?|PD5>11FB%~!Qn0B+R@IQo!9fY=rcC=~Q|u3afDv)@A)PUJ0g z2d7jX#AeJ!gi*^?fMsAuPR_*HhHUpqBIJx4CClSoKly7y@JL z;pb{_kFWjcS@_hf2FlI+s?x(&JtKQe+pB&Yx7*(|2 zbZ4OqEG`A(dQU)Dp=!!hhqoX!6T?@3jMqO~B$r75VZSi<1Ct;74V-t3sNZ0Rd!o%cUkNd5kSTxh zoWli8L8HW<59B*c_^OT}^^Ps4qs<;25FVbaaO?GN5zDbv`L%qo`yTJhLRb{FiHKsY zG<>8X)jGUsfeqF#A=-z1eE84&Hvo6tp#OdzP{&x}9iR8Aj)=~*Gr%*%>`rX7LvQ(B z%Qx_FUb-UUs$2ne_tf>o{ryC?!uxXj7j7Q!57@J?J3%pE{e?Q;uRl-y{J$Ps^*||p zYgb(){m@z*tFN>L><9IpOm((ht_oI>#_RGcKIMdlMv9Ub&O!IS8}AsI^E!O6e`Dy- zT5?APIpJfa!5~sVIMMZv%;JljO9OS*8le#Ko!2?u)azOfTv-fEgWf2kK1Fz3cHvKA zVNg*5Smn8m%;s=yVJE2T`uS(X38{#2GmvIHyN9uoQ_yDaGv^{;K5}t|=kPT0_$*H@ z?Jed)q8~^a?JY1e_@!H8e-$BFS);FAPur1~$7`^Q?1z$be+kpNTLn2QC(*|g8W>lC zha3r;Cxa9UI?axf2FPWpskbcq=h_5eiz%~ym)m{sbQf|kmJYP_V|{+KLW>}=52Can@NUc+s7Es!r4?U zB_9)h8y!7JooQ8j;roglk7y;S?=q~UNX?l9yZ2Z^79hM??Eb14#!v1cXb!vi{ht4& zf=%srG)>7VB`UwESzm*M|BG8B@piG|PBRVIr(u%na|e;Q*T!tuQEA@tbtrJ`g;-q= zm2B;TtF?2qxD>#y1Y&NtY#g>!w}&5<)z92JdK0Hhf$5KZ^j1HfO);2k<8RVZeL23} z-}7h3ni$o@(}o-6KXK!dFf9Rm(qjJOdQzaGKZfI3Z+OEY0r|yCDm3YJkXH3)H?v2z z;Sb_9AsWS^c5-;KxzA9=YSS#hS%4ovUsTLLV~pZ)#_UT2`*N+vyOcRW0bvm7A%+ee z4kMc;uY|`u5{#=`c8V(7>~q9DneZv2*If3l