From f7d9502c0549f02ce33b0d866e5c800d970d9073 Mon Sep 17 00:00:00 2001 From: chenchao2024 Date: Mon, 28 Oct 2024 16:49:58 +0800 Subject: [PATCH] NSFNets002 NSFNets003_241030 NSFNet004 NSFNets005 NSFNets006 NSFNets007 NSFNets008 NSFNets009 --- .../research/NSFNets/NSFNets.ipynb | 414 ++++++++++++++ .../research/NSFNets/NSFNets_CN.ipynb | 426 +++++++++++++++ .../applications/research/NSFNets/README.md | 76 +++ .../research/NSFNets/README_CN.md | 75 +++ .../research/NSFNets/config/NSFNet.yaml | 25 + .../research/NSFNets/config/NSFNet_KF.yaml | 12 + .../research/NSFNets/images/NSFNet.png | Bin 0 -> 98487 bytes .../research/NSFNets/src/datasets.py | 145 +++++ .../research/NSFNets/src/datasets_kf.py | 44 ++ .../research/NSFNets/src/derivatives.py | 504 ++++++++++++++++++ .../research/NSFNets/src/network.py | 213 ++++++++ .../research/NSFNets/src/network_kf.py | 169 ++++++ .../applications/research/NSFNets/test.py | 165 ++++++ .../applications/research/NSFNets/test_KF.py | 87 +++ .../applications/research/NSFNets/train.py | 130 +++++ .../applications/research/NSFNets/train_KF.py | 110 ++++ 16 files changed, 2595 insertions(+) create mode 100644 MindFlow/applications/research/NSFNets/NSFNets.ipynb create mode 100644 MindFlow/applications/research/NSFNets/NSFNets_CN.ipynb create mode 100644 MindFlow/applications/research/NSFNets/README.md create mode 100644 MindFlow/applications/research/NSFNets/README_CN.md create mode 100644 MindFlow/applications/research/NSFNets/config/NSFNet.yaml create mode 100644 MindFlow/applications/research/NSFNets/config/NSFNet_KF.yaml create mode 100644 MindFlow/applications/research/NSFNets/images/NSFNet.png create mode 100644 MindFlow/applications/research/NSFNets/src/datasets.py create mode 100644 MindFlow/applications/research/NSFNets/src/datasets_kf.py create mode 100644 MindFlow/applications/research/NSFNets/src/derivatives.py create mode 100644 MindFlow/applications/research/NSFNets/src/network.py create mode 100644 MindFlow/applications/research/NSFNets/src/network_kf.py create mode 100644 MindFlow/applications/research/NSFNets/test.py create mode 100644 MindFlow/applications/research/NSFNets/test_KF.py create mode 100644 MindFlow/applications/research/NSFNets/train.py create mode 100644 MindFlow/applications/research/NSFNets/train_KF.py diff --git a/MindFlow/applications/research/NSFNets/NSFNets.ipynb b/MindFlow/applications/research/NSFNets/NSFNets.ipynb new file mode 100644 index 000000000..38a01b126 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/NSFNets.ipynb @@ -0,0 +1,414 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "de7a09e7-32fc-4059-9478-6c6fc6667f57", + "metadata": {}, + "source": [ + "# NSFNets: Physics-informed neural networks for the incompressible Navier-Stokes equations\n", + "\n", + "## Environment Setup\n", + "\n", + "This notebook requires **MindSpore version >= 2.0.0** to support new APIs including: *mindspore.jit, mindspore.jit_class, mindspore.data_sink*. Please check [MindSpore Installation](https://www.mindspore.cn/install/en) for details.\n", + "\n", + "In addition, **MindFlow version >=0.1.0** is also required. If it has not been installed in your environment, please select the right version and hardware, then install it as follows." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5a449343-731b-46bb-b8dd-f7a8c1ba8071", + "metadata": {}, + "outputs": [], + "source": [ + "mindflow_version = \"0.1.0\" # update if needed\n", + "# GPU Comment out the following code if you are using NPU.\n", + "!pip uninstall -y mindflow-gpu\n", + "!pip install mindflow-gpu==$mindflow_version\n", + "\n", + "# NPU Uncomment if needed.\n", + "# !pip uninstall -y mindflow-ascend\n", + "# !pip install mindflow-ascend==$mindflow_version" + ] + }, + { + "cell_type": "markdown", + "id": "25fe4081-9d91-44c1-b7de-b50a7645e090", + "metadata": {}, + "source": [ + "## Background\n", + "\n", + "In the industrial application of AI-CFD, the scenarios for solving the NavierStokes equations and fusing multi-precision data are very extensive and have important economic and social significance. Solving ill-posed problems (such as missing some boundary conditions) or inversion problems is one of the key points and difficulties, and it is often costly, requiring the derivation of different formulas applicable to specific problems and the writing of new codes. How to solve the above problems with a set of unified codes at the same computational cost urgently needs in-depth research. Here, Jin Xiaowei, Li Hui and others used physical information neural networks (PINNs) to encode the control equations directly into deep neural networks through automatic differentiation to overcome some of the above-mentioned limitations of simulating incompressible laminar and turbulent flows. And developed Navier-Stokes flow networks (NSFnets, Navier-Stokes flow nets)." + ] + }, + { + "cell_type": "markdown", + "id": "50540e43-3c8e-45f0-8e2b-78831ec89684", + "metadata": {}, + "source": [ + "## Model framework\n", + "\n", + "The model framework is as shown in the following figure:\n", + "\n", + "![NSFNet](images/NSFNet.png)\n", + "\n", + "The figure shows the network for solving the Navier-Stokes equations in the pressure-velocity form. The automatic differentiation of the neural network is used to calculate the partial derivatives required in the equation. The loss function includes boundary condition loss, initial condition loss, and physical loss to satisfy the balance of the equation.\n", + "\n", + "## Preparation\n", + "\n", + "Before practice, ensure that MindSpore of suitable version has been correctly installed. If not, you can run the following command:\n", + "\n", + "* [MindSpore installation page](https://www.mindspore.cn/install) Install MindSpore.\n", + "\n", + "## Datasets Preparation\n", + "\n", + "The dataset can be directly generated by the provided code." + ] + }, + { + "cell_type": "markdown", + "id": "23990483-d613-4c8b-aa7e-0a8b16a89b13", + "metadata": {}, + "source": [ + "## Model Training\n", + "\n", + "Import code packs." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e3d17b8a-bde1-4ccd-9345-ccfaa2cd1bfe", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "\n", + "import mindspore as ms\n", + "import time\n", + "import numpy as np\n", + "from mindspore import set_seed, context, nn, Tensor\n", + "from src.network import VPNSFNets\n", + "from src.datasets import read_training_data" + ] + }, + { + "cell_type": "markdown", + "id": "0b8e1d64-580b-4ae9-a7f6-164b0ee0adf3", + "metadata": {}, + "source": [ + "Setting of model-related parameters and definition of training model" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8a84c0ff-d4cf-41dd-8afc-db8f234d2108", + "metadata": {}, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "parser.add_argument('--model_name', type=str, default='NSFNet')\n", + "parser.add_argument('--case', type=str, default='Three dimensional Beltrami flow')\n", + "parser.add_argument('--device', type=str, default='GPU')\n", + "parser.add_argument('--device_id', type=str, default='1')\n", + "parser.add_argument('--load_params', type=str, default=False)\n", + "parser.add_argument('--second_path', type=str, default='train')\n", + "parser.add_argument('--network_size', type=int, default=[4] + 10 * [100 * 1] + [4])\n", + "parser.add_argument('--learning_rate', type=int, default=[1.0e-03, 1.0e-04, 1.0e-05, 1.0e-06])\n", + "parser.add_argument('--epochs', type=int, default=[5e3, 5e3, 5e4, 5e4])\n", + "parser.add_argument('--batch_size', type=int, default=10000)\n", + "parser.add_argument('--re', type=int, default=1)\n", + "args = parser.parse_known_args()[0]\n", + "# args = parser.parse_args()\n", + "\n", + "model_name = args.model_name\n", + "case = args.case\n", + "device = args.device\n", + "device_id = args.device_id\n", + "network_size = args.network_size\n", + "learning_rate = args.learning_rate\n", + "epochs = args.epochs\n", + "batch_size = args.batch_size\n", + "load_params = args.load_params\n", + "second_path = args.second_path\n", + "re = args.re\n", + "\n", + "use_ascend = context.get_context(attr_key='device_target') == \"Ascend\"\n", + "\n", + "if use_ascend:\n", + " msfloat_type = ms.float16\n", + " npfloat_type = np.float16\n", + "else:\n", + " msfloat_type = ms.float32\n", + " npfloat_type = np.float32\n", + "\n", + "os.environ['CUDA_VISIBLE_DEVICES'] = device_id\n", + "context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=device)\n", + "\n", + "x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, u_i, v_i, w_i, x_f, y_f, z_f, t_f, X_min, X_max = read_training_data()\n", + "\n", + "model = VPNSFNets(x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, u_i, v_i, w_i, x_f, y_f, z_f, t_f, network_size, re, \\\n", + " X_min, X_max, use_ascend, msfloat_type, npfloat_type, load_params, second_path)" + ] + }, + { + "cell_type": "markdown", + "id": "387cb210-078e-41e1-b960-e432af9c8d5f", + "metadata": {}, + "source": [ + "Record the changes in loss." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "482e625a-262a-4f06-a4a7-834766ed6ab4", + "metadata": {}, + "outputs": [], + "source": [ + "# Adam loss history\n", + "loss_history_Adam_Pretrain = np.empty([0])\n", + "loss_b_history_Adam_Pretrain = np.empty([0])\n", + "loss_i_history_Adam_Pretrain = np.empty([0])\n", + "loss_f_history_Adam_Pretrain = np.empty([0])\n", + "np.random.seed(123456)\n", + "set_seed(123456)" + ] + }, + { + "cell_type": "markdown", + "id": "30f8c191-ea21-4055-9d32-dfae30c96a33", + "metadata": {}, + "source": [ + "Code training and output results." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cff1733f-6e6a-47ff-928f-be3a348e98da", + "metadata": {}, + "outputs": [], + "source": [ + "def train(model, NIter, lr, batch_size=batch_size):\n", + " params = model.dnn.trainable_params()\n", + " optimizer_Adam = nn.Adam(params, learning_rate=lr)\n", + " grad_fn = ms.value_and_grad(model.loss_fn, None, optimizer_Adam.parameters, has_aux=True)\n", + " model.dnn.set_train()\n", + " N_data = model.t_f.shape[0]\n", + " start_time = time.time()\n", + " for epoch in range(1, 1+NIter):\n", + " idx_data_0 = np.random.choice(N_data, batch_size)\n", + " idx_data = Tensor(idx_data_0)\n", + " x_batch = model.x_f[idx_data, :]\n", + " y_batch = model.y_f[idx_data, :]\n", + " z_batch = model.z_f[idx_data, :]\n", + " t_batch = model.t_f[idx_data, :]\n", + " (loss, loss_b, loss_i, loss_f), grads = grad_fn(model.xb, model.yb, model.zb, model.tb, model.xi, model.yi, model.zi, model.ti, x_batch, y_batch, z_batch, t_batch, model.ub, model.vb, model.wb, model.ui, model.vi, model.wi)\n", + " optimizer_Adam(grads)\n", + " if epoch % 10 == 0:\n", + " elapsed = time.time() - start_time\n", + " print('It: %d, Total_loss: %.3e, Loss_b: %.3e, Loss_i: %.3e, Loss_f: %.3e, Time: %.2f' %\\\n", + " (epoch, loss.item(), loss_b.item(), loss_i.item(), loss_f.item(), elapsed))\n", + " global loss_history_Adam_Pretrain\n", + " global loss_b_history_Adam_Pretrain\n", + " global loss_i_history_Adam_Pretrain\n", + " global loss_f_history_Adam_Pretrain\n", + "\n", + " loss_history_Adam_Pretrain = np.append(loss_history_Adam_Pretrain, loss.numpy())\n", + " loss_b_history_Adam_Pretrain = np.append(loss_b_history_Adam_Pretrain, loss_b.numpy())\n", + " loss_i_history_Adam_Pretrain = np.append(loss_i_history_Adam_Pretrain, loss_i.numpy())\n", + " loss_f_history_Adam_Pretrain = np.append(loss_f_history_Adam_Pretrain, loss_f.numpy())\n", + "\n", + " start_time = time.time()" + ] + }, + { + "cell_type": "markdown", + "id": "cfedd05b-3d38-4156-b061-8278d232a7f9", + "metadata": {}, + "source": [ + "Run training and save the trained model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c658395b-cade-408e-b999-585ba1ae73d5", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "It: 0, Total_loss: 1.583e+02, Loss_b: 6.385e-01, Loss_i: 7.558e-01, Loss_f: 1.891e+01, Time: 14.63\n", + "It: 10, Total_loss: 1.339e+02, Loss_b: 5.654e-01, Loss_i: 6.433e-01, Loss_f: 1.302e+01, Time: 4.17\n", + "It: 20, Total_loss: 8.946e+01, Loss_b: 3.713e-01, Loss_i: 4.430e-01, Loss_f: 8.031e+00, Time: 4.22\n", + "It: 30, Total_loss: 6.822e+01, Loss_b: 3.025e-01, Loss_i: 2.891e-01, Loss_f: 9.061e+00, Time: 4.23\n", + "It: 40, Total_loss: 5.122e+01, Loss_b: 2.249e-01, Loss_i: 2.195e-01, Loss_f: 6.779e+00, Time: 4.23\n", + "It: 50, Total_loss: 4.095e+01, Loss_b: 1.834e-01, Loss_i: 1.549e-01, Loss_f: 7.122e+00, Time: 4.22\n", + "It: 60, Total_loss: 3.723e+01, Loss_b: 1.664e-01, Loss_i: 1.392e-01, Loss_f: 6.662e+00, Time: 4.25\n", + "It: 70, Total_loss: 3.123e+01, Loss_b: 1.457e-01, Loss_i: 1.092e-01, Loss_f: 5.742e+00, Time: 4.26\n", + "It: 80, Total_loss: 2.830e+01, Loss_b: 1.286e-01, Loss_i: 9.858e-02, Loss_f: 5.586e+00, Time: 4.23\n", + "It: 90, Total_loss: 2.403e+01, Loss_b: 1.046e-01, Loss_i: 7.816e-02, Loss_f: 5.755e+00, Time: 4.28\n", + "It: 100, Total_loss: 2.149e+01, Loss_b: 9.192e-02, Loss_i: 6.689e-02, Loss_f: 5.609e+00, Time: 4.27\n", + "It: 110, Total_loss: 1.913e+01, Loss_b: 8.201e-02, Loss_i: 5.710e-02, Loss_f: 5.223e+00, Time: 4.29\n", + "It: 120, Total_loss: 2.604e+01, Loss_b: 1.202e-01, Loss_i: 9.009e-02, Loss_f: 5.009e+00, Time: 4.31\n", + "It: 130, Total_loss: 1.589e+01, Loss_b: 6.606e-02, Loss_i: 4.568e-02, Loss_f: 4.713e+00, Time: 4.31\n", + "It: 140, Total_loss: 1.695e+01, Loss_b: 7.117e-02, Loss_i: 5.391e-02, Loss_f: 4.445e+00, Time: 4.29\n", + "It: 150, Total_loss: 1.529e+01, Loss_b: 6.290e-02, Loss_i: 4.427e-02, Loss_f: 4.573e+00, Time: 4.33\n", + "It: 160, Total_loss: 1.585e+01, Loss_b: 6.501e-02, Loss_i: 5.110e-02, Loss_f: 4.235e+00, Time: 4.31\n", + "It: 170, Total_loss: 1.325e+01, Loss_b: 5.349e-02, Loss_i: 3.818e-02, Loss_f: 4.081e+00, Time: 4.30\n", + "It: 180, Total_loss: 1.365e+01, Loss_b: 5.630e-02, Loss_i: 4.049e-02, Loss_f: 3.966e+00, Time: 4.37\n", + "It: 190, Total_loss: 1.153e+01, Loss_b: 4.635e-02, Loss_i: 3.206e-02, Loss_f: 3.685e+00, Time: 4.30\n", + "It: 200, Total_loss: 1.048e+01, Loss_b: 4.066e-02, Loss_i: 2.881e-02, Loss_f: 3.531e+00, Time: 4.35\n", + "It: 210, Total_loss: 9.349e+00, Loss_b: 3.611e-02, Loss_i: 2.546e-02, Loss_f: 3.193e+00, Time: 4.35\n", + "It: 220, Total_loss: 3.285e+01, Loss_b: 1.496e-01, Loss_i: 1.481e-01, Loss_f: 3.084e+00, Time: 4.36\n", + "It: 230, Total_loss: 1.309e+01, Loss_b: 5.685e-02, Loss_i: 4.612e-02, Loss_f: 2.790e+00, Time: 4.35\n", + "It: 240, Total_loss: 8.711e+00, Loss_b: 3.276e-02, Loss_i: 2.655e-02, Loss_f: 2.780e+00, Time: 4.37\n", + "It: 250, Total_loss: 8.350e+00, Loss_b: 3.038e-02, Loss_i: 2.387e-02, Loss_f: 2.925e+00, Time: 4.31\n", + "It: 260, Total_loss: 3.457e+01, Loss_b: 1.534e-01, Loss_i: 1.672e-01, Loss_f: 2.513e+00, Time: 4.39\n", + "It: 270, Total_loss: 1.192e+01, Loss_b: 5.266e-02, Loss_i: 3.971e-02, Loss_f: 2.680e+00, Time: 4.35\n", + "It: 280, Total_loss: 1.006e+01, Loss_b: 4.145e-02, Loss_i: 3.491e-02, Loss_f: 2.421e+00, Time: 4.33\n", + "It: 290, Total_loss: 6.854e+00, Loss_b: 2.660e-02, Loss_i: 1.906e-02, Loss_f: 2.288e+00, Time: 4.35\n", + "It: 300, Total_loss: 6.489e+00, Loss_b: 2.398e-02, Loss_i: 1.775e-02, Loss_f: 2.316e+00, Time: 4.31\n", + "It: 310, Total_loss: 5.934e+00, Loss_b: 2.192e-02, Loss_i: 1.608e-02, Loss_f: 2.135e+00, Time: 4.36\n", + "It: 320, Total_loss: 5.373e+00, Loss_b: 1.975e-02, Loss_i: 1.401e-02, Loss_f: 1.997e+00, Time: 4.32\n", + "It: 330, Total_loss: 2.589e+01, Loss_b: 1.093e-01, Loss_i: 1.278e-01, Loss_f: 2.179e+00, Time: 4.35\n", + "It: 340, Total_loss: 7.844e+00, Loss_b: 3.105e-02, Loss_i: 2.841e-02, Loss_f: 1.897e+00, Time: 4.33\n", + "It: 350, Total_loss: 5.789e+00, Loss_b: 2.054e-02, Loss_i: 1.715e-02, Loss_f: 2.020e+00, Time: 4.30\n", + "It: 360, Total_loss: 4.929e+00, Loss_b: 1.746e-02, Loss_i: 1.271e-02, Loss_f: 1.912e+00, Time: 4.35\n", + "It: 370, Total_loss: 4.540e+00, Loss_b: 1.605e-02, Loss_i: 1.103e-02, Loss_f: 1.832e+00, Time: 4.33\n", + "It: 380, Total_loss: 4.350e+00, Loss_b: 1.506e-02, Loss_i: 1.077e-02, Loss_f: 1.767e+00, Time: 4.34\n", + "It: 390, Total_loss: 4.076e+00, Loss_b: 1.387e-02, Loss_i: 9.670e-03, Loss_f: 1.721e+00, Time: 4.32\n", + "It: 400, Total_loss: 3.871e+00, Loss_b: 1.333e-02, Loss_i: 9.054e-03, Loss_f: 1.633e+00, Time: 4.35\n", + "It: 410, Total_loss: 3.836e+00, Loss_b: 1.356e-02, Loss_i: 9.326e-03, Loss_f: 1.547e+00, Time: 4.39\n", + "It: 420, Total_loss: 1.469e+01, Loss_b: 6.135e-02, Loss_i: 7.144e-02, Loss_f: 1.410e+00, Time: 4.35\n", + "It: 430, Total_loss: 9.486e+00, Loss_b: 3.899e-02, Loss_i: 4.083e-02, Loss_f: 1.503e+00, Time: 4.33\n", + "It: 440, Total_loss: 4.276e+00, Loss_b: 1.586e-02, Loss_i: 1.093e-02, Loss_f: 1.597e+00, Time: 4.31\n", + "It: 450, Total_loss: 4.369e+00, Loss_b: 1.655e-02, Loss_i: 1.267e-02, Loss_f: 1.447e+00, Time: 4.38\n", + "It: 460, Total_loss: 3.708e+00, Loss_b: 1.338e-02, Loss_i: 9.417e-03, Loss_f: 1.428e+00, Time: 4.35\n", + "It: 470, Total_loss: 3.409e+00, Loss_b: 1.175e-02, Loss_i: 7.869e-03, Loss_f: 1.447e+00, Time: 4.33\n", + "It: 480, Total_loss: 3.202e+00, Loss_b: 1.058e-02, Loss_i: 7.070e-03, Loss_f: 1.437e+00, Time: 4.30\n", + "It: 490, Total_loss: 3.011e+00, Loss_b: 1.008e-02, Loss_i: 6.602e-03, Loss_f: 1.342e+00, Time: 4.34\n", + "It: 500, Total_loss: 2.883e+00, Loss_b: 9.606e-03, Loss_i: 6.225e-03, Loss_f: 1.300e+00, Time: 4.43\n", + "It: 510, Total_loss: 3.084e+00, Loss_b: 1.055e-02, Loss_i: 7.329e-03, Loss_f: 1.296e+00, Time: 4.34\n", + "It: 520, Total_loss: 5.132e+00, Loss_b: 2.041e-02, Loss_i: 1.931e-02, Loss_f: 1.160e+00, Time: 4.37\n", + "It: 530, Total_loss: 4.985e+00, Loss_b: 1.992e-02, Loss_i: 1.770e-02, Loss_f: 1.223e+00, Time: 4.36\n", + "It: 540, Total_loss: 2.961e+00, Loss_b: 9.838e-03, Loss_i: 6.820e-03, Loss_f: 1.295e+00, Time: 4.35\n", + "It: 550, Total_loss: 2.722e+00, Loss_b: 9.219e-03, Loss_i: 6.062e-03, Loss_f: 1.194e+00, Time: 4.39\n", + "It: 560, Total_loss: 2.651e+00, Loss_b: 8.777e-03, Loss_i: 5.731e-03, Loss_f: 1.200e+00, Time: 4.34\n", + "It: 570, Total_loss: 2.435e+00, Loss_b: 8.007e-03, Loss_i: 5.153e-03, Loss_f: 1.119e+00, Time: 4.31\n", + "It: 580, Total_loss: 2.359e+00, Loss_b: 7.729e-03, Loss_i: 4.839e-03, Loss_f: 1.103e+00, Time: 4.37\n", + "It: 590, Total_loss: 2.411e+00, Loss_b: 7.893e-03, Loss_i: 5.292e-03, Loss_f: 1.093e+00, Time: 4.33\n", + "It: 600, Total_loss: 6.628e+00, Loss_b: 2.613e-02, Loss_i: 2.924e-02, Loss_f: 1.091e+00, Time: 4.33\n", + "It: 610, Total_loss: 3.092e+00, Loss_b: 1.098e-02, Loss_i: 9.677e-03, Loss_f: 1.026e+00, Time: 4.40\n", + "It: 620, Total_loss: 2.359e+00, Loss_b: 7.858e-03, Loss_i: 5.543e-03, Loss_f: 1.018e+00, Time: 4.35\n", + "It: 630, Total_loss: 2.168e+00, Loss_b: 7.162e-03, Loss_i: 4.739e-03, Loss_f: 9.775e-01, Time: 4.34\n", + "It: 640, Total_loss: 2.054e+00, Loss_b: 6.717e-03, Loss_i: 4.270e-03, Loss_f: 9.550e-01, Time: 4.35\n", + "It: 650, Total_loss: 1.949e+00, Loss_b: 6.251e-03, Loss_i: 3.827e-03, Loss_f: 9.413e-01, Time: 4.39\n", + "It: 660, Total_loss: 7.125e+00, Loss_b: 2.947e-02, Loss_i: 3.179e-02, Loss_f: 9.988e-01, Time: 4.35\n", + "It: 670, Total_loss: 8.039e+00, Loss_b: 3.376e-02, Loss_i: 3.641e-02, Loss_f: 1.022e+00, Time: 4.33\n", + "It: 680, Total_loss: 2.239e+00, Loss_b: 7.544e-03, Loss_i: 5.005e-03, Loss_f: 9.844e-01, Time: 4.38\n", + "It: 690, Total_loss: 2.498e+00, Loss_b: 8.721e-03, Loss_i: 7.084e-03, Loss_f: 9.173e-01, Time: 4.37\n", + "It: 700, Total_loss: 2.013e+00, Loss_b: 6.654e-03, Loss_i: 4.436e-03, Loss_f: 9.038e-01, Time: 4.32\n", + "It: 710, Total_loss: 1.930e+00, Loss_b: 6.223e-03, Loss_i: 3.988e-03, Loss_f: 9.092e-01, Time: 4.34\n", + "It: 720, Total_loss: 1.806e+00, Loss_b: 5.569e-03, Loss_i: 3.448e-03, Loss_f: 9.047e-01, Time: 4.42\n", + "It: 730, Total_loss: 1.723e+00, Loss_b: 5.383e-03, Loss_i: 3.242e-03, Loss_f: 8.601e-01, Time: 4.34\n", + "It: 740, Total_loss: 3.905e+00, Loss_b: 1.531e-02, Loss_i: 1.506e-02, Loss_f: 8.681e-01, Time: 4.36\n", + "It: 750, Total_loss: 2.988e+00, Loss_b: 1.194e-02, Loss_i: 1.040e-02, Loss_f: 7.538e-01, Time: 4.35\n", + "It: 760, Total_loss: 2.161e+00, Loss_b: 7.457e-03, Loss_i: 5.830e-03, Loss_f: 8.320e-01, Time: 4.34\n", + "It: 770, Total_loss: 1.972e+00, Loss_b: 6.726e-03, Loss_i: 4.789e-03, Loss_f: 8.208e-01, Time: 4.34\n", + "It: 780, Total_loss: 1.701e+00, Loss_b: 5.803e-03, Loss_i: 3.587e-03, Loss_f: 7.619e-01, Time: 4.29\n", + "It: 790, Total_loss: 1.591e+00, Loss_b: 5.103e-03, Loss_i: 3.114e-03, Loss_f: 7.698e-01, Time: 4.36\n", + "It: 800, Total_loss: 1.535e+00, Loss_b: 4.848e-03, Loss_i: 2.912e-03, Loss_f: 7.591e-01, Time: 4.43\n", + "It: 810, Total_loss: 1.466e+01, Loss_b: 6.288e-02, Loss_i: 7.408e-02, Loss_f: 9.674e-01, Time: 4.37\n", + "It: 820, Total_loss: 4.528e+00, Loss_b: 1.733e-02, Loss_i: 1.952e-02, Loss_f: 8.425e-01, Time: 4.42\n", + "It: 830, Total_loss: 2.498e+00, Loss_b: 9.084e-03, Loss_i: 8.434e-03, Loss_f: 7.457e-01, Time: 4.38\n", + "It: 840, Total_loss: 1.777e+00, Loss_b: 5.724e-03, Loss_i: 4.544e-03, Loss_f: 7.503e-01, Time: 4.41\n", + "It: 850, Total_loss: 1.575e+00, Loss_b: 5.009e-03, Loss_i: 3.406e-03, Loss_f: 7.332e-01, Time: 4.35\n", + "It: 860, Total_loss: 1.418e+00, Loss_b: 4.292e-03, Loss_i: 2.733e-03, Loss_f: 7.161e-01, Time: 4.38\n", + "It: 870, Total_loss: 1.320e+00, Loss_b: 4.021e-03, Loss_i: 2.429e-03, Loss_f: 6.748e-01, Time: 4.34\n", + "It: 880, Total_loss: 1.290e+00, Loss_b: 3.938e-03, Loss_i: 2.351e-03, Loss_f: 6.612e-01, Time: 4.34\n", + "It: 890, Total_loss: 1.261e+00, Loss_b: 3.838e-03, Loss_i: 2.264e-03, Loss_f: 6.513e-01, Time: 4.32\n", + "It: 900, Total_loss: 1.255e+00, Loss_b: 3.763e-03, Loss_i: 2.241e-03, Loss_f: 6.545e-01, Time: 4.34\n", + "It: 910, Total_loss: 1.232e+00, Loss_b: 3.749e-03, Loss_i: 2.343e-03, Loss_f: 6.233e-01, Time: 4.33\n", + "It: 920, Total_loss: 6.618e+00, Loss_b: 2.682e-02, Loss_i: 3.284e-02, Loss_f: 6.520e-01, Time: 4.34\n", + "It: 930, Total_loss: 2.011e+00, Loss_b: 7.589e-03, Loss_i: 6.499e-03, Loss_f: 6.022e-01, Time: 4.39\n", + "It: 940, Total_loss: 1.656e+00, Loss_b: 5.748e-03, Loss_i: 4.692e-03, Loss_f: 6.119e-01, Time: 4.34\n", + "It: 950, Total_loss: 1.524e+00, Loss_b: 5.275e-03, Loss_i: 4.060e-03, Loss_f: 5.907e-01, Time: 4.34\n", + "It: 960, Total_loss: 1.150e+00, Loss_b: 3.567e-03, Loss_i: 2.078e-03, Loss_f: 5.854e-01, Time: 4.36\n", + "It: 970, Total_loss: 1.136e+00, Loss_b: 3.517e-03, Loss_i: 2.214e-03, Loss_f: 5.624e-01, Time: 4.40\n", + "It: 980, Total_loss: 1.098e+00, Loss_b: 3.371e-03, Loss_i: 2.037e-03, Loss_f: 5.576e-01, Time: 4.31\n", + "It: 990, Total_loss: 1.058e+00, Loss_b: 3.218e-03, Loss_i: 1.868e-03, Loss_f: 5.492e-01, Time: 4.34\n", + "It: 1000, Total_loss: 1.048e+00, Loss_b: 3.260e-03, Loss_i: 1.985e-03, Loss_f: 5.237e-01, Time: 4.35\n", + "It: 1010, Total_loss: 9.346e+00, Loss_b: 3.993e-02, Loss_i: 4.750e-02, Loss_f: 6.030e-01, Time: 4.38\n", + "It: 1020, Total_loss: 5.139e+00, Loss_b: 2.211e-02, Loss_i: 2.336e-02, Loss_f: 5.921e-01, Time: 4.34\n", + "It: 1030, Total_loss: 2.316e+00, Loss_b: 8.761e-03, Loss_i: 8.724e-03, Loss_f: 5.671e-01, Time: 4.33\n", + "It: 1040, Total_loss: 1.314e+00, Loss_b: 4.416e-03, Loss_i: 3.425e-03, Loss_f: 5.302e-01, Time: 4.34\n", + "It: 1050, Total_loss: 1.101e+00, Loss_b: 3.623e-03, Loss_i: 2.288e-03, Loss_f: 5.101e-01, Time: 4.35\n", + "It: 1060, Total_loss: 9.831e-01, Loss_b: 2.902e-03, Loss_i: 1.671e-03, Loss_f: 5.259e-01, Time: 4.31\n", + "It: 1070, Total_loss: 9.554e-01, Loss_b: 2.894e-03, Loss_i: 1.657e-03, Loss_f: 5.003e-01, Time: 4.33\n", + "It: 1080, Total_loss: 9.228e-01, Loss_b: 2.796e-03, Loss_i: 1.600e-03, Loss_f: 4.832e-01, Time: 4.33\n", + "It: 1090, Total_loss: 9.017e-01, Loss_b: 2.708e-03, Loss_i: 1.540e-03, Loss_f: 4.769e-01, Time: 4.35\n", + "It: 1100, Total_loss: 9.491e-01, Loss_b: 2.927e-03, Loss_i: 1.858e-03, Loss_f: 4.705e-01, Time: 4.37\n", + "It: 1110, Total_loss: 6.445e+00, Loss_b: 2.654e-02, Loss_i: 3.269e-02, Loss_f: 5.224e-01, Time: 4.38\n", + "It: 1120, Total_loss: 5.876e+00, Loss_b: 2.427e-02, Loss_i: 2.891e-02, Loss_f: 5.584e-01, Time: 4.34\n", + "It: 1130, Total_loss: 1.434e+00, Loss_b: 5.268e-03, Loss_i: 4.353e-03, Loss_f: 4.723e-01, Time: 4.35\n" + ] + } + ], + "source": [ + "for epoch, lr in zip(epochs, learning_rate):\n", + " train(model, int(epoch), lr, batch_size=batch_size)\n", + "ms.save_checkpoint(model.dnn, f'model/{second_path}/model.ckpt')" + ] + }, + { + "cell_type": "markdown", + "id": "7032afa7-b415-4ef1-b49f-97a22e6ba634", + "metadata": {}, + "source": [ + "Save the loss function" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9e56141a-8ec1-4d55-9f4e-cea19c209e8f", + "metadata": {}, + "outputs": [], + "source": [ + "np.save(f'Loss-Coe/{second_path}/loss_history_Adam_Pretrain', loss_history_Adam_Pretrain)\n", + "np.save(f'Loss-Coe/{second_path}/loss_b_history_Adam_Pretrain', loss_b_history_Adam_Pretrain)\n", + "np.save(f'Loss-Coe/{second_path}/loss_i_history_Adam_Pretrain', loss_i_history_Adam_Pretrain)\n", + "np.save(f'Loss-Coe/{second_path}/loss_f_history_Adam_Pretrain', loss_f_history_Adam_Pretrain)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/MindFlow/applications/research/NSFNets/NSFNets_CN.ipynb b/MindFlow/applications/research/NSFNets/NSFNets_CN.ipynb new file mode 100644 index 000000000..5de079922 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/NSFNets_CN.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "de7a09e7-32fc-4059-9478-6c6fc6667f57", + "metadata": {}, + "source": [ + "# NSFNets: 用于不可压缩 Navier-Stokes 方程求解的物理信息神经网络\n", + "\n", + "## 环境安装\n", + "\n", + "本案例要求 MindSpore >= 2.0.0 版本以调用如下接口: mindspore.jit, mindspore.jit_class, mindspore.data_sink。具体请查看MindSpore安装。\n", + "\n", + "此外，你需要安装 MindFlow >=0.1.0 版本。如果当前环境还没有安装，请按照下列方式选择后端和版本进行安装。" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a449343-731b-46bb-b8dd-f7a8c1ba8071", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: Skipping mindflow-gpu as it is not installed.\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "mindflow_version = \"0.1.0\" # update if needed\n", + "# GPU Comment out the following code if you are using NPU.\n", + "!pip uninstall -y mindflow-gpu\n", + "!pip install mindflow-gpu==$mindflow_version\n", + "\n", + "# NPU Uncomment if needed.\n", + "# !pip uninstall -y mindflow-ascend\n", + "# !pip install mindflow-ascend==$mindflow_version" + ] + }, + { + "cell_type": "markdown", + "id": "25fe4081-9d91-44c1-b7de-b50a7645e090", + "metadata": {}, + "source": [ + "## 背景介绍\n", + "\n", + "在AI-CFD的工业应用方面，对NavierStokes方程的求解以及对多精度数据的融合的场景十分广泛，具有重要的经济和社会意义。求解不适定问题（例如部分边界条件缺失）或反演问题是其中的重点和难点之一，且往往代价高昂，需要推导适用于特定问题的不同公式、编写全新代码。如何用一套统一代码以相同计算代价解决上述问题亟需深入研究。在此，金晓威和李惠等使用物理信息神经网络（PINNs），通过自动微分将控制方程直接编码到深度神经网络中，以克服上述一些模拟不可压缩层流和湍流流动的限制。并开发了Navier-Stokes流动网络（NSFnets，Navier-Stokes flow nets）。" + ] + }, + { + "cell_type": "markdown", + "id": "50540e43-3c8e-45f0-8e2b-78831ec89684", + "metadata": {}, + "source": [ + "## 模型框架\n", + "\n", + "模型框架图如下所示:\n", + "\n", + "![NSFNet](images/NSFNet.png)\n", + "\n", + "图中表示的是压力-速度形式的纳维-斯托克斯方程求解网络，利用神经网络的自动微分计算方程中所需要的偏导项，损失函数包括，边界条件损失，初始条件损失以及为了满足方程平衡的物理损失。\n", + "\n", + "## 准备环节\n", + "\n", + "实践前，确保已经正确安装合适版本的MindSpore。如果没有，可以通过：\n", + "\n", + "* [MindSpore安装页面](https://www.mindspore.cn/install) 安装MindSpore。\n", + "\n", + "## 数据集的准备\n", + "\n", + "数据集可以通过提供的代码直接生成。" + ] + }, + { + "cell_type": "markdown", + "id": "23990483-d613-4c8b-aa7e-0a8b16a89b13", + "metadata": {}, + "source": [ + "## 模型训练\n", + "\n", + "引入代码包" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e3d17b8a-bde1-4ccd-9345-ccfaa2cd1bfe", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import argparse\n", + "\n", + "import time\n", + "import numpy as np\n", + "import mindspore as ms\n", + "from mindspore import set_seed, context, nn, Tensor\n", + "from src.network import VPNSFNets\n", + "from src.datasets import read_training_data" + ] + }, + { + "cell_type": "markdown", + "id": "0b8e1d64-580b-4ae9-a7f6-164b0ee0adf3", + "metadata": {}, + "source": [ + "模型相关参数的设置以及训练模型的定义" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8a84c0ff-d4cf-41dd-8afc-db8f234d2108", + "metadata": {}, + "outputs": [], + "source": [ + "parser = argparse.ArgumentParser()\n", + "parser.add_argument('--model_name', type=str, default='NSFNet')\n", + "parser.add_argument('--case', type=str, default='Three dimensional Beltrami flow')\n", + "parser.add_argument('--device', type=str, default='GPU')\n", + "parser.add_argument('--device_id', type=str, default='1')\n", + "parser.add_argument('--load_params', type=str, default=False)\n", + "parser.add_argument('--second_path', type=str, default='train')\n", + "parser.add_argument('--network_size', type=int, default=[4] + 10 * [100 * 1] + [4])\n", + "parser.add_argument('--learning_rate', type=int, default=[1.0e-03, 1.0e-04, 1.0e-05, 1.0e-06])\n", + "parser.add_argument('--epochs', type=int, default=[5e3, 5e3, 5e4, 5e4])\n", + "parser.add_argument('--batch_size', type=int, default=10000)\n", + "parser.add_argument('--re', type=int, default=1)\n", + "args = parser.parse_known_args()[0]\n", + "# args = parser.parse_args()\n", + "\n", + "model_name = args.model_name\n", + "case = args.case\n", + "device = args.device\n", + "device_id = args.device_id\n", + "network_size = args.network_size\n", + "learning_rate = args.learning_rate\n", + "epochs = args.epochs\n", + "batch_size = args.batch_size\n", + "load_params = args.load_params\n", + "second_path = args.second_path\n", + "re = args.re\n", + "\n", + "use_ascend = context.get_context(attr_key='device_target') == \"Ascend\"\n", + "\n", + "if use_ascend:\n", + " msfloat_type = ms.float16\n", + " npfloat_type = np.float16\n", + "else:\n", + " msfloat_type = ms.float32\n", + " npfloat_type = np.float32\n", + "\n", + "os.environ['CUDA_VISIBLE_DEVICES'] = device_id\n", + "context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=device)\n", + "\n", + "x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, u_i, v_i, w_i, x_f, y_f, z_f, t_f, X_min, X_max = read_training_data()\n", + "\n", + "model = VPNSFNets(x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, u_i, v_i, w_i, x_f, y_f, z_f, t_f, network_size, re, \\\n", + " X_min, X_max, use_ascend, msfloat_type, npfloat_type, load_params, second_path)" + ] + }, + { + "cell_type": "markdown", + "id": "387cb210-078e-41e1-b960-e432af9c8d5f", + "metadata": {}, + "source": [ + "记录损失变化" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "482e625a-262a-4f06-a4a7-834766ed6ab4", + "metadata": {}, + "outputs": [], + "source": [ + "# Adam loss history\n", + "loss_history_Adam_Pretrain = np.empty([0])\n", + "loss_b_history_Adam_Pretrain = np.empty([0])\n", + "loss_i_history_Adam_Pretrain = np.empty([0])\n", + "loss_f_history_Adam_Pretrain = np.empty([0])\n", + "np.random.seed(123456)\n", + "set_seed(123456)" + ] + }, + { + "cell_type": "markdown", + "id": "30f8c191-ea21-4055-9d32-dfae30c96a33", + "metadata": {}, + "source": [ + "代码训练与输出结果部分" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cff1733f-6e6a-47ff-928f-be3a348e98da", + "metadata": {}, + "outputs": [], + "source": [ + "def train(model, NIter, lr, batch_size=batch_size):\n", + " params = model.dnn.trainable_params()\n", + " optimizer_Adam = nn.Adam(params, learning_rate=lr)\n", + " grad_fn = ms.value_and_grad(model.loss_fn, None, optimizer_Adam.parameters, has_aux=True)\n", + " model.dnn.set_train()\n", + " N_data = model.t_f.shape[0]\n", + " start_time = time.time()\n", + "\n", + " for epoch in range(1, 1+NIter):\n", + " idx_data_0 = np.random.choice(N_data, batch_size)\n", + " idx_data = Tensor(idx_data_0)\n", + " x_batch = model.x_f[idx_data, :]\n", + " y_batch = model.y_f[idx_data, :]\n", + " z_batch = model.z_f[idx_data, :]\n", + " t_batch = model.t_f[idx_data, :]\n", + " (loss, loss_b, loss_i, loss_f), grads = grad_fn(model.xb, model.yb, model.zb, model.tb, model.xi, model.yi, model.zi, model.ti, x_batch, y_batch, z_batch, t_batch, model.ub, model.vb, model.wb, model.ui, model.vi, model.wi)\n", + " optimizer_Adam(grads)\n", + "\n", + " if epoch % 10 == 0:\n", + " elapsed = time.time() - start_time\n", + " print('It: %d, Total_loss: %.3e, Loss_b: %.3e, Loss_i: %.3e, Loss_f: %.3e, Time: %.2f' %\\\n", + " (epoch, loss.item(), loss_b.item(), loss_i.item(), loss_f.item(), elapsed))\n", + "\n", + " global loss_history_Adam_Pretrain\n", + " global loss_b_history_Adam_Pretrain\n", + " global loss_i_history_Adam_Pretrain\n", + " global loss_f_history_Adam_Pretrain\n", + "\n", + " loss_history_Adam_Pretrain = np.append(loss_history_Adam_Pretrain, loss.numpy())\n", + " loss_b_history_Adam_Pretrain = np.append(loss_b_history_Adam_Pretrain, loss_b.numpy())\n", + " loss_i_history_Adam_Pretrain = np.append(loss_i_history_Adam_Pretrain, loss_i.numpy())\n", + " loss_f_history_Adam_Pretrain = np.append(loss_f_history_Adam_Pretrain, loss_f.numpy())\n", + "\n", + " start_time = time.time()" + ] + }, + { + "cell_type": "markdown", + "id": "cfedd05b-3d38-4156-b061-8278d232a7f9", + "metadata": {}, + "source": [ + "运行训练与保存训练模型" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c658395b-cade-408e-b999-585ba1ae73d5", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "It: 0, Total_loss: 1.583e+02, Loss_b: 6.385e-01, Loss_i: 7.558e-01, Loss_f: 1.891e+01, Time: 14.63\n", + "It: 10, Total_loss: 1.339e+02, Loss_b: 5.654e-01, Loss_i: 6.433e-01, Loss_f: 1.302e+01, Time: 4.17\n", + "It: 20, Total_loss: 8.946e+01, Loss_b: 3.713e-01, Loss_i: 4.430e-01, Loss_f: 8.031e+00, Time: 4.22\n", + "It: 30, Total_loss: 6.822e+01, Loss_b: 3.025e-01, Loss_i: 2.891e-01, Loss_f: 9.061e+00, Time: 4.23\n", + "It: 40, Total_loss: 5.122e+01, Loss_b: 2.249e-01, Loss_i: 2.195e-01, Loss_f: 6.779e+00, Time: 4.23\n", + "It: 50, Total_loss: 4.095e+01, Loss_b: 1.834e-01, Loss_i: 1.549e-01, Loss_f: 7.122e+00, Time: 4.22\n", + "It: 60, Total_loss: 3.723e+01, Loss_b: 1.664e-01, Loss_i: 1.392e-01, Loss_f: 6.662e+00, Time: 4.25\n", + "It: 70, Total_loss: 3.123e+01, Loss_b: 1.457e-01, Loss_i: 1.092e-01, Loss_f: 5.742e+00, Time: 4.26\n", + "It: 80, Total_loss: 2.830e+01, Loss_b: 1.286e-01, Loss_i: 9.858e-02, Loss_f: 5.586e+00, Time: 4.23\n", + "It: 90, Total_loss: 2.403e+01, Loss_b: 1.046e-01, Loss_i: 7.816e-02, Loss_f: 5.755e+00, Time: 4.28\n", + "It: 100, Total_loss: 2.149e+01, Loss_b: 9.192e-02, Loss_i: 6.689e-02, Loss_f: 5.609e+00, Time: 4.27\n", + "It: 110, Total_loss: 1.913e+01, Loss_b: 8.201e-02, Loss_i: 5.710e-02, Loss_f: 5.223e+00, Time: 4.29\n", + "It: 120, Total_loss: 2.604e+01, Loss_b: 1.202e-01, Loss_i: 9.009e-02, Loss_f: 5.009e+00, Time: 4.31\n", + "It: 130, Total_loss: 1.589e+01, Loss_b: 6.606e-02, Loss_i: 4.568e-02, Loss_f: 4.713e+00, Time: 4.31\n", + "It: 140, Total_loss: 1.695e+01, Loss_b: 7.117e-02, Loss_i: 5.391e-02, Loss_f: 4.445e+00, Time: 4.29\n", + "It: 150, Total_loss: 1.529e+01, Loss_b: 6.290e-02, Loss_i: 4.427e-02, Loss_f: 4.573e+00, Time: 4.33\n", + "It: 160, Total_loss: 1.585e+01, Loss_b: 6.501e-02, Loss_i: 5.110e-02, Loss_f: 4.235e+00, Time: 4.31\n", + "It: 170, Total_loss: 1.325e+01, Loss_b: 5.349e-02, Loss_i: 3.818e-02, Loss_f: 4.081e+00, Time: 4.30\n", + "It: 180, Total_loss: 1.365e+01, Loss_b: 5.630e-02, Loss_i: 4.049e-02, Loss_f: 3.966e+00, Time: 4.37\n", + "It: 190, Total_loss: 1.153e+01, Loss_b: 4.635e-02, Loss_i: 3.206e-02, Loss_f: 3.685e+00, Time: 4.30\n", + "It: 200, Total_loss: 1.048e+01, Loss_b: 4.066e-02, Loss_i: 2.881e-02, Loss_f: 3.531e+00, Time: 4.35\n", + "It: 210, Total_loss: 9.349e+00, Loss_b: 3.611e-02, Loss_i: 2.546e-02, Loss_f: 3.193e+00, Time: 4.35\n", + "It: 220, Total_loss: 3.285e+01, Loss_b: 1.496e-01, Loss_i: 1.481e-01, Loss_f: 3.084e+00, Time: 4.36\n", + "It: 230, Total_loss: 1.309e+01, Loss_b: 5.685e-02, Loss_i: 4.612e-02, Loss_f: 2.790e+00, Time: 4.35\n", + "It: 240, Total_loss: 8.711e+00, Loss_b: 3.276e-02, Loss_i: 2.655e-02, Loss_f: 2.780e+00, Time: 4.37\n", + "It: 250, Total_loss: 8.350e+00, Loss_b: 3.038e-02, Loss_i: 2.387e-02, Loss_f: 2.925e+00, Time: 4.31\n", + "It: 260, Total_loss: 3.457e+01, Loss_b: 1.534e-01, Loss_i: 1.672e-01, Loss_f: 2.513e+00, Time: 4.39\n", + "It: 270, Total_loss: 1.192e+01, Loss_b: 5.266e-02, Loss_i: 3.971e-02, Loss_f: 2.680e+00, Time: 4.35\n", + "It: 280, Total_loss: 1.006e+01, Loss_b: 4.145e-02, Loss_i: 3.491e-02, Loss_f: 2.421e+00, Time: 4.33\n", + "It: 290, Total_loss: 6.854e+00, Loss_b: 2.660e-02, Loss_i: 1.906e-02, Loss_f: 2.288e+00, Time: 4.35\n", + "It: 300, Total_loss: 6.489e+00, Loss_b: 2.398e-02, Loss_i: 1.775e-02, Loss_f: 2.316e+00, Time: 4.31\n", + "It: 310, Total_loss: 5.934e+00, Loss_b: 2.192e-02, Loss_i: 1.608e-02, Loss_f: 2.135e+00, Time: 4.36\n", + "It: 320, Total_loss: 5.373e+00, Loss_b: 1.975e-02, Loss_i: 1.401e-02, Loss_f: 1.997e+00, Time: 4.32\n", + "It: 330, Total_loss: 2.589e+01, Loss_b: 1.093e-01, Loss_i: 1.278e-01, Loss_f: 2.179e+00, Time: 4.35\n", + "It: 340, Total_loss: 7.844e+00, Loss_b: 3.105e-02, Loss_i: 2.841e-02, Loss_f: 1.897e+00, Time: 4.33\n", + "It: 350, Total_loss: 5.789e+00, Loss_b: 2.054e-02, Loss_i: 1.715e-02, Loss_f: 2.020e+00, Time: 4.30\n", + "It: 360, Total_loss: 4.929e+00, Loss_b: 1.746e-02, Loss_i: 1.271e-02, Loss_f: 1.912e+00, Time: 4.35\n", + "It: 370, Total_loss: 4.540e+00, Loss_b: 1.605e-02, Loss_i: 1.103e-02, Loss_f: 1.832e+00, Time: 4.33\n", + "It: 380, Total_loss: 4.350e+00, Loss_b: 1.506e-02, Loss_i: 1.077e-02, Loss_f: 1.767e+00, Time: 4.34\n", + "It: 390, Total_loss: 4.076e+00, Loss_b: 1.387e-02, Loss_i: 9.670e-03, Loss_f: 1.721e+00, Time: 4.32\n", + "It: 400, Total_loss: 3.871e+00, Loss_b: 1.333e-02, Loss_i: 9.054e-03, Loss_f: 1.633e+00, Time: 4.35\n", + "It: 410, Total_loss: 3.836e+00, Loss_b: 1.356e-02, Loss_i: 9.326e-03, Loss_f: 1.547e+00, Time: 4.39\n", + "It: 420, Total_loss: 1.469e+01, Loss_b: 6.135e-02, Loss_i: 7.144e-02, Loss_f: 1.410e+00, Time: 4.35\n", + "It: 430, Total_loss: 9.486e+00, Loss_b: 3.899e-02, Loss_i: 4.083e-02, Loss_f: 1.503e+00, Time: 4.33\n", + "It: 440, Total_loss: 4.276e+00, Loss_b: 1.586e-02, Loss_i: 1.093e-02, Loss_f: 1.597e+00, Time: 4.31\n", + "It: 450, Total_loss: 4.369e+00, Loss_b: 1.655e-02, Loss_i: 1.267e-02, Loss_f: 1.447e+00, Time: 4.38\n", + "It: 460, Total_loss: 3.708e+00, Loss_b: 1.338e-02, Loss_i: 9.417e-03, Loss_f: 1.428e+00, Time: 4.35\n", + "It: 470, Total_loss: 3.409e+00, Loss_b: 1.175e-02, Loss_i: 7.869e-03, Loss_f: 1.447e+00, Time: 4.33\n", + "It: 480, Total_loss: 3.202e+00, Loss_b: 1.058e-02, Loss_i: 7.070e-03, Loss_f: 1.437e+00, Time: 4.30\n", + "It: 490, Total_loss: 3.011e+00, Loss_b: 1.008e-02, Loss_i: 6.602e-03, Loss_f: 1.342e+00, Time: 4.34\n", + "It: 500, Total_loss: 2.883e+00, Loss_b: 9.606e-03, Loss_i: 6.225e-03, Loss_f: 1.300e+00, Time: 4.43\n", + "It: 510, Total_loss: 3.084e+00, Loss_b: 1.055e-02, Loss_i: 7.329e-03, Loss_f: 1.296e+00, Time: 4.34\n", + "It: 520, Total_loss: 5.132e+00, Loss_b: 2.041e-02, Loss_i: 1.931e-02, Loss_f: 1.160e+00, Time: 4.37\n", + "It: 530, Total_loss: 4.985e+00, Loss_b: 1.992e-02, Loss_i: 1.770e-02, Loss_f: 1.223e+00, Time: 4.36\n", + "It: 540, Total_loss: 2.961e+00, Loss_b: 9.838e-03, Loss_i: 6.820e-03, Loss_f: 1.295e+00, Time: 4.35\n", + "It: 550, Total_loss: 2.722e+00, Loss_b: 9.219e-03, Loss_i: 6.062e-03, Loss_f: 1.194e+00, Time: 4.39\n", + "It: 560, Total_loss: 2.651e+00, Loss_b: 8.777e-03, Loss_i: 5.731e-03, Loss_f: 1.200e+00, Time: 4.34\n", + "It: 570, Total_loss: 2.435e+00, Loss_b: 8.007e-03, Loss_i: 5.153e-03, Loss_f: 1.119e+00, Time: 4.31\n", + "It: 580, Total_loss: 2.359e+00, Loss_b: 7.729e-03, Loss_i: 4.839e-03, Loss_f: 1.103e+00, Time: 4.37\n", + "It: 590, Total_loss: 2.411e+00, Loss_b: 7.893e-03, Loss_i: 5.292e-03, Loss_f: 1.093e+00, Time: 4.33\n", + "It: 600, Total_loss: 6.628e+00, Loss_b: 2.613e-02, Loss_i: 2.924e-02, Loss_f: 1.091e+00, Time: 4.33\n", + "It: 610, Total_loss: 3.092e+00, Loss_b: 1.098e-02, Loss_i: 9.677e-03, Loss_f: 1.026e+00, Time: 4.40\n", + "It: 620, Total_loss: 2.359e+00, Loss_b: 7.858e-03, Loss_i: 5.543e-03, Loss_f: 1.018e+00, Time: 4.35\n", + "It: 630, Total_loss: 2.168e+00, Loss_b: 7.162e-03, Loss_i: 4.739e-03, Loss_f: 9.775e-01, Time: 4.34\n", + "It: 640, Total_loss: 2.054e+00, Loss_b: 6.717e-03, Loss_i: 4.270e-03, Loss_f: 9.550e-01, Time: 4.35\n", + "It: 650, Total_loss: 1.949e+00, Loss_b: 6.251e-03, Loss_i: 3.827e-03, Loss_f: 9.413e-01, Time: 4.39\n", + "It: 660, Total_loss: 7.125e+00, Loss_b: 2.947e-02, Loss_i: 3.179e-02, Loss_f: 9.988e-01, Time: 4.35\n", + "It: 670, Total_loss: 8.039e+00, Loss_b: 3.376e-02, Loss_i: 3.641e-02, Loss_f: 1.022e+00, Time: 4.33\n", + "It: 680, Total_loss: 2.239e+00, Loss_b: 7.544e-03, Loss_i: 5.005e-03, Loss_f: 9.844e-01, Time: 4.38\n", + "It: 690, Total_loss: 2.498e+00, Loss_b: 8.721e-03, Loss_i: 7.084e-03, Loss_f: 9.173e-01, Time: 4.37\n", + "It: 700, Total_loss: 2.013e+00, Loss_b: 6.654e-03, Loss_i: 4.436e-03, Loss_f: 9.038e-01, Time: 4.32\n", + "It: 710, Total_loss: 1.930e+00, Loss_b: 6.223e-03, Loss_i: 3.988e-03, Loss_f: 9.092e-01, Time: 4.34\n", + "It: 720, Total_loss: 1.806e+00, Loss_b: 5.569e-03, Loss_i: 3.448e-03, Loss_f: 9.047e-01, Time: 4.42\n", + "It: 730, Total_loss: 1.723e+00, Loss_b: 5.383e-03, Loss_i: 3.242e-03, Loss_f: 8.601e-01, Time: 4.34\n", + "It: 740, Total_loss: 3.905e+00, Loss_b: 1.531e-02, Loss_i: 1.506e-02, Loss_f: 8.681e-01, Time: 4.36\n", + "It: 750, Total_loss: 2.988e+00, Loss_b: 1.194e-02, Loss_i: 1.040e-02, Loss_f: 7.538e-01, Time: 4.35\n", + "It: 760, Total_loss: 2.161e+00, Loss_b: 7.457e-03, Loss_i: 5.830e-03, Loss_f: 8.320e-01, Time: 4.34\n", + "It: 770, Total_loss: 1.972e+00, Loss_b: 6.726e-03, Loss_i: 4.789e-03, Loss_f: 8.208e-01, Time: 4.34\n", + "It: 780, Total_loss: 1.701e+00, Loss_b: 5.803e-03, Loss_i: 3.587e-03, Loss_f: 7.619e-01, Time: 4.29\n", + "It: 790, Total_loss: 1.591e+00, Loss_b: 5.103e-03, Loss_i: 3.114e-03, Loss_f: 7.698e-01, Time: 4.36\n", + "It: 800, Total_loss: 1.535e+00, Loss_b: 4.848e-03, Loss_i: 2.912e-03, Loss_f: 7.591e-01, Time: 4.43\n", + "It: 810, Total_loss: 1.466e+01, Loss_b: 6.288e-02, Loss_i: 7.408e-02, Loss_f: 9.674e-01, Time: 4.37\n", + "It: 820, Total_loss: 4.528e+00, Loss_b: 1.733e-02, Loss_i: 1.952e-02, Loss_f: 8.425e-01, Time: 4.42\n", + "It: 830, Total_loss: 2.498e+00, Loss_b: 9.084e-03, Loss_i: 8.434e-03, Loss_f: 7.457e-01, Time: 4.38\n", + "It: 840, Total_loss: 1.777e+00, Loss_b: 5.724e-03, Loss_i: 4.544e-03, Loss_f: 7.503e-01, Time: 4.41\n", + "It: 850, Total_loss: 1.575e+00, Loss_b: 5.009e-03, Loss_i: 3.406e-03, Loss_f: 7.332e-01, Time: 4.35\n", + "It: 860, Total_loss: 1.418e+00, Loss_b: 4.292e-03, Loss_i: 2.733e-03, Loss_f: 7.161e-01, Time: 4.38\n", + "It: 870, Total_loss: 1.320e+00, Loss_b: 4.021e-03, Loss_i: 2.429e-03, Loss_f: 6.748e-01, Time: 4.34\n", + "It: 880, Total_loss: 1.290e+00, Loss_b: 3.938e-03, Loss_i: 2.351e-03, Loss_f: 6.612e-01, Time: 4.34\n", + "It: 890, Total_loss: 1.261e+00, Loss_b: 3.838e-03, Loss_i: 2.264e-03, Loss_f: 6.513e-01, Time: 4.32\n", + "It: 900, Total_loss: 1.255e+00, Loss_b: 3.763e-03, Loss_i: 2.241e-03, Loss_f: 6.545e-01, Time: 4.34\n", + "It: 910, Total_loss: 1.232e+00, Loss_b: 3.749e-03, Loss_i: 2.343e-03, Loss_f: 6.233e-01, Time: 4.33\n", + "It: 920, Total_loss: 6.618e+00, Loss_b: 2.682e-02, Loss_i: 3.284e-02, Loss_f: 6.520e-01, Time: 4.34\n", + "It: 930, Total_loss: 2.011e+00, Loss_b: 7.589e-03, Loss_i: 6.499e-03, Loss_f: 6.022e-01, Time: 4.39\n", + "It: 940, Total_loss: 1.656e+00, Loss_b: 5.748e-03, Loss_i: 4.692e-03, Loss_f: 6.119e-01, Time: 4.34\n", + "It: 950, Total_loss: 1.524e+00, Loss_b: 5.275e-03, Loss_i: 4.060e-03, Loss_f: 5.907e-01, Time: 4.34\n", + "It: 960, Total_loss: 1.150e+00, Loss_b: 3.567e-03, Loss_i: 2.078e-03, Loss_f: 5.854e-01, Time: 4.36\n", + "It: 970, Total_loss: 1.136e+00, Loss_b: 3.517e-03, Loss_i: 2.214e-03, Loss_f: 5.624e-01, Time: 4.40\n", + "It: 980, Total_loss: 1.098e+00, Loss_b: 3.371e-03, Loss_i: 2.037e-03, Loss_f: 5.576e-01, Time: 4.31\n", + "It: 990, Total_loss: 1.058e+00, Loss_b: 3.218e-03, Loss_i: 1.868e-03, Loss_f: 5.492e-01, Time: 4.34\n", + "It: 1000, Total_loss: 1.048e+00, Loss_b: 3.260e-03, Loss_i: 1.985e-03, Loss_f: 5.237e-01, Time: 4.35\n", + "It: 1010, Total_loss: 9.346e+00, Loss_b: 3.993e-02, Loss_i: 4.750e-02, Loss_f: 6.030e-01, Time: 4.38\n", + "It: 1020, Total_loss: 5.139e+00, Loss_b: 2.211e-02, Loss_i: 2.336e-02, Loss_f: 5.921e-01, Time: 4.34\n", + "It: 1030, Total_loss: 2.316e+00, Loss_b: 8.761e-03, Loss_i: 8.724e-03, Loss_f: 5.671e-01, Time: 4.33\n", + "It: 1040, Total_loss: 1.314e+00, Loss_b: 4.416e-03, Loss_i: 3.425e-03, Loss_f: 5.302e-01, Time: 4.34\n", + "It: 1050, Total_loss: 1.101e+00, Loss_b: 3.623e-03, Loss_i: 2.288e-03, Loss_f: 5.101e-01, Time: 4.35\n", + "It: 1060, Total_loss: 9.831e-01, Loss_b: 2.902e-03, Loss_i: 1.671e-03, Loss_f: 5.259e-01, Time: 4.31\n", + "It: 1070, Total_loss: 9.554e-01, Loss_b: 2.894e-03, Loss_i: 1.657e-03, Loss_f: 5.003e-01, Time: 4.33\n", + "It: 1080, Total_loss: 9.228e-01, Loss_b: 2.796e-03, Loss_i: 1.600e-03, Loss_f: 4.832e-01, Time: 4.33\n", + "It: 1090, Total_loss: 9.017e-01, Loss_b: 2.708e-03, Loss_i: 1.540e-03, Loss_f: 4.769e-01, Time: 4.35\n", + "It: 1100, Total_loss: 9.491e-01, Loss_b: 2.927e-03, Loss_i: 1.858e-03, Loss_f: 4.705e-01, Time: 4.37\n", + "It: 1110, Total_loss: 6.445e+00, Loss_b: 2.654e-02, Loss_i: 3.269e-02, Loss_f: 5.224e-01, Time: 4.38\n", + "It: 1120, Total_loss: 5.876e+00, Loss_b: 2.427e-02, Loss_i: 2.891e-02, Loss_f: 5.584e-01, Time: 4.34\n", + "It: 1130, Total_loss: 1.434e+00, Loss_b: 5.268e-03, Loss_i: 4.353e-03, Loss_f: 4.723e-01, Time: 4.35\n" + ] + } + ], + "source": [ + "for epoch, lr in zip(epochs, learning_rate):\n", + " train(model, int(epoch), lr, batch_size=batch_size)\n", + "ms.save_checkpoint(model.dnn, f'model/{second_path}/model.ckpt')" + ] + }, + { + "cell_type": "markdown", + "id": "7032afa7-b415-4ef1-b49f-97a22e6ba634", + "metadata": {}, + "source": [ + "保存损失函数" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "9e56141a-8ec1-4d55-9f4e-cea19c209e8f", + "metadata": {}, + "outputs": [], + "source": [ + "np.save(f'Loss-Coe/{second_path}/loss_history_Adam_Pretrain', loss_history_Adam_Pretrain)\n", + "np.save(f'Loss-Coe/{second_path}/loss_b_history_Adam_Pretrain', loss_b_history_Adam_Pretrain)\n", + "np.save(f'Loss-Coe/{second_path}/loss_i_history_Adam_Pretrain', loss_i_history_Adam_Pretrain)\n", + "np.save(f'Loss-Coe/{second_path}/loss_f_history_Adam_Pretrain', loss_f_history_Adam_Pretrain)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.19" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/MindFlow/applications/research/NSFNets/README.md b/MindFlow/applications/research/NSFNets/README.md new file mode 100644 index 000000000..d9f176ab4 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/README.md @@ -0,0 +1,76 @@ +# NSFNets: Physics-informed neural networks for the incompressible Navier-Stokes equations + +## Background 1 + +In the industrial application of AI-CFD, the scenarios for solving the NavierStokes equations and fusing multi-precision data are very extensive and have important economic and social significance. Solving ill-posed problems (such as missing some boundary conditions) or inversion problems is one of the key points and difficulties, and it is often costly, requiring the derivation of different formulas applicable to specific problems and the writing of new codes. How to solve the above problems with a set of unified codes at the same computational cost urgently needs in-depth research. Here, Jin Xiaowei, Li Hui and others used physical information neural networks (PINNs) to encode the control equations directly into deep neural networks through automatic differentiation to overcome some of the above-mentioned limitations of simulating incompressible laminar and turbulent flows. And developed Navier-Stokes flow networks (NSFnets, Navier-Stokes flow nets).. + +## Model framework + +The model framework is as shown in the following figure: + +![NSFNet](images/NSFNet.png) + +The figure shows the network for solving the Navier-Stokes equations in the pressure-velocity form. The automatic differentiation of the neural network is used to calculate the partial derivatives required in the equation. The loss function includes boundary condition loss, initial condition loss, and physical loss to satisfy the balance of the equation. + +## QuickStart + +The dataset can be directly generated by the provided code. + +The case provides two training methods + +- Run Option 1: Call `train.py` from command line + + ```python + # Call `train.py` from command line + python train.py --device_target GPU --device_id 0 --config_file_path ./config/NSFNet.yaml + + ``` + + `--config_path` indicates the path of the parameter file. Default path "./config/config.yaml". + + In the "./config/config.yaml" parameter file: + + 'case' represents the case name; + + 'device' representsthe type of computing platform used, with options of 'CPU', 'GPU', or 'Ascend', with a default value of 'GPU'; + + 'device_id' represents the index of NPU or GPU. Default 0; + + 'network_size' represents the size of network； + + 'learning_rate' represents the learning rate； + + 'epochs' represents number of training iterations； + +- Run Option 2: Run Jupyter Notebook + + You can use [chinese](./NSFNets_CN.ipynb) or [English](./NSFNets.ipynb) Jupyter Notebook to run the training and evaluation code line-by-line. + +## Case description + +The Jupyter Notebook named NSFNets runs the case of the three-dimensional Beltrami flow. In addition to providing the running code and data of this equation, this open source also provides the two-dimensional Kovasznay flow, which can be run directly by calling the `train_KF.py` script in the command line. + +## Performance + +| Parameters | GPU | NPU | +|:-------------------------:|:-----------------------:|:------------------:| +| hardware | NVIDIA 1080Ti(memory 11G) | Ascend(memory 32G) | +| MindSpore version | 2.2.14 | 2.2.14 | +| data_size | 91648 | 91648 | +| batch_size | 10000 | 10000 | +| epochs | 11w | 11w | +| optimizer | Adam | Adam | +| total loss ((MSE) | 2.6e-3 | 3.3e-4 | +| boundary loss ((MSE) | 5.9e-6 | 1.0e-5 | +| equation loss ((MSE) | 2.6e-6 | 6.3e-6 | +| regularization loss((MSE)| 1.8e-3 | 1.8e-3 | +| speed(s/step) | 0.43 | 0.068 | + +## Contributor + +gitee id: [chenchao2024](https://gitee.com/chenchao2024) +email: chenchao@isrc.iscas.ac.cn + +## Reference + +Jin x, Cai s, Li H, Karniadakis G. NSFnets (Navier-Stokes flow nets): Physics-informed neural networks for the incompressible Navier-Stokes equations[J]. Journal of Computational Physics, 2020, 426: 1. https://doi.org/10.1016/j.jcp.2020.109951 diff --git a/MindFlow/applications/research/NSFNets/README_CN.md b/MindFlow/applications/research/NSFNets/README_CN.md new file mode 100644 index 000000000..5ca7f5b28 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/README_CN.md @@ -0,0 +1,75 @@ +# NSFNets用于不可压缩 Navier-Stokes 方程求解的物理信息神经网络 + +## 背景介绍 + +在AI-CFD的工业应用方面，对NavierStokes方程的求解以及对多精度数据的融合的场景十分广泛，具有重要的经济和社会意义。求解不适定问题（例如部分边界条件缺失）或反演问题是其中的重点和难点之一，且往往代价高昂，需要推导适用于特定问题的不同公式、编写全新代码。如何用一套统一代码以相同计算代价解决上述问题亟需深入研究。在此，金晓威和李惠等使用物理信息神经网络（PINNs），通过自动微分将控制方程直接编码到深度神经网络中，以克服上述一些模拟不可压缩层流和湍流流动的限制。并开发了Navier-Stokes流动网络（NSFnets，Navier-Stokes flow nets）。 + +## 模型框架 + +模型框架如下图所示： + +![NSFNet](images/NSFNet.png) + +图中表示的是压力-速度形式的纳维-斯托克斯方程求解网络，利用神经网络的自动微分计算方程中所需要的偏导项，损失函数包括，边界条件损失，初始条件损失以及为了满足方程平衡的物理损失。 + +## 快速开始 + +数据集可以通过提供的代码直接生成。 + +案例提供两种训练方式 + +- 训练方式一：在命令行中调用`train.py`脚本 + + ```python + # 在命令行调用train.py进行训练示例 + python train.py --device_target GPU --device_id 0 --config_file_path ./config/NSFNet.yaml + + ``` + + `--config_path`表示配置文件的路径，默认值为 "./config/config.yaml"。 + + 在 "./config/config.yaml" 配置文件中： + 'case' 表示案例名称； + + 'device' 表示使用的计算平台类型，可以选择 'CPU'、'GPU' 或 'Ascend'，默认值 'GPU'； + + 'device_id' 表示后端平台端口号，默认值为 0； + + 'network_size' 表示网络大小； + + 'learning_rate' 表示学习率； + + 'epochs' 表示训练迭代次数； + +- 训练方式二：运行Jupyter Notebook + + 您可以使用[中文版](./NSFNets_CN.ipynb) 和[英文版](./NSFNets.ipynb) Jupyter Notebook逐行运行训练和验证代码。 + +## 案例说明 + +其中命名为NSFNets的Jupyter Notebook 运行的案例为三维Beltrami流动，本次开源除了提供该方程的运行代码和数据以外，还提供了二维Kovasznay流动，可以在命令行中调用`train_KF.py` 脚本即可直接运行。 + +## 性能 + +| 参数 | GPU | NPU | +|:-------------------:|:-----------------------:|:------------------:| +| 硬件 | NVIDIA 1080Ti(memory 11G) | Ascend(memory 32G) | +| MindSpore版本 | 2.2.14 | 2.2.14 | +| 数据大小 | 91648 | 91648 | +| batch大小 | 10000 | 10000 | +| 训练步数 | 11w | 11w | +| 优化器 | Adam | Adam | +| total loss 训练精度(MSE) | 2.6e-3 | 3.3e-3 | +| boundary loss 测试精度(MSE) | 5.9e-6 | 1.0e-5 | +| initial condition loss 训练精度(MSE) | 2.6e-6 | 6.3e-6 | +| equation loss 测试精度(MSE) | 1.7e-3 | 1.8e-3 | +| 性能(s/epoch) | 0.43 | 0.068 | + +## 贡献者 + +gitee id: [chenchao2024](https://gitee.com/chenchao2024) +email: chenchao@isrc.iscas.ac.cn + +## 参考文献 + +Jin x, Cai s, Li H, Karniadakis G. NSFnets (Navier-Stokes flow nets): Physics-informed neural networks for the incompressible Navier-Stokes equations[J]. Journal of Computational Physics, 2020, 426: 1. https://doi.org/10.1016/j.jcp.2020.109951 diff --git a/MindFlow/applications/research/NSFNets/config/NSFNet.yaml b/MindFlow/applications/research/NSFNets/config/NSFNet.yaml new file mode 100644 index 000000000..469a881c5 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/config/NSFNet.yaml @@ -0,0 +1,25 @@ +params: + model_name: 'NSFNet' + case: 'Three-dimensional-Beltrami-flow' + device: 'GPU' + device_id: '2' + load_params: False + load_params_test: True + second_path1: 'train' + network_size: [4, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 4] + learning_rate: [1.0e-03, 1.0e-04, 1.0e-05, 1.0e-06] + epochs: [5e3, 5e3, 5e4, 5e4] + batch_size: 10000 + xmin: -1 + xmax: 1 + ymin: -1 + ymax: 1 + zmin: -1 + zmax: 1 + tmin: 0. + tmax: 1. + n_x: 16 + n_y: 16 + n_z: 16 + n_t: 21 + re: 1 \ No newline at end of file diff --git a/MindFlow/applications/research/NSFNets/config/NSFNet_KF.yaml b/MindFlow/applications/research/NSFNets/config/NSFNet_KF.yaml new file mode 100644 index 000000000..910fd6d0a --- /dev/null +++ b/MindFlow/applications/research/NSFNets/config/NSFNet_KF.yaml @@ -0,0 +1,12 @@ +params: + model_name: 'NSFNet' + case: 'Kovasznay-Flow' + device: 'GPU' + device_id: '2' + load_params: False + load_params_test: True + second_path1: 'train' + network_size: [2, 50, 50, 50, 50, 3] + learning_rate: [1.0e-03, 1.0e-04, 1.0e-05, 1.0e-06] + epochs: [5e3, 5e3, 5e4, 5e4] + re: 40 \ No newline at end of file diff --git a/MindFlow/applications/research/NSFNets/images/NSFNet.png b/MindFlow/applications/research/NSFNets/images/NSFNet.png new file mode 100644 index 0000000000000000000000000000000000000000..ab661b771bc9d6c9bab7bf06bdf9f5a7557b7c2b GIT binary patch literal 98487 zcmd43g;$&3)-4PaTHIZO6n8J~?k>fxxVx5O#T|+kcemp19^BpCo!sJU%=G1O$?_l$Z(x#0T*2kRq@j!LKNn)53v&Ksu{P zia=CO5FUY_KwAjQ3qwHE#2~yH!GNE`IY?Tqf|_l{%hHrnv)pP7q4p=4m8fJotC0Gdp)-!8p~ zY|b5TyewZC-NWHNE}nQIkMTdOI=g4J&I{g0J0Y4H{nyX}zk&P_{%aAKS~-UQ{j(!% zxl;9ijT>5nTI;_?2t|iU|6fD?js=hXUkmMmhxuO@7BvhG4E`VMC#@Fg?)e|vM*V-| zZEH2DPxZ^QJI<@LbqPUYRu1)^UW&e~ra$i=E&h9>^w-F>^3)XexCKSTL=`f6Er(na8AnHz3LT)jrgXL+?b-64fz5w*!*Pskgyj+>;B%Uu zm%?~jiqT1&+gfTW;Z1Qkd2Z|4bdxo}=Nf`YH9jLH6EE;O#f!}lPpclE#vZ4@JB|VO zaxWc$BgR#6Y*^mVUcvqH?wUE`Lq^e4n)LDVfG8&vPZF2$AX6qE?%qm`Yut zMLM4Lc|3J%?(X@dfGX>9x$8eWw!J9I2^$PO?payGQ|_lO+hJ404YCQTF{!RBdup?a z$t)~7*}qZT=eJqe*Pb1WPGaoPYI)EvvDKXkDB)H_MIiB|xH}?OFClxoO>ns$eoyy= z_255mReU~WJ8HXW^V0KcP896)7sgFyN)dPxd)`K%;ncwARi`ltoANF!M z*9XnjL{M2{PicQ1gKqf8Q-`2x=`l0pedn47^ukXX6xtaXPqSfu>Tum-@xRU%;d5AI z-eXnf)>R&sDz3My7X|AV65%^|danJtAYcG>)X6+r+@NbzB`-In%vU6IyeJ$`b^DWl z=&GBl$*cEmW%sC-9JqmZJ5OY67JGiOI9hY(kBe1)eW{u)Nw^&(0csjZFX*3i8X3if z))-30D-d+52Im&k$r5{u;gin=)GWiy7Tf zs-tz8Pd(A`DeV=`3jWGw-mirMT-2 z*2R^AEbXr$tKvRwF6Yg_5G3GGoPMgZd5vWP6?QViO^-XR#_a}o&mmA8pGD;PO0vY& z8e~c-RE+h6>*F_qDcUx1zfoO1lm{Es^QX|!$`e>4nXh>vtuZ&v`|lJmgl)?5%d^Ul z;H7LEA)`EDl+cIH;)3`yWLF-$W zUcMXLEMJW@F2(1q&|VCk>_8gw3)$MG)p2vLx%Pt6}$U_G)D8Tg7ihcXnkg}>| zbo5uQ|B1P1FH>;DRn62Ez=hK~<|@Fps*WA$aai|quc;fg?xXn0yEir~8`#KcY`=?v z!aq=}U5jA>i5aw81mwgBwQbhv?+!`d4g^(p2HuBz#eT5@)9t;!Aa}kWXY+{wXkLPG z@)4L4D_bW7JyQ)EsIIXzDz3$T{3l)Y*L*LBgRzKlJC#vUvne-5{)sCC3>mIE3im>C z9gNk|yL~=sDe<8-(5$Y8m;7vf(Ad^c2K93&h?xI6%%3a*fY)NBZvS(d{O3=McIINx z+oWKecr^w6&o#*w{r}0BOQGd(!NuHIO!(v$3wCEnqb`eYqdP6X*QjXUm)YN5`~6)l z-{5Hzb)JQtYUAaVx5Kfn6?c(hcM;$iH7CGV?CX=VUG{k3kKcUJ-e%{;D7#XKU2GdX z|7~Fu_rsyy(nhnP*e6gX!zV{SqqaYz=3n802VPmR6wUFp;7Fy#{EAPujLF4#?cP?F z!_h4jdX{*8r@hL#XfRt+%Z=JAnNU1?iB8(|O83U^J|)5*K;mgqQv>3v9Oe#!cTbH) z9q-4ozE5UM@;XVoo&FF=1YYw#d{~n3vt|1HihuH>#^xg4uoor4eoqz}TmKa!g?*OF zS<~`Z)$uv^WI@Lh(@x9sZ^CnQ0YtR5{l;(jq6veMV!gCHWl;7liC$Gg85FIOU2{_J zen2b#S{PG!;Z|>Q|LqQR{b^jDbY%G1Vo&Cb$st;YV?>uYxG3vcLA{jz)<9gOwo&dR zCmlaoeQBA&WiQkH=v^{vMwZE4^I_qW)qBnnt{oIJcl^&zA9tnbp&_4kcua@c)(7|H z8k0Q#0dSCZtu{M12*%jwyKtj5zgB-R$DupZnr#xWF>&dL}y6MCM^+-3i@q%@|= z)M6zrgDWn|t59#8_D%lCVrAzaNKSsu!yP-zV{o*VW#{s^%$cO9schXj-%aA#x=28R zRQgXQj}X-Uyj8U1yR2*DyRNmko|{{Zrled=5{5r&SCyKihqC8vC*aq4l{Hr4<*rOx z6*Uzx&I5x#xEe%>%HN;%Lr63EOzERHBWM`ubguiOL58YU-JP^|taSJ$zjOxoPY1f4 zEn9v#AvrL*J)V=bZ@)_hjr3L;;FJJ~^e}5_8~*Wm@kX)}L6*-<%1xn)(}~@J)+zq* zVo7`klhI!*8FWFwTY;D|?WNk5`U5LlZB8weZ1u6u5yOireAb%U_Zu6Fy#fZz{Tj;B z?qcQ7YakTiYaZX3J2UyR%K)zjPdNrUf_)M_x#0QxSzTXMgLcbfwG!pL$e_1f!o>^2 zZYawW4Uiahty0zFqt@9{ow)Y;=?d6kbhznpD)_01be~|ddAu)yQNh* z&-#*x_1Fw)%Zq~1AJ%;4ZM^kcz4&2K%&YkPO+GQKY zd~x>hUIO!wR?u%>BQs061eMpTbQESc{)Wy4xp&4q%*oNPU(9j#d6m^NGU)5aYWuAe zFV~~Y5<;W)Y;Em$z1QX3LH+HpAUFSPXsG0YWj1zR#S;*DctfBCUCb2kz!KYN@zK#N0A zK_)LaCn(PLh^x3>^-023bvEWxbZQ))ULn1V#R5%f!vU7s87fW)M)q%%cLf1Mp|I#V z1cq>Hk*L(L$?zYW+|TEe{9bOZxf-VP1BnUDrl5i&@42@N3R{_J!!JsIL2N`G#1c6s zVy9 z8Orngl5G#`?cQsh4D|H1XpT{LySwr-tO>~$YQPPW64x{Yj359{_PVvI+>K%c7Kx>D zQ?Xm)s7!5Q-e>>M-)^_!UyHx3&E=;)t4EJ?kmQ~QSQfdjV9pVgh-DHBVV~IlO`ezl z@9fE7#c6RagA%XLz=*UcAY73_?g`JV3K~dGzx#ZxwP6lrUGOMBVU;@>$ zAOYZe2?;f%Dg=XoJsXZixB7Q2u90(d^VIXC65&%V9FoYUX)}VMLmxBYTKuJR@1Lqr zaDuOkkN6ogoy;`f!2eEv&3c_kO2uTjHo zt;&Abd8d}Ui`Q}%0f&F3L#2+u(qg|v#{XLFGRi=Vcd@bYs${Vh`lZ$JZVu?aAuaeu zk?G&T?%nw1*LSZi`mUKCyRjhini)2Ejo(6$;sj*qBqZ6}N#N2sv?3)Z1^oTld)8-+ z(f1FzG6|?g6)VI}H7WbS(h16WaXMP>6hWO7N3|2l6HBRP@4rB!Ha2}l{X7Tr`r9Bl_K84)2V*Ccy^OLVh+4! z`F@ErpmVy2NXVFhZqvX?<1nvF&(s;uP7L(8EFABM&Rboc4_qs{%jn}-IO~S`=wj3YN zxD+3inE1$H0qE_rNlr|T%Ed~gO++Tf#UdkVmKjN z?XXs0`CSvGfqJt&ns!k?+kOb1ZS?4cgpu&ga_DF+0)yRCvDQx~AzxL2Ay<4H;W)A4 z<@VNKScX&VVa3z?ww|r(5iE8Bwu zyU7-#>pPWBRffUWmF2;s8lM-}!CgHs|CGEwuq(UY3-~sC(MAACPmTS7!e1*b&SHu$ z*2Jd#mShn5#=GM>JnPY)MOZY4gFV+oMQPb(e!(C1ZnPlCJ7UoRYW?EfaFR&2JXce! zKbG07w*_hXF_byzaf2cExN$ojv1c0H$0qB}T~ zRca_zYbx73M=bmvhqwS+%^r?b`dD;i zY5A1$ZOo+v_KDbxi2`r4tpb?+K6a@ zQj1bANu{1Ek2SCz%kyj^J)2mEezX6TM{t6vG$W!+n<}W`o~13Nmm7b3x-Hg^!&FRT zi*RzSRU)$l)sp8D8NL^x(VAeqo7w^+_8KbqU}^az;?*h=89pdcroR{HHN`ioXPWcA zq$2PGdM5hhUS?QAZN6AZ%3hs2_s0OZbK?%gf+y`tcgQt*tEs7>*^=C%){p~tx%b+a z_hbQIH{7#l3nE-rY=JliQk@{i3Z2jVFXd_-ZJ=&GWcF!}1&?)@5ruOo{SU{kZ+BVT z;AYbP=YsosoYma5AnwOylo-TM#KFEVHqce#>)pVD(N&NlqU#TsAp?mdL~Y&xt)ybL z3Dj>OGh9wTw;cFq;=nLT8zg>i7SrKvxphIVxJgt8cK>h6@V^W~#YELUjJjqtQ|hI9 z4FzsADm}mH@Gx3nh8W(}4^BkBs-FCnO8|gvVqFky=MrnH2PMw6v z7D%Y@R#)I7@oi15v--Glt)SM#?PyC~nx{_3Pkl_t@o#fi8$KoZ>=^a-@^Avi)ovAA zK@*swX!$>u8~B~%kw3^r#SeDTidbZt`H`UL0S=3WlW4OZc7Iv4d9`1MQ$?q7ey$(t zq{hl{=z*)`3Q3BgRV&`y_tDbYg|S&_l((LL-Rm6Tk*W9A_~7@dELiA<_$ua3xET1( z9h}!(=5{hB?ajf+lik8aUHtJE#h>92=8N@ z%%#B$mfPcy_LB;xvQLQDIKx$nI+!yf68k^O+)s9;U)V?OX2IlH90Hiz`s7WO$f&22 z&S~pZu~k5(O~?Qg2UMdXn2jbkY)N8!LZLPu%@mcHkE_11#dVhoK?2F6YLe0qw`G~; z3QCAl>Q&aNKXbCVflMo&4|(`GIM6vJyrs%A(VI?Z`kmul2b&$es=Nd;wPOM0V>xbD z-_BNIe_!Q{$7SRjfhcGyJ<5;M@A2yemNsz51DG>?Rg!=;JYUFjDL8dFkW8}MNtJbb zXzM1!!V)v~)nYaNZj?fNhhB{VZ&Vkx9$DwQKvi|+db{GqpD*}-f%X>!I!qr_H??Lb zv$NIa86L=j1J>erKEBK0zvS|h&H zloh%qN$Dxpk0^rPyDXl0^W|$Pl^#+AcK3^#-LRz33}nuZXJJ1uG}c%a@(C5 zr5c1CJb(94Eu^r_qe&WpY|w& zDm_f&9EAq7o~UVPq(Sd5RSE)6rYZT_T7)#GOLdYr`{SxR15sZM& zeCiBTPRvnG9*j9$ph%WLq58+Sg7Opbaf0Lz8#_(g^483lm%xb$~4b3_I|<9dZ-Au>nl8aEh}*R zO8R}s`ZPDqXB^6~UUU@#c*yDy<*MX*SWEcrM($o#)Jk~L4WP=|@C=^yx)8|ZvI1P+11-tPBIEb6PXWH_7KJ zOL%a{Z!d0o)%4mg7#ObUC7@3d#j~YoMQEiq?^h=!{mIm+3pV-1(cI4 zy>kkq-s&Y@#Fz;%U)qk0EjPzxF#7qZD0>yOWC5;`O)O?3m|X7rw=;?hfke_gwyz3N z*7!xqRG!!`PJ1Jk1Ce-k3iA3w+f^thrx#2M_7_>cV(WHhYrZ0TQ2axTMbi;J5W~2j!!^y7symBANspuI(4ixf2#c_-|I%x9YlgPL zlBLns+GPbStTfqcX7aecvUw5>>+t$kT5*=8kfHx`8Tx3QAV$@;Y9!=Wvf7s2bmY%xK|RUUTbJ`3=SO17ac*zME| zJw3Pn%l8;LnmT+%fT52*nXga<-_2T*=OyhfxJ?uFrq;?wUS=XC?OwC79ZhDidk?*% z)!L=h+E1;{d%in)v4}<&%Pt+1e3y(a7CcmJrddB!36tktY?JTGsxrbDZ(Vm$1b zL9bpWlVZT=dmC2|W@#LGgluZEp`a0SAWpO3vA-ZegL1_qT;{bcp8g|XAma?43BcD` zOuAv%!6~MKyVKTvQ2b}vud=40$V9JY$o)*6QlO_8Fqf_JbgeQ|$oc?aG+1s@OcOMC zzSFe4GKTxHFUusopm*TU9kYGWS@4M@(eE`bGpU`YyKW@$stqyG=@yZG}CU8^Y}d;vcD zpTYJWS3UWEwT4$`nmmbjJhpu6%B!AOO1VohQEu21d=9I`))kId`<{yB#w1H#P!#H>G=L=|1Eq~! zI19=XA&ez`)+R3Vu~&{c`AZ|&zjQsa7l&0i7(UpberUSfYJaG{Br)@Qb=VD2$Ux9` z#9ztFeQhs$j3SNWhW=hO#Di2y5}()Ii!4W(yo#?x%rgI{(;gqna596wpD6bw#Z)q3 zkeAVw*@CsJwfI-jg*X^vTigB+0HSBl79}U)E^+qRJT*C0Nqs9H<{N^D0eHe8! z(@0xQ4A+?cl6LAH_S1x}mMM%!g&*k6o!pCnhLf#Q;9tsSgp1=UT(IEnHt<6T7tu%(b)Hf_?a#$`y2{IY+*HnS z!8oKp5Bip~)JIamhZt^s0oHlkT}3aXMt=uoQ6IDKVVhLrPm_+iAqI0 zkdrYjXDI?SKS1Q26q}lFO4X@g=NXlz&d&|Exf4_1eIB>HGqYZO3i!(aciO84J5Q>? z2qb_>V1NC>#TI$BGKy&v@A~71qvfyrgdF7=ZjqQ|-zj!)^ZZ!#sW?+ieyYu6SS?Hl zS_-{Bc76Y_=2oHGRC!`#7)&!nWcX$IZ;?1z2m94W4=6mr3f2>BgrFYroqTwPw}e5! zfPqC{5B$H?_1d5vO3Cj@%Hfza@nqGimNN9^X541KSOO6~P@y6eOrfP87v& zA5VXezQcC|>#?!%AWh8QU!UA&{uL4tgahDR5J^ZCuJ#u}b?PgX{2m6OS}3P17iki9 z_VfLhD4O7jx~t7iETx){iiw8(+&qV<%Zv8e?fq=OPK(3u^D@W7coBMee5}2*pbX{z zMYy*958--huZ8`;3D=K+yF^kB(*gx)X#~A#EcoTbOU59PrCKw+98~Ta*bjM%f5Ylm zcO)L0%W+8&F}uNE1`%wvB!j_9UAz#&6@=(QBrX0>1c62ch+%`^nyCVNVc%I$o8A$0& zu-}msQv=WZYmNwv=mbi&bCBo}us{zeBZUr~Y^K6xzg`ZeQ6F33apv1R%aeEgsiB7n zK9+6#@yPU_{9%P3bbYE<3{!?)4~}~k+1E!f$&gn^4%0#KLtqb9QscP(TO3tSI1W># zf!|aAI2u7!HZOZx()-YQtZYI7wXal5xZE%@WfB#7y@O;<)h9U6O zCx{lMw?tl>b=3AFMiP)_Ik0N0a%uYoV; z2_Z-c<`cJy&qFquUAz|_##_<6;eXD4QgiP_vaRyd!qUJ8fjOU^n9J>{_wR#{LhpY< z+?uMxpa;fUTu&mWIApleyBB$Nu>BFTr`j{EkApC*PD)49iyi!0aen^20rC4C z@aO-|1xe=~01$+PL?a9uxFjFDE=EO{Vlv)N(#)4n&krTJ7d#~|+_yb;D`I~=)5qs1 zjmhe~Vj2Ht@_s%oTtHS&+e#WkaPTvloQT7*h{pYF85qUoRR(|ofA;;ct3DLjO0ob{ zT@MSbk9UCxj1CVah9(?RxTF$ctmx@zC_x@DzblT&TUG18d@XP)=P&4D>YxT|icJ?1 zQqFsxUCr9851}OaU<|{F!!9cP6HQ}m1a`>amw<0eYog<~M?*Kr7>z$EbXD7{&Xcb0 zc>)1?&#qw2aJm3^wME?c7xWFO6V7xpXsrV(wg^~*_o-rsBEfmBcWQ$7f^Pv`|Br)F z%BqNuG@suY$58}R{z2A_Jw)#Ro&1k5TqNLD@;W3v+E3r}qK;;V$Gn6koEFIB2@VUE zn+4bHNj9hJVc;l}_rW(+F`O*uAjq0sLC|a63m4RPw~t}$%eekIO(kuEFJKxb*>^*>Kaq6aB1@~>v!eL}3BkTay5-pl zESm@AyFBdyAFtj&vBElqq(<0GSfIg6uK2wvjHdAyPM9I%ibUg4*fXP)9E^Xc&r+TW zEn$tpKTYAhV@t@3W4A5!XB2*Zle0WrDP(w6jf=faSKh-KgZ_G${m+UC5YC7|0%Pv* zbMYa_2zIFen1PF)oX|fag`CiEeJC$QR{c9~Z)bp@w`|EE*oG_AylwT9634ujQl;Ce zLW-7;agfS^YW~k9APHg!aD&9=m^<6Tk%a?_ayf=xtUe1wXe-bdjEsZFx}L~3U8ING zswWyA#-h`35y#hufrcoXi=`mWnnL{M{Twd9n-$>rcv@FLV0Wp}Lq_t^`gOfMXneRn zQ?kqZR%lZuMgJHsr1SmSKK7GP-y)H_l#PeLx7dT4=OZe6%^M=#Hm4nV6J%wadnpr> zj>z3A4D9;8Vqo0_�iuR`>L|SpqvpW8S%qvY7vPT9sjXlFOD&yaXmY3dN0&?3b~x zpz?s(x>EL2Y|A+n&^CG49h0=6`@`3kL7wOMFG7+2QMQeL~KgtA5(v-o` z+hwSeadHcetsoc@?ZUvo!AU`v2!fzf`W^6sPbzfH9T?~)$sV-rA!`*PR|k(J<{iET zN9d7xX$%wRB;nq?bEe-*3VR2h=dpbEVWN3^*l4$orqHr_7-MS>a_h;*6$~>OfHnER zByt1B#v zHsQ)=QQCxcx3~RY&Aa55oqBsCV8PnmVN4*9^d&5CsvScpHXEYa%H(VyE$RWn9%_rN zPnevt=W6qZetE+dgWXpF=)<;~{7=M}_2p!*Xrep!nc#ah{vx3GDfZX*TD+t*<&@_p*SP z2}AjadzOJv&bhKUil?bYTjA=^-zWh1SLbtLU$hB>{tPHuT^CYl%6(CYe}6A-V66$w zJA%CtqE#M!wSS*50IKHKd>-pPv0|$p`%u;;F(WPUW;QDuryBP2X2j!b*R5fP3vKJ2 zDBDrd6VqQr;zT@Ht^`;o+J>V+5|7OeFe6}rgzE;5qcDmGQ%q z7!Q*A?lQ-=!J)Mr5XUGfW}l;Vw;`8?r;@4#fzaR+GnsMe`%mUQ9dESi;FNn>1;dv-n<^Z=3kNriGhii7|$xG=P;$bo5_csPF8FB!KGjb zj$YjbeQyU5FT3y6Ehm*XVo+PwV-aHkJk0i&a-^l>Q{1p)>k8xv9qrLP_A*~5N!QEjf#bvaDwStM z{WZtUC4270Bn&W6w?JRKKww|s1jVL{dhqq_z!vDfpL2VM2QJZ0+<&$YU|%NBb1g6?K)NEaQ3It&l*_ZDtoHq`-G5(^z{Ofy z_ZgHumpx+ zEUVm9*G3~EMC>4gcQxQPG?WPD2`(&vMz!z{FU$w}Ev+W$KmG6%t|f565ENJFrJ{28 zz=HiJt4Agy+;XLxc+W}#Kj+!+v>U!nIQqeJ_}{C|qu5q!Jx?k>o?w}*AcEsq_M?g6W@joW>rd(wYPt%(3Rc)fYa70_`GHcZ=sGR(@!4U)Ez z^cP??9!fm29W$|)_!jb=g^+d|e__v|3wsudoPDB9x&Kw=Cn`!tzPfaF^$)GUO_{rn8@wI`jUAXvH{pI)-o#K%-l6j!15>2E zEHs!}UKoMl2LMbk-W+u^-rDyKf)vAc3g<|lBm+1Z;|=V9(Kp`*53Eay*jYIN*lkEh z2uj7QQ_46DV@#3@(B-p`VeC#9dcziz?uBrCoQ`0 z|HB;Q<%V&@rW!VhfVXEpw5uX2a&pH4{cL)HJ~-CDv7V^F5?)`9S6>t2vhe=3l$d#I7t3UI{LQnoPkAhx@>Zr!thcX#$RgoTT3^{u-FeA za|#xS`&pGV!ya!B2t9BCh5n$oCRr5( zZ2H4^y@=iqwx3vfodetfu!6a^Xdhn{NybuHdgZMY#>-HB&b4||_8*oIS=PMD{K>Gy zG}{#O7T$`}M+Zw!3rN1rhhT-ckfa?Bg?|k?i>1%jHLMf3#u!?X#+b%(-8gi8zS_SI zXYei&NNo_N6JX9~KrBxUDmr%VHIs4xfkDDtu`;=)gZW2~3N5!3_4q%7e;jw%9X5rK zhsRdM5DxqM@Jm z$-S7EMN?pEF_EYYI+YIMYwd-u21O#);98_}Uhrsh@w*lsr;LzXaXrtf7N{y*^#nK} z>{4tZeK`Q1Gtdchthc>k%=DR2ZK?C4*^5zvn@0kaWWB_v1Qan{3W!7PKvngENr~09 z*}+J>Jjf3p|5CeYg8m)6jkxv{(*_<}!7bb2bhC}Nx|zQBOEg|H}Lt%{%EFZ2jT|< z>z$8-8A@Gj5W@YksPN@csOmjYa!(nlO58nZN-xoau>>WJ<0MmY949jebj$csW z+djqdZd99zbd#N>3qkP=PkDqybBtxsnAyleNa{+WOixnPE5AJF?DvRvqnELnKQcK% z`rvH*LgUtzv>uwIDwcnF{)ETh+x^CAv$$;+yZBr9gS#VSn{Olv=hv6J1p_dXKz#=u z4#WBw1`Ykexmmy6>v*vykH`JI9Lz1YuEQqtO4vXX2W1HTjL@I2jw9z_oy*H6=6#?> zG>W7(OvEg76J6Hs;m#)Au^I0;mK3#Xm&MIqu2Y}|YS0c$h8^j>%OV@a$J+J;ptBu1 zQftik_%+@tYH6jKoX7R?kF{Iq)1S489$#juEx~^5v0#NJ4=P=^7`n!_9A>^DN{U?O zf8^SR2v2?1U=u`q>&vpBkqsb7Q&8m_ z&lJj6{;nntE{D99Fy>x|a{F4Sc*zb;46{9H>F(B3*p7y{wdo8C1G`fb`1mNs#tGv2 zcJWhf%Vs8(go@g~3}KGj*rN~9a82-{?I?}SLSf{)S(eFqo9FrW0)FnW85CqG4+r|M zf0*|I!afad_eZEA_jrw@7#uUEqze$*65Qlb?%Ee1%#=VswBF4dAlAgpJYMbS33q{H zQy5)Emo8b^Z5EYM^u0=WT#xgU8FX8!6u<0YKJf3m&7 zW*Hmu0WU06D*OfO!j;~VvVa0(hk6<) z;^E1bZOprnMAVrS+@?e$5USQ?&AGs88)?rfZL)N=AU#2QV4!}`qKEWmFx(sHgx&>ql>$goOs&`R+K#`NjU+nqBX+)CkCLWWITBktRA(GdBQI|kWq&-0 zlk{UTq5rV;k$1pyio0YfSRM{r3d3x4LM4H6bj^VYnz`nUBGwuDcl(bYjmXG-P7aq( zn}&=qpL)`Myo=82ISnI{yX`-@D4oA)TV4bfA0KcZi}4!Ax5LX`h5O# z1y9M{Pqgj&+7IUwW8#Q-XGZUwH|LYlz^4BENu(uEWl4P(>XoW~5~JVXrGFDwRWF=M6kv%nTdk>et7fMF(ly^RWH$>aLc2a6J71Nb@GRwf!G*v8c;fqPJB>}nqemakje`WoASS@k;<~H26Xi96bv8$LUta(LJ^Y z?j7e|RDn1zEcTuL#D*u-8;6I#gUXJpfDzQ4^0)gvQbVlZsp@+#h#m}y7~(UXYXD%E z%+M&9S=n{|G&IuiZ6L4OALQ4-gU=~A)El6rkjmU0+eyvU7q+04hXzZ;-2j~p-MpXF7W+L8u>-Sf>yOu3pOQXTxCu(m4L|)GQ zsB11N%RyJiWe<4Q0zv7p|m%)9eKVQR_nrxlO~2`dC?c-VXnX=y0&9@hUK&5IE|3Kh>B)_HItnuYDxUrKmlGu2BCy%Zha z4VopL$F-CoODIgXdWAJd0#&sQ1r>c_$5BYmVV%snD*{4z018o<(i0Cdw1|@Z6?Rj- zt2SznN2`Su!V_Mr`lvMzEi7ZJH?-BH9IVGNPOrB#x$wSZPaPy;~>5zyuFyL)i_XS zwuRHqC@{qs6lA`*9c6zJEM5WK%`H>73A!cz;jt!z%=ZViKYL!PVe#IW`||onGG_B? zv9WCi)D8hStBXW%=>C~nVGT3ccl^l(A*ArZ@N>|sNwWkZw1$jG3@Y^NBs^~{KmbL$ z<_S()GIlDG89a#6nlO8+J=-2+-iI%whDxj2>F3Ku%+KEZiy8m<;G3)!rOPLk@j|16 zEbq$nud=Ix)M~}Ko-q#G4`!yPX}^W@+=~4|8);J}5Z~@Jo4wik`Z6}d*?Jm=28Zo7 zu^@dUU0jqOc%{b_h{7_Jmz6tzEElT6u_197K^ap_5RvaHPlXJAMlZBH>|s29|Avvo z7+=0CQ*@MA@4Upo8XnXP%$5ksitkQ}|4RYUM-%0sp34IoGqKbr!1!=MH((E6$ff$U z)&R;0zxHL~HZ$FaYT}M(uUPXf-Ir4-c2A+4Bc80JwCh%^-n?M;n^2zR2E=kM^ z#Rqu`F*Y$0pp;}eQ`k)QeNG8b208paww8{W?#l4Ud`V$_5JOntLlqlTF0WSW`&m;v zQeI`NLdzNFQNse?N%1s5@^dnMT*tf4^Q3=1fnBU!+w_XK#}uHA^IaSd!Wg1Q(R;h1 zyA`#b!!+3sc1TbJa`J;d2_M4gYGl}lud+~dq;hzW)>j)g9*{2x)kWBJP&sJwDG_R+ zDf-4c{Ge?c=p5AqN_i*#PrGWd0TGhWhNLJ1>IE|Wt`BQKU3=-9p6l-k*lwB$7__B# zbe-0U+qAx`Ipt{`E!5jkj9rIl?kNX@Bq_9Nm+>-a>adI>anqc#2)9J(Nsg0-Sfw&a zw1-=v1?%nYzNE+^KoM7ar$E|9Kk?2B>Dkg4-eQ{>%?9cunN4la?n^E=&q|FmiU&e{ z${^{@&wN>^s$~gj3N}S5bFJTm&YEtGQh-Mxw%YkHGhTD!AI|3WfO~HR;rrkJnWTy{ zB1FsS8gbJSS)wkL#%Rv};egodC>GS~JSlCvEL56Ll@&`?p5=+EH3u7ySFVG(`x$@? z$e@@BIiP};b%PqB^RLsj_G;};(``1zkT?F5@ctl(ryMvpr)$um%T;0~TXl+hI-dT9 zvhN6hE=GZV0S`Onh!zyceny-DpAvRGxA8Nf^V%lO*pv{74Gn;Wg{7lgn8HEzhbHv@ zP?>{r$lE5Om8i`nh>Grm43&W$*2uEDy7p)E(p3{v>72wb_WAy9h~4;mzBc_OENSmq z9pZ?T9RJBa+kh4}8?JT0l>{$91g+QuOW$XjKrnV^s)k7QMfxpkv%kz+GO$oknQr@C z3LIB0p3X<>-b-EM2*1b$zh4-u3NNEFO>9A3-P9@f?rL6gndQFTGB`IHi7HAF*E6fj z+44Vmi%1a>ewOnWX&_SzoW+Rz;GuP?z|7Pvm|jAPyVRaONun4=>zLINrCY?Wdr-Q- ze}`!V`u49Gi?87B7iP%i$H%|3GW4q~UiD)6aGvFTy5yLfF%(>GM`Yk5t(k6Vl8-7M zk|zwQ%*&f%k=)E+D0)R!!NWaa1i26qochpVunU)dUQcj4gLTJYlANU9P*q0Y9Scvz zWH0fz9F7XInsV&)EQe4*A3aeWJVwG%7ds7}?{z1g4P6p-QDC`^LtlXwSyyeJQN@-4 zSDRS!C{Vc(s7%WOi@slmR0Fs}LrBmxC(+sl@MBWK2r&YRU~A%ZR(bkfR-M}~$)J(b zw(BPIThFhLUH6jomptvq@fCgio=dpi#~i-c09>?AvabeZv|Z&bKJCy$?8JH1rD>qP z4-gxE_~=0fsrW&L4|ONndO)BDCQi=ta^iCB7fj%fzHcHo zBPmQ5>j^$2V#J>&;st#9sz}U+ZHQ8-Qoe3`VqSsI9rdL`jwyFw#K?I;vG87Q{y-_j zMY`s(>wLRUbo|#>BE-!Mspl{|Q4om!V&d3NJB!fWU z+P>b7XunFr4-?8erPu~)qTY>#!}NXr0zuz88m?eF)@4W^{KE@F03-H9N%6yu@hZ1s z&=uEu%x%w{12Q0du|HLcvZq{x%RUe3>USLK1ju6!wKr(XY@SuI61J~hA(Q9ar+cTD zFlPg5wsZcmBrj^E~hM{x@9gv*+wNd!KXmTKD}~Rd)2boka*@ zS$GstI=x>P9s`ahFjBJ8l3F2E3$Fu}Z9#*GqC%#VZ;IQ%Ix4DQ!I4|UT=F-QX-B8Pam4H|Hp&+Me zKK?J@2b??FpIj6>nLxo9(5vZ6AgcwY(7eK#!c-l~z1rdW5#YnyO-HN>Qd0`+#7PMl zQyq(>cs^Hq>yn4QBWO2BDIyug_szJ*F+XO^wS1oIP~M*^Cl`j6+tM72k{e_=G)_QqyE|}V_ zNf6_x?F>YZK$H5dN-1GD34do`j}zcXoUglAkx{ZZD4=Yw%3kg5oc|OCBt7yXFNio} zaDP>(V&7}qq1fPYQO+FdyB_uN%JqDB(sG{>(Jlpn?t<&Hb0$xF@04>CT ziguRUBfPHH(;E#OL2O7eG~?>mNS@nX|NB-N*6|8!<@7I6;S=H&!6dDkv?1ag_&qjT7Q6FvxA4;-Q{Glvvb?;?Tw?Fgvky9pm1bjJ4y z$TUid_f1@6G@{OF$JKFBni0vn^J;1 zCi-NEusM3Q)FRO=PvF#vtL&(eSxt%M(b}EXVXl011N`%YO=-T}pKd`#IxIrTWjZ-| zOifBjDPPxv#t}S;*tKg*Bcp!VfQn=4)o}I23?V0Pgve?+@tZn>zUqWl=EBX#|W%4?w z83&RHS4;j&!=cA12ua6CBDZu-?q*ZVB#Div$&@07jBba){kd;_OF%hD!T759Py}Dc zWK12h=qMOMCJ>H_E!Ht-q8}QUmw~)2^jSAl^7uGpLlWpDCePO(!T=ai7`!0-x|s$P6Y}z`{Cyz5dZeOCHBW?l}c@a zKc8IX=-z$_w&mp`4N3|5Z_0Ksp7y%s0_xdpfN_FqFgtQYr&HNE_v!70x(5!i+BYk- z$ll0sg0dqp#=9Xw`(aO2Y_5(Y`bX<_n@I9rC(-1HyGkK+%cpQcTXJ7{Ll&%dvX!e| zFUnH(ufa!yHsppE$132&vr+j6RR;X0F}o~EAC|A%8xp-};)L8byV~A!4}5n7jG!ri zt>LSL%P&$4&VyW+c;DTr?dHcU0j@JHv`vU-9b)0Xx*2 zsXRbmA@!mK{OyIt9pBiw=!Lpq00ru7ktRW;O^y?>V~x|4_sdVXp`)FdvQk%r`ONcc zkdFiLM6-CwZS(n$IG0~da&7;cPbjU6(VA}}@OZK-#2QV5AyJrM_xnq>?=4}#WKPPg z)`!_S3A8x2ioZxkH24M^z-qZ-ukx#!(g~@G1uF@Mj^PjzaES3+m8*-TAgqQuNvZpW zDX8uESVQ5bE@Je>?FtLTAjg|cA?iaYWTpMNGDK1n3b<^KYYQ;!WYaL&_J0l9>?h>& z1W9#_lY~~ZcRPS+V7@3>Fu;rpRs_mrjF0lSQ=5K)B})~*=rfdKGY|Xt$6LP*RDn&bKCD2Zt{j#S2ahIbZd*Y2Z7xp7U4pBH(X1BuK;F>2++Ei>hOT3DD`bP$ z8Vbe=6oE#xr_}%NK*f?!1BHVsnjA7zjK7aaOz4#)@;a@|C;dIh92DGa zWafV6Cq|Zhp&1H+l0gbl&k`6(-#0{VWduCI_?cHc*5&Rumw@1svX(m?^NWl;ed4s( zwP4rkcy6Xo;R{?sK`Y!P1o1!mXr6_f=sle!F^s%?J#k1XbXs7|RmQ|6ztO_;Z8qN^gBSu3hPh>Wa z$SjAx=WNkPYsO0*l2Vh>G``xOE7NyG?x;S-k39}^6~u}VO%Xn3twe7(bQJ8&Tw`-@ z=R(By2L-1b$MTQ|qU~{D)zI(QO*p7hix|40w3=7g4QOmh)1xc}{bD(0(MQj^=HsON*u2LVHwTnn8Q`b<=yxwpg`ZU>=z) zk0EQy5v~FTlFTp!$^?cEdjtV8Y|plsCm?}wsm^OWpkEh(n zq5b^#(0WCIOkVE&p>9HX(#0OfLSU7B(8$)<{yV{(Q3reVKMNt6e>%zWKh2#si>C+5 zvIO6~GOrfFc;?WOUEPfS z9U@hWqfE#D_3MxY3Ih&zF|(j^iqAfa2IRc-`@iVAQXYvhVgNCMkMc_Loo0Yd(0J~B z#3f`hYtv-wao*!}-5sx9=bm?WU zIDj`;9J9=$Y07V%FISI02u*!AxbhwHulY7B`Am*HhxXe<)Mzo4d*iU<<*)BNWMpC+b_eYWRtq&(MEgMG-~{Mb|`nzT*`>ZbWW-Ed|LW|p^Fk<5yH^UuV$ z^keW?O~=8VlGz5k7-yNH&x_d&hH4aqk_ya#9nXuIkj&PL#=a`>_{LCXW088YXdDPK zlSO$wt|_xX;AZLfr>L}{pXCwWSq>y0m_DJpLD%s={!P)y#gdG%&=M4oPHW|c755~6 zg`_X!C(l9P*R-MO(pdEo9kG|UEKb^whcyj;5<>CCTm)`*?j4&aFZ{Wwq)nYvH!DuGK+fl=o(X-~p^3eDz>%qZMKlCFaq) z52Ior7uyVIFkxB6Wrm&0gAiRrM-aqtb&(diY;8gnz^Mz3c(0FFde_+f7DRS#PM+!e zeyP2YoYeV%m)Xs2pyCwJlHan+6kQ8QTeey6Lr5XVlud+$oP)n}VkjN_f1rr1mj44q zV+|aHr!3I67lu0VtkYf7moikWpAu_-hrMr=dKX zP5e^hPjn9VWc_}|$Fx{{gHYqjHD*Dci$*99^JOKsQ{5$#5bAa=N`%UTA*h=4)-2Y3 zX?;;A1~8)wcH=y?y2FMP$E=5PaCwBc%I~VujLQsSo;=^`nFmSWwp~Jhee9H@)B)@$ zS_%4*$MF5HdOJ%#5ie3HA&*KBvhnaPX1d5+>9KgrZr@V_OW4{veF_c_EX~g#H|fAZ zw;^*uQy{YlS~Kp|K<(GzJ+dTF=DA<`uC2RT%ub!l!PFEXEeC$(Kf=$`E_h77=;H*` zh)AEm<~c~uZHlG^E;aj<)piGo{{Oi)XR`U#7W(1WFPf4)lI1=JU*|_T2=*jEi8R0D zFF*r80ykKUDl&kTE$BE}a{kCx`q4xZWbuu6x0kl|o9vfufd!9tl-`d5I;~_aT67~y zM*CAK4ANu02?U;&Re$?!k7}Mt=WC!w zctw(HZV4FRI|Tbp8IV*iK%$nkg3KL+vtvSI*D-Ek)3^pQxS>ZY zl4?`N|6}Y~qji^54og7r*2$t2q1#{M?%pF+J{DtZHX9D)n3m>>P>&w6J+#dVQQh{LYWk5bM8~XnlDo zV?JeWKAf;=Z@-PMGMb9*RJ7i;D`dUl@pDtD^Nm3ckC5kT$L+JX7jfbPaR@O`lzfqK z&E8x(($(VXTch8UsSjNPwp>RlwtnNff~k!4_v};7{Gp(R!tMk-uchi4e}0{O{Ard( zyCG&%k0C&<3~K(;W3G|pDg}@fXa-befOk*Ly=QDoCq91mF7}4O6G`9` z^1bxq$yWghDpY%k1rU0MvG{XpUUV3QYrEap`6TmLn5hPodKrKwEqf%CIDyzpgEEW5(TU#=Vofg}9XRF5VBhd~9VKlCuyKSt1PpQR2nFbA2=4>m2tG2U^Z$ppT7- z^Jxb@Dq`?;Q|%%x`XocVs^r^Fyo1?~TK6yuXPh`gBJ+wg%h4xUb{FXH#eJX#Q+nkxX9yF%;=LJyZ4bzQ-ths+D+Zq5hz^Dmj8Ekmcr-d59N&uE)*A{x0R$wHAAo)Z3XKx&`7( zsEkcjz-4fOr$bNuHtXd+m{}apTzr7Tb$E9;micgSaVgt-r|0v^ywIEXt2yErNzyx_ zZf=QHUPU1*AY{ueZGnwy&rHjL43fzWsWxncIqYU8qs6hged9LF;X~jdoUg+%ZeoWe zI^3I_Xx-a1US=-S6T{yVBWQoP*nA)qbyXh(up7!)zd+k>au{d5qz$r&`E@I@>@RPy z&`PlFRc4B^xmb5a&TXEJ(71?dyog^N%-S~d2B;2v78!|m%V=tyovE=29k*@++B;5B zYgEvLxTk5-?{Xi-dV-znf4jw!Ap__NLQqT4hemFtRZZfp&j6ppSYE1l`vfmv3{Gg9*0k68HzwfjM-1nN~ zs$cBSZDtP;@G(%44z|PLaJ|-$z7&?>=?d%nmp?4_jeGQZLx`iW8P2d=9F+r`LM%ap zRjeyD?fjmF|2|VD`o48Vsrf@JDUE-{2TCaqWjrWa`kqhpPGR@-e|U(5@&(Z!&n{oB zkv?|t9l6GPN-1v+qngw`-1JU)@Zm5a=UkY~8E9MYR?I2GLmm?X-?~cP7Nh#xnMy+S z_n!4%e|N{I{-Qsz%@iraC2Vm**Wjr>$K&Fei&5q+>K6(XOuy(WYDs5;-|< zuW-#pot-Gf_U!UjoO{JqK)7Pbc)34}4Zbh8eL_4t*KuYd{FtO`OR{UD0^BI1+A zPrz*Le|==TCASIkqs~cEjVa9DJzR|z2OR+fLQg5wx-{Z3Is|NC>@ zJoPl@S_a)*aE+2rsre#bg6Yld!0!S z_Zy{Gd+%hkL{zBn6!nWOu)$uW64E-CmH&fuJ7dxZ>NsH}%Nd@fn4HX5h{=E)J5i8= zxhq|ejY=_1D){G$Fi`;T|P#h%+-YCQ=DS*%mVAZa-K zGSulvT1!(>JZpfP@2?gG{v_p2N5bikFk$=bvBP+rmK-}{D-0*g~#465WHx5n1Bb|vWGUJ|Hwz8;K5tG*EH_QW$PVb)wTWE*Fn1?n^Xi@iWf z!r~PC@2De|p--HDbbkw-c!oOSsWH6qosBEk1O`(rpVGhDmsx@ZzIob9DRXjkuZwU*{U= z7T5dQ{2Re@+hp$TGg5dO9qi?PL`$a?`chOu-t!PLy)VL5u?HOkvzo z$~N)b4PeOPc1#aOVp`o>aL!HXd6t)Y+pJf%9V#7%4Xrjg~8F&+7tjsM!obvA_N-{ zG6(M)tzDBr$amat>2A;kbqXeo{(8X@L-XR}@=(?3@O!1iv+ha8)6g6jdQq@#ieJRq z8UTbNY%T9u=}*dt_O;&-htG3M%V(W!Cp0uhVX!l-;5>pT&$5&7{BH-pFRKa4ZxDOb zU4V^x0V`%^kw0;;?v~PMye7*^dDMtdp-6!eo^3fT;@bOEYM8a)8-@5PJUDExcv8qZ+g-A zkC@G2&yFD&Tr_gr*ED$iHU#PzZsF`m%eI|*i1rPxz&12Z#Y6@^pxv3wC4ZoLxp_mw0_t?k{sTkh- zo?F8+w2W{8$QNVIciedBFDA%5H_jXO2LL6FCsL5#a+D2qlN&}0$NZ!EX$FG_ z;1xc7;uj;vC7F+j&qGPezjwa7iIZN|n)Y4u5=Q-Lx2$D{ik&{_FW#x*T0YyMgEdGR zXCj8Ge@HI#J67?PF9W;aTBbh&4?&#NE3)K^HHz}X?Rbu>>l`k|uxd&o-bZMCCzNc= z4w3%wp*9eMo^6A@p4j(+apu{_TIRm@5y?O-_B0O#U<3`Oo#KBa_1_ivy_7aoMEp?h z8Mp~4E~Wm=5{`zV6n)^4MVvJ=LigfgE?bEFt5on^*gOG@1#$)tmSu!EC*beOgD~L0 z@aN&&YXrpYvlnnCIc|WZfG;D2$*|Io0gnZP2uK2OkE(0OjkT|RRr}Y&AGl1+Z1c{9 zvS|$5qGC41gc)Q@F;R=6Bl?Oa z7XT@%<`2TG%a#t$t4d|NmtB&ZhQ`MbNROO@FclO4aJFvy8}vA#^LBJ99N%ZqGT$m3 zv`$!(GV?PM^gkVj;WqLY@-Ojl0bz<5+JxQIwgla@s&l>)Kq4ILXF_)unVS{)Pi1a; z2a3YSVU@&u@NKBTHpA>fV8TGiF-FBZs0X8v?1h7}0?p%Uq}+>f-sI3r)VPJ1lLgrf zsoweLv5*axm>%+@ZjAEJaA|Ic9b=}RAR|PQ0m|`rf5SGXJVyLHa9mz7j~>Re^~*_+ zCO(PJaa|tDB?2P1IiGeY13)mLK34zz47jEdGf|X0wm2^sd zLgkAGp*Ms_T)7{F$u75GNAgobo`AfNmmFNq^FweiNkEl zC=ZBpjRy61D8i8Y0s-Tm^7TYH^@dp2F!^IXqS6-(*tdeCJ-HA@%p{sg$CD>u0mEe+ zWXryA84)C3lo5>W>jl}&3EuIGg|%OWE;HGH0-z&|DJ(Q6Ls!tQ*nBOV8KfY}V4hOw zWPHoUw@7Snh$I#UW$n*B3Z1TyGGWm30k8Mv6>z6ekqS0yL+UiXslQ#ow_SBaX(IVS z928buky_CJP3p7&keEl9@Ur+$a)y>{FB8V=0qLNoUuAj@MKlMOJ4I#G(Cg1M;!kjQ zLPerC0eFGn6bc77@*Aq-4pFq^G*tpH_jyKT7Z(v(C^bHI(7W_r`it^tsXfaiCNG6$ z@);n^*>E!DHsr1%M8F9p3IN~}jiXZRZqtSvP~Xd?!(=Hq0Lg5~_HH6-IhpkAH%0QP z0EE;qh!`DA*MtVXM;W;rj*-mGFuih;h&ILqtQ7RX7J@vdeChiHGLsDA1Oi>inHxShaQF9<{D_x((1^3J zrYuk=PKqaoQ>>m->%LaN8R%wAD;35S3zWeLMJY95X4Z~a9w6bj1GFHRPa&RdVzsUy zjL;2>9ev&28FKMDS_KlAASUZeyno-QJt#8o8bPz8kT`OC*jnaP);!s)EZUpisTMu` z? z%*o?T!O!&y2Y5G<{Phj~+4`}W|b!lMPHcl&UPT5!5=_+kivcBZG zDw;C;InQKG?v6T%N?#WcQ8L9dT>O;PQUaQFZLjw1e$8~&N%ebDH=q=W9vOGUeBAk8 zqL#5XET5`U<+YLqV1PF-q05f@n~_2`7&jUZL}v!*0uH#sUZ@6Rf*E?T7|{ft&x^t% zr;6($&fjZMSkjW|?(oh6fuTa;Oq6rIe+aOC`Y%aAV9y^7lRv<4uuzz^NAwEcw z+?TwYMqnsFPGV(9HXf5p68SWj0E|T&#ugI{>S#IQ`Tbk$@t;o%?Dl(p-E zm$5PeMik1-7Gphf7e!zn!Jt7!gF!3ur~QMJDg><>S%vQdF7zBQDBR+ZUlO|lb1VOo zbaK1Xz_Y+4h)LErMrbnMyOKM#WAKI<-KEahCD5o?B+>Ag;1~iC6Q8=2iVrw$8=(sLd*Kqa{U#JSquea8ZcTSWv$F8f8;V`+lAGPItyHTS)2<3j&PBiD*y zmSyqsjqK8&Xp@mF*%uyk_gGibu)1YrE@POm1$}a6TUaDR(sSQzm}p(nAC?1Lt7RoHTQ4L2+LnShXQ zhATKp?ZBZL!$x94+8C@8vN|>xuMWGW27tKF=Xe1`-;d+aqZZD_`D_ieJOM)pBwo>| ziKYcU&`Wq1OZ)H8yQ>x$M|%N=_c;DNmH+AJ%Rv{6xPJKzRKiEZ-El3TF~dRPN|*mGZ=`9z zX#3gKiq7=2;vg^BtTrlEl<%%K+Hp`|U{&*yXF{+7>Yp|#4S zi8#4{u&`A4t4yepoGL+<%oMe?}8uu*b2u!{%s0{|r0{?=SU zg08}`+zc6nkBpXq&2KRH)6GC?-r<#6c5pQ2Psg2TI0V>>ME!B+^j3IcQ8v+xl$jJJ z<^A!U=wf(q3gTgfk2-~Hp_0^+#JQ8cvvymfxz1uR?O)uv(yFD{*cAhEjc!`Pvl%UI zw^1sguRVA%C_R8sFbETNT1VuiWP=gTbR<3MqZn7&wI7!MoI^&8wm5@K9DC+W%t@t@nph2uq^v zzj|~?RpaBCacZB9&K(`-H&1R9T!=@zQ**=dI)L2-&N?&uOIng?Zq0~PS)aq3SpR3pk6>IdV{sXnsv`*^32+b7vLL4e zB(J*kYCs?$iH_&}QpFWZx>6mRARL}%%zmsT7}E3W`8M1yvP6V+jNrAgXGPqMtGliX; z&`*I6x;E>07l{-Ekrr?PH!v)+hpF(k(w2PT5e;WB_1#1l5aBEBFvje|fLb_A#vn~b zNrdZHt@bKQc^nVApx_iPJ;TKhdi1z##c3yS9q4anuodck3-EgNe#p3SyX{cU4E7>{%hP2uLp)cHX(8Yh z>07f#Rln-InYr&Q;#(J3r007*aS0e=KnpbDG{E7)oJ_J1?K#TVs}h6KG9lE{7Ew|m zy>b;&1_n685j~U}ETO^yM<+q+tAarT!fdH5l^QGiH4uOC-*q#-XeCaGuwZh5vxq}C z&v3$%MqFjV?+OZX+;!Oj-DCz{y@K%fw%){e;72XKY?S`GoBkZ@ukC!D+W{p!tEcC1 zV!mF8F!p!BgB+t9iWD5a?flD0I8keR={Pq!_65wt3Zt8SEj!-m#E8Ff0;iI z;BV)dWcJqe?)t`-Ejjp+`0a*WF|k!hBQRgLK;z(Qxy2U(em&@-fW7!ruNn6{!Tj5@ zA$!V~#fFNYG3}(hp&O4qqMP70(Wle?&)~W&4yQTR=be}6&z##QnNF=gInXB9N|UK( zKK`9%XqgIi!%Ovv>aY0oYbMU75aqqo#W`SWXbKF?tPl$pb=rmUFq&Swmc%4Gff+U9=I}^ z9H2dnU5*hb5Uo!(riwCiMU0YLwL|$3EKN8Fp2f~e8bg<-mDp#PYlIzG+uh^nREoux z`vj$7fmX*#+^$c%s2Qoin#0hGj!r*!>UfUgGv6Xg`_zbe+-@>!0xg~$vW#B*c2-(! z#zDg_!BDO6v+m#e?mqe#G6#!E>&mKFQFpLBbH8fU*rLJvKdP@ zOjzjN{oNL3KlQw7Uw|JWsa&!UvLkv(H~1gO0T3S|DIR*n+=8whW%y+gtxz!Nooj!3 zS?o0-Nn!KlmRghho=50w`zqo7e_Ph?aJpkD6dg5XZUp{{|H1H zKSg1=wa&fim9LRd=hp?}m#{(Ll3 zjz-YmI2#r6zDLK`j~tvK226^$Kh;AP*&NpqP|k-oQt?}m(hK{w>(n~;m?*LkSxwr$ zaJ$#RHs+ZdE~9$qk5qYw-|2Aj5zV4Ik-$mA7<42J0wRfnc+$%n9(sh=OA3l%&9T%i zhwOohg!0&H-)&VSLUO4P`1o?n542q3liqD(SqM!yni;(UdN#TOaD`wjVJpH$-qnK? zp$7+AbAsN>@h$s>i=zGqd`1}^FSK&>?$%}QSBVq3{hO1FW3SZ&Mf^YMScIN32fu!! z(wzHI7XL~nmTIBslhKVg+U&1I;Ih6}M~#R5;NTxdRk#O4WLD`DN$ftMvg`m1g8g?} z$i@gM@H$H@@o@ zd$|2!ach|H(OA}NxX{1wgVcgwtNG(FbPFO@FBI z+#K?%_F80oE6#{H)O**vA^g-gW5?j}m345&2f6l7KFMrwpky|knNiN?^JRjiUOtE{ zLIp-s9tE3+s39~#w8$r6`ZSD~GjIS*5(wXe?bz@)bPi!$d|T|@m4x2cc&DjCWCjlflfKuy zuTQjk8{o6ji>4wu^f5)ipYF$8{*Aw7GU_;KA~jE*++Qp}hSXTjnO0*A z-u+4{x*aOu%J0(WndEY*9hD({d2L@ws+9G3@fpkZ$8x=?UBnxwL-ZJ)KL$xYtwz{U z&&f^i`sW;vldi-1y=|%&l4R%XfZEs3e|VRiV8>{ALO+!HGMd4>U-&f!Za7Z>^Z=3$cu^ipmqKF=fxqc7!Jd?pEJ&fSsX{}B8(?EzJIK% zny^kliadOn36D=4@yyl?uxtLNM*Ykn+Q3VXn^nX9((qf|o2TrG_v%Xo83ku?8IyO@ z@9*DI>*`MwDL7rbj92r!u-e&BD!*X2(2s<)y?p+mgOSr~Ynd49D>=T%xT%lo>BcpK z3MN9ixE`zuLd%nZU}QyX$x1`jAP^cu*JD`=s$mqbOK0_uGEM>HTF3DZ%t~Lydyo|v zsV{KYm_+{ej))d>y7^U9VL!r4rJ%O-9%Um@i2hqjFoP-}*+oi3TxX~N8GQW`<_O|i zap{7|DxuksgyZ4-u97#0^%G;q$aZL!8+u4f<%Zg3NKP8p{}<% zvXv8XAO;qG|AbNmtO*BV=aFdzg96IK!68Xhg5CEt{IV)UUf3v7WC}f;4ONecjSo6U zMQ(w5T#>yyG;Q!~ZEO^@K1`IDA20R68XNI}8eHM8M}HK2aPXK-@rP0ZA$nkG(7x;o zU1g92V_49WEhY?x{WhTy>D?L@4fJk}P>2aPZJ>xBy)PL6sU6QW#^>q{5f2xOMG!UN z!qp;iBLUSc=L){%+H~uRg#^k{RFD@ixkdDm$fHv~hKld%L)PLBjRqsviliZ?c|-56 z^~>Z~_UBR)VC_alVUL#?-F7gQf+#~ltgj#P_KP;R(%-c@{@GAiFPoD6a6xzEZp%Ph zLU$G7|1=JBWI=$&#yX$?@+-6{x6;@n_*0=d#ajcb2Y*K`4w?xST!F=d`o&)}po**q*NTlkXx3@VU#wWhOpQDe7*WiDO4LJm#$-6(X4?|OVHu|_p~Hc%;gbgvPf)a_tHE9kd70>4goH5JS$8wGaCq1?wiD#_2V^t@b(lw^L%L$=sAV>|1Ku8b>p}PSd4Tq!YZ43qhb?=_W`S3JlT?L? zL`lmxS%PFKp`u{du=I5>bMZu861`rJAXgFdx#S_0dY=YKfz)iJbBRP0Bg5|5$S)PR zbJD}!J%XqHO6JD@LU?~kQawldA+^Pbx+FZ3FevcQR9uTl#T%DvgMDkK z-QST$WU2w_j?10u`ae|y&Lv$vg65$hbbNmwrH6jVx~v)8=~lP_h>OTc6QN z(8hVOAc~eQho`b2>wBKD_>ra1Q+hK50z&3io>to= zV;d93pG#U(r=%I$25yv}~;1Zu9|zZo)1XODdSR7h)Po^ohFFg{!H&hoT;v zH!fFvGB+Bn!S`8*Ew{t`U1O>$DE3vhpnamCO)t0yjA@T5IxrkV%#?+~me%E5ANMfO z$uDN%kh0d55zKhVSogwj#f6HE*F}Y##5Er75>Hp+605AEAC)+#t9slP{d(M1R%i=! zSbDBcH6Ifs;d#tG_4~qwm8&M6$*U+qb!#%#3te%xoIO!YTv3fOvxmNC77F2WEobB( zMcuwy_hydQx|r9twxRId$Uqd7OXKn3UcGVOEf}S8wf|eFJxnBXw`?Xab8?A-b7g(t zdfAYJQ~o-6HTCJPXug`n@E5sQ%q3SP54R9%eUG64Xo0Dlh@N1x9A#`DpJlW%R| zdhPwl=YYfeaSjvFnIFaz{BM!1+Y$UPsIZCX_b8L!Y>Y_zG?f`ub-%6cd(YPBl_3@I z>$)YP74h&<%;Wp3bh+@APOlactAvd^<83t->kX8dLuPNfP64hWaKZB z)PB0$R^vTuA}RU0DC+Ko`i|hu#wVW=^M_u$%zhHk>So3&L3_krU5D(=u8W@UO7wje z7yE}$du3mGyPj@7KP52~VKeo#_KMb6yh%#bd~{B zjXpNa;vNiT+s{;+mp4){>qFuPH~&ovR&P{#bkfJ(#cUJm-tIW z#hqkTL^Q&sWRJ}G_Qz?`ZPx7e-JP##!Ln+t7*mt-Pcfm{Yt3}F&zihEtEtUjV#W=dR>uPZ1jXWFNHiqpRStW#jRCONyQ*(0g0gWx;#+U&T{6zimL@xIubT^@Wq zFxX5DWXVc2FF7yk#lH}ba1sCWj??Sg=_*vecdbd2KulV?>PHsVu^O9LD>vPqiFqBu z<322GD{!2rRrh{d3~Ifp#XKA||6^wDDck;69;x_i;>5w>bOrl%Jc6N*j$2K2ZL0(v zp`m|^Hd}kq+I)r%V?7S)AKn#1rzCOV6yS`A^`J3k6ySk*DfAO(j1>fqttK^fay_cW zTs&%Me2irw$1EpSE^QsgtW7n*0VOkxBA9U{f?4_!VLWjvF8E=2?)byc!#mOToyrfV zmsSP$BD!H5Y4m~MxKjHd76s;S23PZdZ!uT*Wi2NLqo{>CqMJfLTYD+Vo^Hmu;QP&9 zUxUEcrM%Gkm|*4hT|$p|0ycX&cwEFVv*Knf=ZtR38Z!r_4y7rD>t~lIj~+2SX~{Jo z>@h8qE6{kL*$&;6d7Q7>(S5u@>pnN8?>D}EXo71 zk%FNOq?4VZ1RaSmxm)$moPyF)yaY%e->IWO48Ph&Z@+7a!&Y+`Zyz_*3en;f!)grj5s)2`>~L?X%rp)|;C@1IoA^-+`l>zNv-i>nxrK{xCK` z61pv>Fa1`|sDB-wZGqcYECTmt*}q|<%(Iw82)_-wJ1YBcSPKhAE=n>b$DQVS1>?^- zgAZQcn=Md5G*v{`=r`IUqpzEllAh~js%7}|Jgkx5Tr~gT47$$$=`lJUT+rPd*j(DM!+BVD z!%tu+@XCCCy;ZnpqBaDa!uQ!+i05_Q`8Q>biXj%qodpxFmLSb~pI{tq(B1VkLJq4@I>7!p0r#k&#i=3X`QH026at#w@a^Dj+<8l^$% zsR?}r;rVmh{lD_7-!7VzxNW9=+dr+{%EF%ANiQHQ7{n5rU4$6(Uu)_VU8twJtm~9K zJKWDY`Q`6YRMHKczN<8CoO6vM@!Ln zAo%20zwc?+OcE8&y`L(ytY6^_Fn;?X;dmz}d?@EfQ2XeI=#C`H4=sK%A#RjbBU-he z8f$`k_r=AQ-Z@^h5}ox6dD&#MN?$n5ZZf<+>rJ0P{?*|$$wSpIIL%KxSo(e}^9kGY zdos#&e$m&Lqj}wl!Zk}B>dF%)jlAJ1kb9t}DExEthx)^`Qo1ktAF}gT%_0reJgwEk z?YX=XJ=^QM^8M%2fTy4CBM(l@1%+KuA+iw`Cyz%qLhE2SlgBA089QQ0((%Hf%3g!wzS=Os^sM>x*B1{{42^gTLZ49XLADg!ViLmKt1mtqTiR6?bj+k!X!LVw~ zvTeZ&`sI+DI)f5p=SR#-{`u#%i#w4Eqb0Xs3&l*eTYb1QX0;03h0!CWzQL)(`d@8 zfzj9Ug!0S?Qdl9imqnnt&*#f5#`uI|&1#Yml#ZB9U1-U?(6dN z8>+2PS}K~JmQo`7RD(^8b9^a0#s5dsTZc9IzW@I)1`H53N*Xr0Q$Tu*6r@2y8WE9{ z?i?vCDcvdE(jeWbNOvhM4ZpqKpWpXBj{U!5_wM_=uJd}Hj|XgS-|>4IhWqV6(nrI} z+ONyxZ9)SoT-r<*Wb~h6LLZyuxW3mrr8v6?#mpVjjjQjEXKKf$ACu=EhdJmxj8wk# zU|HP4r)*(W{YFpv67F?HAe}2p7Q+1s&uP+`1mTh z&N5H(uCQ#Fo3{AdCC>~gwl0E7o?Fwf{Doh=Q|)pcKP{r#zaN6XIw!3rF*U0et~x)Q z8!X6DS-2>SGZeu-t*PcW^S0f)*w4pFSLSI$hLDt|oS&@on{jN?T*+P46o-J!u0`8? z#Y%}*_E*BjR*hOW{N}|@W&vLMxK-Tq*DG}WFt8PB-JC(4{Yzg}OLL6kHovj{h`y?X zO7Pz$STNtk%2m__TGsmqw%uA=Gpu7@L>0WS(_9+Tgi{(uLwWje(wk1VKG^5O$1qG> zQdH<`5a+#y2OBTFe{c1ikXHFu>~Y5zEbu>&{=@C#&}+bFrWQQCt8BkTG3-}s3UuZ; zIOdkmE207|$ut9*Na2zmDg!Vg078Kl1Bo%hK_zhdOC8Rrj}97XIYth?*yAv-!KZU1 zZjOC%av*-0sF3*ZP|c736@2JksAQbc?)knb0Gm|=D^9Iq0W#|E6BNrtZicOar%2~J zNC?rD1Y66!1md(Aem=?1-e_NS-?LEDzgcK*h(0K!9J7Iw zn=yuOQCulD;YHeI{TrRB8 zmwoz}6dW#dPuE^weM?H{rwE3&dxz0C8u|)?hnLF=DRsEZ5(K{%C4Sc#IoDySU*pcx zY1PEsUTu6uHUDg(eD(RPQS1w?DSv{_QIX{p^gQLHH>KA)l-JX-Ww$3|Wg4w{>?GH^ zHctj6l60KYnC_WHTsHs86r;6VPt$Bxp0}$mG&$M5F9k zbXCyVx$vcGOmkVGbwBb8V)PVK^-Uc|m8*MjtdHThk@37WRA!Ax(nNHM;DqkKWD3ec2Qc8?laSgUeu5 z>eg}F>GWLO9IVT{@ZFV6YAyIe!1YmN&Vg35Qa(wi{vE#HC92jGwKC>j-m{=uR*w zh8M<(uP_7&uRuS|-brz+(awfBawgN_!++w^hL~yzf|z{?A)-B>PMG19=p3|D;5eR& zU�=Y>2!fJ7Y4^EF6ZwAi6;Xhvix=6t13!;h!gY9dfzp{!3^*Wv&(_z-dEM1FS0O zze={u;EM?&jUo3>qzIw`VJZqHdy4CHyHa2#yA##nw8-?WnVB-NXRj9H^-KK;hjbA` z>O~NNLOax`u8`g9)k%ff_=UK6G{e|x8TDc&5=0;wI?oawj|bwrx4RnW8e4Ar4B$LY zvUn`5m(gG*D;^2(m2MCElt{U!B@~RexBvpVI2}pYQ9=J{WSt6#TnB ze{e6O<-7XCtn>W@BietPnpFJ8Y1!+_;@0fK+v}vxzBQ*8+n~iE_VHj&yR&0ai)B?x zwwG4)XRRAVX}L02tZi~&a6t0g1HB9HK#6U`aOCe!88OXg=|hXW2c*6x_Xb8Er}P|3hKn5>=otMoHavK^7~20pAQ#p*o#jszC5h*q8-Rmw`Oh>bo<5~(?|5P?% z{}(DYTdk+BQ%+s{uyb|PrT!~NqspghW5GB-G1m#?)VN$X+lWrI`*E;ZMfChgi?qR^ zJUPZsOSh?2x6Z&P@0hmAYR~(5e6!By6jR5goWjpSth7dPRmp!3Z{oRQpOlQo_Y0m5 zg^kJ^MLv4{{!`Yz#$@DnNk!S<@#SChw&&Z4f8rUIDHhr_2nwy@Rh4rHmFiJ5lYaqgS7&jc#A4q~17roHJ0TW%9)Q zJ6dKbhKmT8XgyfIWRGiE5q}1$l(Cg8U=f3hi-`Mmq4B(-Aqpf9D3E4`0ODRfhN4M} zL#4t2JrvhsdcL7oGP+_tXG*CUBb1pW zykp_oimD9WT-$7Q_10p;_K)a|_LKdu z&qTW7#VNlz=MUXG${u^e%64tk_QovjW-pTsIvTh&>WoXO&Y~9&r|q>yk2Mp{NF!&4 zjhAPAmGq6gm-iQ2yn@Yj)uyAGvC5Jnen=36KQA2kqkx z9O|4aa z^-W;&&+$an)Hm=8$#>j;l!uEEpE%~q-@2=?d4H=E^DTbk5I%SCN`Pb`C%N{`Y}6Q7 zK-$IK^I7r~`{Ff)3Z;+k^Ihcxq5JQzG)T+K43bIz5Qf6WMIwu{oH7!aGQ&AWsJRWA z+%i&3{=E^P5Opc}PCs0sU$;aW^^DJHqGlN9xGn*&eEvkT{J45U8Lg79Z_4QL1uRO zd?|Kfm-)ln#ZM?V&VN(6%Y6T(n+C!^x^F{KIlbele1-4jAvOGvZNu7k5#BL37|A>) ze;c2cTAY`YN}cc$!y2uAaCYHZ*0Y()F)s?DJ#?47XEQCWw_m2-eYjfg$f46;kC4nU z78_1-PB{KP(fN{#QKWpu`q1UkCwb;cYC(+Wv9|Zr>(h6Ewa{DDUU|~Yyib-H0^Hu? z6zwr{-2cQn-R*WzUOBx;=I_-D`oa#80`}9#<7Zr9l|6~~6J+D0?pPdOxA2dQgeOog z8vaAgB&xZ>fh$q=MJn#N(9 z_{6wRGQyT@CXO;NKyb|u4ny04pet51Ckn+pCTrKBgk}3$>sG~|C30E=9~?U+ZyR?P zZVyIyE-kBG8Ms#Gc41yATLczTW1AsltqZdgGjpOp7kwOE^E)`VQ}caDDxFDbf5C<* z+>3pLkG?q-G)fwAl_Fy&amjkp28=K+ps~FXXYbA#HP`{ zR~Oxi3+q# zQaHTPbim;N{*vlyOwXUu`e8Bp_WYaJY}3lqzW?s?OSk_2L8v(V)ibsl6DCzE9q9xxFI=Gte!Fvc{&So`JqI@Dsu}M$z|(>WMw`v0!d2>qike*8MsfW zyUqpC$FG|wbI5q;lM`!ue_jMSWM<;XJQOoYOF?m`i%*Z$fL`2=jlj8r?y z8$?`2g!x#z*;KV&Z@A6=7`RbL+%wMdZe3qk6fy{DMMm9!JU^wh%Y4cewLL7PmExM5 z1oxNqi#az~-xMD>nihWvb{uB=j=%ou1!GQ-G7=OUzzUB6H3U4X&Y(9VK`#wpJfS=r zxJ=)zRyGzov&-D;GP_V6~j3N|D*@yOinA8Zq`N{R`tVx z*#z8-|HS#zR3BgB>R+W{o348?#Km$4WYM5fc+9zRO{Zy#93kwbRiXK=#`U%4v41Mq z$obAj)%F$F*LM9^ek+4h0SFcfjaI*`_jcu#p@rw+JtEH-D=7*!oEaWOX<$cR%O9Q1 zb)2Sb?NpV7I^hvt&;2i7dz<7eSRGFmU?1-FsDW*JTt}f#A_ub_QOToO#oCR5uZh>s zS&zrd5u&BD_TRFFm4BBBlW4K1O%(PCq6FUdxc*{CH?NaW}QC2UK8LJUgTG z!rx_MaP6w=l71j}Z?TR5WA+4e>YELObaRJ-im+8awO?aCV zdpfZ-3&~hVpaWeoyO*q%fH?b0pIc#Y0mIm>vBkH$n3vw)l9XCIE$#&W)GV^Q z%c(v5n>xQWU+%8A{4Jd6`l$Tn;?qE~myH~^%HJ)WuitaqBJGU*p|5z<7&K9=`o{P=P0d+6Dcp_AKQN~Kk)qDx}l*}_z@&C^t!eg@BX zX0_RkbNflbxih2=oAThPB!8QXA5fZ-HK*=vM*jU>dCA}X{o-%;cMcj0Wo_P9ZvTmc z>^E%d-j5k>3uGOR_3_i2oq#PvL4LXF;LU0}lq$q{$j}kt+9x74xPA%otv7s^>9*Ppsi+^xg?I1$Vcfy%#;P~2!Sxb>WB!<0rpxt)a68)38>chWTG8exH{?^I`uve{|>-Ga- zyzij@yise_@C4-xfh5Z$26`V_iEnq3St;1NbF|TZwdtW;nMI`0-w(Hi3LAK;#tTc# zKi*D1`=cZ@ak7g_5z0T@mjAR%n@4`*nAJ}5CIsdmDxh(qD zzd~^=$RADMy(|U&u?kSpD8RHuma|1q9wr+{fIvV&5ODGaF#e1fP))+}&ie9Tu{2Ra zp9<@0ruq8Zbs|_rYh&X7Va*ndEV%fa>68oKCssB8Y}cG&8J}KU(@#Lz*ICr&^0?6B zm-BTx)n-RKr`=7b=^S}FDF_f}q1^|<+{U}B=Q&?#OA?<~C{3=kpO&`zmCo!<6|*xl z591T!R zTL0F>3zAOq;p)_E&;)Cl#V<+05Iu+;lI)K%D}i%}Bo8lUFW9DLJD&VOg`*eAQUjTRRCFez~u z^`HGVRs-{PHe}84u56kjfBqXq%`B_xSg>p@TOt}uts9Y|qBy~D+fY$F5nAnu40k0l zMGxVH0-)A$ID=Re*;)+Tzn)cN3+;^CYu{JDT($5Jg$lxC61|91G#GSGrq9T9E*q6i`Q4;Y|}3-lUAx(4g@1(NrV;`Gs%Q@_~!h7i}(`qVnZJN1J)hZ9kTwPR5}+>tCn<`0}$cb$nrq{Gx(Y)}%khge$s4+bWGZ`KhlWuE~vDlx4Zj4?3@VCObQu zyzSb(3JnR-_@J+mK0pv;3y=_T2{Fep3xP_Mox&2pgw%f)0; zcVjr_+k=~_;7dIVsS2i|R{NWeL6RLp57F)Hy8qnh-|wzE_Wl2Mbrh@cV*HT7W%^Im z{<=1MgIB}FjM_@aGl)Ousb0-1@gkIi1E+YI(8dt^mplplJ!}!8b<1-i4otQfq?P44 zfhY2q4AbZyEeBjjhL@O&=IZ6Wzoe_eZ8H33U|nG$ZW<6pZu+H;wQ2R#7AgsQKn-$q ze76wBUcq14H06wke`cbkA4$!%4aYk8@)a`1#f#0#V?kFv!Uq5mDv0r$?6K@XlD&ZF z>yj6Eh%|MYmEuahYiFv&h*}0*UaZN+Hu*K)VvO7XsKpmq#6pjR{snqZ_NNFkCP8o> zYIgBQyi8Qt>PaYVrSAv(j2}~BeS&v$TfeO4xTl2<71*5Gm_x(%#>EQNq>`RE-p_wLx@q(zSFA9pxQwupMGHVQB_p)ndI0 z;Do<`r9f~&RDGy^x@!XJC@Q&K;di^dwRZNd>!yXkZ2$dozD#L&^v7#dzwV1c)A-8SGi|pwZjB^xX%=d>l>ecTPBWaMNx$sUWaQI<{ zh_hi2Eo07k8cX(tn^utu1AS|W(p;x>mG6a#pctQPp3TdW-?X;W?*D!{;qY%Fb^KY$ z-lfr`QD}ii;rH-rSWO}^Jr(0P00>|dG-Tao2Kl2P?eF%8iN4gd$#nI)8^~X%zd^*H zsII#mYO9U}r;s7B#2M3Z#9s=Y#Pr5~o37^E*C5!iaw_!Zl?R@uf9$^4dC|pTRV8Xj z9AS%POo|#`{@fM@5WP97yYJ+RdEYlLqPCkX(nH^U@;t>w>J;fuZ!?&*2;VumUG-qd z#dsV_)p9u=Y%mbJb zcPC&;+6CYM00thSGd>3ZK7oEjQ3xzK>v6+8yYC#@nPgH?sqj;THb6FIzvQau*|h&* zlKPN<{R|ng{9c3V9DOD6*zR%r!Z|ZgjGXQj3PgYdbsKSW4#tp8(HOWY`Y`E#SSmXt z;PRV{_{BzpYU$sAh~n)qx&rsr%RWF4{vj6v&77|9APle(CMc%o_hL_27ErST!jPqT z2eA?IRa_EzcrArRAE$m%oRzTRAw&6cvVE>QT*OF?;M!1ETPc%zBf9yYLBdAigw)lz zbk+^Q@HUgGcSOy9k}l;8Sf;c&FEIT~vsnEYeYZq84*W$Q8t~KeYp>e=Ajt=@K5o0H zk|1qj0=W#yZSxLwg<<^W(AIbl+ARY#S*##R!W0T}-)&qCaTSz*Vv8)DdxB(dQ$4rK zr|76&$~@57!}I51(CtYF2d_2nPUVfbYd=O{DxPfo|Ioxg`0x>~$IUVaKc`R=f?JX5 z;h(>FT_*7In)rMQ*BgXF7sHXLXtnzaEUITjA>bu^9%LB^K?5;uVR3vk#latB>goZp zZ9Jesq$+wUMtfJoBhXIpK&N|Cw588T@gmtO%PuL)HI{MNzbYwf{`3AD?{$^D;=lUa z8Onp!9klmC{Lr)}hn%#^KrNbTPA6oR-4gDHmPNk^L?=hHc>$ytk>XG^I}Le>y>>jQ z{v1SGK>Y~|=UI;ebmf9B`5lORT>*(X<&OM^)~1|dNN?ntbn&O@Z|Sq(!6enpwhY1p zgyM5}6kL@slJN#>6F422D(yI)CGndySz2WL<>s?1e>e>UQJn49dUq3rQi4>`fTiH=8iW zBOHP&B-QPe#UMKoEEe}$QHUL);`NDgM{fQ<74K8dCTL3|Gn_D@TSm7LT{C8 zLLiRvroL6?wioHoG(nKn{&eAIWgn;#vJc9v*GRlEd?3X2w$Wv1KUV5-&%T*7t^{q1 z48_iB`UruE!Qz=qdq@JmAxjtF1yna3M)$V1BnEc~)@9)%;Y2|jKyflW<*Zoo) z{Ap&^!`ovN$nwtO-!M~AFiPO7(qD8m8~pfTxwS!zMNcx(EmXZOiVNuVcd5lbt(0eE ze!4#}u4{xB6@y;LCkp@LnEKasfMC18Fr3m2sz4%uLNIJd^a;^o^`0Ab2TZ=kiz2jRjDf-nvs0gpHP#o4nh&Q|WPVe}B(rCm+R=jyC&1kXtRkLR$2bvjPo z1Vd9mchX@+Ll9NN(^vhBJaVnc;sbzzE!^o4eGPt?0^U< z;y+hVWP79d#?EYC8^l%sIzuZ$ne7c2UpXCp(8ee1v|*x9op{shWU8Qd%{ew~ zA$wg~qYH&yj&IM~jal)wNXsSsn;ec!;syc~bHIvwa=>N(&IO)aEWN=7>*p!jGvMEi zuO=i|ByL%sdpzYqOuK)utJq3)-NxuKJO%3@A+CnO1L)Oxgp6^1P(z$p0y74PZbd&t zzsF)vS61elC>8Cre<}E$*g$R^9huHb(uZux{LK@^atr1VMv5hrbk!34k{3r*@Cacz zZ1hENN4#&7E`V)de0YITan*)}n#6UL(}VL6OK9^O#c7(Cdf~*quv{4*WC<;ZOwj5rF|_ zJRqz-j4GK1$mwFAsf8V*Cj}3}^jGLlLeVdjYYiZHLV68albv!@I~lIrFPnvbK()H} z)7Ph(3zf#rI>w|pXs})-%{cX58lz*Co3j<^Jv@uFN1Pt{J?wrV;}Au9bW`cDB|?4x zG(Ar34I;0q8p!ZB$S*MUT-Xi_lVgN`+*csr9E3RZyn;5z!}%MgC@Zai^be~~UKZgU zxkr0_u~Ppympwc{B9V+lXj_RjnBAcl0?E*Hh)u=CEFKu!vUel8OEyt+V3L*Md$)pu51TCfo?d=d{oA2)`O*LwE+mkI$^;5!5#O>6mV+>1@*wU- zVrh69A%7yCy3p`YGx#>_jL$s?V`JGp-myS&0q35nfvJ#0evKP!7gg{)S@w z{3fM!U-fMw6a;_o;U+1Nf`o>5kqfs6R0;;t7nGKCS>`)BX}AXhQ7(nK2?L55_wXc$n&h&W7;Fpz*lsNf<9udM`a zkh%{%;>x}>$#kbju0Mqt6fyVJ8L0=!L!-b&opFcqdHO3yw*^<*JOd2iL*RWf}0q7|- z<8~if3P`}I0h|BEnd{yl;FD{UzJPjm6Q#sS589wr29hAKY64`u2nh_czpA5wUW?O5 zvGqD!{&vtyZLb!*X7p)yFqsHWq6O(oqpitx-SwsRl&QqvUBXCkG-Ktm&ZEBlcg}aY zLD}UB!a>TjLgGD$QZedxBT4-ahB>NUoUd6(psXR+%-i|vJkSaI zc&a0OSe;LE{#j7JJkz}2qi2)<2XMaz?&wb!ofH(14E*11Q#e>_VlD1+5L_pT=R68S z*IH!28)SuvL+laK5PYOAd_v2QxrjK$TU1@oTF(YFNuw+7@H&oB0wSGrD++I*k+liuZ)XcBqgM z3V_bG8n_U03|@8>Co%jt#4^2K?A?N1gkuC9h2Ye4tneYeH?wS8+|T&64ZJLV4dMj? zd$2Ud+|2`bDJ5lo5xz&BB5|dtr9q@^swTeMp2-kV_pcGuqja(uL2>F8vjJgSrvnnb zXdB$r7h+k|KrrefWf3eg@-p`uoBIcE>_5*+1cK5dQ-kH9{2K)R&IINpAk0C$WYcgI3#><1tagwx=p@e6T1An0-nAz# zNTm~607C1dc?ek(TPWcK!L~>fhD8)PfI?gj2)E!}qhunsD& z>SJPX-TA~Y(2KW=hiMn)xYgkyM6O(qy&2@>nAKT|ZKs?b2p>JUk?D!~KZF%0e@MPD zD_ju@(Q}5eBLgAwNG8&9H#i*j$}r5a3o+_c1`3A$;)51E|Ajqh1$jIE^*&B?;~#7u zBHH#$ubz-pFNhEfak=G$dc}=F8JUGc0B}W3zu~@B##RC2YI7}?VZ0JBGSzuDw$BWR zxh{jeg~pvk@%Eocu4lrsnb2fYwl?qnepjROyUs-o(BG0M`x|sog*5?tV$^eK^jnii zgMw-?;na?5JLm=JV?R-Z<1QcA7D2bX2bo=T4i?0Ts2m5j;e0`*d`RRc-)xF@;C?x$ z`+X(e8?`yQ zJ^D0`-eq{BJaAGN$trEXUxjKGQYDN_Kx_5l_g9yI0JK`1A!gk`S+j$nkaRDzLTnUs zu%V4$IMxan*fsrqHvUx5_tsoEB^kJehAL(;kV>ihli(GPO?;I6nu+@qJm_TrrTAK% zDfMJ{;6?hON->LX{EKs7F<@OzKaC+A^OhO96b>E7_Qmv2;Ref6=f1GkOi1q~LL>C} z&o0*UA8+UO3^kQ1lrJ@8rwHbBu<>Y5|1ND(_kuewE(3U*(K6zDoqdLiAR+Nb7U
    Nx&GrA$TdH3sJHAHot|CUI4aMKQCQt+>>&zP{C_2K|030j@1b_jn>s5Mh z>APtbq&p^uGJ2Z?qp(b)$j^+Ze*=JMrqnCg>IFvu8J=(-xaVuS=~0|TH+gUl76J<4 zOcX@h?M1h?wANPSEO1?JwUk$9EI6X;@$gv>AzM1$oBx(B#vROuW37-;fPMB(DP_gF z|CYIF3}P4$PqRK$-=L221gf9`5pf)ZxU$5Txx=o9m7h9=y96f0H@(_BGinsIubsLe#=<>UIw=% zt0Sm-<8qz43f9_!(@$c#_GB7mcG_P{iOIOcXSb+^a?h!AWX zMtc#mxwRm;Fv`EW?Kk^}N{FrPpsZXINYdpU7=})mO8DYK^;fXR zzhOFVk`gldgwwv4xR6&T|0m_pdq=JCz{6I6PJ}bWSOIbZN#R$`9`=?PNjOlPJ?^H~(*4G&y>We|`@LPjrxDd*B)z39Ygb5% zl~b*727A*df`opmjhQA2fa3DR3F@t6R6qBcqb#rYMI(iMyk`M)r+t@4dGAe zl08~vXf?9*{NF{eapv}OrGur$#Fq|3IYWYk1~w7D2ysWR;H-`gi^-nj-L7JzKRd>W7 zJ#R(+TsU%?*N@emH>Q~&h^oSm1h=1DOsj5MOIy1h>)FfY`p;LiNHR^@ud=b%Q2Fd_ zd24L6q0XnrI~0e>;(AJf&efTjxAEyj#)yHO`s(hNj`+0nuVCQrb1s7dE_+zsifQ5FQ|%*OG#NizWv`H zhFX}_Xz_Q+UI^Jev3@V=Cj)Q@^r34~G72Dis^8U7@gl#RJXgeg2hhX~WY`LNXC;a8 zYcoUcUm%2XV?d-@dqy`-Jh?cenH~+AibMhC;f#*>Tk(&K&oHM!gxiMpUL@D5J+3In zOgThQul`h0<4r#ql8pZASM{~~ll9)^+Aof8n;qF?!j%h;7GvPh7kDtD0G(Kr?2if> z)A4v~lfo+XbbtOg)lxuUk&tI(G>!j{{@lG$myrb9k~1+{EX6z20){7{q*NgeEN$`$ z-*Q7Vp;;zyPv!_UYHK|hH?wq%X{h!jJ&gV^|J~ie-8eoG|3v;oU$(0&5Mw!dUp|U_ zedP;3@1IZtxep%7l$*ccB$#G~f$BLK;g}uZZb)006|$NGDCIvRKMT{8_{2@OQJ7L_^ao|deZ=P+SIE`@ZV`*N_TI_m(cO)CwL-J8(=qwdj z2!(7ozX-$wSvi?#+$OQ(wbygeKFv64>5n4osg zzb(5LM|Bgeb_11T|Iv#$SfMvd<&NvD3~e7<5UIxn6209bi(hNhe(h6*3%~o?Sm+a| z4u_yvo3Uue=gaj9`_n8G=(M|)>cBy1s9CyEY74fQK;7BYchvSZ!M&S2XS8YAE!~@D zBh#xoSimA!;#Z@?gw>+xIH|#w%bF*YG5UrYgbDw}>JITE@pukEmrjQ!19IU;{|}%D z)(G9A&hYgl3Cz1XUX^0PY1FTK|5t>wT(7uBPo__AB1=abDv5dlc?IeaPcNW)8Yv#8 z5sat>4&LVqjA;?nd63Z_Povsf87M68g7{S%2 z!!QN(jp!0(aKX#b>|?vvADTO0aYr6UooVRsB??{8^M2yKM9bHuWb5j2x`V*XDyGeKs05g$rR>|ILJp4@@G_xIXK(2BM_urOl z4~IH_EzZlm8k7Ifv?J0b#H(@??S~u7jaC&}LW)w`^2iqsD@&PlAZj2w3emOM-+~A| zes})zHqhoI%IC&X&OP{ri=Y>lSzkg)oYQTD2${z}M4-a@zuyk-$26@-j&Wik$HP_s zBI^{=zga{c2INv6zz+(tAPYta1R0>wp^-1&vJ!$~V=lFmO`x}tUXaowz0QR%UAvPv z=bbslE1knjeJQH5Whe%eWK-f9I7-CZd!V9r`e%xJt@q5Zl z{Zw_dWmil2%+ks*!#T$PR0FUv?tR6a_~UK*>WP$0qd31(=A`;8UQjx1b;%|^bQTYu z468oMMT=krVj?20!qgc#fAsxexQDvxg}y?8eTf?ZYz1ntncNZ2(4Y^AsGcth?@qO{ zN!6#NUY7X!N$F~tC1eY%0JK1_#V;)y4Oi@-(u`6F^pYM$Ux5t2GL#Gg#P>D>qRP#B zLkmIF{mPsDC^?4G(?gZ_t59CkE;Na=K?eJbCy68V_ozOeW|RGj-V4J;6?iB$Lp}Lw zH~~m_{jJ9Y0Jc2w%y78JwSpH(1~7}x+3PD1fPw*^!t~7TYhnQp@Rw7e_(;xI?#oQ= zca`3rt?<2Sk|V2Z-yNa{Q$yZyJ?`i0r%y+jn5iV&bOW943QEBNE))UM3|L&@Aa=z? z#c-UGYcgqdp!5%ULJf^L?tObg*~DZ5=~lv_@7_=6X}<#7EsvXbx^BcL#NEZjzTeFs zJ>J?ax)d+lBj(7RO_y_cbNX_iQ-1TH!{D17l{Vo&=1cn=52)|@`sBg!w_VdO+_Gpv z_xW`7zoVW<>;GNxBx^U!rKvuF9H}FDtQAHa%6svb45}YO7?24p)zE|CYmIthf8koJTXngiQ{p|bcGJ%k7vT&#uGZ4OT!<6uky z(GLsdRl`Kkh*-BpRjPgLVE`^T5{&t&VrQ@Ys^F`jFOO<&4#eev5v9HYEsV&ChL-!c9bzZJy^xyrjT4{n_L$iTrT#KBuVKuC|*pZjCuC;F12rop%|e+M370tMnkZ zO)8Qn5)dTu2PSL%@5tiguUe$BhxF|p!>+*@(@7%VUmxbp;az$khLailg}UBmemc;$ zhZv;)bkB5bS412D{(NR!tdgspn%vW)eyu*46?J0>AE_{`*_Q4Rxks?R~s5 zy)~qo&FQHqE2hVvY}c(?Iz`aeoKFUWGl*$8Vxj*V;VHCVE@)bHYkM|I79@m#9mZZB zmsffcf=ZA%IgB|kOfQ)jv0z_MNc*#IoYwt&JRp(FN>a%C9CF?Pz1SN%H4H?)RFuv6 zPAyWcn27w(I|5X&u}x!%MR=8yP$SsUW@w_31;hj4Kb|e^5I)1`CYB4B&aRve59(P& zz+5ekmmBk{^DZFCy&~dlZP-1Wx%PpPEGs~I2R*8K22qf**G2@(QAUVrGP8mQYK_m5 zb*;$L?vlUE^OuaQ4LgcyGgdQTZS>OqFZv1fpWgHJf5Dx9cicdi0SUb#oOesg7gJIh z;SD|Q0T_U75z%Y${?NN>xzEo(&X&ZtOaGqC-K_9?SC^#L22Un1_b0$sf?51QbYNX^ zSww~n*hwwKJ;CpmQ2>#7QQU^b%zjNTJuBm^pfJ=m;E&tG0a{3tTz027V=T`L$Wdwj zF{37yd^P`#2E&Axms#=}3ge7%l75L}HuyLUSSQjGA>@X{oe)4%>9=w@q4h9REOy)s zXN!W76dnf-N)Rb4z8;_F5m*UP{w5|TVms3j$9&A!X+tgtIywL2GYyk@7g|L1Sz;0A z>#vNcF1|}I{qFo{o=?e59>yN**@Xeq}_;(5fQUT{4#EK_H z{>EwA;Qy}J<}bF9h}Djm%Q|oO0Rx)MAPA`QPOtiO37XSrz3JT(M2!g)M*2O{b3$36 zd(fZI*C$dvL3lmjAYug8iKtK;eFqh~V5)9+!Y5m460`8y`s5>04s|z>7Rr%BDF3_h zQX(GE6%oWA92cg`s0UJYfy~#L1`;!e>MT)?pF@8NV1s-Te{A_bWx8pQtD`N+z9j=1 z06&d0%<{W~)#KKC` z7HUXaOrue=<@r`$I(J|*4BKinwWI~s#$$n8i@Dm`PMOg^tPB@4&TV9gf4uA66}a-S zzYpOG` zc}09ve~fF7y=cc5j~ zj0(O)02${o%6p9+ zo$3RrIKx@?h;x0#&udKTBmiNx z#$EyHcIWyg+A%!~dz@sh%%`&-Bz!nqeL1G%WsVW_0_KMFU-#_&ZpvlkZ4YU_yL5J< zIH8BIyRoUe5oQqH6JFqCmuZ(KEKBys))fAUV)(P=w=OtJv)~LqHMkw7hRp9)9bsa~ zWl<+HDRVC!BlqGvP21bQ{eJ*GLBhU3;90YV()KJ`w`x!rPws03e(qKN2gZ^_Y!pR8 zt9Z<{ghf$4mpMVfa6Jm(davjM_q(h7dw6(wBqfns`l@d3jLOqyh5}M6eMD%>*IhKb z%*$$=FZN=XIXFw5s~1$(YfGw0zt!?{=BFIYLCxx1T?MmVtCCInX?2QhWMYkFR@Hd5 zp?1F2T|YZpKwXt2xuQS}JJv?Rhy230;|(4OtN0%xG20F;14Si(Yya)IE3po7KSk~I}%AyH9*P z0)HR~)Cm3veXfNVrNJ$iiQwug3J7-0F+w|5dMw?{>A{1C?GP7~uY@Q%F61m=@&5A5 zNLvBmGZ8ZJ9uTXKw~9(jw9OT$T}6RwQF0J$U9SSqqon#P4zYnKB`TkO^Nr(qBB`Hn zH^O@GEW{SpKjVo4jgX82%{_pJ1g`@30G<%$040d=MmbffMUe;jJpfx z@hZ6cUeIq34-b!|CbZZKnlQP&Zi?=!^Pj2e^;M`IY+5J|LYSpo%-g7P{a0zr)W@~I zM<)&2dzr0{L5SEgPaDhi*Qza3_4xiC>NISxQeZ0+Z!ELbRjV>{)%EMmS8WJVk}C=b zbkL%O^EaVXsp2G19zE&^`XnJE+#nzz7$9IE6d=^OLJk&D!eFpIVqHbZ!1{>5;SY)i zp)y>DP%&oAX-jBHyuC8;C?F4GS&a+f>-+2R>Mv;I0bhGh&r#Rf59))ZzU=6Tts2;( zy4nuC5Y7`F2yT=L_0RQ42q+1UJR*%4;RyT)*NnHDEbb42DHdJ?Q|84_7I72>6u`4*Pup|H=OzXyfhY@Gwj8k65Q+(HBNqXskxk=RUs$bC z6j)0LsVJe~<^`a9B5b2DP*ICE##LEJEC?~8Jt#3)<>?RQTS*e<9${Qj>QL@zKgu2J zpM$wUKu1~N^~o(@(;hyPKKOf{SQoBR$r^L-ag;d5pY^pL<&!uq-%ZOpMCoLGUHRch z5jwPXmc|Spt=a32r1&4Dh(l{NYs_$sp1n>-u7qFEh(lUCYmA1Eo~?CPM1uoJf$hKI z)IJ!-*ly`q3$!rt7Ao<^)#%gS`3`4uO7L0 zN8dn}EpxmZv?H4wyH%=oTXpW%UJY(5rYdj! ztdtoU8>s^`U(?;CYO8JMZff7)HWjP#mVQpbVZ4zCX6m)OOR09-&g#~_fo>~SMQ^3# z1_zMx+Mj~gz|lc^Abp7aTj3m`3Qd((8FCcR&DEaJyHo#)CR`|yh}^63BNr-f`3lP7 zJLONT<}zZx4rabbwQq>Mw>L3~-Fl1A{)3ryPwg8lvHWrn%Qct?W0|JxT`IkMoQ5w= zA({f!X)sb=DdkZqL+RUO^JP25JU28WWj06im0t7ykj9YG57k%6PIkQh|f1akF zMhwxQ9|mgRci-u|!Sl2&sw`^5zE2xB#<5gth9at1t&H-y=>6R*Ys(5yH`mRyo z3e{Au;HBemm=D&FHAI)-jP|YlS<`+Rp&^5Q(7=IKpS~Ncc|khG7(M*(DoZGhwDS`9 z^2M5tkm_1}nHPj+gi}}OL|{cAXHF3K5dN^p^FN9K!Y$V#;1N25M}e>ys8`QP*1bSh zJvQ!`_KbW)*RKCffA|XKMD7}*H{P3~zkJq$u$!8x?cI^u|4&pmBg?2E1_<9&Pr4j1{&;T~OO!F%2DFcmDefE2Ge3+q z3EK3}Ri3#&cs+B20>Stv7mPdiDt#ba3I!jSe>ZP}`;+HUa!>+TW8D8WYu2>A4tPkI zOCZz@!vSm1&3U-8uzv6+f2CO^zt*~4o3(z!-#V~)kZzy(hWae`D>0Qqh&(V^U(G71 zuh;F;=Jgx&_kqnCbo)%b(Pz2uiWG*WgiZcxmP&rTPP;a**M`3jX!D@kHS>)=N`b?B zF)6V9DS8cLeL~}p@7MI_Z?wKkO6jeYXBBz&sJ6}dP_<7_(3=m|(2G+J$0A%NLcevs zO{vc-92}{2%j+?0ECWZUYs!-Jy7{)k7Yop_n#*u)n=nCHUVL4*Cqn3LoQdUn3*Rh92uT7NY7t(rOWG25zq`0ybOA3oeZh7B8L3j#oLBO6^|u}1J2IdbHA zL1M&+5%%>YEfah0zTW@f8eW$!-8t*CVvy(~59yarA64n=^5~YD&DEyu3wrhKclB>L<;!>6nv-z-^wUrF7&7MU!aH{CSo_=r>t-Y#o+>r$j?GTHB+(+wu_HFZ9l|(Y zwo8{TwS_|FjK_rXP{oHiL*QfXckGC|Hvxo(rO|b@>ZpRYY0C~vNHoW_aNR0tZ(cyXHr`~9GTAN9{9t0Kp}7iaDNLs3Jun5Le{Xx(+@z{ zrXQ|FJUO8aC|?4L`!e z!&*SGN3j9g%b24G(N@~%Di3V%#&^N&*|TkhgZaS=fs%1$Dhi>>R{J&edg@lWw{F@( z_0;>LD%v}Lr|(R{i9GzPhK*aTgs#@mY_;`TucvhDg~qC0?~hbv?|k2h#3NDK9GXqF zU+bl(E;Lw$>h;z~RrYHB&cqIyh}7X z)$4UI5x;g5Zw)5mScVl=(L$lx{Oc0!OYE#vh4j^ zyM4y$G{x`JgH7&J)!e}=t(-cAz($Su{A%2|v3(FGLI@H<2DR3)IyPv~z!nnjzyE$) zNFWr0ong8D!3Q6-1toU7b>GAPz#9DX&+H@|-KGhrwd~#dRr}=yD)9OYZL@^lzsEhT z+wSS3MH@D0-Pjj&V-Qc4I~G>6E(ldIUMfn`;<<=8O3u zq=^5SH$uq}`gqKoVa-OUtzNyls#K|B3xdXkY5DRCrAwobA-@V_JE-0JBXnl<1Z};g zrK;sMI=KI^^%HINZ}}MgUH%c>9Pk~Db7{ngQkKFHX)7|6)#S4PA(fm2*HeWj1VNnh zE(o?LGwd0GSA^#Yd!hZ8Bm4;Ana?JKi+h6m0bE6azbG+`KMFHHa*R2iHxwHb1j3H+ zIxrsyMy*7{|MIQExT zE}Eys>klbBO(~Ttn25r$kHl$nD(kz(Zm|AZxk&RCuh*gQG%8iDpiUo3>@{z+YwTNk ze^!D=*^lIFf2_wl8P`B!5`OxWW8gE^?~HvWo*W&dhLwA2{#W%?3I{u3e{n z%4Nwf!D5cL>Td}TcmdvcM47aQA&J2pV9$g$TyEOK#_jT!|8S{EOx%?A7 zP`0A#)-0*aVHtFL-<`^kXpKy)HFz?S$1;_2<-Atm>sM)2FbnY&jx~%)3n3kTCFX+? zn(I2{I(t9|jyt#JzrPI8wsOr>JerWP)vH6beS1zzAUkYZaS_M}KR~eLK&YV_5{dy< zRl+|IQh5#miR-DN<5tq)vHyVu6QN?&sxr~GJ#yC%>fC#&@;*OU!~S=V3SWZ;n^6Y_ zf1(k4qW^NH1SX+BR#Pqu%m8OO}P|j?}b>#3iOfKpb*X0b+!F>l6d1j5q{c(dHt!XKgzRO+6-T3v_ zVwUn~>>LZ$5DEi=CkH|)R&2tc5I_-N{bSx)TUf8LT$2n=A9KH;@StSS7oMYE+$*l9BspMw-OxZc#|X~{^Wav?Vr|ea zuA{xKqTs?jki5NM!2)|6@D!ma1Y4hHcgzR}^qs*|dOdaUz}>&dkm(*6O}d+B?v zIC@Tp7Jsddum7M;p#j^7`*C(!E9D9cQAk*BJ$a$gn%cIQiaatx@hGd?tjHr8VZZT7tsT-{57c`}->*2TbB7k|>yEGMhfSeMmp(etAt}e% zFW;(6cdb4j?Rip5x<00NS2|v`h;{Glt~}NC$%cR8-2NmRZb=EvZKfQ$xye^r6~9`R(Gvde9<84IfgXS6TkTLDHF&I+E)=(p)WKgq zSIeq5=-O-2DPx{8YTRqC_DBCSZ-+NcIE9DdjDCK(sP80%Yx}mp6_GBBvIVUG7t>nI zpfc^JYS%f8W!8H~>{9<)LEa=*Yd#^we7{^%roE=_I;Tk6!xEug{c9;$S1<1xyyntl z8K-d!)3v#>Ne8#zPB8a4NL#hq&9@k!*#}f9OU4btmB(t()4%c5lO;kjUTFM!kM!9lZYhvFv{W@%^3N@~( z>_HY5IX~sfh1whMVMWJ-fndrm#grMPU4X!iz)9>pf}IQ2Z`#TlM6hK&at=j-zPs`A z$pPUx2-AE4G9P$pNK8gaM$l$Y3fe_`Nf-wDg@Qn+67#{l&=>x?UWMcgKcXGXJAHAJ zvRM!QiULXv!W4Z(8Dfr(TVrikJbLj(`x^R52~|8qF3gQT#}~h)>ZKmhm`pA8<8n(_ zA9J6!elx=NY~ph0$46DvT05+tphv+ndSy=3FUyF{1JtzUtJ=}zYYpu7r0%a(Pc1sV zr^imr)7mV78{wrp?PbArEWT; zsnt(A_Kerl`44OQq$P4%ok2BTeO@;mKca~EHabl$_1m#WF4a z`S-tS)zF4IJ?;(lnUzSMiSSi=yG&Z8$&^>O)$gKtY46c1gJ#x70S+jf>oX7fc*|1HKH&jv8g6@rRNR!6Q*2&^ERHIPvn+5s# z_uri{xm=*6SaPntOLQRNn>T1%iw->@?;8*U-A zQC=1S8e!7~3ohgAdK7rf^LP`m3NzLSgTQ;B>@jbI$S_yTpSyt@VI;TQlG{!SXFrIG z2)RZXuL;pV1*@ojQCb|PoY&>hk=Z|JdC4ZaGt0$V0&8l)0_o|e@%9w5l@&Prj4cW8^at-@prU|6>3Rql=VRnpu!)18 z7D-?4z4u;wegMVI#T*!?whcd2XqP#f(&HiBoc)Ye&G=1e>sbpr3CPiAs!rRMwrGy< zJ^Hq;pVc+;kVd`!KTRv}f_gMB;P-`{M)~sJt@0Jz0Ne|NrB0_@_cl@O97nZ!)lTcD z-NGExn!|UfT{FM9?0;tK-N%~RA%YyuI*!)H72oO^-{YoDo2$)Neu;l+u2+=`do*Ku zxN1Fcmt!T2+^WH!%}}m~A5_WENF5sWy8btB|baeG9?Q|NgV`~oUj&{wI*L|Q>V+Ndz;i&j-#u!%6VgJIM(R!9qrU4 z?HIlOziBG)LJu{M+F<3syS#!7?|EW7&i~AQSC5@ea zVqWvjE=;awzx&wvxoz64qegFBp>LmY=eDVuxA`hcf^y=u z(IUmK!Pt(YL)x@ytS4hO6ek-84oFWuCH3gxJCQJ=_B077QNr0jfZcC>ztb0!Du;4q zIjk*{zR}!5E%ac(1o-p*`-CM>Wyi5NA_O5kV7*1iA!Gr;fpdT$fv`qs2v$b~4!07G z8wbq~G$4#}&Q1KRRm%z0dG9^@p&yp;nLE3}GGZxKQ)8~DwcBTb&{ zdyz{>8l^A#xO#Q0Am`IUl{3$+y1!E&JyS7wP2pEwktHlU8-nq1mIfqrA zq-(732-2)2+K*69;yj80?RArZQ8G|ic^*r#zxUx%Aoo(8icQzXjgw~2mi4~ACu0Fh z34*<=?4S%#w zGcQ(fFQTOIxr{#{fvkZ*MS=bis>WQQU<8AFB>GLiF7zQnYrgEFF{z(W=SFF?W@>+R zs9iy=x3|^!Pip(FNn9fT*{AK6;)C)N^w_>bhy23=k5AR`DMwVgL8GXfH%0E(v`raQ zqj)r5HmOVKw&Ee$v2};l%d^_>eIKo_)lTk|aCXD@>a)I<+TD}&f_FI{ZHCk(Zd37)9op)=vA*xKUbWiY zqtI!?HRVWYHE48`^{sw^$o-sMDY!8Zsoh<45&tUB-9u#e{^W zQto;m>&K4QY5EJz)%$n<8}N*F{&he%y|FAHX*p6SMmO_a6F2PdDC;-mT!1`zd};#M zVgS*t`3p-ky-L1 zy0WnreUkIbFUEw18uQj$z7v6juQYF!%3yBpFxMQ3^qmXBq)ABh#`Nv$Jf1acmazmd zWBu@(p+kq7VZ(+QOE@wkMvO4~_U*H;nL2f<`RS*h?Ehigwryta+_}C-#!Q?z(O814 zv3{hj_n%}eA<*}X!|2h@XMXdI?}U$ty%Wrfm8zI$2TnE9hIBVgs?;{GjM-wMzWM^= z$2;%Sub=P4W#q^aWAf%TrbZ3x>&}0?BlemJFIF;Do*ihWP8(vnH>qN3zcR*biKbJ3 z|7}c`EXFi$>^tFyC6roy9&aq=&8%L%+Wh|e@4jb@nKNgOJr6FHQej4p9BC|tz9P5(Zzf_?=S0p_nKXszc%-##c&&Du7Bkhb85qT=JwPwp@piNO4eV# zIzz(Jo9oKAF~hfoo5%w*%^TGUm@GL;n8shOGAEr+xQK%T>X}TbQyWWh34Ww4U|LUc z<^{t4>~6{^qiHt!!Z|p#^h48a=-(#7_xNH?nQ@O~HrF&BW;RdmZaNR$Y|i?giH`79 zubCSwePo<(&R|5Cm2Va^p*6lW`>dsWZu=zj%{z~n{5e{iIcKb=!~Znh%gAJGHro1h zKTa(*A9fpR{*Jz}=Q%a*5tI3v#%9>&$)YF*5-lw682I&;e zg8KEHr0g%h^quqrPY(y7T>jQ;=7hOnuL2h=#yn5n0${yn9>^zQZtd7}>zsd{0|xJb z10C@bk5zo4V!r)W>d?WN6GEr_alHzx6+8umK;ao7{E3}_@tEL^!aG7$Cagrn)w>=A zAdCqw7i*C9$?Jn*EkE6JkJAt9ZS8wy$`r=}k0XM-=pp`Xio1t zl&;vrdb({#jh#DQ1G-jG-7mK3^vNCi@vg;sZ{SL8nYU5p`mfTKsgG-ak4_pEO(tHB zRugsn>{;8h6ZAN}Lto!_A@nHhx*{r=;heSpFIYJv_fA!>5!v)elYAFk!sZ|SK7Fx{ zoc~yBJ?6}=@Wp-g;jRbNsd+I)SHCURnB7`1bE`_%tEMcwX6T!Rxl|(jcO9-(SA|y0 z*ZOnUsYt;LI%nOh=UW@OS5te9P_{>!C|@joZNhToRQB-2>igj?JpOJ6Hm%>UGu$_+i|hFx`sv{_1J(GBhxFFSrP_Yf z=&qKnRN~K`YS65Yrf)c?lXiPPR<|Dl_wtDisgx(*bqYQGuTDg9&L+lM3?S&5PY6lS zMvD}?23=w`jz%ZR<4l`J`Ep-{E(Kd%NCq&}qgaK78T0w)ewOTrJ!axtZ=EW)~v>qEo;o-!@iR#1lOA0q6Jo0 z-^p08aicM%OFR8qw#;`j2v~t<&z^02dR$KpR(F5z0!)}N!S)1=88gPV5@XfI>%+X@ z?hn&=-F3#~&V8|0AUcj8H>Opq3w@7>X6fNUz=}?rckbM2dlOvn zAh3?GXs=wk(mvMnOf_X+*137a#>*4FK8HaG&ONS>Vm=^^@+<*bj^}NoyMT5fqix&Wo2DqM&`Sa)7 zo)PYG+Q{1A9*3Q4*V+?%B7c7S9QU~;fZNZp79-N8HRiqd?8gwXZ-#lJR#B5BO972a58&RO44Ge7>oOxhW4E$YK&Sj(r)@PlW~u`Sb0m#Rff zqahc(FUA}-gBn&c-Bznnj2G7VWt|O!wO| znh@nQFU)t{f9K4~p7l-B0e?p8-3a88mb!@T=Bw%%O}4U+nr=f^nq!ft%(SPnnap>z zHzU?qH!zS?FkM&vYu0{N%am%r z@VqDX+{&J&e$xTw&sa7()*gj(D^1V(P0fJ7H$Jt(d|I!VDe-V`Gjhs!GqCroreohl z#(#KC#5VKk?J}X&`kP(WdfqvpxoJP=wC~Bdj&mG53g(Cz-1<>7J68j~r1dDG;b>E>^D(Z-QO-`;Q1rK)6lZwly4 zZ00ijck_BdnH##UH2<#s%+$K=5%XvPzsIH8n}ru{rr>KnIxgu^u!z_F(#*7Yp@LCp zHPe4rq>0=)z%*|^$DF@|F4Gz`XJ(pb@`PIR9AYw7=wjym)W+lqwdOn|qp8qkk@fxN zyDnDac-l;9ncIY1-@z>McXwSS9))C66aZP#5zxX-#p|n^8yXHX6MDRCK3^7GN?pI+ z5#;D^)v8gI1V9Kvs9d*hogHq0AcjziV2N;q0E&=-fPf(Cf-r=D0fc7|+7e8l@dst0 zK?A2P4?T40&h z5?m1I7+(Zf!bK1?{RLnIZl32k1aX9N6aoZp1Y3kb<`Y4ex#s#ncI+^wRxM|&@4nmV z$Du>MCw-s{FejZlIl?JQABs{;(@yveVJ`H6&tK1i^n)% zGKPfjpbVh+5N=0Uk3aN{d4L_(fFg_&u7k3~m~)^=(7!`|gS&L86Mj~vj3uP+jI=+i zZQHiicVi=KPHLH6y?WWoUodDwxpL0yt>KLVpU1c^TD0hVUx4=0Htuy40X~nh<^Q$T z1W&c*i-mBeND+HYxuEDV50NZ(KSd#WBFy$rYMX}NEH|TGdd&1#WGOwIJ~rAsUb>vA zkUx{jkoz{%;O!Y^ca$IUjpn1Lx|k(?LQi6FYWHfhaQ1xj+m^#Y!W$#a-ftV3Hj`ud zOPkc3+HF=ZoNea+w#6I{`h5~sb8MpEuXIw8Ur_Aov3(f5LznLw;8*B*` zf0!=SJD8uXrz7{xHZ#^FmLixKT%!HqTg;5fYhtZ1CJL8a0}N>wQ@z8_)_RUK`)1ED z7lt(4tnE?GWUSHG?1&=ST^*cT+0~RzUBtXJKUe_OrPpHQ$zM&ECaugz<5rq~teYqB zanci?`{kthqf2$u!8u`O&sby4auRciH5k5ShM8=&%lF7^nK9X%e;8+un(yvUZ8A0- zVjPo+4=?m6?hw^8V1P7XLWFkg-=|Gu$EeMBU9@gs20IL7^k_EjlK%HUN^Lrc&=fHF z_+x4ATtXmxZ{&mU6gQFC63Fa?MZy&bIUvM>BuBy~IOksykt9h=5V3a&CRBt_1`=jt z;+kum&#t7e1f@pXwR1vXX#1c+(yOmZcilw|cPD|9$~+Wc@m1GMagf+dxC@n# znl&S2$O)7B{Bvp4C~5d`Cvl!iP5JXnZ@=w?fKe%kF}M^Y$r3X1`|rQoWi|;5BazvK zc_9pjxuQZ3p(TXM5Y9um5n)Hn74JiWZm?ukqL^mPa4Kt&6HPx@E7r&A_S>Cs7$Q=b zlg~br8a8yod6+vwn4*Jq!n&f3BnDF!)rIv!=nGZ12;m`wjSv??j98B@gv^kLOL!aM zZuAYfgwPw>Hg#%?DvDWeYrl{GEy#FrZxbfOIJ?jW`o|c% zVMio9pFMllP6jSjri@Nm=Tcd3PMd)_q-1Xy>pA8lc0_2;t^->8QEQDT^_Kp3XC^zj z?%(-SwWr}gE#7%tC-$z?*sitwDzt@b+k^?q^5W~dJ(2fGNtfpq-Cd)K$`{QRB)svM z=Fds18g~XQ%%1$D%cEQFuAwUBiz-{NSWIgq)0eHQ{2LFc(9IbyBrRSx(kag^cdJH~ z@+um*^m znR>1%y7zS8^>?bo_;pn0V|lr(`NZDJAHU zfr!VZ?FiUIhB!hhJ&Yqlyqp1;QW#HUh7kpAa1=2#XiX5|Ewo zuUC&Ux>D)VPTW5gl8Be5{l4)17~wM!{3#nsJ6K09#PhSy0&9o%k&nRc)8tgpPk+Ue zd<=wDgj!edCpW~p)K0}33kx0+|qkqcu^r6!a$+RFV5+_$us{-CsQ57Q>b4@D@CrDQF# zK5svoyAM}Eb$51-9pT#j_fDm%)m!~r6mx|0@B^AQW{qyC8nxnw!#V9+K1Q68 zT28rA_;-46!nJeybghc6`e1UCKBsc9(b@8&Dq1d=r7(tTlgntB#Kt+LT5k9yE;eyx1 zR#5;;KCy*Tk~K+0wd6Uw?Nn{V*g9c?)S`u@(E61_zY@Sh(667fK^}&0<^eesnHNQ( zYgea?Xly~CLx{txgTTjJASk;+A0P}<@)Ag9<~jEDVc2Y7i< z8oG6p>eiL@-oA@s0m3$)iO0ZA`~}t$mU}G!SgjFW=@-H=f;H!6&z@~>+{apDow1P{ z{ouMSTO8pS?~I#Qi=xX;(C=AV*_U5_X(xOm^dqFlM8}R&@#0eN-WT=0OpMTqf{XHa~{h*iV;HkjveRh(zu;FUr@SG@VK8` z5abyf+Qs}u2OfnkU7S+8D01A3T+eH1Kkvgj0mh!W;CZ|qlys$ycu?H_(+0ORFMUU; z!}*Zo`Vjv~FB)bdXrP}z!gRkLJfg=M&1 zeRpO^HcNTr_Ep-SH?t1yIiI?hatU9e*Gt#dsQgX!a-(#b^z8y|zXF;qYc_LSdq!)H z7E#$N%FJzad{JkW&lqZb52aDb0Y_9g8Y6JB6Ou#Mo>`-#Maud-2dPx*t(AUdNUiV0 zHn~%P$0@N%r|K=5Dbu2v@@=vmOTx9}_0p<6D!-b(+(_vreXXyaE9Lk2TP;X2H`B%U zOGu7ub!N>`6)Eetm&@hC9M_`G<&`lsL?LNP>g!#5)vbI+g<8MWX-W>zkvQHYCl<`t zhPxhD^CBt4)0ifWRI#EX3?%|7<{K|#Mp;6LV_}R72s8*|2x$oa2zCf#SVcLHuokE& zP|XGb1Pd<~R1^h&3u|1#&`jIoB7~-NsAgmhi|rlpMR3F!1UKdb!I`iS!bbQ3VEkQN zj{@V*TyZVJELDbFPmgP9cSA{%3j}3X=|l-d*aieotlbE(;J;4-*AoWAZos^r@FD(k z4c-Yn1mApf&R*|$qJZ~q)vA^H^yy=VGSTnnpMTyKh_Qsnge4=}UMs@fHg4Quj}ak9 z>^tFl)Ld@@Fei8`@WQ$K5ul*qEkJ=_?3p81Q2^Rqty(o(x!|7RGe(WtYb(tDRlFFV z5tioPuwg^{Xy3m5`S-gNguwCLh8NNGCNS>68e;v-pFdx7=FG9j2j~aSp=2>u%m;n) zR}@%NcywIlGuDLW*P~t9s-#<9C3PizuqoW`%jsFqDsL3&oAX}6s*(~&RY|>9{GxX%XH)J+f7F;x)m1d+C4QMXsTrNRYVxKf z+FG%f-mR4@<@{*IiV$VKtEZNoU~NX~pV7^9A@K7uadL(_b)BqDOSYbdNMBCYT5$bUwwS)6W?e>8L4 z2Gwf!Xxu9*T_IfY-NNFEKta9&7EJ^O*FuW2fDnYo!Sy;|86^&$&6lu3axK?km2|C^ z9*F}&BGzrj9YGIaloGUzKh|kr%&`npMzwhH;1X&mQ$h9a42+;_ttV^ujgitM5u)r>-%KtpSW5+H#$H3JAdiCmMmoFtp zp-!DTdiddoZ9yL;!4>dhf)KGt>kIGb(F68M$_;FeBBfxsJrK#n3tn-I9U z54eB0-d|Awo})k9Ph5ZY>{ivRc_H_PzQ6kFt9s|1cl7q#Z)@z>v8rCZx~0HOwT1M+ zvcrM!K9q=EW_?-HCfd)sAyHQv62mUoC;`k7apuey z0w{tpo(BX&<`sdJ@Dcy;ljHy)mG-eV=mWwgvF@}LAsgkz1tFMg{KIzK+yZhJ>ej7m z_kq1S!opHnO2#HT5gr($FTVIfk3RaSy&hfgsGxvwTnbCZSLMoh#3F6)OJJCd>rvpd zSm%s0eLz`5Szx?fP%@Y=u1DBr99$5<$?2hOdGqd71;3r}$B!ScfddEXgAYEi!>oWL zYUT|EFgEahxpQx{#;mZdSWcQW$!;IH53C#32y24&p?vTcQ07qJm;>%%okI#{p{|ldNtla|18lbpB>ZF?=|x)YH{^(#dpmW9#Iewg0LVG=0P|MVE_oHB+wET zv258gTR`GEK-fbt!fNR$3W*DX8v-Tczh%poD8eX0D8ek3c5suIUCTG2BFrmFAi^Ks zAm%bT6$Qe9Xcrc0EZA7B`O8`%i2oWOuhyyPOeL(0(=*9xi^V_#?w>=iLna?Nj z8AZtT{?Jc8o6sHRkAw5fMV>qbEd?Oby008T9c603fC2Wnpai;bKaopNkyy!CP`Ge} zs#Mu*w+*ERrGx}=yf?fLpA9GotS9by6aqK2khOBaQr1HH^;5B>OYJ`Udwx7TJUmhk zVJiA~A9X0tdmu>DCKLef6)gPB35p!!3S7f;+$-*V zLi@RvcJMz3|8txfWHqBGsk^H1pvD7)_NO@SR8)SG=Nt)sVPqgiC}Ngd^8ufHj-v5pobJP$&=p{UNji z;lmZ05I8&%4#Fd2ixP%V7Z{_?ydd}^JY(TT_(Ooj69j~rU|HrIb4N%GbK}laQiDf; zwxOsXfYJ`E@5IiN%p3@7kdPPFAg^IPB8&q*=Pa7Tr=HvF;Z?_L#%X|Ox&rz!ncUj*>&YYse(GLI8 zx4bv|C*Wb=vw?6k!jlLS1QaVOJW=flcI-eg$!IAZ%kA~x4~ha~%WG&Ko0^L&|&Sy z6XAIk5(k8NJPM2_ih_UG5g=GDT)5B{>WE)w>6N1%2E@q&TX4M!Sn6laoM~IbNmRb!hQfBmGWM(h_8Op1gvlvj5f!Vsm8b3mwZn4LWoE%r7ay=uihsTA- z2-@@qZ-77a8G)QW_=iw&-w-ZVqC|0}PMudvmpbQ}Cq9F=x-h?ZM%Z+X`;icA90#>&IP z!^6YFBmP{`iUNY18wP?9giz!k!hs-yWgCk&)=pROTk1+UK6s6%C?pP$R+U@1@pVcv`RW1bNT{e{t_0&6&JXFXw&#{$m!a1&#(v?Bxu z;^ryPNBTur5lRC6paPIv=9Mu1Y13H0lsc}cu3fv@9s(-*^zPl;E-^}VC_wOk=9y>o z&_fT|>lcLzSi|hzjAF%}Bj^RZFP48iWOy*ZKZMFZF`Bkv=|||My*%&YCNy*ZFb<4| zJ8yU=Q0}T#b9Q4U8jO37@~(IWrca-)@4x?Eue|b#KL7l4+fxB}NIv@LBR&53v4)~i>~7V-eA_uRR2 z?Q0QIfgF=y$;62XmTSUfXn%tS4eT|9qT&j@Sk|e?LjrPOeSXdp@`Ip`kc}4sVc(zS z%cZhqr5}Hk9(~j~=PJCcWq17sLikaK4jt@7aD;FU#_G;H@r*gUGXw2rJlT_h@usXP z3d{ZX-*1mK7Wm1QcELL5^?2eCwzMw8(cd{#?b?)Xk#f|e~?+GByDf@L1%c<^A!l8Y^+OM2)b--Q;e zWkPxwPlS6EA2&IgcA>=JeE^h{2@@vRIT3D-2>U)De4}(Q?pWs$+PR-xo_x~zT=vfk zq7wv#Feh>=xNlJqdi3aFeSUtcFHYVS{bAlw2*~wd-r2kk<%D~d{?HEK9w*Go4f7%l zE4TGsaLZkH>5SE&WtJv}mnVwC!WPihdwY0zczAduIsCXi_L%QMGI6yj3Y#|*MnR<) zs;@Y^vtlsDcw4HJV*rc}gib7zgr;CwSbuFP-I1w<3o8}rfCjo)!-h1yo zALop)N>~HoNra1B3QJbBv88O{X z*mlyfZFOurIk9c4W821wIp=+U_s%?bo|*a+YFE|XwZ3b879rMNC*f39W`w{2#14)q z6v$IYsDOsJ|I4)RbP{I(Zkv4hrN7P0+vo=Y*Uv-e%K#|<_24F(wbhJ_I8FVN&Zb!= zkA^E^e&m)7ox;{hN&Q2H3-X3=mv+XXySYk*puUN-R#$~g1!gc_lBh~n*@qWGYs&4D zTx9%#RekMc?tHOmDB8hws@WdbFN&6woclt^5rei9lt4Feg=GaS?ch*O{mMM(yxg00 z;pE$#vC}2V3dOa`n-Y2dSE+xO#7(=Av@RbR?uPWH)>xAJ9BgGg{^a61(jPplb+BgjB^+T&LK+e2uc0p3oW*NX*&`4$?JpL6D)AHTmV$k9D7qwbUG z*w&cIZEBjJlWmM^HPkN$IA^>8tht1YA05fHNGhm3ZH?!zNyG|R&vYAC@r=WF z{;%~)MIOodmu{+JQTB-t`gDI?c)q!PQ9Q*qN^~DuK$tT`yyp5#fUfc(Z_@dYS#&Vf zOrH6CjhYSubMx?gC|?^%B@M9Icu}B=)zQEPX(3nMJnbchB!(b3()R2t#g~Me#BI4i z@Bo%U6SnM(XP~#;Hk1&F5O@H2KzBw|)IH7_6`@m0A6Q>Zcdn!iOR^2SMf~IT^D#Ok z?L3xtj`t!8$8Z!;%%%mu&F8C~nQx?ILRe7>my1(kh={^*JpAuKQ27S>R|r1C!CI99 zEhaUsCdR3l;4V=(@w@`N_m}5Q!Y>etL)iX*F~ney1QqV9eFlEC*6u+EQ9Akch13a{ z!$v?{1@`s+$hRj90xl5@byz6Fh=V&`AEd~&r@0&kx46EH^8B==y}vl4bPsBU5E_94 zqq-0jD2hh`GiOu~QKVL2yP?g6p3#Ny=pcYT;UU0tBY}dqU-jP$M$Q%(VIqtnn8t1= zcszS-?0Th7@U~<00v^VFIdA9qGL*zR$t#2pwh6%=&WSvf)pg6mIOPWwv$Vqm^KA?h z7RM@t36FVz2?8R(>z>;%{ZMxQZuRfAcWn+kANaunqF?i3kl}Z0Ib-bu zzC;pQ!r}QSXoN5#8LI@D!ZzzS7TbK}j)34^2;Vf2d7h+lZE(-0iS}c-)z@3?_-^= zw{t_|B3~oKsh8!}=1m{U!*{o>rJTcj3>4dya>01>)(;?G1L#2)h=?`aPMy#GE#Wr3#ZrFQ|>dL1n!0W&hyYMX5QINLTf1M+X}Ex}H78g=x4z%sP@$ey(CR3VmXb|>!20;D4fT0-|&(n%b+CRgsIn+>UuG`aAHvM@xN^cP3E@bF}4uS z8}%L#`lB19@2m1a_``HEQNjA5hWSOZgBip=8das4u_O6W8OkYah7tDfq2Pw~_2P;Q z;tK(|TpxH1`~_tApS*snf(hFq=cBaTxczAs5aQ{h;$#M;VWG=xC1i4Ev7rN)L2K}{ zLl%U#L#LxSI!IDmRTp}yPG`(ZD4H;7>~sK7@t}N_jRd0;oM>bT$CWg3oVk%IZ-q0;PtSGVx<uA@S1+_Zzu)Elk~^0&@PeqW#1j+-&cfZSG(%-uY0jy1l=Ha2%sT@R3SnTTX)2v)1mR86+V!2qE%CW zM|EIMw_h2I*0`jiFUlKkplt326csFU!VkO`9G|bBcs-}n0Sni;$VyHoT_xB5U z@ob{$Ppl(Bvt=TR@rH@Rwltmf%ZKC|iEmE*;@$`d{jg+ov2~p>yr*9)$t{EH(Bw=ps0ioNIdg zy&{MHylXCdIw-4YgseV&fsA)hBKyB?uNdg|s@sOUitAq|K{d_tzh^d6P3i&$*lx>_SFNv2StoZNo8u z&XZ!u0L6~flqX6U`XH(uGGenpCZQ3ru%Q?ETV~uuB1WjbC*{r&3~fc!r>)>e!N2|G z(e@G?@KOQYP+$iTgVBGey5~bV0`mpbF{FNQ)-^yGq`9=`gsOcJ^?ULV(pJ!a8xPtI zHk>3JXd~RO%cf5PJ$7eBO-eFgY@A$fHpo5Tca>NMQCMo|^Z+2h44sV!&TU$|Xq;em z+?jH|179Jeg#x+l{i;m5*9o;Tm;%v`(AWK(eKKW072O=|VtcbF00k)MxqKJX7*ZGB z`=w|*^Zs$fuA8dy9!G8F1|9MrpdM1}Q`~+8*w8COmwlfQs%Fuj<3tcwvit`0NU0~G zkYAJkvGUUoS49yVR)z1`Di6EfCfK_S*bc==@&H2n#5CR}RucY6)UL_B>T?s)z0$Jd z-1$x^>#)0Jqse90>e70!Me=Ah8KBqjQIo;yp|s|kQL^PKa-S#ywZj(VlxI(tD9`UK zwOy4nq$_8gj~r9z)EXM!$|Z-NyT5=2L$Q(xRb7~h4HTBpe2 z;QYEb^lk;01wB*6?(aM>qz@n%tE*!R@K^iVyMR#)qa_5AX9rakCE&FM34$Y;gjf1# z0$u%Q>q#-&cX$Lov5e4J(0QctK``2z!w=|UeF`$l_aK|vDg|`Ex*%s z;6VDAK_d%py57a=15SQycK-H1qQ^~6@5^L^v}1f&8uUNRE_YdCQPlDH82vD{-DM{6 z<>A$^?ZBlT$ej`X4q`TZ$etHzp^~#WEPkJyd-fS>nH3_LkzqH?U7@F!508!)I2`w$ z60FSZf~rP>c_?Y`1z+MOOZ~`GOYV8iA1OUJ^xRCiG|m#l=Rm(cLA*C2VYw9pA?SDL zvda&c-s@?3fiO4~bmSToa~))?G^284CI-^Ok6UTu{7R~Ld&#;SmSGJR00H2}y8Q5R zck%;?l5WPIFw!-W4eYc2Qi11~?)_Q;;xP(T zeOE2nuR$Oh(Xq63e=I5-`VVvrzO5~mVB-GN_@6Og{n`$`j&|+Y_=_;rJ6!BX1OMtA zd*x6hgcxF!pQ-=eBWqT%<)EOJC*dbZ7=VGn(RmG+kovO1SPp1|`3hzL?N3B5`W0E| zT0$S2-_f)kFWC@O6!wQGaLz?<(+MD{`i6~=y{0Dn`=yvh2f;aqNf(diC0?mNqI|HY z_2o3qYibXR&;dKGQI~fsT*QxBzD)V5E?8~|uj1z(CLSfMty}<2P#cKL=Y9zyC^SRY z%4Nd@JoATdhEAlpE)$8USFS8zNweYhJ*PNF!}P^37fz_cUi;JQF`;dPgkdZ>nvlr$Xv-=*I^uRFi_JZ5#h zI&?L0Pey@ZB$R%^pa6ILZ%B~AO$bP~5JPdm5wQmK8gCAHf1tR(sb)VDWjJ09;} zP>Y}WX?x<9$J+#V?*bL_TkoaMf?T3GrV;!pC870gc`F%LKHW zRf8=q7g>vE=e`A~t;+AG#;R>9d9dU6i9V|ws&>3?k-tr*%4X$}7Lv``*b0!?*K73h z^2(+|ITrvLbKm)l-~y=Lg&3hW11%+LWzFSi6}q%z5iUx2{XHr4D=FruHiRK!qxmUK zc|vFXn}fPjgcW;e0(J-Wwtabc0pN)Q<%TnAkDKHa;0ovp^y*raDgX6q?x45|(fV^d zU`O2eF**w=^L?(n68nvGhny8~HlL`bWkuR?{w{yCFdH^tD+@rnRJ zp%j|pRfah2x)AI8Rx#09Rr_6%?ZBR^=mAaqPJQLpa<(kGO7E@X z(#xsoRCc4v=TTci#6{3utJyZSuWz*VzotHiQ(0D?RQNtmJHYSn`}&}6xF#?G zeg&72#_dh9r=t(@`N(dN-K8n#8-kMX*|*nW9N-B5XG4Wr9_cnp?QjlmEWr6cTMrFh z`8h<0C>mHaYwA`1u+K%PIz2HG~@6yft;|{YaU@WP9 z5;Y`XDX1iytB-sWl2Guwf7sL6L9&WOU3rCwh&b@@WWH39uk#`ON(6n%+`Ss7{Ps|8 z;rPCj&$lb2>h+zK_h&x10+CisE!C1zs zMnOu~=bQUN%cX1Q!(Ro}=N-6($YN!zBCC0hQai00h-8DHw%H5 z;^J126FiArz6vl?P_PpJw2tabtD%uo04;2A%xy~CcUA2u6lswq)?Eq4G7q^ zIGHuc%J#aIx0=+ot48{?PiYYtr08>oHWJ37`0)+06eQqv(ZqlMjdYzJXBU_orF6(Y zmXLK&c|CQ zSFOj8;rT(zQh$S|XLNUvVlk)E?qHPF^zwGJT-y{s^d3X8>$5v9QEt#~``-&RCa)AX zm_sA0sISa$3k%qKOy%qp)9dC{GimQAy-h*REyH)ZJ2vEfwRX|Mg8h~(1}IXQ_==i{ zf`v>D3W%A6YC>o#fD>s3-sY&-z>@xc{IbwqR@u~)Ub4ksMVtPkt1e?~qgJ7keR38K zG%{Z}SC=j#BqlPnHz&nbiOx4f_Dt5fdyZ>o&E1r>Y;R0DeE+DW=FU=T{_SdRa3-66 zFVvM=YniA;_Gq}mtRa+jW&9IaEk+f~Ahret!jhf#TQ53I6Ww1?@Ug){$!@LpQmd=6 zA5meAysEY5X4HI6Sw3DI?Z<~{oYQA7uiwiDt9?s7Pa-kCJkaF^9lhLB(0z3Z?$lK< zMXuX}Rm~c45a2p{&tOJUbUa#MA_>9-j1;vx1toY;W;8DQ)D#|(cJwuyP_#DLHzaVQiEnQ0iDf*$CRo#YZaqe=jvc6;`QlqF<%8<8MzZKuLoO zPh`Bf*U3A4yL3h0;8(c;nNE)(o|j{M7Uxryt+Yr^hut7-ZWq8+91|$Y!1_;oF}X*d z_UnSWP}o)4vBm5_DlCQL>|-G;~I3O?^B$&AHR z7Bepj!yZ(zNzg>V_LCo^6dS<=Jbd;zjwjOrd10sWMI$G+d(STT&ex8(v4f%O3+$W| zy0#rnAE>s5>>6XjpvEVn=7LUdE{$R2SH+rt_PUpqr z#$mC~eNVqwYhLqCXEX??MJF|~?U+f$)UDC`&p^y!PzjLAbg}x3|5WSQ%#=if3tV^X zLW3a*`I0k`r^yQFVWMI+Yy5bPy*HbT@o3pDGRmnPm`)39rv9>dA*2 zphR1ek@&WQarvB@d(r)QIj}8Grk$~9r)nHtUY@*#*C(9LVVe9wr>neBa5B8pKZV$% zTJa*M{q3?Pke8&8$;gZ14SK6VSBpft)UxsIh!-l9JZ1*q8(Cp`9}r>dvh19KT96X9 zUuUo${z=xhmIf!SBvpdjZ~z zY1OI6KfK&)8XnY<#dFfWjk^7{CvxX9M_{eBWCOKvA9E+V4+!q|9Y(dzN%cM0>bR`r zd`vcowzk;3JLT`cM1O~c{;R_P$liW~T|p3w%=wffBMDO8l#Pq$(FmcnxKXz>QIaTP zew41@ZuC!d1pUVKnq8Lcx@IG6Z7@f(tv4gsc%hOQiouRjQmpoJy?;$vB#@>rk;T;| z*kq(|q6>wm^A63Mk+np#b4$>q0LI4b6GD2k*7a+bHnQ$!Y24j{nw-|#e5 z+%;FxyFUcNtuvs4pgFk#GWi`P)j(Qo-Dq6y*C2(U<6fw&)^eRSsu}5|#mf!6L2fSS zt$go&wVEj%QTX1Bbx8EwI?PHKkR)Yz(gc>5Ej>>px%Swul z@bb?z9(192#%n7IAbK5OSnNw!JXmi@5*)F47f@t$=iyIAP&bnO2?`3L+MCRwuA99V ztXxWl&&B7JU>1Y^la(u}Wa<>4#VAU|My8~LY?*c(@XMc-vW4W$k)jWdl!=bGh z>7t5)i(Z-T?6b?+5du$T_r!vGEL1NpXS}bXnhUpj2H&^O*1D|9rjUx@v-D&xwEPtf zuMv=Z@4MADhEk}E9xI4UijGKPYkjST{>H8;ye(Whn2(1CT~QVMV+XB*`x*ZuQ-@6& z7>oUWBj>J@QCWI=x1QV1_o<<*%V2f{nusmP3cX3CxInDy|52CC^twM?xQNi0E0&F{ ztI7}L!2-Mh*n_l#w)Cy;ue!6ev3S%;Ray8K#h0^b55pVT&HsqBO}1Z+Dg7ymZN)(J!N3s92;i5h4G23a7LMBS7xzz6dO*RMJB*^oAXibo}AP&wpx;Q179 zAUk4KK>ul1rNYB0nY4Sn0-uHa-{ZBzAey^6|KqDlg3DmhP}^|Q$bZq_MXWgF=|QXb#pN&A&)~XT zXi@qyJM+qLPZDc`9$sSkPe4kf&N?jwf(x5m%B9few*ay{==n_~vD|!tU2-AtMcqD1 zWO91Q@VWhZzS}Ughkexa_AWjx4mpODvEuXQ#`$K)lGFFl2+GurEPFU~2x=dnySS@; zlc3Hfg-v9x0(BSRf0DEy-`#=ISn@ktIYAuSqW z$N^^Fq@JPxax_(Iru%-QdYu2h1rJMu!HLpqqG-!UCa)UQrJGjd=ZCo2^pB-2-zeqk zA$CzM%f~@XJfajn^)}0>8lPn?{|E`vX~VqXK72fYY>@R>0`VPj5LC$64xVwz{lPfN zLf5F*~R6GT4Gzp-0___zalY7+dF8_qLxfJI)~exN)?O`O%W>##i!%8&5M= zAEi!ofpF+r33<1_@tnfnP`3R93K)7sp|DLp04*AMs>c6juK;SQ%5u)>w(ac7SwN&v z34cwdlLt*NI@m$R_nk27ftKGz-je7b7GSxo7X#t-90AWadH?~sPY2{(q|ct{_DfP1 zd(j3KT1zt14(C5Y5W4vO_xs8WEo@-JOw_iVR-}F9=OIG$Vi_LXVbgb-N1+ls?BVF1Ys2JH5s#)3%TWj1tMZSWuBzBqL~P|Z#({d$A_01xpg0)-QS9`$Qd1D&#xTz9HxvB9-uGM`a4+)Y@{A zWkt6kOoSjtk`5#l@m*dRexa2wzJp#nLS`v;KQOMkxCINu8f&g- z(4#^9zvUpK31%>L-k&_TJt~8-8~Sa$-#fq&_8w%vo@qB^uwYaHTy4ml8)Jl4qp|mB z=AMr0EWg`P&W8hgSGSmGhaQj;L$U%xSByz6^XoMufBaj6Pji!3CHqmb>T8I*5krTl z^-M?f!MyLncUf<*ZVxA%;!dh{UDR5WY_{aHYE`Y`)0=nRd5dLTX?Xb^(oqgG)o3+b z%OkCgdq-N}YkYuubz_haw(FR=;n%^4rwvy*8DLS*?&LkRD*yXFG{%QQ7wXPeA}Xv8 zT}U370S>efKSq>{Ymy#N2xI_W9vKxRwZ2?7tt#MnKW^I~ zEO^s6_N${Gdw@~}C4B9mD*C#JX%L6NpZ(V3Qy`1pgu}M(mt%iQPWE+uDw?F};{^)- z1wWG&z&$E3g))yp1%YHH`7c>t$u4Xk2kV=po7f4F?~a|XYZ_7qMIxHBaKO3(c89WT zI>(vOY$3E2t)5D=?Z)}wMacbkUfNrIoc^sWVyZ zG}_=1B;114fj%g)P^J)p0V%8%LVVCxefPw9>jgYdG}IqE;ykYFniP)>76qEg-=&{* zhB9mIqrv|3`8BzXEq*wyu7S!Z)z>C%t~pycZVhQ;vO-jKt16<#TP+Htdb46Ye-ZPj z+KC=ZF9^uC(vS>w!BdI&Jd`VUnEItwIX0oXRLwrj2NAwy-iMgKcES-unvwqn+DQb0 zFty40gvaNKcNph9L_t4Qr*W(-J^(pV!EzML@JM^N1zgZiQ2iQ7@|vRUngrq(MA@=Z zUO;^YlN2~i37Iq(=trZo<-5);t;JD_o|MTrR~`}R?>RHJENP2vI>Eo>9TE2;o8qB} zF-rRhkT&+qQ8;9*=7iG6&O+kI@a-=5I>^#B56qXg*eWjXVnfjhxXXJwC*0>s1kTP)rWJ7b5g_RPxa|C8Igs;)SiTZ{1CHX z=MgS(WHJ52uG~HpPeFU60nA%k1w?4!d;*X&G=RMo{dl!Mgfjywer@~eUNA0yb~*jn z<*{A>>Ypwj*;Y;#N-zMFs}Y~?HT4sXF(u07yNxG0z>_KiXdFcxEJS}(q%gf3bxrtaYIdS+Gi9O{WgTuozi>Ba|)Z1gYK zX)PE{#G-peGB#W4~R zqUWhn8cbCzh=S=Jcz^}5nQN?yp!!jfCqTBkeNW@_w%fDCN|t0~!(@7$Q*K=LVp*2M zkJo4z1cXhOkHu00_%)8Fq(b=_>rHE!*M}kF(cgI>Jr)=3yUWfE0y`FPlp?C^O4K2! z?V;WEiY@tz^_9s^AJ5zKQBIdQrtrIubk75CM0TT9G-wo?3Xe_SkOpGFlxW#973=XE zeY`I{=8FuV3SBs}noeMORzp8r+UNPsKNGN7r>?Z!NAXq>cx`*m26)5ry`HlGr?{_{ zUm-g{uVS(NPG6TDq0I{aCC!MW&QI`HV$yHPem0fQ7DK2{2%gmNcwF;u zH^aX?z3)yB+D8J;XiY|OwtP(g@$2f6CF|R9IiK#g6MHPL5^AwMxQ8_i+wJrPFPgpW zcs_uEnmA)S`6G&uS7NdEK;Q7y92+VI)3@#32az0F^}itE&7?;?`Zv$dFyGt5PU1(! zux0*!rw)>wQLS@wN24woa@8ho+2-J3@9ARes{N*VhVc(YECq0Vbw6^ zuZbI+{Sx191_A=@=9?`wgoZFC!}?zmAvME{k@JwGfg$Rclu&a#A;VI95h|U-U>}rX zVHAQ6p+v#KjQC~b2Vq*@jW*{C1r88ur~+T(F-<2L%8KVR7R%u;;`{IM{W^JYrgt|@ ze6MT4wLoV3ZGp9NA1VoA3a5w3k{l6*I=s%HuUVI;J3<&m?>{ zr9(EDK0cJ#S^rio-5I!~*=<$7X!oQA#rB>X!Lq49;QDv!V(wOBO{HT}ncCgBagiWO zR5&4CeFE6(J}s6Wv|iK48~%4-$i(A=6QTA zL}|qv-V8VOz7}4!iM{5nm?l?jsKZN;HgFWd4WNfTPpUdyuB`p5~IrO>oL@CcJNZXRoR^lo&s z>-TvpyIUJ`b24XMWwIw#74%+J1V?f?Y?$&^W^;!|(wEPVNI-|I4szA7y-w)KwV?@n zwg!1GREzA-_dO$J3Oi~v!$vh&R`<0DM#%-^ikZW|@p-1&w6+c7>{ApIu?Vy2Dysaf zzOU@tmXDOAY)q>Bx6{fXzc+>dfCMUD^>OL}s9-O{Pc-FoL&OxYrS2I%jub>{X88<7 z&ZI~6Crqp27aez+JLqfa;*9^2#(rg+#Bikz8o=-`FFHso+RLdc{7!O~+!VGZqy6|jEoO*I}rOcX`{ z*dHLd!+vl|u5VR^_Ufp0glcX3p_bWFnS_-U+KiDV%@#Lym6ODrR3}CLw7c=ZgOx8( zI)FXB%PYM3ld5BMzZeQkyaw|f-Ev70IT0>z_?`awKf&g1tFD5mhAtN8S_GE5eH1%j zDHZ&+m*WRo3^(L&4RR}btYUFtVK?Lz*{R!ZS<`QCbrG8V+o-Lsm5UekU1zt42$})B zs;dpK{|~r2AE}PA*hR_ZEN6v7hwDJqNBMyl2035&&*`N^i|VrWDUChJ43dTlUpg|? zgtTTIqpjJ{+LjsmEsheUs>~&7uXpN7Slwm{$s4T*HMZU{`L-tt9#kA|6Or)?C@4KYUl+@u0&-r;~_31nEDdu_xmY^v8S883vXWX zKGWqjg?k8s`fYdX?w<=^&00=>@J_9p1ik{zB%Y4!T z#(XAO#~dFsG_hVap=uBJcQ%@CaW`r`uM~A?>$2uoi7xK--(7GTZcC@|Gy&CmX8=K5 z;Y7CgLD+{GK_tpfRwN>evC`L5xR5^y(ru@;AbfV!S(vlOXBAKR$LP1$RyPOBgdvv} zX-?+F8KhTjn3I}sOOi#Xyo=BH!<83^;;hI1cMqCU%Ej&UPCJFkPTn$CRZf{#C}*2X zI_~9X2?#eoA2PFwf3@}jreTyXDJVv=E`3XyRBMT2%^?~QT^Z35M(YT*zKXxbNMYK3>C9%KjExmaM04yWgvK=RRKEwzF!LNNnmQUoNV1oHl1JOU}aAK5G@8x@JwW$ALY37%I+r$6{L zRY)+B1t9W{o&+sltB4MPr`nX7XwYJ(Z_2I8qfJlJ7gV098(X{7s>vi-t`;3%vfiuN zq83YZNC86OR2T>=Vz5ve$^qw4)7YTBUpzuz?qW4!srU^*!oH#lW>-8>a&~-)$=G^D z8cTIEe(18Q+cEz$+=qf?QsczEU(Y+p%6#%qyexJTH1IHyf7fUz4Pb53xX}K($wC#Y zQl?}zSA3q#cxin+6k@uM_~hw>yh~N_GJqz(3`Vd`mN)$o$`6_-JDnybWR+lDoMy-4S!DQ9lQr6$yvN+3+_i9bnhCI1gPhoFUvg9RF?TStM zf(8p_sw6SkR@Rr}aMy)#~N1`<@UPtZ#8SWum7SvT1Cg$;aEXi!*ikwDodL zFP|9|g3G!ap6^QR=nSP+6Tlr*k2)bv;B8#4T472{_V;|fb;X$%Y1MCc8E0@UKjq#L2S;N9_vt=E4`N zU9L@G>vltI-BZRLuSWHo^pq$TF%wpHwp8n5GO?O|dv(YT_Nne1TGYy2!V&q$o2?#Z9mytcE2X%-s{HLwdB_1n*xH?r8Dd1<#X~ zYHw?odjk@4q~i$1d%N|&UcaYaPB$?{+fC@7BJvnd#t-sGPJ~{`{j|m0U4rWC)=FEq zph-hwNe}0B3*~cC*L2 z$|@(5HVEqv>`arIk1W30KW%HDGx1F&K0{!R^Q8OiDgmCL>n<}+<&VH+9>SYBSxCLrugz1rF&SEm^V7HLk2~V_ zzz8|{I09ZYP<3X5XRWxll}T8K<*!AdkiW2vE8^;8(!_9o=6FV#W zu1VSSMz;5SHZH|s&Z-O6MF|`CPeio*3|X^J7txdgfvF=ZtInKo6RIldKgf-YgMtfa zfCY6daF~gI4r##|$e`ZCVFcdWYPTRzhQm%s#^F?r8k+?bfB$?QL4BRhV5unz*eo21 z=yE`EHa^zcvs*iiU*GRR>%a6st}bMsXio>ff40c!=|u`v$iMDU(O{hhOwEKoL`V8! zAnztZ8!VRN{jl;}Lm~u5;J%M?f4(xeNfKZw0XnH=*@(LCyvN|t7;KeTD;x=7#kH^C zJUaM;L7qp*xuHNS|I^^Pkgat=t$2U7)ZB@yBD32`7c6ql7`+tI7-BZuz58n*ongU5 z#*%lYHEtG=R{nu!Ss`?GVaff0wh;4*@0>!Y^m)GKqsWo+S?fJa^lYZ^BZo^{IVSN} zpcuPo83}%?YdV`*oA_XXtf*uG7^LIL-_3U}u1}-r(Z{;Bc}o2ar}jqn+2iYuubL`# zMVH-+W?-^zyss#?RCbjtK;PrRC zDdUM1JHpf^jWi!Nk8#C*u!mfQov*Q$AHzn8R|oXJ+iKL}4r>k()C=S3u2R;5-R^sd zox|5uc#K1vF2nvz+5Ua>Uaa`l9`Rx_fR0DCArPH^o`_r4Imbpb$2;g9YeXF5n0rU9 z(Yz4+&}@+(;SO7IstkfiTfU1d5N6m69y+E|8jj6F+)(7& zWUjF*t&ztpjlr|3l*Idj(iscKKP7}i;KLuDYW1v=!Y^5bg_5XUdA;9^#C6!;Az829 zJriFaCZ@Wbwu{*tXmi^X_rqr_zvo=;BVhdHyV%5~a&*i7MoV3m{HRw-oZ4lq z)5x`T$Y!a;6bq~7efkE1V&^C%WKVYcv^RuKcS`wwhjI2uQ{7>>=>?SaoQ9?rfee7$Kn)4}by}xu`0i0d5PT)V ze7zevv_E!$MSJbEP35r@j5)zF&10nN^K9a27p0#Bxc;(@lJ}Cp4~2yD4@&N;b2uA6 z0M**_C*{);8jv!c~{xKt+JX%Pm?Z==@a2uUWirrCK zH{dFiw}PLwan2ADo@J@~Hj=Y(hu6{5GOgxJEA5e{$b~kD$bkIa`9L;fXKkXjTP%(3 zz%l&}J2E{K$`(;G4T9#-&m%3W2q5BFNcVSDhSTl|p551l!6>md4g10lp0#9R7O6dr zNO4RL0@aqSOK;S;^sUfh=>w{MqwVf)fI{ZsZ91*2!uRLKD;*0{(lesoeR}K7<@6-V z)}Foyz-ta2uF{}u!)y}RdSN=tew8YId&F3Iq(3MnvF05FVgqmmmc1Es&wYemkoRS> zq2-L&-tx0A06uElPnv0q@dv7q*h+@d#F_PsaFEul$z89kLu{KayRZ3WADNl4GoqU` z&xgy34*P9|^MB!oCkIXXFMI{TeqA%B8CLw&z2rcRfAijwLHC9m&i^0MFgcTT1*RG= zm(`-TOYCU&p~sxZtk3YAiG%4ctps)*(1zjMM zldoz}hHCYO`DM(GnDn_rjHhR%gJ(%53{08v=GytuY}i!6pOL3B88s+%+;E4cAek*} zs19fqPGUanGtyP;B!iA0!G)nr3(+<~f^a1{e)HP*4F`|mrMB2i*9;JmvoM!Ale=o~Xfo;lz;1y$ zPcLkrZ&fFevCB6Ih~A^3oDHje^*I&{U)l&=}v3;KX{h zI@|x2_f9!=Xrxc6!Ox_i4gGD3w6xnrV7UaIHQ$ly{28oMLJY}ryneHXNjn(l$=bLC zZQ@y2W;pj&m;;{j-_L9KyJl?iwQ4%!bMDx>PlAvf!AvnWLAl1{QA3A^FM?-fBC4$x z!crUpZsYLF#cH=j=R?kpt#sw?^k=RQQqD}zkn3??vzX<{LjkjEqe=y;Z@F2Z=_!kM zAYwL8B}sp1ryd1kS znf6G^5O5#zIELDFaWsWCSfGzmJ3Zl6kW7_M7A=`k>SOTvNAP5kbBFc1dGX= zz})LXOz~BIzak(MWsQ%unPy()tb1HDmV46g@M7`@zqq;3A>jjl@Upj@^^4Ke#LsCE zmz8uIIiXevvwkF9Ers+-+IlgP)5w%B0Sc}JW>me`NyEutFW~=+pT$I0#^L z^4$Ilq?LJvW7io=JqoiqoLe&ys<>xw2V6a8fv2Z^XHa2;tu%ePiuf7F4V8L$aiqOW zaxkJl_oB!@t-enWRGn)pmOsbwmW&lTA4*$0!cL%=*90fR1}WXZ33;1<>V=l($A+|7 z=4P1Iu0T*SyNWzPe5V`~xL`(M0~d5BBn5-^C$#SM1v11I`a6)%Ebzyr$=0U^3LyUA z(5J}Ny?DEgWXj7Ma=JX{MnM35x7i_K(QEqSm*>wx8b~8r`p#b>qX=}SVkNROPu9~% z)j9z@zXpqFi;Y!RQ1|p=j>jn#$cTXVx@jD9wp4BR58%4FU6)9ga4>~qy6v2H@Wuendy`M~9+4>raMd}NN5o}%yvGYGztxoF{Zv-p_1PnG~sv~yH zCL=bsR@Y5wwHdv!j9UDy8->48N0+w6>q!)k_7Yb*J@V3cKiTUK?2!dYepf#?DdMtl zp16=oeMvEI7yTcwR8-l1DN?x$whGKCHlXjCVZKEdCs6!f{_0_7qtL@-w!tE4gnqh+ zc0T`(mTBs!75$M=+kgkmmZ7PPE}Aj0(!Rb)oHLdH(0Uq{b1eN$XPYE=Gga&?~-+j=#xBC zW*X4!jt%csEu+-8AWhCF=lUT0StCrN?`}M7ir9=3FFq@Pf$K6{^a3wLj2sxVz=6zz z==aG6HHsmz( zac_+{c@vjPt})tvvY$I2y*ZpulQrF$^$tIhy$xeMLZoHHlQv^h<===u?=!l!U!Cx2 z%9T`W?_Rm$MpK)5zpX2qeU1h0p*?XNIIP3lJrrX5o<^-S*q0GuoIYKf9**9WOr*)e zO)AN~sc|&VJ5k3?sXV)BsUYb)MSyL3_@s7br4@uOANK{r=`|>`)Q+DQ2~>eC#Fd`N zTi2^SQ(72!VVgZD=_8irOfT0fCi9;cN^8F)wV^p`pKR(F3LmQJC1ra=INb7rnl(OF zRt8gfLa|v2GblBD)7*9-L19IZlvoIbuMf_K`P>4E&*mER7y5%2 zg%UjIoI_a!BRnov>6~#pGO73)>cn{t@LpWT35b&Z=B3XnLH-}?OW;KaP8ImvkohE? zYQe8Adl<>es8-msklp6fm-Erb1&`Yl4KAI|avk?$sr`X+Ma?wo_Qh(B!myE`1+H$| z!s)UfpO-VI2jZ#?+VrB66RkopTV*G6YMdF?MH2rZVAlCa|LRg5OAnxa(I0e}v{O@h z^zNLFL1=4}sNg056lmkBnLE$_3|@9HOcJYJ5~@WBnBsVrI*$yRGa!?ZP~0X z9+|96e>NnQ!xH+jQLi~3`(2e&u27-R6-2#Mz1-NVQr+r*viC~*bxB?Fat}8^az(nM zofH8^XdRC(Jqv&{`A(H5@hmFkod46UorQufnc2g~3pBqO%0W$fyNb;iI6v2a+uGiX zEn?_)iVs+Et+Y}T{|dc-u3@oAdy*GgSgvXkO~6ge4cmWkc)wJ%)4y^zsi>*o^XG!e z;;Fw<0%m@(GV8W{(aGvmMQY%M9=gw_dBZei%!H&Bl%&p;Q9#882NgTnR|7dVA6@o| zvZ$Y9#Jcp9S1z^+95rWagKmGm2zEc5RoS_|y3#9gcUKlzm(5bC1*0E z+TJRxjwWaq1%eZTLju9w-7O)wyE_4b1a}P{Jh-#a;1=B7Ex5bu!fhdY$ansI_OtKK z#pw$e<{7$sx_hdttKND`H5RMrq~wO)cK6GoBkA}lGO4bNK6BY(?nTMLaAdsAWW9Ff zQ`k{pKFRLiTJzccZ!v#{JSkstQn$3H8(Y|d|M1&Nq4Frt7myTSM?^6nmR@{lte+~} z{y|P>{S+MUVU<6oWx(|~``%HTeQ~>uFTNIPQq4BoECtuX>?61TTZ4-x={chOM>zxa zT&K9JYmM=u__ZxtJnLxXMa~^VZ;-)%bd*|akvOqi+8J7os$xUCa`s&%yls*h!{(}m z*UfRix91jX>vcM`ul3%e$ulHZpt=f{?}{;fA*&Wcw1HcD(I8|u8>u`hBA0AXtRN~O zubnB9@dd1hmH<_3u}VP{7&*EJ7TR`qB*i7e6FZO}NO!Wz{3A@9Tcx-ka34$$*z`=C zl-Y5b40hF-`s^#38^ESB0>~ZHUkl2Hqxce9m5C+#9#S5=AIyaV;$c?7tK}RJm%HrR)lHt6o(|Z~#}5uX$1PQ6JymaaMc3Q_ zyjnEkcks(EqNP{b726EY2mOqdc(vQnQY z-c!9ztuM#vpeM&x%)+6o?GLyxxs=11IHCI<*Diz!4zci|hY2SJH>B<2YZ@s1ppLWL zj>x$&?a8D)B}u*VPl^n$rJQ z?)6dr9!SAm8YXLPgVd~VYMEml|o)c3G0o=fC|JdU#b;y!tu8wqOtDwqPX=`1)l=jBApXZgY4CYUc*#-sly zL18%m(MS~`{{4hukVg3>u?lA1ne!VCfzpg7Dp1-3S-E92jDI(*jImLHPCb`kwxmg4 zMAdk6L8~2FTQVJOWYR6TL?5zC95G`bjZB*`W$Ialy4Ma2`)Fr5uRv_9xB%~il zwr94}@V&G|6_HZ~cF*eI6~E!t4m5{t|AvbIvS=vY+z_`4fUdE+iV zc=hJ&tickI{-NQ)i6%Q2p~9@fAK^ceJF#2}3UZCw94OWxV$ud8K@ScL>>&2kCu^Ue zZQt43yTkR7WAg-IiuTM5LJ%9uyFen+$`}s$iwnT%#60wwBs9hGC7;49y5X}i+dVx1 zrv9m%)j?^{=2eG@pDEO;&SX&T3{d6-Lq)*-hrr{EshdBc^IcJ6ZhP93QoFEON#Q5D zltsL~JC~m{B(s~T;J{45P=6YxXbOW7fkgL73qJiBdMORmjAn`_veoul29TeBI^Qj^F9Q$;kx}T}pbG-xFbTGE} zMz^DxqOV8jw&x)C`~UFECaK`jV6FVR`Vlk*?b{4+h9j*VPOS;7(UQ zUF?n9h7f1dRF-R+V_;xpN_75<=_Sw9of#ZqQeFNJopIz(B5XeP5rA|jgjyveWsX*OP1l{Zzu z{SQ9JH^U_H-RIvmbYjS3Ns^}3$Pw^@o_x{&MfjJC(ooKquk;R%gpT-%DE**S`CTXy zguLhaM!(uDQZEV%46_QqnMS~EW8E*(pHZ#-a2nv@Vr2Rxbd9>!NDr!(S(^zMXHwnq z*}4|*ZP55mmG={!xwHk*7SXLIwwa&V1cyl2r=={*9j8FR7zG*nmfaJS_4T1UlaM9@ zK&aJWr6^$!M7qzfJT%IFF#m!eGx3iu>#V_;*HPSKCqZ;r>}S+VL{6p63g1$WH{_&# zDH_92Dx#-J(DhW8pW-(V4tI0fX-}<752;fuFtDAA?_|Ck)4LC>I4;4>@R>Pjg+B~Z zwnr20O8c~rVZTN5#-u;0cK9z@2Xxi0fJUQ(DWoby!aCFjtM1xrmd0t4!gLU1K!A)} z19o>Ylu!~hAe;F&L0gu=ImGYVwoD(RhJ2~Mw*iPZ7Yi}tL{f>Uk1m7qBJY3Z_p^O? z(cD#~yY5;qk(T$Noi<=Ap8{$N#qB%I7a28j4Jjz>prfKU#~Tm5Zm)X1sl%c)8lZ? zrd$?<<)ed$fPnF8VB{nDMGPYC_SyeX9W-2ml(A_*k~}bjzt-uH<0Zu^J79=}$!dKU z+#=t%j{vfQ#NmZ4l9*M0_@>_*L9#E_j)|*gIGwjz@U%RqL(`L=)sq4{UY3p+ZYeNQ z)EyfxUM(I;LFvDh@RR?AR?JT^5C#r;o*()R0S5+uG)edcx~~ygWU*I+ddRHLzNeS- zuirx<+MeIagi0}Q!z5^sx5@3Y7|>UDsN$Zl=M3u5_tf|G8~g64M@NhnK$!KwR*0-q zSuZbbcdoY%#nZ;^{ZA(mUU-4XK`t1$L~)YGpOxv6wsi@ekxMsE?N-I{;<(i zEkN}Ss0LQk@gC$*vrY*N7@X=SRD{EASR)J|7$u3 z^>Qt8AYGYLKcUMxOiPH(In2MU0@y4LJ6V?ME2gspz@lAu3UtVtAByV7$~)XJ6ft!_ z<3i7o8)cE8pLU9CAOL4{VwY_yR)}rW?naDeJcAb75S2AHlOYIOK!@h-{|+kW7J0k7 z_CgZafzIy>kkVP*U+mQ3cmXI_m!fIUN=c8NG`(gQi=CkamwfUTPS+LZB*H~bAgR-e z*X8I?5;F)ajTYS#1i=rgc@+hMPg4rQ3*yjpwm1cV4kcc<0y>zV<{E$w zkO3&j){y>pt5D&%jDNVH=`|~}UcqdhScp2Gl{jWovFKIrd6YRS=}DR%DccN)*d z)jaP_m(s+1-ddfHq_D3N{TB|aA?$g}CnmU9e+v+|zF3I)X+Kk}=Db&=N~naZxKQdM zC0z!rG13yMLKncoiA8DAu$nEcN_Zo(?sgp1@{blt_geA5_Qv|4e(m29)K>aLOQTx& z>qHHBXIHZu?4Y~u_n4!>A;Df~m{P47KrIBbM8Qy8M!63dw-3HNLzDjrF8#*>95_P- zI856rx41J&hH4Em&Sw^vZC8@y4LOpxp#pHW)MnA!!GUI7(KNoY`x7%A!b`qW#}@ZBTwj$W*HiDZW zC0|LCV?aN$`rxf--VqTEhBhNDfqb7WDthTt2gtSmmZeI^HyT6>-~SlGZBjygD4Wn) zn@S2FAV--l(?&{Ye6bh2|D=V}RORfBUBg>j}Wj+hUP3AG=by}}0+tSgBDM>VL=6S2ht;t%twdtk zD1cLdW1Hr9seYVj3V?OqWnQ02>T8C_Z=2%X&?hOH7EXQ!3+gEGcBS`g$<(U85cSr$ z7-0rVX(M<10hw70=LbIheOo-aAt7I0ndc!mkddmUxl(#xkn64qAsK&O`o^OOIe4ZoEoB zev2vTidL#xn-hLg{;Nn&0$b8wAx`-*Oe*5}%6amqYlS%NFGYgX`4O&(8f{+?yfi4t zVstp}5?nd=V(Vg^hT>ThoS&pl)*3|GUm7K-6O>#N#|yq7V6Nq(H=0)JySqj)t(3|- zq}Cb`Xgrn+XLKsxdxd!6y)qo?_Ew_JqjF2ap*+Y_yao2E&&roTaP)V2RQ;XYJ3p4b z4Rt|dVmN6P;QJ+@c>`{xZoLQt2VPT}(>?>*noxVRNTo5Hx$CG#yUVrOZ$lS7M>(RE zC$mXS5PA3bVz&DPq4{_)q&!0UuFl@0)kv`R-nx$KxmGz>*iB53>Qu?{2tvmv{~f>N zo$HQbAD{3ykj~k`nc<9-nuy@67|^LeyYB?IP584`x#MQG(+2bsM*$ih-!?O+XK%kb z8R?sImZ>oX1qISD+{UG@%lEdc%+zUO2(lShVCtsPd8PHWt}0$Hdi`D?c$plUAl^?P z!m6cqzA|xk`uj%Oww9Jc?t`;|_vb<0#=H_DPuKhzmGJ)q5BOi`o7D5;WMAZDr4gV> zcdKA~Rfe-Pvo%+ZJQ8F1K2dsK#Z0#}susuRwp1P1<%&n8cYV~eNEe`ey34&J68vwUPnY2uq{p@R~P;XX;G3-{x-=O5-1O(e@Q zUTkd@f(5m8QABy9+w;>=I63ka+FG$=}R#1^SnbgVdTHjcA%LsI3o= za(q4%WJX~kJuay3rK$RhkChKEf4_`#SrqOvyQtjar*V49$ghdv#a&m~^VRvbWR6uU zg(rS=TC_8cS+#lcGqYYH*C|7$&)!?L%vp|Ebxj41D?k!ff^h`rxuW_g&W{_3nh?Xs z`vQPDA&{M?>+-jp^8JPKN>h{_fV`g6vDHJbOk%ujWr4c{x62zSiV*wirV8oNsY*88 z>YF^}ll~%;7!Qwj)5-#cnOwC9Y^pP>Uq=CHBq zkmzwSNS(2DP|sm;M2!FUHQ&AVqir(Rg9H{KQHlIXKixQr;7n;pkUFh522PQ_`UEFR zmvVthxo29TOH{6Qt7So)qhXZA=m!6Exz;L*ur&4VK|mtj;@Oo>T+MYfwB?p35I4$r z)o!H3$QIJ?q712WzY-ZY3tKEDE7q?o1uLP8bjMSf%gq$UYcFwqS7~^j=!y>6{}$@c z*JxfL&yyOUUe+cbEw~W`eDZFOu7G+hj40d{hpx9jfj~U)V3ep{7Atfw%4gT11PkPl zeYjGQNY?)Y`&sni!AioSVe4Iy4BinpAFIHVRL5p-l*IN{G$Ex?H@w9#u0;>xLG^fE zn|Hp&RC*ggefA}fI`h}zjN{n7r;33?p z5!1}!t>H4muT%E} zqRRM7wRlQU*KQ{sv`?HA)Z5XdG*s1|mK2-w{@@kn1-JD_;tQ7Y<(FUUz3u1tJ`YRC zumr$%-X*%ot>qtcqTCZnNZ{kpB+G={oX$<$6I;o)CC%iELtCuboJ4Z)3E2L?O|UM4 zJRty#F5O#n{VLdxS?Sd3tpEgep$qnztS*AtadPn9T0v?27;9Y35=|$e- zZ1s>*5K6g-4{CHIWd1x`I!O#DBx8}>51DU&48^)h{+6|mN5qaeoXLNJc;jiV2a#H% zP!3;Gx#g*{AkHTTXgAqaj{yKjqLNuv+o%VfdY6>4gfGPE`}MbUgsewJHh)XP6|A-( z3Gq}EvOy&)#!t0lA%S!=obIX{DHw~V<`%-OIrYVJ5M6%%)uRxOmfq2u)prwwy;j$g zV4qZxo>Rs}AGQ*k23iAbzsI%S?%L|FKQ7RNB%gSVN^exZmo6?aPKi;ktC?Z>cVZ(W!!?+6!T>*?8s;r?VQq_h;2EE{DsiDj3Z0S{1n}ufxvYloOMg%Yv@Mv+3)h1YywR z=~6!7QSf2-wc3?#hOW}mKEPdIpRh%X46R5vQx+uxN8z?dDDm_8>`Q-F6d5|fWL}KN zO?(?f9-nVoiAx!ohTjd2PW9LBxT{E*u}mD*WQ%)L>?gNa9Z`5V(#gr-0i^-pq~JK& zP0qWK*?K8Y?M{O~rm=}USn4nKhvtOP_S?|-CCBrTd=vMIwR1y3cHc6mZf+(+4x}wP zchxmdr78rZz7%D`>H5gxT&ApHV9j3^$L}+(7rr*t!6^^(hqp~N7xzTHn0-QQVejvh zFkxNXV=jMG*>M^Fqeyg@1D?9yTXUyJ5K65NnXAGyMCIb>9Klii1M z&v9zDE$*&z=-HZcgSsA$HmkuAmg*C)Y$FPuVlaEZbGBB@qTso=+~7+C(a481Py)-& zK#JZ*IOMQf53Nw;(imCQU{hAp28Zd_XY(i>6WmQ9pel;2&$5+$&<#A6ySp|@^n3z(a29Ss8i_w0GJnX6xqzi zt}nq z56bS_Vm+GXaz<^faa6(l-`#tKI*m1ao~POgv>=IWPxFII`}-^H3IsJChq5|@m%!9r z{<8L$SwASv|Deji6b-F^xt^4mowkY&K-@S7@Sdw+RT|Jh+wNfS)Fi+ z`}$t~8oGY{Z`~8QIwH{pz?Afv9ql9L&+mviho} z(*n$Ek6_jvkb*C^`+73%uDf zM-g`>yeEjvxDfr*W3`Ew9AXAPzgp&ODX-0-S=>q9S(EcoW(CXEMog*V1BVQ5!PmKm z>$W(dpKuH+;@S<0O%R*4!EZkwWk9R9f7sgQbW4Pmvwj=2W%IKK?cXI+tJcnW5H8;p zjNufO?LBy9Gu1jAR)hxBR2{1nXZZZW=i)C{C}_>|4=tnztpX0TSwgi3vif4Dm{Po; zn`9N4nd>6xq18vRTA#DJ9yRrxotm3c{ol?KQzZjs`Dzqa=u!{I4y{_|m4_pDlCDIV zxmpOt#j~SQ4ECVa+?nCogKrOHO|?!KV&ByR>&DW{Rb1z0Mkh36A_=_aVt(E{9JOJ+ z4qZ~r_oehOenhu$+Rm%A*#GX}Ycq2uSX9 zli>k^kD5>V-Hd>zC8aH2LQ9uN}( zf0(Te+8-xHP587VlFkFrf~o%=D!-~CT-jA0Y;@nCg(W(M@&OFtC!Sl#uyI@G2QVGLJS~9kosnps8n72465%wiF3WJRQS-x{N zcW6YrgHZ{tAYflj-?aUcdgcu#Vb4!TNo9Y7+tK##imH*Lf5uCx$a1D?w9-JIUXEuR zN^HyYYeV0=rX@bRsNSD4(CW7=vANP)xFfIhB&NcJHVOLnYiqe zL=s47CV)k2(r5gxhiiM|gw+}NR^K%3?Oj7w^D*8N`Q z{^;(SG7myU4s&`1m@{dGqwLK43H6?bEBdWho%5W?yVK&qq--4;nzZVH72t~glJ=|^ z1TRv!g1thk6NKx|Nyo9RpJn*-IZcc2R(1rU{Cr;`eO?ekBnkB=H53w?<>wL#%c;I+ zPM}n@lx2c%lq-P5D;h{_$X5r)3oJoFD~J4B8}=!P)$#L_`CIDXIdUYPToZor!`#k| zV%#(M%c{q;HL5S6;U|eOre!ImaKN=Z64c` zM#2=P{dsxmTF;q>bCwm2_xu*S1`WO#`^f2uk(>oWg_zpPaBv2iCV; zfa5$@;kAs#bd`xWwEz>^!nE6f$LOiP+MGxGKMV1ogda?lhSA7-Anba>JE$M385>I- zD`H?JVETTAa_Dvw9rZ->3tUANJiDXE$-@SDRMz!DqnjWZTTxP9JPeAiKAr@Ay9#@^MJ~`!U*ZN!J>}v^G zP~pQl87m#rS8M_AGjgvfkMEHT*~K##AcOb+2Tj`Nqk51aJt`4{;9V9t z@YUq=TKd)eG_Gmj-=|f!mPgW&fstbpi$B^wULhHvZlEPGuK*d z@w=z+c&Ik*;JRcb?v6R%xK@TP7lfV=?3>iiyqqfJ8?>KP^ak&B)7B9i4Q*EC_H;6w`QhJG>53y> z-(4ycII;Wd!<=IWX`c09O%^IeP1=E5+v3peWtZlx|N2#W(dn~+*x($=HPO2b3Zhgd zT>v7(!Qqh_i&q=Lf6c12Z-JCdXGrLsYoHZOP zz9cdC6WILXvbh2}`C9IxE8VLv2{ar3)EN4KzdnYF*Ib{4oZSu=TPxMdXh-T%!8ZQfp-IxY;!j0j%yO~U6+ z#OHM2^2=cDUPxx3JYRN{S7y(|6#3cqBD5=rdvk2hMZ%~t7J5cv+dV9ECTXu)cKEH0 z!5R>!fREq>6X4f>?9E9-bjy~o4@A$4BFwOqbVC{l71g9#^lL>@FLMxyR4XDg0h@1+^O98Tm7owxRqDer3rOV8o5j_L?uYUuoa6wV9D3A=zzvfNTUXU|3@LDPSZRs=77HR`#j5FXHgN{f@Vbc?oh6&WB53r+CkUDE18w9Nt zJpu4AJ4(OB+vSD{LWFmBOjlcT1_1^~L|=mMnBJ0j-mK3#)_mJ&rukt8u$PZ__Azzd z?WV2IH6xt=0?7YlDy>#>6^Pe@px3LATm&WJC+&Qk_R$|bZV?l`giNK+tX2Xnk^KTg z5ZY%2^?epxji72R;o38|>Xy9l;LbV`IxAe!EF}^AH}Q|PZ#&8eB3j%748|u~8o@bG znO??e{-Fbyhf|8QTT;mGI49f8550%T+0O?NGuvObR>o zUF|7`o)8vNSZDbji9YxH>mvl9xjq7Kpt<|K*XFu#VLDA-o6AZ!VnOsobP+_zM(-q9 zsai9)t z^Q>S_iRPC(g~}sd?Yb*HY$&O6;8_~YzreHc`xM9I7zwcde6P_ZSn)yoJ`5O8e@w%l z8x)!EU>fH9^^S-pw2<#&Ui+bz>fd>9IOiye=NKl#J`Z%`IOI+ECt#2R?2*NHM>;F^ zIxI2Y#-{q~fr&cTC0OTcPdhI=5nakWMU!;JngjI4Kl=vQ7diLb!FLI~b}hNzFfZP@ z#~U5-_I0kvo1C$yk|pY+D)r%-R|Wl7nv?J^_fJ<0=b9v(Orq959(zRX)1~&9k1$_* zW3TY#paQS*Q+_{symdGk9;naXn^@11P|UnBV*FM~T1#TWo4D7Uu-0-%wGfG`4ygl0 z@Zz6<`S+s%aXj}QN&G9dA)xwV{ErXu|F1t|49BEx6Jv^f8Q>NSHTkHYSd~3${I3p~ z+Z>U$@8A#O_VPAo@~=M&bHuPrPWmX<668gscAUF-Z!o(S=F(z7-_DUI!zcLs{teQ zi~JhaIi>R9+9!|i#ZJXFHI`NREU*i$oUHTL0Zwxy7L%H^*1^*cFDDr&$x){G{V!N1 z@BtAHKT~Djk$xaltdgh1v9hB4%HI&n=rnI-q7oBUwxW!YEmQSQOp1o=IDs)lGepyT zlqE_o_@jApisuV>V=K+p%*y~b?fgaWQg7|HZTv}K{AqdovhA{c&vVW78J40It;I3T zmSl{80rkBQnp^syzR-)CIwGma1UItCPm!VCj+!?e$3&~`HNN&Z*Vn8hDLJh;aUpn% z(5+@ozAVB=*jCWiy7HagzyDxnHZ?1}p%QjW9N(0c{LL;!Gw~}F5vx@D7G^DvPKw81 zVIa?T?6dC|MAEKY(a(rVZlPhn8(R7HW&6=~j-Y)B-cl&TD!fNcr(eOwL5UaW_U&WT z?|QMTGmLOKhR+%AonzW8jct6l&x<(lR}5C<=YUcUNnpd-z>u9SfnB@&Kz>>j6cz~H zGU=8{$#QIbu$+_^KEn4sjZ7729z*j2?(E>*X+iQT56b=w{x?3BeQjH=7Hb|m88WOu zxAkl^`LLRiqEED$KDqO=D_DX{6RvJc6P$kFKLX7G^jZ}d=o<0Qs<3?TkmBaCw@sjr z^$-P?VH=XAWWAiTBosHDzG6bY=5X3Eo;^W5^l}ljCBD=i_t(RCFG1mK{*adHSf_Mv zFQTnvj;gi&ZhJgNeb@H5#3HRATZPiq*zqqx!lQiB)UX_Z-!p z#VKMqtVe9U4E^2YO(>aN8*mL@Xjj(;j1JAPLfl#xI*CWcT{-AQ(x}_7htQbvy=xZ0RBlVW zDQ3jZX8hK!p;H@5)&0!uRyV&FvQ5lN^f|-xp9(WFTf65Jl6#lG)s2B`NQf`D5Fr&h zoyRk_`zR1?)>!8QgDKKgH@&-!1#Poy{kMyF$WN!Buj&a^I`!k$-4xZSS5x;u8p~To zU4#Bf&DqJ)Z)(-|A+2VFH*8#l!S;^}X3bP-GffNDeP-29IwKxcea2#k#{MV zAN1C&_xH>|LN@A8MAhOut;y7@MqI|_3*!hQt6znywl8Bi{9eTsK=fZQo3!61)#uRY;+Lt%kn z{E*P1sk>x#vy=WQv_6X|>Z=%5IYIj@B+@bRcwkPExovVfR5Hjnm+{1+$k$&4Vnq7t z?FVwQY(=yAlKh8*y38oxRFuL_dR2{^B~lXd{_2PsZJ)8j9W?wvEI>SxkHJOP<;8A0 z8A^7B4AHv`{I+Y)o*xwLq+wZ7zJtSr3rWBZXY~{H^FNCj9@W?0Nidyir^J18Q(_z`H&os}QcY6eV-YePHG_xg?y?IN!F?)`XeY}3=9I1ZadMg=JETO9 z{FL4Vqn8WrUL)ycQ;jBljYisYgcNM$tKpC>c2j;eY zBtNQ&2r3o0l}I3c$`J^Z$nq?Z8B}TPuCkqI)yT|HgTHs*n@S4O^L6f zlk8ZXcbh+8*-+6TJVptbixi4}{&|V1=RpzJ07Zj*WdMDg39XBR`ndaHy&~gBtWQ;U z)lA1B_WJDRDkY(kfS}6@^1vdzHQ!rRH*_~JewKyUdQ^d*+O;2(+T&1-GZQt^uPVf3 zPwa1!5GbX5zdHkmp-2y0)d&vJ^8Wk z-Lc8|9EJEZ2V@^H*Y@=Mj!Uzgq4LPnFnY@>(7TtzuqAH)1+r>tbc0n%MI^cD#hawc zxcAuNSt88srFQu4s~PkWcjJsw`Z=?5C?o*%+|xvP=WzU{hwrk-L(5Ob2pHAuKu2)M zRCN8pszDd=M{_|}{^jH^NLw$RK^>S6Q?kVlv zep{AvfhIettD7}AOuv_3);&8o<9>u`Kr2MU96FY&`5yfBb`7oCAgRHA|Neu2te2P+4|tA5WAS4O0|%t&j`N5t>hCKrxU{6u8F zA}ewJjOGRYSsL8Y@}7z-*YD`#gclvzbxfP7($&erj*r0SB8QP8ntkI1gFIbToc*N) zA?5V3HR!ww7@@D$6-`0L^Y=Jd76vumKQ8~6Gb|Z7R9(wt(d`XCG4?)a)tod(e6NgI zpQ~ZAIdPPpXAQyZd-4;=uds z5rbREP}c@V{fq}wB;9q}RUAv?XO~RwGqzIXk2yXV;_sKPvG1vnLTNjaj-CjGia<~( zY(L`dS92-D6AH4tv64p~-mK<8U7h&TW+3W{baFxn!kB$QDBMx7uqsNo2wI;}8#ZIS z3GD-SFtbpD9P4>Iei32+NPxbL-f6@ik?Rpkg!PGowa!E?MWvHnlgdtps`=`V!HV4= zN0XYS9Dmb-Teyrl(13PV14^rZ%xN2{eZyBhIS@_jf(Bj2bdFM}SYaf)$KsAS*6N__ z7mD&2=_u9kZv@=>va$EPC#2g+h{GOogq>Qyel+5L{CMEO+k5eO@aPA{kbe;fuIz*Q z{9vQ>H?dCQ(63qSTfy$!N=dd*Exg404IkW&4D_qeH(NwP{n2yKztmP6k=LU14pUJ< zr9?^RtRJ%Cej6W0TYh`>0nK`0xY|f>J#@DttJ!bP{EY>6G3_OdUELe*K4s>7Xgx4O z+s;kZQwLjJ(z=0sTrJxCnD+s;55j_m)0g!k0>5z(tbPj0$E;)>!Ehl($~IrLzo{a0 zA)Du6VI5&9$Dm?MtThR5)UVZ%F4$4v32^JxOUTy;IRjiT=;KY72WZ!?r}K4_Txo5# zj;bGc6QUc}qsrfC)#17oqAl^twsS*2=_8ze>^bNA@OE(d4Mo8l;XlU({%wRC^Usq= z4Mr4L&<_q*)axbnekE^`YDDLgP_`cXZLp`ut1|jJLJQg_cQ>ech-<(D9t1pNZ)6Ww z4KYl6BV85cw0q`%M1JN^3VO9 zJLXOhqOcqN{by9A9FK~da^HY%PV~^&aPp$Xq`eTI<=43Zgf1GTe6G8{fhlFZwJo^6 zS4H2LKuUJ159$^zoACBz4XSj{vwS!u5cr^m}LmN;dbWMaSe3#${VQme^fyH2Cu>mAM$?y#4{^@Cifaw3>y>}No(p9?`jqw~ znt`B+?d$2c%RWZx`iz1n+3gl=w6FkAEZabnF|p6jkdHf$LJM6CZ4|?&9FSv>S>%Dv zx|x}IIekg#vBncPI}jYD`wM5n#_o=62^Jfe8x_ z10sT?WU(ZclBHSeNskJe*yOhA8S5d~>LBuw@*R6@y@^st!{eSd;(1AkIoIADt_UKj zd`Nf4-y}a=W@zWfBbz_$KTha>6o^CR_CuYUuRlNxaP|yGcn3dCX@5HT&>{ImZB5$q zg7CTvqHkVliB>(DlEHod+j}o&>=(HBX$JX|tdR`nKoE}^X%Ac|CLC!XdV$otN=vv( z!}lzY^oF;jr$aC|rO^6ECxW=_cNjH6{*D~GUzUDw>>-I3H3CQik)Mt~nFKLK5bYnn zf}vNvG2U~oG)r;U3H;q>${I)c6#n=e@`ZHWI-1uJS@A~y#mq=K+MDqRyduj0tU`23 zU=ae1iu@GVpCo_105n~qSKa8}= in_shape[1]: + raise ValueError("input index should be in range (0, {}), but got {}".format(in_shape[1], in_idx)) + if out_idx >= out_shape[1]: + raise ValueError("output index should be in range (0, {}), but got {}".format(out_shape[1], out_idx)) + + +class Grad(nn.Cell): + """ + Computes and returns the gradients of the specified column of outputs with respect to the specified column of + inputs. + + Args: + model (Cell): a function or network that takes Tensor inputs. + argnum (int): specifies which input the output takes the first derivative of. Default: ``0``. + + Inputs: + - **x** (list) - The input is variable-length argument. The first input is a 2D network inputs (Tensor), + the last three inputs are column index of input (int), column index of output (int) and output of + network (Tensor). + + Outputs: + Tensor. The gradients of the specified column of outputs with respect to the specified column of + inputs. + + Raises: + TypeError: If the type of `argnum` is not int. + + Supported Platforms: + ``Ascend`` + + Examples: + >>> import numpy as np + >>> from mindspore import nn, Tensor + >>> from mindelec.operators import Grad + ... + >>> class Net(nn.Cell): + ... def __init__(self): + ... super(Net, self).__init__() + ... def construct(self, x): + ... return x * x + ... + >>> x = Tensor(np.array([[1.0, -2.0], [-3.0, 4.0]]).astype(np.float32)) + >>> net = Net() + >>> out = net(x) + >>> grad = Grad(net) + >>> print(grad(x, 0, 0, out).asnumpy()) + [[ 2.] + [-6.]] + """ + def __init__(self, model, argnum=0): + super(Grad, self).__init__() + check_mode("Grad") + if not isinstance(model, nn.Cell) and not isfunction(model): + raise TypeError("The type of model should be a function or network, but got {}".format(type(model))) + self.model = model + if isinstance(argnum, bool) or not isinstance(argnum, int): + raise TypeError("The type of argnum should be int, but get {}".format(type(argnum))) + self.argnum = argnum + self.grad = ops.GradOperation(get_all=True, sens_param=True) + self.gather = ops.Gather() + self.cast = ops.Cast() + self.dtype = ops.DType() + + def construct(self, *x): + """ + construct + """ + x = _transfer_tensor_to_tuple(x) + input_idx, output_idx, net_out = x[-3], x[-2], x[-1] + net_in = x[:-3] + _check_type(net_in[0], net_out, input_idx, output_idx) + if net_out is None: + net_out = self.model(*net_in) + net_out = _transfer_tensor_to_tuple(net_out)[0] + _check_dimension(net_in[self.argnum].shape, net_out.shape, input_idx, output_idx) + batch_size, out_chanel = net_out.shape + sens = _generate_sens(batch_size, out_chanel, output_idx) + gradient_function = self.grad(self.model) + sens = self.cast(sens, self.dtype(net_out)) + gradient = gradient_function(*net_in, sens) + if input_idx is None: + output = gradient[self.argnum] + else: + out_indices = _generate_indices(input_idx) + output = self.gather(gradient[self.argnum], out_indices, 1) + return output + + +class SecondOrderGrad(nn.Cell): + """ + Computes and returns the second order gradients of the specified column of outputs with respect to the specified + column of inputs. + + Args: + model (Cell): a function or network that takes a single Tensor input and returns a single Tensor. + input_idx1 (int): specifies the column index of input to take the first derivative, + takes values in [0, model input size - 1]. + input_idx2 (int): specifies the column index of input to take the second derivative, + takes values in [0, model input size - 1]. + output_idx (int): specifies the column index of output, takes values in [0, model output size - 1]. + + Inputs: + - **input** - The input of given function or network `model`. + + Outputs: + Tensor. + + Raises: + TypeError: If the type of `input_idx1`, `input_idx2` or `output_idx` is not int. + + Supported Platforms: + ``Ascend`` + + Examples: + >>> import numpy as np + >>> from mindspore import nn, Tensor + >>> from mindelec.operators import SecondOrderGrad + >>> class Net(nn.Cell): + ... def __init__(self): + ... super(Net, self).__init__() + ... + ... def construct(self, x): + ... return x * x * x + >>> x = Tensor(np.array([[1.0, -2.0], [-3.0, 4.0]]).astype(np.float32)) + >>> net = Net() + >>> out = net(x) + >>> grad = SecondOrderGrad(net, 0, 0, 0) + >>> print(grad(x).asnumpy()) + [[ 6.] + [-18.]] + """ + def __init__(self, model, input_idx1, input_idx2, output_idx): + super(SecondOrderGrad, self).__init__() + check_mode("SecondOrderGrad") + if not isinstance(model, nn.Cell) and not isfunction(model): + raise TypeError("The type of model should be a function or network, but got {}".format(type(model))) + if isinstance(input_idx1, bool) or not isinstance(input_idx1, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(input_idx1))) + if isinstance(input_idx2, bool) or not isinstance(input_idx2, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(input_idx2))) + if isinstance(output_idx, bool) or not isinstance(output_idx, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(output_idx))) + self.jac1 = _FirstOrderGrad(model, input_idx=input_idx1, output_idx=output_idx) + self.jac2 = _FirstOrderGrad(self.jac1, input_idx=input_idx2, output_idx=0) + + def construct(self, x): + hes = self.jac2(x) + return hes + +class ThirdOrderGrad(nn.Cell): + """ + Computes and returns the Third order gradients of the specified column of outputs with respect to the specified + column of inputs. + """ + def __init__(self, model, input_idx1, input_idx2, output_idx): + super(ThirdOrderGrad, self).__init__() + check_mode("SecondOrderGrad") + if not isinstance(model, nn.Cell) and not isfunction(model): + raise TypeError("The type of model should be a function or network, but got {}".format(type(model))) + if isinstance(input_idx1, bool) or not isinstance(input_idx1, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(input_idx1))) + if isinstance(input_idx2, bool) or not isinstance(input_idx2, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(input_idx2))) + if isinstance(output_idx, bool) or not isinstance(output_idx, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(output_idx))) + self.jac1 = _FirstOrderGrad(model, input_idx=input_idx1, output_idx=output_idx) + self.jac2 = _FirstOrderGrad(self.jac1, input_idx=input_idx2, output_idx=0) + self.jac3 = _FirstOrderGrad(self.jac2, input_idx=input_idx2, output_idx=0) + + def construct(self, x): + hes = self.jac3(x) + return hes + +class ForthOrderGrad(nn.Cell): + """ + Computes and returns the Third order gradients of the specified column of outputs with respect to the specified + column of inputs. + """ + def __init__(self, model, input_idx1, input_idx2, output_idx): + super(ForthOrderGrad, self).__init__() + check_mode("SecondOrderGrad") + if not isinstance(model, nn.Cell) and not isfunction(model): + raise TypeError("The type of model should be a function or network, but got {}".format(type(model))) + if isinstance(input_idx1, bool) or not isinstance(input_idx1, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(input_idx1))) + if isinstance(input_idx2, bool) or not isinstance(input_idx2, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(input_idx2))) + if isinstance(output_idx, bool) or not isinstance(output_idx, int): + raise TypeError("The type of input_idx1 should be int, but got {}".format(type(output_idx))) + self.jac1 = _FirstOrderGrad(model, input_idx=input_idx1, output_idx=output_idx) + self.jac2 = _FirstOrderGrad(self.jac1, input_idx=input_idx2, output_idx=0) + self.jac3 = _FirstOrderGrad(self.jac2, input_idx=input_idx2, output_idx=0) + self.jac4 = _FirstOrderGrad(self.jac3, input_idx=input_idx2, output_idx=0) + + def construct(self, x): + hes = self.jac4(x) + return hes + + +class Jacobian(nn.Cell): + r""" + Computes the Jacobian of a given function or network. + + Note: + The output of the given function or network should be a single Tensor. + + Args: + net (Union[function, Cell]): a function or network that takes Tensor inputs. + arg_nums (int): specifies which input the output takes the first derivative of. + out_idx (int): specifies which output to take the first derivative of. + + Inputs: + - **x** (Tensor) - The inputs of the function or network `net`. + + Outputs: + Tensor or tuple of Tensors. If `arg_nums` is int, output will be a Tensor whose shape is the shape of + specified output * the shape of specified input. If `arg_nums` is None, output will be a tuple of Tensors + where output[i] will contain the Jacobian of the specified output and ith input and will have as size the + concatenation of the sizes of the corresponding output and the corresponding input. + + Raises: + TypeError: if the type of `arg_nums` or `out_idx` is not int. + + Supported Platforms: + ``Ascend`` + + Examples: + >>> import numpy as np + >>> from mindelec.operators import Jacobian + >>> from mindspore import Tensor + >>> def func(x, y): + >>> return (x * x * x * y + 3 * y * y * x).sum() + ... + >>> a = Tensor(np.array([[1, 3], [5, 9], [8, 2]], np.float32)) + >>> b = Tensor(np.array([[4, 6], [7, 2], [2, 1]], np.float32)) + >>> jac = Jacobian(func, 0, 0) + >>> output = jac(a, b) + >>> print(output.shape) + (3, 2) + """ + def __init__(self, net, argnums=0, out_idx=0): + super(Jacobian, self).__init__() + if not (isinstance(argnums, int) or argnums is None) or not isinstance(out_idx, int): + raise TypeError("The type of argnums should be int or None and out_idx should be int.") + self.net = net + self.argnums = argnums + self.out_idx = out_idx + self.grad_op = ops.GradOperation(get_all=True, sens_param=True) + self.eye = ops.Eye() + self.concat = ops.Concat() + self.reshape = ops.Reshape() + self.tuple_len = ops.tuple_len + self._merge_output = _MergeOutput() + self._generate_multi_sens = _GenerateMultiSens() + + def construct(self, *x): + """ + forward + + Args: + inputs (tuple): input tensor. + """ + net_out = _transfer_tensor_to_tuple(self.net(*x)) + net_out_target = net_out[self.out_idx] + grad_fn = self.grad_op(self.net) + input_len = self.tuple_len(x) + + identity_matrix = self.eye(net_out_target.size, net_out_target.size, mstype.float32) + identity_matrix = ops.Split(0, net_out_target.size)(identity_matrix) + if self.argnums is None: + out_tmp = [()] * input_len + for line in identity_matrix: + sens = self.reshape(line, net_out_target.shape) + grad_wrt_output = self._generate_multi_sens(self.out_idx, net_out, sens) + grad = grad_fn(*x, grad_wrt_output) + out_tmp = self._merge_output(out_tmp, grad, input_len) + output = () + for i in range(input_len): + out_tmp[i] = self.concat(out_tmp[i]) + output = output + (self.reshape(out_tmp[i], net_out_target.shape + x[i].shape),) + return output + output = () + for line in identity_matrix: + sens = self.reshape(line, net_out_target.shape) + grad_wrt_output = self._generate_multi_sens(self.out_idx, net_out, sens) + grad = grad_fn(*x, grad_wrt_output) + output = output + (grad[self.argnums],) + output = self.concat(output) + return self.reshape(output, net_out_target.shape + x[self.argnums].shape) + + +class Hessian(nn.Cell): + r""" + Computes the Hessian of a given function or network. + + Note: + The output of the given function or network should be a single Tensor. + + Args: + net (Union[function, Cell]): a function or network that takes Tensor inputs and returns a single Tensor. + diff1_idx (int): specifies which input the output takes the first derivative of. + diff2_idx (int): specifies which input the output takes the second derivative of. + + Inputs: + - **x** (Tensor) - The inputs of the function or network `net`. + + Outputs: + Tensor, the shape is the shape output * shape of specified input * the shape of specified input. + + Raises: + TypeError: if the type of `diff1_idx` or `diff2_idx` is not int. + + Supported Platforms: + ``Ascend`` + + Examples: + >>> import numpy as np + >>> from mindelec.operators import Hessian + >>> from mindspore import Tensor + >>> def func(x, y): + >>> return (x * x * x * y + 3 * y * y * x).sum() + >>> a = Tensor(np.array([[1, 3], [5, 9], [8, 2]], np.float32)) + >>> b = Tensor(np.array([[4, 6], [7, 2], [2, 1]], np.float32)) + >>> hes = Hessian(func, 0, 0) + >>> output = hes(a, b) + >>> print(output.shape) + (3, 2, 3, 2) + """ + def __init__(self, net, diff1_idx, diff2_idx, out_idx=0): + super(Hessian, self).__init__() + if not isinstance(diff1_idx, int) or not (isinstance(diff2_idx, int) or diff2_idx is None): + raise TypeError("The type of diff1 should be int and diff2 should be int or None.") + self.jac1 = Jacobian(net, argnums=None, out_idx=out_idx) + self.jac2 = Jacobian(self.jac1, argnums=diff2_idx, out_idx=diff1_idx) + + def construct(self, *x): + return self.jac2(*x) + + +def jacobian(func, inputs): + r""" + Function that computes the Jacobian of a given function or network. + + Parameters: + func: a function or network that takes Tensor inputs. + inputs: The inputs of the function or network `net`. + """ + inputs = _transfer_tensor_to_tuple(inputs) + func_out = _transfer_tensor_to_tuple(func(*inputs)) + output = () + for i in range(len(func_out)): + jac = Jacobian(func, argnums=None, out_idx=i) + output = output + _transfer_tensor_to_tuple(jac(*inputs)) + return output + + +def hessian(func, inputs): + r""" + Function that computes the Hessian of a given function or network. + + Parameters: + func: a function or network that takes Tensor inputs. + inputs: The inputs of the function or network `net`. + """ + inputs = _transfer_tensor_to_tuple(inputs) + func_out = _transfer_tensor_to_tuple(func(*inputs)) + output = () + for i in range(len(func_out)): + out_tmp = () + for j in range(len(inputs)): + hes = Hessian(func, diff1_idx=j, diff2_idx=None, out_idx=i) + out_tmp = out_tmp + _transfer_tensor_to_tuple(hes(*inputs)) + output = output + out_tmp + return output + + +class _FirstOrderGrad(nn.Cell): + """compute first-order derivative""" + def __init__(self, model, argnums=0, input_idx=None, output_idx=1): + super(_FirstOrderGrad, self).__init__() + self.model = model + self.argnums = argnums + self.input_idx = input_idx + self.output_idx = output_idx + self.grad = ops.GradOperation(get_all=True, sens_param=True) + self.gather = ops.Gather() + self.cast = ops.Cast() + self.dtype = ops.DType() + + def construct(self, *x): + """Defines the computation to be performed""" + x = _transfer_tensor_to_tuple(x) + _check_type(x[self.argnums], None) + net_out = self.model(*x) + net_out = _transfer_tensor_to_tuple(net_out)[0] + _check_dimension(x[self.argnums].shape, net_out.shape, self.input_idx, self.output_idx) + batch_size, out_chanel = net_out.shape + sens = _generate_sens(batch_size, out_chanel, self.output_idx) + gradient_function = self.grad(self.model) + sens = self.cast(sens, self.dtype(net_out)) + gradient = gradient_function(*x, sens) + outout_indices = _generate_indices(self.input_idx) + output = self.gather(gradient[self.argnums], outout_indices, 1) + return output diff --git a/MindFlow/applications/research/NSFNets/src/network.py b/MindFlow/applications/research/NSFNets/src/network.py new file mode 100644 index 000000000..8e1c375d5 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/src/network.py @@ -0,0 +1,213 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Network definitions""" +# pylint: disable=C0103 +import mindspore as ms +from mindspore import nn, ops +from mindspore.common.api import jit +from mindspore.amp import DynamicLossScaler +import numpy as np +from src.derivatives import SecondOrderGrad, Grad + + +# the deep neural network +class neural_net(nn.Cell): + """ + neural_net + """ + def __init__(self, layers, msfloat_type, lb, ub): + super(neural_net, self).__init__() + + # normalize the scope + self.lb = lb + self.ub = ub + + # network parameters + self.depth = len(layers) - 1 + + # set up activation + self.activation = nn.Tanh() + + layer_list = [] + for i in range(self.depth - 1): + layer_list.append(nn.Dense(layers[i], layers[i+1]).to_float(msfloat_type)) + layer_list.append(self.activation.to_float(msfloat_type)) + + layer_list.append(nn.Dense(layers[-2], layers[-1]).to_float(msfloat_type)) + + # deploy layers + self.layers = nn.SequentialCell(layer_list).to_float(msfloat_type) + + @jit + def construct(self, x): + H = 2.0 * (x - self.lb) / (self.ub - self.lb) - 1.0 + out = self.layers(H) + return out + +class VPNSFNets: + """ + VPNSFNets + """ + # Initialize the class + def __init__(self, xb, yb, zb, tb, xi, yi, zi, ti, ub, vb, wb, ui, vi, wi, x_f, y_f, z_f, t_f, layers, Re, \ + Xmin, Xmax, use_npu, msfloat_type, npfloat_type, load_params, second_path): + + self.use_npu = use_npu + + self.msfloat_type = msfloat_type + self.npfloat_type = npfloat_type + + self.Xmin = ms.Tensor(np.array(Xmin, self.npfloat_type)) + self.Xmax = ms.Tensor(np.array(Xmax, self.npfloat_type)) + + # The size of network + self.layers = layers + self.Re = Re + + # Training data + self.x_f = ms.Tensor(np.array(x_f, self.npfloat_type)) + self.y_f = ms.Tensor(np.array(y_f, self.npfloat_type)) + self.z_f = ms.Tensor(np.array(z_f, self.npfloat_type)) + self.t_f = ms.Tensor(np.array(t_f, self.npfloat_type)) + + self.xb = ms.Tensor(np.array(xb, self.npfloat_type)) + self.yb = ms.Tensor(np.array(yb, self.npfloat_type)) + self.zb = ms.Tensor(np.array(zb, self.npfloat_type)) + self.tb = ms.Tensor(np.array(tb, self.npfloat_type)) + + self.xi = ms.Tensor(np.array(xi, self.npfloat_type)) + self.yi = ms.Tensor(np.array(yi, self.npfloat_type)) + self.zi = ms.Tensor(np.array(zi, self.npfloat_type)) + self.ti = ms.Tensor(np.array(ti, self.npfloat_type)) + + self.ub = ms.Tensor(np.array(ub, self.npfloat_type)) + self.vb = ms.Tensor(np.array(vb, self.npfloat_type)) + self.wb = ms.Tensor(np.array(wb, self.npfloat_type)) + + self.ui = ms.Tensor(np.array(ui, self.npfloat_type)) + self.vi = ms.Tensor(np.array(vi, self.npfloat_type)) + self.wi = ms.Tensor(np.array(wi, self.npfloat_type)) + + # Initialize NNs---deep neural networks + self.dnn = neural_net(layers, self.msfloat_type, self.Xmin, self.Xmax) + if load_params: + params_dict = ms.load_checkpoint(f'model/{second_path}/model.ckpt') + ms.load_param_into_net(self.dnn, params_dict) + + self.grad = Grad(self.dnn) + self.hessian_u_xx = SecondOrderGrad(self.dnn, 0, 0, output_idx=0) + self.hessian_u_yy = SecondOrderGrad(self.dnn, 1, 1, output_idx=0) + self.hessian_u_zz = SecondOrderGrad(self.dnn, 2, 2, output_idx=0) + + self.hessian_v_xx = SecondOrderGrad(self.dnn, 0, 0, output_idx=1) + self.hessian_v_yy = SecondOrderGrad(self.dnn, 1, 1, output_idx=1) + self.hessian_v_zz = SecondOrderGrad(self.dnn, 2, 2, output_idx=1) + + self.hessian_w_xx = SecondOrderGrad(self.dnn, 0, 0, output_idx=2) + self.hessian_w_yy = SecondOrderGrad(self.dnn, 1, 1, output_idx=2) + self.hessian_w_zz = SecondOrderGrad(self.dnn, 2, 2, output_idx=2) + + self.loss_scaler = DynamicLossScaler(1024, 2, 100) + + def net_u(self, x, y, z, t): + H = ms.ops.concat([x, y, z, t], 1) + Y = self.dnn(H) + return Y + + def net_f(self, x_f, y_f, z_f, t_f): + """ The minspore autograd version of calculating residual """ + U = self.net_u(x_f, y_f, z_f, t_f) + u = U[:, 0:1] + v = U[:, 1:2] + w = U[:, 2:3] + p = U[:, 3:4] + + data = ms.ops.concat([x_f, y_f, z_f, t_f], 1) + + u_x = self.grad(data, 0, 0, U) + u_y = self.grad(data, 1, 0, U) + u_z = self.grad(data, 2, 0, U) + u_t = self.grad(data, 3, 0, U) + u_xx = self.hessian_u_xx(data) + u_yy = self.hessian_u_yy(data) + u_zz = self.hessian_u_zz(data) + + v_x = self.grad(data, 0, 1, U) + v_y = self.grad(data, 1, 1, U) + v_z = self.grad(data, 2, 1, U) + v_t = self.grad(data, 3, 1, U) + v_xx = self.hessian_v_xx(data) + v_yy = self.hessian_v_yy(data) + v_zz = self.hessian_v_zz(data) + + w_x = self.grad(data, 0, 2, U) + w_y = self.grad(data, 1, 2, U) + w_z = self.grad(data, 2, 2, U) + w_t = self.grad(data, 3, 2, U) + w_xx = self.hessian_w_xx(data) + w_yy = self.hessian_w_yy(data) + w_zz = self.hessian_w_zz(data) + + p_x = self.grad(data, 0, 3, U) + p_y = self.grad(data, 1, 3, U) + p_z = self.grad(data, 2, 3, U) + + f_u = u_t + (u * u_x + v * u_y + w * u_z) + p_x - 1. / self.Re * (u_xx + u_yy + u_zz) + f_v = v_t + (u * v_x + v * v_y + w * v_z) + p_y - 1. / self.Re * (v_xx + v_yy + v_zz) + f_w = w_t + (u * w_x + v * w_y + w * w_z) + p_z - 1. / self.Re * (w_xx + w_yy + w_zz) + + f_e = u_x + v_y + w_z + + return u, v, w, p, f_u, f_v, f_w, f_e + + def loss_fn(self, xb, yb, zb, tb, xi, yi, zi, ti, x_f, y_f, z_f, t_f, ub, vb, wb, ui, vi, wi): + """ + loss_fn + """ + Ub = self.net_u(xb, yb, zb, tb) + ub_pred = Ub[:, 0:1] + vb_pred = Ub[:, 1:2] + wb_pred = Ub[:, 2:3] + + Ui = self.net_u(xi, yi, zi, ti) + ui_pred = Ui[:, 0:1] + vi_pred = Ui[:, 1:2] + wi_pred = Ui[:, 2:3] + + _, _, _, _, f_u_pred, f_v_pred, f_w_pred, f_e_pred = self.net_f(x_f, y_f, z_f, t_f) + + loss_ub = ops.reduce_mean(ops.square(ub_pred - ub)) + loss_vb = ops.reduce_mean(ops.square(vb_pred - vb)) + loss_wb = ops.reduce_mean(ops.square(wb_pred - wb)) + + loss_ui = ops.reduce_mean(ops.square(ui_pred - ui)) + loss_vi = ops.reduce_mean(ops.square(vi_pred - vi)) + loss_wi = ops.reduce_mean(ops.square(wi_pred - wi)) + + alpha = ms.Tensor(100., dtype=self.msfloat_type) + + loss_f_u = ops.reduce_mean(ops.square(f_u_pred - ms.ops.zeros_like(f_u_pred))) + loss_f_v = ops.reduce_mean(ops.square(f_v_pred - ms.ops.zeros_like(f_v_pred))) + loss_f_w = ops.reduce_mean(ops.square(f_w_pred - ms.ops.zeros_like(f_w_pred))) + loss_f_e = ops.reduce_mean(ops.square(f_e_pred - ms.ops.zeros_like(f_e_pred))) + + loss_b = loss_ub + loss_vb + loss_wb + loss_i = loss_ui + loss_vi + loss_wi + loss_f = loss_f_u + loss_f_v + loss_f_w + loss_f_e + + loss = alpha * loss_b + alpha * loss_i + loss_f + if self.use_npu: + loss = self.loss_scaler.scale(loss) + return loss, loss_b, loss_i, loss_f diff --git a/MindFlow/applications/research/NSFNets/src/network_kf.py b/MindFlow/applications/research/NSFNets/src/network_kf.py new file mode 100644 index 000000000..8a42b0e87 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/src/network_kf.py @@ -0,0 +1,169 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Network definitions""" +# pylint: disable=C0103 +import mindspore as ms +from mindspore import nn, ops +from mindspore.common.api import jit +from mindspore.amp import DynamicLossScaler +import numpy as np +from src.derivatives import SecondOrderGrad, Grad + + +# the deep neural network +class neural_net(nn.Cell): + """ + neural_net + """ + def __init__(self, layers, msfloat_type, lb, ub): + super(neural_net, self).__init__() + + # normalize the scope + self.lb = lb + self.ub = ub + + # network parameters + self.depth = len(layers) - 1 + + # set up activation + self.activation = nn.Tanh() + + layer_list = [] + for i in range(self.depth - 1): + layer_list.append(nn.Dense(layers[i], layers[i+1]).to_float(msfloat_type)) + layer_list.append(self.activation.to_float(msfloat_type)) + + layer_list.append(nn.Dense(layers[-2], layers[-1]).to_float(msfloat_type)) + + # deploy layers + self.layers = nn.SequentialCell(layer_list).to_float(msfloat_type) + + @jit + def construct(self, x): + H = 2.0 * (x - self.lb) / (self.ub - self.lb) - 1.0 + out = self.layers(H) + return out + +class VPNSFNets: + """ + VPNSFNets + """ + # Initialize the class + def __init__(self, xb, yb, ub, vb, x_f, y_f, layers, use_npu, + msfloat_type, npfloat_type, load_params, second_path): + + self.use_npu = use_npu + + self.msfloat_type = msfloat_type + self.npfloat_type = npfloat_type + + self.Xb_l = ops.concat([ms.Tensor(np.array(xb, self.npfloat_type)), + ms.Tensor(np.array(yb, self.npfloat_type))], 1) + self.X_l = ops.concat([ms.Tensor(np.array(x_f, self.npfloat_type)), + ms.Tensor(np.array(y_f, self.npfloat_type))], 1) + + self.Xmin = ms.Tensor(np.array(self.Xb_l.min(0), self.npfloat_type)) + self.Xmax = ms.Tensor(np.array(self.Xb_l.max(0), self.npfloat_type)) + + + self.layers = layers + + self.x_f = ms.Tensor(np.array(x_f, self.npfloat_type)) + self.y_f = ms.Tensor(np.array(y_f, self.npfloat_type)) + + self.xb = ms.Tensor(np.array(xb, self.npfloat_type)) + self.yb = ms.Tensor(np.array(yb, self.npfloat_type)) + self.ub = ms.Tensor(np.array(ub, self.npfloat_type)) + self.vb = ms.Tensor(np.array(vb, self.npfloat_type)) + + # Initialize NNs---deep neural networks + self.dnn = neural_net(layers, self.msfloat_type, self.Xmin, self.Xmax) + if load_params: + params_dict = ms.load_checkpoint(f'model/{second_path}/model.ckpt') + ms.load_param_into_net(self.dnn, params_dict) + + # The function of Auto-differentiation + self.grad = Grad(self.dnn) + self.hessian_u_xx = SecondOrderGrad(self.dnn, 0, 0, output_idx=0) + self.hessian_u_yy = SecondOrderGrad(self.dnn, 1, 1, output_idx=0) + + self.hessian_v_xx = SecondOrderGrad(self.dnn, 0, 0, output_idx=1) + self.hessian_v_yy = SecondOrderGrad(self.dnn, 1, 1, output_idx=1) + + self.loss_scaler = DynamicLossScaler(1024, 2, 100) + + def net_u(self, x, y): + H = ms.ops.concat([x, y], 1) + Y = self.dnn(H) + return Y + + def net_f(self, x_f, y_f): + """ + net_f + """ + U = self.net_u(x_f, y_f) + u = U[:, 0:1] + v = U[:, 1:2] + p = U[:, 2:3] + + data = ms.ops.concat([x_f, y_f], 1) + + u_x = self.grad(data, 0, 0, U) + u_y = self.grad(data, 1, 0, U) + u_xx = self.hessian_u_xx(data) + u_yy = self.hessian_u_yy(data) + + v_x = self.grad(data, 0, 1, U) + v_y = self.grad(data, 1, 1, U) + v_xx = self.hessian_v_xx(data) + v_yy = self.hessian_v_yy(data) + + p_x = self.grad(data, 0, 2, U) + p_y = self.grad(data, 1, 2, U) + + f_u = (u * u_x + v * u_y) + p_x - (1.0/40) * (u_xx + u_yy) + f_v = (u * v_x + v * v_y) + p_y - (1.0/40) * (v_xx + v_yy) + f_e = u_x + v_y + + return u, v, p, f_u, f_v, f_e + + + def loss_fn(self, xb, yb, x_f, y_f, ub, vb): + """ + loss_fn + """ + Ub = self.net_u(xb, yb) + ub_pred = Ub[:, 0:1] + vb_pred = Ub[:, 1:2] + + + _, _, _, f_u_pred, f_v_pred, f_e_pred = self.net_f(x_f, y_f) + + loss_ub = ops.reduce_mean(ops.square(ub_pred - ub)) + loss_vb = ops.reduce_mean(ops.square(vb_pred - vb)) + + alpha = ms.Tensor(1., dtype=self.msfloat_type) + + loss_f_u = ops.reduce_mean(ops.square(f_u_pred - ms.ops.zeros_like(f_u_pred))) + loss_f_v = ops.reduce_mean(ops.square(f_v_pred - ms.ops.zeros_like(f_v_pred))) + loss_f_e = ops.reduce_mean(ops.square(f_e_pred - ms.ops.zeros_like(f_e_pred))) + + loss_b = loss_ub + loss_vb + loss_f = loss_f_u + loss_f_v + loss_f_e + + loss = alpha * loss_b + loss_f + if self.use_npu: + loss = self.loss_scaler.scale(loss) + return loss, loss_b, loss_f diff --git a/MindFlow/applications/research/NSFNets/test.py b/MindFlow/applications/research/NSFNets/test.py new file mode 100644 index 000000000..4e4efd8bd --- /dev/null +++ b/MindFlow/applications/research/NSFNets/test.py @@ -0,0 +1,165 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ICNet train""" +# pylint: disable=C0103 +# pylint: disable=E1121 +import os +import argparse + +import numpy as np +import scipy.io +import mindspore as ms +from mindspore import set_seed, context +from mindflow.utils import load_yaml_config +from src.network import VPNSFNets +from src.datasets import read_training_data, ThreeBeltramiflow + +# Adam loss history +loss_history_adam_pretrain = np.empty([0]) +loss_b_history_adam_pretrain = np.empty([0]) +loss_i_history_adam_pretrain = np.empty([0]) +loss_f_history_adam_pretrain = np.empty([0]) +np.random.seed(123456) +set_seed(123456) + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config_file_path", type=str, default="./config/NSFNet.yaml") + # args = parser.parse_args() + args = parser.parse_known_args()[0] + + config = load_yaml_config(args.config_file_path) + params = config["params"] + + model_name = params['model_name'] + case = params['case'] + device = params['device'] + device_id = params['device_id'] + network_size = params['network_size'] + learning_rate = params['learning_rate'] + epochs = params['epochs'] + batch_size = params['batch_size'] + load_params = params['load_params_test'] + second_path = params['second_path1'] + xmin = params['xmin'] + xmax = params['xmax'] + ymin = params['ymin'] + ymax = params['ymax'] + zmin = params['zmin'] + zmax = params['zmax'] + tmin = params['tmin'] + tmax = params['tmax'] + n_x = params['n_x'] + n_y = params['n_y'] + n_z = params['n_z'] + n_t = params['n_t'] + + os.environ['CUDA_VISIBLE_DEVICES'] = device_id + context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=device) + + use_ascend = context.get_context(attr_key='device_target') == "Ascend" + + if use_ascend: + msfloat_type = ms.float16 + npfloat_type = np.float16 + else: + msfloat_type = ms.float32 + npfloat_type = np.float32 + + x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, \ + u_i, v_i, w_i, x_f, y_f, z_f, t_f, Re, X_min, X_max = read_training_data() + + model = VPNSFNets(x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, \ + u_i, v_i, w_i, x_f, y_f, z_f, t_f, network_size, \ + Re, X_min, X_max, use_ascend, msfloat_type, npfloat_type, load_params, second_path) + + x_ts_sum = np.arange(xmin, xmax+1e-5, (xmax-xmin)/(n_x-1.)) + y_ts_sum = np.arange(ymin, ymax+1e-5, (ymax-ymin)/(n_y-1.)) + z_ts_sum = np.arange(zmin, zmax+1e-5, (zmax-zmin)/(n_z-1.)) + + x_ts, y_ts, z_ts = np.meshgrid(x_ts_sum, y_ts_sum, z_ts_sum) + x_ts = x_ts.reshape([-1, 1]) + y_ts = y_ts.reshape([-1, 1]) + z_ts = z_ts.reshape([-1, 1]) + x_ts = ms.Tensor(np.array(x_ts, npfloat_type)) + y_ts = ms.Tensor(np.array(y_ts, npfloat_type)) + z_ts = ms.Tensor(np.array(z_ts, npfloat_type)) + + # t1 + t1 = tmin*np.ones([x_ts.shape[0], 1]) + t1 = ms.Tensor(np.array(t1, npfloat_type)) + U_ts = model.net_u(x_ts, y_ts, z_ts, t1) + u_ts, v_ts, w_ts, p_ts = U_ts[:, 0:1], U_ts[:, 1:2], U_ts[:, 2:3], U_ts[:, 3:4] + u_ts_real, v_ts_real, w_ts_real, p_ts_real = \ + ThreeBeltramiflow(x_ts.numpy(), y_ts.numpy(), z_ts.numpy(), t1.numpy(), Re) + scipy.io.savemat('test_result1.mat', {'xts': x_ts.numpy(), 'yts': y_ts.numpy(), 'zts': z_ts.numpy(), + 'uts': u_ts.numpy(), 'vts': v_ts.numpy(), + 'wts': w_ts.numpy(), 'pts': p_ts.numpy(), + 'ureal': u_ts_real, 'vreal': v_ts_real, + 'wreal': w_ts_real, 'preal': p_ts_real}) + # t2 + t2 = (tmin + 0.25*(tmax-tmin))*np.ones([x_ts.shape[0], 1]) + t2 = ms.Tensor(np.array(t2, npfloat_type)) + U_ts = model.net_u(x_ts, y_ts, z_ts, t2) + u_ts, v_ts, w_ts, p_ts = U_ts[:, 0:1], U_ts[:, 1:2], U_ts[:, 2:3], U_ts[:, 3:4] + u_ts_real, v_ts_real, w_ts_real, p_ts_real = ThreeBeltramiflow(x_ts.numpy(), y_ts.numpy(), + z_ts.numpy(), t2.numpy(), Re) + scipy.io.savemat('test_result2.mat', {'xts': x_ts.numpy(), 'yts': y_ts.numpy(), + 'zts': z_ts.numpy(), 'uts': u_ts.numpy(), + 'vts': v_ts.numpy(), 'wts': w_ts.numpy(), + 'pts': p_ts.numpy(), 'ureal': u_ts_real, + 'vreal': v_ts_real, 'wreal': w_ts_real, + 'preal': p_ts_real}) + + # t3 + t3 = (tmin + 0.5*(tmax-tmin))*np.ones([x_ts.shape[0], 1]) + t3 = ms.Tensor(np.array(t3, npfloat_type)) + U_ts = model.net_u(x_ts, y_ts, z_ts, t3) + u_ts, v_ts, w_ts, p_ts = U_ts[:, 0:1], U_ts[:, 1:2], U_ts[:, 2:3], U_ts[:, 3:4] + u_ts_real, v_ts_real, w_ts_real, p_ts_real = ThreeBeltramiflow(x_ts.numpy(), y_ts.numpy(), + z_ts.numpy(), t3.numpy(), Re) + scipy.io.savemat('test_result3.mat', {'xts': x_ts.numpy(), 'yts': y_ts.numpy(), + 'zts': z_ts.numpy(), 'uts': u_ts.numpy(), + 'vts': v_ts.numpy(), 'wts': w_ts.numpy(), + 'pts': p_ts.numpy(), 'ureal': u_ts_real, + 'vreal': v_ts_real, 'wreal': w_ts_real, + 'preal': p_ts_real}) + + # t4 + t4 = (tmin + 0.75*(tmax-tmin))*np.ones([x_ts.shape[0], 1]) + t4 = ms.Tensor(np.array(t4, npfloat_type)) + U_ts = model.net_u(x_ts, y_ts, z_ts, t4) + u_ts, v_ts, w_ts, p_ts = U_ts[:, 0:1], U_ts[:, 1:2], U_ts[:, 2:3], U_ts[:, 3:4] + u_ts_real, v_ts_real, w_ts_real, p_ts_real = ThreeBeltramiflow(x_ts.numpy(), y_ts.numpy(), + z_ts.numpy(), t4.numpy(), Re) + scipy.io.savemat('test_result3.mat', {'xts': x_ts.numpy(), 'yts': y_ts.numpy(), + 'zts': z_ts.numpy(), 'uts': u_ts.numpy(), + 'vts': v_ts.numpy(), 'wts': w_ts.numpy(), + 'pts': p_ts.numpy(), 'ureal': u_ts_real, + 'vreal': v_ts_real, 'wreal': w_ts_real, + 'preal': p_ts_real}) + # t5 + t5 = tmax*np.ones([x_ts.shape[0], 1]) + t5 = ms.Tensor(np.array(t5, npfloat_type)) + U_ts = model.net_u(x_ts, y_ts, z_ts, t5) + u_ts, v_ts, w_ts, p_ts = U_ts[:, 0:1], U_ts[:, 1:2], U_ts[:, 2:3], U_ts[:, 3:4] + u_ts_real, v_ts_real, w_ts_real, p_ts_real = ThreeBeltramiflow(x_ts.numpy(), y_ts.numpy(), + z_ts.numpy(), t5.numpy(), Re) + scipy.io.savemat('test_result3.mat', {'xts': x_ts.numpy(), 'yts': y_ts.numpy(), + 'zts': z_ts.numpy(), 'uts': u_ts.numpy(), + 'vts': v_ts.numpy(), 'wts': w_ts.numpy(), + 'pts': p_ts.numpy(), 'ureal': u_ts_real, + 'vreal': v_ts_real, 'wreal': w_ts_real, + 'preal': p_ts_real}) diff --git a/MindFlow/applications/research/NSFNets/test_KF.py b/MindFlow/applications/research/NSFNets/test_KF.py new file mode 100644 index 000000000..f08057f9f --- /dev/null +++ b/MindFlow/applications/research/NSFNets/test_KF.py @@ -0,0 +1,87 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""NSFNet train""" +# pylint: disable=C0103 +import os +import argparse + +import numpy as np +import scipy.io +import mindspore as ms +from mindspore import set_seed, context +from mindflow.utils import load_yaml_config +from src.network_kf import VPNSFNets +from src.datasets_kf import read_training_data + +# Adam loss history +loss_history_adam_pretrain = np.empty([0]) +loss_b_history_adam_pretrain = np.empty([0]) +loss_f_history_adam_pretrain = np.empty([0]) +np.random.seed(123456) +set_seed(123456) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config_file_path", type=str, default="./config/NSFNet_KF.yaml") + # args = parser.parse_args() + args = parser.parse_known_args()[0] + + config = load_yaml_config(args.config_file_path) + params = config["params"] + + model_name = params['model_name'] + case = params['case'] + device = params['device'] + device_id = params['device_id'] + network_size = params['network_size'] + learning_rate = params['learning_rate'] + epochs = params['epochs'] + load_params = params['load_params_test'] + second_path = params['second_path1'] + re = params['re'] + + os.environ['CUDA_VISIBLE_DEVICES'] = device_id + context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=device) + + use_ascend = context.get_context(attr_key='device_target') == "Ascend" + + if use_ascend: + msfloat_type = ms.float16 + npfloat_type = np.float16 + else: + msfloat_type = ms.float32 + npfloat_type = np.float32 + + xb_train, yb_train, ub_train, vb_train, x_train, y_train = read_training_data() + + model = VPNSFNets(xb_train, yb_train, ub_train, vb_train, x_train, y_train, \ + network_size, use_ascend, msfloat_type, npfloat_type, load_params, second_path) + + x_test = (np.random.rand(1000, 1) - 1 / 3) * 3 / 2 + y_test = (np.random.rand(1000, 1) - 1 / 4) * 2 + lam = 0.5 * re - np.sqrt(0.25 * (re ** 2) + 4 * (np.pi ** 2)) + u_test = 1 - np.exp(lam * x_test) * np.cos(2 * np.pi * y_test) + v_test = (lam / (2 * np.pi)) * np.exp(lam * x_test) * np.sin(2 * np.pi * y_test) + p_test = 0.5 * (1 - np.exp(2 * lam * x_test)) + # to tensor + x_test = ms.Tensor(np.array(x_test, npfloat_type)) + y_test = ms.Tensor(np.array(y_test, npfloat_type)) + + U_pred = model.net_u(x_test, y_test) + u_pred, v_pred, p_pred = U_pred[:, 0:1], U_pred[:, 1:2], U_pred[:, 2:3] + scipy.io.savemat('test_result.mat', {'xts': x_test.numpy(), 'yts': y_test.numpy(), \ + 'uts': u_pred.numpy(), 'vts': v_pred.numpy(), 'pts': p_pred.numpy(), \ + 'ureal': u_test, 'vreal': v_test, 'preal': p_test}) diff --git a/MindFlow/applications/research/NSFNets/train.py b/MindFlow/applications/research/NSFNets/train.py new file mode 100644 index 000000000..d22e86feb --- /dev/null +++ b/MindFlow/applications/research/NSFNets/train.py @@ -0,0 +1,130 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ICNet train""" +# pylint: disable=C0103 +import os +import time +import argparse + +import numpy as np +import mindspore as ms +from mindspore import set_seed, context, nn, Tensor +from mindflow.utils import load_yaml_config +from src.network import VPNSFNets +from src.datasets import read_training_data + +# Adam loss history +loss_history_adam_pretrain = np.empty([0]) +loss_b_history_adam_pretrain = np.empty([0]) +loss_i_history_adam_pretrain = np.empty([0]) +loss_f_history_adam_pretrain = np.empty([0]) +np.random.seed(123456) +set_seed(123456) + +def train(model, niter, lr, batch_size): + """ + Training + """ + # Get the gradients function + params_train = model.dnn.trainable_params() + optimizer_adam = nn.Adam(params_train, learning_rate=lr) + grad_fn = ms.value_and_grad(model.loss_fn, None, optimizer_adam.parameters, has_aux=True) + model.dnn.set_train() + n_data = model.t_f.shape[0] + + start_time = time.time() + + for epoch in range(1, 1+niter): + + idx_data_0 = np.random.choice(n_data, batch_size) + idx_data = Tensor(idx_data_0) + x_batch = model.x_f[idx_data, :] + y_batch = model.y_f[idx_data, :] + z_batch = model.z_f[idx_data, :] + t_batch = model.t_f[idx_data, :] + + (loss, loss_b, loss_i, loss_f), grads = grad_fn(model.xb, model.yb, model.zb, model.tb, + model.xi, model.yi, model.zi, model.ti, + x_batch, y_batch, z_batch, t_batch, + model.ub, model.vb, model.wb, + model.ui, model.vi, model.wi) + optimizer_adam(grads) + + if epoch % 10 == 0: + elapsed = time.time() - start_time + print('It: %d, Total_loss: %.3e, Loss_b: %.3e, Loss_i: %.3e, Loss_f: %.3e, Time: %.2f' %\ + (epoch, loss.item(), loss_b.item(), loss_i.item(), loss_f.item(), elapsed)) + + global loss_history_adam_pretrain + global loss_b_history_adam_pretrain + global loss_i_history_adam_pretrain + global loss_f_history_adam_pretrain + + loss_history_adam_pretrain = np.append(loss_history_adam_pretrain, loss.numpy()) + loss_b_history_adam_pretrain = np.append(loss_b_history_adam_pretrain, loss_b.numpy()) + loss_i_history_adam_pretrain = np.append(loss_i_history_adam_pretrain, loss_i.numpy()) + loss_f_history_adam_pretrain = np.append(loss_f_history_adam_pretrain, loss_f.numpy()) + + start_time = time.time() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config_file_path", type=str, default="./config/NSFNet.yaml") + # args = parser.parse_args() + args = parser.parse_known_args()[0] + + config = load_yaml_config(args.config_file_path) + params = config["params"] + + model_name = params['model_name'] + case = params['case'] + device = params['device'] + device_id = params['device_id'] + network_size = params['network_size'] + learning_rate = params['learning_rate'] + epochs = params['epochs'] + batch_size_train = params['batch_size'] + load_params = params['load_params'] + second_path = params['second_path1'] + re = params['re'] + + os.environ['CUDA_VISIBLE_DEVICES'] = device_id + context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=device) + + use_ascend = context.get_context(attr_key='device_target') == "Ascend" + + if use_ascend: + msfloat_type = ms.float16 + npfloat_type = np.float16 + else: + msfloat_type = ms.float32 + npfloat_type = np.float32 + + x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, \ + u_i, v_i, w_i, x_f, y_f, z_f, t_f, X_min, X_max = read_training_data() + + model_train = VPNSFNets(x_b, y_b, z_b, t_b, x_i, y_i, z_i, t_i, u_b, v_b, w_b, + u_i, v_i, w_i, x_f, y_f, z_f, t_f, network_size, + re, X_min, X_max, use_ascend, msfloat_type, npfloat_type, load_params, second_path) + + for epoch_train, lr_train in zip(epochs, learning_rate): + train(model_train, int(np.float(epoch_train)), lr_train, batch_size=batch_size_train) + ms.save_checkpoint(model_train.dnn, f'model/{second_path}/model.ckpt') + + np.save(f'Loss-Coe/{second_path}/loss_history_Adam_pretrain', loss_history_adam_pretrain) + np.save(f'Loss-Coe/{second_path}/loss_b_history_Adam_pretrain', loss_b_history_adam_pretrain) + np.save(f'Loss-Coe/{second_path}/loss_i_history_Adam_pretrain', loss_i_history_adam_pretrain) + np.save(f'Loss-Coe/{second_path}/loss_f_history_Adam_pretrain', loss_f_history_adam_pretrain) diff --git a/MindFlow/applications/research/NSFNets/train_KF.py b/MindFlow/applications/research/NSFNets/train_KF.py new file mode 100644 index 000000000..d9eb05cd1 --- /dev/null +++ b/MindFlow/applications/research/NSFNets/train_KF.py @@ -0,0 +1,110 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""NSFNet train""" +# pylint: disable=C0103 +import os +import time +import argparse + +import numpy as np +import mindspore as ms +from mindspore import set_seed, context, nn +from mindflow.utils import load_yaml_config +from src.network_kf import VPNSFNets +from src.datasets_kf import read_training_data + +# Adam loss history +loss_history_adam_pretrain = np.empty([0]) +loss_b_history_adam_pretrain = np.empty([0]) +loss_f_history_adam_pretrain = np.empty([0]) +np.random.seed(123456) +set_seed(123456) + +def train(model, Niter, lr): + """ + Training + """ + # Get the gradients function + params_train = model.dnn.trainable_params() + optimizer_Adam = nn.Adam(params_train, learning_rate=lr) + grad_fn = ms.value_and_grad(model.loss_fn, None, optimizer_Adam.parameters, has_aux=True) + model.dnn.set_train() + + start_time = time.time() + + for epoch in range(1, 1+Niter): + + (loss, loss_b, loss_f), grads = grad_fn(model.xb, model.yb, model.x_f, model.y_f, model.ub, model.vb) + optimizer_Adam(grads) + + if epoch % 10 == 0: + elapsed = time.time() - start_time + print('It: %d, Total_loss: %.3e, Loss_b: %.3e, Loss_f: %.3e, Time: %.2f' %\ + (epoch, loss.item(), loss_b.item(), loss_f.item(), elapsed)) + + global loss_history_adam_pretrain + global loss_b_history_adam_pretrain + global loss_f_history_adam_pretrain + + loss_history_adam_pretrain = np.append(loss_history_adam_pretrain, loss.numpy()) + loss_b_history_adam_pretrain = np.append(loss_b_history_adam_pretrain, loss_b.numpy()) + loss_f_history_adam_pretrain = np.append(loss_f_history_adam_pretrain, loss_f.numpy()) + + start_time = time.time() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config_file_path", type=str, default="./config/NSFNet_KF.yaml") + # args = parser.parse_args() + args = parser.parse_known_args()[0] + + config = load_yaml_config(args.config_file_path) + params = config["params"] + + model_name = params['model_name'] + case = params['case'] + device = params['device'] + device_id = params['device_id'] + network_size = params['network_size'] + learning_rate = params['learning_rate'] + epochs = params['epochs'] + load_params = params['load_params'] + second_path = params['second_path1'] + + os.environ['CUDA_VISIBLE_DEVICES'] = device_id + context.set_context(mode=context.GRAPH_MODE, save_graphs=False, device_target=device) + + use_ascend = context.get_context(attr_key='device_target') == "Ascend" + + if use_ascend: + msfloat_type = ms.float16 + npfloat_type = np.float16 + else: + msfloat_type = ms.float32 + npfloat_type = np.float32 + + xb_train, yb_train, ub_train, vb_train, x_train, y_train = read_training_data() + + model_train = VPNSFNets(xb_train, yb_train, ub_train, vb_train, x_train, + y_train, network_size, use_ascend, msfloat_type, + npfloat_type, load_params, second_path) + for epoch_train, lr_train in zip(epochs, learning_rate): + train(model_train, int(np.float(epoch_train)), lr_train) + ms.save_checkpoint(model_train.dnn, f'model/{second_path}/model.ckpt') + + np.save(f'Loss-Coe/{second_path}/loss_history_adam_pretrain', loss_history_adam_pretrain) + np.save(f'Loss-Coe/{second_path}/loss_b_history_adam_pretrain', loss_b_history_adam_pretrain) + np.save(f'Loss-Coe/{second_path}/loss_f_history_adam_pretrain', loss_f_history_adam_pretrain) -- Gitee