From aea616b7a20e600cff76aa4e2438b90c67bea5ac Mon Sep 17 00:00:00 2001 From: Flock-Li <863813609@qq.com> Date: Fri, 30 Apr 2021 12:17:15 +0800 Subject: [PATCH] complete_ass2 --- assignment-2/submission/18307130130/README.md | 227 +++++++++++++++ .../submission/18307130130/img/Adam.png | Bin 0 -> 14151 bytes .../submission/18307130130/img/Adam_train.png | Bin 0 -> 19035 bytes .../18307130130/img/Adam_train_512_01.png | Bin 0 -> 46310 bytes .../submission/18307130130/img/Momentum.png | Bin 0 -> 8347 bytes .../18307130130/img/Momentum_train.png | Bin 0 -> 24670 bytes .../18307130130/img/Momentum_train_512_01.png | Bin 0 -> 50250 bytes .../18307130130/img/SGD_train_128_001.png | Bin 0 -> 76950 bytes .../18307130130/img/SGD_train_128_01.png | Bin 0 -> 69343 bytes .../18307130130/img/SGD_train_512_01.png | Bin 0 -> 50922 bytes .../submission/18307130130/img/model.png | Bin 0 -> 24275 bytes .../submission/18307130130/numpy_fnn.py | 262 ++++++++++++++++++ .../submission/18307130130/numpy_mnist.py | 51 ++++ 13 files changed, 540 insertions(+) create mode 100644 assignment-2/submission/18307130130/README.md create mode 100644 assignment-2/submission/18307130130/img/Adam.png create mode 100644 assignment-2/submission/18307130130/img/Adam_train.png create mode 100644 assignment-2/submission/18307130130/img/Adam_train_512_01.png create mode 100644 assignment-2/submission/18307130130/img/Momentum.png create mode 100644 assignment-2/submission/18307130130/img/Momentum_train.png create mode 100644 assignment-2/submission/18307130130/img/Momentum_train_512_01.png create mode 100644 assignment-2/submission/18307130130/img/SGD_train_128_001.png create mode 100644 assignment-2/submission/18307130130/img/SGD_train_128_01.png create mode 100644 assignment-2/submission/18307130130/img/SGD_train_512_01.png create mode 100644 assignment-2/submission/18307130130/img/model.png create mode 100644 assignment-2/submission/18307130130/numpy_fnn.py create mode 100644 assignment-2/submission/18307130130/numpy_mnist.py diff --git a/assignment-2/submission/18307130130/README.md b/assignment-2/submission/18307130130/README.md new file mode 100644 index 0000000..b7fd7f8 --- /dev/null +++ b/assignment-2/submission/18307130130/README.md @@ -0,0 +1,227 @@ +# Assignment-2 Report + +
------李睿琛 18307130130
+## 算子实现 + +`Matmul` + +实现两个矩阵相乘。对于: +$$ +Y = X \times W +$$ +存在函数`Loss = f(Y)`,根据链式法则,求导得: + + +$$ +\frac{\partial L}{\partial W_{ij}}={\sum_{k,l}}\frac{\partial L}{\partial Y_{kl}}\times\frac{\partial Y_{kl}}{\partial W_{ij}} \\\\ +$$ + +$$ +\frac{\partial Y_{kl}}{\partial X_{ij}}=\frac{\partial {\sum_{s}(X_{ks}\times W_{sl})}}{W_{ij}}=\frac{\partial X_{ki}W_{il}}{W_{ij}}=A_{ki}{\delta_{lj}} +$$ + +$$ +\frac{\partial L}{\partial W}=X^T\frac{\partial L}{\partial Y} +$$ + +于是有: +$$ +grad\_w = X^T \times grad\_y\\\\ +同理有: grad\_x = grad\_y \times W^T +$$ +`Relu` + +作为激活函数,具有单侧抑制、宽兴奋边界等生物学合理性。 +$$ +Y_{ij}=\begin{cases} +X_{ij}&X_{ij}\ge0\\\\ +0&\text{otherwise} +\end{cases} +$$ + +求导得: +$$ +\frac{\partial Y_{ij}}{\partial X_{mn}}=\begin{cases}1&X_{ij}>0,i=m,j=n\\\\0&\text{otherwise}\end{cases} +$$ + +`Log` + +计算公式为: +$$ +Y_{ij}=\ln(X_{ij}+\epsilon),\epsilon=10^{-12} \\\\ +$$ + +求导得: +$$ +\frac{\partial Y_{ij}}{\partial X_{ij}}=\frac1{X_{ij}+\epsilon} +$$ + + +`Softmax` + +称为多项的Logistic回归,表达式为: +$$ +Y_{ij}=\frac{\exp\{X_{ij} \}}{\sum_{k=1}^c\exp\{X_{ik} \}}\\\\ +$$ +向量`Y_i`对向量`X_j`求导得: +$$ +\frac{\partial Y_{i}}{\partial X_{j}}=\begin{cases} +Y_{i}(1-Y_{j})&i=j\\\\ +-Y_{i}Y_{j}&\text otherwise +\end{cases} +$$ + + +## 模型搭建 + +根据`torch_mnist.py`中`TorchModel`方法搭建模型:、 + + + +根据模型计算图,即可得到如下**前馈过程:** + +```python +x = self.matmul_1.forward(x, self.W1) +x = self.relu_1.forward(x) +x = self.matmul_2.forward(x, self.W2) +x = self.relu_2.forward(x) +x = self.matmul_3.forward(x, self.W3) +x = self.softmax.forward(x) +x = self.log.forward(x) +``` + +**反向传播:** + +首先是标量Loss对向量pred的求导: +$$ +\frac{\partial L}{\partial X^T} =[\frac{\partial L}{\partial x_1},...,\frac{\partial L}{\partial x_n}]^T +$$ +于是有: + +```python +class NumpyLoss: +def backward(self): + return -self.target / self.target.shape[0] +``` + +根据计算图,继续反向传播: + +```python +self.log_grad = self.log.backward(y) +self.softmax_grad = self.softmax.backward(self.log_grad) +self.x3_grad, self.W3_grad = self.matmul_3.backward(self.softmax_grad) +self.relu_2_grad = self.relu_2.backward(self.x3_grad) +self.x2_grad, self.W2_grad = self.matmul_2.backward(self.relu_2_grad) +self.relu_1_grad = self.relu_1.backward(self.x2_grad) +self.x1_grad, self.W1_grad = self.matmul_1.backward(self.relu_1_grad) +``` + +## mini_batch实现 + +由于数据集巨大,深度学习往往训练速度很慢,导致难以发挥最大效果。相比batch梯度下降法,mini_batch进一步分割数据集,能在一次遍历训练集的过程中做多次梯度下降,收敛到一个合适的精度。获得mini_batch过程为: + +```python +# 将数据集打乱; +sz = data.shape[0] +index = np.arange(sz) +np.random.shuffle(index) + +# 按照batch_size分割数据集; +for start in range(0, sz, batch_size): + ret.append([data[index[start: start+ batch_size]], label[index[start: start+ batch_size]]]) +``` + +## 模型参数影响 + +探究了学习率、batch_size大小对收敛速度的影响。 + +* learning_rate = 0.1,batch_size=128 + +``` +[0] Accuracy: 0.9436 +[1] Accuracy: 0.9610 +[2] Accuracy: 0.9710 +``` + + + +* learning_rate = 0.01,batch_size=128 + +``` +[0] Accuracy: 0.8730 +[1] Accuracy: 0.9047 +[2] Accuracy: 0.9142 +``` + + + +* learning_rate = 0.1,batch_size=512 + +``` +[0] Accuracy: 0.8233 +[1] Accuracy: 0.9244 +[2] Accuracy: 0.9296 +``` + + + +在一定范围内: + +随着学习率的减小,参数收敛速度减小,在相同迭代次数下准确率更低。 + +随着批处理容量的增大,迭代次数减少,震荡幅度减小,随着容量继续增大,可能达到时间上的最优。 + +## 梯度下降算法优化 + +* SGD算法。learning_rate = 0.1,batch_size=512 + +即优化前的算法。 + +* Momentum算法。learning_rate = 0.1,batch_size=512 + +SGN的损失在一个方向上快速变化而在另一个方向慢慢变化。Momentum算法将一段时间内的梯度向量进行了加权平均 ,有利于帮助SGN加速,冲破沟壑,加速收敛。引入动量能够使得在遇到局部最优的时候在动量的基础上**冲出局部最优**;另外也可使**震荡减弱**,更快运动到最优解。 + +**算法实现:** + + + +```python +# 算法实现 +self.beta1 = self.momentum * self.beta1 + (1 - self.momentum) * self.W1_grad +self.W1 -= learning_rate * self.beta1 + +``` 结果 +[0] Accuracy: 0.9058 +[1] Accuracy: 0.9293 +[2] Accuracy: 0.9391 +``` +``` + + + +* Adam算法。learning_rate = 0.1,batch_size=512 + +相比随机梯度下降中**不变**的学习率,Adam算法通过计算梯度的一阶矩估计和二阶矩估计而为不同的参数设计独立的**自适应性学习率**。 + +**算法实现:** + + + +```python +# 算法实现 +self.m1 = self.theta1 * self.m1 + (1 - self.theta1) * self.W1_grad +self.v1 = self.theta2 * self.v1 + (1 - self.theta1) * np.square(self.W1_grad) +m_ = self.m1 / (1 - np.power(self.theta1, self.n)) +v_ = self.v1 / (1 - np.power(self.theta2, self.n)) +self.W1 -= learning_rate * m_ / (np.sqrt(v_) + self.eps) + +``` 结果 +[0] Accuracy: 0.9639 +[1] Accuracy: 0.9661 +[2] Accuracy: 0.9690 +``` +``` + + + +比较三次迭代后的准确率,收敛速度:Adam > Momentum > SGD 。 \ No newline at end of file diff --git a/assignment-2/submission/18307130130/img/Adam.png b/assignment-2/submission/18307130130/img/Adam.png new file mode 100644 index 0000000000000000000000000000000000000000..cfa321ab862467b331f20692c04e66d4fd692ca0 GIT binary patch literal 14151 zcmb7rWmHvB-zUD!KQn6RB)QesQNOCbub^EW5gEDpJJM_z`xVcZUrt{gH3m
z7Qc2ZFIF1)cW7cU>(pPAS{_&Q1#L>|xFW|4FJp&;n@^b`2yBpvnv`)?=I(l_?RP~*
zf9EQWg)aXXKPFJf{C4zs^XKXUMF05I1tHy8ZTDOmo*z8cHz3r=WaQyPpw*z
zLu~xue+NU<)2?$SL=ws~$H(J!&Xta>#}<>@b)#j&xO& 6>W(0mHA*(L*J1P<_jb`5b4Y|)hV1G_@zk>8WZadjQT
ze(osat7XPV1{2-M cyk_6zJ=AL-@)lV(ox{9lm8DEk70l>Z0E
zUXayp`XAfAkIDhFc2_GP#_`$F7C3ILY1)Fw<&n0E2cw$-{{CYxm*GHG3!I$Abt-2V-%LUxumh{fYRE
zA0AbZ}M#1d&&yO5iZ(b>yz2WHofW}(3Ma!`32J0V#{8q?p^o?4s
fW;#I4S-lqCicj8DDY>dcVX>AUuMr1V
zO-_U3DQ?{D4;+~05)kNn3SLWlDVRVbt+5Jo*`_yJc
(Qq0Bm$8ONAM1vMuK
z{W U2zouHZ
zY*X+?%t}+SSg$L)!f51pFv%r)@eY?PUo-qDA!Un(ud9$UhUO!F
M_iIJ8k#$M+rX^_~&m zm$o#24Nn$V^9|T}YvVk<(mfmCO(5=!81zcYQVeg(n4#iJ2*kqdEM)1>QnK-=x;PX_ zBvQ;H1|suim@$Ep(p#@=@cZ>R(nQMf3TVn=59D-Jnee!^$`Q2WY(WoU%5#wB$NN1= zxZm9Hn2bFlsBxXYYnvW~)x4eQBOE_8j_5a9mX0(VTgCO7%KU^ZdwR?wi@h;NXt^Q2 z`o%t{&==lQFpPwGY4iF~eBjOBLqwuxklWV@_T&wsl#wOcJlWo;dHf*5p{OY-Bj)N{ zsp}|j5HtS4qT84+C;sF_kG4I=$TX_-TIk1r>?4!<#9a8b2o0ILK8l#@xF z`OE1I?jiM8#B8`fuhJvl?;WDh{WpD#t8{jL{R&kF9ojT>MCJ_OOgOzAI=m2*ZfOwQ zd#ERLzi_-zyB@ZRNscsq?(NrSeV(eds;fOV32=%J*wt6_6&DWQ!hm3a_48<%#`*NH z{M^PB)~;f+nW8s6tvkO5wesQN-@? w4?^ 4QZ=+w)cJ zu12(2Vd{(ZAJ;SS%)X?S3bN1wo#DPsESExTzoAX5srf>&%&_5U9K+O$S@R-d8g#0r z@k9)98~JLba%^lChn-gsJp45MY;AKvCU6OFn%P|FOBBP FE&M8z+z)!` zrut>(Ns9eoUoAnpOy7HKb-6D9J*^z#qt#Ix=sW!RL$*Nx0VSX$r=#_8SXs3uWRza4 zq4v}k#}eP$VIk^z1;ShL3!Cd21}4 a4%>-+V&F(F%fdk=FPbj&t*B;+ox=5s 8Ff- z=7ohI6m!s1Amfs-_#WgqiY+|*+_|C%J`^aY_vA3mN$}Eqm@r-%yS29bnYq`bFco?Y-eKd4p8UNurIQBgTtQeiY2M96c_ EQ&hA3QP&NW^UP zO6{Ta<{SReyS1b5D%dj_`1geKVidE75)J75Z)~M~y_{@T@4j_t`jGfq`gbWUxyIm! z$7}~Hc`^BUcJGS~`{RO+7MQC@)L;&C-uE4e24-A ?EsTB4fu=6)-gU%Sp%2o@TZky%m`{jWtK>5<`^+Itfo9H6G6v`ov&vkk#V3R5 zuYt1dNqoIlfZtF!R!b=RDx>O`Z81~-+^YJkd{EF-UGAk*`Pnz|_b%_7cwy)ce#^(w zYCQY#V>c!=l^<|}lK{teae7b6DZJtXU^Hw{= P@UmvnRk&DVr5 zdhu69 N>AE%qy`cHMU1lK?Efij|9zZO zF}~Ux5bVy$>t9-#{2%BEHh}ItI2J`$49HQdf}3IG=72SghczMzs0eB{>ladQxB$?n z>sVtME!1QtaYd$si1@)eXXDg6R$(L>R9dP c=W@b!mfCcrA&BGcG zl&$T6^Npt7cR%x&YP{ZD2}s6aH7xLd6yy7wGdw&7 =NSY)BIBJn-|+ zqsRqjzUZKuyM)xj=e8hQ3<55w#LpMg4!(eg6&py^kmKWVmoPB@uOnsWkA+Qt^Q5?> zZi8y p;#MaI5>h<531ZT0GBhjvbWQoLs`TMN@OuI5v$_hMmZ~(kA-C|wk z$$D1-X8+$WCxUf>_DY_l1CGfk^_EEyv }zN!=sC@K1H txsV zf$S6tgevuBK67WB=DvoG_K43u{Vwxj6o}V*n3P8ImE%jBFfCIIQeLI|`?VyoZQWg_ zz&{o|beA{KB$4(AFsBkAl>;C#y@x4jq~6uM #HNF(Rt{i&^usZ6rw0cMgHYH zXU3DDKg}ds_Ij@rMnv`Yg&?XaK;PFx1_t1)R}R6HmyV08tb;q(CR@#Nqa%~Uti}KZ z=LzGp&nHHuFwIs1PKKjMX{S+M7&PE$t8KsWdnA3js38a*0cwEo=_25M^v$5!R(<_D zz4_vJxIHjBeNxV2@dS&^XmU$1&D_Nt!cUibGljf2O;!Qcf-dwgjf*o@zw$7wdzxAb z)KZ@-%YjCyRU-)(UE4wj_|x1UWAxctTb^%Dae56_mz>hLx1|7sl4Jn;l4jNZetq|j z1yr1EviD&78?ocxO#r*p*5I+*#y9tG6yi!ip}Pbqydq9FNcRsm3 !kJtjy+?nn&rLUFd}?`GAYGU)6<-se8FS 0SGFV0MR=+l$<`T z>;_H^(7fV7%>yMI^aV|3me0?L-Y-69d96U&YL{S1a0Ta-PGHh$-xLPo5BK{N+KjLR zYnJH5#iiM0pkl3AMvRu5b~(}INw;ulWZVXEKar#*sl7oIw0GzPYEOPfNso`JHY6b` zkPX~zlR0ffQ_!XYL#!iMJ=0_Y8bQpP&hn1ARB=KZkX8hxpRbX5{~e-b4y?i3>z|X` zz4c3a)NgPo9u6zrtGx{Z?#lp@_}AQj>w8N;P0!o9pfqV%9imj`V38JP>JZqod%Yp) z)!Xah8mJ&*0L};Y{ODU@#cRs&&y{`6<)%%Odf$v#S2Y+S1R54;V1$18OG~2n2U=y? z+Xc}#^Q3JaMqRbQMl>uX8E?$-wBR $Nu(n-Ma>@*Me!I(ZsV<^w$N+$uboyGUB z!7~9zxsm)AHe^tro?ESh!lnZJ0;ucfHe||8hidENG=SLpX%PnWd-uks+PuJg=x@wd zx{@@`Jxn) 5ZwiW zNvkadvY{!?PoN`^QWg#NoiMI+ZQM_}FS&XIwi4+c9pqa1w4QGj*#M%u`-q+aS}Od8 z|L-sQKsAUB75rX#DO)yTi{bt4&K23pc#!EJ4<3>jxFjjsam@Y~y&TX<|MwkW*n6)S znA`;6R7@4}4)G5owjEl5BsAWDZB%N5Egp(Yo9d3WQaE&dSC;^im!!!D@$N?!pc30+ zh(If&1ZaQ9u0;UK=PZ9f^V}C(O3DYDPXuTKIVVXac?4S3$(d5#Y0?lD(w*D}LC9YO zFD#Bk>ITeD>+rIjK3Vfk1lH`J$2E|Rq_{`KXmDoxSCMuuaRa+;biE{Fx=m7gk+u%V z8pAist%F&vCZfPD5;kC4457dyAE}fR_c@*oxk4mX0cDk>IGAEL0(*uCJ24tn)jnT% zm UE{cWxQ%Krt|>6m z#TTdkY+hhN+%18$D+6|B?6C>Q)0_F^H2?leUS;v^Y_(?)N#y`0&j<|8Hu=>mP?+-p zgD?kd*p@pmiJ&nGi(;TYt@2#gm&!PCACUARi}_VHUBKIP7D>B(0ORNYn?&5?Gp #R$X0*Hn-$yuT<)8nNl zUhIPG-mXkiR@~d|mlv|@y+PvVMgji?@rCCRqi4bmxAvPP#VNGP`ZXZYYk ~bD}{Y1Edz&_cj0z|cfjsBbvlHm)v|59t3N#2~TanNb eqtgU2W mJ*GGR-=>cPmx`r22*U5*b`QHbPzSz3lod4;${#!l{yzZn=Y;nF literal 0 HcmV?d00001 diff --git a/assignment-2/submission/18307130130/img/Adam_train_512_01.png b/assignment-2/submission/18307130130/img/Adam_train_512_01.png new file mode 100644 index 0000000000000000000000000000000000000000..2c3bc63fb7fff6797bb16d17a72359f690c133fa GIT binary patch literal 46310 zcmd?R2T)UM*EWm=J&NU6z=8-Cz|chn5d=|bK> -f`T!H_sa>J;d<;ZE;Hw+)v+9_XD=HPf8 z%C%t437 }W3z3&l)lcYPaMH;Z{vI| zE0%oluz^m>< I>`H7i2dN54)=qjI!_ZmGrB{japrP0h0>N`H#4f&clD5Zo$r6%)id?E3bT4 zy=g6q`3=XhKXEI6G@J=r&HPJh&HwR>jAUM)Eol}r_awVhN~#u}x*Q7{N%^+*WvR>_ z<`oLM&JFM{8+m-Oe&8C+`$SFrfq?Dih561Z33=)p8_zAw%YL}`c@0Iy_f4gdOIw80 zjn~AOdl+}Q){6^I{1NX`>E*eTd0BKZr9MWR=b)MQFy1yqy!gknXEMs9y&w4wcm}Q& z30k@6)rXsly?I{D8v^(I0j7ZbcU3@r74whSJ9|(L)lZn$2I`m3bUW0y8(NYRY|OGt z`n+uu+&hMw%==H~BynLn0tFEB;IQ&GcCGvx#6`l#SakDQ_d1qRqHW8^i$&|B#YdlA z*oKqRSXmG$;$*W@fb{vUtMN{x54 q?|+c@H^fpJ(sQ z$QX(Z|MFaI456g2%(1XV)wa{zQ(SG|&A$e1v+{=$eWGJ31_jBbBPk&gBBO*73%RAa z0pVtTufcGNvuC*}(`Y<;oDBERzNwAklB~Ev%O9?oZ_kzRtei|JcWamL`}NFqGQPh; z-jKp5Z*+3jmFc}v<2q{EU$ze{i!69|Wi|uv#Kj(=(CT*$VRMN>i3@|mF_GGerTcuw zUQ>Gdmu4O}AG zq`(7|P?mG=c9Hhjy|~mjnK+B(?VtM)8ACw7i_!M|dq<>YmfXmzR8nz|v*}Eq&qUp% zp}D)h*SLeFsh0eBp=-m@S}ZJr56#D%mw&E$J*nPuiADqK_op%ZYIysk=*N6v&Z#+f z*aIm2`!OcssMrUTrB4pVD}sX(%)emO<&aAxB~Wjr(_V-AWiNb}t%}h-q(%!qm&H; oz;<#L$vx*z^kqwSx}S*R}g8pU!Pf*yOl0m9BKrqEcvkL&Y?~MST6f z8)v$_h7yEJntl2vNsPYPO6p$L21}02eB$0|UeXJ5H1-UO*m4vZmXwx%yU8t4*J8G; z^WuF}u6KJDX*fMe(Mut5I@>Nv+n*L0DPCkTm9i@;QtMPk`IYw8jbp2}-mboKZAXTT z#)%6e`p PZueSS#K4E32nRLlj}FrWkpi)0pX8Rr=&jor$skYoQ@#doO{{Qq-~QYe8hP$`Ao@DC!9qjjf|);^Lb n#;{?-I4EMLS#!X`sDI2d|e-Kdhv&vl1M+j z$x}%39IJ;Mf<9laM821EFRgviJ5yXD-f8A`kxuoj>VegX(3C)dW6%~}pD&B}O1Yoa zYwBCn_{}YT2JO0Vf7``OP3iK(chv5eS1m72Oh_0PJpdb~NCwa~>3{F4I=Kh8Frc(# z 1Ut6*f9&u)24F+7IP2Oi(1l2 zG;kKh3kyR^l1fXzLhFYVXG^12$LfQ_xF}SmdY_p7)aZNaaoO^&M$-Ifu97i5;_D$d zGwJ>-`mOV# qJBaPaL`|DS^k*qSZ! zj eg_TwiC`eB1drVk^IH?n3-?Oulk@*78WMOn*JXFZQD z+ZcQ8)SW3}Dy${_9Tj5@h0sq*@G}hVm53lE* ULH0zwh2X zvdiMOH4??^!RPjyOWW@2Z01BcoC%YrBut1= GeE$YH{+y(mA^_pJuZJJ#H%a~`Ui&?FEbz1m1mClc3mAgF&DV#JB z7Jm$J9zv@vsaDB?G{A33TBI7146C~N)*>xfY3zkLuK8{(%Il<8a#f9q2&A5^MP)}l zmCfl =AK|4F1v+U!EC=BDGgCz5p4^jXAM0V3=8=&d?jcbt0 zq;mfI59=`bue}#pL1HUiu%2YL%IrcT3#_elVa2dH2GB5ywrtVzu4&&UJtmcK`bN78-4|!2m rX0t(W zh)L|_uL{h&2F{1QtGMQdVL|UFJCrw0rf~kAQmwV9h7g$>7HkthuGaaNtw>2Wu$FD# z22AWO_Ch>^ySUg3p%%3B`#tt*h6~*DvQ}l;lR7-?6MNu?)vYbAu?7xSC?&}rIP+ob zfrC9xzGKfK^01feS^STOz4~X*uCiwFKYPYD5#-5d*~3I0CUuBqbV$#pif5kxw)7EX zQaZXv%SO?qBbjAenG>0+g-j$j%9=f~XW_Gv3z6G1joNobi#25Cm&Fa$E1>;DD2$*@ z{swbPaeTOgd|KC(#p<}(ADCEK_LMq9oACzWDeU1Q4`OfHH-O7sHpQ<;`ho19SXs6a zBX{w#7eW>m;wgK$yf9o<_Ktu(=9S8RGHj{}`+sdJ3FHy%=|}GRz@C2OE}JhbGeYia zVXr0Z^S<3O%z3G1c8 LATXKvTs%>J;qM|}7e%oe%9FxT1gexhS>*Hznir()FB zfVrA_@WbuqrwUwr2Y78~$)1wD&Tp(s--|n6>Vu%<5tAoVIKy;aR=rPdg+-hhgrMvJ z*rQUnk`|SVb;v1AQ}}sjt5}SsznACJQF+E#LA^g?+z^6guIorfdUOnN%obd ? GXOi2})&$tM3w;1u_FDYgak!&X$Whb7we!GWvEO#P9h{iZ4-`s~RRds~)v(X`& zqR$DI7kYoF%0BPa61LXJH1IzCLwT!+aFVYp)8QWyEvJ!0g#+kB;hbE3+}%u;V|@E% zEee(GwKSPzNd+K>MNmJvWwabg---#2DUp-*TpqvP@A6bLh&C3}_|$C=NMTIkr6I2j zKPx m%Z0Dt$0~2)l=#kf*K;1o+-}%K%f EO@NPt@(7lA*>+P zY#M-GY45nRrznz8dEPzI{;z1U50;#UBscqz3=2^HZ6w1I620g;@1`wxzT5TmuSeQr zO;N62@2rn@Ym&2bO=KoGTa*@l(q3-$d$rH^=Ueg#BVNKm6&6xGxqDcEa(cq6d9Wb@ z0C>7%)#6ORX+HPQm&hm3a?1;o^? $pq)PL`YJjaxaCUzasl9m?5UoMB#{aIUqjUrDb;(aH4@V+wz_yuym`+DZS zFxn1@u&>zt#$yU>!-gOoYsmHlTCgES*&7mZ*J!raLY(t;Zj@b;c?n`)98x5>G>bTs z*;BJcltP>!up)IP@^!6JC3aWU8>-T%F@B%_Z8Fn)1YSa%ow%t1>A(XlEqB?ZS!vuJ z@4GPI&d+?JDzfKeq$uMql3?_jND8_^`v^r?X;^{@I|8>szAvynt%5 s&NAM?qFCk?8pvi zh~0?o|By!XTvSMZooBDu0T`!=3i>S8A(y?tA~0ONEcFd;EF^FO29^n~Lv96gKiLcO zR75Q!H*I7&0%Tq9m=rr^im0$+0`nUOHw=3Qufro>uxAhu)E!;c48DR`eS BR1c3b~ z`xe;J|3hwR5%~>|w<~YtO3}x$fhzc^JOQ@#K87j!gKgsn;X*c?Oa(X@#750W0M!2< zxaDErW9*|xEUa1sSTzW+>V``>#A23} +V z(&2yN_3`W#)`RR#%1P_-KQBmSn*zYXx9kf)BNwtw0sIrs6ZS3Oia7rlZfOhJ$)5X- zF!yY1i}*24HnvR#lp4fF&9L?Itk`P@?oDhr`+jhbT&ip@_Frz Hg zL?238?rh5`r*@maOa47CPkh7jCn_1yokXfTC2-A_AI}<^5XJ#`*Yc{w4gkZ{?d#Yn zdFs`=CiT*Kb1J5|-=E=H#USS*IE;L0od&!ASL--Y8?^g?IEm2?m~W%}NDY+$M!)Au zY=p7su~LAR^)yPo;i;pZzgI(3LS5qq4Ztw5cRI&Y+=p6fCpVT?Lpryg7DkIMslb1X$h80-T8+u2CzMylR8HX$R){e=4B&25;vd(pMTx%f z9?6IluezH5b{#LaQMTU~zr*c!bU-FA3HUe~!VS}Q+9mnT3@!owlJ`< $>!jJb-Bs}93-cKeiFVrm}@G^3!S!sEp z>b}U$c74OB;H`>pdVf9h6!RG`GDNszW5s29bIh78CzJoVqy>(b?>Q$Kn^gMJ2S3pp zG+Vwq%v3H-V}Q=Y0Lzc@pGu_)FrBT94KP8C|8Do}Z1lfrb^mUtu?FF;jVJJCp5`nJ zU=3L3ZQ)}qn`pTN)DZHSC8J4g1nZ8S?ilqjkP===YMEE`TD>O{?)~=RoUjY^;v)~T z($Af+Lv>&)voprkuq&=dUL2VT4ahcS4sQ2VxZ`t&Y3I1gu6A+eSDTI@SH(Mx%)H|o z 2bjWHR40T7a-1y78o0GEOikQBeB4*>p_LF fM{>04R9AOu&EkVxYUdblT((7 zrPPlqu1p;;5M0iwPtjes+R9v5gh35p-IRYW+NHu73l?ZYCRPPv6T3695M6h&vjk+x zA9r+XvfKdkIln^?4B-fSyCN5EWP3iC@{r$A>kKrF1PP3g& ha428 zRg>Lw @pfngnI`z8;#@F^=&+)se U4A)2MsGIr1dt4V?Eu C^a4_F zvRw3Pmc4B%P}zWys9K;!iClUfB3b#yG4JiVfB&88)6a+w^&LD!F2w*hVE_TMj0Uc` zbfK|-;i4xNe}--2j+cV4YlSA@dQv5WYMN6vl1WS1MSRt}G*coeoOt{1EA^7mTmIR$ zvrkV?belE=x`F+(M?j2QBV_8wccXe0<+HA~cu56CyxigSPPQe+ipLwh?6HexXYsC} zH$*|is*oo=KRMZeadwNxINwUeNR;AMX!~k)VBsfWjIHk+^rg?)f>C^LmK(x?z#EhQ z?SA(PZ~f$yGD)HI?dF{ROJZw=;z0l{?%~x6!qVMZRPc@k7s<`Je$UI15)TfJLzf|* z?6U2eN0>2mIzq7cf)Xg^38B9}rfPvgH4)= $SVZOL3ZRQiin zzCBtZnD5$|IvwUER7>BRC{ri>*%e0av I5sa47;*@hPspQ5IeJDn@^4C+!NLH>y|9nSDndbe-$5^_14R5sTFdl#STzNZT zMe^Wy^5UkMl4YUw =HFv@YvRKPL=Cxb5$}yrhkw{gCdUEzxacFty !^Z}oTkrz}5m^@QjnsfbY zn;N!et5O9hu}VL3-eZ36_#)WxnwImew4hxT$qR@?i!iAO>1p01KT(knl3tA3>5)t1 zA|nx*zBkUfZ&Fwyk&zfW%=N<_JFM{OVU0+}Xim8^9UzpM4Ea_Y?RjTKTE*~1m*)yi z|MD(KTSD=Un4Ah4scw3nKep4-x)*CfL*bc5892~PEMKc#E zj)-(3`u_~(zZH|WlkL=v#9 4`ikAx`hE*f~ zHtlGnj)t `(!~e6?R-RfZcZG&z&luk0s`5}c(dQ^at7w_H#!4&c;p|nf>6-T`)3^{euqosXSO%q4V2PGc259M z-49|fBn0M{$^Qt-|C4WN`M4ehq3beuq{aFz;3AO_h%5*-BF=AsxBp1_M5RyHjhfB% z;?6fiXi%t9QhxXS?>%$;Ka)@W|9=JK{qNZ;L^7R#LHt3=u_XEhFDT&Y<14M)RW z++?V@-HV&5o;d89tqOTBJi6_X$|DcYEA9U;0kZqe+~1huzA~z%+eOr9wk|8;@D052 zuN=IJJ1pk)bq)WQgslBj$?Kz#OwG)1x}Cg@$0SSrQPl#zhZ(Rf-3Lu*l|fgcO!s zDl(K?`u(} KM4RvTC zg*cuYChb$7ju^0!$7M*eku;f3GOTP($f|@;6;n 0YG$C9HVURcVXD&uZ5ufDA(b<|t2&H63}t z?|+;A>4Bo&7R9-*)V*RnY&;Vmz7?aTNFQ{KDetwV?cAWAzYM)`r7bXd1|2~UC_Hq9 zpcn;2#Y+gVo)Sm^sek!kzsY|3M>UU5tJmzU^+hcHT4YBYqGM@qNocOv=~Vf>dA|>q zMxKa X0(gdhd@T(*)^zX$w(O$@XXC`FkqnqAQ(cf)8zzO#JZ1>m=% z{w3npK!&BmrlTOckkVtR@S8x^ImhONkrd_S?+pze@D*FmN4z6DJzu$nZI9q+by&5J zQp`sCIxKeVVC6hKA@IOf5ex{USf!=cEdK8@(!NahtHiTDl1${q9Cup}t4xC@5^l0# zY1=iRK7V3AUknDt$Hreg(4}YWC59aM ?yMALhzUVU!>Y2GrmTvR5Dl0(kkD9JbY3isO^`jdbuhLF zb|2Tkof_;uf^h#5#}bX1i`vRk!mXhbN7&^YQoy@o*~vO$7ie~}z6qMa1F1u&;LW^j zSp&ESQ*^}X_4~*2cAgW*S*G_MiGYIVOTTR%t(;kzE-@bHN 7TZz h~LoH8R z;uXg2r{Bb_h*Dxu)pMxk=SpD&&vSu$;Vt)@xmf0LBxt9qDi@Miu@LE}wWueXSe{Sp zINX^k`wu!uedThdfiGsT3|s)#eh;sU8R 0m5|XNaOGo`EGBjso`dHE42WUr zr^s{I={`>cbcxN49e`({STqdt2dDyd7iF3H0a#iTs{$i03_ALIHJHz1?SOmWEpV)z zZ1f*LI2P%Xh5j+3BhV8x2s Z1w0ERVDfx}+|>QaclP4>sCiCz!NM)*>MDewe)r{(!Nvs?b-i+`zJ{|M;Ou z0E$srjvwiTdiV9LGJTfRiS$5&?Hc|>8emTXa^?0FC!dG>faAfzXr|XP{aG-xZ D_SyzkdFc;BDFWNv0Zt)4++3F9TX-|FJ~&6=yea^he3IJ1Ud3jHij zWb}6TjfMt@QS7zb{$8n~D+OCFW)A$>5!;&0ngGkL$FD@SSO_jQ;|3)nl+7-$81}#1 zo8(wVclBJ@yDvUXZfH10fTp)4p}qPhS=@ybSJ2K67AC&LX@dLm-5m|+72$LSQkV^% zZ_c6WqoZgKl(4S9-m_Q>qs_`2HQ%Z7MnPWczT9w(MTHyKi3K4gx!FjO3v^6Y3_ xyc>NAmMKdM_jQd!TeTq4eN zXN|!lC{Q0io^zB~&~z&6MZgk0;Pg0 $T7o*l zh#Cl0s+2!P6wR&37qhGcttZt^UzzksxMN(~pXGM{Jj91H^ZoUn0tH&)pn5rZ;z)D^ z^vjpz-xZz!z*UbNBXON-QHwze@1-xuy(Zwa2g1&6L774YvZZ#$ZR@E*0m@i`q4PKQ z>2^F&A@0kaRXHn8=;7<#GD+6Ih(ba_IiA`7rgRMrCeJ@m7(B320sVB;XIRtHL;`a3 zszih&fqGvJT5+SSBn689==A3DU{TygJf|ot7I;6s73KC?%+bi; zeaNF0 Cd{trv68lz+ZM4yYmv2N$+>Xfar0{j=5QkXsVS(ortzYgYxgZeg1Rb zm?;C3`UBC-6v-y)>f546AAxi>=+9j2+I?;fS2#(b0Sg0l^~WSB$-!Gja^9YV60Wjw zf;udyb2=PHxh2nRLCg!x2F_N5#%)rwn0(%em4_<-$a4dsU@`~=bJ4;dI6>`oqK*1c zn~G#8uC?-gug8iyEhzE*3s>n-D{r~w BE<4$CRalts>uEBt$ih*Q?Ns#>tomdem z0*!frXbaC(zf<~#BCZL7sY0w~>DQo)Vv7yB9=)%rtoq7ip!0E#dCDmH2{Wih&f5IE zs)vj&9lkhP$#TRX{YJEX$Gvdx& 9GajK1Hn{PTgk5;sb=RD*1| z5+s pZv&IZ!4*9g}e)!V!&)80ou;t5yo3Kum&A z*oq? DY(-HVEJs%ynp8gP7j54Z?7%ZoIM>5+PEW*?H%<+2MpUdNGbDH >>hBj+oy;FpaB{VEv@KD)z`M; zs*jua