From 59ae1d6ec8a2af7a0dd63de8a55a0327d8037e4f Mon Sep 17 00:00:00 2001 From: ZhidanLiu Date: Fri, 29 May 2020 15:49:49 +0800 Subject: [PATCH] update differential privacy tutorial --- .../advanced_use/differential_privacy.md | 40 ++++++++++-------- .../advanced_use/images/dp_res.png | Bin 27034 -> 0 bytes 2 files changed, 22 insertions(+), 18 deletions(-) delete mode 100644 tutorials/source_zh_cn/advanced_use/images/dp_res.png diff --git a/tutorials/source_zh_cn/advanced_use/differential_privacy.md b/tutorials/source_zh_cn/advanced_use/differential_privacy.md index 96a9e400e8..ead1e0a5a8 100644 --- a/tutorials/source_zh_cn/advanced_use/differential_privacy.md +++ b/tutorials/source_zh_cn/advanced_use/differential_privacy.md @@ -56,6 +56,9 @@ from mindarmour.diff_privacy import PrivacyMonitorFactory from mindarmour.utils.logger import LogUtil from lenet5_net import LeNet5 from lenet5_config import mnist_cfg as cfg + +LOGGER.set_level('INFO') +TAG = 'Lenet5_train' ``` ### 配置环境信息 @@ -72,17 +75,17 @@ from lenet5_config import mnist_cfg as cfg - initial_noise_multiplier:差分隐私参数,高斯噪声的标准差等于initial_noise_multiplier乘以l2_norm_bound。 ```python - parser = argparse.ArgumentParser(description='MindSpore Example') + parser = argparse.ArgumentParser(description='MindSpore MNIST Example') parser.add_argument('--device_target', type=str, default="Ascend", choices=['Ascend', 'GPU', 'CPU'], help='device where the code will be implemented (default: Ascend)') - parser.add_argument('--data_path', type=str, default="./_unzip", + parser.add_argument('--data_path', type=str, default="./MNIST_unzip", help='path where the dataset is saved') parser.add_argument('--dataset_sink_mode', type=bool, default=False, help='dataset_sink_mode is False or True') - parser.add_argument('--micro_batches', type=int, default=None, + parser.add_argument('--micro_batches', type=int, default=32, help='optional, if use differential privacy, need to set micro_batches') - parser.add_argument('--l2_norm_bound', type=float, default=1, + parser.add_argument('--l2_norm_bound', type=float, default=1.0, help='optional, if use differential privacy, need to set l2_norm_bound') - parser.add_argument('--initial_noise_multiplier', type=float, default=0.1, + parser.add_argument('--initial_noise_multiplier', type=float, default=1.5, help='optional, if use differential privacy, need to set initial_noise_multiplier') args = parser.parse_args() ``` @@ -100,7 +103,7 @@ from lenet5_config import mnist_cfg as cfg 加载数据集并处理成MindSpore数据格式。 ```python -def generate__dataset(data_path, batch_size=32, repeat_size=1, +def generate_mnist_dataset(data_path, batch_size=32, repeat_size=1, num_parallel_workers=1, sparse=True): """ create dataset for training or testing @@ -167,7 +170,7 @@ def fc_with_initialize(input_channels, out_channels): def weight_variable(): - return TruncatedNormal(0.02) + return TruncatedNormal(0.05) class LeNet5(nn.Cell): @@ -201,7 +204,7 @@ class LeNet5(nn.Cell): return x ``` -加载`LeNet`网络,定义损失函数、配置checkpoint、用上述定义的数据加载函数`generate__dataset`载入数据。 +加载`LeNet`网络,定义损失函数、配置checkpoint、用上述定义的数据加载函数`generate_mnist_dataset`载入数据。 ```python network = LeNet5() @@ -212,7 +215,7 @@ ckpoint_cb = ModelCheckpoint(prefix="checkpoint_lenet", directory='./trained_ckpt_file/', config=config_ck) -ds_train = generate__dataset(os.path.join(args.data_path, "train"), +ds_train = generate_mnist_dataset(os.path.join(args.data_path, "train"), cfg.batch_size, cfg.epoch_size) ``` @@ -221,12 +224,15 @@ ds_train = generate__dataset(os.path.join(args.data_path, "train"), 1. 配置差分隐私优化器的参数。 + - 判断micro_batches和batch_size参数是否符合要求。 - 实例化差分隐私工厂类。 - 设置差分隐私的噪声机制,目前支持固定标准差的高斯噪声机制:'Gaussian'和自适应调整标准差的自适应高斯噪声机制:'AdaGaussian'。 - 设置优化器类型,目前支持'SGD'和'Momentum'。 - 设置差分隐私预算监测器RDP,用于观测每个step中的差分隐私预算$\epsilon$的变化。 ```python + if args.micro_batches and cfg.batch_size % args.micro_batches != 0: + raise ValueError("Number of micro_batches should divide evenly batch_size") gaussian_mech = DPOptimizerClassFactory(args.micro_batches) gaussian_mech.set_mechanisms('Gaussian', norm_bound=args.l2_norm_bound, @@ -236,9 +242,9 @@ ds_train = generate__dataset(os.path.join(args.data_path, "train"), momentum=cfg.momentum) rdp_monitor = PrivacyMonitorFactory.create('rdp', num_samples=60000, - batch_size=16, - initial_noise_multiplier=5, - target_delta=0.5, + batch_size=cfg.batch_size, + initial_noise_multiplier=args.initial_noise_multiplier* + args.l2_norm_bound, per_print_times=10) ``` @@ -262,13 +268,13 @@ ds_train = generate__dataset(os.path.join(args.data_path, "train"), dataset_sink_mode=args.dataset_sink_mode) LOGGER.info(TAG, "============== Starting Testing ==============") - ckpt_file_name = 'trained_ckpt_file/checkpoint_lenet-10_1875.ckpt' + ckpt_file_name = 'trained_ckpt_file/checkpoint_lenet-10_234.ckpt' param_dict = load_checkpoint(ckpt_file_name) load_param_into_net(network, param_dict) - ds_eval = generate__dataset(os.path.join(args.data_path, 'test'), batch_size=cfg.batch_size) + ds_eval = generate_mnist_dataset(os.path.join(args.data_path, 'test'), batch_size=cfg.batch_size) acc = model.eval(ds_eval, dataset_sink_mode=False) LOGGER.info(TAG, "============== Accuracy: %s ==============", acc) - + ``` 4. 运行命令。 @@ -290,11 +296,9 @@ ds_train = generate__dataset(os.path.join(args.data_path, "train"), ... ============== Starting Testing ============== ... - ============== Accuracy: 0.9635 ============== + ============== Accuracy: 0.9091 ============== ``` - ![dp_res](images/dp_res.png) - ### 引用 [1] C. Dwork and J. Lei. Differential privacy and robust statistics. In STOC, pages 371–380. ACM, 2009. diff --git a/tutorials/source_zh_cn/advanced_use/images/dp_res.png b/tutorials/source_zh_cn/advanced_use/images/dp_res.png deleted file mode 100644 index c661414fcb6dc82206c1d76f813f9cf4e7fdae01..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27034 zcmd?Rg{*HZuO2GGi%mdv(}pH+M%k-@_5*k*boHaJ$fjk4nfEW2tt~|LttpAg5Jf%!O`5-=04{=PM*8;ubiChgt)l=_jgWP2Q#jW9ib5jqK6*INNTtxt&O{T zk?c2#Z}qIzB`Qg6#1Otk{N!MH*oZ^$APaLMP&1(Yw2^V(1e2CUOv+NH5k;0m9&G@@ zSSB79Mn5m9rp?GwA%P%t;Hu*D+&sH@KmO?5?U*&2Z#6#_yDatZ3wwGq3$J+{c>cmn zX9=TcWMoXYktSL}CL|&vQd@ih-3H&Rm{LHYAt52vH_;`o4hTYu0UvV`jsAb|bB-)m z{ShOo_f)PH)@&U%T+-Y(>qdosrHa$BYZbO;M2m$z&C{Ho z@Z4Pg-i7rYgU$=l7ET^KS6BCGMEE8yIxe*UVUOq`wS}dnv^o>xN(!nPNt-IUUx?73 zpYiN(5)y*lcNZ9hg{km^QL?hK{+3%)@!3v9Ow@VKmh_0VWF_dciFYhJH7`3!W%vJB zWl~9rpb>VN+FR}yD-yRDV9C*7Tl@Ppqrd!zLB*H@&cXWVP=2{GiR@o1zMMe|miV78 zb)OG6b)G#7$Yrw~N2aLa)c9 zViXCYjQp%;a+}dYffTr*+Tu-q8)~@-3WmuPQBS^doAEFmn<_M5&(PmjrR_ik%@B-Q->#0YO96FeOi2R24(g@5H=3ZyaPV12t|$SW_#I?K8}AALeO#H z`=R?O^e#SLGE03LjE=>@$6x}wAbLzUFdj${Q!JK>=srmh?oDJcWbfZoL{RbLdme6B zAL!-{hLIb15hdt4U^xlVcpR*S3U5@RbVSkkvxI&B`Lns8Vq#=w_l2?X_Erl4#{8y~ z3{5aELMFg-vyR+x;a5|Fflny#+2Y~zM~@yoE3+Wgcb_L67#cEe@GNmwqWTlIjjBajugmj82w|4$O~%X1E8u9hEV$5vr^swa^erwfg52EP zzYV_P6W)hUVy5-mqp{(d!v5CH+G?s`w3k^;8~{t?S;{491+53{hdw+L_6`tumKPc^ zMe)nfkr8e1{T&uxa>f-2Mj~f-cFlZ3q!0iuzS`c8t*r9hhD{QFR4jHUv!|Sr@|gaj zu`F=e@PRdoPJF^_{L9lknLsIqi^9|ZyP5DNHnwzDF5S1_5u-%gn_Y+Tm=g2}c09odpCmL*xjEqV;_~*|PX~n#r z<>%xO;^5%4xlXvt4_K7FP*4ba#g{0n7I)DNwSwmHc}4=Wg&4O-ys>TA?H2ATS0j1bbi|J5Zh&bq6UvOg;jvPY4#3RO^0@L7pzq*v;Xm(5^GzxLdyVt&P&Av0hPKu`wCbUePp%#5+V?{jf% zMlIq-2<9utgn_>PhNG~s@aUq8tSs7>yu9s=n#~#D#n%0ndmH26Yh7JxXliY^WSy-h z0bE&;t1ci~>?CK{b8ukbX#XRE)|L_m4i9y2Zmo#6RJ7 z>rbIxH~@*8h=H>_^<29S8rSd9jy2t#b*e{^g=6&Dk7CMebynfLF0HSXvwGH5VG zL3HyvuV_3@6{BHRNs%!%Wdg&45!^!bCmUi+m%|0{)1R%90j4^+o|9UyK3E-$zsE2< ztF$u;KQw#!@@ULPdvw8D41y<7|7Ru3ZQhS5<>(T5*|tRIBBO&vKKtxmr|T6*Ep!(M zqn4niR}Y%kU1rG}jskVXe04q7%XV0%{v@3q3Z#NL4`YPq!CF(XQA@GQy7rjcY^2yG zkMD(-$mQ8?8M?7!Rkit^HlZ)RdWdTSi{KW!}LEtjm(@ zS~{H$2W!u*hQFb4)$I}MyG^6HpZADgQkQ;1?X<2K4+^I9d1iwy2B6clCuM#2eYE%m z1@K+bcN{tkCH2+SBg`yd>XA?SUW!uVV+>e0=WX7kql@02?SR*?%Ec7VtuFq_fHl-n znE{_Ql!pUd*5lQrTn%UUzLk{NgU|zY-Pw0P&)`#KCBp#GYgMJEr;iMHtrjSWR2*$jEPB>*U9_O-QE{fPQfG27 zrbo3?N~GJ&g>%(6ev%HJauQP0)C?-nsd%xw(DnV-uT~i92W~J-g0<;cjSB~@`y7$i z`TkJl%9p$;hbp_JUWR}bmTuTv1=pMYJBzJMRNdF7cKdX8~&C>b!0F?1ET_u-=0q} z__?@(!yn6})Evk91omwPe*TJKtQy;@d9q)L`~>dq?v*<~*|*`v4p@GvKRdc-_Oo(w z*x{rvU5RN|bD>_1T^H|AqZQw5UQvCsakS`y<3$4)s9OvSNUMJeS}N^l{YHZ0LWIar z=+*dVtbY!?dJqyF-4VkeUF^2~Vs3t(K}bj_XERHc1$Zm^UF}lL&4x=cvGctr6u9Uh zV2-H$4LT29(APzI6+~ zKA@BgM-Ye|LC43(m1jGhU+rh*M0S7O7mI$v7JqcM+q3Gf!7PAvR zXwkMZmF}mLV;l3Fm4bA#cnKDSHa&|-0L#nHV zGPANIKP{y=8{s051RNJ|@7=p6rH2dhtQ~1JH6jrIkgmK4Kt(ZxY7@HI*w~~VKYkpd zC@mu+W38s66CM!}k!FB=|1CF!-Z;8Ob>l`0^iW2I#HiZwuwY|&O9>n|5Xl1(gLKaM zK7c^*XB-!W9?tIguMU+N=yLi1ReGPgJburOWNvOwh>AqTXH_}nSZ+NU)XCLAtzAF{ z0Fy|wPo$dS zwrX&>GW8a|d7x&3Ai)&yL~fh0N4Ebn$ zwa}2;=`76~4lOHo;N||g-6VXmyo+@l9Z^c_@v`-1%pYljNVejzbC1LuvY*kD#4IIc zlkq6ykv+ppK`!U8xd zUJ9Yk`p_}3`93-2qJnwdh@(z{hC~)gnmS9GCblYTR+ToZXhqiPQ*0#4EaWi04O^FN zDzX|J)0!K2F64Jz$SFlS!J9SC1TzCo_lrUW8S|%+zOH4&Y$m&xPrwaI#zTejTn(Jl zx>u^!hkd(TvTJE9vXCv}ktt1LM-|L3)!FMtncc){+Iu#=QVZHkd{+kg>POTCxmu?k zT4rJ{ILT`E&R^J9(nB1+VoGAZZe;kH`6H9_= zmqoP;-!oh~e}JAD7(|2N{#gM+&aY-Yw5=T-8LpcX??FBq79PG0G71z3UNvfu7ZTl0 zmb2|aGGGLN7hX;HY{n?QC@qzbrCcs?_DKF%c4%3)sahG& z=zC%P8Ctau4ktEl=J~hWnb~{JIz{Tf__=*_5D&j$LT6U+eeOqU7L5s%z?rfAKFN%x zjkm}tU4cS*D5fZhqNQRAO1qF@8B_4pp)m`~rseM(xX+@PpaU z%DISHQIFQlEL2q0SkVOZ)CEfZ>LH!`?)2r5s8eXMPm?ih=1t4x5>g&MjVIPFLl)}M zB}9Feh$^z_zq7Tvr=MEJiAtU zW+rO3a`M~F9)JrVapG}YcwBohV!1S!M}Bco(XdUz?ek4|T)^aBM)5Z!rjo8jtzy@z z?gg2;G9C(dUk2BO!)SU)D4qm;-y&D8lL{ZLvEonHa3%GZ&4+=|lE{3VU%nJ)gKJ$U zM|$OmU%wfZp>w60y0Pq|NYWbR2fm`V^i?qe{*-kyxDP{rH%}8&Sz?Mf+A3wXO~17q zUgON=V|j*4c_%`f6ui<)*`I+Ghp5gdydfTX3<&Qg5 z8aeaXkV92}a5T+&%Q?E!B)v3dZ;xwAe*+gKrsuKskgf0%GRf+PJ zDqZq%R$bXdiXT zv`|k>eOAZi~Sb6p2}+rZhG8kLtBbFT$9V zUUUt4`lpQa-yR@KXG#AyF_Evixi^eX@_|xS*ZK_snOuRwmzxp%-^=`Ori?V}+k42Y z4p{rBBxZ*eG%@fJc@WGg5l|};EaCUOZe1t)bQn;}Qh+U2KrB}<(%+hktq}XIK}Id0 zt4UFu}}X%23u&L3BfP0(?1 z9%Dqf4zFx&G;mgqy;^JkUCJJ8Hj%PPnJ~&BE2N zT&eofqA`?>757;(XYTSKUR7$Kh;8-Vu$(dhwP)sXMp-xa%l}wPq(4x2$t=qts69jx zAu|2&t@)^Nl#8 zm!B28ZZ-p%)(hP|K!(Zgn4a)&_m<%q$==(a^S94$1iN78s7>X81kWd+N1MM&sfx&3PEkXEV z^x&YS+~~s>-jY1f@|eeARCn{amf0;voly?AkJx>GP~|H)cz`HrAh6-*x)Zeg5fg7~@@KoL~6&SE!x2N4tsz zIC@!ld-L|Vb8vg}_IvX(A2G;W43o&H$<$@g@+IJ5RR`eTe7WRMGexWij+FFW--tru z##N&J7}SOFGsg8P-%jB#DP0oRvxiJ!jJoMK3X;PcTn`gQ11UZ6zTADv{agvWb{yi% zu$jWK_M<#+ylZji+bCXvLb!%Bn+K&Q zV_S@0ikj#f%6PV(8+%xbqB+`0?=&x?G_y&C;+4e&JemD}(sSf_58WI3w`eDndX{H}BFQ*@k&#%{k}mVA>yj zcBOLdfxnijmU)3)CjOeafy!C%zu={xmGJJV_EMbCEAVf;$E!HX_cW#&&-1j$F!8s0j~poCvX`7P<+E-{rb4lU0Xsns zaXLTMY&hiuvye^jOkme=_jekdDRp{vb5trz^+yx;j~LXvem1(&FZoYt)aBx(0$H3% z(lyjV`rH;QCLd*~PUr7im3X&I+l-cJ@3Ylyn)T1!4;yOB_@&E)R#*K*>~+ykZPLB< zreRJV{69>lA-mYCofc+>2I&x(76z|>yD-P zpn8+0B0*qH@K={laO(rj0=>257P#pTBRjihL)^y{m>F`ErO%A>nvD95Sl2#e7Hd6w z@^{9JziRTOi15mA+Pg?CWFrU{(>1NIiKAN0lvBDYi<-CmWBU(%e+$d9Go0#nO@E6- zr%|2QzUME>zB&|0_)S!}&}-A+WiRcS%Q;pjJ3RSVakFbQQ*&4~^o1i&*lO7CDJx9v zo5l}#Li*d9(%xEhL|A0#J|WAB@{_bis9DD);IW}-V3WrA5o1dtWu*nEf7+`Wcbb^8 zlE|&nNKyE8XQ#GV{rBgze6?3|4XGkScN)|98dplopWzIYGJW?<9VAU+Uj~$ia3KK+3s7)RKXGNabL=qrMWuU?)4r(PzWJISb7h&#H>(=U3D_c z{e&Ni%%|f(;$7(Aq6uffo27F&4#VYpMyz_X=K~lG3G1hYzf6_bekN)bD z&Odt?5kD;a`o~YQ1nSZU)Ew?tR=A63QsG3ZqttZb=XGKy_A_e(bJ@B)Lod?ZEBRh{Oz!w#d*N}R4CcA#{7vYfnGEpe@$61$OQo$RMt~uuc|11%BX*q z(mjy+8X$t)C&_ONRc)+~QUge&v5E0QX?zUD>9R*d!-BBm_~Ozt>rv7fM*S)zJ8yc~M`?CFIy6a=x zy@-{Yd<{u0f%KRTzBJe>xV|Kh0CHCfov)~p`-`}>W)#oa2^*Ft&i~E;>*K)as(t!_ zQ_A1w|8M~Us1L9$adg%WO;m68a@EtFwCzi4T9Mr|zW3&4a%hTNkO&AW5J)k8P3m|W zH2%+y??=ULN`CkQ>HLJA4GQJP+Q)XcDG5`13XglDqzp48)3&a0S5(D%13-UH_jZD( zuBKJ69K<<7*eLX6kuAvYqfnm8i}v%lZ@kKBWy(OxGfuacSQV5X44tYi(9e~QQ%A|b zlo6y^VrBbn(2s)R^kd0(E%LT8Wk3$91RX}sER?N?omowb@M8dFv79@&i19c5=qvl;#+S_chm@F0tC|vK{L<`Uy;ZX2 zg{sozvG~@ye5F|A(nTsUDhh0=a*`$zal@#_^F7?}Xe9u|oJh=t+MW&SS!Q&}Y6OD%aXglr(Y_6=aZJKx@h}L`OZ7h%PP&U%dPB z^D5G9-2i`4^{?98v*AHE?A(+`KIE&kXI{kG4c&2lWaeAMg%M?g6lL>n;SXC{K1rj^ za+wD{3THjm!#Way5^{lay0oJPMA_%#)Al`NkH{D%?^@=hk0QGz68^b7#oCV%b+m{j z_XsSO$;lJG+`Y)xs#Lc`d)D7dnUy@U(nZfSyg4cVapqf=yOV_*Ds47RjsRxTFiPCo zn+228*Pl^droVk%`8ZAfGYRIb$S&;%2`lup9o^S*B(F#At2bIw9ssNCn=En#Gf_TV z>Zib*xBE-pd${!8y5bMA^BqM0$kFp^J;aib*~H5_1$J+ruXRW!{HNMZHHas|uTfHw zf-mS)J-d&Gqsq3Da$ZgvQY4_=*2m4nMIWfu6jZxCWo`&I@>{;In@*+PN2LCFR6)wd z+PgfPiDGZrc_PplgK4JAR9`p}K8T^2kyb0=Gpu-nt4$PTqec;kz~Kx+FHy7?%&sTY3y##(g6d14NeiG|9&A;savQ*6UPih;eAF{?N zlN6$8KisCK7j5F^L+PbNvM$|wZx|$#jmIM1OL3rZ;L)s)v&c8#aqvWE9Zg>&=3?W- zZE6i`%8R)^N*U!XClWE&o0P?MCenxO?F_r#vhb; z>0h{{{mhWKAjy?`sEW+rfE(eFuY_?QfUGa|Ifj_X-X||&mH4z~rRV%cKM|~z50?85 z60!?%1x`R&gxwci9i2C*F0M(!Qu5wkYBCjxy#N`{94_u>3Pex!$w;Zs z*w0!&$w>=+C+1%sBy1fq(8|G*rJAnl_AXv&A*_AWrDX4B%A)QYRVoqQ_uj7unk0(e zNSxR$-(P#onO)iwPZx7hj~%b!pWEjSutdq$64B?az9d|WRm_`fJ+u#ohE8;S zFSJF#RKGA&0Ucqce@C~6_g0@xw0Bp1IXIc0r2P<2pH5FLPWu6+2=4_R({ixoOxyIh z#=6U+N#8r3S-E@ixpI0JDQbL|6Xrs!aeapijiG&)>xDbBnXxjzwrfy$z{oJVq=A`0 zMH|~3$%9tuc_+r*B#!{Zz8#iP#yY?Gtk_3_>PS83J7wC;XJJ$@Qg3+lFjQeV6EB3G zGngdpabSH@uM>3+d%Q&vaxMO=JSR~~=@eJmg!xHHNfIQ-z)}vpr;%M=6HL>F#e{dGygLU-o62@+rSF0p^ezv);pR(X3F7;uQW027U#j;;g#Y{y( zscn&%L}G`Y-Z=BYezg?#&n5winsIdwi(^UFwRpNXpVp_;g1)NOc{wi)8#x*gp%Q-v zUtew!%jVX$p{a{E-xq&a*2fX(VjZE;nfC`}g{5bVzKb`g-K5Oyx$1+Unjp5Go#A#9 zGj=n)kglz>XUoiJ&8M9#lm7uFuRd zx+=PAk!4cz{A8bNXe{ZzR_c?Ij7X#xp3YsAkXqbH;79f>h%EMxN2Ia*4(<0ZQOHll z74;~?F8XJtSSW}yN17toGu?mE=gJNJhW=7zR<+E0pzQvTn03f1&Vq*S>D@W6Muo_{ zQ15@xTWk9Fx0KS?ezw5EiD-UfdoVE^wnrD!6Kj)P-l3`}at$a(1j`fi# z)u39=inhh2eRQ6sy^|vCUt(LO*JtK>($MFl{!I&ei^KVcsytGzH05N2d{Ak9S$)UE zkM+0p(or_(QC`bB#&pR{gV^}*G52UeA?YiNLIDF!*;0maHSt@_^7tjN?Hf8RQLDrA$ix{K2Qas}J4hNkj#>%VHXD9%ys zV)_<=l(#&iTKDS;JKjg|Jd9LWU*n(;!!0vIr0ez3e@@p^r~CB|fA2!wA~Wr+NaKCH zIAq1>Ut78J=3JYvoD`jL7%JltYc3uqx0oMJcf~*yGav7ED2_9KL=Qxd-Av00Ymt~Y zN~^}?7;42)KYCb`Z?8b(Fl{LXl%Z$A#}>dIbI>9yu2D-bY5dS?X_KYU=3Ro!x}I{XB?hc(vcRnm$A=b4D)^vz=tU2L!ATbZAKsrsh= zyj6FJM1AIKExanJ@s~#OP0O%UZnPuL zcmgt!X2&KWNMkPRZZrGwnsLho1(IO@;>EVN5S(!t9fvl&*E^mRlhM z)yO~=WZ6j+;mNoYM>qMecL#PM;f6ooh*;?%Pr<5NZ1ARr_z{bR-lKk=9HBtgBC z$RCM&tSU*Jc_LXcQ|*~P`S@+L2C<=J*MRso^QPM5T?mRjQED!pGosPa2blu4CoW?8 zr#1>nE!Y0$Zc2(oE?SWCTh7^cB56uoy9W~$Y8qN_PH3ttmVtr;cG6|u_#tsqd+$r6Qb^Bj&=*@be>na;z$CGO#o@qJ&Od1TFCF# zsSj^@oFo|F3aUET!h4^Z8Vxp^j5fxq0R5`5_mlJ+z%2nBDl)puF~^?iWY>u%K%@cm zDg3E7^0ZCs6_ZhPK1ZKo6(^7jh$`LkCqb>_3w!fhQAG$+iFCsbJ!v(37X^|fM%{!e z;)lx0_=S}-*zgV@1XYYiMnvF>Y`5NUTllcR3-+hY-42G$wvT3`J6B=e>C(OjAe2@s zZQhK!s*e#^`(RGSdj%&AP}jQddJ+nqoW#N-dYd|WGPyu;02*0$S(~V%GVX|qA7UlJ z1{>mJ!mb-u+k$+FctXjx{a^Hp0QXW@+nWv$PyyX9m37R^W$nI~)(%an0*5}uIdTgf z*v97Vzk_UOVsb~9lN{`qbCD&#ee2iP*GHBb3ycwnLq4;Z>Kg=j3JYC{UwzI!;^Inu zwl~pm$msd_G*bXC!LOR;^_OoN(><5x_o1n;M$w;KA8!&S04szvt0iC}E?|0=-F6#! z1|Y=szIyOp%v%W9I8C+mg82j3Ew!+)n9I${$+@BEW#oI(ld5W*CkaY5?}h9va>8Ls zgV7g7)W+6U8J2j3xcKd|_&m#qe<{&&OL%u|qjY}=$gejLnV{K{{7)4{5ENniC@_{RWiwr0nvo zJv?#Gxgj}$@J>dq)p)B5!1_qXF@_@SKUYR{OoyOJ&AH&4Mg6#k|jyt#$DSHXE zkrI|nSO|k3P3@o0nbf@nG-%KWKG29?qi1;}ajL8u44io2G+AtL(4W%FRC>(+`oh#+ zL8)==QXcMI6GYK82Yru~fGI7)z}VcIkJ6r_z)Y5_-)g-fJT~bNMJe#bn*bzkzpn%^ z)8`nh41U}9?7+8k(VV2v{|90hz0YRa!haVThy!GL`SPV9pkWnj7bI(pG+y`fIHMD_ z@$Ag&*GFC)$Hi`0Kz4&+cx+4;5T+`Qj$po4kH{|WLU(ewKL){z;cq4|{|&Ghg{tN* zcJYy?oe}WJidH{{bSv!`XuY;10qx8PFmj(&JF()TqX4b2-4QD2Ko5m%>syaHW_XE}Y*A(Nl4t? z5*h|1T`>B;?^r;9{uV?)7sa7d{(TjIO}W*uA5aWb93=$>1(`Ca_$7r(axFLCc#m7+619M`~*{x$nR zY7!8Di8gO1I7N~SJ-ePgEnu(sE@yaczSg(vISw{ArQ8#JqT?wVrODsRa|N4R~dyyJ(jdwnV-~b0` z?ok@Rj_(9|Y!D~lj{=T-@3PbPA3tCosI&-@CeGfp>>T8#MTcDkvz-nNQYxtFpyco^I3#SIw<2uA{tC zy(y(j42&BlqsIhs>Q)j0e4__zJh$DH#9h5=5_EqgAk(qkhW~&`8=u{k$lqefEh7WH zgt?`R*fJm(a;c$##Wp|>GVcsH-XL~s>rNIP-J|S$$I&wPBbJDiba2J^ADQv9>XgY8 zSX)>K^%x;dP0i+c2ZlR$+RVHa6tEQ(6}gVi^lDs;Kug`GUSo8hR{mE|hRjycAb=SF zPDcI>!n-t>>k3cKVv?A^q5%u*k$Uq_gnrwzwHFtqRNf%E#ven&|MV;e3DAcr#JmImz%Uj8 zPCl$g0SYSbiN!*l3VaZeKKcB3dFchZ(tn^*8nVk)5f5dup%B^nN)U5N}zI{6&{jZPF)s#kK%tYb_YM#S!S2~~5(aCf` zzeB3~lldrCRZZVp6fEpIYeFaJwE0}327ukG% zI2jGZ3&!0^f=4TPMO#f6bOF`xjr^~o6&6AUJOmmV+Ow}O(8SKRq*+L?A14cuhLdqf zU0$5ecZ~U*u2;^4u}eMsxn*qs4;HvGk+#Cv3mcXA#{()ReW+kPBVhTLbi6b(Gv1l| z!N8&iXo%O2bw=I?xt$%^zce*XIpk4OQv-LBWsYaLXLl6^0gL!c{Os)92Cd}R0hXMc zZMZzQ#Z(}LWf{+1FRh_tXiN1vz=aCTs%$|>~CJJ$gi=@p_V%?uW!XRKW5HLAAm8g?_$&S9M|}c zoTgnMw>eSgUfAGry()s{W*}0%51#+}a36yymNl*|gk>jQLI4{M@Xtv4#1KH!#uP;> z8dh5R0Qfp18{5yt3qLjuh(DeuChV@|WC=h$Wa21>ej=7%ysz$>Yb^|$T0ErcI z!inBp_XwHP1*!MN*D0V>koWNL$O$KQ`IqHgvHLF8K!9W@Vlvre3`Vj^fI|)A4D)PI0mtRmx2NayIWejiFfKOC3 zoLl$mvx|#gDO&on6k&A@SR!D_98luPIXF0!bO2=@Jdr`Mr5)C1?Cm)viMSJsdhB6p zfAEVuzXYSmLLo&7Y2`md0)kbbqZ~Cm2U?-t-d==x|L3yfGBQrx5LmNw72X1!;0qwA zdsgoynBuGip%8Yt3kwxy9K}x8?nlsyQUYm}01zl}yKNZ) z{q|)ob87jpRZc;{{s*C0YRH}V3iu_!q67l&nB?SfD{o*ikns;L4CUtM2XxL$0exf7 z3GzKT!T)dpfQ}Sc9V9#;>Y1I577yy2ZwAGiDDaoo7l6@r!{up1L46NsEg=;ZyUNX* zH?291xxX{N)^vofW+K3UZb8AD;ulB8CHnx$WTm8}g6k_r*MY6?!KzkVIx%Wkl?2j` zaA2zPlceDiGoYT6O9=}L6Do~ZdX7CHX)9PwkwL;@!Q?4bj0)()yW#h~y-E$o5|2Y9ZSiJ;sAH1?ur z=bOGAKybdjlo8!*#n+kP2LL<_7^E}W7X(X2_dakNx8qU^I-tU--`VjlTc{AcV(X;)QieVPV*{AWZ_DZi7XF=KW0WhTnk;q^Ukw7zYJJSUuAW zpvrzZD*?{KU=h~kSo0%Wy+$pgHQ107uJ_cyl^0}Hhbyh&kUZ?VmIwM*T^ z{}Iao6+^r0r1wzaA?BUy;)x-q`Hd|g^wX(b8v*K^V(-%f!Z?=N9~uA94hhp7$}cLi z{aTc&U9hnidsiO$I>hG=cmUzwyoU;8T3vr&8&C+?8G}9?1p;cI?df2;ncVD@=8f^1X@@E)-RiC8tDIB~_zgkY z5a=O~^~eLDwVE#SJ!iK8J`*A&s%U0(9n}WK3_|Y!ip~Fwe+J@D_=0%`=hz^Y1yZOf zC`R*{?sb9bHO?cvsQ|1SKyWI428I}}|ACmyAC|0=$cD0zVQ`_rrJAebU^y*3Biw*> zQ0h^2U7N4VL)-zG=yo71&B@Da0aZyD>i8$UZUFJoCl&>n|7j9xBai}515u;&qeob@ zK1W7ySPo?1{?`>+H7zajuor=%QONKTWQe)UG6e#TF)+L!<-h=yM&`4N!-fvJ%OiQ9 zS2Lbz4Ta+w`~QllNZuC<;Ne0*983T!5|vZMA_>ILxCh#>geQm^LFwz;Rq5-DoDZp2 zF;`=zeAK1|hV_;708-oCA_Y~+0*&Cx2XL7CqnQ&@@BDiSbE*9TLGtKUgz^hkqRG4l zDI?*3k~kI#Q1+4%!z%rV0vGgHxUWAxkiQaXAwYQ#^?J4!gW?wEzn+0Kk#K@iLLUQO zrjbD&%~0$AZ_&Q`biLwU-SvA0;(E~nB%KJt>qw%h8JzKNw!JV&SR2`Iiy+|sQG-0ITS_9Jk_7V$3rfL1l zF<>}BvQ~ET7vvXGgs7>HvT-Iyc>hhe<$Fln|5iEh001>rRRqwaGJ;iZejYF z00PAEwNp?U>Mb!-aoU`qfuG@_B7W{Px(Y(|wEX;E7OR*Gq>a$jr|GM1mqoK7PYvd2 z0dS>^oC5g`5M1*er-X!HT8&i_!I>9`ki&w4(8qvsx6)}@y=3SSm>v2>6;|Ult_fu{ zT|6q6)e42Z7Ns-}N+94@=X#3TSRN7{-U6PSKnOH|a1kHO!==6~902llrjsQve-MLp zRl7W#oXh25LG6`4&&_>tLUI64{6WZNNZQ+f9lQV;3)7KXZf%r z*qd?^(yOts#@RiD?!$}L8)^+6ooNCJ9R`ikgnY%CEF3{I|C3#wzVmz*824e2(YIMGpQNEAVAJdKWBVSp~tt@`wsA5;rKeX%*@= zeKYAad_W1x4}sum4bRAeKtUFXlk<{htJddS2o~UG{KoY4^#yA3+0)Gi`Ij#<)s42@<%f^gFzOoFLt+`qb6ir0WatfXj7iHC+i_-{8X6*u1LL z3W6UHiJx_zG+yPBw=qB~>`FGokz&GOAbN-n?$KC_{r>QJ*x(b%;X3h~t9Ja)+jZUK zT>SlXTIec*{^5_*+xrs0pf1dmN z5vp`v)r94|Ad2b;y}=CY!$FFdmYF$MzA5Z`Ap++Epx{>I&t{#R$mBx~=1Zxi{$g?9|lb=0gLEg zi_^m(xBZo;?;kUvqMH6pMhYbS;HfOVGpn#ryJ$$K9E2DZc^mJh z7~zejw~1`XhL-UO3C1UT%YTcFP>_+4wa@B-EE-h52fpdmy8RA%I?*gke>MJESvWD$ z5)#k@N<5$(EjS=2CkJ)I2SS%%Ef4@^f%io^>j9)eh`c;@P)X_qB}BQ+{$(dn^1P=t zSu6B}W<^#Mx^|#wHaTc~Yz#=s;r-^T^8m`wb8;5`J_c1QX;7GOaCGDtwe>yKiJ;`g zhPBoA%0`J{kiGT?u`d+=Ph(dCP379Q-*9S2*rZ7V!ZB1DPD!YQOi^S^M23(dl%WVk zp-?1Jgd`zDWC$6eBS}$ai5(>(%E?^*>)Fov|8IT&zy9^LR%>PG-S2+i=eh6ey6)?} zHk7%&mzN5G3P5Bhcncgb^u-GeG8do-s692*z629&XAh5RJg5EO{cvC30xEQDUCrD& z|8hsake?J+jwZHpgix3~cKr<~&%eP|Sx=7-NdT6Imb)^Z{#IBh)9fcUJPS^{_VE8z zLe@?9Et@HZ7%}BR8=;k-3*>(OYZ7~EGbq`>z(Ax_r5$-Bd+W_(S30JK3xu`nXbJqC z5Bii6f`eAY&{;4L-tP2HyNEBP2WRMCL8(VXNSlkl?^)FVq$F-h3js?Zv%oPaWr`gNvZy5B|$e|Y5L`P6gg z&K1OQxf_J#Nj|M{=ArR~_iX7J?d$JPbryY9Gvn3wBw7nw$TG@{BBRVzOHLsVjtY_{oOK1Uen;6y2J%p5OH z#eZT7r>&F?MpKQ#)8&jj}y1rk;dX4N^0i7+5iQ8$0utK+m}u1CpC08 z8JayqMp!8D%{JNFfKtdQSxzSFSj^zi*=X|)?T=DT%aVF~ttO_Yx9?cuaLnhG86usE zGe5qZ5VV#1lWevD*W6bD&#k*8U5DPLE*v>!-x>dtq_Rj#vZE$J4=HMThyb;tNO&L< zNGjn(S1Ad5A^~g&Vacs?D=I8xeiLluT6KC_+6b7NA~JCS3XXzh6r=ZQ)bgYV3e1;+67_j+MK1?V7-`uHL>K z2GKMBi|nfqVL$hm-}RkIZ%XSgE2;`cSL8ZJ{5bA4NtMf9hSod~ufWw-gk zA&y4*eMi2^f$g&t_aX_vJEt`GK6?!585ELh8IC>Lf}|Tbus14C?N? z!iIrqKP4rl4Sf)?j^Zu0mYyv>e_eMUYK+8tqMd4JXljy8{VIO<6#!vN2!qHt62hp~ z53zf-r*YNAB~ibwiEXxeP8SgsWdP#RZey7V>A7Z;hk|7Ur3i%aDT4!eb zU=v9-6(Bi(B;ZcYeXtxEHO-$3(c)m<0om?wTb;9$yf}e4diOKgJ+~>x*zfjT;XHbL zyqfS+s4d4Il_A{YM9H`LfSfqkXLqwehs(GNmzdRaC$FNyRD3uy&FB65_b5tL2s$J) zln{a4M=ID;MP*oQQ%G&sQ~Pq-USxl&eJzoFkSi9|Lc9y4`kpv(f&hREN;zg}{PRqZ z-=L_Z#6UG|Z8d_3^r-Em$6)I@B;id=PFB>`a$poP(R=NHY<7DGsRiNe>Tfdaj6|hZ zoNq%W8cA7{(7k6g_8S`;HrJ~8{^uC4&)q-9=1jsNfbg5M%&y?b@PhF#T(}U%GzD{W z^Q2KZw}AjM(;|~6L_L*Gep*SoF`AE>nwoaPZ_R~d+!v{*t4p9dXax8O8AhQ?-`I4N z<=D#`iKk9C?oV4W&jiVOK=l5Yy$NmE!b|knloT;!QndC?qP0|E4B1fT>x;;)m4shn z>f!<+mn*ZxjtK%qhj7%2iV7MhCz%>>t&;KEX5UOqd64)uqE(0PUg9aol%6>CY9d6V z&Pqv1Ay`P&-F<9z*T#TyTm!;z(Tb$Kj%aCd!pgNi3xmdX!wE;DQuZ){L}goH4|s6< zM@uX`6dff?b2#-wD1;s^f#APq&mMzm|MTZTS8km(gCVNq$rA?X6jYxcIQ0I+q^WCg z5^o!gFe(EgW=GhbJc{1i`Jr@hVA{B7hE~yTZTjcGsW7>vq=LezZcX^l5je z=AUAeE3?AXXf6@Qm5UQf0a?#pMNr@HaIr_3?EgxRnq23i$A5i}v?0N4$=X>4F zKfQs{K6e&x$E9SZf)d*rV$M>m{#egk6+sSr{{yCnOwKB(#$o8P6_u5&SZI}mO9Q?o z-!#9#q(sjRM1%sgi)gC)1_tw=9Jzw{q=~7i7h{h>^%wm*wTTk^625+-sAjy4U^L9x zbddF#oxQ2Bc+~fkT>FsEZ2r{rkeZ$gcS8FOaHKg_F^MWAH2POodM|s%Wdp`>!o3_( zD(OsMsEUxr^ANF6be95+DSaBAQ{F|bZETd#dS@oSNl(szgQ!KYsYi4(80YYB`I(VQ zBr%pXo$h-bUl1#%&MWKjH3Zh64_MvOskJM>i-Z8a2nD-)7efR2$=+)d$g6Xn{BC{A zM0ne_B^ZtqS15)Gi_pIj@6vPK@QB^1uPAT?IUxjaDI$ZGBZw5?MMtj{HHOWr=lZ9D zAJjku9uX-gNBb;shOLk6t~Gr_{FR`%I1Rm?#Khq8WP}Mo1x04pZZL#|mM@zvs%Wt? zE_WK-yuH04rPL>>audm&ByUAYnO?H4jY>>NC?9wS3<{1Q2xW)}`@~d;o%{T~yG=P| z8%P+VGuzc5HLU>p{j~IhR`x@>U8I$g@W8ceS&+FWj_qzc8RFQH4@Utr91@w1Vcp&4 zB_Lo1!y7klRE6TtwOTd+)AFPP@wNrMpKAM!us$f#2HW$}+fK^MJN*m=!E)3u z{=BqFo@T5?)|*D5@zpUx5?a%iY{{Jj^@04KB$EGh2RBDeI9E6eV?kryfi zu2zeaH`(djJO1KbZu8AO7qF5LSSlh%1u#HM@oXy4YTs^Uqluf z2a58It#%Pjzd&3Gl5(_L`jZ_X=)+FH&AXA9dCqUiGOI=hL7Z+zm=@9x`}_NevW=D) zjAplJrSu|fV9Ua z;5{-A24nh~HBO^rvg_Wn+6G>9TYb^S;`Scz*PhAyutdeKVYNWtS7C4$E}^bqX`*Xb zA|7mFG=4S&!WlB)HUdQYB)xtNusWZ6O$c&yd#?D-2(z%Vt~~n*au$8nst`zEB>53O zkmt{zmt$yUpDWEP7~Gu6AqLC97R-tu>^*I1G3<8*JPXq^G)zKamcbF>kO_ZM~ z+rjE@qocVqzf!Rt01j7%i!3DlchMpYXye{|l;O-Us0}ZZV?xxN&7mrl3X1M?^*ZQ( zoo&g2?A0&1W7e-*r&~zeOvfcgyBuwNjJC3x%fYtEN)>qwXF7}G4L2G?eMS$}h)$%b z%bc8XOo0SpQtG_|<;%mn&<08$em4xFC@x@uzvQ5&L&_N$rIqvYbsbyx|H!spd4VpD zI-;gVGck!Bb&?6A^8KOuV)ojcHjcKDHYV_8)9wv`9MUr)m*a zc|X3i9O%z!AHt}(l2GfbSLtdI0SK{tGf+rpfsCQ#?7T@{UY;mPut13GId*iW#^GNM zjb*{X!R-1S(m)?bAEcp()0Zy~Qor^)vuei<3^1(~aHU=>DA*2)bsiw?f~3=}s7EJJ z3qz+)!g<85|4y0&Gy->R;HHH^n8p^}P~B7?T^`qwda-mS4{bytLPY*Yqi)pMCN(`StGHl6*5Vw_k7MM@&@k zQIKgBCQCYp4%zMKJK#IDxb)YQJ#m}F#KaVOPi|Rr=Cjam=}u3g9xbMTIW3#w@p3m6 zmlSA086aE~D6xCuThH>HXSR|cXBZzA`pHi57M_%wS3O)i>o(ZBxdbH`bHs!upS-_P zI(e*gQS6t%f0|t)9izEcL_`_LOTy@S3EaX`^EvtNo?0ROo5J9`$8WzEB_{4cSmiea za~>Y+EOb%A10)Ymj~)}%hl{U2RkCSp2v<=OoqN6YG zIy)8g#B1C*Z{N_*RFN>MT^l)9a4KN6oM`L&@ZtQ#geTlo>x^P%$czw+3)Pcpwnf{C zl&ZOhRBrhAGo}o(S|F(!xSi#o7I3doHa8c%?t6bH-qkeAWLg(YN%G(VHvwL2@|B+?thB?E|Ff4=j5)m%W;BQKeajZMaCb{C#6nt%#~r2BcNzBJe6K*AyM zmk>d-WxI5yb051)i{hqHt1`*eMGp=`Hzm{s;ohEKG79usqU-W7H}pKC?vaP^=dRV z`=)pd_zImF=aqi+#kIFZLVBT`@Bebpz)}I%(R&+Rx4D3MvIzCw^S82nSW|Nm*yFLu zLv=q>#gh#a?{~ue8i1#Huu^$pKU)fbwFIs9sqI!t)0++?jNfIXdk?6JI3!PHOeZnc zqoW>ZL^n&J2NKFa{<8QXlm5EQ>F;+ZU^&6O>w(cjMKtqZXt2RpDZ!X&Tmso~8U3Vn zVcm(ij=bEG5)*jmjj-P@Bggz99BG^3_aTY=I;N%>r}{TYV?P66J%fv6uZ~W*H6+EH z*gue?3zI(gLpb+b;IPEz6oa{n??KUzuZ^#eS9+U%@8* z475{f=I74~DJdy8;cJjHhxW%DDW<+Z0|Xq+A`ko#27+H*;dZ zzaco~{c|5+^8dhw;27mM@B5N#1=kjB02(_O3j3~N#9fuc78ZBjBtv3Yf^m-E z+6Y}pBgA%_epWVae3YANXsRuszK}bes`2qQF0Zrck=q+FifUu=Z;5UTw zw)_TV<%}w*^!u%GUv6x%X7`z3EYQ3q%}Eq-RLQW1Zq@|JZ@oL;1xO?1*CNJ5I z_-igw`|*lIrGou~oxxxjw0Od6Gy?L|)0nDDc;F_OW@e_lX1~_E^YU~?39daDjFOhv zU>gt}9o;mBC}x^a03`@cV)NrRq%0C}E)+lqi7mxzE(2?h>hXnPI{@qp+OQ%`2nD^0 zV`E9Z11D95afIz@Qym>4DhNVlZS#IOcnK@HD@i=hxL^;)+y)wJzMllDY!D3OIYLcVx#(0whkTy|Us;Msnr6C8_DK$ZH283``q z8&`Qq=S1KN3i7*hg&V>kDq%z7w#5*1g~>T1+8=MxVA{mgl!s2ICU)$ua7Ly+H7y{D4h~H5PIoQ5@jx4ZnW< z0>Z9yELUnMH*$f5+?7PK{2mnCrc$^eX0Vhfr;Pee^!GKsGcFL!EfH;wmdHkVo;mXY zEfoH!@7=GQsc3%wzW^lTLG1tl -- Gitee