From f072159b20d99b05a20dc4a52fddb018b8e436a4 Mon Sep 17 00:00:00 2001 From: xiao_yao1994 Date: Fri, 6 Jun 2025 15:02:20 +0800 Subject: [PATCH] add zbv doc --- .../features/parallel/pipeline_parallel.md | 6 ++++++ .../features/parallel/images/zero_bubble_v.png | Bin 0 -> 51956 bytes .../features/parallel/pipeline_parallel.md | 6 ++++++ 3 files changed, 12 insertions(+) create mode 100644 docs/mindspore/source_zh_cn/features/parallel/images/zero_bubble_v.png diff --git a/docs/mindspore/source_en/features/parallel/pipeline_parallel.md b/docs/mindspore/source_en/features/parallel/pipeline_parallel.md index ad447e5954..fd29dce4cb 100644 --- a/docs/mindspore/source_en/features/parallel/pipeline_parallel.md +++ b/docs/mindspore/source_en/features/parallel/pipeline_parallel.md @@ -66,4 +66,10 @@ MindSpore has made memory optimization based on Megatron LM interleaved pipeline *Figure 5: MindSpore Scheduler of Interleaved Pipeline* +### zero_bubble_v Pipeline Scheduler +As shown in Figure 6, zero_bubble_v pipeline parallelism further improves pipeline parallel efficiency and reduces bubble rate by dividing the backward computation into gradient computation and parameter update. For consecutive model layers, the stage value first increases and then decreases. For example, for 8 layers, when the stage size is 4, stage 0 has layer 0 and 7, stage 1 has layer 1 and 6, stage 2 has 2 and 5, stage 3 has layer 3 and 4. + +![mpp2.png](https://mindspore-website.obs.cn-north-4.myhuaweicloud.com/website-images/master/docs/mindspore/source_zh_cn/features/parallel/images/zero_bubble_v.png) + +*Figure 6: zero_bubble_v Pipeline Scheduler* diff --git a/docs/mindspore/source_zh_cn/features/parallel/images/zero_bubble_v.png b/docs/mindspore/source_zh_cn/features/parallel/images/zero_bubble_v.png new file mode 100644 index 0000000000000000000000000000000000000000..ce4a75f259fa1d9e14e6f5388aa617fed85a4021 GIT binary patch literal 51956 zcmeFa2~?9;_wZ|LtE~doIxs`4RvZvy40GC|PzOLn3n+sa8I&PNn8J|MiPRxPMVSIb zL_h_J0c3`#(L@L#C_|Vd0tsOX5JJd&pMcu8|L=Xj@AiL(b-%lCEmy)x@|=@>_TFco zy?@Wi)g#W1%4;;%tXQ!^`M_`cj;>g-+I+=|FD`zw3TU}_^rx%9!xyllj=!!bY0=^U ze|#1Ai__8 zeW0Rx=lk#1f9rW<+wCKd`5v0}~XAM?3?s>K`V1_XC! zvRL>A(O6L38z!V0j;LpZI*;@t@#tyOh5F?|iOaBoh$9iQIT5wn5y_b=E~nX6D{H>OLd0isVetitjFK z-{@FC$+=v&8QBW*oV?-SYw=38wjy``nP*?B+3CHFMXrE|xN_S~+;yiULvkUp=|k|> zdlQ`RE8Y*5GLZ*>53KmvE)Y!MXd11Umc6;HJ?uG)rUbP{-^s9N#wx82b$DbsVS2jh z5A(ct8ID-jb1K#}%8}Lp#mko;ebtk^D^MrYF~f51(}xtoT>R2s#jYxCw;s(K*2KKM zH@U+ERYZoE2ALeB&YW`6nIR5Rr zppY+ZR(g(U&$)gY9qIUxkw&?CKC2bvx9h^^(Iqan`Y*LsM_uM!N8^!Lu`!m*+)0{`!VTNj0Y&BkL?R6b# z&QpAo#nxnF*q;jBso!R(fn7oWIW=x|;++IV_drMC9kDl7DP%Y2lz#9Y?O`PAZNQyO zK%$CPKw65{$%hfFw@Y#cS^*ipQfQ>lZB?|>AAvs{s=s7+@KZ-s=WHANK4Tgeaw=l_$Ikfpj9Em4+=duY`jk1{@mnd{=hG~KP`3-bmV2GllO~a zE)zEtd>>@8D;vHf$i-GbT8dWd$60X2H)#a{tqK{=0H#usHoPCQ;l28squhI#+n+ir z(Ew(6QCi=+eOs=6x5Z{rkcxSyDWs)n)jkfi0wNpzqzwuKWE`5kPxy9P>CB^;R>#Iq z9l0x1X;E4V1-o$&P%wocms*$AZdvU<>i7RcUT;{3Rv>H>I48wZme5u?ZdG9bxA3qf3mjX>ct^R}RT%>I!B zTKziUb{m$$>Bz?s9Tjd{y*|amrk2B zHkzk=ST($F$y!(KSINHimm}8Q#P%t?&-MYFP{6U_)<;4A#G^>=v1kuKn01r4+$Ra{ zn6R>uNi65Q{I}vMdT852v4DX+z=B{pO69iQXU6)n5wn2?GLQq~%7;oU{g9B^cK`eD;wzS;40wcugXrvWQo`X>(h zQmOJCR7GP%*Rza)I)q2x&M-~miowV!wir3YuX|A{=GAh)vg$t^CyuMURW3iI$JA^$ z&(P5+d`+~n%?VlKEz*v*TZh544Pu(H_EfZPt$jo2F7K}C^D&LR+z_N}fjfmrE3CsS z+&C~{zbsDJ=Y<6rJmb-5HA6AWn-*vJA1ut^heKp~V^P~s(Fi%Yt#NC)fwX%l;9aA` zQSsYZ=2NhsCiFY)`Jj-pvxG-w@5Z=dD-n@VXF0svZuE@FBRO=!2M2158({2QTtI2} z-sHNy{cDlR`_MOjY_4w{SP5=i)3Vw``zMRMZu`^}Og=VM!Y9doi!bl^V040NO>;4i zs*A=Rz~Cp}`XJoMhzcZ4rY!!6CGUrdvLrrf2-NF>ju|>mSl?1_7hIhYIY`%`JQ+&N z2iALF7)p!7_#A0;@JPeQY!q#wm#!b%w=MxjxW(R?6E$`(;%d3 z!u&)urm58Wy)=86Md-E91SPYS3tl_4O}Y5UwBt0b*a$ofPv2FOQ4TEi3cEjM`g!40 z*uB*z+fR1yy#Ie;_pF267ZYMAL!sAaHgTCix3=UZhj1-1>2 z>$zg5T@m_sS|s--v?f4sXPx+t-XjF%G*U2I=a&D8u<{9+yiwfI&^trN}G$PhV3SqPLBy>Jk z%#;bH0;w|+ungbAO!WL5KX^W}HHM@St~1w%tM-Jhnn=Dn^V;*|wkbJc-y_QQJ~w$o zVy;+H+m+`cx^iYQNMo(hHas0-)KHr;J5|oKP;VI;qVR#X3)D+H1wUYf{Tf_z^4Kr5;*v#RU^gYnaTcShdFk>`c++GB#c}>d&T54$KR= zhV;qnb>&XEBdq4BTRwWh8+uQbGE#qxpN$!Xj+Xc6u$fd@j~&Pzx?LNDQyCD8l92`6 zMD%}E%hP(DD|YXf$QUwgU?=IW$^S${S(Yq^K6w|2mFo2T2DvYHeO1s78D>9hdW^;D z3$badN|K^9tkj;`x^?gz^EkE*I3iOGYzy-$7JeSWL> z!<8vo%@mVq<{?Ke%cK$vh?%I@8O*pqz&1rTZ#&!b)>2 z27Znb2^gAt3c0VtoNW`=rN!LSjS|Lqh$rNoAGYW9Gvn(;{n>p1w>huGcS9rhP}o_5 zHpRXIu$*lXRY#Y6O>z^C`>s6K?0G&ve54t36~;aI(WcYi3?BSf8wwQS3enZ_8DD`O zW4C64G3-*SP?{z^ARA+ixuL0r2)nK>Qx(XF%qh6j0r4)%{|Z4NER!K^+Y=oTGcJimqh9y>&0pcvTAzuLhvE+Q z$psC!7!7x%)4bO)YLW8EdO`1yb{w#Txtp511P8@5)9`dOh2)K-Fb6Gk=GhozbeL6+ zO;ZI=z(CVRhy-M439O^bi;C8@>8VhRaxLVzGJ6+#G!XbAtjd(Jx7pvFh%QTq1}$c!FQu0I#D#gvoH z7`&}l(3D(TJn)5>D}6nJW#9$A6AfZPPt3*zB5aBoO@#E`43pP4eAmzMAWdb)#9Rnd zt*5T-t+={Zu2OWJC*?-Y$Jhu27%&cBHJj+e8+*6dI@Q;&!eOW8DRistGZ zS^T_5CZGFE=lY?KGw?%9j%UCJug$$k+w6RqxP|ak9858?Y%wfz9^dirY=N*UXiOrL zo{yCW|B=@T@%xV(Q{GM$^vi)lGi%KGP-R=@#`5i_*vtvDpWSiaPbQv*31je~AqEJF z&@E>&CT@%^1GHF}lIe}+jnCJL3#Qj@|Fq|(J?eYSPZ+E@*G^MSWMjj5h1!$+`F!>~ zdmi1$kX@UZ#Li!Bf;o{=kntVgM5+>fiZq-Nv18|}j^L0elcDodF$o{o(HDrKspjLh zfn&}siu3|4Wi0fJK9ko^$0)O+bb3}@*JbZd{=GWunZ(qc%M9ZVN^)>Qld|7&N+%yz zcALrtqjZTw3n!rMkx0{$twOqll{vo#6VQXK0Id@3FKOW%%i%)e$6Ymqm`9}}uW{)pDV zQi<^kWxqno5--j^!qsY|qJ8L%fS|)e`zZ-?vh$!bOQI6HBst#XRm6bk=dV z2fV;!@T~Hk{5VbHH1psRd3PAXGsHamA@LSW5v2f07p-9?__kB_r5ud>oYDTOZUfd zEM|3!sX0{osz9FIP7dI5@rWs8r3uj;lft}9X*KxlU}2Ai_n}Bv5NJ$AHK&2Yl8c*( z{_Ys@s)0+kNl~(`JR;ORD-e4z`#MX=HLYfL2Zw1z8=l6UJKQ3#80bn!kVeOuAnEJ~ z!RdJq^o&SATTExsQX;UM!~{E)k^Ae<`)$oP>(cdg!3&t1;upC7mHoukE`7O(pcY+U z4{63FN&kdiT%5gwu)Cx)2Zr3r{-Jy-O7GV)t@iwi|vYI6gFN*SyaDS~(2GUCBo~G138#GLqG}8dn z?Em_lSorwc%7$;eq#@m}6c#q}_zO6T8BHUN6Cp5mKjQBEqqp<&oS?%w$&}njH6m@g zG*Q5tcTC3hWf22;3y(VHsb=(nt+zkvpCsTlBE-eqV}3|B?9lXqIU`Su4JC9NLx<}U zYvn$~8%TtB5 zcU~8jMY0kF-2&iZlQ)otY)!k(skobG<*W~af0veXB&@P{C>D$5$(}2<j31X@OC?vcqqxR$X((eFrk zerZJN2C%QpfYCld*gpC=`g5kQtX)!{jwT%+?46gU@y6GQt>XpA!3DN8W`k8?bL*_i zFiE!yxn=HroRmeItAd`X0K+&;1g{GEgWX{aUO?+i0TfqO$tJ?$jOuu?UXqNFuX*o+ zW%61am@jZ4mwE3n%q&JjR?bbIZF$XQe_VqOS()AtXvO#?bESS;?+bd4rn3EWBuH$k zoO}@_j6Y>9pFiNk1~Yh|Y9w(aK0{d?KW>!@kGB|>`3a=Y`iU{ulzWX*sN1JGIQYRO zFQ6Y8Vc;3IMO56Z)pE67Kf9qbkr`8>wRV!o0Nr1(ER1A9Q_);w7lA=Ml0Gt>IQj^Q zw4~QQYrne`W%4FYu!N5UM>}A0@rlhQi-DcTHM8fl0%U>N^DByYlWEYoqOgX=Wa8rS z)m>vtNn;vs{HG^h9IgyBGC}=={3qg2DK^u9+wRqBrfGZ4KrZGL{t*L4Sl7d)du^4% zb}S`xs&{)M46#1pAII~vB3lhTsADnuY_C6>Hvuy=>NSL#PWOd1y}A~#r@0TUDmAZ6(p_WvNIf-IA4 zZylOji0#{EB~6}$Z+&BK&UTjw#5T`v!TPmlS-k1EnRMDbbKbWWor5YP4!Ox(I$~>X zMb6gdbla4I+u*|DrEOu~7Qv3Hit`L}u?qe$Mo+z5aHfSk<+hdr=JFDD?l!c({x1av zXc{=s$4ak$+U)t!z?K(`kBaDYS~;A>r3CylMU({yoqmX6)P%m&HxEeGv^um$jxIEh zFbHYUY8Qf3;>KboB>f!@@A8SrzTZwra{|pV4w(lEmWn);L3~+V7*iGAr-$@Ph#BUr zViynTZKBN9#bMmXhB@~NWzrU=cSjs&XPExcCd>(tkb)H{-XZ z_oiEh>12;Bq(22lpVM{Es2YrXT!^3)N^l}E;Eoz99CNFY>}J;6-+*afOb)WPcc*hH zJp;Qnn}tZ}LWL8tjY84ZS;rV?=f}fk^@#4jt4=37i^qPT+OW!Ftl4rwYA_n#!ah@Rt4cX(KRLw`5k@!oBvE=pOFoInJ=Ltt;4yChvu6FIyx%@TU5MKk-l zGJQTmZfns*Oq11j_Y?$p{i{!48X#L{*xz-o@b2&W>=BL^e-R*Lf(-!G`6o$E5EXjg ztLIF2=(!S*d(HwVq|N_~TUn)gJ`4GaDD;TSf_WWpn$HmD{Kzf?# zsVNMo1j5eI;KI26T=suaTt+p_8E8yYe%gwrZF5^J;$>oAYn+m;43x>l^wa;SBqgC) zRbG837KXX;I(GMlScSMw`J4=ST#lwY*3t(Ml7Kj=Xyy#0$^3Rs&*kpWoGCfFo)X66 z)G%l2y}*=6 z1q?6}_Ybw&FdRLvCyq8q%Z*^+BL&aZ2E*au- zL36o|zFbFN#y9?-z&990T%=swCTI}Ub84B=sks@7u8VHs+K$g};!S()h==wnQR!(>Z*o;E+kZq^4Ie>ijhzX|gG&YS zoG+s|N79_RHZBm8Z|Wka-<@5Z7`T4@1-S5l?Hb-J8n;8uUYmTRa7-=WHe$-@=vn;5 zaQ*QIy40^fU1+c2_@_P=J7{cn;EkCgx7;((9(%{hW}gjNbs4V0Of+dlZIY{h)I>f!R2Y*}NKpUE$10XneyO4C&p)5n zNgn82feXnl8~-~Zj>-hDKa!>r#hGj2nVMgEerUB}c64jZ=Jo5-({Sy1dkvWjS1%g1 z`zb{|PWIc*VGnG(JHP82N~yDm;Pu_&!~p~4mHx}b-L%hc4l%_$^Yp$PdzWjFJpj)k z2YeUHd7s~=B5K*P=55kG?1OhcUk`e89ssAV{q^J5 z_w(ueG>eOpNy$L7I?l@W{p@Ykp5JCoZ|hR}(onlDvga43=e~WXd=N)5FY6M9^wyA% zpp4odStVsWz{RH=>nZv{i)zK8MEvqesLj+RCHtfp#$|8otZ2Az9v(u~M8>hy z$j;WI*4O|1rkUz8=9-=cmF#<=HU^COIAw3j91ME zT}Ov`7%KHmA8>~1=*}aj{NfVoWgD(dF!`$2(uAFJ9N&4Q&=8!7cq$UY$@MaX2*s2I ziQ4AyZH2^1$;6i=A79xAiyd#Ho@LI9SR_8(aPLIaDM3TM_#F|EbFvOGEk@R#)q>8P z(<0A`ZY%}_7Ic!S6zlPYxh*fC(egSc`F>~ z^;&NF@wtJ8GF0kSc+LXIRwe>QxaYSSiW@M#UC}ha^ya!s;)zL7KDsu);L^@zb zQl5=I*JygvI9PNxt_!cJV+CIOzCa66U1gw|jxCE=AnDRoLx*_6JwcX|*K=-i-r5Jl z)8`5EH3HtC9~ve-VZ9rLCTRpFB-$OUhF52=6~ytXWbhRvOUY2gvH6;au$DK2bUPl= zOVa0MfLc92{>o7<5(iqKsKNCYV7a$$ky3)|FT-+OjZswFnakCl)IlsT9XD|#d?8MV zTOAHFSK%@FjFpN=4SstaqCp@cnI^Uh;i$oy&D5q#B{pd)4E}DYsZ*O?Hkjd8RWf4b z|GwThY22#LrC^-7+rDU=CSt3$j)^*ehtX^{ph!+o9W23YR}6dDpEoCEVKYP70Xm3d9k|KMbYXtqN2>%TI zRK0!W3%q%GZ$SIcFcJ`3vA-t@`%R@n900Y-=`ta>kQ0OSmS(_|FoU@}LD3idZX^#9 zc7hOuy;ez-0J1IQ~g!n-dky4SL0KjWA zo?f+z);CPY9*l|7*UbPPV)VJtXlqaUx?<+hh}fVVXm73EH`w8p_#3xUNF#N(NggVb zR-;wP#9FPM0P>)u(ihY>E_|B3c0*?usMK0D8Bc}gch*U&x~iD$=_aQmS8}y_qM90Q zLO|Ny#0=dW5)^4Bojs#dfldAZ;suX{zo=7%LoKtQ(I=;Rm73oUTJ|v~qDvVmdmS0n z!P%^IMUOW1=@_m7s)8P#g>i#f9JFTK@LjD4lG$6pa#DCov)W^7=UqLvQ|*CaQ9$<* z@6K)3XHO&cIw7kXJk5_W63VC%@>B2Cv>z9B7&*1k!ClhIdMI)M&ZhI#i-M8*bm7`3 z<0);D{-bGNU2#D`0;Fv&I!^j@x2%ISE~U-ZI=Yd5J9W@b5|Gb?+;4oLA>_{lkf`? z61UFUvn$yk+L1h6hLiAzH)K9}VK@DVmTM%5MBcK7fb{#^Oon@LxEqB2;qVyRcOYc; zz{WY#B`rA8_a%bQG*Z&0npE5p<(s8WO?c!8=iS>>!HAebTvuf9_zSb+2`YgRvmlc! z``WX?Kg1t88nKX#cPp(tvjde{n=<}{plL^fEsSQSU((d5jBry~4PI}Ya+g%>a=H0V zW}5E2<`t9qXy!@#Mm)hK>`g-Ev{sGTJjW^P{GF!38==j_3i3N`Rx8&`DMyn88x`o? zZb6Sk1c4fUIsop%!K`T}>YjbOglT)JPT{M6D zC{JxY^x-1d^o9y&dV5zEIJ-JFQ3fj_vp7t*Gx7%F5gY2*myr`Q1@=CxKWJii7q`5i zosjn&R}!fW`7kr>q~w7zf}lx_cf#g=>?v3XYcoEEsimuHb0W?6=Pdtcd)4D7vVpy} zRKI9Hxohi$VguW98fR*A_J>KGLt-eHBk5MLhs6FtX5H>3hhs*JgGtXLEW>Azy=qeg zuO#43g%|G^)ZeKk8(Xl}K-dFr6Wi|%F7BQ7m!5dewh0T7cp%aXz&(97=6afzlv#G0 zA-GJK5TfI*W}Dgz$rl8RPKK58xPD54zAmo>rx(x3oWlt!s=lE&pPsiexlQ7u9~9!Y zA+FVFO%ZrQ&sViS>AOP$8-&3dig<%Et%dHH+!#+bNk5;pcjqOthl+OeN|yJ?^v2q= zDOv+BNv8D#v(qg-t!gc~{Z&@n{=QzPV&2wU0gJo3>y|Sgg*0oQmGO00~8TKe!ulBRxwxpvR6z!v5Sd{-Rb>)kv_9U z1A7V6X|fH^5zUw+)j8`ZV;NnhTZqW0Y?7vqcZcV`$Xr6Ve~{9;)-JDzUbSTKKx0*? z5;Xo;TPS3d2<2HlFf#RqYiWWogL@_nElOvc)Yrr$*8)%ybZr%jEUHL!XZ zc^!$&Y8(P{>j*-cVL{B3YRifi75d->w5d7zJ@=Hmx^!Uvq&jVIiuKA(`3bvBEx{}O z3B}d9H&g86?^76Us=!|j?&sI$M~o82Q!iJ;El`u&w-#Dwq@nwr910I+q~ZOe zY&90XY3dX zaXYUw@JJFY_LM!^>Euw~OxC;3{V_I$YU6yV|7=7M`xhN}%-Zp1&0S8|%A@Ce=8Me} zAZks6-&2S03b9vPfG1}nWdR3Aru$I4E@CGjmXWy4VdJmI@z^0)tmy1uf82S=iTFE5 zExr7hf;bA>9%AO=yWI{|Sj^_AC6y8~+2pYDP(_Y0?Kp&&<(`N4LMGVs^q9%&UZGW*CwKlgc^6~**qU=gb1Ja$iCj=quyt`z^+6)LH_2jl7 zb6tAL9tVF#RG>e(A&H8{5DJEFGA;F zlr^uVff3jicGZlAvF{8(&f>Q3K>^*d0DmNOEk`lPEmwhn4Sa1rHZ%q1tZ!x*f4d==T`- zBvJKT%fML5tF*-=`uWYZz0d}p>4GaHnmv%9${B@_6zjWzA2ZMb99LnR8U5bq9tqN= zR$G;>u1(Ue^xgpk_sI{)1^&?L9d>Z7A%6!-YiUgBb6a@%ho$GU2L^GHJqsx$au*qA zWR;|0sfi?aRmaekKTqDiURl7kAW9I+KHN%na7sr?fWZbFu(~wT z7$!x9ZLf}m`Pl8Ps4m)@O0osnBz68tAIy0L2z&TQpgu}WIo={RFVX}nKGUC9wGOA| z)4#CZzF&*gA8QJHVh@r5{>2ZlI-t)FxO;m|QqfY&fQD7xwyvYbirpQj40=X(POgpZ z9MzIO=-Z*_Laiux@e7j2dT{Y<>80ZkkufA{Qg@RL^2?}>^oC{+?|7Ep!+?aB6&*0C zUQ%EjTi8lw4Z3CI07s|H09$B1le9SC9vBX z%kF1JNF6JZLxGjQrIZ+pGdr4MG#vG7IK(X$f8$3yIm|7r5ZFlODe4Wc;G2Da1;-Ks zfSfq7d1md1qoIVRnA@^=Nl05GUfc;^=a=A8Gv(j7}o21DUyrXyTO_L<@ zQ}O8Q^tI1qkd1iasD`?Ia0xz2y7erI9NtisZg%q*$QdQ*Y2%GzFJWDYHt+bFW|hWwNMwZlS()^AFxVwlMYEPB$u9AA*#$)7>wFKCyT7b% zRih7v0@@T2kuFv{k#|hRrHVl=z+t)5NqH&NmfWN~{VVKg_0A;n5tVi7T2$s?z!#`D zH!_=J&HH)+_&nZX=xrko@NDBd8AIwy6vMy4i0TDQeXzzknMTl5NgB@tzwL^Y7Mh`R zYgEk89VRBn7`X+6xPYapt8ZNO+;@k=a^+~ba!y3W0oir^tOC}tr$9_CtcX}nXOo? zpLc%5gFf*f%|@n%M8W3$(cUt*3$Axcig`g`*@!qvB4YNZGiF>czkMLfE zLnQk&0j_5WLxOJG*$~=sA7b(wYsB#3{pW}DdZ~st?mzTe;&Q-m z9RM0-2~W~Ia8Lk1Icd(Knd6_KoW(`Cc-`MYq6|oo4}!br+C)tPPygDxs3;%Iy_n?t z1odsaS=D2%@AFe8M^qhLkCiMTFSww*|4Vby0N=cyQ)7j9Y_)a-P7b~e7p{H0WBOg= z{OsC1#mp}vQ7+{Q5NHvq;W^xX2k=lc`6-BmO`mwEC3_kF$cZf?JODYsQp-$Z#$Vr3 zYrwR*em~MXo3V8?J1;0?Rb!px>*E_^IcMU9S6nyV9bcl2^m+hLYzex8Z#}>PP@{!+ z2fz($KA}cSm3aQ&3AC*+B^s4mzyFqZ8Z>9WYfVYZ`1-jPSAdf`<0HGcYfW8bX}#p- zj%!U(OElVsP=LW&;+g7hJ%|KQv;@urUEj|r+M?2uL<+p^6C-3?cFQt}H6j?e|E7cr zqTk=)?at@O|68mk zyUG>i3sCxQ9i0Lc1=QBtf$saU0x zs|2ED+5$gARQQ!*_{0)KG z!3gAbg4dmzTM%(T6fas$G!5Wj+}q}!;;6cxF{+1Z>4t6*v8O737k;po@dflC6_?Jg zmF(E9KcO(!`K|T1b?e5H#~DOVOUWRHef@2g87h_Jdl<|I6MsIa(pw#l2Vur>tM_J+ zz&gm+mrAk#Dyz$)nLfnO&TBuZ+cLqpicwrQTqTX+yORCu)~zV8IAmsga+576t$8rQ z9^@C+8L9`@hds5l?Sv#byw0V&ytMV|=jU3VP1LD}p zEumm&C$oULfb%^C5LKe!Y1=TL>{S54^6Y6Q{{WjF)jAeeCKG<0G%kA2UW*4GZd;uV zhOMHFyN~01fyj4FXoz_|JcLYf80qEEhDfq!6#jYsJinn@SgL`oIM4q`ReAhHRc%mE zRS7zOqpAiKRFyFF#Fu z7#`O}4#<$b@k|XRTPesx*H05F0J@ zitWw1ogWEsQyWE#+|-LZ3T`T455P?cA``leq5#-Q+xIQ@cXI0te-XGUHk=4}JPzUk zTA^U5_U!@~DtL+RRy&jvoWga`3WN%RPSqJ7ys_h^?xGZyL=*%qpduIC$Xh`OO zGhR1(rF3kN&(Q<b!LI8PSV0-5jrwfru~0e66WaV$>}6L|$xp z_o#3DZGiTS6NuMc0)DY%vsV#7V}*PB%1%E_{Rt99yU}3+0g5+ZI_I<>a5(_7dYi+J z{L)u0@COQ~jFPJvrE0pGwHXAXEOp>v5t;hM76Mw|=Q=ckV!~reA8LD#4m9myqu;tX zH!pdL+^Q>6V?>CWRYO3?R{o)2{eH#_AH$SMhCNC(PfeBN4tJUtf^OJ4{Y6dPItNfw zFJTtENqm7JLNetH47Nml6*AWc5zeCEPa|2O0-s8k`#kZs%&9m~t-J zN(nooOdlj6>_f=|Q}Mw&gi+i^r`s+7UG>(b_+k4A-6?|d1f$cVhNLALm%o5PVpaM@ z({m1O7EL3qVF`!ce7w@UGdV0mg`UfdIba1!Z618D*MBUj)Q>yfC~N7WRoFvD89Vhr ztL)HP^lX85NZvC;5Cz3vP1nu{5HJb^;w>J}z{QTuL0`PBX!{0^h_^87vT> z?#3*tbJT0khCMW3hDcu7rCN18ova&pEt^bnD1fg+jCF7_G_dosW?avlax8{x}IjLKOX=2R|xExhgy6)7B0LxuV3_Sg21CE_uD!*?s1RbUxM_mnVF|4 zCF9Qbs;%)zO))5z&LrkYEvK_ zs#!NU6Et5XVelKLstMJxS#kw5<*Wh*K;VogZsbnAHF=gHn0*7jMST&xd&qV63UbPU zwnR?l9FKHolU}=^L1507}W+&`vIusJpfLUuM?`azbnM1q3nZet>9;|(MQ4ytRdGL8WP~sx-cZx7rRW=i6dfK zv=g_5#uI^z&!($Yt@U>YhsH5Hp%kHsMR8qg#N>h}pb~J!jZcJBbcyas7_izY0IIP? zfQpDNhNA(FO}($R>Ta%93+}f}T!qoXZ#_4(S|(bq4h7er1b7me*bF1oB>K#A{l4Kf zko!UAp*{_EmmFeM1i0TsAFOMI77B!_6Gg_t?HY#2tl5IR3(s=b=a4pp_$1Tsxf-BM zCbQDZ9JHDS6-bi=%`FoOrlqht&JIXu5CAqx8ta9PItMThw42C^-niTapL_vP1*=OY z1DU#kUe9?Z9xnNVqAb2|uOC*Rmc;|`w+r*8oi}R-Yj@!NWeA$REktY!&{LW>7U`*B zGbEAq;u4v4^HY$bw zL*rka>{YU9VSe)wj?!!>>La4KcT*_H0XT$*^*ziz0fSA5el$c4VLFZil09dOgktKf z{pZA^P=8ZYEw<3vtbE|j>46(-a#GN_S_XtD?sIn!wYQdDzGZ0J+uYv|K3V6YBZsqr zjOSPJ!C}dhU+-g2e|1uKdxG+xuDeh=wP6R+u_#zqm~li+5xhHRbcR2-A9x@4Mc=QF zv`R|SC^{qX-Fe+z{jXq;gW2_1N^9rNOTN9py=>gc??a*OXGaf2+jpg`nYj8p%Q)Jp zr_M^bT#vA-|9OnP1}Tc4dD@*7Q&+OCfa5-&?qO1$>(>jH>d!xN!tcgDs)x0gnVG{k zPF|eF+X8EJULSO`!raQF$t3k0huHNX+hEf}}13kQD1e-B2(k&fYBP)==i`~L<>Y~4%b1Zur9kpnMDc7w2BD^}&}GuK+?7e-c-m&p z8!t(dX*0D-1R&KUy(9q`4oB1Gza@>Vdg0M_{>}JvfTZ5zL!ne8pTEwy@PgZ2*B@N` zZD_ex^kzrzp29b&Yo!E-_mffTy3X8HlvXjA=3#51xG%J<+~tG5>IDeRk~h0y(z9o) zs?1!XO@pp0xz%w%xmk(8J3Hl4 zucvI&)G%-ZIjqU3G#S700JF?8GC^An;Ew=bu+mb9t+qvxhrDyxF*o{S^8w*UXO}iJ zyUg~TP+5oZZT15|%DkX;cy#s#7%Zp`+fdcZ^WzfAT~bm>84tMKfcsd4m_G7Tv{Znf zdS++yBf9_u*?|g0LNbU1=^uc%dlRqp#1#Z|rHxw`ui%=dwJL0j{bgajDE0hfDr^NR zMYYr$$U=7jU<#|kNDCO^Vtl{rBL>ye@TN3}F#b*vt9b`VjLHE}^AjS~Gvt@XPcWW) zMEeo{Q;~wg;%f2r(X^pLGhGJgZhfM3Rs%q!aBS-tDEuo0KT9_MDZrt=I5p*ZOA4jR zC!l@&O&7W5mLl`5N~l$1Tc3l*Ud|K{Q{N`dg?pn3YctfBLUxr(VHou-B{(Au7i2ht zWoql%_D&rZ_4~RbSz07W5ZuGyxr>f8Hsh^P-}&B)K!_f524qe#S4RVEv=Vi^@x%Nu z;fm3XY|uQj)|~EHr1zay0RVRbYdLxI5ngtxK=fzfN&!4UyRqBGjGT;L)_8e`Tg>Slm3B7acz}UZi5Osd^e>#xSkRg}kgzyqaF7mq zB;19{@{Vo=7^GFJ}s zQI4PFMr51y+~MJ~^W<+j3Fz;D`xxd`tIHhD83RD>Z$Tj~*dGgCTyM>-x$LDzilW7k zX>0nY?GQ7f(XUGt5twl;g>J5F6ZS$)gER%vS!uXN5Ob5J(H*O;Ly~%XmnCth#@c)R z8*XMO_$T1k1UcaQd)XJsy3Dlh&^T_S)N()pNWW7cP{kQxRTkA(a)-t*TLWR4;SYi6 zthSA03|>JRJs1vexD?r0A9`<1u(ASe?hA&r;nzc}X7=3f@GdPF2bn6^Xtb0J+@E@j`hTp`D*Kmp4FY?&Npf-GDx!@EKgEns-G6Bxu~ zD?4TYj1&Ny{woM|P{oDCs?$s^-Izm4IvM{2A(N>$kq2!XCZ+Xfm7u`9ESjA@N?JFw zZWkb?>?{~L>;Z6J`=@F92tg_EDBJEuJnBdMC3-sLFjs3+Pgfo2)5I*-tCs6k%k?S% zfLSi zN@e4-%1;zrnYh&T0mkn$`qMN&tE%IA4B(_1c04?!SE~TaZrQz!8bkj8`)_`u1pD0p z_9bZu*~o!(eFoS7fa!|k#=G^Bi-v2)-Zd_AUk_X&v*We`VmSQH1owb-{5xZON{;aF zfYGA-3ekSI7a61DR;UBJvPs*FDSH7PtMwW{Wf2WE_lwk`Kk;P`LJX0E>w!5UzQ2C4 zCKC9@z#&ka4c>(V-vA#zu%i#h5K>vRvTzClF>x-rIYgT?3^oxs4FsbJw z2Pd(d89y0pUE9!j2kvIO<{vx!yG9a5FS!(4!7Y1K1+;R()hWdN(M-`1c|utky4vHl~v$k4X%A2 z?!7u1o*>9aN^M9{QW4Ea6IAD`xH}j)afR3upUCf+BX!W}0Ce(eO*7F&Zj2goksBpe z05Dq;<>|<2JlJmWvPN~cfD@1sZKeLAkxFaeAge0y6-|`TQr7bj2yWj+RP|jyoQ{2_)!+%hXX=)c z?!fE5Q2c0h^%)g3Nuq_Uzipc&vrXXYIIY_0Yd0YyhFYkY#wCrFo=~A-yuBG=WOY)M zw#FY2)w5&|Z%3Aci-&i`D7X%cshwTL1P*!deYtt&=|3DL4M4=IOnz7hm-W)|f;hVn_RHQ*l+5kUJ{$aZ>Y=Ex(<%_la^Bo&yM{_Fgt&;4rpJ5QA8FdTulmX&s@BNZd0-8FP!>>` zQeS=9GH!=vw05{Lz1zu_0iu%0ttB1FyqBY~N*!j*tnxmtx<#Qm-r23_x>=)hX1ea- zHr*277d<}AxVmD6`C933r3z5QD0W*{s^PZ${59uO@WES9%PsLoE+%Ho$ z&1=Dpo3QweXNTrC8zK#6oM;Q=0?y=Ioa_vXG@dG`#OoGPKw|JI;5TEjST3Qg`;#v5 zvV2;LR2$&!YX1r4JUceb_?vy72<_OEtR|CB z`djb-3T_AcP$_#O@tb&J)z^axX01u`f3$bzQB9rg-fnBFuNC8|s0?9f>ws1ZqD+y= z76m7u4h2*KrIAtQ7-mRqm9$ET7AtcoqF^h65o8JwB}!zB7$M9dA_NGd0Rn`CbMF9F zea~9oS>GRLt#cOtD29Y!_TJBPKhJ&b>lfp5Gs=^EOHw_%h9Me$_2rrIidS~NRkM9s z-@3iT8^y&j68UPatF4x`oL-yCB+gC>v=Z~s;c~3)XqI7xDfl@wv5NT7kEZQ>tAQi? z7Q6%Rf^8R5-p&}F%IY3E3oacHuw_w~j7U&UvWo66hUDqsXI;x^7o5?Ib=A4kpLGS9`n#g=)d*QD$lyu@6 zo{q2I*8X;*YpWTuJ;nSn&*;i$1LiF{%Jgl7+uFoP(}W&qd3zD2f8fO!&x+Yu{q||u zbL}c{o$j&8;Z*uA*VAF>gkMjat<&EH2R=zQ1a`qT7Y~KVyfH<|zXpH67m31@B!#xB ziM^&%D^F*+dwHMQ&}-{MEgvSzG$ctKAO3o*Af7R5R5S3i+o0~BZ#-cQX+;d3_(zO}o2k!E!Rn&MaL?@4RfH(@S~5MSYeY6F88_Xop{+X?%REhp2|_6%A+p;z-?-r2g=z{cG&u&8`Y zJRbMh7FcMKXBh4*q$;QDf+Ow`zH?O(J=qj`+w9mi=-NG+iwdyjU+ocw&C&;9*gz?= zEoj*mjGAj2)9~^Gd%)lhs>q4D5w>gRPyMV3uWZb&Q8P*mf5x(pW1DaG#2>q%Bb4nv z8eNPWQS8OYk4CGvB!UQLVEGAg)eEhi$bpQV(9#Q*l!+82)E}W48;>zm8=9lU!oHC9 z9b1nQ+skHfOW=ezzzsraZ_ls<6HocO@y(czFQu$fT_D?9clPa=DJdmgyeTGLVu~%+ zdwI*olQD`_1KqGZux5UN}a*H=KRp<83k1pOiBxo zCQTU@n08>rbcx-P50II)S+VpT;ny2eSVYRifmYbOJ5$bL1Icw29=?(4bNsTpRIZ)TVv)sO4s*_iA z@QlBctHD%uw@MmMjuS$oE8TEJ-LRo>SA|Bx=jpW2 z*9Yn!SN3+{?#QWdOtql_dHUOd>QeMZ+-(}UGBcn?jCTJk6qu_;>sR7>NH|fM zYWCW9JK4HBdSo0}KxN70p0fhui&xdCKL~H1&)v53b)b=KY8!w9IZb83lTYdIfo~;S zO-Y$um=;7)b^0^5n|Nrgk;DTWmdl z=G3{tqrVhBuiSZ%j~Q#Tq@Rx-F!Ve+OnUWMKS0DopvoCC+ZOQH%o zmN4?)+bM@dut6H{5ngU%VcREs5l&qI-tfignD+O9HyzVUlXczQ0}sUMogeIzZ`r2Y zn$7UIOGRsnnvtWBW7QC3cu5D0F6XDf!lQWGhHK#Ccc-Q^tIKM-?1EK;>K*I)8U=Ft z;~>@Kp;Du;9k4sOfW6sFmfTNIiw`wsjXpIduhuv4i7KzbkHk(ydns(hIR2PeYCzl_ z3iJQxlv4Udz{FB-AgV7DY5u6MdW3Z1kP`|VWGCJcgCzxXl!7D7^qNHDt`+#Al9tHa zw|O^VO`lqhfKN;I_CqEYct%C}$OOX9c~Q%coI}MsY{?KA9{-ATC)8-nvTM^5!RR=r zf(8%jSiIl&XcE_ zG`)*!o0Us!OIkBLGRkqByfD2X*D%3t_p{!^8u}8iu!+%`(f%`V!%3EPU7?>7qi~D+ zoAdp-*+gOK6^*ldP@(F|qJ-in!l@9B?)=&BndWfX>0yliM}B_^A9jhoQH_KjvZ;w} zLe;Z5Xv0QANcJn`1spDWhtgpj*#M~%PPS&Ol;dzok%{9(o4MBk0(KDG597X^;g0uE z*bEgz&txqcm^u&zZ_>{_(3tF}ja|klxJJ!wZS4-j4LX9u z^$q=&N(~opie&eKDk#Q0y6Hd-pN~Ab$3JS_5dQ%U|D!L8<-7GbxWDMKQ@wtJty{k) z?M6ttYfQP1Ebv`2E}5B<+!>RuRuK7 zVpplCGV8|TmEl-1yp}rRz~~A1RrVlDg&vZoGI9Tujds0_MsQf-WE5dTq&tni8^#sC z5WN&JBbH|Ef-B0YZ2qLR7Rz-VX$S=RETIa-uZS)<_~*Ld&9P?D)upo+&i%%l9iHlI z30lD(k(JEs7QnqJ%W&wV%0A^zoBAIKy_2;R-i52CSyTI)!&zJArehz6M?U79m(-S~ zI@@Sl+GwzpnGU^To|a**uAOa65!k=bIHglx6$<=km#4<4X~-eDz#>63$i22<^6_1- zZ%n-kh|!IKyHijp+>$$5@s81Is&uOWO*7rnlCkbx+b||q@ps zmPg^A-;1D6)z>LDIX?`|1GxGY(rF1ul)7bj`p3C9pCuahWjFNxK^#((S|y`=mCtR6 z6twugPFSv!V>OVZW8zMZNz--YMzDCfi@ z*2^fHcSeH&%p*zv$|_uhg5M8BZc8@3?ZrA+HZQFFpTt-3?)}QRm|mOaPbAmLGL-Pz zP&w%2wbk)!x;T~{Bg)aGG7&WHg5X|wB3>%59PZJ0 ze)f_F&#u2B4o9>6W#eZ!6KPF) z(WNLeSSgj^Pk7)&op1Jbwtf{p{fYB@hZ3??l#}}Q$#oZrw#P(Y{VS0RjmmdD6mm%b z2gVJ#l>%w7!G_k_L|g*+2s{ug0WY5XWu$`fqF+H?$^Uxee!TPO{%^2*z$9_d^wRim zImi7zh8Gy3L+`sug5gy*}R-_=COe}2C1k+q04q68Xj2cbo|uUFZP+k-xZNCU*K z83#0(Z}bQ9+UCaX!5*cEqma%Mgvd3%jJzRyf&KOhxZX7Ub_{16S=;1jD-nUELGVqC6Ap5`-;JUk|pBC})F9fzlV}n--e7f?=1Ek3hq~*+OwEeewq1(J-~2t;GtwDzqp`7= zxkH}xrw+uO<@cXUrSB=*7E=NjGV%~;45=^yxa(1y2_OT%vz^;xKBT<=&K=RV$NvH? z7$Q9bB<&;9OLq3|x@dWNUM!?glzr4c@}8<~(WAAfx`Nsgsr0@im3$?5Mmf7K5$8~> z&AWXWeoxP){ofU9>ws7bes!h3qxmI!F1O4_{vio}>kkka@9Dt)(R3|RXb83kMe_F; zQj}u`5OQGtvc#^tn`4uw{c|-T6dY&A0CC$(fwwzZYbnnuMDF6x-8&jT!G&-QTNh2ZY&$|>?H*i<*{$M7}SFswtdG@|+QM3fZ~FpdJgs z{yRms2#ED1ECOPXHaX>H#c*as;m0pU0TlLNfhFUYAYW^0a?Ke(A!%Sz(e!vt?j(a} z?`&=ED4mt=o{ds(PBrh0>~}V|4QEt@dXR97MluM@Av(=I0PYNfnRVuvBS?&}Uk9kO z4nV#NUyUMKweB$!f1BF*fJgS~HK}Qz_#a0x9J607ubWR@qiSV-%hbEP0#G{d-Ja*~X+4htLK1};K>Qibm zDUVi)Qa0g2T4G?CNyrf|0)TXg*X$J5uSx#lT6XL=lTj?%0NKIM#8Zrky$VQqIlSW? zB+QOGZ#~<4=Y5k|u9N8?*M1QWv#sCl=giL41T#VPKIGnPq^y>%E%@y9yx6SZ8S$oU8Y}&=q~%@BPqK7R#A9!&ccy_Q#7$))s^v*7 zUmDu}htgz}JXUg35CKb9aaU+`eV}GA434B?PgkUVjw{MJ~olodvP4NBEMSj`1fh-BIz( zKI`ouqhLrI#`_noW13Wpnojs;8na9O&4OW5xnvA%XOP2mW&sMLr#jlBY8%Edjyp%3 z+fSne$sO0ZHOuhlY*6tFcC+7E(^F>nA``9No!2dlZFPFP>9za_?bgLv*w7F?s8<9B4p2JuZXDI!7jcGy-QWeic zJPS*d2Ske5bKr$qh=>&Cau{DH<o(zSElJr>kuA3{GjO#CRmFgJ_E)Y7{< zT`a+n>$3g;9Y&u3gPqC+@(WpU6i`loo+|?iOztMzZO5$zd&x$Ew5w)~t#-9<$#=q8 z6gFxoDZ$Y3S;$=8?Br=JeTTfrqN7dI>Ln2+ysZf|Hwjk%nvIgOa4y!RPntA6M9B`pHi!bG$%JBQV_0w@8orm>>!G>zySTQ zM}+N#ul&aqzX0*fypXJFQAh@nTQJy&;%Zy!tb)1U1qyF-(0u9^6Ve*V>LitL`pDh% zS6q^o;S>UfxtB{D$DdGm9sODc9RzMyF^KJuU#3;{aBzYJ(G(Nn3$*Fmf-?%sH_5sx zy<_V7m9otinEdusM=_;CAqDZ9DgT8`Os61D5_4;w5X~)`;Pwns&V1X{SM z;adD9U;HAV(sa*BmjA-)`wnZzsv=A@*Cord9l2>ZLrfB^cv1!VA5PhtPq#hGLZ3z! zR+`)oW?$J`r1`2L3Nz0$r&2lN3u%mtp^!1bWf*tGx5 zZ#Lh9#7w?aHLvmFn@>7aA#+BX>w~=-;j=Hl!>;hsH?VJ(7?uPyPOIlsbTS^T))*nK z|3zQLkgM+y2bPFqMBuoT5sr(P=TzP$w*aXL30oS zc;qOT@&tl%EeNn)j*TuuX8|kGjuWtMu3nZFFB)W@hxAEBqP@TZ0IxU$g$SqmJT_lb z$$njMs99~zpI|UKK3G{uuX9Q#wZ6_ouV;q7qb1W8ob9F4 zKg5p_RFj}9ysd|xx zsL6155j7d)qQ1Z)en*t`UdbHSU@it(Wd?5$y4E8P*uw)?OPbQ2nMltJ9EZ04%mum8 z3}oZ%re>3JS#_i-`(;3WQ}ejB?Kf`@9V`6gRPcs8Y$x-h7Kyp_P-U+*dU-03|CMji zy-AsEz*N-jn9AN+(?h{c%_o$U3wO;#Nyn3?i%OW4nMM93fA9gGGjo2{oC5P;?Qpwp zP|g}~Wrh2CYkLSD(=a=9NXC{roBTjV0^y8hl0ZX0GDJ1%kiwViTtOa{$2aar;v2V< z78GZ7HC*>+Kv6`PhKkA7$?>`DKOsChEz*F^61O!wcAnF_Z=eTwBwESmEjJlFC9wme z>rOUWh=~kl$}Bg$Cn;+~px^s*k6$S9TIyIQDK~v^2{*xplJ;?O+Thjk1FeG)Q0t;* zpu=*1_Ef`SPNULA6d19bJm#(DH=u$ICsX(@d*dJJSAT~^Vj-J$7mOIa33U`e^4WlP z-r1Q%%cLYrWM9OXUEc4m>fr?;Z9_nJIoSRH);Gj!_6a&Uvk*yO7G-#Ki2c8MQ-@(Z zyeAQ{Umy-Ys*9`iA&|{uYpx93M|ABKyWy|N?sWDg8a=1r>r=HRiZ?cS4EHI9D*Nlu_4c;xW z{;3bpttv=uj8&aV`e@@@ESDxKLi#gkVcz4tRCj=D2J$0Cm6?%0m+2d_a=wPhS`Y5m zEAb)^RIs5+>VUSum6>yTZEr7MtsprjW%!Jfld}%R^xHAJ`6l`uVGqs9qElYs>{LKB zXV~bgNP(lkY$C06j(3T1XHpXu9A`u17)rY7$qMRTHz!>9zcL7Qyq5A84+IVT54B?SS-j`5LR z9AR~(&pZZsnR3G?i9r&zvsnlQDc=do8{OKfZ9!0f>h0YsIoF+`WXxc?4n~#?eRi>P z?o<={Nqq6$9Me0i8|BOjZ#y~!NT0X|fPiJRNg!lXW4|%+$7W&mmT(xgyt8O6_2LWx zd7Z%v;1ma`^`sSpCQmnx=*j==Tp?WDs0=vi<6WUXGPl=(MeYyMRhi!*NLFqw57;a) zH3ePry-(?xN3vc{aJPdXEm$kaAF-khb1HnwLZk$Tb0{t%*yb>Bz?aFIL{Yt5Qz z$Vbd*w11ed^3e8A`sBv5^s=BjTQl19HMnW`BPk$E0X3`YX_h}A+j>LK&D0A^ccRaJ zyg+q9Y8og^@rSW2)=#kk$FU@qYiz(_>un!*Q|P&ewbvFI2B}|vLvRUu5Q0nR zJ@whUX^&#c4O1r_L5gCVoH3E>(t1)9<)EqVE7Rzv`ElrqSosugE82(XJ1vVHq9~AE z+E}MznaySu|7gtSlOB6}&;)l+iZ?dukO(Ta)e&BK$i_8c;UPKIwE%so{C&1+0Bmd= zV=L3c5E|@RhvNit`yvuFoEa^KdxF!NBO)ly_YsO;eq9`H?#LIVb{Lvg=jJ7tUhW|5 z;e8m%+&EG?$BDsy$5!ArD$_?(RPNSvX2f02i(~!lS#dXzmZ<= zFkDa5Y2oX{d_^Xb>1cr>OS?Z82zwgIn(7I94wfpeb9;~WLNsTL5#Mr4 zC)2U5(Xi8e32wf-y!mM-`Kso;pT|UujKVt?RP& zE(pt+^Q{BP;AH;%SDhFzn}<%F$P~D=JZ$eDN@n%DjJa!QXk>9mXn%RWnPXfhlLIp! z)m@oi;!At~@bu?^`dX@1tH}EOMPRH9T+F zciHzwEc7-^%E*Bff*<;JMZ4=h!M(Y@C&KyqfZX9Gv*EI^&q_g7T0m?pD-J6; zm5e^W7_a^Rd?*aTVJ`ePp7`#M3p@Bf{<=Bh7jEMU2bP(m5K8Odew%$c-~RCPe*%FA BKi~iW literal 0 HcmV?d00001 diff --git a/docs/mindspore/source_zh_cn/features/parallel/pipeline_parallel.md b/docs/mindspore/source_zh_cn/features/parallel/pipeline_parallel.md index 32057ee1b1..74d6fb5988 100644 --- a/docs/mindspore/source_zh_cn/features/parallel/pipeline_parallel.md +++ b/docs/mindspore/source_zh_cn/features/parallel/pipeline_parallel.md @@ -66,4 +66,10 @@ MindSpore在Megatron-LM的interleaved pipeline调度的基础上做了内存优 *图5: MindSpore的interleaved pipeline调度* +### zero_bubble_v pipeline调度 +zero_bubble_v pipeline调度通过将反向计算过程拆分为梯度计算与参数更新,进一步提升流水线并行的效率,减少Bubble率,如图6所示。在zero_bubble_v pipeline调度中,对于连续的模型层,stage的数值会先增大后减小。例如:对于有8个连续层,stage为4时,stage0有第0和第7层,stage1有第1和第6层,stage2有第2和第5层,stage3有第3和第4层。 + +![zero_bubble_v.png](images/zero_bubble_v.png) + +*图6: zero_bubble_v pipeline调度* -- Gitee