From 1a3ddc10db70e1945e6dc753d39058a42aff4430 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 19:45:54 +0800 Subject: [PATCH 01/72] ADD file via upload --- mindspore/ccsrc/新建文件夹.zip | Bin 0 -> 67125 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 mindspore/ccsrc/新建文件夹.zip diff --git a/mindspore/ccsrc/新建文件夹.zip b/mindspore/ccsrc/新建文件夹.zip new file mode 100644 index 0000000000000000000000000000000000000000..060b6d95761937a89135a487430fc590425520bc GIT binary patch literal 67125 zcmaI7Q=xE@fXJ_xEXKZ3*ZQy7^ zXJq7JPYD3{q?o7upQrw};9<@8TWCR)pKib5VqZl{7xI~63an&vL|+Rr?u`gnw`*HP zB8Odd90zD5vjeDulQ@H@yg$%8Q!_JDGcWcm_g|u?xOpCre7D`(7RO|=QU#Lc_1Dfi z8|EM8A4?Waa}RJkf)o|;nu`ruu0crfGNs1zQoV2|Q%0oKh?B%FOZ68+qP|J_T|riN zadXD=o(G;wEptzl%JU+o@n{v!rg%YT8@Nj&I>zgK7w#YoWnnFIXoALOF&un_Xcg-= zzeVd+iOs>Ev-FAC(+3#XpkTZjg+Yln!`^C)2uHQ(BV4Uqv{U*fp=gGTJGSCJ0aM$fw*=}0dMWg z<*UWeJYLZJz+EAO%9Ur`&QD_TkEzk{o(;j+a0Z3Wy)jWIEt)9w=u#VDjD_nPyq`Ou zHsbX<+4L$m_5^0D~TI>mvVxh_P4U|$Pl};_F zk&&PcqHsfxIE%$v=df_5EuHOy@x`cirlb`{k~+2Z!YvrM9!LRW6HH0$GmhE)Q&%UArWhN;1@D9$S+#?K+`Z zA&p@MG}EPiwaN12C*`v>U*sBf8sBKQV{YV394o*IJt+?_WV)nWRQ$<|h+5|gLdzeh zU`>fpEl&p@+2zlHhugl^=|Sb;etR4ozm@0n@p-)29CR!9E@@!9>GpE-!ZZCuV~AD%D(9L`Ldbn=Pzo)D5?#o$P^Ll- znIq6DfNUoiqU1Y2A=Ip`(Pe8>-;mJ0wYl5r(e3u7cz1NM+Bymov^SoyH+76JKErSj zs|)#Pfb(;Cy4md`4>%F;*Lk;h*}2%?IJg_z4SL7mrMH`Xlfwff%^TBSfWG^x<4@bi zk=kvD4VLnz9WZVr9(l~5YLm`tQx7+jqbzDm zh^mO$P$g+>J@{$t);$j`F>hvqY}i56QPb2)dwhaPKPX1?|#LO8K3;;+9$C&N;YR$4Lu`Y=eV4{tkQx` z2ET#z+-VDJ(+49$ChZ>4Qdj| zNQ5zw*>?H$AZoTjutTwZ1xa-ETLaR5gNW#0Ub)`>Uy`&r`Kv(k;}BX0Ng|TprWlp)e8-<vjL^V z6YK(Y8_jSQC}Hf7mu%twbIrdkLpY3ME`q zyDT|zf8;l%2P$?Vqdw9{I;A4To0D0hKgx}@vcj$G{PA(tXPG_aF)RFhMP5gt{WP4d z>1T{czLfEZd;59Fh|gz9=@t1%ONUVvmPL5R4k9zR>%rn@`&9t_gNueV<2ox{qtGKT zom40!?*-7i^SJHoNCIfct&kRo3Jgw4x@eB-TnR!G=gN%(h z#Q@|hqC-CT(IFwMbJQXsYKzpj$YGtU1YWVzAx0MSLI7F}{-lXtp#MwM7+Qg-k4cf4 zvI7GEuz>*p5CYKuyQnd;c5!kxar}p@p#}hy<@F`|Pg(Qd!$Xv1qmTs=x@T)C4orxR zOk;8nz@yj8f--dy74KP`O}UarGH&WSM=?YY?ho@Dz!6nML=bo*3c>FmXVb54#(VW$ z$jGuVo)ZoaRd`hBZ;|76wErqb1s6R7C9Ic5^`k{FhUcc=SmZ3@ z{NBA)A6v5)9b1!x%5;?TdKPQkUJDB)2Q~8$N&%9jKbASD%K?n183bAo%z``*iWe}H zL|h!exREZ!jI&WFto#&FB1UE`A-z2McN?HS z=D!^DXkB*9>RB}(^n|{+oMxSUUCF}01#k5Ll=GNo#ikfULt@X6%06D2g^QObyKQp=L+ zSDkd&Tf&TV$Z@0vjY>XdWuN)z$8<9?0i+y6tO983*bNPAS}|W$uSu?5w{PZ3!;W?W zKXIgz2X>(pw-N-z|AQSJxe8a%d~F9G+Nv9i$BW<7{r))FO|6Bm3)9Q<=l*`tHyUiQ zEONf)0@Gd}FMS!mNqg7s%V`9Oufji=i(FElkKr*=iY>ik;1&tTfbWjfKoj={A^}H- zl3r3a6uZZZt>fF0OB<06aoh9N^EzdO(h*2|$o=v&ZLnah zosN!EzAfD_O8{Cnv3}%=#f?vP`___4=fN@|uY5MgPSDj=rgE7BvCRrw^mCZfW{gDI zsyPb#t=0dU+mg?%@v%2GZSu_L>HGQar5n`vY41C(R=zcNYpdPY=yc_4|8TWvdiwg- zE{%1=!@=oun*Au#cW_#q+l{T-3g9jE`)j94?_VH=vjtIan*+en`wP;pzaS+8VEMm5 zYG>>0XlJcwYHjEC52P{x021m7Y5oJL|L>d>73s=F0hFCnb)1V4DVwF7$SBK7FjgsK zRCNQH(sTXH2xerXc5Q`wzyoL%1w}-`F1loDs)^@kGuw@h2YUB~;`2eGHp|&d?%mfM zi)WinmNhVN_a*eHTdK8Yqf=WBoTOl;y7Em}k3JtU(v~GFdliqhTv;3wI#|+Y0ythO z0T)UZqJsl*#qxeiDw0FOtBPCy>R-Yo6si?o%;9`-1>zqFj-d6$;r%4e8Ug{j&KAY~ zA$t$HC=5s)C`hFpYsz$GW*BBrFurPt*5n)r21^+>Acas1Iwgg?9l0cuInm=Q?RU=v zc1a(KhXRvLnwtssjP^`0Wyu6*@9B^zUM=t>k@o)CqM3k_{?G(H`5MYR#Muy~8fU8l z53y3=`Iyn8#Yk7yRb;-?^G1#}0Y%XWRIt!6l|Y}^sRqkT@r$88Z*P*=a_k$|p%&P* z(6O!ieGZKlYa5+)LSH0n`mjAgVqKwsRe~1vOGKJ zC6-*$=RA(wmtg0}wvP*tcUap4W>y_@}5#7w}Csj7eX6fA& zzuLm<)f!zN2-(=b-khC!dR`BnJp|4Fo((dqMX-L;w{ZeCyMgKL!QJZZe*Y?8sk?q* zN=f>YzV>+pKBV3xvLB13Uy!_7TeG5Xa_ap61ZjA|!1FBPBlbBW)?Tbvu@f*a z^SK0Dz0GyRl1c73mMsW7li_}F@F`TUIm{abZ*~5xb;!{Q5cBDBK2dAcx;SdpZlw%C z>jjVYZP2!z0iHlU;WU%IKpYu;e3L0H{+2{ zd%xe$MKk}uU|nnvqAu~Xaj^6k)+$f{0Js3u{}-&CjsAgVasWVPaMGNAp!px6)vA*d z{|jPDI2R*QwvWpZ>ZNbhD9w_X$Vvv;(s6w*WHYiAB`6|YG%7j~1w=$a=khb5uAb=X zjt5j8i$!Tq7XvnvnPcwV_LYt1Nq-#$Bt>y1{<*qlvs*w8%p^7weP#ANA{4$NC|M}7 z>=nFBa=hOe((Q$KLt8~i&H@D(i0FPwF_7qGDl+zKR968x5rTB8#}8ph;j!TbqFjHg zt%l-|K2#=6u|RD*%R(PVTOn#lw0)XT);WL;86XtoNC}EC3yYvrARJ;(W@D&;vlQ1H z7nk|3Yu8SJP-AGJ4>b-Z<&|@(a_#CgH{$)JeIEF7C4%#mbV-?(c|au5GXX3KW)d3& zq;Pu5izW$(R?@}jPcn-6B^x3Opu@lA)@sO8yB3DGlnD%IaW9^Fr9f%6zuvH0i5)pU z3ib17{gF+ZX2o*2LWgp7-x^lRta1~z)3GJ8_E4cpH&8j{q6IS903sqKn#44I&RCMP zoSjB1X8|Up6hl$zFNBlY)3ILv2g1XuRGg+Dh$*zCiLq29XQT{-R*bUFM4D67kj9uJ za3k9cFLj@qNHq*flMkd3Kw*;7BC+Dz2$RO`8!jTnBez2F%rJ7ADxx4+yM|lJ3Gq*R z{g5!uu#Uc!mx#c#obQog@peY18J&!P%DgH_zf8Hp(8aS zZ~b0Cte2ZOIrFzSQyGAH61yZ#SonV0<;BgzSE`Ij^riFx8$meC^Qptvr~)px?r4mo}$@#XBkKBE){MD2Q_*;cgJdefc)L& zV;jtVqCapU+|)Mzg7Uzp#2gMXM3&rw*(#jQO(5Qy1+Q zEnF=7l1CQ((bMIP4pjZU<_zBXF$5I>;upEU&*R}OcUvv&f;!=uVKQk?gRKNIAXZtzl zuJ`GE4rfriQ5Yb%ehfHp{{}ATuuwgOxL2_}rpD6w+t#6t7^%<$Cr6ET@FntShOm#M z&e=O1YCadr1fav+?od*YvWmor55);|oenS)vN~C(yfTbCe1|arpZZ)93TJvS+9Smr zG#o4n13u0UERUFXm{49hoZ$h0$Fzg#Kz~*wLH1q%K(FSb0D)zqvv_&FapspVzx;sl zqjr-leW2u{yGyl_m5Qt3KoT&yPn;BFVD2Sjz%U!QKp+5oJXz^^0-rr#F)hUh$|~e5 zsPdHEslY?(1+pM(%w#cA)w=ScFX@626X2dkD13$^a77FZ4i$SvOg0&(PIIhncGI47 zyDt4|m1PMawu{DQS+#ogg&1rvoN&7M~9t#cf$?UCK={V>ngkuEI;QAa}DM4eMx)xP7X;(zk zwl>$yGLM&|+(ni0qpx^zy4vsG-=#0!9$&^!-#4f1c5av^lNI#jQ*D+Ab24x&B!;lG z=6B$!1Ty1a;#6pwaJI9itqR+!euFU2=4l@jskjXSoTrjX(#NS%inTAyde_@Vl|j95 zM{spvQ}y)|#@U?WL^4TEx2sLD;t_R{(uxMqQd=O;g^#o68Bw%-muKq{u45wUs`>oz zznhlQ>sK~bnA5@=&txl+rm06VWY*4(i|$Lz?t?}1_N%---ff@Gz~D4lDC(Vh0x`{r zoT4)qt-%&ZN5WNkCG!tq&_gt5GAj5bLnp0{acPbS9L004_5$xIA^`_=B;$4voK_z{ zHrqV*GuA@(EV4_pg;lukN`0KJ2RQO*cv%o*x4Fb|3li)0RSpx%tQYtHT^#` zrd9u^O~KecRrg+sNm)B{IYu_rYA$PTQgl!)9dE*wF*}$V(iHjIrt}Dc?uHZSqEDcr z9)El=v)kyrqi0Kg*mk$s?^k5J&t%_bkH0HfEs_DV38sSL%3;YuN^7;)h5-{=8Uew% zW+lH@3DzmZM?;1c|K9r;B0Mmpsskr{W&mFw(gzQy?Qm(QrTjbU85UGtnS}#WCYcp01HXh z2@oO$^}3cSFtkiO1U>CbD1yUjoiyn@LwP{uPzt?DFs#`_iA7nq#9tdmwy#r|%J7e{ z>|d4z)8?FGis+HN8eUcFcb5{%Ax#wV*HI#`X%NFEwA44RTTE9zh5(w<5;l2qZ#D>LHtOad@$ z%`zzm^UFd)JoPX%gFT7#ac5q-LQUp!nY6Sx(t=nD0i z27r)48A@4HrAZ#Uk&s}rk! zweG6U?p!XfRX}ZYtgb%gN?)16WIAUtOapwc)TF9%cDVVi)8VG0 z^Zv?-21HXHC}MXj9Nk?-=w3y9^e7{^Rj|FgwX!6gDZji5O7u`@mhLa@&eJTr9;XxT#cAt};y$wU zBU3GgYC;1m*GV`I{UkIKg#ydLJ+4)ZW`sn!e!^eEZJmb*%H;<0SZiW2Q1P%(oJnPG zqqD_(7^rB6VAir`21U>kZ{;G6zoq9-*jOa7Vd(l>C>$A}FJyiToT^w#>4TMZSzJV8 zP=%l+!483d7wg1g<^|141=JHi&|_ zSXfRa22O80i1!*kq@x1c(+v8H)WC!|0==9}gg8!+a_W}hr1NP89JVV_s#l>rn55J7 zA2~ZzC{W6vJ2b;P4r6j?P{@_|Gw}%NlgkF6wlQUFN@`xVR?Aq(2<3Ps(j-+>&8W~! z&si5L&b?7m!Bj!WP-RXz+ARqMu%?Nb$&y7l3lF-qgPWk305hv>*X&Lw*KaI8K@fX+cT8bk#V4s?`lMG19n@A&PpCLs@b0reK zeUv>NSe(fjoq2e-!>^rj+2wKI+tKyuZ4m9p#t=@E1q1@>aSgs*km_57dux!YQKF17 zl(uJ107BoNe};DKj1er-K@66J=2p&1zuRnN%+MC7jsBh{XX(G@u;PC}k9OI77KvVf z7Uj$!Y_+SDI(pPP=<;0Re55gtXgxB0T?Rj$N4<)6^Nn{9jg|2n1^x9yQE!Ca{;LP$1J5cvP;FPw%~t!8PitgrOJK$-*lFM8_Nsbk z(=8`VKB3CX%EAToRr}W0egCDRE^nTRz~;2GB79rfTd_#u~P7Z77m5tHjw=qW|jc3#n#CkO+dxPkW-4rrO8 z(^1@!@4L298wpH!sH47{w*uvf)wm6>v;2k}$;O>n< zagE1SVte7bd0P`@D?tudg7xgrEJz0{CcN zKjVFgRbXuRAu-$Vbq|79WNGnut4&WpfEeCyr zg^huk$v^n(uPLPTKBnBK`Rl|S*(&ZFa+C-K=Aa1pJ`)(?u3j~N#tpkmlon$Vdz?%1|Xjh-?Mu|C1+ zotng@r>#w{q~4sjpR>>8UV0Q*fW{?rsDI`|SRDpa63>D=4~!ScOf-~(v}vQxFzm_z zlxI;%lc+GUKA{}Hz@B__Ekrgi|`AY0^?yq<4SrHs)qhuBR# zRf=Lw50O5chO{pTr# zWNT3JT3Z>Z)r2VnxyC|yvfpN+o-LL+YqtM>cN?Q^Ou1~*? z8Q9mtp`@d$IjzUr1OLMJ$J@^R>-{3=_n`W!uW#4(=gZ6b-Qid5?anO=RI?`OuT=ny zVrRY-!0r9XO8~w{Us4fFxQ5dD(DRe81AE(rAte#^XFT|W%I_7G{f($Mj6dyNuR*dh0No!tb$-j&yl zytS9CTVdue`%O2DloVpq0{02KaMjtqcP6DqQ>>(MpmUZD4WCCx_d{3h@r!P5J$~Nr zm$QvHYIQ&DUS{t%$LE{vTRq)jc|K*9le)}kOPF#@j3Z*k?Q94JzVi--czF^}mhW2y zVGZ|)I!i*&k6#nB;pY*ty?GJ@EZ^k6`?S zA-Pg*^=eU1r`62chC0055 z*u^%&)MAdv?{|1d1~(($2=>hQ=>j8JrV~X(@gc)r)58N0J(CUCDp|7?3pmi9sXQ{5 zUr?=mHvH@X79`)FSPcNBF7rj-X}dM&ZR2+zr>|GBM>N?c-zV6kBnW(1jAA&%1@0Kz zA@d4s(KmSbVe8MZN=?7-7AUjlZI-(vvgMv8n#Ft{v{`TjRUW7>2$oALNUy@1@H5-1 z-d|UP2U|j`@5W+yLQ=ky8O|%g$sJwQ*-kjbNPzc#8!`?fmx&)Z5ym}u{~82&r2iTD zH&+}{6}Wm~caE{x--gfsp9^GbJ2Nv2TeE*q;NJ!E363ep{y8Z(UEmJxEWOwm6gHo2!f)bA`Tq}If{thf0)T_d>rZ7 zFFd(wBKH4wHS2jl$$BR{>2Lw73gRWYkwR-?wce<6puUoq2xGD@TRkB{0W5+dJVwG< z;u~m;44Nv^R>ViTQHJc0?T?HI<1dndj8!U6->WO{0$efmAD~dBu$`soAPNX~4uFP@ zC>-lYY0{7g$h|kO3UIg+K?#etj}rp{7I;4c8j}ld6cvlSZVn>34K~R7kGNodX1!YWzR8Go-0(`}g8^SFL8|MzRe-SS- ztT5b!M8w@#Nwy8hC16S*wd#<`B6jjU3uz}sn4#%os}EV?P`5(=WjYbmOi@C^E%o|% z=gk&K)NE!Mltr-`VYAhYv_V?8}P zm>qAu_ow!si$+CG+fKiH4mVpP$kKMH=618|jDz<_Om$D2a<}~h9HS2J=AUnmk7vFI z+bjMK9%#*XXX8ty#nyr`Y2fKsg-_vSTI zW!n`Y>OAv@+8UdWC@bZbpJqEX7eP`*5fW}`Xwor+_9sb{psq`afu@ZB?wyA9<<1qf zMh~>1mOYHQo$=F2(l1HB=-a9yEUq0uliTLJ|;b1PHY7(^&={wLd`Y4oI6W z8z@{7Q=uPxv9)otvR4XQO_X751;91zPZphniWbrq)LaXLK0=B)8n~m)+iavTLTJ$D z8SI2SV;ZGs4j2}Nxr-!x2e^R~crAN*W6A=*BWBAk^A`@#niwI*#B2Tx+yEG!y(pl= zCmcF18G$fR2-8|3i-4isa}>fgY}J@4Uul3mQ{D>5F!dYyJFpGdt8<3h!$9IiYg!Tv z@KALOwzpPlT)loqanQKVKW~a^1=k-q9``9Vw=`iw#UjaZvKhdZ(^k@sG zXyRSduTnJ&0XUW_@jL!e!OfwptxB$Zo=}3t)e{7dRfku*NI6u+z!f^a_~qQB#H%3? zJb=Q-aQ^Z7t)f{OOEHhGPiW_#- zsNSGHDXra$4k4=Zn{OB>SWxxTkrUhvGH9EmOge|`Q9JKyBV{T?tljYM>T5=hHg8I` zf?RGHdvt=FO)M3(2|Y$k3tGY*j&2#w8iz)kCSOsqQBYE+77#hO$HsyW#sl4L(3+ob z_gCk~(`NUmRc+0m4|k`#uj{1dIJe#&uU=F!NCiV$N$S-4O~+$TpJ5u--9q367=Q;* zl;ChIBx;NtgYD7eM&8OAtJ}>s0Mb*1ja`b{r;Zp^h~+xNA}CSZ9w)CLV9lnn=GC|! z9nVx6C72|tp5E`emoqRLVhTcs(Y^?Fq3M7smTZNI!Z7MT|}=;$Xb*o-kMmg=)diojpLxV9+pc?c&W zB53lRRL$d!jX`K=1TuDac=S45nH#)2oh~nbhwb8Pzk8fN%vQFp-0=Une!j%f*Q(@a zFQ}r9UfvJ#*g@XhB^gz-SvU1_R__ix%Ug9Y5OhC+czS+jbX|lS#c((%N{`Ad z&9WWDC(pdyKQQNe``znSY>7yvd=k*`>qrccFBjYO1mKtVvB=9YIuq(NLXbelrU117 z0}hVf>C60`eZe*F^`~n#csY?Z|4~E_RbohEZPk^bQD$`LUQ9+FHKvxxQ?wj&mXTw2 zA?a?EqKuVSMRplkD&{^_Kz8|#$OPK+I2OryPEw90JvK&4pe4VUqWf(xMoGTN$T^*y z^z@yPO~9ABh{)mHso?XfVDHT-oYm!|fT)t`wPI8RKWff2S+4*hB^msg%jmQ7t0dOf z8Jfo8wB-#Y{hfV+{e6xY%iR3^aZq0&%3(>mi_m;SkRj{kUfJM8QF_&o%CUIFdG$#} zG{NGgJW)m(g7p`tv$g2FK122$hLvG#$OBAcS8Z`y-Q1pVhKEM#q{Ij{YSM~D86I!Lg}i|?BUlmn0?ao z2qY0rBG6wW#b%x5AUBTI>_Z()d0z}l2jo;lHAG0FOiVy179y$T7vY8~0ns_?sFx%K zN2Oi0(v)T!(aAQH6_XO{wik@lCgO=iuhy=FakzQXr7f))8#{a!jc&n(_`g5-5x;xe zeyHb^<`)vN%qh4Z1*%EHfb<@;BI^v2$46kP4Bw=kD2n^d zskHXuel<8}R=MnrXGT$2ZKpR7RSd~*mn4~()SHAj=j>N?75cWLhq6q3P|`1d2MgQ# zt-!w!`hSbbzANzSuIu?~%KH_@J{)i?ny>}7j}RqTY2jx; zR7p_W>u*(ui(-^chz5sOB6Ex2PDBu8tg+bT4RiSIS}_iMAYc}Ws!SAM7S3U2l_Ft* zizchp7u{jTo{}RAoERW2tVW6XM{-2_#&M#)4BV?+fM}s30-copoEqQ!+bk1W8bVb= zjs@BW6aawWZ(XGU;QIGj=HFWBXyLABWMFONVr}q0Ymk3bSvCN`=o`Jt|59cD|1-NY zwCpw*P)2v(QP8|7k!fkO$f`Rfs_I~?Wh;fauOTyr{9EX`+Wdp_lAVv&IZXfLHx ze%jyvT&+o-@%Zfs%+8~}vhn56lU(HLb@5&2x^=# z1skj`k;x!{gH93hB|-;!;dD0C}k(rAg19>+9nHj{|Ny71^U~DmhxZAT%aud-c3DOro z(Q)T2Te^f6_2)YH?Kl{*&$r|0!dA`ip@3zB@Gzu71bF|YHa-^aZGtNg1pnU*GU&Ak zuZX#yAl8Xtz`E}$N^Ew-{br6KK1ihwW$}DDU`)p)uyIh+0m9fYtX{+{IIQzwJLVjg zVUj&?qj`4tBWG5(-#unAVL^x;-Qw!4YJMDeJDT>-h|g7aXOnXmt%wB*1K^3d@VLQ) z#e|XvB5Pv7=ECXs?h*sUY&nY9nnsTRvyhpOdP8P&h!5et#YEYPfWXcJEOa_b5hfHV zpHR>)STSbF>wkT{SJL>*fGMdt?Qz}1!5_FcvN(pF{^gBB0{Xo5^;Y1v$xZ{j37NBK z(_%sUq=*c5AheWD!J|Ho9vLR0@-cGYF$FBI;%upjZ}7Q6q!uvMoniM!rEYG9buDw; zPfyPCV)g2N3>{p*T76sRmDLyXiJnM_CZVU1f<1&LZ`osZ+`W4%7xqqU|JWpYc3boX z-{5X~U2Tvv1j z;hJo6m8|c$xc-boJ}MXC(inu~QWw+|))06O;j32RS<39$ojoF#5ypx=kyHNv3ksZdaw24cvy)xhM-1)~#P2id{iksqsss zjaEVrVA4ML(V&Nw+aZ8E9aej6B^rX%CNaCFym0m_=vcK~{SYuaC$N5+BR4J4+fd2~ z@Qn|hMpwB4^7N6Wuun$pVO%K}0k9rXZTG$T)sMN~yf>pa0e$#{t*48}$H#HpZ|G6K zZwKvg$Z09G9OZu#e8?^QcF)ZMA;tPlNUUYQ^C&9BiMODIlt@EC+9yt#q*q?eeKqM@ z&u&uw(G3O-h`-!0BU?_@itmBnPY@|Bki1k4h?RhtP*4CMf=Tsg-WD^TWdMdtJAYk+iW z@VGa0$M707Ptu{^3q~!v)M_ z7Q%FVY4}|u1TK5UtLfBuO)E@ghf$@;6-`X5<7TyR=@regli}*`x-nm3kV)&;*@&2b zbM%F3Vym-}L4)Q~0sxS2{=?B{V_|D-;izZgZg1!K*H!<&KM4MH^!;W6Ao>U6|0DDo z*TQL22RK^1{#Zm`ODvHRa3Hx8=eXzgv-G&M z35P#)O_Ki}-Q8|icR{Kln?nHu17gMURp-<7_1eo$-6&0)NngiM)@zz|SLYXOwNKi0 zr#1Yabu*pShB&U8=A&eFY~5)sha7;5m{^TQ1JDiq#$T;hEI_{vo3T(wTL)A}Gc7kv z-thgH4Y&8+0xnsre2UV332PX{_hRAX3LseNPF8GMsSQF`tL*{6u-8@CPSoDuem`>;D>QxmK0sSXyk+!CZmVh8thZb5}~5goM)76*b`vsCq$i&oZgnCmb% zYiQsI60oDC)@0Gs{bHA+rIDGYMKwjMeh0Pk^u5tHC}-dYmk+BdZ_lc}gmwZKrx^uB z;ge`M2L_iYjY@Q;lCh48MJ6n`K$H|*r0i5}g{fjy(|n0Es&;qd^abW<$c)u<1HPIE zfT$u!&(v}g<{lzSq8DLdW3VsSq*&EN9o^7t%}V+xQhxoz+Fgv=!O9+uvfhk33$AwI zY0$h(L9En?MhS|@#ipWS5(L4Bq17C;jHHjRgS)iT{;dJw?AfEZ)#THu>xiv7>mE`j zP2IDt9n(|M86!3K6Sd93oL@4 zs`0A{>q|iUxnmWhl4G(urCzFXhlcS^lHpPiPC!MqQawbjS~Cnf-w3)SBP(`_o+c$# zKYEHj+4Q$9m<^5WELyBNKxNM=K)G5vj1}we5fkl-b%w3MGvq?THz16&SW+vkSGXo1l~ZBXS-8019+y6-fr&TwI?F)_dRAdS zelJ$eerhx<%H_R~DixV()ul?@7b>}Hh!;gilN$8H!wRA%Tny|WL~y4K&BcBTFspVx z-ayUPUSAZjZ$hibtj6osysNGc+t16e&C%W#IN|02Ao?BeF9XlU=bhicfN&cPNE2^w zE({`Yo6}#m#5naNsM1xf7?>qNuIBWBBa&7nDgnEsB5_4fYi(|Q4L99JHL7~9-c?R9 zr^_Awlse4({GEJARuByx6TMMd^u&=rqMaMhTNtwDm^x}(?ucWNq50N;swOs?| znp1<8j0vlTbf!_`TWC)8S~4fC=~4TJOrsnXj&nZduD+2F>{+Mz*{t%`*+CCMkHc1` zX6~2o$D28h;m1)0@IkFwhr=Zy16$2t7Zy$W*`1xldAa=F#;zx&)U^`8^=^9VA^QyZ zCm$a}0)8X!4cK6$Wwmk7HNh0a*E@%qg|4@HvW%TkDinr(Ci54n7{j{7idT|2!?{O_ zTetx#2F{WZM~6sZ7Vy>$7ar|H6Ao=iu1$W1tvgco@H2F+NI;Gj)23z)CqD z97a)4a6z&xbI|rI7$(xRY^x0ex|LkKels>);UN@St$>o{zf1*?kX3_U7a(-KXHp~BTyzB|In+AF4-kD+Gt|AE7(hWr= z*@SA8j6>$T9-|}>hKu?+2{Y>u#0XbBep{=aUSYEK9w~qqFuj|EZH5viqa(~Q zu+siR1IxJNvZf3Z(K@)&eF^YLw+gT2I^Cfc#0j$%_+`;*xeuZ;?>W=40$FcoZi)w_ zjLCJ4uokgeXc0R`uawGl!&a1PyW8LXo$-X9)Ye`_xlOfZjv zi`4Dn2cooEV_5B>2g~fypy}peJ8X!65esD97OG)J8}wDyeb(()j)?%R^;&n#&XI<^ z@*(i86C<}rb9sIfJm(7L^?V4Gay{@5jVflmPH3{#mcbP4B~tkdQ=DUdcA;Yf_srR3%-8SShGAwjW`9$3Y3! z8FUB%M@jDN3jK{!)<+kRYZoG0j=LBh-fzcIQfBi=Vnk2Qk%~_tno0US<4_1qo~|qfGxsce;X7r5X?K=4Zpeu zSWmEpxH-ho-6pV{)}=cy6SBpQ(GP+qO_clo-U}L6mEAa5zypQ8OWubXOy|)Mf&tbE zt_fk}-F53|SkZGt8VS9Nh*tWsDGDfdOJ}4(v&p&{cL>12EiSww;AR}ggP#rWM=j-# z_Qz)}ynlNjf~$f1PsZXI-6heGVYV2j@}Wo0w|-VR=kvBcWS#Mi(Y!t16i;`dF~koF z720E02kQKEH(&>t>s7uvOm38yhWid5iF4I6v>Ia!jgxQy@>{l!e52t4HX1fvnON%y zBq&IXRoHLT=N}U@ALJ{qeVM&yQ}JaUro;ukD$9 zpj+udig5^6TZamL?AerTA}ruNG1@_RO9Dz(HA#`F^5Xl_o~FhEi#X+2cUm>HAmFy_ zM&1_o2dx?JyhC5G^$i-2(|)=Gr0B zGi(ETMHSUU4dR1%&P4L;3+fh&b5pi506FVniwI>7t*0%#WrHEHVp;m3Y zH)Kj3sGcsGfII~Sf~ET4)>7a!{y?D-%w^P3ryw;_0~5MsrqBrxy~wy`%^)620HIMb z>?AcluELG);IzElwt@k&REeZ_k~-3p_+ji57~<}RZDj~r`B zwFQrC`zPWnJ**}NJLH$?X{75BOr#O~lu>R8MVhng|1xda6DGuNjR*c^ZBJ<3S^;a0 z2?*Q(gKne|=eh_=)HHu^1II6Cg+dM<7!exSOFWUPv64|-?P)_RMei>&YvTz zvX1w=xNEb#9ga_DoMCk>`JW}<6O)P>C9z!3tF6aT#Nk^zYh16$RgWX_3bJ?0-g|5V z0mVOmh)IgAb3VeQ!r2H!lx?hEm#MS#e7hVzHedd_g=2kyF!HxoZs}Wg-LY4X*mncg z)>e}66MnMm5HIUPurytA|3ALoDM+*^%C=0~wr$(CZQHhO>!xknwr$+Boq5wQt0Ow9 zx~ls9o;dsKV6M6MT4Vk@+m5aS1lj2R>Nm#F!k(|DUBcRIeS_*86R(F393)A$Z-#1` zRoFf&8GyE>lduCx1kRdHSp@{ko(d?9Qs9jcq2oXhI6P0HQH24-ccmGI!b}jqX2mJ4 zN+?nfj^X>DGlyZbM1}6$8WG`Rs-Vni)C$&f#F7(*%`!+))`BQTF}tw|d>7}$GL(sl z-wSMkL(I0UQ*0F;6?s1r`pQ>65VR`56@vOZVnZV~-#-|g@nMH-4u!Igb>tQ_F9x3Z zdVwOx?g6y<-0Lm-NJyBU3 zq!RxFc5Z}VzxjD2a(bM{z1Oqsd0F%l4sXx*VeyfeXYlEY=b!)e=zeXE2ciMsa`@V2 zaDi#fe}A)D*JqdWY4vy|oFl&%3<*58Uy-*tw#WOr8J<(ecjeK^d|iewn-nK1<0{7+oY+vdx54#{$}+IMQgC(OepTHLn=xNQS7Z;=dS(ZaA_6Pb z!1XvoQ8O(o+G!ck7^v;)`yi;Ge%T7CjLO;`o^0(Y%VboqQ;uFWc{EIZm~5ZteT|p^ zHE}qnAJ>-0+44MJ*UX_n*z0?dSObsu_Lx3~T9QDcP3`DZcN}+tZO9>%aG&3HR?A{D z&u+KVa)w*|(Kap|Y}rZx9r*|fP+AqN z9G4{}b@K_uCt`FWYY#zBjh5FxegR06hBh3_aZ1G2JYTsN&^8c3qny0iK_3A|>s?{p zmrukdbxS^2x(Z+^FbxwJ!V!BFH4V8|=C%s)6)LK*xjKC}xo-Fu;Byef9ljUnI>uP& zOO+xfSe!Q@hxyn4&!46q0!u z#xk)FHg+-D)MC*cew3LDj@0X*Isv-qGdV)x!SFYPcgpghokH*iX(^II&Elz^;u8rb z40fO2dZ&TP3bW%wO;SD;!Ef>Sc-#iCdTgEUF1Pz3in1lOQ1NF3U@2$~AXUzTmK_QN z;mdOekOUD|(Sw60hhyCKeweml%916jZa!Leg%B;P35y+$C&@L&@$;qZU?E>=XZ9JI z8pDo7TFDGZ_d}lL2yxB%V(YPlT&2u>dJtxsZ>-h@%(qRS+^{%THEp>b^}Z*W7&OIt z>A_NC1wo4k6@OS5D5RyMO40+9su6gP7DQc3zNWhN3oY{e%D-k97c*Ga`hW@wRoyvF z*R`wyb0u@fhRLgd6Djp}MV*|$J45T9E?*8}`fj8GVYEE><*y2^W9_5O}V^X;uh-OsN0(43P0C!uEIL zDY#K*Q3K$~Yh6siFM;+0O)=C_gH{?7i4)m8u_{7wPjx`U*`?X+g*W9*TiW18yqxAr z7QxlCM9Et(%s|Sn|uWm{=bwr;gOA)pYW^0v%FsdT=L&=0* z^jJtIsy%rXNk{{lw3FZAd&y(S&vaDMzhI8StwGZgCk=!`yIP?Mkc#Nwn88 zp(?iwEJ%Gx&|xYNT$om{cT=s+t)EDWJss(BiNZ_qL>R=9cc%o)9w=Jn0YG|1Cq0CHQDALCAl(0ut znO=EnaM<4Q8y1zmy*!lNkZXo6+Gm<)rpx8j?CDwXKP1M8C+LU&{lFPH7vFdPXcOO* z@8042hQbN`eX3e!7O32>qea)|@ojnjQJ>f<65(cmLPmNl#6+$I=Ju9p`>4jm44%%J z+Xk3^?9v`7yo)lZ#zVahlZ_@~Q~PYMnDV_HgEV!6`lJ&CMr8ia)hSuI&Lka*t@P2=T&a+h+&#-E`Z^|2okdjw z+!3)VqZRB!tOpV%DT4Wm>cSd{^LU|LW|v>|t#aRfH>t_{Jta7KGfjCmxnEI6*a&K@69lXg5k*%m2D_s*pT(}2 z_hy|EYv4q9X1cPLWL&T7kuTxo#m1RUOJ~J+(s-X_)#1MpHH-rI`ObmfmwVi>M?{_67dr^j`! z-3aWv?qv(bIFUw00t(@g}zS}N0W>C(}?Q6zMrdlaE@zhdcn zc@&PI7U*cnwp=gjs=nBBjjGX*>l{eBfec3rRv`&!o}#~3CXh(5#(~=boP?@`8gdrv zktwlYV2(SY%ZbOG%Te91=y@rP|gf2&9w9WUQnYFx7sRBlTcsmszIF|}{Iiy0RKQT;a!E`qVQsg+|`>4D~R45If9efVFLE`0u zOSz*7;f)Pwv&6=18)^}w!j3xrC41!jWA-hHK z6ixG3%PKUr1@+riuMpe}{#X*8+VwLwSi}d;xqA#H8jNB>E2|48KmK-`<+`Zzg+5?V zu6q8n6u_pcjJhE#zg1&4+2=`lC3zIB>T$9wfdH&hG9^(j=#MBldKjXtSAEk4 zmuu7!mUj_ zxG2HdCE!I^Aq81>37S5-J(B`2HKnpwszi6cXNyX0ambn~X>mlTm4&t<9`3^!7f@ z$ELu&(tT02o{4;__qv`MR^BN5ile9aP1jneUpWKkm!--zKN1|Ip1ou?kB9erPLMjr zxqAf2jU$y+xz9Rt)ym3q+dlJ#|3JXZkxDmtrmz{KG1w^lW; z(0~9j#Hi~ABWby|o3A)v&$hrot~2eMiR4TG-`lS+u!OA=L9;^&ejTgUBEu2ji${GK zfyDh|t3(^epX^>vh>hSg!G*m0t7r)VQnynyNy+^Y6faz|!Z`-pkaL_b;0os2ig^>Vo@Rcs`5g;kS&sKP=&eMkQo|LAhQnm#_M6R%B);56`rBa zh;$$4hOzIV;L74z+2%7CN5C@|Jc!O|Xl>Ooyzty5GRxiW2MIZK4X4&KjGw)-Ye^{Q zix7`Si(87EuTrABAe?46RS?rTl{}}F>#8nclP65`IgAoVT9jgNZ$yl2L7g5*O1wwv zeWAqm_$Jeg)Xe8dv{R=z$~|7FgvH#;WUO2XnClz5(Hhsw#K`mdRC$qVs|UdHD+h7` zbW@g)!j0^lFnlbfEnMM`bT^uST1p*Iz^~H#?M5l7=(u27Ojto>pLtcF!uH`~&wPlP zJq>OiT*Jo$J@65UZyOP1rphqY(SdlluR96zc0n1{{qL{DC40`06<;#5)rhmw@=ml!g4eg(@WSfynm9 zl{h%ITsJ>xK$K`5-ZoQ&qbGhuw;PGYlZGk9ipEQ?&e+k9Z%^Z?+b?e9?XBCGag1S{ z5rg-Z2NNXMoeu_RZfiuao}2xXN*wq?Q=|p+fpn-_qLS1-+{ZkzQCfKGF-T{N#n~N@ zVBIW}mECS({xkQDu73gPp$51b7k?_sfX}5U^|%1;U!Mi?@2-4ICPTguB7=^o(;xF? zw>c8|t}*Jc_A+;if1fRz3HNw0Y)6Dt#X=4aurTgzG|$4hxy4jr8uzg%HI?>zIZAnb ziqER`Sw<^R=!yx`&?-isXMz{cCLRHd;5_9fgUUvo0f)nnQo@!D7lVi|h6kC-!maC?kNWvGa8Y-&NXv+#q^%%HU zOo{-!CEVGpj)cMc_H!mjBMI%MAqeOf&lKi~lZ@X+=3gk@3u1bzP7(H<_g7#*;HL`D z8l(SCD2kqLGLLvpo;^wEO}gB%d!Ib&!xZDeSFTo03C{;y#S7*xOIhD=^rB1icqlI$F_0M>|bxzMu4_r7wF_9?wUUxCe``4ON?f|F%{^Qr# zGC@Nz@#q-0qN z2eF?d(Zr%Ww-Mn(5^5w4V_jIB@NaX%c}&loDZTo2n8mtN zH+LnJ1qPr&XBHPm!qB5DGUMytI4r}Ct%5Q!%o)<4DA#b0d3^}i_Ebww2!I;aq> z5<>wyzk3o{UftWib3emA4KMgDYl}4|qWaNwmJjOdbc^u6cIvtu%p8RFQz_Rgte(j+ z#tYsH+M+8n-jxMvzA2@lnZk!xV3XCCm8#2g1p1pSQJ=-YS&n~?FLj31 z)eFVBv_4zTAj*nToK)$Vy9yNYitd;A`#!{ZgseZm!~ZDWkw>EENNN*_Yyj5Sg-owMs?!thpchDyf5vboLhRRv!Acd*4o z+l5wMM5Q}Q8-iqxx>0K7^}+-_f<8843Vb~sT%^09fIFb86CfihqoeggSIVgdDsS9f z=+F1T_Lh%%c;eA=(a?rlqRMlvJFLiq3}XL_lWcN|<^lVXj62%v*&2Id(e6u4@~v&I zkLY{t1gK0tt0v}BnX}8;hxTLqFP8||Pa1<9l5Kqzi6LT9x&v}xH1n6ZA9K1Y9i@QCdh@I1Sc8JnLsCQ zGYw*lr#SVNPT&`UzEbfnJA#eq^kbxgQu|Ho(JGu;F6KD8Yyvitd;&8E ziF7wi^h86e?;1$HS?#L^$1Y=py({)&E5eK{XGvk&#nho~^@-O*jPY(&deUy&2s*Ax z%(O?Dx9_x|vGASC%$bd)7r7o|zvc(z=$tLBs?j>z;f4a#6dwIr#NoCZ*|o`gX5FAy=2EJ2G7QVy!>S0+u+CB- z;hVf=|MYnP*ycvkxTj{6F69KHeN=~r)Y1M!+smoG;Azwz?6F>^!T@P?XWSY~n%18?q$nA! zP9v2SlHylIOecS02qLjcko;H%%S#Ju9Y=liQP7EzF9-;_6i196z4*^?-@#b6SiE7y zr5zEtt(kf=l9TWzaGp*a z%s=IZ$Rz{*9A~Wu$2{yT0kV=cY;aJU_C@RTgay^D!pckaoeWOW$)Mfwmb1Mv1Tl0 zxOTAQx?lf*<4No|!zBEqf%WyT<-UbWx$<0ps&sdb-}rII_0Y(Vi7)I%DKI|Tz=wx% z+Sgu-S(TethWd!soy02~De9lOFQy!ds>T41*ysS%T(i$7>@LGC2)(}ec$W%UAOQJ2h_-05^oi_w zQ}9vS4S92!Q1;tJb;x(?C%oJJIH7G{f4B?lz+?>=;R2-n#u=It2A>xq4qyc00#7nl z*O)}<6njcxHoYb%Z&C6k`5CRMWP*dP`)_MWcv~aMRNZA(`6Mai&QZk~9I<#1ZEz8K6kvi=n`9Y*As z$qCd`Mh5XnjHo$nq@3}vb9;Xkd38DPZTzC)GN>eu;0e|6U1_sslS){o-S;$&r@JJxI9Xii9L7Qk@Kml~6q3jB<&ce?A6#RAyC@0`x}2irdf%0A--B?= zayN??)T8gnxbtHxa~Cl#XXy?exyNuz?1V3Ng;~Xt?`m@z(=oh8&(G2j)qvFOQcEHH zkh)1f*&t+?$g*YM$!#`?Q&`s(_`6tP{mTltIwp%HycCvV35etB%Bmj6qqioL@l8Sy zf;q7&C}BE0%=OVTDCz}sKe zCT7ZERTM|mI#57#QC)jvm42^Fp3E`{$#-t;XXDwVKMPgwt6iZTB~@Wdqw3R??y24S z!v9K{QxLGPdK&XwZhDyL3%u*DQWyE+O!J=}d_5_9eDv#5UK~w;OVsre(-*1h&LS6$ z^jn%Uv}L1tFD)DTqO_*#Dn@oyTh{e;X;0Kwjczx!uIf6`nx0!U9UA%0DEUp(6KsTPt9^d`#?tc)HPV9O6yw86qqS+{OZ-t@wS{tB! zSKiU#Xtn|J>U3MI9oxz;o!Hm+GU1;-Aq&MT0j@p{27_{9GAFn{ zv722N=cSvj6SY}mf1cE7D~>%4r-`*K4TJCn6y zDwq?v9qV<+43}xLpCwb9x7N3(xFyV~2=NR_iv`9R&!bsKjYZc|;+9!!FL6 zWPt0K=&b-!)9O<1NumRe{}TxB-lf-)&+%E{=8C|tWM@ug8jYSr8v5W@fJ-mYo45B< z3-%WZIvam>9Pd^~CyRnzpuENKo}O%*N_7ku=p@@lwB_Ur-4&kIg?G1rCR-jfeyE3) z(tNp+qnzo;u&RZ03D-Mhv+;*rxKCU?#s>I1sNk3b#~m?guzJmn^XUhfy}E<1OYsD+ zcPG}7wrt7Oef^#`lrPAfG!7apncnWy>2Yi`7(gZ%MH0I?Vks>iQW7A{vc(MfmuJ0_ zy#pt8x-icZ>e<<-TDzRxB2pAUE6?SN2)JjB3so;TLdv1rgVi)q7DzI0s&2zt4;sl! zE#OwKBvTlC*-&WIE$J%4GG_q8w6P5%LdPnEJ(Gqx5oz~&ugxCISqk+ED{7l7z`W~H z&)6rs`4z&4air%1q8m%;Yf!AL^IL`mFBF<)*xa~rV*m|}Z6LZ4fzFYL@rcaNYsIdb zBRqfCuzOoTX|UWNZ=M3x?l(K2d}&;d6{0|Er&1^!0~SU3pVu<%KEdx*BXhjnLt3;L zkMEpF43$cc2At1-9ETsU@-Jz@@)dHlf=K#@Xt;YV);7e<^4&1HAx!1`au0PO7ajSO9kE%fc|o&G1=gb@G`qgwC~GBuOr|L;>yjGnLJG`!<{IY;XGI1zaQUoymUCyp~?xY5BQ-9}Klz0w-uf6}t14M#@k?Vv*1afq; zli)g-`cs-_mtKt3KV6>{3dob>E#ekz|w z!ALnMRO$)EszKURwrABcJw9?N@;zIhVB^x|cP!ksVSj0xZ$P%VZ8+%_>vF7yECVjz zM0KZu{|Cw0H$hCDRj}LQW#-aRakQhW9#lcT1Q2&R5*9MdX}A}o@TZ+;+!vy zy3E3*sp&lr3Y*|z?AutU7nO6%B~LT)ahCzg34U_J4vPjC>2E?}$*-m58J&2!HdJV~ zQlYYlhgt|JTCl+5-(ZVhjjw|>y_~0O?(FP{`Rwxj(Dq)7Thk4>q5r2@W7Ro02sg2^ zS9soJr;d!55?5A}yg+EAO^aDRL2PLn&Zh*g z*g-DdP<@Lo3zrvcLi zoy+L)eLJ3LY-;_Vq(9ytcAgesE#mZ*Jg|>2_2!avOsWbo8NQ(fx+-EmrTLE|_0I~G z*e^G~w?p?Qg}paz-~xS?Xget#kPc2GIkDB9oqKI*=UTCVU!U6yf5-`m*nbhj3Pas$ z0dU_UL7ONfND*L0Vhl53Tcp_m!lG9P%rg|LgFAGQ%<{$E5#EI-=8S?-N>DvwQEKY} z?M@sDaO&!dDQwC1Gx@bd!-cLrmCj zN&71zeWv-P^|dK}o!4CW?^XxNl}Xy^cmSPLrNy1=GrX$T4x7KCUpc?lZaRHz?^u)m zxf(kM++e|vyxNdVNo|T;aWbi^E6bnk$5fBw* z1ovEQVGhY(E;0lOKX|kxEC4w1j-C|2OBd&d8<+I~kw9r#DhQO)@Ta7H*=4DJ8qAc8 zApI{5vJ@wSQVrwgt9H&N4fm1Il zR}7ZU)WU5+Ey|MJ8C|6g^TJz@9Dz6-eZi&AV?K*1_U zL^4eTGK>re$O8C_((yuqnYcdcCLtXP5^>zW2u#ob`8#6GEBx+DMd(Z0#|+q|CAW^*`D)*!Et&Z>Nased}HhP%I@c0L$lIe$GVwRKTMVT@|Y z(_uKFzSGhXBvN6u0Hm`Kgw>##fTA)e;-^vjVV`8#rFh)H%nN$mg6in1bbDEDX{2xw z6xR@OJPNRrYnU$q4yzm%PzRyF3V;My@x0uYWSj_u*QB{V4%uMy$#L_gU>X}p4ZS`v z%Ad76N`T?2SUap>77eOL&t9uODMzi@tUz}8X)y7j_bc_{dxpbsYd>T=tP_sKqcxgk zd*e-%pqz>g;lwe^wg4Oj7D5*GFPktU@{Q%VfD<#kONrqV2yB)eNijB7)-6`>ec_@o zS4sZ9-8g}Ib&wM)_zPOqvDL1NQ>6~K%asUhHr4D5Ds{`lHx&mP_zP}T%4Ew!1Jt)G zz%TxJ;J~RQ1svf30Six3b%>W?N#sOAXacL1DntbeHnh07N$_eE4Cd{(J8=h{@QvjPi>ZM=9;INWJl_@PMwuq#g;f=dF; zd1c|ih9)_c#$PDVdUQxH#UwbZpy5>NG5O^S@#ryjuVQ5bb`7n~Eb!u%=Dtq*!@=cH zXzqsR_&WD@^}M=m|4o12QFu1QGSvf6TO^%aroBod`$A6)bSmgZ>rdHVV-sSD~&xV7BqKO;%wI2dl zlK(~L!s-5;xmYZm&!gGd{S0_DxMX8^;GP6l7R+8wb9rdz`0k|BTv1|`$BZ_>jGR8hfW$WLvv6Dqo9z22Y~VQv*d%_TTw~zP3w{5mI|TGW(|1CilmC z_=qeJbLpV)ODPOrNOf7m^@_qNTCf;U@KEdw*kOMegpF)j#Iay;v0U`X%-U!$+?_(S zY2rY>P!b0*l*f1<#wVJdTPHW8Ce~pPEnQ}HQdcvFZ?5Wk_i-~!bu+^tSm392&QR?hwVwVF$|7eO-(@sn&kawwL*lO=+g zrQ;7m6n0-=KM;<@U8&AhL))9V1 zsjQy;?Nyuy{k&47JQukpegh7wz{eaZn&&OEodHOB^4~RDZ#sbF}_qImE zV179zST%b|ga^_yXQ%PMBEyo8JlnG}ML(}05n%$(I!!zvL$6uhX+w9)R_^0v2jA@Y zrXb(GxNBkXjY7bVwciuNTRM7~KxhIceBLm8NDtc03}qBQ-@+bpw>FSA&sH25$p$f# zNhn=Gf>ZCVtPbmog*}egnuEfkrT}su=yNy(yboc}#>iLn1xU;?k#kb8OvDa=naivMe4n0B~V{*9MQF6xFLoX?2G;_%|~a%z|tSi z4%7$7(2F;aV2Lq62k3{1{^^5n-}}$X;<*xB{pT!obL!Vg8vHuR|Hk?LHH9XoE~dsV z|6`8)m-7u#2nh1O6czvLg8y)k$c+EN`3~u9P^zIN`WI698y0ZfN>v-?LUs*Omdu#E ziH0cRL!+V-kxxVvRF19aLdf?M^W5+o_`y^s!^OOE3H)StGvj&cb1Kc$)@r81XQNUT{6D}`9Kqfs_2qk$3Y0MjW`z&K9;=AsDD6@o}P*#k%s zJ=5lh6Qzqx5ShxRUJ}VBfl*F`QA6K=VQhbYz$$@FL_Qx(2FoedOi`)p*~B8=Lxsoj&6wICw*Yh@ z^9xm!tZPeG_qtHYW=!^X8!b-l)QhlEJSWUC1Rt8K^|LEBiSOgD*te1JJiDDd{;s?0 zYgN>f$0_i~Hg^7AFJ18ape`jgsTn={WAM~8agZ17S|UsARb&Z){aG&@Kg4| z-U&VjOmq(Snwk<=0fNcG%jAB3ILJVqIE)vgzu)arhqxsDpsB+7>8bt6Yi++DaGnS_ zC7KE;!JyOGtW9;{JO#}b5geM|u}#qE@tP!P0iET`=%SQ$-omysiOURGd0{ecMjOr> zQkfJe((ehShu^uC*z5UdOZ@GPU$67QWyhmFz22Wazt8jcXL|fx+;LfDEsPUh67FfK zHj@m_qF%nyN=={-(IM)c<_6dq^e|TKi66FV!Bh{ce{3%iXl#jSBc-+L-4hUnKOvIY zHb&Swxus#}3i8tBmfvh0NUDu9BKT-EG|ioUQH9M_o)8z)|{SrgHE*>H@-${ZAxQ7pF zuw+#4WhCc7?&pcabdINkX!4NZULF+VQbZZ1@)7H=hs3KSl#Ba;kCoa5@Bx~)$WA7V z*9+93$Vq8}rV#GB4SB$sxv;Q^i5w8ECN+cU+@=E3Ysy^DByK|xsh#`Z9-esr`TIR^ z0TE*pM^9K*L_$r0i~Z_=Sf&U<38<$TWq*nTrQIY=K;2Zk1R#03XM*hGfJ9HXN|R~~ zQ?Ak%U1AW5Y9TOB2MrUy`6SxCED{ck76GYT_$BknXMj^<&|r~jegfM=xS^W5FhM%(ili@DP{;yHi2p_9GRU|o;Y)*m!29HZ+fsord#Df1q_Dq0 zJ;vj|5M7^d6a-c`C2Vb<7Zw$L)$h+;G#15Y2An>d-;}X`h*R~EpFC?~5=+%)6vy&N z9zwK+^uwxAE8)+)9?%A3cd}P!YcZvoK7F4IG-34WM(u}*Xv&f5`jiHi0Gct=z8Pvq zAzy%ktQt_6S$0kZiK`7kubCGDh8a(OXt6GhWJBnPYP3yclz&GHOO9e=S%7X2Iec$R zM?6JxHC-4qP@{q4Rh2jGJ#E4lqo@sKVBS0|n` zmnQhduv;+yk7&1F@%UdiAb-hxD5Oy<|BH0|FSpNVYyU@Pt^aLV#xk^$lrGZRa87SC?DcH*I1>I}ZV#NKizC$Q@A#xZ@X7t$c3~v-h9OoQ)mJl@VxfJ7=@m z+w3Oi){Cih^?9e}AxCmNTpEz_2CL^V)L7Nn6^mO4dZZ|=;x+^BH0Jza@Qzeg7lN~3 zIAv}D@8K6o{MakLdxd&|c{cRd{kjZgVcWlN%MFUg$5hQJkSfM%t{Y@#Mt@`G5#$rD zK`f4t>jiyXlC(e)9pl>p3)wC>h1Uw#ZxEX$f?7`j!Z~{^6alt?vmSAi2a8@Inw83b^%w0ZCS<1{;XRWAg0~tL zU9=Wi>@84j_FkX$7O|Qg{p(%b6&}+%``1b=-pZ=|;f5<39+!_Bp2hvz@`7tMxMbH{ z;|X-Vx3;Y*95RT6mtA~00GIJ6C5PlZe9SEcVInkH?kMRt36EFOyPnmXVGuY7Y!>tH z#;=kf*-QAMDdOW?F*Bp#E&_nZ6yHSCs?@2ZDRh~~z9W3~54FNm!(IZzcR}M`tl+jS zk@G^wRoHFeVte3@JtJ>UhEl2T`Gh?IjcrjW>`(VM)TBnmh7X4aAh8s??YZN|+iYjh zh0m&9xVoVHWgI?qD{8Fs{HIan?67rELTQV+88n-ttt*UZe_C6_ofem#!`}zV(&yj( zeEe`&=Z@#+>3Dwnad~K8hwtm@|MB>z?Ux9vRYQy=j2beldY|#Wf@MUM$jd{<$Fr53 z7H+v8{Q4D(T_kH;>Q?V(OTOOedd>u#XnJa$I^RN1kMGlSN8(EBWdiN*{dO}IY`t$> ztqQn*WaFg@3&_0lEVBN*(pGUU*?Fm&O5FA0(t6EQV{m7?Ed)T{+%pYVdYh zRi3z3^#u`yXZ`@B#MO%Wn~-30=_f#*)ccM7Pk-!9{%%2F2`o?Xrvi4H7Y zFaHk&STC>lA6U$`q2N(UEkQ4ZvGny?*sxOUqf7Yv`O6dgv?)HBlQg22h&7<427wF-3bVWCopbm4KpAuHel zLLr3Y){5Y=E%M%icxO);(Wx?oW$g;rNdVHcBW1yaXxO8YQv-tQgbyFUO-V1|3WQ-M zZW}%z7m8`+Op)Dggd6eE@iL21QWE<|+x_O(l>@WQi8R)w`M|!hBDdT{25vL}k(d&Y z)<_QJH{ZkoC~TZkR|4=UA=V&B1=1BqKB%A!*`_d9WH^A}aSH`s4>zb2neKHy5}@nY z33~wY5va^%W>>n+E_I`+Mdd~)^eDtd3S?V8FtX4PQOBj$e8+LA#?gfHDEcyN9ey&f zO?x!DiI`1`rj6nJ02BObRg-9}ffUJq5KCoNP%4jC>IAmxaayrU18mt(z}G~9h^r$S z)LM`KW{#4b_-d8Oo=<@*ID$&G^}3=84X*5B z7hJJ1rqtN=m?=BKR$^T>Ep=N#A+Onw$N(cokfq8*kw{e{u}U}$l~_6^Jbaj+z;M?l zo-7JM5C$<$D-i7QR0|&Nw{&!St6SG-rgGPDXG%PnCv>_}S#XyWRU(vtnWB)Pkt9dWrU)l; zOe^r$LExnPLO}Byia<(ipJx$;08+m1pJOq5Y3ni3%!d>Dhmh=91yr4Vf@1hXD_ z6pR6N=NxDFo01S7<4Zw^-S-QhNs|>*4)L$s{*ql=vwuHw6XCH)WzGcb%FKp7F$wcy zg-X%{N6>+xvsPR^f^4K7hO#76%$1v!u|1Joc*tF)%o2vY6L}8VLJo=?gQt$_&-b&7 z(z8S~GDIEsWagH5x6W`m^>F#YGx>6Drm8Mr_c4GApl`>rL>dggGz!2x6LJn zVPy*XR9G_1%2-Txm_9=kzKG%82Nvs!0Vv32EvPqz&Wp}rIkm?wq{*1M79E!yt1$I4 znbTWBfzz86INBUw?EG7A23#zt!kpjkVUz&uK6Jy@6o`W^eJ#nk3!g63CFnb(tm7qa zi&v`{KM!GBxS&l4&E+G+-+Ay$5Q z|Jo&YshOK`3<1Lkcg{`?nIPtfLHcRF_rO#C%b_q+9=7_270ceWCjbCw4-f!00Ofx@ zB(S%$^Z5UQfk*)Wd(3nl|9c(w-*3FaU9$rgz2nvY-I=suC};yn6M{fUEFE@9=vY5J z3h_1fF(+VI@J8-oUqF1YoJ8*HPubbpvF>%wR(RZB*Dk%jy1KIA{?0Hqt2V9% z^{@YJ;BptbEJl{&I^g)nG12K@vbW!^wq$wh;WxMEmIE+3vs!ap5A0gM*|_D_-G8n@ zTz3dl<1p560`7+W9hu$f8p#Z!v)3e-#XC@G6c{yLxjZX`(DgljvDKA!8oo;501$?) zq1$$Bu1;^UQ9$5D z4g>ByhVMTF%Rap+a#hX0&DT3N5_FiJj)tbrjvfyjBpMpslZy0uSEU^XO%X;gF@V4!z@(c}yhPYw%M~Xmv}T?Z6u9KpWrwZt*ed?gYHa(l z#w`rb(FitMe+#x=2#ByQ$iUln8|D@vDcOs#aL>#v-l^Bce!;Q2;by;u52X1OmbQ-T zX$xZT`n253%izc5>)&2g79-Sl8YZ^*EXVy64hw5sa_g02un(Vexb^P<9_W2Bh`xy% zwI-jN2M^NH7XErbj>y^o{@QQqxzmq`8bt1^Ad?5lOqU+e>n|(e)7~1)?53banIM!n8&j zjh#(UvD&#BS)NCFs`~}X{H!kSJ;5<3vG;^C+Xl^kmq04dgzOduSMl3~_|V6n<>*Oy z+U3`XBDm zUgPSjHgz;MI!!B`9u1edb*Lq(L?uXYJ{)K8fordm~h6q!k zS0$AMc8uVeKzzV$u%g8qa8*2u%RYMs2!0nKiIg#0P$OQ_S}K$*`l8FVBNH7K{${V|qu%q*4ARgMSu33hhGSsmb zx`uF;sLbqG1Sb^rx} zv;@~P|0K75UtRjPDfbND#AZJ`Xtd3%%+3hn;sLEK__Y@1K@j)ExXSDp4b}4vCyw{v z9cYTLU9M zAN4xq32iwnUdL+-hr`3Vz5Ad4?c=w8d+^R*@BI4buO8X^=BeM__~7Snzy109 zr}sYo`nOMC-}~QOEOsD8We{olU%;J1%n|K;gVaKM*x z$1!EY=@KFNLYE@H5Y%@I)YEQjGp?vk{RYmlX`)~q(vsgp2$4=TG};6h_u;1I=2SZEanRtpgP_T%%wJID$?K_EYW z^XM-hKlbxi?`!b>`0oBczI))ePws;JnE>6Qw#F_(-2fU1YKky?@WEZLMUY4n^;}HF zZl)K)zIek9ddt;x`7kJkWl0eH|F}lm4#Kk+1pG#|CU^*{wOKo?o>)WZx2o8#!l2b=gV314P7aP{h<%p&def&~E11V5X8n5nR>TTe zHvDQKUT}yr{xS>&;pF<{o~0WO7M4AN8Ymzza6}vM)7kY}B(r9tN@*k^w`*0+g6CM? zsMi~WLn-%_&DQeT1=YpN78cjhO|MyJpKVnZppwN^Dv-cmk83pjO(5Ibu5u|<+YtF~ zT3s7jx-(m$b`dlZI}))ZSU{qGRU$ID!NoDP8VOI3hBcf5Qg1D-Hs^%Al>m=x1Wwp; ztE}iOo4L7d-{0;zb8}Q=Yt)^aBag1Z$XRem6#Z=t9sAm|N|AE@^4feRF)2ym7IBhHyVGfbM1uW@?bgY;K=DnD4-#-tROTsE zgVSw~dLfdN(^Ghs1lR>5)D`OBvpYA!WEE7NCO6nmL|=;Zocl^bPbg3I1{{{rz<>0^ zuit+}rC@}^!Aw~?A)_s4yXAInOj@_*XwY*G(Zo&D&V@*O3k;TS0UjRuAFC-$B`hoyn0_25pBtV36gD#;sbw|6y)FBx=8zhGfx=t*X<5Yy(Tvt z^_hdS6KNcdr>M2r+*n$@fZ9mtq2{q*>H@~W=Dm(+g^Mw=H8%&1&eB+fhz?1jDx`3f zs$Rz#lrl>y1ML!5hw=21`Q@ean297CLA65p6?S$lrx!TcMK|YHU?RM}zPKUEYwh~W zr?2fj{LC-!zP|U$Ls7#bQ$R=u8h0P9TD1AnT0gZ(aZ>vCZPw{GXbv!J(prfY7aBqV znStGDQJ4g#n@7WsqwBbYq3j4HwlKzbG>m7ss?j4$5-SlLorax9!wTjzYAf5*!1g?@ zaRFyT9f7q{E23UB7S;_jfbb#(kCfR6B<)|TXAaD>`Cq+DsrT~QrT=C1HgpG&M8Ab@ zFC+tmDl9`${d0_nl2}Or48pyJBHS_w?;2q=s{R0ql5-L{CatCOTdNl@FIQ#N2pAEs z1CrztpxDem8!n4L#dh#_W@x7|(?r3n4WwYxMI$;Rhj{~vVL%2zAelurf||lQ#}qY* zR<$wK(#eZ_z$Q;DYdN$s`W+iWVdx`56_a?R;n3)(CWFOSa6+|4lWb(S4_DDez$wJH zPT-1tt+O@?MU=9d;DA}9-d3luy-$TfY)N9>4$X`)vQ~2MWEeyGMALg!i9?q7q!lxUs|pbpjn9XDS#c5rvvr)sgeB)U64)_wL>kJ|6L^}|Ivt^ma+#D|PX`P2_H5yh`0X~Mr=69=@2XfNMBEBM&i#HB@ z4`S-Lwq|nTh#bj`L{~lWVw@&cjeKm1i)%IdM=KUePJ@$AYWnAc^s2nLjpjRBIOnfJ ztrc7*KKZPi5-7WqPhtfSbux>ecp3pb@r1k*wETS<1EdrLsJBgb0ROh2DKyK6-3oSY zvJ=lSHZD@z2NwL;8+@4i=AI zZEUPtrBiIhX7Kq6Uurl=_mq;!_Y-`LYg=XrASmU zOCD#~2iZ)3T~8J?vC}CKnqV7RTg%clcEnM*brNm$oO5b*-ed(LzM+mH_pj$4R<%)^ z$q#agt$2P{YZuDY+Duq>>i3a-i;w_9g`T6a@n$gC0_f6G#L7k8L$&*gb)0Il$abq* zXq9gzftW*%n4!Y9hx3t#O<2&ihf5wfX1K}18CEmgy1Lj(WK3C`w3mzL5$8d_r+9pk zP`s#r0d5`>v(;_<&XW$(B?E;h<0?8qwPXzyOjg-EplJ!(JPJQlc%JrKoRh(jirb!e z8dDYP9TvCY+?+NnsX2&20n^Bp(MTHWKyoxDMXzbo%Th!N5|db|?Fg%&P9exOT^D=7 zXaIX>P{ol5VWgsHZc@gKg<}UYoU3$h3Ef4e|HShOXZP|iKZ&x`X zKkbxweI@{X^D4_gIIy+I`9_CA*sMAJA7m;A2{fS`Gq=V}#Xg%2Rd9kv%Qi88!&Xgi zm3`Qn{I)D@jrL`aKqvNRmBuqu%x(b@e~tB88>A^bluK23tq`a^Zx$L~j^hT z7@-j*{6U&_W7vY#+8;2#{y=dnGnASOYH)|s3FxMYQlc%H7i>|%A2ccb7fp)57%vC= z=##BW5(Kf%Fz8DX2qz1gl16Hw=MK_ai14NatRza|+u%;{=&OBWedYiR5^>s>JJL9U zbdF${LP;&kR9cCCV~)3Gf~pe>f*Kw7Su+ARGM0@SGRxpLK-Cd+7H-jdD%eJqx^=T? zkvR1x#9N^tQ+I0&qTcMAco=2UcF&Ejh4wH^C_!!&*xs5C)4tIS#$j~^^_dAA7;0Uj zIY?_-&WX2fu1}@`t25**r2@WFkwSVolo6mh!puTv-{_>GgY0+awhs zW|YIIK-CFsc2^<2*7}Ok8k(FG4@_s7t}}oWK}EYx&S*7Gr4db1r$qshd?DM3J_jR| zo)tzO2)g*4ABTZurY$^K(&ah3B__V*dkLN1HtUYajB^XRz?nrCMRgAV6 z3M1hYd<~L0CX4^ISfQNpIjcr;5H2$SYml->;;dC@Y~GeyP_)iTf|E<2LSRvijTvp{sqOuq}5-IU2Fx~(W72=A?R zH?svzC~^c{U^-n~wx#D185S}b<@8{hOSX7vnky%} ztSX~-R!XhJ3Y6=nO*k>Vwm~3FjZmaXyJFzzk3u{{-3kSLs-PBlI8j^3(~B76K5Q^9 zJnYBxEO!!Q54VLJ>N8VNJ5;cJ46w*0=^U0EH@vDc<8)A!OpdC_{G?81Dy0%%q?!s= zs*0)_Nkt9)O03Kzx(Y)|4ivYlr9(Mm*yxP1re#8!b!!p@oUzJG$j~{iTXewgTE`hc z7b)gyPOOWQ=}{4_oT{#&FvwtV?eVG>tDtL|6l}#TIH3n)0?9l%rS|Dd^tC*GK@SSE zie-~rVroU^Ps$v+XL^L598hZx{zXwx`krviuw2qN%RDAs*0^um{l1-=v*m<#G3R1J zClJ|{K{uGax;@`3jV}gNaLQg_?ubR@3Y{G)Jc?1XW(C+VoyR0K3X4|J!eCM`F^ zVx?A0!Aj{dtfIx~-~ODYxhyy6tG?|uzG}*tbPjT zo}kI;0RMXRl(LvCq@;$-3V&O{JX0~SxEA)|BOMv-(gv;_+}4@IDzgBy3O9|ma7q)$ z7ENmmaJObg%B~AW{S=Ipp2g_qsK0sL$R6UH%0N?Rcm+6S`!OVNw0}(Qdb8W@QGbN{ zUMbi&yXC-faNPGuzaWf;-%p%9pPAuur27V6qvQ)JC=_u=qRe6>?8p+zG=_kGM>s9BE;~g2p)~6 z5U8RNOi+ctQ>c;){@nA6u9HX0{xPV>o`dbU<90*A;;p7K#Yv|zRU}r@d63Tq$s(p>w9gR!WHFL*yE0<>xt!tnMqu`#H6r|q!n5NKwU9n6I7ELlYop3 z|8uEgGs}W&^E$|^ReG~jxgZ+~TFQpOj1Btd$NMLpbw%~lqWqDniNk5<`rsHe{mXW5 z*h$Wz#G>%PcfAZv1CJsnm#w8JFjHV=BP+rL*YW5g!SP!!i>NYth`vAUv|yx;);4we zIFb&~JMQ%k&Q3f=*C^ zep7rMx7&^67d4ZTp4~}vvo$r*1c|5>)O_*5j^Qk5)in-jb?`w$3y=+m7Aq$_7((i5A1m5=|Dts^dkD8Es}D20>*a zmjwW6yb2y?8*OFP6xk_Wm)0%>EK-dG4+5n1g}hM(APk(2zI83yT+x;=@otf^cYsYn&%qbRc0!t@-U#|w zXYdDWjM};-cuUj;`5E-z8czU_Ct{-bNVb*Q7MO$`iR}cidzrh}%u{=Oe_X5Q`vW?* z0f%{Rj&junvTZm}i}wsSH{jyU_4#IVabxw$tI_ehYIM?x=blwjfab^ zbIc15^;7b0ZOr*-Di-ZC=OTsgn>ruP=Yif$1xMi&B1N1nEJmDf~-8%gDS(C1~aNSk!e7pA6mM7sLa z@3jKaan+`}Bhq_wyh1&0(#Gt!Ta*_<*%lg1OTe?JmjXAAc{gLvzG)X)qQ{Ypcv}*; zjRZ}{HE{izgo9ySc@1{dbS6-BXQ$dgD68PLsYj^dsh3nhLSRc$N^lQQ>E={yV3TWc zIa?0PF&66b@^QuE5e!3+ycEpC6|TEg?JiwbjN4sLS4m>dZ|Z=o+~Pv z9oUPmNXWcSA~`m2wLFRGxO5~+Vw9LIuyuQg&USH$4XA?AzLHw0sV1%BhK@*ljjAF^ z)8hONsye1Fsp3>)uO!m>V2lBUnNSTcaZ#&-L($xEL_CS#j&K9EWA>2VNx=A@$mDBo z&_E?k)&^V&ZHJjA3klTBPjRcpCLV$3DvAOBxCXOI6%1RV&Q9!ZIW38?P^&TxiJqH( zV@E2UbF8F2uAmHgr!sUHaoE11`$vpl^;fpN)EY1c@3^?sDQ`57CJnSmc9u%O_>T6I z_L=kb<;Ncb$*XJ4#W_+Yv7SoHtL+gesTaZ}(RhtCKYRh+cC-i!?)3mRUbiI`7W>ok zJ#zF>8wX29L*~D-`9e zj;@El7$1)MD9vahHaj~TeRmjjV)aXhA3F=BX zIO?=v`IEk@OTIgBYZTp%FXe=60@h{GhWx09b?zt>tpYOV*G|Af&{BGMgDW4H!7z}6 z0^^hBVrgATQAw8HH%aPVa|n~CL~>#X!D+edO7T487_$m)^}yoc|#F{ zr4Ss~;N8$|aIwJo4|lLplU0Do-FqiNJw~tO=>mOwvF7l1kpFfv2WF=CRnMBYegbtzwJmsinSyaO>0Z#=(UFqL=wl3;nW`wLM9Oj**zFy zOonXWx9q?A0kSeJ9APcj4H3%xJ5~QaRKp)btk^ z^@=?xF_TV#8BNzyXM1TA_c&Vo>6%66Ddy2JW4cV1dG|n=(yhKMtDdJoDc3r zumB`pwo2Z`X7oY*imXO84N8q}Zc8$%X6?!Q+%(}ZGdV+~0XPZ>DIkVW*!T-)LJa{% z88rr&b)!K*k%i+xdR_gpx!J)89x&Uo+t*eT#E3{xS|})6o#(s2dDBNF`*$fbZ@xk9 zCIG|Ft|fZ~QWbTHOg%j8{y$wol3gMRCgBaCd=mXYCaO-rFrr>SC5m-{ zIP&9CWnHvi;If7275w%OY3n)>Qedi)(jkPxgua}r=$4Ca4vx zR5fPfT16D_wU_ey*&1S03`#b&L(oeVQ+UAR#FQ*`A+eb#jiG7;6_@3xCP%NBeu|yy z%2$I08U1l3l_wuqg6AAt;!y&Ow%&3};>9cT$tI;liUI2!+A^%&8lLtbbF&?Nc9#xVy1PS`^ zxxG8zWwL(#_DOg?5NP5z0eSE6Z;yVt_t-PPJo?Vw{a@^T_&AV`gueFmFOPggn4tJ? zKRmg2&s$tDTD^o)d-p!ViF@~a@$*+7Ldi%>!r&x4_}p96lg9gmocqtyK|}c@>O6}V zFIM>e^;AC+CI0cZzdZ26-Vg8p`sl|j6?Q(kWkGD1d;cfDy!`sl-+sY~drv$p%76bP zx4{&=S?s~Z-A@&rQ%Vu1DH_NO)o1Y1ZW&AS4BWRRgo_lTYle_os`Rv!vzLkm=H@Ot#K#$EjCnJ0@rZs;2 z{@1790%S}Ir)IH=-lUuKF|$WfZ4Sd8w7jh_4r2T`s)3^MxDPR}77of)H%Gcmv2ksq z-t4BzqlpwL_udmzKVznis1RjB?UG@Jq|&Lzj<7Q>z^lxby3A#hQ6vPOWLJq*)>G4)Bl7fy(xmc4h}czA(=YMjz!=SqzX7Mf9A}0WFrtvJ z&B=G*ln9eP<#=TjQmIEInYveC%9Y$%3Hud$Hmrs5VFL z{VrOWfg4#{DtBh=LrSsdCZqcLB^HSnypQU7t!>Anf;E9-&Wg~ss?k?uAn2J?eFk6txgM6y=VG_c=+x)_9e=#`-V;{96^iJAdJ`%$Py{qUZQ}$s z7=R-LHDfsB9-K+U3Rv^NLy-u4zH|XlHudtEdzx6vXA<}U!kVY2%7~BgM#bi^#Wp&tr z>ZvIdqKGnSj52GQKqZ~@V-iZCT8?qaq!#-?!7&MP3k^&MIXnr8Rq;4V%^y%=(1K-S zx=Mq`du^c5xQLOQK(1q{6_dRPJ}G<0ptcW-n$$Ygcw z0i?L8wbY|CGCe5|y+S=6-> z#$m!ul{V-%YS(x#jNuh^zt9^p5JWJO)=e{E85AmaE3k0Rjd3x6muKWmGbZ47BG51vSPw5r3Vmren1_735bOAa6TfqMtP|i08TNf4{vqS1U#wqoH8_!-nj>L7 z(1q=|Yw*bMd~^X{h`+ipl+~=O3@zk$F4v%tIE;*m zts{@*NlK*3RCiYQkf=m$)^tgZjIh;6-*f4k@4)sa`cULR{5!uA9IdTe@KLJLI=*`a zAMr~0wKkmPi|f{g(;JmSw&`y7eHXYAU&yc&XnvQCtfr zHJlz@t_1MqPulud7+)SoCz$BOBgcdb$F3G66vM-BMAu`_yc@vL-V$i>ZE9^&E2KIn zmr$Kx`nn@FH~p_c`_`af+A1WPrnn+S@4mA}ML+*nt$&1SMeX%h=$MX6cuWH(1MtLB zN?=y=&H^-y(Q|yGb21mk4#|PJ!0GpM&*{XwNzR{=Rw$dTWrJY&=X_xi6+AbTrv2x_ zHYHU4@`P?hgr0k-gpTv3&c%r%-mO}<_^Y~Rb?o)L6SSPXB(ZMEuf?Q+7s%~L9{7-I zv4&G|pA(8*X*aG+F_g*h!U<#BZo?7x9*ZxfdMF~TTliV3+B(k|)=N(r=7-yK?+W~1 zv2UEdX3_m|bqoI&x6}L3C>jWhsT30kM9B-(r%vFNS6Agu)gq7!LbbB)`{NLhdRIoq zp7D$n?Ay7XchbSkx+tD~yn|CaYf>a`^@O68Zyn#rWn-O4`^hJb?cdc*V0L~op1a125l*Dwx{(7nIxmPTzsh+l zc*hpp0$R7=qf|xM_*u8$qf{m6*=~OXp7fL*oT4r%jVRM{NEzXYhleo510E}I*AoJF zr#e^pU*H)6*TP-$Yq?nm980fT@KLJLrr!+*y%F4YBEOZY-n2c~9FCu*s?Bc!8u*A+ z!ZYk@1(7I%VI41>ro~M_godjCN8}99mr~Z^5T-i?0!Sw`5|UW3&mNGIXneix8%1{%D8zQKFJK9<;K57cx(k+AN*Ht%5XmH&V?}>|CJlM=NEkJ zHf=vBx2nfp!YV`=#kcdy=s`LAS+?czZw%DaPN|J+L8s@S_i`yPSI%Za*P#AUs?r5R zI0J)j7kw$|e*rQVE)ALYF!Jk_i~MdlxJ$qhej9KaODNH-P_js1C2+38icZ~X zgD?YAi2!Y3fK`ynKmLximg`IV?iY9oR^5Wnl7l)S+-%boh8GXuQW{V-`uWea9olx6 zh}H8iUR|Kmk*o77yw#eXfLq$r8 z4KqcGa+s-518702;-rYU3Kz{Us}`Ax0EVT*f{G)<1cihwLIFjJ;;Lss6^9T(kzyPf zD8|0)6)g($s6yhGv!)_2ahg=59KtN9Dd0_%C_TSbh)&@}7wQV=HWw*}ZgL^TBe%Fn zLG*?THIZ<7i|HJCl7$!o$rmc-1chYIu4Ey`BM-6=N2ZHds5>Xee^KpLc^oDh|9C(g z?wY4!ZcF6TR?vl>W5f9{-pMSET3Y6~>N(obY7U@yb`!n0kOMn< zTUUw_v(eEQe852@<{+K#^^EZxFr*Lk=~Tw}Ns8tugIx9ia*7Y-mvfStRbxAOR1^{o z8rUQ0L--)T@WI?1T==sCM~bzl-i#SfgJ_JfK}l`vZicj(Ym%gFSh&>1xv}jGa0k06 zHQVN_#JRsz4Z$*{$QD@ilUnG%f${dSX$-A2a;oy^W-`&-xQ^Uo8FVe>%Nb+t$2m*m zR7wcn9m4g49y`WoMWSQe5v;Lqe~1&qgR|$ouDDmFuG8D#N7hL!1IU}1~@`Sj!0lqjmGcp)vYPUA7;l3{5E#!_n#dnw%T zKsVInT(319A(j^;T07n_H$`zChES1kdPaTPbRRs~G8{5($`6J3P1-ksPuNXe@(0F3 z;sIk;9^BQ!j9uvT;Lu?zJCiZ7ABIYRPO{F#j_o(d`YQPOP7tm-?)J`>KRl4eoLC42 zS~ZoabKM@qjqQWt=HS_YZ}5!Ed2WCFuv?ymgC`$$T}_kjgD04xu^eTcCgF$2zy+`p zFjdMAo@9CjzUZr&#fUx>vh85HL?1lK>l@S7jzb_CX+G_rM)Y|QMccW-4IElVo2n5! zTt=F1Z@P}%!IQiROBKWE5`7qC^D@UJCrz2~!yug&8X=6d1TWoAo%};$!5>^SRniZh z^-dJ0j@4-s( zbm=}6(hnfyaPUOK%44wXUOQ#F4~29-n~HrDHn`BAa!)-lvXNkNA){SW zu^3hI0nqC+-Ui)*ojWbNQ}TV#vWyjlJ7KJfWpRKZ)5*9l+MJ3Jv0ntY)f`Bq;Sg8= zKihF`ntB@?JmJfm&6P6I-O?n@6xI1Lgo?dC2QVin%~E;dd98T?o~ovyIw&LY@jdhq z7}yNq638-Cm!@gbMLUC3TFOPcI9(+v%gY2r!v#1~o-Q5&(Q?1;lqo+f@)P@+4}gtB zWM|plb%y`k)+pl6OhWnrF<@}On}&)y()uPK-j|5)fY)aZh3@!{K=n2iZbekvln%(8 zkMVgm!HPYV6i;jVJc1@(Tf82>na6PixR8DbwpsA{<}5CY`@SZHsVIdr0Nnm)NoXk~ zwye&w;d<#8y;F5@%lib!gY-*z^Ho5}BB2Q$UM}oh?785}u9{cga!EkQg{#LNi#TZZ z#M{wYxGUC#80|YoF`#4^@-($fzcicQwKyk*54q8}YSz6Hf5EP5s;F;8juLLo6;bpy zsm4*1HZ7hoLomkgTQ$YTU1lox?f0lMJ{d~aN-fx)=Z6+(3OtCeUIMW_`KFf@J&jb+ z-~8>2VBW}@l+-_B;`?yB*;;)%ZZ~l(82XHslQ4zscSM}#*@i`$)74aurwG1UX z_~>R-2^3I!f=QB$yeH{VuP^B$PVWQ9Mn$40W0kI$d1@CqY`GAY6?+HuqcvJO25GEl zpEw(#K7MVX-4U;brb-0ziwt^02W8Ij8*SUXR>s&OKQ~u?>pKXTRY`xhH+p-QvXy`; zl|R`#uH6QT;5%P&oCG*vLP94r(Lt0TLg>g`>5MAwZ%$P=!P< zKSO{wet{{sx4XBucYF4kCXj&mc4l^Vc6MfVwxtHc==pPt4hR50`auLbbzlXyobDia zb31;;oC&1NU9mctPJ*3&$ZZ2mC&>$m1GKT>Uk!DR7l3UekHs<*Tx^Bo2yc=MFG{3o zJLE&ExlYuZN?jnR&_!+IPI~iv>VU7hp#w*PwouDYU0Xk=v)(!NR3NS*AzQdrGJT55 z&+e)4Er7BPe|>e$SLD5!-p;=T!?%~pJsuINE_X~kazQk}rm^Vxi@bL(s|SxYi*{H= z^LL%SaER%=iqja_k7Sds*}S54)peI|u6XC2=lK_-J4qqMs|zeadJ~Taq>rGoUAlkd zCk8>3DKH&=_V|Y)Xzux9B7#cFhsPYFSzwY;0s{g+c^|x*^#uLm2DR3N|CI;fv}Xj16z8&GZy~iQ z9$K^}_9wEj)Lc-EX{e#Hoggvu z7+=NDYL_X~1QM@^KsYWo1Cmt?k|jHeur$nLi%@eRDo3YkjnF!dfSD@OWiSl&8P#kq zH&{MCo2#L0f+Jg^SrKkm(d;UNms2b$Z_BAClDzPix}8R!NyTE~Lq7~Fj{ji*^8}V` z0V8Wx7WgWeUhAfy{G=HQpI^G^2gAEh-o=ZuL9=#uWuRYhB#EvBvvY6(Q|AYExz`@J z7j9iw$*BrJDJeX#jU3Hns*~NJ4XI3GupIMAqeH+Yarxsw+l&D>K zM@j+ac4afSE$!T{Zs@kHrQ6j_-L|!LySlO4w${R>1Hr`Q!g1i*g6(p5par*z?MpYe zFTUn7LWT)j!8jQnBVL$h%bF=Z8&r9mlZ_q&NhoPM?Vcx{K@pA@j>mX)*>sppM>N2B zAajV2)^=|0$zs(4+v0@<4}dbXatyROMu2|`kBVkbu?fNU`0=;gVOQctM?6n@shG3YgK%`id`c%0yPS|po*l0`OF`K zc-f=jUO5AgzQsQ3WK!a1JKy{j79x zJ&{UjauubcdRrO6gt);x=iA2gGAnZ;4h9i(@PNxtWe|eV7*;Q3k*7c$iiG`ez#nL+ z5h_;(%*fhY$cfL19CH-os~OT>gke5QRnv4Sa{8&pQh<|ki+^@FeFuNBbSyUQFFIb{ z@agT$?j4^i_RcTFAQ_v}Tw>Um&$6yx!YS4j@4*MXtCHKoaePC#3lo>&1Pa)++Gilr zaCHGtnpqkXmk5cq6swZLov%&paQ%nO z;f86SY^8o<2NK~Yuk$6Nad^Kd5kra|DyQn`-g(TgSK5U6?jGp6uHiRh4ygJ$jDqJ$ zlbf!W0qpI0{5wk*Y)t*&(KMg|jh8xJd)Wdnw<;}msX`Xz)I=@zI(Yn=a z)z}8W>&M>hY$x=kVT90E;)H{_D!FJh>d#sTv6@-F6}Wp197JaP*kGwXskIJu5mIIp z*~W?)CsBm7T{*vMoRH(lB5q_&U-{MfN}u8b%24-O)dT`mauNaYG%Dn5K1B_8_!JfA z+HA!Bclcf~4$0|dVRe>mEeQ+Qkf8*Dx^y3e$yU>lH_KPfA@=c7m`*>28aRy)3wg;k z@bluhH@ks)S$11niGf(^?8UEQ15JtAQ9tO09=?C>#dxkz2qn*xoJ2BO7|;zg>KsKx z8ft)Iw5M#9T_L-o3Pmk&639in;C@jm^(DfqRbGG;PDZlO#XAHKMcyEQ|In{8`Kam- z@+p;v3|x(&r%=)Cjl8mzTOc`6QC9ME(0b8v?RJnHw}&Ix3+=(FW*F(sv`qF+R;sM% zQJ)1EddX!iev93F^+poJRlz;@50DzifnQqA;kQYp*7>a+T5;Q=>|1^O@WP1QAVDhWf zmj(~UGzjg93Z_qe5Mj;Quz7MOs%_L1P7la7ham=JrMoIhm`Xrk-91wPN>hR))aN7{ zV&LaD&YNtyu;qC>ki@w6T%|sJ9wY0^9aULEeR9rISgV*OEu-p!$&uut?7+wpc%fO9 zT?{l4Tap;^Q*g@BDD|!;8Os^T}^-Jm#fs_I`c&`Coo~>lkGxeFtusHBX1swtrCh?=O6P z^x~_(|Mc?F2WzrOZa7LY+}o_h87haNwA{HtHSe|6;;a~F?Tnq=90@W`)kzok;ui2d=yb4O3T ze)RbhK<@V+KmPUk_kQ{2JCEY+0DKQOD9sGWW~8A6Q_;f-W`LVoazsKUR=DG)@Q8_h z?+lhgNRmG;p+@01gemJ-Bw$hQpZ@gQkDqF7=1%J$KBCFM zDsMb3k>Ctx5*1qjeZh(kfcDp?1NbcjV$>N3V9`+gV6?!ebup}Fk=LhhyzIKI%^R3V z@v)|Y{rai`tYrJ^+r)&iX4qQEuaX+Nee~mdzdiHSFW)?0px(be^CsAbqaPkQ`sN!$ zNlhytIdibZ#rdi0!m!7why;Xz(PWq90xPiN2ph$V&l558<7H6_>r*{lO}E@rOMm_H zqoXIDJNohEqoLOa^~qvV}-;?&eULrokyM> ziC|?2z_SByMe1-tE2FG4SiUC1r!S?WL;9?ak`HqaR=k_;N3>bGJOSQ0WDp|Pd54ef z!?hh@QGh=2pl;E28(+0uxTurQmT;}w$w^pRNA>6nZvN2Fh0=gG5(6SYcK+ZppwfVJWxENaggTJ=IkW-B(%eARfy4_^l zJq4qXaM(NL-f%n&`yD)P8TG^QDc#ZZl!V{+Nbm-)4t(e^&}z3^msmU)#KX>X67>_n zwk;smcv)mY|FlFG^-u9qi}@bag(%KIx)Z&Rq(>3Bw9CovA9)RufC`MeLN zf5DjJ%uK;})7svV7e-i&J%NEROswj7JYR*hU^#lUJplVQx!0Ur3wmOHO(+D5p3iE* zmC9EQ6V6@RGHXBHYptWWqWq37l^)n}sz;jPOryZC82Yp~4ns|&0BC-F0_1<7HUU@F zx&%NV1c~9QM*wQoB6Z*4fy3dIB2$%}omvqmG&DBrsV5**D<=SztEP4kYMRxN<7UOK zuUv)HR+bI^P}R09v%=UVjbM_6O_2C9ji5G3N-bK+o%~^cZe-&wkEdg85T$VQ52Td9 zKX5RmoK+8~6cB`>W4aEk6x=enMl$cRdGLlAH z<-aZ4#<`mGZ@o$RK+T0|BL|T1y~{B9u3Gezkh#UgEiUj4ICVlUTWWyWCif*1a*|0f zp;@LRWSfr*V0Ai{jN1(k<6e z2UdUb1t4x5S1lh2_+pbd{_qCD49udJ?<R*F?!AwQ@{fPeql76Ia9&0`x{PupW*< zt*7I0IGprnxFM4rJhQ};&yqmGi_@#ha2xdu;+^z@q$p7je2`htFH#a~?vrS5L*76H(FdoMdINoT&H!hB?%O@(3 zGDijU!jLgXmaK&S&6K&IwNeVqZjC!3fA5lcZOw|6n7sBNX{}kslXQcAm@Fb8rcfv( zE;8iApG!rh5M8N8sV^|5=Pt9_i*O~&Wj=>-explf79i5OaymlRE_1`AV&A{^gujbp`u)J z{r1(gst-O1-7<%jPEgaMtD*O2!&Cf z+P&M4V>EhJZ!U#PlwB9Cb7-b}FYh$Jh{!x4z}Ag7Ty@&IUnd`WiZ$C1~=N^xWMe zMH>~G%?w&M=E~A|%rmv4Tl7U-54Xw(>uCS?WNu9%B znv=dh=n3zPf0uh^0BF`F*Y(9Uee3}VuNfCBXpTUzz?l(ZBk$_|Vi9W#ct5{xRcydB z`ec{&6D*a;7qkSnZIO)VC!K_szDj&;o{xg1V~v#RZgZ&MZEs&IUiZ0&>$|74zI*D{ zch8--uZ#L*uO*JlI9e@8MN-H+QT*nxf`ecVrU$`XOb?MmuyV_jfoUcs<3)p+XT64S zRp%$E75P3ct#p3`)qn>q^naqi+FVWbvdtjr0A#0O-+>>Hu0tnEP_vW38v7cM+1q*k zZQyJO4;)~m{+PamE_UHtG=)!5jtrdSaZ$O8QD`*lYUXu0?v(5Kre)0knC7gKCd>k> zY3lnucCcR{X)&1gJB#oAP^m4n4}`ELLdQCRbeEj+GWK>Tuzn$lof;%|n1rf63Z=A% zh2il1D$tn(%_dk1zpOl3S2r_z$8`qW7^nbX9 zHL~_R1GPFrc0fu^XK~wHh^R)o?3AMWtg@14$7yE6NB=C5Bg?Da_94qLHYM2-$un$D z;AKo87my)iP%$Wge&du=L#rj56!TFqqGxqSG*mI)h@QEO=-J{UdN%)vo|TU1nJbcI zM049*h%9|X&;AV~diGBk(KDxUH_wQ6qbx@BKRda^BYMu0Y>ujD3~o$zt_ZK$&Sp7lt7pK4!neS`EqF>8#5DT;~@iO@h}`t@*4%QLfW;*X^Hhquaf00jbPh*;I!4X!?tXh z-tv^#<_NL@@xBk>saV@OV~0>8I6Krxs6yZy)IFTOU#Ey-7iMWrvLHZWKotpp7BUe7 z5j;%?zSk0sq_0Uxj43BEp$$`n(N>t&#f9J9YsOeSEgEFG6PbR3#zwa@57 z$F0w91W-R$&jW<-)>nCTNgUKg0vdfqp9}j63+nY%K?TsZbw>-t$}C0ECe{kO$U(Cw zd(qfb?1`&lPl^?LVpHr%UB#Z*6uXukN~AtdEc!fg(C6KFI-cyAmOBRA1+&+ZU(HfY zRPC4b6fo+rCy-x#XgI=IN{mxC%2`DA)V&zTaIBQDyV>BI`S&BfMlTq0DQ+|lSXRMs z0{s(@;(mO0)D8MxJnn^KxYfVk=exAE5KJsW6h%npO9+xzGP`>Cp*g+;+j^}W8XZ;U@R+G z>Ro(Q2lR1^lD|@WXa`Bt1A7U>y568}zAF{7lbP-6^oEX;Eg|0^M>O?!Q`a0@@6MxI z|Bk!!CUFDFHud5=M02<}55eRfEgj!D+zj;qqNH5*h?&5*v??+Vo(Y7LFGych2eG@p z>nwJ4iM?DhnSxs5i6@=_gt991Rl5rg4lO%90CD{d%_mHjVRq)SJI$s{m}b(CN7ehJ zT$hcT-dIfB)g@Q3E@Bx2DW`S-rT#1|<1PHSgmQqP*L9a%d2^eEDT?IRU9TSw@1E=d zv?_0eSZ|9xj@!fZ{t)l>_4cWEvbnQuF~++F#Pvq_rV`OB0cF{;Dyc4{{^E|St0+lM zXDwx0>f|DpX4h-j?Ja|ZagR&Mf`d@-G7#zLE7Q`{<0&oWix9y-fB!4k=>Bxh6^c5$ zUo+u2zJ+kj_L=`Z&#_47>z<;?EwyUap73hr?s-zvk47}5k-TYh8psXrw!Jg)zxjIV z&Sc=3(8(T5yZPO$hBrblTV3W=oaKca`;|1E!j0ymjs3c$8jLjL*`0Ftf2`vEAAHc; zuLdi>k|?L!X&^(?;J@*--^1&VPVb2ToVm+ad%M^Ncqi6OBIfYK0OT?WSDV3oJldXe zc>5aBS*Ld@`O7+gf#Jw6HE7r1aRApHhY;}y*0RhQr{-(KNOOBedASdb-34%4%eF9R znHgecW@eNzW@ct)W@bC)n3&j-WRwc7p9Qdqd~iHn~jO- z+LbTHiL_;i* zRvM*48=+@lC9JO1r_Mcf@Y&+KgdzsOjd=smACYUg?S=I%_B_Yb7*FSgehyfPQ=e8D z%fRHX6qq$T658Cvx@ZaRB$_j;dTo}s^PQofqCO$IVVZYd04Y@iDb&ix4lf;n_`TB< z@v;uP<8CdjtORiMd6ru5ZN9IE5HJvL4`lkhc?p6vDxIJTI=r(5 zvH_VSrI#eMQ@;=5>@dL>g2}1tGO3#C)MQ=GX!#7jPy6YCB~lwx*IN@^DCERO-~Y{& zvrGl1Tm(qAxUjauUSCG}vYml3CuAvySX@xNEH_F%)JgLCgI0-5q z;ULLzoofV9XvlaPCP}`-BdsBNH5z3@9YW7i^tsbflxcopFJjV~k9I}B=*hBnf%B1F zhVT2FQA&=oJcZxBl@|R~y|k@hBekAHwVCN&5{kC70~-E{&%zoJjt%j^JHdLhtn=^oDDl`;jy zY{bQINrHHVSP{++yaOR*0$W}M?%CP+5i?zm72_is2^o={S{V?U?fCxRsChU}@w2?u z*E`K&hw=%(;nE;XISXpkqhA972ObE%YNlD#Y;rhnDI7PT(fX+(L7kbXb>Ax=jo#mG zhp>M&9Mlc!TT)(d0aFTwU5=kaj`4QAY`XjPh*P?UHR!r9>qZYs%Ogu^U83yrcq%@U zQoLMdXv$OlJp~^=&0YCt7QO;qfJO{vTPSemBpj~ccUFgtX z$Zd{3CFyD8Y%^Uhj!m#el5(yRxu+M9+Hy)Nf7a}xTkE!F?-%Mal%Xs^3Z+e#vK{Ms!# zuU!H4c6~*V{g(t}jBakbhqC@!qKxPAK}CoUJo9th?nkiqLRj$GS7YP`cLh39k2-=y zQg(yM@t92!$g`hi(7f1b%3UwD=d3F6RP7k$JbS~q$H5G$cYEYx=LA^JqOB9SAy#KWHhk zyYdW(%^1+6#KCS$JAUUe=sLhh#KAu?`OwEXJzr_89?3E)w{*6EBz>hf(^DKIWdvy+ z-ty%n?1{-Toe8Ti9x|fs<5f-ajua6XK5mb1qLEZ&m;{W>h>P_?+{_J*ZHQZDwWT#X z%yW@j{Y9~VPphl0@Jo3xFs5ac&Yz!KdA?;q<0vqp16kqfrbr_XCnU-2g&R)`vP&rk zvv8*+gih9M&5EztBz-Tjvph)^k~t%S7WI9zj1{8;1Srgr8prY&EWoh)@b zBeq&X6MZPxg$r(TwxZm|y>p0R&!a+VZ3D*Tl*={gLVWNsQ#7SJWuR_kja4bcvO}TR zw051_XQXnjREXJzh)zX^_Nf?>kW#rvONANl6kNUP;!~mEBM%yku2r@|>1mBs!{65y z8Wh?U5eucoi-KxY6UuF+cd|`oE3)02WI@xdz9o|2{L}&Ij4jOIlRkPa+wM~6Y6STe z6iUY>DcA~av(DM<~WKCfYF zo?Lx`L)(%yHPyK0Fn#f?=3Af1V6%|ZmD3!@xb(`8hjJWvw|*_pBp%HSsvI{8Z;u)3 z%g2h6PMo@3#)LXTH)e#+Gx1V-MGja9tr3f}zXRK{R5sWZN*Y&mzCD6P1778B zJY-p_6~UzJU=$!>=?;L8`e^@jNrC{Vq=)aR9}#9E?ZHR^f8SlP_~?obx(e?VhqQ#eKaLxr*RS z@z^Fu^l)m>LbohC51$(J$Y!J|#xwHKA{8|P%dt`r7wU9hXVgqP^CTfLc_s?@EXdU? zsL;|I^z>7V3S1Dn=Lc~#0kB3DdS(9 zMK<=F?0I^bWgBy4WXqLNY}VF4?HD^gt-1oU3*yZ}k)f_&TdZ>7lQ+O>u9J8&TZbJN z0=Iu&=6^BArD+h>Dyo>PYmQeL!TMj zMN8^_`!B?!Zd}O36Aw??_&lj*i(jFg1A)zRS)Y|IbG1VjVqWKanFbf7z(1jhd}Lf# zrjBw&!$MOF>8noicXQ9;om%Mm1sqHuj1NYxK<3Ff_9=)ggcVwm{@#Y(PG*AIhDtQ+ zeN{kO5prD68Xm~Ijal21^bp(K;@;g3M}+L-qd6TtyvIQ>9W8o3=CfzmrEFGitmc8K z9g&@r3@oXna|>@1_VY~3r3zT=V_Ub3R*ggn)K0-4U#Ao0?TJ=BuHTse*hD$sIo37q z({4(LCvnCoauq$gG5J!=MzF}5)Nz?%W#h&bpxOahVZ^gx?@}NWR&m5% zQ5#lOrAJ*_8pG$+!jD|NCPK58(&B2?q}+J9q#(!UMAyqdQTKwGE_|p6rs;12_tQVM zM^A6kZ4ArHNrQ)tr2%*H=1H^VNQKH*jS3dj0`0apL{1Jh6WK3)949R8ZEc4CgiU{j95NIr*0e*zAK)s=kKL~ODdDp;J56ho zal>%OhrA_lj5#b(uR~N`4%e6k{HU+s@V@D~S({GVjYg8Fd%u`D(M-P1_2mgjC?9--$DjyXq60)oh`|CC ziHG_HF{xSzT-RdRwrs{}&J#3~UGXkAm$ird%Aym@RAhr4b>+0Y{cBpOEBAFtJHbSl zTg$p(jf$&}odx28wrC*NTB0n2Iy}oQ(!-0@qc;SL8hoxn;X2^``*|&^f;{qI@G=4W z-QSH#zF!bxJ|PLg5}jkj8zLRD#9OLvz$|fMhNDH7L1b3d@lC1?Rug4GWL_1gwnROH zUgt~ZSnSiCB2wl{L=BpKR`p||)L+KTV`b`JOW zD*MmSNzkyNTf0ZdCt)MhU0J!7Ju~yeI_K(w?)upft}o`;GM%gZ8LxBT8#+NBl`g6d zXZKd|s!`SCg$B|OeXMIahExmWIBe?cYYOU6j%J!H8yU(GKJ9(`4oy>E3l+n>VEE%D zQxFUPi&xu7%?O@Cc9uxM6|fGxZmOeg=2cuG_fZAO<5f`;%yp=f;Pl% zb6au$Do^~ko5ky<5%7HlDMg>ZZtvHR#l)2VPkWBpMq4cPH_Gh7^+gz}qerhP_VL@#~lkg&}`P@t;d~ zx(wD@X+^T<<(wTgfVnueB#&LqrgVv+l2yvgv7 z7L{u`ZLyaoIkw+tKFA_>eewy;B}P|D(KG34SO}qLS&_EqwhVIQ!VGRF&k0LWc5KRs zN(?<4$F!m5x^I`t-&qPRUS#02giS6VmSNu+g^aQmSsK^R1cl^Oap*Ti-M>Kgv8dg@ zE710mX+pJ>PvG!?n0_77)$Oru??I0^ED)zo4K6v?nAFvS>8_5xJS!1zf zuSj^WSE@*_OBh=XRe(*>726n9FGS>ON{O#+-+}%|E7&E?kxP%_V6f#Pb6*Ej^D(XG z7skic_MpU?uI{q#yQ$IJ9F*l5_N-d+GFUSlJCICqWFj zAjw9Y0k95d;-a?_%ipyASa+=SeV(#$2JS-LsCCQ|_ekyvx!kQpg#%Y)+nl<@w4E{` zPLR`Jv-)-nKR|n=iH}Z|l)@lWYIK7OC6XMKNjUIqn9zM+Af_k7!4eD#rdE-`AtwlK zndsu_tSRLfgRv)m0fr0fOS!d__41T0Fa0Rrr|D0xX8IM!U_eLScIhE#tHe`Lj>6t7 zrpF{}6m1m3Nf0ibaEvmD?NhbV>g#mUswv|^8SQ}DxX(1Yk<+FzGe@e$7>3k>8O1a- z1s9>VhXkg`S&erK*w9^~YVn4-^W_G++rmi{0;9ALVL0$fd))2)5=%yLUbNdC&4O>?M(?@OxqEd5;&pzfw}{WGHjkgJB{Ce&=29AXuGBt( zfodXTtLUO}=r3X)xqXIj!9fh$40@fn3DzT;8AsKIE8HXkc!L^Iod@FOOHpyoT5Btft1ioXbVlR?SMX0(!tgyd54-DE<8p{4N^={5*}X zMjyWu_~CRu(1{QN9U^6jh**y6yboM^lSd?;#1moV_G7r3grAK8`TsDj7(T| z!D~vb@NeJy6a9f+S67dIkL#_3zi#$9JLS9EUhm(&`d<%R9@(FNdkd@|-3x$&_CVFJ zP2FpOP%_v?fr*AbN7&0Y!Ew;1tK&g(3^S^9{-R`nbLhn*Q7TDTU3OCk`7MCH9*I1_ z*T<#lvo8rg$R1?ICs$`6Da2mUF0G$p16o9(3-!@>&nBayWC4Y zzv;`)+8(l^)ihd^3D@?M zVb9YWX+dR&2R8UgMr@zT$2%9hB82J%!lu_*UDijZcD~lu?7n|-b@MclVZR;|w#8)$ z9^UlWH)A8^d%ZRIVH#_l0gC)y^}GxISA?eN_S?11_9I>22>U4ri`|3u-BS% z>@0zy_SXjbHu^MnZj=!ec}OswPXgF8YCA-A{DaHpTHFmS;Qn|0mxcB>g6rOwpSJwG zCRJ==zHoS+TgV0z4r^MKW?9{b=B1tfKn>%j_3ae+@^JLA?l=|6riS@wwk{9qPPe%* z&Nm%3rFpnwvfSBnO+&)BgXa=stn=~L&--BDBp<7&lFI_UFtkaH2!-E8P|-)`(Cv0D z7YFdI!mTg9@Lq|TgWZw_DbdxMW*oh=+S*xmFIw8?h&n}Kkd3-zPE%j0KL^EZ<^H;E zy~^G2eJ%V3MbKlsRjCh0XqBdb@Kd58C+X;8e=K$5vng+-fn&3tg96}=w2kxm);8$O#{M`iN`bif~&ZK&RfMelb>)}_>c3?2Gt zxuYpLxP27(@?v)N@QB>s6h1;#tE+Rldv)8(5*9M( zxkKXEtpcSW@PR8Pl*ojeDwDt z+d8tcW_*@4Bb-4KRs$(#dM;^`EnfMlh2ZDi#DiX16j{FF6^xy(buDFmvl{U+*O8`Y z6A3PkzGb#p!g&jhUj64&75Sct7DRB?-i}r!zf2ABQOm#3}g7&q8~&>Q#qY0fYZexpc@{A2Rsx) z1}swv1RRfB4IKI*TM0ryS<{+TuZVtbfV6|OJmN7Dp9kla+G4dk$>=Q9{cM18vVV$- z_!Qq(mADH!EShovYA*VQQL4s{$b)vuuce{4cfeljcq{c{$e<_c&6VoOc3K~$DHOIv zhI|X$lW>!_ykv|;9UO$-cp4CqIQLX-*c;rqBP8oB2-OE#t+*2;1k)lqDy1iFLo zwoE8g-IhOT1SZT>(Zj%zDK9?DtO7YpWc$NM#^p!(tghd8!xdS)7+2s1%@iXE@s6g} z{D%zLBcS%_6t=uu*`W=#vi~TpV4L)qkVdo%Y5;Fkx(xm;yGziR9h#Nv+T~&xOvb~~ z3~3s2M)8qy3)ndUckL8=Pm!1oxPl3nfGTS*Dcdj+C9@O(SlSl)N9dFWiKW_>Aguwg z^b@?`XAxy-_RL5eLFhLx43Lp=0|I)iDHAX47&_NTfa#L*bd_)`R!b%CQQzHEdzt%B z&2WbI-rdJHgo{gYCAd?jFOC=ZwMgcM^x%N8G=E1n=SneC%sS==#SpFzZuLBVmy+U> zXb7&&kxi<}t1}Ny)s8b!T@ZqMfd?1fR?ZGt`lNkM+>vHdZ{I{nYeMyni#SViDGuP` zXm%)V=465`U%gALa`wSNd*;IJWC`%3HR)fy2}qi8fYoruf|dv)SdR|ZR0?q=(cI%k z|ClxuOtYc-!w7lw>E%A`7%MFN>zL~Gj#em6RVpw47@Al)5s7J}GP=rO6Wm9dwgg;)K0;dp zgNo_^-F#{gKYy-i11GOVvH%nW82>t_0kTT3vElmO<3n4 z)#BONH8i>m^r=vB#WM4$cn*|iQ`;p~CMX~ncK2{gvSQ9t%3l1heg#%T=Le(3^v8y z^+^@-YEZkcldiK8Cp+62lJfJb+&i4x>T!@~LCU`XCEdf9JktQaT}YyNDR+>Rlb=Uy zHg?+fb}lNEu-c1OnTtifn;g|yf!#yAB^TTLpXN_2X@*helZSFZc(~qF^JHohFyIEieZO_*HSLEY7tZnC7f{oIe=WIZ`$jz^pE8 zzl7Rvy?~_8zzJvK>7WWyzlsq0o9agkvt;GX;=6mv^zgb{%Cz||6+J<13#hEjoZ*{Rz} z4dkv{K$#xT92@v`p2$+{8Ei*~I63&?CInHJ-^4(9@W>VwxwH`&qB2;~mVQRn2GBJ@P$8;D~5cD-DE7UOP zu#PlGm^&(SBd015b$RhodA4}S>NTaB&}9vX$a>l(5|&O1y7(Vj*|qQ2bJsbA;LZ_- z0h6dMt`EqetMbGB&CfiBu;CSw6XWa=-pqL|A4{mzI~1-I55x#r1ia)S2QHz%9^5gu zhdQXu#iQC8KSqMJTrhX4{#0GaM-Av(oSq#{-NumlVKD&f-?i7d?~0jU<%2{_+Kzme z6VyoOCeZ~!1Uc@jP|G)TB&}sy3e_u7lGO^LO)GnK2*|4|3=C1~J$)1mLb9-hV&5p) z<{47#>pYcmscJ;-j*xTxnohZ`w{wE@sSzGxl%+DcNDb8f$0m+|q&dn>mlP$yjnXS; zr3eKEiaQcuNdfgqsD~b;3`g1lLCf#EX{xS`VWwlwSW{V%ae8v~)V1)M&iZI>EoB~B zm1`H?V-K8xD&giG9rkroSQr%%PS7bfnlapM`l~0%Ow3QwP!C>h+_Zgb(xp-mwg{!X z`XH`q#KIC?=H@m?WNbbVQfe*v%@4HlH$G;wgwXy5l0)f zh?u7=@`HIu#*|&}r%{Xy<*CzsrkOOSv^p_Of1*X0wcwA~MZl6_N!MhxuGT;_KFmOJ z>%38+f1PU9+Y*M(b3!1B0OtyHX6Df=eU7wkKJyA12$rZUJ{mJ$B|CZ(nj!fLrrW@@W-j-Xvk*FvnW;n`tn#ZEvYnDZ?fcE!oHWKJWo3J8$@T ztmN*aff^JNwGq-(s-L)tA2I0z72~)b{29K-dBklFT9ilP)ZH^MQ1Yx8!w-@Y&1BZ2 ziPuo|P_HQH++L04*AD6u)@Nd&B_z`6Ntn=IH8qQbGy?5+VCBp^g?GZmBlzemO?C2_ z+*WAS)^XWkIax69(TsY@a0&@`iO!5UD_=`VTx)g-R@3a0fvT0-b zg+U~7d6drjk9;rcj;NY8RPK|O{24V3;VOGUzE>~Mm#{Ok^Dw{W8k1-iC(i4!U@_zQ zX%TGdBqEC`>y(0L3()WLCJH(95jxRtA2kgbOj0pJ7e4{jYiT<))oXF+pN8`SyAFvT zMEI&%mX>I6*1g!X6hE1GIyWQyjjdKmsZELqRf>~ z<++a844lkbo?use+=ncpabB=g*K>dNG(X19isezmUYFfnSv@QeyT8BO#)+@FhsmN& zUaQKeK$d1agaA&Fem+|f02&pMaCg1H*NM=RU>;D#vIO8$c=ASv$EwlKeSZa7IYg#8 zVgwf@+q7ppxX5#Trr&OKNs-L4iR=PJ?nD?H+Lp~fV zS!Zxe~DyqS$bDe z6vM?2C|fwnK+65Vz!QTUZKk*fq8C7;;kI5j%10PSRm__ZV<+k6(L8(1ip3weVSY(k zr1KE17bOnu(h#@8r>N(>%A7LU#=)VZtm7H z`EwXR4_`r;r$-&HXI1rGPbXgc*NeP}hn5)>q-^zLR?{7{VKLt#n02P9AT4q_?YnQa z+pK7-l#^2|VI3#JX7cjc>x))q>9U>4&3$~oW}1((JL5RqswqxGfg2A)1W(QvYERkZ zlx323!#{{UlRcI`j$@zUElTsleqoYcIfC99h=6T=-EPCJ_%`oxTtrH$`w_yNIBW%W zooZ2ki%fS15l_Ql=P^U4nPEF-sIs^WExRGZNgAw>_8~)}oW_TF0*71Svnn!pKDXB0 zcDpaoM2Fn@g4 zGTA}BEGeIgD}$`~De3Y<(I)sB4Qg}ST8{vfuN+>+888oBwL`B2(WNYR2X3Dg!=olzt&L!RxD0fiWH$BHQZ!;s{rs-{r(96cGBvhTItxiU z6Af+xUe230=sQOi{^0fmQ>il#Xo~NZMAGT82t7i%>I!yq+=Ril^b5}OA+>a*R)ZgK zBJq%|hWc9f!-Z_AC@rj?k%+fduc>N9a)xB+3@C}xxdvKr?DsCQoPy>3zTEHMRVY6Y z86<$u7t!vJ^IoMr)ALbp#-qb>D0uo1D<-ZpQ(n39pr6{b`8vsTWY90AbJM7<-RSzM zc@9`H1~l7J@Qhi`fFB;ol5~ALQo3Vz4tmg~Nmron<8)?CHdh)JZ8vt@&t`j2pSfwT4Yd_Gsb%NKqL~~9v-#_)p*u85`J=_b{1E-ZKT;~T_ zL#VUgn_$Lt6G(PmorQ7{SNn|P{i)h60#8E)wol7HLL(bg@q}2Nq>OiTb*DnnWH7Je zrn%fBz%V^SC#hiW9)d+ljR>yqpl)!N2A=yv%kBj^Exok59)_|}-VQ1fP|yE+$p-HNw?ex500}gjHS3y25g`>X6iPUP=Qsou{>iNwCJTS?_0oC~{9jmMk*m`tI=k zx4<0YqaRV*EqJJRWv_&Dr46c7fs2Q92QQOT{MR?&Pa`|f+`^!h#6c*-2Ny}T=9wvc+tw8x(^ywCR2hm0jwuq&iag^9J7G?N(jG;K20qacUC>SI z;A{ORJ>ZStsf(H{8=p8z_UQc{3}C0r*gbB?FCawA41+kvzi1rnL1p5JSxLM#%< zc0AFzEs4n@qxGT+P4iDwWUjGhKbq%^Eqd(2@;v_voKhTqNd@2314aelU-YZyZE6c> zXUhG$hN*T0&q@l83=#LY_`pF_ffNf?m&4c)e)1~+5~PC(ZSMUB_>96a$ioSp6}(F{ zEZrQvyH6B8rgIyuzftiTyoIn#d?uy4rxaLVE~SUu>_W>QN zRI8{8YXL=QLX8nW0Ure73AGqLs6?FVEjHn(b#Ed^Y*M#B z3on=#f^FR|n(O5GYL#tJIFgPiG46<(1Pb+QmVYS@t_HzZTNO)MoDz`tD_tmxHbMx% zRgKjYQ3#nIE#O>!BWFfcnLEnDfAK_@4Fy2L3eNw~NiNR32#Y-AZxOgEuU|e=5qwn_ z93Lf46Ui9>iz;C;)mbSfnkURAW!4&3+Pd%36yh%UEw(UG0v07&ASmbtf{~1B_!OY+ zEVQ*wemGIdos`sYn5>FnG9_b`YF8GWO%sp%V9kN=pp+~Z5=g}SLKI_k+y1c*loz5$ z#()?W>ft4ZP2L(bEWi^;hIXEi8NTQG1VcZys?uC>jmw*XCz<|!Vg1++Ll(G##a9f4 zPxFFFYH;M_V^yd#O{q-5FE(I;ncQX-@Z#{v6-|{!C?eAn!+T>aJ4sK(_U8HUd&u_ z=U5djjAUQeBKv+BYOp5YvMJ%%GAdpiSQVGCX@x{Y!fbi0^k?*Z{3s_Y(N&Qv1EFgfE5d^M6 zV!`br`*0UuTeV5OM&217D7A7O2Y97mK@Dk!+18u7cc2E}VYJU~-Dqof!+>46tXUvc z1OA>o>0{NhMB1=K&S?P+EZ5x(3OTi@)BK`GZ z!<4=Ued;hUu2N67J$sCS?1Y44*Jb5>^(@K!(3#G|M4048Z4KYki9EbW7m~sNMWR8<^&CRf_1nAH>tG}$#KxG z_a?|8(aO@#XXpBGc!MSLR9Ok$%)h0fTTENVdcFuNw+E@n`&LgOceKyLVB%CB8VPPJ zPwh;D-hYJFXpEkl13zxXhbcp*c8jT)ZL99~V-#-r{V-X{dlM4+KZnD`Y@fNl_k&XZ z>C{(Xk)zA3)e>YHi0%5clP0$!`FxZy#Zpl`vzlg{mvNab!aa_1q2jX&^d@VT$1+>t z<@tvJvU}yQ`vsEn8F^r@P~R9VgUksBq2=f36$*Nu=OWXlhz7W2xs*H%W(ZZ)c3lZ^ z>(3CfAzhQ{L#?coPvSZ8t#RT2_4vx3fNZ~}FRaomn~)*tK4&HppOX<2ZHI8+-%cMs z6sYYU+)Jq7F58PP%-x+<>ZB*6y2FJCEC}yhKqx-2hd$2T&}kk>>p!I?x_nk*S9EvN zgj%HD(^~aZ+u_b)U74uU1Rj3gXSDNPFak~KQB7Fk_DV=jnb%2|&JN>BtZ)o8b;P|g zrg+wuOP*1DC_l?%Y&!07#~j5;owGsN3%)-N1+0u3`hzJ*gMbPE&;bAQa}Y+Z{(tWL z=Ku%b`_Db(|D0#~y33{px_)-eFv0?e{?C&j04%^MU3k^f)+)$<{_6idxjLW&Kz`*~ zX8yV5^#Av&&;W=)SoP;kRbOjp0AQ3103ZY?{wDy@`&sz?_qSIybaFEE)VFtV);BRV zwlQ=vWiU4WUDA<=GGRFWHwiTWatDA)W0&B03gJA!dm~u{NLmLZY^>ESC{y;w!83d zRe=Qn@B#9_Srgy8*>7!a>|*>cL^8GV?DW49DFGm?P~_SFMf|UDzsrUTpwyS8SZS!h z0f1LH0Du&r`kQR>eSrKXGcj~A{J*8pH!f?Odh~BCO%DKJY3^|P)Azpx{av}@1*<-? z2(8floz9r0RWGeY+bDn58+DR@*eq=g%}xIWM;cQhsQ*{od%Y>SiD~>3_qTw*>qx4g z)C;<^O~l??IN-nPjRxgEbT; zTahxle%JXE{@J$hwEqy=8oF5g3lII#OW5eI>3eV6ejC<_KY4$N_qz>F8d_CDjt$Ni z0stU-x6uLge(QH8EPvXZEIss%4Q-5FZ4Ccsr2LB%U|yIqSh4b#DUJgG5`C#(`4{QG zNB`X&riH7{K>-JmM-2c_uK!sVO}YN*wzM;`bkaBVaIklJpKbn_0oPedjgnXXR(Ia( zVkrxNBt3^+j~aEG2V3m&EGO=ZS)VBot?gsp^LGFzMZ|(zYS?z z@5k?=6Mr)@13ZN6B;+s$a!0|@_fC`}Xn^xQ6PwoG|w&!CQ zF3*_y84&9e+~M(Qo006opjno4rcwbTFA z_+FFH$Rij3V*E?U--Rr>xazM~Y+Dze002ZcC;%5A`&+vQ1pGs2Z)fN6zm1+oHo?ko ne;qyK0FZ7oUB^Fde+E;KhIp@w007GSP2v50#PrMmJo Date: Tue, 5 Sep 2023 19:49:25 +0800 Subject: [PATCH 02/72] =?UTF-8?q?Delete=20'mindspore/ccsrc/=E6=96=B0?= =?UTF-8?q?=E5=BB=BA=E6=96=87=E4=BB=B6=E5=A4=B9.zip'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mindspore/ccsrc/新建文件夹.zip | Bin 67125 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 mindspore/ccsrc/新建文件夹.zip diff --git a/mindspore/ccsrc/新建文件夹.zip b/mindspore/ccsrc/新建文件夹.zip deleted file mode 100644 index 060b6d95761937a89135a487430fc590425520bc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67125 zcmaI7Q=xE@fXJ_xEXKZ3*ZQy7^ zXJq7JPYD3{q?o7upQrw};9<@8TWCR)pKib5VqZl{7xI~63an&vL|+Rr?u`gnw`*HP zB8Odd90zD5vjeDulQ@H@yg$%8Q!_JDGcWcm_g|u?xOpCre7D`(7RO|=QU#Lc_1Dfi z8|EM8A4?Waa}RJkf)o|;nu`ruu0crfGNs1zQoV2|Q%0oKh?B%FOZ68+qP|J_T|riN zadXD=o(G;wEptzl%JU+o@n{v!rg%YT8@Nj&I>zgK7w#YoWnnFIXoALOF&un_Xcg-= zzeVd+iOs>Ev-FAC(+3#XpkTZjg+Yln!`^C)2uHQ(BV4Uqv{U*fp=gGTJGSCJ0aM$fw*=}0dMWg z<*UWeJYLZJz+EAO%9Ur`&QD_TkEzk{o(;j+a0Z3Wy)jWIEt)9w=u#VDjD_nPyq`Ou zHsbX<+4L$m_5^0D~TI>mvVxh_P4U|$Pl};_F zk&&PcqHsfxIE%$v=df_5EuHOy@x`cirlb`{k~+2Z!YvrM9!LRW6HH0$GmhE)Q&%UArWhN;1@D9$S+#?K+`Z zA&p@MG}EPiwaN12C*`v>U*sBf8sBKQV{YV394o*IJt+?_WV)nWRQ$<|h+5|gLdzeh zU`>fpEl&p@+2zlHhugl^=|Sb;etR4ozm@0n@p-)29CR!9E@@!9>GpE-!ZZCuV~AD%D(9L`Ldbn=Pzo)D5?#o$P^Ll- znIq6DfNUoiqU1Y2A=Ip`(Pe8>-;mJ0wYl5r(e3u7cz1NM+Bymov^SoyH+76JKErSj zs|)#Pfb(;Cy4md`4>%F;*Lk;h*}2%?IJg_z4SL7mrMH`Xlfwff%^TBSfWG^x<4@bi zk=kvD4VLnz9WZVr9(l~5YLm`tQx7+jqbzDm zh^mO$P$g+>J@{$t);$j`F>hvqY}i56QPb2)dwhaPKPX1?|#LO8K3;;+9$C&N;YR$4Lu`Y=eV4{tkQx` z2ET#z+-VDJ(+49$ChZ>4Qdj| zNQ5zw*>?H$AZoTjutTwZ1xa-ETLaR5gNW#0Ub)`>Uy`&r`Kv(k;}BX0Ng|TprWlp)e8-<vjL^V z6YK(Y8_jSQC}Hf7mu%twbIrdkLpY3ME`q zyDT|zf8;l%2P$?Vqdw9{I;A4To0D0hKgx}@vcj$G{PA(tXPG_aF)RFhMP5gt{WP4d z>1T{czLfEZd;59Fh|gz9=@t1%ONUVvmPL5R4k9zR>%rn@`&9t_gNueV<2ox{qtGKT zom40!?*-7i^SJHoNCIfct&kRo3Jgw4x@eB-TnR!G=gN%(h z#Q@|hqC-CT(IFwMbJQXsYKzpj$YGtU1YWVzAx0MSLI7F}{-lXtp#MwM7+Qg-k4cf4 zvI7GEuz>*p5CYKuyQnd;c5!kxar}p@p#}hy<@F`|Pg(Qd!$Xv1qmTs=x@T)C4orxR zOk;8nz@yj8f--dy74KP`O}UarGH&WSM=?YY?ho@Dz!6nML=bo*3c>FmXVb54#(VW$ z$jGuVo)ZoaRd`hBZ;|76wErqb1s6R7C9Ic5^`k{FhUcc=SmZ3@ z{NBA)A6v5)9b1!x%5;?TdKPQkUJDB)2Q~8$N&%9jKbASD%K?n183bAo%z``*iWe}H zL|h!exREZ!jI&WFto#&FB1UE`A-z2McN?HS z=D!^DXkB*9>RB}(^n|{+oMxSUUCF}01#k5Ll=GNo#ikfULt@X6%06D2g^QObyKQp=L+ zSDkd&Tf&TV$Z@0vjY>XdWuN)z$8<9?0i+y6tO983*bNPAS}|W$uSu?5w{PZ3!;W?W zKXIgz2X>(pw-N-z|AQSJxe8a%d~F9G+Nv9i$BW<7{r))FO|6Bm3)9Q<=l*`tHyUiQ zEONf)0@Gd}FMS!mNqg7s%V`9Oufji=i(FElkKr*=iY>ik;1&tTfbWjfKoj={A^}H- zl3r3a6uZZZt>fF0OB<06aoh9N^EzdO(h*2|$o=v&ZLnah zosN!EzAfD_O8{Cnv3}%=#f?vP`___4=fN@|uY5MgPSDj=rgE7BvCRrw^mCZfW{gDI zsyPb#t=0dU+mg?%@v%2GZSu_L>HGQar5n`vY41C(R=zcNYpdPY=yc_4|8TWvdiwg- zE{%1=!@=oun*Au#cW_#q+l{T-3g9jE`)j94?_VH=vjtIan*+en`wP;pzaS+8VEMm5 zYG>>0XlJcwYHjEC52P{x021m7Y5oJL|L>d>73s=F0hFCnb)1V4DVwF7$SBK7FjgsK zRCNQH(sTXH2xerXc5Q`wzyoL%1w}-`F1loDs)^@kGuw@h2YUB~;`2eGHp|&d?%mfM zi)WinmNhVN_a*eHTdK8Yqf=WBoTOl;y7Em}k3JtU(v~GFdliqhTv;3wI#|+Y0ythO z0T)UZqJsl*#qxeiDw0FOtBPCy>R-Yo6si?o%;9`-1>zqFj-d6$;r%4e8Ug{j&KAY~ zA$t$HC=5s)C`hFpYsz$GW*BBrFurPt*5n)r21^+>Acas1Iwgg?9l0cuInm=Q?RU=v zc1a(KhXRvLnwtssjP^`0Wyu6*@9B^zUM=t>k@o)CqM3k_{?G(H`5MYR#Muy~8fU8l z53y3=`Iyn8#Yk7yRb;-?^G1#}0Y%XWRIt!6l|Y}^sRqkT@r$88Z*P*=a_k$|p%&P* z(6O!ieGZKlYa5+)LSH0n`mjAgVqKwsRe~1vOGKJ zC6-*$=RA(wmtg0}wvP*tcUap4W>y_@}5#7w}Csj7eX6fA& zzuLm<)f!zN2-(=b-khC!dR`BnJp|4Fo((dqMX-L;w{ZeCyMgKL!QJZZe*Y?8sk?q* zN=f>YzV>+pKBV3xvLB13Uy!_7TeG5Xa_ap61ZjA|!1FBPBlbBW)?Tbvu@f*a z^SK0Dz0GyRl1c73mMsW7li_}F@F`TUIm{abZ*~5xb;!{Q5cBDBK2dAcx;SdpZlw%C z>jjVYZP2!z0iHlU;WU%IKpYu;e3L0H{+2{ zd%xe$MKk}uU|nnvqAu~Xaj^6k)+$f{0Js3u{}-&CjsAgVasWVPaMGNAp!px6)vA*d z{|jPDI2R*QwvWpZ>ZNbhD9w_X$Vvv;(s6w*WHYiAB`6|YG%7j~1w=$a=khb5uAb=X zjt5j8i$!Tq7XvnvnPcwV_LYt1Nq-#$Bt>y1{<*qlvs*w8%p^7weP#ANA{4$NC|M}7 z>=nFBa=hOe((Q$KLt8~i&H@D(i0FPwF_7qGDl+zKR968x5rTB8#}8ph;j!TbqFjHg zt%l-|K2#=6u|RD*%R(PVTOn#lw0)XT);WL;86XtoNC}EC3yYvrARJ;(W@D&;vlQ1H z7nk|3Yu8SJP-AGJ4>b-Z<&|@(a_#CgH{$)JeIEF7C4%#mbV-?(c|au5GXX3KW)d3& zq;Pu5izW$(R?@}jPcn-6B^x3Opu@lA)@sO8yB3DGlnD%IaW9^Fr9f%6zuvH0i5)pU z3ib17{gF+ZX2o*2LWgp7-x^lRta1~z)3GJ8_E4cpH&8j{q6IS903sqKn#44I&RCMP zoSjB1X8|Up6hl$zFNBlY)3ILv2g1XuRGg+Dh$*zCiLq29XQT{-R*bUFM4D67kj9uJ za3k9cFLj@qNHq*flMkd3Kw*;7BC+Dz2$RO`8!jTnBez2F%rJ7ADxx4+yM|lJ3Gq*R z{g5!uu#Uc!mx#c#obQog@peY18J&!P%DgH_zf8Hp(8aS zZ~b0Cte2ZOIrFzSQyGAH61yZ#SonV0<;BgzSE`Ij^riFx8$meC^Qptvr~)px?r4mo}$@#XBkKBE){MD2Q_*;cgJdefc)L& zV;jtVqCapU+|)Mzg7Uzp#2gMXM3&rw*(#jQO(5Qy1+Q zEnF=7l1CQ((bMIP4pjZU<_zBXF$5I>;upEU&*R}OcUvv&f;!=uVKQk?gRKNIAXZtzl zuJ`GE4rfriQ5Yb%ehfHp{{}ATuuwgOxL2_}rpD6w+t#6t7^%<$Cr6ET@FntShOm#M z&e=O1YCadr1fav+?od*YvWmor55);|oenS)vN~C(yfTbCe1|arpZZ)93TJvS+9Smr zG#o4n13u0UERUFXm{49hoZ$h0$Fzg#Kz~*wLH1q%K(FSb0D)zqvv_&FapspVzx;sl zqjr-leW2u{yGyl_m5Qt3KoT&yPn;BFVD2Sjz%U!QKp+5oJXz^^0-rr#F)hUh$|~e5 zsPdHEslY?(1+pM(%w#cA)w=ScFX@626X2dkD13$^a77FZ4i$SvOg0&(PIIhncGI47 zyDt4|m1PMawu{DQS+#ogg&1rvoN&7M~9t#cf$?UCK={V>ngkuEI;QAa}DM4eMx)xP7X;(zk zwl>$yGLM&|+(ni0qpx^zy4vsG-=#0!9$&^!-#4f1c5av^lNI#jQ*D+Ab24x&B!;lG z=6B$!1Ty1a;#6pwaJI9itqR+!euFU2=4l@jskjXSoTrjX(#NS%inTAyde_@Vl|j95 zM{spvQ}y)|#@U?WL^4TEx2sLD;t_R{(uxMqQd=O;g^#o68Bw%-muKq{u45wUs`>oz zznhlQ>sK~bnA5@=&txl+rm06VWY*4(i|$Lz?t?}1_N%---ff@Gz~D4lDC(Vh0x`{r zoT4)qt-%&ZN5WNkCG!tq&_gt5GAj5bLnp0{acPbS9L004_5$xIA^`_=B;$4voK_z{ zHrqV*GuA@(EV4_pg;lukN`0KJ2RQO*cv%o*x4Fb|3li)0RSpx%tQYtHT^#` zrd9u^O~KecRrg+sNm)B{IYu_rYA$PTQgl!)9dE*wF*}$V(iHjIrt}Dc?uHZSqEDcr z9)El=v)kyrqi0Kg*mk$s?^k5J&t%_bkH0HfEs_DV38sSL%3;YuN^7;)h5-{=8Uew% zW+lH@3DzmZM?;1c|K9r;B0Mmpsskr{W&mFw(gzQy?Qm(QrTjbU85UGtnS}#WCYcp01HXh z2@oO$^}3cSFtkiO1U>CbD1yUjoiyn@LwP{uPzt?DFs#`_iA7nq#9tdmwy#r|%J7e{ z>|d4z)8?FGis+HN8eUcFcb5{%Ax#wV*HI#`X%NFEwA44RTTE9zh5(w<5;l2qZ#D>LHtOad@$ z%`zzm^UFd)JoPX%gFT7#ac5q-LQUp!nY6Sx(t=nD0i z27r)48A@4HrAZ#Uk&s}rk! zweG6U?p!XfRX}ZYtgb%gN?)16WIAUtOapwc)TF9%cDVVi)8VG0 z^Zv?-21HXHC}MXj9Nk?-=w3y9^e7{^Rj|FgwX!6gDZji5O7u`@mhLa@&eJTr9;XxT#cAt};y$wU zBU3GgYC;1m*GV`I{UkIKg#ydLJ+4)ZW`sn!e!^eEZJmb*%H;<0SZiW2Q1P%(oJnPG zqqD_(7^rB6VAir`21U>kZ{;G6zoq9-*jOa7Vd(l>C>$A}FJyiToT^w#>4TMZSzJV8 zP=%l+!483d7wg1g<^|141=JHi&|_ zSXfRa22O80i1!*kq@x1c(+v8H)WC!|0==9}gg8!+a_W}hr1NP89JVV_s#l>rn55J7 zA2~ZzC{W6vJ2b;P4r6j?P{@_|Gw}%NlgkF6wlQUFN@`xVR?Aq(2<3Ps(j-+>&8W~! z&si5L&b?7m!Bj!WP-RXz+ARqMu%?Nb$&y7l3lF-qgPWk305hv>*X&Lw*KaI8K@fX+cT8bk#V4s?`lMG19n@A&PpCLs@b0reK zeUv>NSe(fjoq2e-!>^rj+2wKI+tKyuZ4m9p#t=@E1q1@>aSgs*km_57dux!YQKF17 zl(uJ107BoNe};DKj1er-K@66J=2p&1zuRnN%+MC7jsBh{XX(G@u;PC}k9OI77KvVf z7Uj$!Y_+SDI(pPP=<;0Re55gtXgxB0T?Rj$N4<)6^Nn{9jg|2n1^x9yQE!Ca{;LP$1J5cvP;FPw%~t!8PitgrOJK$-*lFM8_Nsbk z(=8`VKB3CX%EAToRr}W0egCDRE^nTRz~;2GB79rfTd_#u~P7Z77m5tHjw=qW|jc3#n#CkO+dxPkW-4rrO8 z(^1@!@4L298wpH!sH47{w*uvf)wm6>v;2k}$;O>n< zagE1SVte7bd0P`@D?tudg7xgrEJz0{CcN zKjVFgRbXuRAu-$Vbq|79WNGnut4&WpfEeCyr zg^huk$v^n(uPLPTKBnBK`Rl|S*(&ZFa+C-K=Aa1pJ`)(?u3j~N#tpkmlon$Vdz?%1|Xjh-?Mu|C1+ zotng@r>#w{q~4sjpR>>8UV0Q*fW{?rsDI`|SRDpa63>D=4~!ScOf-~(v}vQxFzm_z zlxI;%lc+GUKA{}Hz@B__Ekrgi|`AY0^?yq<4SrHs)qhuBR# zRf=Lw50O5chO{pTr# zWNT3JT3Z>Z)r2VnxyC|yvfpN+o-LL+YqtM>cN?Q^Ou1~*? z8Q9mtp`@d$IjzUr1OLMJ$J@^R>-{3=_n`W!uW#4(=gZ6b-Qid5?anO=RI?`OuT=ny zVrRY-!0r9XO8~w{Us4fFxQ5dD(DRe81AE(rAte#^XFT|W%I_7G{f($Mj6dyNuR*dh0No!tb$-j&yl zytS9CTVdue`%O2DloVpq0{02KaMjtqcP6DqQ>>(MpmUZD4WCCx_d{3h@r!P5J$~Nr zm$QvHYIQ&DUS{t%$LE{vTRq)jc|K*9le)}kOPF#@j3Z*k?Q94JzVi--czF^}mhW2y zVGZ|)I!i*&k6#nB;pY*ty?GJ@EZ^k6`?S zA-Pg*^=eU1r`62chC0055 z*u^%&)MAdv?{|1d1~(($2=>hQ=>j8JrV~X(@gc)r)58N0J(CUCDp|7?3pmi9sXQ{5 zUr?=mHvH@X79`)FSPcNBF7rj-X}dM&ZR2+zr>|GBM>N?c-zV6kBnW(1jAA&%1@0Kz zA@d4s(KmSbVe8MZN=?7-7AUjlZI-(vvgMv8n#Ft{v{`TjRUW7>2$oALNUy@1@H5-1 z-d|UP2U|j`@5W+yLQ=ky8O|%g$sJwQ*-kjbNPzc#8!`?fmx&)Z5ym}u{~82&r2iTD zH&+}{6}Wm~caE{x--gfsp9^GbJ2Nv2TeE*q;NJ!E363ep{y8Z(UEmJxEWOwm6gHo2!f)bA`Tq}If{thf0)T_d>rZ7 zFFd(wBKH4wHS2jl$$BR{>2Lw73gRWYkwR-?wce<6puUoq2xGD@TRkB{0W5+dJVwG< z;u~m;44Nv^R>ViTQHJc0?T?HI<1dndj8!U6->WO{0$efmAD~dBu$`soAPNX~4uFP@ zC>-lYY0{7g$h|kO3UIg+K?#etj}rp{7I;4c8j}ld6cvlSZVn>34K~R7kGNodX1!YWzR8Go-0(`}g8^SFL8|MzRe-SS- ztT5b!M8w@#Nwy8hC16S*wd#<`B6jjU3uz}sn4#%os}EV?P`5(=WjYbmOi@C^E%o|% z=gk&K)NE!Mltr-`VYAhYv_V?8}P zm>qAu_ow!si$+CG+fKiH4mVpP$kKMH=618|jDz<_Om$D2a<}~h9HS2J=AUnmk7vFI z+bjMK9%#*XXX8ty#nyr`Y2fKsg-_vSTI zW!n`Y>OAv@+8UdWC@bZbpJqEX7eP`*5fW}`Xwor+_9sb{psq`afu@ZB?wyA9<<1qf zMh~>1mOYHQo$=F2(l1HB=-a9yEUq0uliTLJ|;b1PHY7(^&={wLd`Y4oI6W z8z@{7Q=uPxv9)otvR4XQO_X751;91zPZphniWbrq)LaXLK0=B)8n~m)+iavTLTJ$D z8SI2SV;ZGs4j2}Nxr-!x2e^R~crAN*W6A=*BWBAk^A`@#niwI*#B2Tx+yEG!y(pl= zCmcF18G$fR2-8|3i-4isa}>fgY}J@4Uul3mQ{D>5F!dYyJFpGdt8<3h!$9IiYg!Tv z@KALOwzpPlT)loqanQKVKW~a^1=k-q9``9Vw=`iw#UjaZvKhdZ(^k@sG zXyRSduTnJ&0XUW_@jL!e!OfwptxB$Zo=}3t)e{7dRfku*NI6u+z!f^a_~qQB#H%3? zJb=Q-aQ^Z7t)f{OOEHhGPiW_#- zsNSGHDXra$4k4=Zn{OB>SWxxTkrUhvGH9EmOge|`Q9JKyBV{T?tljYM>T5=hHg8I` zf?RGHdvt=FO)M3(2|Y$k3tGY*j&2#w8iz)kCSOsqQBYE+77#hO$HsyW#sl4L(3+ob z_gCk~(`NUmRc+0m4|k`#uj{1dIJe#&uU=F!NCiV$N$S-4O~+$TpJ5u--9q367=Q;* zl;ChIBx;NtgYD7eM&8OAtJ}>s0Mb*1ja`b{r;Zp^h~+xNA}CSZ9w)CLV9lnn=GC|! z9nVx6C72|tp5E`emoqRLVhTcs(Y^?Fq3M7smTZNI!Z7MT|}=;$Xb*o-kMmg=)diojpLxV9+pc?c&W zB53lRRL$d!jX`K=1TuDac=S45nH#)2oh~nbhwb8Pzk8fN%vQFp-0=Une!j%f*Q(@a zFQ}r9UfvJ#*g@XhB^gz-SvU1_R__ix%Ug9Y5OhC+czS+jbX|lS#c((%N{`Ad z&9WWDC(pdyKQQNe``znSY>7yvd=k*`>qrccFBjYO1mKtVvB=9YIuq(NLXbelrU117 z0}hVf>C60`eZe*F^`~n#csY?Z|4~E_RbohEZPk^bQD$`LUQ9+FHKvxxQ?wj&mXTw2 zA?a?EqKuVSMRplkD&{^_Kz8|#$OPK+I2OryPEw90JvK&4pe4VUqWf(xMoGTN$T^*y z^z@yPO~9ABh{)mHso?XfVDHT-oYm!|fT)t`wPI8RKWff2S+4*hB^msg%jmQ7t0dOf z8Jfo8wB-#Y{hfV+{e6xY%iR3^aZq0&%3(>mi_m;SkRj{kUfJM8QF_&o%CUIFdG$#} zG{NGgJW)m(g7p`tv$g2FK122$hLvG#$OBAcS8Z`y-Q1pVhKEM#q{Ij{YSM~D86I!Lg}i|?BUlmn0?ao z2qY0rBG6wW#b%x5AUBTI>_Z()d0z}l2jo;lHAG0FOiVy179y$T7vY8~0ns_?sFx%K zN2Oi0(v)T!(aAQH6_XO{wik@lCgO=iuhy=FakzQXr7f))8#{a!jc&n(_`g5-5x;xe zeyHb^<`)vN%qh4Z1*%EHfb<@;BI^v2$46kP4Bw=kD2n^d zskHXuel<8}R=MnrXGT$2ZKpR7RSd~*mn4~()SHAj=j>N?75cWLhq6q3P|`1d2MgQ# zt-!w!`hSbbzANzSuIu?~%KH_@J{)i?ny>}7j}RqTY2jx; zR7p_W>u*(ui(-^chz5sOB6Ex2PDBu8tg+bT4RiSIS}_iMAYc}Ws!SAM7S3U2l_Ft* zizchp7u{jTo{}RAoERW2tVW6XM{-2_#&M#)4BV?+fM}s30-copoEqQ!+bk1W8bVb= zjs@BW6aawWZ(XGU;QIGj=HFWBXyLABWMFONVr}q0Ymk3bSvCN`=o`Jt|59cD|1-NY zwCpw*P)2v(QP8|7k!fkO$f`Rfs_I~?Wh;fauOTyr{9EX`+Wdp_lAVv&IZXfLHx ze%jyvT&+o-@%Zfs%+8~}vhn56lU(HLb@5&2x^=# z1skj`k;x!{gH93hB|-;!;dD0C}k(rAg19>+9nHj{|Ny71^U~DmhxZAT%aud-c3DOro z(Q)T2Te^f6_2)YH?Kl{*&$r|0!dA`ip@3zB@Gzu71bF|YHa-^aZGtNg1pnU*GU&Ak zuZX#yAl8Xtz`E}$N^Ew-{br6KK1ihwW$}DDU`)p)uyIh+0m9fYtX{+{IIQzwJLVjg zVUj&?qj`4tBWG5(-#unAVL^x;-Qw!4YJMDeJDT>-h|g7aXOnXmt%wB*1K^3d@VLQ) z#e|XvB5Pv7=ECXs?h*sUY&nY9nnsTRvyhpOdP8P&h!5et#YEYPfWXcJEOa_b5hfHV zpHR>)STSbF>wkT{SJL>*fGMdt?Qz}1!5_FcvN(pF{^gBB0{Xo5^;Y1v$xZ{j37NBK z(_%sUq=*c5AheWD!J|Ho9vLR0@-cGYF$FBI;%upjZ}7Q6q!uvMoniM!rEYG9buDw; zPfyPCV)g2N3>{p*T76sRmDLyXiJnM_CZVU1f<1&LZ`osZ+`W4%7xqqU|JWpYc3boX z-{5X~U2Tvv1j z;hJo6m8|c$xc-boJ}MXC(inu~QWw+|))06O;j32RS<39$ojoF#5ypx=kyHNv3ksZdaw24cvy)xhM-1)~#P2id{iksqsss zjaEVrVA4ML(V&Nw+aZ8E9aej6B^rX%CNaCFym0m_=vcK~{SYuaC$N5+BR4J4+fd2~ z@Qn|hMpwB4^7N6Wuun$pVO%K}0k9rXZTG$T)sMN~yf>pa0e$#{t*48}$H#HpZ|G6K zZwKvg$Z09G9OZu#e8?^QcF)ZMA;tPlNUUYQ^C&9BiMODIlt@EC+9yt#q*q?eeKqM@ z&u&uw(G3O-h`-!0BU?_@itmBnPY@|Bki1k4h?RhtP*4CMf=Tsg-WD^TWdMdtJAYk+iW z@VGa0$M707Ptu{^3q~!v)M_ z7Q%FVY4}|u1TK5UtLfBuO)E@ghf$@;6-`X5<7TyR=@regli}*`x-nm3kV)&;*@&2b zbM%F3Vym-}L4)Q~0sxS2{=?B{V_|D-;izZgZg1!K*H!<&KM4MH^!;W6Ao>U6|0DDo z*TQL22RK^1{#Zm`ODvHRa3Hx8=eXzgv-G&M z35P#)O_Ki}-Q8|icR{Kln?nHu17gMURp-<7_1eo$-6&0)NngiM)@zz|SLYXOwNKi0 zr#1Yabu*pShB&U8=A&eFY~5)sha7;5m{^TQ1JDiq#$T;hEI_{vo3T(wTL)A}Gc7kv z-thgH4Y&8+0xnsre2UV332PX{_hRAX3LseNPF8GMsSQF`tL*{6u-8@CPSoDuem`>;D>QxmK0sSXyk+!CZmVh8thZb5}~5goM)76*b`vsCq$i&oZgnCmb% zYiQsI60oDC)@0Gs{bHA+rIDGYMKwjMeh0Pk^u5tHC}-dYmk+BdZ_lc}gmwZKrx^uB z;ge`M2L_iYjY@Q;lCh48MJ6n`K$H|*r0i5}g{fjy(|n0Es&;qd^abW<$c)u<1HPIE zfT$u!&(v}g<{lzSq8DLdW3VsSq*&EN9o^7t%}V+xQhxoz+Fgv=!O9+uvfhk33$AwI zY0$h(L9En?MhS|@#ipWS5(L4Bq17C;jHHjRgS)iT{;dJw?AfEZ)#THu>xiv7>mE`j zP2IDt9n(|M86!3K6Sd93oL@4 zs`0A{>q|iUxnmWhl4G(urCzFXhlcS^lHpPiPC!MqQawbjS~Cnf-w3)SBP(`_o+c$# zKYEHj+4Q$9m<^5WELyBNKxNM=K)G5vj1}we5fkl-b%w3MGvq?THz16&SW+vkSGXo1l~ZBXS-8019+y6-fr&TwI?F)_dRAdS zelJ$eerhx<%H_R~DixV()ul?@7b>}Hh!;gilN$8H!wRA%Tny|WL~y4K&BcBTFspVx z-ayUPUSAZjZ$hibtj6osysNGc+t16e&C%W#IN|02Ao?BeF9XlU=bhicfN&cPNE2^w zE({`Yo6}#m#5naNsM1xf7?>qNuIBWBBa&7nDgnEsB5_4fYi(|Q4L99JHL7~9-c?R9 zr^_Awlse4({GEJARuByx6TMMd^u&=rqMaMhTNtwDm^x}(?ucWNq50N;swOs?| znp1<8j0vlTbf!_`TWC)8S~4fC=~4TJOrsnXj&nZduD+2F>{+Mz*{t%`*+CCMkHc1` zX6~2o$D28h;m1)0@IkFwhr=Zy16$2t7Zy$W*`1xldAa=F#;zx&)U^`8^=^9VA^QyZ zCm$a}0)8X!4cK6$Wwmk7HNh0a*E@%qg|4@HvW%TkDinr(Ci54n7{j{7idT|2!?{O_ zTetx#2F{WZM~6sZ7Vy>$7ar|H6Ao=iu1$W1tvgco@H2F+NI;Gj)23z)CqD z97a)4a6z&xbI|rI7$(xRY^x0ex|LkKels>);UN@St$>o{zf1*?kX3_U7a(-KXHp~BTyzB|In+AF4-kD+Gt|AE7(hWr= z*@SA8j6>$T9-|}>hKu?+2{Y>u#0XbBep{=aUSYEK9w~qqFuj|EZH5viqa(~Q zu+siR1IxJNvZf3Z(K@)&eF^YLw+gT2I^Cfc#0j$%_+`;*xeuZ;?>W=40$FcoZi)w_ zjLCJ4uokgeXc0R`uawGl!&a1PyW8LXo$-X9)Ye`_xlOfZjv zi`4Dn2cooEV_5B>2g~fypy}peJ8X!65esD97OG)J8}wDyeb(()j)?%R^;&n#&XI<^ z@*(i86C<}rb9sIfJm(7L^?V4Gay{@5jVflmPH3{#mcbP4B~tkdQ=DUdcA;Yf_srR3%-8SShGAwjW`9$3Y3! z8FUB%M@jDN3jK{!)<+kRYZoG0j=LBh-fzcIQfBi=Vnk2Qk%~_tno0US<4_1qo~|qfGxsce;X7r5X?K=4Zpeu zSWmEpxH-ho-6pV{)}=cy6SBpQ(GP+qO_clo-U}L6mEAa5zypQ8OWubXOy|)Mf&tbE zt_fk}-F53|SkZGt8VS9Nh*tWsDGDfdOJ}4(v&p&{cL>12EiSww;AR}ggP#rWM=j-# z_Qz)}ynlNjf~$f1PsZXI-6heGVYV2j@}Wo0w|-VR=kvBcWS#Mi(Y!t16i;`dF~koF z720E02kQKEH(&>t>s7uvOm38yhWid5iF4I6v>Ia!jgxQy@>{l!e52t4HX1fvnON%y zBq&IXRoHLT=N}U@ALJ{qeVM&yQ}JaUro;ukD$9 zpj+udig5^6TZamL?AerTA}ruNG1@_RO9Dz(HA#`F^5Xl_o~FhEi#X+2cUm>HAmFy_ zM&1_o2dx?JyhC5G^$i-2(|)=Gr0B zGi(ETMHSUU4dR1%&P4L;3+fh&b5pi506FVniwI>7t*0%#WrHEHVp;m3Y zH)Kj3sGcsGfII~Sf~ET4)>7a!{y?D-%w^P3ryw;_0~5MsrqBrxy~wy`%^)620HIMb z>?AcluELG);IzElwt@k&REeZ_k~-3p_+ji57~<}RZDj~r`B zwFQrC`zPWnJ**}NJLH$?X{75BOr#O~lu>R8MVhng|1xda6DGuNjR*c^ZBJ<3S^;a0 z2?*Q(gKne|=eh_=)HHu^1II6Cg+dM<7!exSOFWUPv64|-?P)_RMei>&YvTz zvX1w=xNEb#9ga_DoMCk>`JW}<6O)P>C9z!3tF6aT#Nk^zYh16$RgWX_3bJ?0-g|5V z0mVOmh)IgAb3VeQ!r2H!lx?hEm#MS#e7hVzHedd_g=2kyF!HxoZs}Wg-LY4X*mncg z)>e}66MnMm5HIUPurytA|3ALoDM+*^%C=0~wr$(CZQHhO>!xknwr$+Boq5wQt0Ow9 zx~ls9o;dsKV6M6MT4Vk@+m5aS1lj2R>Nm#F!k(|DUBcRIeS_*86R(F393)A$Z-#1` zRoFf&8GyE>lduCx1kRdHSp@{ko(d?9Qs9jcq2oXhI6P0HQH24-ccmGI!b}jqX2mJ4 zN+?nfj^X>DGlyZbM1}6$8WG`Rs-Vni)C$&f#F7(*%`!+))`BQTF}tw|d>7}$GL(sl z-wSMkL(I0UQ*0F;6?s1r`pQ>65VR`56@vOZVnZV~-#-|g@nMH-4u!Igb>tQ_F9x3Z zdVwOx?g6y<-0Lm-NJyBU3 zq!RxFc5Z}VzxjD2a(bM{z1Oqsd0F%l4sXx*VeyfeXYlEY=b!)e=zeXE2ciMsa`@V2 zaDi#fe}A)D*JqdWY4vy|oFl&%3<*58Uy-*tw#WOr8J<(ecjeK^d|iewn-nK1<0{7+oY+vdx54#{$}+IMQgC(OepTHLn=xNQS7Z;=dS(ZaA_6Pb z!1XvoQ8O(o+G!ck7^v;)`yi;Ge%T7CjLO;`o^0(Y%VboqQ;uFWc{EIZm~5ZteT|p^ zHE}qnAJ>-0+44MJ*UX_n*z0?dSObsu_Lx3~T9QDcP3`DZcN}+tZO9>%aG&3HR?A{D z&u+KVa)w*|(Kap|Y}rZx9r*|fP+AqN z9G4{}b@K_uCt`FWYY#zBjh5FxegR06hBh3_aZ1G2JYTsN&^8c3qny0iK_3A|>s?{p zmrukdbxS^2x(Z+^FbxwJ!V!BFH4V8|=C%s)6)LK*xjKC}xo-Fu;Byef9ljUnI>uP& zOO+xfSe!Q@hxyn4&!46q0!u z#xk)FHg+-D)MC*cew3LDj@0X*Isv-qGdV)x!SFYPcgpghokH*iX(^II&Elz^;u8rb z40fO2dZ&TP3bW%wO;SD;!Ef>Sc-#iCdTgEUF1Pz3in1lOQ1NF3U@2$~AXUzTmK_QN z;mdOekOUD|(Sw60hhyCKeweml%916jZa!Leg%B;P35y+$C&@L&@$;qZU?E>=XZ9JI z8pDo7TFDGZ_d}lL2yxB%V(YPlT&2u>dJtxsZ>-h@%(qRS+^{%THEp>b^}Z*W7&OIt z>A_NC1wo4k6@OS5D5RyMO40+9su6gP7DQc3zNWhN3oY{e%D-k97c*Ga`hW@wRoyvF z*R`wyb0u@fhRLgd6Djp}MV*|$J45T9E?*8}`fj8GVYEE><*y2^W9_5O}V^X;uh-OsN0(43P0C!uEIL zDY#K*Q3K$~Yh6siFM;+0O)=C_gH{?7i4)m8u_{7wPjx`U*`?X+g*W9*TiW18yqxAr z7QxlCM9Et(%s|Sn|uWm{=bwr;gOA)pYW^0v%FsdT=L&=0* z^jJtIsy%rXNk{{lw3FZAd&y(S&vaDMzhI8StwGZgCk=!`yIP?Mkc#Nwn88 zp(?iwEJ%Gx&|xYNT$om{cT=s+t)EDWJss(BiNZ_qL>R=9cc%o)9w=Jn0YG|1Cq0CHQDALCAl(0ut znO=EnaM<4Q8y1zmy*!lNkZXo6+Gm<)rpx8j?CDwXKP1M8C+LU&{lFPH7vFdPXcOO* z@8042hQbN`eX3e!7O32>qea)|@ojnjQJ>f<65(cmLPmNl#6+$I=Ju9p`>4jm44%%J z+Xk3^?9v`7yo)lZ#zVahlZ_@~Q~PYMnDV_HgEV!6`lJ&CMr8ia)hSuI&Lka*t@P2=T&a+h+&#-E`Z^|2okdjw z+!3)VqZRB!tOpV%DT4Wm>cSd{^LU|LW|v>|t#aRfH>t_{Jta7KGfjCmxnEI6*a&K@69lXg5k*%m2D_s*pT(}2 z_hy|EYv4q9X1cPLWL&T7kuTxo#m1RUOJ~J+(s-X_)#1MpHH-rI`ObmfmwVi>M?{_67dr^j`! z-3aWv?qv(bIFUw00t(@g}zS}N0W>C(}?Q6zMrdlaE@zhdcn zc@&PI7U*cnwp=gjs=nBBjjGX*>l{eBfec3rRv`&!o}#~3CXh(5#(~=boP?@`8gdrv zktwlYV2(SY%ZbOG%Te91=y@rP|gf2&9w9WUQnYFx7sRBlTcsmszIF|}{Iiy0RKQT;a!E`qVQsg+|`>4D~R45If9efVFLE`0u zOSz*7;f)Pwv&6=18)^}w!j3xrC41!jWA-hHK z6ixG3%PKUr1@+riuMpe}{#X*8+VwLwSi}d;xqA#H8jNB>E2|48KmK-`<+`Zzg+5?V zu6q8n6u_pcjJhE#zg1&4+2=`lC3zIB>T$9wfdH&hG9^(j=#MBldKjXtSAEk4 zmuu7!mUj_ zxG2HdCE!I^Aq81>37S5-J(B`2HKnpwszi6cXNyX0ambn~X>mlTm4&t<9`3^!7f@ z$ELu&(tT02o{4;__qv`MR^BN5ile9aP1jneUpWKkm!--zKN1|Ip1ou?kB9erPLMjr zxqAf2jU$y+xz9Rt)ym3q+dlJ#|3JXZkxDmtrmz{KG1w^lW; z(0~9j#Hi~ABWby|o3A)v&$hrot~2eMiR4TG-`lS+u!OA=L9;^&ejTgUBEu2ji${GK zfyDh|t3(^epX^>vh>hSg!G*m0t7r)VQnynyNy+^Y6faz|!Z`-pkaL_b;0os2ig^>Vo@Rcs`5g;kS&sKP=&eMkQo|LAhQnm#_M6R%B);56`rBa zh;$$4hOzIV;L74z+2%7CN5C@|Jc!O|Xl>Ooyzty5GRxiW2MIZK4X4&KjGw)-Ye^{Q zix7`Si(87EuTrABAe?46RS?rTl{}}F>#8nclP65`IgAoVT9jgNZ$yl2L7g5*O1wwv zeWAqm_$Jeg)Xe8dv{R=z$~|7FgvH#;WUO2XnClz5(Hhsw#K`mdRC$qVs|UdHD+h7` zbW@g)!j0^lFnlbfEnMM`bT^uST1p*Iz^~H#?M5l7=(u27Ojto>pLtcF!uH`~&wPlP zJq>OiT*Jo$J@65UZyOP1rphqY(SdlluR96zc0n1{{qL{DC40`06<;#5)rhmw@=ml!g4eg(@WSfynm9 zl{h%ITsJ>xK$K`5-ZoQ&qbGhuw;PGYlZGk9ipEQ?&e+k9Z%^Z?+b?e9?XBCGag1S{ z5rg-Z2NNXMoeu_RZfiuao}2xXN*wq?Q=|p+fpn-_qLS1-+{ZkzQCfKGF-T{N#n~N@ zVBIW}mECS({xkQDu73gPp$51b7k?_sfX}5U^|%1;U!Mi?@2-4ICPTguB7=^o(;xF? zw>c8|t}*Jc_A+;if1fRz3HNw0Y)6Dt#X=4aurTgzG|$4hxy4jr8uzg%HI?>zIZAnb ziqER`Sw<^R=!yx`&?-isXMz{cCLRHd;5_9fgUUvo0f)nnQo@!D7lVi|h6kC-!maC?kNWvGa8Y-&NXv+#q^%%HU zOo{-!CEVGpj)cMc_H!mjBMI%MAqeOf&lKi~lZ@X+=3gk@3u1bzP7(H<_g7#*;HL`D z8l(SCD2kqLGLLvpo;^wEO}gB%d!Ib&!xZDeSFTo03C{;y#S7*xOIhD=^rB1icqlI$F_0M>|bxzMu4_r7wF_9?wUUxCe``4ON?f|F%{^Qr# zGC@Nz@#q-0qN z2eF?d(Zr%Ww-Mn(5^5w4V_jIB@NaX%c}&loDZTo2n8mtN zH+LnJ1qPr&XBHPm!qB5DGUMytI4r}Ct%5Q!%o)<4DA#b0d3^}i_Ebww2!I;aq> z5<>wyzk3o{UftWib3emA4KMgDYl}4|qWaNwmJjOdbc^u6cIvtu%p8RFQz_Rgte(j+ z#tYsH+M+8n-jxMvzA2@lnZk!xV3XCCm8#2g1p1pSQJ=-YS&n~?FLj31 z)eFVBv_4zTAj*nToK)$Vy9yNYitd;A`#!{ZgseZm!~ZDWkw>EENNN*_Yyj5Sg-owMs?!thpchDyf5vboLhRRv!Acd*4o z+l5wMM5Q}Q8-iqxx>0K7^}+-_f<8843Vb~sT%^09fIFb86CfihqoeggSIVgdDsS9f z=+F1T_Lh%%c;eA=(a?rlqRMlvJFLiq3}XL_lWcN|<^lVXj62%v*&2Id(e6u4@~v&I zkLY{t1gK0tt0v}BnX}8;hxTLqFP8||Pa1<9l5Kqzi6LT9x&v}xH1n6ZA9K1Y9i@QCdh@I1Sc8JnLsCQ zGYw*lr#SVNPT&`UzEbfnJA#eq^kbxgQu|Ho(JGu;F6KD8Yyvitd;&8E ziF7wi^h86e?;1$HS?#L^$1Y=py({)&E5eK{XGvk&#nho~^@-O*jPY(&deUy&2s*Ax z%(O?Dx9_x|vGASC%$bd)7r7o|zvc(z=$tLBs?j>z;f4a#6dwIr#NoCZ*|o`gX5FAy=2EJ2G7QVy!>S0+u+CB- z;hVf=|MYnP*ycvkxTj{6F69KHeN=~r)Y1M!+smoG;Azwz?6F>^!T@P?XWSY~n%18?q$nA! zP9v2SlHylIOecS02qLjcko;H%%S#Ju9Y=liQP7EzF9-;_6i196z4*^?-@#b6SiE7y zr5zEtt(kf=l9TWzaGp*a z%s=IZ$Rz{*9A~Wu$2{yT0kV=cY;aJU_C@RTgay^D!pckaoeWOW$)Mfwmb1Mv1Tl0 zxOTAQx?lf*<4No|!zBEqf%WyT<-UbWx$<0ps&sdb-}rII_0Y(Vi7)I%DKI|Tz=wx% z+Sgu-S(TethWd!soy02~De9lOFQy!ds>T41*ysS%T(i$7>@LGC2)(}ec$W%UAOQJ2h_-05^oi_w zQ}9vS4S92!Q1;tJb;x(?C%oJJIH7G{f4B?lz+?>=;R2-n#u=It2A>xq4qyc00#7nl z*O)}<6njcxHoYb%Z&C6k`5CRMWP*dP`)_MWcv~aMRNZA(`6Mai&QZk~9I<#1ZEz8K6kvi=n`9Y*As z$qCd`Mh5XnjHo$nq@3}vb9;Xkd38DPZTzC)GN>eu;0e|6U1_sslS){o-S;$&r@JJxI9Xii9L7Qk@Kml~6q3jB<&ce?A6#RAyC@0`x}2irdf%0A--B?= zayN??)T8gnxbtHxa~Cl#XXy?exyNuz?1V3Ng;~Xt?`m@z(=oh8&(G2j)qvFOQcEHH zkh)1f*&t+?$g*YM$!#`?Q&`s(_`6tP{mTltIwp%HycCvV35etB%Bmj6qqioL@l8Sy zf;q7&C}BE0%=OVTDCz}sKe zCT7ZERTM|mI#57#QC)jvm42^Fp3E`{$#-t;XXDwVKMPgwt6iZTB~@Wdqw3R??y24S z!v9K{QxLGPdK&XwZhDyL3%u*DQWyE+O!J=}d_5_9eDv#5UK~w;OVsre(-*1h&LS6$ z^jn%Uv}L1tFD)DTqO_*#Dn@oyTh{e;X;0Kwjczx!uIf6`nx0!U9UA%0DEUp(6KsTPt9^d`#?tc)HPV9O6yw86qqS+{OZ-t@wS{tB! zSKiU#Xtn|J>U3MI9oxz;o!Hm+GU1;-Aq&MT0j@p{27_{9GAFn{ zv722N=cSvj6SY}mf1cE7D~>%4r-`*K4TJCn6y zDwq?v9qV<+43}xLpCwb9x7N3(xFyV~2=NR_iv`9R&!bsKjYZc|;+9!!FL6 zWPt0K=&b-!)9O<1NumRe{}TxB-lf-)&+%E{=8C|tWM@ug8jYSr8v5W@fJ-mYo45B< z3-%WZIvam>9Pd^~CyRnzpuENKo}O%*N_7ku=p@@lwB_Ur-4&kIg?G1rCR-jfeyE3) z(tNp+qnzo;u&RZ03D-Mhv+;*rxKCU?#s>I1sNk3b#~m?guzJmn^XUhfy}E<1OYsD+ zcPG}7wrt7Oef^#`lrPAfG!7apncnWy>2Yi`7(gZ%MH0I?Vks>iQW7A{vc(MfmuJ0_ zy#pt8x-icZ>e<<-TDzRxB2pAUE6?SN2)JjB3so;TLdv1rgVi)q7DzI0s&2zt4;sl! zE#OwKBvTlC*-&WIE$J%4GG_q8w6P5%LdPnEJ(Gqx5oz~&ugxCISqk+ED{7l7z`W~H z&)6rs`4z&4air%1q8m%;Yf!AL^IL`mFBF<)*xa~rV*m|}Z6LZ4fzFYL@rcaNYsIdb zBRqfCuzOoTX|UWNZ=M3x?l(K2d}&;d6{0|Er&1^!0~SU3pVu<%KEdx*BXhjnLt3;L zkMEpF43$cc2At1-9ETsU@-Jz@@)dHlf=K#@Xt;YV);7e<^4&1HAx!1`au0PO7ajSO9kE%fc|o&G1=gb@G`qgwC~GBuOr|L;>yjGnLJG`!<{IY;XGI1zaQUoymUCyp~?xY5BQ-9}Klz0w-uf6}t14M#@k?Vv*1afq; zli)g-`cs-_mtKt3KV6>{3dob>E#ekz|w z!ALnMRO$)EszKURwrABcJw9?N@;zIhVB^x|cP!ksVSj0xZ$P%VZ8+%_>vF7yECVjz zM0KZu{|Cw0H$hCDRj}LQW#-aRakQhW9#lcT1Q2&R5*9MdX}A}o@TZ+;+!vy zy3E3*sp&lr3Y*|z?AutU7nO6%B~LT)ahCzg34U_J4vPjC>2E?}$*-m58J&2!HdJV~ zQlYYlhgt|JTCl+5-(ZVhjjw|>y_~0O?(FP{`Rwxj(Dq)7Thk4>q5r2@W7Ro02sg2^ zS9soJr;d!55?5A}yg+EAO^aDRL2PLn&Zh*g z*g-DdP<@Lo3zrvcLi zoy+L)eLJ3LY-;_Vq(9ytcAgesE#mZ*Jg|>2_2!avOsWbo8NQ(fx+-EmrTLE|_0I~G z*e^G~w?p?Qg}paz-~xS?Xget#kPc2GIkDB9oqKI*=UTCVU!U6yf5-`m*nbhj3Pas$ z0dU_UL7ONfND*L0Vhl53Tcp_m!lG9P%rg|LgFAGQ%<{$E5#EI-=8S?-N>DvwQEKY} z?M@sDaO&!dDQwC1Gx@bd!-cLrmCj zN&71zeWv-P^|dK}o!4CW?^XxNl}Xy^cmSPLrNy1=GrX$T4x7KCUpc?lZaRHz?^u)m zxf(kM++e|vyxNdVNo|T;aWbi^E6bnk$5fBw* z1ovEQVGhY(E;0lOKX|kxEC4w1j-C|2OBd&d8<+I~kw9r#DhQO)@Ta7H*=4DJ8qAc8 zApI{5vJ@wSQVrwgt9H&N4fm1Il zR}7ZU)WU5+Ey|MJ8C|6g^TJz@9Dz6-eZi&AV?K*1_U zL^4eTGK>re$O8C_((yuqnYcdcCLtXP5^>zW2u#ob`8#6GEBx+DMd(Z0#|+q|CAW^*`D)*!Et&Z>Nased}HhP%I@c0L$lIe$GVwRKTMVT@|Y z(_uKFzSGhXBvN6u0Hm`Kgw>##fTA)e;-^vjVV`8#rFh)H%nN$mg6in1bbDEDX{2xw z6xR@OJPNRrYnU$q4yzm%PzRyF3V;My@x0uYWSj_u*QB{V4%uMy$#L_gU>X}p4ZS`v z%Ad76N`T?2SUap>77eOL&t9uODMzi@tUz}8X)y7j_bc_{dxpbsYd>T=tP_sKqcxgk zd*e-%pqz>g;lwe^wg4Oj7D5*GFPktU@{Q%VfD<#kONrqV2yB)eNijB7)-6`>ec_@o zS4sZ9-8g}Ib&wM)_zPOqvDL1NQ>6~K%asUhHr4D5Ds{`lHx&mP_zP}T%4Ew!1Jt)G zz%TxJ;J~RQ1svf30Six3b%>W?N#sOAXacL1DntbeHnh07N$_eE4Cd{(J8=h{@QvjPi>ZM=9;INWJl_@PMwuq#g;f=dF; zd1c|ih9)_c#$PDVdUQxH#UwbZpy5>NG5O^S@#ryjuVQ5bb`7n~Eb!u%=Dtq*!@=cH zXzqsR_&WD@^}M=m|4o12QFu1QGSvf6TO^%aroBod`$A6)bSmgZ>rdHVV-sSD~&xV7BqKO;%wI2dl zlK(~L!s-5;xmYZm&!gGd{S0_DxMX8^;GP6l7R+8wb9rdz`0k|BTv1|`$BZ_>jGR8hfW$WLvv6Dqo9z22Y~VQv*d%_TTw~zP3w{5mI|TGW(|1CilmC z_=qeJbLpV)ODPOrNOf7m^@_qNTCf;U@KEdw*kOMegpF)j#Iay;v0U`X%-U!$+?_(S zY2rY>P!b0*l*f1<#wVJdTPHW8Ce~pPEnQ}HQdcvFZ?5Wk_i-~!bu+^tSm392&QR?hwVwVF$|7eO-(@sn&kawwL*lO=+g zrQ;7m6n0-=KM;<@U8&AhL))9V1 zsjQy;?Nyuy{k&47JQukpegh7wz{eaZn&&OEodHOB^4~RDZ#sbF}_qImE zV179zST%b|ga^_yXQ%PMBEyo8JlnG}ML(}05n%$(I!!zvL$6uhX+w9)R_^0v2jA@Y zrXb(GxNBkXjY7bVwciuNTRM7~KxhIceBLm8NDtc03}qBQ-@+bpw>FSA&sH25$p$f# zNhn=Gf>ZCVtPbmog*}egnuEfkrT}su=yNy(yboc}#>iLn1xU;?k#kb8OvDa=naivMe4n0B~V{*9MQF6xFLoX?2G;_%|~a%z|tSi z4%7$7(2F;aV2Lq62k3{1{^^5n-}}$X;<*xB{pT!obL!Vg8vHuR|Hk?LHH9XoE~dsV z|6`8)m-7u#2nh1O6czvLg8y)k$c+EN`3~u9P^zIN`WI698y0ZfN>v-?LUs*Omdu#E ziH0cRL!+V-kxxVvRF19aLdf?M^W5+o_`y^s!^OOE3H)StGvj&cb1Kc$)@r81XQNUT{6D}`9Kqfs_2qk$3Y0MjW`z&K9;=AsDD6@o}P*#k%s zJ=5lh6Qzqx5ShxRUJ}VBfl*F`QA6K=VQhbYz$$@FL_Qx(2FoedOi`)p*~B8=Lxsoj&6wICw*Yh@ z^9xm!tZPeG_qtHYW=!^X8!b-l)QhlEJSWUC1Rt8K^|LEBiSOgD*te1JJiDDd{;s?0 zYgN>f$0_i~Hg^7AFJ18ape`jgsTn={WAM~8agZ17S|UsARb&Z){aG&@Kg4| z-U&VjOmq(Snwk<=0fNcG%jAB3ILJVqIE)vgzu)arhqxsDpsB+7>8bt6Yi++DaGnS_ zC7KE;!JyOGtW9;{JO#}b5geM|u}#qE@tP!P0iET`=%SQ$-omysiOURGd0{ecMjOr> zQkfJe((ehShu^uC*z5UdOZ@GPU$67QWyhmFz22Wazt8jcXL|fx+;LfDEsPUh67FfK zHj@m_qF%nyN=={-(IM)c<_6dq^e|TKi66FV!Bh{ce{3%iXl#jSBc-+L-4hUnKOvIY zHb&Swxus#}3i8tBmfvh0NUDu9BKT-EG|ioUQH9M_o)8z)|{SrgHE*>H@-${ZAxQ7pF zuw+#4WhCc7?&pcabdINkX!4NZULF+VQbZZ1@)7H=hs3KSl#Ba;kCoa5@Bx~)$WA7V z*9+93$Vq8}rV#GB4SB$sxv;Q^i5w8ECN+cU+@=E3Ysy^DByK|xsh#`Z9-esr`TIR^ z0TE*pM^9K*L_$r0i~Z_=Sf&U<38<$TWq*nTrQIY=K;2Zk1R#03XM*hGfJ9HXN|R~~ zQ?Ak%U1AW5Y9TOB2MrUy`6SxCED{ck76GYT_$BknXMj^<&|r~jegfM=xS^W5FhM%(ili@DP{;yHi2p_9GRU|o;Y)*m!29HZ+fsord#Df1q_Dq0 zJ;vj|5M7^d6a-c`C2Vb<7Zw$L)$h+;G#15Y2An>d-;}X`h*R~EpFC?~5=+%)6vy&N z9zwK+^uwxAE8)+)9?%A3cd}P!YcZvoK7F4IG-34WM(u}*Xv&f5`jiHi0Gct=z8Pvq zAzy%ktQt_6S$0kZiK`7kubCGDh8a(OXt6GhWJBnPYP3yclz&GHOO9e=S%7X2Iec$R zM?6JxHC-4qP@{q4Rh2jGJ#E4lqo@sKVBS0|n` zmnQhduv;+yk7&1F@%UdiAb-hxD5Oy<|BH0|FSpNVYyU@Pt^aLV#xk^$lrGZRa87SC?DcH*I1>I}ZV#NKizC$Q@A#xZ@X7t$c3~v-h9OoQ)mJl@VxfJ7=@m z+w3Oi){Cih^?9e}AxCmNTpEz_2CL^V)L7Nn6^mO4dZZ|=;x+^BH0Jza@Qzeg7lN~3 zIAv}D@8K6o{MakLdxd&|c{cRd{kjZgVcWlN%MFUg$5hQJkSfM%t{Y@#Mt@`G5#$rD zK`f4t>jiyXlC(e)9pl>p3)wC>h1Uw#ZxEX$f?7`j!Z~{^6alt?vmSAi2a8@Inw83b^%w0ZCS<1{;XRWAg0~tL zU9=Wi>@84j_FkX$7O|Qg{p(%b6&}+%``1b=-pZ=|;f5<39+!_Bp2hvz@`7tMxMbH{ z;|X-Vx3;Y*95RT6mtA~00GIJ6C5PlZe9SEcVInkH?kMRt36EFOyPnmXVGuY7Y!>tH z#;=kf*-QAMDdOW?F*Bp#E&_nZ6yHSCs?@2ZDRh~~z9W3~54FNm!(IZzcR}M`tl+jS zk@G^wRoHFeVte3@JtJ>UhEl2T`Gh?IjcrjW>`(VM)TBnmh7X4aAh8s??YZN|+iYjh zh0m&9xVoVHWgI?qD{8Fs{HIan?67rELTQV+88n-ttt*UZe_C6_ofem#!`}zV(&yj( zeEe`&=Z@#+>3Dwnad~K8hwtm@|MB>z?Ux9vRYQy=j2beldY|#Wf@MUM$jd{<$Fr53 z7H+v8{Q4D(T_kH;>Q?V(OTOOedd>u#XnJa$I^RN1kMGlSN8(EBWdiN*{dO}IY`t$> ztqQn*WaFg@3&_0lEVBN*(pGUU*?Fm&O5FA0(t6EQV{m7?Ed)T{+%pYVdYh zRi3z3^#u`yXZ`@B#MO%Wn~-30=_f#*)ccM7Pk-!9{%%2F2`o?Xrvi4H7Y zFaHk&STC>lA6U$`q2N(UEkQ4ZvGny?*sxOUqf7Yv`O6dgv?)HBlQg22h&7<427wF-3bVWCopbm4KpAuHel zLLr3Y){5Y=E%M%icxO);(Wx?oW$g;rNdVHcBW1yaXxO8YQv-tQgbyFUO-V1|3WQ-M zZW}%z7m8`+Op)Dggd6eE@iL21QWE<|+x_O(l>@WQi8R)w`M|!hBDdT{25vL}k(d&Y z)<_QJH{ZkoC~TZkR|4=UA=V&B1=1BqKB%A!*`_d9WH^A}aSH`s4>zb2neKHy5}@nY z33~wY5va^%W>>n+E_I`+Mdd~)^eDtd3S?V8FtX4PQOBj$e8+LA#?gfHDEcyN9ey&f zO?x!DiI`1`rj6nJ02BObRg-9}ffUJq5KCoNP%4jC>IAmxaayrU18mt(z}G~9h^r$S z)LM`KW{#4b_-d8Oo=<@*ID$&G^}3=84X*5B z7hJJ1rqtN=m?=BKR$^T>Ep=N#A+Onw$N(cokfq8*kw{e{u}U}$l~_6^Jbaj+z;M?l zo-7JM5C$<$D-i7QR0|&Nw{&!St6SG-rgGPDXG%PnCv>_}S#XyWRU(vtnWB)Pkt9dWrU)l; zOe^r$LExnPLO}Byia<(ipJx$;08+m1pJOq5Y3ni3%!d>Dhmh=91yr4Vf@1hXD_ z6pR6N=NxDFo01S7<4Zw^-S-QhNs|>*4)L$s{*ql=vwuHw6XCH)WzGcb%FKp7F$wcy zg-X%{N6>+xvsPR^f^4K7hO#76%$1v!u|1Joc*tF)%o2vY6L}8VLJo=?gQt$_&-b&7 z(z8S~GDIEsWagH5x6W`m^>F#YGx>6Drm8Mr_c4GApl`>rL>dggGz!2x6LJn zVPy*XR9G_1%2-Txm_9=kzKG%82Nvs!0Vv32EvPqz&Wp}rIkm?wq{*1M79E!yt1$I4 znbTWBfzz86INBUw?EG7A23#zt!kpjkVUz&uK6Jy@6o`W^eJ#nk3!g63CFnb(tm7qa zi&v`{KM!GBxS&l4&E+G+-+Ay$5Q z|Jo&YshOK`3<1Lkcg{`?nIPtfLHcRF_rO#C%b_q+9=7_270ceWCjbCw4-f!00Ofx@ zB(S%$^Z5UQfk*)Wd(3nl|9c(w-*3FaU9$rgz2nvY-I=suC};yn6M{fUEFE@9=vY5J z3h_1fF(+VI@J8-oUqF1YoJ8*HPubbpvF>%wR(RZB*Dk%jy1KIA{?0Hqt2V9% z^{@YJ;BptbEJl{&I^g)nG12K@vbW!^wq$wh;WxMEmIE+3vs!ap5A0gM*|_D_-G8n@ zTz3dl<1p560`7+W9hu$f8p#Z!v)3e-#XC@G6c{yLxjZX`(DgljvDKA!8oo;501$?) zq1$$Bu1;^UQ9$5D z4g>ByhVMTF%Rap+a#hX0&DT3N5_FiJj)tbrjvfyjBpMpslZy0uSEU^XO%X;gF@V4!z@(c}yhPYw%M~Xmv}T?Z6u9KpWrwZt*ed?gYHa(l z#w`rb(FitMe+#x=2#ByQ$iUln8|D@vDcOs#aL>#v-l^Bce!;Q2;by;u52X1OmbQ-T zX$xZT`n253%izc5>)&2g79-Sl8YZ^*EXVy64hw5sa_g02un(Vexb^P<9_W2Bh`xy% zwI-jN2M^NH7XErbj>y^o{@QQqxzmq`8bt1^Ad?5lOqU+e>n|(e)7~1)?53banIM!n8&j zjh#(UvD&#BS)NCFs`~}X{H!kSJ;5<3vG;^C+Xl^kmq04dgzOduSMl3~_|V6n<>*Oy z+U3`XBDm zUgPSjHgz;MI!!B`9u1edb*Lq(L?uXYJ{)K8fordm~h6q!k zS0$AMc8uVeKzzV$u%g8qa8*2u%RYMs2!0nKiIg#0P$OQ_S}K$*`l8FVBNH7K{${V|qu%q*4ARgMSu33hhGSsmb zx`uF;sLbqG1Sb^rx} zv;@~P|0K75UtRjPDfbND#AZJ`Xtd3%%+3hn;sLEK__Y@1K@j)ExXSDp4b}4vCyw{v z9cYTLU9M zAN4xq32iwnUdL+-hr`3Vz5Ad4?c=w8d+^R*@BI4buO8X^=BeM__~7Snzy109 zr}sYo`nOMC-}~QOEOsD8We{olU%;J1%n|K;gVaKM*x z$1!EY=@KFNLYE@H5Y%@I)YEQjGp?vk{RYmlX`)~q(vsgp2$4=TG};6h_u;1I=2SZEanRtpgP_T%%wJID$?K_EYW z^XM-hKlbxi?`!b>`0oBczI))ePws;JnE>6Qw#F_(-2fU1YKky?@WEZLMUY4n^;}HF zZl)K)zIek9ddt;x`7kJkWl0eH|F}lm4#Kk+1pG#|CU^*{wOKo?o>)WZx2o8#!l2b=gV314P7aP{h<%p&def&~E11V5X8n5nR>TTe zHvDQKUT}yr{xS>&;pF<{o~0WO7M4AN8Ymzza6}vM)7kY}B(r9tN@*k^w`*0+g6CM? zsMi~WLn-%_&DQeT1=YpN78cjhO|MyJpKVnZppwN^Dv-cmk83pjO(5Ibu5u|<+YtF~ zT3s7jx-(m$b`dlZI}))ZSU{qGRU$ID!NoDP8VOI3hBcf5Qg1D-Hs^%Al>m=x1Wwp; ztE}iOo4L7d-{0;zb8}Q=Yt)^aBag1Z$XRem6#Z=t9sAm|N|AE@^4feRF)2ym7IBhHyVGfbM1uW@?bgY;K=DnD4-#-tROTsE zgVSw~dLfdN(^Ghs1lR>5)D`OBvpYA!WEE7NCO6nmL|=;Zocl^bPbg3I1{{{rz<>0^ zuit+}rC@}^!Aw~?A)_s4yXAInOj@_*XwY*G(Zo&D&V@*O3k;TS0UjRuAFC-$B`hoyn0_25pBtV36gD#;sbw|6y)FBx=8zhGfx=t*X<5Yy(Tvt z^_hdS6KNcdr>M2r+*n$@fZ9mtq2{q*>H@~W=Dm(+g^Mw=H8%&1&eB+fhz?1jDx`3f zs$Rz#lrl>y1ML!5hw=21`Q@ean297CLA65p6?S$lrx!TcMK|YHU?RM}zPKUEYwh~W zr?2fj{LC-!zP|U$Ls7#bQ$R=u8h0P9TD1AnT0gZ(aZ>vCZPw{GXbv!J(prfY7aBqV znStGDQJ4g#n@7WsqwBbYq3j4HwlKzbG>m7ss?j4$5-SlLorax9!wTjzYAf5*!1g?@ zaRFyT9f7q{E23UB7S;_jfbb#(kCfR6B<)|TXAaD>`Cq+DsrT~QrT=C1HgpG&M8Ab@ zFC+tmDl9`${d0_nl2}Or48pyJBHS_w?;2q=s{R0ql5-L{CatCOTdNl@FIQ#N2pAEs z1CrztpxDem8!n4L#dh#_W@x7|(?r3n4WwYxMI$;Rhj{~vVL%2zAelurf||lQ#}qY* zR<$wK(#eZ_z$Q;DYdN$s`W+iWVdx`56_a?R;n3)(CWFOSa6+|4lWb(S4_DDez$wJH zPT-1tt+O@?MU=9d;DA}9-d3luy-$TfY)N9>4$X`)vQ~2MWEeyGMALg!i9?q7q!lxUs|pbpjn9XDS#c5rvvr)sgeB)U64)_wL>kJ|6L^}|Ivt^ma+#D|PX`P2_H5yh`0X~Mr=69=@2XfNMBEBM&i#HB@ z4`S-Lwq|nTh#bj`L{~lWVw@&cjeKm1i)%IdM=KUePJ@$AYWnAc^s2nLjpjRBIOnfJ ztrc7*KKZPi5-7WqPhtfSbux>ecp3pb@r1k*wETS<1EdrLsJBgb0ROh2DKyK6-3oSY zvJ=lSHZD@z2NwL;8+@4i=AI zZEUPtrBiIhX7Kq6Uurl=_mq;!_Y-`LYg=XrASmU zOCD#~2iZ)3T~8J?vC}CKnqV7RTg%clcEnM*brNm$oO5b*-ed(LzM+mH_pj$4R<%)^ z$q#agt$2P{YZuDY+Duq>>i3a-i;w_9g`T6a@n$gC0_f6G#L7k8L$&*gb)0Il$abq* zXq9gzftW*%n4!Y9hx3t#O<2&ihf5wfX1K}18CEmgy1Lj(WK3C`w3mzL5$8d_r+9pk zP`s#r0d5`>v(;_<&XW$(B?E;h<0?8qwPXzyOjg-EplJ!(JPJQlc%JrKoRh(jirb!e z8dDYP9TvCY+?+NnsX2&20n^Bp(MTHWKyoxDMXzbo%Th!N5|db|?Fg%&P9exOT^D=7 zXaIX>P{ol5VWgsHZc@gKg<}UYoU3$h3Ef4e|HShOXZP|iKZ&x`X zKkbxweI@{X^D4_gIIy+I`9_CA*sMAJA7m;A2{fS`Gq=V}#Xg%2Rd9kv%Qi88!&Xgi zm3`Qn{I)D@jrL`aKqvNRmBuqu%x(b@e~tB88>A^bluK23tq`a^Zx$L~j^hT z7@-j*{6U&_W7vY#+8;2#{y=dnGnASOYH)|s3FxMYQlc%H7i>|%A2ccb7fp)57%vC= z=##BW5(Kf%Fz8DX2qz1gl16Hw=MK_ai14NatRza|+u%;{=&OBWedYiR5^>s>JJL9U zbdF${LP;&kR9cCCV~)3Gf~pe>f*Kw7Su+ARGM0@SGRxpLK-Cd+7H-jdD%eJqx^=T? zkvR1x#9N^tQ+I0&qTcMAco=2UcF&Ejh4wH^C_!!&*xs5C)4tIS#$j~^^_dAA7;0Uj zIY?_-&WX2fu1}@`t25**r2@WFkwSVolo6mh!puTv-{_>GgY0+awhs zW|YIIK-CFsc2^<2*7}Ok8k(FG4@_s7t}}oWK}EYx&S*7Gr4db1r$qshd?DM3J_jR| zo)tzO2)g*4ABTZurY$^K(&ah3B__V*dkLN1HtUYajB^XRz?nrCMRgAV6 z3M1hYd<~L0CX4^ISfQNpIjcr;5H2$SYml->;;dC@Y~GeyP_)iTf|E<2LSRvijTvp{sqOuq}5-IU2Fx~(W72=A?R zH?svzC~^c{U^-n~wx#D185S}b<@8{hOSX7vnky%} ztSX~-R!XhJ3Y6=nO*k>Vwm~3FjZmaXyJFzzk3u{{-3kSLs-PBlI8j^3(~B76K5Q^9 zJnYBxEO!!Q54VLJ>N8VNJ5;cJ46w*0=^U0EH@vDc<8)A!OpdC_{G?81Dy0%%q?!s= zs*0)_Nkt9)O03Kzx(Y)|4ivYlr9(Mm*yxP1re#8!b!!p@oUzJG$j~{iTXewgTE`hc z7b)gyPOOWQ=}{4_oT{#&FvwtV?eVG>tDtL|6l}#TIH3n)0?9l%rS|Dd^tC*GK@SSE zie-~rVroU^Ps$v+XL^L598hZx{zXwx`krviuw2qN%RDAs*0^um{l1-=v*m<#G3R1J zClJ|{K{uGax;@`3jV}gNaLQg_?ubR@3Y{G)Jc?1XW(C+VoyR0K3X4|J!eCM`F^ zVx?A0!Aj{dtfIx~-~ODYxhyy6tG?|uzG}*tbPjT zo}kI;0RMXRl(LvCq@;$-3V&O{JX0~SxEA)|BOMv-(gv;_+}4@IDzgBy3O9|ma7q)$ z7ENmmaJObg%B~AW{S=Ipp2g_qsK0sL$R6UH%0N?Rcm+6S`!OVNw0}(Qdb8W@QGbN{ zUMbi&yXC-faNPGuzaWf;-%p%9pPAuur27V6qvQ)JC=_u=qRe6>?8p+zG=_kGM>s9BE;~g2p)~6 z5U8RNOi+ctQ>c;){@nA6u9HX0{xPV>o`dbU<90*A;;p7K#Yv|zRU}r@d63Tq$s(p>w9gR!WHFL*yE0<>xt!tnMqu`#H6r|q!n5NKwU9n6I7ELlYop3 z|8uEgGs}W&^E$|^ReG~jxgZ+~TFQpOj1Btd$NMLpbw%~lqWqDniNk5<`rsHe{mXW5 z*h$Wz#G>%PcfAZv1CJsnm#w8JFjHV=BP+rL*YW5g!SP!!i>NYth`vAUv|yx;);4we zIFb&~JMQ%k&Q3f=*C^ zep7rMx7&^67d4ZTp4~}vvo$r*1c|5>)O_*5j^Qk5)in-jb?`w$3y=+m7Aq$_7((i5A1m5=|Dts^dkD8Es}D20>*a zmjwW6yb2y?8*OFP6xk_Wm)0%>EK-dG4+5n1g}hM(APk(2zI83yT+x;=@otf^cYsYn&%qbRc0!t@-U#|w zXYdDWjM};-cuUj;`5E-z8czU_Ct{-bNVb*Q7MO$`iR}cidzrh}%u{=Oe_X5Q`vW?* z0f%{Rj&junvTZm}i}wsSH{jyU_4#IVabxw$tI_ehYIM?x=blwjfab^ zbIc15^;7b0ZOr*-Di-ZC=OTsgn>ruP=Yif$1xMi&B1N1nEJmDf~-8%gDS(C1~aNSk!e7pA6mM7sLa z@3jKaan+`}Bhq_wyh1&0(#Gt!Ta*_<*%lg1OTe?JmjXAAc{gLvzG)X)qQ{Ypcv}*; zjRZ}{HE{izgo9ySc@1{dbS6-BXQ$dgD68PLsYj^dsh3nhLSRc$N^lQQ>E={yV3TWc zIa?0PF&66b@^QuE5e!3+ycEpC6|TEg?JiwbjN4sLS4m>dZ|Z=o+~Pv z9oUPmNXWcSA~`m2wLFRGxO5~+Vw9LIuyuQg&USH$4XA?AzLHw0sV1%BhK@*ljjAF^ z)8hONsye1Fsp3>)uO!m>V2lBUnNSTcaZ#&-L($xEL_CS#j&K9EWA>2VNx=A@$mDBo z&_E?k)&^V&ZHJjA3klTBPjRcpCLV$3DvAOBxCXOI6%1RV&Q9!ZIW38?P^&TxiJqH( zV@E2UbF8F2uAmHgr!sUHaoE11`$vpl^;fpN)EY1c@3^?sDQ`57CJnSmc9u%O_>T6I z_L=kb<;Ncb$*XJ4#W_+Yv7SoHtL+gesTaZ}(RhtCKYRh+cC-i!?)3mRUbiI`7W>ok zJ#zF>8wX29L*~D-`9e zj;@El7$1)MD9vahHaj~TeRmjjV)aXhA3F=BX zIO?=v`IEk@OTIgBYZTp%FXe=60@h{GhWx09b?zt>tpYOV*G|Af&{BGMgDW4H!7z}6 z0^^hBVrgATQAw8HH%aPVa|n~CL~>#X!D+edO7T487_$m)^}yoc|#F{ zr4Ss~;N8$|aIwJo4|lLplU0Do-FqiNJw~tO=>mOwvF7l1kpFfv2WF=CRnMBYegbtzwJmsinSyaO>0Z#=(UFqL=wl3;nW`wLM9Oj**zFy zOonXWx9q?A0kSeJ9APcj4H3%xJ5~QaRKp)btk^ z^@=?xF_TV#8BNzyXM1TA_c&Vo>6%66Ddy2JW4cV1dG|n=(yhKMtDdJoDc3r zumB`pwo2Z`X7oY*imXO84N8q}Zc8$%X6?!Q+%(}ZGdV+~0XPZ>DIkVW*!T-)LJa{% z88rr&b)!K*k%i+xdR_gpx!J)89x&Uo+t*eT#E3{xS|})6o#(s2dDBNF`*$fbZ@xk9 zCIG|Ft|fZ~QWbTHOg%j8{y$wol3gMRCgBaCd=mXYCaO-rFrr>SC5m-{ zIP&9CWnHvi;If7275w%OY3n)>Qedi)(jkPxgua}r=$4Ca4vx zR5fPfT16D_wU_ey*&1S03`#b&L(oeVQ+UAR#FQ*`A+eb#jiG7;6_@3xCP%NBeu|yy z%2$I08U1l3l_wuqg6AAt;!y&Ow%&3};>9cT$tI;liUI2!+A^%&8lLtbbF&?Nc9#xVy1PS`^ zxxG8zWwL(#_DOg?5NP5z0eSE6Z;yVt_t-PPJo?Vw{a@^T_&AV`gueFmFOPggn4tJ? zKRmg2&s$tDTD^o)d-p!ViF@~a@$*+7Ldi%>!r&x4_}p96lg9gmocqtyK|}c@>O6}V zFIM>e^;AC+CI0cZzdZ26-Vg8p`sl|j6?Q(kWkGD1d;cfDy!`sl-+sY~drv$p%76bP zx4{&=S?s~Z-A@&rQ%Vu1DH_NO)o1Y1ZW&AS4BWRRgo_lTYle_os`Rv!vzLkm=H@Ot#K#$EjCnJ0@rZs;2 z{@1790%S}Ir)IH=-lUuKF|$WfZ4Sd8w7jh_4r2T`s)3^MxDPR}77of)H%Gcmv2ksq z-t4BzqlpwL_udmzKVznis1RjB?UG@Jq|&Lzj<7Q>z^lxby3A#hQ6vPOWLJq*)>G4)Bl7fy(xmc4h}czA(=YMjz!=SqzX7Mf9A}0WFrtvJ z&B=G*ln9eP<#=TjQmIEInYveC%9Y$%3Hud$Hmrs5VFL z{VrOWfg4#{DtBh=LrSsdCZqcLB^HSnypQU7t!>Anf;E9-&Wg~ss?k?uAn2J?eFk6txgM6y=VG_c=+x)_9e=#`-V;{96^iJAdJ`%$Py{qUZQ}$s z7=R-LHDfsB9-K+U3Rv^NLy-u4zH|XlHudtEdzx6vXA<}U!kVY2%7~BgM#bi^#Wp&tr z>ZvIdqKGnSj52GQKqZ~@V-iZCT8?qaq!#-?!7&MP3k^&MIXnr8Rq;4V%^y%=(1K-S zx=Mq`du^c5xQLOQK(1q{6_dRPJ}G<0ptcW-n$$Ygcw z0i?L8wbY|CGCe5|y+S=6-> z#$m!ul{V-%YS(x#jNuh^zt9^p5JWJO)=e{E85AmaE3k0Rjd3x6muKWmGbZ47BG51vSPw5r3Vmren1_735bOAa6TfqMtP|i08TNf4{vqS1U#wqoH8_!-nj>L7 z(1q=|Yw*bMd~^X{h`+ipl+~=O3@zk$F4v%tIE;*m zts{@*NlK*3RCiYQkf=m$)^tgZjIh;6-*f4k@4)sa`cULR{5!uA9IdTe@KLJLI=*`a zAMr~0wKkmPi|f{g(;JmSw&`y7eHXYAU&yc&XnvQCtfr zHJlz@t_1MqPulud7+)SoCz$BOBgcdb$F3G66vM-BMAu`_yc@vL-V$i>ZE9^&E2KIn zmr$Kx`nn@FH~p_c`_`af+A1WPrnn+S@4mA}ML+*nt$&1SMeX%h=$MX6cuWH(1MtLB zN?=y=&H^-y(Q|yGb21mk4#|PJ!0GpM&*{XwNzR{=Rw$dTWrJY&=X_xi6+AbTrv2x_ zHYHU4@`P?hgr0k-gpTv3&c%r%-mO}<_^Y~Rb?o)L6SSPXB(ZMEuf?Q+7s%~L9{7-I zv4&G|pA(8*X*aG+F_g*h!U<#BZo?7x9*ZxfdMF~TTliV3+B(k|)=N(r=7-yK?+W~1 zv2UEdX3_m|bqoI&x6}L3C>jWhsT30kM9B-(r%vFNS6Agu)gq7!LbbB)`{NLhdRIoq zp7D$n?Ay7XchbSkx+tD~yn|CaYf>a`^@O68Zyn#rWn-O4`^hJb?cdc*V0L~op1a125l*Dwx{(7nIxmPTzsh+l zc*hpp0$R7=qf|xM_*u8$qf{m6*=~OXp7fL*oT4r%jVRM{NEzXYhleo510E}I*AoJF zr#e^pU*H)6*TP-$Yq?nm980fT@KLJLrr!+*y%F4YBEOZY-n2c~9FCu*s?Bc!8u*A+ z!ZYk@1(7I%VI41>ro~M_godjCN8}99mr~Z^5T-i?0!Sw`5|UW3&mNGIXneix8%1{%D8zQKFJK9<;K57cx(k+AN*Ht%5XmH&V?}>|CJlM=NEkJ zHf=vBx2nfp!YV`=#kcdy=s`LAS+?czZw%DaPN|J+L8s@S_i`yPSI%Za*P#AUs?r5R zI0J)j7kw$|e*rQVE)ALYF!Jk_i~MdlxJ$qhej9KaODNH-P_js1C2+38icZ~X zgD?YAi2!Y3fK`ynKmLximg`IV?iY9oR^5Wnl7l)S+-%boh8GXuQW{V-`uWea9olx6 zh}H8iUR|Kmk*o77yw#eXfLq$r8 z4KqcGa+s-518702;-rYU3Kz{Us}`Ax0EVT*f{G)<1cihwLIFjJ;;Lss6^9T(kzyPf zD8|0)6)g($s6yhGv!)_2ahg=59KtN9Dd0_%C_TSbh)&@}7wQV=HWw*}ZgL^TBe%Fn zLG*?THIZ<7i|HJCl7$!o$rmc-1chYIu4Ey`BM-6=N2ZHds5>Xee^KpLc^oDh|9C(g z?wY4!ZcF6TR?vl>W5f9{-pMSET3Y6~>N(obY7U@yb`!n0kOMn< zTUUw_v(eEQe852@<{+K#^^EZxFr*Lk=~Tw}Ns8tugIx9ia*7Y-mvfStRbxAOR1^{o z8rUQ0L--)T@WI?1T==sCM~bzl-i#SfgJ_JfK}l`vZicj(Ym%gFSh&>1xv}jGa0k06 zHQVN_#JRsz4Z$*{$QD@ilUnG%f${dSX$-A2a;oy^W-`&-xQ^Uo8FVe>%Nb+t$2m*m zR7wcn9m4g49y`WoMWSQe5v;Lqe~1&qgR|$ouDDmFuG8D#N7hL!1IU}1~@`Sj!0lqjmGcp)vYPUA7;l3{5E#!_n#dnw%T zKsVInT(319A(j^;T07n_H$`zChES1kdPaTPbRRs~G8{5($`6J3P1-ksPuNXe@(0F3 z;sIk;9^BQ!j9uvT;Lu?zJCiZ7ABIYRPO{F#j_o(d`YQPOP7tm-?)J`>KRl4eoLC42 zS~ZoabKM@qjqQWt=HS_YZ}5!Ed2WCFuv?ymgC`$$T}_kjgD04xu^eTcCgF$2zy+`p zFjdMAo@9CjzUZr&#fUx>vh85HL?1lK>l@S7jzb_CX+G_rM)Y|QMccW-4IElVo2n5! zTt=F1Z@P}%!IQiROBKWE5`7qC^D@UJCrz2~!yug&8X=6d1TWoAo%};$!5>^SRniZh z^-dJ0j@4-s( zbm=}6(hnfyaPUOK%44wXUOQ#F4~29-n~HrDHn`BAa!)-lvXNkNA){SW zu^3hI0nqC+-Ui)*ojWbNQ}TV#vWyjlJ7KJfWpRKZ)5*9l+MJ3Jv0ntY)f`Bq;Sg8= zKihF`ntB@?JmJfm&6P6I-O?n@6xI1Lgo?dC2QVin%~E;dd98T?o~ovyIw&LY@jdhq z7}yNq638-Cm!@gbMLUC3TFOPcI9(+v%gY2r!v#1~o-Q5&(Q?1;lqo+f@)P@+4}gtB zWM|plb%y`k)+pl6OhWnrF<@}On}&)y()uPK-j|5)fY)aZh3@!{K=n2iZbekvln%(8 zkMVgm!HPYV6i;jVJc1@(Tf82>na6PixR8DbwpsA{<}5CY`@SZHsVIdr0Nnm)NoXk~ zwye&w;d<#8y;F5@%lib!gY-*z^Ho5}BB2Q$UM}oh?785}u9{cga!EkQg{#LNi#TZZ z#M{wYxGUC#80|YoF`#4^@-($fzcicQwKyk*54q8}YSz6Hf5EP5s;F;8juLLo6;bpy zsm4*1HZ7hoLomkgTQ$YTU1lox?f0lMJ{d~aN-fx)=Z6+(3OtCeUIMW_`KFf@J&jb+ z-~8>2VBW}@l+-_B;`?yB*;;)%ZZ~l(82XHslQ4zscSM}#*@i`$)74aurwG1UX z_~>R-2^3I!f=QB$yeH{VuP^B$PVWQ9Mn$40W0kI$d1@CqY`GAY6?+HuqcvJO25GEl zpEw(#K7MVX-4U;brb-0ziwt^02W8Ij8*SUXR>s&OKQ~u?>pKXTRY`xhH+p-QvXy`; zl|R`#uH6QT;5%P&oCG*vLP94r(Lt0TLg>g`>5MAwZ%$P=!P< zKSO{wet{{sx4XBucYF4kCXj&mc4l^Vc6MfVwxtHc==pPt4hR50`auLbbzlXyobDia zb31;;oC&1NU9mctPJ*3&$ZZ2mC&>$m1GKT>Uk!DR7l3UekHs<*Tx^Bo2yc=MFG{3o zJLE&ExlYuZN?jnR&_!+IPI~iv>VU7hp#w*PwouDYU0Xk=v)(!NR3NS*AzQdrGJT55 z&+e)4Er7BPe|>e$SLD5!-p;=T!?%~pJsuINE_X~kazQk}rm^Vxi@bL(s|SxYi*{H= z^LL%SaER%=iqja_k7Sds*}S54)peI|u6XC2=lK_-J4qqMs|zeadJ~Taq>rGoUAlkd zCk8>3DKH&=_V|Y)Xzux9B7#cFhsPYFSzwY;0s{g+c^|x*^#uLm2DR3N|CI;fv}Xj16z8&GZy~iQ z9$K^}_9wEj)Lc-EX{e#Hoggvu z7+=NDYL_X~1QM@^KsYWo1Cmt?k|jHeur$nLi%@eRDo3YkjnF!dfSD@OWiSl&8P#kq zH&{MCo2#L0f+Jg^SrKkm(d;UNms2b$Z_BAClDzPix}8R!NyTE~Lq7~Fj{ji*^8}V` z0V8Wx7WgWeUhAfy{G=HQpI^G^2gAEh-o=ZuL9=#uWuRYhB#EvBvvY6(Q|AYExz`@J z7j9iw$*BrJDJeX#jU3Hns*~NJ4XI3GupIMAqeH+Yarxsw+l&D>K zM@j+ac4afSE$!T{Zs@kHrQ6j_-L|!LySlO4w${R>1Hr`Q!g1i*g6(p5par*z?MpYe zFTUn7LWT)j!8jQnBVL$h%bF=Z8&r9mlZ_q&NhoPM?Vcx{K@pA@j>mX)*>sppM>N2B zAajV2)^=|0$zs(4+v0@<4}dbXatyROMu2|`kBVkbu?fNU`0=;gVOQctM?6n@shG3YgK%`id`c%0yPS|po*l0`OF`K zc-f=jUO5AgzQsQ3WK!a1JKy{j79x zJ&{UjauubcdRrO6gt);x=iA2gGAnZ;4h9i(@PNxtWe|eV7*;Q3k*7c$iiG`ez#nL+ z5h_;(%*fhY$cfL19CH-os~OT>gke5QRnv4Sa{8&pQh<|ki+^@FeFuNBbSyUQFFIb{ z@agT$?j4^i_RcTFAQ_v}Tw>Um&$6yx!YS4j@4*MXtCHKoaePC#3lo>&1Pa)++Gilr zaCHGtnpqkXmk5cq6swZLov%&paQ%nO z;f86SY^8o<2NK~Yuk$6Nad^Kd5kra|DyQn`-g(TgSK5U6?jGp6uHiRh4ygJ$jDqJ$ zlbf!W0qpI0{5wk*Y)t*&(KMg|jh8xJd)Wdnw<;}msX`Xz)I=@zI(Yn=a z)z}8W>&M>hY$x=kVT90E;)H{_D!FJh>d#sTv6@-F6}Wp197JaP*kGwXskIJu5mIIp z*~W?)CsBm7T{*vMoRH(lB5q_&U-{MfN}u8b%24-O)dT`mauNaYG%Dn5K1B_8_!JfA z+HA!Bclcf~4$0|dVRe>mEeQ+Qkf8*Dx^y3e$yU>lH_KPfA@=c7m`*>28aRy)3wg;k z@bluhH@ks)S$11niGf(^?8UEQ15JtAQ9tO09=?C>#dxkz2qn*xoJ2BO7|;zg>KsKx z8ft)Iw5M#9T_L-o3Pmk&639in;C@jm^(DfqRbGG;PDZlO#XAHKMcyEQ|In{8`Kam- z@+p;v3|x(&r%=)Cjl8mzTOc`6QC9ME(0b8v?RJnHw}&Ix3+=(FW*F(sv`qF+R;sM% zQJ)1EddX!iev93F^+poJRlz;@50DzifnQqA;kQYp*7>a+T5;Q=>|1^O@WP1QAVDhWf zmj(~UGzjg93Z_qe5Mj;Quz7MOs%_L1P7la7ham=JrMoIhm`Xrk-91wPN>hR))aN7{ zV&LaD&YNtyu;qC>ki@w6T%|sJ9wY0^9aULEeR9rISgV*OEu-p!$&uut?7+wpc%fO9 zT?{l4Tap;^Q*g@BDD|!;8Os^T}^-Jm#fs_I`c&`Coo~>lkGxeFtusHBX1swtrCh?=O6P z^x~_(|Mc?F2WzrOZa7LY+}o_h87haNwA{HtHSe|6;;a~F?Tnq=90@W`)kzok;ui2d=yb4O3T ze)RbhK<@V+KmPUk_kQ{2JCEY+0DKQOD9sGWW~8A6Q_;f-W`LVoazsKUR=DG)@Q8_h z?+lhgNRmG;p+@01gemJ-Bw$hQpZ@gQkDqF7=1%J$KBCFM zDsMb3k>Ctx5*1qjeZh(kfcDp?1NbcjV$>N3V9`+gV6?!ebup}Fk=LhhyzIKI%^R3V z@v)|Y{rai`tYrJ^+r)&iX4qQEuaX+Nee~mdzdiHSFW)?0px(be^CsAbqaPkQ`sN!$ zNlhytIdibZ#rdi0!m!7why;Xz(PWq90xPiN2ph$V&l558<7H6_>r*{lO}E@rOMm_H zqoXIDJNohEqoLOa^~qvV}-;?&eULrokyM> ziC|?2z_SByMe1-tE2FG4SiUC1r!S?WL;9?ak`HqaR=k_;N3>bGJOSQ0WDp|Pd54ef z!?hh@QGh=2pl;E28(+0uxTurQmT;}w$w^pRNA>6nZvN2Fh0=gG5(6SYcK+ZppwfVJWxENaggTJ=IkW-B(%eARfy4_^l zJq4qXaM(NL-f%n&`yD)P8TG^QDc#ZZl!V{+Nbm-)4t(e^&}z3^msmU)#KX>X67>_n zwk;smcv)mY|FlFG^-u9qi}@bag(%KIx)Z&Rq(>3Bw9CovA9)RufC`MeLN zf5DjJ%uK;})7svV7e-i&J%NEROswj7JYR*hU^#lUJplVQx!0Ur3wmOHO(+D5p3iE* zmC9EQ6V6@RGHXBHYptWWqWq37l^)n}sz;jPOryZC82Yp~4ns|&0BC-F0_1<7HUU@F zx&%NV1c~9QM*wQoB6Z*4fy3dIB2$%}omvqmG&DBrsV5**D<=SztEP4kYMRxN<7UOK zuUv)HR+bI^P}R09v%=UVjbM_6O_2C9ji5G3N-bK+o%~^cZe-&wkEdg85T$VQ52Td9 zKX5RmoK+8~6cB`>W4aEk6x=enMl$cRdGLlAH z<-aZ4#<`mGZ@o$RK+T0|BL|T1y~{B9u3Gezkh#UgEiUj4ICVlUTWWyWCif*1a*|0f zp;@LRWSfr*V0Ai{jN1(k<6e z2UdUb1t4x5S1lh2_+pbd{_qCD49udJ?<R*F?!AwQ@{fPeql76Ia9&0`x{PupW*< zt*7I0IGprnxFM4rJhQ};&yqmGi_@#ha2xdu;+^z@q$p7je2`htFH#a~?vrS5L*76H(FdoMdINoT&H!hB?%O@(3 zGDijU!jLgXmaK&S&6K&IwNeVqZjC!3fA5lcZOw|6n7sBNX{}kslXQcAm@Fb8rcfv( zE;8iApG!rh5M8N8sV^|5=Pt9_i*O~&Wj=>-explf79i5OaymlRE_1`AV&A{^gujbp`u)J z{r1(gst-O1-7<%jPEgaMtD*O2!&Cf z+P&M4V>EhJZ!U#PlwB9Cb7-b}FYh$Jh{!x4z}Ag7Ty@&IUnd`WiZ$C1~=N^xWMe zMH>~G%?w&M=E~A|%rmv4Tl7U-54Xw(>uCS?WNu9%B znv=dh=n3zPf0uh^0BF`F*Y(9Uee3}VuNfCBXpTUzz?l(ZBk$_|Vi9W#ct5{xRcydB z`ec{&6D*a;7qkSnZIO)VC!K_szDj&;o{xg1V~v#RZgZ&MZEs&IUiZ0&>$|74zI*D{ zch8--uZ#L*uO*JlI9e@8MN-H+QT*nxf`ecVrU$`XOb?MmuyV_jfoUcs<3)p+XT64S zRp%$E75P3ct#p3`)qn>q^naqi+FVWbvdtjr0A#0O-+>>Hu0tnEP_vW38v7cM+1q*k zZQyJO4;)~m{+PamE_UHtG=)!5jtrdSaZ$O8QD`*lYUXu0?v(5Kre)0knC7gKCd>k> zY3lnucCcR{X)&1gJB#oAP^m4n4}`ELLdQCRbeEj+GWK>Tuzn$lof;%|n1rf63Z=A% zh2il1D$tn(%_dk1zpOl3S2r_z$8`qW7^nbX9 zHL~_R1GPFrc0fu^XK~wHh^R)o?3AMWtg@14$7yE6NB=C5Bg?Da_94qLHYM2-$un$D z;AKo87my)iP%$Wge&du=L#rj56!TFqqGxqSG*mI)h@QEO=-J{UdN%)vo|TU1nJbcI zM049*h%9|X&;AV~diGBk(KDxUH_wQ6qbx@BKRda^BYMu0Y>ujD3~o$zt_ZK$&Sp7lt7pK4!neS`EqF>8#5DT;~@iO@h}`t@*4%QLfW;*X^Hhquaf00jbPh*;I!4X!?tXh z-tv^#<_NL@@xBk>saV@OV~0>8I6Krxs6yZy)IFTOU#Ey-7iMWrvLHZWKotpp7BUe7 z5j;%?zSk0sq_0Uxj43BEp$$`n(N>t&#f9J9YsOeSEgEFG6PbR3#zwa@57 z$F0w91W-R$&jW<-)>nCTNgUKg0vdfqp9}j63+nY%K?TsZbw>-t$}C0ECe{kO$U(Cw zd(qfb?1`&lPl^?LVpHr%UB#Z*6uXukN~AtdEc!fg(C6KFI-cyAmOBRA1+&+ZU(HfY zRPC4b6fo+rCy-x#XgI=IN{mxC%2`DA)V&zTaIBQDyV>BI`S&BfMlTq0DQ+|lSXRMs z0{s(@;(mO0)D8MxJnn^KxYfVk=exAE5KJsW6h%npO9+xzGP`>Cp*g+;+j^}W8XZ;U@R+G z>Ro(Q2lR1^lD|@WXa`Bt1A7U>y568}zAF{7lbP-6^oEX;Eg|0^M>O?!Q`a0@@6MxI z|Bk!!CUFDFHud5=M02<}55eRfEgj!D+zj;qqNH5*h?&5*v??+Vo(Y7LFGych2eG@p z>nwJ4iM?DhnSxs5i6@=_gt991Rl5rg4lO%90CD{d%_mHjVRq)SJI$s{m}b(CN7ehJ zT$hcT-dIfB)g@Q3E@Bx2DW`S-rT#1|<1PHSgmQqP*L9a%d2^eEDT?IRU9TSw@1E=d zv?_0eSZ|9xj@!fZ{t)l>_4cWEvbnQuF~++F#Pvq_rV`OB0cF{;Dyc4{{^E|St0+lM zXDwx0>f|DpX4h-j?Ja|ZagR&Mf`d@-G7#zLE7Q`{<0&oWix9y-fB!4k=>Bxh6^c5$ zUo+u2zJ+kj_L=`Z&#_47>z<;?EwyUap73hr?s-zvk47}5k-TYh8psXrw!Jg)zxjIV z&Sc=3(8(T5yZPO$hBrblTV3W=oaKca`;|1E!j0ymjs3c$8jLjL*`0Ftf2`vEAAHc; zuLdi>k|?L!X&^(?;J@*--^1&VPVb2ToVm+ad%M^Ncqi6OBIfYK0OT?WSDV3oJldXe zc>5aBS*Ld@`O7+gf#Jw6HE7r1aRApHhY;}y*0RhQr{-(KNOOBedASdb-34%4%eF9R znHgecW@eNzW@ct)W@bC)n3&j-WRwc7p9Qdqd~iHn~jO- z+LbTHiL_;i* zRvM*48=+@lC9JO1r_Mcf@Y&+KgdzsOjd=smACYUg?S=I%_B_Yb7*FSgehyfPQ=e8D z%fRHX6qq$T658Cvx@ZaRB$_j;dTo}s^PQofqCO$IVVZYd04Y@iDb&ix4lf;n_`TB< z@v;uP<8CdjtORiMd6ru5ZN9IE5HJvL4`lkhc?p6vDxIJTI=r(5 zvH_VSrI#eMQ@;=5>@dL>g2}1tGO3#C)MQ=GX!#7jPy6YCB~lwx*IN@^DCERO-~Y{& zvrGl1Tm(qAxUjauUSCG}vYml3CuAvySX@xNEH_F%)JgLCgI0-5q z;ULLzoofV9XvlaPCP}`-BdsBNH5z3@9YW7i^tsbflxcopFJjV~k9I}B=*hBnf%B1F zhVT2FQA&=oJcZxBl@|R~y|k@hBekAHwVCN&5{kC70~-E{&%zoJjt%j^JHdLhtn=^oDDl`;jy zY{bQINrHHVSP{++yaOR*0$W}M?%CP+5i?zm72_is2^o={S{V?U?fCxRsChU}@w2?u z*E`K&hw=%(;nE;XISXpkqhA972ObE%YNlD#Y;rhnDI7PT(fX+(L7kbXb>Ax=jo#mG zhp>M&9Mlc!TT)(d0aFTwU5=kaj`4QAY`XjPh*P?UHR!r9>qZYs%Ogu^U83yrcq%@U zQoLMdXv$OlJp~^=&0YCt7QO;qfJO{vTPSemBpj~ccUFgtX z$Zd{3CFyD8Y%^Uhj!m#el5(yRxu+M9+Hy)Nf7a}xTkE!F?-%Mal%Xs^3Z+e#vK{Ms!# zuU!H4c6~*V{g(t}jBakbhqC@!qKxPAK}CoUJo9th?nkiqLRj$GS7YP`cLh39k2-=y zQg(yM@t92!$g`hi(7f1b%3UwD=d3F6RP7k$JbS~q$H5G$cYEYx=LA^JqOB9SAy#KWHhk zyYdW(%^1+6#KCS$JAUUe=sLhh#KAu?`OwEXJzr_89?3E)w{*6EBz>hf(^DKIWdvy+ z-ty%n?1{-Toe8Ti9x|fs<5f-ajua6XK5mb1qLEZ&m;{W>h>P_?+{_J*ZHQZDwWT#X z%yW@j{Y9~VPphl0@Jo3xFs5ac&Yz!KdA?;q<0vqp16kqfrbr_XCnU-2g&R)`vP&rk zvv8*+gih9M&5EztBz-Tjvph)^k~t%S7WI9zj1{8;1Srgr8prY&EWoh)@b zBeq&X6MZPxg$r(TwxZm|y>p0R&!a+VZ3D*Tl*={gLVWNsQ#7SJWuR_kja4bcvO}TR zw051_XQXnjREXJzh)zX^_Nf?>kW#rvONANl6kNUP;!~mEBM%yku2r@|>1mBs!{65y z8Wh?U5eucoi-KxY6UuF+cd|`oE3)02WI@xdz9o|2{L}&Ij4jOIlRkPa+wM~6Y6STe z6iUY>DcA~av(DM<~WKCfYF zo?Lx`L)(%yHPyK0Fn#f?=3Af1V6%|ZmD3!@xb(`8hjJWvw|*_pBp%HSsvI{8Z;u)3 z%g2h6PMo@3#)LXTH)e#+Gx1V-MGja9tr3f}zXRK{R5sWZN*Y&mzCD6P1778B zJY-p_6~UzJU=$!>=?;L8`e^@jNrC{Vq=)aR9}#9E?ZHR^f8SlP_~?obx(e?VhqQ#eKaLxr*RS z@z^Fu^l)m>LbohC51$(J$Y!J|#xwHKA{8|P%dt`r7wU9hXVgqP^CTfLc_s?@EXdU? zsL;|I^z>7V3S1Dn=Lc~#0kB3DdS(9 zMK<=F?0I^bWgBy4WXqLNY}VF4?HD^gt-1oU3*yZ}k)f_&TdZ>7lQ+O>u9J8&TZbJN z0=Iu&=6^BArD+h>Dyo>PYmQeL!TMj zMN8^_`!B?!Zd}O36Aw??_&lj*i(jFg1A)zRS)Y|IbG1VjVqWKanFbf7z(1jhd}Lf# zrjBw&!$MOF>8noicXQ9;om%Mm1sqHuj1NYxK<3Ff_9=)ggcVwm{@#Y(PG*AIhDtQ+ zeN{kO5prD68Xm~Ijal21^bp(K;@;g3M}+L-qd6TtyvIQ>9W8o3=CfzmrEFGitmc8K z9g&@r3@oXna|>@1_VY~3r3zT=V_Ub3R*ggn)K0-4U#Ao0?TJ=BuHTse*hD$sIo37q z({4(LCvnCoauq$gG5J!=MzF}5)Nz?%W#h&bpxOahVZ^gx?@}NWR&m5% zQ5#lOrAJ*_8pG$+!jD|NCPK58(&B2?q}+J9q#(!UMAyqdQTKwGE_|p6rs;12_tQVM zM^A6kZ4ArHNrQ)tr2%*H=1H^VNQKH*jS3dj0`0apL{1Jh6WK3)949R8ZEc4CgiU{j95NIr*0e*zAK)s=kKL~ODdDp;J56ho zal>%OhrA_lj5#b(uR~N`4%e6k{HU+s@V@D~S({GVjYg8Fd%u`D(M-P1_2mgjC?9--$DjyXq60)oh`|CC ziHG_HF{xSzT-RdRwrs{}&J#3~UGXkAm$ird%Aym@RAhr4b>+0Y{cBpOEBAFtJHbSl zTg$p(jf$&}odx28wrC*NTB0n2Iy}oQ(!-0@qc;SL8hoxn;X2^``*|&^f;{qI@G=4W z-QSH#zF!bxJ|PLg5}jkj8zLRD#9OLvz$|fMhNDH7L1b3d@lC1?Rug4GWL_1gwnROH zUgt~ZSnSiCB2wl{L=BpKR`p||)L+KTV`b`JOW zD*MmSNzkyNTf0ZdCt)MhU0J!7Ju~yeI_K(w?)upft}o`;GM%gZ8LxBT8#+NBl`g6d zXZKd|s!`SCg$B|OeXMIahExmWIBe?cYYOU6j%J!H8yU(GKJ9(`4oy>E3l+n>VEE%D zQxFUPi&xu7%?O@Cc9uxM6|fGxZmOeg=2cuG_fZAO<5f`;%yp=f;Pl% zb6au$Do^~ko5ky<5%7HlDMg>ZZtvHR#l)2VPkWBpMq4cPH_Gh7^+gz}qerhP_VL@#~lkg&}`P@t;d~ zx(wD@X+^T<<(wTgfVnueB#&LqrgVv+l2yvgv7 z7L{u`ZLyaoIkw+tKFA_>eewy;B}P|D(KG34SO}qLS&_EqwhVIQ!VGRF&k0LWc5KRs zN(?<4$F!m5x^I`t-&qPRUS#02giS6VmSNu+g^aQmSsK^R1cl^Oap*Ti-M>Kgv8dg@ zE710mX+pJ>PvG!?n0_77)$Oru??I0^ED)zo4K6v?nAFvS>8_5xJS!1zf zuSj^WSE@*_OBh=XRe(*>726n9FGS>ON{O#+-+}%|E7&E?kxP%_V6f#Pb6*Ej^D(XG z7skic_MpU?uI{q#yQ$IJ9F*l5_N-d+GFUSlJCICqWFj zAjw9Y0k95d;-a?_%ipyASa+=SeV(#$2JS-LsCCQ|_ekyvx!kQpg#%Y)+nl<@w4E{` zPLR`Jv-)-nKR|n=iH}Z|l)@lWYIK7OC6XMKNjUIqn9zM+Af_k7!4eD#rdE-`AtwlK zndsu_tSRLfgRv)m0fr0fOS!d__41T0Fa0Rrr|D0xX8IM!U_eLScIhE#tHe`Lj>6t7 zrpF{}6m1m3Nf0ibaEvmD?NhbV>g#mUswv|^8SQ}DxX(1Yk<+FzGe@e$7>3k>8O1a- z1s9>VhXkg`S&erK*w9^~YVn4-^W_G++rmi{0;9ALVL0$fd))2)5=%yLUbNdC&4O>?M(?@OxqEd5;&pzfw}{WGHjkgJB{Ce&=29AXuGBt( zfodXTtLUO}=r3X)xqXIj!9fh$40@fn3DzT;8AsKIE8HXkc!L^Iod@FOOHpyoT5Btft1ioXbVlR?SMX0(!tgyd54-DE<8p{4N^={5*}X zMjyWu_~CRu(1{QN9U^6jh**y6yboM^lSd?;#1moV_G7r3grAK8`TsDj7(T| z!D~vb@NeJy6a9f+S67dIkL#_3zi#$9JLS9EUhm(&`d<%R9@(FNdkd@|-3x$&_CVFJ zP2FpOP%_v?fr*AbN7&0Y!Ew;1tK&g(3^S^9{-R`nbLhn*Q7TDTU3OCk`7MCH9*I1_ z*T<#lvo8rg$R1?ICs$`6Da2mUF0G$p16o9(3-!@>&nBayWC4Y zzv;`)+8(l^)ihd^3D@?M zVb9YWX+dR&2R8UgMr@zT$2%9hB82J%!lu_*UDijZcD~lu?7n|-b@MclVZR;|w#8)$ z9^UlWH)A8^d%ZRIVH#_l0gC)y^}GxISA?eN_S?11_9I>22>U4ri`|3u-BS% z>@0zy_SXjbHu^MnZj=!ec}OswPXgF8YCA-A{DaHpTHFmS;Qn|0mxcB>g6rOwpSJwG zCRJ==zHoS+TgV0z4r^MKW?9{b=B1tfKn>%j_3ae+@^JLA?l=|6riS@wwk{9qPPe%* z&Nm%3rFpnwvfSBnO+&)BgXa=stn=~L&--BDBp<7&lFI_UFtkaH2!-E8P|-)`(Cv0D z7YFdI!mTg9@Lq|TgWZw_DbdxMW*oh=+S*xmFIw8?h&n}Kkd3-zPE%j0KL^EZ<^H;E zy~^G2eJ%V3MbKlsRjCh0XqBdb@Kd58C+X;8e=K$5vng+-fn&3tg96}=w2kxm);8$O#{M`iN`bif~&ZK&RfMelb>)}_>c3?2Gt zxuYpLxP27(@?v)N@QB>s6h1;#tE+Rldv)8(5*9M( zxkKXEtpcSW@PR8Pl*ojeDwDt z+d8tcW_*@4Bb-4KRs$(#dM;^`EnfMlh2ZDi#DiX16j{FF6^xy(buDFmvl{U+*O8`Y z6A3PkzGb#p!g&jhUj64&75Sct7DRB?-i}r!zf2ABQOm#3}g7&q8~&>Q#qY0fYZexpc@{A2Rsx) z1}swv1RRfB4IKI*TM0ryS<{+TuZVtbfV6|OJmN7Dp9kla+G4dk$>=Q9{cM18vVV$- z_!Qq(mADH!EShovYA*VQQL4s{$b)vuuce{4cfeljcq{c{$e<_c&6VoOc3K~$DHOIv zhI|X$lW>!_ykv|;9UO$-cp4CqIQLX-*c;rqBP8oB2-OE#t+*2;1k)lqDy1iFLo zwoE8g-IhOT1SZT>(Zj%zDK9?DtO7YpWc$NM#^p!(tghd8!xdS)7+2s1%@iXE@s6g} z{D%zLBcS%_6t=uu*`W=#vi~TpV4L)qkVdo%Y5;Fkx(xm;yGziR9h#Nv+T~&xOvb~~ z3~3s2M)8qy3)ndUckL8=Pm!1oxPl3nfGTS*Dcdj+C9@O(SlSl)N9dFWiKW_>Aguwg z^b@?`XAxy-_RL5eLFhLx43Lp=0|I)iDHAX47&_NTfa#L*bd_)`R!b%CQQzHEdzt%B z&2WbI-rdJHgo{gYCAd?jFOC=ZwMgcM^x%N8G=E1n=SneC%sS==#SpFzZuLBVmy+U> zXb7&&kxi<}t1}Ny)s8b!T@ZqMfd?1fR?ZGt`lNkM+>vHdZ{I{nYeMyni#SViDGuP` zXm%)V=465`U%gALa`wSNd*;IJWC`%3HR)fy2}qi8fYoruf|dv)SdR|ZR0?q=(cI%k z|ClxuOtYc-!w7lw>E%A`7%MFN>zL~Gj#em6RVpw47@Al)5s7J}GP=rO6Wm9dwgg;)K0;dp zgNo_^-F#{gKYy-i11GOVvH%nW82>t_0kTT3vElmO<3n4 z)#BONH8i>m^r=vB#WM4$cn*|iQ`;p~CMX~ncK2{gvSQ9t%3l1heg#%T=Le(3^v8y z^+^@-YEZkcldiK8Cp+62lJfJb+&i4x>T!@~LCU`XCEdf9JktQaT}YyNDR+>Rlb=Uy zHg?+fb}lNEu-c1OnTtifn;g|yf!#yAB^TTLpXN_2X@*helZSFZc(~qF^JHohFyIEieZO_*HSLEY7tZnC7f{oIe=WIZ`$jz^pE8 zzl7Rvy?~_8zzJvK>7WWyzlsq0o9agkvt;GX;=6mv^zgb{%Cz||6+J<13#hEjoZ*{Rz} z4dkv{K$#xT92@v`p2$+{8Ei*~I63&?CInHJ-^4(9@W>VwxwH`&qB2;~mVQRn2GBJ@P$8;D~5cD-DE7UOP zu#PlGm^&(SBd015b$RhodA4}S>NTaB&}9vX$a>l(5|&O1y7(Vj*|qQ2bJsbA;LZ_- z0h6dMt`EqetMbGB&CfiBu;CSw6XWa=-pqL|A4{mzI~1-I55x#r1ia)S2QHz%9^5gu zhdQXu#iQC8KSqMJTrhX4{#0GaM-Av(oSq#{-NumlVKD&f-?i7d?~0jU<%2{_+Kzme z6VyoOCeZ~!1Uc@jP|G)TB&}sy3e_u7lGO^LO)GnK2*|4|3=C1~J$)1mLb9-hV&5p) z<{47#>pYcmscJ;-j*xTxnohZ`w{wE@sSzGxl%+DcNDb8f$0m+|q&dn>mlP$yjnXS; zr3eKEiaQcuNdfgqsD~b;3`g1lLCf#EX{xS`VWwlwSW{V%ae8v~)V1)M&iZI>EoB~B zm1`H?V-K8xD&giG9rkroSQr%%PS7bfnlapM`l~0%Ow3QwP!C>h+_Zgb(xp-mwg{!X z`XH`q#KIC?=H@m?WNbbVQfe*v%@4HlH$G;wgwXy5l0)f zh?u7=@`HIu#*|&}r%{Xy<*CzsrkOOSv^p_Of1*X0wcwA~MZl6_N!MhxuGT;_KFmOJ z>%38+f1PU9+Y*M(b3!1B0OtyHX6Df=eU7wkKJyA12$rZUJ{mJ$B|CZ(nj!fLrrW@@W-j-Xvk*FvnW;n`tn#ZEvYnDZ?fcE!oHWKJWo3J8$@T ztmN*aff^JNwGq-(s-L)tA2I0z72~)b{29K-dBklFT9ilP)ZH^MQ1Yx8!w-@Y&1BZ2 ziPuo|P_HQH++L04*AD6u)@Nd&B_z`6Ntn=IH8qQbGy?5+VCBp^g?GZmBlzemO?C2_ z+*WAS)^XWkIax69(TsY@a0&@`iO!5UD_=`VTx)g-R@3a0fvT0-b zg+U~7d6drjk9;rcj;NY8RPK|O{24V3;VOGUzE>~Mm#{Ok^Dw{W8k1-iC(i4!U@_zQ zX%TGdBqEC`>y(0L3()WLCJH(95jxRtA2kgbOj0pJ7e4{jYiT<))oXF+pN8`SyAFvT zMEI&%mX>I6*1g!X6hE1GIyWQyjjdKmsZELqRf>~ z<++a844lkbo?use+=ncpabB=g*K>dNG(X19isezmUYFfnSv@QeyT8BO#)+@FhsmN& zUaQKeK$d1agaA&Fem+|f02&pMaCg1H*NM=RU>;D#vIO8$c=ASv$EwlKeSZa7IYg#8 zVgwf@+q7ppxX5#Trr&OKNs-L4iR=PJ?nD?H+Lp~fV zS!Zxe~DyqS$bDe z6vM?2C|fwnK+65Vz!QTUZKk*fq8C7;;kI5j%10PSRm__ZV<+k6(L8(1ip3weVSY(k zr1KE17bOnu(h#@8r>N(>%A7LU#=)VZtm7H z`EwXR4_`r;r$-&HXI1rGPbXgc*NeP}hn5)>q-^zLR?{7{VKLt#n02P9AT4q_?YnQa z+pK7-l#^2|VI3#JX7cjc>x))q>9U>4&3$~oW}1((JL5RqswqxGfg2A)1W(QvYERkZ zlx323!#{{UlRcI`j$@zUElTsleqoYcIfC99h=6T=-EPCJ_%`oxTtrH$`w_yNIBW%W zooZ2ki%fS15l_Ql=P^U4nPEF-sIs^WExRGZNgAw>_8~)}oW_TF0*71Svnn!pKDXB0 zcDpaoM2Fn@g4 zGTA}BEGeIgD}$`~De3Y<(I)sB4Qg}ST8{vfuN+>+888oBwL`B2(WNYR2X3Dg!=olzt&L!RxD0fiWH$BHQZ!;s{rs-{r(96cGBvhTItxiU z6Af+xUe230=sQOi{^0fmQ>il#Xo~NZMAGT82t7i%>I!yq+=Ril^b5}OA+>a*R)ZgK zBJq%|hWc9f!-Z_AC@rj?k%+fduc>N9a)xB+3@C}xxdvKr?DsCQoPy>3zTEHMRVY6Y z86<$u7t!vJ^IoMr)ALbp#-qb>D0uo1D<-ZpQ(n39pr6{b`8vsTWY90AbJM7<-RSzM zc@9`H1~l7J@Qhi`fFB;ol5~ALQo3Vz4tmg~Nmron<8)?CHdh)JZ8vt@&t`j2pSfwT4Yd_Gsb%NKqL~~9v-#_)p*u85`J=_b{1E-ZKT;~T_ zL#VUgn_$Lt6G(PmorQ7{SNn|P{i)h60#8E)wol7HLL(bg@q}2Nq>OiTb*DnnWH7Je zrn%fBz%V^SC#hiW9)d+ljR>yqpl)!N2A=yv%kBj^Exok59)_|}-VQ1fP|yE+$p-HNw?ex500}gjHS3y25g`>X6iPUP=Qsou{>iNwCJTS?_0oC~{9jmMk*m`tI=k zx4<0YqaRV*EqJJRWv_&Dr46c7fs2Q92QQOT{MR?&Pa`|f+`^!h#6c*-2Ny}T=9wvc+tw8x(^ywCR2hm0jwuq&iag^9J7G?N(jG;K20qacUC>SI z;A{ORJ>ZStsf(H{8=p8z_UQc{3}C0r*gbB?FCawA41+kvzi1rnL1p5JSxLM#%< zc0AFzEs4n@qxGT+P4iDwWUjGhKbq%^Eqd(2@;v_voKhTqNd@2314aelU-YZyZE6c> zXUhG$hN*T0&q@l83=#LY_`pF_ffNf?m&4c)e)1~+5~PC(ZSMUB_>96a$ioSp6}(F{ zEZrQvyH6B8rgIyuzftiTyoIn#d?uy4rxaLVE~SUu>_W>QN zRI8{8YXL=QLX8nW0Ure73AGqLs6?FVEjHn(b#Ed^Y*M#B z3on=#f^FR|n(O5GYL#tJIFgPiG46<(1Pb+QmVYS@t_HzZTNO)MoDz`tD_tmxHbMx% zRgKjYQ3#nIE#O>!BWFfcnLEnDfAK_@4Fy2L3eNw~NiNR32#Y-AZxOgEuU|e=5qwn_ z93Lf46Ui9>iz;C;)mbSfnkURAW!4&3+Pd%36yh%UEw(UG0v07&ASmbtf{~1B_!OY+ zEVQ*wemGIdos`sYn5>FnG9_b`YF8GWO%sp%V9kN=pp+~Z5=g}SLKI_k+y1c*loz5$ z#()?W>ft4ZP2L(bEWi^;hIXEi8NTQG1VcZys?uC>jmw*XCz<|!Vg1++Ll(G##a9f4 zPxFFFYH;M_V^yd#O{q-5FE(I;ncQX-@Z#{v6-|{!C?eAn!+T>aJ4sK(_U8HUd&u_ z=U5djjAUQeBKv+BYOp5YvMJ%%GAdpiSQVGCX@x{Y!fbi0^k?*Z{3s_Y(N&Qv1EFgfE5d^M6 zV!`br`*0UuTeV5OM&217D7A7O2Y97mK@Dk!+18u7cc2E}VYJU~-Dqof!+>46tXUvc z1OA>o>0{NhMB1=K&S?P+EZ5x(3OTi@)BK`GZ z!<4=Ued;hUu2N67J$sCS?1Y44*Jb5>^(@K!(3#G|M4048Z4KYki9EbW7m~sNMWR8<^&CRf_1nAH>tG}$#KxG z_a?|8(aO@#XXpBGc!MSLR9Ok$%)h0fTTENVdcFuNw+E@n`&LgOceKyLVB%CB8VPPJ zPwh;D-hYJFXpEkl13zxXhbcp*c8jT)ZL99~V-#-r{V-X{dlM4+KZnD`Y@fNl_k&XZ z>C{(Xk)zA3)e>YHi0%5clP0$!`FxZy#Zpl`vzlg{mvNab!aa_1q2jX&^d@VT$1+>t z<@tvJvU}yQ`vsEn8F^r@P~R9VgUksBq2=f36$*Nu=OWXlhz7W2xs*H%W(ZZ)c3lZ^ z>(3CfAzhQ{L#?coPvSZ8t#RT2_4vx3fNZ~}FRaomn~)*tK4&HppOX<2ZHI8+-%cMs z6sYYU+)Jq7F58PP%-x+<>ZB*6y2FJCEC}yhKqx-2hd$2T&}kk>>p!I?x_nk*S9EvN zgj%HD(^~aZ+u_b)U74uU1Rj3gXSDNPFak~KQB7Fk_DV=jnb%2|&JN>BtZ)o8b;P|g zrg+wuOP*1DC_l?%Y&!07#~j5;owGsN3%)-N1+0u3`hzJ*gMbPE&;bAQa}Y+Z{(tWL z=Ku%b`_Db(|D0#~y33{px_)-eFv0?e{?C&j04%^MU3k^f)+)$<{_6idxjLW&Kz`*~ zX8yV5^#Av&&;W=)SoP;kRbOjp0AQ3103ZY?{wDy@`&sz?_qSIybaFEE)VFtV);BRV zwlQ=vWiU4WUDA<=GGRFWHwiTWatDA)W0&B03gJA!dm~u{NLmLZY^>ESC{y;w!83d zRe=Qn@B#9_Srgy8*>7!a>|*>cL^8GV?DW49DFGm?P~_SFMf|UDzsrUTpwyS8SZS!h z0f1LH0Du&r`kQR>eSrKXGcj~A{J*8pH!f?Odh~BCO%DKJY3^|P)Azpx{av}@1*<-? z2(8floz9r0RWGeY+bDn58+DR@*eq=g%}xIWM;cQhsQ*{od%Y>SiD~>3_qTw*>qx4g z)C;<^O~l??IN-nPjRxgEbT; zTahxle%JXE{@J$hwEqy=8oF5g3lII#OW5eI>3eV6ejC<_KY4$N_qz>F8d_CDjt$Ni z0stU-x6uLge(QH8EPvXZEIss%4Q-5FZ4Ccsr2LB%U|yIqSh4b#DUJgG5`C#(`4{QG zNB`X&riH7{K>-JmM-2c_uK!sVO}YN*wzM;`bkaBVaIklJpKbn_0oPedjgnXXR(Ia( zVkrxNBt3^+j~aEG2V3m&EGO=ZS)VBot?gsp^LGFzMZ|(zYS?z z@5k?=6Mr)@13ZN6B;+s$a!0|@_fC`}Xn^xQ6PwoG|w&!CQ zF3*_y84&9e+~M(Qo006opjno4rcwbTFA z_+FFH$Rij3V*E?U--Rr>xazM~Y+Dze002ZcC;%5A`&+vQ1pGs2Z)fN6zm1+oHo?ko ne;qyK0FZ7oUB^Fde+E;KhIp@w007GSP2v50#PrMmJo Date: Tue, 5 Sep 2023 19:50:21 +0800 Subject: [PATCH 03/72] ADD file via upload --- ccsrc/transform-update.zip | Bin 0 -> 182640 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 ccsrc/transform-update.zip diff --git a/ccsrc/transform-update.zip b/ccsrc/transform-update.zip new file mode 100644 index 0000000000000000000000000000000000000000..9d43a3c4cd63a56dadf6f1bfe3920b14e7394fd8 GIT binary patch literal 182640 zcmagEQ>-vd(5<;`+qP}nwr#)Lwr$(CZQHhO@A=N;Wd7t_oau}1N+*@RSb4fvs#-xB z7z732e~srHQuY5g_}>%uf2*^Tq1`_-dna337Y7qVXH)wBZ-ZXpe**SEB@bk^6WbMUB16tvG~KnS_}jzsjbB%KuVvRMJ%u|ZizrKT-b zluKBHVL)218H90ktR>?^lnB4S=!WZqH3#fIxO^!}$j*NqTsJWhh@gTr+%UveNw)a0uFN?5e+Z zWtpke;$OPjcJ*rnb>-8$gL-ls)lc^dzGMus|L`WEH_DSE$Z5E6NfO+X+3%@PBfhXc zTb)EswnTWuXvtY~k-bmtYMQUWuq3F(136jwbxCBlUFboqO)FktM; zVChm<1pu>jcxw#s36NGJ|Tw`=(0d3t({#O2M(b z9G}`D!(EA1&_aJ66AcXaBVgD&PT=nJF~1~G?Y{A0UjC7O_xBKW$WhzPB&Y%Ip#a_F zLIdMx#)Ha%D!uH^EwK3Mx`PHlsekOM z`ous008Eeo0MP&6bvJZ!GW5{5clf7oVrpz-=wwP~Y&@c+Z;#E6{PP%|tE zynKPFK+B4lNfAoQe%}MpoSGUg)s>}7)oKMI;VJ9X!{z8OQ(&0|W*g9Y0hrcl0e{x+ z_h@*Ta$V*dLhRO|lh~!DtrqZ3dOCg2KJ)<_cFYx$M9#cVlHPXR8>DRuWG0v63#)TI zz#1%w%eqy%(x8RWv24NxPlp(k*@T(qfSf_cBrwMk4=jNlPzam#j9)tu)Ru5$GzG%E zRMwBeJnp2*=9Vk?T!VK$$j3knTfdaLMCA;`+~!chq+h!_SvT+h z5lq??q=U|Jt_9ES+(7LEV2h=ITszOe?EtF*=T8f+h})piZw*JbZgUb~U6aUqP5@?X zkjCL&BA)CWa(|ae_I%BK32aQo_})rK3Zmc9r4;tD_0> zq+VGf)#FqmC2k*R5j^N zu{f;;--=+Nxt3tOhzf|!se8>Nri(>T1w22XD?{)g+Tg+{9wKuJ!j_ejE4V=p$D(^# zp>4rAZ^vG*Tz?%JkcpdF3gnElZm-dbEmgX;VexpirkJELn@wVf$*lb^BO0=6yu_Jk zex&qZGd0M}JcPMEffB_+Go5Zir4m~DW%T55aG3}DtY2T?<59?`gAnDc;^EgG-=nP$ zkLPo6%l9u%pY1L0r?st#``eCPxq1zKWdj>~OV55#30r}14 zb8Ab6jXh;*r+q-jou3DD`&+FYBJ{WW0G{7Vp05~gNK&~2#*`QBZFIDPXCG?MI#0Oi z{O%U&{P(`Bu3P_C6j0t&gUB1eV$?}cG^r#AxH~%<;_Qgrw86VTk%smjl28k;Te-ya z1m1EV^3y*82NeJNeYvX%(+UiKXY1?b;eP22-mcfp;VbuTVrgJ?^*8FK4J3C?xCYWP zVa;=g&iNAwQwOQqj0_$q=FUr3D_Uf{Q)602Bmx9k0n6&JA{K%-K>dPD5A?+9Ax8Q{ ztcItrS)+o!iE^)--Tl%1{(Wz^N&YR@FCWu=uC=>H9Neu9!_n`LWIU|(y9Zq#&dx`^ zfaT-;1i~CkMXbFb9-M zA@JkplvbYt9Cb2zU&v4>zIQv$`F$g{?IRnkET!U*30^~^sp-K$FT=RUk&0a1K?9Q$ zRtZi+Fd6j-fT~Ir0dxRj1h+zAeXFDG&qjTZEl7ON z3%(bU4W?!@sP9J#+AaF?X-fnWiNXb(jmI@hjXdly_!N6)WlrYo%%JONf!UMW- ztNt(Y-wOd8;j!VI1ql7i$8+S>gf37+1c$dk(MZe|HvS+y)h}wO~aoJMrFLO)iC*W5^g8r(3Jw>9?_> zgs#!_ESNp3A)?-+_&b*#=*5GamIxwVnJOAdhUBo=po$(T9%fBi!;!`;5GkjT>UeBt znym%KGwLW9AWTx^zP13Xj}malkRl9p*t%0Pywgu ze+-^X;|!W&fsMxL28~bl!=aBCaO@~~Mctyu?PMoWK^1v0K$?ftXWxX567v#YGO{X-M!Abz}wT^Oi5E?Z1*&a=@r=F0x!sflhAf zEEOr`X_v zDcw*7?yU49w8dw|OU$G8tIgST*ZXcORSEHvU-l!7TwykRjP$nuMvCTL(R{2nc$scr z6Ag!gG!RL%Cc_#?QHkCWA7Kynq>EDM6Z9Ayqgt*A! zjP$VLCK4=kK1na#B9jTX4D1bx04hYp725Fnbvl<;h!&1bLzB|s z@rqJm=%#3~G+;?m;rt+r7xGmE7xGl?a7Z<6n!V7(byJpO=0=#lPoFPl-#>{>ZXUrF zF*eSpCz=yhln(2Z)DDx7qRVjU`Aq05i=D$|WGmT)`PoU*(W|*jmtuQ~Kx`EAN&!oG zO5~?K#m^IEJxh`AbBru<3`i6zjVoP5Npu1NkjC&5iT%;XP5%M@kFznd22mfEA~O>L z1^{3K0{~$BzdIXa8<&61rcVEdw+T@mu-g(t=y{?ByBuM%SWt6P+61VsW)s2Lh(8D8 zM5J~Sr4_nm>YADiG$f$NK8^?B0|8wugop$X{YQn=97ZI&low(0wXAEDt668C4zDvc5zlmQ$+1s_$XEpB!HVtb9Lz~(8y1pG zjU_lsOeF|U(T)lEpiac*EnH$0DBRNuX&tISSs?WinSc?2qr?P6$8sE52&O!54hias zuqXq3gb>Lhjd4I2Lbw!EC|9AxjMXLuyRZau0fNlL^Kup;;B4uc@3QKUec+%$9I}xa z-;g9@Il1wWS>xK)5)e8S?31fdBCyEKnY3RzTav;;8NFP3p`?Ji0c_IxdgarVwT84HM&~jJRqbmLt7CY zT0JLZ6EnscK{y>2n$4pn*fRki;R;LL{d@#BT`Cd)FL?2$RSxVXsRfI8K*)uGBJGf-wT6tut=MAQ# z1+sgl->S2B=iNL8#aHPc%tbC~K$q?zii#cCJ$MI&W61YNYN+M2i%itfsiuoam4)5s z$=308Ro!vn7;=75)i*kaS7@1JcYyo5?8;_Kp}Ewt$RR4{A4E~w+kyorI{Pw0b_Us| zOoMDuu%KMPs67+3#{BkA)J%9cI|}_0`bs;|r%$G8>66Q$?QQZ=T6Z&EGL;Rh?d`rT z*j9&ittb5>Y_Ur9neXFg*wM9`_}F34KL2mospB1^Xu%SQs`_h zZqC;$N=Mn=`|i*6qqrEu~#O?MBW=T7MI+y(E>?+g7<8D@8hQ6oh-LO%NT5v45g~n2(k%sw4VF0e2x@9vE!Y{>o(BzUE2hS?N!Kq8 z;tc)O2Yv{F#23p^(g^;mABf1LNx1N$0V5dXRH`@^iSj_hX~ai8Ou<=YnxvEg4nE~L zEFtsBS}0Bh9ekYeRJdX@RICdGUu7$%H8*hEF1^k5O}1Uvwxv?GNzs1D4!YGft0i<} z-FERpxSS1szeud96gW}489WXsgNh)N#|0^8x|Qt^CnOd_849@BBW#Og#*O%tsQnO5 zDi1}_zj17cWg#A-^bUY$_b9TW`Z=`LIibMv&SCGBC~Q$Z@q^Iz60_DsaP4prA>btc zCP6lo)|)iM+=T~Gn+SqDMyjpYlJCk;B7*R_{Vq5lzP*EUB*=J%!2rA@N~@@&&0dRp zr6=WDC-kkLx5*q&V`&r?3$MpBJDV5R`^VVH<8ChvUaYUT)7O#p`$MV8cw&*J>A=b&)Ze9rzm81 zf1zlBKW$>;X<>)Fi2oPI`!BgaRGzN3?tFIWK)(+LFHisPCr6ESW|b)=$y4mQ4Iiv* zE;YPZy;83w6}o&Z6d1wn}Sj zijWzM%X_gn{OJyQ#wp_!CRd;24d3Lf0n}vsYH3a{FNb_jCt5d4(Bi_&VhKNVhn6SY z(Gbg=P%4Zvbz&!-dP#naa-3#3OD~73*)&>Qa&Q7;pLxd_kdBspKd}@34OG>PL z`-Z4L`ynI?^r$S(Z1lpfJM&|G6v4xe-O))0hr)!NuA8+RGS(GH0J8T-z3vD6f7tzB z<)K;`Nq{g9008Cj|6bhxUv^zhot*zG4lHy3{6iK`?YpWwUr|NOXr#?=ZN0TAq8jra zMy*#~m$Zv2E*)JQ&lSpDn2C14iR1@!mc+d70Wm(1H^hPV8(&`~(ymA8aEM5PHSFlB z$CeOk?|kd@GV>H8-YC8;=l%o8yj`o?d-M81-Ra{Oc};@j<+UmQ0Y>9=-rwhE!R2x| zC-5L9hNq;uP(1|B%w7Ca9RX=S6>}Pt>c_xw6^fzGtRRsPlcoS`07F^KO+C8Dzq2H{ zqX@0I#Nu4`_7_1QnC=>ctqC{Mn2Q(mNVn(|Whw>YP)QzPXg)CmQ&e3y#!7ir~K08UM@ix3M74b3!_vT3q=7FULu zj?O*5=Ga{M*9-}93cS0j6lsYh{81Kd1<>R(Bq9ea=-2^2E;Au4!XCbDPcd(PA#opH zpjElLCn3XY2|E!QXn|B|iD{)j`ZzvC2H%bbC}7kmHh9BZ+(a&;XMcBbZ7D)>=la^a z@Z(DF(e|@k@mjk(Tf|%Zg>rFz%eA<%wXp-LKPKU|b~llgqbrUT0kjzCkqo~G<^*6; zNg+upII>zqfrpgHMLZ!x?Z{d&8o*k^Py9w*tj}MCf#o2w7|B?kS+0l-A;L{S>7vkL z62+$a5P~c!T+pZ7;!c$WaPx$(I2F%|Y=#s4itXenpcG@Fvu1!*cOo!oxtpmYIsfj|`m-eQmC=R){Grn{lmwhjYNUAwnUX@XT`auPZUMRx>g+ z{Ib8(=0dC)D);~tKH)exyA@?<3pSe(Ehpngc;;Y$5BGY4oi5xnUQy{L4a^kw`7p11 z6adBQW&&K6GM2{I0Q4o$Yj{q=C4PCKEVmUD=()6KNFZEsJ`I(?@j4M!7!U4NNog|P zib_dGZiJcA%5A_usF8L5l>}KT5$DFr?`K%?B0LY_qgopLTuT-TzyD% z^ertfPD3{*x3VR*v}T*FhwjEJq8F)*Fp}pe5;*}3sSYg7WtBFWx@QoIn|*JarxxJ5F2WdRIeUEdHl5<}#^ZBRwM=tvY+ zS8GYTo`WA#O^3%FM=se2WGExxY?z51vomm}EFO-^*#L+pe!)~ES-4s_o+KYcYZ-nV z4}2Xxd`vz^K6gr5ve>2N&HD5^?k|QnR~D19*wLjrfJ9UK(!MT**hI!B@Gj2zU2%M% zgBq7TWG+3wh9e*hY(V2E2ugKA z7pI4hkl1x7Jq+(KpHl;=mSBh+B?5;2aV&d*)4Al^|FpiNd3C_+*-8 zs6bsuLZY*oEeZZoDsIF#8fwT>s6QS}>|SrbU26pp5hMoGM(07Y&v?_bj4CPH6h()L zU{%MXrrONQa=APQIUo6Jc{*Q)w*7W!bhl{!#$Iwh!o10-iaChW7`7cZnlYSAXa-dx z)EHXM_k$%})%-snqSWzr-#r|CFDZSMqa5mQm0O%gnE>sxk!AacVQX-xut;$+TodX@ zR8Ws;-tdsANv3z6;g?v~()`n!5kl(%s7Eu>*rX?X(=l}XLjg&;WLds7f`tI>WgH}U zk+C}mK6PppCKNYf!i(&9C~C~u+4)T08-Zv5jIhTZ%MUg zvF|gItj2|W2x%4bdk5j?#^;jXnU+K^P=Mge>h{y1rVfHKq12GVkz5?Wtwvgeg*Pde zUsEn#y7uQn`Z%$(1cx3pqJR}&6SMQ{sSm|#fmqnZH5y{dF&~?ZI`KLeRp!^jo&w#_omeRR_ojg6`Za1glL`ukY8Q@BYk@qPO{btaEFZwn zqWlJ*Rj&~ZFG1;{!8oY%EunI&nbFyePdnID8&G$gB zcl?1bT|!O$A-fPt3p`0f2bJ6yQ}I2{?fN8W2bzOUi#ujAR?9=Dm#1Yy<+g%r#5dSL zev(|#yxBfx&j0l>t6r5gQbyr`jU9rx5HshAkPBY4hu=t@J;-Awkqp?R9Vn~bYiy!F zuD3PN_72Bl*^z=3% zWd`%o!Z*tjXxrdw_l4fhNB6hATZ1y$*u8Sa^q#*cUWhloSJ8g`8zG#eo zvpC9yWniJOyyGuoAXrWkHeD+T8_fd#Q`Qla^+*JCOlBPiQJj;9VQKjmMzG>=+jSs> zglR)>hHe_!oPn7%Z82s_4xL>tbC|Kb5sO%2Y1wQ6_~*xNy@+kR*ql56#uuTtccar> zT%Jv})}e=qFOIWLsnZ~gUQ8s0z&|1UK$HQ^O@xC=a8`_CTM8t@bnLOH+WNy9#_%7q zvN|FDEpQ1se0!Ekb<%(9PDqCVo&Bn*A*e zukq`!FLM&`?S{D{=1DjhKVKK$D2>gxyj!lRczH7Y2~?Xob&61H`dqA`h)L19n}zSo ztC}NMTRUHv@U2d6PDt(7QWG@1`5&*_S`><9j;1)rsr=4`YwM;c|9#nXYN~O)(E-2; zHLQv|aX|)3YG>u>WOpZIY(=6cHf!4Dh>GiU+M4_2eI0UnB7?h8X`tY~5s?%gl5S~5 z#I;L8zf2hv_~zmw9F#^hUUgriPWv31&=LB=V5x8MQ+`U=`L+?nLFr^Sqp{Yffm@~= z4`Tv;Lk?&_|8#i@5!O~o?XN<1){n_isVHiI~SLT=#dVqRg4GqwW3;6m*iK87*5VY;{h|aUN zdPODaLdhO97?(eqJ|MysZ_m*Vyz zqmfD16Rs&&>_0Bf!@=P86nEf=rew50$fI5^4Ks5b3#KNy-9*@YaERy98vxuKGrI`E zC!f6XB1QYn@z|e-!PPgt6~>N6wp^-ol%8si=izPBS38ykM!n2#O$8Y!M6a_kT9J4M z66_*S=pb~uPN7VnL#PSu-xDaj>0Ix~*op7y)u2B9#u-ihR)BK?Xr&V@14oT!KMrsk;zH5R2@f7ym%GvfO+K3TwifE zqC>juc61!#FWuWq5;Z^w;jL-6re-D)d^5z!*)}=@h1nV{*~1Dw_lrW*)2p9Z|4ywAEHBd*&7rN+?a=yY`Cis(&^x6wgR1Ckdj%Peru>Oj zyM};^DRXv%HhSQ(5JLhRNcb-!qrr6|p_ z*4qqdsspm1zaZdH0p)fK^D0(cts27Ds`q}>8vRYin*}}r2kN>_z&z4;(cRHOV{h${ zVQ(oFz(cquX6v=J@*U2L&kqrbvIam*l^co$b=2o$x`}y}Z}DEI2HKClRjY85b1@rJN_S1Md33|83(( zT^A8M5bOUaEF>Y?Rlvf+)(k?;oN@IHN>n{2&Y_C*24j3%VN@kxxfx1I#IrGjS7j7* z9IVtUQgh!8hNTx>S%jF@{E5nSPvG&=aX(rczwajX?xna&PqD(hn1=ReGy3W%}@lzIxHh(KP&AdU$MB z4OWL4=^MNxTtBZCZ-kVSb&A2%$k!qW@dk4VE{PL0uaUrgRZdi*B3~`3_ShiKgDQpM zvPu@^#3r4J>-)ZF^;iaqcHdvD9(<2`4Dsy`aAn_Q<(C~pWyxnZI?pxN<4Z)~I=&$# z-sIWns4^Uqbj!#fjr6-kwWFM?|BOH&18or?1@A*OP@eA6Ja9Q)*~R=XHTH+Ge zF~XrVu3Zl<6VUM-x%ju+f{ z&m>8>j$iXe=VPD$7LDioeElU1jjMa2QQ}qH2(wkm6Mr1cJ%BR}FmH9T?F-S(LC{9H$eW#;`;Qu&K7AGK64?pKU=fC?susrv?j06F6~*ZBFk zq)MC#{K$Fy#AzS0@<_ni)_fVNxLZhkD$;l+_T<)1S74KVo(zPI*sN1IM-8{9lD_&^ z5~E}InOEmG@J+xUN&8ea@_0=yu&I9H#g2E@m#sO+OKrEpOTb|?F8mAwDi7t!@s0R$C>Swf6e zDb>v{e}t-+gAYohr z(Y{yrc53!|f7Zoz*Bo+5VZ=yygt<)E5~XHIoaJsEJ5|gshQR_%7+%}4udX%8^!G0G zmn5=7P^tocEvE8vAS8GpOrEgK8DfGDswsQXY^)+sQOSJAd*oFk^Yuc)w=h#WNX=?VG0awvBU-*rq8_06tPP&$O9*SSUD*s9;38>A0y8 zZZo1loqscrY_Be_K%iUi{fbDf z0$oaOexrtk!}3&VQH#d-6Q1bE8%uLr0RO@eI&m}L!oe<4p%SfC+<-b2jTf!h)XhG% z@HHJs*JI2YWw!4D8c!GMb??^2L7mv1O6bkM5{%i=`-A`LNN3FKmhI30GqHw>Q&1k; z$pTd_x1jY|5jq_0*Td$EHrB@9M6?O!_CNl3oeZh;L4^+iuA@rs<^9yjfnkkrX4yc0 zM@0aW>&ztDR&ZdxjPdBzjU}Tc&2n$A8@;^_E)%)Yyx)M3L-5@&=i(7j_O4dY&?b9_ zF#_)Zy)Y1EmKyd3KHjUntOaGbJTevjr|fKaUOOZr6ykwpH}%0`BOKFuv&~TR4D5cj zp2sR*U5gJzix0{-_C16G?jx`Xuw?y%Izo>nODq}bid%5sO1igpQ>Oij6kNxnj|`@Z4Xa@0(x?m|yS zP5MNfuB@4TS

X#7O9xWnY_GP13dbN?Ga4&f@ZaPwOHP#ScUDxQ0 z&eQ+w?tb#BxN5x@_U7~(7F0L~(F_jN@4y*6NBm{Wt+CSvCTfgys0eWd8s-UrSE{^R?sm^`49Qvl zUFB8|Tiosbn7&-T_|&9gd~~Bo3Oe7@n>gL zP*d<*i>&f?6R`WV(X8_$!;xjj4o<>x+7hoy-tOoRP7anVM$!%Ql#QTAAg zMhm~i4kT%Rh(Z^u*OS^?RL9VF2Z&@{yX{-FRWIA;d=Sb7LZj zxnB@_YTcXs@11k1W68ZftM2lrX-*&Mjm zYjgto995}R98UxA$7)MVW#}kHhqm=kqxSH?>$0TciIycZj<+u`-@57i?407K5SMJQ z>UuM|Sk0x_OeSE5+ysO!tg1Tln=X`RmRu7Zt!7aXuFci#((Y<-Mc>OzdWy*4LdNb< z$G)rU1+8siiZsJO>Jezl{n2Ol?OqyJe^VEGuyDG0m@0~D8mi-tpxAyqzKreM=uWo% z7hq-=$~7+og>fvmEbyUX-lz9fJ1jZL0`)288EHYbeZYFaLqSJc;Dy`Tf+)eUJttN_ zFL}$@u5x=Vw#d$y167)gh)$%leOoW25R>iB1URm3=mEWXO|zrHhSoK5+!#KMBvi`% zAZI4c*_-_xRnTQ39q|u$JNFa}LZn|VYuD z8b-|d!VQ_9EbC448(Qq^!t~qPqtzAo)T4?+}SngRwOiIWOD{~?Z!d! zx1+tDvBejC@^AEVyJPcl62src{AX~kuG{x(aq@h*?jwQ|{K}!$-*(6M>(Z11WtL!! z!%TC?n5kitr}RQ|eR4en`+RxcZs{Q85Y$J6mJ8kfk(`!;cGo&WEK|<0==!z=s`wd z6%zSikTwnoHmbCw{lAU?PdHj%rzfTnutt0yPp%Wy+;B7sXM8h(bjaeZwoQZx9yXDT zR}Z}Ob%jz!|06$~%WEL)I~%xWDHUUfqU%6y022{Pmpu03?vg2Sfv-S9#sT&epxhP! z{{RWZ5W4U0Ek-Jc;`Mh>q6H3|z}DtcQM1*CMLB%yg%i6;-s>vRKGKBEa@TZo#qCO| z(_Jyr)v6{u=vw95g&SMogGSTsI4R!KOC^zJzdRs~{)`>>_mqoekN@N%vl$X>`n+eGFEIk=@bTka4T) zVqcEm*(K7ZW9|`V#4WS7JxUw@Z0(L#o{w z6PclK0ca#_&LahK7|^~27WRH81{A)%D43!~3oWQ@G9ZF596nYnM<#sqw;CqG1#dG| zWVp>c;0qv;wTRwU<4odYbrLmEIN(IYK&9JW4?KQhZv^bU)^NI+c4CLk51)F4ATrHE z&Ps};!Q%b;@#3NTzEL42Uuugp4Au>_-R?@;aMsZjB?B6_Cc7a!bn>ggfGveh<*JTv z%IeLH>Fg4~D?OLxtaj-)BHh`Bs{uQePDFPxTn0ztl8|k}Jgt||Hbl*H%w2E9jxVi9 ztWDhCq^7UO8A)6AmWNCAHFg)eP$EJr4_4?%^`D465p%s56t^O@*+ z&(2Q6$tM2PEEV;yZECu`q5L z6Azd$ts2-FkKx<;Hs{><`Y=3uBQZ4`AqN;~2-?>@Q3x7_NCD_uyg1zFFSamT93In= z5D8?{0*LH;6H23ljf6<~GkIwK4K5B&JjCTMTLh9d@R(8J&?;98sNMGgW8?PLSoY2N z;53Jg3J8`B8u#PBDgw@Vnv?M4MIXodvNV`*EnFLyC&nhRZFzBYRE}r)fD2p-shBf* z{7XNqjlv)mrG9$B2m!h?>h$n>n&Xg_)8$Q@D}$v%QVZ-P+Z2<3v|s6QobZOulNx5F zT#&%LKLpid++LnjJK8w>lTP7?t6?M8r$Y&2V*f8)Zw>oy1bc-OG85Gt9kY8fkPDN|C>wJE=Q=eu1uhbxO?WM->=0jQlTF$KQ( zrSS5YNy8}UsSv<`0y~?S0r!~4t1o4I(Ms!F2843sbxf{^oae8cWsYciesn9cT++pz zoc&VzzSwP5*JNwL^E8v0l5eW|Og1kW@qx{z(Ynve^79CMwWjijn=RarLxYLWE06%Ie8;yj_By{f>Mj5M60KPVs(oeuN$`WGBOs4)p~k-Wkg z^d3#fZS#;?Doyp{7@%#DvB98>O}z<3M@bN_y$uPQ<`bE+HC9gyk{<$E4(_Y*O|!dt zA&fySaaD*%iR*kumZc#4gi{GwB634gqd@jjU4B3dfeN^kL$|Rn{{<`}8LP z0h0h9$nbsb_It`<@YF|MX8Kz8J+=g=uFI4!pw9(o%cn1Jnra41O?V+Yj&2icK~D`$ zQmNy=7kNy7M%vs-7^m)&dzNKood997K4BH|d(9a}uAE+tL32KCcg)~#Zr45s{ZVs# zFWZRr{YgR7(!*)16SjRVr)%v&(L;Y^f9;QuIFVo?%u9L21BDSeiOF!Un7)fkG35w+ z1iU^O!k`5P9vc5O?s@nXE~wNmCZRTM$#gz6Qk3|#WQAu49a~mHB#BTu>Jg=u0++)@ zYbc|l-2H9y3wT9O=i_rgkis!2Y|%n$rM7|vDG+iWQNa<#31ixduelq4a42qI9Z@72 z9jGI}kE2CdInB+6-)IOUE*uV4r%BKMkg822k&bjM^liz2I?U(CaLm1>gQG_cdG!uk zyatyOcy~hXgn{9^OeY%IxsM12$)--#Ys8>=2ebw1 z5QH@UJ!njG;;sY*9&PCK@;VX~zM-r#@SMKUpcaRAE9wbSRm;w-$nH0crDOhD{R9+_ z6mJ)3x8{McyPpFrO)CSQ5dnS-ZZDxUl5Gq*2;8){Gdw+n6fhL%!^R!wsFh6bqy{T8dtIqoL;noj$11C~zHXuq z5f8|jw&u@Xk-9c)sNsse++1T_dz8@JvaJR9opb@!Wc(~lkBtL~+5nx#MViJb_sN+# z?RMdEHwVQonc1rEK1@3)QxYr(55I$lSu@Kr#7NDOa4eh8LJkW^FMdeQIbtW?(6K2d ztlMrVb`VLvH=lpL0`Mwda61-164Bv2yaq@_B7x&7zop{RqSzPkcyzhGyr@@RSC`@z zh-@1qiio7n&FQ%@p7|GIo1XHePQ4AAm<7XB2zUdKG60vxcEHp5Fv^j>u=40o{v1AnZ>uT%%R1APb zz38A~ZtFg^^+(K~w99Dd0T#ta--l15(R(|Fc;?yHQVHhRacJ(X5bg8QjQ88G$IN}~ zWz2oN@q0hQ4}Vkrz+%~YlFg`sI<)qgkc;?|^ ztykE>c7Vmm_0Jw4LJ>&r_uT{ffjwt(awUC-uib5^~<+J2EO`=^ZH~3uDbLnhH!PzZU*?l7-8-7Hl8TB3d zvuZ_r%#Kx!{Ag@q-o~`R4eM+Bri}laj6*)=N`Yd~u@+DRl|qtC-AB+iN7g_dm0SLN zI5>|NTf5WSPEXg(4hr%$ei`JVEEnq#t^yJb6*9T1WI)oMXt-}^T#T-G^mm!>60lUG z&8*k{#h)?`IAQ$w>O7Um6#7Rd9?_w|xYyj64A}TvjLhE@>uILQr5hrsC`10SD^yN6~_6s$L82s3%ee_X!*2^k6gRK+EBy&)qQ7H4n`&P zQ(Ifk)#b&tTMP62=9HY)ac+vZ%6{)dSd4zb`WKoxT3_lo#D(73hTY!?_3&O;1=jeu z_c`a&%ma8aQC;!elmL!}Q3cHI?lG=zYrz;FsM2uz$JFm)#tAEcG9= z?N99)J%V<-6sxh)Cu;adcP*@^hw7A{c+)p82<(&Kv}@lv{(IvP2!Wf52841XB>=k0tMg)%YP#8=iYKi zUZju_m)=K(E*{7UhM9Rx)Xf+ChJ%kcI$sg3T8R_v%mI~$cU8tvQJfx5#gPp6CBlNl zt=Td^(=sA&3`E&iKN;tFi$o)bXP42Hg%}#aD!cFK9Lli#w4v%1c{JVd*8z#|ojC7z zW6dX%)0dn!#$K;%Ys8I+&B7})n!_>3S9Eq3bx%gcp3NTYOA)|xiMXA2v$Zktf8n2l zLT-4!qm4D-@?1}kCKuF){}YGLQjb}(YZH2D^SEJ$Vb9>$Qox=~GR`P!w9q&MUfN+B zstY9uGhz$9l|qZk*-4^mzWpoT%u;WGRyejVv@|Ex{+a1WeVF;wbrcx_%o&kAy_3@! z%?ZEfPRi!iZM?o!L50Mq=f!U4*WMZXi;XP3K8+9z?sUAJiAhP&W)|E1SHsNVt14eY zqkB8uwvIJ0GH*AC!J1Wi8}obvd&68kB2g+9d3$rgHN&ytRJd)F1cOPAS-O;`qmlXk zqJ5?L9+R!*J#q+)Q^lNDw^MTCQ+X#MubhU$g_xU}_>^nKn6kQXmS4MuwRKXFR z7_tq0Qc0+TlcB{~B01?+7yW|DW_q6=GBRj8Xmrij}06VJw!KJwpBP^UA1oU?b-fR?T>_yzd?_{yvQ;&az}x6Exd_d|aCjX`#cYV%_a z{8cABW2`NoVRvrz_eTCjGt^cKww!o~oCm^M1dqzI#0iIV{UiW^?iYEvp!}}fEr#x+ z9JJuF-{3=ah?Br%LdW_>(KeV!ph*#3Ea4jj7isLZleF09oaEuI1);`5RV~8PxN)@@3oKcF6$@2D0s&Df*6s`h zMO>Ojzb5*Jd*T1Gj>A=?J+asEjLa8yhQGZH0E>^ew7hM(<-s=yB{r(r#G*{&nTCS4 zv)bF|ZF4SSnAu%^xtebh71l9nt)fk+9tumW0n|{JvFYm)g`L=NN^b4O$@A_Yzc)o; z?fyg?fiTaySelj)XHp7K9;Np;vy8iJvsu=?HEWwD=1g3_sJ{gr36Ej0DY=-}ni^g? zBG%!)8AxiF``YxXCOiqoU4FA);TwTG4_*^(4S7N5%-ly@_BC}=n~zr=qxxGYJ<^65 zupF<>3~m=MsVU$wj|GC}kipU5+NyON^?6U2_5h9TE!pD_?)?0$dXS1?0c+NwI+=p& zBYFEc4R+y5t3ohavydN!YT;b1frnDFJ!S`DT*)fxm5-yb$}#2uVcuQ=H~y=AvOIM2 z8doFhy5Q2-R9Ck>#VXPs=7&QCpYjpVd5{dm-~<6WxV8n^H>r^FgaXnxv>4=dNbg3C zWl#W}hu%9I`=+Mna@rXd!4Y?(l4Q-il!WgjIp?<5$6ypeTQ(o{F@kNdaqjjD*bbdmx++@4K%?!0*|LInFfyaE~&B}r`aM)Hv<6*TO8^?Larof zGsGo_Y$E}igt7_E3Fj3CwLOZu9{xkgaJGi*j46&lPT&T~o$r zUIw2h_?Zog&qc^T#>ih6^)1@G`SYm{`5OQOGTYUqm488mV-~DJXm*cP_JfyMdHad& z;iJ~@udQ2mgb%vQ{d|n0=Oj(gBlAv6GEwRI#4MTuG;@h$6}!LO`33yV78PdSxPFht za`DsDBj+*Iw#1yOLH_E}A0U4-Ax9?9)a1H3q->eso6K%p93RGDvV#|9$7gPTb>m{~ z{n>@LX5ruP1JZG9iqz*xJEe{jSx;)(HnZG{^ju8Se%(;VR-@P*$hfrE?(k*5QSnGe zQw}6%+B9>UV5sDBu8)`H$!-NT$|yMjT*&FydDu{QJaLz(R%ABNJ0VHLi)K0 zx|yQmnm!x>#QJTS_>o7l#%dNhmx~Mgc*orZORpCIf(Hp{ zS4r?_aUpucAvbh}jm~!S{yR58BbX>8n{JHP<{*u%PcC1iDXoldTyienIzm`Rxu*8; z6V`A=4B3eg3J6s?Sbct==Tr|u)9JCaY%>R}=hUuUFXOcy!d0nL2he_F#TIvJ-JqnF z?6W7=42&BefN(o48C5XlxU_O_`4lQ$RE}q!ke6}?mAYKGHGg^h-+&Sv`iCrBnlRbE zyj28bXnl0)eFH^HNi`%1axiyDGlCu`+SOZ%Gx&r-Q!T`D#k8J-yui@5HlU?9M>+=# zB0N3R*F~aF*H(vfM8K%=9=$^8CtVYFC)dR7vf%a={G}eQB?SSgm%Zr;%Afn2k7O1K zJ@yk1mH;p>T!#C?t!Pt#{%gj&Ltl;ZDVJ?NR3Gnr8z!bC(*38T*cKGoqgF8N}4Kw-OjB#{2>9g;6N3?tKj^# z6p_dJbRb8i1KrILYCmfN_r*W2-$xO_z1ltIMcE=*rjF##Kao8#&4>a*Q)NwT;f;-J z%o~K%K(X<{DI7mj*YI!LOa#T7!wu2GT{l8V)7jl^1Hc$R>M(J7uMafk+wFGPS-^ks zs?||M-6BO}rD0t;)^Db$^%!=Tux%!@v)s{pILk<4X$Bzm~T}jU=Q<9A$nnCRYH+OMNT&Ha|g{2Dpz&5?QUACqHi>{Oq+TjuuvRo zDEUa)-%nBWm$8>mR3NA?eaWZg`t4;Bm8O^1?^;vr@+_N&Z8*Pqxj24tWBgADG`hNk zOV_!Cz+1nb{n=kjMF{1-2Bx5-&Y)$*&1(%gPOC$*y_Y%=_JVETyRf2%_7c}?9I72z z$-X)C+wFSKJKf8dk=KjlU`Z}dV+qqk0-ziysbE0DS>#_)ir5iMKf4&o z#*)|sN~aMh5FjvOUbUV$+X#;YBPcWE)Tv6J+WD@|YB{$@X#t_GEbz_mqvwB3?v&E= zbqsQfE#(d^m-2BBIXadlE`-gV#NRJkDGmH)?IYD4ZZf<1o9jP* z`<`Mh6nJbK83AbE06YFjT4HV@#%*FJm3|*I)s#=O*xwr8B^UZgGoztLeb|@{-46_< z#Iww+96Y~Sr)=ik!?}H%%-s?f?II6J)3JqW7>D2;O!z-VTlD4A-OX~fdD-0yBYAIl zh!JWK01Au%h;-pM@B9J^;Q0qd%=EBwySu(C{a!faY4eHk3wbhhJy)CQC7U~kBWQ2; zi`>ulmOs{cV5OxBVrJsCc`StC>92Q+o!5bbF{_hMmGgQ#gIwNM`Y@O3co?yNEp@+; zbVFrn3Ql~qzXo8wXK~6(^avZhfj=5|R_;s!e|&loW0TKDp)K%kz*V8^R!X>rw_gG?A{}k)zp@? zYG3@;_>lSAfe2e)Eh84zXif(CUnqC>?(aFZuWv9rSstKrwq51yz*D;ZEu7DG?gpdH zUiNP5?l~3k^$>09UOCj-V-}!NaimO~uPl((YxWr`!%Us)3c!ispF!QYxUun5c{_2I z<1|^6o0?*DvdK<$@0UAl^Q{sZ2Gn2A6bfv8p=bb423kagH=F=IU7xyC-(O0g`2r4SrA&|F%{!;UlXQZnZ$QNbZ=2y zHx}eOT))3O_TJ~o#bH53F>H{LI;418<`Y?E)_#Ebg=>?OYaR(8$T3r5b1dF4Qs51L zGdrmVI5T|vx2vzq+qrY&GYEV`zyl`ga^vVkOWieP^|TmJ$MUJh3S}t$tr$WbR1~Gx zP?Bl^NR&6c`y@pa3E2)OCh3?*Xl3M6ZOZqM4n+ z{IPcV+h9MYLI+0;W~nx)1mOv3{TfMKcv*nC99ov2bnExGlEj;9tW}l5Gm!3Yw_@$I zvN3>xs}b=T#%1%U#^?LZKgu@`2c^p&m5QH@`CQlJ3fIOr#kyPIL-B;4hBs-npJCr zufYZ@AK4~Rj%&Sn&O5o5Tg8nt9FySkdoAhun(LFqU+!&NdjhlU9F9gyw-@+^p0DC- zwd&lW6?ttB_1kNWD6$}b2NpUFt~s(}=dPmS-t)2G&*5e+%4zK6F0Bd@N($;`PYDws zlsxCTs8vF=!s`_fj_K0x0y6~(6d222Uqm654%1*Ac)ZPRG#zLJ&aHjqC%__jaOD>c zgYX!SB{toLDmL1DujVC1Cj5+}S{Punv^{m~8%i0KhQ~^nRM_9mkNG=)d=ghsL#n_K zbLZCYa$*WV>F5$@=R!N2z{E7u8Pm!!|~l> zsms+k35jy4E_(=1t)o$a=;4=LSH?i(MA9X2%wh9GQKIadl0p1B3I!3%mL6UdNdv(5 z^z|i{67$Xf)Y66hQ1kYZyYib?ZM$SIB^6=}Ui)wn*XY|M#VTFVcR{h>Nlf)^XjbZL zJwnmf@hqqhPIvQfEP_P+r49(^1gUYd0CNKYz<0;U8X>AW?(tNtb-bRE<2@Wk&vA1; z)lk*Z-{rI&|57PDs&lXNR_j<-ZIAZ6wA;TA*ee<<&s#32 zIxu`OeWd}|q=2|hPdh9(=QsveVMPN~g-tL!S1%)(%EPdg?f-2wu8ZOEktjJ{DrK|E zr$nZz?@e@JAru8o;JfwSn?Eb?{j9CBmB&vI@y23UX}s`Y$-Y2KZm6Sv0WXD%{$b{R zJ9eq;VKpS+ud|R)ic-DF85RT!_zKp85U?A&Elb{F= zxA6H>yHm?A+VLP#3+JIgYC0O7cndCr-)flMaI6LEYC;s7{nChmQSW>->roRW`e@mw zIg+A3W_X2E#lKfnB_1SYPGLEcwi2MwvVugc4Ua@DWcc;zWlfMHnm+t`PVIF#Lyk=& zgBz(L|2ij_6~W#D+?BzNI*M>7n!QpO8hjY7i#{xx+|dxl;-1qg0%Y=qH?Doi5i49& z$0t#+q1a*-S9%UAp0TUoa@jB-*v&_|{@t!IKjr)}KL*D4d01fqMJ$o;f}krP zxX>G74O{gkf=W2q>LcFWB@S>hz{)mtjRI$=D5PIB(!vBLM#9XWOK|=0Y}$Bv{GzQi z5Ks(D%~3~{xPP<4V+$I*^_M%pLtTO4F*@$!3vc+IRz`zM6ZMWwoE;xNzp?(aE0eW( zVz|@GY~2X5#}i&>c90bwP*+!wt~D_t=#AUB28Obp3@|Xqt4no{cQApX@C?d*G@#!5 z+|$dm6K4@AC8Pn|2$q+n&sTrX?CJ~^I+&L!=Eq!#M{iefHuPEJ74j#v#ZR-C75> z&HGLMfAF2{rDDqqS~jfBkf8MRoy>)U@wO#$EHE1aTTdc|Kxxd>v(*bzO?T^dN*&yz zjY3H;k9?W4U*W?xZJ`nqKVhEWoJc|Ji41N2*w9dXve@n9QL`WQPnBttF8-8ZD14j^m)-Seagd9dY6;Cd~NjD)Z;{fyAuHUi^(9qB*#JgL@=s?Jxq$%y}Oi zsPyJvISa9S$UAbVe?MRE9h&5&DpOp;1E(L%lPh+t-$%d(!IZg}l&=6~YVCF_vIx`d zt55~yAY3JO4+O0&@yv8sdJ$Fd!JmZiS!Y*QWq>Y1GJvFF?NooAs+4ugbxHIoww?={ z8*KUf&0l#6XK)DN#Acm$)^VpAqXoAYD>uY{APDH*r61k-bnT<_!??(b(WEwyzkpyq zB-S3Hbj5eJQ|vB%69I5cx!6de%cZ92dkkp8KniIA((3JTah0x+2Ie~CtSlX#>bdXj zn}Eup&n5>bGz9VY>Ex%pP4#iMsrrG_hE!P+w=pt~Mo9qLcX0Ik!OO4@SwBlj!AHAQ zv)4-TzeIGfJE(7+qen4`;BI4P)A>+EzHJxgNQDLAt2SfmWGAPfA1fP$C5Wds@|ykF z%do!YH{hnOtJ9D|^Y3(4_a3VvQ*iLvN?#|;RuME~aV}l&2T|Ug*(r3PYv5O3?&+7v zLH@{PV1+xl>ijG|%q`A>rAAXIhC{GG&F%t@(RP|)22X}p%0#}aqsfjDYX@O#; zV4GZH-M#d)Xn;%%;C=9PC#R0G+fko4aD`24+aBwSnKd_x?O{mnuV_B(hNGfAk^obYR?YH-^yQ$00ad%Dt<303G!A7`dL*+QG zDCz7i%Pc$l`+HBbTGUx(F^wa)O-slrK~*L%s-1r@JPoldk2794J6}&eR~TLgIu#d1 z9w3D~4fVmqXCu6utlGiw%GdH#H-Rzakktf$u;@O}ghJZnRe{P8C zwe4&}gt6f=Si*DVa`!(MWl{3h+B#W_nf%K%ckM>z25nfs$Px*3{jVa`JPW^dt4*4y^BV-F%T)Rf8 zXpJn(n!0!8;+sM56Q>!OF#Pz37NA>Fx6&x{A9M>2FXnhUAMCvJ<;(|2vb#0&?*FQN zy10Jx3k2nFOf9~-^d(ww9u};G0O8fMvy+G&zVO+~zpVa#?z3An6LacH_56`KI4FgQ zl*pyhqH>}dbEHJ1HNMvt61P(CFz0MeR^Zp~&;HM)PdTcI5(!2YE^l0i!9e0U=ErW_ z#I9Z3c>CVeXLmL(elhvaxbH!TL!O^v3N%5TD}J2t!ptnhk+>NxzoQ#VHN^aQCF9ha z#Ol3z)#3Wqe@gmy)S*qHUHTL|7QY-GK^i^wSL(~1KfAb2@c{icA9U@b$t8Sx>HVvt zjLwWEl*VWN0;1JNr)HZiGIgn3Y*yvWBxtf5sPYr~?yEGj(2wA~1Ftoy;^o-Z|7e=)qnO zF0905_M2~VR)p8g?FIMkD7yM+^5D@|{&2VpnCQ8s2eH$c8%Y{ZyCo5`zj){Oz;ayY z0s^p6lRGTu&f)NE)TmFKmyCnHglw#gHW;C3^4yF@hh)N|!6t_4#f;0~LK(&!9-BN@ z`xP!|;)JO)XC+x`dKsus2QlCzIs3OEF(iGZEani>`zU zQMQ4`DRO|-I){>%90hQ2Sg`sQ7GImswo50K{-7l!m%5UJQoJ8haL|s2!Zq4n+qqIS zf+N6Rdj``-LRwoK;LZ~zD5>JJ+4-6KXtKEA{IKiySsZ~WIkoA^g27I-F4(!P z-BwLV-EYDB%%Q&6XZ&ko=JGp+@{Y_-DsmNWZVGSnQwQ1H^vtsi(>>Uoo#I?l=LFEz44xDI@ORitF=A&Gl|t=g)5`UZ*VptG)}JZAwUH$6$R}l zq%nKHO{^8PKo)F~-6&Gi{0=a_ zt&F-Bf%q(&fDO2Q56-oTv&iJQeh-oXZG=}J>S;@S;&S?l^$gG-NyTha$_ZKb`Mc&7 z1Hj_qg%O4YRyF+zOFZj$gAb&9EkFWEZe{e$8-Vqhahyn~wMe*OA<7C+rn6h?h`8}L zVAA9oDI&H}UTk*aMj!?g5F2O@Vr;b#ZB7y+RBjVA%}HFvg4W6?JtVEzGS}T4e?)hW z{JD0mrfvm=pb3?kU6Q`GhlSR|aAK8954Fe4g{fOW<(s$fTSqEzQv+>WjWAIS?N^GP`9<<%#z>m)Ejo~ zin_X%_4kSYXEy2?bMoj?n#$lws>uuq-j_!ekGwfKgPmqu*y&AbP)SU}d@z{COHin7 zD4TTYi?vk?Png){9Kfc;4XxhT*>}35a=bOJzTk7qq8IGlasVv&IZsyp<9On1G8{hn|$Tlm-iA1+G;-77{U)xgwuS8d8&i*YprUOEXTVWavSz4Z{ zx`VX|=Gr8;Y}9+ihMQxqbq_Z?wRMG9bvPs1!3TTp92Wrg=McITn@^Q-77x$h2B+J< z-T3E)H-^V<-hlpkfGdI>>c_X(LsDNqT&y9Alo5p86!05NeP-za8P7jkUBxJ-Q*Ch- zV-<8j*fiK!VV#37iz~cZY?gXjK&*qst_K0H zyd+jSbX~7+pBSB4eB;h1X1HSkB;VZ#=)u{Cr#C;g(FYceoFqDRjo;Y{9EAfPHS2lF zxogUUI-li-08n5%k7p;#_Or{+gIm_-0T$V3J1h4yox*O!F=?PR6aRYSBk5>(%*+eX zG!rv3A8fooHHnw_ug4wg&DAONCFi-rR@!Jj6F~DFEE=thg8x7Uk?H!~Yk+D`R|fEH zV_J0F$OLT6aN>%8)aoe_X0Y`6;=kACDQyciaFVWH|FL&opE3f;9yQXo_EQ`OMY(}? z0PeOn&vjibTGE$sVx!$)ZYV|4HPZ1LVG|shmFqsU)8ddUHSe~}1y^Okq$$6^PTYL! ztoOV|TS&KXojZ1{|6+0_xzZ#c?Wayyw0|4UZZ@f7s7eU|zEBjobJA^;nK=_550~)e z8L-&3c5UXT3iE|Cgg1-uZ3h^X8G})@@!u+sWeuDHl1J!aRxmBfo2J}Xg<-XMnFjHO z47)0+>wHY7m*q$q08ZTeBl37r1G}_|9DGAjC5^9d2wzoR7_>C(vJWw5W4YyTf87w{IDhB&c_#iJi*trS=FBvvi18LGi$50@f1=m zj!(Qj^MS8A6As6{SewoIeWULE59x$Y(}mgMkBfy|jaF%~Lqy!B3FsppAMK9S{gIhy zP>SX!xhfwH;e?XfEBCIH{?iCWo0p3Ii;6{<_QMKBCB?ZK${9scMiX2%$z+>joXzlu zhvg24P={GkQwUvTOC4N$ddTSL4ha96WF*!etZqWFOhJ*sYqEqCAuOI%0Y56>!+D|% zk__5;XZ^y@7jMu0ob2A5kUVh~Nz4m>xcp#ldjx_t4r_StZ|A5`r%7y4Of_4U0piAdcVFEz#t3t zzOT~TYp5Ni4mCDe=bXb{u!sbD?`sZgyHCvEG72Ou$l3{#QGD1Ams|Ygn;o1 zmn(Kkft-LzDsnPA333ujro$tJrmNxC{S7jul~EBBL5cz*)C9?pOvAe5BB$TSGPIBd zi+n^DU^KVayO3;A-?+$wHDxGF(U>MqQ(zCs9mqPztl)@o5f|=A;Gva7uQJM^!(Nxt z$XRTqBq<|uA>*UuN&_6Uacs6_a!rEqSQpz{u`({!6<8SY3spYBahH~9N7~kIPRBl7 zCFM~IS6i>k49%dCfi-xHE*Kzj>F;`ucbAW6`(N1my^aGI^q^ec zH1nZw&Bbv<*-0PZt0h@Lln;vhfPv0eDatUcw?_#M+uG$eG2ILc)y|L+|&a+xiTrlAvnhwls*3hu8ONmT$eAF zGGf8u;uq65IcJp1TLB!k`CFq4pU_oI!y7-M%ep_f^hIqRHT_oCo2DDVCf6e$Y57D> zCH68QVMTbLdT}Jn^^o~qBElN=qHN$I+^viX$rrE-NzRee;(K+XxA>93! zjaSbkp)Ltl*8bNb_Ttn{CDTxD*h_%0#*!ONPqtYUSQiR%r znS(|akyU$&MFb2WQ*~hrnqK-jh#s8x|I*6y{!b&8h9WR#@U1GCR#q7h*(>mcrqeBS zq!OHKyTbK)!A9Td99xP4cI4eh+btDO5OI3Vl-xO#cI-e3$M`)iz=X;GC1r@QTcbqu z_KfN{biC3zAcEiz`AzG~5YqQnaorjfYlt73`6Qno9y+%5wcY+$lTm+7>keG9p82#< zzP^G*jQE;5pZP=meCI&5jBfOa-sjG9mp{;SrdYdh!B`Cb{_IXr@u z0w~Z=S4N}d9Nqa6Ri?B)^Wa7#KL%4FoB-*}!<(V+0j>!#?U+#^*GKRCLH11QenG!q z`tsTW+mr-}xvl_J_$nsJTnVpVZ(neQ$s@R!BX`q8U{2Hoyx%A(P1y?Zli^glAen$& zktruuPp)CvTqXf`xiawt!qR2cpVFDyL```v?>Eq4-er4=ss$d!VifV)!TvBY=lPMz za6CNuAqvtFTj6_DkgoKQlcIZ@F`*#nwaK#gC2NVLyPu^4jyu8{*vvh<;OFm4w7S{b zxm(MdZpwuk@qTy=QG3Zx?M*HV5j5BW6#_5zELbQ&6IB2a$=*!p)sy8u8y(8=va~|V zs;iag!%_R4N$>{nHFN4wnf@T zj&| z$Q$S!#W~p~yZ)o=cWd+S++15>nF|M&DAxk{$Aqt}edK3}($P16bMcGek%^fAnHg`1 ztV5T&9~y?VI>jYA=gx&^1z$_GMKFI1jy^0yY^C+@$sQ-n0lmqWb#3Lxu}Ln^M)&N| z(2|a7_?TbUIP9(5Bb0;9Nj<5?UfS@?vdvQQ!L{HyO)~ znltw=ZmggJQeDB)hGi1`&?+Zd7D*tqtzP70$miS5F#!MGka)&I=6k>C?2eK@2pXBRiq`2Xd#T>?loX3$ay>XD<>R zY|9QFqJMA`n7#PL^s;@yl+rGeU2fk+O%n|EB)4s+g;1cierY?fZ$qnPK2|lF9=QB- zmA-Ni>+UCsSDc0$#j6xf6<-7sjnY?0x-E#EX*_vUyB3*$cx>*sD-W*U!(kDea}#Pb zDmsbxI8ww4u!d*D({Z%|Dd@O5*n)6lBZ0Miw{xJU^H^`09GdXCyQFbyJ3c&y&}2WqmC;~!rO>n0)zR3u z{2ug*TITQ--l zHY}WLgWWrN2B~yWbiJk5Zj=knwJyBxvQB6sF%k++G>8LHgV2T3)M?34JY|# zXLImXeN~kRYf8lO&XOHbZw1W)q_I@{Lhk6AP#p8lSwFB#O4u5NMgO?R&{OB z4`cJ-jn^g%7$)P3Q8C@QbMz(VlFPg)vGUV}yNXZ@lVxyxes;j- z)^F65o@|>_-12wc;kSe1IpCp~Tw^4vH!`Ka2asaC^P zzq4DNi`qCSa<>U{PHZi)2;!^UA2vIAq;jwix2p27g1*0gSB|sXIu09g$wGm6U{9aD z>94!bU(7?HY0_-{ZqDYov5GJf9D2DJN=%;g5gqy-m`rkC)?w;BgN^T6B#c7gO+Lah zDxYA7uT2_ldR!r!K+*Tc4i%IQyNz6Ug0UrF%v)9fVQq7PX%Tb7b8(Ir*jW77->}X_ zX7dAdU(qRTwPab z@d25IV-h`SP){VMY)L+ry_8y%%Tg99njk@@SA^`^ryL?dMkh8WISUiFO@=4lrmBR| zSd3F#3c%lx@Z6yz9Zw&6@wxq&p1sW5@TGIcT;a5yLErMu&+3 z7e5f1R7)wZq1VBI%yhe7sH>W<>T2#*OJ|ik6)J&y@@O|kjBqp9)7eXZyk3Oa&kt0( z%GIj7p)QBq`6Q~u6^~XW-zuRC2~5Xf<;x9FPSX+=JH-}3Npb1~5nly|3M5TL*+!B4 zkS^R{q0L%Q-eR+Wj}Z&~62#<8Fqx`*FuYi6-#JtJ^R5t2GSJWWO>NxRbZF#$A2Y0P zx2>J3sJF^U`V$(7*Cj>Jvo?DdrBZ95CQ)(r`FBM4Fr64?`!kwLOMwx zL?H{R&Q7&j_$5Ze@a_P(F1D*4GVjCp@BKUemrP{t96Dsd$*!Q}uHklH+ePi|?Zxi2 zZr!eLY27X1#ZNuA_wS#1s$>6)&pzLA@C7RdfeF2RZ{ymV4=#)Vm%Rv7N*c5Eo+*Ry zV_8ebi84cRdv}HJ!vVfOOX*wX9}Gsfw(#vvWN3U6@hsPx`?hYf3($4)ZblcDV%k6We1)zloxT4RBPBs(zdQ}-|!bdxE|Dq?aOhq9|DC(Ik$-QZLqG^Gm-pGx_fSeS-Dy|VJ zJVn-G#eWG2!Lt^<2ySZ6g%;@pCC>7KQkD`W*c0O2CP03V_=2Q19-8&O_!EN%ZI+AK z^6~QLk<9 zEiJ)Sr8xNjaz(vdbOAj1>pxrj|EzE+AF+6O=BzmRgTi^N7RX9m^|Of?shU>G$0N9& z+il`kDkyi`Toe!srmANu0!cyTxTt{jmF?!h0 z8JdT;P33n^%sAmfTt6DLk$@gA@A|!^FMI?IUq7u0%)#fryZ4!c`#bi&aNxyfpL*^{ zp3dZ{jNDFP=^jxTM`Pj+@iJaujv~9NjX9d^0;S;xEf!krKV3g@@R_H}JA6#IR72wI zbV!Q9JeUvN{J6A9X&5TDns)#k#F zCuStuYVMrhCAKLlOmcM}SO-*ZwzZc}QTPU1Eqxy(9JS!4WGpAN`O|FEJ`Ku+@;;4v z4oK(EkR>L4qmTx!F7fupCpxsDrq&y*R2gS;o^LXL z!^`cZS(>_jyT}?R+F5F_;gOd&YLo|2d|e{OA`~FT6yB#)ty1?ZqrOWF@y=h)zk_46 zVpwQx9@V!lenFc{xEr{%pj=aoxyoS22OO1zQL6;`O23U|vom6x!?Z#VS=4}PoL0Bf zO-+`4`jIe#KXBLY@$>>u9UZ8g?@oH8nH}xzJXY@All?K!c7~?7f)zY3%k5aXpN8d){mP@!qVS~kLU=agqbxu5Aoi? zLCXVas1{_{QZOzZBRX^p4hOESb}Dn{^1GvsDx0dri!GBda>>81t~0_~;+a6!O1~vX z@fi6Pxzprme~#kP&OWazg=^_jcARB@NKJ;urW&MZu7ME;P%*`ok+Me^QqI0%@hlR1 zEWMSlQf3YiOys=agtEQ?uAG`4G1F8|40WG@R(g9!cpIKv5-8*?UU=8gfIBJN6l)q9 z?CGuU+T}F5!{vV*D))6U#kH8CqoMRREH1>KFuK5Tu=FM8wQbZ^?_X6#L|?(JisgT_ zRG$XW#6B3_$`(cRT>4S6_!jlpQH%V=1vC?ibOS_q9{JA<1MD00>5J*YebPyj&G>FV zC!F)Fb8ok`_YIvoRvxhT$O7eFevY2dRfpJ7txhnEC7X|90chLVoZ|^?A;#gS@xxKl z)EDjQ?qU4mSW6{vtpV$&~u`vui@-+GYf7#bLk|lx;vjn5j?d?6!am4 z;0~_C4ga@*sfkNjbe_1J(ugn%pW?)Oe+i3?LIp-RFxRYS1Xc5 z5HL~Upp67YEGq5(QcG#p3zitaP?Qukudx)Fm`ET(CaHjptlPnEGKLzZ-~m%gPNI(x z6O5j9FnSC*ESVjYp%_xNUmFnmwVv~i5xs}J6rvL{P4lzU>O)$^ei zs;3DSr`y$ou^ZJ-L^zv!MbUpG)wd8xTl}!`Fc_NJfE+-4FBJ%Y$nnF7wqlPLdo9z< ztgs$*R$tVH$Awue0u~R4Q*NV?e81idA@s|@QD(qml|&*>Bh^b8PJ;vf(} zF#F~3nEo2i3@CFf0<`tp`3nM z|0|NW#?&o8ji!m)LsB6*e|}_5`jYhSKNZ#b8|#WgHpk>?-SS3kqfoS^kne zsw!-8Q>ar%UVQ$Ur#cQib)@6qk*A*RIQZQDr~bQv*td6UDK*uI5hPxhEuA*w%G)YZ zkS+k%!j1v6SMUwAsMF}B%O|XC>77RPex_4Pzzh3M6r1zH@Dq@R-@bGH!L9#!ewao~ zC&2fX`%Vm=gc4w9Xa1$(0n1Wvxmq=PLuX(1L`y)dX)_Y^5@qF5;C{&;`#j8RcW-=U zCC0JYN$h|-`CI%z$v6YY?D#7_6rS0+)?tBjeY;&M1R#K3UEh8YaJ%T@Bk^ZLcqU&h zC71VA`nGl1v3gL$Ce96@SOl_WXxtPrh$wCo1;p#7*kaTfL zO1>fQYtR$!H%UpDa5R-1;kLm~lW8Y?SaK-W!tWpUTmrot7nsZs64I1S9`J7Jt>|cr zO{94LHAupp&+TE+<_9qwy#MqkIwXXYYJCrd0);U!F+Wl41Utc}=do;O)*cfyyDOU7 zxYkte;FGnxwHs@9{YA5(;X-12H%*uI#qoN>Xmso8rhiBK7j2)LtR=#LYCKm9dbIM6 z89C6Y?TL5S-<84z<#eEfjw)NfOTEqCow$rxvWc1HG5*BtuKy-oU`xUdx*8|C7~bO*?ojb9X)Qk-lq(aJ<2_kGd98-rX^_~kmo}#e9~hAqN~CR!};)x087G}!xD4O zCAv_&v7FgESVK2ZI$UF`piF`5n{ALbQY?7QIxUqiF?tto`JIi6UrhcpI8`ZjYkNs{ zw?1jS6Ur469xZ+?3Tr0%D%IyVqaa@0p%L4;F&V#9WdIDF=za?W|GAZ}qY#QfNST6;eAWYGzt57gJ)&IjZyd!jrJpskB-_4wk zhOu(A)4MmZ#|~_{JDCHRNd}Jk@*x0Ny!2wwio8nTMXY3su51Q&O&-3@o<0k4MlKkG zYzZpLGN$@1=O3gPks=Fr4PtMy9mJ=&AZ)@Du)#`y>sARn=7cBUCtNKSYP^T$$v|c3 z#7W}m%+dqp&I)EZfyuMg@J935$s0dD|JBmxYpYr!KjTAqryL1v4K~;-W3p3785ueeY*87lA|3 zrSO>m{&*)!Y&)XV7%ag)c*kxiQVWr=#LU-O%-fJ~hI4ixIm6fQ>3L5tyLv@@1)Z6H z-h{-0Z>-U^S?O=9n%dks^!MJS8y~41^Y`AV6Ao8%Yxuo|pRY@;LW>(i(W9mZVX<7~ za5o3fK#y9ur9SrtP?!wBFNVi9!8uYKLJ34V5xSxhDw}6rq zI^Kw6bD+2Y&hp1d98|AlS$RI7wUI+ zVZ6?-%RMJbT{hNJ-C8=U*7;o52+$M~1j1~VPB);2X3vxe)Y;$Pd)ngdF8{nOJIV-; zX;#}_a!XI1UjgRMmx?7uNB8r{LI&qhYO-skeY6wV+>z$l>QB9sf;~+rIO@&k&jdni zehWQ0wn2ylnEVl@LchCo-j*KR4G{h?04JhZ8+m9V=U)LjBoIPSLJK#Fg6nx5O~c|P z5eUFKSk&OVtUnaRxGv2in8jBw-Kw*cy4@r8GTO6#R$xYU>OKy9^;i)sE z$wS7GY7ep>R@f493xUhs!vaStR$J|_^dY{*o4V+l=^c-jCYLau+Wh=wLz))m&iPs0 zTAzI*zCqiQa*#5-CKH7CpF9EDqntwu5b{2v#%Cg;@Ys~ltd=M0 zg%hsy)AbEiYe4cIC!_*!OyJa~?8GC`yG(fox@0p_$rI{`ssznUe*(;qXa00564dnB z4ASr|pG7J@{qsrPrO(C3JAIpzj~Nu3Z82$Nn{$|DQ*d2cQtYt!FUp4Czr*<8UQhfQ z@tC5`YyDxI@{zgy+#c5F%+n3G@g)~bqKPYlUn1W-#BoV|?GA?|Gaz4nu}Ywx~~4*)+yN|wG9 z+uFqJg)flWN_F!P4nmiw-*all$mp=eN8l65-TUBa zk3j&wEzojt0yian05IHO`oUZ4ya1HuXS84>AoK98b=*wkr+r2uSGYLdo~(uH)5>3p zq`g3c$OSJ2jrIrEKzUPrqUm9yuXLS!rt@@pph2g$SY&*;ltl+;^%g-#kTRF~o=kZ! zS@Ur!=RBXl>@})?j-oScc>|HD1YHGjmK8@w3?}OBMD}=qfi@8Rfsu zoHZyjc=FYrYMCB%Ki3JiI=#%V|K3o?{(%Y?T>b91u|ljVjJz*g4foL)W%J_~V=&!m zoSa+)EN;^Ttiq_Ep^Uv#w_wqhth<=Ywr$(CZQHhO8@X)Twr$(CakK7u*zMFq)oJq| zjE5LAqQ~f;W&7VZ3$qoU4d{)gw2^*ehaEv7WEI&KTO2;(z^}1l4xA#sXYJ%W< zfw665-hI9k%|-HOZMd@7V`O{@b1%5G8MBlUbcG^zUy1^G*xHI4-O`F~AD1$>y3tVB ze};l^^B?nMObGnk8qGJT5cbybPQ(X)xfn z03!V;veU4)hgt-vPSbU@;ED-;x;B^{VdW zPAA*cqV=qiI_kLXPbqds!iiRbm4lEM@x17c@x086;`I!qqAgCl^+L!H*arxzqABq7 zNH?xEsQUJE`Vmme0@>ik(|@^_r~jcS-48af01#N^o1Gl+qLc+!M7daGMG*U{q!(CV zk!2aHDxy)9(Vv(1S=DqFf+Rt7;jp4UBJ7ElLBn?7T<0DS1nUS>oF-{RR1E4Lv~IiK znb;%RO-T}~%wL!Rd}!H1<{+C!TXb0|4X_h`>eYB9M@y{<^aItf^${rRgrLtjSeeHF z#b;;-Vd@CTjn5ZMBO!Cgq@f8E7dm=?xi)@5B_dR z@xqJ=q*@u5`_#QUrnh`5^sJZ5*nqG~1b^5ca+ZvG?C|eS-cr|0e{oZJG-TF5M$Q6C z=@WbEhm{T&dM~e@=9FB~mjB`42Er_rxciyupDFdCsx821fXpNs?CCOAt49!CkJt7B zKAn=5+`SGHV}uIu;l2%Q-)NjihFFJ&5eRb`6GH$@puc5$2I>yNB0t6j!td6yjw)-2 zK_tFTyuf2AZ$Kt+%&ubcqKwUN>1dvr%Xju-S4L?WqY>cmyKMtvi*~iw&#*x@KCCYf zq7-{1d8-`KUhd>gMy;uZYxe2T!kpK4D<|)3SLTy#S9E8SL1yk^oJc1`(1*6r!GMOx zE8WUAcA@9J4km+%bZ{`JmR!|8jgZvS8fk8~wx9*tN>cWxqHbUI4@^g?+N9vGIPO&G zYMO_(=-;U8Z1w<`R&MM#BTW@2rb^yMhB1IupMP%~*oK6J!^*UJ$089 z80ZhEi2WoS2&%wVMh=Ij9fk0u85sKg{^*WbZeF(1tBRhW!nH`RQ9o+(>{*$!iBEwH zY8>5->=Q2|xTgD8E14Vyg`MgaT*W71h*1+WPNIG3-xWW%pAcyRYM(}hwWow}x?3uD z?N zAI@2|J@p&dDt6uXIXN0CJ0|PjzNUok{cc@zK|xL-1(67ndK-he@nc<+j(v~AB~J~O z^paVV#-~ul65~hmtrz<#wR#Lp6P)MzgR>^3uOqC_1Davo%nGb*hgU%=G$Xg!tF0zR zIXhq}q;po_?J=WJN*(GUg|{E1W}Oct3X8U)aMvyOLxz*{qwK&v`=kuE`R+Nh#l^UN zB-Q8@7Ru7+uVS;bww%_k+nDj0aZB2Ha08LRZF*<6sUZ>QGT|-c=83!hn^apbb~_|z zW?+D>Sc7AV%z~7jE^rus9GKD(iF9Uj(v$0t9D?K<0nhd2V~P%CiW}C+C^5!3JN{RN zSMjD&^$#-{khr@%r{C26&qnd25$7}eB85MRK=FdO>J8^cq0#so;B7uNGyY8xuE_)C z;iG9V+Rr9yB45BW+0;`T&4HWEGuOoZo?jr>yScjMxv7BpUDa)LV1XX*(yVvYVY|=n zdk#@emT`>HL}(;7eF$pPrvBlfXdz)fI_gHOrCLbjI8SQxRi3VE9h#e1l*dr&o>qX|7^FdAMB z)tMPheOU;P6P{Ymf&7s8ty|VSZwb)bF>G%9C4T0l%yO~*1T-=5^sF%A+(A9!Xx9@x zNps^Ap|h{~9fc5zN)93yj->6=Al=R8>(c&U1f(wl8ze^p6r;t&+ybjshPKu zif68@H0Gs5P-!rInMiF!3wqJrJG544efO=`ibGSWZt^5|fPrJ}R3v1|J&%TP>v=CN z2bL(DaP}ZWK6r4iW#qvXuclr7VjpoLPW&s_lUI-2?>E+d&e(1vE0k~_yOJR$A9=Bc zj8No14UhzwLt)8O<~iOAeOyh=bZ7s zSEhztaBq*I&f@>@R^%5Xp=DF>C*dtG{d<%53do;GXZL*&5$_S2pEs?Sfyf5~09q4^Xh#D+!_1m~;|6+4?gsrT)j&@Q7Psoijv+vJ zmJ0x@dwhT&r$*;v{EKeCJY6@Z*XvLX)SP)Pmcv!Wv+_IJ$I!~o^;MYH0`3T%u1CG? z4KLGaM6eqGc#|rCK$ncJ!4Ck!cu1^wII9}7cIRG+K|l>>23NmF)5kN<5%OqeyZrC2 zg|`|WU$4O3GxP~S>ajuoi|4)`-R?<2)EsK)sH&)4kBgP5BdC1ar)y)Ns z8*Dtoy7?PxKUsuI9@X`=>H#yQE}YI&rO8a&+46b>+?w4$PdnkrgW$Wc+;SZ>!g_iV zaUf4rh7>pQLBgBfm9vu`rlN38D~^JS3jHlrZ+qPgUVrlMNSfOYImP>ece@qZyu+#K zB*m+M+jaHTg)K zgQ$uBE7%|Yqp&MO0szqbFC=GYqyGk9&sGtL+Z0Cb`BdW$nKU&y*tVv#2BaX)kLfU0 zi%6>=voe&sLYRrpz+j6h5J$1x=dZ-2B<3#@6c5MM>IX~#@+%o{A^QQEjbOtf(=D~6 zouBf$_1g7%>^8GR9u-MNn7pW*gZ(EFBPzuq31RZ|2q~`)sptKOMGM{e(6E+B$ zkyH(SuC+fPqapSWzP@eK6BoSbuY{{u%)dZqL|VVL11=sMti2$VNq`X;&+jcUUm_yt zPM%@9#U>EZgNrDEg#M2wq(Qfm;hViUSfPefXA~?{z7nF^3X;3jQC*Yazh%9?8YgGd zrg7r~HtGdfhd7qzMESOrc<2_{WKR}rsY01UqFZ&7fnhUG#MK|Sam=fD=t zZd|4L#1VdIHJX{Kigux-G0oOz2qO zmb#q$3ZL$Z72J@UEx=k)&Rr?&CdFNe4Db$1dh3TWo% z&E(^uQ_ZhuZM2elytv%cKoed=6Lzj4)!L%daLKx}abC1aUt5qwL{T61xQ!56qNprL zEjiCpigXJ2GzFQX01t0Hk-nPu3If5mOG($+Tw4;EK5wBH#AQ+2f7m+0BIFkgGbx^XvGw%0d)X9}0^-wh{c#<%0;^jgXGiYM#m&gu1Z_S`#bTLpk| z&e@h)b*K>30mP>0N@pw`0%hof(sk*q%WA$nm3Mqci4r}a72SV)KpnrHsq3g-4Kez6 zv+Ekm(d6>_RLFWWl?r^>8w4dj{9gsWzvIgX?6g_TliJubx%Q=5k!fIdCEnu~y8X&= z#*_v#b4o~CdUoUam|)#wBE1Dh)is?IfZb>~*W}ReKWZ^r0H(Go$wbKp0s!y{1pvVG zKdXhYfwO_0skNQkf76U+RRKFJ5#*lRT6e}YcE+8d2zFwELj@lZEM$SmyLo%sis?>$L`mj z*OQp*cMqr&1EFA|@eUrC<`7L7Tj;TPM*FhWLqd2RQ53-nl(iM!f$0dF6NQ>GIN%#4 zhz_xQTtv8Ek{n2H!D7Qeq3R9*r$6$N+QknQ`REaVPvhWibM*4a9O;3uP~>w+aiN&# zU~*4`{s=b;=#1M@2gKAWgAVET;Y2dBcJK^j{3P^$`tWWjetq&SM%WF;Se52;ET{>@WCJpDabyW>A_)%79*V~ z@&$!q!iqwNOvNNzm}gnKY3ByZ8UU1{?8z54<}Ffp3f znK7(aSN$WhR*mP(Zklh;3`<&J_KoWsRRAK*^sqb-#pM8t#-b7zVhzm{)5UWL4w95b zxR>;BQit1s=Za^MMwA??AOtBsCDLLjq~<38Kr3dMXJgDNX%NO*D_+rF{e7c1RuXLk z;y#^J5^GKwO(JI>OHlHpn`D`7+5FHt2o+LR&1xNl>Rcr`sE#~F3>Eh`9VQM7rI1>Z z`Lk`cf&}Lil9=$a)uvW%yAF2CZl~DNvSS^|#>$~^`%ljGK=p2o^E>HES5BI|Tw2Z6 zpT~M|qi%k@zb~WrNnF3&y=L5Kz2D5fKhsgWxip$Cqw7?p)bF0i3yPqG;zO7kbA4j5 z#L|_`)uBPD&+U%y(4AbDw@t#p8>QNv6=J01PC3-g3qKQ*Rm+%%A3-K+wY|LYe1b3; zr-ZX-pK&9Mtdhm^6m7E{C0S!JBnJH1Fe$jY7F+r$sk{44d@qvk~dJ4v# zse#fF6o#p!9L|~h#)Xj_r5CeF`wy8OU}%)_ZA2tqZTdi6p(ymNOqUE;gibuYo~e;& zxe|0|K$+0&PDZboTDCq;Yz7lY@&R&M>9#Y9;M8dhH}AD^qk21&fwOo!e)p$vXuG=} zE}v0qZnts0Ufw@@{gb`9TUo}q zd4-s!u1uVK+Y3Y7pWv*mZ9G~-UBx1C!!FbPYVs&er+iKQ^v9SC6rYOuSL;o?+eyyt zkh{l&PkvVe7=tM&CWak}QA$l(4@tfBUN43p=lh*adQN5TtwTL*I`j-1oWON9Z9i-Y z&vgIMp~GwvwULawU=A1nfLclb0NVdWhfMX%91ZOMZTvT|H83-Aq%|_C*0hnsUPbwt zsckDs)G!?2yeS0|&;)i+9hMqC5LQsB4$bbQpT4?sHC;j1c$8PE8J5NW z69khEnx*ORSGaj|!ljtA;xTf`=_Z?tj*Dp$kxlV#%7^21%PHsIcWeFByoxzyRYaAB zx15r1kd(#wK0-IP32G5cY3yGT)|{1zrC0_iq^TudOEGAlHD6g&ZYrOF%$#vCgk3_M znznEkx4bC0li!hl#Ozq{OdPT2FIrhy-;N^^eF8~L6@y4)mKZ1?5Na47U7#pPS7fn6 z94WfcZmp|AOd~TMZV21xLQPRke#j#wYCD&#-v)y8aF^-;N1SLu!DF5I-eT@-Y+R&b zIMH(QlwURxhB3e6t|mvg{>VvgEyG?~zC<4`>!&8~ymqxmp74+-uZao5qGAzzBfUdzHt%E_Gj_U>StiO5#sLvol6jM(eL54McuFjZa zEkJNP0=9F*#m2uA^ZYuB2z;MIe11u$;}dc7#@|H;XlIw8e!%lE>0ZzoB43AEB}8}R zvISUHo<~wXHI-VlR47O~s>1ldTy31pY%PODDK3-@eLSP26NHn%z1$cXfXl)!Fa)TO zQe`MfYV~jpg%-)m{3SiVU@7qDRtru+DUBIebxJhMnSw7%@u@6J;{LR%sK3JV(!S8j zGz@ShU$F&gld(=V^oiK8)XPh;`#vB?xvG*)ABA6*wNwPEqyTgNtBfOSQ8AJFScY~x zBVO%6S%n$%yV_867z#^}MPqzTiu6NqyZe5zCZoh!R7Oa)xW6XzJaZvWLcY& z91Dv(No%vmE|CSgG2&qI<_3Jq-p3(%Ysej}2-CQO2x{%hkzNKThb=Rz0CNArK?G72UI&=PpZVcW0Yx^Ia*1`vZ|Pfb zQ>}UFq~s8KyDP==qQ?9;TvV-K_dzgyf$8LB(dNhUXi69n#zaf`B_4N9{z z3@~JUN3p0p9)^q}iDlto%wWvRCF--xN{AJ}&1u@Gz4p)VsqIgw&A<1y;0VDLekyek z>QuB^Q5&*+4h7j91z+y3Bxr!$lA*%b>dpYP)Tar}y|dqaVK#ad+?2H{xY^XQH$CcE zZI7*;ziWCs*nWlV?XE@@=(jmJyV$wi<6l?9Yglc?3~#^IjLVyDq=<4|%d66yoZ^LZ ziEv~N7={H1URUSWWkTm7$A~v=T>~r>x*FaQs_L7YdYbCr#`*5c^%+*AgFNjDIlogp z>t>VHXlo3w@7JepJ?pzYuip8A5a65cAN@r_?lS~Rv#V)!rCkkQgHzcDMO^RfKA(+V zvw3pHgle-vQd+-%$|d;u`|mhtNGGvN!j4MzjFEF}sFwnmS<-y@K$A-+ajtlvw7v6_ z%7hSB7N@4VdDS#>eg~)2uy!3+3nbb;7X5tRravxL^vrH_a&`Zb*Q$>RY-?Ju+Qo`y zRtFnl=;eV`H*@$})v`z2tbI+s;LTSQpZ?WjYEthzJHNfXJRL(vN6q8kSLfD0u^>pK z*1E=kuq1D<&xWYWn!VmwL6z%5x{PrvHT*W4$-b6=ZbAy>hnwp+K9b?eR~f|7ac>)d zO;pf}iB<2=F~&E|G&)sJ5_sSMSAu?2+luH$0QVFLJ#!S}jI$QzI+e$Itm;-?Fn7DWJRlYwe8$7m9hKP3wbh}OmDSVD>SGJ$_n5~aOSkqh6x~a}N*$!3>XuU) zp4-PE(*rK`2>etSce-2JrttVoH&sG3=9%2?#`UgQKepNlT1v|=Kx z%>*0P1$x{ghztPSjKB#o&sbPoFrkHMr69g@o`nM%qE~-6py`?8X;u5lgjSAH7n{kepH2(0C1DCS{*hH6jk%v?eIVp}$}T;w47*dd*VH@qodUnap(;;aT>uf>)o5daD!F&L?SaME#^OP+7D3I( zQ&O{8KUUes>6M#Q+%Uk?p;%`M+yoT?HM;iWb+p$$Na5Ea@YV>XY#b@Syt;y0QI&9P zRvOSnBTPJ=M^uO+fu>kECINXEgeG6tw4;$|DZ=~a!ShKeR<{Vs`x3@LGcAZh16 za-?|;uZtUAyVBmy%#*jreCZEC`DnNKb=4vE*_E_;dt&{jtVe|(JEz-e$hQ>;@ZD!7v);sjnyD~@Xk^5I(k zLL+UQ2p|2h4JlWUJ0nh5U0)*ZWUJE!lyI_< z)LEn(QY=NZZxxd(UV9pWRor@cIcAl3v^Kk>5CyGg)j%4IN=>#B7&}H>-6lC(eBgvb zz$j3bzMn6p$p#y%7~udsCil=oR&?17Yf~Pv7ED+wc_)E=95i7my!!xiPcy1lSY6er zYbC21M;^*f`2#c|Iua5gdeZ!p3E%I=kdO=7^IJL@KW7r+5Z4R1u!@6Yev^~g3BcmP zOi;xVa%!rl^LQRByzcTXZ#HZcCtjNJHMuk2kCi&OEAGD=x$54`Gg57B$CH^oW&U9K zo$j<&Z8V`x1K@dkp!hC z>UBcc(9L~j)&1?kP02Q#B!uQlqZX&Na)@$cVm0slQ!qul*vc!pfzq05GBxUH+PD)^HHoX{UG~W z&{GbF4T0sZ^D5bw!Po7WYa4yVSmqy}7I&7mc;_BYuZTGWCv+4}tKj(jZzd3MQ#1S0 zH`f#MJ0u0{f#exjK?Ch-{xL9$9OG_Oh@zfCg({<{kvur#oLyj$i(^uuw9TONI#vaa z4t90ulR19K>h6v-;p}>s#WFoEw73pVD|ic`yIJar%_?=Yp^Ik^`ex73C6OAJA#)gC zmI{2b&9+$P#`?*vNcKMAwle)2TdWmA{Om>TZ2@2i`m{c4wGBVE?xX z)6dNIwfbkmsL22TnEz)JW@>HV>}+DI=lHL4;6yj~C!j(nzy5?R&X!&=C~bTh@?r~1th(M&^Ni-n7XJHp=wqYu=! z#;&t;L#mN!)ACofSYw;-)oylxA zQLS+SAVq>L=9R)yQ)XNlEZ#g=n;DU&KNY{7e6+k^5oDYaq5`2XF7YKAIqbJ88~hz< z*;zwnZ2b%aQPAJN83p_omK?fVV}T9^L4-wIwUl3QMM69!u!W?&{Cfy}BJV1% zqXMHy2O~26#+p<{s|rPejUB%WybMZ%Bv!IpFv+`CPZg2VS^*Hy0Ra3>FVd2RhUUHr zgtVv6q@7y$obe+DelEtf0%D_DA`%wyRf%S6LvYVS!3SM2kW_-2l&S$CD#STcR&$6t z@a2<(9Z6_%1SSu{V2@0*X*je;I;N`R3`X&-iE3}Equ&r#T%0UrSq&GN1+$Uf@Xv@W z)tmPgzReL0niey9T4~A?NR8wxyb3hibDmcCc#kSqAiAH;}fIqVgW&rrVKid-FjOL zzZ$xzlTdJ!mkG_M!+#rpF}wn`Ov}G)ic_(0eKd3azUO)C^H$Pk6fb<8^*XUlAqI9r zCk3+y917_wyxh2>cRn!E66wgCF%$khTq1!>OQsb+ovxx>tRCR=6ju>dv zK!!vu`IN`6dE(7dfR0BTQ$ zfTFVUXJ<5e1`C=#=f{t6tDQtr0^K8vX z0S7dP;RWGYY>gR}7!=t(PSVJqH}7O>lZs@?Rmm4y`;a^y-QaP1Zr$u{-^=?r{hguh zTiL?Mk*VdSyNLkrMp-C#ney;3a;Mr)-TqWp^eKb&UTw9F%7EcMNzlJp-^xC+3B%_C z2|jHbN*Mt!2FJxL3WuhA4C$YONVlqXD_CA&fHtmI(nVb$OqNZBZ)cABx za2f~B<>WTr39x3a8*t6-o&B^x>}puomo-Qkr7dD6V2l%)BF&!)f+L(M-=V~uTmlIf zq|3s9YzkrgG(Hd~AL%rB-95)HF_-QPLlMd>-jUxpyS-;?bJzw;3~QSJ9|G^@wdGyw zZiNlY8GF@#SB(k$?GkC$v@x^rwd6L|svFQ3gs7vN^VQK6z0by?%T}p3eI1I;E)&60 zL#azFmpRWj z&Db{aMvI`_avYJ4v4ik+?ZbqRvzzSqVxK6=rR%#j7v{+&L{D-|v&+KvS5n+n*9tCK zw8Ni^6u{h*!aetzl0pm^JefoT>y9i?q}RpA%o59d0Ox*8m1t(@9LTRiGbY0!{ifi8 zfJ4i?>b%@OS$R@#C(HxaqZLf5CGY5j$=T{Ce_G$(2Q@vE@5C@88gKPK;E3EP(0?Wk zto}y1M?b%|LbhWVz*KUnWE_VKH-3K=PtIYQ?S*=HjP+Fo)SfO{a1>5<45zRemudPm zsSjg^=J53T^zydz-TzE^UUxrGOM+Us+Zh>uDjffDJE#{teym2Aa!V0$*2le2Wv+8Z zT_u0Hl@BvSgla)w;AjXVE7d&+GL55k2F#}F z_a0UEv_8fA0G+~y?K~B#v=7!vbq^jK9RR_Fz%?+bR|v`Agxc!m%cd;4!B(hkrxuM$xm4jIDp0+cA zgIQ51EQigMB@CD!kwc=#%WY!Pe2%e?}^RW9ydBXxFC zJZ$$M4GrWQuUm(Fe+ta@jN%wUpK+}Q4Gko7@v;w4!oAItpl9wDh{ATR-AESbq*^Pl zcJF&qu+wX+`~9aLa|rFDZI5IOZo!2HTlWWC-ZCoN_Bdtk19J@rQO;=)5CpZJK2TJO zu;KI@FNC3*xz?SZ0d7(b$xxX1&9iVYfoz)!)(`}-kh*<=p73Pm8qQa(QK zsyReqo9#?^=ik1FHOF_5KPSj$`c*u1+(UbPE(q9-vzI5=8^4ISWN9_07HYaV$-jJ)ol&Qy^w}P(i&)%fHFG9_PKpQs~AUK~S**JG>t~k!BZOda<_!>zTF>)j7{1c3qf}jg{rB z2R%uIrDJviZRrs9&#oZhFqkG0gJ>%i-|()TYaV&XRT)v~r= z#86LH)3SD@Y>uRpbpFycynUyrz}m=qBv1T`QnGEOCpxk@z7OoY@Uah_Ek@K%4F8#W z?~gt~jKd)E*cw)M8VLL{&{MqkG~`ON0&LiIu_j0ypF&m>n1H8z|491DnZS+9KwGg;h zDvrtv9la|}u=MzQdd-YVFI{B*C%yPs>t7a>DK^Rc!`Ld{G#%*e(U09+{C1BiuXQXkwyT!Mi%x+%{m_PRugA$5IycE?hDBM2=IQ? z@&nnKkl5Dhpuvy6PAPpp1Ls61m0q4w-sMy`+q9_BemgoF({4Hm{x|{-qd>-mJah#K z$xj-54^DJrDntq4mbwYd`U??v-B*jF2Q&3WXdr{bTuLe2$nE_-Fv4HR=Jr5UjdXqz z0AM1=3ytm@g+T(8lcpkWbR)mn!XiTpmYLt^Q37N5Ue+7{dxL_9*K!E`Nj{aT=+xa~ zCCTH#=}Y*xW*n)i2Xk7!Ud@l9yE|7HgQ#6KO6rc9+L;)8w2F+~rYL;(qZ;&5lu21ZbXD>ovS!1iiE%&u9*6@_kpm(=#oTNjIs6b$k$pTWO!CqYH%Ka=rp-8uTzEx}t87h!d{@w6uTij~tkqc#i|Mv|<+ zSPLW0E`~uG$)8K1{+r7F)=Ko;hWkVj#(Peb-Y1S@CTjW;Bt?nxls*CiPg?+-MPOSg z@NdWr>2NY9@_C)8&&2ewshjuMqHQAilg3;XNqNR7BeH?}ti=7!fneO)ETu>RW5Lc^ zl6Eqyo!hpl!p4m?=I63mCzn{o-u2p4|2jZPz*<}RJ#c}(82_p{SUmW+hz;Vv|K5c| z)c`CYA{#BJ1a}5XjA&upZI@gj3ZO(!dsC6slI3 zAdk6@YwvC`8f=!S#x!V9uIM*HzE3;m8>g~PwtXN+KfrpcQlc^P*K4VV-FOO8@MlW9 zByK2dS|-oPn|Cmf>=vS0F+vsJtMY3iYu=zxvQq-ZMZ!KV$9#y(zXM>e7$lCFU&82A z#MfBdGT3qi#I(|Z?(?c%oY4q$xn@IbKVPsmV_PTfK(`gghby`9_;qF5lrqO8!9L&j zkUOKd`plv@+t>vD@ye_b%6U_!N}Izo0i)jf9}=W$(cd_d2xDsxNJQAR;jL2s-X*A{&43yczN32+q(~5K3i3);`l1YlMpHF zE_WeW$s8iAx5Y;*?_Sp{b2Sm2p8jdtK)0aX*Ey!tY_5(-*|X8V$&SUPJfs(j zEU8eA@Ds=$>M@HPEb&V;atzkq@nuZG)03f*G%^W_KUhjP5T++0ghVj)*FEy^cXgNv zHphh9N^kPm)sqkS0_ZgQdZYVyV5wq2P-xbmDu)ey?0A=chZ8p8%NRHDwSDhNSQzUx zR+t;&6M!A%^jEIQ58yxY@LvOVPEo+|G*AElgMURIs{c9L`9ITjLl+BcA(N2br-!KHC92b|<>6X*%_gV}#_gPR7y>ZCCT zHpkcUs{I53=!AY^Nkc~h6Jzv-IU;O7`s$qa7P=8d#ilZ_*lHOlx0JycF@?7~8Npks zCM%OOJ0dSMph2FynB-lVe>yAnSdzsY3~a0nLeP4=DF@_}VPqzDIxGc|-3j`<5;A18 zSuq|Q2{rwU2l@yynYrv;V1(Vdtq}eeQTFk_wfTf0f&|jU>vaYugdD3=vC+e!Vuu%J z1jJW5?1iUgHbEk@TvG%3rT@11iIapbg^^AuBv!+H@j4KeL3gXD`xRfh@;l( z(0+(6IQnF2*=yU|?xt-9AYb(7dtEZui{rwkA0P-r0vuX8QadohN9Brx80Z5e^Y{-; zC;%9D-d)Z@#pSS}MLn{yT}I8it+>_y>7PAKYGKHBsK4$>k!)T!d!@p&{r)k$ykRA; zRw`!KtF58g)-1mb@1oCHvS%BAECHfxQ9aTAe0W%RxegnC*HXj%-PW%7-Bbr7Ng?Tw zb&?>c2BLDR!NX90>k`7DlmikVr(>IR7stMWf{7If6)g^?l(Vi8DMT}m#9Hy! zPwAT=ePcZ9*IqEy-nLrpLls%kK4CCGs6wq?oa@Kr_!vOl~ zXoFU03Je2`23-Q`^@WY8Oc5s$0$ek9z*JXjGO$d$XL0t?O72G8oz_crz5VdLE3CGq zb&&*4h0uIA{r17t+tc{wxi9J$RLQ=Ky=ikZYkm#icB&}HFz;ZEOW$aI$=VGTrO;L+ z<(FCm&N;1EsNkhE9O|Xu$>?Y1^XyYu#SQPq8~HG5Ex`{04?ep>g?Bq;IOf}a2{yk_ z5VYMV^84|U&AM&ShdfQsa}^jRmq>WUt>(k@?_hlL3v<=k^HaWbgZMykB1#ACy=Wx=|Bd3IC03e$l| zc8QDSkv$Y1rE5fq$wB@$cDi>%1`iEmo<3d>fP8keyRZCeeZ$l|QDn77s0E37Y>tZ= zxxlkho17#hB4l!pDki`e_jZ_~Ixtt}VdCbls8C-=e45u=wn50<)A{`gQycbhXl`nM z(Cxz)?&KFGBSl2|OfksZgUchG0IHn=m^NV14djJu#Z#;8z_qlyz;p9@Q_fb>$%DF^ z;da1kU=e*eAH&#*gdUBIA)Z>l(KzBB)K|L1YeeoVgs0S2XL*|m*J&MfU~#)tHk;Gsf=e&^0{AFg zDZ{=xK_M60_x8@~sIY>;R?~QmIyx0+6Qf=lPS{byaf|L|*A3t&;qYG~=4G0oJSnnB z@>3@9K`9FObH}FfP=4;2sze&*bejMT>3pxIvyGP*2~jFJ5)Gl~>aF-JJ!jxiWV}ww z&`X_;lB;f_Xg@;PVv-C+iqZO|umQK4D1?*igfpYD3*Z2Zsk~QH1c^9!DjN6UTYNf~ ze}8|UB@i>saWh8nH^L(<_7#tVmEQTPR-fMPPGX&vC$8;;uQC-Xh=)rc8WLTZekL%Z zg+UfO4V|b4;?^4s2G@<|--2}Gh|}aRhd~wvnfV9rRZgvE7C(qb{iO;3_bKwEnjV4$ zz9o^L9kwODWD%0wXE;CRw{iOSs@tRSo`C%yBO&8EVh%=j`zF6OKYM^1USc2M+$6sz zIP7Ueq%P3!bUjsVY`}Dje6ndmm(6$!OVVU)(?m4}DAjSwzPSk^6{8_yq8W}wn!#}K zVdqh5II|^SU{J2bQ=5C-im})GCn29nl zpgF6yx;gK5)GcoW>))j|2L|stY-^gUT{XvS21e7ADBWQPg9|F$k0A#i$ARGh_0y?f z_@Ui&rI~kn{1ZWByb?Evb#H>%?HC0*WhSpi<2q~D)CY|Xv@Z_BHP$Z-`?tfM2av6d zLut31A6|pDk#vaZ{T4rK>))R5&j%0dG_vlar5_)V8rq6hs>xgV&aQsmBbZBx9YMc& zs`=^Vo3x~oN-p~_jD`{XqDh|~n z%DsA6`m=ZNbq;9c{F5RBz>$$HZf*sJxb~XE7kIQ?Bs^Srq}(}B+mu<17M0-MlnS~5 zU^<&0`vxaNkUHj}S@8i%zzN7kM#s8=gPU{>>O7}sjIFQ$wu-3Gp%Dd5J z*ps|_W26Fhckj+0!>SV*z?L6zU(XGr@%S$~x%4a;TWy+lDhZQ(Mqi zv!HgX{%jceLF(^DSKfxPCaoA&p&&XbvffTC0qj%0q~sWEAv1w0U2!`>4Kv)Nh3l`7 z!p$!98EqqAVLxs^RlQzB>Hgn1`)0kG5UI2bBv)GN-qa`+;Dv4&9|rY)(`udSk98&^ zU3~`XAWo$VAaG`=9%qn55OpquhX`j^@skJyeIcGBs`-PzH9A?ZRXrXsRZzCG1^bn= zf8|mQy7p#FZ6e#_V;XySe^I&7e7%_3nh3mC z2gX$}QxzKewvzMDYm<}#XDHHXOd>tW(ToDq~MB)ghTZM zfrAD|lglwh3n)G?TCK6ji{d=Wqpv1&_FBPh6jlL*Rgl^&=cf{W5Rvkm#3ivY{L{ux zCiuao3xJC2E}+4oSE&g^9L*noq0gz=2wS}<2NMPnX@z8Wv{<2Pt3sWUo}VZ8h{FT) z9bmVFnn9sX2P>_n(RRUyr_iEL&*q;1L9D~UO^2#(4r^dI{YwLAxWda*Y7p*EvK zi07`i#AC6!50Ynbcs=VlIt9JLo<}663h^-EZkJbo2m5%{)vN7l_AW%+e`R2F;QdX& zCbcQ0T94_QhmEf!8i#{MQTA+kH*2uJKJw~rj0S((Ed91s?d~RHibVWV^n$+o2k`o* zo@QdGA`?2#uU;hpU713!9rltM#oc6?)nVr~O3lf2J-1Gj*zcu1PQIz%wO-GS(V&rB z!lXnH;rp+lX)pdN^>009m%pD}NYX!x`Fwf_bD8vMGTN*{Od_VYng5AGMDv+(Ov`NE zKp&=->rBiBES8PJy}l`0G3 zoSFs#bw3aR^;{t7-(Y6S)tLLbwU*zswAM9y1aF)QdpmMf@5Ey%YD(@oF9xcZxVw3@ z|6$Opljb-x$T9EfTG!4=QpfpntgmOU<`Bf7n}wnP{?k4Nbcg9l4a%lMRnDjr++%9` z@XAqZsO9^|znH5;Mw$mtRq74cO_m4hl$NdJhKLs-lxF{mxerQ9q`wd^=n*ck7|imw zuAl(+kUH&Cpr*c!O>?~jKyji$d+$>&7M-qk5yt}3ybEXb^oc%!ap1S~EN-#W)4;9Z zR!v#2{zDEFf_FK7td_EP@gkg=5RW=d6OR)B9Cjq{0wLEl28G;cI_IiOZmRvpdeVGBnm{pCAf7Ncf z$Dfk=M3nrb2{&PWw82ufd42UsvD*`L;+594i%Tr;DJ?fQ%UwWg#%cvS zWgqWLbPo=!Y01`(Y<|S-d=qAb8n6PKztBRUNlI>5oL;y7;YLwi(ydkW^=zWCX%!}D z{~uxR6eL>EW$Tu$UAAr8yKLKb)h^q%ZQHhO+qR8UeQw0<6aPc^>sk@(X=TotGspOb z_m+J@^2N)n6FN*l(X(+09C}lIDBn;O+{R=gMF)G_Imtf;yiobeF5IQ%`<2p^rY&4M z!=MY{JLY_egrh8*I{YWA+Qqv zYHUdE!=|7juDyBbrNWhZz_tn*MJMF3(R27AUvdT)DFYU6ePPnFUjaX4Gu4HZ#DH3r zxiuy}SE`1ex?B!Qy^pVWK71e9uI62y2E~(HRp{rm`0oJFLIup(XhuV+J8}omS|qwza~*@Vb+J zqRLu1ISD&#%p5vUAx3FEpU!58>ipgSCL$@hj52d=1I@+4B|vh6Kc!!js_&r9&O{!J zCKymVHcWZxyWdv`b42qo^DtXKqO$la+%4CSP1K&ZbjU=IrCi49Jm$k#e)j2g)HvK zFr~1nEinBx1KLy{OxmwuD18N~WqvhC$?I^ToWPu(Q_U(k*;fz^7uHN?@bo)QeD5x|cepbZ2u!%AuXFFU@aU(6~uw>5Wceo~jrQYe^i8 zSc_Dn3M_!()%X$arFMX90Ej5Pfy}Mx?_doghWE`H zzr~_&QL%dn5o())c*Iu)Y&xe&oCkH+kJYd-F~?C;($QoGuqZolT!w<|q{oo4odLg; zFI4;@`T4vKH9;EVukCnme4Bu~v@nj&@AZmus+M!^#_A`ofqjWHN9LC@Aq$`ITyClI zu!0@P3iDC@_yv3OZ6Ye#0(Kk;Go1V}G);RVn7HVz%AG{};DZ^fVvL(1(i-QI6S8&W zAMgt_|0XH~J4`fGlQIgXdk1j!hbgpxbPFV1E3@Oml!!zQSz3Q9iOQBALNzU#eI4;R z3M>3~Z^JWtFNcP1uO!)O#IMv=v&*@uUjtHZ6_BenGSg!6;wj)zb&}zhKPPYNlmBcq06C=a85oLM$sDN z0;L9M04G0U)Nfij_+hKz?5M&*9)|*h!y@T52+3W2c98DAbYQ21^+>JMy~Lw2qKH4- zU3ozRxVn@kio$$Jn0Z^_zTB=pJFzV?MV@JW<0aw%v3i9PEoOUKkB(hs4 zsjJ#SHQXVm75RXo4heBPRbaN=mVIT1hlK@YIg@@tvi2#hJ!6RneUex))7Suh?JB(q zYS{ro#&?^+YVPGL^V|f!oC+Q|L&Q6BomD=X71WIuG-!4`>le<_(F5m}g+sCdP@xHwmL=A_Bky+85j zGA~6UiUa1Z=VW(}mWMB8j=F)r7NE?(Gl|xV0E5i4@ysKW$$8B9g4_Fq9LDJ^_adlD zOwcG@{NjD-h0Tzdag0rp!2!5V{Jq%l{_%U~w3-%k(D3JQLEgrxYX&6X+h}<0+L~-Q z>~)LUwNLkfCae1a+6>YbqGX(((xOiWeLx8^iS9|l%*pC0==sKhg+}im)lc^gr z6isQEY>^w|ZTyeYS=kq4S}YS!c~@(5K5{3{hxS{ud1%ZY!q>e%wkuFhao4}=Y??)0mg z-SfO1^s``yE6~8`#o|r+=*^#^j$~8lvMLClHjjs)?IG@0<|-K@hW;MjCq!9_)zyA~ zw|)G8agIz48BBv58p=?0JkJ3|4KXGPJJ9JzimKfX8JKgD=mZKljgah{r#_H^FC}ma z?3zp<)t~6II(a~ZPAz_PI^!xEK_SxKPG3EVaz$*!_{Hb|O5JG#B;(j4emYRI@^2xB zhRmX08)2vPv^zw@`1vtS_TFsw_&7Lt8SV{3>#-!x#rud6%MvP>m3h7L@7EFweRM1@jiN< zN?K2tDsgFcsb z%MdSFtdb$g2#Tpq*+2?R^>I0~EzgC%$QH}h7-DrZBY4|@p1@nQz5b)l#K`nFTS!i2 zzU&bIZM@M=6ZJgkOoZ4&pCeIx)!MpEzUN8W5H@UXZb=>aaa}Var@+kNL_YdrV}BAK zO;=7vE^8=LIJi7IFAV`au&Ahtj)Z*j$JEgHjfTX$#%jw46N7qbwI%q+C+;G(22-%WH?8g|96)r19tgBFOpBQu@B?dui z^~ab4)SC$0pCWMk?Rs6*->PK#K~AAZvBw^d#rxK~z}u%p-jr9k4IPl5pUK0SHBb0d zd^)}tEqIeH9!9^6v*YQh|6w>Ceia?R-T;PJb6O!*8~36wA_UQ2O_?$E$&XwAPQZ1C zaN5siS{9k^&Q#F)d{DOc<%|DScHk_QzFZo5)-!xrx*3#Mm!ZK7Q~c@nbF*XukD6N) z3EbML)kVlotoQh!whiw&xC@L3l`AVXcWr@;2Dna0-E_iOYx*>SnLx+UAW)wG9X%|E zJI{C+G0cc`VLzDk_Nnof#SCaChuW!Ff%5Fxk>9B98G8J}F1ll)%9rIqsOq0>(Qg*^ z>2QMPOx^y@SGByZ+DMb+kC*G&@R=1` zpjM}v5msg`np`}ugru=cI$+kM=#N@<9E3aey9;d%E5g=B!7TY;I#f|AOXLS&8G8*X zcDM%9iRi&6KmD|_%`Naq7X3tbKrr+u9m^wg$LCPg&0KA__o`{7&Ss@{^)m0tvG>iG zgrR=Y!K6$U5v=8BDHS;KK7Z~Zb2Q>rWt3l{;uQ^lpYQDKy&O-i72e zh^+iRL?up?_{X+<2%GS%@j*~Mv);k#c^<6fcUsZPEpVAiX z?-SYfBM8eWTV=a|BGx95E<+thnn_%0b1$K*y3O7HxZ3?>CsW#bs{pC7AG9UTvI^74h`s zgpFt50-=w)u{cB4PzDhZ6lOgD>kz}c<*CU3>|XT57uc{_ZpNdo%%?$H1|||uNShxh z(D=DkYp0tX6OUVRG{X%=Zh5DLeSYXhC~x+w9(6g(-nK$PiXPz{9WP6f$O@%>gqcFj za=6+{T>W5C+%COZJcnsgwb}YIE%nNN?5B+~RiDuzCQNV-G^#*j zOG#ndAr9dV4k>R4#g<}C)mOn<;e&Bo9lDj)aSD73K4o_VGC}_r#>W&IZC6%X31}Fy z4sfb}6FaTlw-@6Re-Ir5=}Ug9ssBk^5HjibOdgOE^Qb2VdZsS~0J~a6HDG`9T59+S z>aGy|=@}LiSpWW@Zu2pO`d99-zt-WoZ+V=gaj*Y?S`fHB@CEIl^H&eEOiNu~ugX2+)rhRnc(qRjQE3zy zG#Hc_3)kcCb;}+1)l{Ot6ohA{029>=pVNaKR00!Y4ql~SZOH@KK*6gB@{duRitg%T z20x!u8n>vR87E6eN5)dg6)&v389u}MN9PN_cW3a`mEP@-0rn9Kw8BPEWsrhGKYrcs zSGLS4nYa^jYQL72Z!AyGL*wV5L0t7^``Is=MeRZkpx90}>Y3>FgmmClFEw$SI?_VG zZaSO|@DvFyR+O=lC!Hz|7{*Tt*nd&N09C$VT8G=GKo!F zlHn^f*FE{M0ljO+cuDp1_DL7Y^R=?q%Gt$-f5fkJA@j#(BU5cBBX?NObtZ2LBuyLA zbGTJi0?k7b%Uuufo;-T=;Q1aO++Ta2X{WoJ5^Ps)Ke}L;ay}b?EIgkmlr(Dn*I@$} z&r)D}fTBLHo0`5cs?qy3>LYDQH~Z*6j~rFWQi-RS^zb@IZx8jaIElfKq$y=;DO#W9 zugt%&xx&W4>y(Dmf*SN>?WI)6u%3ZNM83n%Mx;kgJv9nRcx_Tm1? zA?bs!eQ@@y;cJ0153Ae2j9fWMaM8RDA-{5XTmj_TYo11IUdQ+ zF2d}WwJRVGs6Z?F^xIC-t$&RSZ8bM})vYTU`+6EHRQ!ti5X?vny86JV5i zcvTh1tf4o}9n>J!YUZa|@oh3!Xq@cV?;Hj#d7=zN6zg z%7Nz5d)8Akr-BE2m#C?GEay{!HWy;R%4bU?Z79P8t$`EnIR)3KHBM~FlGZ0JP{L%O zBAl!!+67LVjR$+sx}!6v)b4C{YhzqYr>YP7lQ!?UQnoraq4=x;@@f-`4|*Q~*pj`v zO#5ZrLO9K}iNz1Nh77Tv)+Of@$kjJ9*6fw;5`8+)Z6v%xK1^5%IS}BjLgYW+&qieP z&;vtb+xg=K@df|imzpciP^wCMZ!7x0bef0X)8PL?g)lR;wf+w^TbIg?ES4yOcTbPO zVfXH6leCgDr8yCXU4SzEC~K5!ee%P5=lI)mdyDD;A+bUbeiMNbRMNa|X*qHbZzG{Z zg$S|qH^wxp>=zp6OTfW(Z5DIxy}z@`bB3)+4~FjbI0|TWFAQ6UzxgkB=^+8HH&dE3 zQ6Q?ECg4KLaIrtu3hdYzcwh_bL8>TN7)uWemXg1j99b%Vzo$3BPqwTUoo&~G6&}fk zpVF>qenl&kwk1!xUC$1(S@=(Y`m7NrXnV1OA`lb*k+q`fpd5u`s+F4?P-1079irZ? zvN(O_E+~@hlo>t(My!pvxy+G$mJFFBcfC4oW9D^;dfVH)RGk)0cSK+S^hB@>4K&Et zqzO=j_7xCAT&#pjT(JyiFDQbv+@tt4;xz**_iFluiHO9pX1y3a> zrY{Z=SRs@$xw0a?c4=}s2HK%X*)J7DD0ePJ2 zR=kWm12BB6pWAT?*#=ZBp)IQrw#Hgn!Vo!qv&1M($ymDBz&zLlDawDCucW^>ytgoa ziuy&EhWPXxz;W3mUTns(`j>nsBfqU5T)qM8=|vH$uV3segEVNhls5XWVLP`x+Iqa{ zw(MfFQ3iHR<@`5!!^Q*Vsy!dQP@2Y*3(DGrnK}WQlLduvpgn$~Z*y{AP+;Y*&nk(s z=ziY5s#{JiJ#_%ttj6`3*%RdAA_3#}t6}JTa!|wF@Bot#Mu*p~8TSeI`x1Nm?PRE{ z=^L+yovxe5+nKp+7dp~!!W19k_O|^`=T!kh4c7iql3*jAovN&p)*#L zNDC=M0;+Q{0}D!{DPh(&e^unSrmVUD#y_PQd%(fsD+@xfvyA-I$=I77)fC+96~vP< zZ`+4|tB60NQ{M?E)88wr7<@jbcVf}@?`MLxv65xTwu{gWl6{(k?-_oY#>fhX+=TR5 zBGp|b;L}?k3?4Uq(|0?3&pka3^>0UG%h3;80=6zLx99Y5T&_(gpS3aa)fQP>A!HjJ z<7NCMpnI@MYK#Q4=6zwNE-Z3dYUNdn1HJ_7*q`5slGi`(g_G5Z7lygqRXIb0kJp|~ zvm2dR{LXMU*=M+xp#j&>flw7>D#)w6YFH>RD3TmH!1E5a+_=RymT^XK`M`Ch_ev576U)yfI3EyncSW z1b(q-D~BbXfFM6;M~4RU>&S`yMWEA`>dvM8(i{x=@^Ny%20nP#6lY(4;va_GNQH*v z81XbD!HK{oy?_NketwlTI{JSExZK`Qs<9n=8#%w5)h;3c0NwvQwlcT=1=sw?ZZ(x6 zV2de=JaY3vk$pO(hHPb^T%MpoI4muvZ3XE3i_u!e(OWb&;o?FkuSZrE_b|)H&X=%O z7FI4Q=ODF!S59x%C0!jW)6KJWzU$u)3SzI`k7)WuC-?kxe!k zKtep7VO1OjED|M(YAnLKPx&(*{h&c{V)DTra{I8M8cpR@0T1-TnlI@4X`r-=HN11; zBvdRnam_=2=}|Ivei5=!V}ke=M(st39cdFz9b43QQzk)H$CyBb*zbvgpWCsbe0qrs5gDDu-{JeG{1sX#pS#gz{ z($UW8%e`tcXgh zqDuCge_EBA(@J(qL%>+TcWYS{h2HWUKIWtBgYR$;k{9akuBgOYL$cP}iivF|j2OwZ z7K)Pt49DwOo0)#+g~q3U)yIleDJ!SANMgQ7b1Bo6~q^q00D{9C5*c@w?|F@Ic`0q zkczlAOWT{j=GCZwPkq5<&CM$F(20dHOy1D@wWMq`1RS{IVl`qW3U#!B2UM?IIJ_R7 z_9rWLC4B|YZts^fdk40569v`!jD*-K=E}efI#%bF80bsyIv+@eKksg4#dy(PQ>B-i z^w2|JbN=e*7_XI)i-W}sY1FYY1yKn**)Tord=-a>a{(znU#{;1TPrTzo?bV7w;yC= z@Oa(tCuf(_A;N}m6=S8-_$$(suuopl+0ZI|dr(WjXe4cQ-^ChE?M2~dv*a24z0O!l?o>jMP?Q4@1@r>ODi*X0mvx!4#qKu!8D--K+VhTJ*cTv1fG<#`#~cryi@N*A-qRxx{(LKP z3N_MGl`cAXCRC%v@HXZ!X9prb#wQP6#;Q!CFIpHd+cuGowyAr9*Nm>Y7x%yulq8Ts zk5MX{J(^mk^~f$lEZQqOWP&W4iyJl*#6A8{7Yuo<`yKV4Bl>Ls^-QN;Y7}A;u1GFq04HH{^-;yMJaMXJ=8s*wx-4#` zolR0lqa2MO(<%pr_oB}QdsF-Ys<0qYn2S+_x1q~qPsEigny$=RB-Nrcz8gwUbICq& z<1^V=(NQDs1zJSEfkqU8LY<-O29H91qW5EcCK@+DMWk*LZv(NUoIO4s#=K)lV)so1 zQ}Yd{SK_#T+IcJlQLT#&lvkt9327MaO~+L~Nrn;1DT>YIl`F`%YT*p3x)~ac52=-l6ax6ryS#_>ip(>EFN(-Bf zCNr7;rAJa#x7=5wa(C4V7*dl!eEyRNbat+Q549BFU7u$(Tm&Ws2m-ob8j9S_#t)yZ z1&Mc3hKID$10qbq*DOnqSc&7ldaCq`M$gw%=CR|3POiv4NP-a?F=tE1vOZPPtSaAd zR^c{l&RurAkl8SAcW;7He! zCP#>24XcIRV8#-&CJjv?s*uYi?^8Gd`zvUM&w+&QY~mav@K|j<@Fkn`edl=7H}R#) z=LsfU(w3fAMWhqB*Vod6l~8lz?PGa+Si6o6JxlkN5Wo8Hcj&5hx6HxV;HG>ycsM;#0v&?5r>V#9}p z8oi`S2}?{=b?UDqP-JkQnKa@P0k0?GQp5-kgQjmfq9GFhYN_MV6@0Kt>36hTrLB!p z^5zjWl*g@{0s?iP=MhzPe8$=DHox?QC8Bf$wzI+_@hMpqRS{tuK(rz!?W$3o!S^@Y zyYd1p>G)Ag>Wh-k-4gjtc`3w=H=<2+yU*koZsA&>=pdV8XRPT0q013wqoXSIT07}k z!NOT0t77>0{^1nB_hyj8JHWZ#-LJ+?@Xzw;vr|pvTU3j_Tf!hnIGcCs!Ptw`r@c81 zi+y3>1ZL@a{?hmM=xUBn4yKON!-f;>vdi^zEQ9Oo=i=sOlI1lL(D_lROhG^&*BjCq zx&uIC8OtOaiEM-t>Idk*k5z<~;HvpuSw@P#tJfC{0D$3t?o_O7O-;>hO#h=-DN@|A z-B3dIe$oAhtdmYTXc-i{3>a??)Uezpzt8wI*4|{BNp)%IaKmD*qI^Ev4z9*l2n&fI z1!1n?2H=DlT84FyQiD2uVWcy&h}>M24Zc+R4xh}+)~&T_0kTLupKQVOtCpf@-;l*6bPAX^$syc_7{(O#m|nSb%+-6LaP_aye}F4tSwZBorzCG%j7tc7ZI>nj22 z@10EVH(&$nJ4ilCqDjt#?-dGyper|7MvksO9~!3^<69L4G|f+Tx5*LD)w^n8VJS`>9AI7Uc2md$5L6T3O3KUK<(xQ8dwjs zEM6)F6mF@7--;Gy1spd@OIVz+zL0Yl5oDtz9!qVnoY2CK`hdKnc%3^@vde@ppcEu+ zh>}1~-2w!mHVQB0ejL_PC(>TnHrRwkgm0+<*+b+CYNr$1bV!{oSiLL7%9ZBKPK&Y5 zBu`FI>r$O#mJy`UHPxtr)$FikMss*W*$F>&LF2K& zA+05F`??h1_;Yu=kuzVg@zdGK^?kK8_HZyqH=<#vxE2Jm3YmEwd2nI zeK4~Z6BB3kczb(@Zh0N0^U3LHXLr8>pRT`g^-!ES=%6o)2m6wp$z;^=R%ghpafHqB z7nZka_Tc_Ut6oF6d9PvMaeC`h+l>FH=B3-U%O}>`MQt=&!Hmg+zOc~Xd_OWEawN=t zGt`5cD`$R&gY1M~QT4=`Wy9nx${2+jSN^ca#`?{_N_y@4=G{bD)d(?Is5Ya~pK27L z?Qww|?5!6ykm3#3{qveD`du%NGwtNEKH|ju?DQ<+aZa`EzSHI!1PbK{RtopwERBjB z2rXLxaLyUKbWNFv!BsKB$$OI5-(RlmnElOB5poz-HK8q-sX#dT0vNOe2{l#Qv8>WmQd^p z;K3mRNbB3HPh4>#A(+%D{j@EZ9IopkXbusL0=g{=0A^`{Q1E`kd-Ev#X5{#5PyCmP zjB9&g!v&SKI{W`dcJiL~{juYLdIzaPPxWPWM&SPo{f4ZErlvFFLhnce0+#AL;3bqu z^4nvF?%^noP7*lWu9a6|0*MLFYfF4Z*}KvIX06e$kRIrGz~w8Q0wp6 zopOdEAmq+`usw&`w^ zvsQetG_M0Dt<%^wXlzgd4$c~U5|xBwqJ zXs{D`dW?xC^y!m4Vx2X!WrzKXY#5%q+`-isp{pqn;Zir+RLEOvLm;@MLEKlH!frs# z4F6t(%0L}4mZ~LUUwdnmF2liPRCSH_`NvZG$MlVjNk9svS6vMs_uJjsKxS52FeWR^ z*w(=cZl#NZr|oP}S+%s(!|*yetjuz2Wg?b-y1^<b<)tIDT|3GcYX~ zLsO^Cks@;Io}fhQuaaW^ zqi&`(_aYCWxYNB|ZiN%-2og(T*D;JKRILCO5kWomi}B9#ev=cM&gOQ1c!59J+3|6? ziEx3(&GLS^yu3ST`QcUtMoFwLZ}-m8fzQpdp(}%$X+Xe9nCC1|388X$xAufelO}_a z%0d%nYthAAff-$hDn)7Pa+&R1!puPDJh-y0QBQo()x}h+9!Mzskn=SpoR_1sUh{lq z?4AY^&b*304pNf-8=H#kPa)Mj_Gdd_fLRrB9Agdf#VVdAt&0OilXfwPQWM5_Y7z}e zNPKwk&4fv`Wz0~{U?lY~g>bfJx@fsGCZ&0!cs*nXhgEnYQ$5AU8WUQ^Bz#W997VOI zHzx(U@od-;SDZ}02_>iSFNT^~<*?0oj$&R$maGp;rUc9C^;zw?3x~HGTR+3O1z9j{ zaUnX&3hFA?e7Jv+Wfmt@fp}w_NL4|!d5jm0yOTkPZ6Tx&% z6$d5asl9^d^@V%(G8uSegK0XVX$D{a@Amc2Fdm}58ihv88VxiT#l0y+hhcpO(NOd> zzjl#A-IXhS#uhcX$BZ%Pk`fxE1OiN@4b`POO<}nlFrtO6m5{31A5VKn!8lu}>7!$# z22J@9S!eB2Oe}L(BlI$qH|02!KO`qzS-G*9<_}uSla&(=>TLH6D@XO+5pR=> zVVUSTstcPsk`5j`7qxUHVt4~zJxF{GpV~a->#~?d!>ri!T zlmW9y(R{qre+-%>qc?rF-QqwI@q>GqgMctKkMf7+{*bG4N-cZzj-^ABUvv=|56P)R zR#M|oTOw+xK&<>BNos#T;zdlw$3oXB4~QgirN09I5U?1;G4d)^Wy-i(XnzZHSBM-= zQ`zvtp~)s>^L}MZBJ1g|P^R=tlq?lQY-)BhM;t;F!k~+{2%?U|PoecE*;@*`)I(D& z)X51n+vZg~m|V7WCw&0oh-Sq#K4P@ak8Tu9^(9F885!&M`R}3%EeWBlCc_LJ`fGv3|1GNA|8v{>TU71M zU3Crhtqh%hNoT)a`Tr=fO=)vB`|QZUH*Qc7T@*wb8qSUL%^gGxL2CjF3QA_84na}| zyoS1KUN1b5;Md49aPn%6fV+SSPCIyzr&+g+E= zb1MM05WHpu1o)>64OGZN`fd*`MdKv@9J{Ti_8d#jboha zXyrH1=VHLKXtL!EMT46~xrF`;0ut{HkZ8=s6)?K^6}6AY4y<&GGB`l2%Qc0c57MIL zjl`@5%%B9@!9enA(LC}8Ds!JDDSSJXG^|3~cn7F|vn1jzG#z z5-AW0d_$M|nw)S2dZoq&oX|_d+ajPr(1H3f3=Kfqf+#1_Y>3?lt?LO0Vx>rsoN~*7 z)fl>27x7$er#jIkXg?gd)a%u7+65|y5Zq{7FpQS^T)-VyBa9v%Y?%d zv3ml265*R|*(==<0WlN6seck)1(Zq%CJw{W&pd!fgsR`yPJ{#YI z(3eYeh*Ve1s3i}H1oHxTf0|BNNl3Axs#dyS_9>Ef07_fgjhHH4+IUS3`mD-C$+?-+ zHhA|dQ;qWLbw1Xcf$SW081_no(XC960voU-)Zc>8P&}5%fwuR`YR_L0DFOCK7N;eJ zQwa|x+ZhUH4lcM`aCF|m%25~5Jhgp%cV80uDE-#6_h`26YMQfLnaw49Bq@m7iHgI& z_ls-E5V7U-;#M)Y10shT@L>$#n5FQfD$|U64bxYmCE@ zzG}fD7%L316ZGOgRCCavWZsX#@X-2z#A=A-4)ON|3>?R(IRi>6>C z2OOIv_ODfdX6O^dY@B8wGX@8XQiDvFn)D5&UV;?Tl>m(t{oVF~bvAWqNpN~{X^N?Z zCMXGU8ZuEIYO=m)S`dygnQ2O5X_~fyW*Cm~uMk!bnrZDxi){vOP_aQ$a&dyTsp^;6 zE9D&bUoWKn-Li2)OH5LJzoc!U8H7Wec1igAr8ZOG8T%Qq?XtBZl5^vfbrpSZ@iY%s zWGj8r4qmUZUp36)3A z$1@5ean*?~aZdJO*fjrCE9TfH2fF0*Z0aG8$}vYPn$=^oDyr*7bf_6?A)d_cuqdkH_7o;w#RGI#mzAZVs(q?gsCWq{APBxpo+idX3vS zTjKiaBlieJI4nxk_q1Zf3oA`SYNefZO=g8zJ(~=DiZUhPtP+)?^%W>p1B9a^{y__U ze*q&wB@jArVhr+$fgru<2Ps9WtQz+ij+lK>H0Pfx_sIML+3L~SNrLo@LKJ(j z#W%96ephk~cbr4&nLLfvC7GIj2eFCpm$E!HVEq=<^g!9hO6gJ*kP)n(YLAhk#bq11 z#8x+WKT#X)ttyZNFV{^``#2odogHvob)@1)R7Rp^o!u<#m1j&_REvF@q+<~vfgO>= zGCtmErMrs`b2WcW(wXOj0sTsP8n6$N+s-W-(@6Hc8-C2yNwGE4o#E$U40Al#C_2>m z(lryG)xyLoS4d!bn(d0x(u`4CC4_`OP-0;407>{#NS3?nCyB-{S0&~ELpHcs&cfSjML zuoK~&cHd*+g^D|4T3K0{O%}t3^%_zdV$QJ}Zc^2%PgDsC6LD$A<7IM1Tx1i+I%3brA~KI=M!6NtvMIsQftaxyi6{UV*Vd zC;xR)F(JxDS}eHgQec>ToY;7IR;?}LzYJ}$a->P`Yy*=i<)9MJNly|r?@7rt9vV1h z?IL4#fB(9h_X9=ABro+DEKL&0cOhefqA1{yj@Q8ro*I!lnJNIEZzbT2GYw3)_Ldlr z1pl5tQdkD4HR?pgG%{+R#*JR(*XW^ThU(2pV}8_7LcC^o|GF9sHEU4?EU*k*S4W2A z*n9WUKa4bW8vCDXM@0#v@8S6hwOOU50pZlh7(p%B3mcv$*d=GNBHWdYcM?)t#9;$Q zj7OUR>p8CDerbad?5MuJJP zhC_jQVK2+ibYT1h9MicdMma1qxQlTp7V8wlv2&92S=loxJI!XoCsA+loUa@4{o53j zjbVbE)|jQN?uy|#-=ZQlnF3qb6`-eBu~U)Bavr3dotM`tI2`Mzb$34$@|8Ab-2I2J zP)vn905i>K^^G#QuE_x&?GtLntMy`|U8&u1Rh4;ntUCH9H5up6K*mn?p6t2HbqTiP zdZ&_v>`cfmq3Gi{?|@^6Snm+JCe3iLvz*HlL+)?pLL-ppaxAwyO6WgE;nA^!zf1S* zP5-6=Cn%fv3vkeJaWl`t0sO1K7oq66n4e3Cd$;f!0oy(Th-djO&4C}`J>vQKUJbEw z@je;A(k(?7wel<5nggduBr_}}H?`_J+nPb9NhBvb%UO0pz(PLa(M@-&I81lPdaF42 z*fY84+3|BjMt%~GwXyMnHX|vzIBri@#hg#=A|3fu=pP%F2R}k$uyL?o*##VchOT*N zU`2L))0mhL*mNRtVz8=#$7Yqix=4uT?KEpz4N9w4R2e3hDW72Pp3d{zwT%y(3!*lg z7vnaPdF^dQ2!tPLm5Hi~VBFj+8Bl6$Xh<@!Q4&lw&aV&fQbh}prCA1$?D&w64wMU&d=0wP3RzjSzj4ZSc;H#Y0Fk(jOiPO<+BG}A z3Y%+{I=&V?+>4wqB3rtf>)Ll|bo{wH-JUwuKl5F_zHJ*y6yM_Ej@*`U^*ph6Ap+)! zxHofQAZ7jC2@=*^pA8<)zu4J{f2KMttLO^}wfq8WE0fOItvucJVEIjN$Lx|I&p=dy zoH@l~e&i*Y=8abs3%^RT4i8~=x%F3VlWw-~_T<(Ip*-zoRP`|FvM0b%L;eYy-7N5c8a9g6Ire3_ zOUkQ8J>_2se@q&6;;C4y&`Q?j=|oWbR3w>3WLt^v1x0X69n2@nGko`-j+6~U5+~=I z`!s#yjx1W0p*N~k$W+T_I`uO`1?uZD*VpbzGz?*eyvw`Qemx!O^-^1HZIxR?QM|;b zDSdQcCiMw4&sSmm*}za+!joOG*{b~d_1Zjiwme=9Cp=W0etkd2v-x-(oV9qq51(HS zpZj=3Kx0){LO~<&b+(x{)u<>C@PJY`H6H6}zZou~yTt(4VR%Ags0Rb!fC-;W&9aBn z3iqY%Abb_SOx1Os2U7U&v!qhy4OIn7+;kF)6U}l7e`IdI)7O5PM{cn(HSo)^a&AI_TxK| z`6rnsKJB%Alz;u?2EQYK^QE2{Opc$2+Uxgy>T2J*Wq*=ozlzacKR{0id!K;L1$25- zOg+r1UL=4Js!!?0VkyU?+;~GD9e!+!gCRZ^rBiJYdz(`(s=4|dpX0(d#XuK(gM(s?g!mi%i31f( z?$Nap)c3BzCa7doDe{XXFME^@r9F2ySicuh*J_ktdE*Fvae;%}x&bmpwY*}rXy(|2 z4ig~ZHkF_oS($d{q-kPY?9eRsY}`V0xkCZjnGLq;QQeQ`;pjgxk^LrZKD*@GHkNCD z*X3;N=&v4n*}lS<`bjTrhrD`sOk~F}YFoBHYgoD;skQenQ!F+ufySH2=@r34c1$y&6cM{Yz1lEadlQZ zdj2}E$)imf=cI8=x=f*cD^mG8Z^Y^5bn)K6vCc^)!*spIX*Hs@Z3GA)TeJcWWH2MU zJP^4Ze12+cvd9fA@C&uOP9nS&v9`4QE8)$VLKjwMnzK-2pSffX*xQ9 z0b`Z8V|&Bve@p{evDUIELFeyjuF)-cX`X5@$HaHwFkXm=28c2r>ck}|zw2U+*6q|% z(rAxv!=4aG{fuZSgG#r;Ib)4s3jGz)RKk>~0+yg{N~flip6 z4^oDhC3UuNTA0wTPAAB{p&wz> zDj=|jmMGwSoFSi`Mg?_o2ZW5%Vz&AT<4>;UYTYj-%;{o!Prdq7M+i3J^?;}-AWMk; z`{%ZMqo$;v9lVr0Y$qMPpUk-0?*_a+&ab`0oBH^;>)2Z;b`?T8!$$<1$MoV|`#55%&dl7DX9`him@ z`2Ms;wOj&^PwAyH4PF*iEub)X+6#BuYa`BoC<`Xxg+j+%;Y0{^3_boF&ZnOx`q2Z$%jP#9x%QZ87$cutdUSw zKYKVEC6RoryXD45amAq`ES7U}poYa?{NW!ngD@P_3AE_V_tw;+h$wZZ!0k0+>>BL#Q< z=o3-6jW4}PEzHj%=S*`Tb`M3o6mMi_bRC@)!n``5!0c!>1zU^29Kna%U0vvQuq9S2s?)`QGzH-mu=g&ZQHi(s#msc+qP}vm2KPRo9fMUf73J5 zyV&F^Bl5<%H_v~ReO{{9z`_#lX8GQYfr*RG*pV`+i4%M$9JY~`=ZIXGXY?1(&nLKo zY`*%NrNeAj{L1o!eh=iJrTw*3GHh-nuvp*{jo$AdC95p2IC=7<+b_ zZDU>gk3Yam)a|3`>uqoc?-N^5eDep-idZ=56^}^$6dW!VT%r#FaoIKSd#9l$KL2AU zh09Jk4Hr0?w4PG0Lq-YUU0~|$p3#FACmp)YAo2{rRj2Lc2?#d}n5Gcb&N_Bp(#_SV{bIsQr2pY9h)Ddn zIjldfH!=EWJT(hk4Nr6s_M30wNn)X(G$`z?${_?90m|elQYx;piY%~dC1*-v0)gLI z1Bj~bbY**WNGAZ{NF`Y~eg=URZi7(~*Nwk)$=pJqRB+9M@qs~Wo0D8R1a~4Ju+lZ; zb*Q}RmGU|dD7)tyxv->n469=V03g|q!ggx#S^hftULdIyI~igJm%@9ZSin zl}%mE%J=WgskOZ=;Uq{n=i`ySr|C8D6T-K4==K#fV!7#V-OG2S%V8?(U#aiue+V!4 z%zaIp%TZ2y$DPkHQs3SI@OSAZO45fapLr>97okFQ^{aAD)53#*iL5CTf5$hS`8(AY zgmk89Kz&`M71G5Vdd2ONY48}T3xfs1$_I=!1GWAK6->XZg0oHk&I9bK%Ynp_!1PiW zs2%soZ2bsnKm|)04sO!4P{`-dO_?r1Vc1aftUTw29FsR&a8Ob8@=xAQ%+yl`9Sc+F zo*L!g;!82rjCVY9^#`m9U!i`{>}cOj$8tdfr?&Yi=ij-SPo58&VP+u+F<0m~8;&-I{&dcbCP>@`7$ zc{wCr%*;njgh6RGNj$%@73k(h$f0qfzivvlqX(9Zs*Y?T+=^{Rc>l3!#!pe0B$VCo9Pv?I~A^mZR)O9vuc%FbRHjJ>xLvW3=$EID~nbtD4t=S3WY^0 zJtKa#2?9^$F;V*2iq`1|^}=*~FQm~=^g=_GTmJoQWh-Q3`Skf8ud=o_2<}5MlKzu~ z6(T?XJV6VIS)xPLF`JXj(_V)pBhhO0GNF}G?x}zk*c~*wBOXSu$%(CdNbhwteZqt= z1YdWTHYU;EVpUKtEc3o^#eIGr7n&y_)UXZYKqK_)16Wx{#rR+fYb-kcqb5u=`*+G0 zt$JEc?6vKgK6h*NAbl*!{NA!GeO&t*{VMwHXR|!8Fmx(|^h~3!2b2?2=6WJeFG#hs zT(9!(Ae9yX#N+kXfonA(fiq+ESo*0(l1d#Ro2d8v1R&i>$AvRS}S_L8XI8hQ=8oy zU||Z`&EF5E*cqSCeQ0TiebP-qzkU#z?BCz0>(P?&Z#<@e;FK6)4qi1GPlbwtL$>WO;Ly zvxF;~>~h`?Pc2yxumg8+3ns=MBuRgLFHIX-;Fu3xDxs6HC&hHW5-zSC?etY)F9l^7 zKrW;kk)`wrsK^h(R4~Mk-EZb5I^Qy{i7ar3LivQhfIpj`kELqAdD$)5YYC;kdc+H? zO^Zbj?DbVCjZk%#VMMxrLXChER34k;4ny>DxdDZia75;3CsbWIxbrU!6}Fq*v$}|) zW;r3&BzXkmt3l(RNtt+K?F9DJXhO}vzO)Hw(u4COyGPQ7{83d;%&v}^QW&)GOOcTc z+K(PJHG~hw5p6l>>lk^MRew94=p~+UKwsB_Xl1Dl$=E&a76yoDEmt38X$DyH*4#jV z@OnW&5F#j1U1N|wo1f#gzx@(gs=MevD=xbSB{yQyD*nqc)ER#)3r?0USy^J3opCZt z?7VM%FY#6EFc_ipE@G&FWaF#e{tS5 zm?ZpzBTsR7$ggN?di8sPHJ)-3EImk2*7!(rqFqPDx?rUxuA*iRIc`!DQ3Cmeq&fnz zKBNF@{RCtp{vY9NwYc_9#wI-TP_YyC0y_9v7gwlOU4rLSa~hc9RXVn~#%%}}p7q>k z*R9E%abBr4G}sj#xPm|6Z(?oF5j0pk1-S6a`I$}r$c!%VE~Tj)Fb)b{JZs7Z@;@Hh z!P?mku=a2qIYL>*Mtr~CSY(NH2WvaTl&AK1A^K006P@7_19>=Tn3N2ZVBVv?fm^;S zuGqNT#DEGp3&puGPE*keBC67VQvA>}rGXK)Gu}AIa!hiIJan|sfec%xj?Es8??*dB z1%NEOYTfSs6%sOf`=~5r#)>ODgmnp4T{hR1fztELtSJwJ*HCb?j$!SEMs#-FTycu( zNAFIiavz^a%PD}OGv(~=f|l<;g8AwPoUW52+X9_r5tTSoQdDC+rrHzi5=qj-9LbEO zY815GS0aUOGkc>+C@xY1p9zR#+u-de##xWl?NqM<jwaRfUPLnB9bZ;>-Fbl&s0MA&c^q zNFL#-qmDXwR?>9w%+*3$T~#(^6of$vo)7pDg$e*4heiT|k2DJrxM5p(-)+b_V>LH6 z!iBX#(O4NBxW=1jr0-TUatr5*0FciB1qsj>J&bOiRfgIN5B3g$ z>j`VL0c8aL7 zF?}YUa$FKGTGxP`!!-X+PEmTcItCSExfOFOwQHc+HtmK%^T4uc%*Xfz)8}T7zEQYy z?as#92!aLPe?>kT2&!Wi1gwS%qchi~VfemOP1WqVlHjyf$MQY z-Nn;&cb79Y$uu=V1-D3Aj(5j=EZ@b94b6`EtBSYEym(-^8*C>Gm33G~xzmjaw*;2& zXwB9O^H)8~0VQj$*u6ZpO?v%7rhR9pJK z30OLU$Sd}n2r8Z=O?X6VeBWVJIVUL&C4ZcfkM(^+>!An`!f`z-PWP(}!{C7;3KxBl zjZv4|U#U>sBCHA_=NJw#0`W6P(AMGcu<32;5$GbKn#UEAZedc1`#=jZGDjJN3B-+) z7STY#*W|(t4zCRmS#l`5f_xK6PH`l0*K00(B%)yOWMj~lYe~eG;+!O@nHTgOZD9vJ z-{0c#f1WuPcL?^NEqKRGO*oMGKvOJ66HfLV83?FaNm0xfiCnWfPQViB(k(q9fw)-q z!*UJ#Xl4Nh^WkoXCL3&0*`3X)M%I5BC(eQJF`h_hq)x(2;t0JRJ7q``qiDKy8y6=% z)TSbUmmLZ9+Fm3P;9jHhT>Rlr*f|KleM7U1R(5b03~2}G0P814Ht$BBet|APhrLJy z$($1;ypPURUTv)5ajd<6K(le9D2%!aRJSCJm;d*~SdlyV!M+Uo`MluOKyCy%8VEX7 zq(LX>*~*Vqx%1#Uq+lY)jwD1qA1PF_Pu5oG-S^{@Q{&H~oI{{cAPFFbED7bBV>G}_ zQzKHsT{+QF)W!Pa;^eETcPfOb!&CFHcSMieL7T@>u&1nCp`OW=J)306unL|HU>2;j z#s$w(zKoilqeAyF<3-_C~T5O5{jlz4Ln@=p300z@(E(u765 zjiSt)LuYO-wapS|us8vU4mlRh9R*n1BjtU*SObPu9{V9NTglSjf#LbdhpoY`8cUu5 zN9j`$b<>OMxmOvY!qF68{a@Gk4x{YHp|^x$(;jxwHq&OA+6P7U9p;Sl$E56kKgf>H%|ynYU(n{G_@3HN|`yN)8`U^bUdDyIo0* zjHATL#@}R(XxH-Wa0R&8-nX;9zJL2k;{1k_L|UPtE|=3u_&!26u0no)W3=PlOhe-C z0WbH~A^fNtMYItGNz9Rr%KDwBVKpGhJ5TONXF1K^#SFm7Q61%E5wkgacrtE$==C!3 z<)KM_QV9|oRBK2jsc>k)c!pu*T%?zqY8oPGk2r$M zpbMEU{6r{1tqnj%W-2-*cLo(Z;LF^bwf}03I}(5Vk+TXF&Q(2b6S3Da7yD~6>G=HQ zj{F$c$NT+^zjv9(&kWl7g&OSB=xbl>+G7BRDEMOF;y&^TBC>aUjYIXXLA$n2P4cV& zxU-1){P+_C4&LSUIL$dWi+A@m_&u>YJG#V1CYbu9;t%iOyKe!wriLi}TLhnbefnCG zSeQ%9)yXGhAhwp2kJLq#0gMXaprpu2{%~0@&T<3o*+?JX`>K6x-x~xSurqPq5rVoJ zt)>k^_iC5)+jQ$+oeoZ*B3h}#g{BxB9qHz{p+YUpgr}svJr`;xw?;dAH$f_{-beHK zd0+QHdRL!wFLT=d4=w(a;-+E%Uj+n)1M$rGm7(|qWxVpc)SSsASId(bUNFJ+4lUqH zdRsb&6c5^nnqg8n_xfGg zf)??+w*`Aa4${=thl?WOvsR+ zV(N5#8H!xB_)*2fFN#|i+=58dC8d7+gXMqZO|u!V#T;ofDq&rgaG%9H)-*PW;H_NG z;4IcyPK#uUIi}BZSUCy&C+6>Cc0*Byam+XFFt*fDGL8uF;^lIEAe?#D)VQAn|DAO} zbv;!kL-Y_9#N&nB#(1~a)vs?Xdc{L`dwDKSl61Fv3%k|JoY8H7u=G`U0PVu)YX(r0 zeM)FtpB`2HG?Uieb;PzAx1a2TdXJB|hM=_VKY8 zcKi9cevRPg`98l*9M1X_)k}F(frj~l7f+Ztq86-dK+1Fb>WQ0sj3o^9BmwME4M5GM01nm(WUzM?tAWv<_*G1K_?A*Iua2jw7 z-Y=9q>sYsGI$Baz#O;sO;<12+LMST~9%3S$hqX1EQ=($U5sz%H+Tq6zZpSCWJ>}Xp7Vc2Hu`(;6xrLA|1N{;EsfS$DYu~-? z zBpapD`nzW;b2;*BdRD86bc5V)keA{ur$f%y>1O$t5+s~G9TyO;JXvOjuUYa^ZEOh9 zlVbLW#IR7M!fM54OdLZ9Bg24<85HhrSA?t;=2CGsv>IpXI=0XLjrZUzJ^;FqZicU@ zN4cp`F_OKL(@a#xg>y2SAx@=_6;ynLR6}S(NhF2bl1Y?x({X`m>-h*Y5r%Xc=o@|k z8^Ef7tbN~yKhd(;kx1zPH$7S77Nou{X&vJU5nZ|7aA+5z9DY!&>sn30bXxf+5$7{R zc@Tu=D5a5w%4K&@h=>gCEAo%gG1nnk6?+*lWV@IkY@U+khk_?+lkFs}@YhNiCI7M- zq+Mw!X=*PDh+Rw4r$Z6!j2D~xAeHg@E07uZ;r5^?jA`b!a(hOzS1K9M;qZ%b7_UG$`kkDYQ-40E-r?qQK+mfoC zHW&_zW&_WMrHH08GiL2jWMSd+oZKuq3T@AI+w0oFcu~Eh)z^!e(bt>P$EUfH!rbQh zI(Zp0;OgA-oytpm@Z4k@u=y9hZBIiAu@eQ9bznUck=Hb%nA29&Z}FU_3Dc*jVO_VY z>0Qjk4m>txwCIADnNN$tSm=3hA%$#Tr!x_XB32uE0nAiW@MGgxMZ$4QE1r#F9cqEP zu1qNMuvB=%E?IhW!e*m(rtI1bvnA(m#q~$^hFz-q=K1tS?QG4p*?Noa-@5A_@8+hA zU8R3$0PsxqjuBQamVoKNV0XoWWntg6>lV*kbxbO{W2Y!dt6v=>_QDby8;@T#P3+RU zVwyT79E(YRFp?@NPA=|9QE2E1*p!w3SUI+pUp6v6l|2R^G0<~mBR<=Yq`H|cd-}1; z0J*ife}oP^J{SOACXHF0d};8dOASF{l;BT+*KXW1cu;OB`?JywCTTt^8ghcT$jJWm z@v6pL;C^s7ydXAhm7^0JfG{+Y$E0l7`J43sFp>+vtYrT0<=TH)W;|(w2IT0js-kRV z+I)6*t1G%&JKa1w8DP2>kQ|%UiseAUul#PmO|SJkPP6MbvP#_m$7YK^w&1%+(k%hq z+3}_)ik;K%=$;){!y+s}Vlsi_IT=Rl_68rNkj2s@7vM_`QOXrj=qoTBJ2eBvgzA55 zT4H9FWa*O2!;;JBih1T?)`UJ&;b?tkNCi69HUgEi-Q$^k$%eRp#}Ab{?D72C)G6e; zF5+vk9gv}fdO7LJ^qo1*4o~40fh#S)MTzPNOL>+QTUi7k>_s`T@wmM*Piosr%9JB)*{{ z4!$mF!c|J8e?;O9AeavP>8Q(_JPkB3{Mr0)~mKwLl)#Ao8wR;6y&o-l!yq>7w1zWsA~ zjUK}}1+~pf&@p%}#)orKbZmOC({)(Q|8`gF*=%qMTC!;1B-ZC`*v37-I+|PrY~SqA z-M#x8gNMv3#B|5*+EtWequSDyV{Z-(h7@Yl@Z0(GYU>e98{0=Y^%*hObSp1+cUN#9 zAOek}hat=H=EqJcXsXRsMuip#y2-fn8VRk#*bl3-j~gr6npOkb{hP8EL2!@NmJkE3 z0dCK%Pp*UZ7pR$3pjPSIgLNK$RBDX4TNs9fQ5sM^!}4x`ygic_66b~0;%2R=D+woYBHhKO~B7glJ;%EQ>?Einihn=0ik)eyR zg}&V{s^I^ydMs;BN zAd6NiK!a;ND-Z$-p$Z8-twF(+qr5TwS;&2Zn!c)C<^G(lU6P8IX0hM-_ujMHTin@c z)lYTB#G$6!r}9Qaq^z3KvYfoH7E~!0=oTD{N&9R~T1t zLnQqJWlwcSd4gw%Vr6|XQl-Zko8cL8Fh z_nhC*wQF-BGghS2Fb`L_HQ!w|zDlM;us~(Rlj3V`n#-c^qu4B)tg?fBsR^)SiCx&ge5*K?h z<7(Uo8YW)vwYFv5j2$NH(Fyc?Li^S}zf}d0x}E5Fw{~0!ltHGLGu*n{iZ-eRrHfX4 z)UigSYc#Q7VIS^^;dzDOOhpZZ{s?%#6y?bz7y|4t?hirYtyoEiT5g4Li)h6utlitY z7w*ZlkiT&_ew`Jpo?*SQ=46spnVgABQr#3)ajf`^E|R^CfjcH4Of*k5UXYu&d{L z4u;FCT`a6)AHjWE3J=zH=dmTRYEB#k)TVWnNpO>mt-w5P%~Jf-F9j7 zx~RG}b%~fjvC_Y8pWK2tU|N7~#AV?m974Xgyr&77?^3EGe=iBimG^+5(gW7iF-|v( zuwfkiUM2#qu)p>_YT}TOJDRgsy02r=JZ|>p3yGDT1FR&6|I_&6&3ZjMzxa%8nz{2O zX}U>u9)|OWGheq(!2P_8hf)ZVB7CeRs6!is4*%M+iE}(shW(H7gk(Ct@glyubJ3b> z)p#o!@T>Z4(T_ZlNbS^Ow*c&;Cm_}vGNhGqyo3QC0^a}&c5@V`nm&!>)jab%igke< z`dEf}ZT1+iBD$5POnhN-i5{^iwT*uT23+zGNXRq=tQ?(*(yGy~1w(J@047g!5QW8c zp3{`%P0=R?w|rCm~528-gEZ@*1-he}{Ow$MljtuhTk zwvm)%X+?r`aS74sqFm#Q<$)$c?CAWDYE!QpFH1OQb^mqgSFm-W%!rew7{}-H#M9I# z#R)vsjjv1uhMSSi?4rnYL9Z{ z5_a8BAjN95_~`Zqxf&gTx4YPD=r_9BQPcS=X@}lC(LG%}KNHRPz>tLkuHN2xb1$ z;*#h3{8R=z~i7CQ^87r(a8po@iYYw`Xlq3 z>#4?Yi{%r7GHhC@qLGeJ1dd8yeXNK7g|~&C45waJt|m?f0RV6&3;@9KKYrf--&Vfs z>}~r+HI)9{M|B-jdRi!1%PL9gY9$LO5o60`5Cr)I>s=CcS({ncOjlH-tT%Uz5_pD) z7)aS9GI#87N#W=izf#Mo#h(z}Pp;THpE`b{Rb9S4 z#Wt2DD{KR^k1g{(Dfu&DkP8KyDJgMF!gAx33ZQl?361S5N7M*~l>!jYMi5qmWv}5wwEV9;QyfY|m0X3MpMZ(qqmpKt!h`mw>RbT?l za4Gd`0zovwGvXrK_l6po3^uzBwq#b}{tKX+`~!K}HX;NV?z2|I^2ZUQzTG^eYJ)PA z8r3TV)*knk#&(q&+ME%DW2&NU-2ku!d=c zY5z|wruZ!?09{KgAHkqfOvv-l&=pTGR{f#7yYg))_fZhcBWYHJisOM1r5YvbbTK=X z3-#2x<$;)m3#$sPI-2GF2PzGn+EnW2NheglP7tZ(ti33y799YDS+o~mcGVdiKse+K zW=K-&4J!mCLKak{#9ARl)`ajdvNlg4=)lxQKvv8+h;oHXqHP^TTyTpU;5k8E6Nu2| zBkZ*E^7JX6K$C{Wq`M33OoGcb<+im2kd|!$Ek#9aBouD<%*e(qgpk;^ON|$-$Vzxo ztKJ#~mqiBuQW`Mt01LsXm7U$!_Rq^D#eopoSx zm5O@pGPZuI8ktT_(w=jDk54T?cyqUkcJ^9!_n%9i8=tq=qJeGw*Ss$tcaPINH+VL9 z-+!ieH=jN1_Or)Q&mUBqDmyN881F#!E;K=j1`x0dv02A8on!>UW5fw3r`P*3G8=N> zb-CBOzH-@URgv zjochCX#iA(x8L8NQNN~^eO(8#|3ygh5U90HNM70hdyX3D0k2*87U%Zz`*yiI%zv?i4VHb;au@CvmW zYG%wggEXTYz;eayJbOcC++&&9BWtQ!WZ>Sws2z6#usNMkO(JNal}NhTL^7Gx7ihdc z*BR2#s3+r;0gan{D1uTd3VIjv7>uFpNfJ)WaI zKjw~bFPIQ-H2bUB+{JmYf8~stO?hC&Am2sYLiGCbr>WnEDACA&tq?*WQzLBGm51xK znDA~KIc0CrR}o=VRaTL@afA8GJP)l&BL69RG_wX30CeTt%?%bL05|pYiP?ym{ixcD znn-zNZhn6_ckJ<;~$q z$xx25W@opc4Qk(VcPqO1l3Gm@|WC zJUoeRG%?W|=swqr1FsR>9#e&X}MY`p2c zb5c8I0y!qpl=Pn#b?>CpM+}Y=4w5EDvqljxv}D?Mb{C^YMKaq`j}4AVx=6+AfUiQ? zrv4Ei`|WNl71YOC&+Wy=>P$0ZavM26yJi{wfsD)|9lTP@NE&;vBda1fd`QQRp~<*L z;R}W`ec>_%Rxt%m)9$eQhDCY$Gq2DNqmxD4#M_G`9&)8)HayT0>VaprrE*JvRpg?7gV zBA@-)3tjC00MWc|UF8!vQkxEi3U%$rs0N4n>bRiHL}h7)(oNy%vehh^B8ign#QC5^ z4dfXd&$u=4{WNwz#wvZ{uD(2J1pz4tbYq&qAuM0#S1IJ+OCt9NgU>KSP z`}xN={tf+~D1qlnaP@y@DH}(>($e5pTK)#E{{JFsex;>}sf($x%YT=Z*4W}GeV58G z7b8%SrV&czNmwY_vLZO^8csrTQxD3eRm66f%w zGE4Qz8k3a!gm$lXGnQ*@$|y^E*FJyxUcLI(HCO-UP!~8`BzZ8)g|!OAw0wk}Vq3>> zbux`23n|8>w@%@(6%T`VEmLivp}k48*;*m3lA~dDq3uhofw1!04Aw~~i>^YxQaURd z|ADQM2y!Z>Ql}e7U}h5|<`JetwzxtL7xZDthZ8ihLXs!AAWDki2{T$Hu@DNuJ&Gu- zp2T1;!lFs6NPFQymRsC1-iG-n^bsE8FCP2Nj3= ztQGdgcYK&gcd}^Z7hVerl-UYn!sQW+CgwQ;aS5CVB+rJrfU6oe2WA||9%n#%1}*z; zm^(pD1J|trV_>{0bZUy)*ksl7&_Ofs;Yg)c( zapSssVzhF#E3CTytA>i0j&?+-95)4*FGEExS#4mWE7XhxMng048ZHUG6uh#vj{y&f z)wmhlHHY{Bfv>1fbRitz1RpOfu3~@~j`oolVO6r=wZxj!0D1-;tBcPRVb%3T#~2LH9)&P6>1J-r%xKR|ZJ814es@9VsF=gKL*8MTduoPe5uRqC549L4J8D zDdeJ1xjaV-#v1re%4XX^)+AA}y}o^mxtiHU4c!U#0nv@XRj#gR(YkzLSKCjv5smG` zR)eEf(pgXurZdV2f)CBj#?=ilq2K4t()T(vuf99;%L2U|`DU5EuiKr~_kO=0SB3ItbLo{dK5ec-JpDO?a^>-}r=sHl<>2tJ;}tyWbuK}ohn z1c&Z(O3%N4e=H7ENQ3dJX>t|M9W40V!1?UwD-WQkQz1D-z5(hvN5D?6paSkGlj-%8Ca zuHrFR00>ztCPNgdB$$-}q(P+{ksXj8Ts769=${zl883Y|pkS|aS+k60%2oQ-B?6^V zJpkt6>ucf%H;sC8Ku!e@LZd*f+W&~=Bv+7AQ*cKqQ$$9_RMHJwEHyO0R9_0U%(g^tANcjS|hVxFfp^Zd2Gjs~MhsMP4|ncoQhhB#Fp@x`;MCb5`pN;xzVy+yu+u{UQY@4Loi{(+#gGs<>tpXjPBG zq>AfQkigbZSf$MrFvwuVuUQ16d3YQJk;O-sNiiHFDm{_qphkLU8e4;dB<+ZoV2*|h z^LO~5|1cUwuF2ej2X^sP(WCNsqQDu6rTrNEgcBb%@(X1bri>wipL%D2@c}M2+RcII zE2tg4dpoNWKKO&UUc8TB@6;EA?@F6%@g~777giAaCPEY zb7?_H004{Mit#^S7k`V!|D_W-qpfd`J%;FaTBcD?pDGxYc!g4)2w_0NN=Ty&Bsq46 z+;CJg9&%$>=O&)iY(M~!_*4M-fB+&9MMMFGBB>V$axJ`9e4Z9za{7yjZhDRD^6GLE zUywUQKQ}WoH+S=8ZsMk9x8JEQ$U2MnfX<60C*0Uv|K6_Y$?4I3T0xnG5cOfoY@aqW zmI8|3w98ML3uN?OgTzH-CqT>WM7vkyAS^%LNBolDfPG@S2cFbvZtDrMa|rMpthDc$ zRHa!%;dBqsK`XQr5CDgr<7|N>dPKYz647NbiZxfhiiOCWcR-p+vRFUi!Xx8aV;Clg zHS`D9KP@B}G&Fi5f$8}+b7sr3Jse-NJ&DV5r9oqn-v}Ga*}11>;J|d)ZIMJE$on#s z^3k|*2&e}Ki@8IXZoY?M#NTg__eW>4yPGZ}{4)lNQvc`>>NB{k!2;pKl^wd^TXHf` zDGIpuTS7dm!Zv!iJd2jCf1#-pLBCv$#ZRB7{N?LbuJ0E|_#D^yeaqAGZTmdez023~ zWsmTiR&H@v5rZ}ZJf>IgULb~~ha3{MW?Nn%66C*{HcER5h`eRQ8kyX=2aJNoFX3-= z)es)TZWjH#SpESx;Nzumu#OlBPL~JBtkx~N3T`Iyw0FK|#_awX*(YmbJ0=3(WRGBI znCtbjva7i5{n!lj5|*1uA5L%9apM#wKLG;!UMI2eR;=jUBmiMZM+&P@JSR!t-!@z> zn$~QJ@L&6T@0(sNuBTVNehzhiTz|U(1*`MN$9BLY-9?8x`!Vk;N#u2d(TRrq}0=Uru8Eoyzc%M_J_E}CP`SWDOxND)u_oRE8)x=%hb zLjcunFRzE9J!0E;ZEj%u0|PNNP(<4DSS{ntRrdUSzNb;s&2LHaUss#Ab$tJj*V3aa z0UcnXb}aLQF*Q37)V{~cGA=}0FLJbA-)00a&uB2?E3EUQ;NA$|LrsrFTU!MlpZWW` zKlX;AUB4&9`M*EwW`7=bW^cPcA3yPRzu%`WKceyY5af)?d z@pApXi7DJ)mX&*^+VR#c8AYkn-EhllFS>>EKnEJIKjKU91wy$3HjuzP+%Aq?;P}Wn zvV|fS-PcBijley3ONU5KM}a}wD!J|1tU>jU8kCx~KzZ3D#H6=V&V_k8?+m)})V-IG z6a*wjLUc+)!gm_M9=tEEZuPSV;)Eu&!FgDPm4cEfeyTVW@mxGPkzt|~(f1b&VIaE5 z2?z%j2rd=CMVAYU=0Lc3PZ9-H3DcMh_(%v-rKAML;LxB)J7yd(R3V&w_Ou#55Mbj8 zGP2a71Z-cVLMJVh14z&0lj1}cq9P>_#>vJ`pH8xCQV~O1QWpb!wF|XvuxjH#2^>K& z_gnFBq@Dd7{ZY_3rM48{lasX}NsLk^6J~^A{3)rlHaLy;`q`MQQq=2+I+h7gjr?TT z0OI^pn2J`abxp0*$8kn5i!2IJFhiJ54wT!$00l*zMjWU_*L`yWY9shTFK1z3=@;mItUsQ*%ebgD{(-vf1N-VgS;h`U2j8E?irH5eo^ zu=6?98itwiGc5m(yHO4D^mwfrKAeZY$`@DWC_4A}7auC>^fm2kBF2TO!l4RV@0lLm znr`NcSoQ41Yr~BsUV?)OeNtG(=*rPx#TV+#cd{1xoS_=o*C?hAyvJ+6r#{ z(ACtcg|ZiwW7wp~z%c6dr`{Pq9rt#dBV3b3Bw4^W$@aIg1yV3|A(EH|nv$}ANs49xwoG2PS`%6}pVp1u?P8kk3srYKb*!`{9*DVZ zxlJ`}YXbC%I`eJ&$}?q`2i05esWD*;e=cp^ihfL)DHTAlRzb+HQgWk4M3fWZu?!$V z8A9S2hG3xz(Afj%%vD6fl{++A|NWU!S+Ok~T@LJPk}GAgbXML^!ofaQb9t;&*<6uX`K8u7 zRE0ZTx-awU^WPtGSu{!xz*kRY;a$EHOvI!-!ho1qMHU$VgWB3QF01*OVTJ%f;` zXgJkw4Y-BxXW_@_>0v#*1Asa@sj=NUN3xb_+R6n2na(iGRp))gs6o2b!3Q=87UW+sVC%ZIJKh zWBt@=6$MGE&xHm9>Xv5iP-D^(#avXDioph|AK0T8f;hII%eu|W04Pao%c-?S_O{Ib z8dx+H6f5mVWQIkrTlX^=m#$!eCU-D%@-;l+v~O+7GU8T9;XfbjqL#jqjP*s~?+`gS zrSG)Foe|IE*-5%A>M8~4T$GN;iP_ET?>ML;c@vW&d8dpS1NLJHvr}4*g1Ny+X2pOv z&?a7xdWaHaUYL;8FWCfVn1Qo}WEC?7wPgObeEA0ivmlDct8g=Ki#H zHCv4=PSHKo8SM4=%)$TUyY10EdPt!LbpYLBsVRtQiG`+$lE&Asp;1fiKjmE$$7ojQ zGEVQTFg!x&3qfkXt~|o?+_c>i3k_14V^%Q%ZVM8a3ABC4<5vRG9(S#`^q(9gN-u&F z^(*J_i;;qv!8;oyWRO9B!VzJq2X?KG1;A=fI0mxuZ(VP3fr()Q^0{*D_V7*tt#q7S zWoXlt*dpv0AX4Xc6bI1to5x*kNu4_WO;-Zw8KFyWRa?o(Qts95>)5cU)gy&|v0r0x zIpXLS(|@U#W$WSvnInR>8nSJa!F>s^1TJ3>kM1DTb=V!HV{1lon8s~`w-qr}xoT(I zx#`&Q0G%T4isg<+4g#ZC$;{uO36!^<@oW+iy;;%DtFvwG-`*G%tC_E?V;k52YwuXm zsA#*ehIM+zEN8t5pudPoK@vD>9_O(zSa$(8$w<#7QR*?aCK{$S95a!2qFEZ{ZkKH} zv*0Z!%+Kf=XB&_854oaqJwyI84rGY~fa}%(tvgF{6(y%ahr-t{rhQv+bl%0rg2SZT zR=qxVL9s@cm0DQXDi>Q}gAr+($(13=NK4K@?-H3X_^i9+D*Gu`V9-zJG>$Bi9|gPI zAcIXyANFBTr$86P?NBcV0{uMtVJ6+7tey8r`?!Z(-`1uAWMg95;^Az^(AVb|v-9Wa zW1XKK|NHA>&*AQE;<2r-+w0z-LczL)O(f*N$B9Wo9p?*`D%Jn=C}O(|&4?kC7TajG zBNq0x2p;uybjy3|qW$85X;bM;Q0L@z;xP4llX`(bF=0sdWb+~YZ(!vDtE#W_aJeU7 zu0rW0L$i(GheQ6n$1nLdNq>FF!{gy=qRsy7d4TX0XONkT<7w}8ap-(W7vl|KLI$LYOma?e8YVky2=U^GMC#@q9Wl(tU z$H%^Y!TJ7e-436xlgrcMWtqL-*XJ2*-Jp58Hk*p6X6S}##wf{kkgQ2oq()J>iM|0o2 zppQIHQGW)GVr9y3^14zsX}gj$TO(dEP^;$I5C0vkx@Nv%D*1@VU?zSNI#Q42VOppG zStuq&Y^ZU#iLH9k3PyOZX&_&Y@^}igW$!FJ>bZ;Z%^W=xiw;6`Ok~{wFp?f593o7w zV59)TRf85uosi|Tc|A!5Z;+JKc82FV=Vq?ss7V85nG8B|4@NMgAS0V8BWW*62@0X* zJ7>)uNmu_99zzu{4>(9JZ3s=Exg1dz8#q?RT?nyLr^@8SbUt&+oFWCRWJp)0s125U zg}&$_#Iyfk=(00v889u$eoBlP09F@osFjO$c-5RsE^W3sfEk^F9tuH7zC^zTlpzKK zX0%Cb2_p?>)RLlwqOxLi%oy~E?v<-m>q-c4gNc|ia(CNA3tj(MjN=)?(A#>~6$2it z%Mfaenrvd6S~WRIAKks!_?I&E%mCpdgph!idtt=;0&<)A7D;lR1w|FWNFmQWjoAo4 zf9{~aLKwz}&}ljkZvy5D>1gKYo8x5HsC4CDRMkfR46(9QFj=Ro!?%4JSsn$+zRsDydf|OEFZR0b8OD%3NTbpHuCQ~RYZG) zUC2f?Dbb*=S4EZnr}ih1CS`8C1*r@~-}ji)RH%=qY{fR9Afc~cLRn6(3nOnGX1)Rq zgX#^B{~Y>wc!%P;%uPorLyiiQG-B8_B~>0tCniTx2)Ea(a21KXfP16T?2j->mV)2E zu4OgRFO6>141zWi7CPrCQ@f67TIF$C$lB4rv;}bgXtVXCvx9H$~V=u)3glH4MB60zM;+lEb#-@xvpX-GjRwoNC{p8c#Ayf8(JMoD z-1Cww&$JIjY>W862s@|XP@?Y3$F^olNFs=3n1b&DU@J z(%q*{o!ZrVueE-LZWvQH*IB1&{GAQP)CqUORGlcgg#=JE3u}Yu5?vY5$7-GkmlY3Ja3n|36d8IZ1O6Vd0Y!n*o1xN)9wwMyZSq1GL5U3Ze%_G8Z z0q=U??V2S0n*WCoZf~~xH`BBC#mr3HM(?i%FnT{MKL=dyJO|iy%gXZ0hV?a?hd-40 z7~qEm!?VvSJ}1S%lN$D(8qd&d!l)f5@R6tmMClGfC;=Yv9ONFrPjVVdpD&77+->0V z1y;KN)zV_&`-E%Z!CHP0Bo9~~ez0XoJsdeucw9697uFvFusCqAZxRRGGgyQS&N~iE z^5sS;7oUuafAB^TRl=VF0lm0UsDLrbMWc>5I@WCFP3u%LF}hEKii;)7g(o$+*xFs) zH81u3=fE#K=%R=a^P|e-L%F;hU<`d=uzP@Pi=oVW;Q(<$A(KY=7(CFqYj2yfwc zFkunpm|TiDT;PiuSanl8Th#d3FT=YHjKcfD@P&aHnTd>WazZ2~~VquGJuf)bwpzLbzSGHks52_Z>_ zm=;apFTGavF35|c(F+a9kqIu>*K>ajV^DA zQNb5plo9F+{0U6rf($JZRO1Hs_E0gtC?Dsw=c5Ulx*gubry9RkKr2xGV(^l`Hqh+h zbGVG&vPt88-Iyx~!>PWlW&76UwlrQs3VW|e_sTtLO3Bj{lzQ;%<0d`+n-G=f{nEYe zmlNjqeH>k<_jRs+O7EKcDEPjs%@A|bb_*(uLEIlM%6MIM)!S64nT+eLqN4l!qRId6 z5v6z~4hh#GPo*$*S3CtP$In5UNB|HYowt>6F2WG0{*?fe2G~Xaoo&coIHJhHD2miX-6~gtjicg=cS#X}v-(1kqLdjiUvN3tjxd}2{)!QP4iKiGG*+yt zf`kWNi)_)1Y$&+FDk!2*ocE_dnG&Mf!-R zjDY0F;57&k#KMLF4S75;Aef&Wm5YO}x0?^Snimacpoa zMOUCfBYB&Y?FfZ;lGZ%+rzLg6v?k7pG<1R08zpboVxsLtDjrm3Lo}nZv*wcO(lymJC*xRK)~k4j zoj_?J4U0dwrDTg#neJh`e*7x$srm=pFJsnE8*%uhdSzxfMYWT=5 zLHN>v%W+wAkq_adA{*jqp@qxBDCDG#$@odfF8i<}R()4llf+lp`SjX4o9Nv8-%M14 zPi`kG();mx(4Zp>l1Ve_1jo8EE*#rURi&**gO(D=xltv&BE3jN~63_M_ zq@LEwIIk9yB{=bl2f}prV_=~bBeF&34wPRiHm*_Gm^YzJ>3`AsCsG9DWK67tZwa*W zkeHP~iy(f;!g@Q-p&v>Non`Mb#+>vCH<#exl=P2<2eHu*Cyvvkvf`9;*h{9o^Lo~z z_c1JB?2BMbraL*z&YYR<vtR%o9aeGqSDE+G}+Tka6q@q~@V z7Fg~ILZj`vk=*maf`#J!A%X8SRZN%NUz10-y^pGx>d;ykBdQ|f^=9RlbHjSMgRZc#M)bSIt%8%eK!~w3Fr(+MO$4W|SsGh?e;z#hp7NUhsK#fR)=} z%cBGLbVL^g{*0IUqr@ZbW~}Xz*$d*6+(mZE>-D_1X?@i=K3K6yHDk*1NXwl$U)rHh zh_xl6NYfdumhw}6z%Nzd3t!U-`%uLDq}qkl7>)FXB;4$dGH=Fs!+ZXrmT z^bktA|{EO`?YEzj5mO7TE3eBUk1>G;!;8(Opj2o(%}&%T-05zJ8Z@+2)%} zL|r9-AnKR`$VPXAIx$andsJLXs71{+vrWD2~)Wd-RJK&RO$X&GMVi@|w0? z5j{n{SB~t>kKA*%CVRTGMBxITbpd!`D)P>a!x^Nw7-)|^xEPcP%K*`${WP`AbwSsOM#~#$8=~XxQ}yCR8G1FcoHZ%trjzic32;wWWALEPB0413A@E8MMs)dqNaG> zQMl|!3aka0C>xzMj*ij9pz)Ov+;~)KJ0?{XRQhK2XB#Ke-=t$qVVpMTsGDoBEs??g zG8u{T>TL)iQ(KKkX{DmJn2KIcLMbB$o3``&_pzswml{iKnsxd_(~k zr2i|Y2BXPv&0u=~m+Z%`qSTrsVzSFeRy&3Bt4)HhQwP2tB1xWgeP@q^;U0*T7Y;uk z$T;Y0U4(Of%oCYvbHMh&9=T0&z}J@lApeM6k_>`*Nq4wX*G4Mgag;^=F2Y1z)VKbpnYC9A+EZ&CFg`*9+$d87p+&W*VO%r$ z*YqG~lim__W*e3Vm_lT*TA*K^lFna`6SD#5PasCB6Fcpu2|%`n?5x#qY?SS{B9gq9 zuF5#~!X(q=uK6Y5A-q18FxXFMGL4rGQeij_Q%G+;+UvT?+x&N**8V6*>N2Ad?=j8v zU@KJ0e5a{H2?g5SLexFJs)e%Db)=l=mErxS5Z=@WV$H^X{*9y|rpf30_7TG<{=3h_ z-tM=FXlm%BZ{}+JAA+Fitl3{`*KaHFn>yku3B@j_T_B)D8<74E$=K0IL@p{1W~9)T zsI^$q377qhziB5bPYY$ubS=zPINeaR!H&XhGXM|limSN{h?q3-fSyt{#b~o+ANkGq z=#-aF@zF}XR;3TMe&5R(A6}fWysp~MqoqelA9PchLm0zF(f_F3*gZl~;GI^zU(O<# z(Pa(phL~J{ErW~gUXVlR&wfAO8@Z{bj)WJ(NZfE2{4ahRs?P! z46t?pBF0}`$Ob4dDKFlHl0KRvhDBs4lBF+H3 zOa*+rM9a8+n0CGZVf;k%0%>xd2git9LsXC(f*e77MNDk)g82B-f}v>H_-d^yfe1lx z8Wx(lMzlV1*eX-iim_=mckkOj&~|LkFW6n%YKuVL9|p$u?oG?K*VjoaWq%sHp?y|1 zwyS|eED-t6h91tL##k6hBd;K`Llam+1F_{P|0&?ZJLJ<$W6JF#1QvK>zrkrD{!8HR z2aL54Ad`3ih`9p^XNZy6=qxYND!B)Adq1p@EyRDHgN!JXv^^8HUmEnAe)3|wh=(lA zI^;6GA~<^iTlAZK0(v&_Y*wh?a0miIVgeysYXF{ont$g)2PsR{g1subYhKv6#&Pk! z7Bf#|OYa-qdvs}mGI}su+|c0B?DT%&I`@CRINznm`+B|D2~HlB_w#y~+%$htr;fFA z*T>qnaqn#Nn)-zWp1#%Hwb%JqB?O#(t5p&-h{K(3?H8@y^>I44E3MnxJz-_%7E>>m z1I>;mKRq47y@g}d?)<6J*Qo^RI9x!|eVU}l`ANpIdD;*Ck~wxPmMMg7|3HZRW(x$b*3V62m&^G?&6a1f;Wd}W5=~V=!J@Niv&|$ zz?T+@x1w4)uoq#hEY)u2*OM!p5U&?tY!8Ayc6C-HLDYy4%zXQf5Z*5y+yFg@9WAYs zgxN>YhfJJ2MF%zA+MManC|$oBNt#2BNG7Jq|D#ELPH zM0F6Z@Nc=a+Zz1cbvx8~xjxU&a}~XQ^w=#g0@zS%Ckog}uK<~J6PAI>b-Icv}t*Xkfwqs9b*PEQ<$beZY%}5;9?G!DwQ>k%lfxR{}P7KJ!mKu>f zX)F*4>Mp5MDm5ueTXI-|cvO6L%s(T@%e{$(?3KY-^ z!v^%p8e~phB3l7SM{Zb(t?6;B&Ry%Lvq$u#5}bFSc}iUXcmt{_@mH)#p^)+Q4U!@F zq;29PLjCD)ajGlwC#~;S@3lronJ_bx6er-@OjGe>~(t8xrh~lhNrM4#%$EwA2Nco1^gBkjS#d-QkN|7+BrBDPYz3CF$ zWFm6KJd|&Va)gnh-w*~k7%g@@uqfurD@a(#6cMy8I!IWkNCcv@z0fhSTFU24U?a1< z4A7>293d6ToB}ESrV9~7QO_$*2C~4@N@a=)M!{Asg$aG}LnLv;0s$tO`$zU04s4|I zvYP2`UV?6-WZ?)>N^G$D+0bB7yjgwe2K{CMVFS4oQT$F;7pQZ

UtCiYV zpq1pl3@VlBu&;V!*mD&bv+6N6N#*+ppAW{;VH_c3Qe2e;L9H% zP(k8iOO>)xECf(kcaA(V6GBpGRY6v`$pkhdF0@#Z)5N5tnd75Kr;Q0ieHm5D~~MrTk(Hc*h|o)Eznyb9vaetv1>^; zMEgrHU3Ht0o0+a=D?$)IF*`Bj5$5pU5i3I&{vjSL`vwQged9x!Zh@_vMhtmbxwqc= zZ#^&DHN4!`{WK~bjpP+7&B*FJx^wtk+B!r67UuOI6Mc-Jb(UHWUQhnx`{d0-*^K1!^emVBA4fQ$$G< z{+4u#{OJ!-p%QdgVr{H*_V&8i2kK@V__OD+=LOn+=uxZAlt8p8ogF>Wc^4*T)(K#H+MquxCM11YS?!=b@czcap>pC7-I z%j0P#x0~<#{eV+OZZyQxP8%Nt7;8ybQRQ6PMtWX7mMnF>u)%`}>zoF=F7= z%jbuwpy$(qwY*%~^fysAjW}gbP?p|q;0;xFxRBBN^Yx#%9M`9t&-15h-OKyY^!g-z z_u{w_(eDU|D^Qxc6J{Blh|~{duxv^?za1+E~C*$ zXe-rtm9wdY2B=ktb3memU`fuox>h|#*uv1!wT_>bG9-`)bluXu<7=U`H%DX(rQ_HR zoYVymdVhKIojxG&@K7-NY@e#E$@C@zzgdzVmnYb~-zA`|kbM<9o%m`TW`R zJNFa6TA9{nSKrUqzIs~Y+lRycx8HCn+Z{P59H3p*)NKJ`?68){d}KE!PU9xdV=&KX zPP>aYfWNM64)2M;l{aZuvu&aZQa)BIWs9ts)CJOOVu~<)IH^?D=0Q#A`1}U&s#BBD_H#f%z#8&nU`2b`l4+Q6% z7!H#UB6rlD%mv-;=wbwpGyhvLp>m2i79LaU!Ii22{xEKurmE$;zN5O;E#E#~MjXY? z>^LYbQW_@MLfnYPNuYDa5a@V0I?rE-Vew zk+ASJJ)XsJ=ZF>0*={LAhHrM}v8IYWT*D;-@~&*1vC;n+H`jA0Z5>Kvb|tDgr^FZ2 zn^U1sGmGPf@}=Lq6#g825Pmo6Pj5Ml0y+b^9E)IuwE1bDJKhZBZNhQCLhU3wz&O@T|SY^~%eUU_*a#Gq()sTU_cRceOg@F%R6YECFAhm)j^Ii$GU%uW5xQk7(68 zAfPLt^7Qt>VBn=2abi~Ewr|58f5%=MzLWcUTkcn3yXRW!v$8twIvN*!HywXn7KdIg z%>B~UJ;>Hr5Q{7#txveB7Ja%Lj_WRPexSp=Mz)de;z?qw7@jY-(zT*`YxS z$P`8Jq^t-OQYW+5*)&2AgN@QT*w+kj27Dp7sD1KdSP{H%4#llgJL;$)U5WBwW_*VJ zat16`EXms5k0ocZaR$SDTXVr+{hsZggqIMH;{h~ z)-7cJoIuD-bL*KM1vQIiBDxS|Fmw@Ws79K%7{3bmwn`1M5l8nB-M3#dD=5O;-4MGEoZ*43l`JoU5V|7=e3`@SCN^ZVVc z>hu46N*DiZukw4}E$a7tUa8an><8QPzaG4u{yZK`!s~yZ+UNT}EkE%4JRUUB>wSMG zqwD)WUWfOBjh*U$b^&(#L7wdg*+w1wgZKmh@-A{}67D4J?;GU)k^O1?2 zQ1dFjUGDXW@8vx+Y=nCmsGxoo>nEdDcqbRgtztf|EuLqV6P6?i2&z;^Zo~|lc#Cc5 z*(#ORuB;)9EJbCHBZM&{GtcZm z93x%9u>~3~|4!0QLHDqtKo)ca3BrY%g)mAt@cpB(D)}e*+GH(%x5=al0QE*upJ3-3 zn(F(z4RghJDtW0Sa7t;ogAghRBpe5@N0_-CXy-O_)k@fwW0*~qaP7le9$ZgF1oIz$ zyBn9=b*;NwcK8OFddBXzjT3AzMn)2#*k_kk&U;URw(d%(3TLepwAl3xuxWTTsz_Z3 z@g#Bs1rwuI$=@oG;|OSS3}tG{$E4N$5IrF)HFEBA3GXu~?He$jCY-AKXF}|;BSQ~l z$~zVaV~RX>EaipPYRqcZLnwZ6Kz97=Jm}P2-rZ`H%*L^pm8f+w3}=7x=Z1%sRK5NR zC*yy1WOMwGbvX`7s^pk2;>8kp5l+z+w3?%*2Q9g)Iwb@Rn!4KUVMW<-=ZQgblO__W zNs8ox6eyGnPs2%TYupJsl_`ZEgcwmiJ+P=b`33n(u^4CtgZ!N1_kInQfq*jO$eC%D zCUbEeAWvfEqBU(2JePfX%Q!xE6!M@G-lCDvu#Q7&)|2O7qx^d)>vY_kMiCxY357+I zBavE}#Go+gsgf`+AtaEc0U}IA-2xLJ$fkpWgtE;zi~Q@AQi+^aQ`UDOsX3^CW$Y+q~ zOeHSkK*2khSIq+yLEAQOdx&L~OVR2~hOJtHrk0je z13x6Gm=)dCx2 e$vp3Y1AnAi=uSJ$P4SiZP;HA3cc?A@uk>D?ecs0`0==F`t6{s z=Kp9yAqC{`7k5?{vR-*?;Q0*-yLA1&!OEZ*HDH z%d`rX+e~`!sc>DyDLEwxp_Ko9{yabff{ftEQk^ANppaV_7`nL9+6MRbUN*4^gUM7! zEV*J+-g7zDk7HL2+}iBlEz#zlP6ebYJuQo3tt5jK+c+66l5q#Z4G=t|nMSmLN)S+H ztzc_nj8$5bw(bkm&{#S7M>U4O+0F{fN(3{`2+!kV|Cu}vgeQcUwJDHfl(Hrcqo_)t zC1ID^@B0CHJDj*xQk_KtutKqM(QJE*$WW~$_NlUYEw{f0tJXmEkG>$#11Y0&L&At; zJU0AUneW=5Ya)OnS~ea33M+Q0SqA{Hi!{39uU@OZz5UPHhOO&!Wr&3=4e{DGtys~~ zEmsu+`4lc^Q^NE?g4;MU#k|hElDAcUwiUIMtT+MonpUHHp^{mZjH!(Fu7t@dDSEuq zA)CXMabl8`Wh${en_j*a-1QCBMq;~YSZdHt0z|=V!G`5IP!0|kW6iG4BJ7c~dKgi?66U3}Nz|bnf3CRg z*oysUBby#FnJ1DFqT_l9NQYdd#h-N`7)x^>AV4Txh^sCKT zzF;?^1Hf@0{4^1K5@!4iL8!jHT3yMZ_0tfC)S^^_1PUK7dYOW0l>DcRG{ey?Dn$L~ z&GNqCt%41lfZ~9dn@O3_Sa*46O6)W6UR4DAw(|BEq%d}X66G2zDMg5^RW>MU(5A-1 ztn-{LGYcoCfcb@xdHQQ$X&7rFFT00ycvf#dJH%1ohQduQ#|;KPy_0Ral;Gn*EH|ua zH2pX}EMcg`JCI|0#r8=|%Pa7i}OyksQ}0?0J!As$h0uW44)Vxm+TC zp>XZRn_w1X`Ds>SnN}gFpn~Y;lCA6(^`6057G-jd=nYqze7OAbmilSw=0WSx4tDU)G?K&lOSx!IN_ zw#SKp$6i>&tv6sN2NPj(N@uqA9HSDVH~6`m-GwtQ4Lax#Hn+6r_TTq(5lHm+4fx^B z2~^h6+IoWC-+)OTrht|X26GB%ji92S3@E6?CaeQ{jSC#10?$*0X|alfBr{}iq*FF6 z&!{&*E!Q~J8RQYoT1}HlXR{*dV`9Xc^f9UEP)#~}%NU>h^{Ufs96b)>`F&IZt(n6? zq3HdgoP9TIS7aiOKJ~xJ8g)zXCK14Q|kpY+@q5uY@dO1vNTIdlyxOLh9c2W z9|I*qb0Z}YvcE9wA43nSkr&VuAdG#46(@-iX5-3?lD6`=yEcS{)?4*R?MWMEC++9z zAHNW#S|Y;4;-;fWmWJSFQqa;G5^0xGJxNz%&z91nN@zq7b}CycL{Qrg7dYDsovhZd zN;xj-BHF^TAhyb-b?*ft&>Pm6SpWrx7n*oQMsIw5v5xZX)0#im5@S@zj~ z-B2A??6UPC!u9O_rK(R<=Gt3O#K*{fpvipXlfA6O366erxQ1CF>3IRr88Xa{>y#{6 zs}pc2>uQL)ULaH`Y1jpM0Y>U2I1-vNv|PtD;s+`XnoB}rksv>(NR<2{sgdtg&V4h4 zs{G70NP2W@zacIwDv@jB!5(%Em0U+I^$j*@9IGJF5wg(zq)vfjlM#Ix=@o}Mo}8kp z=oSx_%+r%fgjQy9qciedVZ#(@HRXhxv0jIM9fhRQ$5Ct+^PieRZKrzDDJJbXCMLn3 zjl!}TVX5MDCFhfnj-M5+=nqMpL6)?Z<3;Bt4s<5W7_1r%03R#RD zE2X?`mGjxe9M>CVtQODApmFOn&Q9vuIF`~LgSL@pd|?txq^6-i4u(H1Fzfy*=?j}G z3`m%2?OAmIR+f=Q)uaIDt!~FiSvSF(wL_wft+tquymgafe zd4-vm1=Ny8mvS));Dy_b4eEqU$pox@(vrS}vw8klB>IQ|v4rR~|Ht4Xds1 zl#B?w4Y`{VNZgnNxgKr$=5D~<%aS5iR{uAhu5lC}Fc*0`dZcXs?0_+AcBDHDp;ug+P5yy;IsY%GF|d;4_z?l1Au9FKEI9YkT3% zV;Zc7o!&SLLD&j>3x=LzAXc;7w_Hh@*+M7F5g0G%?#ZVWU!EE|L!pe{h(m9R&?%Hz zV?b5u>r#ghzg;_b8GSM**?jbyYXw`|V$|3iM-nj~HAm7|H`7>=1~yZ&deR|l`UWeL zOZ%^W!XK*6Cco;-4q@&#-G(bj?;eCuC`BB0l;kg@5?w<@4w97x*Wnyf!ndejdKU3` z23691ssSR@vW=c5ohMse5eqGSJ}N{uTBbAmH}W>hJc^oboa1uI^$YdasfH`&Xj#{1 zWR-1nxbv;20tU=q_F1{OSnLK5s8jED3T7`X6%=PL(bg0IQT!DAK>>|DA&~u=z-at# zosm@0&@<2ev7k&Y-c6M^WbX3-Uy#fm4qwYCiIP$&MR9F5T~YEKoW{A1&tM41N-n(| zPB#8vg^YF=x=Tua&4%nK6FPa2uta9OviHfU6f1b0Z3Q!GyFv=4MZK^V8Q;9g$#SFB z`w3};aVC$R`{KI?nH6QX-zi-F!r-Y-eQsW6kSuh%s9^(v#w_yhKGo8!#a*9sFlG#- z*JHTS**Bu7Ek3v>WlEDg;2d!CokZ{TuYm4o!Tt_QdY|I8(ZNl+ ze%{j;f^{<7C;`0oKy1OavHwEV-eUcTS0_LloBO6*i}1)p)m)SZJw?%yMQ0a!y~P@; zU2YYG|Fp(ZHSV)(v7PTsfsppkeVr_MUFAtz^m>}#KA{k~{Wy1v%>jE4e(v(sfhaFv zVA$hooXz)Gyoz(iu3LA=^BL$pbOnQGfgUyi z7}sREVlTqp!h7>_)!$mBjtVum{`?mL;-~XGdK*@-1B5!hr&5kkB@t8j5Tv{ zz7krGo!VR-2vcU644DICBB{fGvTRI~#EP$*%+QzUg-%ZUNn68F9{DO+9)Gln{u&)m zST`#!+zKai^Jf4(L5IfM;J>eB?g`i1EV69ZiC#MwahQMvLC42S$zMM9_4Kuv(fQsukCU=zE( z5W@`YBTUhb5f&||L>XgDsGtd5_^l~_!?f%Y8dN<^zH?`VG&fqtJRD7oR8+KK` z7rh_)0WzoL9CC*5O?5B-pKJbPCHHcYM8=69wN;%PnXvfF@s?tMWg){2sM@v#l6vaW zkSRzE5jC;7?0A|`(*=NXv*K>uq#0Zy%RxQT;#|ihJs6sM?^3Q2I@I7pz^t@KQblqd z;aI?q(UsLSNUTz{0Js&^h5~(BK=Xz?VG2^-b9Zg49hvv(zeiyEKp75|K=o1yKFXOOXLh{n zUyn6UBNpI7AWVU1p-Ef@J6HV|%NkQ=C82P-6V%5I^L) zFKGxJ?-=QiKb?i@R!{ath1t>*C}|Z2hM|Q8+yg^MBR{X@maKAXXlK(=5S@ne6cOCF zq!fhDW%mh}Z>UvtJk{%#6La)HCA9F%{?{JO>7qS1IvSW%JEk_#V)vPDLL2oKDbT_? z%vut%6oFqFBPbFw3H@XmQ;)QK<%|42I5gYcn1VNy$Qa_SsE>tr+ODuYwDs zkcePbSOuYA(C9f$;<{Wb%ZF$IESjg)x+X|)%+TAI)r!m5cMKM3VYV{}axFTmTP2gvF`+#!!_0mNFTXeP%hCz&WQ*NDLr?Wy!2p1Q6&eXXH5 zsP{Mc32G){YK9evSO_Ex^;IOGUj%raepICJU`4kfI)AD&`C67zN*6WgR3!psyeWJ2 z%}|34=8UDeP`WFSoV#Ehax~HkYxjdg%=QI>v%oH+=aXn4cc_omz;`K;dpy??66+{9 z>%82h_3E4P&22p?0X?*LeHMG~peC*3)6s|)0R$;SV31ie*vUvJai1U>pSqgaKIWF32xQFw~ID4q8H`@ zzf$K|45rY7B3W3%RH6ZM)ogth1n z?-MR|8*>s8cljekJJNC6S46_aXs|@|d)+_N|2);W%k{iiw~HPWz2to@v*-K#+aGV^ z_j^C(|7eTf@q53Cm)rTMljnQgNdCnwpC_JfM^FeK-#u>Q=lh(V zlK;H+5a2?5_0#9~JRny~2Ke7!-sb&%A_iLON_V*2YuhOZC%6Cl90&iQAYjct!O5-d zhHbvam?^xWzJ&&Tn;k6gSRL}qFS;ABpgPir-$?Ac%lCVIjQ8`szx4NbzLAflHt|uK z1#%wVy#2g7-~IeI9?oADPVWES8uIhnwFF*?XkKr|Jz=Oe{%NwqNsGcK>qOKfl0SDj zJ|=V-;vDj5N4IK$E*1x_+tTfewAKKbxON{R$o`1#|MB}#0?SJAJa6g0u`2KKxvd); z3Mx5=IBqn}wQm=%_FpR@8vgg+U38bgon_^~TNasnAV_<|$?!X*sRW=X^xR0f_mA}c z4);y=ze}E-{eBZBwWysCB?s5A$&s)9C@g3HK0Y+}|0Y&Y9s;TUIPdzIZNm4_RXv|Y z&y;G*F_wPF8zQ!ABbY5A3;A8iVaKivc@TABb+-vl;kWEd--Ko=cJ zNhh03?&!QFO~zN}WM4iW&_tgxA+<>I=3G54z}eJioi6A!8Hw+CL_$5L*dJ^D@%yQ_ z-LW!55|@Tg5h>*XjYe{N0lRhX`g+z|5byKU9Ca$LvSoi#PpjG`tJ!Y$2L}uUgNq0c zfFUR<+&&@^jW00&WqthVI9+fm);$2!P3Q`{4 z-(ZlQvN0=;j!`entR##QWN@-jt~bLRhQ+lUpOtcfH8?>UuZ01OdRnZ*iy32F!RtdU zE1zU6%Mr&!&jpBQb}bsSeM7>s+Y}kpGC;Uqm9xsEAN#WKW^$n|7%7+)L?pbVHz9mv z0hfyd22%nc^vtx>8+Je~SGE$lJ5R7i&N#*1DOMXgue`9niOW^gq7Rc;# z^KeoWVW$HVOUC2S&^e%5h)bSmAkj3?n##s3q`tAqoMjhK83MCN&Nx20Cm;a6GR@;U zJx0tof}*F6xH(-+G>*;mB)KJ)frF@w3H7l?E}es-9y(nY$M|QX%7A`s#a=q#M6qZpCqKS!74t8$h3a~nkul==FovTq6*MK7L{f+&bH`zG?;O;8k60fdSw_hK z7HEwyxPgA1KdE@!3+D}pQG=-H#JzLeZbQDXMtjx1ZfCu78%(clwUw>c)~NIl#Q%gEyUjnr4#f;Zy&!)!LedhUi4i$ccIX_0Gl3)74E04y4G$1$_MefQH;IB<4lZtVYf*vcGI!V zxpt@9$8osQ!k#6g&ldb$x&_Mx6z#?pf?boJqrS=`A?95&enB`gYDqrxPrH_{C~c>K zdKn#OSf{~_RW$hWMFDrA_6+MSn$uicNLG(*jl9&5TGY%Ws|a~^&(+vo>8{jVCMiXB zE5vzi1(n46j(ZZt!$Vu)Q#OHCXD=Tt=Ao-(w55cJKA6gX zM-13h?9twzqntQ)87RIaZR20*H-<(izdChO`9ViP!lOPejF;FeG#_38ch+3hZ0a17 z^@UZ8Abe4h8m==&}TrjN|kHQ-G@$j+CnW%sa9%_ z_r8$j?zWuCj`3H?2t=sVaB--=HA#Z33N(32*SLG$LCgx59KK=+ zt{#8gd&Y9P^!vJn(Has|;cBD(!+W{f%S(_A{T>ajk+y!yK*+-3InRukgYKiWx$Bm& zyg#}yoK&SfC~IX+Y$tRxeHp=Qkj#=9j(gO;cxGF}_Z`dszNF7R^eOvxpZlDz1DC_S zN#1-OKhs`J*Cw?UM?SFcJoph86W@Eo1~ww_Ni@h6{w+%`+>%%`^!MHP(267&UuwPA zqweQpfwqxz7uvll7bRe;lt7jh{idfpULg__*c>(QyZ4+x)##My7YUKUU>d9Dp>7&3_?qWGy@wDwddXJq(3_G5+OQ^SrCV-O7YQ>EzOY6~z} z%KUFs7S!6Xsp=|p+S>K#Pu_R+=EE!H$omDgY0SEu7o-NgK2mH|8T0U5-<@LBT&yBe z{QReqz7+~uYDsQ;8&n7P?{XU3kRy;JQntZnC_e}W5^O%KTQhl^MC3}oSxo@ti) zRColq_Y72v7vg6LYMw?GGDks4`mIDC+rlywLpO6VlrVO1-9lraza;qmkzK)|-REU} zuWwGU^5V91w%g+h=pIB1XqYBnuTAKuLC{;nkU(Yz>kar(1AQ_7WQU>7cQ`*#BhX3t zb#41Gr=yGolgK-w1=`AZsrK2y;;8}xk2<4rU?wSnrW{CA3iap!D`}6aMkSb+Llwhh zQ~*(J%diT>CRa)ko~skiA!&3ssem%t;;IFw*acDbEuboZs@!;~0dD7AsQ@PD1xo#C zF0BEgT2V}0t5cIzbr@i(fXZQ6sS3ba?15_5a8w0UR&ycMFl18?6lU&#BFGw61?b1B zT@s!6tO-h?8K+7JdPwafU^E5gu#{2*WJNPUwHOttRw7~eQEo7H#u1zGXCaJ>yHaNS zS@(`F_+#Bog2#GlO2UlW5_9F*l3!o3oF{O%bVsL?6j(O8_^n5e3I9GYfnX1JVEB<2 zqOl}ye6zTZ<)NW)ZdGSKeX1brPzbMl-aKkP&6%y!;hYZ5r=(x|N-vv4?g^8EHz+Iz ztVsfH*_$2sVc_}p#Id;>5&l`pZiiS)5ZlSd;WH)VKDd=ula^j*Fe>&X=(HvohR{z} zfX4lIRLk> zGTwRIj?xL@1~=#{7!PwKo#nFJF18B9ojQp;EIeKPql>9dDtfVth+Y%C6=t}rL#Qe9 zBA`?geSWNK=NzDc=aQcJe*u6%f4@MX05Zzg@#XU&yO3t$DepNVQ16>R9~#Xkv~3Uh zPD82COpWFS#uD_En1R|Le}$xkTm5drpLIG-&=zL?rOHUi-F>7_6`BUAq*5x;pL2+3 zVaZ68SX|@68pcQ=E+R(5%JyRd93{Z{)e-y62*i9vVlv%`M-NJ6q@?SByqum3s5LHU zTf>cHw3l>Aubl_QhQD$0%0S)>pE$*9DydEdvRR&QQ&8kCCA5WWe0?*h!zd4JxHPKsa2nJ!QIQ zPquc4OqufMLi{$yP0$r~)2DwnO&|zFMgUUk)BluE!(LB%!(C;vG$VKDr{R7syb$yWyxOtf-!aCj~045sM;aX+(f z%;v^6?6GgeSyd-qt!B)=0o%h~I$e9vYY9H zDbxL2NT;)@FhbB(+L_-3c-#DUmyzX=P z8eq48f`%$G-jEQTqqs!42Rz)0snZ?t($?Kn5Q^CJDV(l4SDzf4moavYVk*&}c4t5{ zs3cG2u!+uLoRn%bi|C5ZkMW0WHZfjH-F7l2Scv;NElxw}YPU=M51ItGLZ~eAiAo}G zu$&SC-U}*yHk{5W;G9QMFd=xE2q<27lF*0b!pYr!Dr{Ml1r|4pkfK5YZoc>;)Pbuf z3P)?t(`6bQqY5zwHUMWB<|#>;l{6cehqrP0?YM+cO}p$p_ytkbP>>*0s%rVbZIF#x z-%ZL|#P2JeN}J$?Y#4lBSh+LUrY?|;D!BXI6n^1KglVPL;(mW1)|JT^FGR<8gWV|Z zt2@0SeKk@;;zu&NlVLlB7-V+YZ#&&>a=I5*^o_svW4iOVOdmwiy>pu2`;wh>y)0ZO z#6%H(E@p{<=)$F&x{we+Ni$r*q)C3^N{AklbBeP#I0jy1RbZK8OIG+h6GeVI$&_Xl zqXS2hJ7VZACm!mq;yhvZ0KRhPk~>x`F`;Nt(lKM`rO*YBSSlE z>WamK$vEEWCDgX2u?-42uQ(3S#9qxo>#p2$3b5t*u_VtV72C-ug-nvkYGzd1u{@+| zccRl&>;j1e6Sa-nGV{EgfY)!gzj*!jHc0u!3rqj8z0|&V5h-AW1X&O&DX*ex=sXpT zTtHPvV`=e1gYkPyX6F~y#d=rreK3?oIp4wYNH5X@XqxOa14?=4R6TgCS+*mWHEvgw zS)%0~UC*9_PX#|_=rjiPqtHn=w{A4oH(%SDk1k)18dqUDk&vmpb%8jDcjD22YyhZf zmpwnu;e`W&l@b^aKS!J#Vl?)A7>UM0bjW+uK`juc$HP5TgA*Odw*&^k3%Gpq5<|ij|lNkg`p*n}vy3%crjxC`fR+{*p zSz8R=n;~@%X_JGxNu!HChuwiVcuyeI=LC6^UKd9hUL7J@rW>)?P_p1tvakY@)=hTf zNpH-BO*B^s#w4+dN_Vb! zAz8b-bK1}k97)2IVEGiR5cUJSV3&IB@q^9D!A>%Q*HL0P<^8-W;~Q0fU-1j*j_l} z0Z@YG$3V44FZzUzit4M_6l1&Cz$?ySS4j;ocuAGOQ*hb`wT)-u$YhLn#Pk?<#-hRS z6xtVg-_+w!u&Em_Z1iE&y*ox^gtZxi&+NxGC)h$`|8t zli!X_{)m;P)p&`Kw~?WwmKtta_DYy%ci9NW zg3Cm+*ET(xQCQC77*{q28*pYR0}xcqz_PcrDL_DEiX^?{fF9sbGvnwy5~FLYfKENm z*qB2bpLs~TDVX`3t6I`^h7MCN*;1m6qE>uullxA5Gv-i0r#+|R<@_^+)7x6_opWA` z9tf2%Jhqm(NU*n_C0WP)NFm|Tn{P(z+5ef02Dg)4v2f|*8k}6O`703ki&f!KcCBV{ zOsJjCl_t|F67Gt%w1fR0nxFXEfo^#OZ~uejD0%}8-#4P}AR)&^a+t!!9$%S=uIgYI zme3~s%)qjZmKcfrMuWE!ob8KE@g zBNWa|kcSIb+f7qmFE)_<_^1;9sc`wKXcp{36s7F)(;!$wNzTnVi4<_SW^^Yx4u{)t ze=IZ>LU(k=4qTssUb*DH8IF>N`i&e?cu_f4*QVRa!M|Rj6NvI^ih;#xV(UsZ0e+^S z4jf~svQkiv5)xG0FB7o$-17HJHf&t^pwVlaT<)8hh2>Dl#`5w5JXfJE_F~W026U^YCKfVpspUYW zvOFg^M{@C6`_Apv&Fx!T+t+I#eEAH-yqrr+MI`D5GhOz^^^J`{S$;0aLcng;(m1ZJ z-dTISEkvLerobg48so<5mv3!t-E4np{m$#{)!R43?>eY;Y!O3wbM^PH-!?>&$#(4m=e=G(XO+41tFy2Zn#a6D(C(QWKy(`_wg18J^{S$v%oG&*XBy||M^aQ}HU zfOCb4DIQIH5h;p^`Mph+mRX}nOT#iy-1bN`!CiKScHUx&`=X2?tGi1DRH=q`5e)*FKmb_MsvAA#z?RR*Gi zdD!A<_#4z`YnZ*DqLMA-Nr)q&y|O{+*fhb^TdHEfOn&M7`MK!4vd-Y^iwg^6jH(~2 z>c^V`A3V?-A?o29;-QYL5f5L=-!KG-=f7WAdAl%gc@DLKMR;d09Be~~Lf%+{+mAY( z^?qj{O!K`~wAyD;wVUie_&+#w3t;@kOXtcWBOIQOYmH2!*M9==|!;|m-$-jN{os;)| z`pF-D{PZt>Y7DaI>39C(ldrvX^48CveE1h|kBu9}W40!NH~-@sPrv`y9IJ)w-#&W# z%M=jrMGlb`(h*~dR|+blZq$-n&Y z>DPZstNzgkCm;W}XMg@15d9Z<}9)@|soj)MGVtnkVMg)OKmy%j915g97 ze7eldFnpHTawd~y<++m0M%{QEPh|?U(ZWS;d2^hCe2^qV*p<(y{;i=_Czm05enrY+ zL&G)Zjqm*|;vASOFUh%$2oCssNDj8qGi_!#QF%kZX)xVX0T*0ipMwluacqwZR1A*? z8DVLmW|qSdHA<7r*&Y#f+9`?;lT2~K^ke=ck39h(6j%|3Ph=TPM!}QPgnCSIst3af z;cC9XuKC!F)i%HejW&*Jvku zUk*9EKn_7P>=5vBNE}w08naEHsy!#{kWOEki4G(K#gRA4jlqJTE{sRa)pQO56*=Su zbBU?*4;s)u*xOMW1@P05i23=(3gfHx6t3%}-6*S8jK^B-c4?Sz4RJ1NF;(0$imip6 z)G}()Rk+7Y81Q_FGT~R7hg`X{oR=S`jW();te;^2v&Akw_)x7vhoXFrhvd`u;X=|vATwWAT6_2&z@^V$ri z7~KAwgOfoCxf&PD-qcH-PB!Yi1Xf7W@4oaP8TFH18;)D1y=3%KXE=EYbaglBW{|7orpP*vOGlAA@&I6p`s@uRNm==Tl7L29t&-&f%ddzJ zE{$|wm>LhzBUW(?xFyay7F+`IrQT{0Nxn4@R$M*)d(l#~94*LWAKtKWpkMSV2u%WH zi@Ju1l3cGfyanN+qTWWkJ)(_=h8{1gT@u-?AzPZZ7>*XFzh&CiVgs~gHz|;Rq?BmS z2ar+TWwH2uGy!B)7+&&ozboGsATf|9mFaOlP-pa8$GSOB|+F z5jd-7ISIwMRC7vbA{q=0D6GCXn2tCGFy#*CCS1@08KD@p;g6Y#^8G+6Q{bC*=_> zL~^++z+QY&HI?z6nQtx*@MbBqrMef5Rf8P|@&^aAMIpykcBz1tz%~{jnYRT?8&nL5 zlpcs*FS3kS31(e^_*{uLlSf+gn-1!E8F0*^wz`bU(_cnHBBM4A+K4nop!5AN>|B51cbSG-FtiIDqbIi(MJQ}28 z;uT5~GcFp=dA-Si*cugzhfGEXEozBiRgC8f*L(>uOl#B$Zr&w7ZOwcUGMV-uYhK7@ zPu7Wh!bmv-ha`p&lYHdJr@T5Ds~FPx@*?#amX9yYv!G%q(g}0<+hq+h;~I`WINO$4 z9*`w-VGx(sVJNP~=Ims^2wNq73W`^`X&SjYMJx7sn6Q=Y*J6RnG4mxye7zS~Q{t$M z;bOOTXE5p~qq3+f3$|hGMi~^<84O0%U!vWAT<%3GBV`fYji)voLJT!hiGc#<6* z93%o?2Q`4O7vJcUARv-Azz~DHJYB(_C@3gvx9)xcca@fxZ^lAe_UV_f=Llcrt*GJT zB{?xE3i`4NZ;Q(c4P)en+B|~$7)3Y0Nu72lNqgAt-JW1xgDDbsHfPK|;Bw#<647 z7UR=#r^Iz zh<;vJp(9H&4!&4Jm~u1m0?te+R8|Pg(>I;7Sy~5BNV=VS#KNXhkLV)P@C-6XMPvo$ znVz)bD1y$re)7ni=BnqCvOGi@PenJB3Pl#^wU1l!5&wHJl1;uj5UTOTXj@cyX8=Du zw2~Z+x(Q-FZwR8@zNUE~Gv)}aP*qFS#8|38wI{XA38hOIB&vfgAC!YR#|M>v;%-ju z@x;?OB8MQ;J{;$&%ikVO3+@LaCr)!L&FWlFi3Pg!`5Y?qqtVaofjfg3<_edb3a8{P zIV^y%1m2ybZ^d0bJB#ndGqV7g?vz}2F0MJp9;yzX!W64;4h3TcWxk0mXt&~vNs<~0 zo|)NFX$KX!#!q@V_XJC@FkI(ArM=4ZJS!lwb|$yJ3g%jej?mVzg^EqLPEj%2-n$Uq z_xTFd_m#r>zEZcouiU@4Wa?ABmQ~!wv1&21C_Hv2QH8Z+2M58*RS$qwQ9Xoi&@I`! z(O{NJ(X_ZQwfAeVtTp*LH9B*j1D`Yf1))wc0|w_m)n6^Yn((&GfdG%?eIDf;Nw0nO*bf^UQ?hmAKq$mR$yht) z(j{6L& z-7XufsF^UWq!!e{E+61nEVr%sSGRh~(7_?Ek*r`@d)4!zc@}boLQ-{x+g1e$SJFkN z6fKSvt}07uuDHyc^3lH>%h3`|*Ey0CK4jBFeK*dU%-NFiGi(*Wi--U!*85`#ss=@X zF=xfh-EGR*#MDP%L?6{1(XxvABl@Vyh(0>|h(0>~h(0PD(MMGa<&5Z~f)Rc6d`GlT z^Jj2GAAJ@^bPf?y7}3=*+ar4JGwNKT5q(PEl9n#hsuv>*WsYGI9;^iusqIcIoxBS) zXtk=3PEqxdtlC}d+(>TT)vKHfi~k`yAAMQ;Yc4Hb5bP3*=ly0g`|4=iy!g^EorgDv z;`xQe3$MKLyGsB)96V|swxXBk;ZwXQg1V->8k)f}=+j^S;Hniy0AY2mL~~KjqmeBq zO3@owgF+>KmrZ>LmmCf)_kb=Zh73ep_mlp(?FI^p3NdPr+!9OI)>#0eg|HPpetGf2 zam%#KZ@ECWHEcF+4IVWFxMXX~CQM*>tSCDUUl$Baz=YuuEZ;X7D|TU(R)rS`NCa?} znDT-rY9NZ4rcL10Srfu98m7inB{qQ$W6Wp;i3WCQs!RC=-?-@PqLMD=m!wAgXLTB{ z^s&ZMKmp}*_PQ*q+GSa|%S?Hv%d)zC7ODXI?05g;?|%OCul$EsFMPgEziFPdf}{i6 zH0=d!1NCh689XRsn55*60Btz+Jl5sQ`^puUtb({0z(8N=@@`)de2u;eR3O^6R8a#- zVb-i@1u4cZsvz0p{dD9D_PDBGkHdm($~(*&*A?tg1t{3H^pNI6D@ZYlRt0(99ZW{! zeP44AuqrpOkMVt|~$nocDHz7J8 zoSO^1ZAisUiBB}hNwG+{18E}t>q6WW=n`LnG8!`${bQYGjuIl?<(1zF1gvdZB5kl^OUO*ocPyo>rP;OOP#v0=0e`x%0+@DLC5{1~gNSSZrT@ z;yeHgCeFk1L;G+u)U(nwm?eBTxt%PZ)2h@vc%}+acG7kVSJ9A~DNie>qPkQpmvN?m z)R6o)m*xQky(;*0%yPl0L(64K=hp4%J4|Mwc21>trt2~xoLO%Wj^Yccq0((*L_D3X zNm}>-b#>7N_eFwZ;E+Xw1_4<8+AO0Ge%L}e1k-DC?e#CoB)967dAb|*lKvjtskCeH zTR<$eRE*=@zRVvI-CJs1yf4*V-Y(jeE0IcX1a~Si3WK1Mx7;4#Q%H@?ovKqsfpc27 zv=M;JnIdMU*K6qQ?Hm*|HCTCELK_Y|xH}JoPxQsJBKEjt2C6Zv*`|%Z&!WlD5IVXy zIpq$8%Z#GvSDRgBGY3o@um1a+InL7g&g_%Bs`abFr8p~@8`HNo z9@RxPMN5O9-6@*>UtrV!n{O(Ti@=Ig5;?Lh4H@D?wm+Ekx{&_pm9Dv|2dHzay{F;; z?_Jj(|j194TU)*wE$D0@a&??%X@>tUN2c*MyQ?1?(#Caz*amtSCJtB)X+GSwZ)^k zt=7?CPrmw!>K7@VK|;w52RH-cO%9`+HWn6DL-jOeN#mWasLX79e_pMmL=nXRHTbHG zyQ;%1Z*(x9)$Izmb8%zxQ2~TEI?ZfXWpJAc40FI;65>o9 z+WnA%FB!A@0OpGG5MD6Kb(RSTC0X6HhX?tyTs?|bXB}ng6yO?F*G7G?O%dt1fkG$G z23YCki(bsSDhUq!3VIh-yDgWJGD8!p#wNa9zfg)wfgZd@-z2r%utw{Bg(ZDB1aMC^g{UdpYN zCyAk+=!vQ6OfqQH$*~bMnRfa#!dB2miM6&;rE*hgWtsEgPTHIHeK9j|GY}i9`WDbs znyzc%VA30>qT`A#=JYaJ=t64X>IY?8jL0@>XjRiHa_TI$<1|<>H%6ZQWK|G=Em$N@ zV>MV|JvfyWVT~jgh8w3&bisSf6WE-7dOis+KsT^-&S!S`o)kvQ|P^;jJi+BN*@3Nw1D0phg*?eY8<=!zjv?MR5u}=s#k339v z8|Oc>AyRlgKbrHI`Ffi1=D^v?SxGM3vrz-tl)j}>K-^5 ziK|VTTNsUNMTC7OrJe-rISIb(mHO(whN`D1*t2MQ<$7|}&nSWGqB`*e928KXccktdfhsdG&f&ro#49ifcED*&a%l?e-D1$R~?&n4==sZioP-t%AuD&NO*&N%7BDB`$;;o zYK1QEMvcb`;c@#hy~ZC6RSs-&p$FE-$pMZ+$nDyRdx96kILkz(aI1^L3%0xX*-s8> zuMt|aoh11y|H1GJ*kQ5gmv}D`vwD0WDx533YL=5V1N{!48R>ZV)YJ1~zToZ~f%z@qn@~GfqJAr!2TL z5ZHQxssfOvCvZ~|8pQFyrp};zF={;UPRCJ^s-q6WwGb?HnvU1piXJ}*opqbOT!Vxa z$u#e+0J8Wv(|l>~D^6}Pa#C6Vy5Q9!nb zgm;H)VP!CkyR|&?UJ=VTk1rcVfVB-T;0)m+;mysHU!ihgwc!gT;c#XF0}>scwJg4? z9fQ2kke4q8q7#K&Tvb{_9 z2Kw3h)Kk{eP^odhvp*QMcYyiRR{gSyZqggaSsm)H5_UI+wy`7BE# zH{zX)zVrrrO$*UHm()og8X*T$h(8>_7(iLKEtEGJ98|`~gEuh?2*pr#p57a~Ez>`k zDZ#?-Sw(K{1slA@1Rs!jl!dG;Y*Z(Ki%E-^VL=#f2o|^T-nF8#E)XkD4S>E;CBqnz zr6>!M3ERV73KOr_M$vA+H7&0u;E&XcEtStqJX;EKT%I~`akRq3wxR_srmcv5tvFER z6e7fXG!8S=f-u~akA+=OSPb5~Yz$9Zf=50vEw7%cymaWKZYm^If_P2rrUyblL1bJ} zX#<4_5U7!%m(4KNo7^UMw^7kf$DN~m%@;LeWwM%2@_T?5EdN8oIeJ234coD z(u1negF#vzhwIJdl?0da?r7c+sWxQ8Z7&Xz3TQZ7a4a4kH3NO#%HeCPpdqbX+H!_s zMXd~aTpin}WX$OTj`?!L%{QHNqj_G#&#T)LbdkgIB1`u@VMOM%4+}+6!gw7^&yOmU zJqig$Nv#MGELU73LXiB3PQ5jNL$qsR>5JQ+O-@#hOjs;wg)B&9w^%7%sH(|5eQ+ky zj!H;7n#Xx{`lR`zCD5JEpJT;j&5~Plb`qV(VF?O%UWju(*~iw&Rk@aSiSxWdh0dGJ zjcssNY{c1v*|=?H=dgMHt{O0QlUjb1I$oM|FlmqXMX&D+)`)4P~ndMUa!N(84nNd{ZGrZ15{ zox!9Z-pv4QAn8R=Gx@l0K-lAzjY1k>S-Ag-(J{fPITcr(zB!*DW(lP%QsNyy?S5#7t(q);znroD1oxv!_a;H~@ z!MnX!G|qzOzo1_Lgn4Voi*Fb4us}KK`@iH+C0YjH|8DFq* z3`p)3AOcc*Vm6GfUfud~d-Lu_`}$2$_7$h6Jz;6(&@6b*Fdf@J1tCki_e(oIuL75& z0@p$gGEAuo`*4s)(koUbS?DA4`m_~h__3LOidz*xXg6jF?TBELd}SXV`W8f<*L7R0 z?&1;mx%!w}c5Q`QORwGBTD=n(cd{R3v{p#-l{qQt;7*Wl|2d<_@vU zQPf%<njgy{Bp8# zb>$o%yi3b!TU$30b$C$&Ie;^ht75*%67f&~&jHX6o8Iue0yiA6O3ZM|FkUEP^1*6y!o%S( zVu_g~_%yIC3a(}lTB#mO_gnfTh_ilhi&k*ABJKaXEs8 zs~$Va!Ek&;Ygx&g`~@zmpR5B$Z?5n#e7HAGR76+w6eSNRZS8JXUPw7K`kV|`T0^?7 zZf)Hx#1t58ii6{k024erNcuS&RsdnNz721gaRF4gEX!Ridx{bSBd>yhwHD|=yL81q z%&W2$Xp&DwZk4C$!2)CfMZtJVa(!Kv1nzor7nqm)_E}@o9YeV$MwwR=H>s!X&zyxP z^ZEHOEr*lDF$LeH!Yj`vIX=s9hIlje%C4RqlovIygaFq#to)VeLUDpP27CZVK6a95 zCA!ccYg`$JNY**#5{oJ)`P*{y1um%EFnUdfOu8IhfQ)>61A=ZdkcV`ll`GsWs4tl- zYTFx7@(nn}BU%Ybghx0{svV)h4%y z7256g>KU4~efnk>vcVf1q zXmc=5vSn<@c`<&Yhsg+B0&sdSi{giS(Qq*6wW3}UKZH~#oxw<~mWKoM;8A{kXn7RF z@j!!RU;G|F8br6Z)+0ev1R0ewA@6YH4CZywCOkqnr1q$Nm#FMCn!vXT&6xJI9nEIu zjy{FSdpC*46L8;5dlr}Yg(%g+{T!pjLCTAKM(lm#Qm9a8(3>3SEg26TtQ-i$E?y8* zXOD2fkCof3Anv>^A?2*7OpJgTyTg^*g%Ex*|?WOC}taSe&}yqNBjKIBOj z{gZ`B4r>4#M0?Ln$dEg{^GOTTWEAv@T$DP9n1neyDm5EAT)=bB4$8F3J46KqgC~}l2_hhj|J4IQm`tcl+cu*E0rEB zpO6=8zJebU_9LggAezN$=(#V2Xug=iT{Am^p8l*BLFgQ)gIi^Hs^)|R%O+%Yi01** z$ZcuMB@b*Pg)%=h3uP4W@h)k3d~8Jw)niSbhqr7pJU>&B_U-Jbj{HdRemdI&;&lax!r+;E8tEr48UA~sP@?mIzJG@W)8@C>?k9qoyWOc@_ngNyPFQuES2-@|=>G`dfNt%iqBe z*vBInyy^K;9yPDNaz<+8{LXBXzC`r@uW9aG=w*3?)GK2wWLeKYB_4a3oVd8Q2Bty*6>X)>#+Gw~2Q_5=b%wt^Y9VyYSzd)`bH*!X% z9MakuWr4aZ9_>LCj8e}9P6`g`cqmAq8v!tP%ka=R0enE|EAU!|;5plVgv8(d^RYXN zpm52YUBCj^x{ID7TzXHiq`4qcAW^4la-cMCwtC&DROXb;&&8KE^>~@rrvOWnJXyiy zW%GKTp~RxW%Eu!+n}uHa?gV398}toEL(9@vEf>8o-k`IE*`^zO3hVv#!PFyjW;1Ji z3X=tiMvVnxohhRWHzd|UUT0-HsJCXWwyssUcX#U`SNB;~!d-73ougQq3Oko2=Pk%u zmurFC$By@GUj-_%1ggbR6vxgBk^nZRP2!cmrDNeXO*(FIWX4(w)a)5RuvFL|REn6g zKphi^ba5jD$88|D4B~uK|Sda%;V>d>X_D!n*UdPbLDH}Gf3W4jI7lp{QdF2HS z_4|u!c+>42*sh19+2ZY&;!$5rcgs5R4kaD~1LHnT?&2>*@xV>E;*sV8Jcf{;#pv~f zl6vb;I#$7bq85tn6HM*wLx>?uTrDpH7I;RJ{y05I@q&2zjGC?) zAbvk=lOlFs?W95(Dv7T)t2>{WSPssovLV6hHOd8b08iRyC|#j0%^qtO`G$f2N>1y? zm*i|_F%e=u`;%S|3==s$Q5}oJNw(jHKs38RFp+QNYe?-5^z9=1mZfSqnY{S;f(jME zDri#!=crN;XD!U6BQVm?3o4G5B|GPNn>@Yf894a3$e@v}tCUt-&1UjSQ&E||%+xTC znMguMK)ep-UlIiFs>2OiL!fP4JZR3#aj^(Gj$I}yY$tOiX^mU(w)5`TlVY~wR7kge zirUndYBlMKtUA`!tqD#g{PIlnlCX<7tJYZ5whL~B`oc;uK}A>KEtHjNqKhW6E=x9V z36T)m#i>VLQ;7LQ>7v6*l~nDj;Bdg14uxqsAYqGQXyH@$LOplRx2Abl_MJIC3g6hF zDOrRnXbQ#`N|(`6n=1F$KdqQZEUx-leWG7T6^>E&6`b$p3{lNcl+(V zq>t~K^9qXNH#DOvvTm)juE?CjARy6O)_l#}d)i*0i2xgVJBfR!g-ql8McllaNnUV~ zEaYB~81`1bCZq%dj}tTqgmsaQy5NABz|M_6x6zT?F;PRxap2Qv-Z~oZ3quJn#G7A? z)H~tb{og)FfBwJaDW7asVbWoCXzv)XLGqWJF`=5o4sT(=M)S?4v~V!LItX~8EjDv zJ(xpKnZ;P~dRNC_p1KE3e`aeC`=+pG1eYxjq-94oYO#-}jHx$9NerhLzdDI~GveqB z*;#p6ac2Ldt@38Dn?;C^sv)l9mW1%_pt}ZMH28_uoAAJkV&Q~A@Pfvd9}lxR7&jcq zGlaWr)7kFnOXq9^^V5>aN`g~KBIjnYK9s%K@SxnyM(yEpfeMy!@nKSMU%+-E>h5=v zyggwkt*`&Q5xOKmIm6*ho4h*@4Hi>!5zl^?n(6>wadwX)18w`@)Gh2jL=DH=wnTxe z60;Am%($W2V%J0cHO|bgCs~9hf&w<+Zf@V%_!0(7%W=ycJLlwDKJbP**?Q&lfr243 zBRnW8%&}r2aPpn+oqYH!p!DhA{NcZT`yC|@GGUH(sLRCyWwXHZn3tgR&%X7^ufKcp z){mb4`Olud^OH~h<$Is}@NI$R;=o4c$Gaqvvv@E(?7GW)m0d5YW$Bd>{RO!cY@B#!95bD8` z_uqZ?SKp*A`{ZA~|McA-J^jx6(5>J8)ts|c5^BYuv`xoKm;$66!<9%UKmABl7J;Zo zC;#*fK@Ca#$#;H6i_ND$_~_)L_h|Ui%xea4=DDLujRr=q*3H4Uj|d$B{^|QqzxHD_xa8W5#!AUvt$}5;oITn~Ffd?-qCYuk_u?Zs6j+MEz;YS{$AE2q zaVz*rfw7W1X*Y#4PwAi^_ezReunzg+jVQe&v=TqDE{I=gTE*;Adt!|QM8Be+!V|?y z196^nS*n4j|NU!E|Mo{u|MI6N@BfddfA-U-Z-4#d@BT^*LL7~AWJoZ^z96V8_HV3_ zlQ0?K+enuHTur`tFmT{QBhYzWwZbKY99tZxicI z-v7?ifBtjHF1t?Ae~q*M#o+(N?&| z3zp~xa^xT5pWVY zwFPaid}}ONpKXqZnDTeueD?mgXqe}W6x1cGnfdy}0g_#8%9ko67fNENGFEj5N8)+( zazvBy?9cxG$*Xw8w&rbbEYgrFZfv<)H&38t1Ow*c1WA!QO=Ps z2YGHchpOcG-7`XTK8>;R=?{MS^qrrO)8pAceR%TyKOwbA-Vb^rnftYmWHAsHd2Fsu zqHmwJ*gh_HLFQt|Gg8q7j2jcL#kc+Vp^#D0cCnBU<}X~bkz@(v#(-cCiJggZW$KD) zV~4da@WWcp3{}a(JkrOZkT{KskHXVWd2GUyjt`vh{_Uf;!8wA1l%9X`KY#J`Z+?U& zEPqINR0gH)SUzWP0l!T7IWJHWQ{HQs!A8S#-DMD+xUnsxwiL>l-P8<;cG@!w(nsu_ zTG*=v>Z=N-;AZ*Myd>PmrC$m(Hrn5F%SJ+jh*kfic*L%PQSw0{u~SJQs@%-USKGY= z=(6g#cj3O_Lp<*6t3I%ymJC!)e~>EXENqLVr`Aafb?Ey>`QZv(CdV)@GFrV=cb>If zXt;*4lo-9z+7(57FEi6Vq`qy#UwVIY7b3Y!hM+cerwVT{Lz*j$jUcxgkTnVt)N)7KDxXh?gf2gxcoq~U1br@V6PisrT4;l{IUCYm zIle3H&15;yHFl_r(4T$M-0Lx%&_-|W_}4dxWdHc z3fM3i9cX6eBX3^89%^y4c%K5=h%IIbwzR!>zR+J-wm!U_KoaTXix;Q7NW{+!R*Lwk zw_KE3rT$ux@8Xo2#+4R@3MEA#6&*gnD6dILkXO=)DusCI_Ta=bj{0pGVy4i@I|(%U zDK2y7^7fr5E2?d)+J@gO%wcLIAmxx zt!WvYb;mq;>edFsBdj@_#n6|ZS!n?I@fE>LJVx9F5QVdt>7hd>4T7M0HgZrZ$~*-l z-&a~~vwCy!F_=<4)nP}w4lDRQB&|8F4vvkeD96aC&C@L89>K(f&>W7Aol&wo!8+s= z48<@A4SVrfxwbV$^?4vhDJD#;NT+hOBACUa*S+# zp{M1<^5PD`Zgp#~Utjw|`?b~en@!Xr*cEbOgIx;IZuRTjIsQw$lhv^I&XddFcq5HZ zPe@ZWwqfrY7A#mB>gg74si+pJ6K)9{NudcG{Lg8>yA9WY2o2UdisD{x@F;^pGM?;B z29qqxP%#7IgXCZ^I-1j&%2Nh_^Ley8pyIG*8pd#oSARLWx73Oj7Z-8YK+v@;e=Xmi zg9}&(JyB{R1cA_8iFV^&hTA0=KrQDxcAW}mrgfo;ihk|l$z##{d+B{K8zQ{;dWnD_ zx|~-i7z_n?bXeq21UHLOX}0AQ#vyw~RA-QdcXx2UQuuMtH7&(SFs;`IY6%W5|{gt;nynDo}+M^g%zY-@pG1>-yIcY2dd49;9L z3FKc}Nbo}%M+gRSBW((FNk&Zzp#ZsepLoNqp+jx~1uIm;M^-9U+GYsqW>u$RE((82 ziL$Pu5HW9-ehrKk(%x22GMW&oA z!_}~v)KtYER<5ocBs>4OplGlVgyDr3GmqYW%3Otw4 zL}|G&nfi4hkv5V->U4)9ZHLO0kVmD}b0i=6y@&7p9(3*1mW!=8aTDFul4L@n3+g5S zRk_)S_xsPQ+~(eK%d0UMv1T>VFQG})8lmpmZF>wceN$er)m^1M*L7F%lEyMetNLI( zZ!*Fww`YM5*}ySJrzLmTKmM^mH@S?E!`DWG19g^f>gNc<-d)T{rJCs?a(uc5Hk>lc zeP6?;8!TUX(*jjp)VzN3s9mRtHau^pdeg{-tJ~ZVjubBrE*bPVoPz^?SNv+Du%2@0 zPhr^S_+2M(L!jIg2cnhj0wlx!&t+(ClQ%ayuNqON)k>SCKNZ@S+ot&y=rx)#Ejt^( z594%nS?5?nyr!`$D>BGwco zoud+46qg3nw82|w(c+URdj$?2Qfydx33f1@-CN4WrW{!}d%AWR}zn@uXkmj2KA?kDcKVx4t}lC&?ZGnJi2CEB>@y|*F()Qu+RGYImsKgTd0kh2P4ehYZ zNec1)XE%|J$_v*aj`dh03r3Pq(3y<2(n4#FFh@2lF_slwH+k^tZA#pAtbShQT8&v0 zE}+3clzlU{64)Wr&<@zH<*xOi-;j+|X9KQ1lKK5%Dr1Se23Dd*tFj|b&~eUiMF`-v zV-3%O*dXm;lT;WjGwP;AKh%+yNcH2eH z2`C2@nDOYC$kzrMoScVcHpRUCLcGX30ftK+5#SC(j9RH#_@FS>*cWua3Vpi<=jA0U zqL50NWE{VTs6=O&X1LWLDwrfN*lNKKzxeP%iz*gCQ$kkpNz5Dsbm@n029PnEKOZgh zH3F}91q7CWboKPcjbDld?Y*0V$>Bq*Bt@pv3$A1p+4gOwYM!l=d15koq#>}k zL@vG|*;m~aolp_(^qw2&9K1RN-)j;Wh-0dE&Y8olFnYH!ks>3V+5)O&m(@=J8ZsE~ z(3Aj=9#*>n&3q`*U=4sTtF$17=^fN`<@cg~Ln(@2uq@Lwr_!W#&KzezG3PZ2&CvdR zpatwv`nfuj>cWqLJy2;^$_nI1d8*SGX+Ptb@Ca3bX2*>S*D(jAX^d2-G+qC*0D6)d zf8mxt*K#S@>_~OyAw^75z9#qY3m2gFx)i`~U;12N!Mv4BbFtKj4x=JM8*J-I9LfrM z*!~s+rYi5RLvk~hMx;1tRcxPuq~b0OXaX_Cp{L2JTE2Gn^t+ImRi*jYqFsDc);4hI znQy*J`H%c0aj#T>v8^dZxe zC`A6bwGw{Oo9(~OAj3oEL$>81TZPVD_FMpAO>kpZwiX4{J*p!!k<mXbnM?Hp@rk{T4$Laf(493Wg(4Nu>F0 zPHT1NPGMEM1X}LN8Rc$hSOdwbX$H@VekPhx`nc5l*PvwCPhstm8Vnf56-bCI2&IuB zY~WJG%BEiT6$u4{1ipC{{U}uC5-NvG+~|6D50(C$Y?wK0b~2LeVx*TK?5;7}Axkq! z#3H=zr5pGD40yX;VExHWl{MbMT@EH9Wi=;BD3#@$=tdrlTfC5ZEmvk~Nv;F0lHCV- zfeITY)#!Jp-b(Z_(jaxlM+lA(su{33l0G}cLeN&aVTvN_Yfn!_1{Au~BzR2z*#3Tv zC=&+;-e-O2O_5P0KlAaX?o-}}?o&ok6L|qs)&jS99a8{m6fVOM#E zz;>nLYJS2;c#eO;iX{_$<|PO3*Kx&Q>pz%La!7#&`xJj5`yvHBxzl>^M|92-8+F9p zQbU|#UH&GidX5cO?mW)8-yKZChQm|grnz~KHQzw|g-_GAKDqi9_{|$J@l2x$m9Ryd z5AheJNDKf%xns6JZ(&s}L&ip)pi`4;PY`Ex3&}#{h<`y~>E@*IQYxxd zA#~IOZ62*mgoyP5ks&sQ5ZK6>{8VsF+F8V0-h8GIkww*HN19ys89&^z-MIZkak07o zAdj|LXyClwrG3&_OmhQ%1XiC-`^DA0M(zP zHkPTFHCt8Dob6MgN`A)Qc##U}A}q##c=G61w* z6sFOGdim9cs5jt>H5|0=)!Lj5$qPl=6bN#cEY+Ka*Bnct6g{}~&tZy5(wdy|IkXeg znj28Li8ISQ3~?&nsZV>nx9y4|okmf%JU!Vce43(FiJ=XGu3tftZv+%cH_Wdg<&iJ8 zO3N1cIuB=YU1=2CKPicBxxJa05RIJ;l|hZ_vvzku^j9MH!aj$01{dyzMY70HizIW#H=8DbUADmp><4R3ot{v z_^jx7Vi<6|P+2_T46 z`M*%JB9&-q(Tn<_b#Q|m*rb7t=nhvEsPZi6T#i3Ar`||E0hRvvm!(;LQMrkR$T*m+ zfLCYm$gat?P1GIm74@ETj6KE4o>KYw-XlhLndfM`U8`dKfj82k8TMA8{=-$Y?g=&M zSeV6yp%Qf&2@661vEk?P<8I`X4|!)18cY?I0-5F#u?aHoEm&$Q7-Om*jpd~@8EXN! zDqSKLCJ}rG?*8++#bU5?EyKp*B|}3)ww5JR++uGF;cLpfAtQX{RKW%)yJXMelVMn7 ztSa3$7TYG%rUuL{b=WkbtUe57IPaZ-nFt34*LjMK*GgwfG}0V&?KLN8 zE5%}%KluDmG^?Q165U18G1M|bq?E^UEX&tkY9Kuv0E~Jtxjg3HaFkM^I|&wv6Y~VO z$8AG{clVA6cN?hO>mf!)`v}hI>C5c)o87?Y<3gKHQo{&nRhF9pCFLzv24{xJthmk+ z=CB@zQl)AQ-VC1IXgD8_GW?>lW;#XAGFqu7PG0WTzf`S}@Na_qveeG{+W(rEALSOt zMEfJJ$Wh(P8^JkTT-uJ@3Gzi`&D`n!N$5FpFV<{*6q!z(QrX7&q{c%7p*erMg6VUd zVYGnwIvn^(df-GMnB1A2ibx=pGap9j=qF`@=wJ3sy$L0W~ zW3|m{k0Zv^nwUypFH|fwX1>BXFCfVz!Wu=!D&7tKGb5xvEgYbr6Mp;U=4kb+>w92k zoRbDfl3_pA&07%D<<>6dD#;{ax4=6&yDQ~JeLLI$9Aq|#*IBDx>Op4jha!=Usr=h| z6{Ukhl=hbUkrPiMco=P(`sBc^2~~=e^(2c#CZ6c?w{=?|5dwg-9hYDM{5sHkE4Qm= zdg~dl0#f~+c|KOeLRsOUnHvTsU$aK*vX+?VLh2)c7N6kHpV%@(xErU~VduJ|fv`2F zZx;(pKQy`;C%4R@6p@b3>=%*OnJq52qMQ`{Bzzc1GE@jze67)wCl?m8fH9rl?a^)K$^;9$S`Ii}k6V2Qih zySc^?X#miW@jew~AUg~lnD{dUka)r`C^CUqLnIO}0O)}Q1!&+I-?PGivSoIHnx)wh z@ULbpw6{u57OHDo{g(FAv-zP+LAgII`G=+gaz)Tx;zO0i=KhRa>D6YX2)CwxI%q;E}+*2W5@0YhEdxz+8%*nzVv8tk9D?kyXqAlUA7B*0dq5Ri?>RZHbF5k_!VMZk-`@sN{_s-&4$9N*@!8k>^x z_mG!q%nwx;_XfW?jEO~m%*=Qw=6i|7SiC7T1oEykvlR8Qqc)}TwJf09NWmX*%WPoN zTLw&ELy)td#z5M$r6AYE8_C*AfLYNo9gb@Kq1ZN!OZzaR)zY>St#2=4{4EX(ODo0B zF*@8Dol){AonXYKeh}qe1vRe9@&N~2sOY}h_#>KAv$r`Hvs=YcXky#tj0dR7SrA*V zRTb5}6V#Y7@^eL0tE(>fA*!_|$6PL}Yy^~-2F-XfSOvqm_jMN1i&XQa}8k(9vxs~Loz`@yXPS9sf<`F}GcPw0y&C`NtBMDn(q$;E!8)7usU8a#Hz;RtD z4EX#rSY@iXpWC;Pizp^WkwlJDs_Fuht9MFfDfe^?vRTd2bm5)XsJ7>RFMdjX*~sFv ziU=)l&i6ReQw=@8m=(CI#?mT1``GH0j{<8Z+8JweWg!1EMTGJ-U4ZKTT{PFb*CH+j z>5+GR(ktW)Fx2cnq0L((BZAImFP#_*158CJzEIpn>%=pf?a zr~4v;90vOr1v`BMJHu2I_me0J)kh`nXxDKNkS>>lRGt|_wNSzQS4({$u{Ra#oN)7T zS)04L`Y!U^B8ugh=9(>zEoo0)zy17?|K@YnA1ukb6}#aw>e+Axdi>#A$VCet$-E82+QL=TaCNLn&mR)nL5p145RhnANc1>`kf}Q zEM16vK>kw!-QYDXezj}3|0BQ*4O)uik3wRh;@rnBG=>Gxubw1lF1di#N4XcOuTE(^ zU9N_*#vD6*#tU&GN9oJvSwZu1^Ra|Xi$@dud>0RgU$4= zVx zaazDJn=HVSumg!yol2E_PubNZZii8v19S71v8F;r^UNaZI>?O~7@qtiiCM1Z0eA@FH6;A&Wy&o zrbL^`a&P|Vg@OzXx?olLD?p%86*{ln6=!$-`eYTg`#RfO(lEMcb_k6*6o}e{r+a%n zB8|wm$=6R$pdc>nASv~*gA4VyCcSSG2<37RDw>L3y6|EY;)w_TzBU7rH?8b*SC}M1 z>7aZZ{4fT|Vork$hU|7v6uaz6B)|}QI(XYczxXmBsj7v93kpRTPKmJD-gt^m7cknB zh2zEx0O=-}>kJsgb00%x-YW!(X4trdcN>vkFcK{T@H52ky_HYilf{)}*3eZ?z!tJ# zDZ@XqL#qB^E08EI9Lr!c6AmClic*jyx@`uVH)`}iqeOkntTL0QM9&>akbbQtCMa&Vm)b$t*(3}EXzsUOq}l=Dj4cly8Mb{)FRaT6^dj} zB;{Dr#sRt`b~!IvTs$B&ck`}LC=XvrOEl38bU40Mok8WSTV~v=y!KSRKMN+37jW3P zuoXATn^7m3LOowug+;K3qvl3tNstettt ztHk@hmC}!EsJW@=G;Vq#k}z&kpfS=p;_H8q&4cn%~4aHz=c7MN1!PLQ06kPT>JcTc% z<;`dJW?kV(Gt9Jh&Q((2bR=5i!9=y~>7vp)S=3tf^Sje- zPSq06t_9F~o+O%@E2{gmuS-2A=R$E7a~u(?SZp$U9OKxk+G1}Leblhh;Wj*tK=q7x ztLzQ{OKddWSx~5fje35b#tf!1;-T#zj{5ZZ!stZN-X?+OJHePdz9A1TRf>iRu9~vN z+&`dqDujPga2C5K?+UY=}ljD|FHae*9C z(D4*!REg%gDH9*sRQbM%)auaFJxyg;B!hy{~N2+b`?m@Zf|zGRfrdy{Euh zgd3{Bsp9Ld2|Y!7hPWOBTcdG;(*{k81Y<8aW{DTs?Z`F8NJd@4(*cjGt&BC(u#{9D zbX1qBD~^y>DWq3N^_I9gg^{^Vj4uG`;1*lB<(ueqlzF0Rge+0s&F|yt+6U+BuuqJK55^I+z%`m@@v~1Ealz zzM+YsgNvyXgR$`rpS82Hc-r1GEsi}pIjV8?)sGJp*dCc6(mEn{lwzY2t+Y#G?M2r_ zGO>>6DufWUN-}0a24ah43~^vRz__8dF*MWhdL~4YA{lMiF4RQQ3`%>eT>jUsa1x)FZ;UvLsJBspe;H&yUSCk5{?|PL#?nx5R>?mX$1*TupzqY;K#4$cN&xL$0bm z@_4cL2AkFzG$SP0xp{W7$NEmXK)iAlS0PT5deNaE+r@=5K^?)x-Gvi4p&j6*cL~@P z-UT>Ztq0EL#SQP2Tw-yruXt7zmlszj`|$`Nxcp7|Dk*+cKy9RMKjV7o&ZJWiSa1nQ zb_;5(-D-nJ^ID2fVlW=h)uWV?lzu*`<S;X3MjC+HjG{Ky+Wh+)UTdl6lFXUj z#qY4(=-wsDXn$bht5Sp^d3N*Ie6!w(=)+nXwv)rBp6mW#$k~Z0 z-+2iXzfzCKn=C(SxcQ(o^iL=>KxCF&H$`B&G-h3ss!k^E7yjStV0$TN+BtRe!jsg+ zV|y4>Ps)?O-_y*P+H~K4rDE^hz+CG&ka zcD8)ooi7mIssuKVmk*9q-CTXyK4m<8v=;}Tb{dcn%A?+=gX5)ujjQvH>Z3kV&Kg=Y za0$=peRT9d;--n)|4%~Ek0n=bs9EU#;V9G^hzp61t=NQMK=ZnQWNmddQP|GKl-H(U zI5GDe?nE>sVXfOZ{NMBb0jEHkF}}D_?sz;w>|(fgAg8iF_Ftj>5M#Z5eX3c1_qn;4q_8!8lzezasxQGgGEcb~ggkqHgW%N}GlbZSDN6x^^@5nl-z!AO%O5 zr@f2{T1kxO@BQ+q<^3A{Mi(@-M4&34&(%W#-}gJ?0N;%WL1J#~Rbw$ff_l_fUj;ud zUHGIeJ-^osh{3Dn1tw@d&FQ^5xH}%gk8KUfiI!Nm;~)gW0h-%>4h~xdn1J4J$M5?* zwI{|eA1!VVl|RNi04ZCXm3$nFJ)vx2aZtS>;Ww0li;EZ1 zsn=8U0&El<^HM1{kT#W^Z#x@U|IWABiDzMl+K_$ooS2<5)#6KAYp>bK2X* z$xx}CVMY&NQ3mJ&3PZBu!S{O$0CHU^T zO-P%>XZ;eJ*qV*(t-*eSJm@-KGDh=@)5@sWNNWt2ho4Y4-8n*xXnGJ#Lz{VcB9BFo z)q^9_X#v&EWQdDp*F~9X_R^8fk(FQa{n{TK-r4a^>>fgk_i=gt7D|Msw+|rE-pY*O z{YC~PGn9=oj%9yAZA0TQpUA(x0`OpF+s+ll@-7fekI%*MqFBnk9ruYi2XWuOMc5Hd zT#RrCPE|QB)Dj?v_@5mYiaM+K5*+8O$e5EaJDveVv6#Ok5KEk0v~?2@xV9k+4Lp63 z{#r3fW&@psH0XAqE&?p|DpWm*Q=BeBwNf#aaw{+uS+P!o=Px8Ek1g>JLubwr+=QtT zx8W^QRFZKHB3d9(q-z?`I)~X#`y}H zKxOia7x{$Y`-gTSadH)Gs<_}hs(8Z$n%u;4isq^HEnj!89clmE@Jx(b#}`d!^nnd@ zxJ8ExWgA`~2U$Xf9t*aIBKkcOa>1_MvNBk>g}n}TF7IF4nG1?DY_8A95GNypV=x6J zKua!37|n2RK3l1Rn1*P{u^x&-B-{vp2t0YiyjjPeP43fj}gbE)_wEMXyf|y_qs+o( z`@|+Fvx`7Mr?5k$?i#Jn#u&hG)a~&-V%F%7$Xvh6DtV&K+Z(+KVu0M_tX#9S%JWrw zd2H1@J_ueJrRMOG-YFHc85^1rN`&xV2l=9c-#dOx)}%$hZFdeO8H1N8GF~3IS~89j z?yVQf?Cj00h95PPkdBA93 z8la~(6LdIQl^T&MSO7!@Vz^aKch{;AXk5v#WHugfEo>k<)!(sbR99Q^CZGxXj7B-$ zf~z{EHDpyl7v)326$zhU%&_N*j_+0_wc{u2Zkh>*5JLVVW~#pv5+taCm_Qa=Hq|vt zlQ!rHUkyi!fbVz0g-8+ys8mK0YM=@XG)G;mBmt+|Z7BDtYZ14t!lbAOppus?E?<%> zLjF()6c>Euup#LV(Q&N)qx?)H-9jHWVIucEcJ}f;I>@Aa+>BFnkGKTmTGkAzw%8)X zI0OcrUFcjamtB~`MU(->0{>S4gec(RE+pw{Tau|`-az`3C`Hoz>5hy9;Bznw6NbAN ztwimPsQ9U8?*qatfa44ikKjq>lFeQLt}YGc9KfC|-&UsNTD=slPL4n&5f3xEm0y=y zzm1)*;nBwiW;3jR&y&J}6bc4xrf{-&)o;F&Z;ecwJ!z9C*1QS? zkH%^VD9arUSjb+CZZP~v+DAKM0X$FWz=WhlT=Bn`*RSSj`j)rsXtGp|U?jadXb~bDfpG(6cG;^&fComa z;DO=_h$|1~$O~~_&jSOUdJCHl9gJfGxJQ^E6k_3aNOo+;jI9W)S%MKQ{MwQQWpMxp zGg%tTX-WDLJD|g*M2fp{1D3*_6NoF3HY~6kEsz@pw@}jp90@@xtg(Zc6VV(Y|JK}? zsM?{Y?NXG~rycxyDC!yX_9bwB4P^vU#9mSW!Ejhz)Mpwp+oeQ$`QvAJrk3cW zfn${PcSwW}m&bOzy556^8!b9d3O6l(M&};^8_7INl>GIxY%WNlbj%$MaNw?#{?a^k zd0Dm2-vS+I)hUHqG+fy@{7ZmY&9b|x1j+BXx|0X&tgOxuYg#cNcyHC7H;?n`huRa^ z(4rtd-p-f7iL8_?D<{8Wf$@dma4t(GUnX^hq9KE;Q&PK~%lI@lyA0Ys!+Yci68H#% zUPSyr0lbMQ4@^&A)RM19=8v1>l95d;5FMD5LPN2AL?e(HI z3@feBp<3=hKN$Kh>2&G!t@0*^;qq`v*Jftds^lhOClu@>BK{gm@i6*I>m%b}v9=OP zFN?R9*ug@T*hNlQbQSn*Crbzt*5ajVX6UT&dc-Y_>cAH@MvE`RZtSQ8N*{k$a%FrS#F?*Wug<^VciTSJW{m0=%;NT_g zl#ni77hZ?Q*7~Q~FUQv0brCLC#{6E|Lak#^-U%Sz1*GcLi5L1jneOkBOOH9+ac_g$ zS`3nnNPKwjP^4f|kRN9h6ice77+xT7TP`wn4nJF=g`<%};QkR99aMx_?4C z6_GJ{Cs;bK42VW?#m}(HPVzbBNvTAmV_0ojyk}fz%La~}E|TW0YMk(6zj+{d4$*Ac zkSG`nRb7{u*=lmiC7`x73lTN5XlsXo*$uzz^lHDzBSh$DGP59AjemYxn7+zDceCDT zivOVRR&g}&Z@^*7)=%R8tMh4oaz2)`w0n6BEmzO8UEezr9s1mkcS|P4~8DC`-APPh;tm8bt(i0ZL;ad`*ku9w0@y{+In!~ zT*zvaj5R^{glmaSBrV>{3^`rf0kjsyjJ+pjL+v`p^MkPk+>~yX`_c1#kqFG;HlTg1 zSfVsy5KSl_N~2Q4^QLAmXHwhru&TjISkl0L1h)m^^m|5&0|%4N;gabfxz#kT;!D!r${^h>@iX zo@Y3ovX0QK9Q=PFm^4go_lBE<3TBQHh6X>{KP?pHauvIgTh^FuS+Y3GBj6*lk&woB zuBnc*+AQzgX|Ke9I{(Fjr?{^fQ8kV1A?wQ>>EAR3al&4SIPna;$1?al>1#b zIZPv>=X=9p*Gvjq5zx_5ROk^VSZ4=$M^IuMu}A3#8CX-)XQIGv=YZ%&;(mBcpjRXD zE=pYG&o=|Y?xEuzF5w)`_-kSy#bKH}adQCLz+oM`dEoQsYpP_I}@-d5*w9eij* zy7{tCCh&gK7>9%$M$=VXQEKe3wFf!+I94Y?!{q@>gNTvf>mfiCISHH)Egtgr7S43c zOJVa<>4CmILokiJ5H9tbd~P)9vq6s7)pY)yvqTYNMIq*wj?Ek40rYI(E}nb!yQGk?c?j%NWKkjCmUys zy*ud86~;MWbBFn}Sbb*aqodP2FfBA_^?Dyd)b<$_;So85vVo0zV@2Y7E}b zzltmmcO4^8y{j&d`!^gh_xGQk9WJ;fZecJ`>avWEiNe#7o2(Y%5;@r&CA|maI`=!+ zzPP;B`&e9FISI(#J*_ItUn6MAw3n4jL)`psUH8q_S5cP~xPbSV#>1LZv9S zZxVgT&n2~;LQ{~3^B;otorR%?g_>d+=RaKD*bk3C`PzwW!In&Fk-tPUJ zOf?@;nzt#TCj z27y{5P~wY+lNVCxBV&{jnsbF%DrbXecx3f){3v~A-Ax679FOAoU0Ef#p4y6{HM@=7 z!P%sbtLDYq(GKyFC!r51yE;|sYW_v-#mimi;JWKUz1>M2#HrW%;Ec*ZZ=v*rHCof) zguxjPoZZ3%D;b_%>#>H!eGkD0d@*7|K7UAM*?q(;56FhkR?2!@B}%yNdOi4VbZI0u zMprnfDE1SgG%@~Lz8a1Rk>K6sSktFK%2}$SACOpKbxE@_g-TxqDBRMHU9v)zF{mvv zYb22{Uw+>Bnq{nc`7CDYfT)P|s3%_yF|*#d=&+e!VrG^^7fOdJIf(;Qg0 z*K0ClwSnp%LR;#kk(-}@hjK=T{t?R7u^Ya7#=)D8%wu;yWR6Qqf#;-pyv);DI04Rc z1-Jy7YGt-RkllcCUo5{S53`j(GYl%GhE*(A0u`PZWreOv0ijKthw{4eHA&Jno&>hZ z0Jk*<3k|u-ySntb@n4|m-MhGhsa0XegNdbt4Q_^iivy%Zug`VyII@3ApLlofwmUUl zt~(9fPecxC+BX#_NRLzNbsw==sp?r2WOcA1ATLlj#kn%k;MC#Z z>gwi$(FZ*!{^D>wSTREAARQh#BD`$|F;`eU=+#jzv@?}l>3w2v*go_iJLp{}5aM6a z(IrR2$VvHVz(w^Ygy1<`5A_1G1smw-%=bcSkxRK9Le=q%gR*ku%|=NQ_L_xy3s6Fj zCMmDVn~PulfMQ#wYh0nFnQZ`<-fo+l#|hOJMnbK&_~=Dc;16t=2PXNA zf`yw6*E^z;E!eX&j?MmvuOq$Uv4>4EENj`1gb7wI#@D^7kB$yj&ky#&?KOPB!YTZ0 zm60z1<0BE}={v`l(2yAJz5u4M6K27kCNZn{Jci6-UdONks{m~!?hUxXE$(U&jd*ne zF~nOIBOFl7h)g&hK+K>u)L=Of=hNu^>vipJC2DJ=hz4u0h8S~N>rB04tz6a4(f1{i z?5yiMmA}^b2KoIQO7|~{p9qpFQ`&DnUcL2O*F{RkG~7<=-&0<7JW_Wo)t6h%hOObP z0J}2^Ug!ZNJJO_RGKkGxHhRMJWH{|yz0TcENxv_0rhRTD?eH!dsDki|(Qq{Mv2hF> z!UpA|?@mX(jx?*X45!g^YP!Ry;!Dj+HS6_xJMl1%6~I0QVnfH!QF9*Rw*f?`85Sd6 z%!kG5v=Nd!PKhNVGhNUm_v?bW9u>xtfC43pq*?$P#5g|!@kY0eKILeX9wN}M&Xq`2Iq=j? zB10G32yjPtJ2~A?hcDI{-C0~In;yjheX!$`eu^adT&dEzh+f;C02nqqp{Ydo`{*4iaUc@y4g<;GsXQPv zcO?(%S?Q*wFf@d@U%9;&Z;>?039R3D2u2>b82aW${YsA^vZ%<^`Vt|5&``mY9wN*E zr1JbGJ%(itPr%vkX<_qy!=&Y(M=PADsaUh zGAXFFaElF6cCyCd-cl~Z9ZqN2M*B5W4az0;M&W!ky^6E)Q50#L>IP_ip5*EmzR2>2 z;6!Ao;E*>_AKvyJp6z(^wrXXypLd~aD7Es7`iJEMs1lBfqDSMJ=OuNxLZ;@oH^+Obj_Dd}@Vh@!}@}mKo zAysb*L>-|OSg)%nWOs(7BMt4DT9n^`m$};3Cwu5(5Jy8c8m;@m8OaGTqxwH?#`eFQ zeMCmBi?Y4Sio=cV3Antz?%rW{!vCGT9UOgKjvU_4=DWD{u1w|^#hl|C;3AC8i-~_t zp27XfHNH10*h{iE3EG@lf2gMo9V~DQjeb;Tz$~~QF#>>^$>yCy%U&AFj1o}r8QrT> zhW`FL;m;@|>j#z7{dGR$x{@|#uC)Zk80cthhM2(d_kj>;KBq=55IeYwkayUWI{|`? zR0_}`8$hyg<%$EmkPkE3y+_Zhk!Kz*$3H3?|Co%bFGwLj64*)qyWq>sp0J?oh&SbfA06DFqms^BOdy6=A7*0-gEdlBqk%Ct=wl|A z=VYzWo7y=W@PXiDrL(!+zVpyg;9%9WXR)=-X%5@R9?k;P!sb_Gu*mYkPiB21)DT`Y zWYRJP82{F;KNWMopS|nqv_jS8L~Nm=xiifMo?%``4N#nqu>mH`DU~%`1fFV+t$;G! zMq?3-Y6REGXi^aui6yDxI{ie7FF)RaHF}>wr36spXlOQYg6XS|!?5$>Ea|gSLO%E# z5XEFX%|=P=3dBFcwcbH^8Ep&-8yoy+n$EZUHZLL&NTVR>yIX+zT3)0j3I@lig+-j} zP{N*(<_qe_M9n9L=8cV+;*oov)@OjjzzNRmm_?DGhonl`emPo~arFXx217J5-~uhW z%t4Revr{9r^+wnTtFyIO{gkHA^H3~U(K66NWRmu{J?7`3HxYPocJ1u!_!kW5GCgNS zqErgdPf;Hb=sBO#p_c7duNhGT`V7?hgy%n_p20n!Ee#dgXmpNEKlhJC=-e$PCm~pq ztrGd`W1|)LYbf+Cb1lF41<~VfOFaFvr9K-`piqGWJH9f1S3|*o>;i5&s0irDnMoc6 zp+W6lg7eG!v}t2iVSbLS^L;vae|5n$tWGKDs-;3>?@Qf3uZaadO#xnBFenFdfP}|r z-fTa%oIFjPI@y%NLfeP-zw*l&2V;ZoQud$*W@J4r4Z9R_N&vf#GJKa)Ohyb&vV~3} z<_|5ou99O@UECh3jLmfo#wmZzU)hsGPPCZyY|FRfUbQRvt70@am~sby6*fNCB^M)P z1dHo?H|)jb0}`b0MiLFu&FX4n{Xy%DlPFsB+(8|v;~jqU$Xmbr!>`|2i(?pcTQW?z z28AvnFyepTV(1H8@R~Lfj&t+;MSn)Ur7T$Ha1i8G`AiRr~4VwudSe zc;yVmB78Hh!=nZ=GU`IT-={H!e4SBFUc2wqj8@S$4(+e|GyGMocGH0~4!Izkp)(s~ zehGLG4xVzqq_SN+OW||#=z6oWV?lyniWxnu|EEQmY(1ANAd6N=w)+7GeFGF`pk2pL z*x9bu=Va(9i**q?y-#Fzqu0s$2rcU5Ahn9PF&iIURK)h~a5}V1)qw8NE~cu+HMcIm^Pr zs|p!c`#_exBZYjB4A)uRPBzglK5>!Z=mm{*h>OFEZ|Dbly4r$6|1nxX;7x>-|Tud0z_RBdn=qHu7CQNrfI z2rNpSy6Qv@Wu0`MlFK`d6sw~9M4DJt?MJn8H>0zg9PPb*h9xFqQ zgWjZQ6CC9jZX$?x!ZVlefTA(MAS{Wef|TVv6-1+59?-=|J+q`Nv>JGai>t%)!H-Ot z*wAJN$7qCM$=W0;`;{8ZaZQq?;^Az~i#H&$o03vhpv2crv09Vr(>3#4Gk=7f7aB8V zg)*9|(G_28BBH3O9 zibkQ+H^w{krhLt@0C5nYH%94yN6hDF901MJoY-QG+hSb%4V(tV-$iraIP~6DGDKtz z=|qthnz`QeyFHbXj-B&`xssfJkn>T3(aS&<;LuEKN$IUJKL(qKLg|<^GhxIRs|XhB z=0vqRl1$goS$^i#oL;(tgz`r8WmAQ5HE%9^?0T8a;+MO`oKk<`vc zgTMR~?i>sq9L_{8h5$$d74^M?-%{~*-G`;B#*MVy<3yeoCW`&`6i0d&hvlymv9pL5 z=)MSL>!edKe0!V3@gfp0Xt49`R`nbo2<{Z!ZPCAD^JwLUMMm1tl__lq&p`4zBoOY< z?XGQak7pxG1g5@k_z|N}L2xa)8B7K_I+EPJZ{)vYirB!G({ExILdN~hSWt?~&e*he zsOorEPkf2uBqaY}|NlV~3W7y%BQg-slr9hu<$rz9WO1$Y&mK=4{ojuN=t$}eH(<*) zG@*klTxZKsj*NfYo#Cl`<-^XV-mATPhtW+2{ck!-U=>DX%7~>fltPlwWfG&65UDB= zF%?uKBML~+8^!(U^v&$dtY2($(1@hujhp%IXAkFXujl8f?P(lg{!c<*badmr`XU#Q z=lhX4H^=+=)I3+PRM;6#zdRm{zlF`ch4EszZG)N%9mW(c!($BJaD7A!b4~oB&=9;3 zWR0*-=vN+@H=bwD&Ee(tV!2ENO7UJhd5#(~(uZ2q*cop_DNq<67g#yemk@5hS3pEE^Qcn?y3U;{w`$7vV@+ut@4I046XmP_QI?dw3`nQcgPP8$TN6b&k1Toy}#=~c0mEiy;{1n8w8 zekhgu=i*)*-`CR6*VnLlOZTeaj!*BVo(}kyb4T~f6@sm1EMRm~OV5G|ZVN!~!!ZN2$uYV26o)@S=561i1z|E_6l>T7V@JgdRoVK+9k zG_39KvCHA0$P3a;i>;OiOM5XuLtxG#JZe0;ZV#pH_CVsw^9Z%r-L->Vq8J{)d3WY1 z>o<;(b$2sY>cI3UyAJLfbKc53zrjl5{L>M}&9(#N4Uws7`c(!Ul{EIq=xp$AJf3H) zaJFZXV;k3qSQfpo-W1+<>4EhlsE+L%Tp64QNxkiYjZueOC9EUi+y(=+lOu*XwP^+FpXSZoB{Sk2-^1zx!J=V*L)c$Df;nA6nAQR{La# zz5n&cB zJfD5Xa~dJ5!noV8A>>;SS61_wN0AS!?_rYiN0Jcj6871qn61VMha!dWtr!j`kVJ(n z6E9MWX;+6NjdJ#WK*3sPh4yW*ZpHj)UldAo$0do{#}7j-0He}?A4jVMZM(PGMhu~( z2`*xT-0X->Jf&35RIwvA4q2FQ&L}=XJ_i!D&TAhh)Pvz60Fw_HUhY*X7go3EhysWS zMBNPB6PXiai}J<5=M1-F`hDJ}P=i&B?4K*QF4gzb)y^Mk5OF9@^ie;H2tMAM5?Hv~ z__=;z;$p!sM)pOu6HBvVE3zMw-FPC|U{@hKV#ZmNH_f2R|I!vguczOS>W2jlDP_eO zLR7TT5JbwXA%*_F+k945mSOFcAnDM;c^xmvr4$4vo8tYx338JWuHMMy30ia zgYN+R*GF|%OP~uj!p@?dP_H-3#v;6Z*Xwog;CtGIbJHaU`Qk0QkdW)BZu+&D;h6mg z=(ewbbh>kn8Z+DNUcdc*x4DIQ>wh;`zx(m$Ab#&->3WS|yZ_-Lyx&ZnYLtNNS7E$b zgRI4V1j|m)z#bQ5M-7wZ15(FnBG}aC=u;PLKa}_W-@m^YdKsN~IwW6qywi98k;OCq z4`t`nEeNnA>uuY%ZQC|_+qQb!wr$(CZQHi3>A7#`>CSnnA5d#$=BkLuuQttMvhIsR z8I9tzCbF4z>RQ#Z2}!Ltv;@?IRZ$8;18X)9bv>z?z&~8_@1cHR?%yRK(5C+qjN)P0 z?$w`-JaoPn({_jpwPkMvB|#{8=L)}Hi?b8W<85_5>lji);aFfgsiXsuMCbjWSfHkr zq$(S9LTO}jCakX$fo)A`Xaat-xzR;)1nZ*!f(q7tpLeSdNjSY@^S=$o z{*>$w!Ityz%kkh&F=^NrmQM+>MTZF*#$w+f3>d>F)F8^?fi`f|O`ebyS)34OC=>(G@p~ z@D^C^VG}AU3zp00jGP_l^>}@&5<$`OPs@y~BR$HM!dUfQnKhiE$~J<+Y`V4ge-Anq^w)c_M+>jK4H4OKv)!I5;Q zk1Gg46_?N%(uqFhcIBG_#;8lcpfeixKqhPm@F|Qxohgq+y76~l@(Mk2cOI3du@6`M zI==c$sY#fDytpJ>IYSlRh4bY7k>O1gK^mVDr7KQKqZaaRz>KURV z*dH1a@f*E?;6a{F6~{NpSj<&Yt`#?zt`#|3xxq^ zFMJGFr!&ow1@NCESIn6fCs)kvixp=JLL!r4!cqY?dK9&;BQet@+K{Wr6jLs@WE*}k zQKnTzJ29RhKaQF%l3ACDNH9(&>9lM3()3e}zfUr>5KySYek08gO3ay*J6(Y<*FD_8 z(VH<5N^qmsm_J)9yNWn?JW5)I3NkSU!%#+LB&XK?!}8q=IX8RA?w=)er4G6)a^$t% z7=qC~MV?a8l~Q?JkQyHvIM=w|FxwQ2`E;MuSfy_3oy0`1UK~Z22uL`bP%F~=Pe~h9 zB%%Z5+Rw`~twYDfM&l7MOPvpizg09H)p!^%^P7*`H-%TGd-c za=Y7TwrbTwmWVU5%mDL6GAm!UTBBr^k~K=UH84qN;!ZS6I<&&e$5gJcb|^Uu`q+W8f(G_Fa*>5!y9aF?=NvYCx*S2P1s(*E^#6o%G4|l zBIxP4Vjk4ODTlJLB29>I5kgRh36ME&o;y?5f`EvR8=0S9PFc{B{(|;Z;ty-tozwkHC z^v9U<1u-#PB6rA#a3<48BH+&usAzQfjvvCgr>2L2%f7YQ{F&|3D%(fBQok=qu^e(W zTMXsrG|iEIKc9ry?ZhNQ_8Er<5&Rz?5mu2JQYuoIbsc{r7Ty&_oR{>R6qNG59L>a| zm_sCdNHsx!ST>ZcR5CNPHix0TKUMS4UwXP0Bo~~L{WDFpyIHq^2FTL|^bB##*Z$|= zkCxdHuz?QDHbP=m8Bgp3w~$~Q6hy+&{Y58cjM>tD(R8)MP+N=Ataax z)Y5NtoCNThKzc+R7eclNHj^cDj6;F@)=;>xrF`ctSQsi^_l)J+01Nfrk%}7*!%bIv zGve`Djot6py}q}nsd~P*uHfwMUxfXU&kqpsl$f>r2ugjE0``BV7__bM_f& z_}TgJ?OeVoA)|t77@T8=C4ek&kqns_GKSQgjFmeH95sSx%bG~#U&D~$ zmq`BOQAYDwZd$}HUX-*{dbaB)rD`P9bN&c;6{@zzn{+2>8cbmviPG;|i46e|`xDsw z@1~&n$(o4vu{Fkw>JyR;Z^c$xO4$^AsgnnkXqYMnR|0>=A-=iG=2XhlJQS;c7H$8B9J9HegseFiCBNjjBKoJw*Ff8hv zAkd`HObbAdHln2suslKab%2Qd<<>b5SSlBjy=&TVf?{li4$@B{XGk#NWQ1^o@paW; z-Q;1ZYX&X~q+l36DuvVtoGeo0`s1Tp+i6_uyI|ZsU$O%QS403}wa!Ey_dE_&PiFoqqR*9Sfw33U;{tS77 zBL`!uY>4JFyj8b}|E^~R1r z%oB+0PQ@)!$z+) zjGqM~=|%To4~1>yf?797T8?v|`Gv4+<{suA*N{VzEM@4psdQ_fu*)Xqi-rZk-#uoU z{`|Xv`g_`3)KtU3;eEpHV-E*sCo(G15NLavI3Z&(b|z1dOrtEgRmm zngSRM-L8y*Qf8FzEm2xV4$!`R

(=vHq+Am<#k`Y2`4>!E>a-K;Vdm1#OGzM`L9L zi_oQO)&T*^h9emikd%kW1W+mDpX-$r2M3BeGiHhFkQR!*a*P%s#!9}Az$&m^ z$O9>@0%(R3&Lq8 z=$`y68f4Y966JP-`H*xoUkWOCZ?rwT#mGF_8x|Y)6C0D4FFjHUXM9Ltz2iXWpK-jO zP^^ydRSN?t2GUjaMd$LL;OJTjpY;Q}{9s7;G^UdY3oohA86J|tPX#Z1Qrz)py>Jux zRdrDsWR0Dp=Z77Q^gdjCY5wr69B~U@FOOLpqz_)h8|xI$85BV_2Butlo8Xf39-I$y z8a6J#jzIGOb~5%zczb53Tb9;0uB$~jJL8jbv<#_NnUdxNlTVHv!%v_=PxvrAom)7% zeI_aLkaX3s&7r+dDLowys*X3}a}?1RshF7Mr!t74C~>h=xzXSh1~mSN2NRuc7qrlj z1{?5*kU@?$sr*_RIi66ectQb}?*iUQlQ&bYEn$rQHiqw}F>esxh6=_|txf!Z1_uTa!%hQ^?FTZbFHf01NIs*uPhAOl0Upf4^oSmjLb zVr1zwSsw(m+B5D9fR%N12A4I*~6S@i(c^jeRX zL23*tQWW$NPr{`-?t#)-i8LB`{Qg@>O4R~qu- z5baQR2t!Uc=Ozz8`AhC}AqQh>ctAPULqJLVO8pzU`V07a$u~=GHeBxWdaF)Hk8)Kayfo-(pMOr9fCMp>5O>5#C{IY@E<^B1k2J=qy1l&JF=crbjhR;C^gdI3k;O%Ndprr| zE;*CnOzO_Fc=TK;&;R-FT+L0WjH1IN=YuvQN+)MDgF1@QI?@!+3$$G2F&3&1GbS=j zIKw2~P71lxo3V(ePgmhVJbJ#TUJ&sQ%yO}J4nU|RA%^ZCBAO>rhGA$UQsy%Y;tir% z4uVD^i*%kThrrZuWW#hTy*v?ZjvVJHs zva=OIdE4Q;f*s|V>o?dwke$Z-Z{Lf5wk=4XoLWfPo}viZ-bpT%BtB>)0Z^X&>`uvX zT6BwuD*b{q8ODI9rs*hAX7xd}4O~$jl)y+OF7PJpX%zfO)^!WN72&9|$CtJ1aa?_S zzKr|svzrk!W^CQwe&_Xb5U`3)!u2l2YaTzV6(Z2uANM6rDhgohC`}SAy(t>AvYkP; zN}9@qF!upL2-!i*NbKM!M-mV*K4~0Oy{!-mXUIw5#H*imIiv{@9qR!r8r=&ji?SWC zb)xXjZN^+~+{j_oYR)$V(T!mrM=UvraKcHl^F%XLW z_pYf(NH7Oom&Sq%y6s89M)r?~iVVAX`H;w?XG?#3>7#Xa$7Y^xvCMKY?5woD6D0_i z1lsERmEhpdVmmtX8~R+5-QsH9ez=I7@EoYJg0{UghP!o;6V&J}jWU-8j1zT{?1D9r zstC?;Jc@75>!se;@#pTZlWb{xWmEQ=Ywy>hFSfC473P4(($EbXr-+$mk=!5&jz6^A zFqFo_VJDcrPa~LzNF5YO?Al=`GD@4gTawmIB+DW(nlNai%?aRQx@GI-LWr->PPHH$ z^EYFf)d9D_r{TnrDA@q)R@rtA5bosLlJzNuC6}^nD`%)Uu45RHeyjg@?MP#c5UVl_nwjk0(Cz%!Xidbt`0PgFX|`#kDFo09Wi4TrXTs?!V|Q6~LEJTqigU%7B!r)A<-B zo*6r^2GLj0hgnpLGkJ{`e|wPdPZh~fVnMZaw9k;PG#I!CXc+jZ7Ff4v9d?TmhiqK$ zvdZn{_oPoUdqSzS63p*&P=3&w9N5~24!qDd9_=-cyeyp`c`ep-zc+omeSc4HvFGv{ z6f%hhg7l?B>GY|WU+8EYu3&0rP!<7$hap=cfcmWKMA!25c-8&^1Yb&Ptt4Wcu3=yh>hQ<0>;}>pe%{neTw(|{NUmnX_R#8h;*chYv z*!7TwH%WU9yQjA02JxjF8;FNJ`)q5=qSL;eKBDd+kH*cUW3L@>J3c?u_aR(@Qx-{fkvVy)hJwV7K zWJ?;jLw~SBUtOEo^0}U}MzidE7Za9gZ7n+V6t_oLybqGtU^2EvuM2b1E!X~OY&pEM z*a72WK$F>P2nkKvfM95~ih0=)vqCCy_94+;eHzcFtui}c8++-+pV2voDw*XX2G+lM zfapA?3vI!{xGL4FQ|+e)L!D%(EXD4i-60H#sZA&~QiT@j8wE?ihTFWRcGK9Ndul>5 z=L~S!!&~VFPs55<3@yauam%k5qf`TZK|0uE-1Pz{I7I`);M-TlzdJvdpUSAz2U^A_ zSyVQv6#as;CiWTlE!zZS_5}aR*U{u>@F{K4`+GsBDPRi|k2?k11VX$;v;08$<(}?G zvh?-APulm(c<930QuUE$;pzURZ()}u{1`H^V%q{QE=7(&2J6xpy*lb9rtCShJQe;{ znH6HOwJx`{o5N}}+@cbbzO%&wTuMEoh0BonIk;5XZ@5(1M|gfnaILL@*-WD;=nnL! zT^enoCKsTI&5m<4qO%Due6$E#G!2lSGu7E$quq{u#s>hpYxQ2s;1fP(QD!>tFi4L* z+pl|q8sVK($sf3u&L+4FI-N#pP`o`S6IV+4J;)C))Jcrg(id`Unl%=tG5t3U;Q9y8 z65y0%_H!Bk_v5%kWp3V4Son2bDx}mPR>5ry!y|?7KuyMdQ z-1RoJ+}Om)=-)h7ownT+L+E*=EMd}L3n40L zSB=rBOF)8lA@NtiIufZG%Wnc`pS+L4w5-eC?reOMW2zC zxnVoV5~pApQ3%68O+yVxZ4C3VEP~d{ThNAbTk?beU&&V%&K@+%G$e)|TLTMXQ7|vU z(H0(12Jyl*u=RruGfpYZ3WqS5p021;HalO(p})j&G{9(Qh8jChV!`!K&v=?h>$>f@ zeC`;PYTzwTvDIw1c=h7JkPVmzCejAyL8r8Yh3chgD=zPTu_A_^^BpE5T1~iNm zBBGY!8Vw`oDOee5(3cJ>_6n9#5RWUIXlp@U7Ma@tmT(_|WiCEf(`$*lL(*>@;F+ygcl;NXC@w6Y9Qj-s-EAQg?G{ zc#DZ(xBLzr&chV2=j{Y@#$ET5iqPBcLS%Cvh*-jD<*Ezd7j>$OW2!#ls}lDXyqrRj zQc-(r!UK~nCTkOhUQ2y?XC_`A{|?*>*LiJP=Hs#eex3Z=YoB`Dd9yd>ahs!?c&`)q zr8mCAdCE=Zt)AI_P@C3&Jz>9X=llLWoyouZa{Vj z|6j0#s-PXVC`R|Ex)&3Y6wzwtNeD7cItc|@zgWHWaw6+`c9r<^4GeQ*TS_uMaE}Nf zQB%Fv0*R6WkpyfIMfg^jGCXYR0Hh)u3(~F|uIF}Az~0DsC-bf6t@mxFhL({LBSxSF zon=D12`uvA>TU-cBcB8A36N4TOvDo1XV-v^80tX+WZEj+N?L=vsj{9TIM8lBl5@I{ z7y`Dh2RGzfoYZg|Jo2~$2BV*w9reF!rm*Azh+XPC#R3wI52XBKhzj)-BO45M5;)DP zXfV`Dsgh3T>f%8oIp|P1+Kj@^Cb?>zAiY`<#9%W0 zkEP@kyl!RQrfxr2t57Vk5DP?BzdnqG;2&b$5XMGb1f>n+KswAj9lc*+obhf8grc{g zN>rZjkhYLO3+6#lEA6@Fg5npBh<}g{WDM=#My@=Dh7@vS+QO0As~D7-9vm!Fla@^@ z-~(9l*q(09m^r(@*%>#!9q`)Fo(`fQ1MPB#ODGF&o|sNjz8GCCa2^*-il?a*uwfm( zzLs$dHdMU#5L^CKPmG}AA(I|X#bQ1KsFF6yGQV9+?TtxD{Q~xQEw#6I=Lw&ZGNev% zrcpyupl!HtP#}Us#F9)J8)nSON0IqZHflnPOL&oD0Gll0#2FORo3oWa!S|xw2&)lRLtAVzq$H~XP^YOqrg{bP23$bRy=4$f0BWnvoBf5XT5(Nn)-vHzgJs1`tVi0CLK zBu#VUDO>Dupp2L^gnG_`Aa0Iq0l6dg3h=zid1cC_km2bb4`JhW#6}w|q%4eb9^bn& zTFUV2>|yNe?C!qMw6epqzWG+&-QE1NJio2?myek^I43=OD8Qbf#x3^BeBoU#_h;z7vaS}+}fb% z%-t$ya3oSt(Nd`Fc3kKh1%2OoP{RNB*^U6xbhn^*J=cOUg&076%Z=2J`t6>=}pX@GRsR3k%LUB`FIw z0{L=EA(*X;YGUNMlJYu4cTLTxSFsVRCURLIfa3$bDY~`v)pls~3CIjyX>jzlXes@c zvYQh}CE69ZxNzP}^Zl7K{9SczZnr?3E18*GAwVw~g(2mqGly=BUukq6xznuS^n8>9 zdCZ%H_?F{UKmLh?o+J0l%m%UfixOhYIG+-#id^G45Hhg{X?tPKwPtZZVU!UYE@G<( zB=Pl1MdX?llNZu13dKU-HEV z7t^1oj>=KuH#`d5j5ycdp*K75a}fpJ;)THEpPz%yWM`#*n03Ici{wWy&kwXUJ;>UW z9FO$9dfl0mlk!rnt7xaWsdn_fS~j&36lv=nZRJ>CC)1ys;sy?IZ+0(=6>2S;#7R?tpwME^n+ot8=ea(GEkEw?+ zEtiKrR*Cpo0+JR>=BWS%p{(_G>I&!oz*ws=$^>i{7o^~kMr8E2g+sJ?db6zo5<7nK z&|~Vy=OydU&w*V{R{^vgiA3533K~z+$^l_qNW{!OStV|rTqnWGL}O*8o-Z@I>L0O$ z-TeLZG&Dih@4(T48V{nQ zF+`*S;4ofOlf<~Hv#34^w(KC$z$>7FcI1H=Z8S5YYHU5ZZUI7$R6# z9+8)5oS`3|hP@?eUbo>@Ka>=+-0B>i%|Gw9@0AfIdEKkMK!yI~dS5PtzJ(mWUN^I@ zzFL2F+U!p=x0d{aJHZ`Ko!*;E8(mA(y*3Ps%yOF@G{Yj%#)6}KR|fmM^~jSsaV+xX ziH>v@Mv?_ehu#RpkDiy|k4e+k0mE0sKl#DFaX7AcIS69+ba+CDX(v=+80qo{nO4@z)W zabTNp$6&jGiQz2(g-zXE?n{#!`NaKt7Ko)0zy%tjIb+#AV30hPFbYEm6#PAJg|PEt z3((^a3S%#+uA2Wk-!Nw*R^nsc1JFV&LqH>;REjC2+G0E}%ao3#w+iK6RrhjENzvLWqU+*K_^KUU-SN+{ciOLw>2H566kqHgn$ zNA`y-I5#GTS=z0Iog+?4v*+x1xt+gnDZp>irBqV@5z0YQjFi*(WymM^6`D(g-E_ zx_m}fUQ)-NqM_NIl+bneIF#0fJLSh;bYZuwdzI&82s=(aV^Mi$>?=Spzz_(IXBCCr zKPOgR1`}nzN2A*gG3FS|$*Cg#BFZ)pka({0RQ=8^S-LPy`OUA4oKCZx0k zceTVkGgI+xn!$Uo)1J0E9iGxqbJvG_i2at>{9H4*yI=w#J^AV8%!8Mlxiqwo?bDWK zwDa)k`O?Vtv3|dw7O%ec_Ga==Rvu_lX0t1`f%F(V3#l@9=uDkfoH)XoILiF$;*Ivk zPHJRja4yN<79DSi!Z$o`q}`ITs^M1hiB7crZ8bG*xHeqYK2H-_{Ov5ss2T6|RM_UF zEWewc-g`6%mzmC&)(J zW#4Al>|5*BNBGdt0XZX& zi?w^)>g(L+tnAb`bv6nLT%J_0zMyv*IH$C_zmY%6Wo^uzu;D6T!RrAjJ<^+!mTXCR zu1!rmWp4ecJdKH}sYj`XCXFo!=OlVOtI)>{j_$+QQS7KBY$y@lUa8L|bV-{(*eT0p zT)yja#{GoD8RyfR+zcZE6)kCID2CyP>=a`D46>+&5UYNH(H2467*zllXUpz{j$hhD zTPC?`LP4;6?F;CA9WtrKdKn_sAOS5$Cb{cIKsoPAaL&*~U`Rj`4EJ#Gej7aO`Fg2| zZJ`F)T0-80>GS*)3Lig`HW~O=pYx}jxN%Ye+3&e&ugQbS{}TU! zK7}&OBTMKxKqiAm)%}%g0s7>6F3MSuCC7}o^yJj))H|Sr1Jy3Oh(Jn+Kx4Wru zvm)9O&zls;6)98xi1VmIjGQiG96iFjpyD}t6oUzpmZ<%jlr}+F${!y%P)76m3VN7a zwewZ0;Wc%kfC-md(8_y(b?l2o3$)iw%rNxRQ~a4 z3%|T1!(nBx@kcn5p*WQ=U$5w6l^0Rn^{%io&$HXZaLUFR-(!$n;v|Mj3}y*xcwU-J zM6wmW0W5!;E8MKVE8Cf8Jg}@JM8w(2o@Xa{2UN5-_x19pGD-k+sq1w+Da^FD(I4u&hDOzDUQv^dsqTW zVWrN)hPW7mYc*XL@Hptz3ABsbVV9WW(Q|0N6+B|p=}{!>@(=ES5*`p>7wSW zeWG?egn=-LqfJzVm=ip{a7}tsS5p(DvpK~h*L-q>`Xo5jhK1B zpghQ6r)7*Z-hNsI?Cr>TK`s0F^V&=!fE9MN&DCArhIn7;@{vLcbnn~J~hQ0 zp8`7wgKVSO3<P{lQS^9I%?Z8`ZFtAt8R$L$mf zsND8ZnvTy~QoNb!3zSOO;gIW3SM02{MfRu(EqaT`Kn=!Qh+<1bDy3-xKfvr9n<4+q zM0SZ;$3we%2r@IZ4b*42v&jyTQ+77mp^{#GmF~TX%7N!f?QJ+V3R7q0oPVcn1_QjtYPoFS+_v6Q-4V-K zecy;{6A3Uk=#y9+JXFp(T0wpS?;6HE-QMYPKk`I))UVfR}?ZE6xauuCh%#Y|!~ zfmf_z(|O=?J5$VUQ!x(oR-sL^BQ;lxvb)FCz}FTE^?1kfp>yv;V<_id!pWq+{jtCe ziGF!`AXB?l?I5~YtzQeZ>0Nq7pb6KqHk#dh*So9z`S$Pq-?HrO+a0{Eq{}we{TI{z z5$M)z^wcz8mB0+m@Ao(|ENlQxlVkO8<|}2uMqB}i^I*71iDH}pzYf|U+-v+Rto0u8 z#~adaSpei=_;m{xHE116n=b87${Dc%)`4jyxY@a-1^CCV0BzixhxwO((p`Ugwu zJC8yD;d;wX`tpY@w2rlKPgO7XX;^|OT$Cur6>VVME@2}8r*Ji3j6$pe_4{x~R(Zhy zuUb8iUXbC!(}0CZm_JeXCU9LQOcmlbLDy)C}Ml<)+9@%^kcju`v7jIb2x9 zr?qrSBoC$z$I?xNH=cF2c&E5qS{j(I!=rnS8l2KRl&F&`iOsdP3S2+F65#5~yv%6O zTLXF8*I?uOWXEB<&Anl6CXKT}y&|(zSge_vWuyTBX4SdyUzgsPql>KfD|cjSfamABb+qgD)T`t~eHY#rZ^Nd3Ogw z0K9tY9l-ola&*4zdKqwY*B_R_ruXEScdp<_BjzmCoJK;E?L3D>yCL5n?8R28cFcW* zTc&8M{JDA^+CA@OaFQevr4n{7luWnl%0|aj6ry+6=(~Kn&y$f+2!M3X^Y^8^3cOdm zDqfTLnlOpmZA0FCBjE((S5dEj_j`%0urh0YLq%*1o+$)7~Ov|`u(x}VQa z7}X00MYn%#O}>-fa`&T~)fY;uzTo)$ikycIpJY?7;!VrA?LhV{Imj){g-oZJ8j?x< z-~q~gA<3|pj?(Wl8l}G5C!&9A_c2TLFC1X}6$@0KrZy>ko$gFmYbEK_hmed-pW$NL z@mI4wX_G|PlBoOJ;njBG#t9y5nCm4j)^EB}n^~w&s}^t`c`tI9FC#2e(KoH8Ds!F^ zO_USGNJgS19gj?9xPna+hyt1CVKU6LIZ(>9!MmzT*wDlr&P&Zs_I$0_|oj}KpPJUXt zjb);FB%%m{S3OP}5E3yN;q$STC`}kd7PjOQY~3OL5+R;MGtb)h3I;>cLD=2-obyvP zzDLk8>DO;k;~7>j*n?J{3?9fPjI{h<3t?^ft9h3~VQQWHHK&?~LNAjul&kGWz{6^} z@gzULpO9Xk$PC6S%1J`9vE5JWB^Ga`^^&Odkl2;OA!>^xWT8nV+Zz}D6KvaR68p6l zmZP7x4(es3vp%S*uChA@Wq8vu8>+)~Fj1Ylt>q}tRalSie51}R{lj6}G`-_9(2($a zgI?zzs+EQc-Y!hbY!7BtUDWaKC1P!^u7K<6=*MeyK#QoCNf1pIim%0TQCMr!AK|v> zQiUQYJs+V#O`DTZa4QUi6y1Ahp^79FwF&v^eT^{j{6ZrY*EKbO%ZszOPvwi)Mp78& z8S@G(_^1 zT@AU6H56Z>zk@)Z$uaRR@wQ=g;HvOTZWc4Ho=+S*QT@A@mrRFWez@>iHm}jpC_xkF zH(z7Y>sK@78eP7VqWhTX`iJ$kfnNBCf2l>+A*_P@Zc*9X1VuHKCWc1_c0khxu>Z(Elf-npdl7*&&Og_`KF~ zuqri)k(HEHs3dkc;*x7Duf=rxKQv`jjlPs!ZeQP6%c$HN_0d!O5rSn*O#?tG&$Gx! z+*eYq@NhQ|mQtOULO~^bp1R_4xGF-3(CcD8&ARnE&16m;`uRp&gU8%Vch#t#*`Mog z-3_G7WuXL)#&VftyT&TS>xdd59Hl;^BXXR|OA)8u(?Y!fs#TWY9_s6zLkcZidQP+H z&z}#>g9tS@j~Ic24s+iO1xp8?JlXR^VN3&8HC4(ZhqS#%pNpA*Eu+N}B z!c>0hFU|j)9yvTS1MCnNCuka9BAKDzDPTz_irosjUoy$jXz6fwMP-MS3Y7-+*fweE z-6LCz$6M903y4aLCbDRvFsb;)prbbF0O%;QTmr?6g(ICRb>3BCI;lV*SQ3Y?&e|L| zIuBCxIJ!^F%8v&gVQGNpG2kEyDv=0DY9TtIR^p5uT!E|bfMZ!l#azcyd6I|~oWU!u z15FJS+5vkKj`|#zcLvO$5$G^*yriH*q)3FcK3ZX3K7d%HN{1$;!J9~#%G41ennVs& zO#ER?sgt#^oS;i_aenWtIGOT|u-_!W8*$`-!X-X0cWQ1NlR!vlLd)2mBS^w9ddJCGMFQ)lnp^9Ov=gJroL`V#MIllMD4eC;&|65Ux?dDyQYdjDr~!SoPRGDUFZT5nb$LNF67@ z!g)b^pp9Kkrhn}2ie+Ymfmi?5akrUZtN?rH&|HO#GIb22Mq&@+W;`)|jAYsVx(-`6 zcB%w#oFtWF+}JPFxy`ws<&OhtI?NHi5t{ESZg%NQk^&G2H59Rs_!dF!9ky{Na_kaW zKY}lPmo89uq}EzS;Dm~7=9OY<_9M6ew=q17osX!% zQfnVsBheqjZm&)a6aktAX^H3$=kz=A;7We$D!pHaZ#IFAb6T{cPosCq6^Z&4M)F~Uja$l~g zGDeq^z=skVvp!klXv4(?mB=1GWU0`zVS{b@5>FF{!{UL(HQQZpM>7M>pYU|sztzh< z&zHxiR_t2W^msYG9+x&0KOt%iQhYq&y2stP*^O>jkN;v+2w z%2F_5V)|ngdvjDg~>l1o`-y zfA$T65Q@WIrQ=2Au37G}uYKKq_dFcgL(_F{>7jFM+K4PwpNXK+`}Hv?Y$STmZ(<@J zn=d=`bn$}5$fW?*io3u5lnsB5Efu{S_LxN9nJ{=ycpa8|-PFwg!@8T`G!W7S z?+ndKI=;R0!fLSQqbK?-Q|(6vxr7K678c@Gl~{*TmMl`Ae-g4Iv8$F6ehD1K{h)ImUySefd~IeUKkHD^w03SybUKStoM1i zhJ|?53)&U=^bF`jYzo7<;guAF4$I&n8ZTyDL&7mrHuzISMTwMp5c-b7C_8qv4g-#P z&qNMAL=?xtIZFL6x~hoq4bb2fqpqRaxvY;41@-Pl{%-JKjPVhs8SejwXTaH&{9s!q z0VCdx812@`<8a^qQHEt^2rMl};x(8-V-fu(W;E?F>Xsz`81%+uJlu^idAgL7BN75S z-6@5ela74CD21E+fht#3h`5Sr;OI}e(NWNGI#pwUkl#cw6BBhbLl+JbX^3oN!o+y! zpNS$)QWL#eUx?;Xt2v5MXbVv!6S|}Ue&H%1e#U~LJ*nKj6OLdGF7T6u898ML?fYP% zMRvZAho8gP6SKdkIsSn7f=pY;coJdaXD8(}~#;MrFhaDo-f?8y~Iy-^J28Ivr$9R_8*cx9qH{@E& zO7AavEqZB4VcN;W2K^O7*fZ_-b}l+Z)=DHGD%2CF;qcG(3mUo#fe5HYT^qtJ-#9}g zxLu9oPTAQ!e0|X-9SYtm-P&)9sg?M%yHqkfJK}TaCtID}T#V0UY@F>*zK9np&UO*L z)VkzzwBEWpguAI(6D2txU6WNA=BCZNMMs|e#t?)#DJ>&x`z zNyoKRontL3VkN=h#et!|rMs!GC3sWSyze;Og>OF#kL_B?z_Mqvur&oG+}sx0;Dl8b zj?7USj(VYM%mZYyR(`)a)$0Io!7!j(3?xZxFwM8L4Ay9dfshY&44e#ULlm7yF^!#p zBM+9)@Tkq?DEtRPD77D00g~m`;LS{(^-;UFwd!yL?J6QDeFI|R&}_dEoZyNYy%t7c zOmW4@5K<<>1I+`6-@*6O-4p%W-_f@#B zk5bTRjs>dHZN%S!-Z_0QH#H!xzW+)%=z#2$g*Tcc=s;=x4ZK0F#TB+GQ?Xaz*t;Y zoCXx_!N8esF}NYH<8*(oW_Gr~OJXzAha=i+Nn3c)KPh{5@?GQ+dH+3UnNskOF{CM> zgLUzgxT$Z$HqJFmp(`=rI5zXrF}!rFn7e^5P!~mf%FsQ#)#|9;2{)keP}jjshm5Zv zeeleAi9O4NPB52Oo?)WvSDT0lEo-fMS#nsyUxL9m_D*9`LvNDA2(e^Kf+ThY9*B%4 zN_1Kj9im1O15N}>_yLrE%eO=?z1bL51msEPuSYq>a)npy3P7$3BZh;qH`uOeZVg&l zbUYt;_YL$9=N1yl;PfvWSW#-59LEyd$jYh#-9B;ogSo}POw~}20ad{ip@a@~V0Qub zr8R%W^=xPhP51(7Gu1`o#(0WcVU4Hc7!xF%$qMx&RF@hVDLKg;PQMCJ&K80Xk9%nz z&i$S)ID?g4__@n1&iF+^u_9|dR5@#N$bo%R67~EMuLEnCU&@oS%eQk?&G=E*1t|jb zy?h|4J=#ggDTt~IIRY-lrmoa~qi&p|#Eyzt!UebCP)sli!&yX zUN={%t(>eloZ|xQAmJOwF7NGv#D!_$p-wv-CHhHIKzK^E|_0R>pN#Cj#iaXw)CBc>5GwQ%wjZt3h6Q~$A;cYY~< zS(-)F&*sj{qE&nCBP^{_tVRM%Q?2ecfk)0HxV(rqJc{usX<={?NIo3J*wi-VKk4{z>S$SZCifmS@d50A- z`R!E0+oDcu1M>12o2>)~^Fu~8z1t{k`QG>0F!<32 zK-t)nObs5~?=vTa3=9nl8!U|2T1`&_zy;}X-znS;wnltEPrka)tlMI&k5{DL6RXpw zJ=kxuXk8RCVN=I?o&rcLva#{Z_TjbCi1Hyhk zJ-iopp5k_F4;@(S_yZcbKXh^RJXbeI)-w}ZUtb46_N8x7N2POTzWl7Ej?v7U&N!aO z7&^0a06QjSs`vw9%J%KxfY>Bh5Nu{D zeq(I(oGtIG^w^XMAXiyaa)^51nv^x2aLyxo=eVHH>FYav}x2j-T z1_>||fo_sc_*V$>e<~Fj5~#S30(N9WgkoQL^S^I8G1zT8=4jh{85%u3eQXBW z5#diyd;W*8cMQ%f+PX$#+qUhFjVHEk+wRzQ(y?vZw$-t1b&@Y{)va6ioVxYBfA_B1 z>(5?utuf{pW9no1clCA|?ZEwt263UaamVEt00Q28kD;kB_*azhM?0!V8pQF4Fsd^Z z|00I$EK0@in0>)JhZsBFCiidcfdp>NV{uppuJ(7fw|M7h2H}lwLaJeYM}4-ZJgMh(2UM;6ygOT)fi zxQzb5z7le+zuMiQVj_r1zuqWMlxc?AWzrS$Z|2}GoKy<*wU`s{MKt-B&stip#45Fd zF#PEBOU#puw$?NDRQz{s2iM#I^eod?B%1B~TKIPbCFB20mBeCAaF?N?!q^6MxO|`L z>{~K0%iXwPKgkQ9W;s$VhYOisZf8sadK{V*v-RFB{5~7ca>H>`QW2UI;iJ?-6WZa7 zNlMZd8MxDzKJnXa^?N_Xitqbpf-pG+3lLXx_Pv!}`IjaTpZMdXY_@p3GpfCNzZ&`9 zy$lPUlM~Y>8XKGajfRJ791Ob{+IqU{-v3=g$gyJh6Ggy&&{n4m_0;*e_cj=X1~No$ zs8*`vuI9*9tMyj$)u_=XZ8o22%i;R=RBo6=!)f*Mobil9Tl#5eza`d|d?EfPWQcq2 zLyiXn0s=<-Um=5op^3h|gR}nsgByMnbIa<}*0>UAy*K)IYkG8r7S*cMLz3{3#cTz! zHdW1m@67Vk&Ayp+a^(5WQKo|R z%b~QlkgX3J=#7kX4Iv?Vf<$)t;Ff{b_w`wngEji*|i;^c?KKmfDLJ<4$66I4p8-P(EL9d?* z7M^d$OTGB1yvx&=D@WCs}kDBlvid&GkN}Mkg6dPnKKW73q_CpdDA1KM^(sg@> z+sX~X%RVR)vd(kE`R?=3x<;+?PaH+UaD0wxzGkhoEcxAm|_gN^{+>+!W6;9%_fYvpdyIw9VRM7SL zxPIgGc)8lD&Zw82>+kyZw4CPurx%xTF|o&FlckhajFW@4B{|~_o`+A`*C;6CEH7Hw zzs>l{UCBE3W7dDB-pzS|;!pD=*8B1`ASk6vNrNgyw$Eb=-wP|TZa} zVG-q0W)m2*l(9a4BgV83Q+fC%t7EQGrmoVC%Oq-{Tz>%um+ayZ6?bAd0zBO0zQqvp zW5XvSc=CeQXNgVq;2i|Z-KU=?3P(4u8ufF&Qf%5ed($vcXx*(#Qy8k#p+wr-Y1^~3 z!4z(;-Au2qbTa@WQ<5Y92mvEH4y`n<+u=US#5+?-_GaV1P7>{{D1inZdj07vzCr%8 zw0UDD0u%m}wG$>FAeR5T7}&wn#lqfB-@(w?S>MUj+|v1fMov0-E^BQoGYtsEvi=VW~IQag>Zfz}O0C;@$G z+IJnsHP}&QO6k3Hi+efz9`!mp`(#&0pEDvOkwsEvm7IzX2sVn3M@fzFAtsWLEhr<) z6Du$jQvzj)pgra^HAcPzj&P?#*(js#w}2jz*kc%qBTbY9C-^medS^Vkf7d2o5ctmK zd#RBQ&k~!B&ABYVx%Ev0B^{QlKGoS|*ru_)1 zYcgT5%3QD$(iJ=1WFQc6477>p(Qgzbki0Z*jHD40mXdD93z{;Kf(8ObIF};e{eTBa zjR>j|lLbmyDdH?t#=G--58e}sDMuzbl>!88(4=v0-fNgMEik&cg{-oYl0gAdG-XgK zJmR7(x^JeBE1#?PpoNiv&#SXy>EcowHBrSN2$^_a;|?_Zmr)sV8;#z=L})mbtbH!G z3-TbRe}9Ukk+JO(p5+Y9YrAx0lJ774yPYVNbvEsbt`J}8CGvPVx+9$P#t6yKFRMPF z>m}*^>fP*5q^^~`C?4dc;ONd?r1$D}w zi+iMQlI>?}7|>dzb#*y5Q4hBn(B_P_x8wM4J)qrPleWv(wNoZD3P^njz6-c7AfI3k zM#hGgT7o+yy~1)^cw9ftS8*63m^Gg&o$!vea~vXEk}3sebxrlwvuBQ2h9{2JuNIX3 zynu`)+sH%g8n`?^nG-E?%*IOe(^O^S!Y5d3c9vi{M&G9=j)v_k+qa6B+@Dg+P_&jF z<3}npz^rDFJrL`OcFk=4-!^niTPXsm6GX4(38hWhh_pFdurqS4y~{t(UoN~S-H)x0 zj{X%&72lh-aAuss1UvddgL#YIcaN6SxLobUJwU+vwJ0<5Bdv~-o!4js2=5%pndJI_ zO;kpH%a=+S;sI8(_bv)lJ|3Bamr2RM=h4LxCCJ z#Z%5_jsOsoL0^E7;4tMkMSyS##f|ejX(34dXx;uq_f=1lm%OvH^8))5Ew9;xT59|O z8aKsJA=WZ#5RNfc;~bC#qjXH(Nglw0q`o7T1m4zl6{je0>aDSUcVs~#N#w_f`h94k z!$K6_%@~4S2LpGMCi_GZSu@NXG=sH;i6c2;i_U$cTf%|Vh^O^a%|iu0Lml&u1S5S6OR z@Z*O!92eewWE*#Y!EGcJr_lD=MMo@o(n9w14ZZe5p{AeER@IgIM8x61=SG~m+X(^p zSIrkwngakB2N@6vJU%%TzC;>+Ko0kN=I4JEO^nLTi7)*Ik@ewRfrZ{A8G?W)pC*z&0V7IPG^xg*C~>v(OmW$f=8T|w&T(cy*L zpFJ{U1>(;-x|FoRK1$(UVOD$&$sNu{!bzL{0ft@PTjkiG*Hi?#$h85^&2#GVh%oV^ zXx5iBkGvW>5X+M3+Gi^6hd(uk{r6{!Uj0@E*%(PYR|Vk!{$7r36O22Wv&BGdC_-+o zAWR$h$6zD9l5@=L(X2}@S00swlNu!pgAw`ExQ$Y;*GbNzjg{q&NjQ5fg&Sc~3XalT zX^fhfVFR*H@5INz3LCNldtAEpIX(Ww1DmlmEt9}laJQ<|-j?|uYix++hi>)LW$VKy zeEs|*3`j8Ux7yqey0ZM0wwHu(H!(j<^M-f<;XMe1D{oC9yB~v4A6_7`Hft89!I(}< zrzLSG-Y7RkFQGPmZ)P5|Mx+w{X|)o@=Wpj4jdjWI?d3Zo<&?K;3zpL=R`%QyIUW`f z!_gb^Z$J!=I(raG0;%w=RDOkF0~N_3Njc^<=Z~@XT#+2s1MpTVk3PS{6W67YOW~$ zFd2LLtKO~%<}lz!?xSh3-emJ0|yFXr4c?dujy98E=?6`<;y$8J9Lt=7lbAEdKUd$$VV}J{bJZ;N_&#LHto?CJs7;v zaDGq5Ad1sPM0fIUFXi4yrcHyQ<_{UFhyBHN>pudSCF`-4QU~}GSb8yo6n>jHHpo&1 z)PK~d7>1rzqmSsGH#jby+2qeXD}I}0t*vm1qFHB7ZO!2@0B%w+*4|6>`V3-3!kKG^ z2ZSnV$8hYQY%GUo&YF;4p9XmCY&VSMi3W%r+&sS=3nLJsI9t^vFHP415gGa~3#%#N zB5In-NH3+n>%IH(=gBSm74lY*qX)(cpMXU3Z3R2B0}z9OdEvnEU1UjcAQk?~OdqU-g^K`jvBvcp*%IAxuMw0E7Fm z1HqYTy*5C*2B-3!T9i4N*Z@;~=xWoggX&>KD;P_xVkl0HAa=R!4Z`k`gxzq~!N1OH z{c&sbEE9Vnurl>-6C6oa0Wcyy6szEQN@?Myom}7~dWuyRC z*>}H+nZ-Q0G=l%j%{Amp^8)|i<&Zip2O3`XKeOqh>??m0c3=Ej2DLG;ktcbi?0`m>lx-#d4r$XD*MaM z|4W%9!fqrv>+b*sx!o?Du^%?a-RtOLV|xj!K5?Lsk!}Hz@v2)K$Rh0M;qNo_}(uv-bCZ)ac9+5@JY;k zq&bpw(|o#XeeRoM)8V=XUFIyxQ zny`FCCfrrR)Fi6w2GB7mfig&Q?7<9VPT)$3Pv{|oNQ`RJY;U`oDX_>zM#C30+u*Cu zTAVIt{pZ_npO+$NK%dRb{)Ax~l;#Ik;w8=8cRbP(H$21gq~ z-DvL@K}Sp2!+wF#_YbQ7vdKY2FDONSF0k0NMaCRrN;aO>w*kY%L$r7-$0U*`WnwgC!^G&$>|FfwIA*k6x>7w#|In_w z_JpPFrOw6yZ7c%5<@>)RV`O2!oos(B`20T>{Qm`RJGvU$xmbFc{uiBCts-mx6Qj}n zpbobbWm;EorJd3QWLyrdsP0DAAtW~tiKi5$)VXxs&_MG#yVTlOiUsavGuLEcLdKK` zBxFy&D-xgs`k|6Hi8x*)us}l32(mNN}BRCEm>$*gYrN~09 zPc|~}41$<(Jtpl=_q8>#xy?NvIS+l@OU=(a8oiMe~E@!?lB2PL< zL^DwwT+>vz%x_qtd_mt54;l}WvI06bp?g8u@V?}cQYC0^fXZ-f2rr*&yC9D^*3u;b zXE)APy|>(~j3fc)b@!DnCc)Zv&9-B<8)8dmUHi5rS54#gynSxawkMQ6XLc%d+4h33 z10%*6m==X>wx~d3b(ls1(ydNn^n4-k)1w8Q3?Llo-d=%;4|#}6T%`mzMav))H2gH2 zrhw&iLdTMHl8L`yDhfs4ULSg(=$frTSZLoc7ba@dm#`bKX%d4vfcA;;!J7(O$6SRC zK}wqJr42FTInK`}O|3Jm0t&G}wLckmld*(PXz)v)P?fK*5J%I2~L?b})_vUsp~GrF!p% z-l=SR)z_C!LHA=k37_<*YvM1O>Zv(xC&*D&6Gx zo|3ePKl-m4pXuOz&P4&;>wl+os~pqrJ!USO?jLT}rBrmt8Mm$y=5>KO+8VW`#=URX zcZB4(clP!(z5d*}-#+h?Z}R>#vkcDIm~XawuNz>PukhF$hAoQyK6o#zL1|HKD7Qg_ zri!sPI*C*SzBbOu=DpF|{v?E0{eB2>QYF_kc^MKZ3fS5}Amp)Oo5k~Un)p1pYE z5w+s=`v?B8#zLwQ2PmESoKc5jB(RTs89F8ogo2$9U*CEM_W#M15p8=r~Hlp{4wnyQ*}6;%n05r2vfc>WmZLll&xkccoi3Rn?4=vSmM_O;T0 zu5&wj`sQSrI>E80)EisLAzAvzHHW6mo5(hJ!~V)+3u`j*2HUS9nx8yl0O%8t_Cv6qbl#0)*dS>G2W> z%+myCLGgmzi3M`7Ne=of(;lM`W$!R2%F2LU)FZErLm=m`AP!6<;xE>kik^m|0g{|X6_cqYsx(H z6u_pQ27Rcpq8;tsT^F;sn&=Rc;8RK}IcLj!7M{0UqKCM7nAbr6FBNJDpmKUl7loQj z^9LO~_pSvdxai`-n2fU-50ax8%1!t){ob}9wpz>Aeb&}i-A#QVlD7-7w3HmzC}`Ts z*;vd~Qt-i?<4>ycy-cer?z}5-W_?eqNvVmX;oNHv1y$6>WCL zhGTdwow`%&NSecrih<_*4&;Vf-J@-OCs(O{O^Kr>bQ<>&^HRe`QXW&1b&=lewSI=hqMRwKYAR zZ(rA00=0JE?7;!dJM<%2p?1o2I1wywg%L~G{}i>r5?+O*rbGmSR3VI~Xbv*!tUO|$ z?{*L=;IiaOJGw6jn#3y}^LpQ1z5LEd%qYUHpoIB;^=RNx{#itTBe99Q8~Z zj~7o|zbhi5Gu{qg=@t?Q>b}QM(^!Su{Ax^sUEtG z+FnBN=XAfL8J>!sp&ukG{vpiiElbA8;yEuW)F7SV z5l@Ca9F)GmxDOOvn*j+_rP2Bf44WpD-;qZTPv11blJ}N<$`^C~ESy{8QJIz-IsSrj z+flbthwk~~Q|yqu(_-VQV5XF7xN|w?XTx34$ga7svASGx8XQRQ87&T9BLZu(5u;x% zvbionEoyb28jF}9Ford^t?US66X579N${p?q4B(clk6_so=zH z%j8#k^Ta?-8^EN!ek{xD>__FId-)waXwdu?7?mD6bA5mWH#i%fit}>u&-`lD(gCFF)A(b@=*n*muWRyW{u%bawX3 zKBxEde0Gtwyn%$LYLjDO1P=>06(1)=%6ZKWyjGFgv&_e)38m4xXxWOJW`Q|7ukUcx zg8jlh)h){&j?E|K7%picMx^i)Cl1ASKcp zPoV;@JXMCr&-Qn@k|;g}2{*eUx+^jwZ6T+aSpDzItAh9r>;$DEuODfk%9E(}yl5n9 z3aXlTg_$T9d8FBdlVq|~0buuDP4dykiGHZ2OC!J`sf(ZDYT{CAM&9OqA&P-k*DVPp zYzdr`9D-n2Q2=-Vt%ufe=i$jZ)N6wW-e}G`2fz1UPyp8BpFvV!Br%p)IcoUzNW6yS zslUTZC9xw zH01&=XoqyaF}ASL7sUoAydwf5nclI48dOk@Ch?^8U$VM#RY-e=>r+$><7L=!_Dpwg zvQTfLV(<-?fdq!qEJ-Snc}h2~i)Bm)oGK;cCMd8geNodGVfkjODAQ~KJsgsEA zr`i@h9W#E>x6cauNkgIQN^8-}F@jCTSz4JSv)oCsq{$4-fP|x&q$0o?Y{4P*a0fzT z(cK<0>OT+hGhOnLlvDKLi$vN;9CHvEDmFyDzX9m=bDjkx1$yyl6l9j%OMkXqx^~Aq zBR3MKqLpKAo#Xf?OMvgIScw5t)1dk8HeCPn@j`$GHGsv0mk^qDDhD1uE;(<6H3t^= zoPaoNtdF>HKlC4B9q~6by9piJqB|i48EHf`XasOofWhViJDr-j_PM&O844Fe))(%{_`#>s zH{CB;62*?6Uhq1Qn?jrs3Oj>&&0AX@rOjJbfiDuzSR``+u_us0d2t3Z$Mp1pn``9O zrwyB90e*10O!vB2z?VNn)CR(vJa+>W|8K=;5W=R^)P8pE{2YX3%)iAn6#YE}=#YZQyN4Mb!+UiYE~ z2TK-+2M&$%SaGA$Zbp|%NKd$+9Q1Zue>eYb+u!tvz{z25y+AU>3V{jW<%i?7@Bq63 z;G8ZQn4Jycgphzcf5h6MpjNk>!dP#gVn~pDye*`FSsOg1RBq}RnPD1PXv!TSBJXho%K!IpiW4GkL#})J@cDu{evUj6TY?d6Bn*_qtl_&*GcGY#~e8U!*VfDB8A6h?diKrO^sQ&A6HqkJ733y<|m#p z%{qVA9|ni?68r=;9~@_b4qjW8P`&C!q~&)tbN1i5WGy`o3Sqlo~nLJUQ|m+(q++3l#7p zg;P>$T@`!m%8o9~g#MhivQTnUd`*Y(jH~&-EBzIw6p!e~G#2l$f4n_8Ql14uXvrj( z6f~(lI&`W&23HDT8cBvZoR8S4%8g(-W{z(<4L|)3(ku2Q333$(S8-0sj%jlHEC6|yUC$?L-i&shiOje{{|ab zT)*2KF)KPF$ZOh$n!}bfM-6%x?U+sho2a-}3ZJO3K%crCykHXbBpH2dC<#A7qAXhAci|dFW(_MZ+zdQWY+Bs1#|p(@5-@}vk?9nXu|Eocoo4w_~IGX0uf&D+T;X!v!i;^i6g zWdV@oQh78T(8PZu@fYeUlUHK3R^Qh(>z*dj|SJyvm{$fWO@vLOGGG?N4AwF zmyUrfvh^zjPXXk~SPGm_uT1k0l0Ov&YIaf3zz^F1@7!%{(DUt~X;$iz;*oMOhGg<6 zW(6bUI*bQoINRnoahA&7@Wtm@=aPsaUQcy=%y~I!e5@t1Pj$n55m7+uUGx)o=Z;O> zc$MpeCL+8t;iU~>l8xi~6w?^Y^uAea>aPby?KFCn)*BZl+e1G&38Ok5iu;dm4x}Ys z2>-&Vb08&*GHcT*nTyq@QqmV_kELbKRGleGT+Kq2$HjYe$%?vM_2=u@*q>=vPkqA+ zVXNlXqAaXc=$CkKH5W-|h1Li`)BXJBxC!X5J|b6u=e-FkYMo7lZ@etjjfv1hd=XmFrj644Teus5PS9N5|9>Z*x#H%5G!|0Y|!GUTEovDO}S^+bFWHYI= z9IXSEV$Qey89%bL$W9kK1n{FKJfwZ1u2<<}XNWxK4c8RXtua-Lvw>q((GwNn6NTer zg~MZo{bR-5o;^m|gx)SZg+1)7GYo72nC4Y(}d}nl1 zeYcVCrgLe@p(P){AFl|AAu5;kTA!n68QQ(^GfX&HC_*Vl^=jz;h0+um=6|2l{uVtL z{tL<2)2N({$)(prU^xvDN;#Fo{FW6QX^%q*RzmV($K~xtIeS>}NCTul6 z@`Jf0k2$}!`3}Tkf#kZKRWl>J5M2l%`UbNB_<5N&PbaCW_yZ%62-x=|ia3xByE)Rr z^@&ET4gYu?>q;{WQdgYgA(ODj}AWzbC6}lv!0jX7(D|uHcr9etm8<`rd+O4*M=Z^>0VNbNjQ^+YsFVkXvNW}^wT~?hE*uDql|~(08R2NAq&`-we#c;68x{aROk*gtM>ou5x(GUb5S{OhlOkU&;z?%m!BZvjx8d|=T$uXfad0ky!h+Pq&UXRLymX7kdWZc z##D<9AxgQ=+I9hk+}Z?ivEokn110;$UE%kNb$k*AGIt&ua!}Z#$)AQsu>k!&Fu-0^ z&Ys)Nud!pvP+d`~RNT|rqVDYly(k+vAgsp59O3gKtn_aTE8brs*2>94mut84mJ zr7O3S;YcRN6$IF714io5ccr{fxj?z;y&h1m*_R_QGixkHJ>Lh3tIeKHd;(ilcX=&| z`%{mrtC>yn6AxbpuMdw0xKiW2*`)mXayL7CIN&z);cpG~V#c`}30Q$0J)Rz#t&NxI zb6mz;2Ik(A&xXy8?dtOTbuaWsJy335pTGSpgOB^m&lB6Z@0pw)y0!!ad|%cV#U1I* z3vn^i`aRk;UvGy5atuJbt^1!ls8<62SuDDNK*>;l@;Jize{#!q9PVc_ed*J+j^Qv4iH+5o^ZXHO}5xXd_=0Wzw(^x0#VuofHZj``? zUO7o@ab#^#I+3s=6MJ*>fS^T6SPQW#xn4n8APuzA&FWOQ`|-@>B~BU@AUxaohVNwS z;Un8wz>EryFB4RvA9~ZW-OP0!ayWlaADYxv36?f0RkyK{;!&ZupO^Cr z*8+~fZKFzXG6=##1HvVs7plLlP<>9IiX3r9WR=wx77a424|z+-SazeYQ$uXh?svyu zkkqT7P{;&?MNn8Xc#yR!!BNN(a2mJm@{A7E0E6{M%n1AvQ>2TwK9{wJp4g5Hi{C*x z7(=mwFBywbJvY(lFq4y$LEMgRb8eL(I9JE$a`M^fbSac^#9`~doz>vI2pslqa7v;m zcy>TKg77L%AI7FLd*Ey^PgOAsICe%57&}YswU&8rFTwpa-HFwV({pYzAK>*P)prRp zo7aHFWLkf7=s_r(!v8bt;#Hk7CMT4c&f)g9!|-9^XJysP%+90>YGmCVT3;Aix?EaE z@p`AnV$=n?kx!wx2P&{2=BYbt^MDwWMU_CoA+s@sBFDtAIZr(4i)=Sgn_H@~5~&qd zXl4{%6=nrNaJeW6Lw<@6N{7rdTand&iJ;&SDmlwHuVlYCFS+S9*ZN|VIQe%V0g*Hz+i@WwFf5ObM@OSOTpzy_SO?$Nt2fn$m<$2`9|K{?#npKvo85Kt=S!&*@iW5`|`<(Ie8XuEq4l>;LZc|0Q` zQ$84W0s90=TvIfFKAo)pA87R-bs->vt`bo@dn;L6S6SotTloPN9CV!bz2)Zp^9`tz z^wZ13jI(uELxiEKzn#rH`3l$7JDvm3M8edwe9)7pIvu~qg=0W&+FFn$ok^r79~^Xe zorItT9bv2BX=wrs3R;!&n6{0XUzyT&$Fk@kAFpPH*X;@k1Y{O0J;c;Acc_H%DMbj6 zv5;)sPa*CB!_8A&xgGB?jU+t^aY*04Q6 ztB=IwiWN$Qp%mwf$Y7~RPDB2hXNdMa6gnhe4a^9Opgc>ml*n{iJFU9vVvI)=Fw+f4 z1sT1|O6q!ck|``4PEq-DJ$Jt0049A2DU$e=oaF(GJV09rs2l7qse+1D$zvcH zgrUkB29oAr!JjeYoi<|zR~QEd>xMc}IzKN3agHFtws8e#BIkjQD6Y7pKY#yOZ-6xf zF8!8zg?NgE%K(hXkZPLQ#R&a{@Q|yzuhPnk^_Ej@9^6$)sP>mt&spBVSpcoAg0V=9 zk?fU{JZS5%4I*Bwx@zoKHl@fh}dZCfi|%4 zKY)4Q^Htnk4UUCMQ2?$s*3&0u*XiFc`_drlfEK8>&bf!bh^yDFjFeh7{!Q(Z%T0`< zm#qRFZ4Ic>cy+O&C?OMM1d8R65?h=l3rYTN+^fCci&5EEzf}yq)%E!P-93*c zd@;L?`;j$9c@IN}vYg(f)DnJQG0yV0|6BhI)8Bc!ZmhNQeK~!4?#*#)>FMyi7{7qb z{p<1k=vCU--uXRTEdb~-m`zBl(5Sb<;Ux@32D&2_oYuV z3mqdUV+>eC7eeaUoEjolBitU=WP+OHM_z$l+uMuSZ0Wpe%bxv((j}Lwb z`F>=(LZhp|5rm5vX{``Bx#&U`3h0WDu97gB6?sAuR;PAplBTXPLO!7iwI?=OyB=kS zAeaaa4g|3$60k`CY>oJ7A-6e|5tq}!;nE|FD7%ylj!-7MBf|*d5HF)vJqcPPlHOJX zF~W?*`Y!<8vFE>_7&+dz5uejvtzcq%%PB;PZik*Ls0v3TT_99+SLkI;#iYoLcBj+3 z;P1fEJP9rtx#-MQlH$=)$@pAGfeP)ANcAXnJQ6L+Fs~XSwNQaQM9IEn(!Z zS@guEfzh~3XzMb=R2P-D&_yPR7Titfhd|}QZ~p$m(kn#Ym!b=uL*aqjeuh|(^N%a= zQ=#H(u-g*DifWNprFR=|;MVcumXRh*BS_EsY4&tv$6JfM^yK+Q`okMclnGlBT9a ztw?1Je#i3lPBmi!wL)Cf(IGG{A}Q=P?KUAa_~mZi3AvgW`ibR-vY05Pa&jG40tMH!>l<5vs`_Q zEpB;4FWZW*W9Re;y^wX2-#i#0lHNDo=Cg$jyx~VIMZl9H=RDMM7+c;w(L?O;1#Bsc zaq6oqfzipF*VMr7nNJ#;hA#o}p8gRNaHSoz!q?*eGWe4g>S26*&>;~vl@ZF)S_tW$ z@9hv=Ix+Al`zYr1VZnaxETf*rm~+$`xzBk9OM0I&Fw^2tC*vS`C+oGq%<4$~>9rtm7d-x`+p0pN`K0u# z81X2D4)g$}`cQ?9r|_4uMyR$w4cS3YmuW0`?4vw0f6A|6rYc2Qrj0~yrUbnI%1KP*R>*ypoVVq1j_&z(to zUV)fB0fIxybtC#zp)95Z&K3;7ze5d{62DvRuKoWLV>r5`>dXDeGbDb7sPOzR7{kHF z(na6c-qy&{&hY;SYjkU9{|`Okx1;aa?boIEgGNc;0PTd9j~06zLK9Vt^JfQS+2rfp(2K^lAplch>4;T?nnJ&Odl3tRYg~jM%_xR9=mZBUKuE6S zmk|dBu%{1sf>}t3Mj~i{C^Ara(g8>UQC6Ho@nMU@jH0!%?t>YS90RDROxVvbz=T@= z%$#+w^gR?Sl}j-Fxk^~3%l$RX_NC2p1_} z655)AC68A{YM>!8?~uV~DAyQO;(*EIEl&eOS4@XnaY2_ybCvJ0c>yfh0VgsGfpDY- zdnAU$8IOCgND0*pD`QI36zyu+*wCc{hQpGWSRjH(Bl10H_!EgL<%v?U!e;j)@~Mat z>5U*|scB+OrwM`gw%QhR3b#dV3c4r3V&4~ zLt5es5@6OboQ|jwF<6D0r@z#Z9Y4%=uSOx58Rw;n&Rx zl`VtUsRFub*}=1)oLrNZuE9UU_f-`cUzx~6ICzh&@@LY2!@_=#3M)#A75vpZCoH#* z8W-C*(Z?ikheXUKD8X6xQk>d@iyfDVgYyPDH}7YrHX%F$1r(|L($ElBruhHi5y61WgAAc5mV%t=4k$)BshoCL%K%beT#10C?U>Sp_ zkCG}#k%K5Q+bPfXI`vd*MP9E)T8DCo{94o?iHHiizx=g`7eT*b@0p&G`Z4tx&6NDzbpD*1R$o zHO`_JGY6XpbIfpqkrpPTUVNevICzh&1bDdBvUjWooa& z4_8(mF;uA6LIw-RL?vR(hKi3?QkFezP9pD=HP06Owr46+U8fb2Q&&B!tHiF;Se5V` zNzi(C6~PIZIXCcKoF_(DE*$Ajc%#&zQ@vAd2Rbt2XTOH3MSLu3y+g}z_hqNe##?8F zk!b9W{d9z6G-lM?U@IDFs7zJh>{myR$;F3;2yr907%rCjDGQhiIP4wWo&8tDYFr56 z?f@U31F>Zl7a6XR+;D~g#tBBl^}=_2k>`FDWHIggCD_;PFx`ikN0~*zx$^Tz_KgK+ z`~g27ER+v8{M}JZT9S-U9!$hn{ofWmHj1|S?MxebIQ?uN$N&-AACnf|I$6LikU4KY zw?5jtl78^4+{X5-vOl?Y?UpLzR}u|?+~*T_%hkAsPZd*>mW^^68z8+OOuaxNA#7U1+sh6bkB7*Uy<}tg`g|7+Et(`x*KIwcB@wZM@*Z!uYs8(^0QR9-%q*Z1MIz zrsi7@DJ%a%rS#Rej&<@im927FLEt@$#|{7Fol^>FdPC`}OgYXrXA8$-Pov$SSEfql3JxCoT_CqDyvmZwxB@S&uw z5(m5tKzmbrT7<4<<1)gL_YccYTO8;exj1*!NU2`>I*$;4Zd047ua#RucIL7`0RReM z$TGt}Y*Y5Y@a5kggL%Y^O5h3n3L>oeA~aFt`fWlGS6;m^SYNqYKH*TB9V|KVXvmyH z;NA@M^|YptMaI|yDwWc>c(t5saZbQQnmB{puB4}m@M+bM+v;u0=+6-~Qc}n;{xo4a zY2OC_v+;5sMuIGGH0yHZJX(E__IF|xI0?}$F$Z`oU7ZTrM7sNWh!qheSR^tYkg4#r z+Vs#_jA;Y`ahM5ypA*xjFzGM_!K%HPVanSE{Gjurk-;ooIb--Kor_Qcz&{`fzSD2! zFj|Y5CX@g}Qi>Ptk8VINkUy$Gj>6}k%?Tr?r4fsv-ETLx-+J$3#la+vO_4*W=2FG?EIh^yHjku$QoJ{WJ zAaxrU!ny7OcJlk+mOpd66C8< zgc3MDjWJN8e|S5a#jYE{2qPDmxIv8hx6^|08YT%^aMf z}ieT)rg}PzEu;CgvL$IqKsC_-g!a^ zQHkRyl5)Fw(ioOg>&b)ILwXduZOO-$o(~DBc3-GFXNJLBBc)xz1*<{lmR8-}>lz-I z9HuEOiU5?~lKq$;|1Egw+1^uq^3%Y8_iE^H0RYaI?fqYY`v2)nHm&kxwK#+Fq~X== zcNL03nQH+Lw(STq)u$2gIz*?R<~){PViZ*;cvG~n25U3YhP5h~lsMSDA;O^hEW3-H zU@&EqAt5@`WWSg$1bk7BQ*Kzg*&K( zsF)5q@;v8l1*_-fN80|BwycatkSQogO^K}^&0zF~0ki{Um7lQC+?Ez-XJbbEfxHgpu7h(o#eZOAozLBJ0^9s!9DFOF#(HV`Yf`-4<7JJ^HP%- zHRPtfKPY1tO1UR%x3g_dg%}?pTxcww$^J~76FVfZ#J_*?Cna)Ah$k8>)B|*^c{Hc< z`#5bVaB`@`E*Me^lrRfqatO2-+6_*Q{_g&n8~x8xx^d`y9jG#n{z9|{oH*3fXx?pL zaofx(9n^vOdRoYz&YV4a&d%)F1qw?=X&GWgamcC4b*X#A!ialNIn&5Zbd%INwIq?Z z0t?@6S-7}dlj19b@A>w2R^rgcQtQU8-}uwQq7o8Xx(dwl!uzerREGGv+^9Ab6IgjR zJ{fyhJUxDwb7)ag=fi>TsH8wAB#7*xhkE~$){W>6qruT)EK*X+n#{J`s>kW!jt3r0 zdLQm%Z+(-&$JutYPjHGytY20i^VH@&HF%0gkhah9s<&(fbCltxab$>b3Cxh#Y+YDg z*=EN0qfRd0u=Fi>(zmM9-`}1(WaN8kB(D~gD16a8=2fQA2D5zQj}~zXb>An4b7O>e zgC&iB#*k9=ooKh~?EwzntphDGeszi(-@RyD-O%aj6y6?u}KBohAf~Y{BBVRa)NnJw^`@<~mMx-;XDT7qmo8 zt3-MuwZ62@NN?K?8r*0;9$f8wfsuIC7BgI`ROlw+w%Aaz%%d{pki1%t->%ev@b-he zjdRv%fSPIjQKdn<)7PEz>Si~sV*)lk^Jevh`$>vMr`gByyG#C3-e#?-vs&25YCp~IXA)N~At zZO+#G${HrXpLRQ+-(?$ePv1+^dgCHlKeltBah+wR$(LL{A8W*6hl4Ka z_*BpU-z?Vgz`mGNoS3bhqXgKTfVV2nWJ)H2dZHs?-KP@>!D?dGCf_-Sm))Jso<1DU z5V3(p_mQJ+iq`5>=o6gj^txBfcaoJD%;Vgynf!>QU{E?R-Rd6<_#QNRNe}~4c&%Q) zXRm;hv71!@%(ry9Eh4YOOh~|{ffW-WWMv}8F?zKf*6|>St}j`wFOPXR6Wq5Fk}F(E zvgiRnFD_zfXrvvB;wif59T6D2n&S9#v>IXVb1gB2+?oE_zF~cDj%85%UBMb z%Gd^J$3Cy#=@ng9o-X4n3y&pKC$5lT8gkQlHHxLj{c<(m#=&*>@_eP47N2tQT5Af# z-j(W|Xifdl6>1xXrG54em?-Z~KeGExBvLqv+)}NuV7jwB%x=roYiOU=m zSp915eHsZI)H3Be);E{Mldwszsg&%qVfx&}8(qF+D%HmdmJ-rGGW6RxBC^^&eh-vR zrhziWmYaomBb{EONC4I+t1=mE_(f{vsKnp_snkqnPXa6R5#Z3k{nL0qh2L-w9})f& z#uxDM5k|pt`H4a3HOOLF&4-fIkBB%(m7u30sX;ZATR8IT`k4uFs56V2I9s<~8Q8cwmtY$+R z_Zuwfhqu;Gc<&1j0yd=6?qgrUCpUuNY0UfZ7}LurVOp9`iPv7IAJnRqIi^p)&0P&( zUy&B*bE!DT#IYwRm%nn#LyePwmc0%Ln|=nuyUrr zx%Qr}tFG0UO7=#5LEGRuQLHf(c2#NRQcXn%MXy90!M^=$;f3(L#{FrxQcaJQFNOPa zB^n6&SSG_;qv%ELBteJE%{%FR$-;VbRidSd_ygM$^#X%cCb7BiC-$<9?RLjaUJn;p z!E96vQ zmo?|4U_`7{M0a3NP%?avxqJyDwuBxF&&>Sz!pijIBd$S$(&~GCfT2vrG@g)Ts{WCX+tb01s9JGhIDuOhqfm9I zJ&w)VuYF8V8I^+?-K5&5!akHQg#*q*Y(k^Q71&95yXotri-7Q?Z4kSs8#2Q8M7oV0 zRwN}d)CCFm-jh-XLsz(0oCD!VR*lGCo1&b2s zj@jgwR+Uw?)I#-?C&TUj0jSW~S;Nst*0lq_rZbWWnyyg!a8{1mOhTRtX{(ah1#5o} zGOA7{Tpjox<2Y(RWQJnaBkQXrkaxP}<>h-H)@%*^fF)@>4T2vj^0~G?A)9#%c~f(V z3g>3S$kv@{DYxk8MR(ttttyU`TEt_`)Ie+WU9O&`2^n3NLu+tp5NPyfZmtQK^7?il z_Oy3+X+GK5cxG4KtnJ4px2E5i>gZi;$HUDI$fq->ZLazI`pDDj&!fTWZQ?%=(Gnr2 zs+eQ$dLc4vB1S1|ReN`;gn7SH?Tqx|aIm<0lS{(PRO13wPxz6LN~50fVdZdWj+vKk zRLaZqp*OOnxm+nNFoT`O-T7ubj$uJC*pt+S8&p5j7b(74B@E&6c4M-?*WR^&SWhcl zL**O5hFBxb&k$h;7uQ^+Lrf>{)(;rVC0=zcnWlTgbbC$GQ!e-<&WI;7maCwaFjr|A zxxGE9(Jk4k>P;jv$<;Yusg*Oav^0~wVc{02d*Hlk#OqcB3m4MJ43k?D3#D?SYbbt3KO)eJ5ZKi`7Y(*fK2FsKmh zQYt-OmA*O}gX%PpcRgkjI7pZ*#{)Ps7ozlEu72=LEHAqp<9CX52uL(h0KNd??LRb- zalwM9c-oqnn%>5|i*C*u{NREb7P{d8&h#;(WisO_vaGMYp!h*FMG);oikUnUyvl-{ zXN`cxTOn`UDwk9gl$-yMDW7!(#GC)M#wx5@RdPJds(Df2Qdnu}L!fU8O=Op<*L5z) z{`JjVX%+aG74zOt$^mkc{?$XWd*1G-&;%1)zZ8QSiF!!N6_sXY#kBV=GbJk?)(?~M z=c2$s|Fwis@`i;&r-9kqt$C}8WIn#@=4_q65tYgjUaB-{G@)Sy7Pb57easu;`%ZzQmXNeSR z`n2MqEBF`eXAzSHP*R@Pma`m&W()g zvg52vcQCQIsHCM3JBpK;&?FX!s3e1JrkpDD4r!7LCYw;aPLsM1C=4UP6?Sr$8L*4r zlIeN>-Yqb!=@za?Qk~Zeq-roqf9eMqDHg?-gdUL(Ytf;kudxxSVNiqPbb4o%3Bgcd zWLHyETv(alWK1M#VFiu+&WbM8Qz(Lt$bKkpT0?bj`-&;056g3K^EKQVxLAJgTx6-# zV&JQH@X7|)ZC^cw7q)RpS4U+BMO0>Zz*p$Km{#G1-J~qe|8zK1wt(7I>`5WD>a$~_ zZ+N=&pdP?bc{Y?KZ;lQ(?WF6jb9;-=6;Eyvz?6X!g_)vEl1DOd)WL=h7qM!Z@Q9Lc8oo z98ksPWDBC3Xqb3x4P6FsqYazYM0Z~AEQ&mY?DbDnyThV=zK#rxL8B>>3>tkhFWerq z8_YQi=r*!{=P0-h(RMYy6UsqDRH6>Bp0xRvbzscVp>%>QB~%<1Za|k*kIw?(ugZQ~ zt=f0Q|G8PI(+6%C(hGb(yk$41pSYxe3CSj1&Hx(gtBCRvi|A`Agpa;$aw-ur`-GDq zr;7aCLgFZy)Kon8@G(;XZ(0H|Y_{&MplQ>=bY}W7RGUJrZRJdJEF_Eu9b zot)t~3W?J}1mGPFE7}d%<~J;I4NTj!U;_~r3E2WlZXK)>=qPF#&cJME$|&%VkOi7N z+0~K!D-+3C&YOO`Gs2ttRXZS6_+`+ijf@rokMjq?Ox|C`g{xU%V_2K4M@}Dn$Y1O2 z_icwFPE>ig#5Pt~D${#~n!zTA7;M4dl@W008#CQvhn{1TirTXBaY~~tr@6}-6L)~- zH|wbYt~lzeY@hD(mO$g3vm#Wo_TtVnDpWPOJg1?kx-EP|8be7M+7` zBTS`U^q|&89aJQ-fr(PC53;d^A&Vf~%6_e|YtrOrEn|JmHRWI78#1zC_Jx&|t5?h_ zlG1K`dWxPvx2rnj{MxJ%G^<_}z%SJbm77WL$B)0w-mxtMjB150yw`Sb^YrAgAQP{n z0OJc-qbXe69s;WZ(NaFqd_riOx1$rf+2OOuM5MJKlal0O#pE;tK33Ps&@@kdvNa_2 z#vwXmv<;a8Q5gI+h8OB$@o4IGDjk#NF*#&jn5%7=%`r$qg z8GIldVCCrP8VtuXt5g|uB2{Q@ZU2yn68tKTScF(>yq2o08d8XvUHPt>vimw;Zdv{o zGEtMb(MR&8PW^Sm1o0dp{pQ-&guAfsg`1V{3D*?w?sDAH-|=}%JDQ&Ff4Pvc0Zm}P z3k3sn$HbmgPc~c_N>J^U)(^!g97E(B$||Tx<QrJ5N5v++1B;mcsItzkm0}_Jh^3 z%n)S58QGLCsJX_4P5W*B#+J)L=nkeOf&4L2thd415 z9+(-@4}s6(L{AXNdAvKQODWhy@gQR1%W!WRr)?*-jp(NFqZUjPx1_%eV~JWk_)*?Z z3MdfV%4@985-4k(eU1%Ll@C_~OaCzz(Y*o!{yEE zPc0;h9>kkjkx+tyVsYRBvw=zxElR^aKUt2gx>F;x0j3#68tRy3N(5>IIcPF?c2u^B zTjb1wl1fuh;>08s^L<|{?3&aL@N?k@4VzKB+r{oI5Lei!)y~bKz8KgPR~S$XGXN0kjARUqq`cw72yfyX$<9jF+EM-6D{sX1|`Lk0!+0 zs_w^bpobi%P%f%pw$fsNiuNwMrz8$PX{P!vB5Kkms>=z08$oQSW&*y4C;C92hH zY37@F&|-9c2Lq|hOntTHHm!2Q1u-9^3CF!P6Te@z>l9``uD`1J)oMM#WyEUSGuiP1 z_L!&{cA==q6Fx+6n%}dW7kCJpW!N~0p4iX1?R6q7vMMlLD$-^Q8~E5Fbo8{~bP%~e zn`T+fhF+VORLB`3|2px6J4%O9<3r~JdUYP(1KBGm&*bO(W`7LfzxDo52M5`Nvlm z^cf1M#5>MKN4{Dumw`Lw`F4DR4c!n62)QaGqN#1F+3N4~n8aZ;NzrT*l6T!l4=`xm zkGs>wEBBuZE65BON#xGv?rsT|$1^q61uL{-DcRyxUR$sB5hz7~pQ-YZ^hSWL=^BzS zxve6COSt;}X!6ugk&4<_(R5R!gxu6nB9&iT$syzjW#?z`zKbXFZNU6Aj+`5Uk8~I` z>zmJCHT-2e5x3YqWqB=B^E7ut?zm@FOc20?Nt+N3LT^v}GAK zdDfJ=)`XBCH+LkzBD6A{WUVLmL5@B`K8euX@VM-(=vBM!$&T6IDbU>!(AZhD&~{PE z78`0nQOLMnXqIG7<5v2s`IBY1J8*YlR28Xy^=aH>Uqei)X)c*s*eeW>+0|a2jaSB( z_4aLaD~k81Gf-nk&C6PL=rx&eKx3r^?V4lJe2;ljcGVv&x~uRdt?}^woAe2sPf2a* zN7kId&+%z;6{Tv2*UJ)<)znzUi=E3S(CYQ-pN}jwNeM=B>y~@uO-N#Ck3+xFPoxh{ zO7yN>PDJ&$%y#o_$1~&mZQ7x;8vFLXB@Pry81C3Z)-68}`>fZPS09yhO*}KS49l|# zX=LD^NX*$e4x+q;aNmbyEkbp!5$D@q*C&s5l`0;c$ySaimbQ0+Kp+s5UN0w8qYcO4 zBU(kZ=+(m%A0n{kW7V$VOv*LQTrivP!SS;uk5K$-)=}Wkvkm7)#|32L6m+8qjKVUu zj+?g-WkY;3Juw%06+in4K(-wi6A0qd5RVspUZ-3rUFFn+KG|y^NW^StYXL&2YEB3Z z+1RFs<}CH(DcJ}q<-MKcZRSgjY-CCo*vQbqO7M+_Bx9&``)J1$AyrZNMCZacm4zD& z?}oEU6uR=|!$tY#Yk%HJO;{lyhBOSU=!rz#okqpugBt29Uo~?P6Zf$*@r-zZ_aSX*KHSMMRD8-`3)k462!?a(!yZnpT{;lI6bSjL-x==US|&N z9qWYV$Y7Z>nPFW3pJ0ACab=H1Hoscx;H!F*=!+7%m$obf0}o~Mk)5;5;e<8-%I_HT zs#{nbnd}uIlaDRiq>C(=E&m%w{=(>bEEiOGNy>L{g<81pKY@*| zn8gcxWEiNqlJTL*P^DDBXl`1&O{)&lQ~;zAvlSL~UQKMG?wkZLesfL1tJ}NqO{lpl z(|m9a zg(9DF<$h8P0@UT&|Lu&0d(^2@T?t(*Qb{nSc79N)3W{&7@fhmE(K;Op#yk%aBNv(j z3ukM9_7MA9j|8qO<0E%uwT!(H3p)3iubSzw3$n#f{Bh!3dD+UPbp@{U_G{Nxx5c`( z7z!bu8HUaMvB4Tej<~UEhJA#Vtv22{ooFE2pn<=I$d%5J0}Jg2v{b;=jtuo zAztWdkh`zehs{X9$%q!^Kr8LN3Nc?tt8*G(y^!<3h^@a$7)-u-p&g8H9mtZmhemTK zHO3Asg<(#J-&I&0MuRMbmh}x)*#~@6H&1F%pMCCSL7NY6tk-Zm1eiEhqQ4o+<10e% z<=uGR@lz75kXo;sTX#BBsZW)LeJOxjMQO2S?@Dr}a(6~DQ=EqMOx=5Jz0Sj;wERvB z7x=#&6stj?FXifPT6KyJJBe^v?VzF$=Ejs*HxO9haJk~=FkE&A8P0-ei`7FbDzKkS zo#};`0_$_Aa-{HDu;L3gCWCAf0k7y#cGh;IVo*?-ahz^5R-jJLA1x3x0YGBmexdO^ms z&Pyc406{JQfP5ba;PoFe0RN+N71V!{;b>rM53B(8?^?;W23Hn+TG;pw?6v|h)6(;W z_UHCXE3d*{M%wGW(A(ZjDCJpk7HSS)yx&itdY`YqKDS?J*{B!M!2G2;|BX6&5>NfP z5CDKuY5;)#7a#ae41CSSy)3{`%g9#8+89_#*;2;{SUKpgdU;m0c2cUR8IVm&0RXW6 zg{_hKBHPeh$KKw+5?GZ;&%xHt#L+;@&fdcQH)Ef3g;nmGhL#-$Y91ZtPYJ)&gIo5p z1P9Cixhkw<{=x)2kAeF=NdFfI06+&g0W^OZ16S3H5`HO1*TKYG|34~cY%;#w7iiUf zz-j#}7^mUC09!w6$ky^VojmtAig6*N1E_UJpmhT$^^Zmj{M_vOFTl1AmX?3(@T?=O zvcmf9tPao=FM#a-wUl4FKAw4zYYe!{V-K`(>)-7ARS}eOW^<~(JP_>t zH*YAs`6Afd%E-vX(&&YLTLL}#_c?K07+{VJFQVTSzO(Zp9@t~!7uawT(26S3tZ<)z zW=#xa^FC`E*na7R_vD}0wkFP6dOGHM4!~9V_uKaW?i>Tv@RZrf;UGGJ22KH-NV;Ew zfbYb>7vYbWC7D?2o7id@I9prUK369DtA8m@f>So=JHieD4nhPI0O0ro{c8}kBp@&C zM!-~ex;plH##)xZ>3u;{SaFETdHG5ra(0*Lf|O=NyvYvr5}Y*%1^uT z3!A{9Xak4xS3e)b@J~|o4eWt=4_+9CL@tyv>8O!}0MJVs1L=R=xtrMjNx(m7>3NM& zVfRxT18sF0cyRe&SN}^(l5hVBZ*65|{_<*gmSCR7kk1251c4+C0I>aKvLQwONdoX- z0z5ChxZs{ed`9^!lL8z>Am#7&6es_(2rEnA2n=*=wG16z+R=XrabCFwp2P}(f}W$A zpW81@1$=y2h_!>3xfQThpB>PwUh;X&>3GUNO<7mPya51+X3&4_F2AHqXuc@L%F@#L z|5|g;-6u`F#|k9_PC+UF094Z-jTrc;zxp!RT1!V?$NKLp`I(A}XxvS^kau%Ujly{I}%4Z&uIR=+>_scti&P2o!$z z!)>qsTk;>#aK9^h)PO<89O%4mfy?u+>VV<-|3d#w(O6uSu)Dx*+Z(uZ{B`qa<^L!0 z9~FCnd$*pg)!#M!dkx@c1Nmig=aMg?|6URJcYXX`gY%c|5d%MlKfH|o&lNd;=l@=G z%2NI^8b$#B){{2uZ!>}|5pD+{(r2J@H_qYWaxj=bxmHT|EJXGztexuc>5=P z(duRTU-IAnj{QAn*)#Uny0mtE8T+>kX1}9<&v5i7I@R+<^#7Ib=yw6XUvK_X07&S| z0{-E$^S=xFPZ!jl1^u$i{OFej{fFynzr%mO4fN+I65?Kj|Ldv5Kb=;ehwyu7@Sjr7 dQeKquFV_eG{4wxc3y2v&^Za9`7P!>|{vQJqqVE6z literal 0 HcmV?d00001 -- 2.34.1 From c72e8fc76e5323463e32a553601fa3904f624f09 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 19:51:14 +0800 Subject: [PATCH 04/72] ADD file via upload --- mindspore/ccsrc/transform-update.zip | Bin 0 -> 182640 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 mindspore/ccsrc/transform-update.zip diff --git a/mindspore/ccsrc/transform-update.zip b/mindspore/ccsrc/transform-update.zip new file mode 100644 index 0000000000000000000000000000000000000000..9d43a3c4cd63a56dadf6f1bfe3920b14e7394fd8 GIT binary patch literal 182640 zcmagEQ>-vd(5<;`+qP}nwr#)Lwr$(CZQHhO@A=N;Wd7t_oau}1N+*@RSb4fvs#-xB z7z732e~srHQuY5g_}>%uf2*^Tq1`_-dna337Y7qVXH)wBZ-ZXpe**SEB@bk^6WbMUB16tvG~KnS_}jzsjbB%KuVvRMJ%u|ZizrKT-b zluKBHVL)218H90ktR>?^lnB4S=!WZqH3#fIxO^!}$j*NqTsJWhh@gTr+%UveNw)a0uFN?5e+Z zWtpke;$OPjcJ*rnb>-8$gL-ls)lc^dzGMus|L`WEH_DSE$Z5E6NfO+X+3%@PBfhXc zTb)EswnTWuXvtY~k-bmtYMQUWuq3F(136jwbxCBlUFboqO)FktM; zVChm<1pu>jcxw#s36NGJ|Tw`=(0d3t({#O2M(b z9G}`D!(EA1&_aJ66AcXaBVgD&PT=nJF~1~G?Y{A0UjC7O_xBKW$WhzPB&Y%Ip#a_F zLIdMx#)Ha%D!uH^EwK3Mx`PHlsekOM z`ous008Eeo0MP&6bvJZ!GW5{5clf7oVrpz-=wwP~Y&@c+Z;#E6{PP%|tE zynKPFK+B4lNfAoQe%}MpoSGUg)s>}7)oKMI;VJ9X!{z8OQ(&0|W*g9Y0hrcl0e{x+ z_h@*Ta$V*dLhRO|lh~!DtrqZ3dOCg2KJ)<_cFYx$M9#cVlHPXR8>DRuWG0v63#)TI zz#1%w%eqy%(x8RWv24NxPlp(k*@T(qfSf_cBrwMk4=jNlPzam#j9)tu)Ru5$GzG%E zRMwBeJnp2*=9Vk?T!VK$$j3knTfdaLMCA;`+~!chq+h!_SvT+h z5lq??q=U|Jt_9ES+(7LEV2h=ITszOe?EtF*=T8f+h})piZw*JbZgUb~U6aUqP5@?X zkjCL&BA)CWa(|ae_I%BK32aQo_})rK3Zmc9r4;tD_0> zq+VGf)#FqmC2k*R5j^N zu{f;;--=+Nxt3tOhzf|!se8>Nri(>T1w22XD?{)g+Tg+{9wKuJ!j_ejE4V=p$D(^# zp>4rAZ^vG*Tz?%JkcpdF3gnElZm-dbEmgX;VexpirkJELn@wVf$*lb^BO0=6yu_Jk zex&qZGd0M}JcPMEffB_+Go5Zir4m~DW%T55aG3}DtY2T?<59?`gAnDc;^EgG-=nP$ zkLPo6%l9u%pY1L0r?st#``eCPxq1zKWdj>~OV55#30r}14 zb8Ab6jXh;*r+q-jou3DD`&+FYBJ{WW0G{7Vp05~gNK&~2#*`QBZFIDPXCG?MI#0Oi z{O%U&{P(`Bu3P_C6j0t&gUB1eV$?}cG^r#AxH~%<;_Qgrw86VTk%smjl28k;Te-ya z1m1EV^3y*82NeJNeYvX%(+UiKXY1?b;eP22-mcfp;VbuTVrgJ?^*8FK4J3C?xCYWP zVa;=g&iNAwQwOQqj0_$q=FUr3D_Uf{Q)602Bmx9k0n6&JA{K%-K>dPD5A?+9Ax8Q{ ztcItrS)+o!iE^)--Tl%1{(Wz^N&YR@FCWu=uC=>H9Neu9!_n`LWIU|(y9Zq#&dx`^ zfaT-;1i~CkMXbFb9-M zA@JkplvbYt9Cb2zU&v4>zIQv$`F$g{?IRnkET!U*30^~^sp-K$FT=RUk&0a1K?9Q$ zRtZi+Fd6j-fT~Ir0dxRj1h+zAeXFDG&qjTZEl7ON z3%(bU4W?!@sP9J#+AaF?X-fnWiNXb(jmI@hjXdly_!N6)WlrYo%%JONf!UMW- ztNt(Y-wOd8;j!VI1ql7i$8+S>gf37+1c$dk(MZe|HvS+y)h}wO~aoJMrFLO)iC*W5^g8r(3Jw>9?_> zgs#!_ESNp3A)?-+_&b*#=*5GamIxwVnJOAdhUBo=po$(T9%fBi!;!`;5GkjT>UeBt znym%KGwLW9AWTx^zP13Xj}malkRl9p*t%0Pywgu ze+-^X;|!W&fsMxL28~bl!=aBCaO@~~Mctyu?PMoWK^1v0K$?ftXWxX567v#YGO{X-M!Abz}wT^Oi5E?Z1*&a=@r=F0x!sflhAf zEEOr`X_v zDcw*7?yU49w8dw|OU$G8tIgST*ZXcORSEHvU-l!7TwykRjP$nuMvCTL(R{2nc$scr z6Ag!gG!RL%Cc_#?QHkCWA7Kynq>EDM6Z9Ayqgt*A! zjP$VLCK4=kK1na#B9jTX4D1bx04hYp725Fnbvl<;h!&1bLzB|s z@rqJm=%#3~G+;?m;rt+r7xGmE7xGl?a7Z<6n!V7(byJpO=0=#lPoFPl-#>{>ZXUrF zF*eSpCz=yhln(2Z)DDx7qRVjU`Aq05i=D$|WGmT)`PoU*(W|*jmtuQ~Kx`EAN&!oG zO5~?K#m^IEJxh`AbBru<3`i6zjVoP5Npu1NkjC&5iT%;XP5%M@kFznd22mfEA~O>L z1^{3K0{~$BzdIXa8<&61rcVEdw+T@mu-g(t=y{?ByBuM%SWt6P+61VsW)s2Lh(8D8 zM5J~Sr4_nm>YADiG$f$NK8^?B0|8wugop$X{YQn=97ZI&low(0wXAEDt668C4zDvc5zlmQ$+1s_$XEpB!HVtb9Lz~(8y1pG zjU_lsOeF|U(T)lEpiac*EnH$0DBRNuX&tISSs?WinSc?2qr?P6$8sE52&O!54hias zuqXq3gb>Lhjd4I2Lbw!EC|9AxjMXLuyRZau0fNlL^Kup;;B4uc@3QKUec+%$9I}xa z-;g9@Il1wWS>xK)5)e8S?31fdBCyEKnY3RzTav;;8NFP3p`?Ji0c_IxdgarVwT84HM&~jJRqbmLt7CY zT0JLZ6EnscK{y>2n$4pn*fRki;R;LL{d@#BT`Cd)FL?2$RSxVXsRfI8K*)uGBJGf-wT6tut=MAQ# z1+sgl->S2B=iNL8#aHPc%tbC~K$q?zii#cCJ$MI&W61YNYN+M2i%itfsiuoam4)5s z$=308Ro!vn7;=75)i*kaS7@1JcYyo5?8;_Kp}Ewt$RR4{A4E~w+kyorI{Pw0b_Us| zOoMDuu%KMPs67+3#{BkA)J%9cI|}_0`bs;|r%$G8>66Q$?QQZ=T6Z&EGL;Rh?d`rT z*j9&ittb5>Y_Ur9neXFg*wM9`_}F34KL2mospB1^Xu%SQs`_h zZqC;$N=Mn=`|i*6qqrEu~#O?MBW=T7MI+y(E>?+g7<8D@8hQ6oh-LO%NT5v45g~n2(k%sw4VF0e2x@9vE!Y{>o(BzUE2hS?N!Kq8 z;tc)O2Yv{F#23p^(g^;mABf1LNx1N$0V5dXRH`@^iSj_hX~ai8Ou<=YnxvEg4nE~L zEFtsBS}0Bh9ekYeRJdX@RICdGUu7$%H8*hEF1^k5O}1Uvwxv?GNzs1D4!YGft0i<} z-FERpxSS1szeud96gW}489WXsgNh)N#|0^8x|Qt^CnOd_849@BBW#Og#*O%tsQnO5 zDi1}_zj17cWg#A-^bUY$_b9TW`Z=`LIibMv&SCGBC~Q$Z@q^Iz60_DsaP4prA>btc zCP6lo)|)iM+=T~Gn+SqDMyjpYlJCk;B7*R_{Vq5lzP*EUB*=J%!2rA@N~@@&&0dRp zr6=WDC-kkLx5*q&V`&r?3$MpBJDV5R`^VVH<8ChvUaYUT)7O#p`$MV8cw&*J>A=b&)Ze9rzm81 zf1zlBKW$>;X<>)Fi2oPI`!BgaRGzN3?tFIWK)(+LFHisPCr6ESW|b)=$y4mQ4Iiv* zE;YPZy;83w6}o&Z6d1wn}Sj zijWzM%X_gn{OJyQ#wp_!CRd;24d3Lf0n}vsYH3a{FNb_jCt5d4(Bi_&VhKNVhn6SY z(Gbg=P%4Zvbz&!-dP#naa-3#3OD~73*)&>Qa&Q7;pLxd_kdBspKd}@34OG>PL z`-Z4L`ynI?^r$S(Z1lpfJM&|G6v4xe-O))0hr)!NuA8+RGS(GH0J8T-z3vD6f7tzB z<)K;`Nq{g9008Cj|6bhxUv^zhot*zG4lHy3{6iK`?YpWwUr|NOXr#?=ZN0TAq8jra zMy*#~m$Zv2E*)JQ&lSpDn2C14iR1@!mc+d70Wm(1H^hPV8(&`~(ymA8aEM5PHSFlB z$CeOk?|kd@GV>H8-YC8;=l%o8yj`o?d-M81-Ra{Oc};@j<+UmQ0Y>9=-rwhE!R2x| zC-5L9hNq;uP(1|B%w7Ca9RX=S6>}Pt>c_xw6^fzGtRRsPlcoS`07F^KO+C8Dzq2H{ zqX@0I#Nu4`_7_1QnC=>ctqC{Mn2Q(mNVn(|Whw>YP)QzPXg)CmQ&e3y#!7ir~K08UM@ix3M74b3!_vT3q=7FULu zj?O*5=Ga{M*9-}93cS0j6lsYh{81Kd1<>R(Bq9ea=-2^2E;Au4!XCbDPcd(PA#opH zpjElLCn3XY2|E!QXn|B|iD{)j`ZzvC2H%bbC}7kmHh9BZ+(a&;XMcBbZ7D)>=la^a z@Z(DF(e|@k@mjk(Tf|%Zg>rFz%eA<%wXp-LKPKU|b~llgqbrUT0kjzCkqo~G<^*6; zNg+upII>zqfrpgHMLZ!x?Z{d&8o*k^Py9w*tj}MCf#o2w7|B?kS+0l-A;L{S>7vkL z62+$a5P~c!T+pZ7;!c$WaPx$(I2F%|Y=#s4itXenpcG@Fvu1!*cOo!oxtpmYIsfj|`m-eQmC=R){Grn{lmwhjYNUAwnUX@XT`auPZUMRx>g+ z{Ib8(=0dC)D);~tKH)exyA@?<3pSe(Ehpngc;;Y$5BGY4oi5xnUQy{L4a^kw`7p11 z6adBQW&&K6GM2{I0Q4o$Yj{q=C4PCKEVmUD=()6KNFZEsJ`I(?@j4M!7!U4NNog|P zib_dGZiJcA%5A_usF8L5l>}KT5$DFr?`K%?B0LY_qgopLTuT-TzyD% z^ertfPD3{*x3VR*v}T*FhwjEJq8F)*Fp}pe5;*}3sSYg7WtBFWx@QoIn|*JarxxJ5F2WdRIeUEdHl5<}#^ZBRwM=tvY+ zS8GYTo`WA#O^3%FM=se2WGExxY?z51vomm}EFO-^*#L+pe!)~ES-4s_o+KYcYZ-nV z4}2Xxd`vz^K6gr5ve>2N&HD5^?k|QnR~D19*wLjrfJ9UK(!MT**hI!B@Gj2zU2%M% zgBq7TWG+3wh9e*hY(V2E2ugKA z7pI4hkl1x7Jq+(KpHl;=mSBh+B?5;2aV&d*)4Al^|FpiNd3C_+*-8 zs6bsuLZY*oEeZZoDsIF#8fwT>s6QS}>|SrbU26pp5hMoGM(07Y&v?_bj4CPH6h()L zU{%MXrrONQa=APQIUo6Jc{*Q)w*7W!bhl{!#$Iwh!o10-iaChW7`7cZnlYSAXa-dx z)EHXM_k$%})%-snqSWzr-#r|CFDZSMqa5mQm0O%gnE>sxk!AacVQX-xut;$+TodX@ zR8Ws;-tdsANv3z6;g?v~()`n!5kl(%s7Eu>*rX?X(=l}XLjg&;WLds7f`tI>WgH}U zk+C}mK6PppCKNYf!i(&9C~C~u+4)T08-Zv5jIhTZ%MUg zvF|gItj2|W2x%4bdk5j?#^;jXnU+K^P=Mge>h{y1rVfHKq12GVkz5?Wtwvgeg*Pde zUsEn#y7uQn`Z%$(1cx3pqJR}&6SMQ{sSm|#fmqnZH5y{dF&~?ZI`KLeRp!^jo&w#_omeRR_ojg6`Za1glL`ukY8Q@BYk@qPO{btaEFZwn zqWlJ*Rj&~ZFG1;{!8oY%EunI&nbFyePdnID8&G$gB zcl?1bT|!O$A-fPt3p`0f2bJ6yQ}I2{?fN8W2bzOUi#ujAR?9=Dm#1Yy<+g%r#5dSL zev(|#yxBfx&j0l>t6r5gQbyr`jU9rx5HshAkPBY4hu=t@J;-Awkqp?R9Vn~bYiy!F zuD3PN_72Bl*^z=3% zWd`%o!Z*tjXxrdw_l4fhNB6hATZ1y$*u8Sa^q#*cUWhloSJ8g`8zG#eo zvpC9yWniJOyyGuoAXrWkHeD+T8_fd#Q`Qla^+*JCOlBPiQJj;9VQKjmMzG>=+jSs> zglR)>hHe_!oPn7%Z82s_4xL>tbC|Kb5sO%2Y1wQ6_~*xNy@+kR*ql56#uuTtccar> zT%Jv})}e=qFOIWLsnZ~gUQ8s0z&|1UK$HQ^O@xC=a8`_CTM8t@bnLOH+WNy9#_%7q zvN|FDEpQ1se0!Ekb<%(9PDqCVo&Bn*A*e zukq`!FLM&`?S{D{=1DjhKVKK$D2>gxyj!lRczH7Y2~?Xob&61H`dqA`h)L19n}zSo ztC}NMTRUHv@U2d6PDt(7QWG@1`5&*_S`><9j;1)rsr=4`YwM;c|9#nXYN~O)(E-2; zHLQv|aX|)3YG>u>WOpZIY(=6cHf!4Dh>GiU+M4_2eI0UnB7?h8X`tY~5s?%gl5S~5 z#I;L8zf2hv_~zmw9F#^hUUgriPWv31&=LB=V5x8MQ+`U=`L+?nLFr^Sqp{Yffm@~= z4`Tv;Lk?&_|8#i@5!O~o?XN<1){n_isVHiI~SLT=#dVqRg4GqwW3;6m*iK87*5VY;{h|aUN zdPODaLdhO97?(eqJ|MysZ_m*Vyz zqmfD16Rs&&>_0Bf!@=P86nEf=rew50$fI5^4Ks5b3#KNy-9*@YaERy98vxuKGrI`E zC!f6XB1QYn@z|e-!PPgt6~>N6wp^-ol%8si=izPBS38ykM!n2#O$8Y!M6a_kT9J4M z66_*S=pb~uPN7VnL#PSu-xDaj>0Ix~*op7y)u2B9#u-ihR)BK?Xr&V@14oT!KMrsk;zH5R2@f7ym%GvfO+K3TwifE zqC>juc61!#FWuWq5;Z^w;jL-6re-D)d^5z!*)}=@h1nV{*~1Dw_lrW*)2p9Z|4ywAEHBd*&7rN+?a=yY`Cis(&^x6wgR1Ckdj%Peru>Oj zyM};^DRXv%HhSQ(5JLhRNcb-!qrr6|p_ z*4qqdsspm1zaZdH0p)fK^D0(cts27Ds`q}>8vRYin*}}r2kN>_z&z4;(cRHOV{h${ zVQ(oFz(cquX6v=J@*U2L&kqrbvIam*l^co$b=2o$x`}y}Z}DEI2HKClRjY85b1@rJN_S1Md33|83(( zT^A8M5bOUaEF>Y?Rlvf+)(k?;oN@IHN>n{2&Y_C*24j3%VN@kxxfx1I#IrGjS7j7* z9IVtUQgh!8hNTx>S%jF@{E5nSPvG&=aX(rczwajX?xna&PqD(hn1=ReGy3W%}@lzIxHh(KP&AdU$MB z4OWL4=^MNxTtBZCZ-kVSb&A2%$k!qW@dk4VE{PL0uaUrgRZdi*B3~`3_ShiKgDQpM zvPu@^#3r4J>-)ZF^;iaqcHdvD9(<2`4Dsy`aAn_Q<(C~pWyxnZI?pxN<4Z)~I=&$# z-sIWns4^Uqbj!#fjr6-kwWFM?|BOH&18or?1@A*OP@eA6Ja9Q)*~R=XHTH+Ge zF~XrVu3Zl<6VUM-x%ju+f{ z&m>8>j$iXe=VPD$7LDioeElU1jjMa2QQ}qH2(wkm6Mr1cJ%BR}FmH9T?F-S(LC{9H$eW#;`;Qu&K7AGK64?pKU=fC?susrv?j06F6~*ZBFk zq)MC#{K$Fy#AzS0@<_ni)_fVNxLZhkD$;l+_T<)1S74KVo(zPI*sN1IM-8{9lD_&^ z5~E}InOEmG@J+xUN&8ea@_0=yu&I9H#g2E@m#sO+OKrEpOTb|?F8mAwDi7t!@s0R$C>Swf6e zDb>v{e}t-+gAYohr z(Y{yrc53!|f7Zoz*Bo+5VZ=yygt<)E5~XHIoaJsEJ5|gshQR_%7+%}4udX%8^!G0G zmn5=7P^tocEvE8vAS8GpOrEgK8DfGDswsQXY^)+sQOSJAd*oFk^Yuc)w=h#WNX=?VG0awvBU-*rq8_06tPP&$O9*SSUD*s9;38>A0y8 zZZo1loqscrY_Be_K%iUi{fbDf z0$oaOexrtk!}3&VQH#d-6Q1bE8%uLr0RO@eI&m}L!oe<4p%SfC+<-b2jTf!h)XhG% z@HHJs*JI2YWw!4D8c!GMb??^2L7mv1O6bkM5{%i=`-A`LNN3FKmhI30GqHw>Q&1k; z$pTd_x1jY|5jq_0*Td$EHrB@9M6?O!_CNl3oeZh;L4^+iuA@rs<^9yjfnkkrX4yc0 zM@0aW>&ztDR&ZdxjPdBzjU}Tc&2n$A8@;^_E)%)Yyx)M3L-5@&=i(7j_O4dY&?b9_ zF#_)Zy)Y1EmKyd3KHjUntOaGbJTevjr|fKaUOOZr6ykwpH}%0`BOKFuv&~TR4D5cj zp2sR*U5gJzix0{-_C16G?jx`Xuw?y%Izo>nODq}bid%5sO1igpQ>Oij6kNxnj|`@Z4Xa@0(x?m|yS zP5MNfuB@4TS

X#7O9xWnY_GP13dbN?Ga4&f@ZaPwOHP#ScUDxQ0 z&eQ+w?tb#BxN5x@_U7~(7F0L~(F_jN@4y*6NBm{Wt+CSvCTfgys0eWd8s-UrSE{^R?sm^`49Qvl zUFB8|Tiosbn7&-T_|&9gd~~Bo3Oe7@n>gL zP*d<*i>&f?6R`WV(X8_$!;xjj4o<>x+7hoy-tOoRP7anVM$!%Ql#QTAAg zMhm~i4kT%Rh(Z^u*OS^?RL9VF2Z&@{yX{-FRWIA;d=Sb7LZj zxnB@_YTcXs@11k1W68ZftM2lrX-*&Mjm zYjgto995}R98UxA$7)MVW#}kHhqm=kqxSH?>$0TciIycZj<+u`-@57i?407K5SMJQ z>UuM|Sk0x_OeSE5+ysO!tg1Tln=X`RmRu7Zt!7aXuFci#((Y<-Mc>OzdWy*4LdNb< z$G)rU1+8siiZsJO>Jezl{n2Ol?OqyJe^VEGuyDG0m@0~D8mi-tpxAyqzKreM=uWo% z7hq-=$~7+og>fvmEbyUX-lz9fJ1jZL0`)288EHYbeZYFaLqSJc;Dy`Tf+)eUJttN_ zFL}$@u5x=Vw#d$y167)gh)$%leOoW25R>iB1URm3=mEWXO|zrHhSoK5+!#KMBvi`% zAZI4c*_-_xRnTQ39q|u$JNFa}LZn|VYuD z8b-|d!VQ_9EbC448(Qq^!t~qPqtzAo)T4?+}SngRwOiIWOD{~?Z!d! zx1+tDvBejC@^AEVyJPcl62src{AX~kuG{x(aq@h*?jwQ|{K}!$-*(6M>(Z11WtL!! z!%TC?n5kitr}RQ|eR4en`+RxcZs{Q85Y$J6mJ8kfk(`!;cGo&WEK|<0==!z=s`wd z6%zSikTwnoHmbCw{lAU?PdHj%rzfTnutt0yPp%Wy+;B7sXM8h(bjaeZwoQZx9yXDT zR}Z}Ob%jz!|06$~%WEL)I~%xWDHUUfqU%6y022{Pmpu03?vg2Sfv-S9#sT&epxhP! z{{RWZ5W4U0Ek-Jc;`Mh>q6H3|z}DtcQM1*CMLB%yg%i6;-s>vRKGKBEa@TZo#qCO| z(_Jyr)v6{u=vw95g&SMogGSTsI4R!KOC^zJzdRs~{)`>>_mqoekN@N%vl$X>`n+eGFEIk=@bTka4T) zVqcEm*(K7ZW9|`V#4WS7JxUw@Z0(L#o{w z6PclK0ca#_&LahK7|^~27WRH81{A)%D43!~3oWQ@G9ZF596nYnM<#sqw;CqG1#dG| zWVp>c;0qv;wTRwU<4odYbrLmEIN(IYK&9JW4?KQhZv^bU)^NI+c4CLk51)F4ATrHE z&Ps};!Q%b;@#3NTzEL42Uuugp4Au>_-R?@;aMsZjB?B6_Cc7a!bn>ggfGveh<*JTv z%IeLH>Fg4~D?OLxtaj-)BHh`Bs{uQePDFPxTn0ztl8|k}Jgt||Hbl*H%w2E9jxVi9 ztWDhCq^7UO8A)6AmWNCAHFg)eP$EJr4_4?%^`D465p%s56t^O@*+ z&(2Q6$tM2PEEV;yZECu`q5L z6Azd$ts2-FkKx<;Hs{><`Y=3uBQZ4`AqN;~2-?>@Q3x7_NCD_uyg1zFFSamT93In= z5D8?{0*LH;6H23ljf6<~GkIwK4K5B&JjCTMTLh9d@R(8J&?;98sNMGgW8?PLSoY2N z;53Jg3J8`B8u#PBDgw@Vnv?M4MIXodvNV`*EnFLyC&nhRZFzBYRE}r)fD2p-shBf* z{7XNqjlv)mrG9$B2m!h?>h$n>n&Xg_)8$Q@D}$v%QVZ-P+Z2<3v|s6QobZOulNx5F zT#&%LKLpid++LnjJK8w>lTP7?t6?M8r$Y&2V*f8)Zw>oy1bc-OG85Gt9kY8fkPDN|C>wJE=Q=eu1uhbxO?WM->=0jQlTF$KQ( zrSS5YNy8}UsSv<`0y~?S0r!~4t1o4I(Ms!F2843sbxf{^oae8cWsYciesn9cT++pz zoc&VzzSwP5*JNwL^E8v0l5eW|Og1kW@qx{z(Ynve^79CMwWjijn=RarLxYLWE06%Ie8;yj_By{f>Mj5M60KPVs(oeuN$`WGBOs4)p~k-Wkg z^d3#fZS#;?Doyp{7@%#DvB98>O}z<3M@bN_y$uPQ<`bE+HC9gyk{<$E4(_Y*O|!dt zA&fySaaD*%iR*kumZc#4gi{GwB634gqd@jjU4B3dfeN^kL$|Rn{{<`}8LP z0h0h9$nbsb_It`<@YF|MX8Kz8J+=g=uFI4!pw9(o%cn1Jnra41O?V+Yj&2icK~D`$ zQmNy=7kNy7M%vs-7^m)&dzNKood997K4BH|d(9a}uAE+tL32KCcg)~#Zr45s{ZVs# zFWZRr{YgR7(!*)16SjRVr)%v&(L;Y^f9;QuIFVo?%u9L21BDSeiOF!Un7)fkG35w+ z1iU^O!k`5P9vc5O?s@nXE~wNmCZRTM$#gz6Qk3|#WQAu49a~mHB#BTu>Jg=u0++)@ zYbc|l-2H9y3wT9O=i_rgkis!2Y|%n$rM7|vDG+iWQNa<#31ixduelq4a42qI9Z@72 z9jGI}kE2CdInB+6-)IOUE*uV4r%BKMkg822k&bjM^liz2I?U(CaLm1>gQG_cdG!uk zyatyOcy~hXgn{9^OeY%IxsM12$)--#Ys8>=2ebw1 z5QH@UJ!njG;;sY*9&PCK@;VX~zM-r#@SMKUpcaRAE9wbSRm;w-$nH0crDOhD{R9+_ z6mJ)3x8{McyPpFrO)CSQ5dnS-ZZDxUl5Gq*2;8){Gdw+n6fhL%!^R!wsFh6bqy{T8dtIqoL;noj$11C~zHXuq z5f8|jw&u@Xk-9c)sNsse++1T_dz8@JvaJR9opb@!Wc(~lkBtL~+5nx#MViJb_sN+# z?RMdEHwVQonc1rEK1@3)QxYr(55I$lSu@Kr#7NDOa4eh8LJkW^FMdeQIbtW?(6K2d ztlMrVb`VLvH=lpL0`Mwda61-164Bv2yaq@_B7x&7zop{RqSzPkcyzhGyr@@RSC`@z zh-@1qiio7n&FQ%@p7|GIo1XHePQ4AAm<7XB2zUdKG60vxcEHp5Fv^j>u=40o{v1AnZ>uT%%R1APb zz38A~ZtFg^^+(K~w99Dd0T#ta--l15(R(|Fc;?yHQVHhRacJ(X5bg8QjQ88G$IN}~ zWz2oN@q0hQ4}Vkrz+%~YlFg`sI<)qgkc;?|^ ztykE>c7Vmm_0Jw4LJ>&r_uT{ffjwt(awUC-uib5^~<+J2EO`=^ZH~3uDbLnhH!PzZU*?l7-8-7Hl8TB3d zvuZ_r%#Kx!{Ag@q-o~`R4eM+Bri}laj6*)=N`Yd~u@+DRl|qtC-AB+iN7g_dm0SLN zI5>|NTf5WSPEXg(4hr%$ei`JVEEnq#t^yJb6*9T1WI)oMXt-}^T#T-G^mm!>60lUG z&8*k{#h)?`IAQ$w>O7Um6#7Rd9?_w|xYyj64A}TvjLhE@>uILQr5hrsC`10SD^yN6~_6s$L82s3%ee_X!*2^k6gRK+EBy&)qQ7H4n`&P zQ(Ifk)#b&tTMP62=9HY)ac+vZ%6{)dSd4zb`WKoxT3_lo#D(73hTY!?_3&O;1=jeu z_c`a&%ma8aQC;!elmL!}Q3cHI?lG=zYrz;FsM2uz$JFm)#tAEcG9= z?N99)J%V<-6sxh)Cu;adcP*@^hw7A{c+)p82<(&Kv}@lv{(IvP2!Wf52841XB>=k0tMg)%YP#8=iYKi zUZju_m)=K(E*{7UhM9Rx)Xf+ChJ%kcI$sg3T8R_v%mI~$cU8tvQJfx5#gPp6CBlNl zt=Td^(=sA&3`E&iKN;tFi$o)bXP42Hg%}#aD!cFK9Lli#w4v%1c{JVd*8z#|ojC7z zW6dX%)0dn!#$K;%Ys8I+&B7})n!_>3S9Eq3bx%gcp3NTYOA)|xiMXA2v$Zktf8n2l zLT-4!qm4D-@?1}kCKuF){}YGLQjb}(YZH2D^SEJ$Vb9>$Qox=~GR`P!w9q&MUfN+B zstY9uGhz$9l|qZk*-4^mzWpoT%u;WGRyejVv@|Ex{+a1WeVF;wbrcx_%o&kAy_3@! z%?ZEfPRi!iZM?o!L50Mq=f!U4*WMZXi;XP3K8+9z?sUAJiAhP&W)|E1SHsNVt14eY zqkB8uwvIJ0GH*AC!J1Wi8}obvd&68kB2g+9d3$rgHN&ytRJd)F1cOPAS-O;`qmlXk zqJ5?L9+R!*J#q+)Q^lNDw^MTCQ+X#MubhU$g_xU}_>^nKn6kQXmS4MuwRKXFR z7_tq0Qc0+TlcB{~B01?+7yW|DW_q6=GBRj8Xmrij}06VJw!KJwpBP^UA1oU?b-fR?T>_yzd?_{yvQ;&az}x6Exd_d|aCjX`#cYV%_a z{8cABW2`NoVRvrz_eTCjGt^cKww!o~oCm^M1dqzI#0iIV{UiW^?iYEvp!}}fEr#x+ z9JJuF-{3=ah?Br%LdW_>(KeV!ph*#3Ea4jj7isLZleF09oaEuI1);`5RV~8PxN)@@3oKcF6$@2D0s&Df*6s`h zMO>Ojzb5*Jd*T1Gj>A=?J+asEjLa8yhQGZH0E>^ew7hM(<-s=yB{r(r#G*{&nTCS4 zv)bF|ZF4SSnAu%^xtebh71l9nt)fk+9tumW0n|{JvFYm)g`L=NN^b4O$@A_Yzc)o; z?fyg?fiTaySelj)XHp7K9;Np;vy8iJvsu=?HEWwD=1g3_sJ{gr36Ej0DY=-}ni^g? zBG%!)8AxiF``YxXCOiqoU4FA);TwTG4_*^(4S7N5%-ly@_BC}=n~zr=qxxGYJ<^65 zupF<>3~m=MsVU$wj|GC}kipU5+NyON^?6U2_5h9TE!pD_?)?0$dXS1?0c+NwI+=p& zBYFEc4R+y5t3ohavydN!YT;b1frnDFJ!S`DT*)fxm5-yb$}#2uVcuQ=H~y=AvOIM2 z8doFhy5Q2-R9Ck>#VXPs=7&QCpYjpVd5{dm-~<6WxV8n^H>r^FgaXnxv>4=dNbg3C zWl#W}hu%9I`=+Mna@rXd!4Y?(l4Q-il!WgjIp?<5$6ypeTQ(o{F@kNdaqjjD*bbdmx++@4K%?!0*|LInFfyaE~&B}r`aM)Hv<6*TO8^?Larof zGsGo_Y$E}igt7_E3Fj3CwLOZu9{xkgaJGi*j46&lPT&T~o$r zUIw2h_?Zog&qc^T#>ih6^)1@G`SYm{`5OQOGTYUqm488mV-~DJXm*cP_JfyMdHad& z;iJ~@udQ2mgb%vQ{d|n0=Oj(gBlAv6GEwRI#4MTuG;@h$6}!LO`33yV78PdSxPFht za`DsDBj+*Iw#1yOLH_E}A0U4-Ax9?9)a1H3q->eso6K%p93RGDvV#|9$7gPTb>m{~ z{n>@LX5ruP1JZG9iqz*xJEe{jSx;)(HnZG{^ju8Se%(;VR-@P*$hfrE?(k*5QSnGe zQw}6%+B9>UV5sDBu8)`H$!-NT$|yMjT*&FydDu{QJaLz(R%ABNJ0VHLi)K0 zx|yQmnm!x>#QJTS_>o7l#%dNhmx~Mgc*orZORpCIf(Hp{ zS4r?_aUpucAvbh}jm~!S{yR58BbX>8n{JHP<{*u%PcC1iDXoldTyienIzm`Rxu*8; z6V`A=4B3eg3J6s?Sbct==Tr|u)9JCaY%>R}=hUuUFXOcy!d0nL2he_F#TIvJ-JqnF z?6W7=42&BefN(o48C5XlxU_O_`4lQ$RE}q!ke6}?mAYKGHGg^h-+&Sv`iCrBnlRbE zyj28bXnl0)eFH^HNi`%1axiyDGlCu`+SOZ%Gx&r-Q!T`D#k8J-yui@5HlU?9M>+=# zB0N3R*F~aF*H(vfM8K%=9=$^8CtVYFC)dR7vf%a={G}eQB?SSgm%Zr;%Afn2k7O1K zJ@yk1mH;p>T!#C?t!Pt#{%gj&Ltl;ZDVJ?NR3Gnr8z!bC(*38T*cKGoqgF8N}4Kw-OjB#{2>9g;6N3?tKj^# z6p_dJbRb8i1KrILYCmfN_r*W2-$xO_z1ltIMcE=*rjF##Kao8#&4>a*Q)NwT;f;-J z%o~K%K(X<{DI7mj*YI!LOa#T7!wu2GT{l8V)7jl^1Hc$R>M(J7uMafk+wFGPS-^ks zs?||M-6BO}rD0t;)^Db$^%!=Tux%!@v)s{pILk<4X$Bzm~T}jU=Q<9A$nnCRYH+OMNT&Ha|g{2Dpz&5?QUACqHi>{Oq+TjuuvRo zDEUa)-%nBWm$8>mR3NA?eaWZg`t4;Bm8O^1?^;vr@+_N&Z8*Pqxj24tWBgADG`hNk zOV_!Cz+1nb{n=kjMF{1-2Bx5-&Y)$*&1(%gPOC$*y_Y%=_JVETyRf2%_7c}?9I72z z$-X)C+wFSKJKf8dk=KjlU`Z}dV+qqk0-ziysbE0DS>#_)ir5iMKf4&o z#*)|sN~aMh5FjvOUbUV$+X#;YBPcWE)Tv6J+WD@|YB{$@X#t_GEbz_mqvwB3?v&E= zbqsQfE#(d^m-2BBIXadlE`-gV#NRJkDGmH)?IYD4ZZf<1o9jP* z`<`Mh6nJbK83AbE06YFjT4HV@#%*FJm3|*I)s#=O*xwr8B^UZgGoztLeb|@{-46_< z#Iww+96Y~Sr)=ik!?}H%%-s?f?II6J)3JqW7>D2;O!z-VTlD4A-OX~fdD-0yBYAIl zh!JWK01Au%h;-pM@B9J^;Q0qd%=EBwySu(C{a!faY4eHk3wbhhJy)CQC7U~kBWQ2; zi`>ulmOs{cV5OxBVrJsCc`StC>92Q+o!5bbF{_hMmGgQ#gIwNM`Y@O3co?yNEp@+; zbVFrn3Ql~qzXo8wXK~6(^avZhfj=5|R_;s!e|&loW0TKDp)K%kz*V8^R!X>rw_gG?A{}k)zp@? zYG3@;_>lSAfe2e)Eh84zXif(CUnqC>?(aFZuWv9rSstKrwq51yz*D;ZEu7DG?gpdH zUiNP5?l~3k^$>09UOCj-V-}!NaimO~uPl((YxWr`!%Us)3c!ispF!QYxUun5c{_2I z<1|^6o0?*DvdK<$@0UAl^Q{sZ2Gn2A6bfv8p=bb423kagH=F=IU7xyC-(O0g`2r4SrA&|F%{!;UlXQZnZ$QNbZ=2y zHx}eOT))3O_TJ~o#bH53F>H{LI;418<`Y?E)_#Ebg=>?OYaR(8$T3r5b1dF4Qs51L zGdrmVI5T|vx2vzq+qrY&GYEV`zyl`ga^vVkOWieP^|TmJ$MUJh3S}t$tr$WbR1~Gx zP?Bl^NR&6c`y@pa3E2)OCh3?*Xl3M6ZOZqM4n+ z{IPcV+h9MYLI+0;W~nx)1mOv3{TfMKcv*nC99ov2bnExGlEj;9tW}l5Gm!3Yw_@$I zvN3>xs}b=T#%1%U#^?LZKgu@`2c^p&m5QH@`CQlJ3fIOr#kyPIL-B;4hBs-npJCr zufYZ@AK4~Rj%&Sn&O5o5Tg8nt9FySkdoAhun(LFqU+!&NdjhlU9F9gyw-@+^p0DC- zwd&lW6?ttB_1kNWD6$}b2NpUFt~s(}=dPmS-t)2G&*5e+%4zK6F0Bd@N($;`PYDws zlsxCTs8vF=!s`_fj_K0x0y6~(6d222Uqm654%1*Ac)ZPRG#zLJ&aHjqC%__jaOD>c zgYX!SB{toLDmL1DujVC1Cj5+}S{Punv^{m~8%i0KhQ~^nRM_9mkNG=)d=ghsL#n_K zbLZCYa$*WV>F5$@=R!N2z{E7u8Pm!!|~l> zsms+k35jy4E_(=1t)o$a=;4=LSH?i(MA9X2%wh9GQKIadl0p1B3I!3%mL6UdNdv(5 z^z|i{67$Xf)Y66hQ1kYZyYib?ZM$SIB^6=}Ui)wn*XY|M#VTFVcR{h>Nlf)^XjbZL zJwnmf@hqqhPIvQfEP_P+r49(^1gUYd0CNKYz<0;U8X>AW?(tNtb-bRE<2@Wk&vA1; z)lk*Z-{rI&|57PDs&lXNR_j<-ZIAZ6wA;TA*ee<<&s#32 zIxu`OeWd}|q=2|hPdh9(=QsveVMPN~g-tL!S1%)(%EPdg?f-2wu8ZOEktjJ{DrK|E zr$nZz?@e@JAru8o;JfwSn?Eb?{j9CBmB&vI@y23UX}s`Y$-Y2KZm6Sv0WXD%{$b{R zJ9eq;VKpS+ud|R)ic-DF85RT!_zKp85U?A&Elb{F= zxA6H>yHm?A+VLP#3+JIgYC0O7cndCr-)flMaI6LEYC;s7{nChmQSW>->roRW`e@mw zIg+A3W_X2E#lKfnB_1SYPGLEcwi2MwvVugc4Ua@DWcc;zWlfMHnm+t`PVIF#Lyk=& zgBz(L|2ij_6~W#D+?BzNI*M>7n!QpO8hjY7i#{xx+|dxl;-1qg0%Y=qH?Doi5i49& z$0t#+q1a*-S9%UAp0TUoa@jB-*v&_|{@t!IKjr)}KL*D4d01fqMJ$o;f}krP zxX>G74O{gkf=W2q>LcFWB@S>hz{)mtjRI$=D5PIB(!vBLM#9XWOK|=0Y}$Bv{GzQi z5Ks(D%~3~{xPP<4V+$I*^_M%pLtTO4F*@$!3vc+IRz`zM6ZMWwoE;xNzp?(aE0eW( zVz|@GY~2X5#}i&>c90bwP*+!wt~D_t=#AUB28Obp3@|Xqt4no{cQApX@C?d*G@#!5 z+|$dm6K4@AC8Pn|2$q+n&sTrX?CJ~^I+&L!=Eq!#M{iefHuPEJ74j#v#ZR-C75> z&HGLMfAF2{rDDqqS~jfBkf8MRoy>)U@wO#$EHE1aTTdc|Kxxd>v(*bzO?T^dN*&yz zjY3H;k9?W4U*W?xZJ`nqKVhEWoJc|Ji41N2*w9dXve@n9QL`WQPnBttF8-8ZD14j^m)-Seagd9dY6;Cd~NjD)Z;{fyAuHUi^(9qB*#JgL@=s?Jxq$%y}Oi zsPyJvISa9S$UAbVe?MRE9h&5&DpOp;1E(L%lPh+t-$%d(!IZg}l&=6~YVCF_vIx`d zt55~yAY3JO4+O0&@yv8sdJ$Fd!JmZiS!Y*QWq>Y1GJvFF?NooAs+4ugbxHIoww?={ z8*KUf&0l#6XK)DN#Acm$)^VpAqXoAYD>uY{APDH*r61k-bnT<_!??(b(WEwyzkpyq zB-S3Hbj5eJQ|vB%69I5cx!6de%cZ92dkkp8KniIA((3JTah0x+2Ie~CtSlX#>bdXj zn}Eup&n5>bGz9VY>Ex%pP4#iMsrrG_hE!P+w=pt~Mo9qLcX0Ik!OO4@SwBlj!AHAQ zv)4-TzeIGfJE(7+qen4`;BI4P)A>+EzHJxgNQDLAt2SfmWGAPfA1fP$C5Wds@|ykF z%do!YH{hnOtJ9D|^Y3(4_a3VvQ*iLvN?#|;RuME~aV}l&2T|Ug*(r3PYv5O3?&+7v zLH@{PV1+xl>ijG|%q`A>rAAXIhC{GG&F%t@(RP|)22X}p%0#}aqsfjDYX@O#; zV4GZH-M#d)Xn;%%;C=9PC#R0G+fko4aD`24+aBwSnKd_x?O{mnuV_B(hNGfAk^obYR?YH-^yQ$00ad%Dt<303G!A7`dL*+QG zDCz7i%Pc$l`+HBbTGUx(F^wa)O-slrK~*L%s-1r@JPoldk2794J6}&eR~TLgIu#d1 z9w3D~4fVmqXCu6utlGiw%GdH#H-Rzakktf$u;@O}ghJZnRe{P8C zwe4&}gt6f=Si*DVa`!(MWl{3h+B#W_nf%K%ckM>z25nfs$Px*3{jVa`JPW^dt4*4y^BV-F%T)Rf8 zXpJn(n!0!8;+sM56Q>!OF#Pz37NA>Fx6&x{A9M>2FXnhUAMCvJ<;(|2vb#0&?*FQN zy10Jx3k2nFOf9~-^d(ww9u};G0O8fMvy+G&zVO+~zpVa#?z3An6LacH_56`KI4FgQ zl*pyhqH>}dbEHJ1HNMvt61P(CFz0MeR^Zp~&;HM)PdTcI5(!2YE^l0i!9e0U=ErW_ z#I9Z3c>CVeXLmL(elhvaxbH!TL!O^v3N%5TD}J2t!ptnhk+>NxzoQ#VHN^aQCF9ha z#Ol3z)#3Wqe@gmy)S*qHUHTL|7QY-GK^i^wSL(~1KfAb2@c{icA9U@b$t8Sx>HVvt zjLwWEl*VWN0;1JNr)HZiGIgn3Y*yvWBxtf5sPYr~?yEGj(2wA~1Ftoy;^o-Z|7e=)qnO zF0905_M2~VR)p8g?FIMkD7yM+^5D@|{&2VpnCQ8s2eH$c8%Y{ZyCo5`zj){Oz;ayY z0s^p6lRGTu&f)NE)TmFKmyCnHglw#gHW;C3^4yF@hh)N|!6t_4#f;0~LK(&!9-BN@ z`xP!|;)JO)XC+x`dKsus2QlCzIs3OEF(iGZEani>`zU zQMQ4`DRO|-I){>%90hQ2Sg`sQ7GImswo50K{-7l!m%5UJQoJ8haL|s2!Zq4n+qqIS zf+N6Rdj``-LRwoK;LZ~zD5>JJ+4-6KXtKEA{IKiySsZ~WIkoA^g27I-F4(!P z-BwLV-EYDB%%Q&6XZ&ko=JGp+@{Y_-DsmNWZVGSnQwQ1H^vtsi(>>Uoo#I?l=LFEz44xDI@ORitF=A&Gl|t=g)5`UZ*VptG)}JZAwUH$6$R}l zq%nKHO{^8PKo)F~-6&Gi{0=a_ zt&F-Bf%q(&fDO2Q56-oTv&iJQeh-oXZG=}J>S;@S;&S?l^$gG-NyTha$_ZKb`Mc&7 z1Hj_qg%O4YRyF+zOFZj$gAb&9EkFWEZe{e$8-Vqhahyn~wMe*OA<7C+rn6h?h`8}L zVAA9oDI&H}UTk*aMj!?g5F2O@Vr;b#ZB7y+RBjVA%}HFvg4W6?JtVEzGS}T4e?)hW z{JD0mrfvm=pb3?kU6Q`GhlSR|aAK8954Fe4g{fOW<(s$fTSqEzQv+>WjWAIS?N^GP`9<<%#z>m)Ejo~ zin_X%_4kSYXEy2?bMoj?n#$lws>uuq-j_!ekGwfKgPmqu*y&AbP)SU}d@z{COHin7 zD4TTYi?vk?Png){9Kfc;4XxhT*>}35a=bOJzTk7qq8IGlasVv&IZsyp<9On1G8{hn|$Tlm-iA1+G;-77{U)xgwuS8d8&i*YprUOEXTVWavSz4Z{ zx`VX|=Gr8;Y}9+ihMQxqbq_Z?wRMG9bvPs1!3TTp92Wrg=McITn@^Q-77x$h2B+J< z-T3E)H-^V<-hlpkfGdI>>c_X(LsDNqT&y9Alo5p86!05NeP-za8P7jkUBxJ-Q*Ch- zV-<8j*fiK!VV#37iz~cZY?gXjK&*qst_K0H zyd+jSbX~7+pBSB4eB;h1X1HSkB;VZ#=)u{Cr#C;g(FYceoFqDRjo;Y{9EAfPHS2lF zxogUUI-li-08n5%k7p;#_Or{+gIm_-0T$V3J1h4yox*O!F=?PR6aRYSBk5>(%*+eX zG!rv3A8fooHHnw_ug4wg&DAONCFi-rR@!Jj6F~DFEE=thg8x7Uk?H!~Yk+D`R|fEH zV_J0F$OLT6aN>%8)aoe_X0Y`6;=kACDQyciaFVWH|FL&opE3f;9yQXo_EQ`OMY(}? z0PeOn&vjibTGE$sVx!$)ZYV|4HPZ1LVG|shmFqsU)8ddUHSe~}1y^Okq$$6^PTYL! ztoOV|TS&KXojZ1{|6+0_xzZ#c?Wayyw0|4UZZ@f7s7eU|zEBjobJA^;nK=_550~)e z8L-&3c5UXT3iE|Cgg1-uZ3h^X8G})@@!u+sWeuDHl1J!aRxmBfo2J}Xg<-XMnFjHO z47)0+>wHY7m*q$q08ZTeBl37r1G}_|9DGAjC5^9d2wzoR7_>C(vJWw5W4YyTf87w{IDhB&c_#iJi*trS=FBvvi18LGi$50@f1=m zj!(Qj^MS8A6As6{SewoIeWULE59x$Y(}mgMkBfy|jaF%~Lqy!B3FsppAMK9S{gIhy zP>SX!xhfwH;e?XfEBCIH{?iCWo0p3Ii;6{<_QMKBCB?ZK${9scMiX2%$z+>joXzlu zhvg24P={GkQwUvTOC4N$ddTSL4ha96WF*!etZqWFOhJ*sYqEqCAuOI%0Y56>!+D|% zk__5;XZ^y@7jMu0ob2A5kUVh~Nz4m>xcp#ldjx_t4r_StZ|A5`r%7y4Of_4U0piAdcVFEz#t3t zzOT~TYp5Ni4mCDe=bXb{u!sbD?`sZgyHCvEG72Ou$l3{#QGD1Ams|Ygn;o1 zmn(Kkft-LzDsnPA333ujro$tJrmNxC{S7jul~EBBL5cz*)C9?pOvAe5BB$TSGPIBd zi+n^DU^KVayO3;A-?+$wHDxGF(U>MqQ(zCs9mqPztl)@o5f|=A;Gva7uQJM^!(Nxt z$XRTqBq<|uA>*UuN&_6Uacs6_a!rEqSQpz{u`({!6<8SY3spYBahH~9N7~kIPRBl7 zCFM~IS6i>k49%dCfi-xHE*Kzj>F;`ucbAW6`(N1my^aGI^q^ec zH1nZw&Bbv<*-0PZt0h@Lln;vhfPv0eDatUcw?_#M+uG$eG2ILc)y|L+|&a+xiTrlAvnhwls*3hu8ONmT$eAF zGGf8u;uq65IcJp1TLB!k`CFq4pU_oI!y7-M%ep_f^hIqRHT_oCo2DDVCf6e$Y57D> zCH68QVMTbLdT}Jn^^o~qBElN=qHN$I+^viX$rrE-NzRee;(K+XxA>93! zjaSbkp)Ltl*8bNb_Ttn{CDTxD*h_%0#*!ONPqtYUSQiR%r znS(|akyU$&MFb2WQ*~hrnqK-jh#s8x|I*6y{!b&8h9WR#@U1GCR#q7h*(>mcrqeBS zq!OHKyTbK)!A9Td99xP4cI4eh+btDO5OI3Vl-xO#cI-e3$M`)iz=X;GC1r@QTcbqu z_KfN{biC3zAcEiz`AzG~5YqQnaorjfYlt73`6Qno9y+%5wcY+$lTm+7>keG9p82#< zzP^G*jQE;5pZP=meCI&5jBfOa-sjG9mp{;SrdYdh!B`Cb{_IXr@u z0w~Z=S4N}d9Nqa6Ri?B)^Wa7#KL%4FoB-*}!<(V+0j>!#?U+#^*GKRCLH11QenG!q z`tsTW+mr-}xvl_J_$nsJTnVpVZ(neQ$s@R!BX`q8U{2Hoyx%A(P1y?Zli^glAen$& zktruuPp)CvTqXf`xiawt!qR2cpVFDyL```v?>Eq4-er4=ss$d!VifV)!TvBY=lPMz za6CNuAqvtFTj6_DkgoKQlcIZ@F`*#nwaK#gC2NVLyPu^4jyu8{*vvh<;OFm4w7S{b zxm(MdZpwuk@qTy=QG3Zx?M*HV5j5BW6#_5zELbQ&6IB2a$=*!p)sy8u8y(8=va~|V zs;iag!%_R4N$>{nHFN4wnf@T zj&| z$Q$S!#W~p~yZ)o=cWd+S++15>nF|M&DAxk{$Aqt}edK3}($P16bMcGek%^fAnHg`1 ztV5T&9~y?VI>jYA=gx&^1z$_GMKFI1jy^0yY^C+@$sQ-n0lmqWb#3Lxu}Ln^M)&N| z(2|a7_?TbUIP9(5Bb0;9Nj<5?UfS@?vdvQQ!L{HyO)~ znltw=ZmggJQeDB)hGi1`&?+Zd7D*tqtzP70$miS5F#!MGka)&I=6k>C?2eK@2pXBRiq`2Xd#T>?loX3$ay>XD<>R zY|9QFqJMA`n7#PL^s;@yl+rGeU2fk+O%n|EB)4s+g;1cierY?fZ$qnPK2|lF9=QB- zmA-Ni>+UCsSDc0$#j6xf6<-7sjnY?0x-E#EX*_vUyB3*$cx>*sD-W*U!(kDea}#Pb zDmsbxI8ww4u!d*D({Z%|Dd@O5*n)6lBZ0Miw{xJU^H^`09GdXCyQFbyJ3c&y&}2WqmC;~!rO>n0)zR3u z{2ug*TITQ--l zHY}WLgWWrN2B~yWbiJk5Zj=knwJyBxvQB6sF%k++G>8LHgV2T3)M?34JY|# zXLImXeN~kRYf8lO&XOHbZw1W)q_I@{Lhk6AP#p8lSwFB#O4u5NMgO?R&{OB z4`cJ-jn^g%7$)P3Q8C@QbMz(VlFPg)vGUV}yNXZ@lVxyxes;j- z)^F65o@|>_-12wc;kSe1IpCp~Tw^4vH!`Ka2asaC^P zzq4DNi`qCSa<>U{PHZi)2;!^UA2vIAq;jwix2p27g1*0gSB|sXIu09g$wGm6U{9aD z>94!bU(7?HY0_-{ZqDYov5GJf9D2DJN=%;g5gqy-m`rkC)?w;BgN^T6B#c7gO+Lah zDxYA7uT2_ldR!r!K+*Tc4i%IQyNz6Ug0UrF%v)9fVQq7PX%Tb7b8(Ir*jW77->}X_ zX7dAdU(qRTwPab z@d25IV-h`SP){VMY)L+ry_8y%%Tg99njk@@SA^`^ryL?dMkh8WISUiFO@=4lrmBR| zSd3F#3c%lx@Z6yz9Zw&6@wxq&p1sW5@TGIcT;a5yLErMu&+3 z7e5f1R7)wZq1VBI%yhe7sH>W<>T2#*OJ|ik6)J&y@@O|kjBqp9)7eXZyk3Oa&kt0( z%GIj7p)QBq`6Q~u6^~XW-zuRC2~5Xf<;x9FPSX+=JH-}3Npb1~5nly|3M5TL*+!B4 zkS^R{q0L%Q-eR+Wj}Z&~62#<8Fqx`*FuYi6-#JtJ^R5t2GSJWWO>NxRbZF#$A2Y0P zx2>J3sJF^U`V$(7*Cj>Jvo?DdrBZ95CQ)(r`FBM4Fr64?`!kwLOMwx zL?H{R&Q7&j_$5Ze@a_P(F1D*4GVjCp@BKUemrP{t96Dsd$*!Q}uHklH+ePi|?Zxi2 zZr!eLY27X1#ZNuA_wS#1s$>6)&pzLA@C7RdfeF2RZ{ymV4=#)Vm%Rv7N*c5Eo+*Ry zV_8ebi84cRdv}HJ!vVfOOX*wX9}Gsfw(#vvWN3U6@hsPx`?hYf3($4)ZblcDV%k6We1)zloxT4RBPBs(zdQ}-|!bdxE|Dq?aOhq9|DC(Ik$-QZLqG^Gm-pGx_fSeS-Dy|VJ zJVn-G#eWG2!Lt^<2ySZ6g%;@pCC>7KQkD`W*c0O2CP03V_=2Q19-8&O_!EN%ZI+AK z^6~QLk<9 zEiJ)Sr8xNjaz(vdbOAj1>pxrj|EzE+AF+6O=BzmRgTi^N7RX9m^|Of?shU>G$0N9& z+il`kDkyi`Toe!srmANu0!cyTxTt{jmF?!h0 z8JdT;P33n^%sAmfTt6DLk$@gA@A|!^FMI?IUq7u0%)#fryZ4!c`#bi&aNxyfpL*^{ zp3dZ{jNDFP=^jxTM`Pj+@iJaujv~9NjX9d^0;S;xEf!krKV3g@@R_H}JA6#IR72wI zbV!Q9JeUvN{J6A9X&5TDns)#k#F zCuStuYVMrhCAKLlOmcM}SO-*ZwzZc}QTPU1Eqxy(9JS!4WGpAN`O|FEJ`Ku+@;;4v z4oK(EkR>L4qmTx!F7fupCpxsDrq&y*R2gS;o^LXL z!^`cZS(>_jyT}?R+F5F_;gOd&YLo|2d|e{OA`~FT6yB#)ty1?ZqrOWF@y=h)zk_46 zVpwQx9@V!lenFc{xEr{%pj=aoxyoS22OO1zQL6;`O23U|vom6x!?Z#VS=4}PoL0Bf zO-+`4`jIe#KXBLY@$>>u9UZ8g?@oH8nH}xzJXY@All?K!c7~?7f)zY3%k5aXpN8d){mP@!qVS~kLU=agqbxu5Aoi? zLCXVas1{_{QZOzZBRX^p4hOESb}Dn{^1GvsDx0dri!GBda>>81t~0_~;+a6!O1~vX z@fi6Pxzprme~#kP&OWazg=^_jcARB@NKJ;urW&MZu7ME;P%*`ok+Me^QqI0%@hlR1 zEWMSlQf3YiOys=agtEQ?uAG`4G1F8|40WG@R(g9!cpIKv5-8*?UU=8gfIBJN6l)q9 z?CGuU+T}F5!{vV*D))6U#kH8CqoMRREH1>KFuK5Tu=FM8wQbZ^?_X6#L|?(JisgT_ zRG$XW#6B3_$`(cRT>4S6_!jlpQH%V=1vC?ibOS_q9{JA<1MD00>5J*YebPyj&G>FV zC!F)Fb8ok`_YIvoRvxhT$O7eFevY2dRfpJ7txhnEC7X|90chLVoZ|^?A;#gS@xxKl z)EDjQ?qU4mSW6{vtpV$&~u`vui@-+GYf7#bLk|lx;vjn5j?d?6!am4 z;0~_C4ga@*sfkNjbe_1J(ugn%pW?)Oe+i3?LIp-RFxRYS1Xc5 z5HL~Upp67YEGq5(QcG#p3zitaP?Qukudx)Fm`ET(CaHjptlPnEGKLzZ-~m%gPNI(x z6O5j9FnSC*ESVjYp%_xNUmFnmwVv~i5xs}J6rvL{P4lzU>O)$^ei zs;3DSr`y$ou^ZJ-L^zv!MbUpG)wd8xTl}!`Fc_NJfE+-4FBJ%Y$nnF7wqlPLdo9z< ztgs$*R$tVH$Awue0u~R4Q*NV?e81idA@s|@QD(qml|&*>Bh^b8PJ;vf(} zF#F~3nEo2i3@CFf0<`tp`3nM z|0|NW#?&o8ji!m)LsB6*e|}_5`jYhSKNZ#b8|#WgHpk>?-SS3kqfoS^kne zsw!-8Q>ar%UVQ$Ur#cQib)@6qk*A*RIQZQDr~bQv*td6UDK*uI5hPxhEuA*w%G)YZ zkS+k%!j1v6SMUwAsMF}B%O|XC>77RPex_4Pzzh3M6r1zH@Dq@R-@bGH!L9#!ewao~ zC&2fX`%Vm=gc4w9Xa1$(0n1Wvxmq=PLuX(1L`y)dX)_Y^5@qF5;C{&;`#j8RcW-=U zCC0JYN$h|-`CI%z$v6YY?D#7_6rS0+)?tBjeY;&M1R#K3UEh8YaJ%T@Bk^ZLcqU&h zC71VA`nGl1v3gL$Ce96@SOl_WXxtPrh$wCo1;p#7*kaTfL zO1>fQYtR$!H%UpDa5R-1;kLm~lW8Y?SaK-W!tWpUTmrot7nsZs64I1S9`J7Jt>|cr zO{94LHAupp&+TE+<_9qwy#MqkIwXXYYJCrd0);U!F+Wl41Utc}=do;O)*cfyyDOU7 zxYkte;FGnxwHs@9{YA5(;X-12H%*uI#qoN>Xmso8rhiBK7j2)LtR=#LYCKm9dbIM6 z89C6Y?TL5S-<84z<#eEfjw)NfOTEqCow$rxvWc1HG5*BtuKy-oU`xUdx*8|C7~bO*?ojb9X)Qk-lq(aJ<2_kGd98-rX^_~kmo}#e9~hAqN~CR!};)x087G}!xD4O zCAv_&v7FgESVK2ZI$UF`piF`5n{ALbQY?7QIxUqiF?tto`JIi6UrhcpI8`ZjYkNs{ zw?1jS6Ur469xZ+?3Tr0%D%IyVqaa@0p%L4;F&V#9WdIDF=za?W|GAZ}qY#QfNST6;eAWYGzt57gJ)&IjZyd!jrJpskB-_4wk zhOu(A)4MmZ#|~_{JDCHRNd}Jk@*x0Ny!2wwio8nTMXY3su51Q&O&-3@o<0k4MlKkG zYzZpLGN$@1=O3gPks=Fr4PtMy9mJ=&AZ)@Du)#`y>sARn=7cBUCtNKSYP^T$$v|c3 z#7W}m%+dqp&I)EZfyuMg@J935$s0dD|JBmxYpYr!KjTAqryL1v4K~;-W3p3785ueeY*87lA|3 zrSO>m{&*)!Y&)XV7%ag)c*kxiQVWr=#LU-O%-fJ~hI4ixIm6fQ>3L5tyLv@@1)Z6H z-h{-0Z>-U^S?O=9n%dks^!MJS8y~41^Y`AV6Ao8%Yxuo|pRY@;LW>(i(W9mZVX<7~ za5o3fK#y9ur9SrtP?!wBFNVi9!8uYKLJ34V5xSxhDw}6rq zI^Kw6bD+2Y&hp1d98|AlS$RI7wUI+ zVZ6?-%RMJbT{hNJ-C8=U*7;o52+$M~1j1~VPB);2X3vxe)Y;$Pd)ngdF8{nOJIV-; zX;#}_a!XI1UjgRMmx?7uNB8r{LI&qhYO-skeY6wV+>z$l>QB9sf;~+rIO@&k&jdni zehWQ0wn2ylnEVl@LchCo-j*KR4G{h?04JhZ8+m9V=U)LjBoIPSLJK#Fg6nx5O~c|P z5eUFKSk&OVtUnaRxGv2in8jBw-Kw*cy4@r8GTO6#R$xYU>OKy9^;i)sE z$wS7GY7ep>R@f493xUhs!vaStR$J|_^dY{*o4V+l=^c-jCYLau+Wh=wLz))m&iPs0 zTAzI*zCqiQa*#5-CKH7CpF9EDqntwu5b{2v#%Cg;@Ys~ltd=M0 zg%hsy)AbEiYe4cIC!_*!OyJa~?8GC`yG(fox@0p_$rI{`ssznUe*(;qXa00564dnB z4ASr|pG7J@{qsrPrO(C3JAIpzj~Nu3Z82$Nn{$|DQ*d2cQtYt!FUp4Czr*<8UQhfQ z@tC5`YyDxI@{zgy+#c5F%+n3G@g)~bqKPYlUn1W-#BoV|?GA?|Gaz4nu}Ywx~~4*)+yN|wG9 z+uFqJg)flWN_F!P4nmiw-*all$mp=eN8l65-TUBa zk3j&wEzojt0yian05IHO`oUZ4ya1HuXS84>AoK98b=*wkr+r2uSGYLdo~(uH)5>3p zq`g3c$OSJ2jrIrEKzUPrqUm9yuXLS!rt@@pph2g$SY&*;ltl+;^%g-#kTRF~o=kZ! zS@Ur!=RBXl>@})?j-oScc>|HD1YHGjmK8@w3?}OBMD}=qfi@8Rfsu zoHZyjc=FYrYMCB%Ki3JiI=#%V|K3o?{(%Y?T>b91u|ljVjJz*g4foL)W%J_~V=&!m zoSa+)EN;^Ttiq_Ep^Uv#w_wqhth<=Ywr$(CZQHhO8@X)Twr$(CakK7u*zMFq)oJq| zjE5LAqQ~f;W&7VZ3$qoU4d{)gw2^*ehaEv7WEI&KTO2;(z^}1l4xA#sXYJ%W< zfw665-hI9k%|-HOZMd@7V`O{@b1%5G8MBlUbcG^zUy1^G*xHI4-O`F~AD1$>y3tVB ze};l^^B?nMObGnk8qGJT5cbybPQ(X)xfn z03!V;veU4)hgt-vPSbU@;ED-;x;B^{VdW zPAA*cqV=qiI_kLXPbqds!iiRbm4lEM@x17c@x086;`I!qqAgCl^+L!H*arxzqABq7 zNH?xEsQUJE`Vmme0@>ik(|@^_r~jcS-48af01#N^o1Gl+qLc+!M7daGMG*U{q!(CV zk!2aHDxy)9(Vv(1S=DqFf+Rt7;jp4UBJ7ElLBn?7T<0DS1nUS>oF-{RR1E4Lv~IiK znb;%RO-T}~%wL!Rd}!H1<{+C!TXb0|4X_h`>eYB9M@y{<^aItf^${rRgrLtjSeeHF z#b;;-Vd@CTjn5ZMBO!Cgq@f8E7dm=?xi)@5B_dR z@xqJ=q*@u5`_#QUrnh`5^sJZ5*nqG~1b^5ca+ZvG?C|eS-cr|0e{oZJG-TF5M$Q6C z=@WbEhm{T&dM~e@=9FB~mjB`42Er_rxciyupDFdCsx821fXpNs?CCOAt49!CkJt7B zKAn=5+`SGHV}uIu;l2%Q-)NjihFFJ&5eRb`6GH$@puc5$2I>yNB0t6j!td6yjw)-2 zK_tFTyuf2AZ$Kt+%&ubcqKwUN>1dvr%Xju-S4L?WqY>cmyKMtvi*~iw&#*x@KCCYf zq7-{1d8-`KUhd>gMy;uZYxe2T!kpK4D<|)3SLTy#S9E8SL1yk^oJc1`(1*6r!GMOx zE8WUAcA@9J4km+%bZ{`JmR!|8jgZvS8fk8~wx9*tN>cWxqHbUI4@^g?+N9vGIPO&G zYMO_(=-;U8Z1w<`R&MM#BTW@2rb^yMhB1IupMP%~*oK6J!^*UJ$089 z80ZhEi2WoS2&%wVMh=Ij9fk0u85sKg{^*WbZeF(1tBRhW!nH`RQ9o+(>{*$!iBEwH zY8>5->=Q2|xTgD8E14Vyg`MgaT*W71h*1+WPNIG3-xWW%pAcyRYM(}hwWow}x?3uD z?N zAI@2|J@p&dDt6uXIXN0CJ0|PjzNUok{cc@zK|xL-1(67ndK-he@nc<+j(v~AB~J~O z^paVV#-~ul65~hmtrz<#wR#Lp6P)MzgR>^3uOqC_1Davo%nGb*hgU%=G$Xg!tF0zR zIXhq}q;po_?J=WJN*(GUg|{E1W}Oct3X8U)aMvyOLxz*{qwK&v`=kuE`R+Nh#l^UN zB-Q8@7Ru7+uVS;bww%_k+nDj0aZB2Ha08LRZF*<6sUZ>QGT|-c=83!hn^apbb~_|z zW?+D>Sc7AV%z~7jE^rus9GKD(iF9Uj(v$0t9D?K<0nhd2V~P%CiW}C+C^5!3JN{RN zSMjD&^$#-{khr@%r{C26&qnd25$7}eB85MRK=FdO>J8^cq0#so;B7uNGyY8xuE_)C z;iG9V+Rr9yB45BW+0;`T&4HWEGuOoZo?jr>yScjMxv7BpUDa)LV1XX*(yVvYVY|=n zdk#@emT`>HL}(;7eF$pPrvBlfXdz)fI_gHOrCLbjI8SQxRi3VE9h#e1l*dr&o>qX|7^FdAMB z)tMPheOU;P6P{Ymf&7s8ty|VSZwb)bF>G%9C4T0l%yO~*1T-=5^sF%A+(A9!Xx9@x zNps^Ap|h{~9fc5zN)93yj->6=Al=R8>(c&U1f(wl8ze^p6r;t&+ybjshPKu zif68@H0Gs5P-!rInMiF!3wqJrJG544efO=`ibGSWZt^5|fPrJ}R3v1|J&%TP>v=CN z2bL(DaP}ZWK6r4iW#qvXuclr7VjpoLPW&s_lUI-2?>E+d&e(1vE0k~_yOJR$A9=Bc zj8No14UhzwLt)8O<~iOAeOyh=bZ7s zSEhztaBq*I&f@>@R^%5Xp=DF>C*dtG{d<%53do;GXZL*&5$_S2pEs?Sfyf5~09q4^Xh#D+!_1m~;|6+4?gsrT)j&@Q7Psoijv+vJ zmJ0x@dwhT&r$*;v{EKeCJY6@Z*XvLX)SP)Pmcv!Wv+_IJ$I!~o^;MYH0`3T%u1CG? z4KLGaM6eqGc#|rCK$ncJ!4Ck!cu1^wII9}7cIRG+K|l>>23NmF)5kN<5%OqeyZrC2 zg|`|WU$4O3GxP~S>ajuoi|4)`-R?<2)EsK)sH&)4kBgP5BdC1ar)y)Ns z8*Dtoy7?PxKUsuI9@X`=>H#yQE}YI&rO8a&+46b>+?w4$PdnkrgW$Wc+;SZ>!g_iV zaUf4rh7>pQLBgBfm9vu`rlN38D~^JS3jHlrZ+qPgUVrlMNSfOYImP>ece@qZyu+#K zB*m+M+jaHTg)K zgQ$uBE7%|Yqp&MO0szqbFC=GYqyGk9&sGtL+Z0Cb`BdW$nKU&y*tVv#2BaX)kLfU0 zi%6>=voe&sLYRrpz+j6h5J$1x=dZ-2B<3#@6c5MM>IX~#@+%o{A^QQEjbOtf(=D~6 zouBf$_1g7%>^8GR9u-MNn7pW*gZ(EFBPzuq31RZ|2q~`)sptKOMGM{e(6E+B$ zkyH(SuC+fPqapSWzP@eK6BoSbuY{{u%)dZqL|VVL11=sMti2$VNq`X;&+jcUUm_yt zPM%@9#U>EZgNrDEg#M2wq(Qfm;hViUSfPefXA~?{z7nF^3X;3jQC*Yazh%9?8YgGd zrg7r~HtGdfhd7qzMESOrc<2_{WKR}rsY01UqFZ&7fnhUG#MK|Sam=fD=t zZd|4L#1VdIHJX{Kigux-G0oOz2qO zmb#q$3ZL$Z72J@UEx=k)&Rr?&CdFNe4Db$1dh3TWo% z&E(^uQ_ZhuZM2elytv%cKoed=6Lzj4)!L%daLKx}abC1aUt5qwL{T61xQ!56qNprL zEjiCpigXJ2GzFQX01t0Hk-nPu3If5mOG($+Tw4;EK5wBH#AQ+2f7m+0BIFkgGbx^XvGw%0d)X9}0^-wh{c#<%0;^jgXGiYM#m&gu1Z_S`#bTLpk| z&e@h)b*K>30mP>0N@pw`0%hof(sk*q%WA$nm3Mqci4r}a72SV)KpnrHsq3g-4Kez6 zv+Ekm(d6>_RLFWWl?r^>8w4dj{9gsWzvIgX?6g_TliJubx%Q=5k!fIdCEnu~y8X&= z#*_v#b4o~CdUoUam|)#wBE1Dh)is?IfZb>~*W}ReKWZ^r0H(Go$wbKp0s!y{1pvVG zKdXhYfwO_0skNQkf76U+RRKFJ5#*lRT6e}YcE+8d2zFwELj@lZEM$SmyLo%sis?>$L`mj z*OQp*cMqr&1EFA|@eUrC<`7L7Tj;TPM*FhWLqd2RQ53-nl(iM!f$0dF6NQ>GIN%#4 zhz_xQTtv8Ek{n2H!D7Qeq3R9*r$6$N+QknQ`REaVPvhWibM*4a9O;3uP~>w+aiN&# zU~*4`{s=b;=#1M@2gKAWgAVET;Y2dBcJK^j{3P^$`tWWjetq&SM%WF;Se52;ET{>@WCJpDabyW>A_)%79*V~ z@&$!q!iqwNOvNNzm}gnKY3ByZ8UU1{?8z54<}Ffp3f znK7(aSN$WhR*mP(Zklh;3`<&J_KoWsRRAK*^sqb-#pM8t#-b7zVhzm{)5UWL4w95b zxR>;BQit1s=Za^MMwA??AOtBsCDLLjq~<38Kr3dMXJgDNX%NO*D_+rF{e7c1RuXLk z;y#^J5^GKwO(JI>OHlHpn`D`7+5FHt2o+LR&1xNl>Rcr`sE#~F3>Eh`9VQM7rI1>Z z`Lk`cf&}Lil9=$a)uvW%yAF2CZl~DNvSS^|#>$~^`%ljGK=p2o^E>HES5BI|Tw2Z6 zpT~M|qi%k@zb~WrNnF3&y=L5Kz2D5fKhsgWxip$Cqw7?p)bF0i3yPqG;zO7kbA4j5 z#L|_`)uBPD&+U%y(4AbDw@t#p8>QNv6=J01PC3-g3qKQ*Rm+%%A3-K+wY|LYe1b3; zr-ZX-pK&9Mtdhm^6m7E{C0S!JBnJH1Fe$jY7F+r$sk{44d@qvk~dJ4v# zse#fF6o#p!9L|~h#)Xj_r5CeF`wy8OU}%)_ZA2tqZTdi6p(ymNOqUE;gibuYo~e;& zxe|0|K$+0&PDZboTDCq;Yz7lY@&R&M>9#Y9;M8dhH}AD^qk21&fwOo!e)p$vXuG=} zE}v0qZnts0Ufw@@{gb`9TUo}q zd4-s!u1uVK+Y3Y7pWv*mZ9G~-UBx1C!!FbPYVs&er+iKQ^v9SC6rYOuSL;o?+eyyt zkh{l&PkvVe7=tM&CWak}QA$l(4@tfBUN43p=lh*adQN5TtwTL*I`j-1oWON9Z9i-Y z&vgIMp~GwvwULawU=A1nfLclb0NVdWhfMX%91ZOMZTvT|H83-Aq%|_C*0hnsUPbwt zsckDs)G!?2yeS0|&;)i+9hMqC5LQsB4$bbQpT4?sHC;j1c$8PE8J5NW z69khEnx*ORSGaj|!ljtA;xTf`=_Z?tj*Dp$kxlV#%7^21%PHsIcWeFByoxzyRYaAB zx15r1kd(#wK0-IP32G5cY3yGT)|{1zrC0_iq^TudOEGAlHD6g&ZYrOF%$#vCgk3_M znznEkx4bC0li!hl#Ozq{OdPT2FIrhy-;N^^eF8~L6@y4)mKZ1?5Na47U7#pPS7fn6 z94WfcZmp|AOd~TMZV21xLQPRke#j#wYCD&#-v)y8aF^-;N1SLu!DF5I-eT@-Y+R&b zIMH(QlwURxhB3e6t|mvg{>VvgEyG?~zC<4`>!&8~ymqxmp74+-uZao5qGAzzBfUdzHt%E_Gj_U>StiO5#sLvol6jM(eL54McuFjZa zEkJNP0=9F*#m2uA^ZYuB2z;MIe11u$;}dc7#@|H;XlIw8e!%lE>0ZzoB43AEB}8}R zvISUHo<~wXHI-VlR47O~s>1ldTy31pY%PODDK3-@eLSP26NHn%z1$cXfXl)!Fa)TO zQe`MfYV~jpg%-)m{3SiVU@7qDRtru+DUBIebxJhMnSw7%@u@6J;{LR%sK3JV(!S8j zGz@ShU$F&gld(=V^oiK8)XPh;`#vB?xvG*)ABA6*wNwPEqyTgNtBfOSQ8AJFScY~x zBVO%6S%n$%yV_867z#^}MPqzTiu6NqyZe5zCZoh!R7Oa)xW6XzJaZvWLcY& z91Dv(No%vmE|CSgG2&qI<_3Jq-p3(%Ysej}2-CQO2x{%hkzNKThb=Rz0CNArK?G72UI&=PpZVcW0Yx^Ia*1`vZ|Pfb zQ>}UFq~s8KyDP==qQ?9;TvV-K_dzgyf$8LB(dNhUXi69n#zaf`B_4N9{z z3@~JUN3p0p9)^q}iDlto%wWvRCF--xN{AJ}&1u@Gz4p)VsqIgw&A<1y;0VDLekyek z>QuB^Q5&*+4h7j91z+y3Bxr!$lA*%b>dpYP)Tar}y|dqaVK#ad+?2H{xY^XQH$CcE zZI7*;ziWCs*nWlV?XE@@=(jmJyV$wi<6l?9Yglc?3~#^IjLVyDq=<4|%d66yoZ^LZ ziEv~N7={H1URUSWWkTm7$A~v=T>~r>x*FaQs_L7YdYbCr#`*5c^%+*AgFNjDIlogp z>t>VHXlo3w@7JepJ?pzYuip8A5a65cAN@r_?lS~Rv#V)!rCkkQgHzcDMO^RfKA(+V zvw3pHgle-vQd+-%$|d;u`|mhtNGGvN!j4MzjFEF}sFwnmS<-y@K$A-+ajtlvw7v6_ z%7hSB7N@4VdDS#>eg~)2uy!3+3nbb;7X5tRravxL^vrH_a&`Zb*Q$>RY-?Ju+Qo`y zRtFnl=;eV`H*@$})v`z2tbI+s;LTSQpZ?WjYEthzJHNfXJRL(vN6q8kSLfD0u^>pK z*1E=kuq1D<&xWYWn!VmwL6z%5x{PrvHT*W4$-b6=ZbAy>hnwp+K9b?eR~f|7ac>)d zO;pf}iB<2=F~&E|G&)sJ5_sSMSAu?2+luH$0QVFLJ#!S}jI$QzI+e$Itm;-?Fn7DWJRlYwe8$7m9hKP3wbh}OmDSVD>SGJ$_n5~aOSkqh6x~a}N*$!3>XuU) zp4-PE(*rK`2>etSce-2JrttVoH&sG3=9%2?#`UgQKepNlT1v|=Kx z%>*0P1$x{ghztPSjKB#o&sbPoFrkHMr69g@o`nM%qE~-6py`?8X;u5lgjSAH7n{kepH2(0C1DCS{*hH6jk%v?eIVp}$}T;w47*dd*VH@qodUnap(;;aT>uf>)o5daD!F&L?SaME#^OP+7D3I( zQ&O{8KUUes>6M#Q+%Uk?p;%`M+yoT?HM;iWb+p$$Na5Ea@YV>XY#b@Syt;y0QI&9P zRvOSnBTPJ=M^uO+fu>kECINXEgeG6tw4;$|DZ=~a!ShKeR<{Vs`x3@LGcAZh16 za-?|;uZtUAyVBmy%#*jreCZEC`DnNKb=4vE*_E_;dt&{jtVe|(JEz-e$hQ>;@ZD!7v);sjnyD~@Xk^5I(k zLL+UQ2p|2h4JlWUJ0nh5U0)*ZWUJE!lyI_< z)LEn(QY=NZZxxd(UV9pWRor@cIcAl3v^Kk>5CyGg)j%4IN=>#B7&}H>-6lC(eBgvb zz$j3bzMn6p$p#y%7~udsCil=oR&?17Yf~Pv7ED+wc_)E=95i7my!!xiPcy1lSY6er zYbC21M;^*f`2#c|Iua5gdeZ!p3E%I=kdO=7^IJL@KW7r+5Z4R1u!@6Yev^~g3BcmP zOi;xVa%!rl^LQRByzcTXZ#HZcCtjNJHMuk2kCi&OEAGD=x$54`Gg57B$CH^oW&U9K zo$j<&Z8V`x1K@dkp!hC z>UBcc(9L~j)&1?kP02Q#B!uQlqZX&Na)@$cVm0slQ!qul*vc!pfzq05GBxUH+PD)^HHoX{UG~W z&{GbF4T0sZ^D5bw!Po7WYa4yVSmqy}7I&7mc;_BYuZTGWCv+4}tKj(jZzd3MQ#1S0 zH`f#MJ0u0{f#exjK?Ch-{xL9$9OG_Oh@zfCg({<{kvur#oLyj$i(^uuw9TONI#vaa z4t90ulR19K>h6v-;p}>s#WFoEw73pVD|ic`yIJar%_?=Yp^Ik^`ex73C6OAJA#)gC zmI{2b&9+$P#`?*vNcKMAwle)2TdWmA{Om>TZ2@2i`m{c4wGBVE?xX z)6dNIwfbkmsL22TnEz)JW@>HV>}+DI=lHL4;6yj~C!j(nzy5?R&X!&=C~bTh@?r~1th(M&^Ni-n7XJHp=wqYu=! z#;&t;L#mN!)ACofSYw;-)oylxA zQLS+SAVq>L=9R)yQ)XNlEZ#g=n;DU&KNY{7e6+k^5oDYaq5`2XF7YKAIqbJ88~hz< z*;zwnZ2b%aQPAJN83p_omK?fVV}T9^L4-wIwUl3QMM69!u!W?&{Cfy}BJV1% zqXMHy2O~26#+p<{s|rPejUB%WybMZ%Bv!IpFv+`CPZg2VS^*Hy0Ra3>FVd2RhUUHr zgtVv6q@7y$obe+DelEtf0%D_DA`%wyRf%S6LvYVS!3SM2kW_-2l&S$CD#STcR&$6t z@a2<(9Z6_%1SSu{V2@0*X*je;I;N`R3`X&-iE3}Equ&r#T%0UrSq&GN1+$Uf@Xv@W z)tmPgzReL0niey9T4~A?NR8wxyb3hibDmcCc#kSqAiAH;}fIqVgW&rrVKid-FjOL zzZ$xzlTdJ!mkG_M!+#rpF}wn`Ov}G)ic_(0eKd3azUO)C^H$Pk6fb<8^*XUlAqI9r zCk3+y917_wyxh2>cRn!E66wgCF%$khTq1!>OQsb+ovxx>tRCR=6ju>dv zK!!vu`IN`6dE(7dfR0BTQ$ zfTFVUXJ<5e1`C=#=f{t6tDQtr0^K8vX z0S7dP;RWGYY>gR}7!=t(PSVJqH}7O>lZs@?Rmm4y`;a^y-QaP1Zr$u{-^=?r{hguh zTiL?Mk*VdSyNLkrMp-C#ney;3a;Mr)-TqWp^eKb&UTw9F%7EcMNzlJp-^xC+3B%_C z2|jHbN*Mt!2FJxL3WuhA4C$YONVlqXD_CA&fHtmI(nVb$OqNZBZ)cABx za2f~B<>WTr39x3a8*t6-o&B^x>}puomo-Qkr7dD6V2l%)BF&!)f+L(M-=V~uTmlIf zq|3s9YzkrgG(Hd~AL%rB-95)HF_-QPLlMd>-jUxpyS-;?bJzw;3~QSJ9|G^@wdGyw zZiNlY8GF@#SB(k$?GkC$v@x^rwd6L|svFQ3gs7vN^VQK6z0by?%T}p3eI1I;E)&60 zL#azFmpRWj z&Db{aMvI`_avYJ4v4ik+?ZbqRvzzSqVxK6=rR%#j7v{+&L{D-|v&+KvS5n+n*9tCK zw8Ni^6u{h*!aetzl0pm^JefoT>y9i?q}RpA%o59d0Ox*8m1t(@9LTRiGbY0!{ifi8 zfJ4i?>b%@OS$R@#C(HxaqZLf5CGY5j$=T{Ce_G$(2Q@vE@5C@88gKPK;E3EP(0?Wk zto}y1M?b%|LbhWVz*KUnWE_VKH-3K=PtIYQ?S*=HjP+Fo)SfO{a1>5<45zRemudPm zsSjg^=J53T^zydz-TzE^UUxrGOM+Us+Zh>uDjffDJE#{teym2Aa!V0$*2le2Wv+8Z zT_u0Hl@BvSgla)w;AjXVE7d&+GL55k2F#}F z_a0UEv_8fA0G+~y?K~B#v=7!vbq^jK9RR_Fz%?+bR|v`Agxc!m%cd;4!B(hkrxuM$xm4jIDp0+cA zgIQ51EQigMB@CD!kwc=#%WY!Pe2%e?}^RW9ydBXxFC zJZ$$M4GrWQuUm(Fe+ta@jN%wUpK+}Q4Gko7@v;w4!oAItpl9wDh{ATR-AESbq*^Pl zcJF&qu+wX+`~9aLa|rFDZI5IOZo!2HTlWWC-ZCoN_Bdtk19J@rQO;=)5CpZJK2TJO zu;KI@FNC3*xz?SZ0d7(b$xxX1&9iVYfoz)!)(`}-kh*<=p73Pm8qQa(QK zsyReqo9#?^=ik1FHOF_5KPSj$`c*u1+(UbPE(q9-vzI5=8^4ISWN9_07HYaV$-jJ)ol&Qy^w}P(i&)%fHFG9_PKpQs~AUK~S**JG>t~k!BZOda<_!>zTF>)j7{1c3qf}jg{rB z2R%uIrDJviZRrs9&#oZhFqkG0gJ>%i-|()TYaV&XRT)v~r= z#86LH)3SD@Y>uRpbpFycynUyrz}m=qBv1T`QnGEOCpxk@z7OoY@Uah_Ek@K%4F8#W z?~gt~jKd)E*cw)M8VLL{&{MqkG~`ON0&LiIu_j0ypF&m>n1H8z|491DnZS+9KwGg;h zDvrtv9la|}u=MzQdd-YVFI{B*C%yPs>t7a>DK^Rc!`Ld{G#%*e(U09+{C1BiuXQXkwyT!Mi%x+%{m_PRugA$5IycE?hDBM2=IQ? z@&nnKkl5Dhpuvy6PAPpp1Ls61m0q4w-sMy`+q9_BemgoF({4Hm{x|{-qd>-mJah#K z$xj-54^DJrDntq4mbwYd`U??v-B*jF2Q&3WXdr{bTuLe2$nE_-Fv4HR=Jr5UjdXqz z0AM1=3ytm@g+T(8lcpkWbR)mn!XiTpmYLt^Q37N5Ue+7{dxL_9*K!E`Nj{aT=+xa~ zCCTH#=}Y*xW*n)i2Xk7!Ud@l9yE|7HgQ#6KO6rc9+L;)8w2F+~rYL;(qZ;&5lu21ZbXD>ovS!1iiE%&u9*6@_kpm(=#oTNjIs6b$k$pTWO!CqYH%Ka=rp-8uTzEx}t87h!d{@w6uTij~tkqc#i|Mv|<+ zSPLW0E`~uG$)8K1{+r7F)=Ko;hWkVj#(Peb-Y1S@CTjW;Bt?nxls*CiPg?+-MPOSg z@NdWr>2NY9@_C)8&&2ewshjuMqHQAilg3;XNqNR7BeH?}ti=7!fneO)ETu>RW5Lc^ zl6Eqyo!hpl!p4m?=I63mCzn{o-u2p4|2jZPz*<}RJ#c}(82_p{SUmW+hz;Vv|K5c| z)c`CYA{#BJ1a}5XjA&upZI@gj3ZO(!dsC6slI3 zAdk6@YwvC`8f=!S#x!V9uIM*HzE3;m8>g~PwtXN+KfrpcQlc^P*K4VV-FOO8@MlW9 zByK2dS|-oPn|Cmf>=vS0F+vsJtMY3iYu=zxvQq-ZMZ!KV$9#y(zXM>e7$lCFU&82A z#MfBdGT3qi#I(|Z?(?c%oY4q$xn@IbKVPsmV_PTfK(`gghby`9_;qF5lrqO8!9L&j zkUOKd`plv@+t>vD@ye_b%6U_!N}Izo0i)jf9}=W$(cd_d2xDsxNJQAR;jL2s-X*A{&43yczN32+q(~5K3i3);`l1YlMpHF zE_WeW$s8iAx5Y;*?_Sp{b2Sm2p8jdtK)0aX*Ey!tY_5(-*|X8V$&SUPJfs(j zEU8eA@Ds=$>M@HPEb&V;atzkq@nuZG)03f*G%^W_KUhjP5T++0ghVj)*FEy^cXgNv zHphh9N^kPm)sqkS0_ZgQdZYVyV5wq2P-xbmDu)ey?0A=chZ8p8%NRHDwSDhNSQzUx zR+t;&6M!A%^jEIQ58yxY@LvOVPEo+|G*AElgMURIs{c9L`9ITjLl+BcA(N2br-!KHC92b|<>6X*%_gV}#_gPR7y>ZCCT zHpkcUs{I53=!AY^Nkc~h6Jzv-IU;O7`s$qa7P=8d#ilZ_*lHOlx0JycF@?7~8Npks zCM%OOJ0dSMph2FynB-lVe>yAnSdzsY3~a0nLeP4=DF@_}VPqzDIxGc|-3j`<5;A18 zSuq|Q2{rwU2l@yynYrv;V1(Vdtq}eeQTFk_wfTf0f&|jU>vaYugdD3=vC+e!Vuu%J z1jJW5?1iUgHbEk@TvG%3rT@11iIapbg^^AuBv!+H@j4KeL3gXD`xRfh@;l( z(0+(6IQnF2*=yU|?xt-9AYb(7dtEZui{rwkA0P-r0vuX8QadohN9Brx80Z5e^Y{-; zC;%9D-d)Z@#pSS}MLn{yT}I8it+>_y>7PAKYGKHBsK4$>k!)T!d!@p&{r)k$ykRA; zRw`!KtF58g)-1mb@1oCHvS%BAECHfxQ9aTAe0W%RxegnC*HXj%-PW%7-Bbr7Ng?Tw zb&?>c2BLDR!NX90>k`7DlmikVr(>IR7stMWf{7If6)g^?l(Vi8DMT}m#9Hy! zPwAT=ePcZ9*IqEy-nLrpLls%kK4CCGs6wq?oa@Kr_!vOl~ zXoFU03Je2`23-Q`^@WY8Oc5s$0$ek9z*JXjGO$d$XL0t?O72G8oz_crz5VdLE3CGq zb&&*4h0uIA{r17t+tc{wxi9J$RLQ=Ky=ikZYkm#icB&}HFz;ZEOW$aI$=VGTrO;L+ z<(FCm&N;1EsNkhE9O|Xu$>?Y1^XyYu#SQPq8~HG5Ex`{04?ep>g?Bq;IOf}a2{yk_ z5VYMV^84|U&AM&ShdfQsa}^jRmq>WUt>(k@?_hlL3v<=k^HaWbgZMykB1#ACy=Wx=|Bd3IC03e$l| zc8QDSkv$Y1rE5fq$wB@$cDi>%1`iEmo<3d>fP8keyRZCeeZ$l|QDn77s0E37Y>tZ= zxxlkho17#hB4l!pDki`e_jZ_~Ixtt}VdCbls8C-=e45u=wn50<)A{`gQycbhXl`nM z(Cxz)?&KFGBSl2|OfksZgUchG0IHn=m^NV14djJu#Z#;8z_qlyz;p9@Q_fb>$%DF^ z;da1kU=e*eAH&#*gdUBIA)Z>l(KzBB)K|L1YeeoVgs0S2XL*|m*J&MfU~#)tHk;Gsf=e&^0{AFg zDZ{=xK_M60_x8@~sIY>;R?~QmIyx0+6Qf=lPS{byaf|L|*A3t&;qYG~=4G0oJSnnB z@>3@9K`9FObH}FfP=4;2sze&*bejMT>3pxIvyGP*2~jFJ5)Gl~>aF-JJ!jxiWV}ww z&`X_;lB;f_Xg@;PVv-C+iqZO|umQK4D1?*igfpYD3*Z2Zsk~QH1c^9!DjN6UTYNf~ ze}8|UB@i>saWh8nH^L(<_7#tVmEQTPR-fMPPGX&vC$8;;uQC-Xh=)rc8WLTZekL%Z zg+UfO4V|b4;?^4s2G@<|--2}Gh|}aRhd~wvnfV9rRZgvE7C(qb{iO;3_bKwEnjV4$ zz9o^L9kwODWD%0wXE;CRw{iOSs@tRSo`C%yBO&8EVh%=j`zF6OKYM^1USc2M+$6sz zIP7Ueq%P3!bUjsVY`}Dje6ndmm(6$!OVVU)(?m4}DAjSwzPSk^6{8_yq8W}wn!#}K zVdqh5II|^SU{J2bQ=5C-im})GCn29nl zpgF6yx;gK5)GcoW>))j|2L|stY-^gUT{XvS21e7ADBWQPg9|F$k0A#i$ARGh_0y?f z_@Ui&rI~kn{1ZWByb?Evb#H>%?HC0*WhSpi<2q~D)CY|Xv@Z_BHP$Z-`?tfM2av6d zLut31A6|pDk#vaZ{T4rK>))R5&j%0dG_vlar5_)V8rq6hs>xgV&aQsmBbZBx9YMc& zs`=^Vo3x~oN-p~_jD`{XqDh|~n z%DsA6`m=ZNbq;9c{F5RBz>$$HZf*sJxb~XE7kIQ?Bs^Srq}(}B+mu<17M0-MlnS~5 zU^<&0`vxaNkUHj}S@8i%zzN7kM#s8=gPU{>>O7}sjIFQ$wu-3Gp%Dd5J z*ps|_W26Fhckj+0!>SV*z?L6zU(XGr@%S$~x%4a;TWy+lDhZQ(Mqi zv!HgX{%jceLF(^DSKfxPCaoA&p&&XbvffTC0qj%0q~sWEAv1w0U2!`>4Kv)Nh3l`7 z!p$!98EqqAVLxs^RlQzB>Hgn1`)0kG5UI2bBv)GN-qa`+;Dv4&9|rY)(`udSk98&^ zU3~`XAWo$VAaG`=9%qn55OpquhX`j^@skJyeIcGBs`-PzH9A?ZRXrXsRZzCG1^bn= zf8|mQy7p#FZ6e#_V;XySe^I&7e7%_3nh3mC z2gX$}QxzKewvzMDYm<}#XDHHXOd>tW(ToDq~MB)ghTZM zfrAD|lglwh3n)G?TCK6ji{d=Wqpv1&_FBPh6jlL*Rgl^&=cf{W5Rvkm#3ivY{L{ux zCiuao3xJC2E}+4oSE&g^9L*noq0gz=2wS}<2NMPnX@z8Wv{<2Pt3sWUo}VZ8h{FT) z9bmVFnn9sX2P>_n(RRUyr_iEL&*q;1L9D~UO^2#(4r^dI{YwLAxWda*Y7p*EvK zi07`i#AC6!50Ynbcs=VlIt9JLo<}663h^-EZkJbo2m5%{)vN7l_AW%+e`R2F;QdX& zCbcQ0T94_QhmEf!8i#{MQTA+kH*2uJKJw~rj0S((Ed91s?d~RHibVWV^n$+o2k`o* zo@QdGA`?2#uU;hpU713!9rltM#oc6?)nVr~O3lf2J-1Gj*zcu1PQIz%wO-GS(V&rB z!lXnH;rp+lX)pdN^>009m%pD}NYX!x`Fwf_bD8vMGTN*{Od_VYng5AGMDv+(Ov`NE zKp&=->rBiBES8PJy}l`0G3 zoSFs#bw3aR^;{t7-(Y6S)tLLbwU*zswAM9y1aF)QdpmMf@5Ey%YD(@oF9xcZxVw3@ z|6$Opljb-x$T9EfTG!4=QpfpntgmOU<`Bf7n}wnP{?k4Nbcg9l4a%lMRnDjr++%9` z@XAqZsO9^|znH5;Mw$mtRq74cO_m4hl$NdJhKLs-lxF{mxerQ9q`wd^=n*ck7|imw zuAl(+kUH&Cpr*c!O>?~jKyji$d+$>&7M-qk5yt}3ybEXb^oc%!ap1S~EN-#W)4;9Z zR!v#2{zDEFf_FK7td_EP@gkg=5RW=d6OR)B9Cjq{0wLEl28G;cI_IiOZmRvpdeVGBnm{pCAf7Ncf z$Dfk=M3nrb2{&PWw82ufd42UsvD*`L;+594i%Tr;DJ?fQ%UwWg#%cvS zWgqWLbPo=!Y01`(Y<|S-d=qAb8n6PKztBRUNlI>5oL;y7;YLwi(ydkW^=zWCX%!}D z{~uxR6eL>EW$Tu$UAAr8yKLKb)h^q%ZQHhO+qR8UeQw0<6aPc^>sk@(X=TotGspOb z_m+J@^2N)n6FN*l(X(+09C}lIDBn;O+{R=gMF)G_Imtf;yiobeF5IQ%`<2p^rY&4M z!=MY{JLY_egrh8*I{YWA+Qqv zYHUdE!=|7juDyBbrNWhZz_tn*MJMF3(R27AUvdT)DFYU6ePPnFUjaX4Gu4HZ#DH3r zxiuy}SE`1ex?B!Qy^pVWK71e9uI62y2E~(HRp{rm`0oJFLIup(XhuV+J8}omS|qwza~*@Vb+J zqRLu1ISD&#%p5vUAx3FEpU!58>ipgSCL$@hj52d=1I@+4B|vh6Kc!!js_&r9&O{!J zCKymVHcWZxyWdv`b42qo^DtXKqO$la+%4CSP1K&ZbjU=IrCi49Jm$k#e)j2g)HvK zFr~1nEinBx1KLy{OxmwuD18N~WqvhC$?I^ToWPu(Q_U(k*;fz^7uHN?@bo)QeD5x|cepbZ2u!%AuXFFU@aU(6~uw>5Wceo~jrQYe^i8 zSc_Dn3M_!()%X$arFMX90Ej5Pfy}Mx?_doghWE`H zzr~_&QL%dn5o())c*Iu)Y&xe&oCkH+kJYd-F~?C;($QoGuqZolT!w<|q{oo4odLg; zFI4;@`T4vKH9;EVukCnme4Bu~v@nj&@AZmus+M!^#_A`ofqjWHN9LC@Aq$`ITyClI zu!0@P3iDC@_yv3OZ6Ye#0(Kk;Go1V}G);RVn7HVz%AG{};DZ^fVvL(1(i-QI6S8&W zAMgt_|0XH~J4`fGlQIgXdk1j!hbgpxbPFV1E3@Oml!!zQSz3Q9iOQBALNzU#eI4;R z3M>3~Z^JWtFNcP1uO!)O#IMv=v&*@uUjtHZ6_BenGSg!6;wj)zb&}zhKPPYNlmBcq06C=a85oLM$sDN z0;L9M04G0U)Nfij_+hKz?5M&*9)|*h!y@T52+3W2c98DAbYQ21^+>JMy~Lw2qKH4- zU3ozRxVn@kio$$Jn0Z^_zTB=pJFzV?MV@JW<0aw%v3i9PEoOUKkB(hs4 zsjJ#SHQXVm75RXo4heBPRbaN=mVIT1hlK@YIg@@tvi2#hJ!6RneUex))7Suh?JB(q zYS{ro#&?^+YVPGL^V|f!oC+Q|L&Q6BomD=X71WIuG-!4`>le<_(F5m}g+sCdP@xHwmL=A_Bky+85j zGA~6UiUa1Z=VW(}mWMB8j=F)r7NE?(Gl|xV0E5i4@ysKW$$8B9g4_Fq9LDJ^_adlD zOwcG@{NjD-h0Tzdag0rp!2!5V{Jq%l{_%U~w3-%k(D3JQLEgrxYX&6X+h}<0+L~-Q z>~)LUwNLkfCae1a+6>YbqGX(((xOiWeLx8^iS9|l%*pC0==sKhg+}im)lc^gr z6isQEY>^w|ZTyeYS=kq4S}YS!c~@(5K5{3{hxS{ud1%ZY!q>e%wkuFhao4}=Y??)0mg z-SfO1^s``yE6~8`#o|r+=*^#^j$~8lvMLClHjjs)?IG@0<|-K@hW;MjCq!9_)zyA~ zw|)G8agIz48BBv58p=?0JkJ3|4KXGPJJ9JzimKfX8JKgD=mZKljgah{r#_H^FC}ma z?3zp<)t~6II(a~ZPAz_PI^!xEK_SxKPG3EVaz$*!_{Hb|O5JG#B;(j4emYRI@^2xB zhRmX08)2vPv^zw@`1vtS_TFsw_&7Lt8SV{3>#-!x#rud6%MvP>m3h7L@7EFweRM1@jiN< zN?K2tDsgFcsb z%MdSFtdb$g2#Tpq*+2?R^>I0~EzgC%$QH}h7-DrZBY4|@p1@nQz5b)l#K`nFTS!i2 zzU&bIZM@M=6ZJgkOoZ4&pCeIx)!MpEzUN8W5H@UXZb=>aaa}Var@+kNL_YdrV}BAK zO;=7vE^8=LIJi7IFAV`au&Ahtj)Z*j$JEgHjfTX$#%jw46N7qbwI%q+C+;G(22-%WH?8g|96)r19tgBFOpBQu@B?dui z^~ab4)SC$0pCWMk?Rs6*->PK#K~AAZvBw^d#rxK~z}u%p-jr9k4IPl5pUK0SHBb0d zd^)}tEqIeH9!9^6v*YQh|6w>Ceia?R-T;PJb6O!*8~36wA_UQ2O_?$E$&XwAPQZ1C zaN5siS{9k^&Q#F)d{DOc<%|DScHk_QzFZo5)-!xrx*3#Mm!ZK7Q~c@nbF*XukD6N) z3EbML)kVlotoQh!whiw&xC@L3l`AVXcWr@;2Dna0-E_iOYx*>SnLx+UAW)wG9X%|E zJI{C+G0cc`VLzDk_Nnof#SCaChuW!Ff%5Fxk>9B98G8J}F1ll)%9rIqsOq0>(Qg*^ z>2QMPOx^y@SGByZ+DMb+kC*G&@R=1` zpjM}v5msg`np`}ugru=cI$+kM=#N@<9E3aey9;d%E5g=B!7TY;I#f|AOXLS&8G8*X zcDM%9iRi&6KmD|_%`Naq7X3tbKrr+u9m^wg$LCPg&0KA__o`{7&Ss@{^)m0tvG>iG zgrR=Y!K6$U5v=8BDHS;KK7Z~Zb2Q>rWt3l{;uQ^lpYQDKy&O-i72e zh^+iRL?up?_{X+<2%GS%@j*~Mv);k#c^<6fcUsZPEpVAiX z?-SYfBM8eWTV=a|BGx95E<+thnn_%0b1$K*y3O7HxZ3?>CsW#bs{pC7AG9UTvI^74h`s zgpFt50-=w)u{cB4PzDhZ6lOgD>kz}c<*CU3>|XT57uc{_ZpNdo%%?$H1|||uNShxh z(D=DkYp0tX6OUVRG{X%=Zh5DLeSYXhC~x+w9(6g(-nK$PiXPz{9WP6f$O@%>gqcFj za=6+{T>W5C+%COZJcnsgwb}YIE%nNN?5B+~RiDuzCQNV-G^#*j zOG#ndAr9dV4k>R4#g<}C)mOn<;e&Bo9lDj)aSD73K4o_VGC}_r#>W&IZC6%X31}Fy z4sfb}6FaTlw-@6Re-Ir5=}Ug9ssBk^5HjibOdgOE^Qb2VdZsS~0J~a6HDG`9T59+S z>aGy|=@}LiSpWW@Zu2pO`d99-zt-WoZ+V=gaj*Y?S`fHB@CEIl^H&eEOiNu~ugX2+)rhRnc(qRjQE3zy zG#Hc_3)kcCb;}+1)l{Ot6ohA{029>=pVNaKR00!Y4ql~SZOH@KK*6gB@{duRitg%T z20x!u8n>vR87E6eN5)dg6)&v389u}MN9PN_cW3a`mEP@-0rn9Kw8BPEWsrhGKYrcs zSGLS4nYa^jYQL72Z!AyGL*wV5L0t7^``Is=MeRZkpx90}>Y3>FgmmClFEw$SI?_VG zZaSO|@DvFyR+O=lC!Hz|7{*Tt*nd&N09C$VT8G=GKo!F zlHn^f*FE{M0ljO+cuDp1_DL7Y^R=?q%Gt$-f5fkJA@j#(BU5cBBX?NObtZ2LBuyLA zbGTJi0?k7b%Uuufo;-T=;Q1aO++Ta2X{WoJ5^Ps)Ke}L;ay}b?EIgkmlr(Dn*I@$} z&r)D}fTBLHo0`5cs?qy3>LYDQH~Z*6j~rFWQi-RS^zb@IZx8jaIElfKq$y=;DO#W9 zugt%&xx&W4>y(Dmf*SN>?WI)6u%3ZNM83n%Mx;kgJv9nRcx_Tm1? zA?bs!eQ@@y;cJ0153Ae2j9fWMaM8RDA-{5XTmj_TYo11IUdQ+ zF2d}WwJRVGs6Z?F^xIC-t$&RSZ8bM})vYTU`+6EHRQ!ti5X?vny86JV5i zcvTh1tf4o}9n>J!YUZa|@oh3!Xq@cV?;Hj#d7=zN6zg z%7Nz5d)8Akr-BE2m#C?GEay{!HWy;R%4bU?Z79P8t$`EnIR)3KHBM~FlGZ0JP{L%O zBAl!!+67LVjR$+sx}!6v)b4C{YhzqYr>YP7lQ!?UQnoraq4=x;@@f-`4|*Q~*pj`v zO#5ZrLO9K}iNz1Nh77Tv)+Of@$kjJ9*6fw;5`8+)Z6v%xK1^5%IS}BjLgYW+&qieP z&;vtb+xg=K@df|imzpciP^wCMZ!7x0bef0X)8PL?g)lR;wf+w^TbIg?ES4yOcTbPO zVfXH6leCgDr8yCXU4SzEC~K5!ee%P5=lI)mdyDD;A+bUbeiMNbRMNa|X*qHbZzG{Z zg$S|qH^wxp>=zp6OTfW(Z5DIxy}z@`bB3)+4~FjbI0|TWFAQ6UzxgkB=^+8HH&dE3 zQ6Q?ECg4KLaIrtu3hdYzcwh_bL8>TN7)uWemXg1j99b%Vzo$3BPqwTUoo&~G6&}fk zpVF>qenl&kwk1!xUC$1(S@=(Y`m7NrXnV1OA`lb*k+q`fpd5u`s+F4?P-1079irZ? zvN(O_E+~@hlo>t(My!pvxy+G$mJFFBcfC4oW9D^;dfVH)RGk)0cSK+S^hB@>4K&Et zqzO=j_7xCAT&#pjT(JyiFDQbv+@tt4;xz**_iFluiHO9pX1y3a> zrY{Z=SRs@$xw0a?c4=}s2HK%X*)J7DD0ePJ2 zR=kWm12BB6pWAT?*#=ZBp)IQrw#Hgn!Vo!qv&1M($ymDBz&zLlDawDCucW^>ytgoa ziuy&EhWPXxz;W3mUTns(`j>nsBfqU5T)qM8=|vH$uV3segEVNhls5XWVLP`x+Iqa{ zw(MfFQ3iHR<@`5!!^Q*Vsy!dQP@2Y*3(DGrnK}WQlLduvpgn$~Z*y{AP+;Y*&nk(s z=ziY5s#{JiJ#_%ttj6`3*%RdAA_3#}t6}JTa!|wF@Bot#Mu*p~8TSeI`x1Nm?PRE{ z=^L+yovxe5+nKp+7dp~!!W19k_O|^`=T!kh4c7iql3*jAovN&p)*#L zNDC=M0;+Q{0}D!{DPh(&e^unSrmVUD#y_PQd%(fsD+@xfvyA-I$=I77)fC+96~vP< zZ`+4|tB60NQ{M?E)88wr7<@jbcVf}@?`MLxv65xTwu{gWl6{(k?-_oY#>fhX+=TR5 zBGp|b;L}?k3?4Uq(|0?3&pka3^>0UG%h3;80=6zLx99Y5T&_(gpS3aa)fQP>A!HjJ z<7NCMpnI@MYK#Q4=6zwNE-Z3dYUNdn1HJ_7*q`5slGi`(g_G5Z7lygqRXIb0kJp|~ zvm2dR{LXMU*=M+xp#j&>flw7>D#)w6YFH>RD3TmH!1E5a+_=RymT^XK`M`Ch_ev576U)yfI3EyncSW z1b(q-D~BbXfFM6;M~4RU>&S`yMWEA`>dvM8(i{x=@^Ny%20nP#6lY(4;va_GNQH*v z81XbD!HK{oy?_NketwlTI{JSExZK`Qs<9n=8#%w5)h;3c0NwvQwlcT=1=sw?ZZ(x6 zV2de=JaY3vk$pO(hHPb^T%MpoI4muvZ3XE3i_u!e(OWb&;o?FkuSZrE_b|)H&X=%O z7FI4Q=ODF!S59x%C0!jW)6KJWzU$u)3SzI`k7)WuC-?kxe!k zKtep7VO1OjED|M(YAnLKPx&(*{h&c{V)DTra{I8M8cpR@0T1-TnlI@4X`r-=HN11; zBvdRnam_=2=}|Ivei5=!V}ke=M(st39cdFz9b43QQzk)H$CyBb*zbvgpWCsbe0qrs5gDDu-{JeG{1sX#pS#gz{ z($UW8%e`tcXgh zqDuCge_EBA(@J(qL%>+TcWYS{h2HWUKIWtBgYR$;k{9akuBgOYL$cP}iivF|j2OwZ z7K)Pt49DwOo0)#+g~q3U)yIleDJ!SANMgQ7b1Bo6~q^q00D{9C5*c@w?|F@Ic`0q zkczlAOWT{j=GCZwPkq5<&CM$F(20dHOy1D@wWMq`1RS{IVl`qW3U#!B2UM?IIJ_R7 z_9rWLC4B|YZts^fdk40569v`!jD*-K=E}efI#%bF80bsyIv+@eKksg4#dy(PQ>B-i z^w2|JbN=e*7_XI)i-W}sY1FYY1yKn**)Tord=-a>a{(znU#{;1TPrTzo?bV7w;yC= z@Oa(tCuf(_A;N}m6=S8-_$$(suuopl+0ZI|dr(WjXe4cQ-^ChE?M2~dv*a24z0O!l?o>jMP?Q4@1@r>ODi*X0mvx!4#qKu!8D--K+VhTJ*cTv1fG<#`#~cryi@N*A-qRxx{(LKP z3N_MGl`cAXCRC%v@HXZ!X9prb#wQP6#;Q!CFIpHd+cuGowyAr9*Nm>Y7x%yulq8Ts zk5MX{J(^mk^~f$lEZQqOWP&W4iyJl*#6A8{7Yuo<`yKV4Bl>Ls^-QN;Y7}A;u1GFq04HH{^-;yMJaMXJ=8s*wx-4#` zolR0lqa2MO(<%pr_oB}QdsF-Ys<0qYn2S+_x1q~qPsEigny$=RB-Nrcz8gwUbICq& z<1^V=(NQDs1zJSEfkqU8LY<-O29H91qW5EcCK@+DMWk*LZv(NUoIO4s#=K)lV)so1 zQ}Yd{SK_#T+IcJlQLT#&lvkt9327MaO~+L~Nrn;1DT>YIl`F`%YT*p3x)~ac52=-l6ax6ryS#_>ip(>EFN(-Bf zCNr7;rAJa#x7=5wa(C4V7*dl!eEyRNbat+Q549BFU7u$(Tm&Ws2m-ob8j9S_#t)yZ z1&Mc3hKID$10qbq*DOnqSc&7ldaCq`M$gw%=CR|3POiv4NP-a?F=tE1vOZPPtSaAd zR^c{l&RurAkl8SAcW;7He! zCP#>24XcIRV8#-&CJjv?s*uYi?^8Gd`zvUM&w+&QY~mav@K|j<@Fkn`edl=7H}R#) z=LsfU(w3fAMWhqB*Vod6l~8lz?PGa+Si6o6JxlkN5Wo8Hcj&5hx6HxV;HG>ycsM;#0v&?5r>V#9}p z8oi`S2}?{=b?UDqP-JkQnKa@P0k0?GQp5-kgQjmfq9GFhYN_MV6@0Kt>36hTrLB!p z^5zjWl*g@{0s?iP=MhzPe8$=DHox?QC8Bf$wzI+_@hMpqRS{tuK(rz!?W$3o!S^@Y zyYd1p>G)Ag>Wh-k-4gjtc`3w=H=<2+yU*koZsA&>=pdV8XRPT0q013wqoXSIT07}k z!NOT0t77>0{^1nB_hyj8JHWZ#-LJ+?@Xzw;vr|pvTU3j_Tf!hnIGcCs!Ptw`r@c81 zi+y3>1ZL@a{?hmM=xUBn4yKON!-f;>vdi^zEQ9Oo=i=sOlI1lL(D_lROhG^&*BjCq zx&uIC8OtOaiEM-t>Idk*k5z<~;HvpuSw@P#tJfC{0D$3t?o_O7O-;>hO#h=-DN@|A z-B3dIe$oAhtdmYTXc-i{3>a??)Uezpzt8wI*4|{BNp)%IaKmD*qI^Ev4z9*l2n&fI z1!1n?2H=DlT84FyQiD2uVWcy&h}>M24Zc+R4xh}+)~&T_0kTLupKQVOtCpf@-;l*6bPAX^$syc_7{(O#m|nSb%+-6LaP_aye}F4tSwZBorzCG%j7tc7ZI>nj22 z@10EVH(&$nJ4ilCqDjt#?-dGyper|7MvksO9~!3^<69L4G|f+Tx5*LD)w^n8VJS`>9AI7Uc2md$5L6T3O3KUK<(xQ8dwjs zEM6)F6mF@7--;Gy1spd@OIVz+zL0Yl5oDtz9!qVnoY2CK`hdKnc%3^@vde@ppcEu+ zh>}1~-2w!mHVQB0ejL_PC(>TnHrRwkgm0+<*+b+CYNr$1bV!{oSiLL7%9ZBKPK&Y5 zBu`FI>r$O#mJy`UHPxtr)$FikMss*W*$F>&LF2K& zA+05F`??h1_;Yu=kuzVg@zdGK^?kK8_HZyqH=<#vxE2Jm3YmEwd2nI zeK4~Z6BB3kczb(@Zh0N0^U3LHXLr8>pRT`g^-!ES=%6o)2m6wp$z;^=R%ghpafHqB z7nZka_Tc_Ut6oF6d9PvMaeC`h+l>FH=B3-U%O}>`MQt=&!Hmg+zOc~Xd_OWEawN=t zGt`5cD`$R&gY1M~QT4=`Wy9nx${2+jSN^ca#`?{_N_y@4=G{bD)d(?Is5Ya~pK27L z?Qww|?5!6ykm3#3{qveD`du%NGwtNEKH|ju?DQ<+aZa`EzSHI!1PbK{RtopwERBjB z2rXLxaLyUKbWNFv!BsKB$$OI5-(RlmnElOB5poz-HK8q-sX#dT0vNOe2{l#Qv8>WmQd^p z;K3mRNbB3HPh4>#A(+%D{j@EZ9IopkXbusL0=g{=0A^`{Q1E`kd-Ev#X5{#5PyCmP zjB9&g!v&SKI{W`dcJiL~{juYLdIzaPPxWPWM&SPo{f4ZErlvFFLhnce0+#AL;3bqu z^4nvF?%^noP7*lWu9a6|0*MLFYfF4Z*}KvIX06e$kRIrGz~w8Q0wp6 zopOdEAmq+`usw&`w^ zvsQetG_M0Dt<%^wXlzgd4$c~U5|xBwqJ zXs{D`dW?xC^y!m4Vx2X!WrzKXY#5%q+`-isp{pqn;Zir+RLEOvLm;@MLEKlH!frs# z4F6t(%0L}4mZ~LUUwdnmF2liPRCSH_`NvZG$MlVjNk9svS6vMs_uJjsKxS52FeWR^ z*w(=cZl#NZr|oP}S+%s(!|*yetjuz2Wg?b-y1^<b<)tIDT|3GcYX~ zLsO^Cks@;Io}fhQuaaW^ zqi&`(_aYCWxYNB|ZiN%-2og(T*D;JKRILCO5kWomi}B9#ev=cM&gOQ1c!59J+3|6? ziEx3(&GLS^yu3ST`QcUtMoFwLZ}-m8fzQpdp(}%$X+Xe9nCC1|388X$xAufelO}_a z%0d%nYthAAff-$hDn)7Pa+&R1!puPDJh-y0QBQo()x}h+9!Mzskn=SpoR_1sUh{lq z?4AY^&b*304pNf-8=H#kPa)Mj_Gdd_fLRrB9Agdf#VVdAt&0OilXfwPQWM5_Y7z}e zNPKwk&4fv`Wz0~{U?lY~g>bfJx@fsGCZ&0!cs*nXhgEnYQ$5AU8WUQ^Bz#W997VOI zHzx(U@od-;SDZ}02_>iSFNT^~<*?0oj$&R$maGp;rUc9C^;zw?3x~HGTR+3O1z9j{ zaUnX&3hFA?e7Jv+Wfmt@fp}w_NL4|!d5jm0yOTkPZ6Tx&% z6$d5asl9^d^@V%(G8uSegK0XVX$D{a@Amc2Fdm}58ihv88VxiT#l0y+hhcpO(NOd> zzjl#A-IXhS#uhcX$BZ%Pk`fxE1OiN@4b`POO<}nlFrtO6m5{31A5VKn!8lu}>7!$# z22J@9S!eB2Oe}L(BlI$qH|02!KO`qzS-G*9<_}uSla&(=>TLH6D@XO+5pR=> zVVUSTstcPsk`5j`7qxUHVt4~zJxF{GpV~a->#~?d!>ri!T zlmW9y(R{qre+-%>qc?rF-QqwI@q>GqgMctKkMf7+{*bG4N-cZzj-^ABUvv=|56P)R zR#M|oTOw+xK&<>BNos#T;zdlw$3oXB4~QgirN09I5U?1;G4d)^Wy-i(XnzZHSBM-= zQ`zvtp~)s>^L}MZBJ1g|P^R=tlq?lQY-)BhM;t;F!k~+{2%?U|PoecE*;@*`)I(D& z)X51n+vZg~m|V7WCw&0oh-Sq#K4P@ak8Tu9^(9F885!&M`R}3%EeWBlCc_LJ`fGv3|1GNA|8v{>TU71M zU3Crhtqh%hNoT)a`Tr=fO=)vB`|QZUH*Qc7T@*wb8qSUL%^gGxL2CjF3QA_84na}| zyoS1KUN1b5;Md49aPn%6fV+SSPCIyzr&+g+E= zb1MM05WHpu1o)>64OGZN`fd*`MdKv@9J{Ti_8d#jboha zXyrH1=VHLKXtL!EMT46~xrF`;0ut{HkZ8=s6)?K^6}6AY4y<&GGB`l2%Qc0c57MIL zjl`@5%%B9@!9enA(LC}8Ds!JDDSSJXG^|3~cn7F|vn1jzG#z z5-AW0d_$M|nw)S2dZoq&oX|_d+ajPr(1H3f3=Kfqf+#1_Y>3?lt?LO0Vx>rsoN~*7 z)fl>27x7$er#jIkXg?gd)a%u7+65|y5Zq{7FpQS^T)-VyBa9v%Y?%d zv3ml265*R|*(==<0WlN6seck)1(Zq%CJw{W&pd!fgsR`yPJ{#YI z(3eYeh*Ve1s3i}H1oHxTf0|BNNl3Axs#dyS_9>Ef07_fgjhHH4+IUS3`mD-C$+?-+ zHhA|dQ;qWLbw1Xcf$SW081_no(XC960voU-)Zc>8P&}5%fwuR`YR_L0DFOCK7N;eJ zQwa|x+ZhUH4lcM`aCF|m%25~5Jhgp%cV80uDE-#6_h`26YMQfLnaw49Bq@m7iHgI& z_ls-E5V7U-;#M)Y10shT@L>$#n5FQfD$|U64bxYmCE@ zzG}fD7%L316ZGOgRCCavWZsX#@X-2z#A=A-4)ON|3>?R(IRi>6>C z2OOIv_ODfdX6O^dY@B8wGX@8XQiDvFn)D5&UV;?Tl>m(t{oVF~bvAWqNpN~{X^N?Z zCMXGU8ZuEIYO=m)S`dygnQ2O5X_~fyW*Cm~uMk!bnrZDxi){vOP_aQ$a&dyTsp^;6 zE9D&bUoWKn-Li2)OH5LJzoc!U8H7Wec1igAr8ZOG8T%Qq?XtBZl5^vfbrpSZ@iY%s zWGj8r4qmUZUp36)3A z$1@5ean*?~aZdJO*fjrCE9TfH2fF0*Z0aG8$}vYPn$=^oDyr*7bf_6?A)d_cuqdkH_7o;w#RGI#mzAZVs(q?gsCWq{APBxpo+idX3vS zTjKiaBlieJI4nxk_q1Zf3oA`SYNefZO=g8zJ(~=DiZUhPtP+)?^%W>p1B9a^{y__U ze*q&wB@jArVhr+$fgru<2Ps9WtQz+ij+lK>H0Pfx_sIML+3L~SNrLo@LKJ(j z#W%96ephk~cbr4&nLLfvC7GIj2eFCpm$E!HVEq=<^g!9hO6gJ*kP)n(YLAhk#bq11 z#8x+WKT#X)ttyZNFV{^``#2odogHvob)@1)R7Rp^o!u<#m1j&_REvF@q+<~vfgO>= zGCtmErMrs`b2WcW(wXOj0sTsP8n6$N+s-W-(@6Hc8-C2yNwGE4o#E$U40Al#C_2>m z(lryG)xyLoS4d!bn(d0x(u`4CC4_`OP-0;407>{#NS3?nCyB-{S0&~ELpHcs&cfSjML zuoK~&cHd*+g^D|4T3K0{O%}t3^%_zdV$QJ}Zc^2%PgDsC6LD$A<7IM1Tx1i+I%3brA~KI=M!6NtvMIsQftaxyi6{UV*Vd zC;xR)F(JxDS}eHgQec>ToY;7IR;?}LzYJ}$a->P`Yy*=i<)9MJNly|r?@7rt9vV1h z?IL4#fB(9h_X9=ABro+DEKL&0cOhefqA1{yj@Q8ro*I!lnJNIEZzbT2GYw3)_Ldlr z1pl5tQdkD4HR?pgG%{+R#*JR(*XW^ThU(2pV}8_7LcC^o|GF9sHEU4?EU*k*S4W2A z*n9WUKa4bW8vCDXM@0#v@8S6hwOOU50pZlh7(p%B3mcv$*d=GNBHWdYcM?)t#9;$Q zj7OUR>p8CDerbad?5MuJJP zhC_jQVK2+ibYT1h9MicdMma1qxQlTp7V8wlv2&92S=loxJI!XoCsA+loUa@4{o53j zjbVbE)|jQN?uy|#-=ZQlnF3qb6`-eBu~U)Bavr3dotM`tI2`Mzb$34$@|8Ab-2I2J zP)vn905i>K^^G#QuE_x&?GtLntMy`|U8&u1Rh4;ntUCH9H5up6K*mn?p6t2HbqTiP zdZ&_v>`cfmq3Gi{?|@^6Snm+JCe3iLvz*HlL+)?pLL-ppaxAwyO6WgE;nA^!zf1S* zP5-6=Cn%fv3vkeJaWl`t0sO1K7oq66n4e3Cd$;f!0oy(Th-djO&4C}`J>vQKUJbEw z@je;A(k(?7wel<5nggduBr_}}H?`_J+nPb9NhBvb%UO0pz(PLa(M@-&I81lPdaF42 z*fY84+3|BjMt%~GwXyMnHX|vzIBri@#hg#=A|3fu=pP%F2R}k$uyL?o*##VchOT*N zU`2L))0mhL*mNRtVz8=#$7Yqix=4uT?KEpz4N9w4R2e3hDW72Pp3d{zwT%y(3!*lg z7vnaPdF^dQ2!tPLm5Hi~VBFj+8Bl6$Xh<@!Q4&lw&aV&fQbh}prCA1$?D&w64wMU&d=0wP3RzjSzj4ZSc;H#Y0Fk(jOiPO<+BG}A z3Y%+{I=&V?+>4wqB3rtf>)Ll|bo{wH-JUwuKl5F_zHJ*y6yM_Ej@*`U^*ph6Ap+)! zxHofQAZ7jC2@=*^pA8<)zu4J{f2KMttLO^}wfq8WE0fOItvucJVEIjN$Lx|I&p=dy zoH@l~e&i*Y=8abs3%^RT4i8~=x%F3VlWw-~_T<(Ip*-zoRP`|FvM0b%L;eYy-7N5c8a9g6Ire3_ zOUkQ8J>_2se@q&6;;C4y&`Q?j=|oWbR3w>3WLt^v1x0X69n2@nGko`-j+6~U5+~=I z`!s#yjx1W0p*N~k$W+T_I`uO`1?uZD*VpbzGz?*eyvw`Qemx!O^-^1HZIxR?QM|;b zDSdQcCiMw4&sSmm*}za+!joOG*{b~d_1Zjiwme=9Cp=W0etkd2v-x-(oV9qq51(HS zpZj=3Kx0){LO~<&b+(x{)u<>C@PJY`H6H6}zZou~yTt(4VR%Ags0Rb!fC-;W&9aBn z3iqY%Abb_SOx1Os2U7U&v!qhy4OIn7+;kF)6U}l7e`IdI)7O5PM{cn(HSo)^a&AI_TxK| z`6rnsKJB%Alz;u?2EQYK^QE2{Opc$2+Uxgy>T2J*Wq*=ozlzacKR{0id!K;L1$25- zOg+r1UL=4Js!!?0VkyU?+;~GD9e!+!gCRZ^rBiJYdz(`(s=4|dpX0(d#XuK(gM(s?g!mi%i31f( z?$Nap)c3BzCa7doDe{XXFME^@r9F2ySicuh*J_ktdE*Fvae;%}x&bmpwY*}rXy(|2 z4ig~ZHkF_oS($d{q-kPY?9eRsY}`V0xkCZjnGLq;QQeQ`;pjgxk^LrZKD*@GHkNCD z*X3;N=&v4n*}lS<`bjTrhrD`sOk~F}YFoBHYgoD;skQenQ!F+ufySH2=@r34c1$y&6cM{Yz1lEadlQZ zdj2}E$)imf=cI8=x=f*cD^mG8Z^Y^5bn)K6vCc^)!*spIX*Hs@Z3GA)TeJcWWH2MU zJP^4Ze12+cvd9fA@C&uOP9nS&v9`4QE8)$VLKjwMnzK-2pSffX*xQ9 z0b`Z8V|&Bve@p{evDUIELFeyjuF)-cX`X5@$HaHwFkXm=28c2r>ck}|zw2U+*6q|% z(rAxv!=4aG{fuZSgG#r;Ib)4s3jGz)RKk>~0+yg{N~flip6 z4^oDhC3UuNTA0wTPAAB{p&wz> zDj=|jmMGwSoFSi`Mg?_o2ZW5%Vz&AT<4>;UYTYj-%;{o!Prdq7M+i3J^?;}-AWMk; z`{%ZMqo$;v9lVr0Y$qMPpUk-0?*_a+&ab`0oBH^;>)2Z;b`?T8!$$<1$MoV|`#55%&dl7DX9`him@ z`2Ms;wOj&^PwAyH4PF*iEub)X+6#BuYa`BoC<`Xxg+j+%;Y0{^3_boF&ZnOx`q2Z$%jP#9x%QZ87$cutdUSw zKYKVEC6RoryXD45amAq`ES7U}poYa?{NW!ngD@P_3AE_V_tw;+h$wZZ!0k0+>>BL#Q< z=o3-6jW4}PEzHj%=S*`Tb`M3o6mMi_bRC@)!n``5!0c!>1zU^29Kna%U0vvQuq9S2s?)`QGzH-mu=g&ZQHi(s#msc+qP}vm2KPRo9fMUf73J5 zyV&F^Bl5<%H_v~ReO{{9z`_#lX8GQYfr*RG*pV`+i4%M$9JY~`=ZIXGXY?1(&nLKo zY`*%NrNeAj{L1o!eh=iJrTw*3GHh-nuvp*{jo$AdC95p2IC=7<+b_ zZDU>gk3Yam)a|3`>uqoc?-N^5eDep-idZ=56^}^$6dW!VT%r#FaoIKSd#9l$KL2AU zh09Jk4Hr0?w4PG0Lq-YUU0~|$p3#FACmp)YAo2{rRj2Lc2?#d}n5Gcb&N_Bp(#_SV{bIsQr2pY9h)Ddn zIjldfH!=EWJT(hk4Nr6s_M30wNn)X(G$`z?${_?90m|elQYx;piY%~dC1*-v0)gLI z1Bj~bbY**WNGAZ{NF`Y~eg=URZi7(~*Nwk)$=pJqRB+9M@qs~Wo0D8R1a~4Ju+lZ; zb*Q}RmGU|dD7)tyxv->n469=V03g|q!ggx#S^hftULdIyI~igJm%@9ZSin zl}%mE%J=WgskOZ=;Uq{n=i`ySr|C8D6T-K4==K#fV!7#V-OG2S%V8?(U#aiue+V!4 z%zaIp%TZ2y$DPkHQs3SI@OSAZO45fapLr>97okFQ^{aAD)53#*iL5CTf5$hS`8(AY zgmk89Kz&`M71G5Vdd2ONY48}T3xfs1$_I=!1GWAK6->XZg0oHk&I9bK%Ynp_!1PiW zs2%soZ2bsnKm|)04sO!4P{`-dO_?r1Vc1aftUTw29FsR&a8Ob8@=xAQ%+yl`9Sc+F zo*L!g;!82rjCVY9^#`m9U!i`{>}cOj$8tdfr?&Yi=ij-SPo58&VP+u+F<0m~8;&-I{&dcbCP>@`7$ zc{wCr%*;njgh6RGNj$%@73k(h$f0qfzivvlqX(9Zs*Y?T+=^{Rc>l3!#!pe0B$VCo9Pv?I~A^mZR)O9vuc%FbRHjJ>xLvW3=$EID~nbtD4t=S3WY^0 zJtKa#2?9^$F;V*2iq`1|^}=*~FQm~=^g=_GTmJoQWh-Q3`Skf8ud=o_2<}5MlKzu~ z6(T?XJV6VIS)xPLF`JXj(_V)pBhhO0GNF}G?x}zk*c~*wBOXSu$%(CdNbhwteZqt= z1YdWTHYU;EVpUKtEc3o^#eIGr7n&y_)UXZYKqK_)16Wx{#rR+fYb-kcqb5u=`*+G0 zt$JEc?6vKgK6h*NAbl*!{NA!GeO&t*{VMwHXR|!8Fmx(|^h~3!2b2?2=6WJeFG#hs zT(9!(Ae9yX#N+kXfonA(fiq+ESo*0(l1d#Ro2d8v1R&i>$AvRS}S_L8XI8hQ=8oy zU||Z`&EF5E*cqSCeQ0TiebP-qzkU#z?BCz0>(P?&Z#<@e;FK6)4qi1GPlbwtL$>WO;Ly zvxF;~>~h`?Pc2yxumg8+3ns=MBuRgLFHIX-;Fu3xDxs6HC&hHW5-zSC?etY)F9l^7 zKrW;kk)`wrsK^h(R4~Mk-EZb5I^Qy{i7ar3LivQhfIpj`kELqAdD$)5YYC;kdc+H? zO^Zbj?DbVCjZk%#VMMxrLXChER34k;4ny>DxdDZia75;3CsbWIxbrU!6}Fq*v$}|) zW;r3&BzXkmt3l(RNtt+K?F9DJXhO}vzO)Hw(u4COyGPQ7{83d;%&v}^QW&)GOOcTc z+K(PJHG~hw5p6l>>lk^MRew94=p~+UKwsB_Xl1Dl$=E&a76yoDEmt38X$DyH*4#jV z@OnW&5F#j1U1N|wo1f#gzx@(gs=MevD=xbSB{yQyD*nqc)ER#)3r?0USy^J3opCZt z?7VM%FY#6EFc_ipE@G&FWaF#e{tS5 zm?ZpzBTsR7$ggN?di8sPHJ)-3EImk2*7!(rqFqPDx?rUxuA*iRIc`!DQ3Cmeq&fnz zKBNF@{RCtp{vY9NwYc_9#wI-TP_YyC0y_9v7gwlOU4rLSa~hc9RXVn~#%%}}p7q>k z*R9E%abBr4G}sj#xPm|6Z(?oF5j0pk1-S6a`I$}r$c!%VE~Tj)Fb)b{JZs7Z@;@Hh z!P?mku=a2qIYL>*Mtr~CSY(NH2WvaTl&AK1A^K006P@7_19>=Tn3N2ZVBVv?fm^;S zuGqNT#DEGp3&puGPE*keBC67VQvA>}rGXK)Gu}AIa!hiIJan|sfec%xj?Es8??*dB z1%NEOYTfSs6%sOf`=~5r#)>ODgmnp4T{hR1fztELtSJwJ*HCb?j$!SEMs#-FTycu( zNAFIiavz^a%PD}OGv(~=f|l<;g8AwPoUW52+X9_r5tTSoQdDC+rrHzi5=qj-9LbEO zY815GS0aUOGkc>+C@xY1p9zR#+u-de##xWl?NqM<jwaRfUPLnB9bZ;>-Fbl&s0MA&c^q zNFL#-qmDXwR?>9w%+*3$T~#(^6of$vo)7pDg$e*4heiT|k2DJrxM5p(-)+b_V>LH6 z!iBX#(O4NBxW=1jr0-TUatr5*0FciB1qsj>J&bOiRfgIN5B3g$ z>j`VL0c8aL7 zF?}YUa$FKGTGxP`!!-X+PEmTcItCSExfOFOwQHc+HtmK%^T4uc%*Xfz)8}T7zEQYy z?as#92!aLPe?>kT2&!Wi1gwS%qchi~VfemOP1WqVlHjyf$MQY z-Nn;&cb79Y$uu=V1-D3Aj(5j=EZ@b94b6`EtBSYEym(-^8*C>Gm33G~xzmjaw*;2& zXwB9O^H)8~0VQj$*u6ZpO?v%7rhR9pJK z30OLU$Sd}n2r8Z=O?X6VeBWVJIVUL&C4ZcfkM(^+>!An`!f`z-PWP(}!{C7;3KxBl zjZv4|U#U>sBCHA_=NJw#0`W6P(AMGcu<32;5$GbKn#UEAZedc1`#=jZGDjJN3B-+) z7STY#*W|(t4zCRmS#l`5f_xK6PH`l0*K00(B%)yOWMj~lYe~eG;+!O@nHTgOZD9vJ z-{0c#f1WuPcL?^NEqKRGO*oMGKvOJ66HfLV83?FaNm0xfiCnWfPQViB(k(q9fw)-q z!*UJ#Xl4Nh^WkoXCL3&0*`3X)M%I5BC(eQJF`h_hq)x(2;t0JRJ7q``qiDKy8y6=% z)TSbUmmLZ9+Fm3P;9jHhT>Rlr*f|KleM7U1R(5b03~2}G0P814Ht$BBet|APhrLJy z$($1;ypPURUTv)5ajd<6K(le9D2%!aRJSCJm;d*~SdlyV!M+Uo`MluOKyCy%8VEX7 zq(LX>*~*Vqx%1#Uq+lY)jwD1qA1PF_Pu5oG-S^{@Q{&H~oI{{cAPFFbED7bBV>G}_ zQzKHsT{+QF)W!Pa;^eETcPfOb!&CFHcSMieL7T@>u&1nCp`OW=J)306unL|HU>2;j z#s$w(zKoilqeAyF<3-_C~T5O5{jlz4Ln@=p300z@(E(u765 zjiSt)LuYO-wapS|us8vU4mlRh9R*n1BjtU*SObPu9{V9NTglSjf#LbdhpoY`8cUu5 zN9j`$b<>OMxmOvY!qF68{a@Gk4x{YHp|^x$(;jxwHq&OA+6P7U9p;Sl$E56kKgf>H%|ynYU(n{G_@3HN|`yN)8`U^bUdDyIo0* zjHATL#@}R(XxH-Wa0R&8-nX;9zJL2k;{1k_L|UPtE|=3u_&!26u0no)W3=PlOhe-C z0WbH~A^fNtMYItGNz9Rr%KDwBVKpGhJ5TONXF1K^#SFm7Q61%E5wkgacrtE$==C!3 z<)KM_QV9|oRBK2jsc>k)c!pu*T%?zqY8oPGk2r$M zpbMEU{6r{1tqnj%W-2-*cLo(Z;LF^bwf}03I}(5Vk+TXF&Q(2b6S3Da7yD~6>G=HQ zj{F$c$NT+^zjv9(&kWl7g&OSB=xbl>+G7BRDEMOF;y&^TBC>aUjYIXXLA$n2P4cV& zxU-1){P+_C4&LSUIL$dWi+A@m_&u>YJG#V1CYbu9;t%iOyKe!wriLi}TLhnbefnCG zSeQ%9)yXGhAhwp2kJLq#0gMXaprpu2{%~0@&T<3o*+?JX`>K6x-x~xSurqPq5rVoJ zt)>k^_iC5)+jQ$+oeoZ*B3h}#g{BxB9qHz{p+YUpgr}svJr`;xw?;dAH$f_{-beHK zd0+QHdRL!wFLT=d4=w(a;-+E%Uj+n)1M$rGm7(|qWxVpc)SSsASId(bUNFJ+4lUqH zdRsb&6c5^nnqg8n_xfGg zf)??+w*`Aa4${=thl?WOvsR+ zV(N5#8H!xB_)*2fFN#|i+=58dC8d7+gXMqZO|u!V#T;ofDq&rgaG%9H)-*PW;H_NG z;4IcyPK#uUIi}BZSUCy&C+6>Cc0*Byam+XFFt*fDGL8uF;^lIEAe?#D)VQAn|DAO} zbv;!kL-Y_9#N&nB#(1~a)vs?Xdc{L`dwDKSl61Fv3%k|JoY8H7u=G`U0PVu)YX(r0 zeM)FtpB`2HG?Uieb;PzAx1a2TdXJB|hM=_VKY8 zcKi9cevRPg`98l*9M1X_)k}F(frj~l7f+Ztq86-dK+1Fb>WQ0sj3o^9BmwME4M5GM01nm(WUzM?tAWv<_*G1K_?A*Iua2jw7 z-Y=9q>sYsGI$Baz#O;sO;<12+LMST~9%3S$hqX1EQ=($U5sz%H+Tq6zZpSCWJ>}Xp7Vc2Hu`(;6xrLA|1N{;EsfS$DYu~-? z zBpapD`nzW;b2;*BdRD86bc5V)keA{ur$f%y>1O$t5+s~G9TyO;JXvOjuUYa^ZEOh9 zlVbLW#IR7M!fM54OdLZ9Bg24<85HhrSA?t;=2CGsv>IpXI=0XLjrZUzJ^;FqZicU@ zN4cp`F_OKL(@a#xg>y2SAx@=_6;ynLR6}S(NhF2bl1Y?x({X`m>-h*Y5r%Xc=o@|k z8^Ef7tbN~yKhd(;kx1zPH$7S77Nou{X&vJU5nZ|7aA+5z9DY!&>sn30bXxf+5$7{R zc@Tu=D5a5w%4K&@h=>gCEAo%gG1nnk6?+*lWV@IkY@U+khk_?+lkFs}@YhNiCI7M- zq+Mw!X=*PDh+Rw4r$Z6!j2D~xAeHg@E07uZ;r5^?jA`b!a(hOzS1K9M;qZ%b7_UG$`kkDYQ-40E-r?qQK+mfoC zHW&_zW&_WMrHH08GiL2jWMSd+oZKuq3T@AI+w0oFcu~Eh)z^!e(bt>P$EUfH!rbQh zI(Zp0;OgA-oytpm@Z4k@u=y9hZBIiAu@eQ9bznUck=Hb%nA29&Z}FU_3Dc*jVO_VY z>0Qjk4m>txwCIADnNN$tSm=3hA%$#Tr!x_XB32uE0nAiW@MGgxMZ$4QE1r#F9cqEP zu1qNMuvB=%E?IhW!e*m(rtI1bvnA(m#q~$^hFz-q=K1tS?QG4p*?Noa-@5A_@8+hA zU8R3$0PsxqjuBQamVoKNV0XoWWntg6>lV*kbxbO{W2Y!dt6v=>_QDby8;@T#P3+RU zVwyT79E(YRFp?@NPA=|9QE2E1*p!w3SUI+pUp6v6l|2R^G0<~mBR<=Yq`H|cd-}1; z0J*ife}oP^J{SOACXHF0d};8dOASF{l;BT+*KXW1cu;OB`?JywCTTt^8ghcT$jJWm z@v6pL;C^s7ydXAhm7^0JfG{+Y$E0l7`J43sFp>+vtYrT0<=TH)W;|(w2IT0js-kRV z+I)6*t1G%&JKa1w8DP2>kQ|%UiseAUul#PmO|SJkPP6MbvP#_m$7YK^w&1%+(k%hq z+3}_)ik;K%=$;){!y+s}Vlsi_IT=Rl_68rNkj2s@7vM_`QOXrj=qoTBJ2eBvgzA55 zT4H9FWa*O2!;;JBih1T?)`UJ&;b?tkNCi69HUgEi-Q$^k$%eRp#}Ab{?D72C)G6e; zF5+vk9gv}fdO7LJ^qo1*4o~40fh#S)MTzPNOL>+QTUi7k>_s`T@wmM*Piosr%9JB)*{{ z4!$mF!c|J8e?;O9AeavP>8Q(_JPkB3{Mr0)~mKwLl)#Ao8wR;6y&o-l!yq>7w1zWsA~ zjUK}}1+~pf&@p%}#)orKbZmOC({)(Q|8`gF*=%qMTC!;1B-ZC`*v37-I+|PrY~SqA z-M#x8gNMv3#B|5*+EtWequSDyV{Z-(h7@Yl@Z0(GYU>e98{0=Y^%*hObSp1+cUN#9 zAOek}hat=H=EqJcXsXRsMuip#y2-fn8VRk#*bl3-j~gr6npOkb{hP8EL2!@NmJkE3 z0dCK%Pp*UZ7pR$3pjPSIgLNK$RBDX4TNs9fQ5sM^!}4x`ygic_66b~0;%2R=D+woYBHhKO~B7glJ;%EQ>?Einihn=0ik)eyR zg}&V{s^I^ydMs;BN zAd6NiK!a;ND-Z$-p$Z8-twF(+qr5TwS;&2Zn!c)C<^G(lU6P8IX0hM-_ujMHTin@c z)lYTB#G$6!r}9Qaq^z3KvYfoH7E~!0=oTD{N&9R~T1t zLnQqJWlwcSd4gw%Vr6|XQl-Zko8cL8Fh z_nhC*wQF-BGghS2Fb`L_HQ!w|zDlM;us~(Rlj3V`n#-c^qu4B)tg?fBsR^)SiCx&ge5*K?h z<7(Uo8YW)vwYFv5j2$NH(Fyc?Li^S}zf}d0x}E5Fw{~0!ltHGLGu*n{iZ-eRrHfX4 z)UigSYc#Q7VIS^^;dzDOOhpZZ{s?%#6y?bz7y|4t?hirYtyoEiT5g4Li)h6utlitY z7w*ZlkiT&_ew`Jpo?*SQ=46spnVgABQr#3)ajf`^E|R^CfjcH4Of*k5UXYu&d{L z4u;FCT`a6)AHjWE3J=zH=dmTRYEB#k)TVWnNpO>mt-w5P%~Jf-F9j7 zx~RG}b%~fjvC_Y8pWK2tU|N7~#AV?m974Xgyr&77?^3EGe=iBimG^+5(gW7iF-|v( zuwfkiUM2#qu)p>_YT}TOJDRgsy02r=JZ|>p3yGDT1FR&6|I_&6&3ZjMzxa%8nz{2O zX}U>u9)|OWGheq(!2P_8hf)ZVB7CeRs6!is4*%M+iE}(shW(H7gk(Ct@glyubJ3b> z)p#o!@T>Z4(T_ZlNbS^Ow*c&;Cm_}vGNhGqyo3QC0^a}&c5@V`nm&!>)jab%igke< z`dEf}ZT1+iBD$5POnhN-i5{^iwT*uT23+zGNXRq=tQ?(*(yGy~1w(J@047g!5QW8c zp3{`%P0=R?w|rCm~528-gEZ@*1-he}{Ow$MljtuhTk zwvm)%X+?r`aS74sqFm#Q<$)$c?CAWDYE!QpFH1OQb^mqgSFm-W%!rew7{}-H#M9I# z#R)vsjjv1uhMSSi?4rnYL9Z{ z5_a8BAjN95_~`Zqxf&gTx4YPD=r_9BQPcS=X@}lC(LG%}KNHRPz>tLkuHN2xb1$ z;*#h3{8R=z~i7CQ^87r(a8po@iYYw`Xlq3 z>#4?Yi{%r7GHhC@qLGeJ1dd8yeXNK7g|~&C45waJt|m?f0RV6&3;@9KKYrf--&Vfs z>}~r+HI)9{M|B-jdRi!1%PL9gY9$LO5o60`5Cr)I>s=CcS({ncOjlH-tT%Uz5_pD) z7)aS9GI#87N#W=izf#Mo#h(z}Pp;THpE`b{Rb9S4 z#Wt2DD{KR^k1g{(Dfu&DkP8KyDJgMF!gAx33ZQl?361S5N7M*~l>!jYMi5qmWv}5wwEV9;QyfY|m0X3MpMZ(qqmpKt!h`mw>RbT?l za4Gd`0zovwGvXrK_l6po3^uzBwq#b}{tKX+`~!K}HX;NV?z2|I^2ZUQzTG^eYJ)PA z8r3TV)*knk#&(q&+ME%DW2&NU-2ku!d=c zY5z|wruZ!?09{KgAHkqfOvv-l&=pTGR{f#7yYg))_fZhcBWYHJisOM1r5YvbbTK=X z3-#2x<$;)m3#$sPI-2GF2PzGn+EnW2NheglP7tZ(ti33y799YDS+o~mcGVdiKse+K zW=K-&4J!mCLKak{#9ARl)`ajdvNlg4=)lxQKvv8+h;oHXqHP^TTyTpU;5k8E6Nu2| zBkZ*E^7JX6K$C{Wq`M33OoGcb<+im2kd|!$Ek#9aBouD<%*e(qgpk;^ON|$-$Vzxo ztKJ#~mqiBuQW`Mt01LsXm7U$!_Rq^D#eopoSx zm5O@pGPZuI8ktT_(w=jDk54T?cyqUkcJ^9!_n%9i8=tq=qJeGw*Ss$tcaPINH+VL9 z-+!ieH=jN1_Or)Q&mUBqDmyN881F#!E;K=j1`x0dv02A8on!>UW5fw3r`P*3G8=N> zb-CBOzH-@URgv zjochCX#iA(x8L8NQNN~^eO(8#|3ygh5U90HNM70hdyX3D0k2*87U%Zz`*yiI%zv?i4VHb;au@CvmW zYG%wggEXTYz;eayJbOcC++&&9BWtQ!WZ>Sws2z6#usNMkO(JNal}NhTL^7Gx7ihdc z*BR2#s3+r;0gan{D1uTd3VIjv7>uFpNfJ)WaI zKjw~bFPIQ-H2bUB+{JmYf8~stO?hC&Am2sYLiGCbr>WnEDACA&tq?*WQzLBGm51xK znDA~KIc0CrR}o=VRaTL@afA8GJP)l&BL69RG_wX30CeTt%?%bL05|pYiP?ym{ixcD znn-zNZhn6_ckJ<;~$q z$xx25W@opc4Qk(VcPqO1l3Gm@|WC zJUoeRG%?W|=swqr1FsR>9#e&X}MY`p2c zb5c8I0y!qpl=Pn#b?>CpM+}Y=4w5EDvqljxv}D?Mb{C^YMKaq`j}4AVx=6+AfUiQ? zrv4Ei`|WNl71YOC&+Wy=>P$0ZavM26yJi{wfsD)|9lTP@NE&;vBda1fd`QQRp~<*L z;R}W`ec>_%Rxt%m)9$eQhDCY$Gq2DNqmxD4#M_G`9&)8)HayT0>VaprrE*JvRpg?7gV zBA@-)3tjC00MWc|UF8!vQkxEi3U%$rs0N4n>bRiHL}h7)(oNy%vehh^B8ign#QC5^ z4dfXd&$u=4{WNwz#wvZ{uD(2J1pz4tbYq&qAuM0#S1IJ+OCt9NgU>KSP z`}xN={tf+~D1qlnaP@y@DH}(>($e5pTK)#E{{JFsex;>}sf($x%YT=Z*4W}GeV58G z7b8%SrV&czNmwY_vLZO^8csrTQxD3eRm66f%w zGE4Qz8k3a!gm$lXGnQ*@$|y^E*FJyxUcLI(HCO-UP!~8`BzZ8)g|!OAw0wk}Vq3>> zbux`23n|8>w@%@(6%T`VEmLivp}k48*;*m3lA~dDq3uhofw1!04Aw~~i>^YxQaURd z|ADQM2y!Z>Ql}e7U}h5|<`JetwzxtL7xZDthZ8ihLXs!AAWDki2{T$Hu@DNuJ&Gu- zp2T1;!lFs6NPFQymRsC1-iG-n^bsE8FCP2Nj3= ztQGdgcYK&gcd}^Z7hVerl-UYn!sQW+CgwQ;aS5CVB+rJrfU6oe2WA||9%n#%1}*z; zm^(pD1J|trV_>{0bZUy)*ksl7&_Ofs;Yg)c( zapSssVzhF#E3CTytA>i0j&?+-95)4*FGEExS#4mWE7XhxMng048ZHUG6uh#vj{y&f z)wmhlHHY{Bfv>1fbRitz1RpOfu3~@~j`oolVO6r=wZxj!0D1-;tBcPRVb%3T#~2LH9)&P6>1J-r%xKR|ZJ814es@9VsF=gKL*8MTduoPe5uRqC549L4J8D zDdeJ1xjaV-#v1re%4XX^)+AA}y}o^mxtiHU4c!U#0nv@XRj#gR(YkzLSKCjv5smG` zR)eEf(pgXurZdV2f)CBj#?=ilq2K4t()T(vuf99;%L2U|`DU5EuiKr~_kO=0SB3ItbLo{dK5ec-JpDO?a^>-}r=sHl<>2tJ;}tyWbuK}ohn z1c&Z(O3%N4e=H7ENQ3dJX>t|M9W40V!1?UwD-WQkQz1D-z5(hvN5D?6paSkGlj-%8Ca zuHrFR00>ztCPNgdB$$-}q(P+{ksXj8Ts769=${zl883Y|pkS|aS+k60%2oQ-B?6^V zJpkt6>ucf%H;sC8Ku!e@LZd*f+W&~=Bv+7AQ*cKqQ$$9_RMHJwEHyO0R9_0U%(g^tANcjS|hVxFfp^Zd2Gjs~MhsMP4|ncoQhhB#Fp@x`;MCb5`pN;xzVy+yu+u{UQY@4Loi{(+#gGs<>tpXjPBG zq>AfQkigbZSf$MrFvwuVuUQ16d3YQJk;O-sNiiHFDm{_qphkLU8e4;dB<+ZoV2*|h z^LO~5|1cUwuF2ej2X^sP(WCNsqQDu6rTrNEgcBb%@(X1bri>wipL%D2@c}M2+RcII zE2tg4dpoNWKKO&UUc8TB@6;EA?@F6%@g~777giAaCPEY zb7?_H004{Mit#^S7k`V!|D_W-qpfd`J%;FaTBcD?pDGxYc!g4)2w_0NN=Ty&Bsq46 z+;CJg9&%$>=O&)iY(M~!_*4M-fB+&9MMMFGBB>V$axJ`9e4Z9za{7yjZhDRD^6GLE zUywUQKQ}WoH+S=8ZsMk9x8JEQ$U2MnfX<60C*0Uv|K6_Y$?4I3T0xnG5cOfoY@aqW zmI8|3w98ML3uN?OgTzH-CqT>WM7vkyAS^%LNBolDfPG@S2cFbvZtDrMa|rMpthDc$ zRHa!%;dBqsK`XQr5CDgr<7|N>dPKYz647NbiZxfhiiOCWcR-p+vRFUi!Xx8aV;Clg zHS`D9KP@B}G&Fi5f$8}+b7sr3Jse-NJ&DV5r9oqn-v}Ga*}11>;J|d)ZIMJE$on#s z^3k|*2&e}Ki@8IXZoY?M#NTg__eW>4yPGZ}{4)lNQvc`>>NB{k!2;pKl^wd^TXHf` zDGIpuTS7dm!Zv!iJd2jCf1#-pLBCv$#ZRB7{N?LbuJ0E|_#D^yeaqAGZTmdez023~ zWsmTiR&H@v5rZ}ZJf>IgULb~~ha3{MW?Nn%66C*{HcER5h`eRQ8kyX=2aJNoFX3-= z)es)TZWjH#SpESx;Nzumu#OlBPL~JBtkx~N3T`Iyw0FK|#_awX*(YmbJ0=3(WRGBI znCtbjva7i5{n!lj5|*1uA5L%9apM#wKLG;!UMI2eR;=jUBmiMZM+&P@JSR!t-!@z> zn$~QJ@L&6T@0(sNuBTVNehzhiTz|U(1*`MN$9BLY-9?8x`!Vk;N#u2d(TRrq}0=Uru8Eoyzc%M_J_E}CP`SWDOxND)u_oRE8)x=%hb zLjcunFRzE9J!0E;ZEj%u0|PNNP(<4DSS{ntRrdUSzNb;s&2LHaUss#Ab$tJj*V3aa z0UcnXb}aLQF*Q37)V{~cGA=}0FLJbA-)00a&uB2?E3EUQ;NA$|LrsrFTU!MlpZWW` zKlX;AUB4&9`M*EwW`7=bW^cPcA3yPRzu%`WKceyY5af)?d z@pApXi7DJ)mX&*^+VR#c8AYkn-EhllFS>>EKnEJIKjKU91wy$3HjuzP+%Aq?;P}Wn zvV|fS-PcBijley3ONU5KM}a}wD!J|1tU>jU8kCx~KzZ3D#H6=V&V_k8?+m)})V-IG z6a*wjLUc+)!gm_M9=tEEZuPSV;)Eu&!FgDPm4cEfeyTVW@mxGPkzt|~(f1b&VIaE5 z2?z%j2rd=CMVAYU=0Lc3PZ9-H3DcMh_(%v-rKAML;LxB)J7yd(R3V&w_Ou#55Mbj8 zGP2a71Z-cVLMJVh14z&0lj1}cq9P>_#>vJ`pH8xCQV~O1QWpb!wF|XvuxjH#2^>K& z_gnFBq@Dd7{ZY_3rM48{lasX}NsLk^6J~^A{3)rlHaLy;`q`MQQq=2+I+h7gjr?TT z0OI^pn2J`abxp0*$8kn5i!2IJFhiJ54wT!$00l*zMjWU_*L`yWY9shTFK1z3=@;mItUsQ*%ebgD{(-vf1N-VgS;h`U2j8E?irH5eo^ zu=6?98itwiGc5m(yHO4D^mwfrKAeZY$`@DWC_4A}7auC>^fm2kBF2TO!l4RV@0lLm znr`NcSoQ41Yr~BsUV?)OeNtG(=*rPx#TV+#cd{1xoS_=o*C?hAyvJ+6r#{ z(ACtcg|ZiwW7wp~z%c6dr`{Pq9rt#dBV3b3Bw4^W$@aIg1yV3|A(EH|nv$}ANs49xwoG2PS`%6}pVp1u?P8kk3srYKb*!`{9*DVZ zxlJ`}YXbC%I`eJ&$}?q`2i05esWD*;e=cp^ihfL)DHTAlRzb+HQgWk4M3fWZu?!$V z8A9S2hG3xz(Afj%%vD6fl{++A|NWU!S+Ok~T@LJPk}GAgbXML^!ofaQb9t;&*<6uX`K8u7 zRE0ZTx-awU^WPtGSu{!xz*kRY;a$EHOvI!-!ho1qMHU$VgWB3QF01*OVTJ%f;` zXgJkw4Y-BxXW_@_>0v#*1Asa@sj=NUN3xb_+R6n2na(iGRp))gs6o2b!3Q=87UW+sVC%ZIJKh zWBt@=6$MGE&xHm9>Xv5iP-D^(#avXDioph|AK0T8f;hII%eu|W04Pao%c-?S_O{Ib z8dx+H6f5mVWQIkrTlX^=m#$!eCU-D%@-;l+v~O+7GU8T9;XfbjqL#jqjP*s~?+`gS zrSG)Foe|IE*-5%A>M8~4T$GN;iP_ET?>ML;c@vW&d8dpS1NLJHvr}4*g1Ny+X2pOv z&?a7xdWaHaUYL;8FWCfVn1Qo}WEC?7wPgObeEA0ivmlDct8g=Ki#H zHCv4=PSHKo8SM4=%)$TUyY10EdPt!LbpYLBsVRtQiG`+$lE&Asp;1fiKjmE$$7ojQ zGEVQTFg!x&3qfkXt~|o?+_c>i3k_14V^%Q%ZVM8a3ABC4<5vRG9(S#`^q(9gN-u&F z^(*J_i;;qv!8;oyWRO9B!VzJq2X?KG1;A=fI0mxuZ(VP3fr()Q^0{*D_V7*tt#q7S zWoXlt*dpv0AX4Xc6bI1to5x*kNu4_WO;-Zw8KFyWRa?o(Qts95>)5cU)gy&|v0r0x zIpXLS(|@U#W$WSvnInR>8nSJa!F>s^1TJ3>kM1DTb=V!HV{1lon8s~`w-qr}xoT(I zx#`&Q0G%T4isg<+4g#ZC$;{uO36!^<@oW+iy;;%DtFvwG-`*G%tC_E?V;k52YwuXm zsA#*ehIM+zEN8t5pudPoK@vD>9_O(zSa$(8$w<#7QR*?aCK{$S95a!2qFEZ{ZkKH} zv*0Z!%+Kf=XB&_854oaqJwyI84rGY~fa}%(tvgF{6(y%ahr-t{rhQv+bl%0rg2SZT zR=qxVL9s@cm0DQXDi>Q}gAr+($(13=NK4K@?-H3X_^i9+D*Gu`V9-zJG>$Bi9|gPI zAcIXyANFBTr$86P?NBcV0{uMtVJ6+7tey8r`?!Z(-`1uAWMg95;^Az^(AVb|v-9Wa zW1XKK|NHA>&*AQE;<2r-+w0z-LczL)O(f*N$B9Wo9p?*`D%Jn=C}O(|&4?kC7TajG zBNq0x2p;uybjy3|qW$85X;bM;Q0L@z;xP4llX`(bF=0sdWb+~YZ(!vDtE#W_aJeU7 zu0rW0L$i(GheQ6n$1nLdNq>FF!{gy=qRsy7d4TX0XONkT<7w}8ap-(W7vl|KLI$LYOma?e8YVky2=U^GMC#@q9Wl(tU z$H%^Y!TJ7e-436xlgrcMWtqL-*XJ2*-Jp58Hk*p6X6S}##wf{kkgQ2oq()J>iM|0o2 zppQIHQGW)GVr9y3^14zsX}gj$TO(dEP^;$I5C0vkx@Nv%D*1@VU?zSNI#Q42VOppG zStuq&Y^ZU#iLH9k3PyOZX&_&Y@^}igW$!FJ>bZ;Z%^W=xiw;6`Ok~{wFp?f593o7w zV59)TRf85uosi|Tc|A!5Z;+JKc82FV=Vq?ss7V85nG8B|4@NMgAS0V8BWW*62@0X* zJ7>)uNmu_99zzu{4>(9JZ3s=Exg1dz8#q?RT?nyLr^@8SbUt&+oFWCRWJp)0s125U zg}&$_#Iyfk=(00v889u$eoBlP09F@osFjO$c-5RsE^W3sfEk^F9tuH7zC^zTlpzKK zX0%Cb2_p?>)RLlwqOxLi%oy~E?v<-m>q-c4gNc|ia(CNA3tj(MjN=)?(A#>~6$2it z%Mfaenrvd6S~WRIAKks!_?I&E%mCpdgph!idtt=;0&<)A7D;lR1w|FWNFmQWjoAo4 zf9{~aLKwz}&}ljkZvy5D>1gKYo8x5HsC4CDRMkfR46(9QFj=Ro!?%4JSsn$+zRsDydf|OEFZR0b8OD%3NTbpHuCQ~RYZG) zUC2f?Dbb*=S4EZnr}ih1CS`8C1*r@~-}ji)RH%=qY{fR9Afc~cLRn6(3nOnGX1)Rq zgX#^B{~Y>wc!%P;%uPorLyiiQG-B8_B~>0tCniTx2)Ea(a21KXfP16T?2j->mV)2E zu4OgRFO6>141zWi7CPrCQ@f67TIF$C$lB4rv;}bgXtVXCvx9H$~V=u)3glH4MB60zM;+lEb#-@xvpX-GjRwoNC{p8c#Ayf8(JMoD z-1Cww&$JIjY>W862s@|XP@?Y3$F^olNFs=3n1b&DU@J z(%q*{o!ZrVueE-LZWvQH*IB1&{GAQP)CqUORGlcgg#=JE3u}Yu5?vY5$7-GkmlY3Ja3n|36d8IZ1O6Vd0Y!n*o1xN)9wwMyZSq1GL5U3Ze%_G8Z z0q=U??V2S0n*WCoZf~~xH`BBC#mr3HM(?i%FnT{MKL=dyJO|iy%gXZ0hV?a?hd-40 z7~qEm!?VvSJ}1S%lN$D(8qd&d!l)f5@R6tmMClGfC;=Yv9ONFrPjVVdpD&77+->0V z1y;KN)zV_&`-E%Z!CHP0Bo9~~ez0XoJsdeucw9697uFvFusCqAZxRRGGgyQS&N~iE z^5sS;7oUuafAB^TRl=VF0lm0UsDLrbMWc>5I@WCFP3u%LF}hEKii;)7g(o$+*xFs) zH81u3=fE#K=%R=a^P|e-L%F;hU<`d=uzP@Pi=oVW;Q(<$A(KY=7(CFqYj2yfwc zFkunpm|TiDT;PiuSanl8Th#d3FT=YHjKcfD@P&aHnTd>WazZ2~~VquGJuf)bwpzLbzSGHks52_Z>_ zm=;apFTGavF35|c(F+a9kqIu>*K>ajV^DA zQNb5plo9F+{0U6rf($JZRO1Hs_E0gtC?Dsw=c5Ulx*gubry9RkKr2xGV(^l`Hqh+h zbGVG&vPt88-Iyx~!>PWlW&76UwlrQs3VW|e_sTtLO3Bj{lzQ;%<0d`+n-G=f{nEYe zmlNjqeH>k<_jRs+O7EKcDEPjs%@A|bb_*(uLEIlM%6MIM)!S64nT+eLqN4l!qRId6 z5v6z~4hh#GPo*$*S3CtP$In5UNB|HYowt>6F2WG0{*?fe2G~Xaoo&coIHJhHD2miX-6~gtjicg=cS#X}v-(1kqLdjiUvN3tjxd}2{)!QP4iKiGG*+yt zf`kWNi)_)1Y$&+FDk!2*ocE_dnG&Mf!-R zjDY0F;57&k#KMLF4S75;Aef&Wm5YO}x0?^Snimacpoa zMOUCfBYB&Y?FfZ;lGZ%+rzLg6v?k7pG<1R08zpboVxsLtDjrm3Lo}nZv*wcO(lymJC*xRK)~k4j zoj_?J4U0dwrDTg#neJh`e*7x$srm=pFJsnE8*%uhdSzxfMYWT=5 zLHN>v%W+wAkq_adA{*jqp@qxBDCDG#$@odfF8i<}R()4llf+lp`SjX4o9Nv8-%M14 zPi`kG();mx(4Zp>l1Ve_1jo8EE*#rURi&**gO(D=xltv&BE3jN~63_M_ zq@LEwIIk9yB{=bl2f}prV_=~bBeF&34wPRiHm*_Gm^YzJ>3`AsCsG9DWK67tZwa*W zkeHP~iy(f;!g@Q-p&v>Non`Mb#+>vCH<#exl=P2<2eHu*Cyvvkvf`9;*h{9o^Lo~z z_c1JB?2BMbraL*z&YYR<vtR%o9aeGqSDE+G}+Tka6q@q~@V z7Fg~ILZj`vk=*maf`#J!A%X8SRZN%NUz10-y^pGx>d;ykBdQ|f^=9RlbHjSMgRZc#M)bSIt%8%eK!~w3Fr(+MO$4W|SsGh?e;z#hp7NUhsK#fR)=} z%cBGLbVL^g{*0IUqr@ZbW~}Xz*$d*6+(mZE>-D_1X?@i=K3K6yHDk*1NXwl$U)rHh zh_xl6NYfdumhw}6z%Nzd3t!U-`%uLDq}qkl7>)FXB;4$dGH=Fs!+ZXrmT z^bktA|{EO`?YEzj5mO7TE3eBUk1>G;!;8(Opj2o(%}&%T-05zJ8Z@+2)%} zL|r9-AnKR`$VPXAIx$andsJLXs71{+vrWD2~)Wd-RJK&RO$X&GMVi@|w0? z5j{n{SB~t>kKA*%CVRTGMBxITbpd!`D)P>a!x^Nw7-)|^xEPcP%K*`${WP`AbwSsOM#~#$8=~XxQ}yCR8G1FcoHZ%trjzic32;wWWALEPB0413A@E8MMs)dqNaG> zQMl|!3aka0C>xzMj*ij9pz)Ov+;~)KJ0?{XRQhK2XB#Ke-=t$qVVpMTsGDoBEs??g zG8u{T>TL)iQ(KKkX{DmJn2KIcLMbB$o3``&_pzswml{iKnsxd_(~k zr2i|Y2BXPv&0u=~m+Z%`qSTrsVzSFeRy&3Bt4)HhQwP2tB1xWgeP@q^;U0*T7Y;uk z$T;Y0U4(Of%oCYvbHMh&9=T0&z}J@lApeM6k_>`*Nq4wX*G4Mgag;^=F2Y1z)VKbpnYC9A+EZ&CFg`*9+$d87p+&W*VO%r$ z*YqG~lim__W*e3Vm_lT*TA*K^lFna`6SD#5PasCB6Fcpu2|%`n?5x#qY?SS{B9gq9 zuF5#~!X(q=uK6Y5A-q18FxXFMGL4rGQeij_Q%G+;+UvT?+x&N**8V6*>N2Ad?=j8v zU@KJ0e5a{H2?g5SLexFJs)e%Db)=l=mErxS5Z=@WV$H^X{*9y|rpf30_7TG<{=3h_ z-tM=FXlm%BZ{}+JAA+Fitl3{`*KaHFn>yku3B@j_T_B)D8<74E$=K0IL@p{1W~9)T zsI^$q377qhziB5bPYY$ubS=zPINeaR!H&XhGXM|limSN{h?q3-fSyt{#b~o+ANkGq z=#-aF@zF}XR;3TMe&5R(A6}fWysp~MqoqelA9PchLm0zF(f_F3*gZl~;GI^zU(O<# z(Pa(phL~J{ErW~gUXVlR&wfAO8@Z{bj)WJ(NZfE2{4ahRs?P! z46t?pBF0}`$Ob4dDKFlHl0KRvhDBs4lBF+H3 zOa*+rM9a8+n0CGZVf;k%0%>xd2git9LsXC(f*e77MNDk)g82B-f}v>H_-d^yfe1lx z8Wx(lMzlV1*eX-iim_=mckkOj&~|LkFW6n%YKuVL9|p$u?oG?K*VjoaWq%sHp?y|1 zwyS|eED-t6h91tL##k6hBd;K`Llam+1F_{P|0&?ZJLJ<$W6JF#1QvK>zrkrD{!8HR z2aL54Ad`3ih`9p^XNZy6=qxYND!B)Adq1p@EyRDHgN!JXv^^8HUmEnAe)3|wh=(lA zI^;6GA~<^iTlAZK0(v&_Y*wh?a0miIVgeysYXF{ont$g)2PsR{g1subYhKv6#&Pk! z7Bf#|OYa-qdvs}mGI}su+|c0B?DT%&I`@CRINznm`+B|D2~HlB_w#y~+%$htr;fFA z*T>qnaqn#Nn)-zWp1#%Hwb%JqB?O#(t5p&-h{K(3?H8@y^>I44E3MnxJz-_%7E>>m z1I>;mKRq47y@g}d?)<6J*Qo^RI9x!|eVU}l`ANpIdD;*Ck~wxPmMMg7|3HZRW(x$b*3V62m&^G?&6a1f;Wd}W5=~V=!J@Niv&|$ zz?T+@x1w4)uoq#hEY)u2*OM!p5U&?tY!8Ayc6C-HLDYy4%zXQf5Z*5y+yFg@9WAYs zgxN>YhfJJ2MF%zA+MManC|$oBNt#2BNG7Jq|D#ELPH zM0F6Z@Nc=a+Zz1cbvx8~xjxU&a}~XQ^w=#g0@zS%Ckog}uK<~J6PAI>b-Icv}t*Xkfwqs9b*PEQ<$beZY%}5;9?G!DwQ>k%lfxR{}P7KJ!mKu>f zX)F*4>Mp5MDm5ueTXI-|cvO6L%s(T@%e{$(?3KY-^ z!v^%p8e~phB3l7SM{Zb(t?6;B&Ry%Lvq$u#5}bFSc}iUXcmt{_@mH)#p^)+Q4U!@F zq;29PLjCD)ajGlwC#~;S@3lronJ_bx6er-@OjGe>~(t8xrh~lhNrM4#%$EwA2Nco1^gBkjS#d-QkN|7+BrBDPYz3CF$ zWFm6KJd|&Va)gnh-w*~k7%g@@uqfurD@a(#6cMy8I!IWkNCcv@z0fhSTFU24U?a1< z4A7>293d6ToB}ESrV9~7QO_$*2C~4@N@a=)M!{Asg$aG}LnLv;0s$tO`$zU04s4|I zvYP2`UV?6-WZ?)>N^G$D+0bB7yjgwe2K{CMVFS4oQT$F;7pQZ

UtCiYV zpq1pl3@VlBu&;V!*mD&bv+6N6N#*+ppAW{;VH_c3Qe2e;L9H% zP(k8iOO>)xECf(kcaA(V6GBpGRY6v`$pkhdF0@#Z)5N5tnd75Kr;Q0ieHm5D~~MrTk(Hc*h|o)Eznyb9vaetv1>^; zMEgrHU3Ht0o0+a=D?$)IF*`Bj5$5pU5i3I&{vjSL`vwQged9x!Zh@_vMhtmbxwqc= zZ#^&DHN4!`{WK~bjpP+7&B*FJx^wtk+B!r67UuOI6Mc-Jb(UHWUQhnx`{d0-*^K1!^emVBA4fQ$$G< z{+4u#{OJ!-p%QdgVr{H*_V&8i2kK@V__OD+=LOn+=uxZAlt8p8ogF>Wc^4*T)(K#H+MquxCM11YS?!=b@czcap>pC7-I z%j0P#x0~<#{eV+OZZyQxP8%Nt7;8ybQRQ6PMtWX7mMnF>u)%`}>zoF=F7= z%jbuwpy$(qwY*%~^fysAjW}gbP?p|q;0;xFxRBBN^Yx#%9M`9t&-15h-OKyY^!g-z z_u{w_(eDU|D^Qxc6J{Blh|~{duxv^?za1+E~C*$ zXe-rtm9wdY2B=ktb3memU`fuox>h|#*uv1!wT_>bG9-`)bluXu<7=U`H%DX(rQ_HR zoYVymdVhKIojxG&@K7-NY@e#E$@C@zzgdzVmnYb~-zA`|kbM<9o%m`TW`R zJNFa6TA9{nSKrUqzIs~Y+lRycx8HCn+Z{P59H3p*)NKJ`?68){d}KE!PU9xdV=&KX zPP>aYfWNM64)2M;l{aZuvu&aZQa)BIWs9ts)CJOOVu~<)IH^?D=0Q#A`1}U&s#BBD_H#f%z#8&nU`2b`l4+Q6% z7!H#UB6rlD%mv-;=wbwpGyhvLp>m2i79LaU!Ii22{xEKurmE$;zN5O;E#E#~MjXY? z>^LYbQW_@MLfnYPNuYDa5a@V0I?rE-Vew zk+ASJJ)XsJ=ZF>0*={LAhHrM}v8IYWT*D;-@~&*1vC;n+H`jA0Z5>Kvb|tDgr^FZ2 zn^U1sGmGPf@}=Lq6#g825Pmo6Pj5Ml0y+b^9E)IuwE1bDJKhZBZNhQCLhU3wz&O@T|SY^~%eUU_*a#Gq()sTU_cRceOg@F%R6YECFAhm)j^Ii$GU%uW5xQk7(68 zAfPLt^7Qt>VBn=2abi~Ewr|58f5%=MzLWcUTkcn3yXRW!v$8twIvN*!HywXn7KdIg z%>B~UJ;>Hr5Q{7#txveB7Ja%Lj_WRPexSp=Mz)de;z?qw7@jY-(zT*`YxS z$P`8Jq^t-OQYW+5*)&2AgN@QT*w+kj27Dp7sD1KdSP{H%4#llgJL;$)U5WBwW_*VJ zat16`EXms5k0ocZaR$SDTXVr+{hsZggqIMH;{h~ z)-7cJoIuD-bL*KM1vQIiBDxS|Fmw@Ws79K%7{3bmwn`1M5l8nB-M3#dD=5O;-4MGEoZ*43l`JoU5V|7=e3`@SCN^ZVVc z>hu46N*DiZukw4}E$a7tUa8an><8QPzaG4u{yZK`!s~yZ+UNT}EkE%4JRUUB>wSMG zqwD)WUWfOBjh*U$b^&(#L7wdg*+w1wgZKmh@-A{}67D4J?;GU)k^O1?2 zQ1dFjUGDXW@8vx+Y=nCmsGxoo>nEdDcqbRgtztf|EuLqV6P6?i2&z;^Zo~|lc#Cc5 z*(#ORuB;)9EJbCHBZM&{GtcZm z93x%9u>~3~|4!0QLHDqtKo)ca3BrY%g)mAt@cpB(D)}e*+GH(%x5=al0QE*upJ3-3 zn(F(z4RghJDtW0Sa7t;ogAghRBpe5@N0_-CXy-O_)k@fwW0*~qaP7le9$ZgF1oIz$ zyBn9=b*;NwcK8OFddBXzjT3AzMn)2#*k_kk&U;URw(d%(3TLepwAl3xuxWTTsz_Z3 z@g#Bs1rwuI$=@oG;|OSS3}tG{$E4N$5IrF)HFEBA3GXu~?He$jCY-AKXF}|;BSQ~l z$~zVaV~RX>EaipPYRqcZLnwZ6Kz97=Jm}P2-rZ`H%*L^pm8f+w3}=7x=Z1%sRK5NR zC*yy1WOMwGbvX`7s^pk2;>8kp5l+z+w3?%*2Q9g)Iwb@Rn!4KUVMW<-=ZQgblO__W zNs8ox6eyGnPs2%TYupJsl_`ZEgcwmiJ+P=b`33n(u^4CtgZ!N1_kInQfq*jO$eC%D zCUbEeAWvfEqBU(2JePfX%Q!xE6!M@G-lCDvu#Q7&)|2O7qx^d)>vY_kMiCxY357+I zBavE}#Go+gsgf`+AtaEc0U}IA-2xLJ$fkpWgtE;zi~Q@AQi+^aQ`UDOsX3^CW$Y+q~ zOeHSkK*2khSIq+yLEAQOdx&L~OVR2~hOJtHrk0je z13x6Gm=)dCx2 e$vp3Y1AnAi=uSJ$P4SiZP;HA3cc?A@uk>D?ecs0`0==F`t6{s z=Kp9yAqC{`7k5?{vR-*?;Q0*-yLA1&!OEZ*HDH z%d`rX+e~`!sc>DyDLEwxp_Ko9{yabff{ftEQk^ANppaV_7`nL9+6MRbUN*4^gUM7! zEV*J+-g7zDk7HL2+}iBlEz#zlP6ebYJuQo3tt5jK+c+66l5q#Z4G=t|nMSmLN)S+H ztzc_nj8$5bw(bkm&{#S7M>U4O+0F{fN(3{`2+!kV|Cu}vgeQcUwJDHfl(Hrcqo_)t zC1ID^@B0CHJDj*xQk_KtutKqM(QJE*$WW~$_NlUYEw{f0tJXmEkG>$#11Y0&L&At; zJU0AUneW=5Ya)OnS~ea33M+Q0SqA{Hi!{39uU@OZz5UPHhOO&!Wr&3=4e{DGtys~~ zEmsu+`4lc^Q^NE?g4;MU#k|hElDAcUwiUIMtT+MonpUHHp^{mZjH!(Fu7t@dDSEuq zA)CXMabl8`Wh${en_j*a-1QCBMq;~YSZdHt0z|=V!G`5IP!0|kW6iG4BJ7c~dKgi?66U3}Nz|bnf3CRg z*oysUBby#FnJ1DFqT_l9NQYdd#h-N`7)x^>AV4Txh^sCKT zzF;?^1Hf@0{4^1K5@!4iL8!jHT3yMZ_0tfC)S^^_1PUK7dYOW0l>DcRG{ey?Dn$L~ z&GNqCt%41lfZ~9dn@O3_Sa*46O6)W6UR4DAw(|BEq%d}X66G2zDMg5^RW>MU(5A-1 ztn-{LGYcoCfcb@xdHQQ$X&7rFFT00ycvf#dJH%1ohQduQ#|;KPy_0Ral;Gn*EH|ua zH2pX}EMcg`JCI|0#r8=|%Pa7i}OyksQ}0?0J!As$h0uW44)Vxm+TC zp>XZRn_w1X`Ds>SnN}gFpn~Y;lCA6(^`6057G-jd=nYqze7OAbmilSw=0WSx4tDU)G?K&lOSx!IN_ zw#SKp$6i>&tv6sN2NPj(N@uqA9HSDVH~6`m-GwtQ4Lax#Hn+6r_TTq(5lHm+4fx^B z2~^h6+IoWC-+)OTrht|X26GB%ji92S3@E6?CaeQ{jSC#10?$*0X|alfBr{}iq*FF6 z&!{&*E!Q~J8RQYoT1}HlXR{*dV`9Xc^f9UEP)#~}%NU>h^{Ufs96b)>`F&IZt(n6? zq3HdgoP9TIS7aiOKJ~xJ8g)zXCK14Q|kpY+@q5uY@dO1vNTIdlyxOLh9c2W z9|I*qb0Z}YvcE9wA43nSkr&VuAdG#46(@-iX5-3?lD6`=yEcS{)?4*R?MWMEC++9z zAHNW#S|Y;4;-;fWmWJSFQqa;G5^0xGJxNz%&z91nN@zq7b}CycL{Qrg7dYDsovhZd zN;xj-BHF^TAhyb-b?*ft&>Pm6SpWrx7n*oQMsIw5v5xZX)0#im5@S@zj~ z-B2A??6UPC!u9O_rK(R<=Gt3O#K*{fpvipXlfA6O366erxQ1CF>3IRr88Xa{>y#{6 zs}pc2>uQL)ULaH`Y1jpM0Y>U2I1-vNv|PtD;s+`XnoB}rksv>(NR<2{sgdtg&V4h4 zs{G70NP2W@zacIwDv@jB!5(%Em0U+I^$j*@9IGJF5wg(zq)vfjlM#Ix=@o}Mo}8kp z=oSx_%+r%fgjQy9qciedVZ#(@HRXhxv0jIM9fhRQ$5Ct+^PieRZKrzDDJJbXCMLn3 zjl!}TVX5MDCFhfnj-M5+=nqMpL6)?Z<3;Bt4s<5W7_1r%03R#RD zE2X?`mGjxe9M>CVtQODApmFOn&Q9vuIF`~LgSL@pd|?txq^6-i4u(H1Fzfy*=?j}G z3`m%2?OAmIR+f=Q)uaIDt!~FiSvSF(wL_wft+tquymgafe zd4-vm1=Ny8mvS));Dy_b4eEqU$pox@(vrS}vw8klB>IQ|v4rR~|Ht4Xds1 zl#B?w4Y`{VNZgnNxgKr$=5D~<%aS5iR{uAhu5lC}Fc*0`dZcXs?0_+AcBDHDp;ug+P5yy;IsY%GF|d;4_z?l1Au9FKEI9YkT3% zV;Zc7o!&SLLD&j>3x=LzAXc;7w_Hh@*+M7F5g0G%?#ZVWU!EE|L!pe{h(m9R&?%Hz zV?b5u>r#ghzg;_b8GSM**?jbyYXw`|V$|3iM-nj~HAm7|H`7>=1~yZ&deR|l`UWeL zOZ%^W!XK*6Cco;-4q@&#-G(bj?;eCuC`BB0l;kg@5?w<@4w97x*Wnyf!ndejdKU3` z23691ssSR@vW=c5ohMse5eqGSJ}N{uTBbAmH}W>hJc^oboa1uI^$YdasfH`&Xj#{1 zWR-1nxbv;20tU=q_F1{OSnLK5s8jED3T7`X6%=PL(bg0IQT!DAK>>|DA&~u=z-at# zosm@0&@<2ev7k&Y-c6M^WbX3-Uy#fm4qwYCiIP$&MR9F5T~YEKoW{A1&tM41N-n(| zPB#8vg^YF=x=Tua&4%nK6FPa2uta9OviHfU6f1b0Z3Q!GyFv=4MZK^V8Q;9g$#SFB z`w3};aVC$R`{KI?nH6QX-zi-F!r-Y-eQsW6kSuh%s9^(v#w_yhKGo8!#a*9sFlG#- z*JHTS**Bu7Ek3v>WlEDg;2d!CokZ{TuYm4o!Tt_QdY|I8(ZNl+ ze%{j;f^{<7C;`0oKy1OavHwEV-eUcTS0_LloBO6*i}1)p)m)SZJw?%yMQ0a!y~P@; zU2YYG|Fp(ZHSV)(v7PTsfsppkeVr_MUFAtz^m>}#KA{k~{Wy1v%>jE4e(v(sfhaFv zVA$hooXz)Gyoz(iu3LA=^BL$pbOnQGfgUyi z7}sREVlTqp!h7>_)!$mBjtVum{`?mL;-~XGdK*@-1B5!hr&5kkB@t8j5Tv{ zz7krGo!VR-2vcU644DICBB{fGvTRI~#EP$*%+QzUg-%ZUNn68F9{DO+9)Gln{u&)m zST`#!+zKai^Jf4(L5IfM;J>eB?g`i1EV69ZiC#MwahQMvLC42S$zMM9_4Kuv(fQsukCU=zE( z5W@`YBTUhb5f&||L>XgDsGtd5_^l~_!?f%Y8dN<^zH?`VG&fqtJRD7oR8+KK` z7rh_)0WzoL9CC*5O?5B-pKJbPCHHcYM8=69wN;%PnXvfF@s?tMWg){2sM@v#l6vaW zkSRzE5jC;7?0A|`(*=NXv*K>uq#0Zy%RxQT;#|ihJs6sM?^3Q2I@I7pz^t@KQblqd z;aI?q(UsLSNUTz{0Js&^h5~(BK=Xz?VG2^-b9Zg49hvv(zeiyEKp75|K=o1yKFXOOXLh{n zUyn6UBNpI7AWVU1p-Ef@J6HV|%NkQ=C82P-6V%5I^L) zFKGxJ?-=QiKb?i@R!{ath1t>*C}|Z2hM|Q8+yg^MBR{X@maKAXXlK(=5S@ne6cOCF zq!fhDW%mh}Z>UvtJk{%#6La)HCA9F%{?{JO>7qS1IvSW%JEk_#V)vPDLL2oKDbT_? z%vut%6oFqFBPbFw3H@XmQ;)QK<%|42I5gYcn1VNy$Qa_SsE>tr+ODuYwDs zkcePbSOuYA(C9f$;<{Wb%ZF$IESjg)x+X|)%+TAI)r!m5cMKM3VYV{}axFTmTP2gvF`+#!!_0mNFTXeP%hCz&WQ*NDLr?Wy!2p1Q6&eXXH5 zsP{Mc32G){YK9evSO_Ex^;IOGUj%raepICJU`4kfI)AD&`C67zN*6WgR3!psyeWJ2 z%}|34=8UDeP`WFSoV#Ehax~HkYxjdg%=QI>v%oH+=aXn4cc_omz;`K;dpy??66+{9 z>%82h_3E4P&22p?0X?*LeHMG~peC*3)6s|)0R$;SV31ie*vUvJai1U>pSqgaKIWF32xQFw~ID4q8H`@ zzf$K|45rY7B3W3%RH6ZM)ogth1n z?-MR|8*>s8cljekJJNC6S46_aXs|@|d)+_N|2);W%k{iiw~HPWz2to@v*-K#+aGV^ z_j^C(|7eTf@q53Cm)rTMljnQgNdCnwpC_JfM^FeK-#u>Q=lh(V zlK;H+5a2?5_0#9~JRny~2Ke7!-sb&%A_iLON_V*2YuhOZC%6Cl90&iQAYjct!O5-d zhHbvam?^xWzJ&&Tn;k6gSRL}qFS;ABpgPir-$?Ac%lCVIjQ8`szx4NbzLAflHt|uK z1#%wVy#2g7-~IeI9?oADPVWES8uIhnwFF*?XkKr|Jz=Oe{%NwqNsGcK>qOKfl0SDj zJ|=V-;vDj5N4IK$E*1x_+tTfewAKKbxON{R$o`1#|MB}#0?SJAJa6g0u`2KKxvd); z3Mx5=IBqn}wQm=%_FpR@8vgg+U38bgon_^~TNasnAV_<|$?!X*sRW=X^xR0f_mA}c z4);y=ze}E-{eBZBwWysCB?s5A$&s)9C@g3HK0Y+}|0Y&Y9s;TUIPdzIZNm4_RXv|Y z&y;G*F_wPF8zQ!ABbY5A3;A8iVaKivc@TABb+-vl;kWEd--Ko=cJ zNhh03?&!QFO~zN}WM4iW&_tgxA+<>I=3G54z}eJioi6A!8Hw+CL_$5L*dJ^D@%yQ_ z-LW!55|@Tg5h>*XjYe{N0lRhX`g+z|5byKU9Ca$LvSoi#PpjG`tJ!Y$2L}uUgNq0c zfFUR<+&&@^jW00&WqthVI9+fm);$2!P3Q`{4 z-(ZlQvN0=;j!`entR##QWN@-jt~bLRhQ+lUpOtcfH8?>UuZ01OdRnZ*iy32F!RtdU zE1zU6%Mr&!&jpBQb}bsSeM7>s+Y}kpGC;Uqm9xsEAN#WKW^$n|7%7+)L?pbVHz9mv z0hfyd22%nc^vtx>8+Je~SGE$lJ5R7i&N#*1DOMXgue`9niOW^gq7Rc;# z^KeoWVW$HVOUC2S&^e%5h)bSmAkj3?n##s3q`tAqoMjhK83MCN&Nx20Cm;a6GR@;U zJx0tof}*F6xH(-+G>*;mB)KJ)frF@w3H7l?E}es-9y(nY$M|QX%7A`s#a=q#M6qZpCqKS!74t8$h3a~nkul==FovTq6*MK7L{f+&bH`zG?;O;8k60fdSw_hK z7HEwyxPgA1KdE@!3+D}pQG=-H#JzLeZbQDXMtjx1ZfCu78%(clwUw>c)~NIl#Q%gEyUjnr4#f;Zy&!)!LedhUi4i$ccIX_0Gl3)74E04y4G$1$_MefQH;IB<4lZtVYf*vcGI!V zxpt@9$8osQ!k#6g&ldb$x&_Mx6z#?pf?boJqrS=`A?95&enB`gYDqrxPrH_{C~c>K zdKn#OSf{~_RW$hWMFDrA_6+MSn$uicNLG(*jl9&5TGY%Ws|a~^&(+vo>8{jVCMiXB zE5vzi1(n46j(ZZt!$Vu)Q#OHCXD=Tt=Ao-(w55cJKA6gX zM-13h?9twzqntQ)87RIaZR20*H-<(izdChO`9ViP!lOPejF;FeG#_38ch+3hZ0a17 z^@UZ8Abe4h8m==&}TrjN|kHQ-G@$j+CnW%sa9%_ z_r8$j?zWuCj`3H?2t=sVaB--=HA#Z33N(32*SLG$LCgx59KK=+ zt{#8gd&Y9P^!vJn(Has|;cBD(!+W{f%S(_A{T>ajk+y!yK*+-3InRukgYKiWx$Bm& zyg#}yoK&SfC~IX+Y$tRxeHp=Qkj#=9j(gO;cxGF}_Z`dszNF7R^eOvxpZlDz1DC_S zN#1-OKhs`J*Cw?UM?SFcJoph86W@Eo1~ww_Ni@h6{w+%`+>%%`^!MHP(267&UuwPA zqweQpfwqxz7uvll7bRe;lt7jh{idfpULg__*c>(QyZ4+x)##My7YUKUU>d9Dp>7&3_?qWGy@wDwddXJq(3_G5+OQ^SrCV-O7YQ>EzOY6~z} z%KUFs7S!6Xsp=|p+S>K#Pu_R+=EE!H$omDgY0SEu7o-NgK2mH|8T0U5-<@LBT&yBe z{QReqz7+~uYDsQ;8&n7P?{XU3kRy;JQntZnC_e}W5^O%KTQhl^MC3}oSxo@ti) zRColq_Y72v7vg6LYMw?GGDks4`mIDC+rlywLpO6VlrVO1-9lraza;qmkzK)|-REU} zuWwGU^5V91w%g+h=pIB1XqYBnuTAKuLC{;nkU(Yz>kar(1AQ_7WQU>7cQ`*#BhX3t zb#41Gr=yGolgK-w1=`AZsrK2y;;8}xk2<4rU?wSnrW{CA3iap!D`}6aMkSb+Llwhh zQ~*(J%diT>CRa)ko~skiA!&3ssem%t;;IFw*acDbEuboZs@!;~0dD7AsQ@PD1xo#C zF0BEgT2V}0t5cIzbr@i(fXZQ6sS3ba?15_5a8w0UR&ycMFl18?6lU&#BFGw61?b1B zT@s!6tO-h?8K+7JdPwafU^E5gu#{2*WJNPUwHOttRw7~eQEo7H#u1zGXCaJ>yHaNS zS@(`F_+#Bog2#GlO2UlW5_9F*l3!o3oF{O%bVsL?6j(O8_^n5e3I9GYfnX1JVEB<2 zqOl}ye6zTZ<)NW)ZdGSKeX1brPzbMl-aKkP&6%y!;hYZ5r=(x|N-vv4?g^8EHz+Iz ztVsfH*_$2sVc_}p#Id;>5&l`pZiiS)5ZlSd;WH)VKDd=ula^j*Fe>&X=(HvohR{z} zfX4lIRLk> zGTwRIj?xL@1~=#{7!PwKo#nFJF18B9ojQp;EIeKPql>9dDtfVth+Y%C6=t}rL#Qe9 zBA`?geSWNK=NzDc=aQcJe*u6%f4@MX05Zzg@#XU&yO3t$DepNVQ16>R9~#Xkv~3Uh zPD82COpWFS#uD_En1R|Le}$xkTm5drpLIG-&=zL?rOHUi-F>7_6`BUAq*5x;pL2+3 zVaZ68SX|@68pcQ=E+R(5%JyRd93{Z{)e-y62*i9vVlv%`M-NJ6q@?SByqum3s5LHU zTf>cHw3l>Aubl_QhQD$0%0S)>pE$*9DydEdvRR&QQ&8kCCA5WWe0?*h!zd4JxHPKsa2nJ!QIQ zPquc4OqufMLi{$yP0$r~)2DwnO&|zFMgUUk)BluE!(LB%!(C;vG$VKDr{R7syb$yWyxOtf-!aCj~045sM;aX+(f z%;v^6?6GgeSyd-qt!B)=0o%h~I$e9vYY9H zDbxL2NT;)@FhbB(+L_-3c-#DUmyzX=P z8eq48f`%$G-jEQTqqs!42Rz)0snZ?t($?Kn5Q^CJDV(l4SDzf4moavYVk*&}c4t5{ zs3cG2u!+uLoRn%bi|C5ZkMW0WHZfjH-F7l2Scv;NElxw}YPU=M51ItGLZ~eAiAo}G zu$&SC-U}*yHk{5W;G9QMFd=xE2q<27lF*0b!pYr!Dr{Ml1r|4pkfK5YZoc>;)Pbuf z3P)?t(`6bQqY5zwHUMWB<|#>;l{6cehqrP0?YM+cO}p$p_ytkbP>>*0s%rVbZIF#x z-%ZL|#P2JeN}J$?Y#4lBSh+LUrY?|;D!BXI6n^1KglVPL;(mW1)|JT^FGR<8gWV|Z zt2@0SeKk@;;zu&NlVLlB7-V+YZ#&&>a=I5*^o_svW4iOVOdmwiy>pu2`;wh>y)0ZO z#6%H(E@p{<=)$F&x{we+Ni$r*q)C3^N{AklbBeP#I0jy1RbZK8OIG+h6GeVI$&_Xl zqXS2hJ7VZACm!mq;yhvZ0KRhPk~>x`F`;Nt(lKM`rO*YBSSlE z>WamK$vEEWCDgX2u?-42uQ(3S#9qxo>#p2$3b5t*u_VtV72C-ug-nvkYGzd1u{@+| zccRl&>;j1e6Sa-nGV{EgfY)!gzj*!jHc0u!3rqj8z0|&V5h-AW1X&O&DX*ex=sXpT zTtHPvV`=e1gYkPyX6F~y#d=rreK3?oIp4wYNH5X@XqxOa14?=4R6TgCS+*mWHEvgw zS)%0~UC*9_PX#|_=rjiPqtHn=w{A4oH(%SDk1k)18dqUDk&vmpb%8jDcjD22YyhZf zmpwnu;e`W&l@b^aKS!J#Vl?)A7>UM0bjW+uK`juc$HP5TgA*Odw*&^k3%Gpq5<|ij|lNkg`p*n}vy3%crjxC`fR+{*p zSz8R=n;~@%X_JGxNu!HChuwiVcuyeI=LC6^UKd9hUL7J@rW>)?P_p1tvakY@)=hTf zNpH-BO*B^s#w4+dN_Vb! zAz8b-bK1}k97)2IVEGiR5cUJSV3&IB@q^9D!A>%Q*HL0P<^8-W;~Q0fU-1j*j_l} z0Z@YG$3V44FZzUzit4M_6l1&Cz$?ySS4j;ocuAGOQ*hb`wT)-u$YhLn#Pk?<#-hRS z6xtVg-_+w!u&Em_Z1iE&y*ox^gtZxi&+NxGC)h$`|8t zli!X_{)m;P)p&`Kw~?WwmKtta_DYy%ci9NW zg3Cm+*ET(xQCQC77*{q28*pYR0}xcqz_PcrDL_DEiX^?{fF9sbGvnwy5~FLYfKENm z*qB2bpLs~TDVX`3t6I`^h7MCN*;1m6qE>uullxA5Gv-i0r#+|R<@_^+)7x6_opWA` z9tf2%Jhqm(NU*n_C0WP)NFm|Tn{P(z+5ef02Dg)4v2f|*8k}6O`703ki&f!KcCBV{ zOsJjCl_t|F67Gt%w1fR0nxFXEfo^#OZ~uejD0%}8-#4P}AR)&^a+t!!9$%S=uIgYI zme3~s%)qjZmKcfrMuWE!ob8KE@g zBNWa|kcSIb+f7qmFE)_<_^1;9sc`wKXcp{36s7F)(;!$wNzTnVi4<_SW^^Yx4u{)t ze=IZ>LU(k=4qTssUb*DH8IF>N`i&e?cu_f4*QVRa!M|Rj6NvI^ih;#xV(UsZ0e+^S z4jf~svQkiv5)xG0FB7o$-17HJHf&t^pwVlaT<)8hh2>Dl#`5w5JXfJE_F~W026U^YCKfVpspUYW zvOFg^M{@C6`_Apv&Fx!T+t+I#eEAH-yqrr+MI`D5GhOz^^^J`{S$;0aLcng;(m1ZJ z-dTISEkvLerobg48so<5mv3!t-E4np{m$#{)!R43?>eY;Y!O3wbM^PH-!?>&$#(4m=e=G(XO+41tFy2Zn#a6D(C(QWKy(`_wg18J^{S$v%oG&*XBy||M^aQ}HU zfOCb4DIQIH5h;p^`Mph+mRX}nOT#iy-1bN`!CiKScHUx&`=X2?tGi1DRH=q`5e)*FKmb_MsvAA#z?RR*Gi zdD!A<_#4z`YnZ*DqLMA-Nr)q&y|O{+*fhb^TdHEfOn&M7`MK!4vd-Y^iwg^6jH(~2 z>c^V`A3V?-A?o29;-QYL5f5L=-!KG-=f7WAdAl%gc@DLKMR;d09Be~~Lf%+{+mAY( z^?qj{O!K`~wAyD;wVUie_&+#w3t;@kOXtcWBOIQOYmH2!*M9==|!;|m-$-jN{os;)| z`pF-D{PZt>Y7DaI>39C(ldrvX^48CveE1h|kBu9}W40!NH~-@sPrv`y9IJ)w-#&W# z%M=jrMGlb`(h*~dR|+blZq$-n&Y z>DPZstNzgkCm;W}XMg@15d9Z<}9)@|soj)MGVtnkVMg)OKmy%j915g97 ze7eldFnpHTawd~y<++m0M%{QEPh|?U(ZWS;d2^hCe2^qV*p<(y{;i=_Czm05enrY+ zL&G)Zjqm*|;vASOFUh%$2oCssNDj8qGi_!#QF%kZX)xVX0T*0ipMwluacqwZR1A*? z8DVLmW|qSdHA<7r*&Y#f+9`?;lT2~K^ke=ck39h(6j%|3Ph=TPM!}QPgnCSIst3af z;cC9XuKC!F)i%HejW&*Jvku zUk*9EKn_7P>=5vBNE}w08naEHsy!#{kWOEki4G(K#gRA4jlqJTE{sRa)pQO56*=Su zbBU?*4;s)u*xOMW1@P05i23=(3gfHx6t3%}-6*S8jK^B-c4?Sz4RJ1NF;(0$imip6 z)G}()Rk+7Y81Q_FGT~R7hg`X{oR=S`jW();te;^2v&Akw_)x7vhoXFrhvd`u;X=|vATwWAT6_2&z@^V$ri z7~KAwgOfoCxf&PD-qcH-PB!Yi1Xf7W@4oaP8TFH18;)D1y=3%KXE=EYbaglBW{|7orpP*vOGlAA@&I6p`s@uRNm==Tl7L29t&-&f%ddzJ zE{$|wm>LhzBUW(?xFyay7F+`IrQT{0Nxn4@R$M*)d(l#~94*LWAKtKWpkMSV2u%WH zi@Ju1l3cGfyanN+qTWWkJ)(_=h8{1gT@u-?AzPZZ7>*XFzh&CiVgs~gHz|;Rq?BmS z2ar+TWwH2uGy!B)7+&&ozboGsATf|9mFaOlP-pa8$GSOB|+F z5jd-7ISIwMRC7vbA{q=0D6GCXn2tCGFy#*CCS1@08KD@p;g6Y#^8G+6Q{bC*=_> zL~^++z+QY&HI?z6nQtx*@MbBqrMef5Rf8P|@&^aAMIpykcBz1tz%~{jnYRT?8&nL5 zlpcs*FS3kS31(e^_*{uLlSf+gn-1!E8F0*^wz`bU(_cnHBBM4A+K4nop!5AN>|B51cbSG-FtiIDqbIi(MJQ}28 z;uT5~GcFp=dA-Si*cugzhfGEXEozBiRgC8f*L(>uOl#B$Zr&w7ZOwcUGMV-uYhK7@ zPu7Wh!bmv-ha`p&lYHdJr@T5Ds~FPx@*?#amX9yYv!G%q(g}0<+hq+h;~I`WINO$4 z9*`w-VGx(sVJNP~=Ims^2wNq73W`^`X&SjYMJx7sn6Q=Y*J6RnG4mxye7zS~Q{t$M z;bOOTXE5p~qq3+f3$|hGMi~^<84O0%U!vWAT<%3GBV`fYji)voLJT!hiGc#<6* z93%o?2Q`4O7vJcUARv-Azz~DHJYB(_C@3gvx9)xcca@fxZ^lAe_UV_f=Llcrt*GJT zB{?xE3i`4NZ;Q(c4P)en+B|~$7)3Y0Nu72lNqgAt-JW1xgDDbsHfPK|;Bw#<647 z7UR=#r^Iz zh<;vJp(9H&4!&4Jm~u1m0?te+R8|Pg(>I;7Sy~5BNV=VS#KNXhkLV)P@C-6XMPvo$ znVz)bD1y$re)7ni=BnqCvOGi@PenJB3Pl#^wU1l!5&wHJl1;uj5UTOTXj@cyX8=Du zw2~Z+x(Q-FZwR8@zNUE~Gv)}aP*qFS#8|38wI{XA38hOIB&vfgAC!YR#|M>v;%-ju z@x;?OB8MQ;J{;$&%ikVO3+@LaCr)!L&FWlFi3Pg!`5Y?qqtVaofjfg3<_edb3a8{P zIV^y%1m2ybZ^d0bJB#ndGqV7g?vz}2F0MJp9;yzX!W64;4h3TcWxk0mXt&~vNs<~0 zo|)NFX$KX!#!q@V_XJC@FkI(ArM=4ZJS!lwb|$yJ3g%jej?mVzg^EqLPEj%2-n$Uq z_xTFd_m#r>zEZcouiU@4Wa?ABmQ~!wv1&21C_Hv2QH8Z+2M58*RS$qwQ9Xoi&@I`! z(O{NJ(X_ZQwfAeVtTp*LH9B*j1D`Yf1))wc0|w_m)n6^Yn((&GfdG%?eIDf;Nw0nO*bf^UQ?hmAKq$mR$yht) z(j{6L& z-7XufsF^UWq!!e{E+61nEVr%sSGRh~(7_?Ek*r`@d)4!zc@}boLQ-{x+g1e$SJFkN z6fKSvt}07uuDHyc^3lH>%h3`|*Ey0CK4jBFeK*dU%-NFiGi(*Wi--U!*85`#ss=@X zF=xfh-EGR*#MDP%L?6{1(XxvABl@Vyh(0>|h(0>~h(0PD(MMGa<&5Z~f)Rc6d`GlT z^Jj2GAAJ@^bPf?y7}3=*+ar4JGwNKT5q(PEl9n#hsuv>*WsYGI9;^iusqIcIoxBS) zXtk=3PEqxdtlC}d+(>TT)vKHfi~k`yAAMQ;Yc4Hb5bP3*=ly0g`|4=iy!g^EorgDv z;`xQe3$MKLyGsB)96V|swxXBk;ZwXQg1V->8k)f}=+j^S;Hniy0AY2mL~~KjqmeBq zO3@owgF+>KmrZ>LmmCf)_kb=Zh73ep_mlp(?FI^p3NdPr+!9OI)>#0eg|HPpetGf2 zam%#KZ@ECWHEcF+4IVWFxMXX~CQM*>tSCDUUl$Baz=YuuEZ;X7D|TU(R)rS`NCa?} znDT-rY9NZ4rcL10Srfu98m7inB{qQ$W6Wp;i3WCQs!RC=-?-@PqLMD=m!wAgXLTB{ z^s&ZMKmp}*_PQ*q+GSa|%S?Hv%d)zC7ODXI?05g;?|%OCul$EsFMPgEziFPdf}{i6 zH0=d!1NCh689XRsn55*60Btz+Jl5sQ`^puUtb({0z(8N=@@`)de2u;eR3O^6R8a#- zVb-i@1u4cZsvz0p{dD9D_PDBGkHdm($~(*&*A?tg1t{3H^pNI6D@ZYlRt0(99ZW{! zeP44AuqrpOkMVt|~$nocDHz7J8 zoSO^1ZAisUiBB}hNwG+{18E}t>q6WW=n`LnG8!`${bQYGjuIl?<(1zF1gvdZB5kl^OUO*ocPyo>rP;OOP#v0=0e`x%0+@DLC5{1~gNSSZrT@ z;yeHgCeFk1L;G+u)U(nwm?eBTxt%PZ)2h@vc%}+acG7kVSJ9A~DNie>qPkQpmvN?m z)R6o)m*xQky(;*0%yPl0L(64K=hp4%J4|Mwc21>trt2~xoLO%Wj^Yccq0((*L_D3X zNm}>-b#>7N_eFwZ;E+Xw1_4<8+AO0Ge%L}e1k-DC?e#CoB)967dAb|*lKvjtskCeH zTR<$eRE*=@zRVvI-CJs1yf4*V-Y(jeE0IcX1a~Si3WK1Mx7;4#Q%H@?ovKqsfpc27 zv=M;JnIdMU*K6qQ?Hm*|HCTCELK_Y|xH}JoPxQsJBKEjt2C6Zv*`|%Z&!WlD5IVXy zIpq$8%Z#GvSDRgBGY3o@um1a+InL7g&g_%Bs`abFr8p~@8`HNo z9@RxPMN5O9-6@*>UtrV!n{O(Ti@=Ig5;?Lh4H@D?wm+Ekx{&_pm9Dv|2dHzay{F;; z?_Jj(|j194TU)*wE$D0@a&??%X@>tUN2c*MyQ?1?(#Caz*amtSCJtB)X+GSwZ)^k zt=7?CPrmw!>K7@VK|;w52RH-cO%9`+HWn6DL-jOeN#mWasLX79e_pMmL=nXRHTbHG zyQ;%1Z*(x9)$Izmb8%zxQ2~TEI?ZfXWpJAc40FI;65>o9 z+WnA%FB!A@0OpGG5MD6Kb(RSTC0X6HhX?tyTs?|bXB}ng6yO?F*G7G?O%dt1fkG$G z23YCki(bsSDhUq!3VIh-yDgWJGD8!p#wNa9zfg)wfgZd@-z2r%utw{Bg(ZDB1aMC^g{UdpYN zCyAk+=!vQ6OfqQH$*~bMnRfa#!dB2miM6&;rE*hgWtsEgPTHIHeK9j|GY}i9`WDbs znyzc%VA30>qT`A#=JYaJ=t64X>IY?8jL0@>XjRiHa_TI$<1|<>H%6ZQWK|G=Em$N@ zV>MV|JvfyWVT~jgh8w3&bisSf6WE-7dOis+KsT^-&S!S`o)kvQ|P^;jJi+BN*@3Nw1D0phg*?eY8<=!zjv?MR5u}=s#k339v z8|Oc>AyRlgKbrHI`Ffi1=D^v?SxGM3vrz-tl)j}>K-^5 ziK|VTTNsUNMTC7OrJe-rISIb(mHO(whN`D1*t2MQ<$7|}&nSWGqB`*e928KXccktdfhsdG&f&ro#49ifcED*&a%l?e-D1$R~?&n4==sZioP-t%AuD&NO*&N%7BDB`$;;o zYK1QEMvcb`;c@#hy~ZC6RSs-&p$FE-$pMZ+$nDyRdx96kILkz(aI1^L3%0xX*-s8> zuMt|aoh11y|H1GJ*kQ5gmv}D`vwD0WDx533YL=5V1N{!48R>ZV)YJ1~zToZ~f%z@qn@~GfqJAr!2TL z5ZHQxssfOvCvZ~|8pQFyrp};zF={;UPRCJ^s-q6WwGb?HnvU1piXJ}*opqbOT!Vxa z$u#e+0J8Wv(|l>~D^6}Pa#C6Vy5Q9!nb zgm;H)VP!CkyR|&?UJ=VTk1rcVfVB-T;0)m+;mysHU!ihgwc!gT;c#XF0}>scwJg4? z9fQ2kke4q8q7#K&Tvb{_9 z2Kw3h)Kk{eP^odhvp*QMcYyiRR{gSyZqggaSsm)H5_UI+wy`7BE# zH{zX)zVrrrO$*UHm()og8X*T$h(8>_7(iLKEtEGJ98|`~gEuh?2*pr#p57a~Ez>`k zDZ#?-Sw(K{1slA@1Rs!jl!dG;Y*Z(Ki%E-^VL=#f2o|^T-nF8#E)XkD4S>E;CBqnz zr6>!M3ERV73KOr_M$vA+H7&0u;E&XcEtStqJX;EKT%I~`akRq3wxR_srmcv5tvFER z6e7fXG!8S=f-u~akA+=OSPb5~Yz$9Zf=50vEw7%cymaWKZYm^If_P2rrUyblL1bJ} zX#<4_5U7!%m(4KNo7^UMw^7kf$DN~m%@;LeWwM%2@_T?5EdN8oIeJ234coD z(u1negF#vzhwIJdl?0da?r7c+sWxQ8Z7&Xz3TQZ7a4a4kH3NO#%HeCPpdqbX+H!_s zMXd~aTpin}WX$OTj`?!L%{QHNqj_G#&#T)LbdkgIB1`u@VMOM%4+}+6!gw7^&yOmU zJqig$Nv#MGELU73LXiB3PQ5jNL$qsR>5JQ+O-@#hOjs;wg)B&9w^%7%sH(|5eQ+ky zj!H;7n#Xx{`lR`zCD5JEpJT;j&5~Plb`qV(VF?O%UWju(*~iw&Rk@aSiSxWdh0dGJ zjcssNY{c1v*|=?H=dgMHt{O0QlUjb1I$oM|FlmqXMX&D+)`)4P~ndMUa!N(84nNd{ZGrZ15{ zox!9Z-pv4QAn8R=Gx@l0K-lAzjY1k>S-Ag-(J{fPITcr(zB!*DW(lP%QsNyy?S5#7t(q);znroD1oxv!_a;H~@ z!MnX!G|qzOzo1_Lgn4Voi*Fb4us}KK`@iH+C0YjH|8DFq* z3`p)3AOcc*Vm6GfUfud~d-Lu_`}$2$_7$h6Jz;6(&@6b*Fdf@J1tCki_e(oIuL75& z0@p$gGEAuo`*4s)(koUbS?DA4`m_~h__3LOidz*xXg6jF?TBELd}SXV`W8f<*L7R0 z?&1;mx%!w}c5Q`QORwGBTD=n(cd{R3v{p#-l{qQt;7*Wl|2d<_@vU zQPf%<njgy{Bp8# zb>$o%yi3b!TU$30b$C$&Ie;^ht75*%67f&~&jHX6o8Iue0yiA6O3ZM|FkUEP^1*6y!o%S( zVu_g~_%yIC3a(}lTB#mO_gnfTh_ilhi&k*ABJKaXEs8 zs~$Va!Ek&;Ygx&g`~@zmpR5B$Z?5n#e7HAGR76+w6eSNRZS8JXUPw7K`kV|`T0^?7 zZf)Hx#1t58ii6{k024erNcuS&RsdnNz721gaRF4gEX!Ridx{bSBd>yhwHD|=yL81q z%&W2$Xp&DwZk4C$!2)CfMZtJVa(!Kv1nzor7nqm)_E}@o9YeV$MwwR=H>s!X&zyxP z^ZEHOEr*lDF$LeH!Yj`vIX=s9hIlje%C4RqlovIygaFq#to)VeLUDpP27CZVK6a95 zCA!ccYg`$JNY**#5{oJ)`P*{y1um%EFnUdfOu8IhfQ)>61A=ZdkcV`ll`GsWs4tl- zYTFx7@(nn}BU%Ybghx0{svV)h4%y z7256g>KU4~efnk>vcVf1q zXmc=5vSn<@c`<&Yhsg+B0&sdSi{giS(Qq*6wW3}UKZH~#oxw<~mWKoM;8A{kXn7RF z@j!!RU;G|F8br6Z)+0ev1R0ewA@6YH4CZywCOkqnr1q$Nm#FMCn!vXT&6xJI9nEIu zjy{FSdpC*46L8;5dlr}Yg(%g+{T!pjLCTAKM(lm#Qm9a8(3>3SEg26TtQ-i$E?y8* zXOD2fkCof3Anv>^A?2*7OpJgTyTg^*g%Ex*|?WOC}taSe&}yqNBjKIBOj z{gZ`B4r>4#M0?Ln$dEg{^GOTTWEAv@T$DP9n1neyDm5EAT)=bB4$8F3J46KqgC~}l2_hhj|J4IQm`tcl+cu*E0rEB zpO6=8zJebU_9LggAezN$=(#V2Xug=iT{Am^p8l*BLFgQ)gIi^Hs^)|R%O+%Yi01** z$ZcuMB@b*Pg)%=h3uP4W@h)k3d~8Jw)niSbhqr7pJU>&B_U-Jbj{HdRemdI&;&lax!r+;E8tEr48UA~sP@?mIzJG@W)8@C>?k9qoyWOc@_ngNyPFQuES2-@|=>G`dfNt%iqBe z*vBInyy^K;9yPDNaz<+8{LXBXzC`r@uW9aG=w*3?)GK2wWLeKYB_4a3oVd8Q2Bty*6>X)>#+Gw~2Q_5=b%wt^Y9VyYSzd)`bH*!X% z9MakuWr4aZ9_>LCj8e}9P6`g`cqmAq8v!tP%ka=R0enE|EAU!|;5plVgv8(d^RYXN zpm52YUBCj^x{ID7TzXHiq`4qcAW^4la-cMCwtC&DROXb;&&8KE^>~@rrvOWnJXyiy zW%GKTp~RxW%Eu!+n}uHa?gV398}toEL(9@vEf>8o-k`IE*`^zO3hVv#!PFyjW;1Ji z3X=tiMvVnxohhRWHzd|UUT0-HsJCXWwyssUcX#U`SNB;~!d-73ougQq3Oko2=Pk%u zmurFC$By@GUj-_%1ggbR6vxgBk^nZRP2!cmrDNeXO*(FIWX4(w)a)5RuvFL|REn6g zKphi^ba5jD$88|D4B~uK|Sda%;V>d>X_D!n*UdPbLDH}Gf3W4jI7lp{QdF2HS z_4|u!c+>42*sh19+2ZY&;!$5rcgs5R4kaD~1LHnT?&2>*@xV>E;*sV8Jcf{;#pv~f zl6vb;I#$7bq85tn6HM*wLx>?uTrDpH7I;RJ{y05I@q&2zjGC?) zAbvk=lOlFs?W95(Dv7T)t2>{WSPssovLV6hHOd8b08iRyC|#j0%^qtO`G$f2N>1y? zm*i|_F%e=u`;%S|3==s$Q5}oJNw(jHKs38RFp+QNYe?-5^z9=1mZfSqnY{S;f(jME zDri#!=crN;XD!U6BQVm?3o4G5B|GPNn>@Yf894a3$e@v}tCUt-&1UjSQ&E||%+xTC znMguMK)ep-UlIiFs>2OiL!fP4JZR3#aj^(Gj$I}yY$tOiX^mU(w)5`TlVY~wR7kge zirUndYBlMKtUA`!tqD#g{PIlnlCX<7tJYZ5whL~B`oc;uK}A>KEtHjNqKhW6E=x9V z36T)m#i>VLQ;7LQ>7v6*l~nDj;Bdg14uxqsAYqGQXyH@$LOplRx2Abl_MJIC3g6hF zDOrRnXbQ#`N|(`6n=1F$KdqQZEUx-leWG7T6^>E&6`b$p3{lNcl+(V zq>t~K^9qXNH#DOvvTm)juE?CjARy6O)_l#}d)i*0i2xgVJBfR!g-ql8McllaNnUV~ zEaYB~81`1bCZq%dj}tTqgmsaQy5NABz|M_6x6zT?F;PRxap2Qv-Z~oZ3quJn#G7A? z)H~tb{og)FfBwJaDW7asVbWoCXzv)XLGqWJF`=5o4sT(=M)S?4v~V!LItX~8EjDv zJ(xpKnZ;P~dRNC_p1KE3e`aeC`=+pG1eYxjq-94oYO#-}jHx$9NerhLzdDI~GveqB z*;#p6ac2Ldt@38Dn?;C^sv)l9mW1%_pt}ZMH28_uoAAJkV&Q~A@Pfvd9}lxR7&jcq zGlaWr)7kFnOXq9^^V5>aN`g~KBIjnYK9s%K@SxnyM(yEpfeMy!@nKSMU%+-E>h5=v zyggwkt*`&Q5xOKmIm6*ho4h*@4Hi>!5zl^?n(6>wadwX)18w`@)Gh2jL=DH=wnTxe z60;Am%($W2V%J0cHO|bgCs~9hf&w<+Zf@V%_!0(7%W=ycJLlwDKJbP**?Q&lfr243 zBRnW8%&}r2aPpn+oqYH!p!DhA{NcZT`yC|@GGUH(sLRCyWwXHZn3tgR&%X7^ufKcp z){mb4`Olud^OH~h<$Is}@NI$R;=o4c$Gaqvvv@E(?7GW)m0d5YW$Bd>{RO!cY@B#!95bD8` z_uqZ?SKp*A`{ZA~|McA-J^jx6(5>J8)ts|c5^BYuv`xoKm;$66!<9%UKmABl7J;Zo zC;#*fK@Ca#$#;H6i_ND$_~_)L_h|Ui%xea4=DDLujRr=q*3H4Uj|d$B{^|QqzxHD_xa8W5#!AUvt$}5;oITn~Ffd?-qCYuk_u?Zs6j+MEz;YS{$AE2q zaVz*rfw7W1X*Y#4PwAi^_ezReunzg+jVQe&v=TqDE{I=gTE*;Adt!|QM8Be+!V|?y z196^nS*n4j|NU!E|Mo{u|MI6N@BfddfA-U-Z-4#d@BT^*LL7~AWJoZ^z96V8_HV3_ zlQ0?K+enuHTur`tFmT{QBhYzWwZbKY99tZxicI z-v7?ifBtjHF1t?Ae~q*M#o+(N?&| z3zp~xa^xT5pWVY zwFPaid}}ONpKXqZnDTeueD?mgXqe}W6x1cGnfdy}0g_#8%9ko67fNENGFEj5N8)+( zazvBy?9cxG$*Xw8w&rbbEYgrFZfv<)H&38t1Ow*c1WA!QO=Ps z2YGHchpOcG-7`XTK8>;R=?{MS^qrrO)8pAceR%TyKOwbA-Vb^rnftYmWHAsHd2Fsu zqHmwJ*gh_HLFQt|Gg8q7j2jcL#kc+Vp^#D0cCnBU<}X~bkz@(v#(-cCiJggZW$KD) zV~4da@WWcp3{}a(JkrOZkT{KskHXVWd2GUyjt`vh{_Uf;!8wA1l%9X`KY#J`Z+?U& zEPqINR0gH)SUzWP0l!T7IWJHWQ{HQs!A8S#-DMD+xUnsxwiL>l-P8<;cG@!w(nsu_ zTG*=v>Z=N-;AZ*Myd>PmrC$m(Hrn5F%SJ+jh*kfic*L%PQSw0{u~SJQs@%-USKGY= z=(6g#cj3O_Lp<*6t3I%ymJC!)e~>EXENqLVr`Aafb?Ey>`QZv(CdV)@GFrV=cb>If zXt;*4lo-9z+7(57FEi6Vq`qy#UwVIY7b3Y!hM+cerwVT{Lz*j$jUcxgkTnVt)N)7KDxXh?gf2gxcoq~U1br@V6PisrT4;l{IUCYm zIle3H&15;yHFl_r(4T$M-0Lx%&_-|W_}4dxWdHc z3fM3i9cX6eBX3^89%^y4c%K5=h%IIbwzR!>zR+J-wm!U_KoaTXix;Q7NW{+!R*Lwk zw_KE3rT$ux@8Xo2#+4R@3MEA#6&*gnD6dILkXO=)DusCI_Ta=bj{0pGVy4i@I|(%U zDK2y7^7fr5E2?d)+J@gO%wcLIAmxx zt!WvYb;mq;>edFsBdj@_#n6|ZS!n?I@fE>LJVx9F5QVdt>7hd>4T7M0HgZrZ$~*-l z-&a~~vwCy!F_=<4)nP}w4lDRQB&|8F4vvkeD96aC&C@L89>K(f&>W7Aol&wo!8+s= z48<@A4SVrfxwbV$^?4vhDJD#;NT+hOBACUa*S+# zp{M1<^5PD`Zgp#~Utjw|`?b~en@!Xr*cEbOgIx;IZuRTjIsQw$lhv^I&XddFcq5HZ zPe@ZWwqfrY7A#mB>gg74si+pJ6K)9{NudcG{Lg8>yA9WY2o2UdisD{x@F;^pGM?;B z29qqxP%#7IgXCZ^I-1j&%2Nh_^Ley8pyIG*8pd#oSARLWx73Oj7Z-8YK+v@;e=Xmi zg9}&(JyB{R1cA_8iFV^&hTA0=KrQDxcAW}mrgfo;ihk|l$z##{d+B{K8zQ{;dWnD_ zx|~-i7z_n?bXeq21UHLOX}0AQ#vyw~RA-QdcXx2UQuuMtH7&(SFs;`IY6%W5|{gt;nynDo}+M^g%zY-@pG1>-yIcY2dd49;9L z3FKc}Nbo}%M+gRSBW((FNk&Zzp#ZsepLoNqp+jx~1uIm;M^-9U+GYsqW>u$RE((82 ziL$Pu5HW9-ehrKk(%x22GMW&oA z!_}~v)KtYER<5ocBs>4OplGlVgyDr3GmqYW%3Otw4 zL}|G&nfi4hkv5V->U4)9ZHLO0kVmD}b0i=6y@&7p9(3*1mW!=8aTDFul4L@n3+g5S zRk_)S_xsPQ+~(eK%d0UMv1T>VFQG})8lmpmZF>wceN$er)m^1M*L7F%lEyMetNLI( zZ!*Fww`YM5*}ySJrzLmTKmM^mH@S?E!`DWG19g^f>gNc<-d)T{rJCs?a(uc5Hk>lc zeP6?;8!TUX(*jjp)VzN3s9mRtHau^pdeg{-tJ~ZVjubBrE*bPVoPz^?SNv+Du%2@0 zPhr^S_+2M(L!jIg2cnhj0wlx!&t+(ClQ%ayuNqON)k>SCKNZ@S+ot&y=rx)#Ejt^( z594%nS?5?nyr!`$D>BGwco zoud+46qg3nw82|w(c+URdj$?2Qfydx33f1@-CN4WrW{!}d%AWR}zn@uXkmj2KA?kDcKVx4t}lC&?ZGnJi2CEB>@y|*F()Qu+RGYImsKgTd0kh2P4ehYZ zNec1)XE%|J$_v*aj`dh03r3Pq(3y<2(n4#FFh@2lF_slwH+k^tZA#pAtbShQT8&v0 zE}+3clzlU{64)Wr&<@zH<*xOi-;j+|X9KQ1lKK5%Dr1Se23Dd*tFj|b&~eUiMF`-v zV-3%O*dXm;lT;WjGwP;AKh%+yNcH2eH z2`C2@nDOYC$kzrMoScVcHpRUCLcGX30ftK+5#SC(j9RH#_@FS>*cWua3Vpi<=jA0U zqL50NWE{VTs6=O&X1LWLDwrfN*lNKKzxeP%iz*gCQ$kkpNz5Dsbm@n029PnEKOZgh zH3F}91q7CWboKPcjbDld?Y*0V$>Bq*Bt@pv3$A1p+4gOwYM!l=d15koq#>}k zL@vG|*;m~aolp_(^qw2&9K1RN-)j;Wh-0dE&Y8olFnYH!ks>3V+5)O&m(@=J8ZsE~ z(3Aj=9#*>n&3q`*U=4sTtF$17=^fN`<@cg~Ln(@2uq@Lwr_!W#&KzezG3PZ2&CvdR zpatwv`nfuj>cWqLJy2;^$_nI1d8*SGX+Ptb@Ca3bX2*>S*D(jAX^d2-G+qC*0D6)d zf8mxt*K#S@>_~OyAw^75z9#qY3m2gFx)i`~U;12N!Mv4BbFtKj4x=JM8*J-I9LfrM z*!~s+rYi5RLvk~hMx;1tRcxPuq~b0OXaX_Cp{L2JTE2Gn^t+ImRi*jYqFsDc);4hI znQy*J`H%c0aj#T>v8^dZxe zC`A6bwGw{Oo9(~OAj3oEL$>81TZPVD_FMpAO>kpZwiX4{J*p!!k<mXbnM?Hp@rk{T4$Laf(493Wg(4Nu>F0 zPHT1NPGMEM1X}LN8Rc$hSOdwbX$H@VekPhx`nc5l*PvwCPhstm8Vnf56-bCI2&IuB zY~WJG%BEiT6$u4{1ipC{{U}uC5-NvG+~|6D50(C$Y?wK0b~2LeVx*TK?5;7}Axkq! z#3H=zr5pGD40yX;VExHWl{MbMT@EH9Wi=;BD3#@$=tdrlTfC5ZEmvk~Nv;F0lHCV- zfeITY)#!Jp-b(Z_(jaxlM+lA(su{33l0G}cLeN&aVTvN_Yfn!_1{Au~BzR2z*#3Tv zC=&+;-e-O2O_5P0KlAaX?o-}}?o&ok6L|qs)&jS99a8{m6fVOM#E zz;>nLYJS2;c#eO;iX{_$<|PO3*Kx&Q>pz%La!7#&`xJj5`yvHBxzl>^M|92-8+F9p zQbU|#UH&GidX5cO?mW)8-yKZChQm|grnz~KHQzw|g-_GAKDqi9_{|$J@l2x$m9Ryd z5AheJNDKf%xns6JZ(&s}L&ip)pi`4;PY`Ex3&}#{h<`y~>E@*IQYxxd zA#~IOZ62*mgoyP5ks&sQ5ZK6>{8VsF+F8V0-h8GIkww*HN19ys89&^z-MIZkak07o zAdj|LXyClwrG3&_OmhQ%1XiC-`^DA0M(zP zHkPTFHCt8Dob6MgN`A)Qc##U}A}q##c=G61w* z6sFOGdim9cs5jt>H5|0=)!Lj5$qPl=6bN#cEY+Ka*Bnct6g{}~&tZy5(wdy|IkXeg znj28Li8ISQ3~?&nsZV>nx9y4|okmf%JU!Vce43(FiJ=XGu3tftZv+%cH_Wdg<&iJ8 zO3N1cIuB=YU1=2CKPicBxxJa05RIJ;l|hZ_vvzku^j9MH!aj$01{dyzMY70HizIW#H=8DbUADmp><4R3ot{v z_^jx7Vi<6|P+2_T46 z`M*%JB9&-q(Tn<_b#Q|m*rb7t=nhvEsPZi6T#i3Ar`||E0hRvvm!(;LQMrkR$T*m+ zfLCYm$gat?P1GIm74@ETj6KE4o>KYw-XlhLndfM`U8`dKfj82k8TMA8{=-$Y?g=&M zSeV6yp%Qf&2@661vEk?P<8I`X4|!)18cY?I0-5F#u?aHoEm&$Q7-Om*jpd~@8EXN! zDqSKLCJ}rG?*8++#bU5?EyKp*B|}3)ww5JR++uGF;cLpfAtQX{RKW%)yJXMelVMn7 ztSa3$7TYG%rUuL{b=WkbtUe57IPaZ-nFt34*LjMK*GgwfG}0V&?KLN8 zE5%}%KluDmG^?Q165U18G1M|bq?E^UEX&tkY9Kuv0E~Jtxjg3HaFkM^I|&wv6Y~VO z$8AG{clVA6cN?hO>mf!)`v}hI>C5c)o87?Y<3gKHQo{&nRhF9pCFLzv24{xJthmk+ z=CB@zQl)AQ-VC1IXgD8_GW?>lW;#XAGFqu7PG0WTzf`S}@Na_qveeG{+W(rEALSOt zMEfJJ$Wh(P8^JkTT-uJ@3Gzi`&D`n!N$5FpFV<{*6q!z(QrX7&q{c%7p*erMg6VUd zVYGnwIvn^(df-GMnB1A2ibx=pGap9j=qF`@=wJ3sy$L0W~ zW3|m{k0Zv^nwUypFH|fwX1>BXFCfVz!Wu=!D&7tKGb5xvEgYbr6Mp;U=4kb+>w92k zoRbDfl3_pA&07%D<<>6dD#;{ax4=6&yDQ~JeLLI$9Aq|#*IBDx>Op4jha!=Usr=h| z6{Ukhl=hbUkrPiMco=P(`sBc^2~~=e^(2c#CZ6c?w{=?|5dwg-9hYDM{5sHkE4Qm= zdg~dl0#f~+c|KOeLRsOUnHvTsU$aK*vX+?VLh2)c7N6kHpV%@(xErU~VduJ|fv`2F zZx;(pKQy`;C%4R@6p@b3>=%*OnJq52qMQ`{Bzzc1GE@jze67)wCl?m8fH9rl?a^)K$^;9$S`Ii}k6V2Qih zySc^?X#miW@jew~AUg~lnD{dUka)r`C^CUqLnIO}0O)}Q1!&+I-?PGivSoIHnx)wh z@ULbpw6{u57OHDo{g(FAv-zP+LAgII`G=+gaz)Tx;zO0i=KhRa>D6YX2)CwxI%q;E}+*2W5@0YhEdxz+8%*nzVv8tk9D?kyXqAlUA7B*0dq5Ri?>RZHbF5k_!VMZk-`@sN{_s-&4$9N*@!8k>^x z_mG!q%nwx;_XfW?jEO~m%*=Qw=6i|7SiC7T1oEykvlR8Qqc)}TwJf09NWmX*%WPoN zTLw&ELy)td#z5M$r6AYE8_C*AfLYNo9gb@Kq1ZN!OZzaR)zY>St#2=4{4EX(ODo0B zF*@8Dol){AonXYKeh}qe1vRe9@&N~2sOY}h_#>KAv$r`Hvs=YcXky#tj0dR7SrA*V zRTb5}6V#Y7@^eL0tE(>fA*!_|$6PL}Yy^~-2F-XfSOvqm_jMN1i&XQa}8k(9vxs~Loz`@yXPS9sf<`F}GcPw0y&C`NtBMDn(q$;E!8)7usU8a#Hz;RtD z4EX#rSY@iXpWC;Pizp^WkwlJDs_Fuht9MFfDfe^?vRTd2bm5)XsJ7>RFMdjX*~sFv ziU=)l&i6ReQw=@8m=(CI#?mT1``GH0j{<8Z+8JweWg!1EMTGJ-U4ZKTT{PFb*CH+j z>5+GR(ktW)Fx2cnq0L((BZAImFP#_*158CJzEIpn>%=pf?a zr~4v;90vOr1v`BMJHu2I_me0J)kh`nXxDKNkS>>lRGt|_wNSzQS4({$u{Ra#oN)7T zS)04L`Y!U^B8ugh=9(>zEoo0)zy17?|K@YnA1ukb6}#aw>e+Axdi>#A$VCet$-E82+QL=TaCNLn&mR)nL5p145RhnANc1>`kf}Q zEM16vK>kw!-QYDXezj}3|0BQ*4O)uik3wRh;@rnBG=>Gxubw1lF1di#N4XcOuTE(^ zU9N_*#vD6*#tU&GN9oJvSwZu1^Ra|Xi$@dud>0RgU$4= zVx zaazDJn=HVSumg!yol2E_PubNZZii8v19S71v8F;r^UNaZI>?O~7@qtiiCM1Z0eA@FH6;A&Wy&o zrbL^`a&P|Vg@OzXx?olLD?p%86*{ln6=!$-`eYTg`#RfO(lEMcb_k6*6o}e{r+a%n zB8|wm$=6R$pdc>nASv~*gA4VyCcSSG2<37RDw>L3y6|EY;)w_TzBU7rH?8b*SC}M1 z>7aZZ{4fT|Vork$hU|7v6uaz6B)|}QI(XYczxXmBsj7v93kpRTPKmJD-gt^m7cknB zh2zEx0O=-}>kJsgb00%x-YW!(X4trdcN>vkFcK{T@H52ky_HYilf{)}*3eZ?z!tJ# zDZ@XqL#qB^E08EI9Lr!c6AmClic*jyx@`uVH)`}iqeOkntTL0QM9&>akbbQtCMa&Vm)b$t*(3}EXzsUOq}l=Dj4cly8Mb{)FRaT6^dj} zB;{Dr#sRt`b~!IvTs$B&ck`}LC=XvrOEl38bU40Mok8WSTV~v=y!KSRKMN+37jW3P zuoXATn^7m3LOowug+;K3qvl3tNstettt ztHk@hmC}!EsJW@=G;Vq#k}z&kpfS=p;_H8q&4cn%~4aHz=c7MN1!PLQ06kPT>JcTc% z<;`dJW?kV(Gt9Jh&Q((2bR=5i!9=y~>7vp)S=3tf^Sje- zPSq06t_9F~o+O%@E2{gmuS-2A=R$E7a~u(?SZp$U9OKxk+G1}Leblhh;Wj*tK=q7x ztLzQ{OKddWSx~5fje35b#tf!1;-T#zj{5ZZ!stZN-X?+OJHePdz9A1TRf>iRu9~vN z+&`dqDujPga2C5K?+UY=}ljD|FHae*9C z(D4*!REg%gDH9*sRQbM%)auaFJxyg;B!hy{~N2+b`?m@Zf|zGRfrdy{Euh zgd3{Bsp9Ld2|Y!7hPWOBTcdG;(*{k81Y<8aW{DTs?Z`F8NJd@4(*cjGt&BC(u#{9D zbX1qBD~^y>DWq3N^_I9gg^{^Vj4uG`;1*lB<(ueqlzF0Rge+0s&F|yt+6U+BuuqJK55^I+z%`m@@v~1Ealz zzM+YsgNvyXgR$`rpS82Hc-r1GEsi}pIjV8?)sGJp*dCc6(mEn{lwzY2t+Y#G?M2r_ zGO>>6DufWUN-}0a24ah43~^vRz__8dF*MWhdL~4YA{lMiF4RQQ3`%>eT>jUsa1x)FZ;UvLsJBspe;H&yUSCk5{?|PL#?nx5R>?mX$1*TupzqY;K#4$cN&xL$0bm z@_4cL2AkFzG$SP0xp{W7$NEmXK)iAlS0PT5deNaE+r@=5K^?)x-Gvi4p&j6*cL~@P z-UT>Ztq0EL#SQP2Tw-yruXt7zmlszj`|$`Nxcp7|Dk*+cKy9RMKjV7o&ZJWiSa1nQ zb_;5(-D-nJ^ID2fVlW=h)uWV?lzu*`<S;X3MjC+HjG{Ky+Wh+)UTdl6lFXUj z#qY4(=-wsDXn$bht5Sp^d3N*Ie6!w(=)+nXwv)rBp6mW#$k~Z0 z-+2iXzfzCKn=C(SxcQ(o^iL=>KxCF&H$`B&G-h3ss!k^E7yjStV0$TN+BtRe!jsg+ zV|y4>Ps)?O-_y*P+H~K4rDE^hz+CG&ka zcD8)ooi7mIssuKVmk*9q-CTXyK4m<8v=;}Tb{dcn%A?+=gX5)ujjQvH>Z3kV&Kg=Y za0$=peRT9d;--n)|4%~Ek0n=bs9EU#;V9G^hzp61t=NQMK=ZnQWNmddQP|GKl-H(U zI5GDe?nE>sVXfOZ{NMBb0jEHkF}}D_?sz;w>|(fgAg8iF_Ftj>5M#Z5eX3c1_qn;4q_8!8lzezasxQGgGEcb~ggkqHgW%N}GlbZSDN6x^^@5nl-z!AO%O5 zr@f2{T1kxO@BQ+q<^3A{Mi(@-M4&34&(%W#-}gJ?0N;%WL1J#~Rbw$ff_l_fUj;ud zUHGIeJ-^osh{3Dn1tw@d&FQ^5xH}%gk8KUfiI!Nm;~)gW0h-%>4h~xdn1J4J$M5?* zwI{|eA1!VVl|RNi04ZCXm3$nFJ)vx2aZtS>;Ww0li;EZ1 zsn=8U0&El<^HM1{kT#W^Z#x@U|IWABiDzMl+K_$ooS2<5)#6KAYp>bK2X* z$xx}CVMY&NQ3mJ&3PZBu!S{O$0CHU^T zO-P%>XZ;eJ*qV*(t-*eSJm@-KGDh=@)5@sWNNWt2ho4Y4-8n*xXnGJ#Lz{VcB9BFo z)q^9_X#v&EWQdDp*F~9X_R^8fk(FQa{n{TK-r4a^>>fgk_i=gt7D|Msw+|rE-pY*O z{YC~PGn9=oj%9yAZA0TQpUA(x0`OpF+s+ll@-7fekI%*MqFBnk9ruYi2XWuOMc5Hd zT#RrCPE|QB)Dj?v_@5mYiaM+K5*+8O$e5EaJDveVv6#Ok5KEk0v~?2@xV9k+4Lp63 z{#r3fW&@psH0XAqE&?p|DpWm*Q=BeBwNf#aaw{+uS+P!o=Px8Ek1g>JLubwr+=QtT zx8W^QRFZKHB3d9(q-z?`I)~X#`y}H zKxOia7x{$Y`-gTSadH)Gs<_}hs(8Z$n%u;4isq^HEnj!89clmE@Jx(b#}`d!^nnd@ zxJ8ExWgA`~2U$Xf9t*aIBKkcOa>1_MvNBk>g}n}TF7IF4nG1?DY_8A95GNypV=x6J zKua!37|n2RK3l1Rn1*P{u^x&-B-{vp2t0YiyjjPeP43fj}gbE)_wEMXyf|y_qs+o( z`@|+Fvx`7Mr?5k$?i#Jn#u&hG)a~&-V%F%7$Xvh6DtV&K+Z(+KVu0M_tX#9S%JWrw zd2H1@J_ueJrRMOG-YFHc85^1rN`&xV2l=9c-#dOx)}%$hZFdeO8H1N8GF~3IS~89j z?yVQf?Cj00h95PPkdBA93 z8la~(6LdIQl^T&MSO7!@Vz^aKch{;AXk5v#WHugfEo>k<)!(sbR99Q^CZGxXj7B-$ zf~z{EHDpyl7v)326$zhU%&_N*j_+0_wc{u2Zkh>*5JLVVW~#pv5+taCm_Qa=Hq|vt zlQ!rHUkyi!fbVz0g-8+ys8mK0YM=@XG)G;mBmt+|Z7BDtYZ14t!lbAOppus?E?<%> zLjF()6c>Euup#LV(Q&N)qx?)H-9jHWVIucEcJ}f;I>@Aa+>BFnkGKTmTGkAzw%8)X zI0OcrUFcjamtB~`MU(->0{>S4gec(RE+pw{Tau|`-az`3C`Hoz>5hy9;Bznw6NbAN ztwimPsQ9U8?*qatfa44ikKjq>lFeQLt}YGc9KfC|-&UsNTD=slPL4n&5f3xEm0y=y zzm1)*;nBwiW;3jR&y&J}6bc4xrf{-&)o;F&Z;ecwJ!z9C*1QS? zkH%^VD9arUSjb+CZZP~v+DAKM0X$FWz=WhlT=Bn`*RSSj`j)rsXtGp|U?jadXb~bDfpG(6cG;^&fComa z;DO=_h$|1~$O~~_&jSOUdJCHl9gJfGxJQ^E6k_3aNOo+;jI9W)S%MKQ{MwQQWpMxp zGg%tTX-WDLJD|g*M2fp{1D3*_6NoF3HY~6kEsz@pw@}jp90@@xtg(Zc6VV(Y|JK}? zsM?{Y?NXG~rycxyDC!yX_9bwB4P^vU#9mSW!Ejhz)Mpwp+oeQ$`QvAJrk3cW zfn${PcSwW}m&bOzy556^8!b9d3O6l(M&};^8_7INl>GIxY%WNlbj%$MaNw?#{?a^k zd0Dm2-vS+I)hUHqG+fy@{7ZmY&9b|x1j+BXx|0X&tgOxuYg#cNcyHC7H;?n`huRa^ z(4rtd-p-f7iL8_?D<{8Wf$@dma4t(GUnX^hq9KE;Q&PK~%lI@lyA0Ys!+Yci68H#% zUPSyr0lbMQ4@^&A)RM19=8v1>l95d;5FMD5LPN2AL?e(HI z3@feBp<3=hKN$Kh>2&G!t@0*^;qq`v*Jftds^lhOClu@>BK{gm@i6*I>m%b}v9=OP zFN?R9*ug@T*hNlQbQSn*Crbzt*5ajVX6UT&dc-Y_>cAH@MvE`RZtSQ8N*{k$a%FrS#F?*Wug<^VciTSJW{m0=%;NT_g zl#ni77hZ?Q*7~Q~FUQv0brCLC#{6E|Lak#^-U%Sz1*GcLi5L1jneOkBOOH9+ac_g$ zS`3nnNPKwjP^4f|kRN9h6ice77+xT7TP`wn4nJF=g`<%};QkR99aMx_?4C z6_GJ{Cs;bK42VW?#m}(HPVzbBNvTAmV_0ojyk}fz%La~}E|TW0YMk(6zj+{d4$*Ac zkSG`nRb7{u*=lmiC7`x73lTN5XlsXo*$uzz^lHDzBSh$DGP59AjemYxn7+zDceCDT zivOVRR&g}&Z@^*7)=%R8tMh4oaz2)`w0n6BEmzO8UEezr9s1mkcS|P4~8DC`-APPh;tm8bt(i0ZL;ad`*ku9w0@y{+In!~ zT*zvaj5R^{glmaSBrV>{3^`rf0kjsyjJ+pjL+v`p^MkPk+>~yX`_c1#kqFG;HlTg1 zSfVsy5KSl_N~2Q4^QLAmXHwhru&TjISkl0L1h)m^^m|5&0|%4N;gabfxz#kT;!D!r${^h>@iX zo@Y3ovX0QK9Q=PFm^4go_lBE<3TBQHh6X>{KP?pHauvIgTh^FuS+Y3GBj6*lk&woB zuBnc*+AQzgX|Ke9I{(Fjr?{^fQ8kV1A?wQ>>EAR3al&4SIPna;$1?al>1#b zIZPv>=X=9p*Gvjq5zx_5ROk^VSZ4=$M^IuMu}A3#8CX-)XQIGv=YZ%&;(mBcpjRXD zE=pYG&o=|Y?xEuzF5w)`_-kSy#bKH}adQCLz+oM`dEoQsYpP_I}@-d5*w9eij* zy7{tCCh&gK7>9%$M$=VXQEKe3wFf!+I94Y?!{q@>gNTvf>mfiCISHH)Egtgr7S43c zOJVa<>4CmILokiJ5H9tbd~P)9vq6s7)pY)yvqTYNMIq*wj?Ek40rYI(E}nb!yQGk?c?j%NWKkjCmUys zy*ud86~;MWbBFn}Sbb*aqodP2FfBA_^?Dyd)b<$_;So85vVo0zV@2Y7E}b zzltmmcO4^8y{j&d`!^gh_xGQk9WJ;fZecJ`>avWEiNe#7o2(Y%5;@r&CA|maI`=!+ zzPP;B`&e9FISI(#J*_ItUn6MAw3n4jL)`psUH8q_S5cP~xPbSV#>1LZv9S zZxVgT&n2~;LQ{~3^B;otorR%?g_>d+=RaKD*bk3C`PzwW!In&Fk-tPUJ zOf?@;nzt#TCj z27y{5P~wY+lNVCxBV&{jnsbF%DrbXecx3f){3v~A-Ax679FOAoU0Ef#p4y6{HM@=7 z!P%sbtLDYq(GKyFC!r51yE;|sYW_v-#mimi;JWKUz1>M2#HrW%;Ec*ZZ=v*rHCof) zguxjPoZZ3%D;b_%>#>H!eGkD0d@*7|K7UAM*?q(;56FhkR?2!@B}%yNdOi4VbZI0u zMprnfDE1SgG%@~Lz8a1Rk>K6sSktFK%2}$SACOpKbxE@_g-TxqDBRMHU9v)zF{mvv zYb22{Uw+>Bnq{nc`7CDYfT)P|s3%_yF|*#d=&+e!VrG^^7fOdJIf(;Qg0 z*K0ClwSnp%LR;#kk(-}@hjK=T{t?R7u^Ya7#=)D8%wu;yWR6Qqf#;-pyv);DI04Rc z1-Jy7YGt-RkllcCUo5{S53`j(GYl%GhE*(A0u`PZWreOv0ijKthw{4eHA&Jno&>hZ z0Jk*<3k|u-ySntb@n4|m-MhGhsa0XegNdbt4Q_^iivy%Zug`VyII@3ApLlofwmUUl zt~(9fPecxC+BX#_NRLzNbsw==sp?r2WOcA1ATLlj#kn%k;MC#Z z>gwi$(FZ*!{^D>wSTREAARQh#BD`$|F;`eU=+#jzv@?}l>3w2v*go_iJLp{}5aM6a z(IrR2$VvHVz(w^Ygy1<`5A_1G1smw-%=bcSkxRK9Le=q%gR*ku%|=NQ_L_xy3s6Fj zCMmDVn~PulfMQ#wYh0nFnQZ`<-fo+l#|hOJMnbK&_~=Dc;16t=2PXNA zf`yw6*E^z;E!eX&j?MmvuOq$Uv4>4EENj`1gb7wI#@D^7kB$yj&ky#&?KOPB!YTZ0 zm60z1<0BE}={v`l(2yAJz5u4M6K27kCNZn{Jci6-UdONks{m~!?hUxXE$(U&jd*ne zF~nOIBOFl7h)g&hK+K>u)L=Of=hNu^>vipJC2DJ=hz4u0h8S~N>rB04tz6a4(f1{i z?5yiMmA}^b2KoIQO7|~{p9qpFQ`&DnUcL2O*F{RkG~7<=-&0<7JW_Wo)t6h%hOObP z0J}2^Ug!ZNJJO_RGKkGxHhRMJWH{|yz0TcENxv_0rhRTD?eH!dsDki|(Qq{Mv2hF> z!UpA|?@mX(jx?*X45!g^YP!Ry;!Dj+HS6_xJMl1%6~I0QVnfH!QF9*Rw*f?`85Sd6 z%!kG5v=Nd!PKhNVGhNUm_v?bW9u>xtfC43pq*?$P#5g|!@kY0eKILeX9wN}M&Xq`2Iq=j? zB10G32yjPtJ2~A?hcDI{-C0~In;yjheX!$`eu^adT&dEzh+f;C02nqqp{Ydo`{*4iaUc@y4g<;GsXQPv zcO?(%S?Q*wFf@d@U%9;&Z;>?039R3D2u2>b82aW${YsA^vZ%<^`Vt|5&``mY9wN*E zr1JbGJ%(itPr%vkX<_qy!=&Y(M=PADsaUh zGAXFFaElF6cCyCd-cl~Z9ZqN2M*B5W4az0;M&W!ky^6E)Q50#L>IP_ip5*EmzR2>2 z;6!Ao;E*>_AKvyJp6z(^wrXXypLd~aD7Es7`iJEMs1lBfqDSMJ=OuNxLZ;@oH^+Obj_Dd}@Vh@!}@}mKo zAysb*L>-|OSg)%nWOs(7BMt4DT9n^`m$};3Cwu5(5Jy8c8m;@m8OaGTqxwH?#`eFQ zeMCmBi?Y4Sio=cV3Antz?%rW{!vCGT9UOgKjvU_4=DWD{u1w|^#hl|C;3AC8i-~_t zp27XfHNH10*h{iE3EG@lf2gMo9V~DQjeb;Tz$~~QF#>>^$>yCy%U&AFj1o}r8QrT> zhW`FL;m;@|>j#z7{dGR$x{@|#uC)Zk80cthhM2(d_kj>;KBq=55IeYwkayUWI{|`? zR0_}`8$hyg<%$EmkPkE3y+_Zhk!Kz*$3H3?|Co%bFGwLj64*)qyWq>sp0J?oh&SbfA06DFqms^BOdy6=A7*0-gEdlBqk%Ct=wl|A z=VYzWo7y=W@PXiDrL(!+zVpyg;9%9WXR)=-X%5@R9?k;P!sb_Gu*mYkPiB21)DT`Y zWYRJP82{F;KNWMopS|nqv_jS8L~Nm=xiifMo?%``4N#nqu>mH`DU~%`1fFV+t$;G! zMq?3-Y6REGXi^aui6yDxI{ie7FF)RaHF}>wr36spXlOQYg6XS|!?5$>Ea|gSLO%E# z5XEFX%|=P=3dBFcwcbH^8Ep&-8yoy+n$EZUHZLL&NTVR>yIX+zT3)0j3I@lig+-j} zP{N*(<_qe_M9n9L=8cV+;*oov)@OjjzzNRmm_?DGhonl`emPo~arFXx217J5-~uhW z%t4Revr{9r^+wnTtFyIO{gkHA^H3~U(K66NWRmu{J?7`3HxYPocJ1u!_!kW5GCgNS zqErgdPf;Hb=sBO#p_c7duNhGT`V7?hgy%n_p20n!Ee#dgXmpNEKlhJC=-e$PCm~pq ztrGd`W1|)LYbf+Cb1lF41<~VfOFaFvr9K-`piqGWJH9f1S3|*o>;i5&s0irDnMoc6 zp+W6lg7eG!v}t2iVSbLS^L;vae|5n$tWGKDs-;3>?@Qf3uZaadO#xnBFenFdfP}|r z-fTa%oIFjPI@y%NLfeP-zw*l&2V;ZoQud$*W@J4r4Z9R_N&vf#GJKa)Ohyb&vV~3} z<_|5ou99O@UECh3jLmfo#wmZzU)hsGPPCZyY|FRfUbQRvt70@am~sby6*fNCB^M)P z1dHo?H|)jb0}`b0MiLFu&FX4n{Xy%DlPFsB+(8|v;~jqU$Xmbr!>`|2i(?pcTQW?z z28AvnFyepTV(1H8@R~Lfj&t+;MSn)Ur7T$Ha1i8G`AiRr~4VwudSe zc;yVmB78Hh!=nZ=GU`IT-={H!e4SBFUc2wqj8@S$4(+e|GyGMocGH0~4!Izkp)(s~ zehGLG4xVzqq_SN+OW||#=z6oWV?lyniWxnu|EEQmY(1ANAd6N=w)+7GeFGF`pk2pL z*x9bu=Va(9i**q?y-#Fzqu0s$2rcU5Ahn9PF&iIURK)h~a5}V1)qw8NE~cu+HMcIm^Pr zs|p!c`#_exBZYjB4A)uRPBzglK5>!Z=mm{*h>OFEZ|Dbly4r$6|1nxX;7x>-|Tud0z_RBdn=qHu7CQNrfI z2rNpSy6Qv@Wu0`MlFK`d6sw~9M4DJt?MJn8H>0zg9PPb*h9xFqQ zgWjZQ6CC9jZX$?x!ZVlefTA(MAS{Wef|TVv6-1+59?-=|J+q`Nv>JGai>t%)!H-Ot z*wAJN$7qCM$=W0;`;{8ZaZQq?;^Az~i#H&$o03vhpv2crv09Vr(>3#4Gk=7f7aB8V zg)*9|(G_28BBH3O9 zibkQ+H^w{krhLt@0C5nYH%94yN6hDF901MJoY-QG+hSb%4V(tV-$iraIP~6DGDKtz z=|qthnz`QeyFHbXj-B&`xssfJkn>T3(aS&<;LuEKN$IUJKL(qKLg|<^GhxIRs|XhB z=0vqRl1$goS$^i#oL;(tgz`r8WmAQ5HE%9^?0T8a;+MO`oKk<`vc zgTMR~?i>sq9L_{8h5$$d74^M?-%{~*-G`;B#*MVy<3yeoCW`&`6i0d&hvlymv9pL5 z=)MSL>!edKe0!V3@gfp0Xt49`R`nbo2<{Z!ZPCAD^JwLUMMm1tl__lq&p`4zBoOY< z?XGQak7pxG1g5@k_z|N}L2xa)8B7K_I+EPJZ{)vYirB!G({ExILdN~hSWt?~&e*he zsOorEPkf2uBqaY}|NlV~3W7y%BQg-slr9hu<$rz9WO1$Y&mK=4{ojuN=t$}eH(<*) zG@*klTxZKsj*NfYo#Cl`<-^XV-mATPhtW+2{ck!-U=>DX%7~>fltPlwWfG&65UDB= zF%?uKBML~+8^!(U^v&$dtY2($(1@hujhp%IXAkFXujl8f?P(lg{!c<*badmr`XU#Q z=lhX4H^=+=)I3+PRM;6#zdRm{zlF`ch4EszZG)N%9mW(c!($BJaD7A!b4~oB&=9;3 zWR0*-=vN+@H=bwD&Ee(tV!2ENO7UJhd5#(~(uZ2q*cop_DNq<67g#yemk@5hS3pEE^Qcn?y3U;{w`$7vV@+ut@4I046XmP_QI?dw3`nQcgPP8$TN6b&k1Toy}#=~c0mEiy;{1n8w8 zekhgu=i*)*-`CR6*VnLlOZTeaj!*BVo(}kyb4T~f6@sm1EMRm~OV5G|ZVN!~!!ZN2$uYV26o)@S=561i1z|E_6l>T7V@JgdRoVK+9k zG_39KvCHA0$P3a;i>;OiOM5XuLtxG#JZe0;ZV#pH_CVsw^9Z%r-L->Vq8J{)d3WY1 z>o<;(b$2sY>cI3UyAJLfbKc53zrjl5{L>M}&9(#N4Uws7`c(!Ul{EIq=xp$AJf3H) zaJFZXV;k3qSQfpo-W1+<>4EhlsE+L%Tp64QNxkiYjZueOC9EUi+y(=+lOu*XwP^+FpXSZoB{Sk2-^1zx!J=V*L)c$Df;nA6nAQR{La# zz5n&cB zJfD5Xa~dJ5!noV8A>>;SS61_wN0AS!?_rYiN0Jcj6871qn61VMha!dWtr!j`kVJ(n z6E9MWX;+6NjdJ#WK*3sPh4yW*ZpHj)UldAo$0do{#}7j-0He}?A4jVMZM(PGMhu~( z2`*xT-0X->Jf&35RIwvA4q2FQ&L}=XJ_i!D&TAhh)Pvz60Fw_HUhY*X7go3EhysWS zMBNPB6PXiai}J<5=M1-F`hDJ}P=i&B?4K*QF4gzb)y^Mk5OF9@^ie;H2tMAM5?Hv~ z__=;z;$p!sM)pOu6HBvVE3zMw-FPC|U{@hKV#ZmNH_f2R|I!vguczOS>W2jlDP_eO zLR7TT5JbwXA%*_F+k945mSOFcAnDM;c^xmvr4$4vo8tYx338JWuHMMy30ia zgYN+R*GF|%OP~uj!p@?dP_H-3#v;6Z*Xwog;CtGIbJHaU`Qk0QkdW)BZu+&D;h6mg z=(ewbbh>kn8Z+DNUcdc*x4DIQ>wh;`zx(m$Ab#&->3WS|yZ_-Lyx&ZnYLtNNS7E$b zgRI4V1j|m)z#bQ5M-7wZ15(FnBG}aC=u;PLKa}_W-@m^YdKsN~IwW6qywi98k;OCq z4`t`nEeNnA>uuY%ZQC|_+qQb!wr$(CZQHi3>A7#`>CSnnA5d#$=BkLuuQttMvhIsR z8I9tzCbF4z>RQ#Z2}!Ltv;@?IRZ$8;18X)9bv>z?z&~8_@1cHR?%yRK(5C+qjN)P0 z?$w`-JaoPn({_jpwPkMvB|#{8=L)}Hi?b8W<85_5>lji);aFfgsiXsuMCbjWSfHkr zq$(S9LTO}jCakX$fo)A`Xaat-xzR;)1nZ*!f(q7tpLeSdNjSY@^S=$o z{*>$w!Ityz%kkh&F=^NrmQM+>MTZF*#$w+f3>d>F)F8^?fi`f|O`ebyS)34OC=>(G@p~ z@D^C^VG}AU3zp00jGP_l^>}@&5<$`OPs@y~BR$HM!dUfQnKhiE$~J<+Y`V4ge-Anq^w)c_M+>jK4H4OKv)!I5;Q zk1Gg46_?N%(uqFhcIBG_#;8lcpfeixKqhPm@F|Qxohgq+y76~l@(Mk2cOI3du@6`M zI==c$sY#fDytpJ>IYSlRh4bY7k>O1gK^mVDr7KQKqZaaRz>KURV z*dH1a@f*E?;6a{F6~{NpSj<&Yt`#?zt`#|3xxq^ zFMJGFr!&ow1@NCESIn6fCs)kvixp=JLL!r4!cqY?dK9&;BQet@+K{Wr6jLs@WE*}k zQKnTzJ29RhKaQF%l3ACDNH9(&>9lM3()3e}zfUr>5KySYek08gO3ay*J6(Y<*FD_8 z(VH<5N^qmsm_J)9yNWn?JW5)I3NkSU!%#+LB&XK?!}8q=IX8RA?w=)er4G6)a^$t% z7=qC~MV?a8l~Q?JkQyHvIM=w|FxwQ2`E;MuSfy_3oy0`1UK~Z22uL`bP%F~=Pe~h9 zB%%Z5+Rw`~twYDfM&l7MOPvpizg09H)p!^%^P7*`H-%TGd-c za=Y7TwrbTwmWVU5%mDL6GAm!UTBBr^k~K=UH84qN;!ZS6I<&&e$5gJcb|^Uu`q+W8f(G_Fa*>5!y9aF?=NvYCx*S2P1s(*E^#6o%G4|l zBIxP4Vjk4ODTlJLB29>I5kgRh36ME&o;y?5f`EvR8=0S9PFc{B{(|;Z;ty-tozwkHC z^v9U<1u-#PB6rA#a3<48BH+&usAzQfjvvCgr>2L2%f7YQ{F&|3D%(fBQok=qu^e(W zTMXsrG|iEIKc9ry?ZhNQ_8Er<5&Rz?5mu2JQYuoIbsc{r7Ty&_oR{>R6qNG59L>a| zm_sCdNHsx!ST>ZcR5CNPHix0TKUMS4UwXP0Bo~~L{WDFpyIHq^2FTL|^bB##*Z$|= zkCxdHuz?QDHbP=m8Bgp3w~$~Q6hy+&{Y58cjM>tD(R8)MP+N=Ataax z)Y5NtoCNThKzc+R7eclNHj^cDj6;F@)=;>xrF`ctSQsi^_l)J+01Nfrk%}7*!%bIv zGve`Djot6py}q}nsd~P*uHfwMUxfXU&kqpsl$f>r2ugjE0``BV7__bM_f& z_}TgJ?OeVoA)|t77@T8=C4ek&kqns_GKSQgjFmeH95sSx%bG~#U&D~$ zmq`BOQAYDwZd$}HUX-*{dbaB)rD`P9bN&c;6{@zzn{+2>8cbmviPG;|i46e|`xDsw z@1~&n$(o4vu{Fkw>JyR;Z^c$xO4$^AsgnnkXqYMnR|0>=A-=iG=2XhlJQS;c7H$8B9J9HegseFiCBNjjBKoJw*Ff8hv zAkd`HObbAdHln2suslKab%2Qd<<>b5SSlBjy=&TVf?{li4$@B{XGk#NWQ1^o@paW; z-Q;1ZYX&X~q+l36DuvVtoGeo0`s1Tp+i6_uyI|ZsU$O%QS403}wa!Ey_dE_&PiFoqqR*9Sfw33U;{tS77 zBL`!uY>4JFyj8b}|E^~R1r z%oB+0PQ@)!$z+) zjGqM~=|%To4~1>yf?797T8?v|`Gv4+<{suA*N{VzEM@4psdQ_fu*)Xqi-rZk-#uoU z{`|Xv`g_`3)KtU3;eEpHV-E*sCo(G15NLavI3Z&(b|z1dOrtEgRmm zngSRM-L8y*Qf8FzEm2xV4$!`R

(=vHq+Am<#k`Y2`4>!E>a-K;Vdm1#OGzM`L9L zi_oQO)&T*^h9emikd%kW1W+mDpX-$r2M3BeGiHhFkQR!*a*P%s#!9}Az$&m^ z$O9>@0%(R3&Lq8 z=$`y68f4Y966JP-`H*xoUkWOCZ?rwT#mGF_8x|Y)6C0D4FFjHUXM9Ltz2iXWpK-jO zP^^ydRSN?t2GUjaMd$LL;OJTjpY;Q}{9s7;G^UdY3oohA86J|tPX#Z1Qrz)py>Jux zRdrDsWR0Dp=Z77Q^gdjCY5wr69B~U@FOOLpqz_)h8|xI$85BV_2Butlo8Xf39-I$y z8a6J#jzIGOb~5%zczb53Tb9;0uB$~jJL8jbv<#_NnUdxNlTVHv!%v_=PxvrAom)7% zeI_aLkaX3s&7r+dDLowys*X3}a}?1RshF7Mr!t74C~>h=xzXSh1~mSN2NRuc7qrlj z1{?5*kU@?$sr*_RIi66ectQb}?*iUQlQ&bYEn$rQHiqw}F>esxh6=_|txf!Z1_uTa!%hQ^?FTZbFHf01NIs*uPhAOl0Upf4^oSmjLb zVr1zwSsw(m+B5D9fR%N12A4I*~6S@i(c^jeRX zL23*tQWW$NPr{`-?t#)-i8LB`{Qg@>O4R~qu- z5baQR2t!Uc=Ozz8`AhC}AqQh>ctAPULqJLVO8pzU`V07a$u~=GHeBxWdaF)Hk8)Kayfo-(pMOr9fCMp>5O>5#C{IY@E<^B1k2J=qy1l&JF=crbjhR;C^gdI3k;O%Ndprr| zE;*CnOzO_Fc=TK;&;R-FT+L0WjH1IN=YuvQN+)MDgF1@QI?@!+3$$G2F&3&1GbS=j zIKw2~P71lxo3V(ePgmhVJbJ#TUJ&sQ%yO}J4nU|RA%^ZCBAO>rhGA$UQsy%Y;tir% z4uVD^i*%kThrrZuWW#hTy*v?ZjvVJHs zva=OIdE4Q;f*s|V>o?dwke$Z-Z{Lf5wk=4XoLWfPo}viZ-bpT%BtB>)0Z^X&>`uvX zT6BwuD*b{q8ODI9rs*hAX7xd}4O~$jl)y+OF7PJpX%zfO)^!WN72&9|$CtJ1aa?_S zzKr|svzrk!W^CQwe&_Xb5U`3)!u2l2YaTzV6(Z2uANM6rDhgohC`}SAy(t>AvYkP; zN}9@qF!upL2-!i*NbKM!M-mV*K4~0Oy{!-mXUIw5#H*imIiv{@9qR!r8r=&ji?SWC zb)xXjZN^+~+{j_oYR)$V(T!mrM=UvraKcHl^F%XLW z_pYf(NH7Oom&Sq%y6s89M)r?~iVVAX`H;w?XG?#3>7#Xa$7Y^xvCMKY?5woD6D0_i z1lsERmEhpdVmmtX8~R+5-QsH9ez=I7@EoYJg0{UghP!o;6V&J}jWU-8j1zT{?1D9r zstC?;Jc@75>!se;@#pTZlWb{xWmEQ=Ywy>hFSfC473P4(($EbXr-+$mk=!5&jz6^A zFqFo_VJDcrPa~LzNF5YO?Al=`GD@4gTawmIB+DW(nlNai%?aRQx@GI-LWr->PPHH$ z^EYFf)d9D_r{TnrDA@q)R@rtA5bosLlJzNuC6}^nD`%)Uu45RHeyjg@?MP#c5UVl_nwjk0(Cz%!Xidbt`0PgFX|`#kDFo09Wi4TrXTs?!V|Q6~LEJTqigU%7B!r)A<-B zo*6r^2GLj0hgnpLGkJ{`e|wPdPZh~fVnMZaw9k;PG#I!CXc+jZ7Ff4v9d?TmhiqK$ zvdZn{_oPoUdqSzS63p*&P=3&w9N5~24!qDd9_=-cyeyp`c`ep-zc+omeSc4HvFGv{ z6f%hhg7l?B>GY|WU+8EYu3&0rP!<7$hap=cfcmWKMA!25c-8&^1Yb&Ptt4Wcu3=yh>hQ<0>;}>pe%{neTw(|{NUmnX_R#8h;*chYv z*!7TwH%WU9yQjA02JxjF8;FNJ`)q5=qSL;eKBDd+kH*cUW3L@>J3c?u_aR(@Qx-{fkvVy)hJwV7K zWJ?;jLw~SBUtOEo^0}U}MzidE7Za9gZ7n+V6t_oLybqGtU^2EvuM2b1E!X~OY&pEM z*a72WK$F>P2nkKvfM95~ih0=)vqCCy_94+;eHzcFtui}c8++-+pV2voDw*XX2G+lM zfapA?3vI!{xGL4FQ|+e)L!D%(EXD4i-60H#sZA&~QiT@j8wE?ihTFWRcGK9Ndul>5 z=L~S!!&~VFPs55<3@yauam%k5qf`TZK|0uE-1Pz{I7I`);M-TlzdJvdpUSAz2U^A_ zSyVQv6#as;CiWTlE!zZS_5}aR*U{u>@F{K4`+GsBDPRi|k2?k11VX$;v;08$<(}?G zvh?-APulm(c<930QuUE$;pzURZ()}u{1`H^V%q{QE=7(&2J6xpy*lb9rtCShJQe;{ znH6HOwJx`{o5N}}+@cbbzO%&wTuMEoh0BonIk;5XZ@5(1M|gfnaILL@*-WD;=nnL! zT^enoCKsTI&5m<4qO%Due6$E#G!2lSGu7E$quq{u#s>hpYxQ2s;1fP(QD!>tFi4L* z+pl|q8sVK($sf3u&L+4FI-N#pP`o`S6IV+4J;)C))Jcrg(id`Unl%=tG5t3U;Q9y8 z65y0%_H!Bk_v5%kWp3V4Son2bDx}mPR>5ry!y|?7KuyMdQ z-1RoJ+}Om)=-)h7ownT+L+E*=EMd}L3n40L zSB=rBOF)8lA@NtiIufZG%Wnc`pS+L4w5-eC?reOMW2zC zxnVoV5~pApQ3%68O+yVxZ4C3VEP~d{ThNAbTk?beU&&V%&K@+%G$e)|TLTMXQ7|vU z(H0(12Jyl*u=RruGfpYZ3WqS5p021;HalO(p})j&G{9(Qh8jChV!`!K&v=?h>$>f@ zeC`;PYTzwTvDIw1c=h7JkPVmzCejAyL8r8Yh3chgD=zPTu_A_^^BpE5T1~iNm zBBGY!8Vw`oDOee5(3cJ>_6n9#5RWUIXlp@U7Ma@tmT(_|WiCEf(`$*lL(*>@;F+ygcl;NXC@w6Y9Qj-s-EAQg?G{ zc#DZ(xBLzr&chV2=j{Y@#$ET5iqPBcLS%Cvh*-jD<*Ezd7j>$OW2!#ls}lDXyqrRj zQc-(r!UK~nCTkOhUQ2y?XC_`A{|?*>*LiJP=Hs#eex3Z=YoB`Dd9yd>ahs!?c&`)q zr8mCAdCE=Zt)AI_P@C3&Jz>9X=llLWoyouZa{Vj z|6j0#s-PXVC`R|Ex)&3Y6wzwtNeD7cItc|@zgWHWaw6+`c9r<^4GeQ*TS_uMaE}Nf zQB%Fv0*R6WkpyfIMfg^jGCXYR0Hh)u3(~F|uIF}Az~0DsC-bf6t@mxFhL({LBSxSF zon=D12`uvA>TU-cBcB8A36N4TOvDo1XV-v^80tX+WZEj+N?L=vsj{9TIM8lBl5@I{ z7y`Dh2RGzfoYZg|Jo2~$2BV*w9reF!rm*Azh+XPC#R3wI52XBKhzj)-BO45M5;)DP zXfV`Dsgh3T>f%8oIp|P1+Kj@^Cb?>zAiY`<#9%W0 zkEP@kyl!RQrfxr2t57Vk5DP?BzdnqG;2&b$5XMGb1f>n+KswAj9lc*+obhf8grc{g zN>rZjkhYLO3+6#lEA6@Fg5npBh<}g{WDM=#My@=Dh7@vS+QO0As~D7-9vm!Fla@^@ z-~(9l*q(09m^r(@*%>#!9q`)Fo(`fQ1MPB#ODGF&o|sNjz8GCCa2^*-il?a*uwfm( zzLs$dHdMU#5L^CKPmG}AA(I|X#bQ1KsFF6yGQV9+?TtxD{Q~xQEw#6I=Lw&ZGNev% zrcpyupl!HtP#}Us#F9)J8)nSON0IqZHflnPOL&oD0Gll0#2FORo3oWa!S|xw2&)lRLtAVzq$H~XP^YOqrg{bP23$bRy=4$f0BWnvoBf5XT5(Nn)-vHzgJs1`tVi0CLK zBu#VUDO>Dupp2L^gnG_`Aa0Iq0l6dg3h=zid1cC_km2bb4`JhW#6}w|q%4eb9^bn& zTFUV2>|yNe?C!qMw6epqzWG+&-QE1NJio2?myek^I43=OD8Qbf#x3^BeBoU#_h;z7vaS}+}fb% z%-t$ya3oSt(Nd`Fc3kKh1%2OoP{RNB*^U6xbhn^*J=cOUg&076%Z=2J`t6>=}pX@GRsR3k%LUB`FIw z0{L=EA(*X;YGUNMlJYu4cTLTxSFsVRCURLIfa3$bDY~`v)pls~3CIjyX>jzlXes@c zvYQh}CE69ZxNzP}^Zl7K{9SczZnr?3E18*GAwVw~g(2mqGly=BUukq6xznuS^n8>9 zdCZ%H_?F{UKmLh?o+J0l%m%UfixOhYIG+-#id^G45Hhg{X?tPKwPtZZVU!UYE@G<( zB=Pl1MdX?llNZu13dKU-HEV z7t^1oj>=KuH#`d5j5ycdp*K75a}fpJ;)THEpPz%yWM`#*n03Ici{wWy&kwXUJ;>UW z9FO$9dfl0mlk!rnt7xaWsdn_fS~j&36lv=nZRJ>CC)1ys;sy?IZ+0(=6>2S;#7R?tpwME^n+ot8=ea(GEkEw?+ zEtiKrR*Cpo0+JR>=BWS%p{(_G>I&!oz*ws=$^>i{7o^~kMr8E2g+sJ?db6zo5<7nK z&|~Vy=OydU&w*V{R{^vgiA3533K~z+$^l_qNW{!OStV|rTqnWGL}O*8o-Z@I>L0O$ z-TeLZG&Dih@4(T48V{nQ zF+`*S;4ofOlf<~Hv#34^w(KC$z$>7FcI1H=Z8S5YYHU5ZZUI7$R6# z9+8)5oS`3|hP@?eUbo>@Ka>=+-0B>i%|Gw9@0AfIdEKkMK!yI~dS5PtzJ(mWUN^I@ zzFL2F+U!p=x0d{aJHZ`Ko!*;E8(mA(y*3Ps%yOF@G{Yj%#)6}KR|fmM^~jSsaV+xX ziH>v@Mv?_ehu#RpkDiy|k4e+k0mE0sKl#DFaX7AcIS69+ba+CDX(v=+80qo{nO4@z)W zabTNp$6&jGiQz2(g-zXE?n{#!`NaKt7Ko)0zy%tjIb+#AV30hPFbYEm6#PAJg|PEt z3((^a3S%#+uA2Wk-!Nw*R^nsc1JFV&LqH>;REjC2+G0E}%ao3#w+iK6RrhjENzvLWqU+*K_^KUU-SN+{ciOLw>2H566kqHgn$ zNA`y-I5#GTS=z0Iog+?4v*+x1xt+gnDZp>irBqV@5z0YQjFi*(WymM^6`D(g-E_ zx_m}fUQ)-NqM_NIl+bneIF#0fJLSh;bYZuwdzI&82s=(aV^Mi$>?=Spzz_(IXBCCr zKPOgR1`}nzN2A*gG3FS|$*Cg#BFZ)pka({0RQ=8^S-LPy`OUA4oKCZx0k zceTVkGgI+xn!$Uo)1J0E9iGxqbJvG_i2at>{9H4*yI=w#J^AV8%!8Mlxiqwo?bDWK zwDa)k`O?Vtv3|dw7O%ec_Ga==Rvu_lX0t1`f%F(V3#l@9=uDkfoH)XoILiF$;*Ivk zPHJRja4yN<79DSi!Z$o`q}`ITs^M1hiB7crZ8bG*xHeqYK2H-_{Ov5ss2T6|RM_UF zEWewc-g`6%mzmC&)(J zW#4Al>|5*BNBGdt0XZX& zi?w^)>g(L+tnAb`bv6nLT%J_0zMyv*IH$C_zmY%6Wo^uzu;D6T!RrAjJ<^+!mTXCR zu1!rmWp4ecJdKH}sYj`XCXFo!=OlVOtI)>{j_$+QQS7KBY$y@lUa8L|bV-{(*eT0p zT)yja#{GoD8RyfR+zcZE6)kCID2CyP>=a`D46>+&5UYNH(H2467*zllXUpz{j$hhD zTPC?`LP4;6?F;CA9WtrKdKn_sAOS5$Cb{cIKsoPAaL&*~U`Rj`4EJ#Gej7aO`Fg2| zZJ`F)T0-80>GS*)3Lig`HW~O=pYx}jxN%Ye+3&e&ugQbS{}TU! zK7}&OBTMKxKqiAm)%}%g0s7>6F3MSuCC7}o^yJj))H|Sr1Jy3Oh(Jn+Kx4Wru zvm)9O&zls;6)98xi1VmIjGQiG96iFjpyD}t6oUzpmZ<%jlr}+F${!y%P)76m3VN7a zwewZ0;Wc%kfC-md(8_y(b?l2o3$)iw%rNxRQ~a4 z3%|T1!(nBx@kcn5p*WQ=U$5w6l^0Rn^{%io&$HXZaLUFR-(!$n;v|Mj3}y*xcwU-J zM6wmW0W5!;E8MKVE8Cf8Jg}@JM8w(2o@Xa{2UN5-_x19pGD-k+sq1w+Da^FD(I4u&hDOzDUQv^dsqTW zVWrN)hPW7mYc*XL@Hptz3ABsbVV9WW(Q|0N6+B|p=}{!>@(=ES5*`p>7wSW zeWG?egn=-LqfJzVm=ip{a7}tsS5p(DvpK~h*L-q>`Xo5jhK1B zpghQ6r)7*Z-hNsI?Cr>TK`s0F^V&=!fE9MN&DCArhIn7;@{vLcbnn~J~hQ0 zp8`7wgKVSO3<P{lQS^9I%?Z8`ZFtAt8R$L$mf zsND8ZnvTy~QoNb!3zSOO;gIW3SM02{MfRu(EqaT`Kn=!Qh+<1bDy3-xKfvr9n<4+q zM0SZ;$3we%2r@IZ4b*42v&jyTQ+77mp^{#GmF~TX%7N!f?QJ+V3R7q0oPVcn1_QjtYPoFS+_v6Q-4V-K zecy;{6A3Uk=#y9+JXFp(T0wpS?;6HE-QMYPKk`I))UVfR}?ZE6xauuCh%#Y|!~ zfmf_z(|O=?J5$VUQ!x(oR-sL^BQ;lxvb)FCz}FTE^?1kfp>yv;V<_id!pWq+{jtCe ziGF!`AXB?l?I5~YtzQeZ>0Nq7pb6KqHk#dh*So9z`S$Pq-?HrO+a0{Eq{}we{TI{z z5$M)z^wcz8mB0+m@Ao(|ENlQxlVkO8<|}2uMqB}i^I*71iDH}pzYf|U+-v+Rto0u8 z#~adaSpei=_;m{xHE116n=b87${Dc%)`4jyxY@a-1^CCV0BzixhxwO((p`Ugwu zJC8yD;d;wX`tpY@w2rlKPgO7XX;^|OT$Cur6>VVME@2}8r*Ji3j6$pe_4{x~R(Zhy zuUb8iUXbC!(}0CZm_JeXCU9LQOcmlbLDy)C}Ml<)+9@%^kcju`v7jIb2x9 zr?qrSBoC$z$I?xNH=cF2c&E5qS{j(I!=rnS8l2KRl&F&`iOsdP3S2+F65#5~yv%6O zTLXF8*I?uOWXEB<&Anl6CXKT}y&|(zSge_vWuyTBX4SdyUzgsPql>KfD|cjSfamABb+qgD)T`t~eHY#rZ^Nd3Ogw z0K9tY9l-ola&*4zdKqwY*B_R_ruXEScdp<_BjzmCoJK;E?L3D>yCL5n?8R28cFcW* zTc&8M{JDA^+CA@OaFQevr4n{7luWnl%0|aj6ry+6=(~Kn&y$f+2!M3X^Y^8^3cOdm zDqfTLnlOpmZA0FCBjE((S5dEj_j`%0urh0YLq%*1o+$)7~Ov|`u(x}VQa z7}X00MYn%#O}>-fa`&T~)fY;uzTo)$ikycIpJY?7;!VrA?LhV{Imj){g-oZJ8j?x< z-~q~gA<3|pj?(Wl8l}G5C!&9A_c2TLFC1X}6$@0KrZy>ko$gFmYbEK_hmed-pW$NL z@mI4wX_G|PlBoOJ;njBG#t9y5nCm4j)^EB}n^~w&s}^t`c`tI9FC#2e(KoH8Ds!F^ zO_USGNJgS19gj?9xPna+hyt1CVKU6LIZ(>9!MmzT*wDlr&P&Zs_I$0_|oj}KpPJUXt zjb);FB%%m{S3OP}5E3yN;q$STC`}kd7PjOQY~3OL5+R;MGtb)h3I;>cLD=2-obyvP zzDLk8>DO;k;~7>j*n?J{3?9fPjI{h<3t?^ft9h3~VQQWHHK&?~LNAjul&kGWz{6^} z@gzULpO9Xk$PC6S%1J`9vE5JWB^Ga`^^&Odkl2;OA!>^xWT8nV+Zz}D6KvaR68p6l zmZP7x4(es3vp%S*uChA@Wq8vu8>+)~Fj1Ylt>q}tRalSie51}R{lj6}G`-_9(2($a zgI?zzs+EQc-Y!hbY!7BtUDWaKC1P!^u7K<6=*MeyK#QoCNf1pIim%0TQCMr!AK|v> zQiUQYJs+V#O`DTZa4QUi6y1Ahp^79FwF&v^eT^{j{6ZrY*EKbO%ZszOPvwi)Mp78& z8S@G(_^1 zT@AU6H56Z>zk@)Z$uaRR@wQ=g;HvOTZWc4Ho=+S*QT@A@mrRFWez@>iHm}jpC_xkF zH(z7Y>sK@78eP7VqWhTX`iJ$kfnNBCf2l>+A*_P@Zc*9X1VuHKCWc1_c0khxu>Z(Elf-npdl7*&&Og_`KF~ zuqri)k(HEHs3dkc;*x7Duf=rxKQv`jjlPs!ZeQP6%c$HN_0d!O5rSn*O#?tG&$Gx! z+*eYq@NhQ|mQtOULO~^bp1R_4xGF-3(CcD8&ARnE&16m;`uRp&gU8%Vch#t#*`Mog z-3_G7WuXL)#&VftyT&TS>xdd59Hl;^BXXR|OA)8u(?Y!fs#TWY9_s6zLkcZidQP+H z&z}#>g9tS@j~Ic24s+iO1xp8?JlXR^VN3&8HC4(ZhqS#%pNpA*Eu+N}B z!c>0hFU|j)9yvTS1MCnNCuka9BAKDzDPTz_irosjUoy$jXz6fwMP-MS3Y7-+*fweE z-6LCz$6M903y4aLCbDRvFsb;)prbbF0O%;QTmr?6g(ICRb>3BCI;lV*SQ3Y?&e|L| zIuBCxIJ!^F%8v&gVQGNpG2kEyDv=0DY9TtIR^p5uT!E|bfMZ!l#azcyd6I|~oWU!u z15FJS+5vkKj`|#zcLvO$5$G^*yriH*q)3FcK3ZX3K7d%HN{1$;!J9~#%G41ennVs& zO#ER?sgt#^oS;i_aenWtIGOT|u-_!W8*$`-!X-X0cWQ1NlR!vlLd)2mBS^w9ddJCGMFQ)lnp^9Ov=gJroL`V#MIllMD4eC;&|65Ux?dDyQYdjDr~!SoPRGDUFZT5nb$LNF67@ z!g)b^pp9Kkrhn}2ie+Ymfmi?5akrUZtN?rH&|HO#GIb22Mq&@+W;`)|jAYsVx(-`6 zcB%w#oFtWF+}JPFxy`ws<&OhtI?NHi5t{ESZg%NQk^&G2H59Rs_!dF!9ky{Na_kaW zKY}lPmo89uq}EzS;Dm~7=9OY<_9M6ew=q17osX!% zQfnVsBheqjZm&)a6aktAX^H3$=kz=A;7We$D!pHaZ#IFAb6T{cPosCq6^Z&4M)F~Uja$l~g zGDeq^z=skVvp!klXv4(?mB=1GWU0`zVS{b@5>FF{!{UL(HQQZpM>7M>pYU|sztzh< z&zHxiR_t2W^msYG9+x&0KOt%iQhYq&y2stP*^O>jkN;v+2w z%2F_5V)|ngdvjDg~>l1o`-y zfA$T65Q@WIrQ=2Au37G}uYKKq_dFcgL(_F{>7jFM+K4PwpNXK+`}Hv?Y$STmZ(<@J zn=d=`bn$}5$fW?*io3u5lnsB5Efu{S_LxN9nJ{=ycpa8|-PFwg!@8T`G!W7S z?+ndKI=;R0!fLSQqbK?-Q|(6vxr7K678c@Gl~{*TmMl`Ae-g4Iv8$F6ehD1K{h)ImUySefd~IeUKkHD^w03SybUKStoM1i zhJ|?53)&U=^bF`jYzo7<;guAF4$I&n8ZTyDL&7mrHuzISMTwMp5c-b7C_8qv4g-#P z&qNMAL=?xtIZFL6x~hoq4bb2fqpqRaxvY;41@-Pl{%-JKjPVhs8SejwXTaH&{9s!q z0VCdx812@`<8a^qQHEt^2rMl};x(8-V-fu(W;E?F>Xsz`81%+uJlu^idAgL7BN75S z-6@5ela74CD21E+fht#3h`5Sr;OI}e(NWNGI#pwUkl#cw6BBhbLl+JbX^3oN!o+y! zpNS$)QWL#eUx?;Xt2v5MXbVv!6S|}Ue&H%1e#U~LJ*nKj6OLdGF7T6u898ML?fYP% zMRvZAho8gP6SKdkIsSn7f=pY;coJdaXD8(}~#;MrFhaDo-f?8y~Iy-^J28Ivr$9R_8*cx9qH{@E& zO7AavEqZB4VcN;W2K^O7*fZ_-b}l+Z)=DHGD%2CF;qcG(3mUo#fe5HYT^qtJ-#9}g zxLu9oPTAQ!e0|X-9SYtm-P&)9sg?M%yHqkfJK}TaCtID}T#V0UY@F>*zK9np&UO*L z)VkzzwBEWpguAI(6D2txU6WNA=BCZNMMs|e#t?)#DJ>&x`z zNyoKRontL3VkN=h#et!|rMs!GC3sWSyze;Og>OF#kL_B?z_Mqvur&oG+}sx0;Dl8b zj?7USj(VYM%mZYyR(`)a)$0Io!7!j(3?xZxFwM8L4Ay9dfshY&44e#ULlm7yF^!#p zBM+9)@Tkq?DEtRPD77D00g~m`;LS{(^-;UFwd!yL?J6QDeFI|R&}_dEoZyNYy%t7c zOmW4@5K<<>1I+`6-@*6O-4p%W-_f@#B zk5bTRjs>dHZN%S!-Z_0QH#H!xzW+)%=z#2$g*Tcc=s;=x4ZK0F#TB+GQ?Xaz*t;Y zoCXx_!N8esF}NYH<8*(oW_Gr~OJXzAha=i+Nn3c)KPh{5@?GQ+dH+3UnNskOF{CM> zgLUzgxT$Z$HqJFmp(`=rI5zXrF}!rFn7e^5P!~mf%FsQ#)#|9;2{)keP}jjshm5Zv zeeleAi9O4NPB52Oo?)WvSDT0lEo-fMS#nsyUxL9m_D*9`LvNDA2(e^Kf+ThY9*B%4 zN_1Kj9im1O15N}>_yLrE%eO=?z1bL51msEPuSYq>a)npy3P7$3BZh;qH`uOeZVg&l zbUYt;_YL$9=N1yl;PfvWSW#-59LEyd$jYh#-9B;ogSo}POw~}20ad{ip@a@~V0Qub zr8R%W^=xPhP51(7Gu1`o#(0WcVU4Hc7!xF%$qMx&RF@hVDLKg;PQMCJ&K80Xk9%nz z&i$S)ID?g4__@n1&iF+^u_9|dR5@#N$bo%R67~EMuLEnCU&@oS%eQk?&G=E*1t|jb zy?h|4J=#ggDTt~IIRY-lrmoa~qi&p|#Eyzt!UebCP)sli!&yX zUN={%t(>eloZ|xQAmJOwF7NGv#D!_$p-wv-CHhHIKzK^E|_0R>pN#Cj#iaXw)CBc>5GwQ%wjZt3h6Q~$A;cYY~< zS(-)F&*sj{qE&nCBP^{_tVRM%Q?2ecfk)0HxV(rqJc{usX<={?NIo3J*wi-VKk4{z>S$SZCifmS@d50A- z`R!E0+oDcu1M>12o2>)~^Fu~8z1t{k`QG>0F!<32 zK-t)nObs5~?=vTa3=9nl8!U|2T1`&_zy;}X-znS;wnltEPrka)tlMI&k5{DL6RXpw zJ=kxuXk8RCVN=I?o&rcLva#{Z_TjbCi1Hyhk zJ-iopp5k_F4;@(S_yZcbKXh^RJXbeI)-w}ZUtb46_N8x7N2POTzWl7Ej?v7U&N!aO z7&^0a06QjSs`vw9%J%KxfY>Bh5Nu{D zeq(I(oGtIG^w^XMAXiyaa)^51nv^x2aLyxo=eVHH>FYav}x2j-T z1_>||fo_sc_*V$>e<~Fj5~#S30(N9WgkoQL^S^I8G1zT8=4jh{85%u3eQXBW z5#diyd;W*8cMQ%f+PX$#+qUhFjVHEk+wRzQ(y?vZw$-t1b&@Y{)va6ioVxYBfA_B1 z>(5?utuf{pW9no1clCA|?ZEwt263UaamVEt00Q28kD;kB_*azhM?0!V8pQF4Fsd^Z z|00I$EK0@in0>)JhZsBFCiidcfdp>NV{uppuJ(7fw|M7h2H}lwLaJeYM}4-ZJgMh(2UM;6ygOT)fi zxQzb5z7le+zuMiQVj_r1zuqWMlxc?AWzrS$Z|2}GoKy<*wU`s{MKt-B&stip#45Fd zF#PEBOU#puw$?NDRQz{s2iM#I^eod?B%1B~TKIPbCFB20mBeCAaF?N?!q^6MxO|`L z>{~K0%iXwPKgkQ9W;s$VhYOisZf8sadK{V*v-RFB{5~7ca>H>`QW2UI;iJ?-6WZa7 zNlMZd8MxDzKJnXa^?N_Xitqbpf-pG+3lLXx_Pv!}`IjaTpZMdXY_@p3GpfCNzZ&`9 zy$lPUlM~Y>8XKGajfRJ791Ob{+IqU{-v3=g$gyJh6Ggy&&{n4m_0;*e_cj=X1~No$ zs8*`vuI9*9tMyj$)u_=XZ8o22%i;R=RBo6=!)f*Mobil9Tl#5eza`d|d?EfPWQcq2 zLyiXn0s=<-Um=5op^3h|gR}nsgByMnbIa<}*0>UAy*K)IYkG8r7S*cMLz3{3#cTz! zHdW1m@67Vk&Ayp+a^(5WQKo|R z%b~QlkgX3J=#7kX4Iv?Vf<$)t;Ff{b_w`wngEji*|i;^c?KKmfDLJ<4$66I4p8-P(EL9d?* z7M^d$OTGB1yvx&=D@WCs}kDBlvid&GkN}Mkg6dPnKKW73q_CpdDA1KM^(sg@> z+sX~X%RVR)vd(kE`R?=3x<;+?PaH+UaD0wxzGkhoEcxAm|_gN^{+>+!W6;9%_fYvpdyIw9VRM7SL zxPIgGc)8lD&Zw82>+kyZw4CPurx%xTF|o&FlckhajFW@4B{|~_o`+A`*C;6CEH7Hw zzs>l{UCBE3W7dDB-pzS|;!pD=*8B1`ASk6vNrNgyw$Eb=-wP|TZa} zVG-q0W)m2*l(9a4BgV83Q+fC%t7EQGrmoVC%Oq-{Tz>%um+ayZ6?bAd0zBO0zQqvp zW5XvSc=CeQXNgVq;2i|Z-KU=?3P(4u8ufF&Qf%5ed($vcXx*(#Qy8k#p+wr-Y1^~3 z!4z(;-Au2qbTa@WQ<5Y92mvEH4y`n<+u=US#5+?-_GaV1P7>{{D1inZdj07vzCr%8 zw0UDD0u%m}wG$>FAeR5T7}&wn#lqfB-@(w?S>MUj+|v1fMov0-E^BQoGYtsEvi=VW~IQag>Zfz}O0C;@$G z+IJnsHP}&QO6k3Hi+efz9`!mp`(#&0pEDvOkwsEvm7IzX2sVn3M@fzFAtsWLEhr<) z6Du$jQvzj)pgra^HAcPzj&P?#*(js#w}2jz*kc%qBTbY9C-^medS^Vkf7d2o5ctmK zd#RBQ&k~!B&ABYVx%Ev0B^{QlKGoS|*ru_)1 zYcgT5%3QD$(iJ=1WFQc6477>p(Qgzbki0Z*jHD40mXdD93z{;Kf(8ObIF};e{eTBa zjR>j|lLbmyDdH?t#=G--58e}sDMuzbl>!88(4=v0-fNgMEik&cg{-oYl0gAdG-XgK zJmR7(x^JeBE1#?PpoNiv&#SXy>EcowHBrSN2$^_a;|?_Zmr)sV8;#z=L})mbtbH!G z3-TbRe}9Ukk+JO(p5+Y9YrAx0lJ774yPYVNbvEsbt`J}8CGvPVx+9$P#t6yKFRMPF z>m}*^>fP*5q^^~`C?4dc;ONd?r1$D}w zi+iMQlI>?}7|>dzb#*y5Q4hBn(B_P_x8wM4J)qrPleWv(wNoZD3P^njz6-c7AfI3k zM#hGgT7o+yy~1)^cw9ftS8*63m^Gg&o$!vea~vXEk}3sebxrlwvuBQ2h9{2JuNIX3 zynu`)+sH%g8n`?^nG-E?%*IOe(^O^S!Y5d3c9vi{M&G9=j)v_k+qa6B+@Dg+P_&jF z<3}npz^rDFJrL`OcFk=4-!^niTPXsm6GX4(38hWhh_pFdurqS4y~{t(UoN~S-H)x0 zj{X%&72lh-aAuss1UvddgL#YIcaN6SxLobUJwU+vwJ0<5Bdv~-o!4js2=5%pndJI_ zO;kpH%a=+S;sI8(_bv)lJ|3Bamr2RM=h4LxCCJ z#Z%5_jsOsoL0^E7;4tMkMSyS##f|ejX(34dXx;uq_f=1lm%OvH^8))5Ew9;xT59|O z8aKsJA=WZ#5RNfc;~bC#qjXH(Nglw0q`o7T1m4zl6{je0>aDSUcVs~#N#w_f`h94k z!$K6_%@~4S2LpGMCi_GZSu@NXG=sH;i6c2;i_U$cTf%|Vh^O^a%|iu0Lml&u1S5S6OR z@Z*O!92eewWE*#Y!EGcJr_lD=MMo@o(n9w14ZZe5p{AeER@IgIM8x61=SG~m+X(^p zSIrkwngakB2N@6vJU%%TzC;>+Ko0kN=I4JEO^nLTi7)*Ik@ewRfrZ{A8G?W)pC*z&0V7IPG^xg*C~>v(OmW$f=8T|w&T(cy*L zpFJ{U1>(;-x|FoRK1$(UVOD$&$sNu{!bzL{0ft@PTjkiG*Hi?#$h85^&2#GVh%oV^ zXx5iBkGvW>5X+M3+Gi^6hd(uk{r6{!Uj0@E*%(PYR|Vk!{$7r36O22Wv&BGdC_-+o zAWR$h$6zD9l5@=L(X2}@S00swlNu!pgAw`ExQ$Y;*GbNzjg{q&NjQ5fg&Sc~3XalT zX^fhfVFR*H@5INz3LCNldtAEpIX(Ww1DmlmEt9}laJQ<|-j?|uYix++hi>)LW$VKy zeEs|*3`j8Ux7yqey0ZM0wwHu(H!(j<^M-f<;XMe1D{oC9yB~v4A6_7`Hft89!I(}< zrzLSG-Y7RkFQGPmZ)P5|Mx+w{X|)o@=Wpj4jdjWI?d3Zo<&?K;3zpL=R`%QyIUW`f z!_gb^Z$J!=I(raG0;%w=RDOkF0~N_3Njc^<=Z~@XT#+2s1MpTVk3PS{6W67YOW~$ zFd2LLtKO~%<}lz!?xSh3-emJ0|yFXr4c?dujy98E=?6`<;y$8J9Lt=7lbAEdKUd$$VV}J{bJZ;N_&#LHto?CJs7;v zaDGq5Ad1sPM0fIUFXi4yrcHyQ<_{UFhyBHN>pudSCF`-4QU~}GSb8yo6n>jHHpo&1 z)PK~d7>1rzqmSsGH#jby+2qeXD}I}0t*vm1qFHB7ZO!2@0B%w+*4|6>`V3-3!kKG^ z2ZSnV$8hYQY%GUo&YF;4p9XmCY&VSMi3W%r+&sS=3nLJsI9t^vFHP415gGa~3#%#N zB5In-NH3+n>%IH(=gBSm74lY*qX)(cpMXU3Z3R2B0}z9OdEvnEU1UjcAQk?~OdqU-g^K`jvBvcp*%IAxuMw0E7Fm z1HqYTy*5C*2B-3!T9i4N*Z@;~=xWoggX&>KD;P_xVkl0HAa=R!4Z`k`gxzq~!N1OH z{c&sbEE9Vnurl>-6C6oa0Wcyy6szEQN@?Myom}7~dWuyRC z*>}H+nZ-Q0G=l%j%{Amp^8)|i<&Zip2O3`XKeOqh>??m0c3=Ej2DLG;ktcbi?0`m>lx-#d4r$XD*MaM z|4W%9!fqrv>+b*sx!o?Du^%?a-RtOLV|xj!K5?Lsk!}Hz@v2)K$Rh0M;qNo_}(uv-bCZ)ac9+5@JY;k zq&bpw(|o#XeeRoM)8V=XUFIyxQ zny`FCCfrrR)Fi6w2GB7mfig&Q?7<9VPT)$3Pv{|oNQ`RJY;U`oDX_>zM#C30+u*Cu zTAVIt{pZ_npO+$NK%dRb{)Ax~l;#Ik;w8=8cRbP(H$21gq~ z-DvL@K}Sp2!+wF#_YbQ7vdKY2FDONSF0k0NMaCRrN;aO>w*kY%L$r7-$0U*`WnwgC!^G&$>|FfwIA*k6x>7w#|In_w z_JpPFrOw6yZ7c%5<@>)RV`O2!oos(B`20T>{Qm`RJGvU$xmbFc{uiBCts-mx6Qj}n zpbobbWm;EorJd3QWLyrdsP0DAAtW~tiKi5$)VXxs&_MG#yVTlOiUsavGuLEcLdKK` zBxFy&D-xgs`k|6Hi8x*)us}l32(mNN}BRCEm>$*gYrN~09 zPc|~}41$<(Jtpl=_q8>#xy?NvIS+l@OU=(a8oiMe~E@!?lB2PL< zL^DwwT+>vz%x_qtd_mt54;l}WvI06bp?g8u@V?}cQYC0^fXZ-f2rr*&yC9D^*3u;b zXE)APy|>(~j3fc)b@!DnCc)Zv&9-B<8)8dmUHi5rS54#gynSxawkMQ6XLc%d+4h33 z10%*6m==X>wx~d3b(ls1(ydNn^n4-k)1w8Q3?Llo-d=%;4|#}6T%`mzMav))H2gH2 zrhw&iLdTMHl8L`yDhfs4ULSg(=$frTSZLoc7ba@dm#`bKX%d4vfcA;;!J7(O$6SRC zK}wqJr42FTInK`}O|3Jm0t&G}wLckmld*(PXz)v)P?fK*5J%I2~L?b})_vUsp~GrF!p% z-l=SR)z_C!LHA=k37_<*YvM1O>Zv(xC&*D&6Gx zo|3ePKl-m4pXuOz&P4&;>wl+os~pqrJ!USO?jLT}rBrmt8Mm$y=5>KO+8VW`#=URX zcZB4(clP!(z5d*}-#+h?Z}R>#vkcDIm~XawuNz>PukhF$hAoQyK6o#zL1|HKD7Qg_ zri!sPI*C*SzBbOu=DpF|{v?E0{eB2>QYF_kc^MKZ3fS5}Amp)Oo5k~Un)p1pYE z5w+s=`v?B8#zLwQ2PmESoKc5jB(RTs89F8ogo2$9U*CEM_W#M15p8=r~Hlp{4wnyQ*}6;%n05r2vfc>WmZLll&xkccoi3Rn?4=vSmM_O;T0 zu5&wj`sQSrI>E80)EisLAzAvzHHW6mo5(hJ!~V)+3u`j*2HUS9nx8yl0O%8t_Cv6qbl#0)*dS>G2W> z%+myCLGgmzi3M`7Ne=of(;lM`W$!R2%F2LU)FZErLm=m`AP!6<;xE>kik^m|0g{|X6_cqYsx(H z6u_pQ27Rcpq8;tsT^F;sn&=Rc;8RK}IcLj!7M{0UqKCM7nAbr6FBNJDpmKUl7loQj z^9LO~_pSvdxai`-n2fU-50ax8%1!t){ob}9wpz>Aeb&}i-A#QVlD7-7w3HmzC}`Ts z*;vd~Qt-i?<4>ycy-cer?z}5-W_?eqNvVmX;oNHv1y$6>WCL zhGTdwow`%&NSecrih<_*4&;Vf-J@-OCs(O{O^Kr>bQ<>&^HRe`QXW&1b&=lewSI=hqMRwKYAR zZ(rA00=0JE?7;!dJM<%2p?1o2I1wywg%L~G{}i>r5?+O*rbGmSR3VI~Xbv*!tUO|$ z?{*L=;IiaOJGw6jn#3y}^LpQ1z5LEd%qYUHpoIB;^=RNx{#itTBe99Q8~Z zj~7o|zbhi5Gu{qg=@t?Q>b}QM(^!Su{Ax^sUEtG z+FnBN=XAfL8J>!sp&ukG{vpiiElbA8;yEuW)F7SV z5l@Ca9F)GmxDOOvn*j+_rP2Bf44WpD-;qZTPv11blJ}N<$`^C~ESy{8QJIz-IsSrj z+flbthwk~~Q|yqu(_-VQV5XF7xN|w?XTx34$ga7svASGx8XQRQ87&T9BLZu(5u;x% zvbionEoyb28jF}9Ford^t?US66X579N${p?q4B(clk6_so=zH z%j8#k^Ta?-8^EN!ek{xD>__FId-)waXwdu?7?mD6bA5mWH#i%fit}>u&-`lD(gCFF)A(b@=*n*muWRyW{u%bawX3 zKBxEde0Gtwyn%$LYLjDO1P=>06(1)=%6ZKWyjGFgv&_e)38m4xXxWOJW`Q|7ukUcx zg8jlh)h){&j?E|K7%picMx^i)Cl1ASKcp zPoV;@JXMCr&-Qn@k|;g}2{*eUx+^jwZ6T+aSpDzItAh9r>;$DEuODfk%9E(}yl5n9 z3aXlTg_$T9d8FBdlVq|~0buuDP4dykiGHZ2OC!J`sf(ZDYT{CAM&9OqA&P-k*DVPp zYzdr`9D-n2Q2=-Vt%ufe=i$jZ)N6wW-e}G`2fz1UPyp8BpFvV!Br%p)IcoUzNW6yS zslUTZC9xw zH01&=XoqyaF}ASL7sUoAydwf5nclI48dOk@Ch?^8U$VM#RY-e=>r+$><7L=!_Dpwg zvQTfLV(<-?fdq!qEJ-Snc}h2~i)Bm)oGK;cCMd8geNodGVfkjODAQ~KJsgsEA zr`i@h9W#E>x6cauNkgIQN^8-}F@jCTSz4JSv)oCsq{$4-fP|x&q$0o?Y{4P*a0fzT z(cK<0>OT+hGhOnLlvDKLi$vN;9CHvEDmFyDzX9m=bDjkx1$yyl6l9j%OMkXqx^~Aq zBR3MKqLpKAo#Xf?OMvgIScw5t)1dk8HeCPn@j`$GHGsv0mk^qDDhD1uE;(<6H3t^= zoPaoNtdF>HKlC4B9q~6by9piJqB|i48EHf`XasOofWhViJDr-j_PM&O844Fe))(%{_`#>s zH{CB;62*?6Uhq1Qn?jrs3Oj>&&0AX@rOjJbfiDuzSR``+u_us0d2t3Z$Mp1pn``9O zrwyB90e*10O!vB2z?VNn)CR(vJa+>W|8K=;5W=R^)P8pE{2YX3%)iAn6#YE}=#YZQyN4Mb!+UiYE~ z2TK-+2M&$%SaGA$Zbp|%NKd$+9Q1Zue>eYb+u!tvz{z25y+AU>3V{jW<%i?7@Bq63 z;G8ZQn4Jycgphzcf5h6MpjNk>!dP#gVn~pDye*`FSsOg1RBq}RnPD1PXv!TSBJXho%K!IpiW4GkL#})J@cDu{evUj6TY?d6Bn*_qtl_&*GcGY#~e8U!*VfDB8A6h?diKrO^sQ&A6HqkJ733y<|m#p z%{qVA9|ni?68r=;9~@_b4qjW8P`&C!q~&)tbN1i5WGy`o3Sqlo~nLJUQ|m+(q++3l#7p zg;P>$T@`!m%8o9~g#MhivQTnUd`*Y(jH~&-EBzIw6p!e~G#2l$f4n_8Ql14uXvrj( z6f~(lI&`W&23HDT8cBvZoR8S4%8g(-W{z(<4L|)3(ku2Q333$(S8-0sj%jlHEC6|yUC$?L-i&shiOje{{|ab zT)*2KF)KPF$ZOh$n!}bfM-6%x?U+sho2a-}3ZJO3K%crCykHXbBpH2dC<#A7qAXhAci|dFW(_MZ+zdQWY+Bs1#|p(@5-@}vk?9nXu|Eocoo4w_~IGX0uf&D+T;X!v!i;^i6g zWdV@oQh78T(8PZu@fYeUlUHK3R^Qh(>z*dj|SJyvm{$fWO@vLOGGG?N4AwF zmyUrfvh^zjPXXk~SPGm_uT1k0l0Ov&YIaf3zz^F1@7!%{(DUt~X;$iz;*oMOhGg<6 zW(6bUI*bQoINRnoahA&7@Wtm@=aPsaUQcy=%y~I!e5@t1Pj$n55m7+uUGx)o=Z;O> zc$MpeCL+8t;iU~>l8xi~6w?^Y^uAea>aPby?KFCn)*BZl+e1G&38Ok5iu;dm4x}Ys z2>-&Vb08&*GHcT*nTyq@QqmV_kELbKRGleGT+Kq2$HjYe$%?vM_2=u@*q>=vPkqA+ zVXNlXqAaXc=$CkKH5W-|h1Li`)BXJBxC!X5J|b6u=e-FkYMo7lZ@etjjfv1hd=XmFrj644Teus5PS9N5|9>Z*x#H%5G!|0Y|!GUTEovDO}S^+bFWHYI= z9IXSEV$Qey89%bL$W9kK1n{FKJfwZ1u2<<}XNWxK4c8RXtua-Lvw>q((GwNn6NTer zg~MZo{bR-5o;^m|gx)SZg+1)7GYo72nC4Y(}d}nl1 zeYcVCrgLe@p(P){AFl|AAu5;kTA!n68QQ(^GfX&HC_*Vl^=jz;h0+um=6|2l{uVtL z{tL<2)2N({$)(prU^xvDN;#Fo{FW6QX^%q*RzmV($K~xtIeS>}NCTul6 z@`Jf0k2$}!`3}Tkf#kZKRWl>J5M2l%`UbNB_<5N&PbaCW_yZ%62-x=|ia3xByE)Rr z^@&ET4gYu?>q;{WQdgYgA(ODj}AWzbC6}lv!0jX7(D|uHcr9etm8<`rd+O4*M=Z^>0VNbNjQ^+YsFVkXvNW}^wT~?hE*uDql|~(08R2NAq&`-we#c;68x{aROk*gtM>ou5x(GUb5S{OhlOkU&;z?%m!BZvjx8d|=T$uXfad0ky!h+Pq&UXRLymX7kdWZc z##D<9AxgQ=+I9hk+}Z?ivEokn110;$UE%kNb$k*AGIt&ua!}Z#$)AQsu>k!&Fu-0^ z&Ys)Nud!pvP+d`~RNT|rqVDYly(k+vAgsp59O3gKtn_aTE8brs*2>94mut84mJ zr7O3S;YcRN6$IF714io5ccr{fxj?z;y&h1m*_R_QGixkHJ>Lh3tIeKHd;(ilcX=&| z`%{mrtC>yn6AxbpuMdw0xKiW2*`)mXayL7CIN&z);cpG~V#c`}30Q$0J)Rz#t&NxI zb6mz;2Ik(A&xXy8?dtOTbuaWsJy335pTGSpgOB^m&lB6Z@0pw)y0!!ad|%cV#U1I* z3vn^i`aRk;UvGy5atuJbt^1!ls8<62SuDDNK*>;l@;Jize{#!q9PVc_ed*J+j^Qv4iH+5o^ZXHO}5xXd_=0Wzw(^x0#VuofHZj``? zUO7o@ab#^#I+3s=6MJ*>fS^T6SPQW#xn4n8APuzA&FWOQ`|-@>B~BU@AUxaohVNwS z;Un8wz>EryFB4RvA9~ZW-OP0!ayWlaADYxv36?f0RkyK{;!&ZupO^Cr z*8+~fZKFzXG6=##1HvVs7plLlP<>9IiX3r9WR=wx77a424|z+-SazeYQ$uXh?svyu zkkqT7P{;&?MNn8Xc#yR!!BNN(a2mJm@{A7E0E6{M%n1AvQ>2TwK9{wJp4g5Hi{C*x z7(=mwFBywbJvY(lFq4y$LEMgRb8eL(I9JE$a`M^fbSac^#9`~doz>vI2pslqa7v;m zcy>TKg77L%AI7FLd*Ey^PgOAsICe%57&}YswU&8rFTwpa-HFwV({pYzAK>*P)prRp zo7aHFWLkf7=s_r(!v8bt;#Hk7CMT4c&f)g9!|-9^XJysP%+90>YGmCVT3;Aix?EaE z@p`AnV$=n?kx!wx2P&{2=BYbt^MDwWMU_CoA+s@sBFDtAIZr(4i)=Sgn_H@~5~&qd zXl4{%6=nrNaJeW6Lw<@6N{7rdTand&iJ;&SDmlwHuVlYCFS+S9*ZN|VIQe%V0g*Hz+i@WwFf5ObM@OSOTpzy_SO?$Nt2fn$m<$2`9|K{?#npKvo85Kt=S!&*@iW5`|`<(Ie8XuEq4l>;LZc|0Q` zQ$84W0s90=TvIfFKAo)pA87R-bs->vt`bo@dn;L6S6SotTloPN9CV!bz2)Zp^9`tz z^wZ13jI(uELxiEKzn#rH`3l$7JDvm3M8edwe9)7pIvu~qg=0W&+FFn$ok^r79~^Xe zorItT9bv2BX=wrs3R;!&n6{0XUzyT&$Fk@kAFpPH*X;@k1Y{O0J;c;Acc_H%DMbj6 zv5;)sPa*CB!_8A&xgGB?jU+t^aY*04Q6 ztB=IwiWN$Qp%mwf$Y7~RPDB2hXNdMa6gnhe4a^9Opgc>ml*n{iJFU9vVvI)=Fw+f4 z1sT1|O6q!ck|``4PEq-DJ$Jt0049A2DU$e=oaF(GJV09rs2l7qse+1D$zvcH zgrUkB29oAr!JjeYoi<|zR~QEd>xMc}IzKN3agHFtws8e#BIkjQD6Y7pKY#yOZ-6xf zF8!8zg?NgE%K(hXkZPLQ#R&a{@Q|yzuhPnk^_Ej@9^6$)sP>mt&spBVSpcoAg0V=9 zk?fU{JZS5%4I*Bwx@zoKHl@fh}dZCfi|%4 zKY)4Q^Htnk4UUCMQ2?$s*3&0u*XiFc`_drlfEK8>&bf!bh^yDFjFeh7{!Q(Z%T0`< zm#qRFZ4Ic>cy+O&C?OMM1d8R65?h=l3rYTN+^fCci&5EEzf}yq)%E!P-93*c zd@;L?`;j$9c@IN}vYg(f)DnJQG0yV0|6BhI)8Bc!ZmhNQeK~!4?#*#)>FMyi7{7qb z{p<1k=vCU--uXRTEdb~-m`zBl(5Sb<;Ux@32D&2_oYuV z3mqdUV+>eC7eeaUoEjolBitU=WP+OHM_z$l+uMuSZ0Wpe%bxv((j}Lwb z`F>=(LZhp|5rm5vX{``Bx#&U`3h0WDu97gB6?sAuR;PAplBTXPLO!7iwI?=OyB=kS zAeaaa4g|3$60k`CY>oJ7A-6e|5tq}!;nE|FD7%ylj!-7MBf|*d5HF)vJqcPPlHOJX zF~W?*`Y!<8vFE>_7&+dz5uejvtzcq%%PB;PZik*Ls0v3TT_99+SLkI;#iYoLcBj+3 z;P1fEJP9rtx#-MQlH$=)$@pAGfeP)ANcAXnJQ6L+Fs~XSwNQaQM9IEn(!Z zS@guEfzh~3XzMb=R2P-D&_yPR7Titfhd|}QZ~p$m(kn#Ym!b=uL*aqjeuh|(^N%a= zQ=#H(u-g*DifWNprFR=|;MVcumXRh*BS_EsY4&tv$6JfM^yK+Q`okMclnGlBT9a ztw?1Je#i3lPBmi!wL)Cf(IGG{A}Q=P?KUAa_~mZi3AvgW`ibR-vY05Pa&jG40tMH!>l<5vs`_Q zEpB;4FWZW*W9Re;y^wX2-#i#0lHNDo=Cg$jyx~VIMZl9H=RDMM7+c;w(L?O;1#Bsc zaq6oqfzipF*VMr7nNJ#;hA#o}p8gRNaHSoz!q?*eGWe4g>S26*&>;~vl@ZF)S_tW$ z@9hv=Ix+Al`zYr1VZnaxETf*rm~+$`xzBk9OM0I&Fw^2tC*vS`C+oGq%<4$~>9rtm7d-x`+p0pN`K0u# z81X2D4)g$}`cQ?9r|_4uMyR$w4cS3YmuW0`?4vw0f6A|6rYc2Qrj0~yrUbnI%1KP*R>*ypoVVq1j_&z(to zUV)fB0fIxybtC#zp)95Z&K3;7ze5d{62DvRuKoWLV>r5`>dXDeGbDb7sPOzR7{kHF z(na6c-qy&{&hY;SYjkU9{|`Okx1;aa?boIEgGNc;0PTd9j~06zLK9Vt^JfQS+2rfp(2K^lAplch>4;T?nnJ&Odl3tRYg~jM%_xR9=mZBUKuE6S zmk|dBu%{1sf>}t3Mj~i{C^Ara(g8>UQC6Ho@nMU@jH0!%?t>YS90RDROxVvbz=T@= z%$#+w^gR?Sl}j-Fxk^~3%l$RX_NC2p1_} z655)AC68A{YM>!8?~uV~DAyQO;(*EIEl&eOS4@XnaY2_ybCvJ0c>yfh0VgsGfpDY- zdnAU$8IOCgND0*pD`QI36zyu+*wCc{hQpGWSRjH(Bl10H_!EgL<%v?U!e;j)@~Mat z>5U*|scB+OrwM`gw%QhR3b#dV3c4r3V&4~ zLt5es5@6OboQ|jwF<6D0r@z#Z9Y4%=uSOx58Rw;n&Rx zl`VtUsRFub*}=1)oLrNZuE9UU_f-`cUzx~6ICzh&@@LY2!@_=#3M)#A75vpZCoH#* z8W-C*(Z?ikheXUKD8X6xQk>d@iyfDVgYyPDH}7YrHX%F$1r(|L($ElBruhHi5y61WgAAc5mV%t=4k$)BshoCL%K%beT#10C?U>Sp_ zkCG}#k%K5Q+bPfXI`vd*MP9E)T8DCo{94o?iHHiizx=g`7eT*b@0p&G`Z4tx&6NDzbpD*1R$o zHO`_JGY6XpbIfpqkrpPTUVNevICzh&1bDdBvUjWooa& z4_8(mF;uA6LIw-RL?vR(hKi3?QkFezP9pD=HP06Owr46+U8fb2Q&&B!tHiF;Se5V` zNzi(C6~PIZIXCcKoF_(DE*$Ajc%#&zQ@vAd2Rbt2XTOH3MSLu3y+g}z_hqNe##?8F zk!b9W{d9z6G-lM?U@IDFs7zJh>{myR$;F3;2yr907%rCjDGQhiIP4wWo&8tDYFr56 z?f@U31F>Zl7a6XR+;D~g#tBBl^}=_2k>`FDWHIggCD_;PFx`ikN0~*zx$^Tz_KgK+ z`~g27ER+v8{M}JZT9S-U9!$hn{ofWmHj1|S?MxebIQ?uN$N&-AACnf|I$6LikU4KY zw?5jtl78^4+{X5-vOl?Y?UpLzR}u|?+~*T_%hkAsPZd*>mW^^68z8+OOuaxNA#7U1+sh6bkB7*Uy<}tg`g|7+Et(`x*KIwcB@wZM@*Z!uYs8(^0QR9-%q*Z1MIz zrsi7@DJ%a%rS#Rej&<@im927FLEt@$#|{7Fol^>FdPC`}OgYXrXA8$-Pov$SSEfql3JxCoT_CqDyvmZwxB@S&uw z5(m5tKzmbrT7<4<<1)gL_YccYTO8;exj1*!NU2`>I*$;4Zd047ua#RucIL7`0RReM z$TGt}Y*Y5Y@a5kggL%Y^O5h3n3L>oeA~aFt`fWlGS6;m^SYNqYKH*TB9V|KVXvmyH z;NA@M^|YptMaI|yDwWc>c(t5saZbQQnmB{puB4}m@M+bM+v;u0=+6-~Qc}n;{xo4a zY2OC_v+;5sMuIGGH0yHZJX(E__IF|xI0?}$F$Z`oU7ZTrM7sNWh!qheSR^tYkg4#r z+Vs#_jA;Y`ahM5ypA*xjFzGM_!K%HPVanSE{Gjurk-;ooIb--Kor_Qcz&{`fzSD2! zFj|Y5CX@g}Qi>Ptk8VINkUy$Gj>6}k%?Tr?r4fsv-ETLx-+J$3#la+vO_4*W=2FG?EIh^yHjku$QoJ{WJ zAaxrU!ny7OcJlk+mOpd66C8< zgc3MDjWJN8e|S5a#jYE{2qPDmxIv8hx6^|08YT%^aMf z}ieT)rg}PzEu;CgvL$IqKsC_-g!a^ zQHkRyl5)Fw(ioOg>&b)ILwXduZOO-$o(~DBc3-GFXNJLBBc)xz1*<{lmR8-}>lz-I z9HuEOiU5?~lKq$;|1Egw+1^uq^3%Y8_iE^H0RYaI?fqYY`v2)nHm&kxwK#+Fq~X== zcNL03nQH+Lw(STq)u$2gIz*?R<~){PViZ*;cvG~n25U3YhP5h~lsMSDA;O^hEW3-H zU@&EqAt5@`WWSg$1bk7BQ*Kzg*&K( zsF)5q@;v8l1*_-fN80|BwycatkSQogO^K}^&0zF~0ki{Um7lQC+?Ez-XJbbEfxHgpu7h(o#eZOAozLBJ0^9s!9DFOF#(HV`Yf`-4<7JJ^HP%- zHRPtfKPY1tO1UR%x3g_dg%}?pTxcww$^J~76FVfZ#J_*?Cna)Ah$k8>)B|*^c{Hc< z`#5bVaB`@`E*Me^lrRfqatO2-+6_*Q{_g&n8~x8xx^d`y9jG#n{z9|{oH*3fXx?pL zaofx(9n^vOdRoYz&YV4a&d%)F1qw?=X&GWgamcC4b*X#A!ialNIn&5Zbd%INwIq?Z z0t?@6S-7}dlj19b@A>w2R^rgcQtQU8-}uwQq7o8Xx(dwl!uzerREGGv+^9Ab6IgjR zJ{fyhJUxDwb7)ag=fi>TsH8wAB#7*xhkE~$){W>6qruT)EK*X+n#{J`s>kW!jt3r0 zdLQm%Z+(-&$JutYPjHGytY20i^VH@&HF%0gkhah9s<&(fbCltxab$>b3Cxh#Y+YDg z*=EN0qfRd0u=Fi>(zmM9-`}1(WaN8kB(D~gD16a8=2fQA2D5zQj}~zXb>An4b7O>e zgC&iB#*k9=ooKh~?EwzntphDGeszi(-@RyD-O%aj6y6?u}KBohAf~Y{BBVRa)NnJw^`@<~mMx-;XDT7qmo8 zt3-MuwZ62@NN?K?8r*0;9$f8wfsuIC7BgI`ROlw+w%Aaz%%d{pki1%t->%ev@b-he zjdRv%fSPIjQKdn<)7PEz>Si~sV*)lk^Jevh`$>vMr`gByyG#C3-e#?-vs&25YCp~IXA)N~At zZO+#G${HrXpLRQ+-(?$ePv1+^dgCHlKeltBah+wR$(LL{A8W*6hl4Ka z_*BpU-z?Vgz`mGNoS3bhqXgKTfVV2nWJ)H2dZHs?-KP@>!D?dGCf_-Sm))Jso<1DU z5V3(p_mQJ+iq`5>=o6gj^txBfcaoJD%;Vgynf!>QU{E?R-Rd6<_#QNRNe}~4c&%Q) zXRm;hv71!@%(ry9Eh4YOOh~|{ffW-WWMv}8F?zKf*6|>St}j`wFOPXR6Wq5Fk}F(E zvgiRnFD_zfXrvvB;wif59T6D2n&S9#v>IXVb1gB2+?oE_zF~cDj%85%UBMb z%Gd^J$3Cy#=@ng9o-X4n3y&pKC$5lT8gkQlHHxLj{c<(m#=&*>@_eP47N2tQT5Af# z-j(W|Xifdl6>1xXrG54em?-Z~KeGExBvLqv+)}NuV7jwB%x=roYiOU=m zSp915eHsZI)H3Be);E{Mldwszsg&%qVfx&}8(qF+D%HmdmJ-rGGW6RxBC^^&eh-vR zrhziWmYaomBb{EONC4I+t1=mE_(f{vsKnp_snkqnPXa6R5#Z3k{nL0qh2L-w9})f& z#uxDM5k|pt`H4a3HOOLF&4-fIkBB%(m7u30sX;ZATR8IT`k4uFs56V2I9s<~8Q8cwmtY$+R z_Zuwfhqu;Gc<&1j0yd=6?qgrUCpUuNY0UfZ7}LurVOp9`iPv7IAJnRqIi^p)&0P&( zUy&B*bE!DT#IYwRm%nn#LyePwmc0%Ln|=nuyUrr zx%Qr}tFG0UO7=#5LEGRuQLHf(c2#NRQcXn%MXy90!M^=$;f3(L#{FrxQcaJQFNOPa zB^n6&SSG_;qv%ELBteJE%{%FR$-;VbRidSd_ygM$^#X%cCb7BiC-$<9?RLjaUJn;p z!E96vQ zmo?|4U_`7{M0a3NP%?avxqJyDwuBxF&&>Sz!pijIBd$S$(&~GCfT2vrG@g)Ts{WCX+tb01s9JGhIDuOhqfm9I zJ&w)VuYF8V8I^+?-K5&5!akHQg#*q*Y(k^Q71&95yXotri-7Q?Z4kSs8#2Q8M7oV0 zRwN}d)CCFm-jh-XLsz(0oCD!VR*lGCo1&b2s zj@jgwR+Uw?)I#-?C&TUj0jSW~S;Nst*0lq_rZbWWnyyg!a8{1mOhTRtX{(ah1#5o} zGOA7{Tpjox<2Y(RWQJnaBkQXrkaxP}<>h-H)@%*^fF)@>4T2vj^0~G?A)9#%c~f(V z3g>3S$kv@{DYxk8MR(ttttyU`TEt_`)Ie+WU9O&`2^n3NLu+tp5NPyfZmtQK^7?il z_Oy3+X+GK5cxG4KtnJ4px2E5i>gZi;$HUDI$fq->ZLazI`pDDj&!fTWZQ?%=(Gnr2 zs+eQ$dLc4vB1S1|ReN`;gn7SH?Tqx|aIm<0lS{(PRO13wPxz6LN~50fVdZdWj+vKk zRLaZqp*OOnxm+nNFoT`O-T7ubj$uJC*pt+S8&p5j7b(74B@E&6c4M-?*WR^&SWhcl zL**O5hFBxb&k$h;7uQ^+Lrf>{)(;rVC0=zcnWlTgbbC$GQ!e-<&WI;7maCwaFjr|A zxxGE9(Jk4k>P;jv$<;Yusg*Oav^0~wVc{02d*Hlk#OqcB3m4MJ43k?D3#D?SYbbt3KO)eJ5ZKi`7Y(*fK2FsKmh zQYt-OmA*O}gX%PpcRgkjI7pZ*#{)Ps7ozlEu72=LEHAqp<9CX52uL(h0KNd??LRb- zalwM9c-oqnn%>5|i*C*u{NREb7P{d8&h#;(WisO_vaGMYp!h*FMG);oikUnUyvl-{ zXN`cxTOn`UDwk9gl$-yMDW7!(#GC)M#wx5@RdPJds(Df2Qdnu}L!fU8O=Op<*L5z) z{`JjVX%+aG74zOt$^mkc{?$XWd*1G-&;%1)zZ8QSiF!!N6_sXY#kBV=GbJk?)(?~M z=c2$s|Fwis@`i;&r-9kqt$C}8WIn#@=4_q65tYgjUaB-{G@)Sy7Pb57easu;`%ZzQmXNeSR z`n2MqEBF`eXAzSHP*R@Pma`m&W()g zvg52vcQCQIsHCM3JBpK;&?FX!s3e1JrkpDD4r!7LCYw;aPLsM1C=4UP6?Sr$8L*4r zlIeN>-Yqb!=@za?Qk~Zeq-roqf9eMqDHg?-gdUL(Ytf;kudxxSVNiqPbb4o%3Bgcd zWLHyETv(alWK1M#VFiu+&WbM8Qz(Lt$bKkpT0?bj`-&;056g3K^EKQVxLAJgTx6-# zV&JQH@X7|)ZC^cw7q)RpS4U+BMO0>Zz*p$Km{#G1-J~qe|8zK1wt(7I>`5WD>a$~_ zZ+N=&pdP?bc{Y?KZ;lQ(?WF6jb9;-=6;Eyvz?6X!g_)vEl1DOd)WL=h7qM!Z@Q9Lc8oo z98ksPWDBC3Xqb3x4P6FsqYazYM0Z~AEQ&mY?DbDnyThV=zK#rxL8B>>3>tkhFWerq z8_YQi=r*!{=P0-h(RMYy6UsqDRH6>Bp0xRvbzscVp>%>QB~%<1Za|k*kIw?(ugZQ~ zt=f0Q|G8PI(+6%C(hGb(yk$41pSYxe3CSj1&Hx(gtBCRvi|A`Agpa;$aw-ur`-GDq zr;7aCLgFZy)Kon8@G(;XZ(0H|Y_{&MplQ>=bY}W7RGUJrZRJdJEF_Eu9b zot)t~3W?J}1mGPFE7}d%<~J;I4NTj!U;_~r3E2WlZXK)>=qPF#&cJME$|&%VkOi7N z+0~K!D-+3C&YOO`Gs2ttRXZS6_+`+ijf@rokMjq?Ox|C`g{xU%V_2K4M@}Dn$Y1O2 z_icwFPE>ig#5Pt~D${#~n!zTA7;M4dl@W008#CQvhn{1TirTXBaY~~tr@6}-6L)~- zH|wbYt~lzeY@hD(mO$g3vm#Wo_TtVnDpWPOJg1?kx-EP|8be7M+7` zBTS`U^q|&89aJQ-fr(PC53;d^A&Vf~%6_e|YtrOrEn|JmHRWI78#1zC_Jx&|t5?h_ zlG1K`dWxPvx2rnj{MxJ%G^<_}z%SJbm77WL$B)0w-mxtMjB150yw`Sb^YrAgAQP{n z0OJc-qbXe69s;WZ(NaFqd_riOx1$rf+2OOuM5MJKlal0O#pE;tK33Ps&@@kdvNa_2 z#vwXmv<;a8Q5gI+h8OB$@o4IGDjk#NF*#&jn5%7=%`r$qg z8GIldVCCrP8VtuXt5g|uB2{Q@ZU2yn68tKTScF(>yq2o08d8XvUHPt>vimw;Zdv{o zGEtMb(MR&8PW^Sm1o0dp{pQ-&guAfsg`1V{3D*?w?sDAH-|=}%JDQ&Ff4Pvc0Zm}P z3k3sn$HbmgPc~c_N>J^U)(^!g97E(B$||Tx<QrJ5N5v++1B;mcsItzkm0}_Jh^3 z%n)S58QGLCsJX_4P5W*B#+J)L=nkeOf&4L2thd415 z9+(-@4}s6(L{AXNdAvKQODWhy@gQR1%W!WRr)?*-jp(NFqZUjPx1_%eV~JWk_)*?Z z3MdfV%4@985-4k(eU1%Ll@C_~OaCzz(Y*o!{yEE zPc0;h9>kkjkx+tyVsYRBvw=zxElR^aKUt2gx>F;x0j3#68tRy3N(5>IIcPF?c2u^B zTjb1wl1fuh;>08s^L<|{?3&aL@N?k@4VzKB+r{oI5Lei!)y~bKz8KgPR~S$XGXN0kjARUqq`cw72yfyX$<9jF+EM-6D{sX1|`Lk0!+0 zs_w^bpobi%P%f%pw$fsNiuNwMrz8$PX{P!vB5Kkms>=z08$oQSW&*y4C;C92hH zY37@F&|-9c2Lq|hOntTHHm!2Q1u-9^3CF!P6Te@z>l9``uD`1J)oMM#WyEUSGuiP1 z_L!&{cA==q6Fx+6n%}dW7kCJpW!N~0p4iX1?R6q7vMMlLD$-^Q8~E5Fbo8{~bP%~e zn`T+fhF+VORLB`3|2px6J4%O9<3r~JdUYP(1KBGm&*bO(W`7LfzxDo52M5`Nvlm z^cf1M#5>MKN4{Dumw`Lw`F4DR4c!n62)QaGqN#1F+3N4~n8aZ;NzrT*l6T!l4=`xm zkGs>wEBBuZE65BON#xGv?rsT|$1^q61uL{-DcRyxUR$sB5hz7~pQ-YZ^hSWL=^BzS zxve6COSt;}X!6ugk&4<_(R5R!gxu6nB9&iT$syzjW#?z`zKbXFZNU6Aj+`5Uk8~I` z>zmJCHT-2e5x3YqWqB=B^E7ut?zm@FOc20?Nt+N3LT^v}GAK zdDfJ=)`XBCH+LkzBD6A{WUVLmL5@B`K8euX@VM-(=vBM!$&T6IDbU>!(AZhD&~{PE z78`0nQOLMnXqIG7<5v2s`IBY1J8*YlR28Xy^=aH>Uqei)X)c*s*eeW>+0|a2jaSB( z_4aLaD~k81Gf-nk&C6PL=rx&eKx3r^?V4lJe2;ljcGVv&x~uRdt?}^woAe2sPf2a* zN7kId&+%z;6{Tv2*UJ)<)znzUi=E3S(CYQ-pN}jwNeM=B>y~@uO-N#Ck3+xFPoxh{ zO7yN>PDJ&$%y#o_$1~&mZQ7x;8vFLXB@Pry81C3Z)-68}`>fZPS09yhO*}KS49l|# zX=LD^NX*$e4x+q;aNmbyEkbp!5$D@q*C&s5l`0;c$ySaimbQ0+Kp+s5UN0w8qYcO4 zBU(kZ=+(m%A0n{kW7V$VOv*LQTrivP!SS;uk5K$-)=}Wkvkm7)#|32L6m+8qjKVUu zj+?g-WkY;3Juw%06+in4K(-wi6A0qd5RVspUZ-3rUFFn+KG|y^NW^StYXL&2YEB3Z z+1RFs<}CH(DcJ}q<-MKcZRSgjY-CCo*vQbqO7M+_Bx9&``)J1$AyrZNMCZacm4zD& z?}oEU6uR=|!$tY#Yk%HJO;{lyhBOSU=!rz#okqpugBt29Uo~?P6Zf$*@r-zZ_aSX*KHSMMRD8-`3)k462!?a(!yZnpT{;lI6bSjL-x==US|&N z9qWYV$Y7Z>nPFW3pJ0ACab=H1Hoscx;H!F*=!+7%m$obf0}o~Mk)5;5;e<8-%I_HT zs#{nbnd}uIlaDRiq>C(=E&m%w{=(>bEEiOGNy>L{g<81pKY@*| zn8gcxWEiNqlJTL*P^DDBXl`1&O{)&lQ~;zAvlSL~UQKMG?wkZLesfL1tJ}NqO{lpl z(|m9a zg(9DF<$h8P0@UT&|Lu&0d(^2@T?t(*Qb{nSc79N)3W{&7@fhmE(K;Op#yk%aBNv(j z3ukM9_7MA9j|8qO<0E%uwT!(H3p)3iubSzw3$n#f{Bh!3dD+UPbp@{U_G{Nxx5c`( z7z!bu8HUaMvB4Tej<~UEhJA#Vtv22{ooFE2pn<=I$d%5J0}Jg2v{b;=jtuo zAztWdkh`zehs{X9$%q!^Kr8LN3Nc?tt8*G(y^!<3h^@a$7)-u-p&g8H9mtZmhemTK zHO3Asg<(#J-&I&0MuRMbmh}x)*#~@6H&1F%pMCCSL7NY6tk-Zm1eiEhqQ4o+<10e% z<=uGR@lz75kXo;sTX#BBsZW)LeJOxjMQO2S?@Dr}a(6~DQ=EqMOx=5Jz0Sj;wERvB z7x=#&6stj?FXifPT6KyJJBe^v?VzF$=Ejs*HxO9haJk~=FkE&A8P0-ei`7FbDzKkS zo#};`0_$_Aa-{HDu;L3gCWCAf0k7y#cGh;IVo*?-ahz^5R-jJLA1x3x0YGBmexdO^ms z&Pyc406{JQfP5ba;PoFe0RN+N71V!{;b>rM53B(8?^?;W23Hn+TG;pw?6v|h)6(;W z_UHCXE3d*{M%wGW(A(ZjDCJpk7HSS)yx&itdY`YqKDS?J*{B!M!2G2;|BX6&5>NfP z5CDKuY5;)#7a#ae41CSSy)3{`%g9#8+89_#*;2;{SUKpgdU;m0c2cUR8IVm&0RXW6 zg{_hKBHPeh$KKw+5?GZ;&%xHt#L+;@&fdcQH)Ef3g;nmGhL#-$Y91ZtPYJ)&gIo5p z1P9Cixhkw<{=x)2kAeF=NdFfI06+&g0W^OZ16S3H5`HO1*TKYG|34~cY%;#w7iiUf zz-j#}7^mUC09!w6$ky^VojmtAig6*N1E_UJpmhT$^^Zmj{M_vOFTl1AmX?3(@T?=O zvcmf9tPao=FM#a-wUl4FKAw4zYYe!{V-K`(>)-7ARS}eOW^<~(JP_>t zH*YAs`6Afd%E-vX(&&YLTLL}#_c?K07+{VJFQVTSzO(Zp9@t~!7uawT(26S3tZ<)z zW=#xa^FC`E*na7R_vD}0wkFP6dOGHM4!~9V_uKaW?i>Tv@RZrf;UGGJ22KH-NV;Ew zfbYb>7vYbWC7D?2o7id@I9prUK369DtA8m@f>So=JHieD4nhPI0O0ro{c8}kBp@&C zM!-~ex;plH##)xZ>3u;{SaFETdHG5ra(0*Lf|O=NyvYvr5}Y*%1^uT z3!A{9Xak4xS3e)b@J~|o4eWt=4_+9CL@tyv>8O!}0MJVs1L=R=xtrMjNx(m7>3NM& zVfRxT18sF0cyRe&SN}^(l5hVBZ*65|{_<*gmSCR7kk1251c4+C0I>aKvLQwONdoX- z0z5ChxZs{ed`9^!lL8z>Am#7&6es_(2rEnA2n=*=wG16z+R=XrabCFwp2P}(f}W$A zpW81@1$=y2h_!>3xfQThpB>PwUh;X&>3GUNO<7mPya51+X3&4_F2AHqXuc@L%F@#L z|5|g;-6u`F#|k9_PC+UF094Z-jTrc;zxp!RT1!V?$NKLp`I(A}XxvS^kau%Ujly{I}%4Z&uIR=+>_scti&P2o!$z z!)>qsTk;>#aK9^h)PO<89O%4mfy?u+>VV<-|3d#w(O6uSu)Dx*+Z(uZ{B`qa<^L!0 z9~FCnd$*pg)!#M!dkx@c1Nmig=aMg?|6URJcYXX`gY%c|5d%MlKfH|o&lNd;=l@=G z%2NI^8b$#B){{2uZ!>}|5pD+{(r2J@H_qYWaxj=bxmHT|EJXGztexuc>5=P z(duRTU-IAnj{QAn*)#Uny0mtE8T+>kX1}9<&v5i7I@R+<^#7Ib=yw6XUvK_X07&S| z0{-E$^S=xFPZ!jl1^u$i{OFej{fFynzr%mO4fN+I65?Kj|Ldv5Kb=;ehwyu7@Sjr7 dQeKquFV_eG{4wxc3y2v&^Za9`7P!>|{vQJqqVE6z literal 0 HcmV?d00001 -- 2.34.1 From 3097502ab48a125fa93f3b7836a68b6e3838ac47 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:50:00 +0800 Subject: [PATCH 05/72] ADD file via upload --- mindspore/ccsrc/transform-update/_init_.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/_init_.py diff --git a/mindspore/ccsrc/transform-update/_init_.py b/mindspore/ccsrc/transform-update/_init_.py new file mode 100644 index 00000000000..59ce74f2ab2 --- /dev/null +++ b/mindspore/ccsrc/transform-update/_init_.py @@ -0,0 +1,16 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Transformers for optimizing ast.""" +from .flatten_recursive_stmt import FlattenRecursiveStmt -- 2.34.1 From 43016e4088b550dc1ed5da7f90974d835ddeef65 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:50:47 +0800 Subject: [PATCH 06/72] ADD file via upload --- .../transform-update/array_ops_declare.cc | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/array_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/array_ops_declare.cc b/mindspore/ccsrc/transform-update/array_ops_declare.cc new file mode 100644 index 00000000000..6fec3a03a81 --- /dev/null +++ b/mindspore/ccsrc/transform-update/array_ops_declare.cc @@ -0,0 +1,204 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/array_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// const +INPUT_MAP(Const) = EMPTY_INPUT_MAP; +//输入映射,设为空 +ATTR_MAP(Const) = {{"value", ATTR_DESC(value, AnyTraits())}}; +//属性映射,属性value类型为AnyValue() +OUTPUT_MAP(Const) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 + +// Constant +INPUT_MAP(Constant) = EMPTY_INPUT_MAP; +//输入映射,设为空 +ATTR_MAP(Constant) = {{"value", ATTR_DESC(value, AnyTraits())}}; +//属性映射,属性value类型为AnyValue() +OUTPUT_MAP(Constant) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Constant, kNameConst, ADPT_DESC(Constant, Const)) +//注册Constant操作的适配器描述KNameConst + +// ScalarSummary +INPUT_MAP(Summary) = {{2, INPUT_DESC(x)}}; +//输入映射,x索引为2 +ATTR_MAP(Summary) = EMPTY_ATTR_MAP +//属性映射,设为空 +#ifndef ENABLE_SECURITY +//如果未定义ENABLE_SECURITY宏变量,则注册适配器描述信息 +//适配器描述用于将特定操作与指定的适配器关联,将ScalarSummaryImageSummaryTensorSummaryHistogramSummary和Debug操作与Summary适配器描述关联 +//操作的名称和prim空间下的kPrimScalarSummary、kPrimImageSummary、kPrimTensorSummary、kPrimHistogramSummary和kPrimDebug相匹配 +REG_ADPT_DESC(ScalarSummary, prim::kPrimScalarSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(ImageSummary, prim::kPrimImageSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(TensorSummary, prim::kPrimTensorSummary->name(), ADPT_DESC(Summary)) +REG_ADPT_DESC(HistogramSummary, prim::kPrimHistogramSummary->name(), ADPT_DESC(Summary)) +#endif +REG_ADPT_DESC(Debug, prim::kPrimDebug->name(), ADPT_DESC(Summary)) +//不论 ENABLE_SECURITY是否定义,都将Debug操作与Summary适配器描述关联 + + +// Data +INPUT_MAP(Data) = EMPTY_INPUT_MAP; +//输入映射,设为空 +ATTR_MAP(Data) = EMPTY_ATTR_MAP; +//属性映射,设为空 +REG_ADPT_DESC(Data, kNameParam, ADPT_DESC(Data)) +//注册Data操作的适配器描述KNameParam + +// Shape +INPUT_MAP(Shape) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Shape) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(Shape) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Shape, kNameShape, ADPT_DESC(Shape)) +//注册Shape操作的适配器描述KNameShape + +// GetShape +INPUT_MAP(GetShape) = EMPTY_INPUT_MAP; +//输入映射,设为空 +DYN_INPUT_MAP(GetShape) = {{1, DYN_INPUT_DESC(x)}}; +//动态输入映射,将索引为1的动态输入与名称为x的动态输入描述关联起来,用于后续操作 +ATTR_MAP(GetShape) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(GetShape) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(GetShape, kNameGetShape, ADPT_DESC(GetShape)); +//注册GetShape操作的适配器描述KNameGetShape + +// Reshape +INPUT_MAP(Reshape) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(shape)}}; +//输入映射,x索引为1,sharp索引为2 +ATTR_MAP(Reshape) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(Reshape) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Reshape, kNameReshape, ADPT_DESC(Reshape)) +//注册ReShape操作的适配器描述KNameReShape +REG_ADPT_DESC(FlattenGrad, kNameFlattenGrad, ADPT_DESC(Reshape)) +//注册FlattenGrad操作的适配器描述kNameFlattenGrad + +// TransShape +INPUT_MAP(TransShape) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +INPUT_ATTR_MAP(TransShape) = {{2, ATTR_DESC(outShape, AnyTraits(), AnyTraits>())}}; +ATTR_MAP(TransShape) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(TransShape) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(TransShape, kNameTransShape, ADPT_DESC(TransShape)) +//注册TransShape操作的适配器描述kNameTransShape + +// MirrorPad +INPUT_MAP(MirrorPad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +//输入映射,x索引为1,paddings索引为2 +ATTR_MAP(MirrorPad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +//属性映射,属性mode类型为string +OUTPUT_MAP(MirrorPad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(MirrorPad, kNameMirrorPad, ADPT_DESC(MirrorPad)) +//注册MirrorPad操作的适配器描述kNameMirrorPad + +// MirrorPadGrad +INPUT_MAP(MirrorPadGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +//输入映射,x索引为1,paddings索引为2 +ATTR_MAP(MirrorPadGrad) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +//属性映射,属性mode类型为string +OUTPUT_MAP(MirrorPadGrad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(MirrorPadGrad, kNameMirrorPadGrad, ADPT_DESC(MirrorPadGrad)) +//注册MirrorPadGrad操作的适配器描述kNameMirrorPadGrad + +// ExpandDims +INPUT_MAP(ExpandDims) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(axis)}}; +//输入映射,x索引为1,axis索引为2 +ATTR_MAP(ExpandDims) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(ExpandDims) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ExpandDims, kNameExpandDims, ADPT_DESC(ExpandDims)) +//注册ExpandDims操作的适配器描述kNameExpandDims + +// Squeeze +INPUT_MAP(Squeeze) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Squeeze) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +//属性映射,属性axis类型为int64_t +OUTPUT_MAP(Squeeze) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Squeeze, prim::kPrimSqueeze->name(), ADPT_DESC(Squeeze)) +//注册Squeeze操作的适配器描述kNameSqueeze返回的name变量 + +// ReverseSequence +INPUT_MAP(ReverseSequence) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(seq_lengths)}}; +//输入映射,x索引为1,seq_lengths索引为2 +ATTR_MAP(ReverseSequence) = {{"seq_dim", ATTR_DESC(seq_dim, AnyTraits())}, + {"batch_dim", ATTR_DESC(batch_dim, AnyTraits())}}; +//属性映射,属性seq_dim类型为int64_t,属性batch_dim类型为int64_t +OUTPUT_MAP(ReverseSequence) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ReverseSequence, kNameReverseSequence, ADPT_DESC(ReverseSequence)) +//注册ReverseSequence操作的适配器描述kNameReverseSequence + +// EditDistance +INPUT_MAP(EditDistance) = {{1, INPUT_DESC(hypothesis_indices)}, {2, INPUT_DESC(hypothesis_values)}, + {3, INPUT_DESC(hypothesis_shape)}, {4, INPUT_DESC(truth_indices)}, + {5, INPUT_DESC(truth_values)}, {6, INPUT_DESC(truth_shape)}}; +//输入映射,hypothesis_indices索引为1,hypothesis_values索引为2,hypothesis_shape索引为3, +// truth_indices索引为4,truth_values索引为5,truth_shape索引为6 +ATTR_MAP(EditDistance) = {{"normalize", ATTR_DESC(normalize, AnyTraits())}}; +//属性映射,属性normalize类型为int64_t +OUTPUT_MAP(EditDistance) = {{0, OUTPUT_DESC(output)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(EditDistance, kNameEditDistance, ADPT_DESC(EditDistance)) +//注册EditDistance操作的适配器描述kNameEditDistance + +// NonZeroWithValue +INPUT_MAP(NonZeroWithValue) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(NonZeroWithValue) = {{"transpose", ATTR_DESC(transpose, AnyTraits())}}; +//属性映射,属性transpose类型为int64_t +OUTPUT_MAP(NonZeroWithValue) = {{0, OUTPUT_DESC(value)}, {1, OUTPUT_DESC(index)}, {2, OUTPUT_DESC(count)}}; +//输出映射,value索引为0,index索引为1,count索引为2 +REG_ADPT_DESC(NonZeroWithValue, kNameNonZeroWithValue, ADPT_DESC(NonZeroWithValue)) +//注册NonZeroWithValue操作的适配器描述kNameNonZeroWithValue + +// NonZeroWithValueShape +INPUT_MAP(NonZeroWithValueShape) = {{1, INPUT_DESC(value)}, {2, INPUT_DESC(index)}, {3, INPUT_DESC(count)}}; +//输入映射,x索引为1,index索引为2,count索引为3 +ATTR_MAP(NonZeroWithValueShape) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(NonZeroWithValueShape) = {{0, OUTPUT_DESC(out_value)}, {1, OUTPUT_DESC(out_index)}}; +//输出映射,out_value索引为0,out_index索引为1,out_count索引为2 +REG_ADPT_DESC(NonZeroWithValueShape, kNameNonZeroWithValueShape, ADPT_DESC(NonZeroWithValueShape)) +//注册NonZeroWithValueShape操作的适配器描述kNameNonZeroWithValueShape + +// Unsqueeze +INPUT_MAP(Unsqueeze) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Unsqueeze) = {{"axis", ATTR_DESC(axes, AnyTraits(), AnyTraits>())}}; +//属性映射,属性axis类型为int64_t +OUTPUT_MAP(Unsqueeze) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Unsqueeze, kNameUnsqueeze, ADPT_DESC(Unsqueeze)) +//注册Unsqueeze操作的适配器描述kNameUnsqueeze +} // namespace mindspore::transform -- 2.34.1 From 5324ac9b68d6d7e2e829d837b0d3cb5e93c85a15 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:51:56 +0800 Subject: [PATCH 07/72] ADD file via upload --- .../transform-update/cluster_ops_declare.cc | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/cluster_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/cluster_ops_declare.cc b/mindspore/ccsrc/transform-update/cluster_ops_declare.cc new file mode 100644 index 00000000000..3a6ce21683c --- /dev/null +++ b/mindspore/ccsrc/transform-update/cluster_ops_declare.cc @@ -0,0 +1,32 @@ +/** + * Copyright 2022-2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/cluster_ops_declare.h" + +namespace mindspore::transform { +// KMeansCentroids +INPUT_MAP(KMeansCentroids) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(sum_square_y)}, {4, INPUT_DESC(sum_square_x)}}; +//输入映射,x索引为1,y索引为2,sum_square_y索引为3,sum_square_x索引为4 +ATTR_MAP(KMeansCentroids) = { + {"use_actual_distance", ATTR_DESC(use_actual_distance, AnyTraits(), AnyTraits())}}; +//属性映射,属性use_actual_distance类型为bool +OUTPUT_MAP(KMeansCentroids) = { + {0, OUTPUT_DESC(segment_sum)}, {1, OUTPUT_DESC(segment_count)}, {2, OUTPUT_DESC(kmean_total_sum)}}; +//输出映射,segment_sum索引为0,segment_count索引为1,kmean_total_sum索引为2 +REG_ADPT_DESC(KMeansCentroids, prim::kPrimKMeansCentroids->name(), ADPT_DESC(KMeansCentroids)) +//注册KMeansCentroids操作的适配器描述kPrimKMeansCentroids预设的name变量 +} // namespace mindspore::transform -- 2.34.1 From 21ab5fcd038b33a87b95948d58f7ca4bcc9b191c Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:53:17 +0800 Subject: [PATCH 08/72] ADD file via upload --- .../control_flow_ops_declare.cc | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/control_flow_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/control_flow_ops_declare.cc b/mindspore/ccsrc/transform-update/control_flow_ops_declare.cc new file mode 100644 index 00000000000..701acadf5fd --- /dev/null +++ b/mindspore/ccsrc/transform-update/control_flow_ops_declare.cc @@ -0,0 +1,41 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/control_flow_ops_declare.h" + +namespace mindspore::transform { +// Merge +INPUT_MAP(Merge) = EMPTY_INPUT_MAP; +//输入映射,设为空 +DYN_INPUT_MAP(Merge) = {{1, DYN_INPUT_DESC(x)}}; +//动态输入映射,将索引为1的动态输入与名称为x的动态输入描述关联起来,用于后续操作 +ATTR_MAP(Merge) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(Merge) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(value_index)}}; +//输出映射,y索引为0,value_index索引为1 +REG_ADPT_DESC(Merge, kNameMerge, ADPT_DESC(Merge)) +//注册Merge操作的适配器描述kNameMerge + +// Switch +INPUT_MAP(Switch) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(pred)}}; +//输入映射,data索引为1,pred索引为2 +OUTPUT_MAP(Switch) = {{0, OUTPUT_DESC(output_false)}, {1, OUTPUT_DESC(output_true)}}; +//输出映射,output_false索引为0,output_true索引为1 +ATTR_MAP(Switch) = EMPTY_ATTR_MAP; +//属性映射,设为空 +REG_ADPT_DESC(Switch, kNameGeSwitch, ADPT_DESC(Switch)) +//注册Switch操作的适配器描述kNameGeSwitch +} // namespace mindspore::transform -- 2.34.1 From f5f8630135180a300f10e4a15ccf93a6b2abdccd Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:53:52 +0800 Subject: [PATCH 09/72] ADD file via upload --- mindspore/ccsrc/transform-update/convert.cc | 2328 +++++++++++++++++++ 1 file changed, 2328 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/convert.cc diff --git a/mindspore/ccsrc/transform-update/convert.cc b/mindspore/ccsrc/transform-update/convert.cc new file mode 100644 index 00000000000..23a5e63278b --- /dev/null +++ b/mindspore/ccsrc/transform-update/convert.cc @@ -0,0 +1,2328 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/convert.h" + +#include +#include +#include +#include "include/common/utils/utils.h" + +#include "base/core_ops.h" +#include "frontend/operator/ops.h" +#include "utils/log_adapter.h" +#include "ir/graph_utils.h" +#include "utils/symbolic.h" +#include "include/common/utils/config_manager.h" +#include "include/common/utils/convert_utils.h" +#include "utils/ms_context.h" +#include "utils/check_convert_utils.h" +#include "include/transform/graph_ir/op_adapter_map.h" +#include "ops/state_ops.h" +#include "ops/array_ops.h" +#include "ops/elewise_calculation_ops.h" +#include "ops/math_ops.h" +#ifdef ENABLE_D +#include "ops/save_ops.h" +#endif +#include "transform/graph_ir/op_adapter.h" +#include "transform/graph_ir/op_adapter_desc.h" + +namespace mindspore { //namespace:ռ +namespace transform { +using std::endl; + +using ge::Operator; +using mindspore::kAnyValue; +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; +using Variable = ge::op::Variable; +using Constant = ge::op::Constant; +using Assign = ge::op::Assign; +using Data = ge::op::Data; + +namespace { +std::vector GetOrderedCNodes(const FuncGraphPtr fg) { //úĹͨȡ˳еCNodeڵ㡣 + MS_EXCEPTION_IF_NULL(fg); ////鴫ǷΪգΪ׳쳣 + auto BelongSameGraph = std::bind(IncludeBelongGraph, fg, std::placeholders::_1); + auto succ_include_fv = [&fg](const AnfNodePtr &node) -> std::vector { + std::vector vecs; + if (node == nullptr) { //Ϊָ룬ֱӷؿյ + return vecs; + } + if (node->isa()) { //һCNodeڵ㣬ж + auto cnode = node->cast(); //ȡCNodeڵ룬ÿ롣 + auto &inputs = cnode->inputs(); + // Check if free variables used. + for (const auto &input : inputs) { + auto input_fg = GetValueNode(input); //һͼֵڵ㣨FuncGraphPtrͣж + if (input_fg) { + for (auto &fv : input_fg->free_variables_nodes()) { + if (fv->func_graph() == fg && fg->nodes().contains(fv)) {//úͼɱڵ㣨free_variables_nodes + vecs.push_back(fv); //ɱڵĺͼ봫ĺͼͬҺͼаɱڵ㣬򽫸ɱڵӵvecs + } + } + } + } + (void)vecs.insert(vecs.end(), inputs.begin(), inputs.end()); //CNodeڵӵvecsĩβ + } + return vecs; //vecs + }; + + return TopoSort(fg->get_return(), succ_include_fv, BelongSameGraph); +} +} // namespace + + +// ---------------implement of DfGraphConvertor------------- +bool IsCaseNode(const CNodePtr node) { //һΪIsCaseNodeCNodePtrĺжϸĽڵǷΪ"case"ڵ㡣 + MS_EXCEPTION_IF_NULL(node); //ʹúȷIJΪգΪ׳쳣 + if (!node->inputs().empty() && node->input(0)->isa() && //ͨжǷǿգҵһǷΪnodeCNode + GetCNodeFuncName(node->input(0)->cast()) == "switch_layer") { //ͨúGetCNodeFuncNameȡһڵĺƣַ"switch_layer"бȽ + return true; //"switch_layer"ȣ򷵻trueʾýڵ"case"ڵ + } + return false; //򷵻falseʾýڵ㲻"case"ڵ +} + +/* +úĿǻȡ CNode Ŀ꺯 "case" ڵ㣬Ŀ꺯 "kNameCase"ڵ㣬Ŀ꺯 +GetCNodeFuncName(cnode) ķֵΪ"switch_layer"򷵻һַ +ھӦУĿ꺯ںĴ߼ +*/ +std::string GetCNodeTargetFuncName(const CNodePtr cnode) { //һΪ CNodePtr ָ cnode Ϊһ std::string ͵Ŀ꺯 + if (IsCaseNode(cnode)) { //жϸcnodeǷcaseڵ㡣 + return string(kNameCase); //caseڵ㣬ֱӷһַ kNameCaseʾĿ꺯ΪkNameCase + } + auto name = GetCNodeFuncName(cnode); //GetCNodeFuncNameڻȡ cnode ĺ䱣һΪ name ľֲС + if (name == "switch_layer") { //麯nameǷΪswitch_layer + name = ""; //ǣ name գֵΪַ + } + return name; //Ŀ꺯name +} + +/* +úǸݽڵͺĿ꺯ҶӦָ롣 +ڴͬ͵IJһģʽͨģʽ +ʹͼתܹشͬ͵ĽڵͲ +*/ +OpAdapterPtr DfGraphConvertor::FindAdapter(const AnfNodePtr node, bool train) { + MS_EXCEPTION_IF_NULL(node); //ָ node ǷΪգΪգ׳쳣 + if (node->isa()) { //䣬ж node ǷΪ CNode ͵Ľڵ + auto cnode = node->cast(); // CNode ͵Ľڵ㣬תΪ CNodePtr ͵ָ룬ֵcnode + + std::string name = kNameCustomOp; //һΪnameַʼΪһΪ kNameCustomOp ַ + if (!IsCustomCNode(cnode)) { //cnodeԶڵ㣨 IsCustomCNode жϣ + name = GetCNodeTargetFuncName(cnode); //nameΪ GetCNodeTargetFuncName(cnode)ķֵ,ȡcnodeĿ꺯 + } + + auto it_adpt = OpAdapterMap::get().find(name); //OpAdapterMapвnameӦOpAdapterMap һ̬ڴ洢ͬӦOpAdapterMap::get() OpAdapterMap + if (it_adpt != OpAdapterMap::get().end()) { //ҵnameӦ + return it_adpt->second->Get(train); //GettrainΪݽȥָ it_adpt->second + } + MS_LOG(EXCEPTION) << "Can't find OpAdapter for " << name; //δҵ쳣־ʾ޷ҵ + } + + if (node->isa()) { //ͬ͵Ľڵ㣺ValueNode Parameter + return OpAdapterMap::get()[kNameConst]->Get(train); //ݽڵͣѡӦضӦָ롣 + } + if (node->isa()) { + return OpAdapterMap::get()[kNameParam]->Get(train); + } + return OpAdapterPtr(nullptr); //ڵͲ CNodeValueNode Parameter򷵻һյ OpAdapterPtr +} + +/* +úڳʼѭصIJOperatorӵ init_input init_ops_ + training_ֵǷʼѭӦıͲĴ͹ +ͼתп漰ѭĴ +*/ +void DfGraphConvertor::InitLoopVar(std::vector *init_input) { + MS_EXCEPTION_IF_NULL(init_input); //ãڼָinit_inputǷΪգΪգ׳쳣 + if (this->training_) { //this->training_ֵΪ棨 training_ Ϊ棩ִ if е + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64); //ͨ std::make_shared(...) ĸΪ var_iter_numvar_loop_condvar_one var_zero ָ롣 + auto var_iter_num = std::make_shared("npu_runconfig/iterations_per_loop"); //Щָָ Variable ʵÿʵһ + auto var_loop_cond = std::make_shared("npu_runconfig/loop_cond"); + auto var_one = std::make_shared("npu_runconfig/one"); + auto var_zero = std::make_shared("npu_runconfig/zero"); + (void)var_iter_num->update_output_desc_y(desc); //ֱΪĸvar_iter_numvar_loop_condvar_one var_zero GeTensorDesc + (void)var_loop_cond->update_output_desc_y(desc); + (void)var_one->update_output_desc_y(desc); + (void)var_zero->update_output_desc_y(desc); + vars_["npu_runconfig/iterations_per_loop"] = var_iter_num; //ĸӵ vars_ У + vars_["npu_runconfig/loop_cond"] = var_loop_cond; //vars_ һԱڴ洢ӳϵ + vars_["npu_runconfig/one"] = var_one; + vars_["npu_runconfig/zero"] = var_zero; + + //ĸΪ const_iter_numconst_loop_condconst_one const_zero ָ롣 + //Щָָ Constant ʵÿʵһ + int64_t value = 0; + auto const_iter_num = std::make_shared("const/npu_runconfig/iterations_per_loop"); + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + value = ConfigManager::GetInstance().iter_num(); + } else { + MS_LOG(INFO) << "Run with normal(non-sink) mode, the iterator number will always be 1"; + ConfigManager::GetInstance().ResetIterNum(); + } + + //ͨ set_attr_value Ϊĸ˲ֵֵͬΪͣ + value -= 1; // iteration start from 0, the max iteration number for n loop should be n-1 + (void)const_iter_num->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_loop_cond = std::make_shared("const/npu_runconfig/loop_cond"); + value = 0; + (void)const_loop_cond->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_one = std::make_shared("const/npu_runconfig/one"); + value = 1; + (void)const_one->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + auto const_zero = std::make_shared("const/npu_runconfig/zero"); + value = 0; + (void)const_zero->set_attr_value(GeTensor(desc, reinterpret_cast(&value), sizeof(int64_t))); + + //ֱΪĸconst_iter_numconst_loop_condconst_one const_zero GeTensorDesc + (void)const_iter_num->update_output_desc_y(desc); + (void)const_loop_cond->update_output_desc_y(desc); + (void)const_one->update_output_desc_y(desc); + (void)const_zero->update_output_desc_y(desc); + + //ĸΪ assign_iter_numassign_loop_condassign_one assign_zero ָ롣 + //Щָָ Assign ʵÿʵһֵ + //ֱͨ set_input_ref set_input_value Ϊĸֵúֵ + auto assign_iter_num = std::make_shared("assign/npu_runconfig/iterations_per_loop"); + (void)assign_iter_num->set_input_ref(*var_iter_num).set_input_value(*const_iter_num); + auto assign_loop_cond = std::make_shared("assign/npu_runconfig/loop_cond"); + (void)assign_loop_cond->set_input_ref(*var_loop_cond).set_input_value(*const_loop_cond); + auto assign_one = std::make_shared("assign/npu_runconfig/one"); + (void)assign_one->set_input_ref(*var_one).set_input_value(*const_one); + auto assign_zero = std::make_shared("assign/npu_runconfig/zero"); + (void)assign_zero->set_input_ref(*var_zero).set_input_value(*const_zero); + + // var_iter_numvar_loop_condvar_one var_zero ӵ init_input Уinit_input һIJڴ洢ʼ + // var_iter_numvar_loop_condvar_onevar_zeroconst_iter_numconst_loop_condconst_oneconst_zeroassign_iter_num + //assign_loop_condassign_one assign_zero ӵ init_ops_ Уinit_ops_ һԱڴ洢ʼ + init_input->push_back(*var_iter_num); + init_input->push_back(*var_loop_cond); + init_input->push_back(*var_one); + init_input->push_back(*var_zero); + init_ops_.push_back(var_iter_num); + init_ops_.push_back(var_loop_cond); + init_ops_.push_back(var_one); + init_ops_.push_back(var_zero); + init_ops_.push_back(const_iter_num); + init_ops_.push_back(const_loop_cond); + init_ops_.push_back(const_one); + init_ops_.push_back(const_zero); + init_ops_.push_back(assign_iter_num); + init_ops_.push_back(assign_loop_cond); + init_ops_.push_back(assign_one); + init_ops_.push_back(assign_zero); + } +} + +/* +úǸݸIJ nameҶӦָ롣 +ڴͬ͵IJһģʽͨģʽʹͼתܹشͬ͵ĽڵͲ +*/ +OpAdapterPtr DfGraphConvertor::FindAdapter(const std::string &name, bool train) { + auto it = OpAdapterMap::get().find(name); // OpAdapterMapвnameӦOpAdapterMapһ̬ڴ洢ͬӦOpAdapterMap::get() OpAdapterMap á + if (it != OpAdapterMap::get().end()) { //ҵ name Ӧget + return it->second->Get(train); //trainΪݽȥ,ָ it->second + } + MS_LOG(EXCEPTION) << "Can't find OpAdapter for " << name; //δҵ쳣־ʾ޷ҵ +} + +/* +úɲʼͼGraphvizʽ init_sout_С +GraphvizһڻͼεĹߣԽͼοӻ͵ԡ +*/ +void DfGraphConvertor::DrawParamInitSubGraph(const std::string &name, const AnfNodePtr &it) { + // draw init subgraph ݲ name ͽڵ it Ʋʼͼ + init_sout_ << "op_assign" << it.get() << "[label=<"; //ʹ<<Ϣӵ init_sout_ + //һ "op_assign{it.get()}[label=<" ַ + // it.get() ǽڵ it ֵָop_assign һͼڵıʶڱʾʼĸֵ + init_sout_ << "" << endl; + init_sout_ << ""; + init_sout_ << ""; //ʹ HTML table ʽͼĽṹ "resource" "value" УӦıǩ + init_sout_ << ""; + init_sout_ << "" << endl; + init_sout_ << "" << endl; + init_sout_ << "
resourcevalue
" + << "\"assign_" << name << "\"
> shape=plaintext]" << endl; + init_sout_ << "param" << it.get() << "[shape=octagon, label=\"" << name << "\"]" << endl; //һ "param{it.get()}[shape=octagon, label="{name}"]" Ľڵ㣬 it.get() ǽڵ it ֵָname Dz + init_sout_ << "const" << it.get() << "[label= \"" << name << "_const" //һ "const{it.get()}[label="{name}_const" shape=ellipse]" Ľڵ㣬 it.get() ǽڵ it ֵָname Dz + << "\" shape=ellipse]" << endl; + init_sout_ << "param" << it.get() << "->" //ƴ param{it.get()} ڵ㵽 op_assign{it.get()}:1 ڵıߣʾֵԴresource֡ + << "op_assign" << it.get() << ":1" << endl; + init_sout_ << "const" << it.get() << "->" //ƴ const{it.get()} ڵ㵽 op_assign{it.get()}:2 ڵıߣʾֵֵvalue֡ + << "op_assign" << it.get() << ":2" << endl; +} + +/* +úòʼͼͼ洢 init_graph_Уڲʼļ㡣 +ͼתп漰ʼͳȲ衣 +*/ +void DfGraphConvertor::SetupParamInitSubGraph(const TensorOrderMap &tensors, std::vector *init_input) { + DfGraphPtr init_graph = std::make_shared("init"); //һΪ init_graph DfGraph 󣬲Ϊ "init" + std::vector nodes = GetOrderedCNodes(anf_graph_); //ͨ GetOrderedCNodes(anf_graph_) ȡͼеڵ㣬洢 nodes С + + for (auto &it : nodes) { //nodesеÿڵit + MS_EXCEPTION_IF_NULL(it); + if (it->isa()) { //ڵǷΪValueNode + if (IsValueNode(it)) { //ڷŽڵ SymbolicKeyInstanceҵӦı Variable洢 op_cache_ Уһʾӵ compute_sout_ 䡣 + auto symbolic = GetValueNode(it); + auto name = std::static_pointer_cast(symbolic->node())->name(); + auto iter = vars_.find(name); // get corresponding variable op + if (iter != vars_.end()) { + op_cache_[it.get()] = iter->second; + // #ifdef DRAW_GE_GRAPH + compute_sout_ << op_draw_name_[params_[name].get()] << " -> " << op_draw_name_[it.get()] + << "[style=\"dotted\"]" << endl; + // #endif + } + } else if (IsValueNode(it)) { //üڵ RefKeyҲҵӦı Variable洢 op_cache_ Уһʾӵ compute_sout_ 䡣 + auto refkey = GetValueNode(it); + MS_EXCEPTION_IF_NULL(refkey); + auto name = refkey->tag(); + auto iter = vars_.find(name); // get corresponding variable op + if (iter != vars_.end()) { + op_cache_[it.get()] = iter->second; + compute_sout_ << op_draw_name_[params_[name].get()] << " -> " << op_draw_name_[it.get()] + << "[style=\"dotted\"]" << endl; + } + } + } + } + + for (auto &it : tensors) { // TensorOrderMap еIJ vars_ӳ䣩еIJӵ vars_ УӦıΪ nullptr + if (vars_.find(it.first) == vars_.end()) { + MS_LOG(WARNING) << "Init parameter " << it.first << " didn't appear in graph."; + vars_[it.first] = nullptr; + } + } + + // set up init sub graph + if (init_input->size()) { + // init sub graph needs no input + MS_LOG(INFO) << "Build data init subgraph."; + (void)init_graph->SetInputs(*init_input); //óʼͼ init_graph_ Ϊ init_input洢 init_graph_ С init_input Ϊգ˵ʼͼҪ룬 init_graph_ Ϊ nullptr + this->init_graph_ = init_graph; + } else { + this->init_graph_ = nullptr; + } +} + +/* +úǸݼơͽڵ㴴ݼ洢 out_handle_cache_Уڹͼʱʹôݼ +ͼתп漰ݼĴ滻 +*/ +void DfGraphConvertor::MakeDatasetHandler(const std::string &name, const size_t &input_idx, const AnfNodePtr &it) { + MS_LOG(INFO) << "The " << name << " is the " << input_idx << "(st/nd/th) input"; //־ʾǰݼƺ + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { //ùеݼģʽǷΪ "DS_SINK_MODE" + auto getnext_idx = static_cast(input_idx); //תΪ int64_t ͵ı getnext_idx + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); //ùлȡݼ洢ڱ param С + if (!param.input_indexes().empty() && input_idx <= param.input_indexes().size()) { //ݼеб input_indexes() Ϊգ input_idx СڵбĴС getnext_idx ӳΪбеֵȥ1Ϊ0ʼ + getnext_idx = param.input_indexes()[input_idx] - 1; // input_idx start from 0. + MS_LOG(INFO) << "remap input_index:" << input_idx << " to getnext_index:" << getnext_idx << "."; + } + // use iterator_getnext op with output_name instead of data op in BuildGraph. + if (dataset_iter_getnext_ != nullptr) { /// dataset_iter_getnext_ Ϊգ򽫴洢 out_handle_cache_ Сdataset_iter_getnext_ һݼڵIJOperator + out_handle_cache_[it.get()] = OutHandler(dataset_iter_getnext_, "y" + std::to_string(getnext_idx)); + } + } +} + +/* +úĿǸݹ㲥㲥͹㲥ͼϢ㲥ͼ洢 broadcast_graph_Уʵֹ㲥 +㲥ָڼнάԶչΪάݣԱڽ㡣 +ͼתУ㲥ͼĹ漰άչݶȴ +*/ +void DfGraphConvertor::SetupBroadcast(const std::shared_ptr &broadcast, + const std::vector &broadcast_desc, + const DfGraphPtr &broadcast_graph, std::vector broadcast_input) { + //const std::shared_ptr &broadcast㲥ָ룩 + //const std::vector &broadcast_desc㲥 + //const DfGraphPtr &broadcast_graph㲥ͼָ룩 + //std::vector broadcast_input㲥 + MS_LOG(INFO) << "build broadcast subgraph"; //־ʾڹ㲥ͼ + if (broadcast_desc.size() != broadcast_input.size()) { //㲥Ƿڹ㲥ȣ׳쳣 + MS_LOG(EXCEPTION) << "Desc number of BroadCast is not equal to number of Input"; + } + //ͨ create_dynamic_input_x create_dynamic_output_y Ϊ㲥̬ + (void)broadcast->create_dynamic_input_x(static_cast(broadcast_input.size())); + (void)broadcast->create_dynamic_output_y(static_cast(broadcast_desc.size())); + for (unsigned int i = 0; i < broadcast_input.size(); i++) { //ʹѭΪ㲥Ķ̬Ӧ + (void)broadcast->set_dynamic_input_x(i, broadcast_input[i]); + (void)broadcast->update_dynamic_output_desc_y(i, broadcast_desc[i]); + } + (void)broadcast_graph->SetInputs(broadcast_input); //㲥ͼ broadcast_graph Ϊ broadcast_input㲥ͼ洢 broadcast_graph_ С + this->broadcast_graph_ = broadcast_graph; +} + +/* +úĿǸݸ TensorOrderMapʼصijʼͼͲ +ͼתУʼһҪIJ裬ú˲ĴʼݵԼʼͼĹ +*/ +void DfGraphConvertor::InitParamWithData(const TensorOrderMap &tensors) { + int index = 0; //ʼһЩ indexinit_inputʼͼȡ + std::vector init_input; + for (auto it : tensors) { // tensors еÿ itݲҶӦĽڵ node + std::string name = it.first; + auto node_itor = params_.find(name); + // if name not in params_, create a node in graph + if (node_itor == params_.end()) { // params_ УʾòڵδʱһΪ name + "_temp" ½ڵ㣬תΪͼ + MS_LOG(WARNING) << name << " is not in params, and create a new node."; + ParameterPtr param = std::make_shared(nullptr); + name = name + "_temp"; + param->set_name(name); + (void)ConvertParameter(param); + node_itor = params_.find(name); + } + auto node = node_itor->second; //ݽڵ node ҶӦIJOperator洢 op_itor Уδҵ׳쳣 + auto op_itor = op_cache_.find(node.get()); + if (op_itor == op_cache_.end()) { + MS_LOG(EXCEPTION) << "Can not find op for node " << node->ToString() << "."; + } + auto adpt = FindAdapter(kNameParam, training_); //Ҳ adptݲ kNameParam ѵ״̬ training_ ȡΪһ + if (adpt == nullptr) continue; + auto param_op = adpt->generate(name + "_data"); //ݲ name һΪ name + "_data" IJ param_op + MS_LOG(INFO) << "Add parameter " << name << " as input, index " << index << "."; + + if (!training_) { //ѵ״̬ training_ʾǰ׶ΣҪʼӵͼС + auto adpt_const = FindAdapter(kNameConst, training_); //ҳ adpt_constݲ kNameConst ѵ״̬ training_ ȡΪһ + if (adpt_const == nullptr) continue; + auto const_op = adpt_const->generate(name + "_const"); + (void)adpt_const->setAttr(const_op, "value", it.second); // const_opó "value" Ϊijʼݡ + + auto const_op_desc = TransformUtil::GetGeTensorDesc(it.second->shape_c(), it.second->data_type(), kOpFormat_NCHW); //ʼݵ const_op_descµ + if (const_op_desc == nullptr) { + MS_LOG(WARNING) << "Create variable " << name << " output descriptor failed!"; + continue; + } + (void)std::static_pointer_cast(const_op)->update_output_desc_y(*const_op_desc); + + vars_[name] = const_op; + op_itor->second = const_op; + continue; + } + + // create tensor descriptor for output descriptor descʾ״ͺ͸ʽ + auto desc = TransformUtil::GetGeTensorDesc(it.second->shape_c(), it.second->data_type(), kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " output descriptor failed!"; + continue; + } + + // we need three variable ops for each graph with same name + // build init subgraph + //ڷdzʼݣit.second->is_init() == 0param_opinit_var assign_op init_ops_ init_input + if (it.second->is_init() == 0) { + (void)std::static_pointer_cast(param_op)->set_attr_index(index++); //ڳʼݣٴֱӽ滻Ϊ param_op + auto init_var = std::make_shared(name); + auto assign_op = std::make_shared("assign_" + name); + (void)init_var->update_output_desc_y(*desc); + (void)assign_op->set_input_ref(*init_var).set_input_value(*param_op); + init_input.push_back(*init_var); + init_ops_.push_back(param_op); + init_ops_.push_back(assign_op); + init_ops_.push_back(init_var); + } + + auto variable = std::make_shared(name); + (void)variable->update_output_desc_y(*desc); + // do not use read variable while variable sink + MS_LOG(DEBUG) << "InitParam, op_name = " << name << ", var = " << variable->GetName() << "."; + op_itor->second = variable; // replace parameter with variable + vars_[name] = variable; // prevent the variable operator from being freed + DrawParamInitSubGraph(name, node); // DrawParamInitSubGraph Ʋʼͼ + } + InitLoopVar(&init_input); // InitLoopVar ʼѭ + SetupParamInitSubGraph(tensors, &init_input); // SetupParamInitSubGraph òʼͼ +} + + +/* +úĿdzʼͼתݸ TensorOrderMapвʼݴ +ͼתУijʼݴͼеҪ衣 +*/ +// convert all parameter need initialize to variable +DfGraphConvertor &DfGraphConvertor::InitParam(const TensorOrderMap &tensors) { + size_t input_idx = 0; //ʼ input_idx + if (error_ != SUCCESS) { // error_ ǷΪ SUCCESSǣֱӷصǰͼת + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { // anf_graph_ anf_graph_->output() ǷϷ + error_ = INVALID_ARGUMENT; //Ϸ error_ Ϊ INVALID_ARGUMENT ϢȻ󷵻صǰͼת + MS_LOG(ERROR) << "Invalid AnfGraph in InitParam."; + return *this; + } + + // Processing input with MakeDatasetHandler + for (auto &it : anf_graph_->parameters()) { // anf_graph_->parameters()ͼIJڵ㡣 + auto op_itor = op_cache_.find(it.get()); // converted node ÿڵ itҶӦIJOperator洢 op_itor С + if (it->isa() && op_itor != op_cache_.end()) { //ڵ Parameter op_cache_ ҵ˶ӦIJʾýڵΪڵ㣬һҪݴ + string name = std::static_pointer_cast(it)->name(); //ȡڵ name + auto tensor_itor = tensors.find(name); // in init value map + if (tensor_itor == tensors.end()) { //Ҹ tensors ǷڸòijʼݣڣҪݴ + DfGraphConvertor::MakeDatasetHandler(name, input_idx, it); // MakeDatasetHandler ݼΪڵ㴴ݼݲơͽڵ㡣 + input_idx++; // input_idxʾһ롣 + } + } + } + InitParamWithData(tensors); // InitParamWithData вʼݸ tensors ɲĴʼݵӺͳʼͼĹ + init_sout_ << "}" << endl; //ʼͼ + return *this; //صǰͼתá +} + +//ǻԤ +/* +úĿǸѳʼıͱ Saveͼ +ͼתУһҪIJ裬ú˱ͱĴԼͼĹ +*/ +#if (defined ENABLE_D) //Ԥָ ENABLE_D ʱŻ´顣 +void DfGraphConvertor::BuildSaveCheckpointGraph() { + std::vector graph_inputs; //ʼ graph_inputsͼ + ge::op::Save save_op("save_parms"); //save_op Save ʵ + int save_op_is_active = 0; //save_op_is_activeǷ񼤻ı־ʼֵΪ0 + size_t index = 0; //indexڱĶ̬ + string name; //nameƣ + + auto count_size = std::count_if(vars_.begin(), vars_.end(), [](const auto &it) { + return LongToUlong(it.second == nullptr || it.first.find("/") != std::string::npos); + }); //ʹ std::count_if ͳ vars_ ֵΪ nullptr а "/" ŵı count_size С + + (void)save_op.create_dynamic_input_tensors(static_cast(vars_.size() - static_cast(count_size))); + // save_op.create_dynamic_input_tensors Save Ķ̬Ϊ vars_.size() - count_size + + // for each "parameter" in anf graph excluding "input" + for (const auto &it : vars_) { // vars_ еÿÿǿƲ "/" ŵıӦıӵ save_op Ķ̬С + name = it.first; + if (it.second == nullptr || name.find("/") != std::string::npos) continue; + Variable variable(name); + (void)variable.update_output_desc_y(it.second->GetOutputDesc(0)); + (void)save_op.set_dynamic_input_tensors(static_cast(index++), variable); + + graph_inputs.push_back(variable); //ÿӵ graph_inputs У save_op + + if (save_op_is_active == 0) { // save_op_is_active Ϊ0ûЧıͼ + checkpoint_sout_ << "op_save" << &save_op << "[label=<"; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "" << endl; + checkpoint_sout_ << "
tensor
" + << "\"saveop" + << "\"
> shape=plaintext]" << endl; + } + + checkpoint_sout_ << "param" << it.second << "[shape=octagon, label=\"" << name << "\"]" << endl; + + checkpoint_sout_ << "param" << it.second << "->" + << "op_save" << &save_op << ":1" << endl; + save_op_is_active = 1; + } + if (save_op_is_active) { // save_op_is_active Ϊ1Чı򴴽ͼ checkpoint_graphΪ graph_inputs Ϊ graph_output save_op + std::vector graph_output; + graph_output.emplace_back(save_op); + DfGraphPtr checkpoint_graph = std::make_shared("checkpoint"); + (void)checkpoint_graph->SetInputs(graph_inputs); + (void)checkpoint_graph->SetOutputs(graph_output); + this->save_ckp_graph_ = checkpoint_graph; //ͼ洢 save_ckp_graph_ С + } else { + this->save_ckp_graph_ = nullptr; + } + + checkpoint_sout_ << "}" << endl; //ͼ + return; +} +#endif + + +/* +úĿɹ㲥ͼڷֲʽѵжԲй㲥 +ͼתУ㲥һҪIJ裬ú˹㲥͹㲥ͼĹ +*/ +DfGraphConvertor &DfGraphConvertor::GenerateBroadcastGraph(const TensorOrderMap &tensors) { + if (error_ != SUCCESS) { // error_ ǷΪ SUCCESSǣֱӷصǰͼת + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { // anf_graph_ anf_graph_->output() ǷϷ + error_ = INVALID_ARGUMENT; //Ϸ error_ Ϊ INVALID_ARGUMENT Ϣ + MS_LOG(ERROR) << "Invalid AnfGraph in generate broadcast graph"; + return *this; //Ȼ󷵻صǰͼת + } + + DfGraphPtr broadcast_graph = std::make_shared("broadcast"); //㲥ͼ broadcast_graph + // collect the operators create for broadcast sub graph, in order to avoid auto release + std::vector broadcast_input; //ʼ broadcast_input㲥ͼ + std::vector broadcast_desc; //broadcast_desc㲥ͼ + auto broadcast = std::make_shared("broadcast_parameter"); //㲥 HcomBroadcastΪ "broadcast_parameter" + (void)broadcast->set_attr_root_rank(0); //ù㲥ĸڵ root_rank Ϊ 0 + (void)broadcast->set_attr_group("hccl_world_group"); //ù㲥ͨ group Ϊ "hccl_world_group" + broadcast_ops_.push_back(broadcast); //㲥 broadcast_ops_ + + // find every parameter, build broadcast subgraph (or initialize the parameter with constant) + for (auto &it : anf_graph_->parameters()) { // anf_graph_->parameters()ͼIJڵ㡣 + auto op_itor = op_cache_.find(it.get()); // converted node ÿڵ itҶӦIJOperator洢 op_itor С + if (it->isa() && op_itor != op_cache_.end()) { //ڵ Parameter op_cache_ ҵ˶ӦIJڸ tensors дڶӦijʼݣʾýڵΪڵ㣬Ҫй㲥 + string name = std::static_pointer_cast(it)->name(); //ȡڵ name + auto tensor_itor = tensors.find(name); // in init tensor map + if (tensor_itor != tensors.end()) { //Ҹ tensors ǷڸòijʼݣڣʾҪй㲥 + auto tensor = tensor_itor->second; + auto shape_ge = tensor->shape_c(); //ȡ״ shape_ge + + // create tensor descriptor for output descriptor + // descʾ״͡ + auto desc = TransformUtil::GetGeTensorDesc(shape_ge, tensor->data_type(), kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create variable " << name << " output descriptor failed!"; + continue; + } + + // build broadcast subgraph + if (distribute_) { // distribute_ Ϊ棨ʾзֲʽѵ򹹽㲥ͼ + auto broadcast_var = std::make_shared(name); //Ҫ㲥IJӦı broadcast_varӵ broadcast_input broadcast_desc + (void)broadcast_var->update_output_desc_y(*desc); + broadcast_input.push_back(*broadcast_var); + broadcast_desc.push_back(*desc); + broadcast_ops_.push_back(broadcast_var); // broadcast_ops_ С + } + } + } + } + + // set up broadcast sub graph + if (!broadcast_input.empty()) { //ù㲥ͼ SetupBroadcast й㲥ͼĹ + DfGraphConvertor::SetupBroadcast(broadcast, broadcast_desc, broadcast_graph, broadcast_input); + } else { + this->broadcast_graph_ = nullptr; + } + return *this; //صǰͼתá +} + +/* +úĿɼͼͼתбģ͵IJ +ͼתУɼͼһҪIJ裬ڽѵеģͲ浽ļУԱҪʱģ͵Ļָͼѵ +*/ +DfGraphConvertor &DfGraphConvertor::GenerateCheckpointGraph() { + if (error_ != SUCCESS) { // error_ ǷΪ SUCCESSǣϢֱӷصǰͼת + MS_LOG(ERROR) << "Generate checkpoint graph failed, found error code " << error_ << "."; + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { // anf_graph_ anf_graph_->output() ǷϷ + error_ = INVALID_ARGUMENT; //Ϸ error_ Ϊ INVALID_ARGUMENT Ϣ + MS_LOG(ERROR) << "Invalid AnfGraph in GenerateCheckpointGraph"; + return *this; //Ȼ󷵻صǰͼת + } +#ifdef ENABLE_D //ָ #ifdef ENABLE_D ڲִ² + auto ms_context = MsContext::GetInstance(); //ȡȫΨһ MsContext ʵ ms_context + MS_EXCEPTION_IF_NULL(ms_context); // ms_context ǷΪգΪգ׳쳣 + if (ms_context->backend_policy() == "ge") { //鵱ǰĺ˲ backend_policy ǷΪ "ge"ʾʹûGraphEngineĺˣ + BuildSaveCheckpointGraph(); //˲Ϊ "ge" BuildSaveCheckpointGraph() ͼ + // Restoring from checkpoint file is done by pyfront, not in graph now. + } +#endif + return *this; //صǰͼת +} + +/* +úҪĿǽеAnfNodeתΪӦӣΪͼת͹ͼ׼ +*/ +DfGraphConvertor &DfGraphConvertor::ConvertAllNode() { + if (error_ != SUCCESS) { // error_ ǷΪ SUCCESSǣֱӷصǰͼת + return *this; + } + if (anf_graph_ == nullptr || anf_graph_->output() == nullptr) { // anf_graph_ anf_graph_->output() ǷϷ + MS_LOG(ERROR) << "Invalid AnfGraph"; + error_ = FAILED; //Ϸ error_ Ϊ FAILED Ϣ + return *this; //صǰͼת + } + //ռͼ compute_sout_ʼͼ init_sout_ָͼrestore_checkpoint_sout_ ͼͼ checkpoint_sout_ + //ݣʼΪµͼ + compute_sout_.clear(); + compute_sout_ << "digraph {" << endl; + init_sout_.clear(); + init_sout_ << "digraph {" << endl; +#ifdef ENABLE_D //ָ #ifdef ENABLE_D ڲִ² + auto ms_context = MsContext::GetInstance(); //ȡȫΨһ MsContext ʵ ms_context + MS_EXCEPTION_IF_NULL(ms_context); // ms_context ǷΪգΪգ׳쳣 + if (ms_context->backend_policy() == "ge") { //鵱ǰĺ˲ backend_policy ǷΪ "ge"ʾʹûGraphEngineĺˣ + checkpoint_sout_.clear(); // ˲Ϊ "ge"ռͼ checkpoint_sout_ ݣʼΪµͼ + checkpoint_sout_ << "digraph {" << endl; //ָ + } +#endif + restore_checkpoint_sout_.clear(); //ջָͼ restore_checkpoint_sout_ ݣʼΪµͼ + restore_checkpoint_sout_ << "digraph {" << endl; + + // Convert all anf node to Operator + MS_LOG(DEBUG) << "convert all node"; + std::vector nodes = GetOrderedCNodes(anf_graph_); //ȡAnfNodeڵб nodesڰ˳еAnfNode + for (auto &it : nodes) { + (void)Convert(it); //ÿAnfNode it Convert תΪӦӣOperator + if (this->error_ != SUCCESS) { //תǷɹִϢ + MS_LOG(ERROR) << "failed to convert node: " << it->DebugString() << "."; + } + } + + // Create dataset iterator and iterator_getnext node + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { //ݼSinkģʽ£򴴽ݼGetNextӡ + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); + MS_LOG(INFO) << "Dataset param is " << param.ToString() << "."; + // GetNext + auto iter_getnext_op = make_shared("get_next_tmp"); + std::vector getnext_types; + const auto &origin_ge_types = param.ge_types(); + (void)std::transform( + origin_ge_types.begin(), origin_ge_types.end(), std::back_inserter(getnext_types), + [](int64_t t_num) -> enum ge::DataType { return static_cast(t_num); }); + (void)iter_getnext_op->set_attr_output_types(getnext_types); + (void)iter_getnext_op->set_attr_output_shapes(param.shapes()); + (void)iter_getnext_op->set_attr_channel_name(param.queue_name()); + + // save iter_getnext_op for later use + dataset_iter_getnext_ = iter_getnext_op; + } + + // return the data flow graph + return *this; //صǰͼתá +} + +/* +úĿǴӻлȡضAnfNodeϢӵͼбԱͼʱʹá +ڹͼʱԸͼбȷͼڵ㡣 +*/ +void DfGraphConvertor::TraceOutputFromTupleGetItem(const AnfNodePtr &anf_out) { + auto it = out_handle_cache_.find(anf_out.get()); //ͨ anf_outڻ out_handle_cache_ вҶӦϢ + if (it != out_handle_cache_.end()) { //ҵ˶ӦϢ it out_handle_cache_.end()ȡϢ OutHandler handle + OutHandler handle = it->second; + auto op = handle.op; // handle лȡӣOperatorָ op + if (op != nullptr) { // op ΪգӵơԼӵͼб graph_outputs_ С + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType() << ", out_name: " << handle.out; + (void)graph_outputs_.emplace_back(*op, handle.out); + } else { // op ΪգʾӦAnfNodeûбɹתΪӣʱ׳쳣 + MS_LOG(EXCEPTION) << "tuple_getitem: " << anf_out->fullname_with_scope() << " is not converted"; + } + } else { //ڻҲӦϢ it out_handle_cache_.end()ϢʾЧ tuple_getitem + // invalid tuple_getitem e.g. tuple_getitem(tuple_getitem())/tuple_getitem(depend())/tuple_getitem(make_tuple()) + MS_LOG(WARNING) << "Invalid tuple_getitem: " << anf_out->fullname_with_scope(); + } +} + +/* +úĿǸٸAnfNodeеӵͼбԱڹͼʱʹá +ͨݹãԴӵļͼṹȷϢȷؼ¼ͼбС +*/ +void DfGraphConvertor::TraceOutput(const AnfNodePtr node) { + MS_EXCEPTION_IF_NULL(node); //AnfNodeǷΪգΪգ׳쳣 + AnfNodePtr anf_out = node; + AnfNodePtr pre_node = nullptr; + + // Trace value node + if (node->isa()) { //AnfNodeһValueNodeֵڵ㣩 + auto op = Convert(anf_out); // Convert תΪ,ӵͼб graph_outputs_ С + if (op != nullptr) { + (void)graph_outputs_.emplace_back(*op, ""); + AddGraphConstInput(op); + } + return; + } + + // Trace Parameter node + TraceOutputFromParameter(anf_out); //AnfNodeһParameterڵ㣨ڵ㣩 TraceOutputFromParameter ýڵ㡣 + + // Then trace cnode + if (!node->isa()) { //AnfNodeǷCNodeڵ㣩 + return; + } + + // trace tuple_getitem + // tuple_getitem ڵ㣬ͨϸе tuple_getitem ڵ㣬ֱҵԴͷCNodeΪֹ TraceOutputFromTupleGetItem Ϣ + while (anf_out->isa() && IsPrimitiveCNode(anf_out, prim::kPrimTupleGetItem)) { + pre_node = anf_out; + anf_out = anf_out->cast()->input(1); + } + // trace every element of make_tuple + //AnfNodeCNodeĿ꺯Ϊ "MakeTuple"еԪزݹ TraceOutput ÿԪء + auto c = anf_out->cast(); + std::string name = ""; + if (anf_out->isa()) { + name = GetCNodeTargetFuncName(c); + } + + if (name == "MakeTuple") { + for (unsigned int i = 1; i < c->inputs().size(); i++) { + TraceOutput(c->input(i)); + } + } else if (name == prim::kPrimDepend->name()) { //Ŀ꺯Ϊ "Depend"ٵһԪء + if (c->inputs().size() < 3) { // "Depend" primitive have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs is " << c->inputs().size() << ", which is less than 3"; + } + TraceOutput(c->input(1)); + } else if (name == prim::kTupleGetItem) { //Ŀ꺯Ϊ "prim::kPrimTupleGetItem" TraceOutputFromTupleGetItem Ϣ + TraceOutputFromTupleGetItem(anf_out); + } else { //򣬽AnfNodeתΪӣӵͼб graph_outputs_ С + //ڴ tuple_getitem ʱҵǰýڵ㣨pre_nodeϢ򽫸ϢΪǰڵ + // add outputs + auto op = Convert(anf_out); + std::string index; + if (op != nullptr) { + if ((pre_node != nullptr) && IsPrimitiveCNode(pre_node, prim::kPrimTupleGetItem)) { + auto item = out_handle_cache_.find(pre_node.get()); + if (item != out_handle_cache_.end()) { + index = item->second.out; + } else { + MS_LOG(WARNING) << "Can't get operator: " << anf_out->fullname_with_scope() << " 's output item"; + } + } + MS_LOG(INFO) << "Add graph output: " << anf_out->fullname_with_scope() << ":" << index; + (void)graph_outputs_.emplace_back(*op, index); + } + } +} + +/* +úĿǴParameterڵ㣬ΪͼӵбС +ڴͨDatasetͼģʽµʱֱ˲ͬĴ߼ȷϢȷؼ¼ͼбС +*/ +void DfGraphConvertor::TraceOutputFromParameter(const AnfNodePtr &anf_out) { + MS_EXCEPTION_IF_NULL(anf_out); //AnfNodeǷΪգΪգ׳쳣 + if (anf_out->isa()) { //AnfNodeǷParameterڵ㡣Parameterڵ㣬ʾýڵͼڵ㡣 + MS_LOG(INFO) << "Add graph output: " << anf_out->fullname_with_scope(); + auto it = out_handle_cache_.find(anf_out.get()); + if (it != out_handle_cache_.end()) { // out_handle_cache_ ҵParameterڵOutHandler + //˵ParameterڵDatasetͼģʽµҪ⴦Ϊͼӵб graph_outputs_ У¼ӦӣopԼƣout_name + // For dataset graph mode, input parameter is converted to a "iterator_get_next:yn" OutHandler. + OutHandler handle = it->second; + auto op = handle.op; + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType() << ", out_name: " << handle.out; + (void)graph_outputs_.emplace_back(*op, handle.out); + } else { // out_handle_cache_ δҵParameterڵ˵Parameterڵͨ + //תΪӲӵб graph_outputs_ С + // common parameter case + auto op = Convert(anf_out); + if (op != nullptr) { + MS_LOG(INFO) << "op name: " << op->GetName() << ", op type: " << op->GetOpType(); + (void)graph_outputs_.emplace_back(*op, ""); + } + } + } +} + +/* +úĿDatasetͼģʽ£DatasetͼIJϢ iterator_getnext +ӵϢȷDatasetͼƥ䡣 +*/ +void SetupDatasetIterGetNextNode(const OperatorPtr &op) { + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { //ù ConfigManager dataset_mode() ǷΪ DS_SINK_MODEǷDatasetͼģʽ¡ + DatasetGraphParam param = ConfigManager::GetInstance().dataset_param(); //Datasetͼģʽ£ùлȡDatasetͼIJ param + size_t output_num = param.ge_types().size();//ݲ param еϢȷ iterator_getnext ӵ output_numҪöٸ + MS_LOG(INFO) << "Set iterator_getnext op's output num = " << output_num << "."; + // set iterator_getnext op's output num op תΪ ge::op::GetNext ͵ӣԱ + shared_ptr iter_getnext = std::static_pointer_cast(op); + (void)iter_getnext->create_dynamic_output_y(static_cast(output_num)); // create_dynamic_output_y iterator_getnext ӵΪ output_num + + //ÿ param е״ϢϢӦ ge::TensorDesc 󣬲ʹ update_dynamic_output_desc_y ÿϢ + for (uint32_t i = 0; i < output_num; i++) { + ge::TensorDesc desc(GeShape(param.shapes()[i]), ge::FORMAT_NCHW, (ge::DataType)param.ge_types()[i]); + // we don't SetRealDimCnt here since GE do not use this output's real-dim + (void)iter_getnext->update_dynamic_output_desc_y((i), desc); + } + } + return; +} + +/* +úĿǴCaseڵͼCaseڵз֧ͼΪӦĸӵͼ +*/ +void DfGraphConvertor::SetSubgraph(const AnfNodePtr &node) { + if (!node->isa()) { //鴫Ľڵ node ǷΪCNodeͣǣֱӷأ + return; + } + auto cnode = node->cast(); + if (!IsCaseNode(cnode)) { //жϽڵ node ǷΪCaseڵ㣬ͨ IsCaseNode жϡ + return; //Caseڵ㣬ֱͬӷأ + } + std::vector case_inputs; //ڵ node Caseڵ㣬ôCaseڵлȡеCase֧ڵ㣬 case_inputsЩڵ㽫ںͼ + for (size_t i = 1; i < cnode->inputs().size(); i++) { + case_inputs.emplace_back(cnode->input(i)); + } + std::shared_ptr> branches = std::make_shared>();//һ洢DfGraphָ branchesڴ洢Caseڵз֧ͼ + auto bnode = cnode->input(0)->cast()->input(2)->cast(); + + for (size_t i = 1; i < bnode->inputs().size(); i++) { //CaseڵлȡCaseڵĵڶ룬CaseڵconditionֵbnodeתΪCNode + auto branch_node = bnode->input(i)->cast(); + for (size_t j = 2; j < branch_node->inputs().size(); j++) {// bnode 루Caseڵÿ֧ȡÿ֧CNodeڵ branch_node + if (std::find(case_inputs.begin(), case_inputs.end(), branch_node->input(j)) == case_inputs.end()) { + case_inputs.emplace_back(branch_node->input(j)); //ÿ֧ڵ㣬 case_inputs еڵӵ case_inputs Уȷ case_inputs з֧ڵ㡣 + } + } + } + //ֱÿ֧ڵ ProcessSubgraph д÷ᴦ֧ڵͼ洢 branches_map_ + for (size_t i = 1; i < bnode->inputs().size(); i++) { + ProcessSubgraph(bnode->input(i), case_inputs); + } + // bnode 루Caseڵÿ֧ÿ֧ͼ branches_map_ ȡӵ branches + for (size_t i = 1; i < bnode->inputs().size(); i++) { + (void)branches->emplace_back(branches_map_[bnode->input(i).get()]); + } + + if (op_cache_.find(node.get()) == op_cache_.end()) { + return; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + MS_LOG(DEBUG) << "Not found adapter"; + return; + } + //ͨ Convert ڵ node תΪ OperatorPtr ͵ op + OperatorPtr op = Convert(node); + (void)adpt->setSubgraph(op, 0, branches); //ڵ node Ӧ adpt֧ͼ branches Ϊ op ͼ + return; +} + +/* +úĿǴCaseڵ룬ÿCase֧Ϣ洢 tuple_out_handle_cache_ У +Caseڵ洢case_input_handle_cache_ СЩϢںͼõ +*/ +void DfGraphConvertor::GetCaseNodeInput(const CNodePtr node, const CNodePtr input_node) { + std::vector case_inputs; + for (size_t i = 1; i < node->inputs().size(); i++) { //CaseڵлȡеCase֧ڵ㣬洢 case_inputs + case_inputs.emplace_back(node->input(i)); + } + auto bnode = input_node->input(2)->cast(); + MS_EXCEPTION_IF_NULL(bnode); + for (size_t i = 1; i < bnode->inputs().size(); i++) { //CaseڵлȡCaseڵĵڶ룬Caseڵconditionֵinput_nodeתΪCNode͡ + auto branch_node = bnode->input(i)->cast(); + MS_EXCEPTION_IF_NULL(branch_node); + for (size_t j = 2; j < branch_node->inputs().size(); j++) { + if (std::find(case_inputs.begin(), case_inputs.end(), branch_node->input(j)) == case_inputs.end()) { + case_inputs.emplace_back(branch_node->input(j)); + } + } + } + + const size_t case_index = 1; + const size_t make_tuple_index = 2; + + AnfNodePtr case_index_iter = input_node->input(case_index); + AnfNodePtr make_tuple_iter = input_node->input(make_tuple_index); + auto make_tuple_node = make_tuple_iter->cast(); //ȡ input_node ĵڶ루Caseڵmake_tupleתΪCNodeͣ洢 make_tuple_node С + std::shared_ptr> tuple_items = std::make_shared>();//һ洢OutHandlerָ tuple_itemsڴ洢ÿCase֧ + + for (size_t i = 0; i < case_inputs.size(); i++) { + auto item = case_inputs[i]; + auto op = Convert(item); + if (op != nullptr) { //ÿڵ itemԽתΪ opӵ tuple_items С + (void)tuple_items->emplace_back(OutHandler(op, "", item)); + } else if (out_handle_cache_.find(item.get()) != out_handle_cache_.end()) { // item Ѿ out_handle_cache_ жӦOutHandler棬ֱӽӵ tuple_items С + tuple_items->push_back(out_handle_cache_[item.get()]); + } else { ////ȲתΪҲ out_handle_cache_ УôһյOutHandler tuple_items С + MS_LOG(DEBUG) << "Add an empty out handler: " << item->ToString(); + tuple_items->emplace_back(OutHandler()); + } + } + + tuple_out_handle_cache_[make_tuple_node.get()] = tuple_items;// tuple_items 洢 tuple_out_handle_cache_ УΪ make_tuple_node + + std::shared_ptr> case_input_items = std::make_shared>(); + //һ洢AnfNodePtrָ case_input_itemsڴ洢Caseڵ + //Caseڵĵһ루case_index_iter͵ڶ루make_tuple_iterӵ case_input_items У + // case_input_items 洢 case_input_handle_cache_ УΪ node + (void)case_input_items->emplace_back(case_index_iter); + (void)case_input_items->emplace_back(make_tuple_iter); + case_input_handle_cache_[node.get()] = case_input_items; +} + +/* +úĿǽǰĴгɹתΪӵOutHandlerµ tuple_out_handle_cache_У +ȷںĴܹȷȡصϢ +*/ +void DfGraphConvertor::UpdateTupleOutCache() { + for (auto &it : tuple_out_handle_cache_) { // tuple_out_handle_cache_ еÿֵ + std::size_t len = it.second->size(); //мΪ itֵΪ it.secondָ std::vector ָ + for (std::size_t i = 0; i < len; i++) { //ÿOutHandler飬С len + OutHandler handle = (*it.second)[i]; //OutHandler飬ÿOutHandler + if (handle.op == nullptr) { // op ΪnullptrʾûжӦӣOutHandler + continue; + } + string name = handle.op->GetName(); //򣬻ȡOutHandlerӦӵ name + if (vars_.count(name) && (vars_[name] != nullptr)) { // vars_ ǷƣҶӦӲΪnullptrǰĴѾɹתΪӣ + (*it.second)[i] = OutHandler(vars_[name], handle.out, handle.node);// µǰOutHandlerΪ vars_[name] Ӧӣԭ out node Ϣ + MS_LOG(INFO) << "update tuple_out_handle_cache_ " << name; //־ʾɹ tuple_out_handle_cache_ еϢ + } + } + } +} + +/* +úͨANFͼΪͼеÿڵ롢ͼ +*/ +DfGraphConvertor &DfGraphConvertor::BuildGraph() { + SetupDatasetIterGetNextNode(dataset_iter_getnext_); //ݼģʽΪDS_SINK_MODEݼͼģʽеGetNext + + if (error_ != SUCCESS) { + return *this; + } + + // Case node set input. + std::vector nodes = GetOrderedCNodes(anf_graph_); + for (auto &it : nodes) { + if (it->isa() && IsCaseNode(it->cast())) { + auto node = it->cast(); + auto input_node = node->input(0)->cast(); + GetCaseNodeInput(node, input_node); //ANFͼеÿCaseڵ㣬ͨڵCaseڵ롣 + } + } + + // update tuple_out_handle_cache_ + UpdateTupleOutCache(); //tuple_out_handle_cache_԰ɹתOutHandler + + // set up dependencies 룺úANFͼенڵ㣬ΪÿڵͿ룬ͬʱκͼ² + MS_LOG(DEBUG) << "set up dependencies"; + nodes = GetOrderedCNodes(anf_graph_); + for (auto &it : nodes) { + SetNodeInput(it); + SetOpControlInput(it); + SetSubgraph(it); + UpdateOpDesc(it); + } + + if (error_ == SUCCESS) { //ûдʹANFͼƴͼ(df_graph_) + df_graph_ = make_shared(anf_graph_->ToString()); + } else { + return *this; + } + + // set graph input according to the order from anf graph + //ͼ룺ݼģʽǷʹԶ룬ͼ룬κڹ볣IJijڵΪͼ롣 + std::vector inputs; + if (ConfigManager::GetInstance().dataset_mode() == DS_SINK_MODE) { + inputs.push_back(*dataset_iter_getnext_); + } else { + auto params = anf_graph_->parameters(); + if (use_inputs_) { + params = inputs_; + auto anf_params = anf_graph_->parameters(); + for (size_t i = 0; i < params.size(); i++) { + for (size_t j = 0; j < anf_params.size(); j++) { + if (params[i]->ToString() == anf_params[j]->ToString()) { + params[i] = anf_params[j]; + } + } + } + } + + int index = 0; + for (auto &it : params) { + auto name = std::static_pointer_cast(it)->name(); + // the parameters which has not been converted to var + if (vars_.find(name) == vars_.end()) { + if (HasAbstractMonad(it)) { + MS_LOG(INFO) << it->DebugString() << " is a monad parameter, skip."; + continue; + } + auto op = Convert(it); + MS_EXCEPTION_IF_NULL(op); + MS_LOG(INFO) << "add not var input " << it->ToString() << ", index " << index; + if (op == nullptr) { + MS_LOG(ERROR) << "Convert graph failed!"; + return *this; + } + UpdateDataOpDesc(it, op); + MS_LOG(INFO) << "add input " << it->ToString() << ", index " << index; + (void)std::static_pointer_cast(op)->set_attr_index(index++); + inputs.push_back(*op); + } else if (vars_[name] != nullptr) { + MS_LOG(INFO) << "add var input " << it->ToString(); + auto op = Convert(it); + UpdateConstOpDesc(it, vars_[name]); + MS_EXCEPTION_IF_NULL(op); + inputs.push_back(*op); + } + } + } + + + MS_LOG(DEBUG) << "trace output"; + graph_outputs_.clear(); + TraceOutput(anf_graph_->get_return()->input(1));//ͼڵи٣graph_outputs_ + + // Add const nodes as graph input for some operator work with constant + MS_LOG(INFO) << "graph const input size: " << graph_const_inputs_.size(); + (void)std::transform(graph_const_inputs_.begin(), graph_const_inputs_.end(), std::back_inserter(inputs), + [](const OperatorPtr &x) { return *x; }); + + MS_LOG(INFO) << "set graph input num: " << inputs.size(); + (void)df_graph_->SetInputs(inputs); + + // set graph output + // set the value of finale return apply node as the output of dataflow graph + //ͼʹgraph_outputs_ͼ + MS_LOG(DEBUG) << "set output"; + MS_LOG(INFO) << "set graph output num: " << graph_outputs_.size(); + (void)df_graph_->SetOutputs(graph_outputs_); + + compute_sout_ << "}" << endl; + // For the graph(e.g. eval_subgraph) whose IterNum is 1, donot set NeedIteration flag. + //NeedIteration־(iter_num)1NeedIteration־Ϊtrueͼ + if (ConfigManager::GetInstance().iter_num() > 1) { + df_graph_->SetNeedIteration(true); + } + return *this; +} + +/* +˺ +ȷ˵ConstantkOpFormat_NCHWConstantΪָĸʽƥ䣨ã +*/ +void DfGraphConvertor::UpdateConstOpDesc(const AnfNodePtr &it, const OperatorPtr &op) const { + if (!it->isa()) { //ǷΪڵ㡣ǣ¼һϢָʾDzҺضһ + MS_LOG(DEBUG) << "It is not parameter, name: " << it->DebugString(); + return; + } + auto para = it->cast(); //ǽڵ㣬ӦĶǿתĬϸʽַ + MS_EXCEPTION_IF_NULL(para); + std::string format = kOpFormat_NCHW; + std::string param_debug_info = para->DebugString(); + auto param_format = param_format_.find(param_debug_info); //param_debug_infoĵϢ map ҵ˲ĸʽҵʽӦظ±¼Ϣ + if (param_format != param_format_.end()) { + format = param_format->second; //ʽδģ˵ú¼Ϣ + MS_LOG(DEBUG) << "Parameter debug info: " << param_debug_info << ", format is " << format; + } + if (format == kOpFormat_NCHW) { + MS_LOG(DEBUG) << "Format is not changed, no need to update op desc, name: " << param_debug_info; + return; + } + if (!para->has_default()) { + MS_LOG(DEBUG) << "Parameter has no default, no need to update op desc, name: " << param_debug_info; + return; + } + auto value = para->default_param(); + MS_EXCEPTION_IF_NULL(value); + auto tensor = value->cast>(); //Ĭֵúֵvalue ǿתΪstd::shared_ptr + MS_EXCEPTION_IF_NULL(tensor); //ʹøúµ ״ͺ͸¸ʽãconst_op_descTransformUtil::GetGeTensorDesc + auto const_op_desc = TransformUtil::GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + if (const_op_desc == nullptr) { //ʧܣ nullptrú¼沢ء + MS_LOG(WARNING) << "Create parameter " << para->name() << " output descriptor failed!"; + return; + } + (void)std::static_pointer_cast(op)->update_output_desc_y(*const_op_desc); //ʹ´ .Constantopconst_op_desc +} + +void DfGraphConvertor::UpdateDataOpDesc(const AnfNodePtr &it, const OperatorPtr &op) const { + auto node = std::static_pointer_cast(it); //ڵitתΪstd::shared_ptr + if (node == nullptr) { //תʧܻnodenullptr¼󲢷ء + MS_LOG(ERROR) << "Update data op descriptor failed! Invalid node."; + return; + } + + std::vector shape; //abstract::Shapeȡڵ״޷ȡ״ڵЧ״¼һϢabstract::NoShapeء + if (auto normal_shape_ptr = dyn_cast(node->Shape()); normal_shape_ptr != nullptr) { + shape = normal_shape_ptr->shape(); + } else if (auto no_shape_ptr = dyn_cast(node->Shape()); no_shape_ptr != nullptr) { + shape = {}; + } else { + MS_LOG(INFO) << "Invalid shape to update data op descriptor."; + return; + } + + if (node->Type() == nullptr) { //ڵ͡Ͳãnullptr¼һϢء + MS_LOG(INFO) << "Invalid type to update data op descriptor."; + return; + } + TypeId me_type = node->Type()->type_id(); + if (kObjectTypeTensorType == me_type) { //ڵΪkObjectTypeTensorTypeԴȡԪ͡ + me_type = dyn_cast(node->Type())->element()->type_id(); + } + std::ostringstream buf; + buf << "[" << shape << "]"; + MS_LOG(INFO) << "input shape is " << buf.str() << ", type is " << me_type;//ʹMS_LOG(INFO)־дӡڵ״Ϣ + std::string format = "NCHW"; + if (it->isa()) { //ڵΪParameterparam_format_ȡƲʽ + auto param = it->cast(); + std::string param_name = param->DebugString(); + auto param_format = param_format_.find(param_name); + if (param_format != param_format_.end()) { //ȷڵĸʽparam_format_ҲĬΪNCHW״ͺ͸ʽԻȡ + format = param_format->second; + MS_LOG(DEBUG) << "parameter: " << param_name << ", format is " << format; + } + } + auto desc = TransformUtil::GetGeTensorDesc(shape, me_type, format); + if (desc == nullptr) { //Ϊ null¼;ʹûõ¶ + MS_LOG(ERROR) << "Update data op descriptor failed! TensorDesc is null."; + } else { + (void)std::static_pointer_cast(op)->update_input_desc_x(*desc); + (void)std::static_pointer_cast(op)->update_output_desc_y(*desc); + } +} + +DfGraphPtr DfGraphConvertor::GetComputeGraph() { return df_graph_; } + +DfGraphPtr DfGraphConvertor::GetInitGraph() { return init_graph_; } + +DfGraphPtr DfGraphConvertor::GetSaveCheckpointGraph() { return save_ckp_graph_; } + +DfGraphPtr DfGraphConvertor::GetBroadcastGraph() { return broadcast_graph_; } + +/* +úжһڵǷΪԴ߽ڵ +*/ +bool DfGraphConvertor::IsSourceEdgeNode(const AnfNodePtr &node) { + if (!node->isa()) { //жϸýڵǷΪ CNode ͣ򷵻 false + return false; + } + auto cnode = node->cast(); + if (!IsCustomCNode(cnode)) { //ȡ CNode Ŀ꺯Ϊ򷵻 false + std::string name = GetCNodeTargetFuncName(cnode); + if (name.empty()) { + return false; + } + + // Ignore apply node Depend, UpdateState, make_tuple. make_tuple in ge pipeline. + //һЩضĽڵ㣬 DependUpdateStatemake_tuple Return + //ڵĿ꺯Щضڵ֮һ򷵻 false + if ((name == prim::kPrimDepend->name()) || (name == prim::kPrimUpdateState->name()) || + (name == prim::kPrimReturn->name()) || (name == prim::kPrimMakeTuple->name())) { + return false; + } + } + // Load and other normal primitives which contain monad node. + //ýڵǷ monad ڵ㣬򷵻 true + auto has_monad = std::any_of(cnode->inputs().begin(), cnode->inputs().end(), + [](const AnfNodePtr &node) -> bool { return HasAbstractMonad(node); }); + if (has_monad) { + return true; + } + + // primitive with make_tuple as input + //ýڵǷ make_tuple ΪĿ꺯 CNodeǣ make_tuple Ƿ monad ڵ㣬򷵻 true + for (auto &input : cnode->inputs()) { + if (IsPrimitiveCNode(input, prim::kPrimMakeTuple)) { + auto tuple = input->cast(); + auto ret = std::any_of(tuple->inputs().begin(), tuple->inputs().end(), + [](const AnfNodePtr &node) -> bool { return HasAbstractMonad(node); }); + if (ret) { + return true; + } + } + } + //㣬򷵻 falseʾýڵ㲻Դ߽ڵ㡣 + return false; +} + +/* +úжһڵǷΪƱ߽ڵ +*/ +bool DfGraphConvertor::IsControlEdgeNode(const AnfNodePtr &node) { + if (!node->isa()) { //жϸýڵǷΪ CNode ͣ򷵻 false + return false; + } + auto cnode = node->cast(); + if (!IsCustomCNode(cnode)) { //ȡ CNode Ŀ꺯Ϊ򷵻 false + std::string name = GetCNodeTargetFuncName(cnode); + if (name.empty()) { + return false; + } + + // Ignore apply node of Load, Depend, UpdateState, make_tuple, return + //һЩضĽڵ㣬 LoadDependUpdateStatemake_tuple Return + //ڵĿ꺯Щضڵ֮һ򷵻 false + if ((name == prim::kPrimLoad->name()) || (name == prim::kPrimDepend->name()) || + (name == prim::kPrimUpdateState->name()) || (name == prim::kPrimMakeTuple->name()) || + (name == prim::kPrimReturn->name())) { + return false; + } + } + //㣬򷵻 trueʾýڵǿƱ߽ڵ + return true; +} + +/* +AnfNodePtrתΪOperatorPtr +֮ǰȵGetRealOpNodeȡʵڵ㣬䴫ݸConvertת +תʧܣ¼־nullptr򷵻תOperatorPtr +*/ +OperatorPtr DfGraphConvertor::ToOperatorPtr(const AnfNodePtr &node) { + auto op = Convert(GetRealOpNode(node)); // ȡʵڵ + if (op == nullptr) { //// תʧܣ¼־error_ΪFAILED + MS_LOG(ERROR) << "Convert real op node to operator failed, " << node->ToString(); + error_ = FAILED; + return nullptr; + } + //תOperatorPtr + return op; +} + +/* +ΪDfGraphConvertorάmonad_control_edge_cache_ӿߡ +УÿԴڵsrcһӦĿڵ㼯ϣʾsrcĿƽڵ㡣 +*/ +void DfGraphConvertor::AddEdgeToCache(const AnfNodePtr &src, const AnfNodePtr &dest) { + auto item = monad_control_edge_cache_.find(src); //ԴڵǷѴڿ߻ + if (item == monad_control_edge_cache_.end()) { // Դڵ㲻ڻУ򴴽һµĻĿڵӵû + monad_control_edge_cache_[src] = std::set{dest}; + } else { // ԴڵѴڻУĿڵӵԴڵڵ㼯 + // ʹinsertĿڵ㣬setȷظظĿڵ + (void)item->second.insert(dest); + } +} + +//úΪLoadͽڵӿ +void DfGraphConvertor::AddEdgeForLoad(const AnfNodePtr &node) { + auto func_graph = node->func_graph(); // ȡڵĺͼ + MS_EXCEPTION_IF_NULL(func_graph); + auto mng = func_graph->manager(); // ȡͼĹ + if (mng == nullptr) { // Ϊգ򴴽һµĹΪͼĹ + mng = Manage(func_graph, true); + func_graph->set_manager(mng); + } + auto manager = func_graph->manager(); //ٴλȡͼĹ + MS_EXCEPTION_IF_NULL(manager); + if (manager->node_users().find(node) == manager->node_users().end()) { // ڵǷڹĽڵû + MS_LOG(EXCEPTION) << "Can't find node in nodes_users."; + } + auto &users = manager->node_users()[node]; // ȡڵû + // ڴ洢ԴڵĿڵĹָб + std::shared_ptr> src_node_list = std::make_shared>(); + std::shared_ptr> dst_node_list = std::make_shared>(); + for (const auto &iter : users) { // ڵûϣصԴڵĿڵӵӦб + auto user_node = iter.first; + auto name = GetCNodeTargetFuncName(user_node->cast()); + if (name == prim::kPrimUpdateState->name()) { // ûڵprim::kPrimUpdateStateͣΪĿڵ㣬Ŀڵ + FindDestOps(user_node, dst_node_list, false); + continue; + } + if (IsControlEdgeNode(user_node)) { // ûڵǿƱ߽ڵ㣨ControlDependͣΪԴڵ + src_node_list->push_back(user_node); + continue; + } + FindDestOps(user_node, src_node_list, false); // 򣬽ûڵΪͨԴڵ㣬Ŀڵ + } + + // add to cache + // ԴڵĿڵӵ߻ + for (auto &dest : *dst_node_list) { + for (auto &src : *src_node_list) { + AddEdgeToCache(src, dest); + } + } +} + +/* +úҪĿǵݹزҸڵĿڵ㣬ЩĿڵӵnode_listС +topڱʶǰڵǷΪڵ㣬ΪtrueֻеûڵǿƱ߽ڵʱŻὫӵnode_listС +ΪfalseûڵͣὫӵnode_listС +*/ +void DfGraphConvertor::FindDestOps(const AnfNodePtr &node, const std::shared_ptr> &node_list, + bool top) { + MS_EXCEPTION_IF_NULL(node); // ڵǷΪ + auto func_graph = node->func_graph(); // ȡڵĺͼ + MS_EXCEPTION_IF_NULL(func_graph); + auto mng = func_graph->manager(); // ȡͼĹ + if (mng == nullptr) { // Ϊգ򴴽һµĹΪͼĹ + mng = Manage(func_graph, true); + func_graph->set_manager(mng); + } + auto manager = func_graph->manager(); // ٴλȡͼĹ + MS_EXCEPTION_IF_NULL(manager); + + auto users = manager->node_users()[node]; // ȡڵû + for (const auto &iter : users) { // ڵû + auto user_node = iter.first; + if (IsControlEdgeNode(user_node)) { // ûڵǿƱ߽ڵ㣨ControlDependͣҲڵ㣬ӵnode_list + if (!top) { + node_list->push_back(user_node); + } + } else { // 򣬵ݹزҸûڵĿڵ㣬ӵnode_list + FindDestOps(user_node, node_list, false); + } + } +} + +/* +úҪԶռMonad룬ӦĿߡ +ѧϰУMonadָͨһϵڿƼͼִ˳ +*/ +void DfGraphConvertor::AutoMonadCollectInput(const AnfNodePtr &node) { + if (!IsSourceEdgeNode(node)) { // ڵǷΪԴ߽ڵ㣬ǣҪ + return; + } + + // Add control edge if contain monad input. + // Loadͽڵ㣬Ϊӿ + std::string name = GetCNodeTargetFuncName(node->cast()); + if (name == prim::kPrimLoad->name()) { + AddEdgeForLoad(node); + } else { // 򣬻ȡڵӦIJ + auto src_ops = ToOperatorPtr(node); + if (src_ops != nullptr) { // ڣĿڵ㲢Ϊӿ + // Find dest ops list + // Ŀڵб + std::shared_ptr> dst_node_list = std::make_shared>(); + FindDestOps(node, dst_node_list, true); + for (auto &dest : *dst_node_list) { // ԴڵĿڵһΪ + AddEdgeToCache(node, dest); + } + } + } +} + +/* +úԶMonad룬ݿ߻棨monad_control_edge_cache_еϢ +Ϊڵ㽨ߡ +*/ +void DfGraphConvertor::AutoMonadSetInput(const AnfNodePtr &node) { + // ڵǷڿ߻УڣҪ + if (monad_control_edge_cache_.find(node) == monad_control_edge_cache_.end()) { + return; + } + + auto src_ops = ToOperatorPtr(node); // ȡڵӦIJ + if (src_ops != nullptr) { // ڣӦĿĿڵ㣬ΪĿڵӿ + for (auto &dest : monad_control_edge_cache_[node]) { + auto dest_ops = ToOperatorPtr(dest); + if (dest_ops == nullptr) { // Ŀ󲻴ڣĿڵ + continue; + } + (void)dest_ops->AddControlInput(*src_ops); // ΪĿӿ룬 +#ifdef DRAW_GE_GRAPH // DEBUGģʽ£Ƽͼʱϵ + compute_sout_ << op_draw_name_[node.get()] << " -> " << op_draw_name_[dest.get()] << "[style=\"dotted\"]" << endl; +#endif + } + } +} + +/* +úҪԶÿߡ + ЩΪȷѧϰУͼִ˳ϵҪԱ֤ȷԡ +*/ +void DfGraphConvertor::AutoMonadSetControlInput(const AnfNodePtr &node) { + AutoMonadCollectInput(node); // ԶռMonad룬 + AutoMonadSetInput(node); // ԶMonad룬 +} + +//úҪΪڵÿ룬ߡ +void DfGraphConvertor::SetOpControlInput(const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(node); // ڵǷΪ + AutoMonadSetControlInput(node); // ԶMonad룬 + if (control_edge_cache_.find(node.get()) == control_edge_cache_.end()) { // 鵱ǰڵǷڿƱ߻ + return; //ڣֱӷ + } + // ȡǰڵĿƱ߻Ϣ + std::vector control_edges = control_edge_cache_[node.get()]; + if ((control_edges.empty())) { // Ʊ߻Ϊգ¼־ + MS_LOG(ERROR) << "Get control edge node's src or dest operator failed"; + return; + } + + for (auto &item : control_edges) { // ΪǰڵĿڵӿ + (void)item.dest_op->AddControlInput(*item.src_op); + } +} +//ɱij +const std::vector trans_var_list = {string(kNameAssign), string(kNameAssignAdd), string(kNameAssignSub)}; + +//úڴLoadͽڵлȡӦijڵ +AnfNodePtr DfGraphConvertor::ParseLoadInput(const CNodePtr &cnode) { + if (cnode->inputs().size() < 3) { // CNodeǷС3 + MS_LOG(EXCEPTION) << "input size error, " << cnode->ToString(); + } + const size_t para_index = 1; // 峣Ϊ1LoadڵͨΪcnode->inputs()[1] + return cnode->input(para_index); // LoadڵijӦAnfNodePtr +} + +//úڴԪͽڵ룬ΪĿ롣 +void DfGraphConvertor::SetTupleOpInput(const OpAdapterPtr &adpt, const CNodePtr &node, const AnfNodePtr &pred, + const OperatorPtr &src, int index) { + // Ԫлȡ + std::shared_ptr> handler_vec = tuple_out_handle_cache_[pred.get()]; + // һµĴڱûMonad͵Ԫ + std::shared_ptr> handler_vec_without_monad = std::make_shared>(); + bool with_monad = false; // ڱǴǷMonadԪ + // еÿԪأжǷMonadԪأMonad͵ԪӵµĴ + for (auto &handler : *handler_vec) { + // when tuple with monad type element, the handler operator is nullptr, should be ignored. + if (handler.op == nullptr) { + if ((handler.node != nullptr) && !HasAbstractMonad(handler.node)) { + MS_LOG(WARNING) << "Unsupported node in tuple : " << node->ToString(); + } + continue; + } + with_monad = true; + handler_vec_without_monad->push_back(handler); + } + // ʹOpAdaptersetInputµĴΪøĿ + int ret = adpt->setInput(src, index, handler_vec_without_monad); + // óɹԤڵνڵһCNode봦СMonad͵Ԫأ + // ӿߣƼͼͬʱеԪΪͼij + if ((ret == 0) && pred->isa() && (pred->cast()->inputs().size() == handler_vec->size() + 1)) { + for (unsigned int j = 0; j < handler_vec_without_monad->size(); j++) { + AnfNodePtr input_node = pred->cast()->input(j + 1); + if (with_monad) { + input_node = handler_vec_without_monad->at(j).node; + } + compute_sout_ << op_draw_name_[input_node.get()] << " -> " << op_draw_name_[node.get()] << ":" << index << endl; + AddGraphConstInput(handler_vec_without_monad->at(j).op); + } + return; + } + // ʧܻԤڵνڵ㲻¼־ + MS_LOG(WARNING) << "This anf node is not supported as a tuple item : " << node->ToString(); +} + +//úҪڻȡʵʵڵ㣬Աк +AnfNodePtr DfGraphConvertor::GetRealInputNode(const CNodePtr &node, const AnfNodePtr &input) { + if (input == nullptr || node == nullptr) { // ڵCNodeǷΪ + return nullptr; + } + AnfNodePtr pred = input; // ȡνڵ + while (pred->isa() && GetCNodeTargetFuncName(pred->cast()) == prim::kPrimDepend->name()) { + pred = pred->cast()->input(1); + } + // skip input of UMonad, IOMonad + // UMonadIOMonad͵Ľڵ + if (IsValueNode(pred) || IsValueNode(pred)) { + return nullptr; + } + // skip input of the None, UpdateState + // NoneͺUpdateState͵Ľڵ + if (IsValueNode(pred) || IsPrimitiveCNode(pred, prim::kPrimUpdateState)) { + return nullptr; + } + // Loadڵ㣬ʵڵ + if (IsPrimitiveCNode(pred, prim::kPrimLoad)) { + pred = ParseLoadInput(pred->cast()); + } + + // transform "Const" op to "Variable" op when the next node is "Assign" op. + // ǰڵ"Assign"ͽڵ㣬һڵ"Const"ͻ"Constant"͵Parameterڵʱת"Const" opΪ"Variable" op + std::string c_name = GetCNodeTargetFuncName(node); + auto pos = std::find(trans_var_list.begin(), trans_var_list.end(), c_name); + if (!training_ && pos != trans_var_list.end() && pred->isa()) { + std::string name = std::static_pointer_cast(pred)->name(); + auto op_itor = op_cache_.find(pred.get()); + if (op_itor == op_cache_.end()) { + MS_LOG(EXCEPTION) << "Can not find op for node " << pred->ToString() << "."; + } + if (op_itor->second != nullptr && + (op_itor->second->GetOpType() == "Constant" || op_itor->second->GetOpType() == "Const") && + vars_.find(name) != vars_.end()) { + auto variable = std::make_shared(name); + auto desc = vars_[name]->GetOutputDesc("y"); + (void)variable->update_output_desc_y(desc); + MS_LOG(DEBUG) << "Trans to variable, var = " << variable->GetName() << "."; + op_itor->second = variable; // replace parameter with variable + vars_[name] = variable; + } + } + return pred; // ʵʵڵ +} + +//úòڵ +void DfGraphConvertor::SetOpInput(const OpAdapterPtr &adpt, const CNodePtr &node) { + OperatorPtr src = Convert(node); // CNodeڵתΪOperatorPtr + int case_flag = 0; // case_flagڱǷ⴦ + auto &inputs = node->inputs(); // ȡCNodeڵб + size_t input_size = inputs.size(); + // ýڵcase_input_handle_cache_Уcase_flagΪ1ͬʱΪcacheеĴС+1 + if (case_input_handle_cache_.find(node.get()) != case_input_handle_cache_.end()) { + case_flag = 1; + input_size = case_input_handle_cache_[node.get()]->size() + 1; + } + + for (size_t i = 1; i < input_size; i++) { // ڵÿ + AnfNodePtr pred = nullptr; + if (case_flag != 0) { // ⴦case_input_handle_cache_лȡڵ + pred = case_input_handle_cache_[node.get()]->at(i - 1); + } else { // ֱӴinputsлȡڵ + pred = inputs[i]; + } + pred = GetRealInputNode(node, pred); // ȡʵʵڵ㣬˵Ҫ + if (pred == nullptr) { + continue; + } + + int index = SizeToInt(i); // Operatorе + // find in out_hadnle_cache_ first + // out_handle_cache_вǷжӦ + auto it = out_handle_cache_.find(pred.get()); + if (it != out_handle_cache_.end()) { // ҵΪ + int ret = adpt->setInput(src, index, it->second); + if (ret == 0) { // ɹ룬ƼͼеĿߣеIJΪͼij + if (pred->isa() && GetCNodeTargetFuncName(pred->cast()) == prim::kTupleGetItem) { + compute_sout_ << op_draw_name_[pred->cast()->input(1).get()] << " -> " << op_draw_name_[node.get()] + << ":" << i << endl; + } else if (pred->isa()) { + compute_sout_ << op_draw_name_[pred.get()] << " -> " << op_draw_name_[node.get()] << ":" << i << endl; + } else { + // don't draw anything. + // κ + MS_LOG(INFO) << "DRAW_GE_GRAPH: Shouldn't have this case."; + } + AddGraphConstInput(it->second.op); + } + } else if (tuple_out_handle_cache_.find(pred.get()) != tuple_out_handle_cache_.end()) { + // tuple_out_handle_cache_ҵԪͽڵ + SetTupleOpInput(adpt, node, pred, src, index); + } else { + // out_handle_cache_tuple_out_handle_cache_жûҵֱӽڵתΪ󣬲Ϊ + auto op = Convert(pred); + int ret = adpt->setInput(src, index, op); + if (ret == 0) { + // ɹ룬ƼͼеĿߣΪͼij + compute_sout_ << op_draw_name_[pred.get()] << " -> " << op_draw_name_[node.get()] << ":" << i << endl; + AddGraphConstInput(op); + } + } + } +} + +//úgraph_const_inputs_ӳ룬ԱڼͼʹЩΪ롣 +void DfGraphConvertor::AddGraphConstInput(const OperatorPtr &op) { + if (op->GetOpType() == "Constant" || op->GetOpType() == "Const") { // жϲǷΪ"Constant""Const" + graph_const_inputs_.push_back(op); // dz͵IJӵgraph_const_inputs_ + } +} + + +//ݽڵͺȷòڵ룬ڼͼлӦĿߡ +void DfGraphConvertor::SetNodeInput(const AnfNodePtr node) { + if (!node->isa()) { // жϽڵǷCNode򷵻 + return; + } + if (op_cache_.find(node.get()) == op_cache_.end()) { // жϽڵǷop_cache_У򷵻 + return; + } + auto cnode = node->cast(); // ȡCNodeڵ㣬ҶӦOpAdapter + OpAdapterPtr adpt = FindAdapter(cnode, training_); + if (adpt == nullptr) { // ҲӦOpAdaptererror_־ΪNOT_FOUND + error_ = NOT_FOUND; + return; + } + + // get Operator from op_cache_, use adapter to set Inputs + // ʹOpAdapterSetOpInputCNodeڵ + DfGraphConvertor::SetOpInput(adpt, cnode); +} + +//úڴͼڵ㣨Partialڵ㣩 +void DfGraphConvertor::ProcessSubgraph(const AnfNodePtr &node, const std::vector &inputs) { + // жϽڵǷCNodeҺǷΪ"Partial"ֱӷ + if (!node->isa() || GetCNodeFuncName(node->cast()) != "Partial") { + return; + } + // ȡͼڵӦFuncGraph + auto graph_node = node->cast()->input(1)->cast(); + MS_EXCEPTION_IF_NULL(graph_node); + FuncGraphPtr anf_graph = graph_node->value()->cast(); + + // µDfGraphConvertor󣬲ʹͼFuncGraphΪ + DfGraphConvertor converter(anf_graph); + + // converteruse_inputs_ΪtrueʾʹøinputsΪͼ + converter.use_inputs_ = true; + converter.inputs_ = inputs; + + // ͼתΪDfGraph + (void)converter.ConvertAllNode().BuildGraph(); +#ifdef ENABLE_DUMP_IR // þǷƼͼ + std::string name = graph_node->ToString() + "_ge_graph.dot"; + if (MsContext::GetInstance()->get_param(MS_CTX_SAVE_GRAPHS_FLAG)) { + converter.DrawComputeGraph(name); + } +#endif // תDfGraph洢branches_map_УΪͼڵֵַָΪתDfGraph + branches_map_[node.get()] = *(converter.df_graph_); +} + +// Update GE op's shape and type info +//úڸ²Ϣ +//ڵ״ShapeͣTypeͽڵ㱾Ϊ¶ӦIJ +void DfGraphConvertor::UpdateOpDesc(const AnfNodePtr node) { + if (node == nullptr || !node->isa()) { // жϽڵǷΪջCNodeֱͣӷ + return; + } + + if (op_cache_.find(node.get()) == op_cache_.end()) { // жϽڵǷop_cache_Уֱӷ + return; + } + + OpAdapterPtr adpt = FindAdapter(node, training_); // ҽڵӦOpAdapter + if (adpt == nullptr) { // ҲӦOpAdaptererror_־ΪNOT_FOUND + error_ = NOT_FOUND; + return; + } + + // get Operator from op_cache_ + // ȡڵӦOperator + OperatorPtr op = Convert(node); + + // ʹOpAdapterupdateOutputDesc²Ϣ + adpt->updateOutputDesc(op, node->Shape(), node->Type(), node); +} + +//úڽAnfNodeڵתΪӦOperator +OperatorPtr DfGraphConvertor::Convert(const AnfNodePtr node) { + if (node == nullptr) { // жϽڵǷΪգΪerror_־ΪNOT_FOUNDnullptr + MS_LOG(ERROR) << "node is nullptr"; + error_ = NOT_FOUND; + return nullptr; + } + // find in cache + // op_cache_вҽڵӦOperatorҵֱӷ + if (op_cache_.count(node.get())) { + return op_cache_[node.get()]; + } + + // do not convert primitive node, Load, UpdateState + // ԭڵ㣨Primitiveڵ㣩Loadڵ㡢UpdateStateڵ㣬ֱӷnullptrת + if (IsValueNode(node) || IsPrimitiveCNode(node, prim::kPrimLoad) || + IsPrimitiveCNode(node, prim::kPrimUpdateState)) { + return nullptr; + } + + // convert a new one + // CNodeڵ㣬ConvertCNodeת + if (node->isa()) { + return ConvertCNode(node->cast()); + } + // Parameterڵ㣬ConvertParameterת + if (node->isa()) { + return ConvertParameter(node); + } + // ValueNodeڵ㣬ݽڵǷΪMonadǷת + if (node->isa()) { + if (IsValueNode(node)) { + return nullptr; + } + return ConvertValueNode(node->cast()); + } + // ͵Ľڵ㣬error_־ΪINVALID_ARGUMENTnullptr + MS_LOG(ERROR) << "Invalid AnfNode"; + error_ = INVALID_ARGUMENT; + return nullptr; +} + +//úڽMakeTupleڵתΪӦOutHandlerб +void DfGraphConvertor::ConvertMakeTuple(const CNodePtr node) { + // һָ룬ڴ洢MakeTupleڵ + std::shared_ptr> tuple_items = std::make_shared>(); + // convert each tuple item to a OutHandler + // MakeTupleڵתΪOutHandler + for (size_t i = 1; i < node->inputs().size(); i++) { + AnfNodePtr item = node->input(i); + if (IsPrimitiveCNode(item, prim::kPrimLoad)) { // Loadڵ㣬Ҫ룬صݽڵ + item = ParseLoadInput(item->cast()); + } + OperatorPtr op = Convert(item); // AnfNodeڵתΪӦOperator + if (op != nullptr) { // תõOperatorΪգOutHandlerӵtuple_items + (void)tuple_items->emplace_back(OutHandler(op, "", item)); + } else if (out_handle_cache_.find(item.get()) != out_handle_cache_.end()) { + // out_handle_cache_ҵӦOutHandlerӵtuple_items + tuple_items->push_back(out_handle_cache_[item.get()]); + } else { // 򣬽һյOutHandlerӵtuple_items + tuple_items->emplace_back(OutHandler(nullptr, "", item)); + } + } + // ӡϢתõOutHandlerб洢tuple_out_handle_cache_ + MS_LOG(DEBUG) << "ConvertMakeTuple: " << node.get() << " " << tuple_items->size(); + tuple_out_handle_cache_[node.get()] = tuple_items; +} + +//úڽTopKڵתΪӦOperator󣬲ڶת +void DfGraphConvertor::ConvertTopK(const CNodePtr node) { + MS_EXCEPTION_IF_NULL(node); // жϽڵǷΪ + MS_LOG(INFO) << "Convert TopK second input's type from int64 to int32."; // ӡ־ϢʾTopKڵĵڶʹint64תΪint32 + auto value_ptr = node->input(2)->cast(); // ȡTopKڵĵڶ루kֵ + MS_EXCEPTION_IF_NULL(value_ptr); + std::ostringstream ss; // ΪڶڵһΨһıʶ洢op_draw_name_УڻƼͼʱʶýڵ + ss << "op" << value_ptr.get(); + op_draw_name_[value_ptr.get()] = ss.str(); + // ƼͼڵϢ洢compute_sout_ + compute_sout_ << ss.str() << "[label= \"" << value_ptr->value()->ToString() << "\" shape=ellipse]" << endl; + // ȡڶڵֵתΪint64 + auto input_value = value_ptr->value(); + auto int64_value = GetValue(input_value); + OpAdapterPtr adpt = FindAdapter(value_ptr, training_); // ҵڶڵӦOpAdapter + auto op = adpt->generate(value_ptr); // ʹOpAdaptergenerateɵڶڵӦOperator + (void)adpt->setAttr(op, "value", static_cast(int64_value)); // ڶڵֵתΪint32ͣΪOperator + op_cache_[value_ptr.get()] = op; // ڶڵӦOperator洢op_cache_ +} + +//úڽValuePtrתΪstd::vector͵ݡ +std::vector DfGraphConvertor::CastToInt(const ValuePtr &value) { + if (value == nullptr) { // жValuePtrǷΪգΪӡϢؿյstd::vector + MS_LOG(WARNING) << "Value ptr is nullptr."; + return {}; + } + std::vector cur_value = {}; + if (utils::isa(value)) { // ValuePtrValueSequencePtrͣʾһֵ + auto val_seq_ptr = value->cast(); + MS_EXCEPTION_IF_NULL(val_seq_ptr); + if (!val_seq_ptr->value().empty()) { + auto first_val = val_seq_ptr->value().front(); + MS_EXCEPTION_IF_NULL(first_val); + MS_EXCEPTION_IF_NULL(first_val->type()); + if (first_val->type()->number_type() == kNumberTypeInt64) { // ֵеԪint64ֱӽתΪstd::vector + cur_value = GetValue>(value); + } else { // 򣬽ֵеԪתΪintͣתΪstd::vector + auto origin_value = GetValue>(value); + (void)std::transform(origin_value.begin(), origin_value.end(), std::back_inserter(cur_value), + [](int index) { return static_cast(index); }); + } + } + } else { // ValuePtrֵУֱӽתΪstd::vector + MS_EXCEPTION_IF_NULL(value->type()); + if (value->type()->number_type() == kNumberTypeInt64) { + cur_value.push_back(GetValue(value)); + } else { + cur_value.push_back(static_cast(GetValue(value))); + } + } + return cur_value; +} + +//úڽReshapeڵתΪӦOperator󣬲ڶ롣 +void DfGraphConvertor::ConvertReshape(const CNodePtr node) { + // ӡ־ϢʾReshapeڵĵڶתΪOp + MS_LOG(INFO) << "Convert the second input of reshape to op attr."; + const auto kInputNum = 3; // 峣kInputNumʾReshapeڵӦþе + if (node->size() < kInputNum) { // жReshapeڵǷСkInputNumСڣӡϢ + MS_LOG(WARNING) << "Reshape must have two inputs."; + return; + } + OpAdapterPtr adpt = FindAdapter(node, training_); // ReshapeڵӦOpAdapter + if (adpt == nullptr) { + return; + } + auto op = adpt->generate(node); // ʹOpAdaptergenerateReshapeڵӦOperator + MS_EXCEPTION_IF_NULL(op); + // get shape form attr + // ȡReshapeڵĵһ루shapeֵӦValueNodePtr + auto value_node = node->input(0)->cast(); + MS_EXCEPTION_IF_NULL(value_node); + MS_EXCEPTION_IF_NULL(value_node->value()); + auto primitive = value_node->value()->cast(); // ȡValueNodePtrеPrimitivePtr + MS_EXCEPTION_IF_NULL(primitive); + auto value = primitive->GetAttr("shape"); // ȡPrimitivePtrеshapeԵֵ + std::vector list; + list = CastToInt(value); // shapeԵֵתΪstd::vector + + (void)op->SetAttr("shape", list); // תõshapeֵΪOperator + op_cache_[node.get()] = op; // ReshapeڵӦOperator洢op_cache_УReshapeڵַָΪOperatorΪֵ +} + +//úڽConv2DڵתΪӦOperator󣬲paddingԡ +void DfGraphConvertor::ConvertConv2D(const CNodePtr node) { + MS_EXCEPTION_IF_NULL(node); // жConv2DڵǷΪգΪ򷵻 + OpAdapterPtr adpt = FindAdapter(node, training_); // Conv2DڵӦOpAdapter + if (adpt == nullptr) { + return; + } + auto op = adpt->generate(node); // ʹOpAdaptergenerateConv2DڵӦOperator + MS_EXCEPTION_IF_NULL(op); + auto value_node = node->input(0)->cast(); // ȡConv2DڵĵһӦValueNodePtr + MS_EXCEPTION_IF_NULL(value_node); + MS_EXCEPTION_IF_NULL(value_node->value()); + auto primitive = value_node->value()->cast(); // ȡValueNodePtrеPrimitivePtr + MS_EXCEPTION_IF_NULL(primitive); + auto value = primitive->GetAttr("padding"); // ȡPrimitivePtrеpaddingԵֵ + if (value != nullptr) { // paddingԵֵΪգʾConv2Dڵpadding + std::string pad_mode = GetValue(value); + (void)op->SetAttr("padding", pad_mode); // paddingԵֵΪOperator + } + op_cache_[node.get()] = op; // Conv2DڵӦOperator洢op_cache_УConv2DڵַָΪOperatorΪֵ +} + +//ú׷ٴTupleGetItemڵ㣬ȡItem룬ظڵ㡣 +AnfNodePtr DfGraphConvertor::TraceTupleGetItem(const CNodePtr &node, uint64_t *index) { + const int TUPLE_GET_ITEM_INDEX = 2; // 峣TUPLE_GET_ITEM_INDEXʾTupleGetItemڵλ + if (node->inputs().size() < 3) { // "tuple_getitem" primitive must have 3 inputs + // // ж"tuple_getitem" primitiveǷС3С3׳쳣 + MS_LOG(EXCEPTION) << "length of inputs of TupleGetItem is less than 3"; + } + auto index_node = node->inputs()[TUPLE_GET_ITEM_INDEX]; // ȡTupleGetItemڵĵ룬ֵ + if (!index_node->isa()) { // жֵӦĽڵǷΪValueNodeǣerror_ΪINVALID_ARGUMENT׳쳣 + error_ = INVALID_ARGUMENT; + MS_LOG(EXCEPTION) << "can't convert get item with non-constant index"; + } + // ȡֵʾindexָָı + *index = LongToUlong(GetValue(GetValueNode(index_node))); + return node->inputs()[1]; // TupleGetItemڵĵڶ룬ȡItem +} + +//ú׷ٴDependڵ㣬ȡcontrol룬ظڵ㡣 +AnfNodePtr DfGraphConvertor::TraceDepend(const CNodePtr &node) { + auto cnode = node->cast(); // ȡDependڵָcnode + // ж"Depend" primitiveǷС3С3׳쳣 + if (cnode->inputs().size() < 3) { // "Depend" primitive have 3 inputs + MS_LOG(EXCEPTION) << "length of inputs of depend is less than 3"; + } + return cnode->inputs()[1]; // Dependڵĵڶ룬control +} + +//ú׷ٴMakeTupleڵ㣬ȡTupleĵindexԪأظڵ㡣 +AnfNodePtr DfGraphConvertor::TraceMakeTuple(const CNodePtr &node, uint64_t index) { + if (index + 1 >= node->inputs().size()) { // жindex + 1Ƿڵmake_tupleڵǣ׳쳣 + MS_LOG(EXCEPTION) << "length of make_tuple is less than index: " << index; + } + return node->inputs()[index + 1]; // make_tupleڵĵindex + 1ڵ㣬ȡTupleĵindexԪ +} + +//úڻȡڵĴOutHandlerݽڵǷTupleڲвͬĴ +OutHandler DfGraphConvertor::GetHandler(const AnfNodePtr &node, const std::stack &index_stack, + AnfNode *const draw_index) { + if (node == nullptr) { // жϽڵǷΪnullptrǣ־һյOutHandler + MS_LOG(ERROR) << "Get nullptr while trace real op"; + return OutHandler(nullptr, ""); + } + std::ostringstream ss; // һostringstreamɽڵַʾ + ss << "op" << node.get(); + if (index_stack.empty()) { // жindex_stackǷΪ ΪգʾTupleڲֱOutHandler + op_draw_name_[draw_index] = ss.str(); // ڵַʾ浽op_draw_name_ + return OutHandler(Convert(node), ""); // ConvertڵתΪOperatorPtrOutHandler + } else { + // index_stackΪգʾTupleڲ + // ҸýڵOpAdapterPtr + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { // Ϊգ־һյOutHandler + MS_LOG(ERROR) << "Can not get node output as adpt is nullptr!"; + error_ = NOT_FOUND; + return OutHandler(nullptr, ""); + } + OperatorPtr op = Convert(node); // ConvertڵתΪOperatorPtr + if (op == nullptr) { // תOperatorPtrΪգ־һյOutHandler + error_ = NOT_FOUND; + MS_LOG(ERROR) << "Can not convert node for trace real op"; + return OutHandler(nullptr, ""); + } + op_draw_name_[draw_index] = ss.str(); // ڵַʾ浽op_draw_name_ + // getOutputȡĴOutHandler + return adpt->getOutput(Convert(node), static_cast(index_stack.top())); + } +} + +// get the real operator through maketuple tuple_getitem depend +//ú׷ٻȡڵʵڵ㣬ȥTupleGetItemMakeTupleDependڵ㣬䴦OutHandler +OutHandler DfGraphConvertor::TraceRealOp(AnfNodePtr node) { + // жϽڵǷΪTupleGetItemMakeTupleDependڵ + bool flag = IsPrimitiveCNode(node, prim::kPrimTupleGetItem) || IsPrimitiveCNode(node, prim::kPrimMakeTuple) || + IsPrimitiveCNode(node, prim::kPrimDepend); + std::stack index_stack; // һջindex_stackڱTupleGetItemڵ + auto draw_index = node.get(); // 浱ǰڵַָںͼ + while (flag) { // ѭ׷ʵڵ + flag = false; + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { + uint64_t index; + // ǰڵTupleGetItemڵ㣬TraceTupleGetItemȡʵڵ + node = TraceTupleGetItem(node->cast(), &index); + // ѹindex_stack + index_stack.push(index); + flag = true; + } else if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { + if (index_stack.empty()) { + // ǰڵMakeTupleڵindex_stackΪգʾڴ־һյOutHandler + MS_LOG(ERROR) << "TraceRealOp find a make_tuple node"; + return OutHandler(nullptr, ""); + } else { + // ǰڵMakeTupleڵindex_stackΪգTraceMakeTupleȡʵڵ㲢 + node = TraceMakeTuple(node->cast(), index_stack.top()); + index_stack.pop(); + flag = true; + } + } else if (IsPrimitiveCNode(node, prim::kPrimDepend)) { + // ǰڵDependڵ㣬TraceDependȡʵڵ + node = TraceDepend(node->cast()); + flag = true; + } + } + return GetHandler(node, index_stack, draw_index); // GetHandlerȡڵĴOutHandler +} + +//úڽTupleGetItemڵתΪӦӴ +void DfGraphConvertor::ConvertTupleGetItem(const CNodePtr node) { + auto handle = TraceRealOp(node); // TraceRealOpȡTupleGetItemڵʵڵ㴦OutHandler + if (handle.op == nullptr) { // ʵڵ㴦Ϊգ־ + MS_LOG(ERROR) << "Failed to trace tuple get item"; + return; + } + out_handle_cache_[node.get()] = handle; // TupleGetItemڵʵڵ㴦OutHandlerӵout_handle_cache_л +} + +// Get the real op for tuple_getitem through make tuple, or depend +//úڴTupleGetItemڵDependڵݹػȡЩڵʵڵ㡣 +AnfNodePtr DfGraphConvertor::GetRealOpNode(AnfNodePtr node) { + const int TUPLE_GET_ITEM_INDEX = 2; + if (IsPrimitiveCNode(node, prim::kPrimTupleGetItem)) { //// ǰڵTupleGetItemڵ + auto node_inputs = node->cast()->inputs(); + if (node_inputs.size() != 3) { // "tuple_getitem" primitive must have 3 inputs + MS_LOG(ERROR) << "tuple get item node not correct!"; + error_ = FAILED; + return node; + } + MS_EXCEPTION_IF_NULL(node_inputs[TUPLE_GET_ITEM_INDEX]); + if (!node_inputs[TUPLE_GET_ITEM_INDEX]->isa()) { // ȡTupleGetItemڵֵ + error_ = INVALID_ARGUMENT; + MS_LOG(EXCEPTION) << "can't convert get item with non-constant index"; + } + auto value_ptr = GetValueNode(node_inputs[TUPLE_GET_ITEM_INDEX])->cast(); + if (value_ptr == nullptr) { + MS_LOG(ERROR) << "Can not convert get item as value is nullptr!"; + error_ = FAILED; + return node; + } + int64_t index = value_ptr->value(); + + // make_tuple apply inputs:make_tuple, [tuple_items,] + if (IsPrimitiveCNode(node_inputs[1], prim::kPrimMakeTuple)) { // TupleGetItemڵMakeTupleڵ + auto tuple_inputs = node->cast()->inputs(); + if (tuple_inputs.size() < LongToSize(index + 1L)) { + MS_LOG(ERROR) << "make tuple input items node not correct! size:" << tuple_inputs.size() + << ", item index:" << index; + error_ = FAILED; + return node; + } + return GetRealOpNode(tuple_inputs[LongToSize(index + 1L)]); // ݹGetRealOpNodeȡMakeTupleڵʵڵ + } + return GetRealOpNode(node_inputs[1]); // ݹGetRealOpNodeȡTupleGetItemڵʵڵ + } + + // depend apply inputs: depend,output,depended_node + if (IsPrimitiveCNode(node, prim::kPrimDepend)) { // ǰڵDependڵ + auto depend_inputs = node->cast()->inputs(); + if (depend_inputs.size() != 3) { // "Depend" primitive have 3 inputs + MS_LOG(ERROR) << "depend input items not correct"; + error_ = FAILED; + return node; + } + return GetRealOpNode(depend_inputs[1]); // ݹGetRealOpNodeȡDependڵʵڵ + } + return node; // ֱӷصǰڵ +} + +// convert the anf node to corresponding operator list +/* +˺ĿǽDependڵԼMakeTupleڵеӽڵתΪ +ڹͼʱԽDependڵתΪƱߣMakeTupleڵӽڵںбȷ +*/ +std::vector DfGraphConvertor::ConvertDependNode(const AnfNodePtr node) { + if (IsPrimitiveCNode(node, prim::kPrimMakeTuple)) { //жϽڵǷΪMakeTupleڵ + std::vector op_lists; //ǣĸԪؽڵתΪ洢op_listsУȻ󷵻op_lists + auto node_inputs = node->cast()->inputs(); + for (size_t index = 1; index < node_inputs.size(); index++) { + auto op = Convert(GetRealOpNode(node_inputs[index])); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert real op node to operator failed"; + error_ = FAILED; + return std::vector({}); + } + op_lists.push_back(op); + } + return op_lists; + } + // ǰڵ㲻MakeTupleڵ㣬תΪ + auto op = Convert(GetRealOpNode(node)); + if (op == nullptr) { + MS_LOG(ERROR) << "Convert real op node to operator failed"; + error_ = FAILED; + return std::vector({}); + } + return std::vector({op}); +} + +/* +úCNodeڵ㣩ͣݽڵӦضIJ +һֵʾǷҪһĽڵ㡣 +*/ +bool DfGraphConvertor::CheckCNode(const std::string &name, const CNodePtr node) { + // ignore apply node of return + // ضڵ㣬falseһ + if (name == "" || name == prim::kPrimReturn->name() || name == prim::kPrimDepend->name() || + name == prim::kPrimSwitchLayer->name() || name == prim::kPrimPartial->name()) { + return false; + } + + // Convert TopK second input from int64 to int32. + // TopKڵĵڶint64תΪint32 + if (name == prim::kPrimTopK->name()) { + ConvertTopK(node); + return true; + } + + // Convert Reshape add const input to attr(shape) + // תReshapeڵ㣬ӵԣshapeС + if (name == prim::kPrimReshape->name()) { + ConvertReshape(node); + return true; + } + + // Add attr pad mode to Conv2D + // ΪConv2DDepthwiseConv2dNativeConv2DBackpropInputV2ڵpaddingԡ + if (name == prim::kPrimConv2D->name() || name == prim::kPrimDepthwiseConv2dNative->name() || + name == kNameConv2DBackpropInputV2) { + ConvertConv2D(node); + return true; + } + + // make_tuple is used for a dynamic_input, convert it to a vector of OutHandlers + // ڶ̬make_tupleڵ㣬תΪOutHandler + if (name == prim::kPrimMakeTuple->name()) { + ConvertMakeTuple(node); + return false; // falsemake_tupleڵĽһ + } + + // As for nodes with multi outputs, convert tuple_getitem to OutHandle + // жtuple_getitemڵ㣬תΪOutHandler + if (name == prim::kPrimTupleGetItem->name()) { + ConvertTupleGetItem(node); + return false; // falsetuple_getitemڵĽһ + } + // δ򷵻trueʾҪһýڵ㡣 + return true; +} + +//úConvertCNodeڽCNodeڵ㣩תΪӦ +OperatorPtr DfGraphConvertor::ConvertCNode(const CNodePtr node) { + SaveParamFormat(node); // SaveParamFormatڵIJʽ + std::string name = GetCNodeTargetFuncName(node); //ȡڵ + if (!CheckCNode(name, node)) { //ͨCheckCNodeýڵ㣬ݽڵӦӦIJ + return nullptr; //CheckCNodefalseʾýڵΪڵ㣬Ҫһֱӷnullptr + } + + // get corresponding OpAdapter + // ȡӦOpAdapter + OpAdapterPtr adpt = FindAdapter(node, training_); //ͨFindAdapterȡڸýڵOpAdapter + if (adpt == nullptr) { //δҵerror_ΪNOT_FOUNDnullptr + error_ = NOT_FOUND; + return nullptr; + } + + // get operator + // ȡ + OperatorPtr op = nullptr; + auto it_op = op_cache_.find(node.get()); + if (it_op != op_cache_.end()) { //Ѿֱʹ + op = it_op->second; + } else { + op = adpt->generate(node); //ͨgenerate + } + + // set attribute for primitive + // ԭ + (void)adpt->setAttr(op, node); //ݽڵ⴦òͬԡ + + // add into cache + // ӵ + (void)op_cache_.emplace(node.get(), op); + + DrawCNode(node, adpt); // ƽڵϢڿӻ + + return op_cache_[node.get()]; //ؽڵӦ +} + +//úڽANFеParameterڵ㣩תΪDataFlowеı +OperatorPtr DfGraphConvertor::ConvertParameter(const AnfNodePtr node) { + // convert Parameter in ANF to variable in DataFlow + // ANFеParameterתΪDataFlowеı + auto adpt = FindAdapter(node, training_); //ͨFindAdapterȡڸýڵadpt + if (adpt == nullptr) { //δҵ׳쳣 + MS_LOG(EXCEPTION) << "Can not find adapter for Parameter"; + } + auto op = adpt->generate(node); //ͨgenerateӵ + op_cache_[node.get()] = op; + + // build index for parameter using name + // ʹΪ ڵӵparams_ + std::string name = std::static_pointer_cast(node)->name(); + params_[name] = node; + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); //Ϊýڵ㴴һıʶ䱣op_draw_name_Уڿӻ + compute_sout_ << ss.str() << "[shape=octagon, label=\"" << name << "\"]" << endl; + return op_cache_[node.get()]; //ڿӻлƸýڵ㣬ؽڵӦ +} + +//úڱĸʽϢ +void DfGraphConvertor::SaveParamFormat(const CNodePtr node) { + AnfNodePtr op = node->input(0); + if (IsValueNode(op)) { //ڵĵһǷValueNode + auto prim = GetValueNode(op); //ȡValueNodeӦPrimitiveͣԡ + for (auto attr : prim->attrs()) { + if (attr.first == "format") { //Ϊ"format"ԣֵлȡʽϢ + std::string format; //ʽϢַͻͣͽд + if (attr.second->isa()) { + bool converted = CheckAndConvertUtils::ConvertAttrValueToString(prim->name(), "format", &attr.second); + if (converted) { + format = attr.second->ToString(); + } else { + CheckAndConvertUtils::GetFormatStringVal(prim, &format); + } + } else if (attr.second->isa()) { + format = attr.second->ToString(); + } + if (format != "NCDHW" && format != "NHWC") { //ʽϢ"NCDHW""NHWC"ڵ + break; + } + for (size_t i = 1; i < node->size(); i++) { + auto input = node->input(i); + if (input->isa()) { //ΪParameter͵룬Ӧĸʽparam_format_С + param_format_[input->DebugString()] = format; + MS_LOG(DEBUG) << "Save Param " << input->DebugString() << " format: " << format; //ӡIJʽϢڵ + } + } + } + } + } +} + +//úڽValueNodeתɶ(Constant)ڵ㡣 +Status DfGraphConvertor::TryConvertValueNodeToMultiConst(const ValueNodePtr node) { + MS_EXCEPTION_IF_NULL(node); + ValuePtr value = node->value(); + MS_EXCEPTION_IF_NULL(value); + if (!value->isa() && !value->isa()) { + return FAILED; + } + // ֵǷΪ ValueList ValueTuple ͣǣ򷵻 FAILED + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + if (vec.empty()) { + return FAILED; + } + //ȡValueListValueTupleеԪأЩԪءκһԪزMeTensorͣ᷵FAILED + std::shared_ptr> tuple_items = std::make_shared>(); + for (size_t i = 0; i < vec.size(); i++) { + MS_EXCEPTION_IF_NULL(vec[i]); + if (vec[i]->isa()) { + // MeTensor ת GeTensor + GeTensorPtr ge_tensor = transform::TransformUtil::ConvertTensor(vec[i]->cast(), kOpFormat_NCHW); + auto const_op = std::make_shared(node->fullname_with_scope() + "/const/inputs/" + std::to_string(i)); + (void)const_op->set_attr_value(*ge_tensor); + (void)const_op->update_output_desc_y(ge_tensor->GetTensorDesc()); + (void)tuple_items->emplace_back(OutHandler(const_op, "")); + } else { // бԪеκһԪز MeTensor ͣ򷵻 FAILED + return FAILED; + } + } + if (tuple_items->empty()) { // תбԪΪգ򷵻 FAILED + return FAILED; + } + + tuple_out_handle_cache_[node.get()] = tuple_items; // תбԪ鱣Ϊ OutHandler + return SUCCESS; +} + +//úڽValueNodeתɳ(Constant) +OperatorPtr DfGraphConvertor::ConvertValueNode(const ValueNodePtr node) { + // convert valuenode in ANF to Const in DataFlow + // find paramerte referenced by SymbolicKeyInstance of valuenode + // ANF е ValueNode ת DataFlow е Const + // ûͼϢ + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + compute_sout_ << ss.str() << "[label= \"" << node->value()->ToString() << "\" shape=ellipse]" << endl; + // Խ ValueNode תɶ(Constant)ڵ + if (TryConvertValueNodeToMultiConst(node) == SUCCESS) { + MS_LOG(INFO) << "Convert value node to multi Constant OP success"; + return nullptr; + } + // ȡӦ OpAdapter + OpAdapterPtr adpt = FindAdapter(node, training_); + if (adpt == nullptr) { + error_ = NOT_FOUND; + return nullptr; + } + // ɶӦIJ Operator + auto op = adpt->generate(node); + // set const's attrs + // óֵ + if (adpt->setAttr(op, "value", node->value()) != 0) { + MS_LOG(WARNING) << "set attr value for const failed"; + } + // תΪ Constant + auto const_op = std::static_pointer_cast(op); + if (const_op == nullptr) { + MS_LOG(ERROR) << "Get Constant operator failed"; + return nullptr; + } + // + auto ge_tensor = const_op->get_attr_value(); + auto ge_desc = ge_tensor.GetTensorDesc(); + (void)const_op->update_output_desc_y(ge_desc); + // op_cache_ У + op_cache_[node.get()] = op; + return op_cache_[node.get()]; +} + +//úڻCNodeڵͼαʾ +void DfGraphConvertor::DrawCNode(const CNodePtr node, const OpAdapterPtr adpt) { + // apply node CNode ڵͼαʾ + if (adpt == nullptr || node == nullptr) { + MS_LOG(ERROR) << "Failed to draw apply node as adpt or node is nullptr!"; + return; + } + std::ostringstream ss; + ss << "op" << node.get(); + op_draw_name_[node.get()] = ss.str(); + // ƽڵıʾ + compute_sout_ << ss.str() << "[label=<"; + compute_sout_ << "" << endl; + // ˿ڵıǩ + auto input_map = adpt->getInputMap(); + auto dyn_input_map = adpt->getDynInputMap(); + if (input_map.size() + dyn_input_map.size() > 0) { + compute_sout_ << ""; + for (auto &it : input_map) { + compute_sout_ << ""; + } + for (auto &it : dyn_input_map) { + compute_sout_ << ""; + } + compute_sout_ << "" << endl; + } + // ƽڵĹƺ + compute_sout_ << "" << endl; + + // print attrs' values + // ƽڵֵ + auto atts = adpt->GetAttrsFromDrawGraph(); + for (auto &it : atts) { + compute_sout_ << ""; + } + // ԵΪһλ׼ + adpt->clearAttrVect(); + + compute_sout_ << "
" << it.second.name << "" << it.second.name << "
\"" << node->ToString() + << ":" << GetCNodeTargetFuncName(node) << "\"
\"" << it + << "\"
> shape=plaintext]" << endl; +} + +//úע +void DfGraphConvertor::RegisterAdapter(const std::string &name, OpAdapterPtr adpt) { + // עӵOpAdapterMap + // ʹOpAdapterDescаװ + OpAdapterMap::get()[name] = std::make_shared(adpt); +} + +//úעͬʱִ֧ѵ +void DfGraphConvertor::RegisterAdapter(const std::string &name, OpAdapterPtr train_adpt, OpAdapterPtr infer_adpt) { + // עѵӵOpAdapterMap + // ʹOpAdapterDescѵаװ + OpAdapterMap::get()[name] = std::make_shared(train_adpt, infer_adpt); +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From a5c9cf55a09cf842782ac26250a42d3ce10be92a Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:54:28 +0800 Subject: [PATCH 10/72] ADD file via upload --- .../ccsrc/transform-update/ctc_ops_declare.cc | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/ctc_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/ctc_ops_declare.cc b/mindspore/ccsrc/transform-update/ctc_ops_declare.cc new file mode 100644 index 00000000000..0ff3d3c3a21 --- /dev/null +++ b/mindspore/ccsrc/transform-update/ctc_ops_declare.cc @@ -0,0 +1,48 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/ctc_ops_declare.h" + +namespace mindspore::transform { +// CTCLoss +INPUT_MAP(CTCLoss) = {{1, INPUT_DESC(inputs)}, + {2, INPUT_DESC(labels_indices)}, + {3, INPUT_DESC(labels_values)}, + {4, INPUT_DESC(sequence_length)}}; +//输入映射,inputs索引为1,labels_indices索引为2,labels_values索引为3,sequence_length索引为4 +ATTR_MAP(CTCLoss) = { + {"preprocess_collapse_repeated", ATTR_DESC(preprocess_collapse_repeated, AnyTraits())}, + {"ctc_merge_repeated", ATTR_DESC(ctc_merge_repeated, AnyTraits())}, + {"ignore_longer_outputs_than_inputs", ATTR_DESC(ignore_longer_outputs_than_inputs, AnyTraits())}}; +//属性映射,属性preprocess_collapse_repeated类型为bool,属性ctc_merge_repeated类型为bool,属性ignore_longer_outputs_than_inputs类型为bool +OUTPUT_MAP(CTCLoss) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(gradient)}}; +//输出映射,loss索引为0,gradient索引为1 +REG_ADPT_DESC(CTCLoss, kNameCTCLoss, ADPT_DESC(CTCLoss)) +//注册CTCLoss操作的适配器描述KNameCTCLoss + +// CTCGreedyDecoder +INPUT_MAP(CTCGreedyDecoder) = {{1, INPUT_DESC(inputs)}, {2, INPUT_DESC(sequence_length)}}; +//输入映射,inputs索引为1,sequence_length索引为2 +ATTR_MAP(CTCGreedyDecoder) = {{"merge_repeated", ATTR_DESC(merge_repeated, AnyTraits())}}; +//属性映射,属性merge_repeated类型为bool +OUTPUT_MAP(CTCGreedyDecoder) = {{0, OUTPUT_DESC(decoded_indices)}, + {1, OUTPUT_DESC(decoded_values)}, + {2, OUTPUT_DESC(decoded_shape)}, + {3, OUTPUT_DESC(log_probability)}}; +//输出映射,decoded_indices索引为0,decoded_values索引为1,decoded_shape索引为2,log_probability索引为1 +REG_ADPT_DESC(CTCGreedyDecoder, kNameCTCGreedyDecoder, ADPT_DESC(CTCGreedyDecoder)) +//注册CTCGreedyDecoder操作的适配器描述KNameCTCGreedyDecoder +} // namespace mindspore::transform -- 2.34.1 From 067065579da78cc75491d1480c583388e485184f Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:55:55 +0800 Subject: [PATCH 11/72] ADD file via upload --- .../transform-update/data_flow_ops_declare.cc | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/data_flow_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/data_flow_ops_declare.cc b/mindspore/ccsrc/transform-update/data_flow_ops_declare.cc new file mode 100644 index 00000000000..e8ccf5c9c61 --- /dev/null +++ b/mindspore/ccsrc/transform-update/data_flow_ops_declare.cc @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/data_flow_ops_declare.h" +#include +#include + +namespace mindspore::transform { +INPUT_MAP(TensorArray) = {{1, INPUT_DESC(size)}}; +//输入映射,size索引为1 +ATTR_MAP(TensorArray) = {{"dtype", ATTR_DESC(dtype, AnyTraits())}, + {"element_shape", ATTR_DESC(element_shape, AnyTraits>())}, + {"dynamic_size", ATTR_DESC(dynamic_size, AnyTraits())}, + {"clear_after_read", ATTR_DESC(clear_after_read, AnyTraits())}, + {"identical_element_shapes", ATTR_DESC(identical_element_shapes, AnyTraits())}, + {"tensor_array_name", ATTR_DESC(tensor_array_name, AnyTraits())}}; +//属性映射,属性dtype类型为GEType,属性element_shape类型为int64_t,属性dynamic_size类型为bool,属性identical_element_shapes类型为bool +//属性clear_after_read类型为bool,属性tensor_array_name类型为bool +OUTPUT_MAP(TensorArray) = {{0, OUTPUT_DESC(handle)}, {1, OUTPUT_DESC(flow)}}; +//输出映射,handle索引为0,flow索引为1 +REG_ADPT_DESC(TensorArray, kNameTensorArray, ADPT_DESC(TensorArray)) +//注册TensorArray,操作的适配器描述KNameTensorArray, + +INPUT_MAP(TensorArrayWrite) = { + {1, INPUT_DESC(handle)}, {2, INPUT_DESC(index)}, {3, INPUT_DESC(value)}, {4, INPUT_DESC(flow_in)}}; +//输入映射,handle索引为1,index索引为2,value索引为3,flow_in索引为4 +ATTR_MAP(TensorArrayWrite) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TensorArrayWrite) = {{0, OUTPUT_DESC(flow_out)}}; +REG_ADPT_DESC(TensorArrayWrite, kNameTensorArrayWrite, ADPT_DESC(TensorArrayWrite)) + +INPUT_MAP(TensorArrayGather) = {{1, INPUT_DESC(handle)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(flow_in)}}; +//输入映射,handle索引为1,indices索引为2,flow_in索引为3 +ATTR_MAP(TensorArrayGather) = {{"dtype", ATTR_DESC(dtype, AnyTraits())}, + {"element_shape", ATTR_DESC(element_shape, AnyTraits>())}}; +//属性映射,属性dtype类型为GEType,属性element_shape类型为int64_t +OUTPUT_MAP(TensorArrayGather) = {{0, OUTPUT_DESC(value)}}; +//输出映射,value索引为0 +REG_ADPT_DESC(TensorArrayGather, kNameTensorArrayGather, ADPT_DESC(TensorArrayGather)) +//注册TensorArrayGather操作的适配器描述KNameTensorArrayGather +} // namespace mindspore::transform -- 2.34.1 From 9d4bc71b5f6985b5fbaf2439b9e349f2d58d8b29 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:56:24 +0800 Subject: [PATCH 12/72] ADD file via upload --- .../transform-update/df_graph_manager.cc | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/df_graph_manager.cc diff --git a/mindspore/ccsrc/transform-update/df_graph_manager.cc b/mindspore/ccsrc/transform-update/df_graph_manager.cc new file mode 100644 index 00000000000..1e1daaf565c --- /dev/null +++ b/mindspore/ccsrc/transform-update/df_graph_manager.cc @@ -0,0 +1,245 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/df_graph_manager.h" + +#include + +#ifndef ENABLE_LITE_ACL +#include "include/common/utils/python_adapter.h" +#include "pipeline/jit/pipeline.h" +#endif +#ifndef NO_DLIB +#include "tdt/tsd_client.h" +#endif + +namespace mindspore { +namespace transform { +// ˹캯ڳʼ DfGraphWrapper һʵ +// ĸ'name''id''graph_ptr' 'options' +DfGraphWrapper::DfGraphWrapper(const std::string &name, const int &id, const DfGraphPtr &graph_ptr, + const OptionMap &options) + : name_(name), id_(id), graph_ptr_(graph_ptr), options_(options) {} + +DfGraphManager::DfGraphManager() { //캯 + graph_id_ = 0; + graph_runner_ptr_ = nullptr; + sess_ptr_ = nullptr; +} + +DfGraphManager::~DfGraphManager() { // + // in python first destroy after atexit but in c++ destoy before atexit + DeleteGraphRunner(); + DeleteGeSession(); + ClearGraph(); +#ifndef ENABLE_LITE_ACL + python_adapter::set_python_env_flag(false); +#endif +} + +DfGraphManager &DfGraphManager::GetInstance() { + static DfGraphManager instance; + return instance; +} + +// úͼID +int DfGraphManager::GenerateId() { + graph_id_++; // ͼID + if (graph_id_ <= 0) { // ͼIDСڵ0Ϊ1ȷIDΪ + graph_id_ = 1; + } + MS_LOG(INFO) << "Generate graph Id : " << graph_id_; // ӡɵͼID־¼ + return graph_id_; // ɵͼID +} + +// úͼιһͼΡ +// 'name' ʾͼεƣ'graph_ptr' ʾͼεָ룬'options' ʾͼεѡ +Status DfGraphManager::AddGraph(const std::string &name, const DfGraphPtr &graph_ptr, const OptionMap &options) { + std::lock_guard lg(lock_); // ʹûȷͼεIJ̰߳ȫ + if (name.empty()) { // ͼΪգЧ + MS_LOG(ERROR) << "The graph name is null, add graph failed"; + return Status::INVALID_ARGUMENT; + } + + if (graph_ptr == nullptr) { // ͼָΪգЧ + MS_LOG(INFO) << "The new graph {" << name << "}'s pointer is null, add graph failed"; + return Status::INVALID_ARGUMENT; + } + + int id = GenerateId(); // һµͼID + // һ DfGraphWrapperPtr ڰװͼϢͼӵͼι + DfGraphWrapperPtr wrap_ptr = std::make_shared(name, id, graph_ptr, options); + // ͼӵͼι + auto ret = graphs_.emplace(name, wrap_ptr); + if (ret.second == false) { // ͼѾڣɵͼθ + MS_LOG(WARNING) << "The graph name:{ " << name << " }is already exists! The old graph will be overwritten!!"; + ret.first->second = wrap_ptr; + } + MS_LOG(INFO) << "Add graph " << name << " to GraphManager success!"; // ɹͼκ¼־ + return Status::SUCCESS; // سɹ״̬ +} + +// úڻȡͼιеͼΣһ DfGraphWrapperPtr ͵ЩͼΡ +std::vector DfGraphManager::GetAllGraphs() { + std::lock_guard lg(lock_); // ʹûȷȡͼεIJ̰߳ȫ + std::vector ret; // ڴ洢ͼε + std::stringstream ss; + ss << "{ "; + for (auto it = graphs_.begin(); it != graphs_.end(); ++it) { // ͼιеͼ + ss << it->first << ", "; // ͼӵ־¼ַ + ret.emplace_back(it->second); // ͼָӵ + } + ss << "}"; + MS_LOG(INFO) << "Return graphs: " << ss.str(); // ¼ȡͼƵ־ + + return ret; // ش洢ͼε +} + +// úڻȡѱͼƼϡ +// Ϊ std::setʾһ洢ΨһͼƵļϡ +std::set DfGraphManager::GetSavedGraphs() { return saved_graphs_; } // ֱӷرͼƼ + +// úѱͼƼµͼơ +// 'id' ʾҪӵͼơ +void DfGraphManager::AddSavedGraphs(const std::string &id) { saved_graphs_.insert(id); } // µͼ 'id' ѱͼƼ + +// úڸͼƻȡӦ DfGraphWrapperPtr +// 'name' ʾҪȡͼơ +DfGraphWrapperPtr DfGraphManager::GetGraphByName(const std::string &name) { + std::lock_guard lg(lock_); // ʹûȷȡͼεIJ̰߳ȫ + if (name.empty()) { + MS_LOG(ERROR) << "The graph name is null"; + return nullptr; // ͼΪգؿָ + } + + auto it = graphs_.find(name); + if (it == graphs_.end()) { + MS_LOG(INFO) << "Can't found graph name: " << name; + return nullptr; // ͼͼιҲؿָ + } + MS_LOG(INFO) << "Return graph: " << name; // ¼ȡͼƵ־ + return it->second; // ҵͼε DfGraphWrapperPtr +} + +// úͼιеͼΣͷԴ +void DfGraphManager::ClearGraph() noexcept { + std::lock_guard lg(lock_); // ʹûȷͼεIJ̰߳ȫ + graphs_.clear(); // ͼιеͼ + anf_graphs_.clear(); // ͼιе ANF ͼΣһͼαʾ + MS_LOG(INFO) << "Remove all graphs in GraphManager"; // ¼ͼε־ +} + +// úڽ ANF ͼָضͼ +// 'name' ʾͼεƣ'anf_graph_ptr' ʾҪ ANF ͼָ롣 +void DfGraphManager::SetAnfGraph(const std::string &name, const AnfGraphPtr &anf_graph_ptr) { + DfGraphWrapperPtr df_graph = GetGraphByName(name); // ȡƵͼΰװ + if (df_graph == nullptr) { + MS_LOG(ERROR) << "Can't found graph name: " << name; + return; // ҲƵͼΣ򷵻ش˳ + } + std::lock_guard lg(lock_); // ʹûȷ ANF ͼεIJ̰߳ȫ + anf_graphs_[df_graph->id_] = anf_graph_ptr; // ANF ͼָͼΰװ ID 洢 anf_graphs_ +} + +// úڸݸͼ ID ȡӦ ANF ͼָ롣 +// 'graph_id' ʾҪȡͼ ID +AnfGraphPtr DfGraphManager::GetAnfGraph(uint32_t graph_id) { + std::lock_guard lg(lock_); // ʹûȷȡ ANF ͼεIJ̰߳ȫ + auto iter = anf_graphs_.find(graph_id); + if (iter == anf_graphs_.end()) { + MS_LOG(ERROR) << "Can't found anf graph, graph_id = " << graph_id; + return nullptr; // Ҳͼ ID Ӧ ANF ͼΣ¼־ؿָ + } + + return iter->second; // ҵͼ ID Ӧ ANF ͼָ +} + +// ú ANF ͼƳѹ ANF ͼΡ +void DfGraphManager::EraseAnfGraph() { + std::lock_guard lg(lock_); // ʹûȷ ANF ͼIJ̰߳ȫ + anf_graphs_.clear(); // ANF ͼƳѹ ANF ͼ +} + +// úͼι GEGraphEngineỰָ롣 +// 'sess_ptr' ʾҪõ GE Ựָ롣 +void DfGraphManager::SetGeSession(const std::shared_ptr &sess_ptr) { + std::lock_guard lg(lock_); // ʹûȷ GE ỰIJ̰߳ȫ + if (sess_ptr == nullptr) { + MS_LOG(WARNING) << "You are adding a empty Ge Session"; // GE ỰָΪգ¼־ + } + + if (sess_ptr_ == nullptr) { + MS_LOG(INFO) << "Add a new Ge Session success"; // ֮ǰδù GE Ự¼óɹ־ + } else { // ֮ǰѾù GE Ự¼óɹ־ʾ֮ǰ GE Ự + MS_LOG(INFO) << "Add a new Ge Session success, the old Ge Session will be overwritten!!"; + } + sess_ptr_ = sess_ptr; // GE ỰָΪͼι GE Ựָ +} + +// úڻȡͼι GEGraphEngineỰָ롣 +// Ϊ std::shared_ptrʾ GE Ựָ롣 +std::shared_ptr DfGraphManager::GetGeSession() { + std::lock_guard lg(lock_); // ʹûȷȡ GE ỰָIJ̰߳ȫ + return sess_ptr_; // ͼι GE Ựָ +} + +// úɾͼι GEGraphEngineỰûỰصݡ +void DfGraphManager::DeleteGeSession() noexcept { + std::lock_guard lg(lock_); // ʹûȷɾ GE ỰIJ̰߳ȫ + if (sess_ptr_ == nullptr) { + MS_LOG(INFO) << "Ge Session is not exist"; // ǰûй GE Ự¼־ֱӷ + } else { + sess_ptr_ = nullptr; // GE ỰָΪָ룬ʾɾ GE Ự + saved_graphs_.clear(); // ѱͼƼϣƳѱͼϢ + MS_LOG(INFO) << "Delete Ge Session success"; // ¼ɾɹ־ + } +} + +// úͼιͼGraphRunnerָ롣 +// 'graph_runner_ptr' ʾҪõͼָ롣 +void DfGraphManager::SetGraphRunner(const std::shared_ptr &graph_runner_ptr) noexcept { + std::lock_guard lg(lock_); // ʹûȷͼIJ̰߳ȫ + if (graph_runner_ptr == nullptr) { // ͼָΪգ¼־ + MS_LOG(WARNING) << "You are adding a empty GraphRunner"; + } + + if (graph_runner_ptr_ == nullptr) { // ֮ǰδùͼ¼óɹ־ + MS_LOG(INFO) << "Add a new GraphRunner success"; + } else { // ֮ǰѾùͼ¼óɹ־ʾ֮ǰͼ + MS_LOG(INFO) << "Add a new GraphRunner success, the old GraphRunner will be overwritten!!"; + } + graph_runner_ptr_ = graph_runner_ptr; // ͼָΪͼιͼָ +} + +// úڻȡͼιͼGraphRunnerָ롣 +// Ϊ std::shared_ptrʾͼָ +std::shared_ptr DfGraphManager::GetGraphRunner() { + std::lock_guard lg(lock_); // ʹûȷȡͼָIJ̰߳ȫ + return graph_runner_ptr_; // ͼιͼָ +} + +// úɾͼιͼGraphRunner +void DfGraphManager::DeleteGraphRunner() noexcept { + std::lock_guard lg(lock_); // ʹûȷɾͼIJ̰߳ȫ + if (graph_runner_ptr_ == nullptr) { + MS_LOG(INFO) << "GraphRunner is not exist"; // ǰûйͼ¼־ֱӷ + } else { + graph_runner_ptr_ = nullptr; // ͼָΪָ룬ʾɾͼ + MS_LOG(INFO) << "Delete GraphRunner success"; // ¼ɾɹ־ + } +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 34887821072a1dc3dcad165d276f80803b1ee366 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:56:47 +0800 Subject: [PATCH 13/72] ADD file via upload --- .../flatten_recursive_stmt.py | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/flatten_recursive_stmt.py diff --git a/mindspore/ccsrc/transform-update/flatten_recursive_stmt.py b/mindspore/ccsrc/transform-update/flatten_recursive_stmt.py new file mode 100644 index 00000000000..f0544ae5675 --- /dev/null +++ b/mindspore/ccsrc/transform-update/flatten_recursive_stmt.py @@ -0,0 +1,164 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Ast optimizer for flatten recursive call.""" +from typing import Any, Tuple +import ast +from ast import FunctionDef +from mindspore import log as logger + + +class FlattenRecursiveStmt(ast.NodeTransformer): + """Ast optimizer for flatten recursive call.""" + + def __init__(self): + """ + Constructor of FlattenRecursiveStmt. + + Returns: + An instance of ast optimizer for flatten recursive call. + """ + self._flatten_table: dict = { + ast.Return: ["value"], + ast.Call: ["args"], + ast.BinOp: ["left", "right"], + ast.BoolOp: ["values"], + ast.unaryop: ["operand"], + } + + @staticmethod + def _generate_target_name(node: ast.AST, target_names): + """Generate unique target name.""" + if isinstance(node, ast.Call): #ڵast.Call,ڵԣʾõĿ꣩ + func = node.func #funcͣʹΪĿ + if isinstance(func, ast.Name): #ast.NameͣʹidΪĿ + target_name = func.id + elif isinstance(func, ast.Attribute): #ast.AttributeͣʹattrΪĿ + target_name = func.attr + else: #ͣ¼һ棬ĿΪ"function" + logger.warning("unhandled type of func of ast.Call while generating new target name: %s ", type(func)) + target_name = "function" + elif isinstance(node, ast.Return): #ڵast.ReturnͣĿΪ"return_value" + target_name = "return_value" + elif isinstance(node, (ast.BinOp, ast.boolop, ast.UnaryOp)): #ڵast.BinOpastast.boolopast.UnaryOpͣʹӦΪĿơ + target_name = type(node.op).__name__ + else: #͵Ľڵ㣬¼һ棬ĿΪýڵ + logger.warning("unhandled type of node while generating new target name: %s ", type(node)) + target_name = type(node).__name__ + suffix = 0 + result = target_name + while result in target_names: #ɵĿֺ׺ȷĿбΨһ + suffix += 1 + result = f"{target_name}_{suffix}" + target_names.append(result) + return result #ɵĿƲӵб + + @staticmethod + def _fill_in_original_target_names(target_names, node): + """Fill in original target names before getting unique names.""" + for function_index in range(len(node.body)): #ASTAbstract Syntax Treeڵnode.body + child = node.body[function_index] + if not isinstance(child, ast.Assign): #ÿڵУǷһֵ(ast.Assign) + continue #ǣһڵ㡣 + targets = child.targets #ڵǸֵ䣬ȡֵĿ(targets) + for target in targets: #Ŀast.Nameͣȡ(target.id)ǷѾtarget_namesбС + if not isinstance(target, ast.Name): + raise RuntimeError("currently only support ast.Name targets") + target_name = target.id + if target_name not in target_names: #target_namesбУͽӽȥ + target_names.append(target_name) + + @staticmethod + # nodeǷ֮һast.Name, ast.Constant, ast.Num, ast.Str, ast.NameConstant, ast.Bytes, ast.Ellipsis + # ǣֱӷһַԭʼnodeΪЩڵDzҪ¸ֵڵġ + def _create_new_assign_node(node: ast.AST, target_names) -> Tuple[str, ast.AST]: + """Create new assign node to be inserted into ast.FunctionDef.""" + if isinstance(node, (ast.Name, ast.Constant, ast.Num, ast.Str, ast.NameConstant, ast.Bytes, ast.Ellipsis)): + return "", node + #͵Ľڵ㣬FlattenRecursiveStmt._generate_target_nameһµĿơ + new_target_name = FlattenRecursiveStmt._generate_target_name(node, target_names) + return new_target_name, ast.Assign(targets=[ast.Name(id=new_target_name, ctx=ast.Store())], value=node) #һµĸֵڵast. Assign.nodeΪֵֵ(value) µĿƺʹĸֵڵ + + def _flatten_statement(self, node: ast.AST, target_names) -> [ast.AST]: + """Flatten recursive statement according to different node type.""" + flatten_config = self._flatten_table.get(type(node)) #ͨ_flatten_tableֵȡnodeͶӦչflatten_configûҵӦãһб + if flatten_config is None: + return [] + results = [] + for todo_name in flatten_config: + todos = getattr(node, todo_name) #չеÿչ(todo_name) + #б͵ԣбеÿԪأFlattenRecursiveStmt._create_new_assign_nodeһµĸֵڵ㣬ɵĿ滻ԭʼԪصλá + #ɵ½ڵԭʼڵͬԭʼڵ㡣 򣬽½ڵӵбС + if isinstance(todos, list): + new_list = [] + for todo in todos: + new_target_name, new_node = FlattenRecursiveStmt._create_new_assign_node(todo, target_names) + if id(new_node) == id(todo): + new_list.append(todo) + else: + new_list.append(ast.Name(id=new_target_name, ctx=ast.Load())) + results.append(new_node) + setattr(node, todo_name, new_list) + #ֵ͵ԣֵеÿֵԣFlattenRecursiveStmt._create_new_assign_nodeµĸֵڵ㣬ɵĿ滻ԭʼֵλá + #ͬɵ½ڵԭʼڵͬԭʼڵ㡣 򣬽½ڵӵбС + elif isinstance(todos, dict): + new_dict = [] + for key, value in todos: + new_target_name, new_node = FlattenRecursiveStmt._create_new_assign_node(value, target_names) + if id(new_node) == id(value): + new_dict[key] = value + else: + new_dict[key] = ast.Name(id=new_target_name, ctx=ast.Load()) + results.append(new_node) + setattr(node, todo_name, new_dict) + else: + new_target_name, new_node = FlattenRecursiveStmt._create_new_assign_node(todos, target_names) + if id(new_node) != id(todos): + setattr(node, todo_name, ast.Name(id=new_target_name, ctx=ast.Load())) + results.append(new_node) + return results #ؽбаɵ¸ֵڵ + + #nodeǷΪ"construct",ǣֱӷԭʼnode + def visit_FunctionDef(self, node: FunctionDef) -> Any: + """Traverse construct node and flatten recursive nodes.""" + if node.name != "construct": + return node + + target_names = [] #һյtarget_namesб_fill_in_original_target_namesԭʼĿ + self._fill_in_original_target_names(target_names, node) + index = len(node.body) - 1 #Ӻһ俪ʼǰÿ䣨node.bodyбʾ + while index >= 0: + child = node.body[index] #ÿ䣬Ƿһֵ䣨ast.Assignһʽ䣨ast.Expr + if isinstance(child, ast.Assign): + stmt = child.value #Ǹֵ,ȡֵֵ + elif isinstance(child, ast.Expr): + stmt = child.value #DZʽ䣬ȡʽֵ + else: + stmt = child #򣬽䱾ΪҪĽڵ + results = self._flatten_statement(stmt, target_names) #_flatten_statementչеĵݹڵ㣬target_namesб úһɵ¸ֵڵб + #_flatten_statement˽Щÿ½ڵ뵽ԭʼڵǰ棬ȷչĸֵڵ㰴ȷ˳롣 + if results: + results.reverse() + for result in results: + node.body.insert(index, result) + index += 1 + index -= 1 + return node + + def transform(self, ast_root): #FlattenRecursiveStmttransformFlattenRecursiveStmtĽӿڷ + #һASTAbstract Syntax Treeĸڵast_rootΪ룬ؾչݹڵ㴦ASTڵ㡣 + """Interface of FlattenRecursiveStmt.""" + ast_root = self.visit(ast_root) #self.visit(ast_root)һݹASṬvisit_FunctionDefȷASTеĸֽڵ㡣 + ast_root = ast.fix_missing_locations(ast_root) #ʹast.fix_missing_locations(ast_root)޸ASTڵȱʧλϢ ڴASTʱܻԽڵв롢ɾȲ½ڵλϢʧast.fix_missing_locationsASTΪȱʧλõĽڵĬϵλϢ + return ast_root #ؾչݹڵ㴦λ޸ASTڵ -- 2.34.1 From b0736de2be130baf299d1bed018b5cefb7a2f405 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:57:22 +0800 Subject: [PATCH 14/72] ADD file via upload --- .../functional_ops_declare.cc | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/functional_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/functional_ops_declare.cc b/mindspore/ccsrc/transform-update/functional_ops_declare.cc new file mode 100644 index 00000000000..2ea6af2dacc --- /dev/null +++ b/mindspore/ccsrc/transform-update/functional_ops_declare.cc @@ -0,0 +1,45 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/functional_ops_declare.h" + +namespace mindspore::transform { +// Case +INPUT_MAP(Case) = {{1, INPUT_DESC(branch_index)}}; +//输入映射,branch_index索引为1 +DYN_INPUT_MAP(Case) = {{2, DYN_INPUT_DESC(input)}}; +//动态输入映射,将索引为2的动态输入与名称为input的动态输入描述关联起来,用于后续操作 +ATTR_MAP(Case) = EMPTY_ATTR_MAP; +//属性映射,设为空 +DYN_OUTPUT_MAP(Case) = {{0, DYN_OUTPUT_DESC(output)}}; +//动态输出映射,将索引为0的动态输出与名称为output的动态输出描述关联起来,用于后续操作 +DYN_SUBGRAPH_MAP(Case) = {{0, DYN_SUBGRAPH_DESC(branches)}}; +//动态子图映射,将索引为0的动态子图与名称为branches的动态描述关联起来,用于后续操作 +REG_ADPT_DESC(Case, kNameCase, ADPT_DESC(Case)); +//注册Case操作的适配器描述kNameCase + +// While +DYN_INPUT_MAP(While) = {{1, DYN_INPUT_DESC(input)}}; +//输入映射,input索引为1 +ATTR_MAP(While) = {{"parallel_iterations", ATTR_DESC(parallel_iterations, AnyTraits())}}; +//属性映射,属性parallel_iterations类型为int32_t +DYN_OUTPUT_MAP(While) = {{0, DYN_OUTPUT_DESC(output)}}; +//动态输出映射,将索引为0的动态输出与名称为output的动态输出描述关联起来,用于后续操作 +SUBGRAPH_MAP(While) = {{0, SUBGRAPH_DESC(cond)}, {1, SUBGRAPH_DESC(body)}}; +//动态子图映射,将索引为0的动态子图与名称为cond的动态描述关联,将索引为1的动态子图与名称为body的动态描述关联,用于后续操作 +REG_ADPT_DESC(While, kNameWhile, ADPT_DESC(While)); +//注册While操作的适配器描述kNameWhile +} // namespace mindspore::transform -- 2.34.1 From 55894bcb491457aa4597bac89db8f13c5e3c8f6c Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 21:59:54 +0800 Subject: [PATCH 15/72] ADD file via upload --- .../ccsrc/transform-update/graph_builder.cc | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/graph_builder.cc diff --git a/mindspore/ccsrc/transform-update/graph_builder.cc b/mindspore/ccsrc/transform-update/graph_builder.cc new file mode 100644 index 00000000000..e40f894e1b9 --- /dev/null +++ b/mindspore/ccsrc/transform-update/graph_builder.cc @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/graph_builder.h" + +#include + +#include "ops/math_ops.h" + +namespace mindspore { +namespace transform { +// úڹ MDDatasetMindSpore DatasetͼΣGraph +DfGraphPtr BuildMDDatasetGraph(const DatasetGraphParam ¶m) { + MS_LOG(INFO) << "BuildMDDatasetGraph."; // ¼־ʾڹ MDDataset ͼ + + // InitData + // һ "InitData"ʹò "init_data_tmp" Ϊƣ "channel_name" Ϊ + // param.queue_name() ֵ + auto d = ge::op::InitData("init_data_tmp").set_attr_channel_name(param.queue_name()); + + // set graph inputs & outputs + // ͼε + std::vector inputs{d}; // "InitData" Ϊͼε + std::vector outputs{d}; // "InitData" Ϊͼε + + // һΪ "dataset" MDDataset ͼΣʹ "dataset_graph" ָָͼ + DfGraphPtr dataset_graph = std::make_shared("dataset"); + + // õ MDDataset ͼ + (void)dataset_graph->SetInputs(inputs); + (void)dataset_graph->SetOutputs(outputs); + + return dataset_graph; // عõ MDDataset ͼεָ +} + +// úڹݼͼΣGraph +Status BuildDatasetGraph(const DatasetGraphParam ¶m, const std::string &phase) { + Status ret; // 洢ִн״̬ + std::string graph_name = phase; // Ը 'phase' Ϊͼε + + MS_LOG(INFO) << "BuildDatasetGraph begin. phase is " << phase; // ¼־ʾʼݼͼ + MS_LOG(INFO) << "param is " << param.ToString() << "."; // ¼־ӡ 'param' ϸϢ + + // BuildMDDatasetGraph MDDataset ͼΣõͼָ洢 'dataset_graph' + DfGraphPtr dataset_graph = BuildMDDatasetGraph(param); + // õ MDDataset ͼӵͼιУʹ 'graph_name' Ϊͼε + ret = DfGraphManager::GetInstance().AddGraph(graph_name, dataset_graph); + // AddGraph ִнӦ־¼ + if (ret != Status::SUCCESS) { // ͼʧܣ¼־ + MS_LOG(ERROR) << "BuildDatasetGraph failed."; + } else { // ͼγɹ¼־ + MS_LOG(INFO) << "BuildDatasetGraph end."; + } + return ret; // غִн״̬ +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 2b388dfe7479949b846b6936bd74a8e2e1653ea4 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:02:36 +0800 Subject: [PATCH 16/72] ADD file via upload --- .../ccsrc/transform-update/graph_pattern.py | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/graph_pattern.py diff --git a/mindspore/ccsrc/transform-update/graph_pattern.py b/mindspore/ccsrc/transform-update/graph_pattern.py new file mode 100644 index 00000000000..5e09fe47584 --- /dev/null +++ b/mindspore/ccsrc/transform-update/graph_pattern.py @@ -0,0 +1,249 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Patterns for describing graphs""" +#描述图形的模式 +from mindspore.ops import Primitive +from mindspore.common.tensor import Tensor +from mindspore._c_expression import Pattern, OneOf_, Prim_, Call_, NoneOf_, Any, NewTensor_, NewParameter_, Imm + +__all__ = [ + "OneOf", + "Prim", + "Call", + "NoneOf", + "Any", + "NewTensor", + "NewParameter", + "Imm" +] + + +class OneOf(OneOf_): + r""" + Express a pattern which allows a list of patterns. + 表达一个允许模式列表的模式 + """ + def __init__(self, patterns=None): + r""" + Args: + patterns(Union[:class:`mindspore.graph_utils.graph_pattern`, + tuple[:class:`mindspore.graph_utils.graph_pattern`], + list[:class:`mindspore.graph_utils.graph_pattern`]]): list of allowed patterns, + each element should be one of the exposed Pattern instance. + + Raises: + TypeError: raise type error for invalid inputs. + """ + self.patterns = patterns + if isinstance(patterns, Pattern): + OneOf_.__init__(self, [patterns]) + #如果patterns是mindspore.graph_utils.graph_pattern类的实例,则使用包含patterns的单一元素列表初始化基类OneOf_。 + elif isinstance(patterns, (tuple, list)) and all(isinstance(pattern, Pattern) for pattern in patterns): + OneOf_.__init__(self, patterns) + #如果patterns是元组或列表,并且其中的所有元素都是mindspore.graph_utils.graph_pattern类的实例, + #则使用包含patterns中模式的列表初始化基类OneOf_ + else: + raise TypeError(f"Expect patterns to be a list of Patterns/Pattern, got : {patterns}") + #如果patterns是其他类型的对象,或者其中包含不是mindspore.graph_utils.graph_pattern类的实例, + #则抛出TypeError并附带相应的错误消息。 + + +class Prim(Prim_): + r""" + Express a pattern of certain primitive type(s). + 表示某种基本类型的模式 + + NOTE: + This pattern will match and only match the primitive value node. If matching primitive CNode is needed, + please refer to CallWith pattern. + """ + def __init__(self, types, name=None): + r""" + Args: + 支持三种不同的types参数输入方式: + types (Union[str, :class:`mindspore.ops.Primitive`, list[:class:`mindspore.ops.Primitive`], + tuple[:class:`mindspore.ops.Primitive`]): + Specify allowed types. + If it is a string, the form could be + 1) a single primitive type, e.g. 'Conv2D'若types是一个字符串,则可以是单个基本类型,例如 'Conv2D' + 2) a set of primitive types separated by '|', e.g. 'MatMul|Conv2D' + 多个基本类型,用|符号分隔,例如 'MatMul|Conv2D' + It can also be a Primitive or a list/tuple of Primitives, e.g. [ops.Conv2D(1, 6)] + 如果types是一个Primitive对象,则表示仅匹配该具体的基本类型。 + 如果types是一个列表或元组,并且列表中的元素都是Primitive对象,则表示匹配多个基本类型。 + name (str): name of the pattern, optional. Default: None. + + Raises: + TypeError: raise type error for invalid argument. + """ + # 如果提供了 name 参数,但不是字符串类型,则抛出 TypeError + if name is not None and not isinstance(name, str): + raise TypeError(f"Expect string, got : {name}") + # 将参数 name 赋值给对象的 name 属性 + self.name = name + # 根据 types 参数的类型进行不同的处理 + if isinstance(types, str): + if self.name is None: + self.name = types + self.types = types.split('|') + # 如果 types 是一个字符串,则将其按照 '|' 符号拆分,并存储在 self.types 中 + elif isinstance(types, Primitive): + # 如果 types 是一个 Primitive 对象,则表示只允许匹配这个具体的基本类型 + if self.name is None: + self.name = types.name + self.types = [types] + elif isinstance(types, (tuple, list)) and all(isinstance(tp, Primitive) for tp in types): + # 如果 types 是一个包含 Primitive 对象的列表或元组,则表示允许匹配多个基本类型 + if self.name is None: + self.name = "" + for prim in types: + self.name += prim.name + self.types = types + else: + # 如果 types 参数不是允许的类型,则抛出 TypeError + raise TypeError(f"Expecting a primitive type string or a list of Primitives, got : {types}") + # 调用父类 Prim_ 的初始化方法,将处理好的 types 和 name 作为参数传递给父类 + Prim_.__init__(self, self.types, self.name) + + +class Call(Call_): + #初始化对象的属性 + r""" + Express a primitive CNode. + """ + def __init__(self, prim_pattern, inputs=None): + r""" + Args: + prim_pattern (Union[str, :class:`mindspore.graph_utils.graph_pattern.IsPrimTypeOf`, + :class:`mindspore.ops.Primitive`]): Primitive ValueNode in the Primitive CNode. + inputs (Union[list[:class:`mindspore.graph_utils.graph_pattern`], + tuple[:class:`mindspore.graph_utils.graph_pattern`]]): + Specify inputs pattern for the primitive(s), optional. If None, accepts any inputs; if specified, input + patterns should be of right order and each element should be one of the exposed Pattern instance. + 模式的顺序应该正确,每个元素应该是公开的Pattern实例之一 + Raises: + TypeError: raise type error for invalid argument. + """ + #检查prim_pattern的类型是否是Pattern、Primitive或字符串类型。 + #如果不是这些类型之一,则抛出TypeError,表示期望prim_pattern是Pattern、Primitive或字符串类型。 + if not isinstance(prim_pattern, (Pattern, str, Primitive)): + raise TypeError(f"Expect prim_pattern to be Pattern, Primitive or string, got : {prim_pattern}") + #将传入的prim_pattern赋值给实例变量self.prim_pattern,以保存原语模式(Primitive Pattern)或原语名称(Primitive name) + self.prim_pattern = prim_pattern + #将self.inputs初始化为空列表 + self.inputs = [] + #None:什么都不做 + if inputs is None: + pass + #检查inputs是否为Pattern的列表或元组,且其中的所有元素都是Pattern类型。 + elif isinstance(inputs, (tuple, list)) and all(isinstance(input, Pattern) for input in inputs): + self.inputs = inputs + #如果inputs不满足上述条件,抛出TypeError,表示期望inputs是Pattern的列表。 + else: + raise TypeError(f"Expect inputs to be a list of Patterns, got : {inputs}") + Call_.__init__(self, self.prim_pattern, self.inputs) + #调用基类Call_的构造函数来完成初始化,传入self.prim_pattern作为原语模式或名称, + #以及self.inputs作为输入模式列表。 + +class NoneOf(NoneOf_): + r""" + Express a pattern which forbids a list of patterns. + 表达一个禁止模式列表的模式。 + NOTE: + NoneOf pattern should not be the root pattern. + #NoneOf模式不是根模式 + """ + def __init__(self, patterns=None): + r""" + Args: + patterns(Union[list[:class:`mindspore.graph_utils.graph_pattern`]]: list of forbidden patterns, each + element should be one of the exposed Pattern instance. + 禁用模式列表,每个元素应该是公开的Pattern实例之一。 + Raises: + TypeError: raise type error for invalid argument. + """ + self.patterns = patterns + if patterns is None: + NoneOf_.__init__(self, ()) + #如果 patterns 参数为 None,即未提供 patterns,则将 None 传递给 NoneOf_ 类的初始化方法。 + elif isinstance(patterns, Pattern): + NoneOf_.__init__(self, [patterns]) + #如果 patterns 参数是一个单独的 Pattern 对象,则将包含这个 Pattern 对象的列表传递给 NoneOf_ 类的初始化方法。 + elif isinstance(patterns, (tuple, list)) and all(isinstance(pattern, Pattern) for pattern in patterns): + NoneOf_.__init__(self, patterns) + #如果 patterns 参数是一个列表或元组,并且列表中的元素都是 Pattern 对象, + #则将整个列表传递给 NoneOf_ 类的初始化方法。 + else: + raise TypeError(f"Expect list of Patterns/Pattern, got : {patterns}") + + +class NewTensor(NewTensor_): + r""" + New Tensor to be used in the target. + 要在目标中使用的新张量 + """ + def __init__(self, input_tensor): + r""" + Args: + input_tensor(:class:`mindspore.common.tensor.Tensor`): new tensor to be used in the target. + 要在目标中使用的新张量 + Raises: + TypeError: raise type error for invalid argument. + """ + self.input_tensor = input_tensor + # 将传入的 input_tensor 参数赋值给对象的 input_tensor 属性 + if isinstance(input_tensor, Tensor): + # 检查 input_tensor 参数的类型 + NewTensor_.__init__(self, input_tensor) + # 如果 input_tensor 是一个 Tensor 对象,则调用 NewTensor_ 类的初始化方法,并传入 input_tensor 作为参数 + else: + raise TypeError(f"Expect input_tensor to be a Tensor, got : {input_tensor}") + # 如果 input_tensor 参数不是 Tensor 对象,则抛出 TypeError,指示输入参数类型错误 + + +class NewParameter(NewParameter_): + r""" + New Parameter to be used in the target. + """ + def __init__(self, para_name, default_tensor, requires_grad=False, layerwise_parallel=False): + r""" + Args: + para_name(str): name for the new Parameter. + default_tensor(:class:`mindspore.common.tensor.Tensor`): default value for the new Parameter. + requires_grad(bool): True if the parameter requires gradient. Default: True. + 如果参数需要梯度,则为True + layerwise_parallel(bool): switch for layerwise parallel mode. Default: False. + Layerwise_parallel (bool):分层并行模式开关。 + + Raises: + TypeError: raise type error for invalid argument. + """ + # 将传入的 para_name、default_tensor、requires_grad 和 layerwise_parallel 参数赋值给对象的相应属性 + self.para_name = para_name + self.default_tensor = default_tensor + self.requires_grad = requires_grad + self.layerwise_parallel = layerwise_parallel + # 检查传入的参数类型是否正确,并根据结果选择初始化父类 NewParameter_ + if isinstance(para_name, str) and isinstance(default_tensor, Tensor) and isinstance(requires_grad, bool) and\ + isinstance(layerwise_parallel, bool): + # 如果 para_name 是一个字符串,default_tensor 是一个 Tensor 对象,requires_grad 和 layerwise_parallel 都是布尔值, + # 则调用 NewParameter_ 类的初始化方法,并传入 para_name、default_tensor、requires_grad 和 layerwise_parallel 作为参数。 + NewParameter_.__init__(self, self.para_name, self.default_tensor, self.requires_grad, + self.layerwise_parallel) + else: + # 如果有任何一个参数类型不正确,则抛出 TypeError,指示输入参数类型错误。 + raise TypeError(f"Expect para_name(str), default_tensor(Tensor), requires_grad(bool), \ + layerwise_parallel(bool), got : {para_name}, {default_tensor}, \ + {requires_grad}, {layerwise_parallel}") -- 2.34.1 From 03afa7be681f3af3471b26d0a29c722b99f927cc Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:03:02 +0800 Subject: [PATCH 17/72] ADD file via upload --- .../ccsrc/transform-update/graph_runner.cc | 234 ++++++++++++++++++ 1 file changed, 234 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/graph_runner.cc diff --git a/mindspore/ccsrc/transform-update/graph_runner.cc b/mindspore/ccsrc/transform-update/graph_runner.cc new file mode 100644 index 00000000000..b6ded43aa63 --- /dev/null +++ b/mindspore/ccsrc/transform-update/graph_runner.cc @@ -0,0 +1,234 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + */ + +#include "include/transform/graph_ir/graph_runner.h" +#include +#include +#include + +#ifndef ENABLE_LITE_ACL +#include "pybind11/pybind11.h" +#endif +#include "utils/log_adapter.h" +#include "include/common/utils/config_manager.h" +#include "sys/time.h" +#include "include/common/utils/utils.h" +#include "include/common/utils/callbacks.h" +#ifdef ENABLE_D +#include "include/common/utils/callbacks_ge.h" +#endif +#include "utils/ms_context.h" + +#ifndef ENABLE_LITE_ACL +namespace py = pybind11; +#endif +namespace mindspore { +namespace transform { +// úڴµ GEGraphEngineỰ +// 'sess_options' ʾỰѡ +std::shared_ptr GraphRunner::NewSession(const SessionOptions &sess_options) { +#ifdef ENABLE_D + std::shared_ptr ret; // ڴ洢 GE Ựָ + auto ms_context = MsContext::GetInstance(); // ȡ MindSpore ʵ + MS_EXCEPTION_IF_NULL(ms_context); // ʵǷΪ + if (ms_context->backend_policy() == "ge") { // 鵱ǰĺ˲ǷΪ GE + ret = std::make_shared(sess_options); // һµ GE Ựʹôѡ 'sess_options' + if (ret == nullptr) { // GE Ựʧܣ׳쳣¼־ + MS_LOG(EXCEPTION) << "Create GE session failed!"; + } + MS_LOG(INFO) << "Create new GE session success!"; // ¼ɹ GE Ự־ + return ret; // ´ GE Ựָ + } +#endif + + MS_LOG(WARNING) << "no GE client, return nullptr!"; // û GE ˣ¼־ؿָ + return nullptr; // ؿָ룬ʾûд GE Ự +} + +// ù캯ڳʼGraphRunner +GraphRunner::GraphRunner(const GraphRunnerOptions &options) + : options_(options), graph_manager_(DfGraphManager::GetInstance()) { + // 鲢¼MindSporeIJвǷΪONE_DEVICE + if (ConfigManager::GetInstance().parallel_strategy() == ParallelStrategy::ONE_DEVICE) { + MS_LOG(INFO) << "ME run in ONE_DEVICE strategy mode"; + } + + if (options.sess_ptr != nullptr) { // optionsдsess_ptrжǷлỰ + sess_ = options.sess_ptr; + } else { // sess_ptrΪգNewSessionµGEỰ + sess_ = NewSession(options.options); + if (sess_ == nullptr) { + MS_LOG(WARNING) << "graph runner sess_ is nullptr!"; + } + } + +#ifdef ENABLE_D + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + // register the callback function + // עص + if (sess_->RegisterCallBackFunc(callbacks::kCheckPoint, callbacks::CheckpointSaveCallback) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "register callback failed!"; + } + + if (sess_->RegisterCallBackFunc(callbacks::kSummary, callbacks::SummarySaveCallback) != ge::GRAPH_SUCCESS) { + MS_LOG(EXCEPTION) << "register summary callback failed!"; + } + } +#endif + // ͼιȡеͼΰװ + std::vector wrappers = graph_manager_.GetAllGraphs(); + if (wrappers.empty()) { // ͼΰװΪգ¼־ֱӷ + MS_LOG(INFO) << "The GraphManager is empty!!"; + return; + } +#ifdef ENABLE_D + if (ms_context->backend_policy() != "ge") { + return; + } + + for (auto &it : wrappers) { // ͼΰװδͼӵGEỰ + std::set saved_graph = graph_manager_.GetSavedGraphs(); + auto iter_find = saved_graph.find(std::to_string(it->id_)); + if (iter_find != saved_graph.end()) { + continue; + } + MS_LOG(INFO) << "Add the graph " << (*it).name_ << " to GE, it's id is: " << (*it).id_; + graph_manager_.AddSavedGraphs(std::to_string(it->id_)); + (void)sess_->AddGraph(static_cast(it->id_), *(it->graph_ptr_), it->options_); + } +#endif +} + +// úָƵͼΣGraph +Status GraphRunner::RunGraph(const RunOptions &options, const std::vector &inputs, + std::vector *outputs) { + std::string name = options.name; // ȡѡеͼ + if (name.empty()) { // ͼΪգ¼־Ч״̬ + MS_LOG(ERROR) << "The graph name is null"; + return Status::INVALID_ARGUMENT; + } + // ͼιȡָƵͼΰװ + DfGraphWrapperPtr wrap_ptr = graph_manager_.GetGraphByName(name); + if (wrap_ptr == nullptr) { + MS_LOG(ERROR) << "Get graph form DfGraphManager failed!"; // ȡͼΰװʧܣ¼δҵ״̬Ĵ־ + return Status::NOT_FOUND; + } + + if (wrap_ptr->graph_ptr_ == nullptr) { // ͼΪգ¼־δҵ״̬ + MS_LOG(WARNING) << "The graph is null"; + return Status::NOT_FOUND; + } + + // call ge::RunGraph() to exec a graph; + // ge::RunGraph() ִͼμ + std::vector ge_inputs; + std::vector ge_outputs; + + // 'inputs' תΪ 'ge_inputs'ڵ GE ӿ + (void)std::transform(inputs.begin(), inputs.end(), std::back_inserter(ge_inputs), + [](const GeTensorPtr &i) { return *i; }); + + MS_LOG(INFO) << "Run the graph in GE with " << ge_inputs.size() << " inputs"; // ¼־ʾ GE ͼΣԼ + + struct timeval start_time, end_time; + (void)gettimeofday(&start_time, nullptr); + +#ifdef ENABLE_D + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + if (ms_context->backend_policy() == "ge") { + if (sess_ == nullptr) { + MS_LOG(ERROR) << "The GE session is null, can't run the graph!"; // GE ỰΪգ¼־ִʧ״̬ + return Status::FAILED; + } + ge::Status ret = sess_->RunGraph(static_cast(wrap_ptr->id_), ge_inputs, ge_outputs); // GE ӿͼ + if (ret != ge::GRAPH_SUCCESS) { + MS_LOG(ERROR) << "Call GE RunGraph Failed, ret is: " << ret; // ͼʧܣ¼־ִʧ״̬ + return Status::FAILED; + } + } +#else + ge_outputs.swap(ge_inputs); // δ GE ˣֱӽں +#endif + + (void)gettimeofday(&end_time, nullptr); + const uint64_t kUSecondInSecond = 1000000; + uint64_t cost = kUSecondInSecond * static_cast(end_time.tv_sec - start_time.tv_sec); + cost += static_cast(end_time.tv_usec - start_time.tv_usec); + MS_LOG(INFO) << "Call GE RunGraph Success in " << cost << " us, the GE outputs num is: " << ge_outputs.size(); + // ¼־ʾͼμɹִУӡִʱ + + // GE תΪ 'outputs'ڷظ + (void)std::transform(ge_outputs.begin(), ge_outputs.end(), std::back_inserter(*outputs), + [](const GeTensor &ge_tensor) { return std::make_shared(ge_tensor); }); + + return Status::SUCCESS; // ִгɹ״̬ +} + +// úָƵͼΣתΪ MeTensorPtr +Status GraphRunner::RunGraph(const RunOptions &options, const std::vector &inputs, + std::vector *const outputs) { + std::vector ge_inputs; // ڴ洢ת GeTensorPtr + for (auto it : inputs) { + MS_EXCEPTION_IF_NULL(it); + MS_LOG(INFO) << "inputs tensor's data size is: " << (*it).DataSize(); // ӡ MeTensor ݴС + auto shape = (*it).shape(); + std::string shape_str; + for (const auto &elem : shape) { + shape_str += std::to_string(elem); + shape_str += " "; + } + MS_LOG(INFO) << "inputs tensor's shape is: { " << shape_str << "}"; // ӡ MeTensor ״ + + // MeTensor תΪ GeTensorתĸʽΪ kOpFormat_NCHW + auto ge_tensor_ptr = TransformUtil::ConvertTensor(it, kOpFormat_NCHW); + if (ge_tensor_ptr != nullptr) { + ge_inputs.emplace_back(ge_tensor_ptr); // ת GeTensorPtr ӵ ge_inputs + } else { // תʧܣ¼־ִʧ״̬ + MS_LOG(INFO) << "Convert input Me tensor to Ge tensor failed. Abort this graph"; + return Status::FAILED; + } + } + + std::vector ge_outputs; // ڴ洢ͼκ GeTensorPtr + Status ret; + { + // Release GIL before calling into (potentially long-running) C++ code + // ͷ GILȻ C++ 루dzʱеĴ룩 +#ifndef ENABLE_LITE_ACL + py::gil_scoped_release release; +#endif + ret = RunGraph(options, ge_inputs, &ge_outputs); // RunGraph ͼμ + } + if (ret != Status::SUCCESS) { + return ret; // ͼʧܣֱӷִʧ״̬ + } else { + // convert GeTensor to MeTensor + // GeTensor תΪ MeTensorת MeTensorPtr ӵ outputs + for (auto &it : ge_outputs) { + auto tensor = TransformUtil::ConvertGeTensor(it); + if (tensor != nullptr) { + (void)outputs->emplace_back(tensor); + } + } + MS_LOG(INFO) << "Return Me tensor outputs num is: " << outputs->size(); // ӡص MeTensor + return Status::SUCCESS; // ִгɹ״̬ + } +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From 3caf864d7ed03faf35559128aa08079a6a6291f7 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:03:26 +0800 Subject: [PATCH 18/72] ADD file via upload --- .../transform-update/hcom_ops_declare.cc | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/hcom_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/hcom_ops_declare.cc b/mindspore/ccsrc/transform-update/hcom_ops_declare.cc new file mode 100644 index 00000000000..537a31fa10f --- /dev/null +++ b/mindspore/ccsrc/transform-update/hcom_ops_declare.cc @@ -0,0 +1,68 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/hcom_ops_declare.h" +#include + +namespace mindspore::transform { +// HCOMAllreduce +INPUT_MAP(HcomAllReduce) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +OUTPUT_MAP(HcomAllReduce) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +ATTR_MAP(HcomAllReduce) = {{"op", ATTR_DESC(reduction, AnyTraits())}, + {"group", ATTR_DESC(group, AnyTraits())}, + {"fusion", ATTR_DESC(fusion, AnyTraits())}}; +//属性映射,属性op类型为string,属性group类型为string,属性fusion类型为int64_t +REG_ADPT_DESC(HcomAllReduce, kNameAllReduce, ADPT_DESC(HcomAllReduce)) +//注册HcomAllReduce操作的适配器描述kNameAllReduce返回的name变量 + +// HCOMBraodcast +INPUT_MAP(HcomBroadcast) = EMPTY_INPUT_MAP; +//输入映射,设为空 +DYN_INPUT_MAP(HcomBroadcast) = {{1, DYN_INPUT_DESC(x)}}; +//动态输入映射,将索引为1的动态输入与名称为x的动态输入描述关联起来,用于后续操作 +DYN_OUTPUT_MAP(HcomBroadcast) = {{0, DYN_OUTPUT_DESC(y)}}; +//动态输出映射,将索引为0的动态输出与名称为y的动态输出描述关联起来,用于后续操作 +ATTR_MAP(HcomBroadcast) = {{"root_rank", ATTR_DESC(root_rank, AnyTraits())}, + {"group", ATTR_DESC(group, AnyTraits())}}; +//属性映射,属性root_rank类型为int64_t,属性group类型为string +REG_ADPT_DESC(HcomBroadcast, kNameBroadcast, ADPT_DESC(HcomBroadcast)) +//注册HcomBroadcast操作的适配器描述kNameBroadcast返回的name变量 + +// HcomAllGather +INPUT_MAP(HcomAllGather) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +OUTPUT_MAP(HcomAllGather) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +ATTR_MAP(HcomAllGather) = {{"group", ATTR_DESC(group, AnyTraits())}, + {"rank_size", ATTR_DESC(rank_size, AnyTraits())}}; +//属性映射,属性group类型为string,属性rank_size类型为int64_t +REG_ADPT_DESC(HcomAllGather, kNameAllgather, ADPT_DESC(HcomAllGather)) +//注册HcomAllGather操作的适配器描述kNameAllgather返回的name变量 + +// HCOMReduceScatter +INPUT_MAP(HcomReduceScatter) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +OUTPUT_MAP(HcomReduceScatter) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +ATTR_MAP(HcomReduceScatter) = {{"group", ATTR_DESC(group, AnyTraits())}, + {"op", ATTR_DESC(reduction, AnyTraits())}, + {"rank_size", ATTR_DESC(rank_size, AnyTraits())}}; +//属性映射,属性group类型为string,属性op类型为string>,属性rank_size类型为int64_t +REG_ADPT_DESC(HcomReduceScatter, kNameReduceScatter, ADPT_DESC(HcomReduceScatter)) +//注册HcomReduceScatter操作的适配器描述kNameReduceScatter返回的name变量 +} // namespace mindspore::transform -- 2.34.1 From cda2b4cfef33ad6c9c1a94bed3a111e8ad372a44 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:05:14 +0800 Subject: [PATCH 19/72] ADD file via upload --- .../transform-update/image_ops_declare.cc | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/image_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/image_ops_declare.cc b/mindspore/ccsrc/transform-update/image_ops_declare.cc new file mode 100644 index 00000000000..d70ba1622be --- /dev/null +++ b/mindspore/ccsrc/transform-update/image_ops_declare.cc @@ -0,0 +1,88 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/image_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// ResizeNearestNeighborV2D +INPUT_MAP(ResizeNearestNeighborV2D) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(ResizeNearestNeighborV2D) = { + {"size", ATTR_DESC(size, AnyTraits>(), AnyTraits>())}, + {"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +//属性映射,属性size类型为int64_t,属性align_corners类型为bool +OUTPUT_MAP(ResizeNearestNeighborV2D) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ResizeNearestNeighborV2D, kNameResizeNearestNeighborD, ADPT_DESC(ResizeNearestNeighborV2D)) +//注册ResizeNearestNeighborV2D操作的适配器描述kNameResizeNearestNeighborVD + +// ResizeNearestNeighborV2 +INPUT_MAP(ResizeNearestNeighborV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(size)}}; +//输入映射,x索引为1,size索引为2 +ATTR_MAP(ResizeNearestNeighborV2) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}, + {"half_pixel_centers", ATTR_DESC(half_pixel_centers, AnyTraits())}}; +//属性映射,属性align_corners类型为bool,属性half_pixel_centers类型为bool +OUTPUT_MAP(ResizeNearestNeighborV2) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ResizeNearestNeighborV2, kNameResizeNearestNeighborV2, ADPT_DESC(ResizeNearestNeighborV2)) +//注册ResizeNearestNeighborV2操作的适配器描述kNameResizeNearestNeighborV2 + +// ResizeNearestNeighborV2Grad +INPUT_MAP(ResizeNearestNeighborV2Grad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(size)}}; +//输入映射,grads索引为1,size索引为2 +ATTR_MAP(ResizeNearestNeighborV2Grad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +//属性映射,属性align_corners类型为bool +OUTPUT_MAP(ResizeNearestNeighborV2Grad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ResizeNearestNeighborV2Grad, kNameResizeNearestNeighborGrad, ADPT_DESC(ResizeNearestNeighborV2Grad)) +//注册ResizeNearestNeighborV2Grad操作的适配器描述kNameResizeNearestNeighborGrad + +// ResizeBilinearV2Grad +INPUT_MAP(ResizeBilinearV2Grad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(original_image)}}; +//输入映射,grads索引为1,original_image索引为2 +ATTR_MAP(ResizeBilinearV2Grad) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +//属性映射,属性align_corners类型为bool +OUTPUT_MAP(ResizeBilinearV2Grad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ResizeBilinearV2Grad, kNameResizeBilinearGrad, ADPT_DESC(ResizeBilinearV2Grad)) +//注册ResizeBilinearV2Grad操作的适配器描述kNameResizeBilinearV2Grad + +// ResizeBilinearV2 +INPUT_MAP(ResizeBilinearV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(size)}}; +//输入映射,x索引为1,size索引为2 +ATTR_MAP(ResizeBilinearV2) = {{"align_corners", ATTR_DESC(align_corners, AnyTraits())}}; +//属性映射,属性align_corners类型为bool +OUTPUT_MAP(ResizeBilinearV2) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(ResizeBilinearV2, kNameResizeBilinear, ADPT_DESC(ResizeBilinearV2)) +//注册ResizeBilinearV2操作的适配器描述kNameResizeBilinearV2 +REG_ADPT_DESC(ResizeBilinearV2New, kNameResizeBilinearV2, ADPT_DESC(ResizeBilinearV2)) +//注册ResizeBilinearV2New操作的适配器描述kNameResizeBilinearV2 + +// CropAndResize +INPUT_MAP(CropAndResize) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(boxes)}, {3, INPUT_DESC(box_index)}, {4, INPUT_DESC(crop_size)}}; + //输入映射,x索引为1,boxes索引为2,box_index索引为3,crop_size索引为4 +ATTR_MAP(CropAndResize) = {{"extrapolation_value", ATTR_DESC(extrapolation_value, AnyTraits())}, + {"method", ATTR_DESC(method, AnyTraits())}}; +//属性映射,属性extrapolation_value类型为float,属性method类型为string +OUTPUT_MAP(CropAndResize) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(CropAndResize, kNameCropAndResize, ADPT_DESC(CropAndResize)) +//注册CropAndResize操作的适配器描述kNameCropAndResize +} // namespace mindspore::transform -- 2.34.1 From 26de0688669c9213e6d33bacc2e97b4b30c47c25 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:05:47 +0800 Subject: [PATCH 20/72] ADD file via upload --- .../ccsrc/transform-update/io_format_map.cc | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/io_format_map.cc diff --git a/mindspore/ccsrc/transform-update/io_format_map.cc b/mindspore/ccsrc/transform-update/io_format_map.cc new file mode 100644 index 00000000000..1a4e0fa2098 --- /dev/null +++ b/mindspore/ccsrc/transform-update/io_format_map.cc @@ -0,0 +1,46 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/io_format_map.h" + +namespace mindspore { +namespace transform { +// һΪ 'IOFormatMap' +// ̬Ա 'io_format_map_'ڴ洢ʽ֮ӳϵ +// 'io_format_map_' ӳijʼֵ +mindspore::HashMap IOFormatMap::io_format_map_ = {{"BasicLSTMCell", "ND"}, + {"BasicLSTMCellInputGrad", "ND"}, + {"BasicLSTMCellCStateGrad", "ND"}, + {"Dequant", "ND"}, + {"DynamicGRUV2", "ND"}, + {"DynamicGRUV2Grad", "ND"}, + {"DynamicRNN", "ND"}, + {"DynamicRNNGrad", "ND"}, + {"MatMul", "ND"}, + {"BatchMatMul", "ND"}, + {"BatchMatMulV2", "ND"}, + {"Quant", "ND"}, + {"BasicLSTMCellWeightGrad", "HWCN"}, + {"ExtractImagePatches", "NCHW"}, + {"Conv3D", "format"}, + {"MaxPool3D", "NCDHW"}, + {"Conv3DBackpropFilter", "format"}, + {"Conv3DBackpropInput", "format"}, + {"Conv3DTranspose", "format"}}; +// ̬Ա 'get()'ڻȡ 'io_format_map_' ӳ +mindspore::HashMap &IOFormatMap::get() { return io_format_map_; } +} // namespace transform +} // namespace mindspore -- 2.34.1 From 9278c58e1dec0e8d457dda97ec83b5d3d7c6e1b8 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:06:11 +0800 Subject: [PATCH 21/72] ADD file via upload --- .../transform-update/logging_ops_declare.cc | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/logging_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/logging_ops_declare.cc b/mindspore/ccsrc/transform-update/logging_ops_declare.cc new file mode 100644 index 00000000000..dbf30650cc2 --- /dev/null +++ b/mindspore/ccsrc/transform-update/logging_ops_declare.cc @@ -0,0 +1,38 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/logging_ops_declare.h" + +namespace mindspore::transform { +// Print +INPUT_MAP(Print) = EMPTY_INPUT_MAP; +//输入映射,设为空 +DYN_INPUT_MAP(Print) = {{1, DYN_INPUT_DESC(x)}}; +//动态输入映射,将索引为1的动态输入与名称为x的动态输入描述关联起来,用于后续操作 +ATTR_MAP(Print) = EMPTY_ATTR_MAP; +//属性映射,设为空 +REG_ADPT_DESC(Print, kNamePrint, ADPT_DESC(Print)) +//注册Print操作的适配器描述kNamePrint + +INPUT_MAP(Assert) = {{1, INPUT_DESC(input_condition)}}; +//输入映射,input_condition索引为1 +DYN_INPUT_MAP(Assert) = {{2, DYN_INPUT_DESC(input_data)}}; +//动态输入映射,将索引为2的动态输入与名称为input_data的动态输入描述关联起来,用于后续操作 +ATTR_MAP(Assert) = {{"summarize", ATTR_DESC(summarize, AnyTraits())}}; +//属性映射,属性summarize类型为int64_t +REG_ADPT_DESC(Assert, kNameAssert, ADPT_DESC(Assert)) +//注册Assert操作的适配器描述kNameAssert +} // namespace mindspore::transform -- 2.34.1 From 49e7505fe6b3a0704dfc83e5a1648c83ef2972f3 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:06:41 +0800 Subject: [PATCH 22/72] ADD file via upload --- .../transform-update/math_ops_declare.cc | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/math_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/math_ops_declare.cc b/mindspore/ccsrc/transform-update/math_ops_declare.cc new file mode 100644 index 00000000000..d2401069cb3 --- /dev/null +++ b/mindspore/ccsrc/transform-update/math_ops_declare.cc @@ -0,0 +1,168 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/math_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// ActsULQ +INPUT_MAP(ActsULQ) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(clamp_min)}, {3, INPUT_DESC(clamp_max)}}; +//输入映射,x索引为1,clamp_min索引为2,clamp_max索引为3 +ATTR_MAP(ActsULQ) = {{"fixed_min", ATTR_DESC(fixed_min, AnyTraits())}, + {"num_bits", ATTR_DESC(num_bits, AnyTraits())}}; +//属性映射,属性fixed_min类型为bool,属性num_bits类型为int64_t +OUTPUT_MAP(ActsULQ) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(clamp_min_mask)}, + {2, OUTPUT_DESC(clamp_max_mask)}, + {3, OUTPUT_DESC(x_clamped_loss)}}; +//输出映射,y索引为0,clamp_min_mask索引为1,clamp_max_mask索引为2,x_clamped_loss索引为3 +REG_ADPT_DESC(ActsULQ, kNameActsULQ, ADPT_DESC(ActsULQ)) +//注册ActsULQ操作的适配器描述kNameActsULQ + +// ActsULQInputGrad +INPUT_MAP(ActsULQInputGrad) = { + {1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_min_mask)}, {3, INPUT_DESC(clamp_max_mask)}}; +//输入映射,y_grad索引为1,clamp_min_mask索引为2,clamp_max_mask索引为3 +ATTR_MAP(ActsULQInputGrad) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(ActsULQInputGrad) = {{0, OUTPUT_DESC(x_grad)}}; +//输出映射,x_grad索引为0 +REG_ADPT_DESC(ActsULQInputGrad, kNameActsULQInputGrad, ADPT_DESC(ActsULQInputGrad)) +//注册ActsULQInputGrad操作的适配器描述kNameActsULQInputGrad + +// ActULQClampMaxGrad +INPUT_MAP(ActULQClampMaxGrad) = { + {1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_max_mask)}, {3, INPUT_DESC(x_clamped_loss)}}; +//输入映射,y_grad索引为1,clamp_max_mask索引为2,x_clamped_loss索引为3 +ATTR_MAP(ActULQClampMaxGrad) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(ActULQClampMaxGrad) = {{0, OUTPUT_DESC(clamp_max_grad)}}; +//输出映射,clamp_max_grad索引为0 +REG_ADPT_DESC(ActULQClampMaxGrad, kNameActULQClampMaxGrad, ADPT_DESC(ActULQClampMaxGrad)) +//注册ActsULQClampMaxGrad操作的适配器描述kNameActsULQClampMaxGrad + +// ActULQClampMinGrad +INPUT_MAP(ActULQClampMinGrad) = { + {1, INPUT_DESC(y_grad)}, {2, INPUT_DESC(clamp_min_mask)}, {3, INPUT_DESC(x_clamped_loss)}}; +//输入映射,y_grad索引为1,clamp__min_mask索引为2,x_clamped_loss索引为3 +ATTR_MAP(ActULQClampMinGrad) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(ActULQClampMinGrad) = {{0, OUTPUT_DESC(clamp_min_grad)}}; +//输出映射,clamp_min_grad索引为0 +REG_ADPT_DESC(ActULQClampMinGrad, kNameActULQClampMinGrad, ADPT_DESC(ActULQClampMinGrad)) +//注册ActsULQClampMinGrad操作的适配器描述kNameActsULQClampMinGrad + +// HistogramFixedWidthD +INPUT_MAP(HistogramFixedWidthD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(range)}}; +//输入映射,x索引为1,range)索引为2 +ATTR_MAP(HistogramFixedWidthD) = {{"nbins", ATTR_DESC(nbins, AnyTraits())}, + {"dtype", ATTR_DESC(dtype, AnyTraits())}}; +//属性映射,属性nbins类型为int64_t,属性dtype类型为int64_t +OUTPUT_MAP(HistogramFixedWidthD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(HistogramFixedWidthD, kNameHistogramFixedWidthD, ADPT_DESC(HistogramFixedWidthD)) +//注册HistogramFixedWidthD操作的适配器描述kNameHistogramFixedWidthD + +// IFMR +INPUT_MAP(IFMR) = { + {1, INPUT_DESC(data)}, {2, INPUT_DESC(data_min)}, {3, INPUT_DESC(data_max)}, {4, INPUT_DESC(cumsum)}}; +//输入映射,data索引为1,data_min索引为2,data_max索引为3,cumsum索引为4 +ATTR_MAP(IFMR) = {{"min_percentile", ATTR_DESC(min_percentile, AnyTraits())}, + {"max_percentile", ATTR_DESC(max_percentile, AnyTraits())}, + {"search_range", ATTR_DESC(search_range, AnyTraits>())}, + {"search_step", ATTR_DESC(search_step, AnyTraits())}}; +//属性映射,属性min_percentile类型为float,属性max_percentile类型为float,属性search_range类型为float,属性search_step类型为float +OUTPUT_MAP(IFMR) = {{0, OUTPUT_DESC(scale)}, {1, OUTPUT_DESC(offset)}}; +//输出映射,scale索引为0,offset索引为1 +REG_ADPT_DESC(IFMR, kNameIFMR, ADPT_DESC(IFMR)) +//注册IFMR操作的适配器描述kNameIFMR + +// NLLLoss +INPUT_MAP(NLLLoss) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(weight)}}; +//输入映射,x索引为1,target索引为2,weight索引为3 +ATTR_MAP(NLLLoss) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +//属性映射,属性reduction类型为string +OUTPUT_MAP(NLLLoss) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(total_weight)}}; +//输出映射,y索引为0,total_weight索引为1 +REG_ADPT_DESC(NLLLoss, kNameNLLLoss, ADPT_DESC(NLLLoss)) +//注册NLLLoss操作的适配器描述kNameNLLLoss + +// NLLLossGrad +INPUT_MAP(NLLLossGrad) = {{1, INPUT_DESC(x)}, + {2, INPUT_DESC(y_grad)}, + {3, INPUT_DESC(target)}, + {4, INPUT_DESC(weight)}, + {5, INPUT_DESC(total_weight)}}; +//输入映射,x索引为1,y_grad索引为2,target索引为3,weight索引为4,total_weight索引为5 +ATTR_MAP(NLLLossGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +//属性映射,属性reduction类型为string +OUTPUT_MAP(NLLLossGrad) = {{0, OUTPUT_DESC(x_grad)}}; +//输出映射,x_grad索引为0 +REG_ADPT_DESC(NLLLossGrad, kNameNLLLossGrad, ADPT_DESC(NLLLossGrad)) +//注册NLLLosGrad操作的适配器描述kNameNLLLossGrad + +// Erf +INPUT_MAP(Erf) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Erf) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(Erf) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Erf, kNameErf, ADPT_DESC(Erf)) +//注册Erf操作的适配器描述kNameErf + +// Erfc +INPUT_MAP(Erfc) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Erfc) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(Erfc) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Erfc, kNameErfc, ADPT_DESC(Erfc)) +//注册Erfc操作的适配器描述kNameErfc + +// WtsARQ +INPUT_MAP(WtsARQ) = {{1, INPUT_DESC(w)}, {2, INPUT_DESC(w_min)}, {3, INPUT_DESC(w_max)}}; +//输入映射,x索引为1,w_min索引为2,w_max索引为3 +ATTR_MAP(WtsARQ) = {{"num_bits", ATTR_DESC(num_bits, AnyTraits())}, + {"offset_flag", ATTR_DESC(offset_flag, AnyTraits())}}; +//输入映射,num_bits索引为1,offset_flag索引为2 +OUTPUT_MAP(WtsARQ) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(WtsARQ, kNameWtsARQ, ADPT_DESC(WtsARQ)) +//注册WtsARQ操作的适配器描述kNameWtsARQ + +// IsFinite +INPUT_MAP(IsFinite) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(IsFinite) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(IsFinite) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(IsFinite, kNameIsFinite, ADPT_DESC(IsFinite)) +//注册IsFinite操作的适配器描述kNameIsFinite + +// IsNan +INPUT_MAP(IsNan) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(IsNan) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(IsNan) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(IsNan, kNameIsNan, ADPT_DESC(IsNan)) +//注册IsNan操作的适配器描述kNameIsNan +} // namespace mindspore::transform -- 2.34.1 From ea93d3ff4033715c0b8a18ed94cdbb9051e204bd Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:07:31 +0800 Subject: [PATCH 23/72] ADD file via upload --- .../matrix_calculation_ops_declare.cc | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/matrix_calculation_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/matrix_calculation_ops_declare.cc b/mindspore/ccsrc/transform-update/matrix_calculation_ops_declare.cc new file mode 100644 index 00000000000..45a56f49105 --- /dev/null +++ b/mindspore/ccsrc/transform-update/matrix_calculation_ops_declare.cc @@ -0,0 +1,160 @@ +/** + * Copyright 2019-2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/matrix_calculation_ops_declare.h" + +namespace mindspore::transform { +// TensorScatterUpdate +INPUT_MAP(TensorScatterUpdate) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(TensorScatterUpdate) = EMPTY_ATTR_MAP; +OUTPUT_MAP(TensorScatterUpdate) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(TensorScatterUpdate, kNameTensorScatterUpdate, ADPT_DESC(TensorScatterUpdate)) + +// ScatterUpdate +INPUT_MAP(ScatterUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterUpdate) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterUpdate, kNameScatterUpdate, ADPT_DESC(ScatterUpdate)) + +// ScatterNdUpdate +INPUT_MAP(ScatterNdUpdate) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdUpdate) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdUpdate) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterNdUpdate, kNameScatterNdUpdate, ADPT_DESC(ScatterNdUpdate)) + +// ScatterMax +INPUT_MAP(ScatterMax) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterMax) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterMax) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterMax, kNameScatterMax, ADPT_DESC(ScatterMax)) + +// ScatterMin +INPUT_MAP(ScatterMin) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterMin) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterMin) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterMin, kNameScatterMin, ADPT_DESC(ScatterMin)) + +// ScatterAdd +INPUT_MAP(ScatterAdd) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterAdd) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterAdd) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterAdd, kNameScatterAdd, ADPT_DESC(ScatterAdd)) + +// ScatterSub +INPUT_MAP(ScatterSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterSub) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterSub) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterSub, kNameScatterSub, ADPT_DESC(ScatterSub)) + +// ScatterMul +INPUT_MAP(ScatterMul) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterMul) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterMul) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterMul, kNameScatterMul, ADPT_DESC(ScatterMul)) + +// ScatterDiv +INPUT_MAP(ScatterDiv) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterDiv) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterDiv) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterDiv, kNameScatterDiv, ADPT_DESC(ScatterDiv)) + +// ScatterNdAdd +INPUT_MAP(ScatterNdAdd) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdAdd) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdAdd) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterNdAdd, kNameScatterNdAdd, ADPT_DESC(ScatterNdAdd)) + +// ScatterNdSub +INPUT_MAP(ScatterNdSub) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterNdSub) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +OUTPUT_MAP(ScatterNdSub) = {{0, OUTPUT_DESC(var)}}; +REG_ADPT_DESC(ScatterNdSub, kNameScatterNdSub, ADPT_DESC(ScatterNdSub)) + +// MatMul +INPUT_MAP(MatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(MatMul) = {{"transpose_x1", ATTR_DESC(transpose_x1, AnyTraits())}, + {"transpose_x2", ATTR_DESC(transpose_x2, AnyTraits())}}; +OUTPUT_MAP(MatMul) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatMul, kNameMatMul, ADPT_DESC(MatMul)) + +// MatMulV2 +INPUT_MAP(MatMulV2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(bias)}}; +ATTR_MAP(MatMulV2) = {{"transpose_a", ATTR_DESC(transpose_x1, AnyTraits())}, + {"transpose_b", ATTR_DESC(transpose_x2, AnyTraits())}}; +OUTPUT_MAP(MatMulV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatMulV2, prim::kPrimMatMul->name(), ADPT_DESC(MatMulV2)) + +// MatrixDiag +INPUT_MAP(MatrixDiag) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(MatrixDiag) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MatrixDiag) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatrixDiag, kNameMatrixDiagD, ADPT_DESC(MatrixDiag)) + +// MatrixDiagPartD +INPUT_MAP(MatrixDiagPartD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(assist)}}; +ATTR_MAP(MatrixDiagPartD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MatrixDiagPartD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatrixDiagPartD, kNameMatrixDiagPartD, ADPT_DESC(MatrixDiagPartD)) + +// MatrixSetDiagD +INPUT_MAP(MatrixSetDiagD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(diagonal)}, {3, INPUT_DESC(assist)}}; +ATTR_MAP(MatrixSetDiagD) = EMPTY_ATTR_MAP; +OUTPUT_MAP(MatrixSetDiagD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(MatrixSetDiagD, kNameMatrixSetDiagD, ADPT_DESC(MatrixSetDiagD)) + +// DiagPart +INPUT_MAP(DiagPart) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(DiagPart) = EMPTY_ATTR_MAP; +OUTPUT_MAP(DiagPart) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(DiagPart, kNameDiagPart, ADPT_DESC(DiagPart)) + +// BatchMatMul +INPUT_MAP(BatchMatMul) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BatchMatMul) = {{"transpose_x1", ATTR_DESC(adj_x1, AnyTraits())}, + {"transpose_x2", ATTR_DESC(adj_x2, AnyTraits())}}; +OUTPUT_MAP(BatchMatMul) = {{0, OUTPUT_DESC(y)}}; + +// BatchMatMul->BatchMatMulV2 +INPUT_MAP(BatchMatMulV2) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +ATTR_MAP(BatchMatMulV2) = {{"transpose_x1", ATTR_DESC(adj_x1, AnyTraits())}, + {"transpose_x2", ATTR_DESC(adj_x2, AnyTraits())}}; +OUTPUT_MAP(BatchMatMulV2) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(BatchMatMul, kNameBatchMatMul, ADPT_DESC(BatchMatMul)) +REG_ADPT_DESC(BatchMatMulV2, kNameBatchMatMulV2, ADPT_DESC(BatchMatMulV2)) + +// L2Loss +INPUT_MAP(L2Loss) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(L2Loss) = EMPTY_ATTR_MAP; +OUTPUT_MAP(L2Loss) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(L2Loss, kNameL2Loss, ADPT_DESC(L2Loss)) + +// ScatterElements +INPUT_MAP(ScatterElements) = {{1, INPUT_DESC(data)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +ATTR_MAP(ScatterElements) = {{"axis", ATTR_DESC(axis, AnyTraits())}}; +OUTPUT_MAP(ScatterElements) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ScatterElements, kNameTensorScatterElements, ADPT_DESC(ScatterElements)) + +// FullyConnection +INPUT_MAP(FullyConnection) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, {4, INPUT_DESC(offset_w)}}; + +ATTR_MAP(FullyConnection) = {{"num_output", ATTR_DESC(num_output, AnyTraits())}, + {"transpose", ATTR_DESC(transpose, AnyTraits())}, + {"axis", ATTR_DESC(axis, AnyTraits())}, + {"offset_x", ATTR_DESC(offset_x, AnyTraits())}}; + +OUTPUT_MAP(FullyConnection) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(FullyConnection, kNameFullConnection, ADPT_DESC(FullyConnection)) +} // namespace mindspore::transform -- 2.34.1 From eaaecac92cf7f86f53e57b16a9124c7e06dfdcde Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:08:38 +0800 Subject: [PATCH 24/72] ADD file via upload --- .../ccsrc/transform-update/mindir_exporter.cc | 1311 +++++++++++++++++ 1 file changed, 1311 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/mindir_exporter.cc diff --git a/mindspore/ccsrc/transform-update/mindir_exporter.cc b/mindspore/ccsrc/transform-update/mindir_exporter.cc new file mode 100644 index 00000000000..a2e6b397df6 --- /dev/null +++ b/mindspore/ccsrc/transform-update/mindir_exporter.cc @@ -0,0 +1,1311 @@ +/** + * Copyright 2020-2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "utils/hash_map.h" +#include "ir/tensor.h" +#include "ir/param_info.h" +#include "ir/func_graph.h" +#include "mindspore/core/ops/core_ops.h" +#include "proto/mind_ir.pb.h" +#include "utils/check_convert_utils.h" +#include "include/common/debug/dump_proto.h" +#include "utils/ms_utils.h" +#include "include/common/utils/utils.h" +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP +#include "frontend/parallel/tensor_layout/tensor_layout.h" +#endif +#include "abstract/abstract_function.h" +#include "mindspore/core/utils/file_utils.h" + +namespace mindspore { +using FloatPtr = std::shared_ptr; +using IntPtr = std::shared_ptr; +using UIntPtr = std::shared_ptr; +using ModelProtoPtr = std::shared_ptr; + +// anf type to mindir type map将 ANF 类型映射到 MindIR 类型的映射表 +static mindspore::HashMap g_data_type_map = { + {kNumberTypeBool, mind_ir::TensorProto_DataType_BOOL}, + {kNumberTypeInt8, mind_ir::TensorProto_DataType_INT8}, + {kNumberTypeInt16, mind_ir::TensorProto_DataType_INT16}, + {kNumberTypeInt32, mind_ir::TensorProto_DataType_INT32}, + {kNumberTypeInt64, mind_ir::TensorProto_DataType_INT64}, + {kNumberTypeUInt8, mind_ir::TensorProto_DataType_UINT8}, + {kNumberTypeUInt16, mind_ir::TensorProto_DataType_UINT16}, + {kNumberTypeUInt32, mind_ir::TensorProto_DataType_UINT32}, + {kNumberTypeUInt64, mind_ir::TensorProto_DataType_UINT64}, + {kNumberTypeFloat16, mind_ir::TensorProto_DataType_FLOAT16}, + {kNumberTypeFloat32, mind_ir::TensorProto_DataType_FLOAT}, + {kNumberTypeFloat64, mind_ir::TensorProto_DataType_DOUBLE}, + {kObjectTypeString, mind_ir::TensorProto_DataType_STRING}, + {kNumberTypeComplex64, mind_ir::TensorProto_DataType_COMPLEX64}, + {kNumberTypeComplex128, mind_ir::TensorProto_DataType_COMPLEX128}}; + +static mindspore::HashMap g_data_bits_int_map = { + {8, mind_ir::TensorProto_DataType_INT8}, + {16, mind_ir::TensorProto_DataType_INT16}, + {32, mind_ir::TensorProto_DataType_INT32}, + {64, mind_ir::TensorProto_DataType_INT64}, +}; + +static mindspore::HashMap g_data_bits_uint_map = { + {8, mind_ir::TensorProto_DataType_UINT8}, + {16, mind_ir::TensorProto_DataType_UINT16}, + {32, mind_ir::TensorProto_DataType_UINT32}, + {64, mind_ir::TensorProto_DataType_UINT64}, +}; + +static mindspore::HashMap g_data_bits_float_map = { + {16, mind_ir::TensorProto_DataType_FLOAT16}, + {32, mind_ir::TensorProto_DataType_FLOAT}, + {64, mind_ir::TensorProto_DataType_FLOAT64}, +}; + +static std::set g_export_attr_blacklist = {kAttrDump}; + +// Can build different builder according to format根据格式构建不同的生成器。 +class IrExportBuilder; +using IrExportBuilderPtr = std::shared_ptr; +//使用IrExportBuilderPtr表示std::shared_ptr + +class IrExporter { + public: + explicit IrExporter(IrExportBuilderPtr builder) : builder_(std::move(builder)) {} + virtual ~IrExporter() = default; + std::string GetDumpString(const FuncGraphPtr &func_graph); + ModelProtoPtr GetDumpProto(const FuncGraphPtr &func_graph, const FuncGraphPtr ¶m_layout_fg = nullptr); + + private: + IrExportBuilderPtr builder_; +}; +//声明IrExporter类 +using IrExporterPtr = std::shared_ptr; + +class IrExportBuilder { + public: + IrExportBuilder() : model_(std::make_shared()) {} + ~IrExportBuilder() = default; + std::string GetProtoString() const; + void BuildModelInfo(); + bool BuildModel(const FuncGraphPtr &func_graph); + ModelProtoPtr Model() { return model_; } + +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP + void BuildLayout(const FuncGraphPtr &func_graph); +#endif +//如果没有定义MINDIR_EXPORT_TENSOR_LAYOUT_CLIP,则会定义一个名为BuildLayout的函数 + + bool BuildFuncGraph(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); + bool BuildFuncGraphAttrs(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); + bool BuildParameters(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); + bool BuildNodes(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto); + bool BuildOutput(const CNodePtr &node, mind_ir::GraphProto *const graph_proto); + bool BuildCNode(const CNodePtr &node, mind_ir::GraphProto *const graph_proto); + std::string BuildInputNode(const AnfNodePtr &node, mind_ir::GraphProto *const graph_proto); + + bool SetValueInfoProto(const AnfNodePtr &node, mind_ir::ValueInfoProto *const value_proto); + bool SetParamToTensorProto(const ParameterPtr ¶m, mind_ir::TensorProto *const tensor_proto); + bool SetTensorProto(const AbstractBasePtr &abstract, mind_ir::TensorProto *const tensor_proto); + bool SetCSRTensorToProto(const AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto); + bool SetCOOTensorToProto(const AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto); + bool SetAttributeProto(const AnfNodePtr &node, mind_ir::NodeProto *const node_proto); + bool SetAbstractToNodeProto(const CNodePtr &node, mind_ir::NodeProto *const node_proto); + bool SetAbstractToNodeProto(const abstract::AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto); + bool SetValueToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetTypeToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetScalarToAttributeProto_ir(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) const; + bool SetScalarToAttributeProtoForInt_ir(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) const; + bool SetScalarToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) const; + bool SetScalarToAttributeProtoForInt_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) const; + bool SetTypeToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetTensorToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetSequenceToAttributeProto(const ValueSequencePtr &value, mind_ir::AttributeProto *const attr_proto); + bool SetSeqElemToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto); + + mind_ir::TensorProto_DataType GetMindirDataType(TypeId type_id) const; + mind_ir::TensorProto_DataType GetMindirDataBitsIntType(int bits) const; + mind_ir::TensorProto_DataType GetMindirDataBitsFloatType(int bits) const; + mind_ir::TensorProto_DataType GetMindirDataBitsUIntType(int bits) const; + std::string GetNodeName(const AnfNodePtr &node) const; + std::string GetUniqueNodeName(const AnfNodePtr &node); + std::string GetOpTypeName(const AnfNodePtr &node); + size_t GetUniqueID() { return ++unique_id_; } + + private: + bool SetAbstractFuncToAttributeProto(const abstract::AbstractBasePtr &abstract, + mind_ir::AttributeProto *const attr_proto); + std::string GetPrimitiveUniqueName(const PrimitivePtr &primitive_ptr); + bool BuildPrimitives(); + + ModelProtoPtr model_; + mind_ir::NodeProto *last_node_{nullptr}; + std::list todo_; + std::map node_name_map_; + std::map primitive_name_map_; + std::set nodeName_; + size_t unique_id_{0}; + bool top_graph{true}; +}; +//声明IrExportBuilder类 + +bool IrExportBuilder::SetAbstractFuncToAttributeProto(const abstract::AbstractBasePtr &abstract, + mind_ir::AttributeProto *const attr_proto) { + MS_EXCEPTION_IF_NULL(abstract); + MS_EXCEPTION_IF_NULL(attr_proto); + //如果abstract、attr_proto为空,则抛出异常 + if (abstract->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_FUNCGRAPHCLOSURE); + auto func_name = abstract->cast()->func_graph()->ToString(); + attr_proto->set_s(func_name); + } else if (abstract->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_PRIMITIVECLOSURE); + auto prim = abstract->cast()->prim(); + attr_proto->set_s(GetPrimitiveUniqueName(prim)); + } else if (abstract->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_PARTIALCLOSURE); + auto node_ptr = abstract->cast()->node(); + MS_EXCEPTION_IF_NULL(node_ptr); + attr_proto->set_s(GetUniqueNodeName(node_ptr)); + } else if (abstract->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UNIONFUNCCLOSURE); + auto visit_func = [this, &attr_proto](const abstract::AbstractFuncAtomPtr &poss) { + auto element_attr_proto = attr_proto->add_values(); + if (!this->SetAbstractFuncToAttributeProto(poss, element_attr_proto)) { + MS_LOG(EXCEPTION) << "Set union function abstract to proto error." << poss->ToString(); + } + }; + abstract->cast()->Visit(visit_func); + } else { + MS_LOG(ERROR) << "The parameter abstract is not an abstractFunction: " << abstract->ToString(); + return false; + } + return true; +} +//实例化IrExportBuilder类中的函数SetAbstractFuncToAttributeProto +//根据abstract变量下的各项事例是否存在,设置attr_proto的类型为对应名称,然后从abstract变量中获取对应类型的指针,进而获取其对应的函数图名称 +//并将该名称设置为 attr_proto的字符串属性,并返回为真。若没有对应事例则抛出异常并报错,返回为假。 + +std::string IrExportBuilder::GetPrimitiveUniqueName(const PrimitivePtr &primitive_ptr) { + auto it = primitive_name_map_.find(primitive_ptr); + if (it != primitive_name_map_.end()) { + return it->second; + } + // Remove this check if we find a way to handle save/load training model with flattened parameters. + if (IsPrimitiveEquals(primitive_ptr, prim::kPrimFlattenConcat)) { + MS_LOG(EXCEPTION) << "Export model with operator '" << primitive_ptr->name() << "' is not supported yet.\n" + << "Please remove 'net.flatten_weights()' in your script and try again."; + } + auto answer = primitive_ptr->name() + ":" + std::to_string(GetUniqueID()); + primitive_name_map_[primitive_ptr] = answer; + return answer; +} +//实例化IrExportBuilder类中的函数GetPrimitiveUniqueName +//如果找到一种方法来处理具有扁平参数的保存/加载训练模型,请删除此检查 + +bool IrExportBuilder::BuildPrimitives() { + // 遍历 primitive_name_map_ 中的每个原语 + for (auto it = primitive_name_map_.begin(); it != primitive_name_map_.end(); ++it) { + auto prim_proto = model_->add_primitives(); + auto prim = it->first; + prim_proto->set_name(it->second); + prim_proto->set_op_type(prim->name()); + // 获取实际的原语(可能存在原语的包装) + auto real_prim = GetValueWithoutDoSignature(prim)->cast(); + if (real_prim != nullptr) { + prim = real_prim; + } + + // Set primitive attributes遍历设置原语的属性 + for (const auto &attr : prim->attrs()) { + // 检查当前属性是否在黑名单中,如果是则跳过 + MS_LOG(DEBUG) << "attr: " << attr.first << " " << attr.second->DumpText() << " " << attr.second->type_name(); + auto iter = g_export_attr_blacklist.find(attr.first); + if (iter != g_export_attr_blacklist.end()) { + continue; + } + // 向原语的 proto 中添加一个属性 + mind_ir::AttributeProto *attr_proto = prim_proto->add_attribute(); + attr_proto->set_name(attr.first); + auto attr_value = attr.second; + // 转换并检查属性值 + CheckAndConvertUtils::ConvertAttrValueInExport(prim->name(), attr.first, &attr_value); + if (!SetValueToAttributeProto(attr_value, attr_proto)) { + MS_LOG(ERROR) << "Set value to AttributeProto failed."; + return false; + } + } // Loop of attrs + } // Loop of primitives + return true; +} + +std::string IrExporter::GetDumpString(const FuncGraphPtr &func_graph) { + auto dump_proto = GetDumpProto(func_graph); + if (dump_proto == nullptr) { + MS_LOG(EXCEPTION) << "Get dump proto for graph " << func_graph->ToString() << " failed."; + } + return builder_->GetProtoString(); +} +//获取DumpString + +ModelProtoPtr IrExporter::GetDumpProto(const FuncGraphPtr &func_graph, const FuncGraphPtr ¶m_layout_fg) { + if ((builder_ == nullptr) || (func_graph == nullptr)) { + MS_LOG(EXCEPTION) << "Input params is null."; + } + + // Export model info + builder_->BuildModelInfo(); + + // Export model and return string + if (!builder_->BuildModel(func_graph)) { + return nullptr; + } + +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP + // Export layout information + if (param_layout_fg) { + builder_->BuildLayout(param_layout_fg); + } +#endif + return builder_->Model(); +} + +std::string IrExportBuilder::GetProtoString() const { + MS_LOG(DEBUG) << "BuildModel complete!"; + return model_->SerializeAsString(); +} + +void IrExportBuilder::BuildModelInfo() { + // 构建模型信息 + constexpr auto ir_version = "0.1.1"; + constexpr auto mindspore_name = "MindSpore"; + model_->set_ir_version(ir_version);// 设置IR版本 + model_->set_producer_name(mindspore_name);// 设置生产者名称 + model_->set_model_version(VERSION);// 设置模型版本 + model_->set_little_endian(common::IsLittleByteOrder());// 设置字节序 + model_->set_mind_ir_version(mind_ir::Version_MAX);// 设置Mind IR版本 +} + +#ifndef MINDIR_EXPORT_TENSOR_LAYOUT_CLIP +void IrExportBuilder::BuildLayout(const FuncGraphPtr &func_graph) { + // 构建张量布局信息 + MS_EXCEPTION_IF_NULL(func_graph); + std::vector graph_params = func_graph->parameters();// 获取图的参数节点 + mind_ir::ParallelProto *parallel_proto = model_->mutable_parallel();// 获取模型的并行信息 + // 遍历图的参数节点 + for (auto para : graph_params) { + std::string name = std::static_pointer_cast(para)->name();// 获取参数节点的名称 + auto tensor_layout = para->user_data();// 获取参数节点的张量布局信息 + if (tensor_layout == nullptr) { + MS_LOG(INFO) << "GetParameterLayout nullptr name = " << name; + } else { + mind_ir::LayoutProto *layoutProto = parallel_proto->add_layout();// 添加张量布局信息到模型的并行信息中 + + // Get all the information for layput + // 获取张量布局的各种信息 + auto device_arrangement = tensor_layout->device_arrangement().array(); + auto tensor_map = tensor_layout->tensor_map().array(); + auto slice_shape = tensor_layout->slice_shape().array(); + int64_t field_size = tensor_layout->get_field_size(); + bool uniform_split = tensor_layout->uniform_split(); + std::string opt_shard_group = tensor_layout->opt_shard_group(); + + // Save all information to Layout Proto + // 将信息保存到布局信息中 + layoutProto->set_name(name); + for (auto device_arrangement_element : device_arrangement) { + layoutProto->add_device_arrangement_int(device_arrangement_element); + } + for (auto tensor_map_element : tensor_map) { + layoutProto->add_tensor_map_int(tensor_map_element); + } + for (auto slice_shape_element : slice_shape) { + layoutProto->add_slice_shape_int(slice_shape_element); + } + layoutProto->set_field_size(field_size); + layoutProto->set_uniform_split(uniform_split); + layoutProto->set_opt_shard_group(opt_shard_group); + } + } +} +#endif + +bool IrExportBuilder::BuildModel(const FuncGraphPtr &func_graph) { + // 构建模型的函数 + MS_EXCEPTION_IF_NULL(func_graph);// 检查输入函数图是否为空 + // 清空待办列表、节点名称集合和原语名称映射 + mind_ir::GraphProto *graph_proto = model_->mutable_graph(); + graph_proto->set_name(func_graph->ToString()); + graph_proto->set_bprop_hash(func_graph->bprop_hash()); + // 清空待办列表、节点名称集合和原语名称映射 + todo_.clear(); + nodeName_.clear(); + primitive_name_map_.clear(); + // Build the main funcGraph + // 构建主函数图 + // 将主函数图名称添加到节点名称集合 + (void)nodeName_.insert(func_graph->ToString()); + top_graph = true; + if (!BuildFuncGraph(func_graph, graph_proto)) { + MS_LOG(ERROR) << "Build func_graph " << func_graph->ToString() << " failed."; + return false; + } + + // Build child funcGraphs + // 构建子函数图 + std::set graphVisited; + (void)graphVisited.insert(func_graph); + top_graph = false; + while (!todo_.empty()) { + // 从待办列表中取出一个函数图 + FuncGraphPtr fg = todo_.back(); + todo_.pop_back(); + // 如果函数图已经被访问过,则继续处理下一个函数图 + if (graphVisited.count(fg) > 0) { + continue; + } + // 检查节点名称是否重复,如果重复则报错 + if (nodeName_.count(fg->ToString()) > 0) { + MS_LOG(ERROR) << "There is a duplicate name: " << fg->ToString(); + return false; + } + // 将函数图名称添加到节点名称集合和已访问的函数图集合 + (void)nodeName_.insert(fg->ToString()); + (void)graphVisited.insert(fg); + // 创建一个新的函数图对象,并构建该函数图 + auto graph = model_->add_functions(); + if (!BuildFuncGraph(fg, graph)) { + MS_LOG(ERROR) << "Build func_graph " << fg->ToString() << " failed."; + return false; + } + } + // 构建原语信息 + if (!BuildPrimitives()) { + return false; + } + // Release resource + // 释放资源,清空节点名称集合、节点名称映射和原语名称映射 + nodeName_.clear(); + node_name_map_.clear(); + primitive_name_map_.clear(); + return true; +} + +bool IrExportBuilder::BuildFuncGraph(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + // Export funcGraph name. + graph_proto->set_name(func_graph->ToString()); + // Export parameters + // 1. parameters should be mapped to ValueInfoProto + // 2. parameters with default value should be mapped to Initializer + //导出参数 + //1.参数应映射到ValueInfoProto + //2.具有默认值的参数应映射到Initializer + if (!BuildParameters(func_graph, graph_proto)) { + MS_LOG(ERROR) << "Build parameters failed."; + return false; + } + + // Export graph attributes + //导出图形属性 + if (!BuildFuncGraphAttrs(func_graph, graph_proto)) { + MS_LOG(ERROR) << "Build attributes for graph failed."; + return false; + } + + // Export operator nodes(include output) + //导出操作员节点(包括输出) + return BuildNodes(func_graph, graph_proto); +} + +bool IrExportBuilder::BuildFuncGraphAttrs(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(graph_proto); + // 遍历函数图的所有属性 + for (const auto &attr : func_graph->attrs()) { + // 输出调试信息,打印属性名、属性值的文本表示和属性值的类型名 + MS_LOG(DEBUG) << "attr: " << attr.first << " " << attr.second->DumpText() << " " << attr.second->type_name(); + // 在导出属性黑名单中查找当前属性名 + auto iter = g_export_attr_blacklist.find(attr.first); + if (iter != g_export_attr_blacklist.end()) { + continue; + } + // 创建一个AttributeProto对象,并设置属性名称 + mind_ir::AttributeProto *attr_proto = graph_proto->add_attribute(); + attr_proto->set_name(attr.first); + // 将属性值转换并设置到AttributeProto中 + if (!SetValueToAttributeProto(attr.second, attr_proto)) { + MS_LOG(ERROR) << "Set value to AttributeProto for GraphProto failed."; + return false; + } + } + return true; +} + +bool IrExportBuilder::BuildParameters(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + // 构建函数图的参数信息并添加到GraphProto中 + MS_EXCEPTION_IF_NULL(func_graph); + MS_EXCEPTION_IF_NULL(graph_proto); + // 遍历函数图的所有参数节点 + for (auto &item : func_graph->parameters()) { + MS_EXCEPTION_IF_NULL(item); + auto param = item->cast(); + // 如果无法将节点转换为参数节点,输出错误信息并返回失败 + if (param == nullptr) { + MS_LOG(ERROR) << "Parameter: '" << item->ToString() << "' could not cast to parameter."; + return false; + } + // 获取唯一的参数名称 + std::string param_name = GetUniqueNodeName(param); + // 如果是顶层函数图且参数具有默认值 + if (top_graph && param->has_default()) { + MS_LOG(DEBUG) << "Parameter: '" << item->DebugString(); + mind_ir::TensorProto *parameter_proto = graph_proto->add_parameter(); + // 设置参数节点的名称,并将参数转换为TensorProto + parameter_proto->set_name(param_name); + if (!SetParamToTensorProto(param, parameter_proto)) { + MS_LOG(ERROR) << "Set parameter " << param->DebugString() << " to TensorProto failed."; + return false; + } + } else { + mind_ir::ValueInfoProto *input_proto = graph_proto->add_input(); + // 设置参数节点的名称,并将参数转换为ValueInfoProto + input_proto->set_name(param_name); + // 检查参数名称是否重复,如果是则输出错误信息并返回失败 + if (!SetValueInfoProto(param, input_proto)) { + MS_LOG(ERROR) << "Set parameter " << param->DebugString() << " to TensorProto failed."; + return false; + } + } + if (nodeName_.count(param_name) > 0) { + MS_LOG(ERROR) << "parameter name is duplicate:" << param_name; + return false; + } + (void)nodeName_.insert(param_name); + } + return true; +} + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataType(TypeId type_id) const { + auto iter = g_data_type_map.find(type_id); + if (iter == g_data_type_map.end()) { + MS_LOG(ERROR) << "Convert type error, unsupported type! " << type_id; + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} +//获取Mindir数据类型 + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataBitsIntType(int bits) const { + auto iter = g_data_bits_int_map.find(bits); + if (iter == g_data_bits_int_map.end()) { + MS_LOG(ERROR) << "Convert bits int error, unsupported bits! " << bits; + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} +//获取Mindir数据是否为int类型 + +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataBitsUIntType(int bits) const { + auto iter = g_data_bits_uint_map.find(bits); + if (iter == g_data_bits_uint_map.end()) { + MS_LOG(ERROR) << "Convert bits uint error, unsupported bits! " << bits; + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} +//获取Mindir数据是否为uint类型 +mind_ir::TensorProto_DataType IrExportBuilder::GetMindirDataBitsFloatType(int bits) const { + auto iter = g_data_bits_float_map.find(bits); + if (iter == g_data_bits_float_map.end()) { + MS_LOG(ERROR) << "Convert bits float error, unsupported bits! " << bits; + return mind_ir::TensorProto_DataType_UNDEFINED; + } + return iter->second; +} +//获取Mindir数据是否为float类型 +bool IrExportBuilder::SetValueInfoProto(const AnfNodePtr &node, mind_ir::ValueInfoProto *const value_proto) { + if (node == nullptr || value_proto == nullptr) { + MS_LOG(EXCEPTION) << "AnfNode or ValueInfo is null!"; + } + MS_LOG(DEBUG) << "SetValueInfoProto: " << node->DebugString(); + const TypePtr &type = node->Type(); + const BaseShapePtr &shape = node->Shape(); + // For the bprop fg which has not been renormalized. + if (type == nullptr || shape == nullptr) { + return true; + } + if (type->isa() && shape->isa()) { + mind_ir::TensorProto *tensor_proto = value_proto->add_tensor(); + if (!SetTensorProto(node->abstract(), tensor_proto)) { + return false; + } + } else if (type->isa()) { + mind_ir::AttributeProto *attribute = value_proto->mutable_attr_info(); + if (!SetAbstractToNodeProto(node->abstract(), attribute)) { + MS_LOG(ERROR) << "Set shape to Proto for " << node->DebugString() << " failed."; + return false; + } + attribute->set_name("shape"); + } else { + value_proto->set_denotation(type->type_name()); + } + MS_LOG(DEBUG) << "Value type: " << type->type_name(); + return true; +} +//设置proto参数 + +bool IrExportBuilder::SetTensorToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; + } + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + tensor_proto->set_name("value0"); + auto data = value->cast(); + MS_EXCEPTION_IF_NULL(data); + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); + auto dtype = data->data_type(); + auto shape = data->shape_c(); + auto data_type = GetMindirDataType(dtype); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + for (const auto &dim : shape) { + tensor_proto->add_dims(dim); + } + return true; +} +//设置proto参数 +bool IrExportBuilder::SetCSRTensorToProto(const AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto) { + abstract::AbstractCSRTensorPtr csr_tensor_abs = abstract->cast(); + MS_EXCEPTION_IF_NULL(csr_tensor_abs); + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_CSR_TENSOR); + mind_ir::AttributeProto *indptr = attr_proto->add_values(); + bool res = SetAbstractToNodeProto(csr_tensor_abs->indptr(), indptr); + mind_ir::AttributeProto *indices = attr_proto->add_values(); + res = res && SetAbstractToNodeProto(csr_tensor_abs->indices(), indices); + mind_ir::AttributeProto *values = attr_proto->add_values(); + res = res && SetAbstractToNodeProto(csr_tensor_abs->values(), values); + mind_ir::AttributeProto *shape = attr_proto->add_values(); + res = res && SetAbstractToNodeProto(csr_tensor_abs->shape(), shape); + return res; +} +//设置proto参数 +bool IrExportBuilder::SetCOOTensorToProto(const AbstractBasePtr &abstract, mind_ir::AttributeProto *const attr_proto) { + abstract::AbstractCOOTensorPtr coo_tensor_abs = abstract->cast(); + MS_EXCEPTION_IF_NULL(coo_tensor_abs); + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_COO_TENSOR); + mind_ir::AttributeProto *indices = attr_proto->add_values(); + bool res = SetAbstractToNodeProto(coo_tensor_abs->indices(), indices); + mind_ir::AttributeProto *values = attr_proto->add_values(); + res = res && SetAbstractToNodeProto(coo_tensor_abs->values(), values); + mind_ir::AttributeProto *shape = attr_proto->add_values(); + res = res && SetAbstractToNodeProto(coo_tensor_abs->shape(), shape); + return res; +} +//设置proto参数 +bool IrExportBuilder::SetTensorProto(const AbstractBasePtr &abstract, mind_ir::TensorProto *const tensor_proto) { + auto type = abstract->BuildType(); + auto shape = abstract->BuildShape(); + if (!type->isa() || !shape->isa()) { + MS_LOG(ERROR) << "Type or shape is not supported! " << type->ToString(); + return false; + } + auto tensor = type->cast(); + auto tensor_shape = shape->cast(); + const auto &dims = tensor_shape->shape(); + auto data_type = GetMindirDataType(tensor->element()->type_id()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + for (const auto &dim : dims) { + tensor_proto->add_dims(dim); + } + if (tensor_shape->IsDynamic()) { + auto min_shape = tensor_shape->min_shape(); + auto max_shape = tensor_shape->max_shape(); + for (auto item : min_shape) { + tensor_proto->add_min_dims(item); + } + for (auto item : max_shape) { + tensor_proto->add_max_dims(item); + } + } + if (!abstract->name().empty()) { + tensor_proto->set_name(abstract->name()); + } + // Deal Ref + if (!type->isa()) { + return true; + } + + auto abs_ref = abstract->cast(); + if (abs_ref == nullptr) { + MS_LOG(ERROR) << "The abstract " << abstract->ToString() << " should be AbstractRefTensor."; + return false; + } + auto ref_key_value = abs_ref->ref_key_value()->cast(); + if (ref_key_value == nullptr) { + MS_LOG(INFO) << "The ref_key_value of abstract ref " << abstract->ToString() << " is nullptr"; + return true; + } + tensor_proto->set_ref_key(ref_key_value->value()); + return true; +} +//设置proto参数 +bool IrExportBuilder::SetParamToTensorProto(const ParameterPtr ¶m, mind_ir::TensorProto *const tensor_proto) { + if (param == nullptr || tensor_proto == nullptr) { + MS_LOG(EXCEPTION) << "Parameter or TensorProto is null!"; + } + MS_LOG(DEBUG) << "SetParamToTensorProto: " << param->DebugString(); + return SetTensorProto(param->abstract(), tensor_proto); +} +//设置proto参数 +bool IrExportBuilder::BuildNodes(const FuncGraphPtr &func_graph, mind_ir::GraphProto *const graph_proto) { + // 构建函数图中的节点信息并添加到GraphProto中 + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude);// 使用拓扑排序获取函数图中的节点顺序 + for (const AnfNodePtr &node : nodes) {// 遍历所有节点 + MS_EXCEPTION_IF_NULL(node); + // 如果节点不是CNode类型,则输出调试信息并继续处理下一个节点 + if (!node->isa()) { + MS_LOG(DEBUG) << "Node: '" << node->ToString() << "' is not cnode"; + continue; + } + auto cnode = node->cast(); + // 如果节点是函数图的返回节点 + if (cnode == func_graph->get_return()) { + // 构建返回节点的输出信息并添加到GraphProto + if (!BuildOutput(cnode, graph_proto)) { + MS_LOG(ERROR) << "Build output for graph " << func_graph->ToString() << " failed."; + return false; + } + } else { + // 构建普通CNode节点的信息并添加到GraphProto + if (!BuildCNode(cnode, graph_proto)) { + MS_LOG(ERROR) << "Build proto for cnode " << cnode->DebugString() << " failed."; + return false; + } + } + } + return true; +} + +bool IrExportBuilder::BuildOutput(const CNodePtr &node, mind_ir::GraphProto *const graph_proto) { + MS_EXCEPTION_IF_NULL(node); + const int OutputSize = 2; + if (node->size() != OutputSize) { + MS_LOG(ERROR) << "Number of inputs of return node is not equal to 2."; + return false; + } + AnfNodePtr arg = node->input(1); + std::string node_name = BuildInputNode(arg, graph_proto); + if (node_name.empty()) { + MS_LOG(ERROR) << "Build input node failed for arg " << arg->DebugString(); + return false; + } + mind_ir::ValueInfoProto *output_proto = graph_proto->add_output(); + output_proto->set_name(node_name); + return SetValueInfoProto(arg, output_proto); +} +//建立输出 +std::string IrExportBuilder::GetOpTypeName(const AnfNodePtr &node) { + // May be ValueNode/CNode/Parameter + std::string type_name = ""; + if (IsValueNode(node)) { + PrimitivePtr prim = GetValueNode(node); + MS_EXCEPTION_IF_NULL(prim); + type_name = "REF::" + GetPrimitiveUniqueName(prim); + } else if (IsValueNode(node)) { + FuncGraphPtr fg = GetValueNode(node); + MS_EXCEPTION_IF_NULL(fg); + todo_.push_back(fg); + type_name = "REF::" + fg->ToString(); + } else if (node->isa() || node->isa()) { + auto nodeName = GetUniqueNodeName(node); + type_name = "REF::" + nodeName; + if (nodeName_.count(nodeName) == 0) { + MS_LOG(ERROR) << "There is not the name: " << nodeName; + return ""; + } + } else { + MS_LOG(ERROR) << "Need to support op type: " << node->type_name(); + return ""; + } + MS_LOG(DEBUG) << "ExportType: " << type_name; + return type_name; +} +//获取OpType的类型名,可能为ValueNode/CNode/Parameter +bool IrExportBuilder::SetAbstractToNodeProto(const AbstractBasePtr &abs, mind_ir::AttributeProto *const attr_proto) { + auto type = abs->BuildType(); + auto shape = abs->BuildShape(); + if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TUPLE); + auto tuple_abs = abs->cast(); + for (size_t i = 0; i < tuple_abs->size(); i++) { + mind_ir::AttributeProto *attr_values = attr_proto->add_values(); + if (!SetAbstractToNodeProto((*tuple_abs)[i], attr_values)) { + return false; + } + } + } else if (type->isa() && shape->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + return SetTensorProto(abs, tensor_proto); + } else if (type->isa()) { + if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_BOOL); + } else { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto data_type = GetMindirDataType(type->type_id()); + tensor_proto->set_data_type(data_type); + tensor_proto->add_dims(1); + } + } else if (type->isa()) { + if (!SetAbstractFuncToAttributeProto(abs, attr_proto)) { + return false; + } + } else if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_STRING); + } else if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UMONAD); + } else if (type->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_IOMONAD); + } else if (type->isa()) { + auto csr_tensor_abs = abs->cast(); + if (!SetCSRTensorToProto(csr_tensor_abs, attr_proto)) { + return false; + } + } else if (type->isa()) { + auto coo_tensor_abs = abs->cast(); + if (!SetCOOTensorToProto(coo_tensor_abs, attr_proto)) { + return false; + } + } else { + MS_LOG(ERROR) << "Type of cnode need to be supported: " << type->type_name(); + return false; + } + return true; +} +//设置proto参数 +bool IrExportBuilder::SetAbstractToNodeProto(const CNodePtr &node, mind_ir::NodeProto *const node_proto) { + // Get shape of cnode + // 1. need to get shape from tuple element + // 2. save shape in TensorProto + MS_EXCEPTION_IF_NULL(node); + auto type = node->Type(); + auto shape = node->Shape(); + auto abs = node->abstract(); + // For the bprop fg which has not been renormalized. + if (type == nullptr || shape == nullptr) { + return true; + } + mind_ir::AttributeProto *attr_proto = node_proto->add_attribute(); + if (!SetAbstractToNodeProto(abs, attr_proto)) { + MS_LOG(ERROR) << "Set shape to NodeProto for " << node->DebugString() << " failed."; + return false; + } + attr_proto->set_name("shape"); + return true; +} +//设置proto参数 +bool IrExportBuilder::BuildCNode(const CNodePtr &node, mind_ir::GraphProto *const graph_proto) { + // 构建计算图中的一个 CNode 节点,并将其表示添加到图的 proto 中 + auto inputs_size = node->size();// 获取 CNode 的输入数量 + if (inputs_size < 1) { + MS_LOG(ERROR) << "Inputs of node " << node->DebugString() << " is empty"; + return false; + } + + // Need to build input node before dealing with cnode + // 需要先构建输入节点,然后再处理 CNode + std::vector input_names; + for (size_t i = 1; i < inputs_size; i++) { + auto input = node->input(i); + std::string node_name = BuildInputNode(input, graph_proto);// 构建输入节点并获取节点名 + if (node_name.empty()) { + MS_LOG(ERROR) << "Build input node for " << input->DebugString() << " failed."; + return false; + } + input_names.push_back(node_name);// 将输入节点名加入列表 + } + + // Build cnode + // 构建 CNode + mind_ir::NodeProto *node_proto = graph_proto->add_node();// 添加一个节点表示到图的 proto 中 + std::string output_name = GetUniqueNodeName(node);// 获取唯一的节点名 + if (nodeName_.count(output_name) > 0) { + MS_LOG(EXCEPTION) << "There is a duplicate name: " << output_name; + } + (void)nodeName_.insert(output_name);// 将节点名加入已用名字集合 + node_proto->add_output(output_name);// 设置节点的输出名 + node_proto->set_name(output_name);// 设置节点的名字 + node_proto->set_domain(node->fullname_with_scope());// 设置节点的域 + AnfNodePtr op = node->input(0);// 获取操作节点 + std::string type_name = GetOpTypeName(op);// 获取操作类型名 + if (type_name.empty()) { + MS_LOG(ERROR) << "Get op type name for " << op->DebugString() << " failed."; + return false; + } + node_proto->set_op_type(type_name);// 设置节点的操作类型 + last_node_ = node_proto;// 记录最后一个节点 + // Maybe Tensor or Function or nullptr + if (!SetAbstractToNodeProto(node, node_proto)) { + return false; + } + // 将输入节点名加入节点的输入列表中 + (void)std::for_each(input_names.begin(), input_names.end(), + [&node_proto](const string &name) { node_proto->add_input(name); }); + return true; +} + +std::string IrExportBuilder::BuildInputNode(const AnfNodePtr &node, mind_ir::GraphProto *const graph_proto) { + // Return the NodeName that the node has been processed. + auto iter = node_name_map_.find(node); + if (iter != node_name_map_.end()) { + return iter->second; + } + + std::string node_name = GetUniqueNodeName(node); + // FuncGraph will be added to functions and the input name is the function name. + if (IsValueNode(node)) { + FuncGraphPtr fg = GetValueNode(node); + todo_.push_back(fg); + return fg->ToString(); + } + if (node->isa()) { + (void)nodeName_.insert(node_name); + // When node input is a ValueNode, need to create a Constant Node + mind_ir::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_name(node_name); + node_proto->add_output(node_name); + if (!SetAttributeProto(node, node_proto)) { + return ""; + } + } + return node_name; +} +//建立输入节点 +std::string IrExportBuilder::GetUniqueNodeName(const AnfNodePtr &node) { + // Naming anfnode + // 1. parameter is unique in one func_graph + // 2. cnode and valuenode may be reduplicative, so add index to identify. + auto iter = node_name_map_.find(node); + if (iter != node_name_map_.end()) { + return iter->second; + } else { + std::string node_name = GetNodeName(node); + // Compatible before. CNode = FuncGraphName:CNodeName:index ,Parameter = FuncGraphName:ParameterName + if (node->isa()) { + node_name = node_name + ":" + std::to_string(GetUniqueID()); + } + // Avoid duplicate name. + while (nodeName_.count(node_name) > 0) { + node_name = node_name + "_" + std::to_string(GetUniqueID()); + } + node_name_map_[node] = node_name; + return node_name; + } +} +//获得未确定节点的名字 +std::string IrExportBuilder::GetNodeName(const AnfNodePtr &node) const { + MS_EXCEPTION_IF_NULL(node); + std::string node_name = ""; + if (node->func_graph() != nullptr) { + node_name = node->func_graph()->ToString() + ":"; + } + if (node->isa()) { + // Needn't value + node_name += node->AnfNode::ToString(); + } else { + node_name += node->ToString(); + } + MS_LOG(DEBUG) << "GetNodeName: " << node_name; + return node_name; +} + +bool IrExportBuilder::SetAttributeProto(const AnfNodePtr &node, mind_ir::NodeProto *const node_proto) { + if (node == nullptr || node_proto == nullptr) { + MS_LOG(EXCEPTION) << "AnfNode or NodeProto is null!"; + } + auto value_node = node->cast(); + MS_EXCEPTION_IF_NULL(value_node); + auto value = value_node->value(); + node_proto->set_op_type("Constant"); + mind_ir::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + MS_LOG(DEBUG) << "Set Constant attribute: " << value->ToString(); + return SetValueToAttributeProto(value, attr_proto); +} +//获得节点名 +bool IrExportBuilder::SetTypeToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; + } + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + if (value->isa()) { + tensor_proto->set_name("value0"); + auto int_value = value->cast(); + auto data_type = GetMindirDataBitsIntType(int_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + tensor_proto->set_name("value0"); + auto float_value = value->cast(); + auto data_type = GetMindirDataBitsUIntType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + tensor_proto->set_name("value0"); + auto float_value = value->cast(); + auto data_type = GetMindirDataBitsFloatType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + tensor_proto->set_name("value0"); + tensor_proto->set_data_type(mind_ir::TensorProto_DataType_BOOL); + } else if (value->isa()) { + tensor_proto->set_name("tensor0"); + auto elem_type = value->cast()->element(); + if (elem_type->isa()) { + auto int_value = elem_type->cast(); + auto data_type = GetMindirDataBitsIntType(int_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (elem_type->isa()) { + auto float_value = elem_type->cast(); + auto data_type = GetMindirDataBitsFloatType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else { + MS_LOG(ERROR) << "Unsupported type " << elem_type->type_name(); + return false; + } + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << value->type_name(); + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetValueToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; + } + if (value->isa() || value->isa()) { + return SetScalarToAttributeProto_ir(value, attr_proto); + } else if (value->isa() || value->isa()) { + return SetTypeToAttributeProto(value, attr_proto); + } else if (value->isa()) { + if (!SetSequenceToAttributeProto(value->cast(), attr_proto)) { + MS_LOG(ERROR) << "Set sequence to AttributeProto failed."; + return false; + } + MS_LOG(DEBUG) << "Attr string: " << value->type_name(); + } else if (value->isa()) { + return SetTensorToAttributeProto(value, attr_proto); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_NONE); + MS_LOG(DEBUG) << "Attr string: " << value->type_name(); + } else if (value->isa()) { + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UMONAD); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_IOMONAD); + } else { + MS_LOG(ERROR) << "Unsupported Monad type: " << value->type_name(); + return false; + } + } else { + MS_LOG(ERROR) << "Unsupported type: " << value->type_name(); + return false; + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetScalarToAttributeProto_ir(const ValuePtr &value, + mind_ir::AttributeProto *const attr_proto) const { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValuePtr or AttributeProto is null!"; + } + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_STRING); + attr_proto->set_s(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_BOOL); + int64_t attr_value = GetValue(value) ? 1 : 0; + attr_proto->set_i(attr_value); + } else if (SetScalarToAttributeProtoForInt_ir(value, attr_proto)) { + return true; + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_FLOAT); + attr_proto->set_f(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_DOUBLE); + attr_proto->set_d(GetValue(value)); + } else { + MS_LOG(ERROR) << "Unsupported scalar type: " << value->type_name(); + return false; + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetScalarToAttributeProtoForInt_ir(const ValuePtr &value, + mind_ir::AttributeProto *const attr_proto) const { + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT8); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT16); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT32); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT64); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT8); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT16); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT32); + attr_proto->set_i(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT64); + attr_proto->set_i(UlongToLong(value->cast()->value())); + } else { + return false; + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetTypeToAttributeProto_irs(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "AttributeProto is null!"; + } + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto int_value = value->cast(); + auto data_type = GetMindirDataBitsIntType(int_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto float_value = value->cast(); + auto data_type = GetMindirDataBitsFloatType(float_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + auto uint_value = value->cast(); + auto data_type = GetMindirDataBitsFloatType(uint_value->nbits()); + if (data_type == mind_ir::TensorProto_DataType_UNDEFINED) { + return false; + } + tensor_proto->set_data_type(data_type); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + mind_ir::TensorProto *tensor_proto = attr_proto->add_tensors(); + tensor_proto->set_data_type(mind_ir::TensorProto_DataType_BOOL); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TENSORS); + return SetTensorToAttributeProto(value, attr_proto); + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << value->type_name(); + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetScalarToAttributeProto_irs(const ValuePtr &value, + mind_ir::AttributeProto *const attr_proto) const { + if (attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "AttributeProto is null!"; + } + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_STRING); + attr_proto->add_strings(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_BOOL); + attr_proto->add_ints(GetValue(value)); + } else if (SetScalarToAttributeProtoForInt_irs(value, attr_proto)) { + return true; + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_FLOAT); + attr_proto->add_floats(GetValue(value)); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_DOUBLE); + attr_proto->add_doubles(GetValue(value)); + } else { + MS_LOG(ERROR) << "Unsupported scalar type: " << value->type_name(); + return false; + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetScalarToAttributeProtoForInt_irs(const ValuePtr &value, + mind_ir::AttributeProto *const attr_proto) const { + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT8); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT16); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT32); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_INT64); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT8); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT16); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT32); + attr_proto->add_ints(value->cast()->value()); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_UINT64); + attr_proto->add_ints(SizeToInt(value->cast()->value())); + } else { + return false; + } + return true; +} +//设置proto名 +bool IrExportBuilder::SetSeqElemToAttributeProto(const ValuePtr &value, mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr) { + MS_LOG(ERROR) << "Value is nullptr"; + return false; + } + if (value->isa() || value->isa()) { + return SetScalarToAttributeProto_irs(value, attr_proto); + } + return SetTypeToAttributeProto_irs(value, attr_proto); +} + +bool IrExportBuilder::SetSequenceToAttributeProto(const ValueSequencePtr &value, + mind_ir::AttributeProto *const attr_proto) { + if (value == nullptr || attr_proto == nullptr) { + MS_LOG(EXCEPTION) << "ValueSequencePtr or AttributeProto is null!"; + } + if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_TUPLE); + } else if (value->isa()) { + attr_proto->set_type(mind_ir::AttributeProto_AttributeType_LIST); + } else { + MS_LOG(EXCEPTION) << "The sequance value should be ValueTuple or ValueList, but it is " << value->ToString(); + } + auto value_sequence = value->cast(); + MS_EXCEPTION_IF_NULL(value_sequence); + const auto &values = value_sequence->value(); + if (values.empty()) { + MS_LOG(DEBUG) << "SetSequenceToAttributeProto sequence size is 0"; + return true; + } + for (const auto &item : values) { + mind_ir::AttributeProto *attr_values = attr_proto->add_values(); + MS_EXCEPTION_IF_NULL(item); + if (item->isa()) { + if (!SetSequenceToAttributeProto(item->cast(), attr_values)) { + MS_LOG(ERROR) << "Set sequence to AttributeProto failed."; + return false; + } + } else { + if (!SetSeqElemToAttributeProto(item, attr_values)) { + MS_LOG(ERROR) << "Set seq elem to AttributeProto failed."; + return false; + } + } + } + return true; +} +//设置proto名 +std::string GetBinaryProtoString(const FuncGraphPtr &func_graph) { + auto builder = std::make_shared(); + if (builder == nullptr) { + MS_LOG(ERROR) << "Create ir exporter failed!"; + return ""; + } + auto exporter = std::make_shared(builder); + if (exporter == nullptr) { + return ""; + } + auto ret = exporter->GetDumpString(func_graph); + return ret; +} +//获得protostring +bool DumpBinaryProto(const FuncGraphPtr &func_graph, const std::string &file_path, + const FuncGraphPtr ¶m_layout_fg) { + auto exporter = std::make_shared(std::make_shared()); + auto proto = exporter->GetDumpProto(func_graph, param_layout_fg); + if (proto == nullptr) { + MS_LOG(ERROR) << "Get binary proto for graph " << func_graph->ToString() << " failed."; + return false; + } + + auto realpath = Common::CreatePrefixPath(file_path, true); + if (!realpath.has_value()) { + MS_LOG(ERROR) << "Get real path of file " << file_path << " failed."; + return false; + } + + ChangeFileMode(realpath.value(), S_IWUSR); + std::ofstream fout(realpath.value()); + if (!fout.is_open()) { + MS_LOG(ERROR) << "Open the file '" << realpath.value() << "' failed!" << ErrnoToString(errno); + return false; + } + + if (!proto->SerializeToOstream(&fout)) { + MS_LOG(ERROR) << "Failed to write the mindir proto to file " << realpath.value(); + fout.close(); + return false; + } + fout.close(); + ChangeFileMode(realpath.value(), S_IRUSR); + return true; +} +} // namespace mindspore -- 2.34.1 From 821dc204d7ed54cea709268339a63c520b665443 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Tue, 5 Sep 2023 22:09:04 +0800 Subject: [PATCH 25/72] ADD file via upload --- .../nn_batch_norm_ops_declare.cc | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nn_batch_norm_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nn_batch_norm_ops_declare.cc b/mindspore/ccsrc/transform-update/nn_batch_norm_ops_declare.cc new file mode 100644 index 00000000000..184ea8efe78 --- /dev/null +++ b/mindspore/ccsrc/transform-update/nn_batch_norm_ops_declare.cc @@ -0,0 +1,99 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_batch_norm_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// BatchNorm +INPUT_MAP(BatchNorm) = {{1, INPUT_DESC(x)}, + {2, INPUT_DESC(scale)}, + {3, INPUT_DESC(offset)}, + {4, INPUT_DESC(mean)}, + {5, INPUT_DESC(variance)}}; +//输入映射,x索引为1,scale索引为2,offset索引为3,mean索引为4,variance索引为5 +ATTR_MAP(BatchNorm) = {{"format", ATTR_DESC(data_format, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +//属性映射,属性format类型为string,属性epsilon类型为float,属性format类型为bool +OUTPUT_MAP(BatchNorm) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(batch_mean)}, + {2, OUTPUT_DESC(batch_variance)}, + {3, OUTPUT_DESC(reserve_space_1)}, + {4, OUTPUT_DESC(reserve_space_2)}}; +//输出映射,y索引为0,batch_mean索引为1,batch_variance索引为2,reserve_space_1索引为3,reserve_space_2索引为4 +// BNInference is BatchNorm for caffe +INPUT_MAP(BNInference) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mean)}, {3, INPUT_DESC(variance)}, + {4, INPUT_DESC(momentum)}, {5, INPUT_DESC(scale)}, {6, INPUT_DESC(offset)}}; +//输入映射,x索引为1,mean索引为2,variance索引为3,momentum索引为4,scale索引为5,offset索引为5 +ATTR_MAP(BNInference) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"use_global_stats", ATTR_DESC(use_global_stats, AnyTraits())}, + {"mode", ATTR_DESC(mode, AnyTraits())}}; +//属性映射,属性epsilon类型为float,属性use_global_stats类型为bool,属性mode类型为bool +OUTPUT_MAP(BNInference) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(BNInference, kNameBNInference, ADPT_DESC(BNInference)) +//注册BNInference操作的适配器描述kNameBNInference +REG_ADPT_DESC(BatchNorm, kNameBatchNorm, ADPT_DESC(BatchNorm)) +//注册BatchNorm操作的适配器描述kNameBatchNorm +REG_ADPT_DESC(FusedBatchNorm, kNameFusedBatchNorm, ADPT_DESC(BatchNorm)) +//注册FusedBatchNorm操作的适配器描述kNameFusedBatchNorm + +// BatchNormGrad +INPUT_MAP(BatchNormGrad) = {{1, INPUT_DESC(y_backprop)}, + {2, INPUT_DESC(x)}, + {3, INPUT_DESC(scale)}, + {4, INPUT_DESC(reserve_space_1)}, + {5, INPUT_DESC(reserve_space_2)}}; +//输入映射,y_backprop索引为1,x索引为2,scale索引为3,reserve_space_1索引为4,reserve_space_2索引为5 +ATTR_MAP(BatchNormGrad) = {{"format", ATTR_DESC(data_format, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +//属性映射,属性format类型为string,属性epsilon类型为float,属性format类型为bool +OUTPUT_MAP(BatchNormGrad) = {{0, OUTPUT_DESC(x_backprop)}, + {1, OUTPUT_DESC(scale_backprop)}, + {2, OUTPUT_DESC(offset_backprop)}, + {3, OUTPUT_DESC(reserve_space_4)}, + {4, OUTPUT_DESC(reserve_space_5)}}; +//输出映射,x_backprop索引为0,scale_backprop索引为1,offset_backprop索引为2,reserve_space_4索引为3,reserve_space_5索引为4 +REG_ADPT_DESC(BatchNormGrad, kNameBatchNormGrad, ADPT_DESC(BatchNormGrad)) +//注册BatchNormGrad操作的适配器描述kNameBatchNormGrad + +// L2NormalizeGrad +INPUT_MAP(L2NormalizeGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(dy)}}; +//输入映射,x索引为1,y索引为2,dy索引为3 +ATTR_MAP(L2NormalizeGrad) = { + {"axis", ATTR_DESC(dim, AnyTraits>(), AnyTraits>())}, + {"epsilon", ATTR_DESC(eps, AnyTraits())}}; +//属性映射,属性axis类型为int64_t,属性epsilon类型为float +OUTPUT_MAP(L2NormalizeGrad) = {{0, OUTPUT_DESC(dx)}}; +//输出映射,x_backprop索引为0 +REG_ADPT_DESC(L2NormalizeGrad, kNameL2NormalizeGrad, ADPT_DESC(L2NormalizeGrad)) +//注册L2NormalizeGrad操作的适配器描述kNameL2NormalizeGrad + +// L2Normalize +INPUT_MAP(L2Normalize) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(L2Normalize) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}, + {"epsilon", ATTR_DESC(eps, AnyTraits())}}; +//属性映射,属性axis类型为int64_t,属性epsilon类型为float +OUTPUT_MAP(L2Normalize) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(L2Normalize, kNameL2Normalize, ADPT_DESC(L2Normalize)) +//注册L2Normalize操作的适配器描述kNameL2Normalize +} // namespace mindspore::transform -- 2.34.1 From 2d2707971cb856a9e0325179ea5f94f4d4ba5f2c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:12:12 +0800 Subject: [PATCH 26/72] ADD file via upload --- .../nn_calculation_ops_declare.cc | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nn_calculation_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nn_calculation_ops_declare.cc b/mindspore/ccsrc/transform-update/nn_calculation_ops_declare.cc new file mode 100644 index 00000000000..54869592a8c --- /dev/null +++ b/mindspore/ccsrc/transform-update/nn_calculation_ops_declare.cc @@ -0,0 +1,262 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_calculation_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// BiasAddGrad +INPUT_MAP(BiasAddGrad) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(BiasAddGrad) = {{"format", ATTR_DESC(data_format, AnyTraits())}}; +//属性映射,属性format类型为string +OUTPUT_MAP(BiasAddGrad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(BiasAddGrad, prim::kPrimBiasAddGrad->name(), ADPT_DESC(BiasAddGrad)) +//注册BiasAddGrad操作的适配器描述kPrimBiasAddGrad返回的name变量 + +// Conv2D +INPUT_MAP(Conv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}}; +//输入映射,x索引为1,filter索引为2,bias索引为3 +ATTR_MAP(Conv2D) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"group", ATTR_DESC(groups, AnyTraits())}, +}; +//属性映射,属性stride类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t,属性format类型为string,属性group类型为int64_t +OUTPUT_MAP(Conv2D) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv2D, prim::kPrimConv2D->name(), ADPT_DESC(Conv2D)) +//注册Conv2D操作的适配器描述kPrimConv2D返回的name变量 + +// Conv2DBackpropInputD +INPUT_MAP(Conv2DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filter)}}; +//输入映射,out_backprop索引为1,filter索引为2 +INPUT_ATTR_MAP(Conv2DBackpropInputD) = { + {3, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}}; +//输入属性映射,将索引为3的输入与属性为input_size的变量相关联,用于反卷积操作的输入属性映射 +ATTR_MAP(Conv2DBackpropInputD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"group", ATTR_DESC(groups, AnyTraits())}, +}; +//属性映射,属性pad_list类型为int64_t,属性stride类型为int64_t,属性dilations类型为int64_t,属性format类型为string,属性group类型为int64_t +OUTPUT_MAP(Conv2DBackpropInputD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv2DBackpropInputD, prim::kPrimConv2DBackpropInput->name(), ADPT_DESC(Conv2DBackpropInputD)) +//注册Conv2DBackpropInputD操作的适配器描述kPrimConv2DBackpropInput返回的name变量 + +// Conv2DBackpropInput for tf inference +INPUT_MAP(Conv2DBackpropInput) = {{1, INPUT_DESC(input_size)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}}; +//输入映射,input_size索引为1,filter索引为2,out_backprop索引为3 +ATTR_MAP(Conv2DBackpropInput) = { + {"stride", ATTR_DESC(strides, AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}, +}; +//属性映射,属性stride类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t,属性pad_list类型为int64_t,属性data_format类型为string +OUTPUT_MAP(Conv2DBackpropInput) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv2DBackpropInput, kNameConv2DBackpropInputV2, ADPT_DESC(Conv2DBackpropInput)) +//注册Conv2DBackpropInput操作的适配器描述kNameConv2DBackpropInputV2返回的name变量 + +// Deconvolution for caffe inference +INPUT_MAP(Deconvolution) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +//输入映射,x索引为1,filter索引为2,bias索引为3,offset_w索引为4 +ATTR_MAP(Deconvolution) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"group", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"offset", ATTR_DESC(offset_x, AnyTraits())}}; +//属性映射,属性stride类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t +//属性groups类型为int64_t,属性format类型为string,属性offset类型为int64_t +OUTPUT_MAP(Deconvolution) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Deconvolution, kNameDeconvolution, ADPT_DESC(Deconvolution)) +//注册Deconvolution操作的适配器描述kNameDeconvolution,返回的name变量 +REG_ADPT_DESC(Conv2DTranspose, kConv2DTransposeOpName, ADPT_DESC(Conv2DBackpropInputD)) +//注册Conv2DTranspose操作的适配器描述kConv2DTransposeOpName返回的name变量 + +// Conv2DTransposeD for tf onnx inference +INPUT_MAP(Conv2DTransposeD) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +//输入映射,x索引为1,filter索引为2,bias索引为3,offset_w索引为4 +ATTR_MAP(Conv2DTransposeD) = { + {"input_size", ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"group", ATTR_DESC(groups, AnyTraits())}, + {"data_format", ATTR_DESC(data_format, AnyTraits())}, + {"output_paddings", ATTR_DESC(output_padding, AnyTraits>(), AnyTraits>())}, + {"offset", ATTR_DESC(offset_x, AnyTraits())}}; +//属性映射,属性input_size类型为int64_t,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t +//属性groups类型为int64_t,属性data_format类型为string,属性output_paddings类型为int64_t,属性offset类型为int64_t +OUTPUT_MAP(Conv2DTransposeD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv2DTransposeD, kNameConv2DTransposeD, ADPT_DESC(Conv2DTransposeD)) +//注册Conv2DTransposeD操作的适配器描述kNameConv2DTransposeD返回的name变量 + +// Conv2DBackpropFilterD +INPUT_MAP(Conv2DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}}; +//输入映射,out_backprop索引为1,x索引为2 +INPUT_ATTR_MAP(Conv2DBackpropFilterD) = { + {3, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; + +ATTR_MAP(Conv2DBackpropFilterD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"group", ATTR_DESC(groups, AnyTraits())}, +}; +//属性映射,属性pad_list类型为int64_t,属性stride类型为int64_t,属性dilations类型为int64_t,属性groups类型为int64_t,属性group类型为int64_t +OUTPUT_MAP(Conv2DBackpropFilterD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv2DBackpropFilterD, prim::kPrimConv2DBackpropFilter->name(), ADPT_DESC(Conv2DBackpropFilterD)) +//注册Conv2DBackpropFilterD操作的适配器描述kPrimConv2DBackpropFilter返回的name变量 + +// Conv3DTransposeD +INPUT_MAP(Conv3DTransposeD) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +//输入映射,x索引为1,filter索引为2,bias索引为3,offset_w索引为4 +ATTR_MAP(Conv3DTransposeD) = { + {"input_size", ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"output_padding", ATTR_DESC(output_padding, AnyTraits>(), AnyTraits>())}, +}; +//属性映射,属性input_size类型为int64_t,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t +//属性groups类型为int64_t,属性format类型为int64_t,属性output_padding类型为int64_t +OUTPUT_MAP(Conv3DTransposeD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv3DTransposeD, kNameConv3DTransposeD, ADPT_DESC(Conv3DTransposeD)) +//注册Conv3DTransposeD操作的适配器描述kNameConv3DTransposeD返回的name变量 + +// Conv3D +INPUT_MAP(Conv3D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}, {4, INPUT_DESC(offset_w)}}; +//输入映射,x索引为1,filter索引为2,bias索引为3,offset_w索引为4 +ATTR_MAP(Conv3D) = { + {"strides", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"offset_x", ATTR_DESC(offset_x, AnyTraits())}, +}; +//属性映射,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t,属性groups类型为int64_t,属性format类型为int64_t,属性offset_x类型为int64_t +OUTPUT_MAP(Conv3D) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv3D, kNameConv3D, ADPT_DESC(Conv3D)) +//注册Conv3D操作的适配器描述kNameConv3D返回的name变量 + +// Conv3DBackpropInputD +INPUT_MAP(Conv3DBackpropInputD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(filter)}}; +//输入映射,out_backprop索引为1,filter索引为2 +INPUT_ATTR_MAP(Conv3DBackpropInputD) = { + {3, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv3DBackpropInputD) = { + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, +}; +//属性映射,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t,属性groups类型为int64_t,属性format类型为int64_t +OUTPUT_MAP(Conv3DBackpropInputD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv3DBackpropInputD, kNameConv3DBackpropInputD, ADPT_DESC(Conv3DBackpropInputD)) +//注册Conv3DBackpropInputD操作的适配器描述kNameConv3DBackpropInputD返回的name变量 + +// Conv3DBackpropFilterD +INPUT_MAP(Conv3DBackpropFilterD) = {{1, INPUT_DESC(out_backprop)}, {2, INPUT_DESC(x)}}; +//输入映射,out_backprop索引为1,filter索引为2 +INPUT_ATTR_MAP(Conv3DBackpropFilterD) = { + {3, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(Conv3DBackpropFilterD) = { + {"strides", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilations", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"groups", ATTR_DESC(groups, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, +}; +//属性映射,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t,属性groups类型为int64_t,属性format类型为int64_t +OUTPUT_MAP(Conv3DBackpropFilterD) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Conv3DBackpropFilterD, kNameConv3DBackpropFilterD, ADPT_DESC(Conv3DBackpropFilterD)) +//注册Conv3DBackpropFilterD操作的适配器描述kNameConv3DBackpropFilterD返回的name变量 + +// DepthwiseConv2D +INPUT_MAP(DepthwiseConv2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(filter)}, {3, INPUT_DESC(bias)}}; +//输入映射,x索引为1,filter索引为2,bias索引为3 +ATTR_MAP(DepthwiseConv2D) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, +}; +//属性映射,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t,属性format类型为int64_t +OUTPUT_MAP(DepthwiseConv2D) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(DepthwiseConv2D, prim::kPrimDepthwiseConv2dNative->name(), ADPT_DESC(DepthwiseConv2D)) +//注册DepthwiseConv2D操作的适配器描述kPrimDepthwiseConv2dNative返回的name变量 + +// DepthwiseConv2DBackpropInputD +INPUT_MAP(DepthwiseConv2DBackpropInputD) = {{2, INPUT_DESC(filter)}, {3, INPUT_DESC(out_backprop)}}; +//输入映射,filter索引为2,out_backprop索引为3 +INPUT_ATTR_MAP(DepthwiseConv2DBackpropInputD) = { + {1, ATTR_DESC(input_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(DepthwiseConv2DBackpropInputD) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, +}; +//属性映射,属性strides类型为int64_t,属性pad_list类型为int64_t,属性dilations类型为int64_t +OUTPUT_MAP(DepthwiseConv2DBackpropInputD) = {{0, OUTPUT_DESC(input_grad)}}; +//输出映射,input_grad索引为0 +REG_ADPT_DESC(DepthwiseConv2DBackpropInputD, prim::kPrimDepthwiseConv2dNativeBackpropInput->name(), + ADPT_DESC(DepthwiseConv2DBackpropInputD)) +//注册DepthwiseConv2DBackpropInputD操作的适配器描述kPrimDepthwiseConv2dNativeBackpropInput返回的name变量 + +// DepthwiseConv2DBackpropFilterD +INPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{1, INPUT_DESC(input)}, {3, INPUT_DESC(out_backprop)}}; +//输入映射,input索引为1,out_backprop索引为3 +INPUT_ATTR_MAP(DepthwiseConv2DBackpropFilterD) = { + {2, ATTR_DESC(filter_size, AnyTraits>(), AnyTraits>())}}; +ATTR_MAP(DepthwiseConv2DBackpropFilterD) = { + {"stride", ATTR_DESC(strides, AnyTraits>(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits>(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilations, AnyTraits>(), AnyTraits>())}, +}; +//属性映射,属性strides类型为int64_t,属性pads类型为int64_t,属性dilations类型为int64_t +OUTPUT_MAP(DepthwiseConv2DBackpropFilterD) = {{0, OUTPUT_DESC(filter_grad)}}; +//输出映射,filter_grad索引为0 +REG_ADPT_DESC(DepthwiseConv2DBackpropFilterD, prim::kPrimDepthwiseConv2dNativeBackpropFilter->name(), + ADPT_DESC(DepthwiseConv2DBackpropFilterD)) +//注册DepthwiseConv2DBackpropFilterD操作的适配器描述kPrimDepthwiseConv2dNativeBackpropFilter返回的name变量 +} // namespace mindspore::transform -- 2.34.1 From 0a94cddbc51782461f2645d6f22eb08343d6a0c7 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:12:35 +0800 Subject: [PATCH 27/72] ADD file via upload --- .../transform-update/nn_detect_ops_declare.cc | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nn_detect_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nn_detect_ops_declare.cc b/mindspore/ccsrc/transform-update/nn_detect_ops_declare.cc new file mode 100644 index 00000000000..588b20e949a --- /dev/null +++ b/mindspore/ccsrc/transform-update/nn_detect_ops_declare.cc @@ -0,0 +1,115 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_detect_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// BoundingBoxEncode +INPUT_MAP(BoundingBoxEncode) = { + {1, INPUT_DESC(anchor_box)}, + {2, INPUT_DESC(ground_truth_box)}, +}; +//输入映射,anchor_box索引为1,ground_truth_box索引为2 +ATTR_MAP(BoundingBoxEncode) = { + {"means", ATTR_DESC(means, AnyTraits>(), AnyTraits())}, + {"stds", ATTR_DESC(stds, AnyTraits>(), AnyTraits())}, +}; +//属性映射,属性means类型为float,属性stds类型为float +OUTPUT_MAP(BoundingBoxEncode) = {{0, OUTPUT_DESC(delats)}}; +//输出映射,delats索引为0 +REG_ADPT_DESC(BoundingBoxEncode, kNameBoundingBoxEncode, ADPT_DESC(BoundingBoxEncode)) +//注册BoundingBoxEncode操作的适配器描述KNameBoundingBoxEncode + +// BoundingBoxDecode +INPUT_MAP(BoundingBoxDecode) = { + {1, INPUT_DESC(rois)}, + {2, INPUT_DESC(deltas)}, +}; +//输入映射,rois索引为1,deltas索引为2 +ATTR_MAP(BoundingBoxDecode) = { + {"means", ATTR_DESC(means, AnyTraits>(), AnyTraits())}, + {"stds", ATTR_DESC(stds, AnyTraits>(), AnyTraits())}, + {"max_shape", ATTR_DESC(max_shape, AnyTraits>(), AnyTraits>())}, + {"wh_ratio_clip", ATTR_DESC(wh_ratio_clip, AnyTraits())}, +}; +//属性映射,属性means类型为float,属性stds类型为float,属性max_shape类型为int64_t,属性wh_ratio_clip类型为float +OUTPUT_MAP(BoundingBoxDecode) = {{0, OUTPUT_DESC(bboxes)}}; +//输出映射,bboxes索引为0 +REG_ADPT_DESC(BoundingBoxDecode, kNameBoundingBoxDecode, ADPT_DESC(BoundingBoxDecode)) +//注册BoundingBoxDecode操作的适配器描述KNameBoundingBoxDecode + +// Iou +INPUT_MAP(Iou) = {{1, INPUT_DESC(bboxes)}, {2, INPUT_DESC(gtboxes)}}; +//输入映射,bboxes索引为1,gtboxes索引为2 +ATTR_MAP(Iou) = {{"mode", ATTR_DESC(mode, AnyTraits())}}; +//属性映射,属性mode类型为string +OUTPUT_MAP(Iou) = {{0, OUTPUT_DESC(overlap)}}; +//输出映射,overlap索引为0 +REG_ADPT_DESC(Iou, kNameIOU, ADPT_DESC(Iou)) +//注册IOU操作的适配器描述KNameIOU + +// CheckValid +INPUT_MAP(CheckValid) = {{1, INPUT_DESC(bbox_tensor)}, {2, INPUT_DESC(img_metas)}}; +//输入映射,bbox_tensor索引为1,img_metas索引为2 +ATTR_MAP(CheckValid) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(CheckValid) = {{0, OUTPUT_DESC(valid_tensor)}}; +//输出映射,CheckValid索引为0 +REG_ADPT_DESC(CheckValid, kNameCheckValid, ADPT_DESC(CheckValid)) +//注册CheckValid操作的适配器描述KNameCheckValid + +// Sort +INPUT_MAP(Sort) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Sort) = {{"axis", ATTR_DESC(axis, AnyTraits())}, + {"descending", ATTR_DESC(descending, AnyTraits())}}; +//属性映射,属性axis类型为int64_t,属性descending类型为bool +OUTPUT_MAP(Sort) = {{0, OUTPUT_DESC(y1)}, {1, OUTPUT_DESC(y2)}}; +//输出映射,y1索引为0,y2索引为1 +REG_ADPT_DESC(Sort, kNameSort, ADPT_DESC(Sort)) +//注册Sort操作的适配器描述KNameSort + +// ROIAlign +INPUT_MAP(ROIAlign) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(rois)}}; +//输入映射,features索引为1,rois索引为2 +OUTPUT_MAP(ROIAlign) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +ATTR_MAP(ROIAlign) = {{"pooled_height", ATTR_DESC(pooled_height, AnyTraits())}, + {"pooled_width", ATTR_DESC(pooled_width, AnyTraits())}, + {"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits())}, + {"sample_num", ATTR_DESC(sample_num, AnyTraits())}, + {"roi_end_mode", ATTR_DESC(roi_end_mode, AnyTraits())}}; +//属性映射,属性pooled_height类型为int64_t,属性pooled_width类型为int64_t,属性spatial_scale类型为float,属性sample_num类型为int64_t,属性roi_end_mode类型为int64_t +REG_ADPT_DESC(ROIAlign, kNameROIAlign, ADPT_DESC(ROIAlign)) +//注册ROIAlign操作的适配器描述KNameROIAlign + +// ROIAlignGrad +INPUT_MAP(ROIAlignGrad) = {{1, INPUT_DESC(ydiff)}, {2, INPUT_DESC(rois)}}; +//输入映射,ydiff索引为1,rois索引为2 +OUTPUT_MAP(ROIAlignGrad) = {{0, OUTPUT_DESC(xdiff)}}; +//输出映射,xdiff索引为0 +ATTR_MAP(ROIAlignGrad) = { + {"xdiff_shape", ATTR_DESC(xdiff_shape, AnyTraits>(), AnyTraits>())}, + {"pooled_height", ATTR_DESC(pooled_height, AnyTraits())}, + {"pooled_width", ATTR_DESC(pooled_width, AnyTraits())}, + {"spatial_scale", ATTR_DESC(spatial_scale, AnyTraits())}, + {"sample_num", ATTR_DESC(sample_num, AnyTraits())}}; +//属性映射,属性xdiff_shape类型为int64_t,属性pooled_height类型为int64_t,属性pooled_width类型为int64_t,属性spatial_scale类型为float,属性sample_num类型为int64_t +REG_ADPT_DESC(ROIAlignGrad, kNameROIAlignGrad, ADPT_DESC(ROIAlignGrad)) +//注册ROIAlignGrad操作的适配器描述KNameROIAlignGrad +} // namespace mindspore::transform -- 2.34.1 From 3ac8b24aaf6f91897d933556d8fc8331d1fc1cf0 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:13:02 +0800 Subject: [PATCH 28/72] ADD file via upload --- .../transform-update/nn_norm_ops_declare.cc | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nn_norm_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nn_norm_ops_declare.cc b/mindspore/ccsrc/transform-update/nn_norm_ops_declare.cc new file mode 100644 index 00000000000..bf2a05566fa --- /dev/null +++ b/mindspore/ccsrc/transform-update/nn_norm_ops_declare.cc @@ -0,0 +1,245 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_norm_ops_declare.h" +#include +#include + +namespace mindspore::transform { +// SoftmaxV2 +INPUT_MAP(SoftmaxV2) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(SoftmaxV2) = { + {"axis", ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}, +}; +//属性映射,属性axis类型为int64_t +OUTPUT_MAP(SoftmaxV2) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(SoftmaxV2, kNameSoftmax, ADPT_DESC(SoftmaxV2)) +//注册SoftmaxV2操作的适配器描述KNameSoftmax + +// SoftmaxGrad +INPUT_MAP(SoftmaxGrad) = {{1, INPUT_DESC(softmax)}, {2, INPUT_DESC(grad_softmax)}}; +//输入映射,softmax索引为1,grad_softmax索引为2 +OUTPUT_MAP(SoftmaxGrad) = {{0, OUTPUT_DESC(grad_x)}}; +//输出映射,SoftmaxGrad索引为0 +ATTR_MAP(SoftmaxGrad) = EMPTY_ATTR_MAP; +//属性映射,设为空 +REG_ADPT_DESC(SoftmaxGrad, kNameSoftmaxGrad, ADPT_DESC(SoftmaxGrad)) +//注册SoftmaxGrad操作的适配器描述KNameSoftmax + +// SoftmaxCrossEntropyWithLogits +INPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{1, INPUT_DESC(features)}, {2, INPUT_DESC(labels)}}; +//输入映射,features索引为1,labels索引为2 +ATTR_MAP(SoftmaxCrossEntropyWithLogits) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(SoftmaxCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}, {1, OUTPUT_DESC(backprop)}}; +//输出映射,loss索引为0,backprop索引为1 +REG_ADPT_DESC(SoftmaxCrossEntropyWithLogits, prim::kPrimSoftmaxCrossEntropyWithLogits->name(), + ADPT_DESC(SoftmaxCrossEntropyWithLogits)) +//注册SoftmaxCrossEntropyWithLogits操作的适配器描述SoftmaxCrossEntropyWithLogits + +// SmoothL1Loss +INPUT_MAP(SmoothL1Loss) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}}; +//输入映射,predict索引为1,labels索引为2 +ATTR_MAP(SmoothL1Loss) = {{"beta", ATTR_DESC(sigma, AnyTraits())}}; +//属性映射,属性beta类型为float +OUTPUT_MAP(SmoothL1Loss) = {{0, OUTPUT_DESC(loss)}}; +//输出映射,SmoothL1loss索引为0 +REG_ADPT_DESC(SmoothL1Loss, kNameSmoothL1Loss, ADPT_DESC(SmoothL1Loss)) +//注册SmoothL1Loss操作的适配器描述kNameSmoothL1Loss + +// SmoothL1LossGrad +INPUT_MAP(SmoothL1LossGrad) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(label)}, {3, INPUT_DESC(dout)}}; +//输入映射,predict索引为1,labels索引为2,dout索引为3 +ATTR_MAP(SmoothL1LossGrad) = {{"beta", ATTR_DESC(sigma, AnyTraits())}}; +//属性映射,属性beta类型为float +OUTPUT_MAP(SmoothL1LossGrad) = {{0, OUTPUT_DESC(gradient)}}; +//输出映射,gradient索引为0 +REG_ADPT_DESC(SmoothL1LossGrad, kNameSmoothL1LossGrad, ADPT_DESC(SmoothL1LossGrad)) +//注册SmoothL1LossGrad操作的适配器描述kNameSmoothL1LossGrad + +// SigmoidCrossEntropyWithLogits +INPUT_MAP(SigmoidCrossEntropyWithLogits) = {{1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}}; +//输入映射,predict索引为1,labels索引为2 +ATTR_MAP(SigmoidCrossEntropyWithLogits) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(SigmoidCrossEntropyWithLogits) = {{0, OUTPUT_DESC(loss)}}; +//输出映射,loss索引为0 +REG_ADPT_DESC(SigmoidCrossEntropyWithLogits, kNameSigmoidCrossEntropyWithLogits, + ADPT_DESC(SigmoidCrossEntropyWithLogits)) +//注册SigmoidCrossEntropyWithLogits操作的适配器描述kNameSigmoidCrossEntropyWithLogits + +// SigmoidCrossEntropyWithLogitsGrad +INPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = { + {1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(dout)}}; +//输入映射,predict索引为1,target索引为2,dout索引为3 +ATTR_MAP(SigmoidCrossEntropyWithLogitsGrad) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(SigmoidCrossEntropyWithLogitsGrad) = {{0, OUTPUT_DESC(gradient)}}; +//输出映射,gradient索引为0 +REG_ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad, kNameSigmoidCrossEntropyWithLogitsGrad, + ADPT_DESC(SigmoidCrossEntropyWithLogitsGrad)) +//注册SigmoidCrossEntropyWithLogitsGrad操作的适配器描述kNameSigmoidCrossEntropyWithLogitsGrad + +// SigmoidCrossEntropyWithLogitsV2 +INPUT_MAP(SigmoidCrossEntropyWithLogitsV2) = { + {1, INPUT_DESC(predict)}, {2, INPUT_DESC(target)}, {3, INPUT_DESC(weight)}, {4, INPUT_DESC(pos_weight)}}; +//输入映射,predict索引为1,target索引为2,weight索引为3,pos_weight索引为4 +ATTR_MAP(SigmoidCrossEntropyWithLogitsV2) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +//属性映射,属性reduction类型为string +OUTPUT_MAP(SigmoidCrossEntropyWithLogitsV2) = {{0, OUTPUT_DESC(loss)}}; +//输出映射,loss索引为0 +REG_ADPT_DESC(SigmoidCrossEntropyWithLogitsV2, kNameSigmoidCrossEntropyWithLogitsV2, + ADPT_DESC(SigmoidCrossEntropyWithLogitsV2)) +//注册SigmoidCrossEntropyWithLogitsV2操作的适配器描述kNameSigmoidCrossEntropyWithLogitsV2 + +// LogSoftmaxGrad +INPUT_MAP(LogSoftmaxGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}}; +//输入映射,x索引为1,grad索引为2 +ATTR_MAP(LogSoftmaxGrad) = { + {"axis", ATTR_DESC(axis, AnyTraits>(), AnyTraits>())}}; +//属性映射,属性axis类型为int64_t +OUTPUT_MAP(LogSoftmaxGrad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(LogSoftmaxGrad, prim::kPrimLogSoftmaxGrad->name(), ADPT_DESC(LogSoftmaxGrad)) +//注册LogSoftmaxGrad操作的适配器描述kPrimLogSoftmaxGrad + +// LogSoftmaxV2 +INPUT_MAP(LogSoftmaxV2) = {{1, INPUT_DESC(logits)}}; +//输入映射,logits索引为1 +ATTR_MAP(LogSoftmaxV2) = { + {"axis", ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +//属性映射,属性axes类型为int64_t +OUTPUT_MAP(LogSoftmaxV2) = {{0, OUTPUT_DESC(logsoftmax)}}; +//输出映射,logsoftmax索引为0 +REG_ADPT_DESC(LogSoftmaxV2, prim::kPrimLogSoftmax->name(), ADPT_DESC(LogSoftmaxV2)) +//注册LogSoftmaxV2操作的适配器描述kPrimLogSoftmax + +// LayerNorm +INPUT_MAP(LayerNorm) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(gamma)}, {3, INPUT_DESC(beta)}}; +//输入映射,x索引为1,gamma索引为2,beta索引为3 +ATTR_MAP(LayerNorm) = {{"begin_norm_axis", ATTR_DESC(begin_norm_axis, AnyTraits())}, + {"begin_params_axis", ATTR_DESC(begin_params_axis, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}}; +//属性映射,属性begin_norm_axis类型为int64_t,属性begin_params_axis类型为int64_t,属性epsilon类型为float +OUTPUT_MAP(LayerNorm) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mean)}, {2, OUTPUT_DESC(variance)}}; +//输出映射,y索引为0,mean索引为1,variance索引为2 +REG_ADPT_DESC(LayerNorm, prim::kPrimLayerNorm->name(), ADPT_DESC(LayerNorm)) +//注册LayerNorm操作的适配器描述kPrimLayerNorm + +// LayerNormGrad +INPUT_MAP(LayerNormGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(dy)}, {3, INPUT_DESC(variance)}, {4, INPUT_DESC(mean)}, {5, INPUT_DESC(gamma)}}; +//输入映射,x索引为1,dy索引为2,variance索引为3,mean索引为4,gamma索引为5 +ATTR_MAP(LayerNormGrad) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(LayerNormGrad) = {{0, OUTPUT_DESC(pd_x)}, {1, OUTPUT_DESC(pd_gamma)}, {2, OUTPUT_DESC(pd_beta)}}; +//输出映射,pd_x索引为0,pd_gamma索引为1,pd_beta索引为2 +REG_ADPT_DESC(LayerNormGrad, prim::kPrimLayerNormGrad->name(), ADPT_DESC(LayerNormGrad)) +//注册LayerNormGrad操作的适配器描述kPrimLayerNormGrad + +// LRN +INPUT_MAP(LRN) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(LRN) = {{"depth_radius", ATTR_DESC(depth_radius, AnyTraits())}, + {"bias", ATTR_DESC(bias, AnyTraits())}, + {"alpha", ATTR_DESC(alpha, AnyTraits())}, + {"beta", ATTR_DESC(beta, AnyTraits())}, + {"norm_region", ATTR_DESC(norm_region, AnyTraits())}}; +//属性映射,属性depth_radius类型为int64_t,属性bias类型为float,属性alpha类型为float,属性beta类型为float,属性norm_region类型为float +OUTPUT_MAP(LRN) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(LRN, kNameLRN, ADPT_DESC(LRN)) +//注册LRN操作的适配器描述kNameLRN + +// LRNGrad +INPUT_MAP(LRNGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}}; +//输入映射,grads索引为1,x索引为2,y索引为3 +ATTR_MAP(LRNGrad) = {{"depth_radius", ATTR_DESC(depth_radius, AnyTraits())}, + {"bias", ATTR_DESC(bias, AnyTraits())}, + {"alpha", ATTR_DESC(alpha, AnyTraits())}, + {"beta", ATTR_DESC(beta, AnyTraits())}}; +//属性映射,属性depth_radius类型为int64_t,属性bias类型为float,属性alpha类型为float,属性beta类型为float +OUTPUT_MAP(LRNGrad) = {{0, OUTPUT_DESC(z)}}; +//输出映射,z索引为0 +REG_ADPT_DESC(LRNGrad, kNameLRNGrad, ADPT_DESC(LRNGrad)) +//注册LRNGrad操作的适配器描述kNameLRNGrad + +// DropoutDoMask +INPUT_MAP(DropOutDoMask) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(mask)}, {3, INPUT_DESC(keep_prob)}}; +//输入映射,x索引为1,mask索引为2,keep_prob索引为3 +ATTR_MAP(DropOutDoMask) = EMPTY_ATTR_MAP; +//属性映射,设为空 +OUTPUT_MAP(DropOutDoMask) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(DropOutDoMask, kNameDropoutDoMask, ADPT_DESC(DropOutDoMask)) +//注册DropOutDoMask操作的适配器描述kNameDropOutDoMask + +// BinaryCrossEntropy +INPUT_MAP(BinaryCrossEntropy) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(weight)}}; +//输入映射,x索引为1,y索引为2,weight索引为3 +ATTR_MAP(BinaryCrossEntropy) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +//属性映射,属性reduction类型为string +OUTPUT_MAP(BinaryCrossEntropy) = {{0, OUTPUT_DESC(output)}}; +//输出映射,output索引为0 +REG_ADPT_DESC(BinaryCrossEntropy, kNameBinaryCrossEntropy, ADPT_DESC(BinaryCrossEntropy)) +//注册BinaryCrossEntropy操作的适配器描述kNameBinaryCrossEntropy + +// BinaryCrossEntropyGrad +INPUT_MAP(BinaryCrossEntropyGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(y)}, {3, INPUT_DESC(grad_output)}, {4, INPUT_DESC(weight)}}; +//输入映射,x索引为1,y索引为2,grad_output索引为3,weight索引为3 +ATTR_MAP(BinaryCrossEntropyGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}}; +//属性映射,属性reduction类型为string +OUTPUT_MAP(BinaryCrossEntropyGrad) = {{0, OUTPUT_DESC(output)}}; +//输出映射,output索引为0 +REG_ADPT_DESC(BinaryCrossEntropyGrad, kNameBinaryCrossEntropyGrad, ADPT_DESC(BinaryCrossEntropyGrad)) +//注册BinaryCrossEntropyGrad操作的适配器描述kNameBinaryCrossEntropyGrad + +// Centralization +INPUT_MAP(Centralization) = {{1, INPUT_DESC(x)}}; +//输入映射,x索引为1 +ATTR_MAP(Centralization) = {{"axes", ATTR_DESC(axes, AnyTraits>())}}; +//属性映射,属性axes类型为int64_t +OUTPUT_MAP(Centralization) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Centralization, kNameCentralization, ADPT_DESC(Centralization)) +//注册Centralization操作的适配器描述kNameCentralization + +// Scale +INPUT_MAP(Scale) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(scale)}, {3, INPUT_DESC(bias)}}; +//输入映射,x索引为1,scale索引为2,bias索引为3 +ATTR_MAP(Scale) = {{"axis", ATTR_DESC(axis, AnyTraits())}, + {"num_axes", ATTR_DESC(num_axes, AnyTraits())}, + {"scale_from_blob", ATTR_DESC(scale_from_blob, AnyTraits())}}; +//属性映射,属性axes类型为int64_t,属性num_axes类型为int64_t,属性scale_from_blob类型为bool +OUTPUT_MAP(Scale) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(Scale, kNameScale, ADPT_DESC(Scale)) +//注册Scale操作的适配器描述kNameScale + +// KlDivLossGrad +INPUT_MAP(KlDivLossGrad) = {{1, INPUT_DESC(grad)}, {2, INPUT_DESC(input)}, {3, INPUT_DESC(target)}}; +//输入映射,grad索引为1,input索引为2,target索引为3 +ATTR_MAP(KlDivLossGrad) = {{"reduction", ATTR_DESC(reduction, AnyTraits())}, + {"log_target", ATTR_DESC(log_target, AnyTraits())}}; +//属性映射,属性reduction类型为string,属性log_target类型为bool +OUTPUT_MAP(KlDivLossGrad) = {{0, OUTPUT_DESC(y)}}; +//输出映射,y索引为0 +REG_ADPT_DESC(KlDivLossGrad, kNameKlDivLossGrad, ADPT_DESC(KlDivLossGrad)) +//注册KlDivLossGrad操作的适配器描述kNameKlDivLossGrad +} // namespace mindspore::transform -- 2.34.1 From 89b0165779c072d1846ed88c7347d52143f26663 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:13:26 +0800 Subject: [PATCH 29/72] ADD file via upload --- .../nn_pooling_ops_declare.cc | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nn_pooling_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nn_pooling_ops_declare.cc b/mindspore/ccsrc/transform-update/nn_pooling_ops_declare.cc new file mode 100644 index 00000000000..e13224296a7 --- /dev/null +++ b/mindspore/ccsrc/transform-update/nn_pooling_ops_declare.cc @@ -0,0 +1,240 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_pooling_ops_declare.h" +#include + +namespace mindspore::transform { +// MaxPool +INPUT_MAP(MaxPool) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(MaxPool) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides"Ϊint64_tstd::vectorͣ"pad_mode""format"Ϊstd::string +OUTPUT_MAP(MaxPool) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPool, kNameMaxPool, ADPT_DESC(MaxPool)) + +// MaxPool3D +INPUT_MAP(MaxPool3D) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(MaxPool3D) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"pad_list", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilation, AnyTraits(), AnyTraits>())}, + {"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬߸ԣ"kernel_size""pad_list""strides""format""ceil_mode"Ϊint64_tstd::vectorͣ"pad_mode""format"Ϊstd::string +OUTPUT_MAP(MaxPool3D) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPool3D, kNameMaxPool3D, ADPT_DESC(MaxPool3D)) +// עMaxPool3DkNameMaxPool3D + +// MaxPool3DGrad +INPUT_MAP(MaxPool3DGrad) = {{1, INPUT_DESC(orig_x)}, {2, INPUT_DESC(orig_y)}, {3, INPUT_DESC(grads)}}; +// ӳ䣬orig_xΪ1orig_yΪ2gradsΪ3 +ATTR_MAP(MaxPool3DGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides""pad_list"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +OUTPUT_MAP(MaxPool3DGrad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPool3DGrad, kNameMaxPool3DGrad, ADPT_DESC(MaxPool3DGrad)) +// עMaxPool3DGradkNameMaxPool3DGrad + +// MaxPool3DGradGrad +INPUT_MAP(MaxPool3DGradGrad) = {{1, INPUT_DESC(orig_x)}, {2, INPUT_DESC(orig_y)}, {3, INPUT_DESC(grads)}}; +// ӳ䣬orig_xΪ1orig_yΪ2gradsΪ3 +ATTR_MAP(MaxPool3DGradGrad) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_list", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides""pad_list"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +OUTPUT_MAP(MaxPool3DGradGrad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPool3DGradGrad, kNameMaxPool3DGradGrad, ADPT_DESC(MaxPool3DGradGrad)) +// עMaxPool3DGradGradkNameMaxPool3DGradGrad + +// AvgPool +INPUT_MAP(AvgPool) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(AvgPool) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides""pad_list"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +OUTPUT_MAP(AvgPool) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(AvgPool, kNameAvgPool, ADPT_DESC(AvgPool)) +// עAvgPoolkNameAvgPool + +// MaxPoolGrad +INPUT_MAP(MaxPoolGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}}; +// ӳ䣬orig_xΪ1orig_yΪ2gradsΪ3 +ATTR_MAP(MaxPoolGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides""pad_list"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +OUTPUT_MAP(MaxPoolGrad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPoolGrad, kNameMaxPoolGrad, ADPT_DESC(MaxPoolGrad)) +// עMaxPoolGradkNameMaxPoolGrad + +// MaxPoolGradGrad +INPUT_MAP(MaxPoolGradGrad) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}, {3, INPUT_DESC(grad)}}; +// ӳ䣬orig_xΪ1orig_yΪ2gradsΪ3 +ATTR_MAP(MaxPoolGradGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides""pad_list"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +OUTPUT_MAP(MaxPoolGradGrad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPoolGradGrad, kNameMaxPoolGradGrad, ADPT_DESC(MaxPoolGradGrad)) +// עMaxPoolGradGradkNameMaxPoolGradGrad + +// avgpoolgrad +INPUT_MAP(AvgPoolGrad) = {{1, INPUT_DESC(orig_input_shape)}, {2, INPUT_DESC(input_grad)}}; +// ӳ䣬orig_input_shapeΪ1input_gradΪ2 +ATTR_MAP(AvgPoolGrad) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +// ӳ䣬ĸԣ"kernel_size""strides""pad_list"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +OUTPUT_MAP(AvgPoolGrad) = {{0, OUTPUT_DESC(out_grad)}}; +// ӳ䣬out_gradΪ0 +REG_ADPT_DESC(AvgPoolGrad, kNameAvgPoolGrad, ADPT_DESC(AvgPoolGrad)) +// עAvgPoolGradkNameAvgPoolGrad + +// MaxPoolWithArgmax +INPUT_MAP(MaxPoolWithArgmax) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(MaxPoolWithArgmax) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}}; +// ӳ䣬ԣ"kernel_size""strides"Ϊint64_tstd::vectorͣ"pad_mode"Ϊstd::string +OUTPUT_MAP(MaxPoolWithArgmax) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(argmax)}}; +// ӳ䣬yΪ0argmaxΪ1 +REG_ADPT_DESC(MaxPoolWithArgmax, kNameMaxPoolWithArgmax, ADPT_DESC(MaxPoolWithArgmax)) +// עMaxPoolWithArgmaxkNameMaxPoolWithArgmax + +// MaxPoolGradWithArgmax +INPUT_MAP(MaxPoolGradWithArgmax) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}, {3, INPUT_DESC(argmax)}}; +// ӳ䣬xΪ1gradΪ2argmaxΪ3 +ATTR_MAP(MaxPoolGradWithArgmax) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}}; +// ӳ䣬ԣ"kernel_size""strides"Ϊint64_tstd::vectorͣ"pad_mode"Ϊstd::string +OUTPUT_MAP(MaxPoolGradWithArgmax) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPoolGradWithArgmax, kNameMaxPoolGradWithArgmax, ADPT_DESC(MaxPoolGradWithArgmax)) +// עMaxPoolGradWithArgmaxkNameMaxPoolGradWithArgmax + +// MaxPoolGradGradWithArgmax +INPUT_MAP(MaxPoolGradGradWithArgmax) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(grad)}, {3, INPUT_DESC(argmax)}}; +// ӳ䣬xΪ1gradΪ2argmaxΪ3 +ATTR_MAP(MaxPoolGradGradWithArgmax) = { + {"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"pad_mode", ATTR_DESC(padding, AnyTraits())}}; +// ӳ䣬ԣ"kernel_size""strides"Ϊint64_tstd::vectorͣ"pad_mode"Ϊstd::string +OUTPUT_MAP(MaxPoolGradGradWithArgmax) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPoolGradGradWithArgmax, kNameMaxPoolGradGradWithArgmax, ADPT_DESC(MaxPoolGradGradWithArgmax)) +// עMaxPoolGradGradWithArgmaxkNameMaxPoolGradGradWithArgmax + +// Pooling +INPUT_MAP(Pooling) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Pooling) = {{"mode", ATTR_DESC(mode, AnyTraits())}, + {"global", ATTR_DESC(global_pooling, AnyTraits())}, + {"kernel_size", ATTR_DESC(window, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(stride, AnyTraits(), AnyTraits>())}, + {"pad", ATTR_DESC(pad, AnyTraits(), AnyTraits>())}, + {"dilation", ATTR_DESC(dilation, AnyTraits(), AnyTraits>())}, + {"round_mode", ATTR_DESC(ceil_mode, AnyTraits())}, + {"format", ATTR_DESC(data_format, AnyTraits())}}; +//ӳ䣬а˸ԣ"kernel_size""strides""pad""dilation"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +//"mode""round_mode"Ϊint64_tͣ"global"Ϊbool +OUTPUT_MAP(Pooling) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Pooling, kNamePooling, ADPT_DESC(Pooling)) +// עPoolingkNamePooling + +// MaxPoolV3 +INPUT_MAP(MaxPoolV3) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(MaxPoolV3) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding_mode", ATTR_DESC(padding_mode, AnyTraits())}, + {"pad", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"global", ATTR_DESC(global_pooling, AnyTraits())}, + {"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits())}}; +// ӳ䣬߸ԣ"kernel_size""strides""pad"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +//"round_mode"Ϊint64_tͣ"global""ceil_mode"Ϊbool +OUTPUT_MAP(MaxPoolV3) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(MaxPoolV3, kNameMaxPoolV3, ADPT_DESC(MaxPoolV3)) +// עMaxPoolV3kNameMaxPoolV3 + +// AvgPoolV2 +INPUT_MAP(AvgPoolV2) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(AvgPoolV2) = {{"kernel_size", ATTR_DESC(ksize, AnyTraits(), AnyTraits>())}, + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + {"padding_mode", ATTR_DESC(padding_mode, AnyTraits())}, + {"pad", ATTR_DESC(pads, AnyTraits(), AnyTraits>())}, + {"format", ATTR_DESC(data_format, AnyTraits())}, + {"global", ATTR_DESC(global_pooling, AnyTraits())}, + {"ceil_mode", ATTR_DESC(ceil_mode, AnyTraits())}}; +// ӳ䣬߸ԣ"kernel_size""strides""pad"Ϊint64_tstd::vectorͣ"format"Ϊstd::string +//"round_mode"Ϊint64_tͣ"global""ceil_mode"Ϊbool +OUTPUT_MAP(AvgPoolV2) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(AvgPoolV2, kNameAvgPoolV2, ADPT_DESC(AvgPoolV2)) +// עAvgPoolV2kNameAvgPoolV2 + +// GlobalAveragePool +INPUT_MAP(GlobalAveragePool) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(GlobalAveragePool) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(GlobalAveragePool) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(GlobalAveragePool, kNameGlobalAvgPool, ADPT_DESC(GlobalAveragePool)) +// עGlobalAveragePoolkNameGlobalAvgPool + +// Upsample +INPUT_MAP(Upsample) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Upsample) = {{"scale", ATTR_DESC(scale, AnyTraits())}, + {"stride_h", ATTR_DESC(stride_h, AnyTraits())}, + {"stride_w", ATTR_DESC(stride_w, AnyTraits())}}; +// ӳ䣬ԣ"stride_h""stride_w"Ϊint64_tͣ"scale"Ϊfloat +OUTPUT_MAP(Upsample) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Upsample, kNameUpsample, ADPT_DESC(Upsample)) +// עUpsamplekNameUpsample +} // namespace mindspore::transform -- 2.34.1 From 1092a7e133603ea791cfcc2d2b39964dbeea664c Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:13:45 +0800 Subject: [PATCH 30/72] ADD file via upload --- .../nn_training_ops_declare.cc | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nn_training_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nn_training_ops_declare.cc b/mindspore/ccsrc/transform-update/nn_training_ops_declare.cc new file mode 100644 index 00000000000..c8b78ecfe30 --- /dev/null +++ b/mindspore/ccsrc/transform-update/nn_training_ops_declare.cc @@ -0,0 +1,315 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nn_training_ops_declare.h" + +namespace mindspore::transform { +// ApplyMomentum +INPUT_MAP(ApplyMomentum) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}, {5, INPUT_DESC(momentum)}}; +// ӳ䣬ĸ룬accumΪ2lrΪ3gradΪ4momentumΪ5 +ATTR_MAP(ApplyMomentum) = {{"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬"use_nesterov""use_locking"δbool +OUTPUT_MAP(ApplyMomentum) = {{0, OUTPUT_DESC(var)}};' + //ӳ䣬varΪ0 + REG_ADPT_DESC(ApplyMomentum, kNameApplyMomentum, ADPT_DESC(ApplyMomentum)) + // עApplyMomentumkNameApplyMomentum + // + // LarsV2Update +INPUT_MAP(LarsV2Update) = {{1, INPUT_DESC(w)}, + {2, INPUT_DESC(g)}, + {3, INPUT_DESC(w_square_sum)}, + {4, INPUT_DESC(g_square_sum)}, + {5, INPUT_DESC(weight_decay)}, + {6, INPUT_DESC(learning_rate)}}; +// ӳ䣬룬wΪ1gΪ2w_square_sumΪ3g_square_sumΪ4weight_decayΪ5learning_rateΪ6 +ATTR_MAP(LarsV2Update) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"hyperpara", ATTR_DESC(hyperpara, AnyTraits())}, + {"use_clip", ATTR_DESC(use_clip, AnyTraits())}}; +// ӳ䣬ԣ"epsilon""hyperpara"Ϊfloatͣ"use_clip"Ϊbool +OUTPUT_MAP(LarsV2Update) = {{0, OUTPUT_DESC(g_new)}}; +// ӳ䣬g_newΪ0 +REG_ADPT_DESC(LarsV2Update, kNameLARSUpdate, ADPT_DESC(LarsV2Update)) +// עLarsV2UpdatekNameLARSUpdate + +// ApplyAdam +INPUT_MAP(ApplyAdam) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)}, + {10, INPUT_DESC(grad)}}; +// ӳ䣬ʮ룬varΪ1mΪ2vΪ3beta1_powerΪ4beta2_powerΪ5lrΪ6 +// betalΪ7beta2Ϊ8epsilonΪ9gradΪ10 +ATTR_MAP(ApplyAdam) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}}; +// ӳ䣬"use_locking""use_nesterov"Ϊbool +OUTPUT_MAP(ApplyAdam) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +// ApplyAdamD +INPUT_MAP(ApplyAdamD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(beta2_power)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(beta1)}, {8, INPUT_DESC(beta2)}, {9, INPUT_DESC(epsilon)}, + {10, INPUT_DESC(grad)}}; +// ӳ䣬ʮ룬varΪ1mΪ2vΪ3beta1_powerΪ4beta2_powerΪ5lrΪ6 +// betalΪ7beta2Ϊ8epsilonΪ9gradΪ10 +ATTR_MAP(ApplyAdamD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"use_nesterov", ATTR_DESC(use_nesterov, AnyTraits())}}; +// ӳ䣬"use_locking""use_nesterov"Ϊbool +OUTPUT_MAP(ApplyAdamD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}}; +// ӳ䣬varΪ0mΪ1vΪ2 +REG_ADPT_DESC(ApplyAdamD, kNameApplyAdam, ADPT_DESC(ApplyAdamD)) +// עApplyAdamDkNameApplyAdam +REG_ADPT_DESC(ApplyAdam, kNameApplyAdam, ADPT_DESC(ApplyAdam)) +// עApplyAdamkNameApplyAdam + +// ApplyAdagradD +INPUT_MAP(ApplyAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}}; +// ӳ䣬ĸ룬varΪ1accumΪ2lrΪ3gradΪ4 +ATTR_MAP(ApplyAdagradD) = {{"update_slots", ATTR_DESC(update_slots, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬"use_locking""use_nesterov"Ϊbool +OUTPUT_MAP(ApplyAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +// ӳ䣬varΪ0accumΪ1 +REG_ADPT_DESC(ApplyAdagradD, kNameApplyAdagrad, ADPT_DESC(ApplyAdagradD)) +// עApplyAdagradD kNameApplyAdagrad +// +// ApplyAdagradV2D +INPUT_MAP(ApplyAdagradV2D) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, {4, INPUT_DESC(grad)}}; +// ӳ䣬ĸ룬varΪ1accumΪ2lrΪ3gradΪ4 +ATTR_MAP(ApplyAdagradV2D) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"update_slots", ATTR_DESC(update_slots, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬ԣ"epsilon"Ϊfloatͣ"update_slots""use_locking"Ϊbool +OUTPUT_MAP(ApplyAdagradV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +// ӳ䣬varΪ0accumΪ1 +REG_ADPT_DESC(ApplyAdagradV2D, kNameApplyAdagradV2D, ADPT_DESC(ApplyAdagradV2D)) +// עApplyAdagradV2D kNameApplyAdagradV2D + +// ApplyAddSignD +INPUT_MAP(ApplyAddSignD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(alpha)}, {5, INPUT_DESC(sign_decay)}, {6, INPUT_DESC(beta)}, + {7, INPUT_DESC(grad)}}; +// ӳ䣬߸룬varΪ1mΪ2lrΪ3alphaΪ4sign_decayΪ5betaΪ6gradΪ7 +ATTR_MAP(ApplyAddSignD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +//ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyAddSignD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}}; +// ӳ䣬varΪ0mΪ1 +REG_ADPT_DESC(ApplyAddSignD, kNameApplyAddSignD, ADPT_DESC(ApplyAddSignD)) +// עApplyAddSignDkNameApplyAddSignD + +// SparseApplyAdagradV2D +INPUT_MAP(SparseApplyAdagradV2D) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}}; +// ӳ䣬ĸ룬varΪ1accumΪ2gradΪ3indicesΪ4 +ATTR_MAP(SparseApplyAdagradV2D) = {{"lr", ATTR_DESC(lr, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}, + {"update_slots", ATTR_DESC(update_slots, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬ĸԣ"lr""epsilon"Ϊfloatͣ"update_slots""use_locking"Ϊbool +OUTPUT_MAP(SparseApplyAdagradV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +// ӳ䣬varΪ0accumΪ1 +REG_ADPT_DESC(SparseApplyAdagradV2D, kNameSparseApplyAdagradV2D, ADPT_DESC(SparseApplyAdagradV2D)) +// עSparseApplyAdagradV2DkNameSparseApplyAdagradV2D + +// DataFormatDimMap +INPUT_MAP(DataFormatDimMap) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(DataFormatDimMap) = {{"src_format", ATTR_DESC(src_format, AnyTraits())}, + {"dst_format", ATTR_DESC(dst_format, AnyTraits())}}; +// ӳ䣬ԣ"src_format""dst_format"Ϊtd::string +OUTPUT_MAP(DataFormatDimMap) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(DataFormatDimMap, kNameDataFormatDimMap, ADPT_DESC(DataFormatDimMap)) +// עDataFormatDimMapkNameDataFormatDimMap + +// ApplyAdadeltaD +INPUT_MAP(ApplyAdadeltaD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(accum_update)}, + {4, INPUT_DESC(lr)}, {5, INPUT_DESC(rho)}, {6, INPUT_DESC(epsilon)}, + {7, INPUT_DESC(grad)}}; +// ӳ䣬߸룬varΪ1accumΪ2accum_updateΪ3lrΪ4rhoΪ5epsilonΪ6gradΪ7 +ATTR_MAP(ApplyAdadeltaD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyAdadeltaD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}, {2, OUTPUT_DESC(accum_update)}}; +// ӳ䣬varΪ0accumΪ1accum_updateΪ2 +REG_ADPT_DESC(ApplyAdadeltaD, kNameApplyAdadelta, ADPT_DESC(ApplyAdadeltaD)) +// עApplyAdadeltaD kNameApplyAdadelta + +// ApplyAdaMaxD +INPUT_MAP(ApplyAdaMaxD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(v)}, + {4, INPUT_DESC(beta1_power)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(beta1)}, + {7, INPUT_DESC(beta2)}, {8, INPUT_DESC(epsilon)}, {9, INPUT_DESC(grad)}}; +// ӳ䣬ʮ룬varΪ1mΪ2vΪ3beta1_powerΪ4lrΪ5 +// betalΪ6beta2Ϊ7epsilonΪ8gradΪ9 +ATTR_MAP(ApplyAdaMaxD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyAdaMaxD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}, {2, OUTPUT_DESC(v)}}; +// ӳ䣬varΪ0mΪ1vΪ2 +REG_ADPT_DESC(ApplyAdaMaxD, kNameApplyAdaMax, ADPT_DESC(ApplyAdaMaxD)) +// עApplyAdaMaxD kNameApplyAdaMax + +// ApplyGradientDescent +INPUT_MAP(ApplyGradientDescent) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(alpha)}, {3, INPUT_DESC(delta)}}; +// ӳ䣬룬varΪ1alphaΪ2deltaΪ3 +ATTR_MAP(ApplyGradientDescent) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyGradientDescent) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(ApplyGradientDescent, kNameApplyGradientDescent, ADPT_DESC(ApplyGradientDescent)) +// עApplyGradientDescentkNameApplyGradientDescent + +// ApplyPowerSignD +INPUT_MAP(ApplyPowerSignD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(m)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(logbase)}, {5, INPUT_DESC(sign_decay)}, {6, INPUT_DESC(beta)}, + {7, INPUT_DESC(grad)}}; +// ӳ䣬߸룬varΪ1mΪ2lrΪ3logbaseΪ4sign_decayΪ5 +// betaΪ6gradΪ7 +ATTR_MAP(ApplyPowerSignD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyPowerSignD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(m)}}; +// ӳ䣬varΪ0mΪ1 +REG_ADPT_DESC(ApplyPowerSignD, kNameApplyPowerSign, ADPT_DESC(ApplyPowerSignD)) +// עApplyPowerSignDkNameApplyPowerSign + +// ApplyProximalGradientDescent +INPUT_MAP(ApplyProximalGradientDescent) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(alpha)}, {3, INPUT_DESC(l1)}, {4, INPUT_DESC(l2)}, {5, INPUT_DESC(delta)}}; +// ӳ䣬룬varΪ1alphaΪ2l1Ϊ3l2Ϊ4deltaΪ5 +ATTR_MAP(ApplyProximalGradientDescent) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyProximalGradientDescent) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(ApplyProximalGradientDescent, kNameApplyProximalGradientDescent, ADPT_DESC(ApplyProximalGradientDescent)) +// עApplyProximalGradientDescentkNameApplyProximalGradientDescent +// +// SGD +INPUT_MAP(SGD) = {{1, INPUT_DESC(parameters)}, {2, INPUT_DESC(gradient)}, {3, INPUT_DESC(learning_rate)}, + {4, INPUT_DESC(accum)}, {5, INPUT_DESC(momentum)}, {6, INPUT_DESC(stat)}}; +// ӳ䣬룬parametersΪ1gradientΪ2lrΪ3logbaseΪ4sign_decayΪ5statΪ6 +ATTR_MAP(SGD) = {{"dampening", ATTR_DESC(dampening, AnyTraits())}, + {"weight_decay", ATTR_DESC(weight_decay, AnyTraits())}, + {"nesterov", ATTR_DESC(nesterov, AnyTraits())}}; +// ӳ䣬ԣ"dampening""weight_decay"Ϊfloatͣ"nesterov"Ϊbool +OUTPUT_MAP(SGD) = {{0, OUTPUT_DESC(parameters)}}; +// ӳ䣬parametersΪ0 +REG_ADPT_DESC(SGD, kNameSGD, ADPT_DESC(SGD)) +// עSGDkNameSGD + +// SparseApplyAdagradD +INPUT_MAP(SparseApplyAdagradD) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(grad)}, {4, INPUT_DESC(indices)}}; +// ӳ䣬ĸ룬varΪ1accumΪ2gradΪ3indicesΪ4 +ATTR_MAP(SparseApplyAdagradD) = {{"lr", ATTR_DESC(lr, AnyTraits())}, + {"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬ԣ"lr"Ϊfloatͣ"use_locking"Ϊbool +OUTPUT_MAP(SparseApplyAdagradD) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(SparseApplyAdagradD, kNameSparseApplyAdagrad, ADPT_DESC(SparseApplyAdagradD)) +// עSparseApplyAdagradD kNameSparseApplyAdagrad + +// ApplyProximalAdagradD +INPUT_MAP(ApplyProximalAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(l1)}, {5, INPUT_DESC(l2)}, {6, INPUT_DESC(grad)}}; +// ӳ䣬룬varΪ1accumΪ2lrΪ3l1Ϊ4l2Ϊ5gradΪ6 +ATTR_MAP(ApplyProximalAdagradD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyProximalAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +// ӳ䣬varΪ0accumΪ1 +REG_ADPT_DESC(ApplyProximalAdagradD, kNameApplyProximalAdagrad, ADPT_DESC(ApplyProximalAdagradD)) +// עApplyProximalAdagradDkNameApplyProximalAdagrad +// +// SparseApplyProximalAdagradD +INPUT_MAP(SparseApplyProximalAdagradD) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(lr)}, + {4, INPUT_DESC(l1)}, {5, INPUT_DESC(l2)}, {6, INPUT_DESC(grad)}, + {7, INPUT_DESC(indices)}}; +// ӳ䣬߸룬varΪ1accumΪ2lrΪ3l1Ϊ4l2Ϊ5gradΪ6indicesΪ7 +ATTR_MAP(SparseApplyProximalAdagradD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(SparseApplyProximalAdagradD) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}}; +// ӳ䣬varΪ0accumΪ1 +REG_ADPT_DESC(SparseApplyProximalAdagradD, kNameSparseApplyProximalAdagradD, ADPT_DESC(SparseApplyProximalAdagradD)) +// עSparseApplyProximalAdagradDkNameSparseApplyProximalAdagradD + +// SparseApplyFtrlD +INPUT_MAP(SparseApplyFtrlD) = {{1, INPUT_DESC(var)}, + {2, INPUT_DESC(accum)}, + {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, + {5, INPUT_DESC(indices)}}; +// ӳ䣬룬varΪ1accumΪ2linearΪ3gradΪ4indicesΪ5 +ATTR_MAP(SparseApplyFtrlD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}, + {"lr", ATTR_DESC(lr, AnyTraits())}, + {"l1", ATTR_DESC(l1, AnyTraits())}, + {"l2", ATTR_DESC(l2, AnyTraits())}, + {"lr_power", ATTR_DESC(lr_power, AnyTraits())}}; +// ӳ䣬ԣ"use_locking""lr""l1""l2""lr_power"Ϊfloat +OUTPUT_MAP(SparseApplyFtrlD) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(SparseApplyFtrlD, kNameSparseApplyFtrlD, ADPT_DESC(SparseApplyFtrlD)) +// עSparseApplyFtrlDkNameSparseApplyFtrlD + +// SparseApplyFtrlV2D +INPUT_MAP(SparseApplyFtrlV2D) = {{1, INPUT_DESC(var)}, + {2, INPUT_DESC(accum)}, + {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, + {5, INPUT_DESC(indices)}}; +// ӳ䣬룬varΪ1accumΪ2linearΪ3gradΪ4indicesΪ5 +ATTR_MAP(SparseApplyFtrlV2D) = {{"lr", ATTR_DESC(lr, AnyTraits())}, {"l1", ATTR_DESC(l1, AnyTraits())}}; +// ӳ䣬ԣ"l1""l2"Ϊfloat +OUTPUT_MAP(SparseApplyFtrlV2D) = {{0, OUTPUT_DESC(var)}, {1, OUTPUT_DESC(accum)}, {2, OUTPUT_DESC(linear)}}; +// ӳ䣬varΪ0accumΪ1linearΪ2 +REG_ADPT_DESC(SparseApplyFtrlV2D, kNameSparseApplyFtrlV2D, ADPT_DESC(SparseApplyFtrlV2D)) +// עSparseApplyFtrlV2D kNameSparseApplyFtrlV2D + +// ApplyFtrl +INPUT_MAP(ApplyFtrl) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(accum)}, {3, INPUT_DESC(linear)}, + {4, INPUT_DESC(grad)}, {5, INPUT_DESC(lr)}, {6, INPUT_DESC(l1)}, + {7, INPUT_DESC(l2)}, {8, INPUT_DESC(lr_power)}}; +// ӳ䣬а˸룬varΪ1accumΪ2linearΪ3gradΪ4lrΪ5l1Ϊ6l2Ϊ7lr_powerΪ8 +ATTR_MAP(ApplyFtrl) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬һԣ"use_locking"Ϊbool +OUTPUT_MAP(ApplyFtrl) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(ApplyFtrl, kNameApplyFtrl, ADPT_DESC(ApplyFtrl)) +// עApplyFtrlkNameApplyFtrl + +// ApplyRMSPropD +INPUT_MAP(ApplyRMSPropD) = { + {1, INPUT_DESC(var)}, {2, INPUT_DESC(ms)}, {3, INPUT_DESC(mom)}, {4, INPUT_DESC(lr)}, {5, INPUT_DESC(grad)}}; +// ӳ䣬룬varΪ1msΪ2momΪ3lrΪ4gradΪ5 +INPUT_ATTR_MAP(ApplyRMSPropD) = {{6, ATTR_DESC(rho, AnyTraits())}, + {7, ATTR_DESC(momentum, AnyTraits())}, + {8, ATTR_DESC(epsilon, AnyTraits())}}; +//ӳ䣬3rhoΪ6momentumΪ7epsilonΪ8 +ATTR_MAP(ApplyRMSPropD) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}} +// ӳ䣬һԣ"use_locking"Ϊbool; +OUTPUT_MAP(ApplyRMSPropD) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(ApplyRMSPropD, kNameApplyRMSProp, ADPT_DESC(ApplyRMSPropD)) +// עApplyRMSPropD kNameApplyRMSProp + +// ApplyCenteredRMSProp +INPUT_MAP(ApplyCenteredRMSProp) = {{1, INPUT_DESC(var)}, {2, INPUT_DESC(mg)}, {3, INPUT_DESC(ms)}, + {4, INPUT_DESC(mom)}, {5, INPUT_DESC(grad)}, {6, INPUT_DESC(lr)}, + {7, INPUT_DESC(rho)}, {8, INPUT_DESC(momentum)}, {9, INPUT_DESC(epsilon)}}; +// ӳ䣬оŸ룬varΪ1mgΪ2msΪ3momΪ4gradΪ5lrΪ6rhoΪ7momentumΪ8epsilonΪ9 +ATTR_MAP(ApplyCenteredRMSProp) = {{"use_locking", ATTR_DESC(use_locking, AnyTraits())}}; +// ӳ䣬rhoΪ6momentumΪ7epsilonΪ8 +OUTPUT_MAP(ApplyCenteredRMSProp) = {{0, OUTPUT_DESC(var)}}; +// ӳ䣬varΪ0 +REG_ADPT_DESC(ApplyCenteredRMSProp, kNameApplyCenteredRMSProp, ADPT_DESC(ApplyCenteredRMSProp)) +// עApplyCenteredRMSPropkNameApplyCenteredRMSProp +} // namespace mindspore::transform -- 2.34.1 From dcfd6500ea439704cf2d841c5d0ac0e8db31595f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:14:08 +0800 Subject: [PATCH 31/72] ADD file via upload --- .../nonlinear_fuc_ops_declare.cc | 290 ++++++++++++++++++ 1 file changed, 290 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/nonlinear_fuc_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/nonlinear_fuc_ops_declare.cc b/mindspore/ccsrc/transform-update/nonlinear_fuc_ops_declare.cc new file mode 100644 index 00000000000..eb95dfc5e58 --- /dev/null +++ b/mindspore/ccsrc/transform-update/nonlinear_fuc_ops_declare.cc @@ -0,0 +1,290 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/nonlinear_fuc_ops_declare.h" + +namespace mindspore::transform { +// Relu +INPUT_MAP(Relu) = {{1, INPUT_DESC(x)}}; +//ӳ䣬xΪ1 +ATTR_MAP(Relu) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(Relu) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(Relu, prim::kPrimRelu->name(), ADPT_DESC(Relu)) +// עReluprim::kPrimRelu->name() + +// ReluV2 +INPUT_MAP(ReluV2) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(ReluV2) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(ReluV2) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}}; +// ӳ䣬yΪ0maskΪ1 +REG_ADPT_DESC(ReluV2, kNameReluV2, ADPT_DESC(ReluV2)) +// עReluV2kNameReluV2 + +// Elu +INPUT_MAP(Elu) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Elu) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}}; +// ӳ䣬"alpha"Ϊfloat +OUTPUT_MAP(Elu) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Elu, kNameElu, ADPT_DESC(Elu)) +// עElukNameElu + +// EluGrad +INPUT_MAP(EluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(activations)}}; +// ӳ䣬xΪ1activationsΪ2 +ATTR_MAP(EluGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(EluGrad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(EluGrad, kNameEluGrad, ADPT_DESC(EluGrad)) +// עEluGradkNameEluGrad + +// PRelu +INPUT_MAP(PRelu) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(weight)}}; +// ӳ䣬xΪ1weightΪ2 +ATTR_MAP(PRelu) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(PRelu) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(PRelu, kNamePrelu, ADPT_DESC(PRelu)) +// עPRelukNamePrelu + +// PReluGrad +INPUT_MAP(PReluGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(features)}, {3, INPUT_DESC(weights)}}; +// ӳ䣬gradsΪ1featuresΪ2weightsΪ3 +ATTR_MAP(PReluGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(PReluGrad) = {{0, OUTPUT_DESC(dx)}, {1, OUTPUT_DESC(da)}}; +// ӳ䣬dxΪ0daΪ1 +REG_ADPT_DESC(PReluGrad, kNamePreluGrad, ADPT_DESC(PReluGrad)) +// עPReluGradkNamePreluGrad + +// Selu +INPUT_MAP(Selu) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Selu) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Selu) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Selu, kNameSelu, ADPT_DESC(Selu)) +// עSelukNameSelu + +// Sigmoid +INPUT_MAP(Sigmoid) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Sigmoid) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Sigmoid) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Sigmoid, kNameSigmoid, ADPT_DESC(Sigmoid)) +// עSigmoidkNameSigmoid + +// SigmoidGrad +INPUT_MAP(SigmoidGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +// ӳ䣬yΪ1dyΪ2 +ATTR_MAP(SigmoidGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(SigmoidGrad) = {{0, OUTPUT_DESC(z)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(SigmoidGrad, kNameSigmoidGrad, ADPT_DESC(SigmoidGrad)) +// עSigmoidGradkNameSigmoidGrad + +// HardSwish +INPUT_MAP(HardSwish) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(HardSwish) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(HardSwish) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(HardSwish, kNameHSwish, ADPT_DESC(HardSwish)) +// עHardSwishkNameHSwish + +// HardSwishGrad +INPUT_MAP(HardSwishGrad) = {{1, INPUT_DESC(grad)}, {2, INPUT_DESC(x)}}; +// ӳ䣬gradΪ1xΪ2 +ATTR_MAP(HardSwishGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(HardSwishGrad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(HardSwishGrad, kNameHSwishGrad, ADPT_DESC(HardSwishGrad)) +// עHardSwishGradkNameHSwishGrad + +// HSigmoid +INPUT_MAP(HardSigmoid) = {{1, INPUT_DESC(input_x)}}; +// ӳ䣬input_xΪ1 +ATTR_MAP(HardSigmoid) = {{"alpha", ATTR_DESC(alpha, AnyTraits())}, + {"beta", ATTR_DESC(beta, AnyTraits())}}; +// ӳ䣬"alpha""beta"Ϊfloat +OUTPUT_MAP(HardSigmoid) = {{0, OUTPUT_DESC(output_y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(HardSigmoid, kNameHSigmoid, ADPT_DESC(HardSigmoid)) +// עHardSigmoidkNameHSigmoid + +// Relu6 +INPUT_MAP(Relu6) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Relu6) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Relu6) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Relu6, kNameReLU6, ADPT_DESC(Relu6)) +// עRelu6kNameReLU6 + +// Relu6Grad +INPUT_MAP(Relu6Grad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +// ӳ䣬gradientsΪ1featuresΪ2 +ATTR_MAP(Relu6Grad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Relu6Grad) = {{0, OUTPUT_DESC(backprops)}}; +// ӳ䣬backpropsΪ0 +REG_ADPT_DESC(Relu6Grad, kNameReLU6Grad, ADPT_DESC(Relu6Grad)) +// עRelu6GradkNameReLU6Grad + +// Softsign +INPUT_MAP(Softsign) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Softsign) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Softsign) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Softsign, kNameSoftsign, ADPT_DESC(Softsign)) +// עSoftsignkNameSoftsign + +// Softplus +INPUT_MAP(Softplus) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Softplus) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Softplus) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Softplus, kNameSoftplus, ADPT_DESC(Softplus)) +// עSoftplus kNameSoftplus + +// SoftplusGrad +INPUT_MAP(SoftplusGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +// ӳ䣬gradientsΪ1featuresΪ2 +ATTR_MAP(SoftplusGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(SoftplusGrad) = {{0, OUTPUT_DESC(backprops)}}; +// ӳ䣬backpropsΪ0 +REG_ADPT_DESC(SoftplusGrad, kNameSoftplusGrad, ADPT_DESC(SoftplusGrad)) +// עSoftplusGradkNameSoftplusGrad + +// ReluGrad +INPUT_MAP(ReluGrad) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(features)}}; +// ӳ䣬gradientsΪ1featuresΪ2 +ATTR_MAP(ReluGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(ReluGrad) = {{0, OUTPUT_DESC(backprops)}}; +// ӳ䣬backpropsΪ0 +REG_ADPT_DESC(ReluGrad, prim::kPrimReluGrad->name(), ADPT_DESC(ReluGrad)) +// עReluGradprim::kPrimReluGrad->name() + +// ReluGradV2 +INPUT_MAP(ReluGradV2) = {{1, INPUT_DESC(gradients)}, {2, INPUT_DESC(mask)}}; +// ӳ䣬gradientsΪ1maskΪ2 +ATTR_MAP(ReluGradV2) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(ReluGradV2) = {{0, OUTPUT_DESC(backprops)}}; +// ӳ䣬backpropsΪ0 +REG_ADPT_DESC(ReluGradV2, kNameReluGradV2, ADPT_DESC(ReluGradV2)) +// עReluGradV2kNameReluGradV2 + +// Tanh +INPUT_MAP(Tanh) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Tanh) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Tanh) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Tanh, prim::kPrimTanh->name(), ADPT_DESC(Tanh)) +// עTanhprim::kPrimTanh->name() + +// TanhGrad +INPUT_MAP(TanhGrad) = {{1, INPUT_DESC(y)}, {2, INPUT_DESC(dy)}}; +// ӳ䣬yΪ1dyΪ2 +ATTR_MAP(TanhGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(TanhGrad) = {{0, OUTPUT_DESC(z)}}; +// ӳ䣬zΪ0 +REG_ADPT_DESC(TanhGrad, prim::kPrimTanhGrad->name(), ADPT_DESC(TanhGrad)) +// עNPUTanhGradprim::kPrimTanhGrad->name() + +// Mish +INPUT_MAP(Mish) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Mish) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Mish) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Mish, kNameMish, ADPT_DESC(Mish)) +// עMishkNameMish + +// GeLU +INPUT_MAP(Gelu) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Gelu) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Gelu) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Gelu, prim::kPrimGeLU->name(), ADPT_DESC(Gelu)) +// עGeluprim::kPrimGeLU->name() + +// GeLUGrad +INPUT_MAP(GeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(y)}}; +// ӳ䣬xΪ1activationsΪ2yΪ3 +ATTR_MAP(GeluGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(GeluGrad) = {{0, OUTPUT_DESC(z)}}; +// ӳ䣬zΪ0 +REG_ADPT_DESC(GeluGrad, prim::kPrimGeLUGrad->name(), ADPT_DESC(GeluGrad)) +// עGeluGradprim::kPrimGeLUGrad->name() + +// FastGeLU +INPUT_MAP(FastGelu) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(FastGelu) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(FastGelu) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(FastGelu, prim::kPrimFastGeLU->name(), ADPT_DESC(FastGelu)) +// עFastGelu prim::kPrimFastGeLU->name() + +// FastGeLUGrad +INPUT_MAP(FastGeluGrad) = {{1, INPUT_DESC(dy)}, {2, INPUT_DESC(x)}}; +// ӳ䣬dyΪ1xΪ2 +ATTR_MAP(FastGeluGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(FastGeluGrad) = {{0, OUTPUT_DESC(z)}}; +// ӳ䣬zΪ0 +REG_ADPT_DESC(FastGeluGrad, prim::kPrimFastGeLUGrad->name(), ADPT_DESC(FastGeluGrad)) +// עFastGeluGrad prim::kPrimFastGeLUGrad->name() + +// LeakyRelu +INPUT_MAP(LeakyRelu) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(LeakyRelu) = {{"alpha", ATTR_DESC(negative_slope, AnyTraits())}}; +// ӳ䣬"alpha"Ϊfloat +OUTPUT_MAP(LeakyRelu) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(LeakyRelu, prim::kPrimLeakyRelu->name(), ADPT_DESC(LeakyRelu)) +// עLeakyRelu prim::kPrimLeakyRelu->name() +} // namespace mindspore::transform -- 2.34.1 From d076ece04e365309630cca6f211f0015aacb953d Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:14:25 +0800 Subject: [PATCH 32/72] ADD file via upload --- .../npu_loss_scale_ops_declare.cc | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/npu_loss_scale_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/npu_loss_scale_ops_declare.cc b/mindspore/ccsrc/transform-update/npu_loss_scale_ops_declare.cc new file mode 100644 index 00000000000..5ff7ccd9d66 --- /dev/null +++ b/mindspore/ccsrc/transform-update/npu_loss_scale_ops_declare.cc @@ -0,0 +1,49 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/npu_loss_scale_ops_declare.h" + +namespace mindspore::transform { +// NPUGetFloatStatus +INPUT_MAP(NPUGetFloatStatus) = {{1, INPUT_DESC(addr)}}; +// ӳ䣬addrΪ1 +OUTPUT_MAP(NPUGetFloatStatus) = {{0, OUTPUT_DESC(data)}}; +// ӳ䣬dataΪ0 +ATTR_MAP(NPUGetFloatStatus) = EMPTY_ATTR_MAP; +//ӳ䣬 +REG_ADPT_DESC(NPUGetFloatStatus, kNameNPUGetFloatStatus, ADPT_DESC(NPUGetFloatStatus)) +// עNPUGetFloatStatuskNameNPUGetFloatStatus + +// NPUAllocFloatStatus +INPUT_MAP(NPUAllocFloatStatus) = EMPTY_INPUT_MAP; +// ӳ䣬 +ATTR_MAP(NPUAllocFloatStatus) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(NPUAllocFloatStatus) = {{0, OUTPUT_DESC(data)}}; +// ӳ䣬dataΪ0 +REG_ADPT_DESC(NPUAllocFloatStatus, kNameNPUAllocFloatStatus, ADPT_DESC(NPUAllocFloatStatus)) +// עNPUAllocFloatStatuskNameNPUAllocFloatStatus + +// NPUClearFloatStatus +INPUT_MAP(NPUClearFloatStatus) = {{1, INPUT_DESC(addr)}}; +// ӳ䣬addrΪ1 +OUTPUT_MAP(NPUClearFloatStatus) = {{0, OUTPUT_DESC(data)}}; +// ӳ䣬dataΪ0 +ATTR_MAP(NPUClearFloatStatus) = EMPTY_ATTR_MAP; +//ӳ䣬 +REG_ADPT_DESC(NPUClearFloatStatus, kNameNPUClearFloatStatus, ADPT_DESC(NPUClearFloatStatus)) +// עNPUClearFloatStatuskNameNPUClearFloatStatus +} // namespace mindspore::transform -- 2.34.1 From 2ace1c5911f99b946da08d2bad50ac89a22aa906 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:14:43 +0800 Subject: [PATCH 33/72] ADD file via upload --- .../ccsrc/transform-update/onnx_exporter.cc | 3762 +++++++++++++++++ 1 file changed, 3762 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/onnx_exporter.cc diff --git a/mindspore/ccsrc/transform-update/onnx_exporter.cc b/mindspore/ccsrc/transform-update/onnx_exporter.cc new file mode 100644 index 00000000000..0d521a8ab5b --- /dev/null +++ b/mindspore/ccsrc/transform-update/onnx_exporter.cc @@ -0,0 +1,3762 @@ +/** + * Copyright 2020-2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "mindspore/core/ops/core_ops.h" +#include "ir/func_graph.h" +#include "ir/param_info.h" +#include "ir/tensor.h" +#include "proto/onnx.pb.h" +#include "utils/check_convert_utils.h" +#include "utils/hash_map.h" +#include "utils/ms_context.h" + +namespace mindspore { +const int ONNX_VERSION = 11; +const int kZeroNum = 0; +const int kOneNum = 1; +const int kTwoNum = 2; +const int kThreeNum = 3; +const int kFourNum = 4; +const int kFiveNum = 5; +const int64_t kOneNumLong = 1; +const float weight_for_mul = 0.5; +enum OpMergeMode { + OP_MERGE_UNDEFINED = 0, // undefined behavior + OP_MERGE_IGNORE = 1, // indicate an input op merged into other op in compute node list + OP_MERGE_CONV = 2, // indicate `MindSpore Conv + BiasAdd` --> `ONNX Conv` + OP_MERGE_GEMM = 3, // indicate `MindSpore MatMul + BiasAdd` --> `ONNX Gemm` + OP_MERGE_BATCH_NORM = 4, // indicate `MindSpore BatchNorm(x)[0]` --> `ONNX Batch Normalization` + OP_MERGE_MAXPOOL_WITH_ARGMAX = 5, // indicate `MindSpore MaxPoolWithArgmax(x)[0]` --> `ONNX MaxPool` + OP_MERGE_LAYER_NORM = 6, // indicate `MindSpore LayerNorm(x)[0]` --> `ONNX MeanVarianceNormalization` + OP_MERGE_CONV2D_TRANSPOSE = 7, // indicate `MindSpore ConvTranspose + BiasAdd` --> `ONNX ConvTranspose` +}; + +struct OpMergedInfo { + OpMergeMode mode = OP_MERGE_UNDEFINED; + int referred_count = 0; +}; + +using GenAttrFuncType = + std::function; + +bool IsIgnoredIdentityNode(const AnfNodePtr &node) { + return IsPrimitiveCNode(node, prim::kPrimDepend) || IsPrimitiveCNode(node, prim::kPrimLoad); +} + +/* + If true, the node should not be referenced by anything and should not be contributing to any + ref counts itself + 如果返回值为true,则该节点不应被任何对象引用,也不应参与任何ref自身计数 + */ +bool IsZeroRefcountNode(const AnfNodePtr &node) { return HasAbstractMonad(node) || IsIgnoredIdentityNode(node); } + +// Ideally this should be applied to every node->input() call, not only inside GetNodeInputName +//这应该应用于每个node->input()调用 +static AnfNodePtr GetRealInput(const AnfNodePtr &origin_input) { + AnfNodePtr input = origin_input; + while (IsIgnoredIdentityNode(input)) { + input = input->cast()->inputs().at(1); + } + return input; +} + +template +void SetAttrValueToProto(const ValuePtr &value, onnx::AttributeProto_AttributeType attr_type, + onnx::AttributeProto *const attr_proto, const PrimitivePtr &) { + auto casted_value = dyn_cast(value); + if (casted_value == nullptr) { + MS_LOG(EXCEPTION) << "Cast value " << value->ToString() << " to type T failed."; + } + auto attr_value = casted_value->value(); + switch (attr_type) { + case onnx::AttributeProto_AttributeType_INT: + attr_proto->set_i(static_cast<::google::protobuf::int64>(attr_value)); + break; + case onnx::AttributeProto_AttributeType_FLOAT: + attr_proto->set_f(static_cast(attr_value)); + break; + case onnx::AttributeProto_AttributeType_INTS: + for (size_t i = 0; i < rep_cnt; ++i) { + attr_proto->add_ints(static_cast<::google::protobuf::int64>(attr_value)); + } + break; + case onnx::AttributeProto_AttributeType_FLOATS: + for (size_t i = 0; i < rep_cnt; ++i) { + attr_proto->add_floats(static_cast(attr_value)); + } + break; + default: + MS_LOG(EXCEPTION) << "Convert attribute fail, unexpected ONNX type " << attr_type; + } + attr_proto->set_type(attr_type); +} +//设置proto参数 +template +void SetAttrTupleValueToProto(const ValuePtr &value, onnx::AttributeProto_AttributeType attr_type, + onnx::AttributeProto *const attr_proto, const PrimitivePtr &) { + auto tuple_ptr = dyn_cast(value); + if (tuple_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Cast value from type " << value->type_name() << " to ValueTuple failed."; + } + switch (attr_type) { + case onnx::AttributeProto_AttributeType_INTS: + for (size_t i = beg_idx; i < tuple_ptr->size(); ++i) { + attr_proto->add_ints(GetValue((*tuple_ptr)[i])); + } + break; + case onnx::AttributeProto_AttributeType_INT: + attr_proto->set_i(GetValue((*tuple_ptr)[beg_idx])); + break; + case onnx::AttributeProto_AttributeType_FLOATS: + for (size_t i = beg_idx; i < tuple_ptr->size(); ++i) { + attr_proto->add_floats(GetValue((*tuple_ptr)[i])); + } + break; + default: + MS_LOG(EXCEPTION) << "Convert attribute fail, unexpected ONNX type " << attr_type; + } + attr_proto->set_type(attr_type); +} +//设置proto参数 +void SetPoolingPadMode(const ValuePtr &value, onnx::AttributeProto_AttributeType, + onnx::AttributeProto *const attr_proto, const PrimitivePtr &) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + int64_t attr_value; + CheckAndConvertUtils::GetPadModEnumValue(value, &attr_value, true); + if (attr_value == PadMode::VALID) { + attr_proto->set_s("VALID"); + } else { + attr_proto->set_s("SAME_UPPER"); + } +} +//设置池化模型 +void SetConvPadding(const ValuePtr &value, onnx::AttributeProto_AttributeType, onnx::AttributeProto *const attr_proto, + const PrimitivePtr &prim) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + int64_t attr_value; + CheckAndConvertUtils::GetPadModEnumValue(value, &attr_value); + if (attr_value == PadMode::VALID) { + attr_proto->set_s("VALID"); + } else if (attr_value == PadMode::SAME) { + attr_proto->set_s("SAME_UPPER"); + } else { // pad_mode is 'pad', use attribute 'pad_list' to fill ONNX attribute 'pads' + attr_proto->set_name("pads"); + SetAttrTupleValueToProto(prim->GetAttr("pad_list"), onnx::AttributeProto_AttributeType_INTS, attr_proto, prim); + } +} + +void SetConvTransposePadding(const ValuePtr &value, onnx::AttributeProto_AttributeType, + onnx::AttributeProto *const attr_proto, const PrimitivePtr &prim) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + int64_t attr_value; + CheckAndConvertUtils::GetPadModEnumValue(value, &attr_value); + if (attr_value == PadMode::VALID) { + attr_proto->set_s("VALID"); + } else if (attr_value == PadMode::SAME) { + attr_proto->set_s("SAME_LOWER"); + } else { // pad_mode is 'pad', use attribute 'pad_list' to fill ONNX attribute 'pads' + attr_proto->set_name("pads"); + SetAttrTupleValueToProto(prim->GetAttr("pad_list"), onnx::AttributeProto_AttributeType_INTS, attr_proto, prim); + } +} + +PrimitivePtr GetPrimitive(const CNodePtr &node) { + AnfNodePtr op = node->input(kZeroNum); + auto op_value = dyn_cast(op); + MS_EXCEPTION_IF_NULL(op_value); + auto prim = dyn_cast(op_value->value()); + MS_EXCEPTION_IF_NULL(prim); + return prim; +} + +template +T GetOpAttribute(const CNodePtr &node, const std::string &name) { + ValuePtr attr = GetPrimitive(node)->GetAttr(name); + return GetValue(attr); +} + +template +std::shared_ptr GetOpAttributePtr(const CNodePtr &node, const std::string &name) { + ValuePtr attr = GetPrimitive(node)->GetAttr(name); + auto result = dyn_cast(attr); + MS_EXCEPTION_IF_NULL(result); + return result; +} + +std::string MakeOutputName(const std::string &node_name, int output_index) { + return node_name + "_" + std::to_string(output_index); +} + +int64_t RavelIndex(const std::vector &index, const std::vector &shape) { + MS_EXCEPTION_IF_CHECK_FAIL(index.size() <= shape.size(), "Index ndims must be <= shape ndims"); + int64_t result = 0; + int64_t stride = 1; + for (size_t i = 0; i < shape.size() - index.size(); ++i) { + stride *= shape[shape.size() - 1 - i]; + } + for (size_t i = 0; i < index.size(); ++i) { + size_t rev_i = index.size() - 1 - i; + result += index[rev_i] * stride; + stride *= shape[rev_i]; + } + return result; +} + +namespace fp16 { +uint32_t FieldMask(unsigned int field_size) { + const unsigned int BYTE_SIZE = 8; + uint32_t mask = std::numeric_limits::max(); + return mask >> (BYTE_SIZE * sizeof(mask) - field_size); +} + +uint32_t ExponentBias(unsigned int exponent_size) { return (1U << (exponent_size - 1U)) - 1U; } + +uint32_t Fp32ToFp16(float value) { + const unsigned int FP32_M = 23; + const unsigned int FP32_E = 32 - 1 - FP32_M; + const unsigned int FP16_M = 10; + const unsigned int FP16_E = 16 - 1 - FP16_M; + + uint32_t fp32_bits; + auto ret = memcpy_s(reinterpret_cast(&fp32_bits), sizeof(fp32_bits), + reinterpret_cast(&value), sizeof(value)); + if (ret != 0) { + MS_LOG(ERROR) << "Set data memcpy_s failed, ret = " << ret; + } + + uint32_t mantissa = fp32_bits & FieldMask(FP32_M); + uint32_t fp32_exp_mask = FieldMask(FP32_E); + uint32_t fp32_exponent = (fp32_bits >> FP32_M) & fp32_exp_mask; + if (fp32_exponent == fp32_exp_mask) { + MS_LOG(EXCEPTION) << "Tried to convert inf or nan to float16: " << value; + } + uint32_t sign = fp32_bits >> (FP32_E + FP32_M); + + uint32_t fp16_bits = 0; + fp16_bits |= sign << (FP16_E + FP16_M); + uint32_t fp16_exponent = 0; + if (fp32_exponent != 0) { + fp16_exponent = fp32_exponent - ExponentBias(FP32_E) + ExponentBias(FP16_E); + } + if (fp16_exponent >= FieldMask(FP16_E)) { // inf, nan (==), underflow, or overflow (>) + MS_LOG(EXCEPTION) << "Conversion of " << value << " to float16 resulted in exponent overflow or underflow"; + } + fp16_bits |= fp16_exponent << FP16_M; + fp16_bits |= mantissa >> (FP32_M - FP16_M); + + return fp16_bits; +} +} // namespace fp16 + +void AddFloatScalarInitializer(const std::string &name, float value, onnx::TensorProto_DataType type, + onnx::GraphProto *graph_proto) { + onnx::TensorProto *initializer = graph_proto->add_initializer(); + initializer->set_name(name); + if (type == onnx::TensorProto_DataType_FLOAT16) { + uint32_t fp16 = fp16::Fp32ToFp16(value); + initializer->add_int32_data(static_cast(fp16)); + } else if (type == onnx::TensorProto_DataType_FLOAT) { + initializer->add_float_data(value); + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << type; + } + initializer->set_data_type(type); +} + +void AddInt64Tensor1DInitializer(const std::string &name, const std::vector &values, + onnx::GraphProto *graph_proto) { + onnx::TensorProto *initializer = graph_proto->add_initializer(); + initializer->set_name(name); + initializer->set_data_type(onnx::TensorProto_DataType_INT64); + initializer->add_dims(values.size()); + for (auto value : values) { + initializer->add_int64_data(value); + } +} + +void AddFloatTensor1DInitializer(const std::string &name, const std::vector &values, + onnx::TensorProto_DataType type, onnx::GraphProto *graph_proto) { + onnx::TensorProto *initializer = graph_proto->add_initializer(); + initializer->set_name(name); + initializer->add_dims(values.size()); + if (type == onnx::TensorProto_DataType_FLOAT16) { + for (auto value : values) { + uint32_t fp16 = fp16::Fp32ToFp16(value); + initializer->add_int32_data(static_cast(fp16)); + } + } else if (type == onnx::TensorProto_DataType_FLOAT) { + for (auto value : values) { + initializer->add_float_data(value); + } + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << type; + } + initializer->set_data_type(type); +} + +void AddOp(const std::string &type, const std::vector &inputs, const std::vector &outputs, + onnx::GraphProto *graph_proto) { + onnx::NodeProto *op = graph_proto->add_node(); + op->set_op_type(type); + op->set_name(outputs.at(0) + type); + for (const auto &input : inputs) { + op->add_input(input); + } + for (const auto &output : outputs) { + op->add_output(output); + } +} + +void AddClipOp(const std::string &input, const std::string &output, float min, float max, + onnx::TensorProto_DataType type, onnx::GraphProto *graph_proto) { + auto min_input_name = output + "__min_initializer"; + AddFloatScalarInitializer(min_input_name, min, type, graph_proto); + + auto max_input_name = output + "__max_initializer"; + AddFloatScalarInitializer(max_input_name, max, type, graph_proto); + + AddOp("Clip", {input, min_input_name, max_input_name}, {output}, graph_proto); +} + +void AddSliceOp(const std::string &input, const std::string &output, const std::vector &start, + const std::vector &end, const std::vector &axis, const std::vector &step, + onnx::GraphProto *graph_proto) { + auto starts_name = output + "__starts_initializer"; + AddInt64Tensor1DInitializer(starts_name, start, graph_proto); + + auto ends_name = output + "__ends_initializer"; + AddInt64Tensor1DInitializer(ends_name, end, graph_proto); + + auto axes_name = output + "__axes_initializer"; + AddInt64Tensor1DInitializer(axes_name, axis, graph_proto); + + auto steps_name = output + "__steps_initializer"; + AddInt64Tensor1DInitializer(steps_name, step, graph_proto); + + AddOp("Slice", {input, starts_name, ends_name, axes_name, steps_name}, {output}, graph_proto); +} + +void AddSplitOp(const std::string &input, const std::vector &outputs, const std::vector &split, + int64_t axis, onnx::GraphProto *graph_proto) { + if (outputs.size() != split.size()) { + MS_LOG(EXCEPTION) << "Number of splits and number of outputs do not match"; + } + + onnx::NodeProto *split_proto = graph_proto->add_node(); + std::string op_type = "Split"; + split_proto->set_op_type(op_type); + split_proto->set_name(outputs.at(0) + op_type); + split_proto->add_input(input); + for (const auto &output : outputs) { + split_proto->add_output(output); + } + onnx::AttributeProto *axis_attr_proto = split_proto->add_attribute(); + axis_attr_proto->set_name("axis"); + axis_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + axis_attr_proto->set_i(axis); + onnx::AttributeProto *split_attr_proto = split_proto->add_attribute(); + split_attr_proto->set_name("split"); + split_attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + for (int64_t n : split) { + split_attr_proto->add_ints(n); + } +} + +void AddReshapeOp(const std::string &input, const std::string &output, const std::vector &shape, + onnx::GraphProto *graph_proto) { + auto shape_name = output + "__shape_initializer"; + AddInt64Tensor1DInitializer(shape_name, shape, graph_proto); + AddOp("Reshape", {input, shape_name}, {output}, graph_proto); +} + +onnx::TensorProto *AddConstantOfShapeOp(const std::string &shape, const std::string &output, + onnx::GraphProto *graph_proto) { + onnx::NodeProto *op = graph_proto->add_node(); + std::string op_type = "ConstantOfShape"; + op->set_op_type(op_type); + op->set_name(output + op_type); + op->add_input(shape); + op->add_output(output); + onnx::AttributeProto *value_attr = op->add_attribute(); + value_attr->set_name("value"); + value_attr->set_type(onnx::AttributeProto_AttributeType_TENSOR); + onnx::TensorProto *value_proto = value_attr->mutable_t(); + value_proto->add_dims(1); + return value_proto; +} + +void AddCastOp(const std::string &input, const std::string &output, onnx::TensorProto_DataType target_type, + onnx::GraphProto *graph_proto) { + onnx::NodeProto *node_proto = graph_proto->add_node(); + std::string op_type = "Cast"; + node_proto->set_op_type(op_type); + node_proto->set_name(output + op_type); + node_proto->add_input(input); + node_proto->add_output(output); + + onnx::AttributeProto *target_type_attr = node_proto->add_attribute(); + target_type_attr->set_name("to"); + target_type_attr->set_type(onnx::AttributeProto_AttributeType_INT); + target_type_attr->set_i(target_type); +} + +void AddReduceOp(const std::string &op_type, const std::string &input, const std::string &output, + const std::vector &axes, bool keepdims, onnx::GraphProto *graph_proto) { + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_name(output + op_type); + node_proto->set_op_type(op_type); + node_proto->add_input(input); + node_proto->add_output(output); + + onnx::AttributeProto *keep_dims_proto = node_proto->add_attribute(); + keep_dims_proto->set_name("keepdims"); + keep_dims_proto->set_type(onnx::AttributeProto_AttributeType_INT); + keep_dims_proto->set_i(static_cast(keepdims)); + + onnx::AttributeProto *axes_proto = node_proto->add_attribute(); + axes_proto->set_name("axes"); + axes_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + + for (auto axis : axes) { + axes_proto->add_ints(axis); + } +} + +void AddMeanVarianceNormalizationOp(const std::string &input, const std::string &gamma, const std::string &beta, + const std::string &output, const std::vector &axes, float epsilon, + const std::vector &input_shape, onnx::TensorProto_DataType input_type, + onnx::GraphProto *graph_proto) { + auto input_name = output + "_input"; + AddCastOp(input, input_name, onnx::TensorProto_DataType_FLOAT, graph_proto); + auto gamma_name = output + "_gamma"; + AddCastOp(gamma, gamma_name, onnx::TensorProto_DataType_FLOAT, graph_proto); + auto beta_name = output + "_beta"; + AddCastOp(beta, beta_name, onnx::TensorProto_DataType_FLOAT, graph_proto); + + // MeanVarianceNormalization is replaced with equivalent ops because it is not supported by CUDAExecutionProvider + auto meanvariancenormal_node_name = output + "_normalized"; + + auto mean_name = output + "_mean"; + AddReduceOp("ReduceMean", input_name, mean_name, axes, true, graph_proto); + auto centered_name = output + "_centered"; + AddOp("Sub", {input_name, mean_name}, {centered_name}, graph_proto); + + auto sqsum_name = output + "_sqsum"; + AddReduceOp("ReduceSumSquare", centered_name, sqsum_name, axes, true, graph_proto); + float reduce_size = std::accumulate(axes.begin(), axes.end(), 1.0f, + [&input_shape](auto acc, auto axis) { return acc * input_shape[axis]; }); + auto reduce_size_name = output + "_reduce_size"; + AddFloatScalarInitializer(reduce_size_name, reduce_size, onnx::TensorProto_DataType_FLOAT, graph_proto); + auto variance_name = output + "_variance"; + AddOp("Div", {sqsum_name, reduce_size_name}, {variance_name}, graph_proto); + + auto epsilon_name = output + "_epsilon"; + AddFloatScalarInitializer(epsilon_name, epsilon, onnx::TensorProto_DataType_FLOAT, graph_proto); + auto variance_with_epsilon_name = output + "_variance_with_epsilon"; + AddOp("Add", {variance_name, epsilon_name}, {variance_with_epsilon_name}, graph_proto); + auto std_name = output + "_std"; + AddOp("Sqrt", {variance_with_epsilon_name}, {std_name}, graph_proto); + + AddOp("Div", {centered_name, std_name}, {meanvariancenormal_node_name}, graph_proto); + + // Add mul and add node + auto mul_node_name = output + "_rescaled"; + AddOp("Mul", {meanvariancenormal_node_name, gamma_name}, {mul_node_name}, graph_proto); + + // add beta + auto add_node_name = output; + if (input_type == onnx::TensorProto_DataType_FLOAT16) { + add_node_name += "_shifted"; + } + AddOp("Add", {mul_node_name, beta_name}, {add_node_name}, graph_proto); + + if (input_type == onnx::TensorProto_DataType_FLOAT16) { + AddCastOp(add_node_name, output, onnx::TensorProto_DataType_FLOAT16, graph_proto); + } +} + +void AddConcatOp(const std::vector &inputs, const std::string &output, int axis, + onnx::GraphProto *graph_proto) { + onnx::NodeProto *concat_proto = graph_proto->add_node(); + auto op_type = "Concat"; + concat_proto->set_op_type(op_type); + concat_proto->set_name(output + op_type); + for (const auto &input : inputs) { + concat_proto->add_input(input); + } + concat_proto->add_output(output); + onnx::AttributeProto *axis_proto = concat_proto->add_attribute(); + axis_proto->set_name("axis"); + axis_proto->set_type(onnx::AttributeProto_AttributeType_INT); + axis_proto->set_i(axis); +} + +void ConvertBoxesToXywh(const std::string &startpoints, const std::string &endpoints, const std::string ¢erpoints, + const std::string &dimensions, onnx::TensorProto_DataType type, onnx::GraphProto *graph_proto) { + auto coord_sums_name = centerpoints + "__to_div"; + AddOp("Add", {startpoints, endpoints}, {coord_sums_name}, graph_proto); + auto two_name = centerpoints + "__two_initializer"; + AddFloatScalarInitializer(two_name, 2.0f, type, graph_proto); + AddOp("Div", {coord_sums_name, two_name}, {centerpoints}, graph_proto); + + auto coord_diffs_name = dimensions + "__to_add"; + AddOp("Sub", {endpoints, startpoints}, {coord_diffs_name}, graph_proto); + auto one_name = dimensions + "__one_initializer"; + AddFloatScalarInitializer(one_name, 1.0f, type, graph_proto); + AddOp("Add", {coord_diffs_name, one_name}, {dimensions}, graph_proto); +} + +void ConvertBoxesToXyxy(const std::string ¢erpoints, const std::string &dimensions, const std::string &startpoints, + const std::string &endpoints, onnx::TensorProto_DataType type, onnx::GraphProto *graph_proto) { + auto half_name = startpoints + "__half_initializer"; + AddFloatScalarInitializer(half_name, 0.5f, type, graph_proto); + + auto half_dim_name = startpoints + "__half_dim"; + auto half_dim_to_sub_name = startpoints + "__to_sub"; + AddOp("Mul", {dimensions, half_name}, {half_dim_to_sub_name}, graph_proto); + AddOp("Sub", {half_dim_to_sub_name, half_name}, {half_dim_name}, graph_proto); + + AddOp("Sub", {centerpoints, half_dim_name}, {startpoints}, graph_proto); + AddOp("Add", {centerpoints, half_dim_name}, {endpoints}, graph_proto); +} + +void ClipPointsComponent(const std::string &points, const std::string &clipped, float max, int64_t component_idx, + onnx::TensorProto_DataType type, onnx::GraphProto *graph_proto) { + auto res_to_clip_name = clipped + "__clip"; + AddSliceOp(points, res_to_clip_name, {component_idx}, {component_idx + 1}, {1}, {1}, graph_proto); + AddClipOp(res_to_clip_name, clipped, 0.0f, max, type, graph_proto); +} + +namespace while_loop_export { +namespace { +const char CONTROL_PATTERN[] = "\u21B5"; // ↵ +const char LOOP_BODY_PATTERN[] = "\u21BB"; // ↻ +const char AFTER_LOOP_PATTERN[] = "\u2193"; // ↓ + +const size_t LOOP_BODY_INPUT = 2; +const size_t AFTER_LOOP_INPUT = 3; + +bool IsSubgraphNameCorrect(const FuncGraphPtr &func_graph, const std::string &part_pattern) { + auto name = func_graph->ToString(); + return name.find("construct") != std::string::npos && name.find(part_pattern) != std::string::npos; +} + +template +const std::shared_ptr GetNodeInput(const CNodePtr &node, size_t i) { + auto input = GetRealInput(node->input(i)); + auto result = dyn_cast(input); + if (result == nullptr) { + MS_LOG(EXCEPTION) << "Failed to get input " << i << " of node " << node->DebugString(); + } + return result; +} + +template +const std::shared_ptr GetNodeInputValue(const CNodePtr &node, size_t i) { + auto input = GetNodeInput(node, i); + auto result = dyn_cast(input->value()); + if (result == nullptr) { + MS_LOG(EXCEPTION) << "Failed to get a value from input " << i << " of node " << node->DebugString(); + } + return result; +} + +CNodePtr FindLoopSwitchNode(const FuncGraphPtr &control_subgraph) { + if (!IsSubgraphNameCorrect(control_subgraph, CONTROL_PATTERN)) { + MS_LOG(EXCEPTION) << "Expected a loop control structure"; + } + auto lazy_call_node = GetNodeInput(control_subgraph->get_return(), kOneNum); + if (lazy_call_node->inputs().size() != kOneNum || !lazy_call_node->input(kZeroNum)->isa()) { + MS_LOG(EXCEPTION) << "Expected a lazy call node"; + } + auto switch_node = GetNodeInput(lazy_call_node, kZeroNum); + if (!switch_node->IsApply(prim::kPrimSwitch)) { + MS_LOG(EXCEPTION) << "Expected a switch node"; + } + return switch_node; +} + +FuncGraphPtr GetSubgraph(const CNodePtr &switch_node, size_t input_index, const std::string &name_pattern) { + auto input_node = GetNodeInput(switch_node, input_index); + if (!input_node->IsApply(prim::kPrimPartial)) { + MS_LOG(EXCEPTION) << "Expected a partial node"; + } + + auto subgraph = GetNodeInputValue(input_node, kOneNum); + if (!IsSubgraphNameCorrect(subgraph, name_pattern)) { + MS_LOG(EXCEPTION) << "Expected a loop part: " << name_pattern; + } + + return subgraph; +} + +// The inputs of this node are the outputs of ONNX Loop +CNodePtr FindLoopRepeatNode(const FuncGraphPtr &loop_subgraph, const FuncGraphPtr &control_subgraph) { + auto repeat_node = GetNodeInput(loop_subgraph->return_node(), kOneNum); + auto maybe_control_graph = GetNodeInputValue(repeat_node, kZeroNum); + MS_EXCEPTION_IF_CHECK_FAIL(maybe_control_graph == control_subgraph, "Loop matching failed"); + return repeat_node; +} + +struct LoopConditionInfo { + int64_t begin; + int64_t end; + int64_t step; +}; + +/* + NOTE: loop support is currently very limited, because proper condition export requires more graph surgery (copying + condition expression before and inside Loop subgraph) + The only while loop form supported currently is the one used in GNMT v2's Beam Search. Python example: + i = begin + while i < end + ... + i += step + To enable proper support for arbitrary while loop contitions, condition calculation should be duplicated inside the + Loop supgraph. But exporting the same ops twice with different names is not currently supported. + */ +LoopConditionInfo TraceLoopConditionInfo(const CNodePtr &start_node, const CNodePtr &cond_node, + const FuncGraphPtr &control_subgraph, const CNodePtr &loop_repeat_node) { + MS_EXCEPTION_IF_CHECK_FAIL(cond_node->IsApply(prim::kPrimLess), "Expected Less node"); + + auto counter = GetNodeInput(cond_node, kOneNum); + auto end_tensor = GetNodeInputValue(cond_node, kTwoNum); + MS_EXCEPTION_IF_CHECK_FAIL(end_tensor->shape_c().empty(), "Expected a scalar tensor"); + auto end = *reinterpret_cast(end_tensor->data_c()); + + const auto &subgraph_args = control_subgraph->parameters(); + auto counter_input_pos = std::find(subgraph_args.begin(), subgraph_args.end(), counter) - subgraph_args.begin(); + + auto begin_tensor = GetNodeInputValue(start_node, 1UL + static_cast(counter_input_pos)); + MS_EXCEPTION_IF_CHECK_FAIL(begin_tensor->shape_c().empty(), "Expected a scalar tensor"); + auto begin = *reinterpret_cast(begin_tensor->data_c()); + + auto increment_node = GetNodeInput(loop_repeat_node, 1UL + static_cast(counter_input_pos)); + MS_EXCEPTION_IF_CHECK_FAIL(increment_node->IsApply(prim::kPrimAdd), "Expected Add node"); + auto step_tensor = GetNodeInputValue(increment_node, kTwoNum); + MS_EXCEPTION_IF_CHECK_FAIL(step_tensor->shape_c().empty(), "Expected a scalar tensor"); + auto step = *reinterpret_cast(step_tensor->data_c()); + + return LoopConditionInfo{begin, end, step}; +} + +// result[i] is which control subgraph input should be taken for pos i to match the order of loop subgraph inputs +std::vector TraceLoopToControlMap(const FuncGraphPtr &control_subgraph) { + std::vector result; + + auto switch_node = FindLoopSwitchNode(control_subgraph); + auto loop_partial_node = GetNodeInput(switch_node, kTwoNum); + const auto &control_params = control_subgraph->parameters(); + int64_t auxiliary_inputs_num = 2; + for (size_t i = static_cast(auxiliary_inputs_num); i < loop_partial_node->inputs().size(); ++i) { + auto loop_param = GetNodeInput(loop_partial_node, i); + auto control_param_pos = + std::find(control_params.begin(), control_params.end(), loop_param) - control_params.begin(); + result.push_back(control_param_pos); + } + + return result; +} + +std::vector TraceAfterToLoopMap(const FuncGraphPtr &control_subgraph) { + std::vector result; + + auto switch_node = FindLoopSwitchNode(control_subgraph); + auto loop_partial_node = GetNodeInput(switch_node, kTwoNum); + auto after_partial_node = GetNodeInput(switch_node, kThreeNum); + const auto &loop_params = loop_partial_node->inputs(); + int64_t auxiliary_inputs_num = 2; + for (size_t i = static_cast(auxiliary_inputs_num); i < after_partial_node->inputs().size(); ++i) { + auto after_param = GetNodeInput(after_partial_node, i); + auto after_param_pos = std::find(loop_params.begin(), loop_params.end(), after_param) - loop_params.begin(); + result.push_back(after_param_pos - auxiliary_inputs_num); + } + + return result; +} + +std::vector TraceIgnoredLoopParams(const CNodePtr &start_node, const std::vector &loop_to_control_map) { + auto inputs_num = start_node->inputs().size() - 1; + std::vector result(inputs_num); + for (size_t loop_i = 0; loop_i < inputs_num; ++loop_i) { + auto control_i = loop_to_control_map.at(loop_i); + const auto &input = start_node->input(control_i + 1); + if ((input->isa() && input->cast()->has_default()) || HasAbstractMonad(input)) { + result.at(loop_i) = true; + } + } + return result; +} +} // namespace + +bool IsControlSubgraph(const ValuePtr &func_graph_node) { + auto func_graph = dyn_cast(func_graph_node); + return func_graph != nullptr && IsSubgraphNameCorrect(func_graph, CONTROL_PATTERN); +} + +bool IsLoopBodyReturnNode(const CNodePtr &node, const FuncGraphPtr &func_graph) { + return IsSubgraphNameCorrect(func_graph, LOOP_BODY_PATTERN) && node == func_graph->get_return(); +} + +bool IsAfterLoopReturnNode(const CNodePtr &node, const FuncGraphPtr &func_graph) { + return IsSubgraphNameCorrect(func_graph, AFTER_LOOP_PATTERN) && node == func_graph->get_return(); +} + +struct LoopParts { + LoopConditionInfo loop_condition_info; + std::vector> after_param_to_output_indices; + std::vector ignored_loop_param_indices; + std::vector> used_loop_to_control_param_indices; + CNodePtr repeat_node; + FuncGraphPtr loop_subgraph; + FuncGraphPtr after_loop_subgraph; +}; +// 匹配图模式的主函数,接受一个CNode节点作为起始节点 +LoopParts MatchGraph(const CNodePtr &start_node) { + LoopParts result;// 存储匹配结果的对象 + // 获取控制子图,根据CNode的输入获取ValueNode,再获取其中的FuncGraph + auto control_subgraph_value = dyn_cast(start_node->input(0)); + MS_EXCEPTION_IF_NULL(control_subgraph_value); + auto control_subgraph = dyn_cast(control_subgraph_value->value()); + MS_EXCEPTION_IF_NULL(control_subgraph); + // 寻找循环中的Switch节点,找到条件节点和循环体子图 + auto switch_node = FindLoopSwitchNode(control_subgraph); + auto cond_node = GetNodeInput(switch_node, kOneNum); + // 获取循环体子图 + result.loop_subgraph = GetSubgraph(switch_node, LOOP_BODY_INPUT, LOOP_BODY_PATTERN); + // 寻找循环中的Repeat节点 + result.repeat_node = FindLoopRepeatNode(result.loop_subgraph, control_subgraph); + // 跟踪循环条件信息 + result.loop_condition_info = TraceLoopConditionInfo(start_node, cond_node, control_subgraph, result.repeat_node); + // 获取循环后子图 + result.after_loop_subgraph = GetSubgraph(switch_node, AFTER_LOOP_INPUT, AFTER_LOOP_PATTERN); + // 跟踪循环与控制节点参数的映射关系 + auto loop_to_control_order_map = TraceLoopToControlMap(control_subgraph); + // 跟踪忽略的循环参数掩码 + auto ignored_loop_params_mask = TraceIgnoredLoopParams(start_node, loop_to_control_order_map); + // 处理循环输入参数 + auto loop_inputs_num = start_node->inputs().size() - 1; + for (size_t i = 0; i < loop_inputs_num; ++i) { + if (ignored_loop_params_mask.at(i)) { + result.ignored_loop_param_indices.push_back(i); + } else { + result.used_loop_to_control_param_indices.push_back(std::make_pair(i, loop_to_control_order_map.at(i))); + } + } + // 跟踪循环后子图到循环内参数的映射 + auto after_to_loop_order_map = TraceAfterToLoopMap(control_subgraph); + // 处理循环后参数到循环输出参数的映射 + for (size_t after_i = 0; after_i < result.after_loop_subgraph->parameters().size(); ++after_i) { + auto loop_i = after_to_loop_order_map.at(after_i); + if (!ignored_loop_params_mask.at(loop_i)) { + auto output_i = loop_i; + for (size_t i = 0; i < loop_i; ++i) { + output_i -= static_cast(ignored_loop_params_mask.at(i)); + } + result.after_param_to_output_indices.push_back(std::make_pair(after_i, output_i)); + } + } + + return result; +} +} // namespace while_loop_export + +class OpAttrInfo { + public: + OpAttrInfo(const std::string &attr_name, const string &onnx_attr_name, + onnx::AttributeProto_AttributeType onnx_attr_type, const GenAttrFuncType &fn_gen_attr) + : attr_name_(attr_name), + onnx_attr_name_(onnx_attr_name), + onnx_attr_type_(onnx_attr_type), + fn_gen_attr_(fn_gen_attr) {} + ~OpAttrInfo() {} + + const std::string &attr_name() const { return attr_name_; } + const std::string &onnx_attr_name() const { return onnx_attr_name_; } + onnx::AttributeProto_AttributeType onnx_attr_type() const { return onnx_attr_type_; } + GenAttrFuncType fn_gen_attr() const { return fn_gen_attr_; } + + private: + std::string attr_name_; // attribute name of MindSpore + std::string onnx_attr_name_; // corresponding attribute name of ONNX + onnx::AttributeProto_AttributeType onnx_attr_type_; // corresponding attribute type of ONNX + GenAttrFuncType fn_gen_attr_; // function used convert +}; + +struct InputConversion { + int input_index; + onnx::TensorProto_DataType input_type; + onnx::TensorProto_DataType target_type; +}; + +struct OutputConversion { + int output_index; + enum class Mode { FIXED, INPUT } mode; + union { + onnx::TensorProto_DataType target_type; + int input_with_matching_type; + }; +}; + +class OpNameInfo { + public: + OpNameInfo &set_op_type(const std::string &op_type) { + op_type_ = op_type; + return *this; + } + + const std::string &op_type() const { return op_type_; } + + OpNameInfo &set_onnx_type(const std::string &onnx_type) { + onnx_type_ = onnx_type; + return *this; + } + + const std::string &onnx_type() const { return onnx_type_; } + + OpNameInfo &Attr(const std::string &attr_name, const std::string &onnx_attr_name, + onnx::AttributeProto_AttributeType onnx_attr_type, const GenAttrFuncType &fn_gen_attr) { + (void)op_attrs_.emplace_back(OpAttrInfo(attr_name, onnx_attr_name, onnx_attr_type, fn_gen_attr)); + return *this; + } + + const std::vector &op_attrs() const { return op_attrs_; } + + const std::vector &input_casts() const { return input_casts_; } + + OpNameInfo &CastInput(int input_index, onnx::TensorProto_DataType input_type, + onnx::TensorProto_DataType target_type) { + input_casts_.push_back({input_index, input_type, target_type}); + return *this; + } + + const std::vector &output_casts() const { return output_casts_; } + + OpNameInfo &CastOutputToFixedType(onnx::TensorProto_DataType type, int output_index = 0) { + output_casts_.push_back({output_index, OutputConversion::Mode::FIXED, {type}}); + return *this; + } + + OpNameInfo &CastOutputToInputType(int input_index, int output_index = 0) { + auto rule = OutputConversion{output_index, OutputConversion::Mode::INPUT}; + rule.input_with_matching_type = input_index; + output_casts_.push_back(rule); + return *this; + } + + int num_outputs() const { return num_outputs_; } + + OpNameInfo &set_num_outputs(int n) { + num_outputs_ = n; + return *this; + } + + private: + std::string op_type_; // operator type of MindSpore + std::string onnx_type_; // corresponding ONNX operator type + std::vector op_attrs_; // operator attributes map info + std::vector input_casts_; // if input input_index has type input_type, cast it to target_type + std::vector output_casts_; // cast output output_index to fixed type or input type + int num_outputs_ = 1; +}; + +#define OPERATOR_ONNX_CONVERT_DEFINE(name, onnx_name, impl) \ + OpNameInfo GetOpOnnxConvertInfo_##name() { return impl.set_op_type(#name).set_onnx_type(#onnx_name); } + +OPERATOR_ONNX_CONVERT_DEFINE(Add, Add, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Mul, Mul, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Pow, Pow, OpNameInfo()) + +OPERATOR_ONNX_CONVERT_DEFINE(ReLU, Relu, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Sigmoid, Sigmoid, OpNameInfo()) + +OPERATOR_ONNX_CONVERT_DEFINE(Flatten, Flatten, OpNameInfo()) + +OPERATOR_ONNX_CONVERT_DEFINE( + Conv2D, Conv, + OpNameInfo() + .Attr("dilation", "dilations", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>) + .Attr("group", "group", onnx::AttributeProto_AttributeType_INT, SetAttrValueToProto) + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<0>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetConvPadding) + .Attr("stride", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>)) +OPERATOR_ONNX_CONVERT_DEFINE( + Conv3D, Conv, + OpNameInfo() + .Attr("dilations", "dilations", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto) + .Attr("group", "group", onnx::AttributeProto_AttributeType_INT, SetAttrValueToProto) + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<0>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetConvPadding) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto)) +OPERATOR_ONNX_CONVERT_DEFINE( + Conv3DTranspose, ConvTranspose, + OpNameInfo() + .Attr("dilations", "dilations", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto) + .Attr("group", "group", onnx::AttributeProto_AttributeType_INT, SetAttrValueToProto) + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<0>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetConvTransposePadding) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto) + .Attr("output_padding", "output_padding", onnx::AttributeProto_AttributeType_INTS, + SetAttrTupleValueToProto)) +OPERATOR_ONNX_CONVERT_DEFINE(BiasAdd, Add, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(MatMul, Gemm, + OpNameInfo() + .Attr("transpose_a", "transA", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto) + .Attr("transpose_b", "transB", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto)) + +OPERATOR_ONNX_CONVERT_DEFINE(BatchNorm, BatchNormalization, + OpNameInfo() + .Attr("epsilon", "epsilon", onnx::AttributeProto_AttributeType_FLOAT, + SetAttrValueToProto) + .CastInput(0, onnx::TensorProto_DataType_FLOAT16, onnx::TensorProto_DataType_FLOAT) + .CastOutputToInputType(0)) + +OPERATOR_ONNX_CONVERT_DEFINE(Reshape, Reshape, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Cast, Cast, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(PReLU, PRelu, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Argmax, ArgMax, + OpNameInfo() + .Attr("axis", "axis", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto) + .Attr("", "keepdims", onnx::AttributeProto_AttributeType_INT, + [](ValuePtr, onnx::AttributeProto_AttributeType, + onnx::AttributeProto *const attr_proto, const PrimitivePtr &) { + attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + attr_proto->set_i(0); + }) + .CastOutputToFixedType(onnx::TensorProto_DataType_INT32)) + +OPERATOR_ONNX_CONVERT_DEFINE(SimpleMean, AveragePool, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE( + MaxPool, MaxPool, + OpNameInfo() + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetPoolingPadMode) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>)) + +OPERATOR_ONNX_CONVERT_DEFINE( + MaxPoolWithArgmax, MaxPool, + OpNameInfo() + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetPoolingPadMode) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>)) + +OPERATOR_ONNX_CONVERT_DEFINE( + AvgPool, AveragePool, + OpNameInfo() + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetPoolingPadMode) + .Attr("strides", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<2>)) + +OPERATOR_ONNX_CONVERT_DEFINE(Gather, Gather, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(MakeTuple, SequenceConstruct, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(RealDiv, Div, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Sub, Sub, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Maximum, Max, + OpNameInfo() + .CastInput(0, onnx::TensorProto_DataType_INT32, onnx::TensorProto_DataType_FLOAT) + .CastInput(1, onnx::TensorProto_DataType_INT32, onnx::TensorProto_DataType_FLOAT) + .CastOutputToInputType(0)) +OPERATOR_ONNX_CONVERT_DEFINE(Minimum, Min, + OpNameInfo() + .CastInput(0, onnx::TensorProto_DataType_INT32, onnx::TensorProto_DataType_FLOAT) + .CastInput(1, onnx::TensorProto_DataType_INT32, onnx::TensorProto_DataType_FLOAT) + .CastOutputToInputType(0)) +OPERATOR_ONNX_CONVERT_DEFINE(Transpose, Transpose, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Exp, Exp, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Softplus, Softplus, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Tanh, Tanh, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Abs, Abs, OpNameInfo()) + +// MindSpore Softmax axis(int, Tuple) +OPERATOR_ONNX_CONVERT_DEFINE(Softmax, Softmax, + OpNameInfo().Attr("axis", "axis", onnx::AttributeProto_AttributeType_INT, + SetAttrTupleValueToProto<0>)) + +// MindSpore LogSoftmax axis(int) +OPERATOR_ONNX_CONVERT_DEFINE(LogSoftmax, LogSoftmax, + OpNameInfo().Attr("axis", "axis", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto)) + +OPERATOR_ONNX_CONVERT_DEFINE(Softsign, Softsign, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Sqrt, Sqrt, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Equal, Equal, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Floor, Floor, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(ACos, Acos, OpNameInfo()) + +OPERATOR_ONNX_CONVERT_DEFINE(GatherNd, GatherND, + OpNameInfo().CastInput(1, onnx::TensorProto_DataType_INT32, + onnx::TensorProto_DataType_INT64)) +OPERATOR_ONNX_CONVERT_DEFINE(Select, Where, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Log, Log, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(Greater, Greater, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(LogicalAnd, And, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(ReverseSequence, ReverseSequence, + OpNameInfo() + .Attr("seq_dim", "time_axis", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto) + .Attr("batch_dim", "batch_axis", onnx::AttributeProto_AttributeType_INT, + SetAttrValueToProto) + .CastInput(1, onnx::TensorProto_DataType_INT32, onnx::TensorProto_DataType_INT64)) +OPERATOR_ONNX_CONVERT_DEFINE(Less, Less, OpNameInfo()) +OPERATOR_ONNX_CONVERT_DEFINE(TensorScatterUpdate, ScatterND, + OpNameInfo().CastInput(1, onnx::TensorProto_DataType_INT32, + onnx::TensorProto_DataType_INT64)) + +#define OP_CONVERT_FUNCTION_NAME(name) GetOpOnnxConvertInfo_##name + +void RegisterOpConverters(const std::function &fn) { + fn(OP_CONVERT_FUNCTION_NAME(Add)()); + fn(OP_CONVERT_FUNCTION_NAME(Mul)()); + fn(OP_CONVERT_FUNCTION_NAME(Pow)()); + fn(OP_CONVERT_FUNCTION_NAME(ReLU)()); + fn(OP_CONVERT_FUNCTION_NAME(Sigmoid)()); + fn(OP_CONVERT_FUNCTION_NAME(Conv2D)()); + fn(OP_CONVERT_FUNCTION_NAME(Conv3D)()); + fn(OP_CONVERT_FUNCTION_NAME(Conv3DTranspose)()); + fn(OP_CONVERT_FUNCTION_NAME(Argmax)()); + fn(OP_CONVERT_FUNCTION_NAME(Flatten)()); + fn(OP_CONVERT_FUNCTION_NAME(MaxPool)()); + fn(OP_CONVERT_FUNCTION_NAME(MaxPoolWithArgmax)()); + fn(OP_CONVERT_FUNCTION_NAME(AvgPool)()); + + fn(OP_CONVERT_FUNCTION_NAME(BatchNorm)()); + fn(OP_CONVERT_FUNCTION_NAME(MatMul)()); + fn(OP_CONVERT_FUNCTION_NAME(MakeTuple)()); + fn(OP_CONVERT_FUNCTION_NAME(RealDiv)()); + fn(OP_CONVERT_FUNCTION_NAME(BiasAdd)()); + fn(OP_CONVERT_FUNCTION_NAME(Sub)()); + fn(OP_CONVERT_FUNCTION_NAME(Maximum)()); + fn(OP_CONVERT_FUNCTION_NAME(Minimum)()); + fn(OP_CONVERT_FUNCTION_NAME(Exp)()); + + fn(OP_CONVERT_FUNCTION_NAME(Softplus)()); + fn(OP_CONVERT_FUNCTION_NAME(Tanh)()); + fn(OP_CONVERT_FUNCTION_NAME(Softmax)()); + fn(OP_CONVERT_FUNCTION_NAME(LogSoftmax)()); + fn(OP_CONVERT_FUNCTION_NAME(Abs)()); + fn(OP_CONVERT_FUNCTION_NAME(Softsign)()); + fn(OP_CONVERT_FUNCTION_NAME(Sqrt)()); + fn(OP_CONVERT_FUNCTION_NAME(Equal)()); + fn(OP_CONVERT_FUNCTION_NAME(Floor)()); + fn(OP_CONVERT_FUNCTION_NAME(ACos)()); + + fn(OP_CONVERT_FUNCTION_NAME(GatherNd)()); + fn(OP_CONVERT_FUNCTION_NAME(Select)()); + fn(OP_CONVERT_FUNCTION_NAME(Log)()); + fn(OP_CONVERT_FUNCTION_NAME(Less)()); + fn(OP_CONVERT_FUNCTION_NAME(Greater)()); + fn(OP_CONVERT_FUNCTION_NAME(LogicalAnd)()); + fn(OP_CONVERT_FUNCTION_NAME(ReverseSequence)()); + fn(OP_CONVERT_FUNCTION_NAME(TensorScatterUpdate)()); +} + +class OpConvertRegistry { + public: + ~OpConvertRegistry() { Clear(); } + + static void RegisterOneOpConverter(OpNameInfo &&op_info) { GetSingleton().op_map_[op_info.op_type()] = op_info; } + + static void RegisterAllOpConverters() { RegisterOpConverters(RegisterOneOpConverter); } + + static OpConvertRegistry &GetSingleton() { + static OpConvertRegistry registry = OpConvertRegistry(); + return registry; + } + + static const mindspore::HashMap &GetOpConvertMap() { return GetSingleton().op_map_; } + + void Clear() noexcept { op_map_.clear(); } + + private: + OpConvertRegistry() {} + + mindspore::HashMap op_map_; +}; + +class OnnxExporter { + public: + OnnxExporter() {} + ~OnnxExporter() {} + + std::string GetOnnxProtoString(const FuncGraphPtr &func_graph); + + private: + void InitModelInfo(); + + void ExportFuncGraph(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + onnx::GraphProto *graph_proto, bool export_inputs = true); + void ExportInputs(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + onnx::GraphProto *graph_proto); + + std::string ExportPrimitive(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + const PrimitivePtr &prim, const std::vector &inputs, + onnx::GraphProto *graph_proto); + + static onnx::TensorProto_DataType GetOnnxDataType(TypeId type_id); + static onnx::TensorProto_DataType GetOutputType(const AnfNodePtr &node, int64_t output_index = -1); + void SetValueInfoType(const AnfNodePtr &node, onnx::ValueInfoProto *value_proto, int64_t output_index = -1) const; + + void MatchAndMark(const FuncGraphPtr &func_graph, const std::vector &nodes, + mindspore::HashMap *op_merged_infos_ptr); + void MatchAndMarkCNode(const FuncGraphPtr &func_graph, const CNodePtr &cnode, + mindspore::HashMap *op_merged_infos_ptr) const; + void ExportNodes(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + onnx::GraphProto *graph_proto); + + void ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportWhileLoop(const CNodePtr &start_node, std::map *node_map_ptr, + onnx::GraphProto *graph_proto); + + void ExportPrimReshape(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimReduce(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimTranspose(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimStridedSlice(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + onnx::NodeProto *PrimResizeExportHelper(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto); + void ExportPrimResizeNearestNeighbor(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimResizeBilinear(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimExpandDims(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimPad(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimBatchMatMul(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimGeLU(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimConcat(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimCast(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimPReLU(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimReLU6(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimDepthwiseConv2d(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimTile(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimSquare(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimGatherV2(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimTupleGetItem(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimTopK(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimBoundingBoxDecode(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimNMSWithMask(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimSplit(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimROIAlign(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimSlice(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimOnesLike(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimArgMaxWithValue(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimOneHot(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void PrimConv2DTransposeExportHelper(const CNodePtr &conv_node, const CNodePtr &bias_add_node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto); + void ExportPrimConv2DTranspose(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimGreaterEqual(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimSqueeze(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimLSTM(const FuncGraphPtr &, const CNodePtr &node, std::map *node_map_ptr, + onnx::GraphProto *graph_proto); + void ExportPrimReverseV2(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimTensorCopySlices(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportPrimStack(const FuncGraphPtr &, const CNodePtr &node, std::map *node_map_ptr, + onnx::GraphProto *graph_proto); + void ExportMergeConv(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportMergeGemm(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportMergeBatchNorm(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportMergeMaxPoolWithArgmax(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportMergeLayerNorm(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + void ExportMergeConv2DTranspose(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + + void ExportOutput(const FuncGraphPtr &func_graph, const AnfNodePtr &return_arg, + std::map *node_map_ptr, onnx::GraphProto *graph_proto); + std::string GetNodeInputName(const AnfNodePtr &node, std::map *node_map_ptr, + onnx::GraphProto *const); + + void ConvertTupleToTensor(const ValuePtr &value, onnx::TensorProto *tensor_proto) const; + void SetTensorData(const ValuePtr &value, onnx::TensorProto *tensor_proto); + + void AddOutputWithCast(onnx::NodeProto *node_proto, const std::string &output_name, + onnx::TensorProto_DataType target_type, onnx::GraphProto *graph_proto) const; + + std::string GenerateUniqueName() { return std::to_string(++onnx_node_index_); } + std::string RegisterNodeWithUniqueName(const AnfNodePtr &node, std::map *node_map_ptr) { + auto name = GenerateUniqueName(); + (*node_map_ptr)[node] = name; + return name; + } + std::string GenerateUniqueParameterName(const ParameterPtr &node, std::map *node_map_ptr) { + auto node_name = node->ToString(); + MS_EXCEPTION_IF_CHECK_FAIL(node_name != "", "Cannot get the name of an ignored parameter"); + auto dup_iter = std::find_if(node_map_ptr->begin(), node_map_ptr->end(), + [&node_name](const auto &pair) { return pair.second == node_name; }); + if (dup_iter != node_map_ptr->end()) { + node_name = GenerateUniqueName() + node_name; + } + return node_name; + } + + void ResetNodeIndex() { onnx_node_index_ = 0; } + + static int64_t GetInt64Value(const AnfNodePtr &node) { + auto value_node_ptr = dyn_cast(node); + MS_EXCEPTION_IF_NULL(value_node_ptr); + return GetValue(value_node_ptr->value()); + } + + onnx::ModelProto model_; + + size_t onnx_node_index_ = 0; + + std::map renamed_node_map_; +}; + +std::string OnnxExporter::GetOnnxProtoString(const FuncGraphPtr &func_graph) { + if (func_graph == nullptr) { + return ""; + } + ResetNodeIndex(); + OpConvertRegistry::GetSingleton().Clear(); + OpConvertRegistry::RegisterAllOpConverters(); + InitModelInfo(); + onnx::GraphProto *graph_proto = model_.mutable_graph(); + std::map node_map; + ExportFuncGraph(func_graph, &node_map, graph_proto); + return model_.SerializeAsString(); +} + +void OnnxExporter::InitModelInfo() { + model_.set_ir_version(onnx::IR_VERSION_2019_1_22); + model_.set_producer_name("MindSpore"); + model_.set_producer_version("1.0"); + onnx::OperatorSetIdProto *opset_proto = model_.add_opset_import(); + opset_proto->set_version(ONNX_VERSION); +} + +void OnnxExporter::ExportFuncGraph(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + onnx::GraphProto *const graph_proto, bool export_inputs) { + MS_LOG(INFO) << "Begin exporting onnx model for graph " << func_graph->ToString(); + + // set graph name + graph_proto->set_name(func_graph->ToString()); + + // export inputs if graph is not inlined + if (export_inputs) { + ExportInputs(func_graph, node_map_ptr, graph_proto); + } + + // export computational nodes and output nodes + ExportNodes(func_graph, node_map_ptr, graph_proto); + + // add names for easier debugging + for (auto &node : *graph_proto->mutable_node()) { + if (!node.has_name()) { + node.set_name(node.output(0) + node.op_type()); + } + } + + MS_LOG(INFO) << "End exporting onnx model for graph " << func_graph->ToString(); +} + +void OnnxExporter::ExportInputs(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + for (auto ¶m : func_graph->parameters()) { + const ParameterPtr param_ptr = dyn_cast(param); + if (param_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Parameter '" << param->ToString() << "' could not cast to parameter."; + } + + if (param_ptr->has_default()) { + continue; + } + + // set onnx input. + std::string name; + auto renamed_iter = renamed_node_map_.find(param_ptr); + if (renamed_iter != renamed_node_map_.end()) { + name = renamed_iter->second; + if (name == "") { + continue; + } + } else { + name = GenerateUniqueParameterName(param_ptr, node_map_ptr); + (*node_map_ptr)[param_ptr] = name; + } + + onnx::ValueInfoProto *input_proto = graph_proto->add_input(); + input_proto->set_name(name); + SetValueInfoType(param_ptr, input_proto); + } +} + +onnx::TensorProto_DataType OnnxExporter::GetOnnxDataType(TypeId type_id) { + // clang-format off + static mindspore::HashMap type_map = { + {kNumberTypeBool, onnx::TensorProto_DataType_BOOL}, + {kNumberTypeInt8, onnx::TensorProto_DataType_INT8}, + {kNumberTypeInt16, onnx::TensorProto_DataType_INT16}, + {kNumberTypeInt32, onnx::TensorProto_DataType_INT32}, + {kNumberTypeInt64, onnx::TensorProto_DataType_INT64}, + {kNumberTypeUInt8, onnx::TensorProto_DataType_UINT8}, + {kNumberTypeUInt16, onnx::TensorProto_DataType_UINT16}, + {kNumberTypeUInt32, onnx::TensorProto_DataType_UINT32}, + {kNumberTypeUInt64, onnx::TensorProto_DataType_UINT64}, + {kNumberTypeFloat16, onnx::TensorProto_DataType_FLOAT16}, + {kNumberTypeFloat32, onnx::TensorProto_DataType_FLOAT}, + {kNumberTypeFloat64, onnx::TensorProto_DataType_DOUBLE}, + }; + // clang-format on + + auto iter = type_map.find(type_id); + if (iter == type_map.end()) { + MS_LOG(EXCEPTION) << "Convert type error, unsupported type " << type_id; + } + + return iter->second; +} + +void OnnxExporter::SetValueInfoType(const AnfNodePtr &node, onnx::ValueInfoProto *const value_proto, + int64_t output_index) const { + auto dtype = GetOutputType(node, output_index); + auto shape = node->Shape(); + + abstract::ShapePtr output_shape; + if (shape->isa()) { + auto tuple_shape = dyn_cast(shape); + auto base_shape = tuple_shape->shape().at(static_cast(output_index)); + output_shape = dyn_cast(base_shape); + if (output_shape == nullptr) { + MS_LOG(EXCEPTION) << "Expected " << node->ToString() << " to output a tuple of tensors. Instead got " + << base_shape->ToString() << " from output " << output_index; + } + } else if (shape->isa()) { + output_shape = dyn_cast(shape); + } else { + MS_LOG(EXCEPTION) << "Unsupported shape: " << shape->ToString(); + } + + auto *type_proto = value_proto->mutable_type(); + type_proto->mutable_tensor_type()->set_elem_type(dtype); + auto *shape_proto = type_proto->mutable_tensor_type()->mutable_shape(); + + for (const auto dim : output_shape->shape()) { + shape_proto->add_dim()->set_dim_value(dim); + } +} + +void OnnxExporter::MatchAndMark(const FuncGraphPtr &func_graph, const std::vector &nodes, + mindspore::HashMap *op_merged_infos_ptr) { + auto &op_merged_infos = *op_merged_infos_ptr; + + for (auto &node : nodes) { + if (!node->isa() || IsZeroRefcountNode(node)) { + continue; + } + auto cnode = node->cast(); + if (cnode == func_graph->get_return()) { + // if the key `input` does not exist, just create a new one + op_merged_infos[cnode].referred_count += 1; + } + for (auto &orig_input : cnode->inputs()) { + auto input = GetRealInput(orig_input); + if (!input->isa() || IsZeroRefcountNode(input)) { + continue; + } + // if the key `input` does not exist, just create a new one + op_merged_infos[input].referred_count += 1; + } + MatchAndMarkCNode(func_graph, cnode, op_merged_infos_ptr); + } +} + +struct MergeRule { + PrimitivePtr node_type; + PrimitivePtr prev_type; + OpMergeMode merge_mode; +}; + +void OnnxExporter::MatchAndMarkCNode(const FuncGraphPtr &func_graph, const CNodePtr &cnode, + mindspore::HashMap *op_merged_infos_ptr) const { + auto &op_merged_infos = *op_merged_infos_ptr; + const auto ignore = [&op_merged_infos](const AnfNodePtr &node) { + op_merged_infos[node].mode = OP_MERGE_IGNORE; + op_merged_infos[node].referred_count -= 1; + }; + + const std::vector first_input_merge_rules = { + {prim::kPrimBiasAdd, prim::kPrimConv2D, OP_MERGE_CONV}, + {prim::kPrimBiasAdd, prim::kPrimConv2DTranspose, OP_MERGE_CONV2D_TRANSPOSE}, + {prim::kPrimBiasAdd, prim::kPrimConv3D, OP_MERGE_CONV}, + {prim::kPrimBiasAdd, prim::kPrimConv3DTranspose, OP_MERGE_CONV}, + {prim::kPrimBiasAdd, prim::kPrimMatMul, OP_MERGE_GEMM}, + {prim::kPrimTupleGetItem, prim::kPrimBatchNorm, OP_MERGE_BATCH_NORM}, + {prim::kPrimTupleGetItem, prim::kPrimMaxPoolWithArgmax, OP_MERGE_MAXPOOL_WITH_ARGMAX}, + {prim::kPrimTupleGetItem, prim::kPrimLayerNorm, OP_MERGE_LAYER_NORM}, + }; + + auto rule = std::find_if(first_input_merge_rules.begin(), first_input_merge_rules.end(), [&cnode](const auto &rule) { + return cnode->IsApply(rule.node_type) && IsPrimitiveCNode(cnode->input(1), rule.prev_type); + }); + if (rule != first_input_merge_rules.end()) { + if (cnode->IsApply(prim::kPrimTupleGetItem) && GetInt64Value(cnode->input(kTwoNum)) != 0) { + MS_LOG(EXCEPTION) << "Multiple outputs for node \"" << cnode->input(1)->ToString() << "\" are not supported"; + } + op_merged_infos[cnode].mode = rule->merge_mode; + ignore(cnode->input(1)); + } else if (while_loop_export::IsLoopBodyReturnNode(cnode, func_graph)) { + // Ignore to replace with other outputs + ignore(cnode); + auto repeat_node = dyn_cast(GetRealInput(cnode->input(1))); + MS_EXCEPTION_IF_NULL(repeat_node); + ignore(repeat_node); + } else if (while_loop_export::IsAfterLoopReturnNode(cnode, func_graph)) { + // Ignore to inline after-loop subgraph in main graph + ignore(cnode); + auto first_input = GetRealInput(cnode->input(1)); + if (IsPrimitiveCNode(first_input, prim::kPrimMakeTuple)) { + ignore(first_input); + } + } else if (cnode == func_graph->get_return()) { + auto first_input = GetRealInput(cnode->input(1)); // Unpack Depend + if (IsPrimitiveCNode(first_input, prim::kPrimMakeTuple)) { + // Ignore MakeTuple output node to avoid exporting it to SequenceConstruct + // and handle multiple outputs in ExportOutput + ignore(first_input); + } + } else if (cnode->IsApply(prim::kPrimConcat) && IsPrimitiveCNode(cnode->input(1), prim::kPrimMakeTuple)) { + // Ignore MakeTuple to handle it in ExportPrimConcat + ignore(cnode->input(1)); + } +} + +/** + * AnfNode + * +-- CNode + * +-- ANode + * | +-- Parameter + * | `-- ValueNode + */ +void OnnxExporter::ExportNodes(const FuncGraphPtr &func_graph, std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + std::vector nodes = TopoSort(func_graph->get_return(), SuccIncoming, AlwaysInclude); + + mindspore::HashMap op_merged_infos; + MatchAndMark(func_graph, nodes, &op_merged_infos); + for (const AnfNodePtr &node : nodes) { + if (!node->isa()) { + continue; + } + auto cnode = node->cast(); + + auto iter = op_merged_infos.find(cnode); + // the node is not referenced by any other nodes, skip it + if (iter == op_merged_infos.end()) { + continue; + } + auto merged_info = iter->second; + // the op node is merged with other node and not used any more, skip it + if (merged_info.mode == OP_MERGE_IGNORE && merged_info.referred_count == 0) { + continue; + } + if (cnode == func_graph->get_return()) { + ExportOutput(func_graph, cnode->input(kOneNum), node_map_ptr, graph_proto); + continue; + } + switch (merged_info.mode) { + case OP_MERGE_CONV: + ExportMergeConv(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_GEMM: + ExportMergeGemm(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_BATCH_NORM: + ExportMergeBatchNorm(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_MAXPOOL_WITH_ARGMAX: + ExportMergeMaxPoolWithArgmax(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_LAYER_NORM: + ExportMergeLayerNorm(func_graph, cnode, node_map_ptr, graph_proto); + break; + case OP_MERGE_CONV2D_TRANSPOSE: + ExportMergeConv2DTranspose(func_graph, cnode, node_map_ptr, graph_proto); + break; + default: + ExportCNode(func_graph, cnode, node_map_ptr, graph_proto); + break; + } + } +} + +void OnnxExporter::ExportPrimReshape(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + // 导出 ONNX 格式中的 Reshape 操作 + auto name_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto);// 获取输入节点的名称 + auto input_shape = node->input(kTwoNum);// 获取输入类型(可能是 ValueNode 或者其他) + std::string name_shape; + // 如果输入类型是 ValueNode,则将其转换为 Constant 节点 + if (input_shape->isa()) { + name_shape = RegisterNodeWithUniqueName(input_shape, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + auto name = prim::kPrimReshape->name(); + // 设置 Constant 节点的信息 + node_proto->set_name(name_shape + name); + node_proto->add_output(name_shape); + node_proto->set_op_type("Constant"); + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + ConvertTupleToTensor(dyn_cast(input_shape)->value(), attr_proto->mutable_t()); + } else { + // 如果输入形状不是 ValueNode,则抛出异常 + name_shape = GetNodeInputName(input_shape, node_map_ptr, graph_proto); + MS_LOG(EXCEPTION) << "Need to insert op convert variable from tuple to tensor for Reshape."; + } + // 注册当前 Reshape 节点并设置相应的信息 + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type(prim::kPrimReshape->name()); + node_proto->add_output(node_name); + node_proto->add_input(name_x); + node_proto->add_input(name_shape); +} + +void OnnxExporter::ExportPrimReduce(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_data = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_axis = node->input(kTwoNum); + auto keep_dims = GetOpAttribute(node, "keep_dims"); + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + std::string name; + if (node->IsApply(prim::kPrimReduceSum)) { + name = "ReduceSum"; + } else if (node->IsApply(prim::kPrimReduceMean)) { + name = "ReduceMean"; + } else { + MS_LOG(EXCEPTION) << "Unsupported reduce op: " << node->ToString(); + } + + std::vector axes; + if (input_axis->isa()) { + auto axis_value = dyn_cast(input_axis)->value(); + if (axis_value->isa()) { + auto int_ptr = dyn_cast(axis_value); + axes.push_back(int_ptr->value()); + } else if (axis_value->isa()) { + auto int_ptr = dyn_cast(axis_value); + axes.push_back(int_ptr->value()); + } else if (axis_value->isa()) { + auto tuple_ptr = dyn_cast(axis_value); + axes = GetValue>(tuple_ptr); + } else { + MS_LOG(EXCEPTION) << "Cannot convert value " << axis_value->ToString() << " of type " + << axis_value->type()->ToString() << " for \"axes\" attribute of " << name; + } + } else { + MS_LOG(EXCEPTION) << "Need to insert op convert variable from tuple to attributes for " << name; + } + + AddReduceOp(name, input_data, node_name, axes, keep_dims, graph_proto); +} +//导出Reduce操作 +void OnnxExporter::ExportPrimTranspose(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_data = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_perm = node->input(kTwoNum); + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + auto name = prim::kPrimTranspose->name(); + + node_proto->set_name(node_name + name); + node_proto->set_op_type(name); + node_proto->add_output(node_name); + node_proto->add_input(input_data); + + if (input_perm->isa()) { + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("perm"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + auto perm_value = dyn_cast(input_perm)->value(); + auto int_ptr = dyn_cast(perm_value); + if (int_ptr == nullptr) { + auto tuple_ptr = dyn_cast(perm_value); + MS_EXCEPTION_IF_NULL(tuple_ptr); + for (size_t i = 0; i < tuple_ptr->size(); ++i) { + attr_proto->add_ints(GetValue((*tuple_ptr)[i])); + } + } else { + attr_proto->add_ints(int_ptr->value()); + } + } else { + MS_LOG(EXCEPTION) << "The input input_perm of Transpose is not a ValueNode! " + << "Need to insert op convert variable from tuple to attributes for " << name; + } +} + +/* + See: + - mindspore/ccsrc/backend/kernel_compiler/cpu/stridedslice_cpu_kernel.cc + - mindspore/ccsrc/backend/kernel_compiler/common_utils.cc + */ +void OnnxExporter::ExportPrimStridedSlice(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_data = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto name = node_name + prim::kPrimStridedSlice->name(); + + auto begin = node->input(kTwoNum); + if (!begin->isa()) { + MS_LOG(EXCEPTION) << "The input begin of StridedSlice is not a ValueNode! " + << "Need to insert op convert variable from tuple to tensor for " << name; + } + auto begin_value_node = dyn_cast(begin); + auto begin_value = GetValue>(begin_value_node->value()); + auto begin_ignore_mask = GetOpAttribute(node, "begin_mask"); + for (size_t i = 0; i < begin_value.size(); ++i) { + if ((begin_ignore_mask & (1 << i)) != 0) { + begin_value[i] = 0; + } + } + + auto end = node->input(kThreeNum); + if (!end->isa()) { + MS_LOG(EXCEPTION) << "The input end of StridedSlice is not a ValueNode! " + << "Need to insert op convert variable from tuple to tensor for " << name; + } + auto end_value_node = dyn_cast(end); + auto end_value = GetValue>(end_value_node->value()); + const auto &x_shape = dyn_cast(node->input(kOneNum)->Shape())->shape(); + auto end_ignore_mask = GetOpAttribute(node, "end_mask"); + for (size_t i = 0; i < end_value.size(); ++i) { + if ((static_cast(end_ignore_mask) & (1 << i)) != 0) { + end_value[i] = x_shape[i]; + } + } + + std::vector axes_value; + for (size_t i = 0; i < x_shape.size(); ++i) { + axes_value.push_back(static_cast(i)); + } + + auto strides = node->input(kFourNum); + if (!strides->isa()) { + MS_LOG(EXCEPTION) << "The input strides of StridedSlice is not a ValueNode! " + << "Need to insert op convert variable from tuple to tensor for " << name; + } + auto strides_value_node = dyn_cast(strides); + auto strides_value = GetValue>(strides_value_node->value()); + + auto shrink_axis_mask = GetOpAttribute(node, "shrink_axis_mask"); + for (size_t i = 0; i < end_value.size(); ++i) { + if ((shrink_axis_mask & (1 << i)) != 0) { + strides_value[i] = end_value[i] > begin_value[i] ? 1 : -1; + end_value[i] = begin_value[i] + strides_value[i]; + } + } + + auto slice_name = node_name; + if (shrink_axis_mask != 0) { + slice_name = node_name + "__reshape"; + } + + AddSliceOp(input_data, slice_name, begin_value, end_value, axes_value, strides_value, graph_proto); + + if (shrink_axis_mask != 0) { + onnx::NodeProto *squeeze_op = graph_proto->add_node(); + squeeze_op->set_op_type("Squeeze"); + squeeze_op->add_input(slice_name); + squeeze_op->add_output(node_name); + onnx::AttributeProto *axes_attr = squeeze_op->add_attribute(); + axes_attr->set_name("axes"); + axes_attr->set_type(onnx::AttributeProto_AttributeType_INTS); + for (size_t i = 0; i < x_shape.size(); ++i) { + if ((shrink_axis_mask & (1 << i)) != 0) { + axes_attr->add_ints(i); + } + } + } +} + +onnx::NodeProto *OnnxExporter::PrimResizeExportHelper(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_data = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto x_shape = dyn_cast(node->input(kOneNum)->Shape()); + + AnfNodePtr op = node->input(kZeroNum); + auto op_value = dyn_cast(op); + auto prim = dyn_cast(op_value->value()); + std::vector resize_size; + + auto tuple_ptr = dyn_cast(prim->GetAttr("size")); // size may be Tuple or List + if (tuple_ptr == nullptr) { + MS_LOG(EXCEPTION) << "Got null pointer, currently the " << prim->name() + << " operator in your model is not support for exporting onnx."; + } + + for (size_t i = 0; i < x_shape->shape().size() - kTwoNum; i++) { + resize_size.push_back(x_shape->shape()[i]); + } + for (size_t i = 0; i < tuple_ptr->size(); i++) { + ValuePtr elem = (*tuple_ptr)[i]; + resize_size.push_back(dyn_cast(elem)->value()); + } + auto resize_size_ptr = MakeValue>(resize_size); + auto size = NewValueNode(resize_size_ptr)->cast(); + + auto name_size = RegisterNodeWithUniqueName(size, node_map_ptr); + onnx::NodeProto *node_proto_size = graph_proto->add_node(); + node_proto_size->add_output(name_size); + node_proto_size->set_op_type("Constant"); + onnx::AttributeProto *attr_proto = node_proto_size->add_attribute(); + attr_proto->set_name("value"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + ConvertTupleToTensor(resize_size_ptr, attr_proto->mutable_t()); + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + onnx::TensorProto *roi_initializer_proto = graph_proto->add_initializer(); + auto roi_name = node_name + "roi_initializer"; + roi_initializer_proto->set_name(roi_name); + roi_initializer_proto->set_data_type(GetOnnxDataType(kNumberTypeFloat32)); + roi_initializer_proto->add_dims(0); + + onnx::TensorProto *scales_initializer_proto = graph_proto->add_initializer(); + auto scales_name = node_name + "scales_initializer"; + scales_initializer_proto->set_name(scales_name); + scales_initializer_proto->set_data_type(GetOnnxDataType(kNumberTypeFloat32)); + scales_initializer_proto->add_dims(0); + + onnx::NodeProto *node_proto = graph_proto->add_node(); + + node_proto->set_op_type("Resize"); + node_proto->add_output(node_name); + node_proto->add_input(input_data); + node_proto->add_input(roi_name); + node_proto->add_input(scales_name); + node_proto->add_input(name_size); + + return node_proto; +} + +void OnnxExporter::ExportPrimResizeNearestNeighbor(const FuncGraphPtr &graph, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + onnx::NodeProto *node_proto = PrimResizeExportHelper(graph, node, node_map_ptr, graph_proto); + + auto align_corners = GetOpAttribute(node, "align_corners"); + std::string coordinate_transformation_mode = align_corners ? "align_corners" : "asymmetric"; + // `nearest_mode` is based on ResizeNearestNeighborCPUKernel::LaunchKernel in + // mindspore/ccsrc/backend/kernel_compiler/cpu/resize_nearest_neighbor_cpu_kernel.cc + std::string nearest_mode = align_corners ? "round_prefer_ceil" : "floor"; + + onnx::AttributeProto *coordinate_mode_proto = node_proto->add_attribute(); + coordinate_mode_proto->set_name("coordinate_transformation_mode"); + coordinate_mode_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + coordinate_mode_proto->set_s(coordinate_transformation_mode); + + onnx::AttributeProto *nearest_mode_proto = node_proto->add_attribute(); + nearest_mode_proto->set_name("nearest_mode"); + nearest_mode_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + nearest_mode_proto->set_s(nearest_mode); +} + +void OnnxExporter::ExportPrimResizeBilinear(const FuncGraphPtr &graph, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + onnx::NodeProto *node_proto = PrimResizeExportHelper(graph, node, node_map_ptr, graph_proto); + + auto align_corners = GetOpAttribute(node, "align_corners"); + std::string coordinate_transformation_mode = align_corners ? "align_corners" : "asymmetric"; + + onnx::AttributeProto *coordinate_mode_proto = node_proto->add_attribute(); + coordinate_mode_proto->set_name("coordinate_transformation_mode"); + coordinate_mode_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + coordinate_mode_proto->set_s(coordinate_transformation_mode); + + onnx::AttributeProto *mode_proto = node_proto->add_attribute(); + mode_proto->set_name("mode"); + mode_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + mode_proto->set_s("linear"); +} + +// MindSpore ExpandDims -> ONNX Reshape +void OnnxExporter::ExportPrimExpandDims(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto axis = GetInt64Value(node->input(kTwoNum)); + auto x_shape = dyn_cast(node->input(kOneNum)->Shape()); + auto name = prim::kPrimExpandDims->name(); + + std::vector new_shape; + for (size_t i = 0; i < x_shape->shape().size(); i++) { + new_shape.push_back(x_shape->shape()[i]); + } + if (axis < 0) { + axis = axis + kOneNumLong + SizeToLong(x_shape->shape().size()); + } + (void)new_shape.insert(new_shape.begin() + axis, kOneNum); + auto new_shape_value = MakeValue>(new_shape); + auto shape = NewValueNode(new_shape_value)->cast(); + std::string name_shape; + + if (shape->isa()) { + name_shape = RegisterNodeWithUniqueName(shape, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->add_output(name_shape); + node_proto->set_op_type("Constant"); + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + ConvertTupleToTensor(dyn_cast(shape)->value(), attr_proto->mutable_t()); + } else { + name_shape = GetNodeInputName(shape, node_map_ptr, graph_proto); + MS_LOG(EXCEPTION) << "Need to insert op convert variable from tuple to tensor for " << name; + } + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Reshape"); + node_proto->add_output(node_name); + node_proto->add_input(input_x); + node_proto->add_input(name_shape); +} + +// MindSpore Pad -> ONNX Pad +void OnnxExporter::ExportPrimPad(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *const graph_proto) { + auto x_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + auto paddings = GetOpAttributePtr(node, "paddings"); + std::vector> paddings_values = GetValue>>(paddings); + std::vector pads_sequence; + for (size_t i = 0; i < paddings_values.size(); ++i) { + pads_sequence.push_back(paddings_values[i][0]); + } + for (size_t j = 0; j < paddings_values.size(); ++j) { + pads_sequence.push_back(paddings_values[j][1]); + } + auto pads_ptr = MakeValue>(pads_sequence); + auto pads = NewValueNode(pads_ptr)->cast(); + + auto pads_name = RegisterNodeWithUniqueName(pads, node_map_ptr); + onnx::NodeProto *pads_node = graph_proto->add_node(); + pads_node->add_output(pads_name); + pads_node->set_op_type("Constant"); + onnx::AttributeProto *pads_attr_proto = pads_node->add_attribute(); + pads_attr_proto->set_name("value"); + pads_attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + ConvertTupleToTensor(pads_ptr, pads_attr_proto->mutable_t()); + + auto ms_pad_node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *onnx_pad_node = graph_proto->add_node(); + onnx_pad_node->set_op_type("Pad"); + onnx_pad_node->add_output(ms_pad_node_name); + onnx_pad_node->add_input(x_name); + onnx_pad_node->add_input(pads_name); +} + +// MindSpore BatchMatMul -> ONNX Transpose + MatMul +void OnnxExporter::ExportPrimBatchMatMul(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_y = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + + AnfNodePtr batchmatmul_op = node->input(kZeroNum); + auto op_value = dyn_cast(batchmatmul_op); + auto prim = dyn_cast(op_value->value()); + auto transpose_a = GetValue(prim->GetAttr("transpose_a")); + auto transpose_b = GetValue(prim->GetAttr("transpose_b")); + std::string transpose_input_x_name = ""; + std::string transpose_input_y_name = ""; + + if (transpose_a) { + auto input_x_shape = dyn_cast(node->input(kOneNum)->Shape()); + // Add Transpose node after input_x of BatchMatMul + transpose_input_x_name = GenerateUniqueName(); + onnx::NodeProto *transpose_inputx_node_proto = graph_proto->add_node(); + transpose_inputx_node_proto->add_input(input_x); + transpose_inputx_node_proto->add_output(transpose_input_x_name); + transpose_inputx_node_proto->set_op_type(prim::kPrimTranspose->name()); + onnx::AttributeProto *attr_proto = transpose_inputx_node_proto->add_attribute(); + attr_proto->set_name("perm"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + for (size_t i = 0; i < input_x_shape->shape().size() - kTwoNum; i++) { + attr_proto->add_ints(SizeToLong(i)); + } + attr_proto->add_ints(SizeToLong(input_x_shape->shape().size()) - IntToLong(kOneNum)); + attr_proto->add_ints(SizeToLong(input_x_shape->shape().size()) - IntToLong(kTwoNum)); + } + if (transpose_b) { + auto input_y_shape = dyn_cast(node->input(kTwoNum)->Shape()); + // Add Transpose node after input_y of BatchMatMul + transpose_input_y_name = GenerateUniqueName(); + onnx::NodeProto *transpose_inputy_node_proto = graph_proto->add_node(); + transpose_inputy_node_proto->add_input(input_y); + transpose_inputy_node_proto->add_output(transpose_input_y_name); + transpose_inputy_node_proto->set_op_type(prim::kPrimTranspose->name()); + onnx::AttributeProto *attr_proto = transpose_inputy_node_proto->add_attribute(); + attr_proto->set_name("perm"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + for (size_t i = 0; i < input_y_shape->shape().size() - kTwoNum; i++) { + attr_proto->add_ints(SizeToLong(i)); + } + attr_proto->add_ints(SizeToLong(input_y_shape->shape().size()) - IntToLong(kOneNum)); + attr_proto->add_ints(SizeToLong(input_y_shape->shape().size()) - IntToLong(kTwoNum)); + } + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("MatMul"); + node_proto->add_output(node_name); + node_proto->set_name(node_name + "MatMul"); + if (transpose_a) { + node_proto->add_input(transpose_input_x_name); + } else { + node_proto->add_input(input_x); + } + if (transpose_b) { + node_proto->add_input(transpose_input_y_name); + } else { + node_proto->add_input(input_y); + } +} + +// MindSpore GeLU -> ONNX 0.5 * X * (1.0 + tanh((sqrt(2/pi) * (x + 0.044715 * pow(x, 3))))) +void OnnxExporter::ExportPrimGeLU(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto onnx_type = GetOutputType(node->input(kOneNum)); + + // Add pow node + auto pow_name = GenerateUniqueName(); + auto exp_node_name = pow_name + "exponent_initializer"; + AddFloatTensor1DInitializer(exp_node_name, {3.0}, onnx_type, graph_proto); + AddOp("Pow", {input_x, exp_node_name}, {pow_name}, graph_proto); + + // Add first Mul Node + auto fmul_name = GenerateUniqueName(); + auto fmul_input_node_name = fmul_name + "input_y_for_mul_initializer"; + AddFloatTensor1DInitializer(fmul_input_node_name, {0.044715}, onnx_type, graph_proto); + AddOp("Mul", {pow_name, fmul_input_node_name}, {fmul_name}, graph_proto); + + // Add first Add node + auto fadd_name = GenerateUniqueName(); + AddOp("Add", {input_x, fmul_name}, {fadd_name}, graph_proto); + + // Add second Mul Node + auto smul_name = GenerateUniqueName(); + auto smul_input_node_name = smul_name + "input_y_for_smul_initializer"; + AddFloatTensor1DInitializer(smul_input_node_name, {0.7978845608}, onnx_type, graph_proto); + AddOp("Mul", {fadd_name, smul_input_node_name}, {smul_name}, graph_proto); + + // Add tanh node + auto tanh_name = GenerateUniqueName(); + AddOp("Tanh", {smul_name}, {tanh_name}, graph_proto); + + // Add second Add node + auto sadd_name = GenerateUniqueName(); + auto sadd_input_node_name = sadd_name + "input_y_for_sadd_initializer"; + AddFloatTensor1DInitializer(sadd_input_node_name, {1.0}, onnx_type, graph_proto); + AddOp("Add", {tanh_name, sadd_input_node_name}, {sadd_name}, graph_proto); + + // Add third Mul Node + auto tmul_name = GenerateUniqueName(); + auto tmul_input_node_name = tmul_name + "input_y_for_tmul_initializer"; + AddFloatTensor1DInitializer(tmul_input_node_name, {0.5}, onnx_type, graph_proto); + AddOp("Mul", {sadd_name, tmul_input_node_name}, {tmul_name}, graph_proto); + + // Add fourth Mul Node + auto fomul_node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + AddOp("Mul", {input_x, tmul_name}, {fomul_node_name}, graph_proto); +} + +void OnnxExporter::ExportPrimConcat(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + // Get inputs first: otherwise if an input is a constant, topological order will break + auto input_node = node->input(kOneNum)->cast(); + std::vector input_names; + if (input_node->IsApply(prim::kPrimMakeTuple)) { + for (size_t i = 1; i < input_node->inputs().size(); ++i) { + auto input_name = GetNodeInputName(input_node->input(i), node_map_ptr, graph_proto); + input_names.push_back(input_name); + } + } else { + auto input_data = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + input_names.push_back(input_data); + } + + AddConcatOp(input_names, node_name, GetOpAttribute(node, "axis"), graph_proto); +} + +void OnnxExporter::ExportPrimCast(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_data = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_type = node->input(kTwoNum); + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type(prim::kPrimCast->name()); + node_proto->add_output(node_name); + node_proto->add_input(input_data); + + if (input_type->isa()) { + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("to"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + auto type_value = dyn_cast(input_type)->value(); + auto type_ptr = dyn_cast(type_value); + MS_EXCEPTION_IF_NULL(type_ptr); + attr_proto->set_i(GetOnnxDataType(type_ptr->type_id())); + } else { + MS_LOG(EXCEPTION) << "Need to convert MindSpore Cast input(1) to ONNX Cast to attribute."; + } +} + +void OnnxExporter::ExportPrimPReLU(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_slope = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + + auto x_shape = dyn_cast(node->input(kOneNum)->Shape()); + auto slope_shape = dyn_cast(node->input(kTwoNum)->Shape()); + MS_EXCEPTION_IF_NULL(x_shape); + MS_EXCEPTION_IF_NULL(slope_shape); + + // format of x is NCHW, input format is NCHW, if length of input_slope is 1, insert Unsqueeze [1,2] + if (x_shape->shape().size() == kFourNum && slope_shape->shape().size() == kOneNum) { + auto node_name = GenerateUniqueName(); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Unsqueeze"); + node_proto->add_output(node_name); + + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + attr_proto->set_name("axes"); + attr_proto->add_ints(kOneNum); + attr_proto->add_ints(kTwoNum); + + node_proto->add_input(input_slope); + input_slope = node_name; + } + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("PRelu"); + node_proto->add_output(node_name); + node_proto->add_input(input_x); + node_proto->add_input(input_slope); +} + +void OnnxExporter::ExportPrimReLU6(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto input_x_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto onnx_input_type = GetOutputType(node->input(kOneNum)); + AddClipOp(input_x_name, node_name, 0.0f, 6.0f, onnx_input_type, graph_proto); +} + +void OnnxExporter::ExportPrimDepthwiseConv2d(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto input_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_w = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto x_shape = dyn_cast(node->input(kOneNum)->Shape()); + auto w_shape = dyn_cast(node->input(kTwoNum)->Shape()); + MS_EXCEPTION_IF_NULL(x_shape); + MS_EXCEPTION_IF_NULL(w_shape); + if (x_shape->shape().size() != kFourNum || w_shape->shape().size() != kFourNum) { + MS_LOG(EXCEPTION) << "DepthwiseConv2d input shape should be 4d."; + } + if (w_shape->shape()[kZeroNum] != kOneNum && w_shape->shape()[kOneNum] != kOneNum) { + MS_LOG(EXCEPTION) << "DepthwiseConv2d weight shape[0] != 1 and shape[1] != 1, cannot reshape"; + } + // create w_shape constant node + auto node_name = GenerateUniqueName(); + onnx::NodeProto *node_proto = graph_proto->add_node(); + auto name_w_shape = node_name; + node_proto->add_output(name_w_shape); + node_proto->set_op_type("Constant"); + // create Value Tensor + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + onnx::TensorProto *tensor_proto = attr_proto->mutable_t(); + tensor_proto->add_dims(static_cast<::google::protobuf::int64>(w_shape->shape().size())); + tensor_proto->set_data_type(onnx::TensorProto_DataType_INT64); + // reshape + tensor_proto->add_int64_data(w_shape->shape()[kOneNum]); + tensor_proto->add_int64_data(w_shape->shape()[kZeroNum]); + tensor_proto->add_int64_data(w_shape->shape()[kTwoNum]); + tensor_proto->add_int64_data(w_shape->shape()[kThreeNum]); + + // add reshape node + node_name = GenerateUniqueName(); + node_proto = graph_proto->add_node(); + node_proto->set_op_type(prim::kPrimReshape->name()); + node_proto->add_input(input_w); + node_proto->add_input(name_w_shape); + input_w = node_name; + node_proto->add_output(input_w); + + // add conv node + node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + node_proto = graph_proto->add_node(); + node_proto->set_op_type("Conv"); + node_proto->add_input(input_x); + node_proto->add_input(input_w); + node_proto->add_output(node_name); + // set attributes + AnfNodePtr op = node->input(0); + auto op_value = dyn_cast(op); + auto prim = dyn_cast(op_value->value()); + // set dilations + onnx::AttributeProto *onnx_attr_proto = node_proto->add_attribute(); + onnx_attr_proto->set_name("dilations"); + SetAttrTupleValueToProto<2>(prim->GetAttr("dilation"), onnx::AttributeProto_AttributeType_INTS, onnx_attr_proto, + prim); + // set group + onnx_attr_proto = node_proto->add_attribute(); + onnx_attr_proto->set_name("group"); + onnx_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + onnx_attr_proto->set_i(x_shape->shape()[1]); + // set kernel_shape + onnx_attr_proto = node_proto->add_attribute(); + onnx_attr_proto->set_name("kernel_shape"); + SetAttrTupleValueToProto<0>(prim->GetAttr("kernel_size"), onnx::AttributeProto_AttributeType_INTS, onnx_attr_proto, + prim); + + // set pad + onnx_attr_proto = node_proto->add_attribute(); + int64_t attr_value; + CheckAndConvertUtils::GetPadModEnumValue(prim->GetAttr("pad_mode"), &attr_value); + onnx_attr_proto->set_name("auto_pad"); + onnx_attr_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + if (attr_value == PadMode::VALID) { + onnx_attr_proto->set_s("VALID"); + } else if (attr_value == PadMode::SAME) { + onnx_attr_proto->set_s("SAME_UPPER"); + } else { + onnx_attr_proto->set_name("pads"); + SetAttrTupleValueToProto(prim->GetAttr("pad_list"), onnx::AttributeProto_AttributeType_INTS, onnx_attr_proto, prim); + } + // set strides + onnx_attr_proto = node_proto->add_attribute(); + onnx_attr_proto->set_name("strides"); + SetAttrTupleValueToProto<2>(prim->GetAttr("stride"), onnx::AttributeProto_AttributeType_INTS, onnx_attr_proto, prim); +} + +void OnnxExporter::ExportPrimTile(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto name_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto multiples = node->input(kTwoNum); + std::string name_multiples; + if (multiples->isa()) { + onnx::NodeProto *node_proto = graph_proto->add_node(); + name_multiples = RegisterNodeWithUniqueName(multiples, node_map_ptr); + node_proto->add_output(name_multiples); + node_proto->set_op_type("Constant"); + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("value"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + ConvertTupleToTensor(dyn_cast(multiples)->value(), attr_proto->mutable_t()); + } else { + name_multiples = GetNodeInputName(multiples, node_map_ptr, graph_proto); + MS_LOG(EXCEPTION) << "Need to insert op convert variable from tuple to tensor for Tile."; + } + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Tile"); + node_proto->add_output(node_name); + node_proto->add_input(name_x); + node_proto->add_input(name_multiples); +} + +void OnnxExporter::ExportPrimSquare(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto name_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto name_exponent = GenerateUniqueName(); + onnx::NodeProto *node_proto_exp = graph_proto->add_node(); + node_proto_exp->add_output(name_exponent); + + node_proto_exp->set_op_type("Constant"); + onnx::AttributeProto *attr_proto = node_proto_exp->add_attribute(); + attr_proto->set_name("value"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_TENSOR); + onnx::TensorProto *tensor_proto = attr_proto->mutable_t(); + const float exponent_value = 2.0; + tensor_proto->set_name("exponent"); + tensor_proto->add_dims(static_cast<::google::protobuf::int64>(1)); + tensor_proto->set_data_type(GetOnnxDataType(kNumberTypeFloat32)); + tensor_proto->add_float_data(exponent_value); + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Pow"); + node_proto->add_output(node_name); + node_proto->add_input(name_x); + node_proto->add_input(name_exponent); +} + +void OnnxExporter::ExportPrimGatherV2(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto name_x = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto name_indices = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto axis = node->input(kThreeNum)->cast()->value(); + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Gather"); + node_proto->add_output(node_name); + node_proto->add_input(name_x); + node_proto->add_input(name_indices); + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name("axis"); + attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + attr_proto->set_i(static_cast<::google::protobuf::int64>(dyn_cast(axis)->value())); +} + +/* + This is a workaround for nodes with several outputs used at once + MatchAndMark cannot help here, because it only supports a single output + Proposed convention: + * Nodes with several outputs are registered as + `(*node_map_ptr)[node] = node_idx;`, just like nodes with a single output + * Their outputs are named "{node_idx}_{output_idx}" + * TupleGetItem automatically passes the outputs to the next nodes + See OnnxExporter::ExportPrimTopK for a usage example +*/ +void OnnxExporter::ExportPrimTupleGetItem(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto index = GetInt64Value(node->input(kTwoNum)); + + auto input_node_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_name = MakeOutputName(input_node_name, index); + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Identity"); + node_proto->add_input(input_name); + node_proto->add_output(node_name); +} + +void OnnxExporter::ExportPrimTopK(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto x_input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto k_input_name = node_name + "k_initializer"; + auto k = GetInt64Value(node->input(kTwoNum)); + AddInt64Tensor1DInitializer(k_input_name, {k}, graph_proto); + + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("TopK"); + node_proto->add_input(x_input_name); + node_proto->add_input(k_input_name); + node_proto->add_output(MakeOutputName(node_name, kZeroNum)); // Values + auto indices_name = MakeOutputName(node_name, kOneNum); + auto indices_cast_name = indices_name + "_cast"; + node_proto->add_output(indices_cast_name); + + onnx::AttributeProto *sorted_attr_proto = node_proto->add_attribute(); + sorted_attr_proto->set_name("sorted"); + sorted_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + auto sorted = GetOpAttribute(node, "sorted"); + sorted_attr_proto->set_i(sorted); + AddCastOp(indices_cast_name, indices_name, onnx::TensorProto_DataType_INT32, graph_proto); +} + +// Based on mindspore/ccsrc/backend/kernel_compiler/cpu/boundingbox_decode_cpu_kernel.cc +void OnnxExporter::ExportPrimBoundingBoxDecode(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto anchor_bbox_input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto deltas_input_name = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto onnx_input_type = GetOutputType(node->input(kOneNum)); + + auto means = GetOpAttributePtr(node, "means"); + std::vector mean_values = GetValue>(means); + auto means_name = node_name + "means_initializer"; + AddFloatTensor1DInitializer(means_name, mean_values, onnx_input_type, graph_proto); + + auto stds = GetOpAttributePtr(node, "stds"); + std::vector std_values = GetValue>(stds); + auto stds_name = node_name + "stds_initializer"; + AddFloatTensor1DInitializer(stds_name, std_values, onnx_input_type, graph_proto); + + auto wh_ratio_clip = GetOpAttribute(node, "wh_ratio_clip"); + auto max_ratio = static_cast(std::abs(std::log(wh_ratio_clip))); + + auto unstd_deltas_name = node_name + "unstd_deltas"; + auto sd_to_add_name = unstd_deltas_name + "__add"; + AddOp("Mul", {deltas_input_name, stds_name}, {sd_to_add_name}, graph_proto); + AddOp("Add", {sd_to_add_name, means_name}, {unstd_deltas_name}, graph_proto); + + auto center_deltas_name = node_name + "center_deltas"; + auto log_scale_deltas_name = node_name + "log_scale_deltas"; + auto lsd_to_clip_name = log_scale_deltas_name + "__clip"; + AddSplitOp(unstd_deltas_name, {center_deltas_name, lsd_to_clip_name}, {kTwoNum, kTwoNum}, 1, graph_proto); + AddClipOp(lsd_to_clip_name, log_scale_deltas_name, -max_ratio, max_ratio, onnx_input_type, graph_proto); + + auto anchor_starts_name = node_name + "anchor_starts"; + auto anchor_ends_name = node_name + "anchor_ends"; + AddSplitOp(anchor_bbox_input_name, {anchor_starts_name, anchor_ends_name}, {kTwoNum, kTwoNum}, 1, graph_proto); + + auto anchor_centers_name = node_name + "anchor_centers"; + auto anchor_dimensions_name = node_name + "anchor_dimensions"; + ConvertBoxesToXywh(anchor_starts_name, anchor_ends_name, anchor_centers_name, anchor_dimensions_name, onnx_input_type, + graph_proto); + + auto anchor_shifts_name = node_name + "anchor_shifts"; + AddOp("Mul", {anchor_dimensions_name, center_deltas_name}, {anchor_shifts_name}, graph_proto); + auto result_centers_name = node_name + "result_centers"; + AddOp("Add", {anchor_centers_name, anchor_shifts_name}, {result_centers_name}, graph_proto); + + auto anchor_scales_name = node_name + "anchor_scales"; + AddOp("Exp", {log_scale_deltas_name}, {anchor_scales_name}, graph_proto); + auto result_dimensions_name = node_name + "result_dimensions"; + AddOp("Mul", {anchor_dimensions_name, anchor_scales_name}, {result_dimensions_name}, graph_proto); + + auto result_starts_to_clip_name = node_name + "result_starts_to_clip"; + auto result_ends_to_clip_name = node_name + "result_ends_to_clip"; + ConvertBoxesToXyxy(result_centers_name, result_dimensions_name, result_starts_to_clip_name, result_ends_to_clip_name, + onnx_input_type, graph_proto); + + auto max_shape = GetOpAttributePtr(node, "max_shape"); + auto max_y = GetValue((*max_shape)[0]); + auto max_x = GetValue((*max_shape)[1]); + auto result_start_xs_name = node_name + "result_start_x"; + auto result_start_ys_name = node_name + "result_start_y"; + auto result_end_xs_name = node_name + "result_end_x"; + auto result_end_ys_name = node_name + "result_end_y"; + ClipPointsComponent(result_starts_to_clip_name, result_start_xs_name, static_cast(max_x), 0, onnx_input_type, + graph_proto); + ClipPointsComponent(result_starts_to_clip_name, result_start_ys_name, static_cast(max_y), 1, onnx_input_type, + graph_proto); + ClipPointsComponent(result_ends_to_clip_name, result_end_xs_name, static_cast(max_x), 0, onnx_input_type, + graph_proto); + ClipPointsComponent(result_ends_to_clip_name, result_end_ys_name, static_cast(max_y), 1, onnx_input_type, + graph_proto); + + AddConcatOp({result_start_xs_name, result_start_ys_name, result_end_xs_name, result_end_ys_name}, node_name, kOneNum, + graph_proto); +} + +void OnnxExporter::ExportPrimNMSWithMask(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto bboxes_input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto iou_threshold = GetOpAttribute(node, "iou_threshold"); + auto selected_boxes_output_name = MakeOutputName(node_name, kZeroNum); + auto selected_idx_output_name = MakeOutputName(node_name, kOneNum); + auto selected_mask_output_name = MakeOutputName(node_name, kTwoNum); + auto onnx_input_type = GetOutputType(node->input(kOneNum)); + + // Preprocessing + + auto boxes_count_name = node_name + "max_output_boxes"; + auto max_output_boxes_to_squeeze_name = boxes_count_name + "_to_reshape"; + auto input_shape_name = node_name + "input_shape"; + AddOp("Shape", {bboxes_input_name}, {input_shape_name}, graph_proto); + AddSliceOp(input_shape_name, max_output_boxes_to_squeeze_name, {0}, {1}, {0}, {1}, graph_proto); + AddReshapeOp(max_output_boxes_to_squeeze_name, boxes_count_name, {}, graph_proto); + + auto scores_name = node_name + "scores"; + auto flat_scores_name = scores_name + "_flat"; + auto sorted_scores_name = flat_scores_name + "_sorted"; + auto scores_to_flatten_name = scores_name + "_to_reshape"; + auto descending_order_name = node_name + "descending_indices"; + const int BBOX_NUM_EL = 4; + AddSliceOp(bboxes_input_name, scores_to_flatten_name, {BBOX_NUM_EL}, {BBOX_NUM_EL + 1}, {1}, {1}, graph_proto); + AddReshapeOp(scores_to_flatten_name, flat_scores_name, {-1}, graph_proto); + AddOp("TopK", {flat_scores_name, max_output_boxes_to_squeeze_name}, {sorted_scores_name, descending_order_name}, + graph_proto); + AddReshapeOp(sorted_scores_name, scores_name, {1, 1, -1}, graph_proto); + auto iou_threshold_name = node_name + "iou_threshold_initializer"; + AddFloatScalarInitializer(iou_threshold_name, iou_threshold, onnx::TensorProto_DataType_FLOAT, graph_proto); + + AddOp("Gather", {bboxes_input_name, descending_order_name}, {selected_boxes_output_name}, + graph_proto); // Output 0: boxes + auto boxes_name = node_name + "boxes"; + auto boxes_to_reshape_name = boxes_name + "_to_reshape"; + AddSliceOp(selected_boxes_output_name, boxes_to_reshape_name, {0}, {BBOX_NUM_EL}, {1}, {1}, graph_proto); + AddReshapeOp(boxes_to_reshape_name, boxes_name, {1, -1, BBOX_NUM_EL}, graph_proto); + + if (onnx_input_type == onnx::TensorProto_DataType_FLOAT16) { + auto fp32_boxes_name = boxes_name + "_fp32"; + AddCastOp(boxes_name, fp32_boxes_name, onnx::TensorProto_DataType_FLOAT, graph_proto); + boxes_name = fp32_boxes_name; + + auto fp32_scores_name = scores_name + "_fp32"; + AddCastOp(scores_name, fp32_scores_name, onnx::TensorProto_DataType_FLOAT, graph_proto); + scores_name = fp32_scores_name; + } + + // NMS op + + auto selected_indices_name = node_name + "selected_indices"; + AddOp("NonMaxSuppression", {boxes_name, scores_name, boxes_count_name, iou_threshold_name}, {selected_indices_name}, + graph_proto); + + // Output 1: indices + + auto flat_indices_name = node_name + "flat_indices"; + auto flat_indices_to_squeeze_name = flat_indices_name + "__reshape"; + const int BOX_INDEX_POS = 2; + AddSliceOp(selected_indices_name, flat_indices_to_squeeze_name, {BOX_INDEX_POS}, {BOX_INDEX_POS + 1}, {1}, {1}, + graph_proto); + AddReshapeOp(flat_indices_to_squeeze_name, flat_indices_name, {-1}, graph_proto); + + auto zero_name = node_name + "zero_initializer"; + onnx::TensorProto *zero_initializer = graph_proto->add_initializer(); + zero_initializer->set_name(zero_name); + zero_initializer->set_data_type(onnx::TensorProto_DataType_INT32); + zero_initializer->add_int32_data(0); + auto one_name = node_name + "one_initializer"; + onnx::TensorProto *one_initializer = graph_proto->add_initializer(); + one_initializer->set_name(one_name); + one_initializer->set_data_type(onnx::TensorProto_DataType_INT32); + one_initializer->add_int32_data(1); + auto int32_boxes_count_name = boxes_count_name + "_int32"; + AddCastOp(boxes_count_name, int32_boxes_count_name, onnx::TensorProto_DataType_INT32, graph_proto); + AddOp("Range", {zero_name, int32_boxes_count_name, one_name}, {selected_idx_output_name}, graph_proto); + + // Output 2: mask + + auto empty_mask_name = selected_mask_output_name + "__scatter"; + onnx::TensorProto *empty_mask_value_proto = + AddConstantOfShapeOp(max_output_boxes_to_squeeze_name, empty_mask_name, graph_proto); + empty_mask_value_proto->set_data_type(onnx::TensorProto_DataType_BOOL); + empty_mask_value_proto->add_int32_data(0); + + auto true_elements_name = node_name + "true"; + auto true_elements_shape_name = true_elements_name + "_shape"; + AddOp("Shape", {flat_indices_name}, {true_elements_shape_name}, graph_proto); + onnx::TensorProto *true_elements_value_proto = + AddConstantOfShapeOp(true_elements_shape_name, true_elements_name, graph_proto); + true_elements_value_proto->set_data_type(onnx::TensorProto_DataType_BOOL); + true_elements_value_proto->add_int32_data(1); + + AddOp("ScatterElements", {empty_mask_name, flat_indices_name, true_elements_name}, {selected_mask_output_name}, + graph_proto); +} + +void OnnxExporter::ExportPrimSplit(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + auto axis = GetOpAttribute(node, "axis"); + auto output_num = GetOpAttribute(node, "output_num"); + if (output_num == 0) { + MS_LOG(EXCEPTION) << "output_num must be > 0"; + } + const auto &input_shape = dyn_cast(node->input(kOneNum)->Shape())->shape(); + + if (axis < 0 || static_cast(axis) >= input_shape.size()) { + MS_LOG(EXCEPTION) << "`axis` is out of range"; + } + if (input_shape[static_cast(axis)] % output_num != 0) { + MS_LOG(EXCEPTION) << "Input dim is not divisible by `output_num`"; + } + + onnx::NodeProto *split_proto = graph_proto->add_node(); + split_proto->set_op_type("Split"); + split_proto->add_input(input_name); + for (int64_t i = 0; i < output_num; ++i) { + split_proto->add_output(MakeOutputName(node_name, i)); + } + + onnx::AttributeProto *axis_attr_proto = split_proto->add_attribute(); + axis_attr_proto->set_name("axis"); + axis_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + axis_attr_proto->set_i(axis); + + onnx::AttributeProto *split_attr_proto = split_proto->add_attribute(); + split_attr_proto->set_name("split"); + split_attr_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + for (int64_t i = 0; i < output_num; ++i) { + split_attr_proto->add_ints(input_shape[static_cast(axis)] / output_num); + } +} + +/* + Based on mindspore-project/mindspore/ccsrc/backend/kernel_compiler/cpu/roi_align_cpu_kernel.cc + Notes: + * MS version uses avg pool, leaving corresponding ONNX attr as is + * MS has two ROI end modes, implemented with pre-processing + */ +void OnnxExporter::ExportPrimROIAlign(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto features_input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto rois_input_name = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto onnx_input_type = GetOutputType(node->input(kOneNum)); + + auto roi_indices_name = node_name + "roi_indices"; + auto roi_indices_column_name = roi_indices_name + "_column"; + auto roi_starts_name = node_name + "roi_starts"; + auto roi_ends_name = node_name + "roi_ends"; + AddSplitOp(rois_input_name, {roi_indices_column_name, roi_starts_name, roi_ends_name}, {1, kTwoNum, kTwoNum}, 1, + graph_proto); + + // Indices transformation + + auto flat_roi_indices_name = roi_indices_name + "_flat"; + AddReshapeOp(roi_indices_column_name, flat_roi_indices_name, {-1}, graph_proto); + auto int_roi_indices_name = roi_indices_name + "_int"; + // This should be fine if indices are whole numbers less than 2^23 + AddCastOp(flat_roi_indices_name, int_roi_indices_name, onnx::TensorProto_DataType_INT64, graph_proto); + + // ROI end mode + + auto roi_end_mode = GetOpAttribute(node, "roi_end_mode"); + auto roi_end_mode_name = node_name + "roi_end_mode_initializer"; + AddFloatScalarInitializer(roi_end_mode_name, roi_end_mode, onnx_input_type, graph_proto); + + auto corrected_roi_ends_name = roi_ends_name + "_corrected"; + AddOp("Add", {roi_ends_name, roi_end_mode_name}, {corrected_roi_ends_name}, graph_proto); + + // Contatenate ROIs + + auto corrected_rois_name = node_name + "corrected_rois"; + AddConcatOp({roi_starts_name, corrected_roi_ends_name}, corrected_rois_name, kOneNum, graph_proto); + + // RoiAlign op + + onnx::NodeProto *roi_align_proto = graph_proto->add_node(); + roi_align_proto->set_op_type("RoiAlign"); + roi_align_proto->add_input(features_input_name); + roi_align_proto->add_input(corrected_rois_name); + roi_align_proto->add_input(int_roi_indices_name); + roi_align_proto->add_output(node_name); + onnx::AttributeProto *height_attr_proto = roi_align_proto->add_attribute(); + height_attr_proto->set_name("output_height"); + height_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + height_attr_proto->set_i(GetOpAttribute(node, "pooled_height")); + onnx::AttributeProto *width_attr_proto = roi_align_proto->add_attribute(); + width_attr_proto->set_name("output_width"); + width_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + width_attr_proto->set_i(GetOpAttribute(node, "pooled_width")); + onnx::AttributeProto *scale_attr_proto = roi_align_proto->add_attribute(); + scale_attr_proto->set_name("spatial_scale"); + scale_attr_proto->set_type(onnx::AttributeProto_AttributeType_FLOAT); + scale_attr_proto->set_f(GetOpAttribute(node, "spatial_scale")); + onnx::AttributeProto *sampling_ratio_attr_proto = roi_align_proto->add_attribute(); + sampling_ratio_attr_proto->set_name("sampling_ratio"); + sampling_ratio_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + sampling_ratio_attr_proto->set_i(GetOpAttribute(node, "sample_num")); +} + +void OnnxExporter::ExportPrimSlice(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto input_x_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto begin_input_name = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto size_input_name = GetNodeInputName(node->input(kThreeNum), node_map_ptr, graph_proto); + + auto end_name = node_name + "end"; + AddOp("Add", {begin_input_name, size_input_name}, {end_name}, graph_proto); + AddOp("Slice", {input_x_name, begin_input_name, end_name}, {node_name}, graph_proto); +} + +void OnnxExporter::ExportPrimOnesLike(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto input_x_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + auto shape_name = node_name + "shape"; + AddOp("Shape", {input_x_name}, {shape_name}, graph_proto); + + auto dtype = node->input(kOneNum)->Type(); + auto elem_type = dyn_cast(dtype)->element()->type_id(); + + onnx::TensorProto *one_proto = AddConstantOfShapeOp(shape_name, node_name, graph_proto); + switch (elem_type) { + case kNumberTypeInt32: + one_proto->set_data_type(onnx::TensorProto_DataType_INT32); + one_proto->add_int32_data(1); + break; + case kNumberTypeInt64: + one_proto->set_data_type(onnx::TensorProto_DataType_INT64); + one_proto->add_int64_data(1); + break; + case kNumberTypeFloat32: + one_proto->set_data_type(onnx::TensorProto_DataType_FLOAT); + one_proto->add_float_data(1.0f); + break; + case kNumberTypeFloat64: + one_proto->set_data_type(onnx::TensorProto_DataType_DOUBLE); + one_proto->add_double_data(1.0); + break; + default: + MS_LOG(EXCEPTION) << "Unsupported dtype: " << elem_type; + } +} + +void OnnxExporter::ExportPrimArgMaxWithValue(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto input_x_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto axis = GetOpAttribute(node, "axis"); + auto keep_dims = GetOpAttribute(node, "keep_dims"); + + auto indices_output_name = MakeOutputName(node_name, kZeroNum); + auto indices_cast_name = indices_output_name + "_cast"; + + onnx::NodeProto *argmax_proto = graph_proto->add_node(); + argmax_proto->set_op_type("ArgMax"); + argmax_proto->add_input(input_x_name); + argmax_proto->add_output(indices_cast_name); + onnx::AttributeProto *argmax_axis_attr_proto = argmax_proto->add_attribute(); + argmax_axis_attr_proto->set_name("axis"); + argmax_axis_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + argmax_axis_attr_proto->set_i(axis); + onnx::AttributeProto *argmax_keepdims_attr_proto = argmax_proto->add_attribute(); + argmax_keepdims_attr_proto->set_name("keepdims"); + argmax_keepdims_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + argmax_keepdims_attr_proto->set_i(keep_dims); + + AddCastOp(indices_cast_name, indices_output_name, onnx::TensorProto_DataType_INT32, graph_proto); + + auto max_output_name = MakeOutputName(node_name, kOneNum); + AddReduceOp("ReduceMax", input_x_name, max_output_name, {axis}, keep_dims, graph_proto); +} + +void OnnxExporter::ExportPrimOneHot(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto indices_input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto depth_input_name = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto on_input_name = GetNodeInputName(node->input(kThreeNum), node_map_ptr, graph_proto); + auto off_input_name = GetNodeInputName(node->input(kFourNum), node_map_ptr, graph_proto); + auto axis = GetOpAttribute(node, "axis"); + + if (GetOutputType(node->input(kOneNum)) == onnx::TensorProto_DataType_INT32) { + auto indices_cast_name = node_name + "_indices_as_int32"; + AddCastOp(indices_input_name, indices_cast_name, onnx::TensorProto_DataType_INT64, graph_proto); + indices_input_name = indices_cast_name; + } + + auto on_1d_name = node_name + "on_1d"; + AddReshapeOp(on_input_name, on_1d_name, {-1}, graph_proto); + auto off_1d_name = node_name + "off_1d"; + AddReshapeOp(off_input_name, off_1d_name, {-1}, graph_proto); + + auto on_off_name = node_name + "on_off"; + AddConcatOp({off_1d_name, on_1d_name}, on_off_name, kZeroNum, graph_proto); + + onnx::NodeProto *one_hot_proto = graph_proto->add_node(); + one_hot_proto->set_op_type("OneHot"); + one_hot_proto->add_input(indices_input_name); + one_hot_proto->add_input(depth_input_name); + one_hot_proto->add_input(on_off_name); + one_hot_proto->add_output(node_name); + onnx::AttributeProto *one_hot_axis_attr_proto = one_hot_proto->add_attribute(); + one_hot_axis_attr_proto->set_name("axis"); + one_hot_axis_attr_proto->set_type(onnx::AttributeProto_AttributeType_INT); + one_hot_axis_attr_proto->set_i(axis); +} + +/* + Based on nn.Conv2dTranspose + Warning: `output_shape` is an input in MS and an attribute in ONNX. Hence + it is not possible to change the output shape in runtime + */ +void OnnxExporter::PrimConv2DTransposeExportHelper(const CNodePtr &conv_node, const CNodePtr &bias_add_node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + std::string node_name; + + std::vector inputs{conv_node->input(kOneNum), conv_node->input(kTwoNum)}; + if (bias_add_node != nullptr) { + inputs.push_back(bias_add_node->input(kTwoNum)); + node_name = RegisterNodeWithUniqueName(bias_add_node, node_map_ptr); + } else { + node_name = RegisterNodeWithUniqueName(conv_node, node_map_ptr); + } + + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("ConvTranspose"); + for (const auto &input : inputs) { + node_proto->add_input(GetNodeInputName(input, node_map_ptr, graph_proto)); + } + node_proto->add_output(node_name); + + auto prim = GetPrimitive(conv_node); + auto attrs_convert_info = + OpNameInfo() + .Attr("dilation", "dilations", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto) + .Attr("group", "group", onnx::AttributeProto_AttributeType_INT, SetAttrValueToProto) + .Attr("kernel_size", "kernel_shape", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto<0>) + .Attr("pad_mode", "auto_pad", onnx::AttributeProto_AttributeType_STRING, SetConvTransposePadding) + .Attr("stride", "strides", onnx::AttributeProto_AttributeType_INTS, SetAttrTupleValueToProto); + for (const auto &attr_info : attrs_convert_info.op_attrs()) { + onnx::AttributeProto *attr_proto = node_proto->add_attribute(); + attr_proto->set_name(attr_info.onnx_attr_name()); + auto ms_attr = GetOpAttributePtr(conv_node, attr_info.attr_name()); + MS_EXCEPTION_IF_NULL(ms_attr); + attr_info.fn_gen_attr()(ms_attr, attr_info.onnx_attr_type(), attr_proto, prim); + } + + // Set output shape + + auto input_shape_node = GetRealInput(conv_node->input(kThreeNum)); + if (!input_shape_node->isa()) { + MS_LOG(EXCEPTION) << "For ONNX export third argument must be constant " + "(Python tuple). Instead got " + << input_shape_node->ToString(); + } + auto input_shape_value_ptr = input_shape_node->cast()->value(); + if (!input_shape_value_ptr->isa()) { + MS_LOG(EXCEPTION) << "Expected ValueTuple, got " << input_shape_value_ptr->ToString() << " of type " + << input_shape_value_ptr->type()->ToString(); + } + + onnx::AttributeProto *output_shape_attr_proto = node_proto->add_attribute(); + output_shape_attr_proto->set_name("output_shape"); + SetAttrTupleValueToProto<0>(input_shape_value_ptr, onnx::AttributeProto_AttributeType_INTS, output_shape_attr_proto, + prim); +} + +void OnnxExporter::ExportPrimConv2DTranspose(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *graph_proto) { + PrimConv2DTransposeExportHelper(node, nullptr, node_map_ptr, graph_proto); +} + +void OnnxExporter::ExportPrimGreaterEqual(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto input_x_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto input_y_name = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto less_name = node_name + "less"; + + AddOp("Less", {input_x_name, input_y_name}, {less_name}, graph_proto); + AddOp("Not", {less_name}, {node_name}, graph_proto); +} + +void OnnxExporter::ExportPrimSqueeze(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_op_type("Squeeze"); + node_proto->add_input(input_name); + node_proto->add_output(node_name); + + auto axes = GetOpAttributePtr(node, "axis"); + auto axes_value = GetValue>(axes); + if (!axes_value.empty()) { + onnx::AttributeProto *axes_proto = node_proto->add_attribute(); + axes_proto->set_name("axes"); + axes_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + for (auto axis : axes_value) { + axes_proto->add_ints(axis); + } + } +} + +void MakeLSTMWeight(const std::string &input, const std::string &output, const std::vector &output_shape, + onnx::GraphProto *graph_proto) { + // 创建LSTM权重 + auto reshaped_name = output + "__split";// 将输入数据进行重塑 + AddReshapeOp(input, reshaped_name, output_shape, graph_proto); + // 分割输入数据为不同部分 + auto split_i_name = output + "__concat_i"; + auto split_o_name = output + "__concat_o"; + auto split_f_name = output + "__concat_f"; + auto split_c_name = output + "__concat_c"; + int64_t hidden_size = output_shape[kOneNum] / kFourNum; + AddSplitOp(reshaped_name, {split_i_name, split_f_name, split_c_name, split_o_name}, + {hidden_size, hidden_size, hidden_size, hidden_size}, 1, graph_proto); + // 将分割后的部分连接成输出 + AddConcatOp({split_i_name, split_o_name, split_f_name, split_c_name}, output, 1, graph_proto); +} +// 导出 LSTM 层的权重到 ONNX 格式 +void ExportLSTMWeights(const CNodePtr &node, const std::string &node_name, const std::string &weights_name, + onnx::TensorProto_DataType dtype, const std::string &onnx_input_weights_name, + const std::string &onnx_hidden_weights_name, const std::string &onnx_bias_name, + onnx::GraphProto *graph_proto) { + // 从节点中获取各种属性 + auto input_size = GetOpAttribute(node, "input_size"); + auto hidden_size = GetOpAttribute(node, "hidden_size"); + auto num_layers = GetOpAttribute(node, "num_layers"); + auto has_bias = GetOpAttribute(node, "has_bias"); + auto bidirectional = GetOpAttribute(node, "bidirectional"); + auto num_dir = 1 + static_cast(bidirectional); + auto num_gates = 4; + auto gate_size = num_gates * hidden_size; + // 检查是否支持多层 LSTM + if (num_layers != 1) { + MS_LOG(EXCEPTION) << "Converter for multilayer LSTM is not implemented"; + } + // 检查是否支持双向模式 + if (bidirectional) { + MS_LOG(EXCEPTION) << "Bidirectional mode for P.LSTM is not implemented"; + } + auto ms_context = MsContext::GetInstance(); + MS_EXCEPTION_IF_NULL(ms_context); + auto target_device = ms_context->get_param(MS_CTX_DEVICE_TARGET); + // 检查设备目标是否合法 + if (target_device != "CPU" && target_device != "GPU") { + MS_LOG(EXCEPTION) << "Unsupported target device: " << target_device; + } + // 生成输入权重、隐藏权重、输入偏差和隐藏偏差的名称 + auto input_weights_name = node_name + "_input_weights"; + auto hidden_weights_name = node_name + "_hidden_weights"; + auto input_bias_name = node_name + "_input_bias"; + auto hidden_bias_name = node_name + "_hidden_bias"; + // 准备权重和偏差的分割参数 + std::vector split_sizes = {input_size * gate_size, hidden_size * gate_size}; + std::vector split_outputs = {input_weights_name, hidden_weights_name}; + if (has_bias) { + if (target_device == "GPU") { + (void)split_sizes.insert(split_sizes.end(), {gate_size, gate_size}); + (void)split_outputs.insert(split_outputs.end(), {input_bias_name, hidden_bias_name}); + } else if (target_device == "CPU") { + split_sizes.push_back(gate_size); + split_outputs.push_back(input_bias_name); + } else { + MS_LOG(EXCEPTION) << "Impossible branch"; + } + } + // 添加分割操作 + AddSplitOp(weights_name, split_outputs, split_sizes, 0, graph_proto); + // 创建输入权重和隐藏权重的 ONNX 张量 + MakeLSTMWeight(input_weights_name, onnx_input_weights_name, {num_dir, gate_size, input_size}, graph_proto); + MakeLSTMWeight(hidden_weights_name, onnx_hidden_weights_name, {num_dir, gate_size, hidden_size}, graph_proto); + // 处理偏差 + if (has_bias) { + auto onnx_input_bias_name = node_name + "_onnx_input_bias"; + auto onnx_hidden_bias_name = node_name + "_onnx_hidden_bias"; + if (target_device == "GPU") { + // 创建 GPU 下的偏差张量 + MakeLSTMWeight(input_bias_name, onnx_input_bias_name, {num_dir, gate_size}, graph_proto); + MakeLSTMWeight(hidden_bias_name, onnx_hidden_bias_name, {num_dir, gate_size}, graph_proto); + } else if (target_device == "CPU") { + // 创建 CPU 下的偏差张量 + MakeLSTMWeight(input_bias_name, onnx_input_bias_name, {num_dir, gate_size}, graph_proto); + // 创建用于填充的零张量 + auto bias_shape_name = node_name + "_bias_shape"; + AddOp("Shape", {onnx_input_bias_name}, {bias_shape_name}, graph_proto); + onnx::TensorProto *zero_padding = AddConstantOfShapeOp(bias_shape_name, onnx_hidden_bias_name, graph_proto); + zero_padding->set_data_type(dtype); + // 根据数据类型添加不同类型的零值 + if (dtype == onnx::TensorProto_DataType_FLOAT16) { + zero_padding->add_int32_data(0); // float 0 and int 0 have identical representations + } else if (dtype == onnx::TensorProto_DataType_FLOAT) { + zero_padding->add_float_data(0.0f); + } else { + MS_LOG(EXCEPTION) << "Unsupported type: " << dtype; + } + } else { + MS_LOG(EXCEPTION) << "Impossible branch"; + } + // 添加连接操作,将偏差连接起来 + AddConcatOp({onnx_input_bias_name, onnx_hidden_bias_name}, onnx_bias_name, 1, graph_proto); + } +} + +void OnnxExporter::ExportPrimLSTM(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + // MS inputs + auto x_input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + auto init_h_input_name = GetNodeInputName(node->input(kTwoNum), node_map_ptr, graph_proto); + auto init_c_input_name = GetNodeInputName(node->input(kThreeNum), node_map_ptr, graph_proto); + + auto hidden_size = GetOpAttribute(node, "hidden_size"); + auto has_bias = GetOpAttribute(node, "has_bias"); + auto bidirectional = GetOpAttribute(node, "bidirectional"); + std::string direction = bidirectional ? "bidirectional" : "forward"; + auto x_input_shape = dyn_cast(node->input(kOneNum)->Shape())->shape(); + auto seq_len = x_input_shape[0]; + auto batch_size = x_input_shape[1]; + auto num_dir = 1 + static_cast(bidirectional); + + auto weights_name = GetNodeInputName(node->input(kFourNum), node_map_ptr, graph_proto); + auto dtype = GetOutputType(node->input(kOneNum)); + auto onnx_input_weights_name = node_name + "_onnx_input_weights"; + auto onnx_hidden_weights_name = node_name + "_onnx_hidden_weights"; + auto onnx_bias_name = node_name + "_onnx_bias"; + + ExportLSTMWeights(node, node_name, weights_name, dtype, onnx_input_weights_name, onnx_hidden_weights_name, + onnx_bias_name, graph_proto); + + // Create LSTM node + onnx::NodeProto *lstm_node_proto = graph_proto->add_node(); + lstm_node_proto->set_op_type("LSTM"); + lstm_node_proto->add_input(x_input_name); + lstm_node_proto->add_input(onnx_input_weights_name); + lstm_node_proto->add_input(onnx_hidden_weights_name); + lstm_node_proto->add_input(has_bias ? onnx_bias_name : ""); + lstm_node_proto->add_input(""); // seqlens + lstm_node_proto->add_input(init_h_input_name); + lstm_node_proto->add_input(init_c_input_name); + + auto Y_output_name = node_name + "_Y"; + lstm_node_proto->add_output(Y_output_name); + lstm_node_proto->add_output(MakeOutputName(node_name, kOneNum)); + lstm_node_proto->add_output(MakeOutputName(node_name, kTwoNum)); + + onnx::AttributeProto *hidden_size_proto = lstm_node_proto->add_attribute(); + hidden_size_proto->set_name("hidden_size"); + hidden_size_proto->set_type(onnx::AttributeProto_AttributeType_INT); + hidden_size_proto->set_i(hidden_size); + + onnx::AttributeProto *direction_proto = lstm_node_proto->add_attribute(); + direction_proto->set_name("direction"); + direction_proto->set_type(onnx::AttributeProto_AttributeType_STRING); + direction_proto->set_s(direction); + + // Transpose 1st output of the LSTM node + onnx::NodeProto *transpose_node_proto = graph_proto->add_node(); + auto transpose_node_name = node_name + "_Y_transposed"; + transpose_node_proto->set_name(transpose_node_name); + transpose_node_proto->set_op_type("Transpose"); + transpose_node_proto->add_input(Y_output_name); + transpose_node_proto->add_output(transpose_node_name); + + onnx::AttributeProto *perm_proto = transpose_node_proto->add_attribute(); + perm_proto->set_name("perm"); + perm_proto->set_type(onnx::AttributeProto_AttributeType_INTS); + perm_proto->add_ints(kZeroNum); + perm_proto->add_ints(kTwoNum); + perm_proto->add_ints(kOneNum); + perm_proto->add_ints(kThreeNum); + + // Reshape + auto output_name = MakeOutputName(node_name, kZeroNum); + AddReshapeOp(transpose_node_name, output_name, {seq_len, batch_size, num_dir * hidden_size}, graph_proto); +} + +void OnnxExporter::ExportPrimReverseV2(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto output = RegisterNodeWithUniqueName(node, node_map_ptr); + auto input = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + auto axes_ptr = GetOpAttributePtr(node, "axis"); + auto axes_vec = GetValue>(axes_ptr); + size_t n_axes = axes_vec.size(); + auto shape = dyn_cast(node->input(kOneNum)->Shape())->shape(); + + std::vector starts_vec(n_axes, -1); + std::vector ends_vec(n_axes); + (void)std::transform(axes_vec.begin(), axes_vec.end(), ends_vec.begin(), + [&shape](size_t ax) { return -shape.at(ax) - 1; }); + std::vector steps_vec(n_axes, -1); + + AddSliceOp(input, output, starts_vec, ends_vec, axes_vec, steps_vec, graph_proto); +} + +void OnnxExporter::ExportPrimTensorCopySlices(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto x_input = node->input(kOneNum); + auto value_input = node->input(kTwoNum); + + auto x_input_name = GetNodeInputName(x_input, node_map_ptr, graph_proto); + auto value_input_name = GetNodeInputName(value_input, node_map_ptr, graph_proto); + + const auto &x_shape = dyn_cast(x_input->Shape())->shape(); + const auto &value_shape = dyn_cast(value_input->Shape())->shape(); + + auto begin_node = dyn_cast(node->input(kThreeNum)); + MS_EXCEPTION_IF_NULL(begin_node); + auto begin = GetValue>(begin_node->value()); + + auto end_node = dyn_cast(node->input(kFourNum)); + MS_EXCEPTION_IF_NULL(end_node); + auto end = GetValue>(end_node->value()); + + auto strides_node = dyn_cast(node->input(kFiveNum)); + MS_EXCEPTION_IF_NULL(strides_node); + auto strides = GetValue>(strides_node->value()); + + MS_EXCEPTION_IF_CHECK_FAIL( + begin.size() == end.size() && end.size() == strides.size() && strides.size() <= x_shape.size(), + "Sizes of begin, end, and strides must be equal"); + // MindSpore only allows contuguous slices of memory + // Contiguous slice size follows the pattern: [1, ..., 1, n, :, ..., :] + bool found_slice = false; + for (size_t i = 0; i < begin.size(); ++i) { + int64_t dim = end[i] - begin[i]; + if (!found_slice && dim != 1) { + found_slice = true; + } else if (found_slice && dim != x_shape[i]) { + MS_LOG(EXCEPTION) << "Slice must be contiguous"; + } + } + for (auto stride : strides) { + MS_EXCEPTION_IF_CHECK_FAIL(stride == 1, "Slice must be contiguous"); + } + + int64_t flat_begin_index = RavelIndex(begin, x_shape); + + std::vector end_inclusive; + (void)std::transform(end.begin(), end.end(), std::back_inserter(end_inclusive), [](auto x) { return x - 1; }); + (void)std::transform(x_shape.begin() + end.size(), x_shape.end(), std::back_inserter(end_inclusive), + [](auto x) { return x - 1; }); + int64_t flat_end_index = RavelIndex(end_inclusive, x_shape) + 1; + + int64_t x_size = std::accumulate(x_shape.begin(), x_shape.end(), 1, std::multiplies()); + int64_t value_size = std::accumulate(value_shape.begin(), value_shape.end(), 1, std::multiplies()); + MS_EXCEPTION_IF_CHECK_FAIL(value_size == flat_end_index - flat_begin_index, "Cannot copy 'value' to target slice"); + + auto flat_x_name = node_name + "_flat_x"; + AddReshapeOp(x_input_name, flat_x_name, {-1}, graph_proto); + auto begin_slice_name = node_name + "_begin_slice"; + AddSliceOp(flat_x_name, begin_slice_name, {0}, {static_cast(flat_begin_index)}, {0}, {1}, graph_proto); + auto end_slice_name = node_name + "_end_slice"; + AddSliceOp(flat_x_name, end_slice_name, {static_cast(flat_end_index)}, {x_size}, {0}, {1}, graph_proto); + + auto flat_value_name = node_name + "_flat_value"; + AddReshapeOp(value_input_name, flat_value_name, {-1}, graph_proto); + + auto flat_result_name = node_name + "_flat_result"; + AddConcatOp({begin_slice_name, flat_value_name, end_slice_name}, flat_result_name, 0, graph_proto); + AddReshapeOp(flat_result_name, node_name, x_shape, graph_proto); +} + +void OnnxExporter::ExportPrimStack(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *graph_proto) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto input_name = GetNodeInputName(node->input(kOneNum), node_map_ptr, graph_proto); + + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_name(node_name + "Stack"); + node_proto->set_op_type("ConcatFromSequence"); + node_proto->add_input(input_name); + node_proto->add_output(node_name); + + onnx::AttributeProto *axis_proto = node_proto->add_attribute(); + axis_proto->set_name("axis"); + axis_proto->set_type(onnx::AttributeProto_AttributeType_INT); + axis_proto->set_i(GetOpAttribute(node, "axis")); + + onnx::AttributeProto *new_axis_proto = node_proto->add_attribute(); + new_axis_proto->set_name("new_axis"); + new_axis_proto->set_type(onnx::AttributeProto_AttributeType_INT); + new_axis_proto->set_i(true); +} + +void OnnxExporter::ExportCNode(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, onnx::GraphProto *const graph_proto) { + using ExportFunc = std::function *, onnx::GraphProto *const)>; + static std::vector> export_table = { + {prim::kPrimReshape, &OnnxExporter::ExportPrimReshape}, + {prim::kPrimReduceMean, &OnnxExporter::ExportPrimReduce}, + {prim::kPrimReduceSum, &OnnxExporter::ExportPrimReduce}, + {prim::kPrimTranspose, &OnnxExporter::ExportPrimTranspose}, + {prim::kPrimStridedSlice, &OnnxExporter::ExportPrimStridedSlice}, + {prim::kPrimResizeNearestNeighbor, &OnnxExporter::ExportPrimResizeNearestNeighbor}, + {prim::kPrimResizeBilinear, &OnnxExporter::ExportPrimResizeBilinear}, + {prim::kPrimConcat, &OnnxExporter::ExportPrimConcat}, + {prim::kPrimCast, &OnnxExporter::ExportPrimCast}, + {prim::kPrimPRelu, &OnnxExporter::ExportPrimPReLU}, + {prim::kPrimRelu6, &OnnxExporter::ExportPrimReLU6}, + {prim::kPrimDepthwiseConv2dNative, &OnnxExporter::ExportPrimDepthwiseConv2d}, + {prim::kPrimTile, &OnnxExporter::ExportPrimTile}, + {prim::kPrimSquare, &OnnxExporter::ExportPrimSquare}, + {prim::kPrimGather, &OnnxExporter::ExportPrimGatherV2}, + {prim::kPrimTupleGetItem, &OnnxExporter::ExportPrimTupleGetItem}, + {prim::kPrimTopK, &OnnxExporter::ExportPrimTopK}, + {prim::kPrimBoundingBoxDecode, &OnnxExporter::ExportPrimBoundingBoxDecode}, + {prim::kPrimNMSWithMask, &OnnxExporter::ExportPrimNMSWithMask}, + {prim::kPrimSplit, &OnnxExporter::ExportPrimSplit}, + {prim::kPrimROIAlign, &OnnxExporter::ExportPrimROIAlign}, + {prim::kPrimSlice, &OnnxExporter::ExportPrimSlice}, + {prim::kPrimOnesLike, &OnnxExporter::ExportPrimOnesLike}, + {prim::kPrimArgMaxWithValue, &OnnxExporter::ExportPrimArgMaxWithValue}, + {prim::kPrimOneHot, &OnnxExporter::ExportPrimOneHot}, + {prim::kPrimConv2DTranspose, &OnnxExporter::ExportPrimConv2DTranspose}, + {prim::kPrimGreaterEqual, &OnnxExporter::ExportPrimGreaterEqual}, + {prim::kPrimSqueeze, &OnnxExporter::ExportPrimSqueeze}, + {prim::kPrimExpandDims, &OnnxExporter::ExportPrimExpandDims}, + {prim::kPrimPad, &OnnxExporter::ExportPrimPad}, + {prim::kPrimBatchMatMul, &OnnxExporter::ExportPrimBatchMatMul}, + {prim::kPrimGeLU, &OnnxExporter::ExportPrimGeLU}, + {prim::kPrimLstm, &OnnxExporter::ExportPrimLSTM}, + {prim::kPrimReverseV2, &OnnxExporter::ExportPrimReverseV2}, + {prim::kPrimTensorCopySlices, &OnnxExporter::ExportPrimTensorCopySlices}, + {prim::kPrimStack, &OnnxExporter::ExportPrimStack}, + }; + + auto iter = std::find_if(export_table.begin(), export_table.end(), + [&node](const auto &item) { return node->IsApply(item.first); }); + if (iter != export_table.end()) { + iter->second(this, func_graph, node, node_map_ptr, graph_proto); + return; + } + + auto inputs = node->inputs(); + if (inputs.size() < 1) { + MS_LOG(EXCEPTION) << "Inputs of apply node is empty"; + } + + AnfNodePtr op = inputs[kZeroNum]; + std::vector op_inputs; + // first process node input 1,2,..., since when node input is a ValueNode, here need to create a Constant Operator + for (size_t i = 1; i < inputs.size(); i++) { + if (!HasAbstractMonad(inputs[i])) { + op_inputs.push_back(inputs[i]); + } + } + + if (!op->isa()) { + MS_LOG(EXCEPTION) << "Need to support node op type " << op->type_name(); + } + + auto op_value = dyn_cast(op)->value(); + if (op_value->isa()) { + auto prim = dyn_cast(op_value); + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim, op_inputs, graph_proto); + } else if (while_loop_export::IsControlSubgraph(op_value)) { + ExportWhileLoop(node, node_map_ptr, graph_proto); + } else { + MS_LOG(EXCEPTION) << "Need to support node op value type " << op_value->type_name(); + } +} + +void OnnxExporter::ExportWhileLoop(const CNodePtr &start_node, std::map *node_map_ptr, + onnx::GraphProto *graph_proto) { + auto node_name = RegisterNodeWithUniqueName(start_node, node_map_ptr); + auto loop_parts = while_loop_export::MatchGraph(start_node); + + // 1. Make Loop op + + onnx::NodeProto *loop_proto = graph_proto->add_node(); + loop_proto->set_op_type("Loop"); + + auto loop_count_name = node_name + "_M"; + const auto &loop_counter_params = loop_parts.loop_condition_info; + int64_t loop_count = (loop_counter_params.end - loop_counter_params.begin) / loop_counter_params.step; + onnx::TensorProto *loop_count_proto = graph_proto->add_initializer(); + loop_count_proto->set_name(loop_count_name); + loop_count_proto->set_data_type(onnx::TensorProto_DataType_INT64); + loop_count_proto->add_int64_data(loop_count); + + auto loop_cond_name = node_name + "_cond"; + auto *cond_value = graph_proto->add_initializer(); + cond_value->set_name(loop_cond_name); + cond_value->set_data_type(onnx::TensorProto_DataType_BOOL); + cond_value->add_int32_data(true); + + loop_proto->add_input(loop_count_name); + loop_proto->add_input(loop_cond_name); + for (const auto &[loop_i, control_i] : loop_parts.used_loop_to_control_param_indices) { + auto name = GetNodeInputName(start_node->input(control_i + 1), node_map_ptr, graph_proto); + loop_proto->add_input(name); + loop_proto->add_output(MakeOutputName(node_name + "_loop", loop_i)); + } + + onnx::AttributeProto *subgraph_attr = loop_proto->add_attribute(); + subgraph_attr->set_type(onnx::AttributeProto_AttributeType_GRAPH); + subgraph_attr->set_name("body"); + onnx::GraphProto *loop_subgraph_proto = subgraph_attr->mutable_g(); + + // 2. Create subgraph for loop body + + auto subgraph_name = loop_parts.loop_subgraph->ToString(); + auto subgraph_input_cond_name = subgraph_name + "_input_cond"; + + auto *iter_num_input = loop_subgraph_proto->add_input(); + iter_num_input->set_name(subgraph_name + "_input_M"); + (void)iter_num_input->mutable_type()->mutable_tensor_type()->mutable_shape(); // side-effect: shape created + iter_num_input->mutable_type()->mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT64); + + auto *cond_input = loop_subgraph_proto->add_input(); + cond_input->set_name(subgraph_input_cond_name); + cond_input->mutable_type()->mutable_tensor_type()->set_elem_type(cond_value->data_type()); + + auto *cond_output = loop_subgraph_proto->add_output(); + cond_output->set_name(cond_input->name()); + cond_output->mutable_type()->mutable_tensor_type()->set_elem_type(cond_value->data_type()); + + MS_EXCEPTION_IF_CHECK_FAIL(renamed_node_map_.empty(), "renamed_nodes must be cleared after subgraph export"); + for (size_t i : loop_parts.ignored_loop_param_indices) { + const auto ¶m = loop_parts.loop_subgraph->parameters().at(i); + renamed_node_map_[param] = ""; + } + + // Export everything except the control call and the output (see MatchAndMark) + ExportFuncGraph(loop_parts.loop_subgraph, node_map_ptr, loop_subgraph_proto); + + // Export outputs manually + for (const auto &loop_to_control_i : loop_parts.used_loop_to_control_param_indices) { + const auto &input = loop_parts.repeat_node->input(loop_to_control_i.second + 1); + ExportOutput(loop_parts.loop_subgraph, input, node_map_ptr, loop_subgraph_proto); + } + renamed_node_map_.clear(); + + // 3. Export part after loop + + MS_EXCEPTION_IF_CHECK_FAIL(renamed_node_map_.empty(), "renamed_nodes must be cleared after subgraph export"); + const auto &after_loop_params = loop_parts.after_loop_subgraph->parameters(); + for (const auto &[after_i, output_i] : loop_parts.after_param_to_output_indices) { + MS_EXCEPTION_IF_CHECK_FAIL(static_cast(output_i) < loop_proto->output_size(), "Output index out of bounds"); + renamed_node_map_[after_loop_params.at(after_i)] = loop_proto->output(output_i); + } + ExportFuncGraph(loop_parts.after_loop_subgraph, node_map_ptr, graph_proto, false); + + auto after_loop_retval = GetRealInput(loop_parts.after_loop_subgraph->get_return()->input(1)); + if (after_loop_retval->isa() && after_loop_retval->cast()->IsApply(prim::kPrimMakeTuple)) { + auto tuple_retval = dyn_cast(after_loop_retval); + for (size_t i = 1; i < tuple_retval->inputs().size(); ++i) { + auto output_name = GetNodeInputName(tuple_retval->input(i), node_map_ptr, graph_proto); + AddOp("Identity", {output_name}, {MakeOutputName(node_name, i - 1)}, graph_proto); + } + } else { + auto output_name = GetNodeInputName(after_loop_retval, node_map_ptr, graph_proto); + AddOp("Identity", {output_name}, {node_name}, graph_proto); + } + renamed_node_map_.clear(); +} + +onnx::TensorProto_DataType OnnxExporter::GetOutputType(const AnfNodePtr &node, int64_t output_index) { + auto unpacked = GetRealInput(node); + if (IsPrimitiveCNode(unpacked, prim::kPrimTupleGetItem)) { + if (output_index != -1) { + MS_LOG(EXCEPTION) << "Unexpected output index for TupleGetItem: " << output_index; + } + auto cnode = dyn_cast(unpacked); + unpacked = cnode->input(kOneNum); + output_index = GetInt64Value(cnode->input(kTwoNum)); + } + + /* + Special cases (MS and ONNX type differences) go here + Example: + if (IsPrimitiveCNode(unpacked, prim::kPrim) && output_index == ) { + return onnx::TensorProto_DataType_; + } + */ + + if (output_index == -1) { + auto tensor = dyn_cast(unpacked->Type()); + if (tensor == nullptr) { + MS_LOG(EXCEPTION) << "Expected output of node " << unpacked->ToString() + << " to be a single tensor. Instead got: " << unpacked->Type()->ToString(); + } + return GetOnnxDataType(tensor->element()->type_id()); + } else { + auto tuple_type = dyn_cast(unpacked->Type()); + if (tuple_type == nullptr) { + MS_LOG(EXCEPTION) << "Expected output of node " << unpacked->ToString() + << " to be a tuple. Instead got: " << unpacked->Type()->ToString(); + } + auto element_type = tuple_type->elements()[static_cast(output_index)]; + MS_EXCEPTION_IF_NULL(element_type); + auto tensor_type = dyn_cast(element_type); + if (tensor_type == nullptr) { + MS_LOG(EXCEPTION) << "Expected output " << output_index << " of node " << unpacked->ToString() + << " to be a tensor. Instead got: " << element_type->ToString(); + } + return GetOnnxDataType(tensor_type->element()->type_id()); + } +} + +void OnnxExporter::AddOutputWithCast(onnx::NodeProto *node_proto, const std::string &output_name, + onnx::TensorProto_DataType target_type, onnx::GraphProto *graph_proto) const { + if (target_type == onnx::TensorProto_DataType_UNDEFINED) { + node_proto->add_output(output_name); + } else { + auto output_to_cast_name = output_name + "_output_to_cast"; + node_proto->add_output(output_to_cast_name); + AddCastOp(output_to_cast_name, output_name, target_type, graph_proto); + } +} + +std::string OnnxExporter::ExportPrimitive(const FuncGraphPtr &, std::map *node_map_ptr, + const PrimitivePtr &prim, const std::vector &inputs, + onnx::GraphProto *const graph_proto) { + auto op_map = OpConvertRegistry::GetOpConvertMap(); + MS_EXCEPTION_IF_NULL(prim); + auto op_iter = op_map.find(prim->name()); + if (op_iter == op_map.end()) { + MS_LOG(EXCEPTION) << "Can not find key " << prim->name() << " in convert map. " + << "Exporting " << prim->name() << " operator is not yet supported."; + } + // Get input first, because input maybe valuenode which need create constant node + std::vector input_list; + for (const auto &input : inputs) { + auto input_name = GetNodeInputName(input, node_map_ptr, graph_proto); + input_list.push_back(input_name); + } + + const OpNameInfo &op_convert_info = op_iter->second; + auto node_name = GenerateUniqueName(); + + std::vector output_cast_types(op_convert_info.num_outputs(), + onnx::TensorProto_DataType_UNDEFINED); + // Cast inputs if needed + for (const auto &rule : op_convert_info.input_casts()) { + auto original_type = GetOutputType(inputs[static_cast(rule.input_index)]); + if (original_type != rule.input_type) { + continue; + } + + auto cast_input_name = node_name + "cast_input_" + std::to_string(rule.input_index); + AddCastOp(input_list[static_cast(rule.input_index)], cast_input_name, rule.target_type, graph_proto); + input_list[static_cast(rule.input_index)] = cast_input_name; + + auto output_cast = std::find_if( + op_convert_info.output_casts().begin(), op_convert_info.output_casts().end(), [&rule](const OutputConversion &x) { + return x.mode == OutputConversion::Mode::INPUT && x.input_with_matching_type == rule.input_index; + }); + if (output_cast != op_convert_info.output_casts().end()) { + output_cast_types[static_cast(output_cast->output_index)] = original_type; + } + } + + for (const auto &output_cast : op_convert_info.output_casts()) { + if (output_cast.mode == OutputConversion::Mode::FIXED) { + output_cast_types[static_cast(output_cast.output_index)] = output_cast.target_type; + } + } + + onnx::NodeProto *node_proto = graph_proto->add_node(); + node_proto->set_name(node_name + op_convert_info.onnx_type()); + node_proto->set_op_type(op_convert_info.onnx_type()); + + // Set outputs + if (op_convert_info.num_outputs() == 1) { + AddOutputWithCast(node_proto, node_name, output_cast_types[0], graph_proto); + } else { + for (int i = 0; i < op_convert_info.num_outputs(); ++i) { + auto output_name = MakeOutputName(node_name, i); + AddOutputWithCast(node_proto, output_name, output_cast_types[static_cast(i)], graph_proto); + } + } + + // Set inputs + for (const auto &input_name : input_list) { + node_proto->add_input(input_name); + } + + // Set node attribute + for (const OpAttrInfo &attr : op_convert_info.op_attrs()) { + const std::string &attr_name = attr.attr_name(); + ValuePtr attr_value = nullptr; + if (!attr_name.empty()) { + attr_value = prim->GetAttr(attr_name); + if (attr_value == nullptr) { + MS_LOG(EXCEPTION) << "Primitive " << prim->name() << " does not have attribute " << attr_name; + } + } + onnx::AttributeProto *onnx_attr_proto = node_proto->add_attribute(); + onnx_attr_proto->set_name(attr.onnx_attr_name()); + attr.fn_gen_attr()(attr_value, attr.onnx_attr_type(), onnx_attr_proto, prim); + } + return node_name; +} + +void OnnxExporter::ExportMergeConv(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto conv_node = dyn_cast(node->input(kOneNum)); + auto input_x = conv_node->input(kOneNum); // conv input x + auto input_w = conv_node->input(kTwoNum); // conv weight(filter) + auto input_b = node->input(kTwoNum); // conv bias + + PrimitivePtr prim_conv = dyn_cast((dyn_cast(conv_node->input(kZeroNum)))->value()); + std::vector inputs{input_x, input_w, input_b}; + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_conv, inputs, graph_proto); +} + +void OnnxExporter::ExportMergeGemm(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto matmul_node = dyn_cast(node->input(kOneNum)); + auto input_x = matmul_node->input(kOneNum); // matmul input x + auto input_y = matmul_node->input(kTwoNum); // matmul input y + auto input_b = node->input(kTwoNum); // matmul bias + + PrimitivePtr prim_matmul = dyn_cast((dyn_cast(matmul_node->input(kZeroNum)))->value()); + std::vector inputs{input_x, input_y, input_b}; + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_matmul, inputs, graph_proto); +} + +void OnnxExporter::ExportMergeBatchNorm(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto batch_norm_node = dyn_cast(node->input(kOneNum)); + + auto is_training = GetOpAttribute(batch_norm_node, "is_training"); + if (is_training) { + auto input_x_name = GetNodeInputName(batch_norm_node->input(kOneNum), node_map_ptr, graph_proto); + auto scale_input_name = GetNodeInputName(batch_norm_node->input(kTwoNum), node_map_ptr, graph_proto); + auto bias_input_name = GetNodeInputName(batch_norm_node->input(kThreeNum), node_map_ptr, graph_proto); + + auto onnx_type = GetOutputType(batch_norm_node->input(kOneNum)); + + auto output_name = RegisterNodeWithUniqueName(node, node_map_ptr); + + auto input_shape_ptr = batch_norm_node->input(kOneNum)->Shape(); + auto input_shape = input_shape_ptr->cast()->shape(); + + std::vector normalize_axes = {0}; + for (size_t i = kTwoNum; i < input_shape.size(); ++i) { + normalize_axes.push_back(static_cast(i)); + } + + std::vector scale_bias_shape(input_shape.size(), 1); + scale_bias_shape[1] = -1; + auto reshaped_scale_name = output_name + "_reshaped_scale"; + AddReshapeOp(scale_input_name, reshaped_scale_name, scale_bias_shape, graph_proto); + auto reshaped_bias_name = output_name + "_reshaped_bias"; + AddReshapeOp(bias_input_name, reshaped_bias_name, scale_bias_shape, graph_proto); + auto epsilon = GetOpAttribute(batch_norm_node, "epsilon"); + + AddMeanVarianceNormalizationOp(input_x_name, reshaped_scale_name, reshaped_bias_name, output_name, normalize_axes, + epsilon, input_shape, onnx_type, graph_proto); + } else { + PrimitivePtr prim_batch_norm = GetPrimitive(batch_norm_node); + std::vector inputs; + for (size_t i = 1; i < batch_norm_node->inputs().size(); i++) { + inputs.push_back(batch_norm_node->input(i)); + } + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_batch_norm, inputs, graph_proto); + } +} + +void OnnxExporter::ExportMergeMaxPoolWithArgmax(const FuncGraphPtr &func_graph, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto maxpool_with_argmax_node = dyn_cast(node->input(kOneNum)); + + PrimitivePtr prim_maxpool_with_argmax = + dyn_cast((dyn_cast(maxpool_with_argmax_node->input(kZeroNum)))->value()); + std::vector inputs; + for (size_t i = 1; i < maxpool_with_argmax_node->inputs().size(); i++) { + inputs.push_back(maxpool_with_argmax_node->input(i)); + } + (*node_map_ptr)[node] = ExportPrimitive(func_graph, node_map_ptr, prim_maxpool_with_argmax, inputs, graph_proto); +} + +// LayerNorm(N, C1, H, W) --> reshape(1, C2, 1, W) + MeanVarianceNormalization + reshape(N, C1, H, W) +void OnnxExporter::ExportMergeLayerNorm(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto LayerNormNode = dyn_cast(node->input(kOneNum)); + auto layernorm_input_x = GetNodeInputName(LayerNormNode->input(kOneNum), node_map_ptr, graph_proto); + auto layernorm_input_gamma = GetNodeInputName(LayerNormNode->input(kTwoNum), node_map_ptr, graph_proto); + auto layernorm_input_beta = GetNodeInputName(LayerNormNode->input(kThreeNum), node_map_ptr, graph_proto); + + auto begin_norm_axis = GetOpAttribute(LayerNormNode, "begin_norm_axis"); + auto begin_params_axis = GetOpAttribute(LayerNormNode, "begin_params_axis"); + if (begin_norm_axis != -1 || begin_params_axis != -1) { + MS_LOG(EXCEPTION) << "begin_norm_axis != -1 and begin_params_axis != -1 are not implemented"; + } + + auto onnx_type = GetOutputType(LayerNormNode->input(kOneNum)); + auto input_shape = dyn_cast(LayerNormNode->input(kOneNum)->Shape())->shape(); + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto epsilon = GetOpAttribute(LayerNormNode, "epsilon"); + std::vector reduce_axes = {static_cast(input_shape.size()) - 1}; + + AddMeanVarianceNormalizationOp(layernorm_input_x, layernorm_input_gamma, layernorm_input_beta, node_name, reduce_axes, + epsilon, input_shape, onnx_type, graph_proto); +} + +void OnnxExporter::ExportMergeConv2DTranspose(const FuncGraphPtr &, const CNodePtr &node, + std::map *node_map_ptr, + onnx::GraphProto *const graph_proto) { + auto conv_node = dyn_cast(node->input(kOneNum)); + PrimConv2DTransposeExportHelper(conv_node, node, node_map_ptr, graph_proto); +} + +/* + Kinds of return values: + 1) A single Tensor + 2) A Tuple returned by an op with multiple outputs like TopK + 3) A Tuple returned by MakeTuple. This corresponds to `return x, y` + or equivalent in Python, where x and y are Tensors + In this case MakeTuple itself is not exported, so this case must be handled + separately from the previous one + 4) A constant tuple (ValueNode). Example: + class MyCell(nn.Cell): + def __init__(self): + super().__init__() + self.x = ms.Tensor(np.zeros((1, 2, 3))) + + def construct(self): + return self.x, self.x + + */ +void OnnxExporter::ExportOutput(const FuncGraphPtr &, const AnfNodePtr &return_arg, + std::map *node_map_ptr, onnx::GraphProto *const graph_proto) { + AnfNodePtr arg = GetRealInput(return_arg); + if (IsPrimitiveCNode(arg, prim::kPrimMakeTuple)) { + auto arg_cnode = dyn_cast(arg); + for (size_t i = 1; i < arg_cnode->inputs().size(); ++i) { + const auto &output = arg_cnode->input(i); + auto output_name = GetNodeInputName(output, node_map_ptr, graph_proto); + onnx::ValueInfoProto *output_proto = graph_proto->add_output(); + output_proto->set_name(output_name); + SetValueInfoType(output, output_proto); + } + } else if (arg->isa() && arg->cast()->value()->isa()) { + // Several outputs, all constants + auto tuple = arg->cast()->value()->cast(); + for (size_t i = 0; i < tuple->value().size(); ++i) { + const auto &element = tuple->value().at(i); + std::string output_name = GenerateUniqueName(); + + onnx::TensorProto *initializer = graph_proto->add_initializer(); + initializer->set_name(output_name); + SetTensorData(element, initializer); + + onnx::ValueInfoProto *output_proto = graph_proto->add_output(); + output_proto->set_name(output_name); + SetValueInfoType(arg, output_proto, i); + } + } else if (arg->Type()->isa()) { + auto arg_name = GetNodeInputName(arg, node_map_ptr, graph_proto); + auto tuple = dyn_cast(arg->Type()); + + for (size_t i = 0; i < tuple->size(); ++i) { + auto output_name = MakeOutputName(arg_name, i); + onnx::ValueInfoProto *output_proto = graph_proto->add_output(); + output_proto->set_name(output_name); + SetValueInfoType(arg, output_proto, i); + } + } else if (arg->Type()->isa()) { + auto arg_name = GetNodeInputName(arg, node_map_ptr, graph_proto); + onnx::ValueInfoProto *output_proto = graph_proto->add_output(); + output_proto->set_name(arg_name); + SetValueInfoType(arg, output_proto); + } else { + MS_LOG(EXCEPTION) << "Unsupported network output type " << arg->Type()->ToString() << " in node " + << arg->ToString(); + } +} + +std::string OnnxExporter::GetNodeInputName(const AnfNodePtr &orig_node, std::map *node_map_ptr, + onnx::GraphProto *const) { + auto node = GetRealInput(orig_node); + + // if node is renamed and not ignored, use alternative name + // if it is ignored, try to find the actual name in global map + auto renamed_iter = renamed_node_map_.find(node); + if (renamed_iter != renamed_node_map_.end() && renamed_iter->second != "") { + return renamed_iter->second; + } + + auto iter = node_map_ptr->find(node); + if (iter != node_map_ptr->end()) { + return iter->second; + } + + if (node->isa() || (node->isa() && !node->cast()->has_default())) { + MS_LOG(EXCEPTION) << "Can not find node '" << node->DebugString() << "' in node_map"; + } + + // for ValueNode or Parameter with default input, create an initializer + // same value can be used in several subgraphs, so create initializers in root graph + if (node->isa()) { + auto node_name = RegisterNodeWithUniqueName(node, node_map_ptr); + auto value = node->cast()->value(); + + onnx::TensorProto *initializer_proto = model_.mutable_graph()->add_initializer(); + initializer_proto->set_name(node_name); + SetTensorData(value, initializer_proto); + + (*node_map_ptr)[node] = node_name; + return node_name; + } + + if (node->isa()) { + auto param = dyn_cast(node); + auto node_name = GenerateUniqueParameterName(param, node_map_ptr); + + onnx::TensorProto *initializer_proto = model_.mutable_graph()->add_initializer(); + initializer_proto->set_name(node_name); + SetTensorData(param->default_param(), initializer_proto); + + (*node_map_ptr)[node] = node_name; + return node_name; + } + + MS_LOG(EXCEPTION) << "Unexpected node type " << node->type_name(); +} + +void OnnxExporter::ConvertTupleToTensor(const ValuePtr &value, onnx::TensorProto *const tensor_proto) const { + auto tuple_ptr = dyn_cast(value); + MS_EXCEPTION_IF_NULL(tuple_ptr); + if (tuple_ptr->size() == 0) { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, the size of converted tuple is 0."; + } + + ValuePtr first_element = (*tuple_ptr)[0]; + if (!first_element->isa()) { // For non-scalars x->type() contains nullptr + MS_LOG(EXCEPTION) << "Expected tuple elements to be scalars. Got: " << value->ToString(); + } + auto type_id = first_element->type()->type_id(); + for (size_t i = 1; i < tuple_ptr->size(); ++i) { + const auto element_type = (*tuple_ptr)[i]->type(); + if (element_type == nullptr || element_type->type_id() != type_id) { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, type of tuple elements is not same."; + } + } + + onnx::TensorProto_DataType result_type = onnx::TensorProto_DataType_UNDEFINED; + if (first_element->isa()) { + result_type = onnx::TensorProto_DataType_INT64; + } else if (first_element->isa()) { + result_type = onnx::TensorProto_DataType_FLOAT; + } else { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, unexpected tuple element type " + << first_element->type()->type_name() << "."; + } + + tensor_proto->add_dims(static_cast<::google::protobuf::int64>(tuple_ptr->size())); + tensor_proto->set_data_type(result_type); + for (size_t i = 0; i < tuple_ptr->size(); ++i) { + ValuePtr elem = (*tuple_ptr)[i]; + if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_int64_data(dyn_cast(elem)->value()); + } else if (elem->isa()) { + tensor_proto->add_float_data(dyn_cast(elem)->value()); + } else { + MS_LOG(EXCEPTION) << "Convert tuple to tensor fail, unexpected tuple element type " << elem->type()->type_name() + << "."; + } + } +} + +void OnnxExporter::SetTensorData(const ValuePtr &value, onnx::TensorProto *tensor_proto) { + if (value->isa()) { + auto attr_value = dyn_cast(value)->value(); + tensor_proto->set_data_type(onnx::TensorProto_DataType_INT32); + tensor_proto->add_int32_data(attr_value); + } else if (value->isa()) { + auto attr_value = dyn_cast(value)->value(); + tensor_proto->set_data_type(onnx::TensorProto_DataType_INT64); + tensor_proto->add_int64_data(attr_value); + } else if (value->isa()) { + auto data = dyn_cast(value); + tensor_proto->set_raw_data(data->data_c(), static_cast(data->data().nbytes())); + auto dtype = data->data_type(); + auto shape = data->shape_c(); + + tensor_proto->set_data_type(GetOnnxDataType(dtype)); + for (const auto dim : shape) { + tensor_proto->add_dims(dim); + } + } else if (value->isa()) { // Note: this is a tuple of primitives, not Tensors + ConvertTupleToTensor(value, tensor_proto); + } else { + MS_LOG(EXCEPTION) << "Need to set value " << value->ToString() << " attribute for Constant node"; + } +} + +std::string GetOnnxProtoString(const FuncGraphPtr &func_graph) { + OnnxExporter exporter; + return exporter.GetOnnxProtoString(func_graph); +} +} // namespace mindspore -- 2.34.1 From 5047cc9701d542cc51d014c08be18ff98b728b80 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:15:08 +0800 Subject: [PATCH 34/72] ADD file via upload --- .../ccsrc/transform-update/op_adapter.cc | 824 ++++++++++++++++++ 1 file changed, 824 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter.cc diff --git a/mindspore/ccsrc/transform-update/op_adapter.cc b/mindspore/ccsrc/transform-update/op_adapter.cc new file mode 100644 index 00000000000..58bbb025bdd --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter.cc @@ -0,0 +1,824 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "transform/graph_ir/op_adapter.h" +#include "utils/check_convert_utils.h" + +namespace mindspore { +namespace transform { +// ̬ CustomInferFunc +// const Operator &һ +// أuint32_tһ޷ +static uint32_t CustomInferFunc(const Operator &) { return 0; } // ع̶ֵ 0Ըݾʵֲͬ߼㷵ֵ + +// úжϸIJǷΪԶ +// opOperatorPtr͵IJָ +// أboolͣʾǷΪԶ +bool OpAdapterImpl::IsCustomOp(const OperatorPtr &op) { + MS_EXCEPTION_IF_NULL(op); // ָǷΪգΪ׳쳣 + auto it = cus_input_map_->find(op->GetOpType()); // ԶӳвҸ + if (it == cus_input_map_->end()) { + return false; // ӳδҵòͣ򷵻falseʾòԶ + } + return true; // ӳҵ˸òͣ򷵻trueʾòԶ +} + +// úԶӳ +// opCusOperatorPtr͵Զָ +// primPrimitivePtr͵IJԭָ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::GenerateCustomOpInputMap(const CusOperatorPtr &op, const PrimitivePtr &prim) { + MS_EXCEPTION_IF_NULL(op); // ԶָǷΪգΪ׳쳣 + MS_EXCEPTION_IF_NULL(prim); // ԭָǷΪգΪ׳쳣 + // Create the map of custom op from input index to input name. + // ڴ洢Զӳ mapΪֵΪ + mindspore::HashMap input_map; + auto value = prim->GetAttr("input_names"); // ȡԭΪ "input_names" ֵ + if (value == nullptr) { // ȡֵΪգʾԶûӳ + (*cus_output_map_)[prim->name()] = input_map; //һյӳ洢 cus_output_map_ + return NOT_FOUND; // NOT_FOUND ״̬ + } + // ֵתΪ std::vector + auto input_names = GetValue>(value); + for (size_t i = 0; i < input_names.size(); ++i) { // бƵӳ洢 input_map УעᵽԶ + // input_map begin form 1 + // 1 ʼ + input_map[i + 1] = input_names[i]; + op->CustomInputRegister(input_names[i]); + } + // cus_input_map_ δҵԶӳ䣬 input_map 洢 cus_input_map_ + if (cus_input_map_->find(prim->name()) == cus_input_map_->end()) { + (*cus_input_map_)[prim->name()] = input_map; + } + return SUCCESS; // ִгɹ SUCCESS ״̬ +} + +// úԶӳ +// opCusOperatorPtr͵Զָ +// primPrimitivePtr͵IJԭָ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::GenerateCustomOpOutputMap(const CusOperatorPtr &op, const PrimitivePtr &prim) { + MS_EXCEPTION_IF_NULL(op); // ԶָǷΪգΪ׳쳣 + MS_EXCEPTION_IF_NULL(prim); // ԭָǷΪգΪ׳쳣 + // Create the map of custom op from output index to output name. + // ڴ洢Զӳ mapΪֵΪ + mindspore::HashMap output_map; + // ȡԭΪ "output_names" ֵ + auto value = prim->GetAttr("output_names"); + if (value == nullptr) { // ȡֵΪգʾԶûӳ + // generate a empty output_map for it + (*cus_output_map_)[prim->name()] = output_map; //һյӳ洢 cus_output_map_ + return NOT_FOUND; // NOT_FOUND ״̬ + } + // ֵתΪ std::vector + auto output_names = GetValue>(value); + for (size_t i = 0; i < output_names.size(); ++i) { // бƵӳ洢 output_map УעᵽԶ + // output_map begin form 0 + // 0 ʼ + output_map[i] = output_names[i]; + op->CustomOutputRegister(output_names[i]); + } + // cus_output_map_ δҵԶӳ䣬 output_map 洢 cus_output_map_ + if (cus_output_map_->find(prim->name()) == cus_output_map_->end()) { + (*cus_output_map_)[prim->name()] = output_map; + } + return SUCCESS; // ִгɹ SUCCESS ״̬ +} + +// úԶ +// anfAnfNodePtr͵Ľڵָ +// أOperatorPtrͣʾɵԶָ +OperatorPtr OpAdapterImpl::GenerateCustomOp(const AnfNodePtr anf) { + MS_EXCEPTION_IF_NULL(anf); // ڵָǷΪգΪ׳쳣 + auto node = anf->cast(); // ڵתΪ CNodePtr + if (node == nullptr) { // תʧܣؿָ + return nullptr; + } + + if (node->inputs().empty()) { // ڵΪգ׳쳣 + MS_LOG(EXCEPTION) << "length of node inputs is empty"; + } + + auto prim = GetValueNode(node->inputs()[0]); // ȡڵԭָ + MS_EXCEPTION_IF_NULL(prim); // ԭָǷΪգΪ׳쳣 + // ge::CustomOperator ͵Զڵȫԭ + auto op = std::make_shared(node->fullname_with_scope(), prim->name()); + // ԶӳעᵽԶ + if (GenerateCustomOpInputMap(op, prim) != SUCCESS) { + MS_LOG(WARNING) << "Custom op node has no input_names, op[" << prim->name() << "]."; + } + // ԶӳעᵽԶ + if (GenerateCustomOpOutputMap(op, prim) != SUCCESS) { + MS_LOG(WARNING) << "Custom op node has no output_names, op[" << prim->name() << "]."; + } + // עԶ + op->CustomInferFuncRegister(CustomInferFunc); + // ɵԶָ + return op; +} + +// úòͼ +// opOperatorPtr͵IJָ +// indexint͵ֵ +// branchesstd::shared_ptr>͵ͼָ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::SetOpSubgraphFunc(const OperatorPtr &op, int index, + const std::shared_ptr> &branches) { + MS_EXCEPTION_IF_NULL(op); // ָǷΪգΪ׳쳣 + auto it = dyn_subgraph_map_.find(index); // ڶ̬ͼӳвҸӳ + if (it != dyn_subgraph_map_.end()) { // ҵӳ + auto size = branches->size(); // ȡͼָеͼ + it->second.create_dyn_subgraph(op, static_cast(size)); // Ķ̬ͼ + for (size_t i = 0; i < size; i++) { // òͼ + it->second.set_subgraph(op, static_cast(i), std::make_shared((*branches)[i])); + } + return SUCCESS; // ִгɹ SUCCESS ״̬ + } + return NOT_FOUND; // δҵӦӳ䣬 NOT_FOUND ״̬ +} + +// úԶ +// opCusOperatorPtr͵Զָ +// indexint͵ +// inputOperatorPtr͵ָ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::SetCustomOpInput(const CusOperatorPtr &op, int index, const OperatorPtr &input) { + MS_EXCEPTION_IF_NULL(op); // ԶָǷΪգΪ׳쳣 + MS_EXCEPTION_IF_NULL(input); // ָǷΪգΪ׳쳣 + auto it = cus_input_map_->find(op->GetOpType()); // ԶӳвҸԶ͵ӳ + if (it == cus_input_map_->end()) { // δҵӳ䣬򷵻 NOT_FOUND ״̬ + return NOT_FOUND; + } + mindspore::HashMap &input_map = it->second; // ȡԶӳ + + if ((input_map.find(index) != input_map.end())) { // ӳҵӳ + MS_LOG(DEBUG) << "Link op " << input->GetName() << " to " << op->GetName() << ":" << input_map[index]; + (void)op->SetInput(input_map[index], *input); // Զ + return SUCCESS; // ִгɹ SUCCESS ״̬ + } + return NOT_FOUND; // δҵӳ䣬 NOT_FOUND ״̬ +} + +// úͨ +// opOperatorPtr͵ָͨ +// indexint͵ +// inputOperatorPtr͵ָ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::SetNormalOpInput(const OperatorPtr &op, int index, const OperatorPtr &input) { + MS_EXCEPTION_IF_NULL(op); // ָͨǷΪգΪ׳쳣 + auto it = input_map_.find(index); // ӳвҸӳ + if (input != nullptr && it != input_map_.end()) { // ָ벻Ϊգҵӳ + MS_LOG(DEBUG) << "Link op " << input->GetName() << " to " << op->GetName() << ":" << it->second.name; + it->second.set_op(op, input); // ͨ + return SUCCESS; // ִгɹ SUCCESS ״̬ + } + return NOT_FOUND; // ָΪջδҵӳ䣬 NOT_FOUND ״̬ +} + +// úò +// opOperatorPtr͵IJָ +// indexint͵ +// inputOperatorPtr͵ָ +// أintͣʾִе״̬зǸΪɹ״̬Ϊʧ״̬ +int OpAdapterImpl::setInput(const OperatorPtr &op, int index, const OperatorPtr &input) { + if (IsCustomOp(op)) { // Զ + auto cus_op = std::dynamic_pointer_cast(op); // ָתΪԶָ + return static_cast(SetCustomOpInput(cus_op, index, input)); // SetCustomOpInput Զ룬״̬ + } else { // ͨ + return static_cast(SetNormalOpInput(op, index, input));// SetNormalOpInput ͨ룬״̬ + } +} + +// úԶ +// opCusOperatorPtr͵Զָ +// indexint͵ +// handleOutHandler͵ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::SetCustomOpInput(const CusOperatorPtr &op, int index, const OutHandler &handle) { + MS_EXCEPTION_IF_NULL(op); // ԶָǷΪգΪ׳쳣 + auto it = cus_input_map_->find(op->GetOpType()); // ӳвҸԶ͵ӳ + if (it == cus_input_map_->end()) { // δҵԶ͵ӳ + return NOT_FOUND; // NOT_FOUND ״̬ʾδҵ͵Զ + } + + mindspore::HashMap &input_map = it->second; // ȡӳиԶ͵ӳ + if ((handle.op != nullptr) && (input_map.find(index) != input_map.end())) { // еIJָ벻Ϊգҵ˸ӳ + if (handle.out.empty()) { // Ϊ + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << " to " << op->GetName() << ":" << input_map[index]; + (void)op->SetInput(input_map[index], *(handle.op)); // Զ + } else { // ƲΪ + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << ":" << handle.out << " to " << op->GetName() << ":" + << input_map[index]; + (void)op->SetInput(input_map[index], *(handle.op), handle.out); // Զ + } + return SUCCESS; // ִгɹ SUCCESS ״̬ + } + return NOT_FOUND; // еIJָΪջδҵӳ䣬 NOT_FOUND ״̬ +} + +// úͨ +// opOperatorPtr͵IJָ +// indexint͵ +// handleOutHandler͵ +// أStatusͣʾִе״̬ +Status OpAdapterImpl::SetNormalOpInput(const OperatorPtr &op, int index, const OutHandler &handle) { + MS_EXCEPTION_IF_NULL(op); // ָǷΪգΪ׳쳣 + auto it = input_map_.find(index); // ӳвҸӳ + if ((handle.op != nullptr) && (it != input_map_.end())) { // еIJָ벻Ϊգҵ˸ӳ + if (handle.out.empty()) { // Ϊ + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << " to " << op->GetName() << ":" << it->second.name; + it->second.set_op(op, handle.op); // ͨ + } else { // ƲΪ + MS_LOG(DEBUG) << "Link op " << handle.op->GetName() << ":" << handle.out << " to " << op->GetName() << ":" + << it->second.name; + it->second.set_handle(op, handle); // ͨ + } + return SUCCESS; // ִгɹ SUCCESS ״̬ + } + return NOT_FOUND; // еIJָΪջδҵӳ䣬 NOT_FOUND ״̬ +} + +// úò +// opOperatorPtr͵IJָ +// indexint͵ +// handleOutHandler͵ +// أintͣʾִеĽ +int OpAdapterImpl::setInput(const OperatorPtr &op, int index, const OutHandler &handle) { + if (IsCustomOp(op)) { // Զ + auto cus_op = std::dynamic_pointer_cast(op); // ָתΪԶָ + return static_cast(SetCustomOpInput(cus_op, index, handle)); // SetCustomOpInput Զ룬תΪ + } else { // Զ + return static_cast(SetNormalOpInput(op, index, handle)); // SetNormalOpInput ͨ룬תΪ + } +} + +// úòĶ̬ + +// opOperatorPtr͵IJָ +// indexint͵ +// handler_vecstd::shared_ptr>͵ +// أintͣʾִеĽ +int OpAdapterImpl::setInput(const OperatorPtr &op, int index, + const std::shared_ptr> &handler_vec) { + MS_EXCEPTION_IF_NULL(handler_vec); // Ч + if (IsCustomOp(op)) { // Զ + MS_LOG(ERROR) << "Custom Op do not support dynamic input"; // ϢԶֶ֧̬ + return static_cast(FAILED); // ִʧܵ״̬ + } + MS_EXCEPTION_IF_NULL(op); // ָЧ + auto it = dyn_input_map_.find(index); // ҶӦĶ̬Ϣ + if (it != dyn_input_map_.end()) { // ҵ˶ӦĶ̬Ϣ + it->second.create_dyn_input(op, static_cast(handler_vec->size())); // ̬ + for (unsigned int i = 0; i < handler_vec->size(); ++i) { // + OutHandler h = (*handler_vec)[i]; // ȡ + MS_EXCEPTION_IF_NULL(h.op); // IJָЧ + if (h.out.empty()) { // ûָ + MS_LOG(DEBUG) << "Link op " << h.op->GetName() << " to " << op->GetName() << ":" << it->second.name; + // ϢеIJΪ̬һ + it->second.set_op(op, (i), h.op); + } else { // ָ + MS_LOG(DEBUG) << "Link op " << h.op->GetName() << ":" << h.out << " to " << op->GetName() << ":" + << it->second.name; + // ϢеΪ̬һ + it->second.set_handle(op, i, h); + } + } + return 0; // ִгɹ״̬ + } + return static_cast(NOT_FOUND); // δҵ״̬ +} + +// úڻȡ +// opOperatorPtr͵IJָ +// indexint͵ +// أOutHandler͵ +OutHandler OpAdapterImpl::getOutput(const OperatorPtr &op, int index) { + MS_EXCEPTION_IF_NULL(op);// ָЧ + if (IsCustomOp(op)) { // Զ + return getCustomOutput(op, index); // getCustomOutput ȡԶ + } + return getNormalOutput(op, index); // 򣬵 getNormalOutput ȡͨ +} + +// úڻȡԶ +// opOperatorPtr͵IJָ +// indexint͵ +// أOutHandler͵ +OutHandler OpAdapterImpl::getCustomOutput(const OperatorPtr &op, int index) { + MS_EXCEPTION_IF_NULL(op); // ָЧ + auto it = cus_output_map_->find(op->GetOpType()); + if (it == cus_output_map_->end()) { // Զӳ + MS_LOG(ERROR) << "OpAdpator(" << op->GetName() << ") has both OUTPUT is not supported!"; // ûҵӳ䣬־ + return OutHandler(); // ؿյ + } + + mindspore::HashMap &output_map = it->second; // ȡӳ + + if ((output_map.find(index) != output_map.end())) { // ǷҵӦ + return OutHandler(op, output_map[index]); // ҵ + } + MS_LOG(ERROR) << "OpAdpator(" << op->GetName() << ") has no OUTPUT index(" << index << ")!"; // ûҵ־ + return OutHandler(); // ؿյ +} + +// úڻȡͨ +// opOperatorPtr͵IJָ +// indexint͵ +// أOutHandler͵ +OutHandler OpAdapterImpl::getNormalOutput(const OperatorPtr &op, int index) { + MS_EXCEPTION_IF_NULL(op); // ָЧ + if (!dyn_output_map_.empty() && !output_map_.empty()) { // Ƿͬʱڶ̬ӳͨӳ + MS_LOG(ERROR) << "OpAdpator(" << op->GetName() << ") has both OUTPUT and DYN_OUTPUT is not supported!"; // ͬʱڶ̬ӳͨӳ䣬־ + return OutHandler(); // ؿյ + } + auto it = output_map_.find(index); // ͨӳвָ + if (it != output_map_.end()) { // ҵ + return OutHandler(op, it->second.name); // + } else if (!dyn_output_map_.empty()) { // ͨӳΪգ̬ӳ䲻Ϊ + return OutHandler(op, dyn_output_map_.begin()->second.name + std::to_string(index)); // ݶ̬ӳĵһƹ + } else { // ûͨӳ䣬Ҳûж̬ӳ + MS_LOG(ERROR) << "OpAdpator(" << op->GetName() << ") has no OUTPUT and DYN_OUTPUT index(" << index << ")!"; // ־ + return OutHandler(); // ؿյ + } +} + +// úڸµ +// opOperatorPtr͵IJָ +// shpabstract::BaseShapePtr͵״ָ +// typeTypePtr͵ +// formatstring͵ݸʽ +// أStatus͵״̬SUCCESSʾɹFAILEDʾʧ +Status OpAdapterImpl::UpdateSingleOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, + const TypePtr &type, const std::string &format) { + MS_EXCEPTION_IF_NULL(type); // ͵Ч + + auto desc = CreateOutputDesc(dyn_cast(shp), type, format); + if (desc == nullptr) { // ǷΪ + MS_LOG(ERROR) << "Update output descriptor failed!"; // ־ + return FAILED; // ʧ״̬ + } + + if (IsCustomOp(op)) { // ǷΪԶ + if (cus_output_map_->find(op->GetOpType()) == cus_output_map_->end() || + ((*cus_output_map_)[op->GetOpType()].empty())) { // ǷԶӳ䣬ҲΪ + MS_LOG(ERROR) << "This op does not create custom output map"; // ־ + return FAILED; // ʧ״̬ + } + auto cus_op = std::dynamic_pointer_cast(op); // ָתΪԶָ + MS_EXCEPTION_IF_NULL(cus_op); // ԶָЧ + mindspore::HashMap output_map = (*cus_output_map_)[op->GetOpType()]; // ȡԶӳ + (void)cus_op->UpdateOutputDesc(output_map[0], *desc); // Զ + } else { // Զ + if (output_map_.empty()) { // ͨӳǷΪ + MS_LOG(INFO) << "This op does not have output map"; // ʾϢ + return FAILED; // ʧ״̬ + } + output_map_.begin()->second.update_out_desc(op, *desc); // ͨ + } + return SUCCESS; // سɹ״̬ +} + +// úڻȡԶ +// cus_opCusOperatorPtr͵Զָ +// أsize_t͵ʾԶ +size_t OpAdapterImpl::GetCustomOpOutputSize(const CusOperatorPtr &cus_op) { + MS_EXCEPTION_IF_NULL(cus_op); // ԶָЧ + if (cus_output_map_->find(cus_op->GetOpType()) == cus_output_map_->end()) { // ǷԶӳ + MS_LOG(ERROR) << "This op does not create custom output map"; // ־ + return 0; // 0ʾԶΪ0 + } + size_t output_size = (*cus_output_map_)[cus_op->GetOpType()].size(); // ȡԶ + return output_size; // Զ +} + +// úڴGeTensorDesc״ͺ͸ʽ +// +// - shape_ptr: abstract::ShapePtr͵״ָ룬ʾ״ +// - type: TypePtr͵ָ룬ʾ͡ +// - format: std::string͵ĸʽַʾݸʽ +// أ +// - std::shared_ptr͵ָ룬ʾGeTensorDesc +std::shared_ptr OpAdapterImpl::CreateOutputDesc(const abstract::ShapePtr &shape_ptr, const TypePtr &type, + const std::string &format) { + if (type == nullptr) { // ǷΪ + MS_LOG(ERROR) << "Type ptr is nullptr"; // ־ + return nullptr; // ؿָ룬ʾʧ + } + + TypeId me_type = type->type_id(); // ȡID + if (kObjectTypeTensorType == me_type) { // TensorType + me_type = dyn_cast(type)->element()->type_id(); // ȡTensorԪصID + } + // TransformUtil::GetGeTensorDescGeTensorDesc󲢷 + return TransformUtil::GetGeTensorDesc((shape_ptr == nullptr) ? ShapeVector{} : shape_ptr->shape(), me_type, format); +} + +// úڸ¶ +// +// - op: OperatorPtr͵ָ룬ʾҪ +// - shp: abstract::BaseShapePtr͵״ָ룬ʾ״ +// - type: TypePtr͵ָ룬ʾ͡ +// - format: std::string͵ĸʽַʾݸʽ +// أ +// - StatusͣʾIJ״̬ +Status OpAdapterImpl::UpdateMultiOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, + const TypePtr &type, const std::string &format) { + auto tuple_shp = dyn_cast(shp); // תTupleShape + MS_EXCEPTION_IF_NULL(tuple_shp); + + size_t output_size = 0; + bool is_custom_op = IsCustomOp(op); // ǷΪԶ + if (is_custom_op) { + output_size = GetCustomOpOutputSize(std::dynamic_pointer_cast(op)); // ȡԶ + } else { + output_size = output_map_.size(); // ȡͨ + } + + if (output_size == 0) { // Ϊ0ʧ״̬ + MS_LOG(INFO) << "This op does not have output map"; + return FAILED; + } + + if (output_size != tuple_shp->shape().size()) { // ǷTupleShapeĴС + MS_LOG(ERROR) << "output_map is not equal tuple_shape size"; + return FAILED; // ȣʧ״̬ + } + + for (size_t i = 0; i < tuple_shp->shape().size(); ++i) { + auto tuple_type = dyn_cast(type); // תTuple + MS_EXCEPTION_IF_NULL(tuple_type); + TypePtr type_elem = tuple_type->elements()[i]; // ȡTupleеiԪص + // CreateOutputDescGeTensorDesc + auto desc = CreateOutputDesc(dyn_cast(tuple_shp->shape()[i]), type_elem, format); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create output descriptor failed!"; + return FAILED; // ʧܣʧ״̬ + } + + if (is_custom_op) { // Զ + // CustomOperatorUpdateOutputDesc + (void)std::dynamic_pointer_cast(op)->UpdateOutputDesc((*cus_output_map_)[op->GetOpType()][i], + *desc); + } else { + auto it = output_map_.find(i); + if (it != output_map_.end()) { + it->second.update_out_desc(op, *desc); // ͨ + } + } + } + return SUCCESS; // سɹ״̬ +} + +// úڴGeTensorDesc󣬱ʾAnfNode +// +// - node: AnfNodePtr͵ָ룬ʾҪĽڵ㡣 +// - format: std::string͵ĸʽַʾݸʽ +// أ +// - std::shared_ptr͵ָ룬ʾGeTensorDesc +std::shared_ptr OpAdapterImpl::CreateNodeDesc(const AnfNodePtr &node, const std::string &format) { + MS_EXCEPTION_IF_NULL(node); + TypeId me_type = node->Type()->type_id(); // ȡڵ + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(node->Type())->element()->type_id(); + } + // ǷЧЧ򷵻nullptr + if (me_type <= kNumberTypeBegin || me_type >= kNumberTypeEnd) { + return nullptr; + } + + std::vector shape; + auto shape_ptr = dyn_cast(node->Shape()); + if (shape_ptr != nullptr) { // ȡڵ״ + shape = shape_ptr->shape(); + } + // TransformUtilGetGeTensorDescGeTensorDesc + auto desc = TransformUtil::GetGeTensorDesc(shape, me_type, format); + // ǷɹGeTensorDescʧܷnullptr + if (desc == nullptr) { + MS_LOG(ERROR) << "Update output descriptor failed!"; + return nullptr; + } + return desc; // شGeTensorDescָ +} + +// úڸͨӵ +// +// - op: OperatorPtr͵ָ룬ʾҪӡ +// - node: AnfNodePtr͵ָ룬ʾӶӦCNodeڵ㡣 +// - format: std::string͵ĸʽַʾݸʽ +void OpAdapterImpl::UpdateNormalOpInputDesc(const OperatorPtr &op, const AnfNodePtr &node, const std::string format) { + if (op == nullptr) { //ǷΪգΪӡ־ + MS_LOG(ERROR) << "op is nullptr"; + return; + } + MS_EXCEPTION_IF_NULL(node); + + auto inputs = node->cast()->inputs(); //CNodeڵ룬ӵڶ뿪ʼΪһӱ + for (size_t i = 1; i < inputs.size(); ++i) { // ڵ㣬ǷжӦ + auto it = input_map_.find(i); + if (it != input_map_.end()) { //ҵCreateNodeDescµʹøڵ + auto desc = CreateNodeDesc(inputs[i], format); + if (desc == nullptr) { // ʧܣһڵ㡣 + continue; + } + + it->second.update_input_desc(op, *desc); + } + } +} + +// úڸԶӵ +// +// - op: CusOperatorPtr͵ָ룬ʾҪԶӡ +// - node: AnfNodePtr͵ָ룬ʾԶӶӦCNodeڵ㡣 +// - format: std::string͵ĸʽַʾݸʽ +void OpAdapterImpl::UpdateCustomOpInputDesc(const CusOperatorPtr &op, const AnfNodePtr &node, + const std::string format) { + if (op == nullptr) { //ǷΪգΪӡ־ + MS_LOG(ERROR) << "op is nullptr"; + return; + } + MS_EXCEPTION_IF_NULL(node); + //ԶǷ񴴽ӳ䣬δӡ־ + if (cus_input_map_->find(op->GetOpType()) == cus_input_map_->end() || ((*cus_input_map_)[op->GetOpType()].empty())) { + MS_LOG(ERROR) << "This op does not create custom input map"; + return; + } + //CNodeڵ룬ӵڶ뿪ʼΪһӱ + mindspore::HashMap &input_map = (*cus_input_map_)[op->GetOpType()]; + auto inputs = node->cast()->inputs(); + for (size_t i = 1; i < inputs.size(); ++i) { // ڵ㣬ǷжӦ + if (input_map.find(i) != input_map.end()) { //ҵCreateNodeDescµʹøԶӵ + auto desc = CreateNodeDesc(inputs[i], format); + if (desc == nullptr) { // ʧܣһڵ㡣 + continue; + } + (void)op->UpdateInputDesc(input_map[i], *desc); + } + } +} + +// úڸӵ +// +// - op: OperatorPtr͵ָ룬ʾҪӡ +// - node: AnfNodePtr͵ָ룬ʾӶӦCNodeڵ㡣 +void OpAdapterImpl::updateInputDesc(const OperatorPtr &op, const AnfNodePtr &node) { + MS_EXCEPTION_IF_NULL(op); //ӺͽڵǷΪգΪ׳쳣 + MS_EXCEPTION_IF_NULL(node); + std::string format = GetOpIOFormat(node); //ȡڵݸʽIOʽͨGetOpIOFormat + if (IsCustomOp(op)) { //ǷΪԶ + auto cus_op = std::dynamic_pointer_cast(op); //ǣתΪCustomOperator + UpdateCustomOpInputDesc(cus_op, node, format); //UpdateCustomOpInputDesc + } else { //ԶӣUpdateNormalOpInputDesc + UpdateNormalOpInputDesc(op, node, format); + } +} + +// úڸ״Ϣ +// : +// - op: ָOperatorPtrָ룬ʾҪ +// - shp: ָBaseShapePtrָ룬ʾ״Ϣ +// - type: ָTypePtrָ룬ʾϢ +// - node: ָAnfNodePtrָ룬ʾӦCNodeڵ㡣 +void OpAdapterImpl::updateOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const AnfNodePtr &node) { + if (op == nullptr) { //ͽڵָǷΪգΪգ쳣 + MS_LOG(ERROR) << "op is nullptr"; + return; + } + MS_EXCEPTION_IF_NULL(node); + MS_LOG(INFO) << "Op name is " << op->GetName() << " anf is " << node->DebugString(); + + auto normal_shape_ptr = dyn_cast(shp); + auto no_shape_ptr = dyn_cast(shp); + std::string format = GetOpIOFormat(node); //ͨGetOpIOFormatȡڵݸʽIOʽ + //״Ϣ + if ((normal_shape_ptr != nullptr) || (no_shape_ptr != nullptr)) { //״ΪShapeNoShape + if (UpdateSingleOutputDesc(op, shp, type, format) != SUCCESS) { //UpdateSingleOutputDesc + return; + } + } else if (dyn_cast(shp) != nullptr) { // ״ΪTupleShape + if (UpdateMultiOutputDesc(op, shp, type, format) != SUCCESS) { //UpdateMultiOutputDesc + return; + } + } else { //״δ֪¼沢ء + MS_LOG(WARNING) << "Update output desc failed, unknown output shape type"; + return; + } + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { //󣬺ڵǷΪCNodeڵ㣩 + return; //ڵ㲻CNodeأΪǼڵ㲻Ҫ + } + + // Need to update input_desc while the output_desc is updated + // ڵCNodeupdateInputDescͬʱ + updateInputDesc(op, node); +} + +// úΪԡ +// : +// - op: ָOperatorPtrָ룬ʾҪԵ +// - attr_key: ԵļʾҪõơ +// - attr_value: ָValuePtrָ룬ʾҪõֵ +int OpAdapterImpl::setAttr(const OperatorPtr &op, const std::string &attr_key, const ValuePtr &attr_value) { + auto it = attr_map_.find(attr_key); //ͨattr_map_вҸattr_keyǷжӦϢ + if (it != attr_map_.end()) { // ҵƥϢӦset_attrԡ + // switch case for each avalilable attribute type + // ֮ǰӡƺֵϢӵadpt_ԻͼΡ + MS_LOG(INFO) << "Set attr: " << attr_key << "(" << it->second.name << "), value: " << attr_value->ToString(); + adpt_->AddAttrToDrawGraph(attr_key + std::string("=") + attr_value->ToString()); + it->second.set_attr(op, attr_value); + return 0; // ɹԷ0 + } + return static_cast(NOT_FOUND); // δҵƥϢNOT_FOUND +} + +//úԶԡ +int OpAdapterImpl::SetCustomOpAttr(const CusOperatorPtr &op, const PrimitivePtr &prim) { + enum ValueType { //һöValueTypeöֵSINGLE_VALUESEQUEUE_VALUEUNKNOWN_VALUE + SINGLE_VALUE = 0, + SEQUEUE_VALUE, + UNKNOWN_VALUE, + }; + + MS_EXCEPTION_IF_NULL(prim); //ʹöԣMS_EXCEPTION_IF_NULLȷprimopָ벻Ϊ + MS_EXCEPTION_IF_NULL(op); + + ValueType value_type = SINGLE_VALUE; //ʼһvalue_typeΪSINGLE_VALUE + for (auto item : prim->attrs()) { //ͨprimò + //ÿԣȼͣȻ͵ʵĺֵõС + //ֵ֧ͰInt32ImmStringImmBoolImmFP32Imm + /* ԵValueSequencevalue_typeΪSEQUEUE_VALUEеһԪصֵ + ԵͲֵ֧бУ׳쳣*/ + if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + (void)op->SetAttr(item.first, GetValue(item.second)); + } else if (item.second->isa()) { + value_type = SEQUEUE_VALUE; + auto val_seq = item.second->cast(); + if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else if ((*val_seq)[0]->isa()) { + (void)op->SetAttr(item.first, GetValue>(item.second)); + } else { + MS_LOG(EXCEPTION) << "Unsupported custom attribute type in adaptor, prim name: " << prim->name() + << ", attr name: " << item.first << ", value: " << item.second->ToString(); + } + } else { + MS_LOG(WARNING) << "Unsupported custom attribute type in adaptor, prim name: " << prim->name() + << ", attr name: " << item.first << ", value: " << item.second->ToString(); + return static_cast(NOT_FOUND); + } + /*ֵ֮󣬸value_typeֵʹʵַʾӵͼС + value_typeΪSINGLE_VALUEֵԵȺӲӵͼУ + value_typeΪSEQUEUE_VALUEʡԺӵͼС*/ + if (value_type == SINGLE_VALUE) { + adpt_->AddAttrToDrawGraph(item.first + std::string("=") + item.second->ToString()); + } else if (value_type == SEQUEUE_VALUE) { + adpt_->AddAttrToDrawGraph(item.first + std::string("=") + "[...]"); + } + } + return 0; //0ʾɹԡ +} + +// úΪͨԡ +// : +// - op: ָOperatorPtrָ룬ʾҪԵ +// - prim: ָPrimitivePtrָ룬ʾԭPrimitiveϢ +int OpAdapterImpl::SetNormalOpAttr(const OperatorPtr &op, const PrimitivePtr &prim) { + MS_EXCEPTION_IF_NULL(prim); //primopǷΪգΪգ׳쳣 + MS_EXCEPTION_IF_NULL(op); + for (auto &it : attr_map_) { //attr_map_ + auto value = prim->GetAttr(it.first); //ÿԣattr_map_еÿֵԣȴprimлȡӦֵ + if (value != nullptr) { //primҵƥֵֵһϵת罫תΪַʽIRתΪOpԣ + // convert parts of attr to str eg. data_format or change ir attr to op attr eg. axis[0] + (void)CheckAndConvertUtils::ConvertAttrValueToString(prim->name(), it.first, &value); + (void)CheckAndConvertUtils::CheckIrAttrtoOpAttr(prim->name(), it.first, &value); + // set attr from primitive + int ret = setAttr(op, it.first, value); //ȻsetAttr + if (ret) { + return ret; + } + } else { //// primδҵƥֵǷextra_attr_Ϣ + // set attr from extra_attr + auto it_extra = extra_attr_->find(it.first); + if (it_extra != extra_attr_->end()) { // extra_attr_ҵƥֵͬsetAttr + int ret = setAttr(op, it.first, it_extra->second); + if (ret) { // зʱ˷ֵʧܣظô롣 + return ret; + } + } + } + } + return 0; //ɹԷ0 +} + +// úΪԡ +// : +// - op: ָOperatorPtrָ룬ʾҪԵ +// - prim: ָPrimitivePtrָ룬ʾԭPrimitiveϢ +int OpAdapterImpl::setAttr(const OperatorPtr &op, const PrimitivePtr &prim) { + int ret = 0; + if (IsCustomPrim(prim)) { //жprimǷΪԶԭCustomPrimitive + auto cus_op = std::dynamic_pointer_cast(op); + ret = SetCustomOpAttr(cus_op, prim); //ԶԭSetCustomOpAttrΪԡ + } else { // 򣬵SetNormalOpAttrΪ + ret = SetNormalOpAttr(op, prim); + } + return ret; //ֵΪʾʧܣضӦĴ롣 +} + +// úΪԡ +// : +// - op: ָOperatorPtrָ룬ʾҪԵ +// - node: ָAnfNodePtrָ룬ʾӦͼڵ㣨AnfNode +int OpAdapterImpl::setAttr(const OperatorPtr &op, const AnfNodePtr &node) { + // no attribute for lonely node + MS_EXCEPTION_IF_NULL(node); + if (!node->isa()) { //жnodeǷΪCNodeڵ㣩 + return 0; //CNodeʾýڵûԣֱӷ0 + } + // ڵתΪCNodePtrڵıʾ + auto cnode = node->cast(); + if (cnode == nullptr) { + return 0; + } + // ȡڵ롣 + auto &inputs = cnode->inputs(); + if (inputs.empty()) { + return 0; + } + + // get Attr T from abstract of anfnode first, + // if attr "T" appears in primitive, the primitive T will cover this one + // ȴAnfNodeijϢлȡ"T" + // ԭprimitive"T"ԭе"T"Ǵ˴"T"ԡ + if (attr_map_.find("T") != attr_map_.end()) { + // get dtype from inputs[1], if the node has no inputs, set the attr T with output dtype + // inputs[1]лȡͣdtypeڵû룬ʹ"T" + TypePtr type; + if (inputs.size() > 1) { + type = inputs[1]->Type(); + } else { + type = node->Type(); + } + if (type != nullptr) { // ʹõ"T" + (void)setAttr(op, "T", MakeValue(type)); + } + } + + // set attr from primitive and ExtraAttr + // ԭExtraAttrԡ + if (IsValueNode(inputs[0])) { + // set attr from primitive + // ԭԡ + PrimitivePtr prim = GetValueNode(inputs[0]); + int ret = setAttr(op, prim); + if (ret != 0) { + return ret; + } + } + + // set attr from const input + // ӳԡ + for (auto &it : input_attr_map_) { + // ǷڷΧڣǷΪValueNodeڵ㣩 + if (inputs.size() <= it.first || !inputs[it.first]->isa()) { + continue; + } + // лȡֵ + auto const_value = GetValueNode(inputs[it.first]); + MS_LOG(INFO) << "Set attr: input_" << it.first << "(" << it.second.name << "), value: " << const_value->ToString(); + if (const_value->isa()) { // ֵΪNoneԡ + continue; + } + // ϢӵͼͼС + adpt_->AddAttrToDrawGraph(it.second.name + std::string("=") + const_value->ToString()); + // ʹṩit.second.set_attrͳֵԡ + it.second.set_attr(op, const_value); + } + return 0; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From d8913ceb95ac5407dc1da95937b181177c1e6b69 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:15:25 +0800 Subject: [PATCH 35/72] ADD file via upload --- mindspore/ccsrc/transform-update/op_adapter.h | 478 ++++++++++++++++++ 1 file changed, 478 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter.h diff --git a/mindspore/ccsrc/transform-update/op_adapter.h b/mindspore/ccsrc/transform-update/op_adapter.h new file mode 100644 index 00000000000..302f9855804 --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter.h @@ -0,0 +1,478 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_H_ + +#include +#include +#include + +#include "utils/hash_map.h" +#include "transform/graph_ir/op_adapter_util.h" +#include "transform/graph_ir/op_adapter_base.h" +#include "include/common/utils/utils.h" +namespace mindspore { +namespace transform { +class OpAdapterImpl { + public: + // 캯ڳʼOpAdapterImpl󡣽һϵDZ浽ӦijԱС + OpAdapterImpl(const mindspore::HashMap &input_map, + const mindspore::HashMap &dyn_input_map, + const mindspore::HashMap &output_map, + const mindspore::HashMap &dyn_output_map, + const mindspore::HashMap &dyn_subgraph_map, + const mindspore::HashMap &attr_map, + const mindspore::HashMap &enum_map, + const mindspore::HashMap &input_attr_map, + mindspore::HashMap> *cus_input_map, + mindspore::HashMap> *cus_output_map, + mindspore::HashMap *extra_attr, + mindspore::HashMap *name_counts, BaseOpAdapter *adpt) + : input_map_(input_map), + dyn_input_map_(dyn_input_map), + output_map_(output_map), + dyn_output_map_(dyn_output_map), + dyn_subgraph_map_(dyn_subgraph_map), + attr_map_(attr_map), + enum_map_(enum_map), + input_attr_map_(input_attr_map), + cus_input_map_(cus_input_map), + cus_output_map_(cus_output_map), + extra_attr_(extra_attr), + name_counts_(name_counts), + adpt_(adpt) { + MS_EXCEPTION_IF_NULL(cus_input_map_); + MS_EXCEPTION_IF_NULL(cus_output_map_); + MS_EXCEPTION_IF_NULL(extra_attr_); + MS_EXCEPTION_IF_NULL(name_counts_); + MS_EXCEPTION_IF_NULL(adpt_); + } + ~OpAdapterImpl() {} // ͷԴ + bool IsCustomOp(const OperatorPtr &op); // жһǷΪԶ + Status GenerateCustomOpInputMap(const CusOperatorPtr &op, const PrimitivePtr &prim); // Զӳ䡣 + Status GenerateCustomOpOutputMap(const CusOperatorPtr &op, const PrimitivePtr &prim); // Զӳ䡣 + OperatorPtr GenerateCustomOp(const AnfNodePtr anf); // Զ + Status SetOpSubgraphFunc(const OperatorPtr &op, int index, const std::shared_ptr> &branches); // ͼ + Status SetCustomOpInput(const CusOperatorPtr &op, int index, const OperatorPtr &input); // Զ롣 + Status SetNormalOpInput(const OperatorPtr &op, int index, const OperatorPtr &input); // ͨ롣 + int setInput(const OperatorPtr &op, int index, const OperatorPtr &input); // 롣 + Status SetCustomOpInput(const CusOperatorPtr &op, int index, const OutHandler &handle); // Զ롣 + Status SetNormalOpInput(const OperatorPtr &op, int index, const OutHandler &handle); // ͨ롣 + int setInput(const OperatorPtr &op, int index, const OutHandler &handle); // 롣 + int setInput(const OperatorPtr &op, int index, const std::shared_ptr> &handler_vec); // 롣 + OutHandler getOutput(const OperatorPtr &op, int index); // ȡ + OutHandler getCustomOutput(const OperatorPtr &op, int index); // ȡԶ + OutHandler getNormalOutput(const OperatorPtr &op, int index); // ȡͨ + Status UpdateSingleOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const std::string &format); // µϢ + size_t GetCustomOpOutputSize(const CusOperatorPtr &cus_op); // ȡԶ + std::shared_ptr CreateOutputDesc(const abstract::ShapePtr &shape_ptr, const TypePtr &type, + const std::string &format); // + Status UpdateMultiOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const std::string &format); // ¶Ϣ + std::shared_ptr CreateNodeDesc(const AnfNodePtr &node, const std::string &format); // ڵ + void UpdateNormalOpInputDesc(const OperatorPtr &op, const AnfNodePtr &node, const std::string format); // ͨ + void UpdateCustomOpInputDesc(const CusOperatorPtr &op, const AnfNodePtr &node, const std::string format);// Զ + void updateInputDesc(const OperatorPtr &op, const AnfNodePtr &node);// + void updateOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const AnfNodePtr &node); // + int setAttr(const OperatorPtr &op, const std::string &attr_key, const ValuePtr &attr_value); // ԡ + int SetCustomOpAttr(const CusOperatorPtr &op, const PrimitivePtr &prim); // Զԡ + int SetNormalOpAttr(const OperatorPtr &op, const PrimitivePtr &prim); // ͨԡ + int setAttr(const OperatorPtr &op, const PrimitivePtr &prim); // ԡ + int setAttr(const OperatorPtr &op, const AnfNodePtr &node); // ԡ + + private: // һϵӳӳ䡣 + const mindspore::HashMap &input_map_; + const mindspore::HashMap &dyn_input_map_; + const mindspore::HashMap &output_map_; + const mindspore::HashMap &dyn_output_map_; + const mindspore::HashMap &dyn_subgraph_map_; + const mindspore::HashMap &attr_map_; + const mindspore::HashMap &enum_map_; + const mindspore::HashMap &input_attr_map_; + // Զӳӳ䡣 + mindspore::HashMap> *const cus_input_map_; + mindspore::HashMap> *const cus_output_map_; + mindspore::HashMap *const extra_attr_; + mindspore::HashMap *const name_counts_; + BaseOpAdapter *const adpt_; +}; + +template +class OpAdapter : public BaseOpAdapter { + public: + // ʹOpTypeΪģĹ캯ʼOpAdapterImpl + using OpType = T; + OpAdapter() + : impl_(std::make_shared(input_map_, dyn_input_map_, output_map_, dyn_output_map_, + dyn_subgraph_map_, attr_map_, enum_map_, input_attr_map_, &cus_input_map_, + &cus_output_map_, &extra_attr_, &name_counts_, this)) { + MS_EXCEPTION_IF_NULL(impl_); + } + // ʹExtraAttrΪĹ캯ʼOpAdapterImpl + explicit OpAdapter(const ExtraAttr &extra_attr) + : extra_attr_(extra_attr), + impl_(std::make_shared(input_map_, dyn_input_map_, output_map_, dyn_output_map_, + dyn_subgraph_map_, attr_map_, enum_map_, input_attr_map_, &cus_input_map_, + &cus_output_map_, &extra_attr_, &name_counts_, this)) { + MS_EXCEPTION_IF_NULL(impl_); + } + // ͷԴ + ~OpAdapter() override {} + // жһǷΪԶ + bool IsCustomOp(const OperatorPtr &op) { return impl_->IsCustomOp(op); } + // Զӳ䡣 + Status GenerateCustomOpInputMap(const CusOperatorPtr &op, const PrimitivePtr &prim) { + return impl_->GenerateCustomOpInputMap(op, prim); + } + // Զӳ䡣 + Status GenerateCustomOpOutputMap(const CusOperatorPtr &op, const PrimitivePtr &prim) { + return impl_->GenerateCustomOpOutputMap(op, prim); + } + + // Convert ME UserCustom AnfNode to GE CustomOp. And set it's attrs. + // ME UserCustom AnfNodeתΪGE CustomOpԡ + OperatorPtr GenerateCustomOp(const AnfNodePtr anf) { return impl_->GenerateCustomOp(anf); } + // ͨ + OperatorPtr GenerateNormalOp(const AnfNodePtr &anf) { + OperatorPtr op = nullptr; + // There are duplicate names in ANF graph, do not assign ANF node name to GE + // GE will generate unique name automatically + if (anf != nullptr && anf->fullname_with_scope() != "") { + MS_LOG(DEBUG) << anf->fullname_with_scope(); + op = std::make_shared(anf->fullname_with_scope()); + } else { + MS_LOG(DEBUG) << "no fullname_with_scope"; + op = std::make_shared(); + } + + // set dynamic output num if op use DYNAMIC_OUTPUT + if ((op != nullptr) && (!dyn_output_map_.empty()) && (anf != nullptr)) { + TypePtr type = anf->Type(); + if (type == nullptr) { + MS_LOG(EXCEPTION) << "Dynamic output node:" << op->GetName() << "'s Type is a nullptr!"; + } + size_t num = type->isa() ? (type->cast>()->size()) : 1; + MS_LOG(INFO) << "create_dyn_output for node:" << anf->ToString() << ", type:" << type->ToString() + << ", num:" << num; + dyn_output_map_.begin()->second.create_dyn_output(op, static_cast(num)); + } + return op; + } + // ʵgenerateݴAnfNodeɶӦOperatorPtr + OperatorPtr generate(const AnfNodePtr &anf) override { + OperatorPtr op = nullptr; + if (IsCustomCNode(anf)) { + op = GenerateCustomOp(anf); + } else { + op = GenerateNormalOp(anf); + } + if (op == nullptr) { + MS_LOG(EXCEPTION) << "Can not generate op for " << anf->fullname_with_scope(); + } + return op; + } + // ʵgenerateݴop_nameɶӦOperatorPtr + OperatorPtr generate(const std::string &op_name) override { return std::make_shared(op_name); } + // ȡӳ䡣 + const mindspore::HashMap &getInputMap() override { return input_map_; } + // ȡӳ䡣 + const mindspore::HashMap &getInputAttrMap() override { return input_attr_map_; } + // ȡ̬ӳ䡣 + const mindspore::HashMap &getDynInputMap() override { return dyn_input_map_; } + // ȡӳ䡣 + const mindspore::HashMap &getOutputMap() override { return output_map_; } + // ȡ̬ͼӳ䡣 + const mindspore::HashMap &getDynSubgraphMap() override { return dyn_subgraph_map_; } + // ͼ + Status SetOpSubgraphFunc(const OperatorPtr &op, int index, const std::shared_ptr> &branches) { + return impl_->SetOpSubgraphFunc(op, index, branches); + } + // ͼ + int setSubgraph(const OperatorPtr &op, int index, const std::shared_ptr> &branches) override { + return static_cast(SetOpSubgraphFunc(op, index, branches)); + } + // Զ롣 + Status SetCustomOpInput(const CusOperatorPtr &op, int index, const OperatorPtr &input) { + return impl_->SetCustomOpInput(op, index, input); + } + // ͨ롣 + Status SetNormalOpInput(const OperatorPtr &op, int index, const OperatorPtr &input) { + return impl_->SetNormalOpInput(op, index, input); + } + // 롣 + int setInput(const OperatorPtr &op, int index, const OperatorPtr &input) override { + return impl_->setInput(op, index, input); + } + // Զ롣 + Status SetCustomOpInput(const CusOperatorPtr &op, int index, const OutHandler &handle) { + return impl_->SetCustomOpInput(op, index, handle); + } + // ͨ롣 + Status SetNormalOpInput(const OperatorPtr &op, int index, const OutHandler &handle) { + return impl_->SetNormalOpInput(op, index, handle); + } + // + int setInput(const OperatorPtr &op, int index, const OutHandler &handle) override { + return impl_->setInput(op, index, handle); + } + // 롣 + int setInput(const OperatorPtr &op, int index, const std::shared_ptr> &handler_vec) override { + return impl_->setInput(op, index, handler_vec); + } + // ȡ + OutHandler getOutput(const OperatorPtr &op, int index) override { return impl_->getOutput(op, index); } + // ȡԶ + OutHandler getCustomOutput(const OperatorPtr &op, int index) { return impl_->getCustomOutput(op, index); } + // ȡͨ + OutHandler getNormalOutput(const OperatorPtr &op, int index) { return impl_->getNormalOutput(op, index); } + // µϢ + Status UpdateSingleOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const std::string &format) { + return impl_->UpdateSingleOutputDesc(op, shp, type, format); + } + // ȡԶ + size_t GetCustomOpOutputSize(const CusOperatorPtr &cus_op) { return impl_->GetCustomOpOutputSize(cus_op); } + // + std::shared_ptr CreateOutputDesc(const abstract::ShapePtr &shape_ptr, const TypePtr &type, + const std::string &format) { + return impl_->CreateOutputDesc(shape_ptr, type, format); + } + // ¶Ϣ + Status UpdateMultiOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const std::string &format) { + return impl_->UpdateMultiOutputDesc(op, shp, type, format); + } + // ڵ + std::shared_ptr CreateNodeDesc(const AnfNodePtr &node, const std::string &format) { + return impl_->CreateNodeDesc(node, format); + } + // ͨ + void UpdateNormalOpInputDesc(const OperatorPtr &op, const AnfNodePtr node, const std::string format) { + return impl_->UpdateNormalOpInputDesc(op, node, format); + } + // Զ + void UpdateCustomOpInputDesc(const CusOperatorPtr &op, const AnfNodePtr &node, const std::string format) { + return impl_->UpdateCustomOpInputDesc(op, node, format); + } + // + void updateInputDesc(const OperatorPtr &op, const AnfNodePtr &node) { impl_->updateInputDesc(op, node); } + // + void updateOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const AnfNodePtr &node) override { + impl_->updateOutputDesc(op, shp, type, node); + } + // ԡ + int setAttr(const OperatorPtr &op, const std::string &attrKey, const ValuePtr &attrValue) override { + return impl_->setAttr(op, attrKey, attrValue); + } + // Զԡ + int SetCustomOpAttr(const CusOperatorPtr &op, const PrimitivePtr &prim) { return impl_->SetCustomOpAttr(op, prim); } + // ͨԡ + int SetNormalOpAttr(const OperatorPtr &op, const PrimitivePtr &prim) { return impl_->SetNormalOpAttr(op, prim); } + // ԡ + int setAttr(const OperatorPtr &op, const PrimitivePtr &prim) override { return impl_->setAttr(op, prim); } + // ԡ + int setAttr(const OperatorPtr &op, const AnfNodePtr &node) override { return impl_->setAttr(op, node); } + // ȡԡ + mindspore::HashMap GetExtraAttr() override { return extra_attr_; } + + private: + template + static S ConvertAny(const ValuePtr &value, const AnyTraits &) { + return GetValue(value); + } + + // specialization for reverse bool + static bool ConvertAny(const ValuePtr &value, const AnyTraits &, bool reverse) { + return reverse != GetValue(value); + } + + template + static Q ConvertAny(const ValuePtr &value, const AnyTraits

&traits_from, const AnyTraits &traits_to) { + return ConvertAnyUtil(value, traits_from, traits_to); + } + + // specialization for tensor + static GeTensor ConvertAny(const ValuePtr &value, const AnyTraits &traits) { + // To-DO the format may read from ME tensor + return ConvertAnyUtil(value, traits); + } + + // specialization for int + static int64_t ConvertAny(const ValuePtr &value, const AnyTraits) { + return static_cast(GetValue(value)); + } + + // specialization for int or tuple broadcast to Vector + static std::vector ConvertAny(const ValuePtr &value, const std::string &name, + const AnyTraits> anyTraitsInt) { + return ConvertAnyUtil(value, name, anyTraitsInt); + } + + static std::vector> ConvertAny(const ValuePtr &value, + const AnyTraits>>) { + MS_EXCEPTION_IF_NULL(value); + MS_LOG(INFO) << "Value: " << value->type_name(); + std::vector> list; + if (!value->isa()) { + MS_LOG(EXCEPTION) << "Value should be ValueTuple, but got " << value->type_name(); + } + auto vec = value->cast(); + MS_EXCEPTION_IF_NULL(vec); + for (auto &it : vec->value()) { + MS_EXCEPTION_IF_NULL(it); + if (!it->isa()) { + MS_LOG(EXCEPTION) << "It should be ValueTuple, but got " << it->type_name(); + } + auto sub_vector = it->cast(); + std::vector sublist; + for (auto &item : sub_vector->value()) { + sublist.push_back(static_cast(GetValue(item))); + } + list.push_back(sublist); + } + return list; + } + + static std::vector ConvertAny(const ValuePtr &value, const AnyTraits>>, + const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); + MS_LOG(DEBUG) << "Value: " << value->type_name(); + if (!value->isa()) { + MS_LOG(EXCEPTION) << "Value should be ValueList, but got " << value->type_name(); + } + auto vec = value->cast(); + std::vector list; + for (auto &it : vec->value()) { + MS_EXCEPTION_IF_NULL(it); + if (!it->isa()) { + MS_LOG(EXCEPTION) << "It should be ValueList, but got " << it->type_name(); + } + auto sub_vector = it->cast(); + for (auto &item : sub_vector->value()) { + list.push_back(static_cast(GetValue(item))); + } + } + return list; + } + + static std::vector ConvertAny(const ValuePtr &value, const AnyTraits>, + const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); + MS_LOG(INFO) << "Value: " << value->type_name(); + std::vector list; + if (value->isa()) { + auto vec = value->cast(); + MS_EXCEPTION_IF_NULL(vec); + for (auto &it : vec->value()) { + list.push_back(static_cast(GetValue(it))); + } + return list; + } + if (value->isa()) { + list.push_back(static_cast(GetValue(value))); + return list; + } + MS_LOG(EXCEPTION) << "Value should be ValueTuple or Scalar, but got " << value->type_name(); + } + + static std::string ConvertAny(const ValuePtr &value, const AnyTraits> anyTraitsVec, + const AnyTraits anyTraitsStr) { + return ConvertAnyUtil(value, anyTraitsVec, anyTraitsStr); + } + + static std::vector ConvertAny(const ValuePtr &value, const AnyTraits> anyTraitsVec, + const AnyTraits anyTraitsFlo) { + return ConvertAnyUtil(value, anyTraitsVec, anyTraitsFlo); + } + + static std::vector ConvertAny(const ValuePtr &value, const std::string &format, + const AnyTraits> anyTraitsVec, + const AnyTraits anyTraitsInt) { + return ConvertAnyUtil(value, format, anyTraitsVec, anyTraitsInt); + } + + // convert value list for value tuple to vector + template + static std::vector ConvertAny(const ValuePtr &value, const AnyTraits

&anyTraitsP, + const AnyTraits> anyTraitsQ) { + return ConvertAnyUtil(value, anyTraitsP, anyTraitsQ); + } + + static int64_t ConvertAny(const ValuePtr &value, const AnyTraits) { + auto name = GetValue(value); + auto it = enum_map_.find(name); + int v = 0; + if (it != enum_map_.end()) { + v = it->second; + } + return v; + } + + static GeDataType ConvertAny(const ValuePtr &value, const AnyTraits anyTraitsGE) { + return ConvertAnyUtil(value, anyTraitsGE); + } + + // convert any value to tensor + static GeTensor ConvertAny(const ValuePtr &value, const AnyTraits anyTraitsValue) { + return ConvertAnyUtil(value, anyTraitsValue); + } + + static const mindspore::HashMap input_map_; + static const mindspore::HashMap dyn_input_map_; + static const mindspore::HashMap output_map_; + static const mindspore::HashMap dyn_output_map_; + static const mindspore::HashMap dyn_subgraph_map_; + static const mindspore::HashMap attr_map_; + static const mindspore::HashMap enum_map_; + // convert input from anf graph to Attr in Operators + static const mindspore::HashMap input_attr_map_; + static mindspore::HashMap> cus_input_map_; + static mindspore::HashMap> cus_output_map_; + mindspore::HashMap extra_attr_; + mindspore::HashMap name_counts_; + const std::shared_ptr impl_; +}; + +template +const mindspore::HashMap OpAdapter::input_map_; +template +const mindspore::HashMap OpAdapter::dyn_input_map_; +template +const mindspore::HashMap OpAdapter::output_map_; +template +const mindspore::HashMap OpAdapter::dyn_output_map_; +template +const mindspore::HashMap OpAdapter::dyn_subgraph_map_; +template +const mindspore::HashMap OpAdapter::attr_map_; +template +const mindspore::HashMap OpAdapter::enum_map_; +template +const mindspore::HashMap OpAdapter::input_attr_map_; +template +mindspore::HashMap> OpAdapter::cus_input_map_; +template +mindspore::HashMap> OpAdapter::cus_output_map_; + +// specialization for method +} // namespace transform +} // namespace mindspore + +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_H_ -- 2.34.1 From 419206d9e48fdf95d199cd46143f9215cf5c81e4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:15:44 +0800 Subject: [PATCH 36/72] ADD file via upload --- .../ccsrc/transform-update/op_adapter_base.h | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter_base.h diff --git a/mindspore/ccsrc/transform-update/op_adapter_base.h b/mindspore/ccsrc/transform-update/op_adapter_base.h new file mode 100644 index 00000000000..7cb85816dae --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter_base.h @@ -0,0 +1,175 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_ + +#include +#include +#include +#include +#include + +#include "utils/hash_map.h" +#include "include/transform/graph_ir/util.h" +#include "ir/anf.h" +#include "ir/primitive.h" +#include "ir/value.h" +#include "include/transform/graph_ir/types.h" +#include "graph/operator_reg.h" +#include "external/ge/ge_api.h" +#include "graph/tensor.h" + +namespace ge { +class CustomOperator : public Operator { // CustomOperator + public: // + CustomOperator(const string &name, const string &type) : Operator(name, type) {} + + ~CustomOperator() override{}; + + void CustomInputRegister(const string &name) { Operator::InputRegister(name); } + + void CustomOutputRegister(const string &name) { Operator::OutputRegister(name); } + + void CustomInferFuncRegister(const std::function &func) { + Operator::InferFuncRegister(func); + } +}; +} // namespace ge + +namespace mindspore { +namespace transform { +using CusOperatorPtr = std::shared_ptr; +using CustomOperator = ge::CustomOperator; +using AttrFunc = std::function; +using OutputFunc = std::function; +using InputOpFunc = std::function; +using InputHandleFunc = std::function; +using CreateDynInputOpFunc = std::function; +using DynInputOpFunc = std::function; +using DynInputHandleFunc = std::function; +using UpdateOutputDescFunc = std::function; +using CreateDynOutputOpFunc = std::function; +using CreateDynSubGraphFunc = std::function; +using DynSubGraphFunc = std::function; + +//ṹ +struct AttrDesc { + std::string name; + AttrFunc set_attr; +}; + +struct InputDesc { + std::string name; + InputOpFunc set_op; + InputHandleFunc set_handle; + UpdateOutputDescFunc update_input_desc; +}; + +struct DynInputDesc { + std::string name; + CreateDynInputOpFunc create_dyn_input; + DynInputOpFunc set_op; + DynInputHandleFunc set_handle; +}; + +struct DynSubGraphDesc { + std::string name; + CreateDynSubGraphFunc create_dyn_subgraph; + DynSubGraphFunc set_subgraph; +}; + +struct OutputDesc { + std::string name; + UpdateOutputDescFunc update_out_desc; +}; + +struct DynOutputDesc { + std::string name; + CreateDynOutputOpFunc create_dyn_output; +}; + +class BaseOpAdapter { //BaseOpAdapter + public: // + virtual ~BaseOpAdapter() {} + virtual OperatorPtr generate(const AnfNodePtr &anf) = 0; + virtual OperatorPtr generate(const std::string &type) { return std::make_shared(type); } + virtual int setSubgraph(const OperatorPtr &op, int index, const std::shared_ptr> &branches) = 0; + virtual int setInput(const OperatorPtr &op, int index, const OperatorPtr &input) = 0; + virtual int setInput(const OperatorPtr &op, int index, const OutHandler &handle) = 0; + virtual int setInput(const OperatorPtr &op, int index, + const std::shared_ptr> &handler_vec) = 0; + virtual int setAttr(const OperatorPtr &op, const std::string &attrKey, const ValuePtr &attrValue) = 0; + virtual int setAttr(const OperatorPtr &op, const PrimitivePtr &prim) = 0; + virtual int setAttr(const OperatorPtr &op, const AnfNodePtr &node) = 0; + virtual mindspore::HashMap GetExtraAttr() = 0; + template ::value>::type> + int setAttr(const OperatorPtr &op, const std::string &attrKey, const std::shared_ptr &attrValue) { + return setAttr(op, attrKey, MakeValue(attrValue)); + } + template ::value>::type> + int setAttr(const OperatorPtr &op, const std::string &attrKey, const T &attrValue) { + return setAttr(op, attrKey, MakeValue(attrValue)); + } + virtual OutHandler getOutput(const OperatorPtr &op, int index) = 0; + virtual void updateOutputDesc(const OperatorPtr &op, const abstract::BaseShapePtr &shp, const TypePtr &type, + const AnfNodePtr &node) = 0; + virtual const mindspore::HashMap &getInputMap() = 0; + virtual const mindspore::HashMap &getInputAttrMap() = 0; + virtual const mindspore::HashMap &getDynInputMap() = 0; + virtual const mindspore::HashMap &getOutputMap() = 0; + virtual const mindspore::HashMap &getDynSubgraphMap() = 0; + void AddAttrToDrawGraph(const std::string &attr_str) { attrs_vec_.push_back(attr_str); } + const std::vector &GetAttrsFromDrawGraph() const { return attrs_vec_; } + void clearAttrVect() { attrs_vec_.clear(); } + + private: //Ա + std::vector attrs_vec_; +}; + +using OpAdapterPtr = std::shared_ptr; + +enum AttrType { //enumؼ + ATTR_INT = 0, + ATTR_FLOAT, + ATTR_DOUBLE, + ATTR_STRING, + ATTR_TENSOR, + ATTR_BOOL, + ATTR_LIST_INT, + ATTR_LIST_ANY_INT, + ATTR_ENUM +}; + +struct GeEnum {}; +struct TFType {}; +struct GEType {}; + +// declare Any type +template +struct AnyTraits { + using type = T; +}; + +template <> +struct AnyTraits { + using type = int64_t; +}; + +using ExtraAttr = mindspore::HashMap; +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_BASE_H_ -- 2.34.1 From e6fcc172e1ef5bbe881b49cb12bda033b0c60c4d Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:16:03 +0800 Subject: [PATCH 37/72] ADD file via upload --- .../ccsrc/transform-update/op_adapter_desc.h | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter_desc.h diff --git a/mindspore/ccsrc/transform-update/op_adapter_desc.h b/mindspore/ccsrc/transform-update/op_adapter_desc.h new file mode 100644 index 00000000000..1b87cae93c4 --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter_desc.h @@ -0,0 +1,75 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_ + +#include +#include "transform/graph_ir/op_adapter.h" + +namespace mindspore { +namespace transform { +class OpAdapterDesc { //һOpAdapterDesc + public: // + OpAdapterDesc() : train_(nullptr), infer_(nullptr) {} + + OpAdapterDesc(const OpAdapterPtr &train, const OpAdapterPtr &infer) : train_(train), infer_(infer) {} + + explicit OpAdapterDesc(const OpAdapterPtr &common) : train_(common), infer_(common) {} + + OpAdapterDesc(const OpAdapterDesc &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + } + + OpAdapterDesc(OpAdapterDesc &&desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + desc.train_ = nullptr; + desc.infer_ = nullptr; + } + + ~OpAdapterDesc() = default; + + OpAdapterPtr Get(bool train) const { return train ? train_ : infer_; } + + OpAdapterDesc &operator=(const OpAdapterDesc &desc) { + if (this != &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + } + return *this; + } + + OpAdapterDesc &operator=(OpAdapterDesc &&desc) { + if (this != &desc) { + this->train_ = desc.train_; + this->infer_ = desc.infer_; + desc.train_ = nullptr; + desc.infer_ = nullptr; + } + return *this; + } + + private: //Ա + OpAdapterPtr train_; + OpAdapterPtr infer_; +}; + +using OpAdapterDescPtr = std::shared_ptr; +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_DESC_H_ -- 2.34.1 From 2375e97925220f552d0665d58c5c1cbcc44c29ce Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:16:44 +0800 Subject: [PATCH 38/72] ADD file via upload --- .../ccsrc/transform-update/op_adapter_map.cc | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter_map.cc diff --git a/mindspore/ccsrc/transform-update/op_adapter_map.cc b/mindspore/ccsrc/transform-update/op_adapter_map.cc new file mode 100644 index 00000000000..c5d1e9536ec --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter_map.cc @@ -0,0 +1,43 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "include/transform/graph_ir/op_adapter_map.h" +#include +#include "graph/operator.h" +#include "transform/graph_ir/op_adapter_desc.h" + +namespace mindspore { +namespace transform { +namespace { +// һ HashMap 洢ַ OpAdapterDescPtr ӳϵ +// std::string ֵͣ OpAdapterDesc shared_ptr +mindspore::HashMap adpt_map_ = { + {kNameCustomOp, std::make_shared(std::make_shared>())}}; +// ʹóʼбһԪز뵽 HashMap С +// "kNameCustomOp"ֵһʹ OpAdapter Ϊģ OpAdapterDesc shared_ptr +} // namespace + +// ģ壬Ϊ ge::Operator ͵ OpAdapter һƵӳ䡣 +// ʹ mindspore::HashMap Ϊֵ HashMapȻʹ std::string Ϊ HashMap +template <> +mindspore::HashMap> OpAdapter::cus_input_map_{}; +// ģ壬Ϊ ge::Operator ͵ OpAdapter һƵӳ䡣 +// ʹ mindspore::HashMap Ϊֵ HashMapȻʹ std::string Ϊ HashMap +template <> +mindspore::HashMap> OpAdapter::cus_output_map_{}; +// OpAdapterMap ijԱڷ OpAdapterMap adpt_map_ Աá +mindspore::HashMap &OpAdapterMap::get() { return adpt_map_; } +} // namespace transform +} // namespace mindspore -- 2.34.1 From 5eba83c33b4ac28354213001364a62013c076fb4 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:17:14 +0800 Subject: [PATCH 39/72] ADD file via upload --- .../ccsrc/transform-update/op_adapter_util.cc | 392 ++++++++++++++++++ 1 file changed, 392 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter_util.cc diff --git a/mindspore/ccsrc/transform-update/op_adapter_util.cc b/mindspore/ccsrc/transform-update/op_adapter_util.cc new file mode 100644 index 00000000000..a98852cc66d --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter_util.cc @@ -0,0 +1,392 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_adapter_util.h" + +#include +#include +#include + +#include "include/common/utils/utils.h" +#include "utils/check_convert_utils.h" +#include "transform/graph_ir/op_adapter_base.h" +#include "transform/graph_ir/io_format_map.h" + +namespace mindspore { +namespace transform { +// ConvertAnyUtil ڽ MindSpore е Tensormindspore::tensor::TensorתΪ GEGraphEngineеTensorGeTensor +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits &) { + // To-DO the format may read from ME tensor + // TODOҪ MEMindSpore Execution Tensor ȡʽϢformat + MS_EXCEPTION_IF_NULL(value); + //// value ǿתΪ MeTensorPtr ͣMeTensorPtr һָ룬ʾ ME TensorMindSpore Execution Tensor + auto me_tensor = value->cast(); + // TransformUtil::ConvertTensor ME Tensor תΪ GE Tensor + // kOpFormat_ND ָת GE Tensor ʹõĸʽ ND ʽN-Dimensional + auto ge_tensor = TransformUtil::ConvertTensor(me_tensor, kOpFormat_ND); + // ת GE Tensor Ϊգ򷵻һյ GeTensor 󣬷򷵻ת GE Tensor + return ge_tensor == nullptr ? GeTensor() : *ge_tensor; +} + +// ConvertAnyUtil ڽһ ValuePtr ͵ֵתΪ std::vector ͡ +// תķʽȡڴ name AnyTraits> +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &name, + const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); + std::vector list; // һ int64_t ͵ vectorڴ洢תĽ + if (name == "pad") { // name "pad"ִضת߼ + if (!value->isa()) { // ȷ value ValueSequence ͡ + MS_LOG(EXCEPTION) << "Value should be ValueTuple, but got" << value->type_name(); + } + auto vec = value->cast(); // value תΪ ValueSequencePtr ͡ + // vector ĴСתĽ + // ڽԪأ11˱ ValueSequence ĴС2 + list.resize(vec->value().size() + 2); + // ԪΪ 1 + list[0] = 1; + list[1] = 1; + // ʹ std::transform ValueSequence еԪתΪ int64_t洢 vector С + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin() + 2, + [](const ValuePtr &val) { return static_cast(GetValue(val)); }); + } else { // name "pad"ִͨõת߼ + int64_t data = GetValue(value); // value лȡ int64_t ͵ݡ + int size = 2; // 2 int in list // vector ĴСΪ2 int64_t ͵Ԫء + // TransformUtil::ConvertIntToList int64_t תΪ std::vector + list = TransformUtil::ConvertIntToList(data, size); + } + + return list; // ת std::vector +} + +// ConvertAnyUtil ڽһ ValuePtr ͵ֵתΪ std::string ͡ +// תķʽȡڴ AnyTraits> AnyTraits +std::string ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); // value תΪ ValueTuplePtr ͡ + if (vec == nullptr) { // vec Ϊָ룬׳쳣˵ value ValueTuplePtr ͡ + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::ostringstream buffer; // һ ostringstream ڹַ + int i = 0; // ڸַļ + for (auto &it : vec->value()) { // value еԪء + if (i != 0) { // ÿԪ֮ǰ붺ţ˵һԪأ + buffer << ","; + } + buffer << GetValue(it); // ԪصֵתΪ int64_tӵַС + i++; // Ӽ + } + return buffer.str(); // عַ +} + +// ConvertAnyUtil ڽһ ValuePtr ͵ֵתΪ std::vector ͡ +// תķʽȡڴ AnyTraits> AnyTraits +std::vector ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); // value תΪ ValueTuplePtr ͡ + if (vec == nullptr) { // vec Ϊָ룬׳쳣˵ value ValueTuplePtr ͡ + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::vector list; // һ std::vector ڴ洢תĽ + list.resize(vec->value().size()); // vector ĴСתĽС ValueTuple еԪظͬ + // ʹ std::transform ValueTuple еÿԪתΪ float洢 vector С + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin(), + [](const ValuePtr &val) { return static_cast(GetValue(val)); }); + return list; // ת std::vector +} + +// ConvertAnyUtil ڽһ ValuePtr ͵ֵתΪ std::vector ͡ +// תķʽȡڴ format AnyTraits> AnyTraits +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &format, + const AnyTraits>, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + auto vec = value->cast(); // value תΪ ValueTuplePtr ͡ + if (vec == nullptr) { // vec Ϊָ룬׳쳣˵ value ValueTuplePtr ͡ + MS_LOG(EXCEPTION) << "not ValueTuplePtr"; + } + std::vector list; // һ std::vector ڴ洢תĽ + list.resize(vec->value().size()); // vector ĴСתĽС ValueTuple еԪظͬ + // ʹ std::transform ValueTuple еÿԪתΪ int64_t洢 vector С + (void)std::transform(vec->value().begin(), vec->value().end(), list.begin(), + [](const ValuePtr &val) { return static_cast(GetValue(val)); }); + if (format == kOpFormat_NHWC) { // ݴ format ִضĸʽת + if (list.size() < 4) { // ʽΪ NHWCбСС4׳쳣 + MS_LOG(EXCEPTION) << "The size of list is less than 4"; + } else { // ʽΪ NHWCбСڵ4иʽת + // беĵ1Ԫغ͵2Ԫؽλã3Ԫغ͵4Ԫؽλá + int64_t temp = list[1]; + list[1] = list[2]; + list[2] = list[3]; + list[3] = temp; + } + } + return list; // ת std::vector +} + +// ConvertAnyUtil ڽһ ValuePtr ͵ֵתΪ GeDataTypeGraphEngine ͣ +// תķʽȡڴ AnyTraits +GeDataType ConvertAnyUtil(const ValuePtr &value, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + if (!value->isa()) { // ȷ value Type ͡ + MS_LOG(EXCEPTION) << "error convert Value to TypePtr for value: " << value->ToString() + << ", type: " << value->type_name() << ", value should be a Typeptr"; + } + auto type = value->cast(); // value תΪ TypePtr ͡ + MS_EXCEPTION_IF_NULL(type); // ȷת type Ϊָ롣 + TypeId me_type = type->type_id(); // ȡ TypePtr TypeIdMindSpore еͱʶ + // TypePtr TypeId kObjectTypeTensorTypeʾΪ TensorType ͡ + // ҪһȡԪ͵ TypeIdԱк GraphEngine ת + if (kObjectTypeTensorType == me_type) { + me_type = dyn_cast(type)->element()->type_id(); + } + return TransformUtil::ConvertDataType(me_type); // TransformUtil::ConvertDataType MindSpore תΪ GraphEngine ͡ +} + +// VectorToTensorUtil ڽһ ValuePtr ͵ֵתΪ GeTensorGraphEngine Tensor +// úֽ֧ tuple list תΪ GeTensorĿǰ֧һάݡ +GeTensor VectorToTensorUtil(const ValuePtr &value) { + // convert tuple or list to ge tensor, only supported one dim for now + // ת tuple list ge tensorĿǰ֧һά + MS_EXCEPTION_IF_NULL(value); + // ȡ tuple list еԪֵ + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + if (vec.empty()) { // tuple list Ϊգ򷵻һյ GeTensor + MS_LOG(WARNING) << "Convert a none tuple to an empty ge tensor"; + return GeTensor(GeTensorDesc(ge::Shape({0}))); + } + MS_EXCEPTION_IF_NULL(vec[0]); // ȡһԪأȷΪա + // ݵһԪصִӦת߼ + // һԪ Int32Imm ͣʾҪתΪ int32_t ͵ GeTensor + if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Int32"; + // תΪ int32_t ͵ std::vector + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + // ȡӦ GeTensorDesc Ϣ + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeInt32, kOpFormat_NCHW); + // ȡϢʧܣ׳쳣 + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + // GeTensorʹ int32_t ͵ Tensor ݡ + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(int32_t)); + // һԪ Int64Imm ͣʾҪתΪ int64_t ͵ GeTensor + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Int64"; + // תΪ int64_t ͵ std::vector + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + // ȡӦ GeTensorDesc Ϣ + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeInt64, kOpFormat_NCHW); + if (desc == nullptr) { // ȡϢʧܣ׳쳣 + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + // GeTensorʹ int64_t ͵ Tensor ݡ + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(int64_t)); + // һԪ FP32Imm ͣʾҪתΪ float ͵ GeTensor + } else if (vec[0]->isa()) { + MS_LOG(INFO) << "convert value to tensor with data type = Float32"; + // תΪ float ͵ std::vector + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + // ȡӦ GeTensorDesc Ϣ + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeFloat32, kOpFormat_NCHW); + if (desc == nullptr) { // ȡϢʧܣ׳쳣 + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + // GeTensorʹ float ͵ Tensor ݡ + return GeTensor(*desc, reinterpret_cast(data.data()), data.size() * sizeof(float)); + } else if (vec[0]->isa()) { // һԪ BoolImm ͣʾҪתΪ bool ͵ GeTensor + MS_LOG(INFO) << "convert value to tensor with data type = Bool"; + // We use uint8_t to save bool type data + // תΪ bool ͵ std::vector + // ʹ uint8_t + auto data = ConvertAnyUtil(value, AnyTraits(), AnyTraits>()); + auto desc = TransformUtil::GetGeTensorDesc({static_cast(vec.size())}, kNumberTypeBool, kOpFormat_NCHW); + if (desc == nullptr) { + MS_LOG(EXCEPTION) << "Update conversion descriptor failed!"; + } + return GeTensor(*desc, static_cast(data.data()), data.size() * sizeof(uint8_t)); + } else { + MS_LOG(EXCEPTION) << "Unsupported data type of tuple or list elements: " << vec[0]->type_name(); + } +} + +// ConvertAnyUtil ڽһ ValuePtr ͵ֵתΪ GeTensorGraphEngine Tensor +// תķʽȡڴ AnyTraits +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits) { + MS_EXCEPTION_IF_NULL(value); + if (value->isa()) { // ValuePtr Ƿ MeTensor ͣǣִ MeTensor GeTensor ת + // convert me tensor to ge tensor + // MeTensor תΪ GeTensor + return ConvertAnyUtil(value, AnyTraits()); + // ValuePtr Ƿ ValueList ValueTuple ͣǣִ List Tuple GeTensor ת + } else if (value->isa() || value->isa()) { + return VectorToTensorUtil(value); + // ValuePtr Ƿ Int32Imm ͣǣִ Int32Imm GeTensor ת + } else if (value->isa()) { + // convert scalar Int to GeTensor + // Int32 תΪ GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Int32"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT32); // GeTensorDesc Ϣ + auto v = GetValue(value); + desc.SetRealDimCnt(0); // ϢʵάΪ0 + return GeTensor(desc, reinterpret_cast(&v), sizeof(int32_t)); // GeTensorʹ int32_t ͵ Tensor ݡ + } + // ValuePtr Ƿ Int64Imm ͣǣִ Int64Imm GeTensor ת + else if (value->isa()) { + // convert scalar Int64 to GeTensor + // Int64 תΪ GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Int64"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_INT64); // GeTensorDesc Ϣ + auto v = GetValue(value); + desc.SetRealDimCnt(0); // ϢʵάΪ0 + return GeTensor(desc, reinterpret_cast(&v), sizeof(int64_t)); // GeTensorʹ int64_t ͵ Tensor ݡ + } + // ValuePtr Ƿ FP32Imm ͣǣִ FP32Imm GeTensor ת + else if (value->isa()) { + // convert scalar FP32 to GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = FP32"; // FP32 תΪ GeTensor + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_FLOAT); // GeTensorDesc Ϣ + auto v = GetValue(value); + desc.SetRealDimCnt(0); // ϢʵάΪ0 + return GeTensor(desc, reinterpret_cast(&v), sizeof(float)); // GeTensorʹ float ͵ Tensor ݡ + } + // ValuePtr Ƿ BoolImm ͣǣִ BoolImm GeTensor ת + else if (value->isa()) { + // convert scalar FP32 to GeTensor + // Bool תΪ GeTensor + MS_LOG(INFO) << "convert scalar to tensor with data type = Bool"; + GeTensorDesc desc(GeShape(), ge::FORMAT_NCHW, ge::DT_BOOL); // GeTensorDesc Ϣ + auto v = GetValue(value); + desc.SetRealDimCnt(0); // ϢʵάΪ0 + return GeTensor(desc, reinterpret_cast(&v), sizeof(bool)); // GeTensorʹ bool ͵ Tensor ݡ + } + // ValuePtr Ƿ StringImm ͣǣִ StringImm GeTensor ת + else if (value->isa()) { + // convert String to GeTensor + // String תΪ GeTensor + MS_LOG(INFO) << "convert string to tensor with data type = String"; + std::string v = GetValue(value); // ȡ string ͵ֵ + std::vector ge_shape; // GeTensorDesc Ϣ + GeShape shape(ge_shape); + GeTensorDesc desc(shape, ge::FORMAT_NCHW, ge::DT_STRING); + GeTensor str_tensor(desc); + (void)str_tensor.SetData(v); + return str_tensor; + } else { + MS_LOG(WARNING) << "Unsupported value type: " << value->type_name() + << " to convert to tensor. Value: " << value->ToString(); + } + return GeTensor(); +} + +// IsCustomPrim жϸ PrimitivePtr ǷΪԶIJCustom Primitive +bool IsCustomPrim(const PrimitivePtr &prim) { + if (prim == nullptr) { // PrimitivePtr Ϊָ룬򷵻 false + return false; + } + // Primitive лȡΪ "_custom_op_flag" ֵ + ValuePtr flag = prim->GetAttr("_custom_op_flag"); + if (flag == nullptr) { // ȡֵΪָ룬򷵻 false + return false; + } + // ֵתΪ bool ͣ洢ڱ is_custom_op С + bool is_custom_op = GetValue(flag); + // is_custom_op Ϊ falseͬʱ Primitive Ϊ "_custom_op_impl_config_path" ԣ + // ׳쳣ʾԶӦ÷ "_custom_op_impl_config_path" ԡ + if (!is_custom_op && prim->GetAttr("_custom_op_impl_config_path") != nullptr) { + MS_LOG(EXCEPTION) << "The custom op flag is false, but the op information config path is not null, non-custom op " + "can not assign the op information config path."; + } + + return is_custom_op; // is_custom_opʾ Primitive ǷΪԶIJ +} + +// IsCustomCNode жϸ AnfNodePtr ǷΪԶ CNode +// Զ CNode ָһ ValueNode ValueNode һԶ PrimitivePtr +bool IsCustomCNode(const AnfNodePtr &anf) { + if (anf == nullptr) { // AnfNodePtr Ϊָ룬򷵻 false + return false; + } + auto node = anf->cast(); // AnfNodePtr תΪ CNodePtr + if (node == nullptr) { // תʧܣ˵ AnfNodePtr CNode false + return false; + } + if (node->inputs().empty()) { // CNode ǷΪգΪգ׳쳣 + MS_LOG(EXCEPTION) << "Length of node inputs is empty"; + } + MS_EXCEPTION_IF_NULL(node->inputs()[0]); // CNode ĵһǷΪָ룬ǣ׳쳣 + // CNode ĵһǷΪ ValueNodeǣ falseʾԶ CNode + if (!node->inputs()[0]->isa()) { + return false; + } + // Խ CNode ĵһתΪ ValueNodeȡ PrimitivePtr + auto cus_prim = GetValueNode(node->inputs()[0]); + if (cus_prim == nullptr) { // ȡ PrimitivePtr Ϊָ룬 falseʾԶ CNode + return false; + } + + return IsCustomPrim(cus_prim); // IsCustomPrim жϻȡ PrimitivePtr ǷΪԶIJжϽ +} + +// GetOpIOFormat ڻȡ AnfNodePtr ӦIJʽIO Format +std::string GetOpIOFormat(const AnfNodePtr &anf) { + std::string ret; + if (anf == nullptr) { // AnfNodePtr ǷΪָ룬ǣ־ؿַ + MS_LOG(ERROR) << "The anf is nullptr"; + return ret; + } + auto node = anf->cast(); // Խ AnfNodePtr תΪ CNodePtr + if (node == nullptr) { // תʧܣ˵ AnfNodePtr CNode־ؿַ + MS_LOG(ERROR) << "The anf is not a cnode."; + return ret; + } + if (node->inputs().empty()) { // CNode ǷΪգΪգ׳쳣 + MS_LOG(EXCEPTION) << "Length of node inputs is empty."; + } + MS_EXCEPTION_IF_NULL(node->inputs()[0]); // CNode ĵһǷΪָ룬ǣ׳쳣 + if (!node->inputs()[0]->isa()) { // CNode ĵһǷΪ ValueNodeǣ־ؿַ + MS_LOG(ERROR) << "The anf is not a value node."; + return ret; + } + auto prim = GetValueNode(node->inputs()[0]); // Խ CNode ĵһתΪ ValueNodeȡ PrimitivePtr + if (prim == nullptr) { // ȡ PrimitivePtr Ϊָ룬־ؿַ + MS_LOG(ERROR) << "The anf is not a Primitive."; + return ret; + } + if (prim->HasAttr("io_format")) { // PrimitivePtr ǷΪ "io_format" ԣУ򷵻ֵΪ IO Format + return GetValue(prim->GetAttr("io_format")); + } + // PrimitivePtr ûΪ "io_format" ԣ IOFormatMap вҲӦ IO Format + auto io_format_map = IOFormatMap::get(); + auto iter = io_format_map.find(prim->name()); + if (iter == io_format_map.end()) { // IOFormatMap ûҵӦ IO FormatĬϷ "NCHW" + return "NCHW"; + } + // IO Format Ƿ "format" ͵ԣǣһ󷵻ؾĸʽֵ + if (iter->second == "format") { + ValuePtr format = prim->GetAttr("format"); + MS_EXCEPTION_IF_NULL(format); + if (format->isa()) { + bool converted = CheckAndConvertUtils::ConvertAttrValueToString(prim->name(), "format", &format); + if (converted) { + return GetValue(format); + } + } else { + return GetValue(format); + } + } + return iter->second; // "format" ͵ԣֱӷ IO Format +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From f0490dd62e60687b4cc9ad52f59e4118ab52174a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:17:37 +0800 Subject: [PATCH 40/72] ADD file via upload --- .../ccsrc/transform-update/op_adapter_util.h | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_adapter_util.h diff --git a/mindspore/ccsrc/transform-update/op_adapter_util.h b/mindspore/ccsrc/transform-update/op_adapter_util.h new file mode 100644 index 00000000000..3ac31810cba --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_adapter_util.h @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_ + +#include +#include + +#include "transform/graph_ir/op_adapter_base.h" + +namespace mindspore { +namespace transform { +template +static Q ConvertAnyUtil(const ValuePtr &value, const AnyTraits

&, const AnyTraits &) { + return static_cast(GetValue

(value)); +} + +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits &traits); + +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &name, + const AnyTraits>); + +std::string ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits); + +std::vector ConvertAnyUtil(const ValuePtr &value, const AnyTraits>, const AnyTraits); + +std::vector ConvertAnyUtil(const ValuePtr &value, const std::string &format, + const AnyTraits>, const AnyTraits); + +GeDataType ConvertAnyUtil(const ValuePtr &value, const AnyTraits); + +template +// ConvertAnyUtil ڽ ValuePtr תΪ P Ԫص std::vector +// P Q Dzͬ͡ +std::vector ConvertAnyUtil(const ValuePtr &value, AnyTraits

, const AnyTraits>) { + MS_EXCEPTION_IF_NULL(value); // ValuePtr ǷΪָ룬ǣ׳쳣 + // ValuePtr ǷΪ ValueTuple ValueListǣ׳쳣 + if (!value->isa() && !value->isa()) { + MS_LOG(EXCEPTION) << "error convert Value to vector for value: " << value->ToString() + << ", type: " << value->type_name() << ", value should be a tuple or list"; + } + // ȡ ValuePtr еݼϣ ValueTuple ValueList + auto vec = value->isa() ? value->cast()->value() : value->cast()->value(); + std::vector data; // std::vectorڴ洢תĽ + for (auto &it : vec) { // еÿԪأÿԪص ConvertAnyUtil תӵ data С + data.push_back(ConvertAnyUtil(it, AnyTraits

(), AnyTraits())); + } + return data; // ת std::vector +} + +GeTensor ConvertAnyUtil(const ValuePtr &value, const AnyTraits); + +bool IsCustomPrim(const PrimitivePtr &prim); +bool IsCustomCNode(const AnfNodePtr &node); +std::string GetOpIOFormat(const AnfNodePtr &node); +} // namespace transform +} // namespace mindspore +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_ADAPTER_UTIL_H_ -- 2.34.1 From 88f8baa48040e2e72864f86206643488e2d48ac6 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:17:53 +0800 Subject: [PATCH 41/72] ADD file via upload --- .../ccsrc/transform-update/op_declare_macro.h | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/op_declare_macro.h diff --git a/mindspore/ccsrc/transform-update/op_declare_macro.h b/mindspore/ccsrc/transform-update/op_declare_macro.h new file mode 100644 index 00000000000..1c290a2ca29 --- /dev/null +++ b/mindspore/ccsrc/transform-update/op_declare_macro.h @@ -0,0 +1,213 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_ +#define MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_ + +#include +#include +#include "utils/hash_map.h" +#include "transform/graph_ir/op_adapter.h" +#include "transform/graph_ir/op_adapter_desc.h" +#include "include/transform/graph_ir/op_adapter_map.h" +#include "mindspore/core/base/core_ops.h" + +namespace mindspore::transform { +//һ DECLARE_OP_ADAPTER(T)ӦOp Adapter +#define DECLARE_OP_ADAPTER(T) \ + using T = ge::op::T; \//һ TʾӦGEеIJge::op::T + template <> \ + const mindspore::HashMap OpAdapter::input_map_; \//һģػһ̬Ա input_map_ڴ洢 T ͵IJӳϢ + // InputDesc һԶĽṹ壬ϢƺӦĴ + template <> \ + const mindspore::HashMap OpAdapter::attr_map_; //һģػһ̬Ա attr_map_ڴ洢 T ͵IJӳϢ + //AttrDesc һԶĽṹ壬ԵϢԵƺӦĴ + +#define DECLARE_OP_USE_OUTPUT(T) \ + template <> \ + const mindspore::HashMap OpAdapter::output_map_;// OpAdapter һģػʹģ T output_map_ + // Ϊ OpAdapter һ output_map_ ģػ OutputDesc ֵ output_map_ ģڽʶӳ䵽 OutputDesc + + +#define DECLARE_OP_USE_ENUM(T) \ + template <> \ + const mindspore::HashMap OpAdapter::enum_map_{};// OpAdapter һģػʹģ Tһյ enum_map_ + // Ϊ OpAdapter һ enum_map_ ģػֵַģ T ʾ͡ + +#define DECLARE_OP_USE_INPUT_ATTR(T) \ + template <> \ + const mindspore::HashMap OpAdapter::input_attr_map_;// OpAdapter һģػʹģ T input_attr_map_ + // Ϊ OpAdapter һ input_attr_map_ ģػ޷ AttrDesc ֵ + + +#define DECLARE_OP_USE_DYN_INPUT(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_input_map_;// OpAdapter һģػʹģ T dyn_input_map_ + // Ϊ OpAdapter һ dyn_input_map_ ģػ DynInputDesc ֵ + +#define DECLARE_OP_USE_DYN_SUBGRAPH(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_subgraph_map_;// OpAdapter һģػʹģ Tdyn_subgraph_map_ +// Ϊ OpAdapter һ dyn_input_map_ ģػ DynInputDescֵ + +#define DECLARE_OP_USE_DYN_OUTPUT(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_output_map_;// OpAdapter һģػʹģ Tdyn_output_map_ +// Ϊ OpAdapter һ dyn_input_map_ ģػ DynInputDescֵ + +#define INPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::input_map_ + // EMPTY_INPUT_MAPʾһյӳ䣬ʹ mindspore::HashMap() ʼ + + // INPUT_DESC(name)Ϊһ +#define EMPTY_INPUT_MAP mindspore::HashMap() +#define INPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, const OperatorPtr input) { \// + auto p = std::static_pointer_cast(op); \ + (void)p->set_input_##name(*input); \ + }, \ + [](const OperatorPtr op, const OutHandler& handle) { \// + auto p = std::static_pointer_cast(op); \ + (void)p->set_input_##name(*(handle.op), handle.out); \ + }, \ + [](const OperatorPtr op, const GeTensorDesc desc) { \// + auto p = std::static_pointer_cast(op); \ + (void)p->update_input_desc_##name(desc); \ + } \ + }//ΪṩעͺͲ + + + // DYN_INPUT_MAP(T)Ϊ OpAdapter ̬ӳػ +#define DYN_INPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_input_map_ + + // DYN_INPUT_DESC(name)Ϊ̬һ +#define DYN_INPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \// + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_input_##name(num); \ + }, \ + [](const OperatorPtr op, unsigned int index, const OperatorPtr input) { \// + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_input_##name(index, *input); \ + }, \ + [](const OperatorPtr op, unsigned int index, const OutHandler& handle) { \// + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_input_##name(index, *(handle.op), handle.out); \ + } \ + }//ΪṩעͺͲ + +// DYN_SUBGRAPH_MAP(T)Ϊ OpAdapter ̬ͼӳػ +#define DYN_SUBGRAPH_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_subgraph_map_ + +// DYN_SUBGRAPH_DESC(name)Ϊ̬ͼһ +#define DYN_SUBGRAPH_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \//̬ͼ + auto p = std::static_pointer_cast(op); \ + (void)p->create_dynamic_subgraph_##name(num); \ + }, \ + [](const OperatorPtr op, unsigned int index, const DfGraphPtr graph) { \//ͼ + auto p = std::static_pointer_cast(op); \ + (void)p->set_dynamic_subgraph_builder_##name(index, [graph](){return *graph;}); \ + } \ + }// Ϊ̬ͼṩעͺͲ + + // ATTR_MAP(T)Ϊ OpAdapter ӳػ +#define ATTR_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::attr_map_ +#define EMPTY_ATTR_MAP mindspore::HashMap() + // EMPTY_ATTR_MAPʾһյӳ䣬ʹ mindspore::HashMap() ʼ +#define ATTR_DESC(name, ...) \ + // ATTR_DESC(name, ...)Ϊһ + { \ +#name, \ + [](const OperatorPtr op, const ValuePtr& value) { \ //ֵ + auto p = std::static_pointer_cast(op); \ + (void)p->set_attr_##name(ConvertAny(value, __VA_ARGS__)); \ + } \ + }// ΪṩעͺͲ + + // INPUT_ATTR_MAP(T)Ϊ OpAdapter ӳػ + + //INPUT_ATTR_MAP 궨һ T ģػػУһΪ input_attr_map_ ijϣӳ䣬޷ӳ䵽 AttrDesc +#define INPUT_ATTR_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::input_attr_map_ //ӳ + + //OUTPUT_MAP 궨һ T ģػػУһΪ output_map_ ijϣӳ䣬ӳ䵽 OutputDesc +#define OUTPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::output_map_ //ӳ +#define OUTPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, const GeTensorDesc desc) { \ //һ Lambda ʽ OperatorPtr GeTensorDesc + auto p = std::static_pointer_cast(op); \ // OperatorPtr תΪ OpType ָ + (void)p->update_output_desc_##name(desc); \ // OpType ijԱ update_output_desc_name name ǺչIJ + } \ + } + + //DYN_OUTPUT_MAP 궨һ T ģػػУһΪ dyn_output_map_ ijϣӳ䣬ӳ䵽 DynOutputDesc +#define DYN_OUTPUT_MAP(T) \ + template <> \ + const mindspore::HashMap OpAdapter::dyn_output_map_ //̬ӳ + +#define DYN_OUTPUT_DESC(name) \ + { \ +#name, \ + [](const OperatorPtr op, unsigned int num) { \ //һ Lambda ʽ OperatorPtr unsigned int + auto p = std::static_pointer_cast(op); \ // OperatorPtr תΪ OpType ָ + (void)p->create_dynamic_output_##name(num); \// OpType ijԱ create_dynamic_output_name name ǺչIJnum Ǵݸ unsigned int + } \ + } + +#define ADPT_DESC_ONE(T) std::make_shared(std::make_shared>()) +//ʹ std::make_shared>() һ OpAdapter ͵ָ룬Ϊݸ std::make_shared() OpAdapterDesc ͵ָ롣 +#define ADPT_DESC_TWO(T, I) \ + std::make_shared(std::make_shared>(), std::make_shared>()) +//궨巵һ std::shared_ptr +//ʹ std::make_shared>()`` std::make_shared>()ͬ͵ָ룬ȻΪݸstd::make_shared()OpAdapterDesc͵ָ롣 +//ʾģTI` +#define GET_MACRO(_1, _2, DESC, ...) DESC +//궨һ꣬ڸݲѡͬĺ궨塣ݴIJѡ ADPT_DESC_TWO ADPT_DESC_ONE +#define ADPT_DESC(...) GET_MACRO(__VA_ARGS__, ADPT_DESC_TWO, ADPT_DESC_ONE, ...)(__VA_ARGS__) +//궨ǸݴIJѡ ADPT_DESC_TWO ADPT_DESC_ONE ꡣIJԭݸ GET_MACRO ꣬Ȼݲѡȷĺꡣ +#define REG_ADPT_DESC(name, name_str, adpt_desc) \ + static struct RegAdptDesc##name { \ + public: \ + RegAdptDesc##name() { OpAdapterMap::get()[name_str] = adpt_desc; } \ + \ + private: \ + int ph_{0}; \// ph_{0} һõijԱȷṹжһ޶ʵ + } g_reg_adpt_desc_##name; + + //궨עھ̬洢һṹ RegAdptDesc##name name ǴIJ + //ȻڽṹĹ캯У adpt_desc ӵ OpAdapterMap ӳУӳļ name_str + //ڳʱԶע + +} // namespace mindspore::transform +#endif // MINDSPORE_CCSRC_TRANSFORM_GRAPH_IR_OP_DECLARE_MACRO_H_ -- 2.34.1 From 3c42e42e9b79148c89d436e023172d26045a4fd2 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:18:19 +0800 Subject: [PATCH 42/72] ADD file via upload --- .../ccsrc/transform-update/pad_ops_declare.cc | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/pad_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/pad_ops_declare.cc b/mindspore/ccsrc/transform-update/pad_ops_declare.cc new file mode 100644 index 00000000000..72846124e76 --- /dev/null +++ b/mindspore/ccsrc/transform-update/pad_ops_declare.cc @@ -0,0 +1,103 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/pad_ops_declare.h" +#include + +namespace mindspore::transform { +// PadD +INPUT_MAP(PadD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(PadD) = {{"paddings", ATTR_DESC(paddings, AnyTraits>>())}}; +// ӳ䣬"paddings"Ϊfloat,"sqrt_mode"Ϊstd::vector> +OUTPUT_MAP(PadD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(PadD, kNamePadD, ADPT_DESC(PadD)) +// עPadDkNamePadD + +// Pad +INPUT_MAP(Pad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}}; +// ӳ䣬xΪ1paddingsΪ2 +ATTR_MAP(Pad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Pad) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Pad, kNamePadV1, ADPT_DESC(Pad)) +// עPadkNamePadV1 + +// BroadcastToD +INPUT_MAP(BroadcastToD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(BroadcastToD) = {{"shape", ATTR_DESC(shape, AnyTraits(), AnyTraits>())}}; +// ӳ䣬"shape"Ϊint64_tstd::vector> +OUTPUT_MAP(BroadcastToD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(BroadcastToD, kNameBroadcastTo, ADPT_DESC(BroadcastToD)) +// עBroadcastToDkNameBroadcastTo + +// Diag +INPUT_MAP(Diag) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(Diag) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Diag) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(Diag, kNameDiag, ADPT_DESC(Diag)) +// עDiagkNameDiag + +// FillD +INPUT_MAP(FillD) = {{1, INPUT_DESC(value)}}; +// ӳ䣬valueΪ1 +ATTR_MAP(FillD) = {{"dims", ATTR_DESC(dims, AnyTraits>())}}; +// ӳ䣬"dims"std::vector +OUTPUT_MAP(FillD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(FillD, kNameFillD, ADPT_DESC(FillD)) +// עFillDkNameFillD + +// Fill +INPUT_MAP(Fill) = {{1, INPUT_DESC(dims)}, {2, INPUT_DESC(value)}}; +// ӳ䣬dimsΪ1valueΪ2 +ATTR_MAP(Fill) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(Fill) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(Fill, kNameFillV1, ADPT_DESC(Fill)) +// עFillkNameFillV1 + + +// PadV3 +INPUT_MAP(PadV3) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}, {3, INPUT_DESC(constant_values)}}; +// ӳ䣬xΪ1paddingsΪ2constant_valuesΪ3 +ATTR_MAP(PadV3) = {{"mode", ATTR_DESC(mode, AnyTraits())}, + {"pad_contiguous", ATTR_DESC(paddings_contiguous, AnyTraits())}}; +// ӳ䣬"dims""pad_contiguous"ͷֱstd::stringbool +OUTPUT_MAP(PadV3) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(PadV3, kNamePadV3, ADPT_DESC(PadV3)) +// עPadV3kNamePadV3 + + +// PadV2 +INPUT_MAP(PadV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(paddings)}, {3, INPUT_DESC(constant_values)}}; +// ӳ䣬xΪ1paddingsΪ2constant_valuesΪ3 +ATTR_MAP(PadV2) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(PadV2) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(PadV2, kNamePadV2, ADPT_DESC(PadV2)) +// עPadV2kNamePadV2 +} // namespace mindspore::transform -- 2.34.1 From 54b288a0f5311081e6cb670d7d4db3a486c85b8f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:18:34 +0800 Subject: [PATCH 43/72] ADD file via upload --- .../transform-update/python_pass_register.py | 265 ++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/python_pass_register.py diff --git a/mindspore/ccsrc/transform-update/python_pass_register.py b/mindspore/ccsrc/transform-update/python_pass_register.py new file mode 100644 index 00000000000..9bc19273822 --- /dev/null +++ b/mindspore/ccsrc/transform-update/python_pass_register.py @@ -0,0 +1,265 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Python pass register""" +from inspect import isfunction # ǷΪ +from mindspore.graph_utils.graph_pattern import Pattern, NewParameter # ͼģʽƥ +from mindspore._c_expression import PyPassManager_ # ڹŻݵ + +# __all__ бָʹ "from import *" ʱķ +# űжĺƣԱⲿ + +__all__ = [ + "register_pass", # עµŻݵݹеĺ + "unregister_pass", # ӴݹעŻݵĺ + "gen_new_parameter", # ͼת²ĺ + "cancel_new_parameter", # ȡ²ĺ + "set_renorm", # ͼת¹һ־ĺ + "set_reopt" # ͼתŻ־ĺ +] + +# PyPassManager̳PyPassManager_עעPythonŻݣԱڱڼͼ޸ġ +class PyPassManager(PyPassManager_): + r""" + Used to register and unregister python passes which can be used to alter graphs. + + Args: + requires_grad(bool): Do automatic-differentiation after modified graph if true. Default: True + run_only_once (bool): Specify whether or not to run pass only once. Default: False. + + Raises: + TypeError: If argument has invalid type. + """ + #һPyPassManager󣬲ָIJrequires_gradrun_only_onceгʼ + #PyPassManagerڹŻݵ࣬עעPythonŻݺԱڱڼԼͼ޸ġ + #ͨPyPassManager󣬿ԽPythonŻݺעᵽضı׶ΣָǷԶ΢ֺǷֻһΡ + def __init__(self, requires_grad=True, run_only_once=False): + # IJrun_only_onceǷΪboolͣ׳TypeError쳣 + if not isinstance(requires_grad, bool): + raise TypeError(f"Expect bool, got : ({type(requires_grad)}){requires_grad}") + # IJrequires_gradǷΪboolͣ׳TypeError쳣 + if not isinstance(run_only_once, bool): + raise TypeError(f"Expect bool, got : ({type(run_only_once)}){run_only_once}") + # IJrun_only_onceǷΪboolͣ׳TypeError쳣 + self.requires_grad = requires_grad + self.run_only_once_ = run_only_once + # IJrequires_gradrun_only_once浽ʵ + PyPassManager_.__init__(self) + # øPyPassManager_Ĺ캯ʼPyPassManager_ + + # registerǽPythonŻݺעᵽŻݹУԱڱڼԼͼ޸ĺŻ + + # һPyPassManager󣬿ʹøöregisterдPythonŻݺעᵽݹС + # ע󣬸Żݺڱڼض׶αԶãԼͼ޸ġ + def register(self, py_pass): + if not isfunction(py_pass): + raise TypeError(f"Expect function pass, got : ({type(py_pass)}){py_pass}") + # py_passǷΪͣǣ׳TypeError쳣 + pattern, target = py_pass() + # py_passȡ䷵صͼģʽƥģʽĿ + pass_name = py_pass.__name__ + # ȡpy_passΪŻݵ + if not isinstance(pattern, Pattern): + raise TypeError(f"Expect pattern of Pattern type, got : ({type(pattern)}){pattern}") + # patternǷΪPatternͣǣ׳TypeError쳣 + if not isinstance(target, Pattern): + raise TypeError(f"Expect target of Pattern type, got : ({type(target)}){target}") + # targetǷΪPatternͣǣ׳TypeError쳣 + super().register(pass_name, pattern, target, self.requires_grad, self.run_only_once_) + # øPyPassManager_registerŻݺעᵽݹ + # ŻơͼģʽƥģʽĿꡢǷԶ΢ֺǷֻһεı־ + + #unregisterڴӴݹעעŻݺŻݺעͨregisterɵġ + def unregister(self, py_pass): + #py_passַͣ˵ҪעһעŻݺ + if isinstance(py_pass, str): + super().unregister(py_pass) + return + # øPyPassManager_unregisterŻƣӴݹעŻݺ + + # py_passǺͣ˵ҪעһעŻݺ + if isfunction(py_pass): + super().unregister(py_pass.__name__) + return + # øPyPassManager_unregisterŻݺƣӴݹעŻݺ + + raise TypeError(f"Expect py_pass to be string or function, got ({type(py_pass)}){py_pass}") + # py_passȲַҲǺ׳TypeError쳣 + + #__call__PythonбΪ""ʹøʵһá + #PyPassManagerУ__call__ڽPythonŻݺpy_passעᵽݹУڳɹע󷵻ظŻݺ + def __call__(self, py_pass): + self.register(py_pass) + # registerpy_passעᵽݹ + return py_pass + # py_pass + + + #ͼת² + #ȣ鴫 pattern Ƿ NewParameter ͵ʵǣͻ׳쳣 + #Ȼø gen_new_parameter ²ɹ̡ + def gen_new_parameter(self, pattern): + if not isinstance(pattern, NewParameter): + raise TypeError(f"Expect pattern to be a NewParameter Pattern, got {pattern}") + # patternǷΪNewParameterͣǣ׳TypeError쳣 + super().gen_new_parameter(pattern) + # øPyPassManager_gen_new_parameterͼת² + + #һ should_renormָʾǷ¹淶 + #ȣ鴫 should_renorm ǷDzͣǣͻ׳쳣 + #Ȼø set_renorm should_renorm ݸ¹淶õIJ + def set_renorm(self, should_renorm): + if not isinstance(should_renorm, bool): + raise TypeError(f"Expect should_renorm to be a bool, got {should_renorm}") + # should_renormǷΪboolͣǣ׳TypeError쳣 + super().set_renorm(should_renorm) + # øset_renormshould_renormݸ + + #һ do_reoptָʾǷŻ + #ȣ鴫 do_reopt ǷDzͣǣͻ׳쳣 + #Ȼø set_reopt do_reopt ݸŻõIJ + + def set_reopt(self, do_reopt): + if not isinstance(do_reopt, bool): + raise TypeError(f"Expect do_reopt to be a bool, got {do_reopt}") + # do_reoptǷΪboolͣǣ׳TypeError쳣 + super().set_reopt(do_reopt) + # øset_reoptdo_reoptݸ + +def register_pass(requires_grad=True, run_only_once=False): + """ + Register python pass to specified pipeline phase which would be used in compilation. + pythonעᵽָĹܵ׶Σý׶νڱʹá + + Args: : + requires_grad(bool): Do automatic-differentiation after modified graph if true. Default: True. + ޸ĺͼΪtrueִԶ΢֡Ĭֵ:True + run_only_once(bool): Run this pass only once if set true. Otherwise run the pass until converge. Default: + False. + ΪtrueֻһΡֱͨĬֵ:False + + Returns: + This function should be used as a decorator, return the decoratorated pass function. + ӦñװؾװεĴݺ + + Examples: + >>> from mindspore.graph_utils.graph_pattern import Call, Any + >>> from mindspore.ops import operations as P + >>> @register_pass() + >>> def toy_pass(): + >>> x = Any() + >>> pattern = Call(P.Softmax(), [x]) + >>> target = Call(P.ReLU(), [x]) + >>> return pattern, target + """ + return PyPassManager(requires_grad, run_only_once) + + +def unregister_pass(py_pass): + """ + Unregister python pass. + עpython· + + Args: + py_pass(Union(str, function)): target python pass to unregister. + עָpython· + """ + ppm = PyPassManager() + ppm.unregister(py_pass) + + +def gen_new_parameter(pattern): + """ + Generate specified parameter every time a network gets compiled. + ÿαʱָIJ + + NOTE: + In this way, every pass uses this pattern would be using the same Parameter. If use NewParameter without + gen_new_parameter, every pass match would build a new Parameter. + This would register a pass to add new parameter in the compilation pipeline, so later compilation would + ALSO add this parameter unless the pass is unregistered. To unregister this pass, call + cancel_new_parameter(pattern) + ÿʹôģʽpassʹͬParameterʹgen_new_parameter֮NewParameter + ÿͨƥ䶼ṹһµParameter + ⽫עһڱܵ²passԺı뽫passδעᣬҲӴ˲ + Ҫעpasscancel_new_parameter(ģʽ) + + Args: + pattern (NewParameter): NewParameter type, could be used to build nested patterns across multiple passes + after gen_new_parameter. + NewParameterͣgen_new_parameter֮passǶģʽ + + Raises: + TypeError: If argument has invalid type. + Ч + + Examples: + >>> from mindspore.graph_utils.graph_pattern import NewParameter + >>> abc = NewParameter("abc") + >>> gen_new_parameter(abc) + """ + ppm = PyPassManager() + ppm.gen_new_parameter(pattern) + + +def cancel_new_parameter(pattern): + """ + Use with gen_new_parameter to unregister gen_new_parameter pass. + + Args: + pattern (NewParameter): NewParameter type, cancel the pass which would add new parameter as this pattern + describes. + NewParameterͣȡ²Ĵݣģʽġ + Examples: + >>> from mindspore.graph_utils.graph_pattern import NewParameter + >>> abc = NewParameter("abc") + >>> gen_new_parameter(abs) + >>> # some compilations + >>> cancel_new_parameter(abc) + """ + if not isinstance(pattern, NewParameter): + raise TypeError(f"Expect pattern to be a NewParameter Pattern, got {pattern}") + ppm = PyPassManager() + ppm.unregister(pattern.para_name) + + +def set_renorm(should_renorm): + """ + Set whether or not to do renormalization after modified graph in python pass(es). + + Args: + should_renorm(bool): whether or not to do renormalization after modified graph in python pass(es). + + NOTE: + This interface is mainly intended for testing modifying graph without worrying about its validity. Turn off + renormalization may BREAK the network. + """ + ppm = PyPassManager() + ppm.set_renorm(should_renorm) + + +def set_reopt(do_reopt): + """ + Set whether or not to do optimization after modified graph in python pass(es). + + Args: + do_reopt(bool): whether or not to do optimization after modified graph in python pass(es). + pythonpass޸ͼκǷ¹淶 + NOTE: + This interface is mainly intended for testing modifying graph without worrying about its validity. Turn off + renormalization may BREAK the network. + ýӿҪڲ޸ͼõ޸ͼЧԡرܻƻ硣 + """ + ppm = PyPassManager() + ppm.set_reopt(do_reopt) -- 2.34.1 From ce3dc76a69bb378b74880cf51487c57a9a896407 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:18:52 +0800 Subject: [PATCH 44/72] ADD file via upload --- .../transform-update/quantize_ops_declare.cc | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/quantize_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/quantize_ops_declare.cc b/mindspore/ccsrc/transform-update/quantize_ops_declare.cc new file mode 100644 index 00000000000..8023ebde77d --- /dev/null +++ b/mindspore/ccsrc/transform-update/quantize_ops_declare.cc @@ -0,0 +1,44 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/quantize_ops_declare.h" + +namespace mindspore::transform { +// AscendQuant +INPUT_MAP(AscendQuant) = {{1, INPUT_DESC(x)}}; +//ӳ䣬xΪ1 +ATTR_MAP(AscendQuant) = {{"scale", ATTR_DESC(scale, AnyTraits())}, + {"offset", ATTR_DESC(offset, AnyTraits())}, + {"sqrt_mode", ATTR_DESC(sqrt_mode, AnyTraits())}, + {"round_mode", ATTR_DESC(round_mode, AnyTraits())}}; +// ӳ䣬гĸԣ"scale""offset"Ϊfloat,"sqrt_mode"Ϊbool,"round_mode"Ϊstd::string +OUTPUT_MAP(AscendQuant) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(AscendQuant, kNameAscendQuant, ADPT_DESC(AscendQuant)) +// עAscendQuantkNameAscendQuant + +// AscendDequant +INPUT_MAP(AscendDequant) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(deq_scale)}}; +// ӳ䣬xΪ1deq_scaleΪ2 +ATTR_MAP(AscendDequant) = {{"sqrt_mode", ATTR_DESC(sqrt_mode, AnyTraits())}, + {"relu_flag", ATTR_DESC(relu_flag, AnyTraits())}, + {"dtype", ATTR_DESC(dtype, AnyTraits())}}; +// ӳ䣬гĸԣ"sqrt_mode""relu_flag"Ϊbool,"dtype"ΪGEType +OUTPUT_MAP(AscendDequant) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(AscendDequant, kNameAscendDequant, ADPT_DESC(AscendDequant)) +//עAscendDequantkNameAscendDequant +} // namespace mindspore::transform -- 2.34.1 From 1b84efbfa2a62973fb86aa836bbf8844834287b9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:19:10 +0800 Subject: [PATCH 45/72] ADD file via upload --- .../transform-update/random_ops_declare.cc | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/random_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/random_ops_declare.cc b/mindspore/ccsrc/transform-update/random_ops_declare.cc new file mode 100644 index 00000000000..959841812d1 --- /dev/null +++ b/mindspore/ccsrc/transform-update/random_ops_declare.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/random_ops_declare.h" + +namespace mindspore::transform { +// DropOutGenMask +INPUT_MAP(DropOutGenMask) = {{1, INPUT_DESC(shape)}, {2, INPUT_DESC(prob)}}; +// ӳ䣬shapeΪ1probΪ2 +ATTR_MAP(DropOutGenMask) = {{"Seed0", ATTR_DESC(seed, AnyTraits())}, + {"Seed1", ATTR_DESC(seed2, AnyTraits())}}; +//ӳ䣬гԣ"Seed0""Seed1"Ϊint64_t +OUTPUT_MAP(DropOutGenMask) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(DropOutGenMask, prim::kPrimDropoutGenMask->name(), ADPT_DESC(DropOutGenMask)) +// עDropOutGenMaskprim::kPrimDropoutGenMask->name() +// +// LinSpace +INPUT_MAP(LinSpace) = {{1, INPUT_DESC(start)}, {2, INPUT_DESC(stop)}, {3, INPUT_DESC(num)}}; +// ӳ䣬startΪ1stopΪ2numΪ3 +ATTR_MAP(LinSpace) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(LinSpace) = {{0, OUTPUT_DESC(output)}}; +//ӳ䣬outputΪ +REG_ADPT_DESC(LinSpace, kNameLinSpace, ADPT_DESC(LinSpace)) +// עLinSpacekNameLinSpace + +// RandomChoiceWithMask +INPUT_MAP(RandomChoiceWithMask) = {{1, INPUT_DESC(x)}}; +//ӳ䣬xΪ1 +ATTR_MAP(RandomChoiceWithMask) = {{"count", ATTR_DESC(count, AnyTraits())}, + {"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}}; +// ӳ䣬гԣ"count""seed""seed2"Ϊint64_t +OUTPUT_MAP(RandomChoiceWithMask) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(mask)}}; +// ӳ䣬yΪ0maskΪ1 +REG_ADPT_DESC(RandomChoiceWithMask, kNameRandomChoiceWithMask, ADPT_DESC(RandomChoiceWithMask)) +// עRandomChoiceWithMaskkNameRandomChoiceWithMask + +// TruncatedNormal +INPUT_MAP(TruncatedNormal) = {{1, INPUT_DESC(shape)}}; +// ӳ䣬shapeΪ1 +ATTR_MAP(TruncatedNormal) = {{"seed", ATTR_DESC(seed, AnyTraits())}, + {"seed2", ATTR_DESC(seed2, AnyTraits())}}; +// ӳ䣬гԣ"seed""seed2"Ϊint64_t +OUTPUT_MAP(TruncatedNormal) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(TruncatedNormal, kNameTruncatedNormal, ADPT_DESC(TruncatedNormal)) +// עTruncatedNormalkNameTruncatedNormal +} // namespace mindspore::transform -- 2.34.1 From 6c819f6c8d6af53193a258349373ccbfd16425bd Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:19:31 +0800 Subject: [PATCH 46/72] ADD file via upload --- .../transform-update/reduce_ops_declare.cc | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/reduce_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/reduce_ops_declare.cc b/mindspore/ccsrc/transform-update/reduce_ops_declare.cc new file mode 100644 index 00000000000..a54607887bc --- /dev/null +++ b/mindspore/ccsrc/transform-update/reduce_ops_declare.cc @@ -0,0 +1,161 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/reduce_ops_declare.h" +#include + +namespace mindspore::transform { +// BNTrainingReduce +INPUT_MAP(BNTrainingReduce) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(BNTrainingReduce) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(BNTrainingReduce) = {{0, OUTPUT_DESC(sum)}, {1, OUTPUT_DESC(square_sum)}}; +// ӳ䣬sumΪ0square_sumΪ1 +REG_ADPT_DESC(BNTrainingReduce, kNameBNTrainingReduce, ADPT_DESC(BNTrainingReduce)) +//עBNTrainingReducekNameBNTrainingReduce + +// BNTrainingReduceGrad +INPUT_MAP(BNTrainingReduceGrad) = {{1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(diff_scale)}, + {4, INPUT_DESC(diff_offset)}, {5, INPUT_DESC(scale)}, {6, INPUT_DESC(batch_mean)}, + {7, INPUT_DESC(batch_variance)}}; +//ӳ䣬߸gradΪ1xΪ2diff_scaleΪ3diff_offsetΪ4scaleΪ5batch_meanΪ6batch_varianceΪ7 +ATTR_MAP(BNTrainingReduceGrad) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}}; +// ӳ䣬г"epsilon"Ϊfloat +OUTPUT_MAP(BNTrainingReduceGrad) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(BNTrainingReduceGrad, kNameBNTrainingReduceGrad, ADPT_DESC(BNTrainingReduceGrad)) +// עBNTrainingReduceGradkNameBNTrainingReduceGrad + +// BNTrainingUpdate +INPUT_MAP(BNTrainingUpdate) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(sum)}, {3, INPUT_DESC(square_sum)}, + {4, INPUT_DESC(scale)}, {5, INPUT_DESC(offset)}, {6, INPUT_DESC(mean)}, + {7, INPUT_DESC(variance)}}; +// ӳ䣬߸gradΪ1xΪ2diff_scaleΪ3diff_offsetΪ4scaleΪ5batch_meanΪ6batch_varianceΪ7 +ATTR_MAP(BNTrainingUpdate) = {{"factor", ATTR_DESC(factor, AnyTraits())}, + {"epsilon", ATTR_DESC(epsilon, AnyTraits())}}; +// ӳ䣬г"factor""epsilon"Ϊfloat +OUTPUT_MAP(BNTrainingUpdate) = {{0, OUTPUT_DESC(y)}, + {1, OUTPUT_DESC(mean)}, + {2, OUTPUT_DESC(variance)}, + {3, OUTPUT_DESC(batch_mean)}, + {4, OUTPUT_DESC(batch_variance)}}; +// ӳ䣬yΪ0meanΪ1varianceΪ2batch_meanΪ3batch_varianceΪ4 +REG_ADPT_DESC(BNTrainingUpdate, kNameBNTrainingUpdate, ADPT_DESC(BNTrainingUpdate)) +// עBNTrainingUpdatekNameBNTrainingUpdate + +// BNTrainingUpdateGrad +INPUT_MAP(BNTrainingUpdateGrad) = { + {1, INPUT_DESC(grads)}, {2, INPUT_DESC(x)}, {3, INPUT_DESC(batch_mean)}, {4, INPUT_DESC(batch_variance)}}; +// ӳ䣬gradsΪ1xΪ2batch_meanΪ3batch_varianceΪ4 +ATTR_MAP(BNTrainingUpdateGrad) = {{"epsilon", ATTR_DESC(epsilon, AnyTraits())}}; +// ӳ䣬"epsilon"Ϊfloat +OUTPUT_MAP(BNTrainingUpdateGrad) = {{0, OUTPUT_DESC(diff_scale)}, {1, OUTPUT_DESC(diff_offset)}}; +// ӳ䣬diff_scaleΪ0diff_offsetΪ1 +REG_ADPT_DESC(BNTrainingUpdateGrad, kNameBNTrainingUpdateGrad, ADPT_DESC(BNTrainingUpdateGrad)) +// עBNTrainingUpdateGradkNameBNTrainingUpdateGrad + +// ReduceAnyD +INPUT_MAP(ReduceAnyD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceAnyD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceAnyD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceAnyD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(ReduceAnyD, kNameReduceAnyD, ADPT_DESC(ReduceAnyD)) +// עReduceAnyD kNameReduceAnyD + +// ReduceSumD +INPUT_MAP(ReduceSumD) = {{1, INPUT_DESC(x)}}; +//ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceSumD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceSumD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceSumD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(ReduceSumD, prim::kPrimReduceSum->name(), ADPT_DESC(ReduceSumD)) +// עReduceSumD prim::kPrimReduceSum->name() + +// ReduceProdD +INPUT_MAP(ReduceProdD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceProdD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceProdD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceProdD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(ReduceProdD, kNameReduceProd, ADPT_DESC(ReduceProdD)) +// עReduceProdDkNameReduceProd + + +// ReduceAllD +INPUT_MAP(ReduceAllD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceAllD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceAllD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceAllD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(ReduceAllD, prim::kPrimReduceAll->name(), ADPT_DESC(ReduceAllD)) +// עReduceAllDprim::kPrimReduceAll->name() + +// ReduceMeanD +INPUT_MAP(ReduceMeanD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceMeanD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceMeanD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceMeanD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(ReduceMeanD, prim::kPrimReduceMean->name(), ADPT_DESC(ReduceMeanD)) +// עReduceMeanDprim::kPrimReduceAll->name() +// +// ReduceMinD +INPUT_MAP(ReduceMinD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceMinD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceMinD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceMinD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(ReduceMinD, prim::kPrimReduceMin->name(), ADPT_DESC(ReduceMinD)) +// עReduceMinDprim::kPrimReduceAll->name() + +// ReduceMaxD +INPUT_MAP(ReduceMaxD) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +INPUT_ATTR_MAP(ReduceMaxD) = { + {2, ATTR_DESC(axes, AnyTraits>(), AnyTraits>())}}; +// ӳ䣬axesΪ2Ϊint64_t +ATTR_MAP(ReduceMaxD) = {{"keep_dims", ATTR_DESC(keep_dims, AnyTraits())}}; +// ӳ䣬"keep_dims"Ϊbool +OUTPUT_MAP(ReduceMaxD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(ReduceMaxD, prim::kPrimReduceMax->name(), ADPT_DESC(ReduceMaxD)) +// עReduceMaxDprim::kPrimReduceAll->name() +} // namespace mindspore::transform -- 2.34.1 From 5ee8a81a100ef9cd614a2263f12dd82722b3419f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:19:48 +0800 Subject: [PATCH 47/72] ADD file via upload --- .../ccsrc/transform-update/rnn_declare.cc | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/rnn_declare.cc diff --git a/mindspore/ccsrc/transform-update/rnn_declare.cc b/mindspore/ccsrc/transform-update/rnn_declare.cc new file mode 100644 index 00000000000..fcb4e37e45d --- /dev/null +++ b/mindspore/ccsrc/transform-update/rnn_declare.cc @@ -0,0 +1,192 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/rnn_declare.h" + +namespace mindspore::transform { +// BasicLSTMCell +INPUT_MAP(BasicLSTMCell) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(h)}, {3, INPUT_DESC(c)}, {4, INPUT_DESC(w)}, {5, INPUT_DESC(b)}}; +// ӳ䣬xΪ1hΪ2cΪ3wΪ4bΪ5 +ATTR_MAP(BasicLSTMCell) = {{"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}, + {"state_is_tuple", ATTR_DESC(state_is_tuple, AnyTraits())}, + {"activation", ATTR_DESC(activation, AnyTraits())}}; +// ӳ䣬г"keep_prob""forget_bias""state_is_tuple""activation"ĸԣͷֱΪboolstd::string +OUTPUT_MAP(BasicLSTMCell) = {{0, OUTPUT_DESC(ct)}, {1, OUTPUT_DESC(ht)}, {2, OUTPUT_DESC(it)}, {3, OUTPUT_DESC(jt)}, + {4, OUTPUT_DESC(ft)}, {5, OUTPUT_DESC(ot)}, {6, OUTPUT_DESC(tanhct)}} +// ӳ䣬߸ctyΪ0htΪ1itΪ2jtΪ3ftΪ4otΪ5tanhctΪ6 +REG_ADPT_DESC(BasicLSTMCell, kNameBasicLSTMCell, ADPT_DESC(BasicLSTMCell)) +// עBasicLSTMCellkNameBasicLSTMCell + +// BasicLSTMCellInputGrad +INPUT_MAP(BasicLSTMCellInputGrad) = {{1, INPUT_DESC(dgate)}, {2, INPUT_DESC(w)}}; +// ӳ䣬dgateΪ1wΪ2 +ATTR_MAP(BasicLSTMCellInputGrad) = {{"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}}; +// ӳ䣬г"keep_prob"ԣΪfloat +OUTPUT_MAP(BasicLSTMCellInputGrad) = {{0, OUTPUT_DESC(dxt)}, {1, OUTPUT_DESC(dht)}}; +// ӳ䣬dxtΪ0dhtΪ1 +REG_ADPT_DESC(BasicLSTMCellInputGrad, kNameBasicLSTMCellInputGrad, ADPT_DESC(BasicLSTMCellInputGrad)) +// עBasicLSTMCellInputGradkNameBasicLSTMCellInputGrad + + +// BasicLSTMCellWeightGrad +INPUT_MAP(BasicLSTMCellWeightGrad) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(h)}, {3, INPUT_DESC(dgate)}}; +// ӳ䣬dgateΪ1wΪ2dgateΪ3 +ATTR_MAP(BasicLSTMCellWeightGrad) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(BasicLSTMCellWeightGrad) = {{0, OUTPUT_DESC(dw)}, {1, OUTPUT_DESC(db)}}; +// ӳ䣬dwΪ0dbΪ1 +REG_ADPT_DESC(BasicLSTMCellWeightGrad, kNameBasicLSTMCellWeightGrad, ADPT_DESC(BasicLSTMCellWeightGrad)) +// עBasicLSTMCellWeightGradkNameBasicLSTMCellWeightGrad + +// BasicLSTMCellCStateGrad +INPUT_MAP(BasicLSTMCellCStateGrad) = {{1, INPUT_DESC(c)}, {2, INPUT_DESC(dht)}, {3, INPUT_DESC(dct)}, + {4, INPUT_DESC(it)}, {5, INPUT_DESC(jt)}, {6, INPUT_DESC(ft)}, + {7, INPUT_DESC(ot)}, {8, INPUT_DESC(tanhct)}}; +// ӳ䣬˸cΪ1dhtΪ2dctΪ3itΪ4jtΪ5ftΪ6otΪ7tanhctΪ8 +ATTR_MAP(BasicLSTMCellCStateGrad) = {{"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}, + {"activation", ATTR_DESC(activation, AnyTraits())}}; +// ӳ䣬г"forget_bias""activation"ԣͷֱΪfloatstd::string +OUTPUT_MAP(BasicLSTMCellCStateGrad) = {{0, OUTPUT_DESC(dgate)}, {1, OUTPUT_DESC(dct_1)}}; +// ӳ䣬dgateΪ0dct_1Ϊ1 +REG_ADPT_DESC(BasicLSTMCellCStateGrad, kNameBasicLSTMCellCStateGrad, ADPT_DESC(BasicLSTMCellCStateGrad)) +// עBasicLSTMCellCStateGradkNameBasicLSTMCellCStateGrad + + +// LSTMInputGrad +INPUT_MAP(LSTMInputGrad) = {{1, INPUT_DESC(w)}, {2, INPUT_DESC(init_c)}, {3, INPUT_DESC(c)}, {4, INPUT_DESC(dy)}, + {5, INPUT_DESC(dh)}, {6, INPUT_DESC(dc)}, {7, INPUT_DESC(i)}, {8, INPUT_DESC(j)}, + {9, INPUT_DESC(f)}, {10, INPUT_DESC(o)}, {11, INPUT_DESC(tanhct)}}; +//ӳ䣬ʮһwΪ1init_cΪ2diΪ3dyΪ4dhΪ5dcΪ6iΪ7jΪ8fΪ9oΪ10tanhctΪ11 +ATTR_MAP(LSTMInputGrad) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(LSTMInputGrad) = { + {0, OUTPUT_DESC(dx)}, {1, OUTPUT_DESC(dh_prev)}, {2, OUTPUT_DESC(dc_prev)}, {4, OUTPUT_DESC(dgate)}}; +// ӳ䣬ĸdh_prevΪ1dc_prevΪ2dgateΪ4 +REG_ADPT_DESC(LSTMInputGrad, kNameLSTMInputGrad, ADPT_DESC(LSTMInputGrad)) +// עLSTMInputGradkNameLSTMInputGrad + +// DynamicRNN +INPUT_MAP(DynamicRNN) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, + {4, INPUT_DESC(seq_length)}, {5, INPUT_DESC(init_h)}, {6, INPUT_DESC(init_c)}, + {7, INPUT_DESC(wci)}, {8, INPUT_DESC(wcf)}, {9, INPUT_DESC(wco)}, + {10, INPUT_DESC(mask)}}; +// ӳ䣬ʮxΪ1wΪ2bΪ3seq_lengthΪ4init_hΪ5init_cΪ6wciΪ7wcfΪ8wcoΪ9maskΪ10 +ATTR_MAP(DynamicRNN) = {{"cell_type", ATTR_DESC(cell_type, AnyTraits())}, + {"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"use_peephole", ATTR_DESC(use_peephole, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"ivation", ATTR_DESC(activation, AnyTraits())}, + {"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +// ӳ䣬г"cell_type""ivation""direction"ԣΪstd::string +//"cell_depth""num_proj",Ϊint64_t +//"use_peephole""is_training""time_major"Ϊbool +//"keep_prob""cell_clip""forget_bias"Ϊfloat +OUTPUT_MAP(DynamicRNN) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(output_h)}, {2, OUTPUT_DESC(output_c)}, + {3, OUTPUT_DESC(i)}, {4, OUTPUT_DESC(j)}, {5, OUTPUT_DESC(f)}, + {6, OUTPUT_DESC(o)}, {7, OUTPUT_DESC(tanhc)}}; +// ӳ䣬ʮxΪ1wΪ2bΪ3seq_lengthΪ4init_hΪ5init_cΪ6wciΪ7wcfΪ8wcoΪ9maskΪ10 +REG_ADPT_DESC(DynamicRNN, kNameDynamicRNN, ADPT_DESC(DynamicRNN)) +// עDynamicRNNkNameDynamicRNN + +// DynamicRNNGrad +INPUT_MAP(DynamicRNNGrad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(w)}, {3, INPUT_DESC(b)}, {4, INPUT_DESC(y)}, + {5, INPUT_DESC(init_h)}, {6, INPUT_DESC(init_c)}, {7, INPUT_DESC(h)}, {8, INPUT_DESC(c)}, + {9, INPUT_DESC(dy)}, {10, INPUT_DESC(dh)}, {11, INPUT_DESC(dc)}, {12, INPUT_DESC(i)}, + {13, INPUT_DESC(j)}, {14, INPUT_DESC(f)}, {15, INPUT_DESC(o)}, {16, INPUT_DESC(tanhct)}}; +// ӳ䣬ʮxΪ1wΪ2bΪ3yΪ4init_hΪ5init_cΪ6hΪ7cΪ8dyΪ9dhΪ10,dcΪ11iΪ12 +//jΪ13fΪ14oΪ15tanhctΪ16 +ATTR_MAP(DynamicRNNGrad) = {{"cell_type", ATTR_DESC(cell_type, AnyTraits())}, + {"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"use_peephole", ATTR_DESC(use_peephole, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"forget_bias", ATTR_DESC(forget_bias, AnyTraits())}}; +// ӳ䣬г"cell_type""direction"ԣΪstd::string +//"cell_depth""num_proj",Ϊint64_t +//"use_peephole""is_training""time_major"Ϊbool +//"keep_prob""cell_clip""forget_bias"Ϊfloat +OUTPUT_MAP(DynamicRNNGrad) = {{0, OUTPUT_DESC(dw)}, + {1, OUTPUT_DESC(db)}, + {2, OUTPUT_DESC(dx)}, + {3, OUTPUT_DESC(dh_prev)}, + {4, OUTPUT_DESC(dc_prev)}}; +// ӳ䣬dwΪ0dbΪ1dh_prevΪ2dh_prevΪ3dc_prevΪ4 +REG_ADPT_DESC(DynamicRNNGrad, kNameDynamicRNNGrad, ADPT_DESC(DynamicRNNGrad)) +// עDynamicRNNGradkNameDynamicRNNGrad + +// DynamicGRUV2 +INPUT_MAP(DynamicGRUV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(weight_input)}, {3, INPUT_DESC(weight_hidden)}, + {4, INPUT_DESC(bias_input)}, {5, INPUT_DESC(bias_hidden)}, {6, INPUT_DESC(seq_length)}, + {7, INPUT_DESC(init_h)}}; +// ӳ䣬߸xΪ1weight_inputΪ2weight_hiddenΪ3bias_inputΪ4bias_hiddenΪ5seq_lengthΪ6init_hΪ7 +ATTR_MAP(DynamicGRUV2) = {{"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"activation", ATTR_DESC(activation, AnyTraits())}, + {"gate_order", ATTR_DESC(gate_order, AnyTraits())}, + {"reset_after", ATTR_DESC(reset_after, AnyTraits())}, + {"is_training", ATTR_DESC(is_training, AnyTraits())}}; +// ӳ䣬г"direction""activation""gate_order"ԣΪstd::string +//"cell_depth""num_proj",Ϊint64_t +//"reset_after""is_training""time_major"Ϊbool +//"keep_prob""cell_clip""forget_bias"Ϊfloat +OUTPUT_MAP(DynamicGRUV2) = {{0, OUTPUT_DESC(y)}, {1, OUTPUT_DESC(output_h)}, {2, OUTPUT_DESC(update)}, + {3, OUTPUT_DESC(reset)}, {4, OUTPUT_DESC(new)}, {5, OUTPUT_DESC(hidden_new)}}; +// ӳ䣬yΪ0output_hΪ1updateΪ2resetΪ3newΪ4hidden_newΪ5 +REG_ADPT_DESC(DynamicGRUV2, kNameDynamicGRUV2, ADPT_DESC(DynamicGRUV2)) +// עDynamicGRUV2kNameDynamicGRUV2 +// +// DynamicGRUV2Grad +INPUT_MAP(DynamicGRUV2Grad) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(weight_input)}, {3, INPUT_DESC(weight_hidden)}, + {4, INPUT_DESC(y)}, {5, INPUT_DESC(init_h)}, {6, INPUT_DESC(h)}, + {7, INPUT_DESC(dy)}, {8, INPUT_DESC(dh)}, {9, INPUT_DESC(update)}, + {10, INPUT_DESC(reset)}, {11, INPUT_DESC(new)}, {12, INPUT_DESC(hidden_new)}, + {13, INPUT_DESC(seq_length)}, {14, INPUT_DESC(mask)}}; +// ӳ䣬ʮĸxΪ1weight_inputΪ2weight_hiddenΪ3yΪ4init_hΪ5hΪ6dyΪ7dhΪ8 +//updateΪ9resetΪ10newΪ11hidden_newΪ12seq_lengthΪ13maskΪ14 +ATTR_MAP(DynamicGRUV2Grad) = {{"direction", ATTR_DESC(direction, AnyTraits())}, + {"cell_depth", ATTR_DESC(cell_depth, AnyTraits())}, + {"keep_prob", ATTR_DESC(keep_prob, AnyTraits())}, + {"cell_clip", ATTR_DESC(cell_clip, AnyTraits())}, + {"num_proj", ATTR_DESC(num_proj, AnyTraits())}, + {"time_major", ATTR_DESC(time_major, AnyTraits())}, + {"gate_order", ATTR_DESC(gate_order, AnyTraits())}, + {"reset_after", ATTR_DESC(reset_after, AnyTraits())}}; +// ӳ䣬г"direction""activation""gate_order"ԣΪstd::string +//"cell_depth""num_proj",Ϊint64_t +//"reset_after""is_training""time_major"Ϊbool +//"keep_prob""cell_clip"Ϊfloat +OUTPUT_MAP(DynamicGRUV2Grad) = {{0, OUTPUT_DESC(dw_input)}, {1, OUTPUT_DESC(dw_hidden)}, {2, OUTPUT_DESC(db_input)}, + {3, OUTPUT_DESC(db_hidden)}, {4, OUTPUT_DESC(dx)}, {5, OUTPUT_DESC(dh_prev)}}; +// ӳ䣬dw_inputΪ0dw_hiddenΪ1db_inputΪ2db_hiddenΪ3dxΪ4dh_prevΪ5 +REG_ADPT_DESC(DynamicGRUV2Grad, kNameDynamicGRUV2Grad, ADPT_DESC(DynamicGRUV2Grad)) +// עDynamicGRUV2GradkNameDynamicGRUV2Grad +} // namespace mindspore::transform -- 2.34.1 From febf01acae7faa9054194e5c09f688f421724f64 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:20:09 +0800 Subject: [PATCH 48/72] ADD file via upload --- .../ccsrc/transform-update/rpn_ops_declare.cc | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/rpn_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/rpn_ops_declare.cc b/mindspore/ccsrc/transform-update/rpn_ops_declare.cc new file mode 100644 index 00000000000..3cf19fc7276 --- /dev/null +++ b/mindspore/ccsrc/transform-update/rpn_ops_declare.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/rpn_ops_declare.h" + +namespace mindspore::transform { +// NMSWithMask +INPUT_MAP(NMSWithMask) = {{1, INPUT_DESC(box_scores)}}; +//ӳ䣬βbox_scores +ATTR_MAP(NMSWithMask) = {{"iou_threshold", ATTR_DESC(iou_threshold, AnyTraits())}}; +//ӳ䣬аһ "iou_threshold"ʹΪ "iou_threshold" (ATTR_DESC) +//ֵָΪ float"iou_threshold" ԿָǼֵ (NMS) е IoU ֵ +OUTPUT_MAP(NMSWithMask) = { + {0, OUTPUT_DESC(selected_boxes)}, {1, OUTPUT_DESC(selected_idx)}, {2, OUTPUT_DESC(selected_mask)}}; +// "NMSWithMask" ӳ(OUTPUT_MAP)0ӳΪΪ "selected_boxes" (OUTPUT_DESC) +//1ӳΪΪ "selected_idx" 2ӳΪΪ "selected_mask" +//ʾ "NMSWithMask" ڼл +REG_ADPT_DESC(NMSWithMask, kNameNMSWithMask, ADPT_DESC(NMSWithMask)) +//ע "NMSWithMask" (REG_ADPT_DESC) +//а "kNameNMSWithMask" (ADPT_DESC) +//⽫ "NMSWithMask" ڿеʵֹ +} // namespace mindspore::transform -- 2.34.1 From 5a17ea7fe9b659791fdb8c72133e62258ee2aec1 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:20:30 +0800 Subject: [PATCH 49/72] ADD file via upload --- .../transform-update/selection_ops_declare.cc | 320 ++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/selection_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/selection_ops_declare.cc b/mindspore/ccsrc/transform-update/selection_ops_declare.cc new file mode 100644 index 00000000000..f7ab6952b0d --- /dev/null +++ b/mindspore/ccsrc/transform-update/selection_ops_declare.cc @@ -0,0 +1,320 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "transform/graph_ir/op_declare/selection_ops_declare.h" + +namespace mindspore::transform { +// CumsumD +INPUT_MAP(CumsumD) = {{1, INPUT_DESC(x)}}; +//һӳx +INPUT_ATTR_MAP(CumsumD) = {{2, ATTR_DESC(axis, AnyTraits())}}; +//CumsumDӳ䣬һΪ"axis"ԣΪint64_t +ATTR_MAP(CumsumD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits())}, + {"reverse", ATTR_DESC(reverse, AnyTraits())}}; +// CumsumDӳ䣬г"exclusive""reverse"ԣͷֱΪbool +OUTPUT_MAP(CumsumD) = {{0, OUTPUT_DESC(y)}}; +// CumsumDӳ䣬һ"y"Ϊ0 +REG_ADPT_DESC(CumsumD, kNameCumSum, ADPT_DESC(CumsumD)) +// עCumsumDkNameCumSum +// +// GatherV2 +INPUT_MAP(GatherV2) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(axis)}}; +// GatherV2ӳ䣬룺"x""indices""axis"ֱΪ123 +ATTR_MAP(GatherV2) = EMPTY_ATTR_MAP; +// GatherV2ûԣΪյӳ +OUTPUT_MAP(GatherV2) = {{0, OUTPUT_DESC(y)}}; +// GatherV2ӳ䣬һ"y"Ϊ0 +// +// CumprodD +INPUT_MAP(CumprodD) = {{1, INPUT_DESC(x)}}; +// CumprodDӳ䣬һ"x"Ϊ1 +INPUT_ATTR_MAP(CumprodD) = {{2, ATTR_DESC(axis, AnyTraits())}}; +// CumprodDӳ䣬һΪ"axis"ԣΪint64_tz +ATTR_MAP(CumprodD) = {{"exclusive", ATTR_DESC(exclusive, AnyTraits())}, + {"reverse", ATTR_DESC(reverse, AnyTraits())}}; +// CumprodDӳ䣬г"exclusive""reverse"ԣͷֱΪbool +OUTPUT_MAP(CumprodD) = {{0, OUTPUT_DESC(y)}}; +// CumprodDӳ䣬һ"y"Ϊ0 +REG_ADPT_DESC(CumprodD, kNameCumProd, ADPT_DESC(CumprodD)) +// עCumprodDkNameCumProd + +INPUT_MAP(SliceD) = {{1, INPUT_DESC(x)}}; +// SliceDӳ䣬һ"x"Ϊ1 +INPUT_ATTR_MAP(SliceD) = {{2, ATTR_DESC(offsets, AnyTraits(), AnyTraits>())}, + {3, ATTR_DESC(size, AnyTraits(), AnyTraits>())}}; +// ԣֱ"offsets""size"Ӧͷֱint64_tstd::vector +ATTR_MAP(SliceD) = EMPTY_ATTR_MAP; +//SliceDûԣΪյӳ +OUTPUT_MAP(SliceD) = {{0, OUTPUT_DESC(y)}}; +// SliceDӳ +REG_ADPT_DESC(SliceD, kNameSlice, ADPT_DESC(SliceD)) +//עSliceDkNameSlice + +// TopK +INPUT_MAP(TopK) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(k)}}; +// TopKӳ䣬룺"x""k"ֱΪ12 +ATTR_MAP(TopK) = {{"sorted", ATTR_DESC(sorted, AnyTraits())}}; +//ӳ䣬һsortbool +OUTPUT_MAP(TopK) = {{0, OUTPUT_DESC(values)}, {1, OUTPUT_DESC(indices)}}; +//ӳ䣬:"values""indices"ֱΪ01 +REG_ADPT_DESC(TopK, kNameTopK, ADPT_DESC(TopK)) +// עTopKkNameTopK + +// InTopK +INPUT_MAP(InTopKD) = {{1, INPUT_DESC(x1)}, {2, INPUT_DESC(x2)}}; +// InTopKDӳ䣬룺"x1""x2"ֱΪ12 +ATTR_MAP(InTopKD) = {{"k", ATTR_DESC(k, AnyTraits())}}; +//ӳ䣬һkint64_t +OUTPUT_MAP(InTopKD) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬һy0 +REG_ADPT_DESC(InTopKD, kNameInTopKD, ADPT_DESC(InTopKD)) +//עInTopKDkNameInTopK + +// TileD +INPUT_MAP(TileD) = {{1, INPUT_DESC(x)}}; +//ӳ䣬һx1 +INPUT_ATTR_MAP(TileD) = {{2, ATTR_DESC(multiples, AnyTraits(), AnyTraits>())}}; +// multiplesӦint64_tstd::vector2 +ATTR_MAP(TileD) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(TileD) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬һyʱ0 +REG_ADPT_DESC(TileD, kNameTile, ADPT_DESC(TileD)) +// עTileDkNameTile + +// OneHot +INPUT_MAP(OneHot) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(depth)}, {3, INPUT_DESC(on_value)}, {4, INPUT_DESC(off_value)}}; +// ӳ䣬ĸxdepthon_valueoff_valueֱ1234 +ATTR_MAP(OneHot) = {{"axis", ATTR_DESC(axis, AnyTraits())}}; +//ӳ䣬һaxisΪint64_t +OUTPUT_MAP(OneHot) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬һyΪ0 +REG_ADPT_DESC(OneHot, prim::kPrimOneHot->name(), ADPT_DESC(OneHot)) +//Ϊ "OneHot" IJעᡣ +//"OneHot" ƣprim::kPrimOneHot->name()ADPT_DESC(OneHot) + +// GatherV2D +INPUT_MAP(GatherV2D) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}}; +//ӳ䣬xindices,ֱΪ12 +INPUT_ATTR_MAP(GatherV2D) = {{3, ATTR_DESC(axis, AnyTraits())}}; +//ӳ䣬int64_t͵axisΪ3 +ATTR_MAP(GatherV2D) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(GatherV2D) = {{0, OUTPUT_DESC(y)}}; +//һӳyΪ0 +REG_ADPT_DESC(GatherV2D, prim::kPrimGather->name(), ADPT_DESC(GatherV2D)) +//дǽΪ "GatherV2D" IJעᡣ +// "GatherV2D" ƣprim::kPrimGather->name()ADPT_DESC(GatherV2D)ԱضܻܹȷִкŻ "GatherV2D" +REG_ADPT_DESC(Gather, kNameGather, ADPT_DESC(GatherV2D)) +//Ϊ "Gather" IJעᡣ +// "Gather" ƣkNameGatherADPT_DESC(GatherV2D)ԱضܻܹȷִкŻ "Gather" + +// ScatterNdD +INPUT_MAP(ScatterNdD) = {{1, INPUT_DESC(indices)}, {2, INPUT_DESC(x)}}; +//ӳ䣬indicesxΪ12 +INPUT_ATTR_MAP(ScatterNdD) = { + {3, ATTR_DESC(shape, AnyTraits>(), AnyTraits>())}}; +//ӳ䣬int64_t͵shapeΪ3 +ATTR_MAP(ScatterNdD) = EMPTY_ATTR_MAP; +//ӳΪ +OUTPUT_MAP(ScatterNdD) = {{0, OUTPUT_DESC(y)}}; +//ӳyΪ0 +REG_ADPT_DESC(ScatterNdD, kNameScatterNdD, ADPT_DESC(ScatterNdD)) +// Ϊ "ScatterNdD" IJעᡣ +// "ScatterNdD" ƣkNameScatterNdDADPT_DESC(ScatterNdD)ԱضܻܹȷִкŻ "ScatterNdD" +// +// ScatterNonAliasingAdd +INPUT_MAP(ScatterNonAliasingAdd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}, {3, INPUT_DESC(updates)}}; +//ӳ䣬xΪ1indicesΪ2updatesΪ3 +ATTR_MAP(ScatterNonAliasingAdd) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(ScatterNonAliasingAdd) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(ScatterNonAliasingAdd, kNameScatterNonAliasingAdd, ADPT_DESC(ScatterNonAliasingAdd)) +// עScatterNonAliasingAddkNameScatterNonAliasingAdd +// +// GatherNd +INPUT_MAP(GatherNd) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(indices)}}; +//ӳ䣬xΪ1indicesΪ2 +ATTR_MAP(GatherNd) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(GatherNd) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(GatherNd, kNameGatherNd, ADPT_DESC(GatherNd)) +// עGatherNd kNameGatherNd + +// GatherD +INPUT_MAP(GatherD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(dim)}, {3, INPUT_DESC(index)}}; +//ӳ䣬xΪ1dimΪ2,indexΪ3 +ATTR_MAP(GatherD) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(GatherD) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(GatherD, kNameGatherD, ADPT_DESC(GatherD)) +// עkNameGatherD GatherD + +// Range +INPUT_MAP(RangeD) = {{1, INPUT_DESC(x)}}; +//ӳ䣬xΪ1 +ATTR_MAP(RangeD) = {{"start", ATTR_DESC(start, AnyTraits())}, + {"limit", ATTR_DESC(limit, AnyTraits())}, + {"delta", ATTR_DESC(delta, AnyTraits())}}; +//ӳ䣬г"start""limit""delta"ԣͷֱΪfloat +REG_ADPT_DESC(RangeD, kNameRange, ADPT_DESC(RangeD)) +//עRangeD kNameRange + +// InplaceAddD +INPUT_MAP(InplaceAddD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}}; +// ӳ䣬xΪ1yΪ2 +ATTR_MAP(InplaceAddD) = {{"indices", ATTR_DESC(indices, AnyTraits>())}}; +//ӳ,indicesΪint64_t +OUTPUT_MAP(InplaceAddD) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(InplaceAddD, kNameInplaceAddD, ADPT_DESC(InplaceAddD)) +// עRangeD kNameRange + +// InplaceSubD +INPUT_MAP(InplaceSubD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}}; +//ӳ䣬xΪ1yΪ2 +ATTR_MAP(InplaceSubD) = {{"indices", ATTR_DESC(indices, AnyTraits>())}}; +// ӳ,indicesΪint64_t +OUTPUT_MAP(InplaceSubD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(InplaceSubD, kNameInplaceSubD, ADPT_DESC(InplaceSubD)) +// עInplaceSubDkNameInplaceSubD + +// InplaceUpdateD +INPUT_MAP(InplaceUpdateD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(v)}}; +// ӳ䣬xΪ1yΪ2 +ATTR_MAP(InplaceUpdateD) = {{"indices", ATTR_DESC(indices, AnyTraits>())}}; +// ӳ,indicesΪint64_t +OUTPUT_MAP(InplaceUpdateD) = {{0, OUTPUT_DESC(y)}}; +// ӳ,yΪ0 +REG_ADPT_DESC(InplaceUpdateD, kNameInplaceUpdateD, ADPT_DESC(InplaceUpdateD)) +// עInplaceUpdateDkNameInplaceUpdateD + +// Select +INPUT_MAP(Select) = {{1, INPUT_DESC(condition)}, {2, INPUT_DESC(x1)}, {3, INPUT_DESC(x2)}}; +// ӳ䣬conditionΪ1x1Ϊ2x2Ϊ3 +ATTR_MAP(Select) = EMPTY_ATTR_MAP; +//ӳ䣬 +OUTPUT_MAP(Select) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(Select, prim::kPrimSelect->name(), ADPT_DESC(Select)) +// עInplaceUpdateDprim::kPrimSelect->name() + +// StridedSliceGrad +INPUT_MAP(StridedSliceGrad) = { + {1, INPUT_DESC(dy)}, {2, INPUT_DESC(shape)}, {3, INPUT_DESC(begin)}, {4, INPUT_DESC(end)}, {5, INPUT_DESC(strides)}}; +// ӳ䣬dyΪ1shapeΪ2beginΪ3endΪ4stridesΪ5 +ATTR_MAP(StridedSliceGrad) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}}; +// ӳ䣬г"begin_mask""end_mask""ellipsis_mask""new_axis_mask""shrink_axis_mask"ԣͷֱΪint64_t +OUTPUT_MAP(StridedSliceGrad) = {{0, OUTPUT_DESC(output)}}; +// ӳ䣬outputΪ0 +REG_ADPT_DESC(StridedSliceGrad, kNameStridedSliceGrad, ADPT_DESC(StridedSliceGrad)) +//עStridedSliceGradDkNameStridedSliceGrad + +// StridedSlice +INPUT_MAP(StridedSlice) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(strides)}}; +// ӳ䣬ĸxΪ1beginΪ2endΪ3stridesΪ4 +ATTR_MAP(StridedSlice) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}}; +// ӳ䣬г"begin_mask""end_mask""ellipsis_mask""new_axis_mask""shrink_axis_mask"ԣͷֱΪint64_t +OUTPUT_MAP(StridedSlice) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(StridedSlice, kNameStridedSlice, ADPT_DESC(StridedSlice)) +// עStridedSlicekNameStridedSlice + +// StridedSliceV2 +INPUT_MAP(StridedSliceV2) = { + {1, INPUT_DESC(x)}, {2, INPUT_DESC(begin)}, {3, INPUT_DESC(end)}, {4, INPUT_DESC(axes)}, {5, INPUT_DESC(strides)}}; +// ӳ䣬xΪ1beginΪ2endΪ3axesΪ4stridesΪ5 +ATTR_MAP(StridedSliceV2) = {{"begin_mask", ATTR_DESC(begin_mask, AnyTraits())}, + {"end_mask", ATTR_DESC(end_mask, AnyTraits())}, + {"ellipsis_mask", ATTR_DESC(ellipsis_mask, AnyTraits())}, + {"new_axis_mask", ATTR_DESC(new_axis_mask, AnyTraits())}, + {"shrink_axis_mask", ATTR_DESC(shrink_axis_mask, AnyTraits())}}; +// ӳ䣬г"begin_mask""end_mask""ellipsis_mask""new_axis_mask""shrink_axis_mask"ԣͷֱΪint64_t +OUTPUT_MAP(StridedSliceV2) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(StridedSliceV2, kNameStridedSliceV2, ADPT_DESC(StridedSliceV2)) +// עStridedSliceV2StridedSliceV2 + +// UnsortedSegmentSum +INPUT_MAP(UnsortedSegmentSumD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +// ӳ䣬xΪ1segment_idsΪ2 +INPUT_ATTR_MAP(UnsortedSegmentSumD) = {{3, ATTR_DESC(num_segments, AnyTraits())}}; +// ӳ䣬num_segmentsint64_t,3 +ATTR_MAP(UnsortedSegmentSumD) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(UnsortedSegmentSumD) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(UnsortedSegmentSumD, prim::kPrimUnsortedSegmentSum->name(), ADPT_DESC(UnsortedSegmentSumD)) +//עUnsortedSegmentSumDprim::kPrimUnsortedSegmentSum->name() + +// UnsortedSegmentProdD +INPUT_MAP(UnsortedSegmentProdD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +// ӳ䣬xΪ1segment_idsΪ2 +INPUT_ATTR_MAP(UnsortedSegmentProdD) = {{3, ATTR_DESC(num_segments, AnyTraits())}}; +// ӳ䣬num_segmentsint64_t,3 +ATTR_MAP(UnsortedSegmentProdD) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(UnsortedSegmentProdD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(UnsortedSegmentProdD, kNameUnsortedSegmentProdD, ADPT_DESC(UnsortedSegmentProdD)) +// עUnsortedSegmentSumDkNameUnsortedSegmentProdD + + +// UnsortedSegmentMaxD +INPUT_MAP(UnsortedSegmentMaxD) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}}; +// ӳ䣬xΪ1segment_idsΪ2 +INPUT_ATTR_MAP(UnsortedSegmentMaxD) = {{3, ATTR_DESC(num_segments, AnyTraits())}}; +// ӳ䣬num_segmentsint64_t,3 +ATTR_MAP(UnsortedSegmentMaxD) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(UnsortedSegmentMaxD) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(UnsortedSegmentMaxD, kNameUnsortedSegmentMaxD, ADPT_DESC(UnsortedSegmentMaxD)) +// עUnsortedSegmentMaxDkNameUnsortedSegmentMaxD + +// UnsortedSegmentMin +INPUT_MAP(UnsortedSegmentMin) = {{1, INPUT_DESC(x)}, {2, INPUT_DESC(segment_ids)}, {3, INPUT_DESC(num_segments)}}; +// ӳ䣬xΪ1segment_idsΪ2num_segmentsΪ3 +ATTR_MAP(UnsortedSegmentMin) = EMPTY_ATTR_MAP; +// ӳ䣬 +OUTPUT_MAP(UnsortedSegmentMin) = {{0, OUTPUT_DESC(y)}}; +// ӳ䣬yΪ0 +REG_ADPT_DESC(UnsortedSegmentMin, prim::kPrimUnsortedSegmentMin->name(), ADPT_DESC(UnsortedSegmentMin)) +// עUnsortedSegmentMinprim::kPrimUnsortedSegmentMin->name() + +// ReverseV2 +INPUT_MAP(ReverseV2D) = {{1, INPUT_DESC(x)}}; +// ӳ䣬xΪ1 +ATTR_MAP(ReverseV2D) = {{"axis", ATTR_DESC(axis, AnyTraits(), AnyTraits>())}}; +//ӳ䣬axisΪint64_tstd::vector +OUTPUT_MAP(ReverseV2D) = {{0, OUTPUT_DESC(y)}}; +//ӳ䣬yΪ0 +REG_ADPT_DESC(ReverseV2D, kNameReverseV2, ADPT_DESC(ReverseV2D)) +// עReverseV2DkNameReverseV2 +} // namespace mindspore::transform -- 2.34.1 From 7c5ab75646479c1efbea9aec27fdabfb7e35a3da Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:20:49 +0800 Subject: [PATCH 50/72] ADD file via upload --- .../split_combination_ops_declare.cc | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/split_combination_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/split_combination_ops_declare.cc b/mindspore/ccsrc/transform-update/split_combination_ops_declare.cc new file mode 100644 index 00000000000..a7ed72213fa --- /dev/null +++ b/mindspore/ccsrc/transform-update/split_combination_ops_declare.cc @@ -0,0 +1,94 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/split_combination_ops_declare.h" +#include + +namespace mindspore::transform { +// SplitD +INPUT_MAP(SplitD) = {{1, INPUT_DESC(x)}}; +//ӳ䣬1ӳΪΪx(INPUT_DESC) +ATTR_MAP(SplitD) = {{"axis", ATTR_DESC(split_dim, AnyTraits())},//ָά + {"output_num", ATTR_DESC(num_split, AnyTraits())}};//ָ +// ӳ +DYN_OUTPUT_MAP(SplitD) = {{0, DYN_OUTPUT_DESC(y)}}; +//̬ӳ //0ӳΪΪ "y" Ķ̬(DYN_OUTPUT_DESC) +REG_ADPT_DESC(SplitD, kNameSplitD, ADPT_DESC(SplitD)) +//ע "SplitD" (REG_ADPT_DESC),а "kNameSplitD" (ADPT_DESC) +// ⽫ "SplitD" ڿеʵֹ +// +// Pack +INPUT_MAP(Pack) = EMPTY_INPUT_MAP; +// "Pack" ӳ(INPUT_MAP)Ϊաʾ "Pack" ûʽ룬ûκ +DYN_INPUT_MAP(Pack) = {{1, DYN_INPUT_DESC(x)}}; +// "Pack" Ķ̬ӳ(DYN_INPUT_MAP)1ӳΪΪ "x" Ķ̬(DYN_INPUT_DESC) +ATTR_MAP(Pack) = {{"num", ATTR_DESC(N, AnyTraits())}, {"axis", ATTR_DESC(axis, AnyTraits())}}; +// "Pack" ӳ(ATTR_MAP) ԣ/ά +OUTPUT_MAP(Pack) = {{0, OUTPUT_DESC(y)}};//ӳ(OUTPUT_MAP)0ӳΪΪ "y" (OUTPUT_DESC) +REG_ADPT_DESC(Pack, prim::kStack, ADPT_DESC(Pack))//ע "Pack" (REG_ADPT_DESC) + +// ParallelConcat +INPUT_MAP(ParallelConcat) = EMPTY_INPUT_MAP; +// "ParallelConcat" ӳ(INPUT_MAP)Ϊա +//ʾ "ParallelConcat" ûʽ룬ûκ +DYN_INPUT_MAP(ParallelConcat) = {{1, DYN_INPUT_DESC(values)}}; +// "ParallelConcat" Ķ̬ӳ(DYN_INPUT_MAP)1ӳΪΪ "values" Ķ̬(DYN_INPUT_DESC) +//ʾ "ParallelConcat" һ̬룬ÿ붼ʹ "values" ʶ +ATTR_MAP(ParallelConcat) = {//ӳ(ATTR_MAP), + {"shape", ATTR_DESC(shape, AnyTraits>())},//Ӳʱ״ + {"N", ATTR_DESC(N, AnyTraits())},//ӵ +}; +OUTPUT_MAP(ParallelConcat) = {{0, OUTPUT_DESC(output_data)}}; +//ӳ(OUTPUT_MAP) +REG_ADPT_DESC(ParallelConcat, kNameParallelConcat, ADPT_DESC(ParallelConcat)) +//ע "ParallelConcat" (REG_ADPT_DESC) +//а "kNameParallelConcat" (ADPT_DESC) +//⽫ "ParallelConcat" ڿеʵֹʹ "kNameParallelConcat" Ϊʶ + + + +// ConcatD +INPUT_MAP(ConcatD) = EMPTY_INPUT_MAP; +// "ConcatD" ӳ(INPUT_MAP)Ϊա +//ʾ "ConcatD" ûʽ룬ûκ +DYN_INPUT_MAP(ConcatD) = {{1, DYN_INPUT_DESC(x)}}; +// "ConcatD" Ķ̬ӳ(DYN_INPUT_MAP)1ӳΪΪ "x" Ķ̬(DYN_INPUT_DESC) +//ʾ "ConcatD" һ̬룬ÿ붼ʹ "x" ʶ +ATTR_MAP(ConcatD) = {//ӳ + {"axis", ATTR_DESC(concat_dim, AnyTraits())},//Ӳʱά + {"inputNums", ATTR_DESC(N, AnyTraits())},// +}; +OUTPUT_MAP(ConcatD) = {{0, OUTPUT_DESC(y)}}; +// "ConcatD" ӳ(OUTPUT_MAP)0ӳΪΪ "y" (OUTPUT_DESC) +//ʾ "ConcatD" һʹ "y" ʶ +REG_ADPT_DESC(ConcatD, prim::kPrimConcat->name(), ADPT_DESC(ConcatD)) +//ע "ConcatD" (REG_ADPT_DESC)а "prim::kPrimConcat->name()" (ADPT_DESC) +// ⽫ "ConcatD" ڿеʵֹʹ "prim::kPrimConcat->name()" Ϊʶ +// +// ConcatV2D Inference for tf +INPUT_MAP(ConcatV2D) = EMPTY_INPUT_MAP;//ӳΪ +DYN_INPUT_MAP(ConcatV2D) = {{1, DYN_INPUT_DESC(x)}};//̬ӳ䣬жӳ +ATTR_MAP(ConcatV2D) = { + {"axis", ATTR_DESC(concat_dim, AnyTraits())},//Ӳʱά + {"N", ATTR_DESC(N, AnyTraits())},//ӵ +}; +OUTPUT_MAP(ConcatV2D) = {{0, OUTPUT_DESC(y)}}; +// "ConcatV2D" ӳ(OUTPUT_MAP)0ӳΪΪ "y" (OUTPUT_DESC) +//ʾ "ConcatV2D" һʹ "y" ʶ +REG_ADPT_DESC(ConcatV2D, kNameConcatV2D, ADPT_DESC(ConcatV2D)) +//ע "ConcatV2D" (REG_ADPT_DESC)а "kNameConcatV2D" (ADPT_DESC) +//⽫ "ConcatV2D" ڿеʵֹʹ "kNameConcatV2D" Ϊʶ +} // namespace mindspore::transform -- 2.34.1 From d39a15cd042edc1f8171e3de9211936a0a9d803e Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:21:13 +0800 Subject: [PATCH 51/72] ADD file via upload --- .../transform-update/state_ops_declare.cc | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/state_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/state_ops_declare.cc b/mindspore/ccsrc/transform-update/state_ops_declare.cc new file mode 100644 index 00000000000..8e77f0ef5e5 --- /dev/null +++ b/mindspore/ccsrc/transform-update/state_ops_declare.cc @@ -0,0 +1,25 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/state_ops_declare.h" +//"Variable" ͨڱʾģеıȨغƫ +namespace mindspore::transform { +// Variable +INPUT_MAP(Variable) = {{1, INPUT_DESC(x)}}; +// "Variable" ӳ(INPUT_MAP)1ӳΪΪ "x" (INPUT_DESC) +ATTR_MAP(Variable) = EMPTY_ATTR_MAP; +// "Variable" ӳ(ATTR_MAP)˴Ϊ(EMPTY_ATTR_MAP) +} // namespace mindspore::transform -- 2.34.1 From dac5730b965bc613ed6c3c25428d81af334db5f9 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:21:37 +0800 Subject: [PATCH 52/72] ADD file via upload --- .../transformation_ops_declare.cc | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/transformation_ops_declare.cc diff --git a/mindspore/ccsrc/transform-update/transformation_ops_declare.cc b/mindspore/ccsrc/transform-update/transformation_ops_declare.cc new file mode 100644 index 00000000000..edf4a344e2e --- /dev/null +++ b/mindspore/ccsrc/transform-update/transformation_ops_declare.cc @@ -0,0 +1,133 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transform/graph_ir/op_declare/transformation_ops_declare.h" +#include + +namespace mindspore::transform { +// Flatten +// չƽΪһ1D +INPUT_MAP(Flatten) = {{1, INPUT_DESC(x)}};//ӳ䣬FlattenһΪ1Ҹβx +ATTR_MAP(Flatten) = EMPTY_ATTR_MAP;//ӳ䣬գòҪκζԲ +OUTPUT_MAP(Flatten) = {{0, OUTPUT_DESC(y)}};//ӳ䣬FlattenһΪ0Ҹβy +REG_ADPT_DESC(Flatten, prim::kPrimFlatten->name(), ADPT_DESC(Flatten)) +// ͨ"prim::kPrimFlatten"name()ȡ"Flatten" һǰMindSporeʱִ"Flatten" +// "Flatten"עᵽ(REG_ADPT_DESC)УԱMindSporeѧϰܹʹøòͼ +//˲עᵽΪ prim::kPrimFlatten->name() ͼ + +// Unpack +INPUT_MAP(Unpack) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(Unpack) = {{"axis", ATTR_DESC(axis, AnyTraits())}, {"num", ATTR_DESC(num, AnyTraits())}}; + // //axisʾֵ //numʾֺɵ +DYN_OUTPUT_MAP(Unpack) = {{0, DYN_OUTPUT_DESC(y)}};///̬ yʾ +REG_ADPT_DESC(Unpack, prim::kUnstack, ADPT_DESC(Unpack)) +//˲עᵽΪ prim::kUnstack ͼ + +// ExtractImagePatches +INPUT_MAP(ExtractImagePatches) = {{1, INPUT_DESC(x)}}; +//һx +ATTR_MAP(ExtractImagePatches) = { +//ĸ + {"ksizes", ATTR_DESC(ksizes, AnyTraits(), AnyTraits>())}, + //ksizesʾݵÿάϻĴڴС + {"strides", ATTR_DESC(strides, AnyTraits(), AnyTraits>())}, + //stridesʾݵÿάϻIJ + {"rates", ATTR_DESC(rates, AnyTraits(), AnyTraits>())}, + //ratesʾݵÿάϵdilationţ + {"padding", ATTR_DESC(padding, AnyTraits())}}; + //paddingʾݵΧӵ͡ +OUTPUT_MAP(ExtractImagePatches) = {{0, OUTPUT_DESC(y)}}; + //һy +REG_ADPT_DESC(ExtractImagePatches, kNameExtractImagePatches, ADPT_DESC(ExtractImagePatches)) + //˲עᵽΪ kNameExtractImagePatches ͼ + +// Transpose +INPUT_MAP(TransposeD) = {{1, INPUT_DESC(x)}}; +// һx +INPUT_ATTR_MAP(TransposeD) = {{2, ATTR_DESC(perm, AnyTraits(), AnyTraits>())}}; +//2ʾDz"TransposeD"ĵڶ +// perm:Ե ԵΪint64_t һstd::vector͵ֵһά +ATTR_MAP(TransposeD) = EMPTY_ATTR_MAP; +// Do not set Transpose operator output descriptor +REG_ADPT_DESC(TransposeD, prim::kPrimTranspose->name(), ADPT_DESC(TransposeD)) + +// SpaceToDepth +INPUT_MAP(SpaceToDepth) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(SpaceToDepth) = {{"block_size", ATTR_DESC(block_size, AnyTraits())}}; +// block_sizeʾռ䵽תĿС +OUTPUT_MAP(SpaceToDepth) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(SpaceToDepth, kNameSpaceToDepth, ADPT_DESC(SpaceToDepth)) + +// DepthToSpace +INPUT_MAP(DepthToSpace) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(DepthToSpace) = {{"block_size", ATTR_DESC(block_size, AnyTraits())}}; +// block_sizeʾȵռתĿС +OUTPUT_MAP(DepthToSpace) = {{0, OUTPUT_DESC(y)}}; +//һ y +REG_ADPT_DESC(DepthToSpace, kNameDepthToSpace, ADPT_DESC(DepthToSpace)) +//˲עᵽΪ kNameDepthToSpace ͼ +// +// SpaceToBatchD +INPUT_MAP(SpaceToBatchD) = {{1, INPUT_DESC(x)}}; + +ATTR_MAP(SpaceToBatchD) = { + {"block_size", ATTR_DESC(block_size, AnyTraits())}, + //block_sizeʾռ䵽תĿС + {"paddings", ATTR_DESC(paddings, AnyTraits>>(), AnyTraits>())}}; + //paddingsʾռ䵽ת䷽ʽ +OUTPUT_MAP(SpaceToBatchD) = {{0, OUTPUT_DESC(y)}}; +REG_ADPT_DESC(SpaceToBatchD, kNameSpaceToBatch, ADPT_DESC(SpaceToBatchD)) + +// SpaceToBatchNDD +INPUT_MAP(SpaceToBatchNDD) = {{1, INPUT_DESC(x)}}; +ATTR_MAP(SpaceToBatchNDD) = { + {"block_shape", ATTR_DESC(block_shape, AnyTraits>())}, + //block_shapeʾռ䵽תĿ״ + {"paddings", ATTR_DESC(paddings, AnyTraits>>(), AnyTraits>())}}; + //paddingsʾռ䵽ת䷽ʽ +OUTPUT_MAP(SpaceToBatchNDD) = {{0, OUTPUT_DESC(y)}}; +//һ y +REG_ADPT_DESC(SpaceToBatchNDD, kNameSpaceToBatchNDD, ADPT_DESC(SpaceToBatchNDD)) +//˲עᵽΪ kNameSpaceToBatchNDD ͼ +// +// BatchToSpaceD +INPUT_MAP(BatchToSpaceD) = {{1, INPUT_DESC(x)}}; +//òһ x +ATTR_MAP(BatchToSpaceD) = { + {"block_size", ATTR_DESC(block_size, AnyTraits())}, + //block_sizeʾռתĿС + {"crops", ATTR_DESC(crops, AnyTraits>>(), AnyTraits>())}}; + //cropsʾռתIJüʽ +OUTPUT_MAP(BatchToSpaceD) = {{0, OUTPUT_DESC(y)}}; +// һ y +REG_ADPT_DESC(BatchToSpaceD, kNameBatchToSpace, ADPT_DESC(BatchToSpaceD)) +//˲עᵽΪ kNameBatchToSpace ͼ + +// BatchToSpaceNDD +INPUT_MAP(BatchToSpaceNDD) = {{1, INPUT_DESC(x)}}; +//һ x +ATTR_MAP(BatchToSpaceNDD) = { + {"block_shape", ATTR_DESC(block_shape, AnyTraits>())}, + //block_shapeʾռתĿ״ + {"crops", ATTR_DESC(crops, AnyTraits>>(), AnyTraits>())}}; + //cropsʾռתIJüʽ +OUTPUT_MAP(BatchToSpaceNDD) = {{0, OUTPUT_DESC(y)}}; +//һ y +REG_ADPT_DESC(BatchToSpaceNDD, kNameBatchToSpaceNd, ADPT_DESC(BatchToSpaceNDD)) +//˲עᵽΪ kNameBatchToSpaceNd ͼ +} // namespace mindspore::transform +// һϵͼÿвͬ롢ԣ +//ҽЩעᵽӦУԱMindSporeѧϰʹЩͼ㡣 \ No newline at end of file -- 2.34.1 From a55500e409556dcc31f049cfc92b71dbafe551cb Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 5 Sep 2023 22:21:56 +0800 Subject: [PATCH 53/72] ADD file via upload --- mindspore/ccsrc/transform-update/util.cc | 524 +++++++++++++++++++++++ 1 file changed, 524 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/util.cc diff --git a/mindspore/ccsrc/transform-update/util.cc b/mindspore/ccsrc/transform-update/util.cc new file mode 100644 index 00000000000..d66b03b5b1c --- /dev/null +++ b/mindspore/ccsrc/transform-update/util.cc @@ -0,0 +1,524 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "include/transform/graph_ir/util.h" + +#include +#include + +#include "securec/include/securec.h" +#include "include/common/utils/convert_utils.h" +#include "include/common/utils/utils.h" + +namespace mindspore { +namespace transform { +using std::make_shared; +using std::shared_ptr; +using std::string; +using std::vector; + +const size_t kErrorSize = 0; + +//úҪһͬ ݵ int64_tvectorһЩʹá +vector TransformUtil::ConvertIntToList(int64_t data, int size) { + vector list{}; //һյ ͵ vector list + if (size <= 0) { // size ǷСڵ 0 + MS_LOG(WARNING) << "size <= 0"; //ǣ־ؿյ list + return list; + } + for (int i = 0; i < size; ++i) { // ʹ for ѭ data size ΣÿθƵĽӵlistС + list.push_back(data); + } + return list;//ѭ󣬷ش洢иƽlist +} + +static std::map datatype_trans_map = { + {MeDataType::kNumberTypeFloat16, GeDataType::DT_FLOAT16}, {MeDataType::kNumberTypeFloat32, GeDataType::DT_FLOAT}, + {MeDataType::kNumberTypeFloat64, GeDataType::DT_DOUBLE}, {MeDataType::kNumberTypeInt8, GeDataType::DT_INT8}, + {MeDataType::kNumberTypeInt16, GeDataType::DT_INT16}, {MeDataType::kNumberTypeInt32, GeDataType::DT_INT32}, + {MeDataType::kNumberTypeInt64, GeDataType::DT_INT64}, {MeDataType::kNumberTypeUInt8, GeDataType::DT_UINT8}, + {MeDataType::kNumberTypeUInt16, GeDataType::DT_UINT16}, {MeDataType::kNumberTypeUInt32, GeDataType::DT_UINT32}, + {MeDataType::kNumberTypeUInt64, GeDataType::DT_UINT64}, {MeDataType::kNumberTypeBool, GeDataType::DT_BOOL}}; + +// ConvertDataType ڽ MeDataType ͵תΪ GeDataType ͡ +GeDataType TransformUtil::ConvertDataType(const MeDataType &type) { + MS_LOG(DEBUG) << "Convert me data type: " << TypeIdLabel(type) << " to ge data type"; + // datatype_trans_map в MeDataType Ӧ GeDataType + if (datatype_trans_map.find(type) != datatype_trans_map.end()) { + return datatype_trans_map[type]; + } else { // ҲӦӳϵ򷵻 GeDataType::DT_UNDEFINED + return GeDataType::DT_UNDEFINED; + } +} + +static std::map datatype_size_map = { + {MeDataType::kNumberTypeFloat16, sizeof(float) / 2}, {MeDataType::kNumberTypeFloat32, sizeof(float)}, // 1/2 of float + {MeDataType::kNumberTypeFloat64, sizeof(double)}, {MeDataType::kNumberTypeInt8, sizeof(int8_t)}, + {MeDataType::kNumberTypeInt16, sizeof(int16_t)}, {MeDataType::kNumberTypeInt32, sizeof(int32_t)}, + {MeDataType::kNumberTypeInt64, sizeof(int64_t)}, {MeDataType::kNumberTypeUInt8, sizeof(uint8_t)}, + {MeDataType::kNumberTypeUInt16, sizeof(uint16_t)}, {MeDataType::kNumberTypeUInt32, sizeof(uint32_t)}, + {MeDataType::kNumberTypeUInt64, sizeof(uint64_t)}, {MeDataType::kNumberTypeBool, sizeof(bool)}}; + +// GetDataTypeSize ڻȡ MeDataType ڴռݵֽڴС +size_t TransformUtil::GetDataTypeSize(const MeDataType &type) { + if (datatype_size_map.find(type) != datatype_size_map.end()) { // datatype_size_map в MeDataType ӦֽڴС + return datatype_size_map[type]; + } else { // ҲӦĴС־һضĴСkErrorSize + MS_LOG(ERROR) << "Illegal tensor data type!"; + return kErrorSize; + } +} + +// ConvertFormat ڽַʽ `format` תΪӦ GeFormat ö͡ +GeFormat TransformUtil::ConvertFormat(const string &format) { + // ͨȽַʽ `format`ж϶Ӧ GeFormat öͣתء + if (format == kOpFormat_NCHW) { + return GeFormat::FORMAT_NCHW; + } else if (format == kOpFormat_NDHWC) { + return GeFormat::FORMAT_NDHWC; + } else if (format == kOpFormat_NCDHW) { + return GeFormat::FORMAT_NCDHW; + } else if (format == kOpFormat_DHWNC) { + return GeFormat::FORMAT_DHWNC; + } else if (format == kOpFormat_DHWCN) { + return GeFormat::FORMAT_DHWCN; + } else if (format == kOpFormat_NC1HWC0) { + return GeFormat::FORMAT_NC1HWC0; + } else if (format == kOpFormat_NHWC) { + return GeFormat::FORMAT_NHWC; + } else if (format == kOpFormat_HWCN) { + return GeFormat::FORMAT_HWCN; + } else if (format == kOpFormat_ND) { + return GeFormat::FORMAT_ND; + } else { // `format` ֵ֧ݸʽ־Ĭϵ GeFormat::FORMAT_ND ʽ + MS_LOG(ERROR) << "Illegal tensor data format: (" << format << "). Use ND format instead."; + return GeFormat::FORMAT_ND; + } +} + +static int64_t IntegerCastFunc(size_t temp) { return static_cast(temp); } + +// GetGeTensorDesc ڸݸ MeTensor ״ShapeVectorͣMeDataType͸ʽformat +// Ӧ GeTensorDesc 󣬲һָö shared_ptr +// GeTensorDesc Ascend AI Core ж࣬״ͺݸʽ +std::shared_ptr TransformUtil::GetGeTensorDesc(const ShapeVector &me_shape, const MeDataType &me_type, + const std::string &format) { + // convert me shape to ge shape + // MeTensor ״ShapeVectorתΪ GeShape std::vector + std::vector ge_shape; + + if (me_shape.size() == 1) { + ge_shape.push_back(static_cast(me_shape[0])); + } else { + ge_shape.resize(me_shape.size()); + (void)std::transform(me_shape.begin(), me_shape.end(), ge_shape.begin(), IntegerCastFunc); + } + + GeShape shape(ge_shape); + if (shape.GetDimNum() == 0) { // GeShape άΪ 0ʾϢ־ + MS_LOG(INFO) << "The dims size of Ge tensor is zero"; + } + // convert me format to ge format + // MeTensor ĸʽformatתΪ GeFormat ö͡ + GeFormat ge_format = ConvertFormat(format); + if (ge_format == GeFormat::FORMAT_ND) { + MS_LOG(INFO) << "Set ND data format"; + } + // convert me datatype to ge datatype + // MeTensor ͣme_typeתΪ GeDataType ö͡ + GeDataType data_type = ConvertDataType(me_type); + if (data_type == GeDataType::DT_UNDEFINED) { // תʧܣ־ؿָ롣 + MS_LOG(ERROR) << "undefined data type :" << me_type; + return nullptr; + } + // GeTensorDesc 󣬲Ӧ״ͺݸʽϢ + auto desc = std::make_shared(shape, ge_format, data_type); + if (desc == nullptr) { + MS_LOG(ERROR) << "Create GeTensorDesc failed!"; + return nullptr; + } + // ʵά GeTensorDesc ״ά + MS_LOG(INFO) << "SetRealDimCnt is :" << me_shape.size(); + desc->SetRealDimCnt(SizeToInt(me_shape.size())); + return desc; // ָ򴴽 GeTensorDesc shared_ptr +} + +// if failed, return empty vector. +// ConvertInputTensors ڽ MeTensor бme_tensorsתΪӦ GeTensor бתĽ +// MeTensor MindSpore ж࣬ڴ洢ݺϢ +// GeTensor Ascend AI Core ж࣬ڴ洢ݺϢ +// MeTensor бÿ MeTensor תת GeTensor ӵб ge_tensors С +// תӦĴ־ؿб +std::vector TransformUtil::ConvertInputTensors(const std::vector &me_tensors, + const std::string &format) { + std::vector ge_tensors; + // MeTensor бÿ MeTensor תת GeTensor ӵб ge_tensors С + for (size_t index = 0; index < me_tensors.size(); index++) { + MS_EXCEPTION_IF_NULL(me_tensors[index]); + // ǰ MeTensor ݴС״Ϣ + MS_LOG(INFO) << "me_tensor " << index << " 's data size is: " << me_tensors[index]->DataSize(); + auto shape = me_tensors[index]->shape(); + std::string shape_str; + for (size_t i = 0; i < shape.size(); i++) { + shape_str += std::to_string(shape[i]); + shape_str += " "; + } + MS_LOG(INFO) << "me_tensor " << index << " 's shape is: { " << shape_str << "}"; + MS_LOG(INFO) << "me_tensor " << index << " 's type is: " << me_tensors[index]->data_type(); + // ConvertTensor ǰ MeTensor תΪӦ GeTensor + auto ge_tensor_ptr = TransformUtil::ConvertTensor(me_tensors[index], format); + if (ge_tensor_ptr != nullptr) { // תɹת GeTensor ӵб ge_tensors С + ge_tensors.emplace_back(ge_tensor_ptr); + } else { // תӦĴ־սб ge_tensorsؿб + MS_LOG(ERROR) << "Convert me_tensor " << index << " to Ge Tensor failed!"; + ge_tensors.clear(); + return ge_tensors; + } + } + return ge_tensors; // ת GeTensor б +} + +// ConvertTensor ڽ MeTensor`tensor`תΪӦ GeTensorתĽ +// MeTensor MindSpore ж࣬ڴ洢ݺϢ +// GeTensor Ascend AI Core ж࣬ڴ洢ݺϢ +GeTensorPtr TransformUtil::ConvertTensor(const MeTensorPtr &tensor, const std::string &format) { + // get tensor data type size + // ȡ MeTensor ʹСtype_sizeռõֽ + MS_EXCEPTION_IF_NULL(tensor); + size_t type_size = GetDataTypeSize(tensor->data_type()); + if (type_size == kErrorSize) { // ʹСȡʧܣ־ؿָ롣 + MS_LOG(ERROR) << "The Me Tensor data type size is wrong, type size is: " << type_size; + return nullptr; + } + // ȡ MeTensor ԪݻСdata_buff_size + size_t elements_num = IntToSize(tensor->ElementsNum()); + + // get tensor buff size + size_t data_buff_size = elements_num * type_size; + if (data_buff_size == 0) { // ݻСΪ 0ʾϢ־ + MS_LOG(INFO) << "The Me Tensor data buff size is 0."; + } + // create ge tensor + // GeTensorDesc 󣬲 MeTensor ״ͺݸʽϢӦ GeTensor + auto desc = GetGeTensorDesc(tensor->shape_c(), tensor->data_type(), format); + if (desc == nullptr) { // GeTensorDesc ʧܣ־ؿָ롣 + MS_LOG(ERROR) << "Failed to get Tensor Desc"; + return nullptr; + } + // GeTensor 󣬲Ӧݻ״͵Ϣ + GeTensorPtr tensor_ptr = make_shared(*desc, static_cast(tensor->data_c()), data_buff_size); + if (tensor_ptr != nullptr) { // GeTensor ɹתɹʾϢָ򴴽 GeTensor shared_ptr + MS_LOG(INFO) << "Convert Me Tensor to Ge Tensor success!"; + } + return tensor_ptr; +} + +// ConvertGeTensors ڽ GeTensor`ge_tensors`תΪӦ MeTensorMindSpore +// еתĽ `ge_tensors` Ascend AI Core еڴ洢 +// `request_dims` һ `ge_tensors` ͬСָת MeTensor ״ShapeVector +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors, + const std::vector &request_dims) { + std::vector outputs; + // `ge_tensors` еÿ GeTensorתΪӦ MeTensor + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = nullptr; + // ֵ `index` ȡӦ״`request_dims` + if (index < request_dims.size()) { + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], request_dims[index]); + } else { // ״Сڵǰ `index`ʹÿյ״ת + ShapeVector empty_shape; + me_tensor_ptr = ConvertGeTensor(ge_tensors[index], empty_shape); + } + + if (me_tensor_ptr != nullptr) { // תɹת MeTensor 洢 `outputs` С + outputs.emplace_back(me_tensor_ptr); + } else { // תʧܣӦĴ־Ѿɹת MeTensor `outputs` + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; // ת MeTensor `outputs` +} + +// ConvertGeTensors ڽ GeTensor`ge_tensors`תΪӦ MeTensorMindSpore еתĽ +//`ge_tensors` Ascend AI Core еڴ洢 +std::vector TransformUtil::ConvertGeTensors(const std::vector &ge_tensors) { + std::vector outputs; + // `ge_tensors` еÿ GeTensorתΪӦ MeTensor + for (size_t index = 0; index < ge_tensors.size(); index++) { + MeTensorPtr me_tensor_ptr = ConvertGeTensor(ge_tensors[index]); + if (me_tensor_ptr != nullptr) { // תɹת MeTensor 洢 `outputs` С + outputs.emplace_back(me_tensor_ptr); + } else { // תʧܣӦĴ־Ѿɹת MeTensor `outputs` + MS_LOG(ERROR) << "Convert Ge Tensor " << index << " to Me Tensor failed!"; + return outputs; + } + } + return outputs; // ת MeTensor `outputs` +} + +// ConvertGeDataType ڽ GeDataTypeAscend AI Core еͣתΪӦ MeDataTypeMindSporeеͣ +//`type` Ascend AI Core еͣҪתΪӦ MeDataType +MeDataType TransformUtil::ConvertGeDataType(const GeDataType &type) { + switch (type) { // ڲͬ GeDataType ֵӦͽת + case GeDataType::DT_FLOAT16: + return MeDataType::kNumberTypeFloat16; + case GeDataType::DT_FLOAT: + return MeDataType::kNumberTypeFloat32; + case GeDataType::DT_DOUBLE: + return MeDataType::kNumberTypeFloat64; + case GeDataType::DT_INT64: + return MeDataType::kNumberTypeInt64; + case GeDataType::DT_INT32: + return MeDataType::kNumberTypeInt32; + case GeDataType::DT_INT16: + return MeDataType::kNumberTypeInt16; + case GeDataType::DT_INT8: + return MeDataType::kNumberTypeInt8; + case GeDataType::DT_BOOL: + return MeDataType::kNumberTypeBool; + case GeDataType::DT_UINT8: + return MeDataType::kNumberTypeUInt8; + case GeDataType::DT_UINT16: + return MeDataType::kNumberTypeUInt16; + case GeDataType::DT_UINT32: + return MeDataType::kNumberTypeUInt32; + case GeDataType::DT_UINT64: + return MeDataType::kNumberTypeUInt64; + // δг GeDataType ֵ޷תֵ MeDataType::kTypeUnknownʾδ֪͡ + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL_SUB_UINT8: + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_DUAL: + return MeDataType::kTypeUnknown; + default: + return MeDataType::kTypeUnknown; + } +} + +namespace { +// IsGeShapeCompatible ڼ GeTensor ״ `ge_shape` Ƿ״ `request_dims` ݡ +// `ge_shape` GeTensor ״`request_dims` ״ +bool IsGeShapeCompatible(const GeShape &ge_shape, const ShapeVector &request_dims) { + MS_LOG(INFO) << "GeTensor's shape is " << TransformUtil::PrintVector(ge_shape.GetDims()); + MS_LOG(INFO) << "Me request shape is " << TransformUtil::PrintVector(request_dims); + + const int GE_DIMS = 4; + std::vector ge_dims = ge_shape.GetDims(); + if (request_dims.size() > ge_dims.size()) { // ά GeTensor ά˵״ݣ false + MS_LOG(ERROR) << "Request shape's dims count greater than ge shape's"; + return false; + } + + // convert NHWC to NCHW + // ά 1 GeTensor ά 4ҶӦάֵ NHWC NCHW תؼݡ + if ((request_dims.size() == 1) && (ge_dims.size() == GE_DIMS) && (request_dims[0] == ge_dims[1]) && + (ge_dims[0] == 1) && (ge_dims[2] == 1) && (ge_dims[3] == 1)) { + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; + } + + std::string::size_type i = 0; + // һȽ `request_dims` `ge_shape` άֵκάֵȣ˵״ݣ false + for (; i < request_dims.size(); i++) { + if (ge_dims[i] != request_dims[i]) { + MS_LOG(ERROR) << "Request shape's dims value not equal to ge shape's"; + return false; + } + } + // `request_dims` ѱȽϵάȺδṩάȣҪ `ge_shape` жӦάֵΪ1˵״ݣ false + for (; i < ge_dims.size(); i++) { + if (ge_dims[i] != 1) { + MS_LOG(ERROR) << "GeShape's extend dims is not equal to 1"; + return false; + } + } + // ״ݣ true + MS_LOG(INFO) << "Ge tensor shape and request shape is compatible"; + return true; +} +} // namespace + +// ConvertMeShape ڽ MeTensor ״ʾתΪ GeTensor ״ʾ +GeShape TransformUtil::ConvertMeShape(const ShapeVector &me_dims) { + std::vector ge_dims; + // `me_dims` еάֵһµ vector `ge_dims` Уʹµ vector һ GeShape + (void)std::copy(me_dims.begin(), me_dims.end(), std::back_inserter(ge_dims)); + return GeShape(ge_dims); // GeShape 󣬱ʾ MeTensor ״תΪ GeTensor ״Ľ +} + +// ConvertGeShape ڽ GeTensor ״ʾתΪ MeTensor ״ʾ +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape) { + // GeShape еάֵһµ ShapeVector `me_dims` Уʹµ ShapeVector ʾ MeTensor + ShapeVector me_dims; + std::vector ge_dims = ge_shape.GetDims(); + (void)std::copy(ge_dims.begin(), ge_dims.end(), std::back_inserter(me_dims)); + return me_dims; // `me_dims`ʾ GeTensor ״תΪ MeTensor ״Ľ +} + +// ConvertGeShape ڽ GeTensor ״ʾתΪ MeTensor ״ʾ +// `ge_shape` GeTensor ״ GeShape ʾ +// `request_dims` MeTensor ״ ShapeVector ʾ +ShapeVector TransformUtil::ConvertGeShape(const GeShape &ge_shape, const ShapeVector &request_dims) { + vector ret; + if (ge_shape.GetDimNum() == 0) { + MS_LOG(DEBUG) << "GeTensor's shape is scalar"; + return ret; + } + if (IsGeShapeCompatible(ge_shape, request_dims) == true) { //Ƚ GeShape MeTensor ״Ƿ + ret = request_dims; //򷵻 MeTensor ״ `request_dims` + } else { //򷵻 GeTensor ״ת MeTensor ״ `ret` + MS_LOG(ERROR) << "GeShape and Me request shape are incompatible, return GeShape"; + ret = ConvertGeShape(ge_shape); + } + return ret; +} + +// GenerateMeTensor ڸݸ GeTensor `ge_tensor`Ӧ MeTensor 󣬲״ݸƵMeTensor С +// `ge_tensor` Ǹ GeTensor 󣬱ʾת GeTensor +// `me_dims` MeTensor ״ShapeVector ʾ +// `me_type` MeTensor ͣ TypeId ʾ +MeTensorPtr TransformUtil::GenerateMeTensor(const GeTensorPtr &ge_tensor, const ShapeVector &me_dims, + const TypeId &me_type) { + MeTensor me_tensor(me_type, me_dims); + + // Get the writable data pointer of the tensor and cast it to its data type + // ȡ MeTensor Ŀдָ룬תΪָ + auto me_data_ptr = reinterpret_cast(me_tensor.data_c()); + size_t me_data_size = static_cast(me_tensor.data().nbytes()); + MS_EXCEPTION_IF_NULL(me_data_ptr); + MS_EXCEPTION_IF_NULL(ge_tensor); + if (me_data_size < ge_tensor->GetSize()) { + MS_LOG(ERROR) << "ME tensor data size[" << me_data_size << " bytes] is less than GE tensor [" + << ge_tensor->GetSize() << " bytes]"; + return nullptr; + } + + // Copy or use the writable data pointer of the ME tensor + // ƻʹ MeTensor Ŀдָ + MS_EXCEPTION_IF_NULL(ge_tensor->GetData()); + if (ge_tensor->GetSize() == 0) { + MS_LOG(ERROR) << "GE tensor data size is zero!"; + return nullptr; + } + + // Use memcpy here, not memcpy_s, just because the size of ge_tensor may be bigger than 2GB + // which is the size limit of memcpy_s + // ʹ memcpy ݿʹ memcpy_sΪ ge_tensor ĴСܴ 2GB + // memcpy_s 2GB ĴСơ + (void)memcpy(me_data_ptr, ge_tensor->GetData(), ge_tensor->GetSize()); + + return make_shared(me_tensor); +} + +// ConvertGeTensor ڽ GeTensor `ge_tensor` תΪӦ MeTensor +// `ge_tensor` Ǹ GeTensor 󣬱ʾת GeTensor +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr &ge_tensor) { + MS_EXCEPTION_IF_NULL(ge_tensor); + // ȡ GeTensor ״תΪ MeTensor ״ + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape(); + vector me_dims = ConvertGeShape(ge_shape); + // ȡ GeTensor ͣתΪ MeTensor + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType()); + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; + } + // GenerateMeTensor ת MeTensor ״ͣӦ MeTensor + return GenerateMeTensor(ge_tensor, me_dims, type_id); +} + +// if request_dims is empty, use ge tensor's shape,otherwise convert to request shape +// ConvertGeTensor ڽ GeTensor `ge_tensor` תΪӦ MeTensor 󣬲ݸ MeTensor ״ `request_dims` ת +// `ge_tensor` Ǹ GeTensor 󣬱ʾת GeTensor +// `request_dims` Ҫ MeTensor ״ GeTensor ״мԼת +MeTensorPtr TransformUtil::ConvertGeTensor(const GeTensorPtr ge_tensor, const ShapeVector &request_dims) { + MS_EXCEPTION_IF_NULL(ge_tensor); + // ȡ GeTensor ״ MeTensor ״ `request_dims` мԼת + GeShape ge_shape = ge_tensor->GetTensorDesc().GetShape(); + vector me_dims = ConvertGeShape(ge_shape, request_dims); + // GeTensor + MS_LOG(INFO) << "GE tensor type is " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + // Create a tensor with wanted data type and shape + // ָͺ״ MeTensor + TypeId type_id = ConvertGeDataType(ge_tensor->GetTensorDesc().GetDataType()); + if (type_id == MeDataType::kTypeUnknown) { + MS_LOG(ERROR) << "Could not convert Ge Tensor because of unsupported data type: " + << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + return nullptr; //תʧܣ򷵻 nullptr + } + return GenerateMeTensor(ge_tensor, me_dims, type_id); //ָ룬ʾתɹ +} + +//PrintGeTensor ڴӡ GeTensor +std::string TransformUtil::PrintGeTensor(const GeTensorPtr ge_tensor) { + std::string ret; + if (ge_tensor == nullptr) { // ge_tensor ǷΪ + MS_LOG(ERROR) << "Input ge tensor is nullptr"; //Ϊգ־ؿַ + return ret; + } + //ȡge_tensor ͣʹ MakeVector ge_tensor תӦ + //͵IJͬתɲͬ͵ uint32_tfloat_tint32_tdouble_tint64_tuint64_tint16_tuint16_tint8_t uint8_t + //ʹ PrintVector תӡַظַ + MS_LOG(INFO) << "Ge Tensor data type is : " << static_cast(ge_tensor->GetTensorDesc().GetDataType()); + switch (static_cast(ge_tensor->GetTensorDesc().GetDataType())) { + case GeDataType::DT_UINT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT32: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DOUBLE: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT64: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_INT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT16: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_DUAL_SUB_INT8: + case GeDataType::DT_INT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_UINT8: + case GeDataType::DT_DUAL_SUB_UINT8: + ret = PrintVector(MakeVector(ge_tensor->GetData(), ge_tensor->GetSize())); + break; + case GeDataType::DT_FLOAT16: + case GeDataType::DT_BOOL: + case GeDataType::DT_UNDEFINED: + case GeDataType::DT_DUAL: + // ge_tensor Ͳֵ֧бУ DT_FLOAT16DT_BOOLDT_UNDEFINED DT_DUAL־ؿַ + default: + MS_LOG(ERROR) << "Unsupported to print type:" << static_cast(ge_tensor->GetTensorDesc().GetDataType()) + << " ge tensor"; + break; + } + return ret; +} +} // namespace transform +} // namespace mindspore -- 2.34.1 From b4da936e0a3147437773dfd10912e0805dd1f6c4 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Wed, 27 Sep 2023 20:44:42 +0800 Subject: [PATCH 54/72] ADD file via upload --- mindspore/ccsrc/transform-update/adasum.py | 330 +++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/adasum.py diff --git a/mindspore/ccsrc/transform-update/adasum.py b/mindspore/ccsrc/transform-update/adasum.py new file mode 100644 index 00000000000..df34aa94ab6 --- /dev/null +++ b/mindspore/ccsrc/transform-update/adasum.py @@ -0,0 +1,330 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""adasum""" +import copy +import hashlib +import math +from mindspore.nn.cell import Cell +from mindspore.communication.management import create_group +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.ops.operations._inner_ops import Send, Receive + + +__all__ = ["AdaSum"] + + +MAX_NUM_HASH = 2 ** 31 + + +_update_parameters = C.MultitypeFuncGraph("update_parameters") + + +@_update_parameters.register("Tensor", "Tensor", "Tensor", "Tensor") + #һΪ _update_parameters_after_broadcast ĺڸ² +def _update_parameters_after_broadcast(delta_weight, update_delta_weight, parameter, old_parameter): + shape = F.shape(delta_weight) # ȡ delta_weight ״ + update_delta_weight = P.Reshape()(update_delta_weight, shape) # update_delta_weight ܳ delta_weight ͬ״ + new_parameter = old_parameter - update_delta_weight # µIJֵͨӾɲмȥ update_delta_weight + return P.Assign()(parameter, new_parameter) # ʹ P.Assign() ²ֵ parameter + + # һΪ _send_before_receive ĺڷݲӦ +def _send_before_receive(send_part, send, recv): + send_ok = send(send_part) # ͨ send send_partؽ洢 send_ok + return recv(send_ok) # recv send_ok ΪݣڽӦ + +# һΪ _receive_before_send ĺȽݺ +def _receive_before_send(send_part, send, recv): + receive_ok = recv(send_part) # ͨ recv send_part Ϊݣڽݣս洢 receive_ok + send_part = F.depend(send_part, receive_ok) # ʹ F.depend() ϵȷ send_part ֮ǰִнղ + return F.depend(receive_ok, send(send_part)) # ͨ send send_part Ϊݣڷݣͽ + +# һΪ _send_recv_res ĺڷͺͽݣѧʹ +def _send_recv_res(left_send, recv_part, local_part, allreduce, parameter_divisibility, allreduce_node_num): + """send result and receive result.""" + if parameter_divisibility: # ɷָִ² + recv_part = P.Squeeze()(recv_part) # ȥ recv_part еά + local_part = F.depend(local_part, recv_part) # ϵȷ local_part ʹǰ recv_part ѱ + eps = 1e-12 + # һЩֵЩֵںļ + value_0 = P.ReduceSum()(local_part * recv_part) + eps + if left_send: + value_1 = P.ReduceSum()(local_part * local_part) + eps + value_2 = P.ReduceSum()(recv_part * recv_part) + eps + else: + value_1 = P.ReduceSum()(recv_part * recv_part) + eps + value_2 = P.ReduceSum()(local_part * local_part) + eps + # ԼֵȫֹԼallreduce + value_0 = allreduce(value_0) + value_1 = F.depend(allreduce(value_1), value_0) + value_2 = F.depend(allreduce(value_2), value_1) + # ҷͣleft_sendյĽ res + if left_send: + res = (1 - (value_0 / (2 * value_1))) * local_part + (1 - (value_0 / (2 * value_2))) * recv_part + else: + res = (1 - (value_0 / (2 * value_1))) * recv_part + (1 - (value_0 / (2 * value_2))) * local_part + else: + res = allreduce(local_part) # ɷָֱִȫֹԼallreduce + res /= allreduce_node_num + return res + + +_adasum_opt_forward = C.MultitypeFuncGraph("adasum_opt_forward") + + +@_adasum_opt_forward.register("Bool", "Function", "Bool", "Int64", "Function", "Function", "Tensor") +# һΪ _adasum_opt_forward_process ĺ Adasum Żǰ +def _adasum_opt_forward_process(left_send, allreduce, parameter_divisibility, allreduce_node_num, send, recv, delta_w): + """adasum optimizer process.""" + if parameter_divisibility: # ɷָִ² + delta_w = P.Squeeze()(delta_w) # ȥ delta_w еά + ori_len = F.shape(delta_w)[0] # ȡ delta_w ԭʼ + divide_len = ori_len / 2 # ָλ + left_part = delta_w[:divide_len] # delta_w Ϊ벿 + right_part = delta_w[divide_len:] # delta_w ΪҰ벿 + else: # ɷָֱӽ delta_w Ƶ벿ֺҰ벿 + left_part = delta_w + right_part = delta_w + + if left_send: # ߷ + if parameter_divisibility: # ɷָִзͲݲ + recv_part = _send_before_receive(left_part, send, recv) + else: # ɷָҰ벿Ϊݲ + recv_part = right_part + # µ delta_w + update_delta_w = _send_recv_res(left_send, recv_part, right_part, allreduce, parameter_divisibility, + allreduce_node_num) + else: # ұ߷ + if parameter_divisibility: # ɷָִнղݲ + recv_part = _receive_before_send(right_part, send, recv) + else: # ɷָ벿Ϊݲ + recv_part = left_part + # µ delta_w + update_delta_w = _send_recv_res(left_send, recv_part, left_part, allreduce, parameter_divisibility, + allreduce_node_num) + + return update_delta_w + + +_adasum_opt_rollback = C.MultitypeFuncGraph("adasum_opt_rollback") + + +@_adasum_opt_rollback.register("Bool", "Bool", "Tensor", "Function", "Function") +# һΪ _adasum_opt_rollback_process ĺ Adasum ŻĻع +def _adasum_opt_rollback_process(left_send, parameter_divisibility, delta_w, send, recv): + """adasum optimizer rollback process.""" + if parameter_divisibility: # ɷָִ² + if left_send: # ߷ݣִзͲݲ + recv_part = _send_before_receive(delta_w, send, recv) + else: # ұ߷ݣִнղݲ + recv_part = _receive_before_send(delta_w, send, recv) + + recv_part = P.Squeeze()(recv_part) # ȥݲֵά + recv_part = P.Reshape()(recv_part, (-1,)) # ܽݲֺ delta_w Ϊһά + delta_w = P.Reshape()(delta_w, (-1,)) + # ҷ͵ݲֺ delta_w ƴ + if left_send: + res = P.Concat()((recv_part, delta_w)) + else: + res = P.Concat()((delta_w, recv_part)) + else: + res = delta_w # ɷָֱӷ delta_w + return res + +#һΪ AdaSum ࣬һԶ㣨CellִAdasum㷨 +#㷨ڷֲʽݲѵѧϰģ͡ +class AdaSum(Cell): + r""" + The Adaptive Summation, or AdaSum, is a novel algorithm for improving distributed data + parallel training of Deep Learning models. + + Args: + rank (int): Rank number. + device_number (int): Device number. + group_number (int): Group number. + parameter_tuple (Tuple(Parameter)): Tuple of parameters. + + Inputs: + - **delta_weights** (Tuple(Tensor)) - Tuple of gradients. + - **parameters** (Tuple(Parameter)) - Tuple of current parameters. + - **old_parameters** (Tuple(Parameter)) - Tuple of last parameters. + + Outputs: + - **adasum_parameters** (Tuple(Tensor)) - Tuple of parameters after adasum process. + """ + def __init__(self, rank, device_number, group_number, parameter_tuple): + super(AdaSum, self).__init__() + self.rank = rank + self.device_number = device_number + self.group_number = group_number + self.parameter_tuple = parameter_tuple + self._generate_communication_op() + self.hyper_map = C.HyperMap() + # ͨŲ + # ÷ڴAdasum㷨ͨŲ͡աȫֹԼȲ + def _generate_communication_op(self): + """generate communication op.""" + self.calc_times = int(math.log(self.group_number, 2)) + self.send_node = [] + self.send_list_forward = [] + self.recv_list_forward = [] + self.send_list_rollback = [] + self.recv_list_rollback = [] + self.allreduce_list = [] + self.broadcast_list = [] + self.parameter_divisibility_list = [] + self.allreduce_node_num_list = [] + last_delta_weights = [] + group_start_rank = (self.rank // self.device_number) * self.device_number + + for step in range(self.calc_times): + current_group = self.device_number * (2 ** step) + sr_target = self.rank + if (sr_target // current_group) % 2 == 0: + dest_target = sr_target + current_group + self.send_node.append(True) + else: + dest_target = sr_target - current_group + self.send_node.append(False) + + neighbor_ids = [] + group_name_last = 0 + for index in range(2 ** (step + 1)): + node_rank = self.rank // self.device_number + double_d = 2 ** (step + 1) + neighbor_id = (node_rank // double_d * double_d + index) * self.device_number + \ + self.rank % self.device_number + neighbor_ids.append(neighbor_id) + group_name_last += neighbor_id + group_name = "adasum_" + str(step) + "_" + str(group_name_last) + create_group(group_name, neighbor_ids) + + send_left = [] + send_right = [] + recv_left = [] + recv_right = [] + allreduce_node_num = () + left_delta_weights, right_delta_weights, delta_weights_divisibility = \ + self._get_delta_weights_info(last_delta_weights) + self.parameter_divisibility_list.append(delta_weights_divisibility) + weights_index = 0 + fusion_id = (step + 1) * 3 + for shape, dtype in left_delta_weights: + send_tag = self._hash(step, sr_target, weights_index) + send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") + send.add_prim_attr("fusion", fusion_id) + recv_tag = self._hash(step, dest_target, weights_index) + recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, + group="hccl_world_group") + recv.add_prim_attr("fusion", fusion_id) + send_left.append(send) + recv_left.append(recv) + weights_index += 1 + for shape, dtype in right_delta_weights: + send_tag = self._hash(step, sr_target, weights_index) + send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") + send.add_prim_attr("fusion", fusion_id + 1) + recv_tag = self._hash(step, dest_target, weights_index) + recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, + group="hccl_world_group") + recv.add_prim_attr("fusion", fusion_id + 1) + send_right.append(send) + recv_right.append(recv) + weights_index += 1 + + if self.send_node and self.send_node[-1]: + self.send_list_forward.append(send_left) + self.send_list_rollback.append(send_right) + self.recv_list_forward.append(recv_right) + self.recv_list_rollback.append(recv_left) + last_delta_weights = right_delta_weights + else: + self.send_list_forward.append(send_right) + self.send_list_rollback.append(send_left) + self.recv_list_forward.append(recv_left) + self.recv_list_rollback.append(recv_right) + last_delta_weights = left_delta_weights + + server_all_reduce = P.AllReduce("sum", group_name) + server_all_reduce.add_prim_attr("fusion", fusion_id + 2) + self.allreduce_list.append(server_all_reduce) + + for param_divisibility in delta_weights_divisibility: + if param_divisibility: + allreduce_node_num += (0,) + else: + allreduce_node_num += (2 ** (step + 1),) + self.allreduce_node_num_list.append(allreduce_node_num) + + broadcast_group = [x for x in range(group_start_rank, group_start_rank + self.device_number)] + broadcast_group_name = "broadcast_group_" + str(group_start_rank) + create_group(broadcast_group_name, broadcast_group) + for b_rank in range(len(broadcast_group)): + self.broadcast_list.append(P.Broadcast(b_rank, group=broadcast_group_name)) + self.sync_barrier = P.AllReduce("sum", group=broadcast_group_name) + # ȡݶϢ + # ÷ڻȡݶϢݶȻΪҲ֣жǷɷָ + def _get_delta_weights_info(self, last_delta_weights): + """get delta weights info.""" + half_delta_weights = [] + if last_delta_weights: + half_delta_weights = last_delta_weights + else: + for parameter in self.parameter_tuple: + new_shape = [int(x) for x in parameter.shape] + half_delta_weights.append((new_shape, parameter.dtype)) + left_delta_weights = [] + right_delta_weights = [] + delta_weights_divisibility = () + for shape, dtype in half_delta_weights: + left_shape = copy.deepcopy(shape) + right_shape = copy.deepcopy(shape) + divisibility_flag = False + for i in range(len(shape)): + if shape[i] > 1: + left_shape[i] = int(shape[i] // 2) + right_shape[i] = shape[i] - int(shape[i] // 2) + divisibility_flag = True + break + left_delta_weights.append((left_shape, dtype)) + right_delta_weights.append((right_shape, dtype)) + delta_weights_divisibility += (divisibility_flag,) + return left_delta_weights, right_delta_weights, delta_weights_divisibility + # ϣֵ + # ÷ڼϣֵͨűǩ + def _hash(self, step, target, weights_index): + target = "tag" + str(step) + str(target) + str(weights_index) + target_hash = hashlib.sha1(target.encode()).hexdigest() + hash_res = int(int(target_hash, 16) % MAX_NUM_HASH) + return hash_res + # ִAdasumǰ + # òִִAdasum㷨ǰݵķ͡աȡ + def construct(self, delta_weights, parameters, old_parameters): + forward_weights = [delta_weights] + for i in range(self.calc_times): + process_weights = self.hyper_map(F.partial(_adasum_opt_forward, self.send_node[i], self.allreduce_list[i]), + self.parameter_divisibility_list[i], self.allreduce_node_num_list[i], + self.send_list_forward[i], self.recv_list_forward[i], forward_weights[-1]) + forward_weights.append(process_weights) + for i in range(self.calc_times): + j = self.calc_times - i - 1 + process_weights = self.hyper_map(F.partial(_adasum_opt_rollback, self.send_node[j]), + self.parameter_divisibility_list[j], forward_weights[j + 1], + self.send_list_rollback[j], self.recv_list_rollback[j]) + forward_weights[j] = process_weights + adasum_parameters = self.hyper_map(F.partial(_update_parameters), delta_weights, forward_weights[0], + parameters, old_parameters) + return adasum_parameters -- 2.34.1 From 6bd5d6b02825e73d2d2bcddf806e0d6a152502cb Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Wed, 27 Sep 2023 20:47:40 +0800 Subject: [PATCH 55/72] ADD file via upload --- mindspore/ccsrc/transform-update/base.py | 538 +++++++++++++++++++++++ 1 file changed, 538 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/base.py diff --git a/mindspore/ccsrc/transform-update/base.py b/mindspore/ccsrc/transform-update/base.py new file mode 100644 index 00000000000..08e18ac51c0 --- /dev/null +++ b/mindspore/ccsrc/transform-update/base.py @@ -0,0 +1,538 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""base process""" +import os +import time +import math +import copy +import numpy as np +from scipy import linalg as la +from mindspore.context import ParallelMode +import mindspore.nn as nn +from mindspore.nn.optim import LARS +from mindspore import log as logger +from mindspore.common import Parameter +from mindspore.communication.management import get_group_size +from mindspore.train.serialization import load_checkpoint +from mindspore.parallel._utils import _get_global_rank +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from .less_batch_normalization import CommonHeadLastFN + + +__all__ = ["OptimizerProcess", "ParameterProcess"] + +#δ붨һΪ OptimizerProcess ࣬ڴŻһЩ +#ݶĻGradient CentralizationǩµŻ +class OptimizerProcess: + r""" + Process optimizer for Boost. Currently, this class supports adding GC(grad centralization) tags + and creating new optimizers. + + Args: + opt (Cell): Optimizer used. + + Examples: + >>> import numpy as np + >>> from mindspore import Tensor, Parameter, nn + >>> from mindspore import ops + >>> from mindspore.boost import OptimizerProcess + >>> + >>> class Net(nn.Cell): + ... def __init__(self, in_features, out_features): + ... super(Net, self).__init__() + ... self.weight = Parameter(Tensor(np.ones([in_features, out_features]).astype(np.float32)), + ... name='weight') + ... self.matmul = ops.MatMul() + ... + ... def construct(self, x): + ... output = self.matmul(x, self.weight) + ... return output + ... + >>> size, in_features, out_features = 16, 16, 10 + >>> network = Net(in_features, out_features) + >>> optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> optimizer_process = OptimizerProcess(optimizer) + >>> optimizer_process.add_grad_centralization(network) + >>> optimizer = optimizer_process.generate_new_optimizer() + """ + # ʼ OptimizerProcess + # ݴŻ opt ʼ OptimizerProcess ʵزŻࡣ + def __init__(self, opt): + if isinstance(opt, LARS): + self.is_lars = True + self.single_opt = opt.opt + self.opt_class = type(opt.opt) + self.opt_init_args = opt.opt.init_args + self.lars_init_args = opt.init_args + self.learning_rate = opt.opt.init_learning_rate + else: + self.is_lars = False + self.single_opt = opt + self.opt_class = type(opt) + self.opt_init_args = opt.init_args + self.learning_rate = opt.init_learning_rate + self.origin_params = opt.init_params["params"] + # ֵ + # ڹѵеIJֵ䡣 + def build_params_dict(self, network): + r""" + Build the parameter's dict of the network. + + Args: + network (Cell): The training network. + """ + cells = network.cells_and_names() + params_dict = {} + for _, cell in cells: + for par in cell.get_parameters(expand=False): + params_dict[id(par)] = cell + return params_dict + # ݶĻIJ + # ڹݶĻIJ顣 + def build_gc_params_group(self, params_dict, parameters): + r""" + Build the parameter's group with grad centralization. + + Args: + params_dict (dict): The network's parameter dict. + parameters (list): The network's parameter list. + """ + group_params = [] + for group_param in parameters: + if 'order_params' in group_param.keys(): + group_params.append(group_param) + continue + params_gc_value = [] + params_value = [] + for param in group_param['params']: + if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name: + param_cell = params_dict[id(param)] + if (isinstance(param_cell, nn.Conv2d) and param_cell.group > 1) or \ + isinstance(param_cell, CommonHeadLastFN): + params_value.append(param) + else: + params_gc_value.append(param) + else: + params_value.append(param) + if params_gc_value: + new_group_param = copy.deepcopy(group_param) + new_group_param['params'] = params_gc_value + new_group_param['grad_centralization'] = True + group_params.append(new_group_param) + if params_value: + new_group_param = copy.deepcopy(group_param) + new_group_param['params'] = params_value + group_params.append(new_group_param) + return group_params + # ݶĻ + # ݴݶĻǩ޸IJ顣 + def add_grad_centralization(self, network): + r""" + Add gradient centralization. + + Args: + network (Cell): The training network. + """ + params_dict = self.build_params_dict(network) + + parameters = self.origin_params + if parameters is not None and not isinstance(parameters, list): + parameters = list(parameters) + + if not parameters: + raise ValueError("Optimizer got an empty parameter list.") + + if not isinstance(parameters[0], (dict, Parameter)): + raise TypeError("Only a list of Parameter or dict can be supported.") + + if isinstance(parameters[0], Parameter): + logger.warning("Only group parameters support gradient centralization.") + return + + self.origin_params = self.build_gc_params_group(params_dict, parameters) + # µŻ + # ݱIJͳʼµŻʵظŻ + def generate_new_optimizer(self): + """Generate new optimizer.""" + if self.learning_rate is None: + self.learning_rate = self.single_opt.learning_rate + if not self.is_lars: + opt = self.opt_class(params=self.origin_params, learning_rate=self.learning_rate, **self.opt_init_args) + else: + opt = LARS(self.opt_class(params=self.origin_params, learning_rate=self.learning_rate, \ + **self.opt_init_args), **self.lars_init_args) + + return opt + +#δ붨һΪ ParameterProcess ࣬ڴһЩԶݶȷָ㡣 +class ParameterProcess: + r""" + Process parameter for Boost. Currently, this class supports creating group parameters + and automatically setting gradient segmentation point. + + Examples: + >>> from mindspore import Tensor, Parameter, nn + >>> import mindspore.ops as ops + >>> from mindspore.boost import OptimizerProcess + >>> + >>> class Net(nn.Cell): + ... def __init__(self, in_features, out_features): + ... super(Net, self).__init__() + ... self.weight = Parameter(Tensor(np.ones([in_features, out_features]).astype(np.float32)), + ... name='weight') + ... self.weight2 = Parameter(Tensor(np.ones([in_features, out_features]).astype(np.float32)), + ... name='weight2') + ... self.matmul = ops.MatMul() + ... self.matmul2 = ops.MatMul() + ... + ... def construct(self, x): + ... output = self.matmul(x, self.weight) + ... output2 = self.matmul2(x, self.weight2) + ... return output + output2 + ... + >>> size, in_features, out_features = 16, 16, 10 + >>> network = Net(in_features, out_features) + >>> new_parameter = net.trainable_params()[:1] + >>> parameter_process = ParameterProcess() + >>> group_params = parameter_process.generate_group_params(new_parameter, net.trainable_params()) + """ + def __init__(self): + self._parameter_indices = 1 + # + # 䵽ͬ飬ݶȷָ㡣 + def assign_parameter_group(self, parameters, split_point=None): + r""" + Assign parameter group. + + Args: + parameters (list): The network's parameter list. + split_point (list): The gradient split point of this network. default: None. + """ + if not isinstance(parameters, (list, tuple)) or not parameters: + return parameters + + parameter_len = len(parameters) + if split_point: + split_parameter_index = split_point + else: + split_parameter_index = [parameter_len // 2] + for i in range(parameter_len): + if i in split_parameter_index: + self._parameter_indices += 1 + parameters[i].comm_fusion = self._parameter_indices + return parameters + # ɲ + # ɴвIJбŻá + def generate_group_params(self, parameters, origin_params): + r""" + Generate group parameters. + + Args: + parameters (list): The network's parameter list. + origin_params (list): The network's origin parameter list. + """ + origin_params_copy = origin_params + if origin_params_copy is not None: + if not isinstance(origin_params_copy, list): + origin_params_copy = list(origin_params_copy) + + if not origin_params_copy: + raise ValueError("Optimizer got an empty parameter list.") + + if not isinstance(origin_params_copy[0], (dict, Parameter)): + raise TypeError("Only a list of Parameter or dict can be supported.") + + if isinstance(origin_params_copy[0], Parameter): + group_params = [{"params": parameters}] + return group_params + + group_params = [] + params_name = [param.name for param in parameters] + new_params_count = copy.deepcopy(params_name) + new_params_clone = {} + max_key_number = 0 + for group_param in origin_params_copy: + if 'order_params' in group_param.keys(): + new_group_param = copy.deepcopy(group_param) + new_group_param['order_params'] = parameters + group_params.append(new_group_param) + continue + params_value = [] + for param in group_param['params']: + if param.name in params_name: + index = params_name.index(param.name) + params_value.append(parameters[index]) + new_params_count.remove(param.name) + new_group_param = copy.deepcopy(group_param) + new_group_param['params'] = params_value + group_params.append(new_group_param) + if len(group_param.keys()) > max_key_number: + max_key_number = len(group_param.keys()) + new_params_clone = copy.deepcopy(group_param) + if new_params_count: + params_value = [] + for param in new_params_count: + index = params_name.index(param) + params_value.append(parameters[index]) + if new_params_clone: + new_params_clone['params'] = params_value + group_params.append(new_params_clone) + else: + group_params.append({"params": params_value}) + return group_params + +#úҪڻȡ PCA · +def _get_local_pca_mat_path(weight_load_dir, pca_mat_path, n_component, device_number, network): + """ + get local pca mat path. + + Args: + weight_load_dir (str): The weight(ckpt) file directory to be load. + pca_mat_path (str): the path to load pca mat. Default: None. + n_component (int): pca component. + device_number (int): device number. + network (Cell): The network. + """ + if pca_mat_path is not None and os.path.exists(pca_mat_path) and os.path.isfile(pca_mat_path) and \ + pca_mat_path.endswith(".npy"): + # ṩ pca_mat_path һЧ .npy ļʹø· + full_pca_mat_path = pca_mat_path + pca_mat_exist = True + + else: # weight_load_dir һĿ¼򹹽ʱ PCA ļ· + if weight_load_dir is None or not os.path.exists(weight_load_dir) or not os.path.isdir(weight_load_dir): + raise ValueError("The weight_load_dir: {} is None / not exists / not directory.".format(weight_load_dir)) + + full_pca_mat_path = os.path.join(weight_load_dir, "pca_mat_temp.npy") + pca_mat_exist = False + # PCA ־ļ· + save_pca_end_path = os.path.join(os.path.dirname(full_pca_mat_path), "save_pca_end.txt") + if os.path.exists(save_pca_end_path): + os.remove(save_pca_end_path) + # ȡȫrankͱ PCA · + rank = _get_global_rank() + local_pca_mat_path = full_pca_mat_path[:-4] + "_rank_" + str(rank) + ".npy" + if os.path.exists(local_pca_mat_path): + os.remove(local_pca_mat_path) + # 豸ı򷵻ر PCA · + if rank % device_number != 0: + return local_pca_mat_path + # PCA ѴڣȨݼ PCA 󲢱 + if pca_mat_exist: + pca_mat = np.load(full_pca_mat_path) + else: + data = _load_weights(weight_load_dir, network) + pca_mat = _compute_pca_mat(data, n_component) + np.save(full_pca_mat_path, pca_mat) + # 汾 PCA + _save_local_pca_mat(pca_mat, full_pca_mat_path, n_component) + return local_pca_mat_path + +#úڼȨأckptļеIJݡ +def _load_weights(weight_load_dir, network): + """ + load weights. + + Args: + weight_load_dir (str): The weight(ckpt) file directory to be load. + network (Cell): The network. + """ + # ȡҪݶȵIJб + param_requires_grad_list = [] + for param in network.trainable_params(): + param_requires_grad_list.append(param.name) + # ʼԪ + param_mat_tuple = () + # ȡȨļб + weight_file_list = os.listdir(weight_load_dir) + for file in weight_file_list: # Ȩļб + if not file.endswith('.ckpt'): + continue + file_path = os.path.join(weight_load_dir, file) + # Ȩļ + param_dict = load_checkpoint(file_path) + param_tuple = () + for key, value in param_dict.items(): # صIJֵ + if key in param_requires_grad_list: + param_tuple += (value.asnumpy().reshape((1, -1)),) # ҪݶȣתΪ NumPy 鲢ӵԪ + param = np.concatenate(param_tuple, axis=1) # ԪϲΪһ󣬲ӵԪ + param_mat_tuple += (param,) + param_mat = np.concatenate(param_mat_tuple, axis=0) # мصIJϲΪһ + return param_mat + +#úڼPCAɷַı任 +def _compute_pca_mat(data, n_component, randomized=True): + """ + compute pca mat. + + Args: + data (array): array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + n_component (int): pca component. + randomized (bool) if use randomized svd. + """ + if data.shape[0] < n_component: + raise ValueError("The samples: {} is less than: n_component {}.".format(data.shape[0], n_component)) + + if randomized: # ʹSVDPCAɷ + components = _randomized_svd(data, n_component) + else: # ʹñ׼SVDPCAɷ + components = _full_svd(data, n_component) + + return components + +#úʹSVDPCAɷ֡ +def _randomized_svd(data, n_component, n_oversample=10, n_iter=1): + """ + compute pca mat use randomized svd. + + Args: + data (array): array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + n_component (int): pca component. + n_oversample (int): oversample num + n_iter (int): iteration count + """ + mean = np.mean(data, axis=0) # ݵľֵݾл + data -= mean + n_random = n_component + n_oversample # 㳬Ŀ + n_samples, n_features = data.shape # ȡݵ + transpose = n_samples < n_features # жǷҪתݾ + if transpose: + data = data.T + q_mat = _randomized_range_finder(data, n_random, n_iter) # ʹķȡռͶӰQ + b_mat = q_mat.T @ data # BB = Q^T * X + u_hat, _, vt_mat = la.svd(b_mat, full_matrices=False) # BSVDֽ⣬õU_hatV^T + del b_mat + # յUV^T + u_mat = np.dot(q_mat, u_hat) + u_mat, vt_mat = _svd_flip(u_mat, vt_mat, transpose) + if transpose: # ҪתãUתã򷵻V^Tǰn_component + components = u_mat[:, :n_component].T + else: + components = vt_mat[:n_component, :] + return components + +#úʹñ׼SVDSingular Value DecompositionPCAɷ֡ +def _full_svd(data, n_component): + """ + compute pca mat use full svd. + + Args: + data (array): array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + n_component (int): pca component. + """ + mean = np.mean(data, axis=0) # ݵľֵݾл + data -= mean + u, _, v = la.svd(data, full_matrices=False) # ʹñ׼SVDֽݾ󣬵õUSV^T + _, v = _svd_flip(u, v) # UV^Tķ + components = v[:n_component] # ѡȡǰn_componentɷ֣V^Tǰn_component + return components + +#úʹSVDPCAɷ֡ +def _randomized_range_finder(data, size, n_iter=1): + """ + compute pca mat use randomized svd. + + Args: + data (array): array-like of shape (n_samples, n_features) + Training data, where `n_samples` is the number of samples + and `n_features` is the number of features. + size (int): n_component + n_oversample. + n_iter (int): iteration count + """ + # һʼQ״Ϊ (n_features, size) + q_mat = np.random.normal(size=(data.shape[1], size)) + # жεQ + for _ in range(n_iter): + q_mat, _ = la.lu(data @ q_mat, permute_l=True) # ʹLUֽ data @ Q + q_mat, _ = la.lu(data.T @ q_mat, permute_l=True) # ʹLUֽ data^T @ Q + + q_mat, _ = la.qr(data @ q_mat, mode="economic") # data @ Q оQRֽ⣬õQ + return q_mat + +#úSVDֵֽ⣩жԽзתȷһԡ +def _svd_flip(u, v, transpose=True): + """ + svd flip. + + Args: + u (ndarray): the output of `linalg.svd`. + v (ndarray): the output of `linalg.svd`. + transpose (bool): if data is transposed. + """ + if not transpose: + # δתãֵԪصķת + max_abs_cols = np.argmax(np.abs(u), axis=0) + signs = np.sign(u[max_abs_cols, range(u.shape[1])]) + u *= signs + v *= signs[:, np.newaxis] + else: + # ѾתãֵԪصķת + max_abs_rows = np.argmax(np.abs(v), axis=1) + signs = np.sign(v[range(v.shape[0]), max_abs_rows]) + u *= signs + v *= signs[:, np.newaxis] + return u, v + +#úڽ PCAɷַ󱣴浽ļС +def _save_local_pca_mat(pca_mat, full_pca_mat_path, n_component): + """ + save pca mat. + + Args: + pca_mat (numpy.ndarray): pca mat to be saved. + full_pca_mat_path (str): the path of full pca mat. + n_component (int): pca component. + """ + parallel_mode = auto_parallel_context().get_parallel_mode() + rank_size = 1 if parallel_mode == ParallelMode.STAND_ALONE else get_group_size() + local_dim = math.ceil(n_component / rank_size) + for rank_id in range(rank_size): + start_index = rank_id * local_dim + end_index = (rank_id + 1) * local_dim + pca_start_index = min(n_component, start_index) + pca_end_index = min(n_component, end_index) + p_local = np.zeros([local_dim, pca_mat.shape[1]]) + if pca_start_index != pca_end_index: + p_local[0: pca_end_index - pca_start_index, :] = pca_mat[pca_start_index: pca_end_index, :] + local_pca_mat_path = full_pca_mat_path[:-4] + "_rank_" + str(rank_id) + ".npy" + np.save(local_pca_mat_path, p_local) + save_pca_end_path = os.path.join(os.path.dirname(full_pca_mat_path), "save_pca_end.txt") + os.mknod(save_pca_end_path) + +#úڴӱؼ PCAɷַ +def _load_local_pca_mat(local_pca_mat_path, timeout): + """ + load pca mat. + + Args: + local_pca_mat_path (str): local pca mat file path. + """ + save_pca_end_path = os.path.join(os.path.dirname(local_pca_mat_path), "save_pca_end.txt") + start_time = time.time() + while True: + current_time = time.time() + if (current_time - start_time) > timeout: + raise RuntimeError("the time of waiting to load local pca mat is larger than {} second.".format(timeout)) + if os.path.exists(save_pca_end_path): + break + time.sleep(5) + pca_mat = np.load(local_pca_mat_path) + return pca_mat -- 2.34.1 From dbf45f752c13c73e2feaf86f5d9feb61f456b830 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Wed, 27 Sep 2023 20:48:20 +0800 Subject: [PATCH 56/72] ADD file via upload --- mindspore/ccsrc/transform-update/boost.py | 409 ++++++++++++++++++++++ 1 file changed, 409 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/boost.py diff --git a/mindspore/ccsrc/transform-update/boost.py b/mindspore/ccsrc/transform-update/boost.py new file mode 100644 index 00000000000..e4caa9f2e61 --- /dev/null +++ b/mindspore/ccsrc/transform-update/boost.py @@ -0,0 +1,409 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""boost""" +import threading +from mindspore.nn.optim import SGD +from .less_batch_normalization import LessBN +from .grad_freeze import GradientFreeze +from .base import OptimizerProcess, ParameterProcess +from .base import _get_local_pca_mat_path + + +__all__ = ["AutoBoost"] + +_boost_config_mode = ["auto", "manual", "enable_all", "disable_all"] +_boost_config_level = { + "O0": { + "less_bn": False, + "grad_freeze": False, + "adasum": False, + "grad_accumulation": False, + "dim_reduce": False}, + "O1": { + "less_bn": True, + "grad_freeze": True, + "adasum": False, + "grad_accumulation": False, + "dim_reduce": False}, + "O2": { + "less_bn": True, + "grad_freeze": True, + "adasum": True, + "grad_accumulation": False, + "dim_reduce": False}} + +#һΪ AutoBoost ࣬ṩԶѵĹܡ +class AutoBoost: + r""" + Provide auto accelerating for network. + + Args: + level (str): Boost config level. Default: "O0". + boost_config_dict (dict): User config hyperparameter dict, recommended config format: + + .. code-block:: + + { + "boost": { + "mode": "auto", + "less_bn": False, + "grad_freeze": False, + "adasum": False, + "grad_accumulation": False, + "dim_reduce": False + }, + "common": { + "gradient_split_groups": [50, 100], + "device_number": 8 + }, + "less_bn": { + "fn_flag": True, + "gc_flag": True + }, + "grad_freeze": { + "param_groups": 10, + "freeze_type": 1, + "freeze_p": 0.7, + "total_steps": 65536 + } + "grad_accumulation": { + "grad_accumulation_step": 1 + }, + "dim_reduce": { + "rho": 0.55, + "gamma": 0.9, + "alpha": 0.001, + "sigma": 0.4, + "n_components": 32, + "pca_mat_path": None, + "weight_load_dir": None, + "timeout": 1800 + } + } + + - boost: + + - mode (str): How to set the boost. Supports ["auto", "manual", "enable_all", "disable_all"]. + Default: "auto". + + - auto: Depend on the argument "boost_level" in class Model. + - manual: Depend on "boost_config_dict". + - enable_all: Set all boost functions true. + - disable_all: Set all boost functions false. + + - less_bn (bool): Whether to apply less_bn function. Default: False. + - grad_freeze: (bool): Whether to apply grad_freeze function. Default: False. + - adasum (bool): Whether to apply adasum function. Default: False. + - grad_accumulation (bool): Whether to apply grad_accumulation function. Default: False. + - dim_reduce (bool): Whether to apply dim_reduce function. Default: False. + + If set dim_reduce true, other functions will be false. + If set grad_freeze true and dim_reduce false, other functions will be false. + + - common: + + - gradient_split_groups (list): The gradient split point of this network. Default: [50, 100]. + - device_number (int): Device number. Default: 8. + + - less_bn: + + - fn_flag (bool): Whether changing fc to fn. Default: True. + - gc_flag (bool): Whether to apply gc. Default: True. + + - grad_freeze: + + - param_groups (int): The number of parameter groups. Default: 10. + - freeze_type (int): Gradient freeze grouping strategy, select from [0, 1]. Default: 1. + - freeze_p (float): Gradient freezing probability. Default: 0.7. + - total_steps (int): Total training steps. Default: 65536. + + - grad_accumulation: + + - grad_accumulation_step (int): Steps to accumulate gradients. Default: 1. + + - dim_reduce: + + The leading principles of dim_reduce: + + .. math:: + + \begin{align} + grad\_k &= pca\_mat \cdot grad\\ + dk &= - bk \cdot grad\_k\\ + sk &= rho ^ m \cdot dk\\ + delta\_loss &= sigma \cdot grad\_k.T \cdot sk + \end{align} + + Here: + + - pca_mat (array): Shape (k*n), k is part of n_components, n is the size of weight. + - bk (array): Shape (k*k), is the symmetric positive definite matrix in Quasi-Newton method. + + we need to find the m satisfy: + + .. math:: + new\_loss < old\_loss + delta\_loss + + Then, get delta_grad to update the weights for model: + + .. math:: + + \begin{align} + grad\_k\_proj &= pca\_mat.T \cdot grad\_k\\ + new\_grad\_momentum &= gamma \cdot old\_grad\_momentum + grad - grad\_k\_proj\\ + delta\_grad &= alpha \cdot new\_grad\_momentum - pca\_mat.T \cdot sk + \end{align} + + - rho (float): Generally, it does not need to be modified. Default: 0.55. + - gamma (float): Generally, it does not need to be modified. Default: 0.9. + - alpha (float): Generally, it does not need to be modified. Default: 0.001. + - sigma (float): Generally, it does not need to be modified. Default: 0.4. + - n_components (int): PCA component. Default: 32. + - pca_mat_path (str): The path to load pca mat. Default: None. + - weight_load_dir (str): The directory to load weight files saved as ckpt. Default: None. + - timeout (int): Waiting time to load local pca mat. Default: 1800 (second). + + User can load the config through the JSON file or use the dictionary directly. + The unconfigured parameters will adopt the default values. + + Raises: + ValueError: The boost mode not in ["auto", "manual", "enable_all", "disable_all"]. + + Supported Platforms: + ``Ascend`` + + Examples: + >>> from mindspore.boost import AutoBoost + >>> #1) when configuring the dict directly: + >>> boost_config_dict = {"boost": {"mode": "auto"}} + >>> boost = AutoBoost("O1", boost_config_dict) + >>> + >>> #2) when loading the dict from a json file: + >>> import json + >>> boost_json = "/path/boost_config.json" + >>> with open(boost_json, 'r') as fp: + >>> boost_config_dict = json.load(fp) + >>> boost = AutoBoost("O1", boost_config_dict) + """ + _instance_lock = threading.Lock() + _instance = None + + def __init__(self, level="O0", boost_config_dict=""): + if level not in _boost_config_level.keys(): + level = "O0" + if self._instance.level is None: + # ʼһЩĬϲ + self.level = level + self.boost_config_dict = boost_config_dict + self._fn_flag = True + self._gc_flag = True + self._param_groups = 10 + self._freeze_type = 1 + self._freeze_p = 0.7 + self._total_steps = 65536 + self.gradient_groups = None + self.device_number = 8 + self.grad_accumulation_step = 1 + self.rho = 0.55 + self.gamma = 0.9 + self.alpha = 0.001 + self.sigma = 0.4 + self.n_components = 32 + self.pca_mat_path = None + self.weight_load_dir = None + self.local_pca_mat_path = None + self.timeout = 1800 + self.boost_config = self._get_configuration(level, self.boost_config_dict) + self._param_processer = ParameterProcess() + + # pylint: disable=unused-argument + def __new__(cls, *args, **kwargs): + if AutoBoost._instance is None: + with AutoBoost._instance_lock: + if AutoBoost._instance is None: + AutoBoost._instance = object.__new__(cls) + AutoBoost._instance.level = None + AutoBoost._instance.boost_config_dict = None + return AutoBoost._instance + + # Զѵ + def network_auto_process_train(self, network, optimizer): + r""" + Boost network train. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + """ + # άȽ͹ + if self.boost_config["dim_reduce"]: + # ȡPCA· + self.local_pca_mat_path = _get_local_pca_mat_path(self.weight_load_dir, self.pca_mat_path, + self.n_components, self.device_number, network) + optimizer = SGD(network.trainable_params(), learning_rate=1) # һµSGDŻάȽ + setattr(optimizer, "dim_reduce", True) # Żdim_reduceΪTrue + return network, optimizer # ޸ĺŻ + # ˼Batch NormalizationLessBN + if self.boost_config["less_bn"]: + network = LessBN(network, fn_flag=self._fn_flag) # ʹLessBN޸ģѡǷıȫӲΪȫ׼ + optimizer_process = OptimizerProcess(optimizer) # Ż + group_params = self._param_processer.assign_parameter_group(network.trainable_params(), # Ŀѵ + self.gradient_groups) + # µIJ鲢Ż + optimizer_process.origin_params = \ + self._param_processer.generate_group_params(group_params, optimizer_process.origin_params) + if self._gc_flag: # ݶĻgrad centralizationݶĻ + optimizer_process.add_grad_centralization(network) + optimizer = optimizer_process.generate_new_optimizer() # µŻ + # ݶȶᣨGradient Freeze + if self.boost_config["grad_freeze"]: + # ݶȶᴦ + freeze_processer = GradientFreeze(self._param_groups, self._freeze_type, + self._freeze_p, self._total_steps) + network, optimizer = freeze_processer.freeze_generate(network, optimizer) # Żݶȶᴦ + # Adasum + if self.boost_config["adasum"]: + setattr(optimizer, "adasum", True) # ŻadasumΪTrue + return network, optimizer # ޸ĺŻ + + # Զ + def network_auto_process_eval(self, network): + r""" + Boost network eval. + + Args: + network (Cell): The inference network. + """ + # άȽ͹ + if self.boost_config["dim_reduce"]: + return network # ֱӷ磬 + # ˼Batch NormalizationLessBN + if self.boost_config["less_bn"]: + network = LessBN(network) # ʹLessBN޸ + + return network # ޸ĺ + + def set_fn_flag(self, fn_flag): + self._fn_flag = fn_flag + + def set_gc_flag(self, gc_flag): + self._gc_flag = gc_flag + + def set_param_groups(self, param_groups): + self._param_groups = param_groups + + def set_freeze_type(self, freeze_type): + self._freeze_type = freeze_type + + def set_freeze_p(self, freeze_p): + self._freeze_p = freeze_p + + def set_total_steps(self, total_steps): + self._total_steps = total_steps + + def set_device_number(self, device_number): + self.device_number = device_number + + def set_grad_accumulation_step(self, grad_accumulation_step): + self.grad_accumulation_step = grad_accumulation_step + + def set_gradient_split_groups(self, gradient_groups): + # ݶȷָǷΪбֵ쳣 + if not isinstance(gradient_groups, (list, int)): + raise ValueError(f"gradient_groups `{gradient_groups}` is not in (list, int)") + # תΪһԪصб + if isinstance(gradient_groups, int): + gradient_groups = list(gradient_groups) + self.gradient_groups = gradient_groups # ݶȷָ + + def set_rho(self, rho): + self.rho = rho + + def set_gamma(self, gamma): + self.gamma = gamma + + def set_alpha(self, alpha): + self.alpha = alpha + + def set_sigma(self, sigma): + self.sigma = sigma + + def set_n_components(self, n_components): + self.n_components = n_components + + def set_pca_mat_path(self, pca_mat_path): + self.pca_mat_path = pca_mat_path + + def set_weight_load_dir(self, weight_load_dir): + self.weight_load_dir = weight_load_dir + + def set_timeout(self, timeout): + self.timeout = timeout + + def _get_configuration(self, level, boost_config_dict): + """Get configuration.""" + level_config = _boost_config_level[level] # ȡָϢ + if not boost_config_dict: # ûṩûֵ䣬ֱӷָϢ + return level_config + + mode = "auto" # Ĭʹ "auto" ģʽ + # ûֵǷ boost ãԼǷָ boost ģʽ + if 'boost' in boost_config_dict and 'mode' in boost_config_dict['boost']: + mode = boost_config_dict['boost']['mode'] + # boost ģʽǷϷϷֵ쳣 + if mode not in _boost_config_mode: + raise ValueError("The boost mode must be in {}, but got {}".format(_boost_config_mode, mode)) + # ݲͬģʽϢ + if mode == "manual": + # ֶģʽʹûṩ boost Ĭ + for key, value in boost_config_dict["boost"].items(): + if key in level_config: + level_config[key] = value + elif mode == "enable_all": # ģʽ boost Ϊ True + level_config = {key: True for key in level_config} + elif mode == "disable_all": # ǽģʽ boost Ϊ False + level_config = {key: False for key in level_config} + # ȡЧ boost Ϣָֻе "common" е + valid_boost_each_mode_config = [] + for key, boost_each_mode_config in boost_config_dict.items(): + if key in level_config.keys() and level_config[key] or key == "common": + valid_boost_each_mode_config.append(boost_each_mode_config) + # ЧϢöӦ÷ + for boost_each_mode_config in valid_boost_each_mode_config: + for key_s in boost_each_mode_config.keys(): + if key_s in self._boost_config_func_map: + self._boost_config_func_map[key_s](self, boost_each_mode_config[key_s]) + # յϢ + return level_config + + _boost_config_func_map = { + "fn_flag": set_fn_flag, + "gc_flag": set_gc_flag, + "param_groups": set_param_groups, + "freeze_type": set_freeze_type, + "freeze_p": set_freeze_p, + "total_steps": set_total_steps, + "device_number": set_device_number, + "gradient_split_groups": set_gradient_split_groups, + "grad_accumulation_step": set_grad_accumulation_step, + "rho": set_rho, + "gamma": set_gamma, + "alpha": set_alpha, + "sigma": set_sigma, + "n_components": set_n_components, + "pca_mat_path": set_pca_mat_path, + "weight_load_dir": set_weight_load_dir, + "timeout": set_timeout + } -- 2.34.1 From 4ce7ba963723bf40d84700a7f88ccd4021fa6dc4 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Wed, 27 Sep 2023 20:49:04 +0800 Subject: [PATCH 57/72] ADD file via upload --- .../transform-update/boost_cell_wrapper.py | 585 ++++++++++++++++++ 1 file changed, 585 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/boost_cell_wrapper.py diff --git a/mindspore/ccsrc/transform-update/boost_cell_wrapper.py b/mindspore/ccsrc/transform-update/boost_cell_wrapper.py new file mode 100644 index 00000000000..30750411257 --- /dev/null +++ b/mindspore/ccsrc/transform-update/boost_cell_wrapper.py @@ -0,0 +1,585 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Boost Mode Cell Wrapper.""" +from mindspore.nn.wrap import TrainOneStepCell +import mindspore.context as context +from mindspore.context import ParallelMode +from mindspore.parallel._utils import _get_global_rank, _get_device_num, _get_gradients_mean +from mindspore.communication.management import get_group_size, create_group +from mindspore.nn.cell import Cell +from mindspore.common import Tensor, RowTensor +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +from mindspore.ops import functional as F +from mindspore.ops import composite as C +from mindspore.ops import operations as P +from mindspore.common import dtype as mstype +from .boost import AutoBoost +from .grad_freeze import FreezeOpt, freeze_cell +from .adasum import AdaSum +from .dim_reduce import DimReduce +from .grad_accumulation import gradient_accumulation_op, gradient_clear_op +from .base import _load_local_pca_mat + + +__all__ = ["BoostTrainOneStepCell", "BoostTrainOneStepWithLossScaleCell"] + + +_get_delta_weight = C.MultitypeFuncGraph("_get_delta_weight") + + +@_get_delta_weight.register("Tensor", "Tensor") +def _get_delta_weight_process(new_parameter, old_parameter): + delta_w = old_parameter - new_parameter + return delta_w + + +_save_weight = C.MultitypeFuncGraph("_save_weight") + + +@_save_weight.register("Tensor", "Tensor") +def _save_weight_process(new_parameter, old_parameter): + return P.Assign()(new_parameter, old_parameter) + + +_grad_scale = C.MultitypeFuncGraph("grad_scale") +reciprocal = P.Reciprocal() + + +@_grad_scale.register("Tensor", "Tensor") +def tensor_grad_scale(scale, grad): + return grad * F.cast(reciprocal(scale), F.dtype(grad)) + + +@_grad_scale.register("Tensor", "RowTensor") +def tensor_grad_scale_row_tensor(scale, grad): + return RowTensor(grad.indices, + grad.values * F.cast(reciprocal(scale), F.dtype(grad.values)), + grad.dense_shape) + + +_grad_overflow = C.MultitypeFuncGraph("_grad_overflow") +grad_overflow = P.FloatStatus() + + +@_grad_overflow.register("Tensor") +def _tensor_grad_overflow(grad): + return grad_overflow(grad) + + +@_grad_overflow.register("RowTensor") +def _tensor_grad_overflow_row_tensor(grad): + return grad_overflow(grad.values) + +#һΪ BoostTrainOneStepCell ࣬ڰװѵŻʵѵеһ +class BoostTrainOneStepCell(TrainOneStepCell): + r""" + Boost Network training package class. + + Wraps the network with an optimizer. The resulting Cell is trained with input '\*inputs'. + The backward graph will be created in the construct function to update the parameter. Different + parallel modes are available for training. + + Args: + network (Cell): The training network. The network only supports single output. + optimizer (Union[Cell]): Optimizer for updating the weights. + sens (numbers.Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0. + + Inputs: + - **(\*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`. + + Outputs: + Tensor, a tensor means the loss value, the shape of which is usually :math:`()`. + + Raises: + TypeError: If `sens` is not a number. + + Supported Platforms: + ``Ascend`` ``GPU`` ``CPU`` + + Examples: + >>> from mindspore import boost + >>> net = Net() + >>> loss_fn = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> #1) Using the WithLossCell existing provide + >>> loss_net = nn.WithLossCell(net, loss_fn) + >>> train_net = boost.BoostTrainOneStepCell(loss_net, optim) + >>> + >>> #2) Using user-defined WithLossCell + >>> class MyWithLossCell(Cell): + ... def __init__(self, backbone, loss_fn): + ... super(MyWithLossCell, self).__init__(auto_prefix=False) + ... self._backbone = backbone + ... self._loss_fn = loss_fn + ... + ... def construct(self, x, y, label): + ... out = self._backbone(x, y) + ... return self._loss_fn(out, label) + ... + ... @property + ... def backbone_network(self): + ... return self._backbone + ... + >>> loss_net = MyWithLossCell(net, loss_fn) + >>> train_net = boost.BoostTrainOneStepCell(loss_net, optim) + """ + + def __init__(self, network, optimizer, sens=1.0): + # øijʼѵ磨networkŻoptimizerжȣsens + super(BoostTrainOneStepCell, self).__init__(network, optimizer, sens) + self.hyper_map = C.HyperMap() # һӳ䣨HyperMapڲв + self.freeze = isinstance(optimizer, FreezeOpt) # ǷʹݶȶᣨFreezeOptȡŻIJΪȨ + if not self.freeze: + self.weights = self.optimizer.parameters + self.train_strategy = getattr(self.optimizer, 'train_strategy', None) # ȡŻѵԣtrain_strategyĬΪNone + # ǷݲлģʽʹݶۻϢӦIJ + auto_boost = AutoBoost() + self.use_grad_accumulation = self.parallel_mode in (ParallelMode.DATA_PARALLEL, ParallelMode.STAND_ALONE) + self.use_grad_accumulation = self.use_grad_accumulation & auto_boost.boost_config["grad_accumulation"] + self.max_accumulation_step = 1 + if self.use_grad_accumulation: + self.max_accumulation_step = auto_boost.grad_accumulation_step + if self.max_accumulation_step <= 1: + self.max_accumulation_step = 1 + self.use_grad_accumulation = False + # ݶۻ򴴽ۻݶȵIJͱ + self.accumulation_step = Parameter(Tensor(0, dtype=mstype.int32), name="accumulation_step") + if self.use_grad_accumulation: + self.grad_accumulation = self.weights.clone(prefix="grad_accumulation", init='zeros') + # ǷάȽάdimension reduction + self.enable_dim_reduce = self.check_dim_reduce_enable() + if self.enable_dim_reduce: + local_pca_mat_path = auto_boost.local_pca_mat_path + rho = auto_boost.rho + gamma = auto_boost.gamma + alpha = auto_boost.alpha + sigma = auto_boost.sigma + _rank = _get_global_rank() + _rank_size = 1 if self.parallel_mode == ParallelMode.STAND_ALONE else get_group_size() + _device_number = auto_boost.device_number + n_components = auto_boost.n_components + timeout = auto_boost.timeout + pca_mat = _load_local_pca_mat(local_pca_mat_path, timeout) # رص PCA + self.weights_clone = ParameterTuple(self.weights).clone(prefix="weights_clone", init="same") # ȨصĿ¡άȽά + # άȽά + self.dim_reduce = DimReduce(self.network, self.optimizer, self.weights, pca_mat, n_components, rho, gamma, + alpha, sigma, _rank, _rank_size) + # ʼݶȶصIJͱ + self.freeze_nets = None + self.step = Parameter(Tensor(0, dtype=mstype.int32)) + if self.freeze: + if self.reducer_flag: + self.mean = _get_gradients_mean() + self.degree = _get_device_num() + else: + self.mean = None + self.degree = None + # ݶȶ + self.freeze_nets = freeze_cell(self.reducer_flag, self.network, self.optimizer, self.sens, + self.grad, self.use_grad_accumulation, self.mean, self.degree, + self.max_accumulation_step) + # Ƿ AdasumAll-reduce㷨ز + self.enable_adasum = self.check_adasum_enable() + self.sync_tensor = Parameter(Tensor(0, dtype=mstype.int32)) + if self.enable_adasum: + _rank = _get_global_rank() + _rank_size = get_group_size() + _device_number = auto_boost.device_number + self.device_number = _device_number + group_number = _rank_size // _device_number + # 㵱ǰ豸 + self.server_rank = _rank % _device_number + # ÿ豸IJΧ + parameter_rank_number = len(self.weights) // _device_number + self.start = [x * parameter_rank_number for x in range(_device_number)] + self.end = [(x + 1) * parameter_rank_number for x in range(_device_number)] + self.end[-1] = len(self.weights) + # ȡǰ豸IJ + current_weights = self.weights[self.start[self.server_rank]: self.end[self.server_rank]] + # ݶȵĿ¡ Adasum 㷨 + self.grad_clone = ParameterTuple(current_weights).clone(prefix="delta_weight") + # Adasum 㷨 + self.adasum = AdaSum(_rank, _device_number, group_number, self.grad_clone) + # ÿIJ + self.degree = int(self.degree / group_number) + # б + group_list = [list(range(x * self.degree, (x + 1) * self.degree)) for x in range(group_number)] + # 㵱ǰ豸 + current_index = _rank // _device_number + server_group_name = "allreduce_" + str(current_index) + # ֲʽݶȹԼ + create_group(server_group_name, group_list[current_index]) + self.grad_reducer = DistributedGradReducer(self.weights, self.mean, self.degree, group=server_group_name) + + def construct(self, *inputs): + if self.freeze: # ݶȶ + loss = self.gradient_freeze_process(*inputs) # ִݶȶ㷨Ĵ + else: # ִѵ + loss = self.network(*inputs) # ǰ򴫲ʧ loss + sens = F.fill(loss.dtype, loss.shape, self.sens) # ʹʧ loss ͺ״֪ȣsens + grads = self.grad(self.network, self.weights)(*inputs, sens) # ݶ grads + grads = self.grad_reducer(grads) # ݶ grads йԼӦѵ + if self.use_grad_accumulation: # ݶۻ + loss = self.gradient_accumulation_process(loss, grads, sens, *inputs) # ִݶۻ㷨Ĵ + else: + if self.enable_dim_reduce: # άȽͣDimension Reduction + # ִάȽ㷨Ĵ + loss = F.depend(loss, self.dim_reduce(loss, grads, sens, self.weights, self.weights_clone, *inputs)) + elif self.enable_adasum: # Adasum 㷨 + loss = F.depend(loss, self.adasum_process(loss, grads)) # ִ Adasum 㷨Ĵ + else: # ִݲ + loss = F.depend(loss, self.optimizer(grads)) + return loss # ʧֵ loss + + def gradient_freeze_process(self, *inputs): + r""" + Gradient freeze algorithm process. + + Args: + inputs (tuple(Tensor)): Tuple of input tensors with shape :math:`(N, \ldots)`. + + Outputs: + - **loss** (Tensor) - Network loss, tensor with shape :math:`()`. + """ + if self.train_strategy is None: # δѵԣʹĬϵ step max_index + step = self.step + max_index = len(self.freeze_nets) + else: # 򣬸ѵԻȡǰ step max_index + step = self.train_strategy[self.step] + max_index = len(self.train_strategy) + loss = self.freeze_nets[step](*inputs) # ʹõǰ step ִݶȶǰ򴫲ʧ loss + if self.step + 1 >= max_index: # step max_indexΪ 0 + self.step = 0 + else: + self.step += 1 + return loss # step max_indexΪ 0 + + def gradient_accumulation_process(self, loss, grads, sens, *inputs): + r""" + Gradient accumulation algorithm process. + + Args: + loss (Tensor): Tensor with shape :math:`()`. + grads (tuple(Tensor)): Tuple of gradient tensors. + sens (Tensor): Tensor with shape :math:`()`. + inputs (tuple(Tensor)): Tuple of input tensors with shape :math:`(N, \ldots)`. + + Outputs: + - **loss** (Tensor) - Network loss, tensor with shape :math:`()`. + """ + # ʹݶۻݶۻ grad_accumulation Уڵǰʧ loss + loss = F.depend(loss, self.hyper_map(F.partial(gradient_accumulation_op, self.max_accumulation_step), + self.grad_accumulation, grads)) + self.accumulation_step += 1 + + if self.accumulation_step >= self.max_accumulation_step: # ۻﵽۻִ² + if self.enable_dim_reduce: # άȼ٣άȼٲ + loss = F.depend(loss, self.dim_reduce(loss, self.grad_accumulation, sens, self.weights, + self.weights_clone, *inputs)) + elif self.enable_adasum: # Adasum Adasum + loss = F.depend(loss, self.adasum_process(loss, self.grad_accumulation)) + else: # ʹŻݶۻ grad_accumulation + loss = F.depend(loss, self.optimizer(self.grad_accumulation)) + self.accumulation_step = 0 + + if self.accumulation_step == 0: # ۻΪ 0ݶۻ grad_accumulation + loss = F.depend(loss, self.hyper_map(F.partial(gradient_clear_op), self.grad_accumulation)) + + return loss # ʧֵ loss + + def adasum_process(self, loss, grads): + r""" + Adasum algorithm process. + + Args: + loss (Tensor): Tensor with shape :math:`()`. + grads (tuple(Tensor)): Tuple of gradient tensors. + + Outputs: + - **loss** (Tensor) - Network loss, tensor with shape :math:`()`. + """ + loss = F.depend(loss, self.optimizer(grads)) # ʹŻݶ gradsڵǰʧ loss + rank_weights = self.weights[self.start[self.server_rank]: self.end[self.server_rank]] # 㵱ǰ豸Ȩ rank_weights + grad_clone = F.depend(self.grad_clone, loss) # ʧ loss ݶȸ grad_clone + delta_w = self.hyper_map(F.partial(_get_delta_weight), rank_weights, grad_clone) # ʹóӳݶȱ仯 delta_w + adasum_res = self.adasum(delta_w, rank_weights, grad_clone) # Adasum adasum_res + sync_tensor = F.depend(self.sync_tensor, adasum_res) # ͬ sync_tensor adasum_res + sync_flag = self.adasum.sync_barrier(sync_tensor) # ִ Adasum ͬϲ + # ÿ豸ͬȨ + for i in range(self.device_number): + weight_tuple = self.weights[self.start[i]: self.end[i]] + node_rank = F.depend(weight_tuple, sync_flag) + update_weights = self.adasum.broadcast_list[i](node_rank) + if i == self.server_rank: # Ƿ豸ʹ grad_clone Ȩ + self.hyper_map(F.partial(_save_weight), self.grad_clone, update_weights) + else: # 豸ʹȨԪȨ + self.hyper_map(F.partial(_save_weight), weight_tuple, update_weights) + return loss # ʧֵ loss + + #÷ڼǷ Adasum 㷨 + def check_adasum_enable(self): + r""" + Check adasum enable. + """ + if not getattr(self.optimizer, "adasum", None) or not self.reducer_flag: # ŻǷ adasum ԣҼǷ reducer_flag + return False + _rank_size = get_group_size() # ȡǰѵܴС + _device_number = 8 # 趨豸 + group_number = _rank_size // _device_number # ѵ + is_enable = bool(group_number > 1 and group_number & (group_number - 1) == 0) # жǷAdasum ҪѵΪ 2 + return is_enable + + #÷ڼǷάȽͣdim_reduceܡ + def check_dim_reduce_enable(self): + r""" + Check dim_reduce enable. + """ + if not getattr(self.optimizer, "dim_reduce", None): # ŻǷ dim_reduce + return False + return True # dim_reduce ԣʾάȽ͹ + + +class BoostTrainOneStepWithLossScaleCell(BoostTrainOneStepCell): + r""" + Boost Network training with loss scaling. + + This is a training step with loss scaling. It takes a network, an optimizer and possibly a scale update + Cell as args. The loss scale value can be updated in both host side or device side. The + BoostTrainOneStepWithLossScaleCell will be compiled to be graph which takes `*inputs` as input data. + The Tensor type of `scale_sense` is acting as loss scaling value. If you want to update it on host side, + the value must be provided. If the Tensor type of `scale_sense` is not given, the loss scale update logic + must be provide by Cell type of `scale_sense`. + + Args: + network (Cell): The training network. The network only supports single output. + optimizer (Cell): Optimizer for updating the weights. + scale_sense (Union[Tensor, Cell]): If this value is Cell type, the loss scaling update logic cell.If this value + is Tensor type, Tensor with shape :math:`()` or :math:`(1,)`. + + Inputs: + - **(*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`. + + Outputs: + Tuple of 3 Tensor, the loss, overflow flag and current loss scaling value. + + - **loss** (Tensor) - Tensor with shape :math:`()`. + - **overflow** (Tensor) - Tensor with shape :math:`()`, type is bool. + - **loss scaling value** (Tensor) - Tensor with shape :math:`()` + + Raises: + TypeError: If `scale_sense` is neither Cell nor Tensor. + ValueError: If shape of `scale_sense` is neither (1,) nor (). + + Supported Platforms: + ``Ascend`` ``GPU`` + + Examples: + >>> import numpy as np + >>> from mindspore import Tensor, Parameter, nn + >>> import mindspore.ops as ops + >>> from mindspore.nn import WithLossCell + >>> from mindspore import dtype as mstype + >>> from mindspore import boost + >>> + >>> class Net(nn.Cell): + ... def __init__(self, in_features, out_features): + ... super(Net, self).__init__() + ... self.weight = Parameter(Tensor(np.ones([in_features, out_features]).astype(np.float32)), + ... name='weight') + ... self.matmul = ops.MatMul() + ... + ... def construct(self, x): + ... output = self.matmul(x, self.weight) + ... return output + ... + >>> size, in_features, out_features = 16, 16, 10 + >>> #1) when the type of scale_sense is Cell: + >>> net = Net(in_features, out_features) + >>> loss = nn.MSELoss() + >>> optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> net_with_loss = WithLossCell(net, loss) + >>> manager = nn.DynamicLossScaleUpdateCell(loss_scale_value=2**12, scale_factor=2, scale_window=1000) + >>> train_network = boost.BoostTrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_sense=manager) + >>> input = Tensor(np.ones([out_features, in_features]), mstype.float32) + >>> labels = Tensor(np.ones([out_features,]), mstype.float32) + >>> output = train_network(input, labels) + >>> + >>> #2) when the type of scale_sense is Tensor: + >>> net = Net(in_features, out_features) + >>> loss = nn.MSELoss() + >>> optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> net_with_loss = WithLossCell(net, loss) + >>> inputs = Tensor(np.ones([size, in_features]).astype(np.float32)) + >>> label = Tensor(np.zeros([size, out_features]).astype(np.float32)) + >>> scaling_sens = Tensor(np.full((1), np.finfo(np.float32).max), dtype=mstype.float32) + >>> train_network = boost.BoostTrainOneStepWithLossScaleCell(net_with_loss, optimizer, scale_sense=scaling_sens) + >>> output = train_network(inputs, label) + """ + def __init__(self, network, optimizer, scale_sense): + super(BoostTrainOneStepWithLossScaleCell, self).__init__(network, optimizer, sens=None) + self.base = Tensor(1, mstype.float32) # Tensorڼʧ + self.reduce_sum = P.ReduceSum(keep_dims=False) # ReduceSum P.ReduceSum ʵ + self.less_equal = P.LessEqual() # ڱȽϲ P.LessEqual ʵ + self.allreduce = P.AllReduce() # AllReduce P.AllReduce ʵ + self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE) # ǷǷֲʽѵ + self.gpu_target = (context.get_context("device_target") == "GPU") # Ƿ GPU 豸 + self.loss_scaling_manager = None # ʼʧŹ + if isinstance(scale_sense, Cell): # scale_sense Cell ͣʹָʧŹ + self.loss_scaling_manager = scale_sense + self.scale_sense = Parameter(Tensor(scale_sense.get_loss_scale(), dtype=mstype.float32), + name="scale_sense") + elif isinstance(scale_sense, Tensor): # scale_sense Tensor ֶָͣʧֵ + if scale_sense.shape == (1,) or scale_sense.shape == (): # ʧֵ״ǷϷ + self.scale_sense = Parameter(scale_sense, name='scale_sense') + else: + raise ValueError("The shape of scale_sense must be (1,) or (), but got {}".format(scale_sense.shape)) + else: # scale_sense ǺϷͣ׳쳣 + raise TypeError("The scale_sense must be Cell or Tensor, but got {}".format(type(scale_sense))) + + def construct(self, *inputs): + weights = self.weights # ȡȨ + loss = self.network(*inputs) # ʧ + scaling_sens = self.scale_sense # ȡʧֵ + + status, scaling_sens = self._start_overflow_check(loss, scaling_sens) # ʼʧ + + scaling_sens_filled = C.ones_like(loss) * F.cast(scaling_sens, F.dtype(loss)) # ʧʧ + grads = self.grad(self.network, weights)(*inputs, scaling_sens_filled) # ݶȣʧֵݶȽ + grads = self.hyper_map(F.partial(_grad_scale, scaling_sens), grads) + + # get the overflow buffer + # ȡ״̬־ + cond = self._get_overflow_status(status, grads) + overflow = self._process_loss_scale(cond) # ʧţȷǷ + # if there is no overflow, do optimize + # ûִŻ + if not overflow: + if self.use_grad_accumulation: + loss = self.gradient_accumulation_process(loss, grads, scaling_sens_filled, *inputs) + else: + if self.enable_dim_reduce: + loss = F.depend(loss, self.dim_reduce(loss, grads, scaling_sens_filled, self.weights, + self.weights_clone, *inputs)) + elif self.enable_adasum: + loss = F.depend(loss, self.adasum_process(loss, grads)) + else: + loss = F.depend(loss, self.optimizer(grads)) + return loss, cond, scaling_sens # ʧ־ʧֵ + + #úʧֵ + def _set_sense_scale(self, sens): + """ + If the user has set the sens in the training process and wants to reassign the value, he can call + this function again to make modification, and sens needs to be of type Tensor. + + Inputs: + - **sens** (Tensor) - The new sense whose shape and type are the same with original `scale_sense`. + """ + if self.scale_sense and isinstance(sens, Tensor): + self.scale_sense.set_data(sens) + else: + raise TypeError("The input type must be Tensor, but got {}".format(type(sens))) + + #úڿʼ + def _start_overflow_check(self, pre_cond, compute_input): + """ + Start floating-point overflow detection. Create and clear the overflow detection state. + + Specify the argument 'pre_cond' and 'compute_input' to make sure overflow status is cleared at the right time. + Taking this situation as an example, we need to execute state clearing after loss calculation and then detect + overflow in the process of gradient calculation. In this case, pre_cond should be the output of the loss + function, and compute_input should be the input of gradients-computing function. + + Inputs: + - **pre_cond** (Tensor) - A precondition for starting overflow detection. It determines the executing order + of overflow state clearing and prior processions. It makes sure that the function 'start_overflow' + clears status after finishing the process of precondition. + - **compute_input** (object) - The input of subsequent process. Overflow detection should be performed on a + certain computation. Set `compute_input` as the input of the computation, to ensure overflow status is + cleared before executing the computation. + + Outputs: + Tuple[object, object], the first value is False for GPU backend, while it is an instance of + NPUAllocFloatStatus for other backend. The status is used to detect overflow during overflow detection. + The second value is the same as the input of `compute_input`, but contains some information about the + execution order. + """ + status = False + if not self.gpu_target: + # init overflow buffer + # ʼ + status = P.NPUAllocFloatStatus()() + status = F.depend(status, pre_cond) + # clear overflow buffer + # + clear_status = P.NPUClearFloatStatus()(status) + compute_input = F.depend(compute_input, clear_status) + return status, compute_input + + #úڻȡ״̬ + def _get_overflow_status(self, status, compute_output): + """ + Get floating-point overflow status. + + Get overflow results after executing the target process for overflow detection. + + Inputs: + - **status** (object) - A status instance used to detect the overflow. + - **compute_output** - Overflow detection should be performed on a certain computation. Set `compute_output` + as the output of the computation, to ensure overflow status is acquired before executing the + computation. + + Outputs: + bool, whether the overflow occurs or not. + """ + if not self.gpu_target: + + status = F.depend(status, compute_output) + get_status = P.NPUGetFloatStatus()(status) + status = F.depend(status, get_status) + # sum overflow buffer elements, 0:not overflow , >0:overflow + # Ԫأ0 ʾû>0 ʾ + flag_sum = self.reduce_sum(status, (0,)) + else: + flag_sum = self.hyper_map(F.partial(_grad_overflow), compute_output) + flag_sum = P.AddN()(flag_sum) + # convert flag_sum to scalar + # flag_sum תΪ + flag_sum = P.Reshape()(flag_sum, (())) + + if self.is_distributed: + # sum overflow flag over devices + # 豸ϻ־ + flag_reduce = self.allreduce(flag_sum) + overflow = self.less_equal(self.base, flag_reduce) + else: + overflow = self.less_equal(self.base, flag_sum) + return overflow + + #úڸʧű + def _process_loss_scale(self, overflow): + """ + Calculate loss scale according to the overflow. + + Inputs: + - **overflow** (bool) - Whether the overflow occurs or not. + + Outputs: + bool, overflow value. + """ + if self.loss_scaling_manager is not None: + return self.loss_scaling_manager(self.scale_sense, overflow) + return overflow -- 2.34.1 From 645e43f2f3890770cdcf363d5364327f3883db73 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Wed, 27 Sep 2023 20:49:57 +0800 Subject: [PATCH 58/72] ADD file via upload --- .../ccsrc/transform-update/dim_reduce.py | 331 ++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/dim_reduce.py diff --git a/mindspore/ccsrc/transform-update/dim_reduce.py b/mindspore/ccsrc/transform-update/dim_reduce.py new file mode 100644 index 00000000000..1ff7bfd2765 --- /dev/null +++ b/mindspore/ccsrc/transform-update/dim_reduce.py @@ -0,0 +1,331 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""dim_reduce""" +import math +import numpy as np +from mindspore.nn.cell import Cell +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter, ParameterTuple +from mindspore.common import dtype as mstype + + +__all__ = ["DimReduce"] + + +_scale_grad = C.MultitypeFuncGraph("_scale_grad") + + +@_scale_grad.register("Tensor", "Tensor") +def _scale_grad_process(scale, grad): + grad = F.cast(grad, mstype.float32) + grad = P.Div()(grad, scale) + return grad + + +_save_weight = C.MultitypeFuncGraph("_save_weight") + + +@_save_weight.register("Tensor", "Tensor") +def _save_weight_process(parameter, new_parameter): + return P.Assign()(parameter, new_parameter) + + +_pca_projection = C.MultitypeFuncGraph("_pca_projection") + + +@_pca_projection.register("Tensor", "Tensor") +def _pca_projection_process(pca_mat, grad): + grad_k = P.MatMul()(pca_mat, F.reshape(grad, (-1, 1))) + return grad_k + + +_pca_back_projection = C.MultitypeFuncGraph("_pca_back_projection") + + +@_pca_back_projection.register("Tensor", "Tensor", "Tensor") +#úPCAͶӰ +def _pca_back_projection_process(grad_k, pca_mat, grad): + grad_proj = P.MatMul()(F.transpose(pca_mat, (1, 0)), grad_k) + grad_proj_reshape = F.reshape(grad_proj, F.shape(grad)) + return grad_proj_reshape + + +_update_grad_res_momentum = C.MultitypeFuncGraph("_update_grad_res_momentum") + + +@_update_grad_res_momentum.register("Float32", "Float32", "Tensor", "Tensor", "Tensor") +#úڸݶȲв +def _update_grad_res_momentum_process(gamma, alpha, grad_res_momentum, grad, grad_proj): + grad_res_momentum_new = gamma * grad_res_momentum + grad - grad_proj + P.Assign()(grad_res_momentum, grad_res_momentum_new) + res = alpha * grad_res_momentum_new + return res + + +_get_delta_weight = C.MultitypeFuncGraph("_get_delta_weight") + + +@_get_delta_weight.register("Tensor", "Tensor", "Tensor") +def _get_delta_weight_process(rho, dn, grad_res_momentum): + delta_weight = grad_res_momentum - rho * dn + return delta_weight + + +class DimReduce(Cell): + r""" + The dimension reduce training, is a novel algorithm for accelerating convergence of Deep Learning models. + + .. math:: + + \begin{align} + grad\_k &= pca\_mat \cdot grad\\ + dk &= - bk \cdot grad\_k\\ + sk &= rho ^ m \cdot dk\\ + delta\_loss &= sigma \cdot grad\_k.T \cdot sk + \end{align} + + Here: + + - pca_mat (array): Shape (k*n), k is part of n_components, n is the size of weight. + - bk (array): Shape (k*k), is the symmetric positive definite matrix in Quasi-Newton method. + + we need to find the m satisfy: + + .. math:: + new\_loss < old\_loss + delta\_loss + + Then, get delta_grad to update the weights for model: + + .. math:: + + \begin{align} + grad\_k\_proj &= pca\_mat.T \cdot grad\_k\\ + new\_grad\_momentum &= gamma \cdot old\_grad\_momentum + grad - grad\_k\_proj\\ + delta\_grad &= alpha \cdot new\_grad\_momentum - pca\_mat.T \cdot sk + \end{align} + + Args: + network (Cell): The training network. The network only supports single output. + optimizer (Union[Cell]): Optimizer for updating the weights. + weight (Tuple(Parameter)): Tuple of parameters. + pca_mat_local (numpy.ndarray): For PCA operation, k*n, k is part of n_components, n is the size of weight. + n_components (int): PCA.components. + rho (float): Coefficient. + gamma (float): Coefficient. + alpha (float): Coefficient. + sigma (float): Coefficient. + rank (int): Rank number. + rank_size (int): Rank size. + + Inputs: + - **loss** (Tensor) - Tensor with shape :math:`()`. + - **old_grad** (Tuple(Tensor)) - Tuple of gradient tensors. + - **weight** (Tuple(Tensor)) - Tuple of parameters. + - **weight_clone** (Tuple(Tensor)) - clone of weight + - **(\*inputs)** (Tuple(Tensor)) - Tuple of input tensors with shape :math:`(N, \ldots)`. + + Outputs: + - **loss** (Tensor) - Tensor with shape :math:`()`. + """ + def __init__(self, network, optimizer, weight, pca_mat_local, n_components, rho, gamma, alpha, sigma, rank, + rank_size): + super(DimReduce, self).__init__() + self.network = network + self.optimizer = optimizer + self.rank = rank + self.rank_size = rank_size + self.gamma = gamma + self.alpha = alpha + self.sigma = sigma + + self.float_type = mstype.float32 + self._set_rho_list(rho) + self._set_local_pca_mat(pca_mat_local, n_components, weight) + self._set_init_parameter(weight) + + self.hyper_map = C.HyperMap() + self.concat = P.Concat() + self.matmul = P.MatMul() + self.mul = P.Mul() + self.add = P.Add() + + #úΪrhoѣֵбԱŻ㷨ʹá + def _set_rho_list(self, rho): + """set rho list info.""" + self.max_search_time = 2 # + self.rho_list = [] + for i in range(self.max_search_time): + self.rho_list.append(Tensor(np.power(rho, i), dtype=self.float_type)) + self.rho_list.append(Tensor(0, dtype=self.float_type)) + + #úñPCAϢ + def _set_local_pca_mat(self, pca_mat_local, n_components, parameter_tuple): + """set pca info.""" + self.n_components = n_components + local_dim = math.ceil(self.n_components / self.rank_size) + + self.start_index = self.rank * local_dim + self.end_index = (self.rank + 1) * local_dim + + start = 0 + self.pca_list_local = () + # ģͲPCA󰴲Сֿ飬洢pca_list_local + for param in parameter_tuple: + size = np.shape(param.asnumpy().reshape((-1, 1)))[0] + self.pca_list_local += (Tensor(pca_mat_local[:, start:start + size], dtype=self.float_type),) + start += size + + self.dk_pad_flag = False + pad_num = self.rank_size * local_dim - self.n_components + if pad_num: # Ҫ䣬dk_pad_flagΪTrue䲿 + self.dk_pad_flag = True + self.dk_pad_part = Tensor(np.zeros([pad_num, 1]), dtype=self.float_type) + + if self.rank_size > 1: + self.broadcast_list = [] + for i in range(self.rank_size): # 㲥бڿ豸ͨ + broadcast = P.Broadcast(i) + self.broadcast_list.append(broadcast) + self.allreduce = P.AllReduce() # ִȫֹԼ + self.allgather = P.AllGather() # ִȫռ + + def _set_init_parameter(self, parameter_tuple): + """init parameters.""" + self.true_flag = Tensor(True) + self.false_flag = Tensor(False) + self.epsilon = np.power(10.0, -20) + # ʼgk_last + self.gk_last = Parameter(Tensor(np.zeros([self.n_components, 1]), dtype=self.float_type), name="gk_last") + self.gk_last_init = Parameter(Tensor(False), name="gk_last_init") + # ʼbksk + self.bk = Parameter(Tensor(np.eye(self.n_components), dtype=self.float_type), name="bk") + self.sk = Parameter(Tensor(np.zeros([self.n_components, 1]), dtype=self.float_type), name="sk") + self.eye = Tensor(np.eye(self.n_components), dtype=self.float_type) + # ʼgrad_res_momentum + self.grad_res_momentum = ParameterTuple(parameter_tuple).clone(prefix="grad_res_momentum", init="zeros") + # ʼgk_last_backbk_back + self.gk_last_back = Parameter(Tensor(np.zeros([self.n_components, 1]), dtype=self.float_type), + name="gk_last_back") + self.bk_back = Parameter(Tensor(np.eye(self.n_components), dtype=self.float_type), name="bk_back") + # ʼgrad_proj_initdn_init + self.grad_proj_init = ParameterTuple(parameter_tuple).clone(prefix="grad_proj_init", init="zeros") + self.dn_init = ParameterTuple(parameter_tuple).clone(prefix="dn_init", init="zeros") + + def construct(self, loss, old_grad, loss_scale, weight, weight_clone, *inputs): + # Ȩغݶ + weight = F.depend(weight, loss) + old_grad = F.depend(old_grad, weight) + old_grad = self.hyper_map(F.partial(_scale_grad, loss_scale), old_grad) + old_loss = self.allreduce(loss) / self.rank_size if self.rank_size > 1 else loss + # gk_local + gk_local = self.hyper_map(_pca_projection, self.pca_list_local, old_grad) + gk_local = F.addn(gk_local) + gk_pad = self.allgather(gk_local) if self.rank_size > 1 else gk_local + gk_pad = F.reshape(gk_pad, (-1, 1)) + gk = gk_pad[0:self.n_components, :] + # Ȩغݶ + _save_weight(self.gk_last_back, self.gk_last) + _save_weight(self.bk_back, self.bk) + # dk + dk = self._apply_quasi_newton_update(gk) + if self.dk_pad_flag: + dk_pad = self.concat((dk, self.dk_pad_part)) + else: + dk_pad = dk + dk_local = dk_pad[self.start_index: self.end_index, :] + # dn_localgrad_proj_local + dn_local = self.hyper_map(F.partial(_pca_back_projection, dk_local), self.pca_list_local, old_grad) + grad_proj_local = self.hyper_map(F.partial(_pca_back_projection, gk_local), self.pca_list_local, old_grad) + dn = self.dn_init if self.rank_size > 1 else dn_local + grad_proj = self.grad_proj_init if self.rank_size > 1 else grad_proj_local + if self.rank_size > 1: + for broadcast in self.broadcast_list: + dn_part = broadcast(dn_local) + dn = self.hyper_map(self.add, dn, dn_part) + grad_proj_part = broadcast(grad_proj_local) + grad_proj = self.hyper_map(self.add, grad_proj, grad_proj_part) + # ʹrhoֵ + rho, find = self._line_search(gk, dk, dn, old_loss, weight, weight_clone, *inputs) + if not find: + _save_weight(self.gk_last, self.gk_last_back) + _save_weight(self.bk, self.bk_back) + # ݶȲִŻ + update_grad = self.hyper_map(F.partial(_update_grad_res_momentum, self.gamma, self.alpha), + self.grad_res_momentum, old_grad, grad_proj) + delta_weight = self.hyper_map(F.partial(_get_delta_weight, rho), dn, update_grad) + update = self.optimizer(delta_weight) + weight = F.depend(weight, update) + clone = self.hyper_map(_save_weight, weight_clone, weight) + loss = F.depend(loss, clone) + return loss + + #úѰѵ rho ֵȷģͲĸ²ʧӡ + def _line_search(self, gk, dk, dn, old_loss, weight, weight_clone, *inputs): + """line search rho.""" + res = self.rho_list[-1] # ʼrhoΪСֵ + find = self.false_flag # ʼfind־ΪFalse + for i in range(self.max_search_time): # rhoѡбԲͬrhoֵ + find = self._find_rho(gk, dk, dn, old_loss, weight, weight_clone, self.rho_list[i], *inputs) + if find: # ҵrhoֵresѭ + res = self.rho_list[i] + break + return res, find + + #úڲ rho ֵķȷģͲĸ²ᵼʧӡ + def _find_rho(self, gk, dk, dn, old_loss, weight, weight_clone, rho, *inputs): + """search rho.""" + res = self.false_flag # ʼres־ΪFalse + # snʧӦrhoֵĸ + sn = self.hyper_map(F.partial(self.mul, -1 * rho), dn) + sn = F.depend(sn, old_loss) + update = self.optimizer(sn) + # ʹrhoֵºʧ + new_loss = F.depend(self.network(*inputs), update) + # Ƿֲʽѵʧȫƽ + if self.rank_size > 1: + new_loss = self.allreduce(new_loss) / self.rank_size + # ʧı仯 + old_loss_delta = old_loss + self.sigma * rho * F.squeeze(self.matmul(F.transpose(gk, (1, 0)), dk)) + # ʹrhoֵºʧСھɵʧʾҵһrhoֵ + if old_loss_delta > new_loss: + _save_weight(self.sk, rho * dk) + res = self.true_flag + # weight_cloneweightֵ + weight_clone = F.depend(weight_clone, old_loss_delta) + restore = self.hyper_map(_save_weight, weight, weight_clone) + res = F.depend(res, restore) + return res + + #úӦţٸ¡ + def _apply_quasi_newton_update(self, gk): + """apply quasi_newton update.""" + if self.gk_last_init: + yk = gk - self.gk_last # gkһεgkIJֵ + g = self.matmul(F.transpose(yk, (1, 0)), self.sk) + g = F.squeeze(g) + if g > self.epsilon: + pk = 1. / g + t1 = self.eye - self.matmul(pk * yk, F.transpose(self.sk, (1, 0))) + new_bk = self.matmul(self.matmul(F.transpose(t1, (1, 0)), self.bk), t1) + \ + self.matmul(pk * self.sk, F.transpose(self.sk, (1, 0))) + _save_weight(self.bk, new_bk) # ţپbk + else: + _save_weight(self.gk_last_init, self.true_flag) # һεʱʼgk_last_init + _save_weight(self.gk_last, gk) # 浱ǰgkֵ + dk = -1 * self.matmul(self.bk, gk) # ţٸºdkֵ + return dk -- 2.34.1 From cd77c72c71518bad5eb927d6ced7ae6e7d524b0f Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sat, 30 Sep 2023 21:42:50 +0800 Subject: [PATCH 59/72] ADD file via upload --- .../ccsrc/transform-update/grad_freeze.py | 454 ++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/grad_freeze.py diff --git a/mindspore/ccsrc/transform-update/grad_freeze.py b/mindspore/ccsrc/transform-update/grad_freeze.py new file mode 100644 index 00000000000..3efff8fdbba --- /dev/null +++ b/mindspore/ccsrc/transform-update/grad_freeze.py @@ -0,0 +1,454 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""grad freeze""" + +import numpy as np + +from mindspore.nn.cell import Cell +from mindspore.nn.optim import Optimizer +from mindspore.common import Tensor +from mindspore.common import dtype as mstype +from mindspore.nn.optim import LARS +from mindspore.nn.wrap.grad_reducer import DistributedGradReducer +from mindspore.ops import functional as F + +from .base import ParameterProcess +from .grad_accumulation import GradientAccumulation + +__all__ = ['GradientFreeze', 'FreezeOpt', 'freeze_cell'] + + +CONTINUOUS_STRATEGY = 0 +INTERVAL_STRATEGY = 1 + + +class FreezeOpt(Cell): + """ + Optimizer that supports gradients freezing training. + + Args: + opt (Cell): non-freezing optimizer instance, such as 'Momentum', 'SGD'. + train_parameter_groups (Union[tuple, list]): Groups of parameters for gradients freezing training. + train_strategy (Union[tuple(int), list(int), Tensor]): Strategy for gradients freezing training. + + Supported Platforms: + ``Ascend`` + """ + def __init__(self, opt, train_parameter_groups=None, train_strategy=None): + super(FreezeOpt, self).__init__() + # ǶŻǷ Optimizer ʵ + # ʼֲǷʹ LARS ŻŻ͵ + # һŻб self.optsڴ洢ݶȶѵеŻ + + # Ƿ opt Optimizer ʵ׳쳣 + if not isinstance(opt, Optimizer): + raise TypeError( + f"The first arg 'opt' must be an Optimizer instance, but got {type(opt)}") + + # train_strategy ָ train_parameter_groups δָ׳쳣 + if train_strategy is not None and train_parameter_groups is None: + raise ValueError("When the 'train_strategy' is specified, the value of 'train_parameter_groups' " + "must also be specified") + + # 鴫 opt ǷΪ LARSLayer-wise Adaptive Rate ScalingŻʵ + if isinstance(opt, LARS): + self.is_lars = True + self.opt_class = type(opt.opt)# ȡ LARS ڲŻ + self.opt_init_args = opt.opt.init_args # ȡ LARS ڲŻijʼ + self.lars_init_args = opt.init_args# ȡ LARS ijʼ + self.single_opt = opt.opt# ȡ LARS ڲŻʵ + self.parameters = opt.opt.parameters# ȡŻIJб + self.learning_rate = opt.opt.init_learning_rate# ȡŻѧϰ + self.dynamic_lr = opt.opt.dynamic_lr# ȡŻǷʹö̬ѧϰ + else: + self.is_lars = False + self.opt_class = type(opt)# ȡķ LARS Ż + self.opt_init_args = opt.init_args# ȡ LARS Żijʼ + self.single_opt = opt# ȡķ LARS Żʵ + self.parameters = opt.parameters# ȡŻIJб + self.learning_rate = opt.init_learning_rate# ȡŻѧϰ + self.dynamic_lr = opt.dynamic_lr# ȡŻǷʹö̬ѧϰ + + # һյŻб self.opts + self.opts = [] + # train_parameter_groups δָĬϴ 10 飬ÿŻIJ + if train_parameter_groups is None: + self.groups_num = 10 # + step = 6 # ÿIJ + parameters = opt.parameters # ȡģ͵IJб + # бָɶ飬ÿ step + train_parameter_groups = (tuple(parameters[(i * step):]) for i in range(self.groups_num)) + else: + # ûָ train_parameter_groupsͲȡ + if not isinstance(train_parameter_groups, (tuple, list)): + raise TypeError( + "The specified 'train_parameter_groups' should be tuple or list") + self.groups_num = len(train_parameter_groups) + + # ʼѵ + self._init_train_strategy(train_strategy) + # ͬȫֲµѧϰ + self._create_new_group_learning_rate() + + # ʼŻ + self.opt_index = 0 + + # ÿ飬ΪÿһӦŻʵ洢 self.opts б + for params in train_parameter_groups: + if not isinstance(params, (tuple, list)): + raise TypeError("The each element of 'train_parameter_groups' should be tuple or list " + "to store the Parameter") + # generate one-to-one opt corresponding to the parameter group + # һӦŻʵӵ self.opts б + self.opts.append(self._generate_new_optimizer(params)) + self.opt_index += 1 + + """ʼݶȶѵԡ + + Args: + train_strategy: ָݶȶѵԵIJ֮һ + - Noneʾʹκضѵԡ + - бԪ飺бԪ飬ָЩݶӦñᡣ + - TensorһÿʾЩݶӦñᡣ + + Raises: + ValueError: train_strategyͲϷϷԪء + TypeError: train_strategyͲNonetuplelistTensor + """ + def _init_train_strategy(self, train_strategy): + """Init train strategy for gradient freeze.""" + # 鴫train_strategyǷԪб + if isinstance(train_strategy, (tuple, list)): + # ǣеԪ + for ele in train_strategy: + # ÿԪǷΪ쳣 + if not isinstance(ele, int): + raise ValueError( + "The element in train_strategy should be int number") + # Ԫضtrain_strategyתΪint32͵ֵself.train_strategy + self.train_strategy = Tensor(train_strategy, mstype.int32) + # train_strategyTensor + elif isinstance(train_strategy, Tensor): + # TensorάǷΪ1dtypeǷΪint32쳣 + if train_strategy.ndim != 1 or train_strategy.dtype != mstype.int32: + raise ValueError("When train_strategy is a Tensor, the dimension should be 1 and " + "the dtype should be int32") + # ֱӸֵself.train_strategy + self.train_strategy = train_strategy + # train_strategyΪNone + elif train_strategy is None: + # ֱӽself.train_strategyΪNone + self.train_strategy = None + # train_strategy֮һʹ쳣 + else: + raise TypeError( + "The specified 'train_strategy' should be None, tuple, list or Tensor") + + def _create_new_group_learning_rate(self): + """Create new learning rate for different global step.""" + """ͬȫֲѧϰʡ""" + # ʼһյĶ̬ѧϰбб(groups_num) + self.dynamic_learning_rate = [[] for _ in range(self.groups_num)] + # ǰѧϰΪNoneѧϰΪһŻsingle_optѧϰ + if self.learning_rate is None: + self.learning_rate = self.single_opt.learning_rate + return + # ˶̬ѧϰʣdynamic_lrѧϰбͣlistѵTensor + if self.dynamic_lr and isinstance(self.learning_rate, list) and isinstance(self.train_strategy, Tensor): + train_strategy = list(self.train_strategy.asnumpy()) + if len(self.learning_rate) <= len(train_strategy): + for i, lr in enumerate(self.learning_rate): + self.dynamic_learning_rate[train_strategy[i]].append(lr) + + def _generate_new_optimizer(self, params): + """Generate new optimizer.""" + + if self.dynamic_learning_rate[self.opt_index]: + lr = self.dynamic_learning_rate[self.opt_index] + else: + lr = self.learning_rate + if not self.is_lars: + opt = self.opt_class(params=params, learning_rate=lr, **self.opt_init_args) + opt._update_local_parameters_name("boost_{}".format(self.opt_index)) # pylint: disable=W0212 + else: + opt = LARS(self.opt_class(params=params, learning_rate=lr, **self.opt_init_args), + **self.lars_init_args) + opt.opt._update_local_parameters_name("boost_{}".format(self.opt_index)) # pylint: disable=W0212 + opt._update_local_parameters_name("boost_{}".format(self.opt_index)) # pylint: disable=W0212 + return opt + + +class _TrainFreezeCell(Cell): + r""" + Gradient freezing training network. + + Args: + net (Cell): The training network.ѵ硣һģΪ + sens (numbers.Number): The scaling number to be filled as the input of backpropagation. Default value is 1.0.򴫲ֵĬֵΪ 1.0 + grad (tuple(Tensor)): The gradients of network parameters and inputs.ݶȡ grad һݶȵԪ顣 + grad_reducer (Cell): Constructs a gradient reducer Cell, which applies communication and average operations on + single-process gradient values.һݶ Cellڵݶִֵͨźƽ + use_grad_accumulation (bool): Whether use grad accumulation.ǷʹݶۻָʾǷݶȵۻ + optimizer (Union[Cell]): Optimizer for updating the weights.ڸȨصŻһŻΪ + max_accumulation_step (numbers.Number): Max grad accumulation steps. Default: 1.0ݶۻ衣ĬֵΪ 1.0 + + + Supported Platforms: + ``Ascend`` + """ + # ʼһϵвѵ + def __init__(self, net, sens, grad, grad_reducer, use_grad_accumulation, optimizer, max_accumulation_step=1): + # ø Cell ijʼرԶǰ׺ + super(_TrainFreezeCell, self).__init__(auto_prefix=False) + # IJΪԱ + self.net = net + self.grad = grad + self.grad_reducer = grad_reducer + self.opt = optimizer + self.parameters = optimizer.parameters + self.sens = sens + self.use_grad_accumulation = use_grad_accumulation + self.max_accumulation_step = max_accumulation_step + # ݶۻ + if use_grad_accumulation: + # GradientAccumulation ڹݶۻ + self.grad_accumulation = GradientAccumulation( + self.max_accumulation_step, self.optimizer) + # ѵеľ + def construct(self, *inputs): + # ʧ + loss = self.net(*inputs) + # һʧͬͺ״ĸ֪ + sens = F.fill(loss.dtype, loss.shape, self.sens) + # ݶȣgrad һݶȵԪ + grads = self.grad(self.net, self.parameters)(*inputs, sens) + # ʹݶݶȽ + grads = self.grad_reducer(grads) + # ݶۻ + if self.use_grad_accumulation: + # ʧݶȴݸݶۻµʧֵ + loss = self.grad_accumulation(loss, grads) + else: + # ִŻȨأʧݶ֮佨ϵ + loss = F.depend(loss, self.opt(grads)) + # ʧֵ + return loss + + +class GradientFreeze: + r""" + Freezing the gradients of some layers randomly. The number and + probability of frozen layers can be configured by users + + Args: + param_groups (Union[tuple, list]): Groups of parameters for gradients freezing training. + freeze_type (int): Strategy of gradients freezing training. + freeze_p (float): probability of gradients freezing training. + total_steps (numbers.Number): Steps of the whole training. + + Examples: + >>> gradient_freeze_class = boost.GradientFreeze(10, 1, 0.5, 2000) + >>> network, optimizer = gradient_freeze_class.freeze_generate(network, optimizer) + """ + # ʼһϵвö + def __init__(self, param_groups, freeze_type, freeze_p, total_steps): + # 洫IJΪijԱ + self._param_groups = param_groups # 飬ҪвIJ + self._freeze_type = freeze_type # ָͣζܵȡֵ 'epoch' 'step' + self._freeze_p = freeze_p # ڿƲı + self._total_steps = total_steps # ܲڼѵܲ + self.grad_reducer = F.identity # grad_reducer ʼΪ F.identityͨݶȵıʶ + self._param_processer = ParameterProcess() # ParameterProcess 󣬲䱣ΪԱ + + # һڽԽݶȶѵ + def split_parameters_groups(self, net, freeze_para_groups_number): + r""" + Split parameter groups for gradients freezing training. + + Args: + net (Cell): The training network. + freeze_para_groups_number (int): The number of gradient freeze groups. + """ + # һбڴ洢IJ + grouped_params = [] + # ѵ + tmp = [] + for para in net.trainable_params(): + name = para.name + # ensure 'bn' after 'conv' is not split + # а 'bn' 'bias'򽫲ӵʱб + if 'bn' in name or 'bias' in name: + tmp.append(para) + # ʱбеIJѾﵽ 3 + elif len(tmp) >= 3: + # ʱбеIJӵбУ³ʼʱб + grouped_params.append(tmp) + tmp = [para] + # 򣬼ӵʱб + else: + tmp.append(para) + # ʣIJ + if tmp: + grouped_params.append(tmp) + # ÿ֮IJȷƽͬĶ + stride = len(grouped_params) // freeze_para_groups_number + # IJбÿԪһб + freeze_grouped_params = [sum(grouped_params[i * stride:], []) + for i in range(freeze_para_groups_number)] + # طĶб + return freeze_grouped_params + + #ǩ + def generate_freeze_index_sequence(self, parameter_groups_number, freeze_strategy, freeze_p, total_steps): + r""" + Generate index sequence for gradient freezing training. + + Args: + parameter_groups_number (int): The number of parameter groups. + freeze_strategy (int): Gradient freeze grouping strategy, select from [0, 1]. + freeze_p (float): Gradient freezing probability. + total_steps (int): Total training steps. + """ + #total_stepԸtotal_steps101% + total_step = int(total_steps * 1.01) + #parameter_groups_numberСڵ1򷵻һΪtotal_stepб + if parameter_groups_number <= 1: + return [0 for _ in range(total_step)] + # local continuous freezing training strategy, as '00001234' + #freeze_strategyǷCONTINUOUS_STRATEGY + if freeze_strategy == CONTINUOUS_STRATEGY: + #ṩĹʽʹfreeze_pparameter_groups_numberzero_cnt + zero_cnt = int( + freeze_p * (parameter_groups_number - 1) / (1 - freeze_p) + 0.5) + #һбsub_idxаzero_cntظ㣬ȻǴ1parameter_groups_numberС + sub_idx = [0] * zero_cnt + list(range(1, parameter_groups_number)) + #ͨظsub_idxֱ䳤ȵtotal_stepfreeze_idxesб + freeze_idxes = [] + while len(freeze_idxes) < total_step: + freeze_idxes += sub_idx + #freeze_idxesб + return freeze_idxes + # interval freezing training strategy, as '01020304' + #freeze_strategyǷINTERVAL_STRATEGY + if freeze_strategy == INTERVAL_STRATEGY: + #ʼindex_allа1parameter_groups_numberУÿĸprobʼfreeze_idxeszero_cntfreeze_cnt + index_all = list(range(1, parameter_groups_number)) + prob = [x / sum(index_all) for x in index_all] + freeze_idxes = [0] + zero_cnt = 1 + freeze_cnt = 0 + #һѭֱfreeze_idxesﵽijȡ㵱ǰfreeze_p_cur + while len(freeze_idxes) < total_step: + freeze_p_cur = 1.0 * freeze_cnt / (zero_cnt + freeze_cnt) + #鵱ǰĶǷС1 - freeze_p + if freeze_p_cur < 1 - freeze_p: + #ṩĸʴindex_allѡһ丽ӵfreeze_idxesȻfreeze_cnt + freeze_idxes.append( + int(np.random.choice(index_all[::-1], p=prob))) + freeze_cnt += 1 + #㸽ӵfreeze_idxeszero_cnt + else: + freeze_idxes.append(0) + zero_cnt += 1 + #freeze_idxesб + return freeze_idxes + #freeze_strategyȲCONTINUOUS_STRATEGYҲINTERVAL_STRATEGYһValueErrorаһϢָʾֵ֧IJԡ + raise ValueError( + f"Unsupported freezing training strategy '{freeze_strategy}'") + + # ɶŻķ + def freeze_generate(self, network, optimizer): + r""" + Generate freeze network and optimizer. + + Args: + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + """ + # IJֳɶ + train_para_groups = self.split_parameters_groups( + network, self._param_groups) + # ÿдÿIJ + for i in range(self._param_groups): + train_para_groups[i] = self._param_processer.generate_group_params(train_para_groups[i], + optimizer.init_params['params']) + # ɶ + train_strategy = self.generate_freeze_index_sequence( + self._param_groups, self._freeze_type, self._freeze_p, self._total_steps) + # жṦܵŻ + optimizer = FreezeOpt(optimizer, train_para_groups, train_strategy) + # Ż + return network, optimizer + + +# ɴжṦܵŻķ +def freeze_cell(reducer_flag, network, optimizer, sens, grad, use_grad_accumulation, mean=None, degree=None, + max_accumulation_step=1): + r""" + Generate freeze network and optimizer. + + Args: + reducer_flag (bool): Reducer flag. + network (Cell): The training network. + optimizer (Cell): Optimizer for updating the weights. + sens (numbers.Number): The scaling number. + grad (tuple(Tensor)): Tuple of gradient tensors. + use_grad_accumulation (bool): Use gradient accumulation flag. + mean (bool): Gradients mean flag. default: None. + degree (int): Device number. default: None. + max_accumulation_step (int): Max accumulation steps. default: 1. + + Examples: + >>> import numpy as np + >>> from mindspore import Tensor, Parameter, nn + >>> import mindspore.ops as ops + >>> from mindspore.boost.grad_freeze import freeze_cell + >>> + >>> class Net(nn.Cell): + ... def __init__(self, in_features, out_features): + ... super(Net, self).__init__() + ... self.weight = Parameter(Tensor(np.ones([in_features, out_features]).astype(np.float32)), + ... name='weight') + ... self.matmul = ops.MatMul() + ... + ... def construct(self, x): + ... output = self.matmul(x, self.weight) + ... return output + ... + >>> in_features, out_features = 16, 10 + >>> network = Net(in_features, out_features) + >>> optimizer = nn.Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> grad = ops.GradOperation(get_by_list=True, sens_param=True) + >>> freeze_nets = freeze_cell(False, network, optimizer, 1.0, grad, False, None, None, 1) + """ + # reducer_flagΪ棬ʾʹ˷ֲʽݶȾۺ + if reducer_flag: + # + param_processer = ParameterProcess() + # ݶreducersÿoptimizerIJݸ + grad_reducers = (DistributedGradReducer(param_processer.assign_parameter_group(opt.parameters), + mean, degree) for opt in optimizer.opts) + # Ԫ飬ÿӦһoptimizer + freeze_nets = tuple(_TrainFreezeCell(network, sens, grad, reducer, + use_grad_accumulation, opt, max_accumulation_step) + for reducer, opt in zip(grad_reducers, optimizer.opts)) + else: + # ûʹ÷ֲʽݶȾۺϣֱӴԪ飬ÿӦһoptimizer + freeze_nets = tuple(_TrainFreezeCell(network, sens, grad, F.identity, + use_grad_accumulation, opt, max_accumulation_step) + for opt in optimizer.opts) + + # ضԪ + return freeze_nets -- 2.34.1 From 47db2ab436e102843bfb59a11720a2efb11963f8 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sat, 30 Sep 2023 21:43:13 +0800 Subject: [PATCH 60/72] ADD file via upload --- .../less_batch_normalization.py | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/less_batch_normalization.py diff --git a/mindspore/ccsrc/transform-update/less_batch_normalization.py b/mindspore/ccsrc/transform-update/less_batch_normalization.py new file mode 100644 index 00000000000..94b78c9014a --- /dev/null +++ b/mindspore/ccsrc/transform-update/less_batch_normalization.py @@ -0,0 +1,172 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""less Batch Normalization""" +# ҪĿģ +import numpy as np +from mindspore.nn.cell import Cell +from mindspore.nn.layer import Dense +from mindspore.ops import operations as P +from mindspore.common import Tensor, Parameter +from mindspore.common import dtype as mstype +from mindspore.common.initializer import initializer + + +__all__ = ["CommonHeadLastFN", "LessBN"] + +# LessBNģ飬ࣺCommonHeadLastFNLessBN +# ԶBatch NormalizationBNܲ׼ȷԡ + +# һΪCommonHeadLastFN࣬һȫӲ +class CommonHeadLastFN(Cell): + r""" + The last full Normalization layer. + + This layer implements the operation as: + + .. math:: + \text{inputs} = \text{norm}(\text{inputs}) + \text{kernel} = \text{norm}(\text{kernel}) + \text{outputs} = \text{multiplier} * (\text{inputs} * \text{kernel} + \text{bias}), + + Args: + in_channels (int): The number of channels in the input space. + out_channels (int): The number of channels in the output space. + weight_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable weight_init parameter. The dtype + is same as input x. The values of str refer to the function `initializer`. Default: 'normal'. + bias_init (Union[Tensor, str, Initializer, numbers.Number]): The trainable bias_init parameter. The dtype is + same as input x. The values of str refer to the function `initializer`. Default: 'zeros'. + has_bias (bool): Specifies whether the layer uses a bias vector. Default: True. + + Supported Platforms: + ``Ascend`` ``GPU`` ``CPU`` + + Examples: + >>> input = Tensor(np.array([[180, 234, 154], [244, 48, 247]]), mindspore.float32) + >>> net = CommonHeadLastFN(3, 4) + >>> output = net(input) + """ + + def __init__(self, + in_channels, + out_channels, + weight_init='normal', + bias_init='zeros', + has_bias=True): + + super(CommonHeadLastFN, self).__init__() + # ʼȨز + weight_shape = [out_channels, in_channels] + self.weight = Parameter(initializer(weight_init, weight_shape), requires_grad=True, name='weight') + # L2һֱӦxȨkernel + self.x_norm = P.L2Normalize(axis=1) + self.w_norm = P.L2Normalize(axis=1) + # ˲ڼxwij˻ + self.fc = P.MatMul(transpose_a=False, transpose_b=True) + # ʼӲ + self.multiplier = Parameter(Tensor(np.ones([1]), mstype.float32), requires_grad=True, name='multiplier') + # Ƿʹƫ + self.has_bias = has_bias + if self.has_bias: + # ʹƫʼƫò + bias_shape = [out_channels] + self.bias_add = P.BiasAdd() + self.bias = Parameter(initializer(bias_init, bias_shape), requires_grad=True, name='bias') + + def construct(self, x): + # xL2һ + x = self.x_norm(x) + # ȨkernelL2һ + w = self.w_norm(self.weight) + # xwľ + x = self.fc(x, w) + if self.has_bias: + # ʹƫƫӵx + x = self.bias_add(x, self.bias) + # ԳӲ + x = self.multiplier * x + # ؽ + return x + +# LessBN࣬ԶBatch NormalizationBNܲ׼ȷ +class LessBN(Cell): + """ + Reduce the number of BN automatically to improve the network performance + and ensure the network accuracy. + + Args: + network (Cell): Network to be modified. + fn_flag (bool): Replace FC with FN. default: False. + + Examples: + >>> network = boost.LessBN(network) + """ + + def __init__(self, network, fn_flag=False): + super(LessBN, self).__init__() + # 洢 + self.network = network + # "less_bn" + self.network.set_boost("less_bn") + # ĵԪǰ׺ + self.network.update_cell_prefix() + # fn_flagΪTrueеȫӲ滻ΪFN + if fn_flag: + self._convert_to_less_bn_net(self.network) + # ӳ־IJֲӳٵʱִ + self.network.add_flags(defer_inline=True) + + def _convert_dense(self, subcell): + """ + convert dense cell to FN cell + """ + prefix = subcell.param_prefix + # µFN㣬ԭʼȫӲͬ + new_subcell = CommonHeadLastFN(subcell.in_channels, + subcell.out_channels, + subcell.weight, + subcell.bias, + False) + new_subcell.update_parameters_name(prefix + '.') + + return new_subcell + + def _convert_to_less_bn_net(self, net): + """ + convert network to less_bn network + """ + cells = net.name_cells() + dense_name = [] + dense_list = [] + # еԪ + for name in cells: + subcell = cells[name] + if subcell == net: + continue + elif isinstance(subcell, (Dense)): + # ԪȫӲ㣬¼ƺʵ + dense_name.append(name) + dense_list.append(subcell) + else: + # ݹãӵԪеȫӲ + self._convert_to_less_bn_net(subcell) + + if dense_list: + # ȫӲ㣬һȫӲ滻ΪFN + new_subcell = self._convert_dense(dense_list[-1]) + net.insert_child_to_cell(dense_name[-1], new_subcell) + + def construct(self, *inputs): + # ǰ򴫲봫ݸ粢ؽ + return self.network(*inputs) -- 2.34.1 From 6eef24c510112d7a05891f07e49bcf9896c2b076 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Sat, 30 Sep 2023 21:43:33 +0800 Subject: [PATCH 61/72] ADD file via upload --- .../transform-update/grad_accumulation.py | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/grad_accumulation.py diff --git a/mindspore/ccsrc/transform-update/grad_accumulation.py b/mindspore/ccsrc/transform-update/grad_accumulation.py new file mode 100644 index 00000000000..ca955ca04ba --- /dev/null +++ b/mindspore/ccsrc/transform-update/grad_accumulation.py @@ -0,0 +1,82 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""grad accumulation""" +from mindspore.nn.cell import Cell +from mindspore.common import Parameter, Tensor +from mindspore.common import dtype as mstype +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P + + +__all__ = ["GradientAccumulation", "gradient_accumulation_op", "gradient_clear_op"] + + +# ݶȻ۲ +gradient_accumulation_op = C.MultitypeFuncGraph("gradient_accumulation_op") + +# עݶȻ۲ļͼǻ۲ۻݶԼǰݶ +@gradient_accumulation_op.register("Int64", "Tensor", "Tensor") +def cumulative_grad_process(accumulation_step, cumulative_grad, grad): + """Apply gradient accumulation to cumulative grad.""" + # ǰݶԻ۲ӵۻݶ + return P.AssignAdd()(cumulative_grad, grad / accumulation_step) + +# ݶ +gradient_clear_op = C.MultitypeFuncGraph("gradient_clear_op") + +# עݶļͼۻݶ +@gradient_clear_op.register("Tensor") +def clear_grad(cumulative_grad): + # һۻݶ״һȫ丳ֵۻݶȣʵݶ + zero_grad = P.ZerosLike()(cumulative_grad) + return F.assign(cumulative_grad, zero_grad) + +# ݶȻ +class GradientAccumulation(Cell): + """ + After accumulating the gradients of multiple steps, call to optimize its update. + + Args: + max_accumulation_step (int): Steps to accumulate gradients. + optimizer (Cell): Optimizer used. + """ + def __init__(self, max_accumulation_step, optimizer): + super(GradientAccumulation, self).__init__() + self._max_accumulation_step = max_accumulation_step + self.optimizer = optimizer + self.weights = optimizer.parameters + self.hyper_map = C.HyperMap() + # ۻݶȵʼֵΪȫ + self._grad_accumulation = self.weights.clone(prefix="grad_accumulation", init='zeros') + # ڼ¼ǰۻIJʼֵΪ0 + self._accumulation_step = Parameter(Tensor(0, dtype=mstype.int32), name="accumulation_step") + + def construct(self, loss, grads): + # ʹݶȻ۲ǰݶӵۻݶ + loss = F.depend(loss, self.hyper_map(F.partial(gradient_accumulation_op, self._max_accumulation_step), + self._grad_accumulation, grads)) + # ۻ + self._accumulation_step += 1 + # ۻﵽۻִŻݶȸ£ۻ + if self._accumulation_step >= self._max_accumulation_step: + loss = F.depend(loss, self.optimizer(self._grad_accumulation)) + self._accumulation_step = 0 + + # ۻΪ0ִݶ + if self._accumulation_step == 0: + loss = F.depend(loss, self.hyper_map(F.partial(gradient_clear_op), self._grad_accumulation)) + + return loss -- 2.34.1 From f0e49149b4dbf213a34bf0eabda811510aecf367 Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Sat, 30 Sep 2023 21:52:42 +0800 Subject: [PATCH 62/72] Delete 'mindspore/ccsrc/transform-update/adasum.py' --- mindspore/ccsrc/transform-update/adasum.py | 330 --------------------- 1 file changed, 330 deletions(-) delete mode 100644 mindspore/ccsrc/transform-update/adasum.py diff --git a/mindspore/ccsrc/transform-update/adasum.py b/mindspore/ccsrc/transform-update/adasum.py deleted file mode 100644 index df34aa94ab6..00000000000 --- a/mindspore/ccsrc/transform-update/adasum.py +++ /dev/null @@ -1,330 +0,0 @@ -# Copyright 2021 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""adasum""" -import copy -import hashlib -import math -from mindspore.nn.cell import Cell -from mindspore.communication.management import create_group -from mindspore.ops import composite as C -from mindspore.ops import functional as F -from mindspore.ops import operations as P -from mindspore.ops.operations._inner_ops import Send, Receive - - -__all__ = ["AdaSum"] - - -MAX_NUM_HASH = 2 ** 31 - - -_update_parameters = C.MultitypeFuncGraph("update_parameters") - - -@_update_parameters.register("Tensor", "Tensor", "Tensor", "Tensor") - #һΪ _update_parameters_after_broadcast ĺڸ² -def _update_parameters_after_broadcast(delta_weight, update_delta_weight, parameter, old_parameter): - shape = F.shape(delta_weight) # ȡ delta_weight ״ - update_delta_weight = P.Reshape()(update_delta_weight, shape) # update_delta_weight ܳ delta_weight ͬ״ - new_parameter = old_parameter - update_delta_weight # µIJֵͨӾɲмȥ update_delta_weight - return P.Assign()(parameter, new_parameter) # ʹ P.Assign() ²ֵ parameter - - # һΪ _send_before_receive ĺڷݲӦ -def _send_before_receive(send_part, send, recv): - send_ok = send(send_part) # ͨ send send_partؽ洢 send_ok - return recv(send_ok) # recv send_ok ΪݣڽӦ - -# һΪ _receive_before_send ĺȽݺ -def _receive_before_send(send_part, send, recv): - receive_ok = recv(send_part) # ͨ recv send_part Ϊݣڽݣս洢 receive_ok - send_part = F.depend(send_part, receive_ok) # ʹ F.depend() ϵȷ send_part ֮ǰִнղ - return F.depend(receive_ok, send(send_part)) # ͨ send send_part Ϊݣڷݣͽ - -# һΪ _send_recv_res ĺڷͺͽݣѧʹ -def _send_recv_res(left_send, recv_part, local_part, allreduce, parameter_divisibility, allreduce_node_num): - """send result and receive result.""" - if parameter_divisibility: # ɷָִ² - recv_part = P.Squeeze()(recv_part) # ȥ recv_part еά - local_part = F.depend(local_part, recv_part) # ϵȷ local_part ʹǰ recv_part ѱ - eps = 1e-12 - # һЩֵЩֵںļ - value_0 = P.ReduceSum()(local_part * recv_part) + eps - if left_send: - value_1 = P.ReduceSum()(local_part * local_part) + eps - value_2 = P.ReduceSum()(recv_part * recv_part) + eps - else: - value_1 = P.ReduceSum()(recv_part * recv_part) + eps - value_2 = P.ReduceSum()(local_part * local_part) + eps - # ԼֵȫֹԼallreduce - value_0 = allreduce(value_0) - value_1 = F.depend(allreduce(value_1), value_0) - value_2 = F.depend(allreduce(value_2), value_1) - # ҷͣleft_sendյĽ res - if left_send: - res = (1 - (value_0 / (2 * value_1))) * local_part + (1 - (value_0 / (2 * value_2))) * recv_part - else: - res = (1 - (value_0 / (2 * value_1))) * recv_part + (1 - (value_0 / (2 * value_2))) * local_part - else: - res = allreduce(local_part) # ɷָֱִȫֹԼallreduce - res /= allreduce_node_num - return res - - -_adasum_opt_forward = C.MultitypeFuncGraph("adasum_opt_forward") - - -@_adasum_opt_forward.register("Bool", "Function", "Bool", "Int64", "Function", "Function", "Tensor") -# һΪ _adasum_opt_forward_process ĺ Adasum Żǰ -def _adasum_opt_forward_process(left_send, allreduce, parameter_divisibility, allreduce_node_num, send, recv, delta_w): - """adasum optimizer process.""" - if parameter_divisibility: # ɷָִ² - delta_w = P.Squeeze()(delta_w) # ȥ delta_w еά - ori_len = F.shape(delta_w)[0] # ȡ delta_w ԭʼ - divide_len = ori_len / 2 # ָλ - left_part = delta_w[:divide_len] # delta_w Ϊ벿 - right_part = delta_w[divide_len:] # delta_w ΪҰ벿 - else: # ɷָֱӽ delta_w Ƶ벿ֺҰ벿 - left_part = delta_w - right_part = delta_w - - if left_send: # ߷ - if parameter_divisibility: # ɷָִзͲݲ - recv_part = _send_before_receive(left_part, send, recv) - else: # ɷָҰ벿Ϊݲ - recv_part = right_part - # µ delta_w - update_delta_w = _send_recv_res(left_send, recv_part, right_part, allreduce, parameter_divisibility, - allreduce_node_num) - else: # ұ߷ - if parameter_divisibility: # ɷָִнղݲ - recv_part = _receive_before_send(right_part, send, recv) - else: # ɷָ벿Ϊݲ - recv_part = left_part - # µ delta_w - update_delta_w = _send_recv_res(left_send, recv_part, left_part, allreduce, parameter_divisibility, - allreduce_node_num) - - return update_delta_w - - -_adasum_opt_rollback = C.MultitypeFuncGraph("adasum_opt_rollback") - - -@_adasum_opt_rollback.register("Bool", "Bool", "Tensor", "Function", "Function") -# һΪ _adasum_opt_rollback_process ĺ Adasum ŻĻع -def _adasum_opt_rollback_process(left_send, parameter_divisibility, delta_w, send, recv): - """adasum optimizer rollback process.""" - if parameter_divisibility: # ɷָִ² - if left_send: # ߷ݣִзͲݲ - recv_part = _send_before_receive(delta_w, send, recv) - else: # ұ߷ݣִнղݲ - recv_part = _receive_before_send(delta_w, send, recv) - - recv_part = P.Squeeze()(recv_part) # ȥݲֵά - recv_part = P.Reshape()(recv_part, (-1,)) # ܽݲֺ delta_w Ϊһά - delta_w = P.Reshape()(delta_w, (-1,)) - # ҷ͵ݲֺ delta_w ƴ - if left_send: - res = P.Concat()((recv_part, delta_w)) - else: - res = P.Concat()((delta_w, recv_part)) - else: - res = delta_w # ɷָֱӷ delta_w - return res - -#һΪ AdaSum ࣬һԶ㣨CellִAdasum㷨 -#㷨ڷֲʽݲѵѧϰģ͡ -class AdaSum(Cell): - r""" - The Adaptive Summation, or AdaSum, is a novel algorithm for improving distributed data - parallel training of Deep Learning models. - - Args: - rank (int): Rank number. - device_number (int): Device number. - group_number (int): Group number. - parameter_tuple (Tuple(Parameter)): Tuple of parameters. - - Inputs: - - **delta_weights** (Tuple(Tensor)) - Tuple of gradients. - - **parameters** (Tuple(Parameter)) - Tuple of current parameters. - - **old_parameters** (Tuple(Parameter)) - Tuple of last parameters. - - Outputs: - - **adasum_parameters** (Tuple(Tensor)) - Tuple of parameters after adasum process. - """ - def __init__(self, rank, device_number, group_number, parameter_tuple): - super(AdaSum, self).__init__() - self.rank = rank - self.device_number = device_number - self.group_number = group_number - self.parameter_tuple = parameter_tuple - self._generate_communication_op() - self.hyper_map = C.HyperMap() - # ͨŲ - # ÷ڴAdasum㷨ͨŲ͡աȫֹԼȲ - def _generate_communication_op(self): - """generate communication op.""" - self.calc_times = int(math.log(self.group_number, 2)) - self.send_node = [] - self.send_list_forward = [] - self.recv_list_forward = [] - self.send_list_rollback = [] - self.recv_list_rollback = [] - self.allreduce_list = [] - self.broadcast_list = [] - self.parameter_divisibility_list = [] - self.allreduce_node_num_list = [] - last_delta_weights = [] - group_start_rank = (self.rank // self.device_number) * self.device_number - - for step in range(self.calc_times): - current_group = self.device_number * (2 ** step) - sr_target = self.rank - if (sr_target // current_group) % 2 == 0: - dest_target = sr_target + current_group - self.send_node.append(True) - else: - dest_target = sr_target - current_group - self.send_node.append(False) - - neighbor_ids = [] - group_name_last = 0 - for index in range(2 ** (step + 1)): - node_rank = self.rank // self.device_number - double_d = 2 ** (step + 1) - neighbor_id = (node_rank // double_d * double_d + index) * self.device_number + \ - self.rank % self.device_number - neighbor_ids.append(neighbor_id) - group_name_last += neighbor_id - group_name = "adasum_" + str(step) + "_" + str(group_name_last) - create_group(group_name, neighbor_ids) - - send_left = [] - send_right = [] - recv_left = [] - recv_right = [] - allreduce_node_num = () - left_delta_weights, right_delta_weights, delta_weights_divisibility = \ - self._get_delta_weights_info(last_delta_weights) - self.parameter_divisibility_list.append(delta_weights_divisibility) - weights_index = 0 - fusion_id = (step + 1) * 3 - for shape, dtype in left_delta_weights: - send_tag = self._hash(step, sr_target, weights_index) - send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") - send.add_prim_attr("fusion", fusion_id) - recv_tag = self._hash(step, dest_target, weights_index) - recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, - group="hccl_world_group") - recv.add_prim_attr("fusion", fusion_id) - send_left.append(send) - recv_left.append(recv) - weights_index += 1 - for shape, dtype in right_delta_weights: - send_tag = self._hash(step, sr_target, weights_index) - send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") - send.add_prim_attr("fusion", fusion_id + 1) - recv_tag = self._hash(step, dest_target, weights_index) - recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, - group="hccl_world_group") - recv.add_prim_attr("fusion", fusion_id + 1) - send_right.append(send) - recv_right.append(recv) - weights_index += 1 - - if self.send_node and self.send_node[-1]: - self.send_list_forward.append(send_left) - self.send_list_rollback.append(send_right) - self.recv_list_forward.append(recv_right) - self.recv_list_rollback.append(recv_left) - last_delta_weights = right_delta_weights - else: - self.send_list_forward.append(send_right) - self.send_list_rollback.append(send_left) - self.recv_list_forward.append(recv_left) - self.recv_list_rollback.append(recv_right) - last_delta_weights = left_delta_weights - - server_all_reduce = P.AllReduce("sum", group_name) - server_all_reduce.add_prim_attr("fusion", fusion_id + 2) - self.allreduce_list.append(server_all_reduce) - - for param_divisibility in delta_weights_divisibility: - if param_divisibility: - allreduce_node_num += (0,) - else: - allreduce_node_num += (2 ** (step + 1),) - self.allreduce_node_num_list.append(allreduce_node_num) - - broadcast_group = [x for x in range(group_start_rank, group_start_rank + self.device_number)] - broadcast_group_name = "broadcast_group_" + str(group_start_rank) - create_group(broadcast_group_name, broadcast_group) - for b_rank in range(len(broadcast_group)): - self.broadcast_list.append(P.Broadcast(b_rank, group=broadcast_group_name)) - self.sync_barrier = P.AllReduce("sum", group=broadcast_group_name) - # ȡݶϢ - # ÷ڻȡݶϢݶȻΪҲ֣жǷɷָ - def _get_delta_weights_info(self, last_delta_weights): - """get delta weights info.""" - half_delta_weights = [] - if last_delta_weights: - half_delta_weights = last_delta_weights - else: - for parameter in self.parameter_tuple: - new_shape = [int(x) for x in parameter.shape] - half_delta_weights.append((new_shape, parameter.dtype)) - left_delta_weights = [] - right_delta_weights = [] - delta_weights_divisibility = () - for shape, dtype in half_delta_weights: - left_shape = copy.deepcopy(shape) - right_shape = copy.deepcopy(shape) - divisibility_flag = False - for i in range(len(shape)): - if shape[i] > 1: - left_shape[i] = int(shape[i] // 2) - right_shape[i] = shape[i] - int(shape[i] // 2) - divisibility_flag = True - break - left_delta_weights.append((left_shape, dtype)) - right_delta_weights.append((right_shape, dtype)) - delta_weights_divisibility += (divisibility_flag,) - return left_delta_weights, right_delta_weights, delta_weights_divisibility - # ϣֵ - # ÷ڼϣֵͨűǩ - def _hash(self, step, target, weights_index): - target = "tag" + str(step) + str(target) + str(weights_index) - target_hash = hashlib.sha1(target.encode()).hexdigest() - hash_res = int(int(target_hash, 16) % MAX_NUM_HASH) - return hash_res - # ִAdasumǰ - # òִִAdasum㷨ǰݵķ͡աȡ - def construct(self, delta_weights, parameters, old_parameters): - forward_weights = [delta_weights] - for i in range(self.calc_times): - process_weights = self.hyper_map(F.partial(_adasum_opt_forward, self.send_node[i], self.allreduce_list[i]), - self.parameter_divisibility_list[i], self.allreduce_node_num_list[i], - self.send_list_forward[i], self.recv_list_forward[i], forward_weights[-1]) - forward_weights.append(process_weights) - for i in range(self.calc_times): - j = self.calc_times - i - 1 - process_weights = self.hyper_map(F.partial(_adasum_opt_rollback, self.send_node[j]), - self.parameter_divisibility_list[j], forward_weights[j + 1], - self.send_list_rollback[j], self.recv_list_rollback[j]) - forward_weights[j] = process_weights - adasum_parameters = self.hyper_map(F.partial(_update_parameters), delta_weights, forward_weights[0], - parameters, old_parameters) - return adasum_parameters -- 2.34.1 From 073f1dedcb0055fb77c6529a10a8b2031c86690c Mon Sep 17 00:00:00 2001 From: zyf1234 Date: Sat, 30 Sep 2023 21:53:33 +0800 Subject: [PATCH 63/72] ADD file via upload --- mindspore/ccsrc/transform-update/adasum.py | 330 +++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/adasum.py diff --git a/mindspore/ccsrc/transform-update/adasum.py b/mindspore/ccsrc/transform-update/adasum.py new file mode 100644 index 00000000000..df34aa94ab6 --- /dev/null +++ b/mindspore/ccsrc/transform-update/adasum.py @@ -0,0 +1,330 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""adasum""" +import copy +import hashlib +import math +from mindspore.nn.cell import Cell +from mindspore.communication.management import create_group +from mindspore.ops import composite as C +from mindspore.ops import functional as F +from mindspore.ops import operations as P +from mindspore.ops.operations._inner_ops import Send, Receive + + +__all__ = ["AdaSum"] + + +MAX_NUM_HASH = 2 ** 31 + + +_update_parameters = C.MultitypeFuncGraph("update_parameters") + + +@_update_parameters.register("Tensor", "Tensor", "Tensor", "Tensor") + #һΪ _update_parameters_after_broadcast ĺڸ² +def _update_parameters_after_broadcast(delta_weight, update_delta_weight, parameter, old_parameter): + shape = F.shape(delta_weight) # ȡ delta_weight ״ + update_delta_weight = P.Reshape()(update_delta_weight, shape) # update_delta_weight ܳ delta_weight ͬ״ + new_parameter = old_parameter - update_delta_weight # µIJֵͨӾɲмȥ update_delta_weight + return P.Assign()(parameter, new_parameter) # ʹ P.Assign() ²ֵ parameter + + # һΪ _send_before_receive ĺڷݲӦ +def _send_before_receive(send_part, send, recv): + send_ok = send(send_part) # ͨ send send_partؽ洢 send_ok + return recv(send_ok) # recv send_ok ΪݣڽӦ + +# һΪ _receive_before_send ĺȽݺ +def _receive_before_send(send_part, send, recv): + receive_ok = recv(send_part) # ͨ recv send_part Ϊݣڽݣս洢 receive_ok + send_part = F.depend(send_part, receive_ok) # ʹ F.depend() ϵȷ send_part ֮ǰִнղ + return F.depend(receive_ok, send(send_part)) # ͨ send send_part Ϊݣڷݣͽ + +# һΪ _send_recv_res ĺڷͺͽݣѧʹ +def _send_recv_res(left_send, recv_part, local_part, allreduce, parameter_divisibility, allreduce_node_num): + """send result and receive result.""" + if parameter_divisibility: # ɷָִ² + recv_part = P.Squeeze()(recv_part) # ȥ recv_part еά + local_part = F.depend(local_part, recv_part) # ϵȷ local_part ʹǰ recv_part ѱ + eps = 1e-12 + # һЩֵЩֵںļ + value_0 = P.ReduceSum()(local_part * recv_part) + eps + if left_send: + value_1 = P.ReduceSum()(local_part * local_part) + eps + value_2 = P.ReduceSum()(recv_part * recv_part) + eps + else: + value_1 = P.ReduceSum()(recv_part * recv_part) + eps + value_2 = P.ReduceSum()(local_part * local_part) + eps + # ԼֵȫֹԼallreduce + value_0 = allreduce(value_0) + value_1 = F.depend(allreduce(value_1), value_0) + value_2 = F.depend(allreduce(value_2), value_1) + # ҷͣleft_sendյĽ res + if left_send: + res = (1 - (value_0 / (2 * value_1))) * local_part + (1 - (value_0 / (2 * value_2))) * recv_part + else: + res = (1 - (value_0 / (2 * value_1))) * recv_part + (1 - (value_0 / (2 * value_2))) * local_part + else: + res = allreduce(local_part) # ɷָֱִȫֹԼallreduce + res /= allreduce_node_num + return res + + +_adasum_opt_forward = C.MultitypeFuncGraph("adasum_opt_forward") + + +@_adasum_opt_forward.register("Bool", "Function", "Bool", "Int64", "Function", "Function", "Tensor") +# һΪ _adasum_opt_forward_process ĺ Adasum Żǰ +def _adasum_opt_forward_process(left_send, allreduce, parameter_divisibility, allreduce_node_num, send, recv, delta_w): + """adasum optimizer process.""" + if parameter_divisibility: # ɷָִ² + delta_w = P.Squeeze()(delta_w) # ȥ delta_w еά + ori_len = F.shape(delta_w)[0] # ȡ delta_w ԭʼ + divide_len = ori_len / 2 # ָλ + left_part = delta_w[:divide_len] # delta_w Ϊ벿 + right_part = delta_w[divide_len:] # delta_w ΪҰ벿 + else: # ɷָֱӽ delta_w Ƶ벿ֺҰ벿 + left_part = delta_w + right_part = delta_w + + if left_send: # ߷ + if parameter_divisibility: # ɷָִзͲݲ + recv_part = _send_before_receive(left_part, send, recv) + else: # ɷָҰ벿Ϊݲ + recv_part = right_part + # µ delta_w + update_delta_w = _send_recv_res(left_send, recv_part, right_part, allreduce, parameter_divisibility, + allreduce_node_num) + else: # ұ߷ + if parameter_divisibility: # ɷָִнղݲ + recv_part = _receive_before_send(right_part, send, recv) + else: # ɷָ벿Ϊݲ + recv_part = left_part + # µ delta_w + update_delta_w = _send_recv_res(left_send, recv_part, left_part, allreduce, parameter_divisibility, + allreduce_node_num) + + return update_delta_w + + +_adasum_opt_rollback = C.MultitypeFuncGraph("adasum_opt_rollback") + + +@_adasum_opt_rollback.register("Bool", "Bool", "Tensor", "Function", "Function") +# һΪ _adasum_opt_rollback_process ĺ Adasum ŻĻع +def _adasum_opt_rollback_process(left_send, parameter_divisibility, delta_w, send, recv): + """adasum optimizer rollback process.""" + if parameter_divisibility: # ɷָִ² + if left_send: # ߷ݣִзͲݲ + recv_part = _send_before_receive(delta_w, send, recv) + else: # ұ߷ݣִнղݲ + recv_part = _receive_before_send(delta_w, send, recv) + + recv_part = P.Squeeze()(recv_part) # ȥݲֵά + recv_part = P.Reshape()(recv_part, (-1,)) # ܽݲֺ delta_w Ϊһά + delta_w = P.Reshape()(delta_w, (-1,)) + # ҷ͵ݲֺ delta_w ƴ + if left_send: + res = P.Concat()((recv_part, delta_w)) + else: + res = P.Concat()((delta_w, recv_part)) + else: + res = delta_w # ɷָֱӷ delta_w + return res + +#һΪ AdaSum ࣬һԶ㣨CellִAdasum㷨 +#㷨ڷֲʽݲѵѧϰģ͡ +class AdaSum(Cell): + r""" + The Adaptive Summation, or AdaSum, is a novel algorithm for improving distributed data + parallel training of Deep Learning models. + + Args: + rank (int): Rank number. + device_number (int): Device number. + group_number (int): Group number. + parameter_tuple (Tuple(Parameter)): Tuple of parameters. + + Inputs: + - **delta_weights** (Tuple(Tensor)) - Tuple of gradients. + - **parameters** (Tuple(Parameter)) - Tuple of current parameters. + - **old_parameters** (Tuple(Parameter)) - Tuple of last parameters. + + Outputs: + - **adasum_parameters** (Tuple(Tensor)) - Tuple of parameters after adasum process. + """ + def __init__(self, rank, device_number, group_number, parameter_tuple): + super(AdaSum, self).__init__() + self.rank = rank + self.device_number = device_number + self.group_number = group_number + self.parameter_tuple = parameter_tuple + self._generate_communication_op() + self.hyper_map = C.HyperMap() + # ͨŲ + # ÷ڴAdasum㷨ͨŲ͡աȫֹԼȲ + def _generate_communication_op(self): + """generate communication op.""" + self.calc_times = int(math.log(self.group_number, 2)) + self.send_node = [] + self.send_list_forward = [] + self.recv_list_forward = [] + self.send_list_rollback = [] + self.recv_list_rollback = [] + self.allreduce_list = [] + self.broadcast_list = [] + self.parameter_divisibility_list = [] + self.allreduce_node_num_list = [] + last_delta_weights = [] + group_start_rank = (self.rank // self.device_number) * self.device_number + + for step in range(self.calc_times): + current_group = self.device_number * (2 ** step) + sr_target = self.rank + if (sr_target // current_group) % 2 == 0: + dest_target = sr_target + current_group + self.send_node.append(True) + else: + dest_target = sr_target - current_group + self.send_node.append(False) + + neighbor_ids = [] + group_name_last = 0 + for index in range(2 ** (step + 1)): + node_rank = self.rank // self.device_number + double_d = 2 ** (step + 1) + neighbor_id = (node_rank // double_d * double_d + index) * self.device_number + \ + self.rank % self.device_number + neighbor_ids.append(neighbor_id) + group_name_last += neighbor_id + group_name = "adasum_" + str(step) + "_" + str(group_name_last) + create_group(group_name, neighbor_ids) + + send_left = [] + send_right = [] + recv_left = [] + recv_right = [] + allreduce_node_num = () + left_delta_weights, right_delta_weights, delta_weights_divisibility = \ + self._get_delta_weights_info(last_delta_weights) + self.parameter_divisibility_list.append(delta_weights_divisibility) + weights_index = 0 + fusion_id = (step + 1) * 3 + for shape, dtype in left_delta_weights: + send_tag = self._hash(step, sr_target, weights_index) + send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") + send.add_prim_attr("fusion", fusion_id) + recv_tag = self._hash(step, dest_target, weights_index) + recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, + group="hccl_world_group") + recv.add_prim_attr("fusion", fusion_id) + send_left.append(send) + recv_left.append(recv) + weights_index += 1 + for shape, dtype in right_delta_weights: + send_tag = self._hash(step, sr_target, weights_index) + send = Send(sr_tag=send_tag, dest_rank=dest_target, group="hccl_world_group") + send.add_prim_attr("fusion", fusion_id + 1) + recv_tag = self._hash(step, dest_target, weights_index) + recv = Receive(sr_tag=recv_tag, src_rank=dest_target, shape=shape, dtype=dtype, + group="hccl_world_group") + recv.add_prim_attr("fusion", fusion_id + 1) + send_right.append(send) + recv_right.append(recv) + weights_index += 1 + + if self.send_node and self.send_node[-1]: + self.send_list_forward.append(send_left) + self.send_list_rollback.append(send_right) + self.recv_list_forward.append(recv_right) + self.recv_list_rollback.append(recv_left) + last_delta_weights = right_delta_weights + else: + self.send_list_forward.append(send_right) + self.send_list_rollback.append(send_left) + self.recv_list_forward.append(recv_left) + self.recv_list_rollback.append(recv_right) + last_delta_weights = left_delta_weights + + server_all_reduce = P.AllReduce("sum", group_name) + server_all_reduce.add_prim_attr("fusion", fusion_id + 2) + self.allreduce_list.append(server_all_reduce) + + for param_divisibility in delta_weights_divisibility: + if param_divisibility: + allreduce_node_num += (0,) + else: + allreduce_node_num += (2 ** (step + 1),) + self.allreduce_node_num_list.append(allreduce_node_num) + + broadcast_group = [x for x in range(group_start_rank, group_start_rank + self.device_number)] + broadcast_group_name = "broadcast_group_" + str(group_start_rank) + create_group(broadcast_group_name, broadcast_group) + for b_rank in range(len(broadcast_group)): + self.broadcast_list.append(P.Broadcast(b_rank, group=broadcast_group_name)) + self.sync_barrier = P.AllReduce("sum", group=broadcast_group_name) + # ȡݶϢ + # ÷ڻȡݶϢݶȻΪҲ֣жǷɷָ + def _get_delta_weights_info(self, last_delta_weights): + """get delta weights info.""" + half_delta_weights = [] + if last_delta_weights: + half_delta_weights = last_delta_weights + else: + for parameter in self.parameter_tuple: + new_shape = [int(x) for x in parameter.shape] + half_delta_weights.append((new_shape, parameter.dtype)) + left_delta_weights = [] + right_delta_weights = [] + delta_weights_divisibility = () + for shape, dtype in half_delta_weights: + left_shape = copy.deepcopy(shape) + right_shape = copy.deepcopy(shape) + divisibility_flag = False + for i in range(len(shape)): + if shape[i] > 1: + left_shape[i] = int(shape[i] // 2) + right_shape[i] = shape[i] - int(shape[i] // 2) + divisibility_flag = True + break + left_delta_weights.append((left_shape, dtype)) + right_delta_weights.append((right_shape, dtype)) + delta_weights_divisibility += (divisibility_flag,) + return left_delta_weights, right_delta_weights, delta_weights_divisibility + # ϣֵ + # ÷ڼϣֵͨűǩ + def _hash(self, step, target, weights_index): + target = "tag" + str(step) + str(target) + str(weights_index) + target_hash = hashlib.sha1(target.encode()).hexdigest() + hash_res = int(int(target_hash, 16) % MAX_NUM_HASH) + return hash_res + # ִAdasumǰ + # òִִAdasum㷨ǰݵķ͡աȡ + def construct(self, delta_weights, parameters, old_parameters): + forward_weights = [delta_weights] + for i in range(self.calc_times): + process_weights = self.hyper_map(F.partial(_adasum_opt_forward, self.send_node[i], self.allreduce_list[i]), + self.parameter_divisibility_list[i], self.allreduce_node_num_list[i], + self.send_list_forward[i], self.recv_list_forward[i], forward_weights[-1]) + forward_weights.append(process_weights) + for i in range(self.calc_times): + j = self.calc_times - i - 1 + process_weights = self.hyper_map(F.partial(_adasum_opt_rollback, self.send_node[j]), + self.parameter_divisibility_list[j], forward_weights[j + 1], + self.send_list_rollback[j], self.recv_list_rollback[j]) + forward_weights[j] = process_weights + adasum_parameters = self.hyper_map(F.partial(_update_parameters), delta_weights, forward_weights[0], + parameters, old_parameters) + return adasum_parameters -- 2.34.1 From 545d5b255fd303d069865c774e0a7355f1f8a340 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:20:41 +0800 Subject: [PATCH 64/72] ADD file via upload --- .../group_loss_scale_manager.py | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/group_loss_scale_manager.py diff --git a/mindspore/ccsrc/transform-update/group_loss_scale_manager.py b/mindspore/ccsrc/transform-update/group_loss_scale_manager.py new file mode 100644 index 00000000000..1476f9e1276 --- /dev/null +++ b/mindspore/ccsrc/transform-update/group_loss_scale_manager.py @@ -0,0 +1,187 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Group Loss Scale Manager""" +from __future__ import absolute_import +from __future__ import division + +from mindspore.nn.cell import Cell +import mindspore.common.dtype as mstype +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from mindspore.common.parameter import Parameter, ParameterTuple + + +__all__ = ["GroupLossScaleManager"] + + +class GroupLossScaleManager(Cell): + """ + Enhanced hybrid precision algorithm supports multi-layer application of different loss scales and + dynamic updating of loss scales. + 增强型混合精度算法支持不同损耗规模的多层应用损失规模的动态更新。 + Args: + init_loss_scale (Number): The initialized loss scale value. + loss_scale_groups (List): The loss scale groups, which are divided from the param list. + + Inputs: + - **x** (Tensor) - The output of last operator. + - **layer1** (Int) - Current network layer value. + - **layer2** (Int) - Last network layer value. + + Outputs: + - **x** (Tensor) - The output of `_DynamicLossScale` operator. + + Supported Platforms: + ``Ascend`` + + Examples: + >>> import mindspore as ms + >>> from mindspore import boost, nn + >>> + >>> class Net(nn.Cell): + ... def __init__(self, enhanced_amp, num_class=10, num_channel=1): + ... super(Net, self).__init__() + ... self.conv1 = nn.Conv2d(num_channel, 6, 5, pad_mode='valid') + ... self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid') + ... self.fc1 = nn.Dense(16*5*5, 120, weight_init='ones') + ... self.fc2 = nn.Dense(120, 84, weight_init='ones') + ... self.fc3 = nn.Dense(84, num_class, weight_init='ones') + ... self.relu = nn.ReLU() + ... self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2) + ... self.flatten = nn.Flatten() + ... self.enhanced_amp = enhanced_amp + ... + ... def construct(self, x): + ... x = self.enhanced_amp(x, 0, 1) + ... x = self.max_pool2d(self.relu(self.conv1(x))) + ... x = self.max_pool2d(self.relu(self.conv2(x))) + ... x = self.flatten(x) + ... x = self.enhanced_amp(x, 1, 2) + ... x = self.relu(self.fc1(x)) + ... x = self.relu(self.fc2(x)) + ... x = self.fc3(x) + ... x = self.enhanced_amp(x, 2, 3) + ... return x + >>> + >>> loss_scale_manager = boost.GroupLossScaleManager(4096, []) + >>> net = Net(loss_scale_manager) + >>> param_group1 = [] + >>> param_group2 = [] + >>> for param in net.trainable_params(): + >>> if 'conv' in param.name: + >>> param_group1.append(param) + >>> else: + >>> param_group2.append(param) + >>> loss_scale_manager.loss_scale_groups = [param_group1, param_group2] + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = nn.Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> boost_config_dict = {"boost": {"mode": "manual", "less_bn": False, "grad_freeze": False, "adasum": False, \ + >>> "grad_accumulation": False, "dim_reduce": False, "loss_scale_group": True}} + >>> model = ms.Model(net, loss_fn=loss, optimizer=optim, metrics=None, loss_scale_manager=loss_scale_manager, \ + >>> boost_level="O1", boost_config_dict=boost_config_dict) + >>> # For details about how to build the dataset, please refer to the variable `dataset_train` in tutorial + >>> # document on the official website: + >>> # https://www.mindspore.cn/tutorials/zh-CN/master/beginner/quick_start.html + >>> dataset = create_custom_dataset() + >>> model.train(2, dataset) + """ + def __init__(self, init_loss_scale, loss_scale_groups): + super(GroupLossScaleManager, self).__init__() + self._loss_scale = init_loss_scale + self.loss_scale_groups = loss_scale_groups + self.loss_scale_number = 0 + self.layer_loss_scale = None + self.dynamic_loss_scale = None + + def set_loss_scale_status(self, loss_scale_number, init_loss_scale): + """ + Generate dynamic loss scale tuple and set overflow status list. + 生成动态损失规模元组并设置溢出状态列表。 + Args: + loss_scale_number (int): The number of loss scale. + init_loss_scale (float): The initialized loss scale. + """ + # 初始化动态损失尺度管理器 +self.loss_scale_number = loss_scale_number # 设置损失尺度的数量 + +# 创建一个包含动态损失尺度的列表 inner_list +inner_list = [P._DynamicLossScale(layer=x) for x in range(loss_scale_number + 1)] # pylint: disable=W0212 + +# 将 inner_list 转换为元组,作为 layer_loss_scale 的值 +self.layer_loss_scale = tuple(inner_list) + +# 创建动态损失尺度的参数元组 dynamic_loss_scale +self.dynamic_loss_scale = ParameterTuple( + Parameter(Tensor(1, mstype.float32), name='layer_loss_scale_{}'.format(x), requires_grad=False) + for x in range(loss_scale_number + 2)) + +# 初始化动态损失尺度的值 +if isinstance(init_loss_scale, list): + # 如果 init_loss_scale 是列表,将列表中的值设置为动态损失尺度的初始值 + for i, value in enumerate(init_loss_scale): + self.dynamic_loss_scale[i + 1].set_data(value) +else: + # 如果 init_loss_scale 不是列表,使用同一初始值设置所有动态损失尺度的值 + for i in range(self.loss_scale_number): + self.dynamic_loss_scale[i + 1].set_data(init_loss_scale) + + self.dynamic_loss_scale[i + 1].set_data(init_loss_scale) + + def update_loss_scale_status(self, layer, update_ratio): + """ + Update dynamic loss scale. + 更新动态损失规模。 + Args: + layer (int): Current layer. + update_ratio (float): The ratio of loss scale update. + + Outputs: + float, new loss scale. + """ + +# 增加层次计数器 layer +layer = layer + 1 + +# 计算新的损失尺度值 new_loss_scale,它是当前层次的动态损失尺度乘以更新比例 update_ratio 的结果 +new_loss_scale = self.dynamic_loss_scale[layer] * update_ratio + +# 使用 P.Assign() 操作将新的损失尺度值 new_loss_scale 赋值给当前层次的动态损失尺度 +P.Assign()(self.dynamic_loss_scale[layer], new_loss_scale) + +# 返回新的损失尺度值 new_loss_scale,用于后续的损失尺度管理 +return new_loss_scale + + + def construct(self, x, layer1, layer2): + x = self.layer_loss_scale[layer1](x, self.dynamic_loss_scale[layer1] / self.dynamic_loss_scale[layer2]) + return x + + def get_loss_scale(self): + """ + Get loss scale value. + 获取损失规模值。 + Returns: + bool, `loss_scale` value. + """ + return self._loss_scale + + def get_update_cell(self): + """ + Returns the instance of :class:`mindspore.boost.GroupLossScaleManager`. + 返回:class:`mindspore.boost.GroupLossScaleManager`的实例。 + Returns: + :class:`mindspore.boost.GroupLossScaleManager`. + """ + return self -- 2.34.1 From 2bebfc7849dcb65f47680bd06f51b8e9e8e959cf Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:21:16 +0800 Subject: [PATCH 65/72] ADD file via upload --- .../ccsrc/transform-update/transforms.py | 2170 +++++++++++++++++ 1 file changed, 2170 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/transforms.py diff --git a/mindspore/ccsrc/transform-update/transforms.py b/mindspore/ccsrc/transform-update/transforms.py new file mode 100644 index 00000000000..5252119e2cd --- /dev/null +++ b/mindspore/ccsrc/transform-update/transforms.py @@ -0,0 +1,2170 @@ +# Copyright 2021-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The module audio.transforms is inherited from _c_dataengine and is +implemented based on C++. It's a high performance module to process +audio. Users can apply suitable augmentations on audio data to improve +their training models. +对音频数据应用适当的增强功能以改进他们的训练模式。 +""" + +import numpy as np + +import mindspore._c_dataengine as cde +from .utils import BorderType, DensityFunction, FadeShape, GainType, Interpolation, MelType, Modulation, NormType, \ + ResampleMethod, ScaleType, WindowType +from .validators import check_allpass_biquad, check_amplitude_to_db, check_band_biquad, check_bandpass_biquad, \ + check_bandreject_biquad, check_bass_biquad, check_biquad, check_complex_norm, check_compute_deltas, \ + check_contrast, check_db_to_amplitude, check_dc_shift, check_deemph_biquad, check_detect_pitch_frequency, \ + check_dither, check_equalizer_biquad, check_fade, check_flanger, check_gain, check_griffin_lim, \ + check_highpass_biquad, check_inverse_mel_scale, check_lfilter, check_lowpass_biquad, check_magphase, \ + check_mask_along_axis, check_mask_along_axis_iid, check_masking, check_mel_scale, check_mu_law_coding, \ + check_overdrive, check_phase_vocoder, check_phaser, check_resample, check_riaa_biquad, check_sliding_window_cmn, \ + check_spectral_centroid, check_spectrogram, check_time_stretch, check_treble_biquad, check_vad, check_vol +from ..transforms.py_transforms_util import Implementation +from ..transforms.transforms import TensorOperation + + +class AudioTensorOperation(TensorOperation): + """ + Base class of Audio Tensor Ops. + 音频常量运算的基类 + """ + + def __init__(self): + super().__init__() + self.implementation = Implementation.C + + def __call__(self, *input_tensor_list): + for tensor in input_tensor_list: + if not isinstance(tensor, (np.ndarray,)): + raise TypeError("Input should be NumPy audio, got {}.".format(type(tensor))) + return super().__call__(*input_tensor_list) + + def parse(self): + raise NotImplementedError("AudioTensorOperation has to implement parse() method.") + + +class AllpassBiquad(AudioTensorOperation): + r""" + Design two-pole all-pass filter with central frequency and bandwidth for audio waveform. + + An all-pass filter changes the audio's frequency to phase relationship without changing + its frequency to amplitude relationship. The system function is: + + .. math:: + H(s) = \frac{s^2 - \frac{s}{Q} + 1}{s^2 + \frac{s}{Q} + 1} + 针对音频波形设计了具有中心频率和带宽的双极全通滤波器。 + 全通滤波器在不改变的情况下改变音频的频率-相位关系其频率与振幅的关系。系统功能为: + H(s)=\frac{s^2-\frac{s}{Q}+1} + + Similar to `SoX `_ implementation. + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + sample_rate (int): Sampling rate (in Hz), which can't be zero. + central_freq (float): Central frequency (in Hz). + Q (float, optional): `Quality factor `_ , + in range of (0, 1]. Default: 0.707. + + Raises: + TypeError: If `sample_rate` is not of type integer. + ValueError: If `sample_rate` is 0. + TypeError: If `central_freq` is not of type float. + TypeError: If `Q` is not of type float. + ValueError: If `Q` is not in range of (0, 1]. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.AllpassBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_allpass_biquad + # 定义一个类的构造函数,用于初始化对象的属性 +def __init__(self, sample_rate, central_freq, Q=0.707): + super().__init__() # 调用父类的构造函数 + self.sample_rate = sample_rate # 初始化sample_rate属性 + self.central_freq = central_freq # 初始化central_freq属性 + self.quality_factor = Q # 初始化quality_factor属性 + +# 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.AllpassBiquadOperation对象,该对象使用初始化时传入的属性值 + return cde.AllpassBiquadOperation(self.sample_rate, self.central_freq, self.quality_factor) + +# 定义一个名为DE_C_SCALE_TYPE的字典,其中包含一个键值对 +# ScaleType.POWER对应cde.ScaleType.DE_SCALE_TYPE_POWER +DE_C_SCALE_TYPE = {ScaleType.POWER: cde.ScaleType.DE_SCALE_TYPE_POWER} + +class AmplitudeToDB(AudioTensorOperation): + r""" + Turn the input audio waveform from the amplitude/power scale to decibel scale. + + Note: + The dimension of the audio waveform to be processed needs to be (..., freq, time). + 将输入音频波形从振幅/功率刻度转到分贝刻度。 + 注: + 要处理的音频波形的维度需要为(…,freq,time)。 + Args: + stype (ScaleType, optional): Scale of the input waveform, which can be + ScaleType.POWER or ScaleType.MAGNITUDE. Default: ScaleType.POWER. + ref_value (float, optional): Multiplier reference value for generating + `db_multiplier`. Default: 1.0. The formula is + + :math:`\text{db_multiplier} = Log10(max(\text{ref_value}, amin))`. + + amin (float, optional): Lower bound to clamp the input waveform, which must + be greater than zero. Default: 1e-10. + top_db (float, optional): Minimum cut-off decibels, which must be non-negative. Default: 80.0. + + Raises: + TypeError: If `stype` is not of type :class:`mindspore.dataset.audio.utils.ScaleType`. + TypeError: If `ref_value` is not of type float. + ValueError: If `ref_value` is not a positive number. + TypeError: If `amin` is not of type float. + ValueError: If `amin` is not a positive number. + TypeError: If `top_db` is not of type float. + ValueError: If `top_db` is not a positive number. + RuntimeError: If input tensor is not in shape of <..., freq, time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> from mindspore.dataset.audio import ScaleType + >>> + >>> waveform = np.random.random([1, 400 // 2 + 1, 30]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.AmplitudeToDB(stype=ScaleType.POWER)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_amplitude_to_db + def __init__(self, stype=ScaleType.POWER, ref_value=1.0, amin=1e-10, top_db=80.0): + super().__init__() + self.stype = stype + self.ref_value = ref_value + self.amin = amin + self.top_db = top_db + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.AmplitudeToDBOperation对象,该对象使用以下参数进行初始化: + # - DE_C_SCALE_TYPE.get(self.stype): 使用self.stype作为键从DE_C_SCALE_TYPE字典中获取对应的值, + # 用于指定幅度到分贝转换的比例尺类型 + # - self.ref_value: 用于指定幅度到分贝转换的参考值 + # - self.amin: 用于指定幅度到分贝转换中的最小输入值 + # - self.top_db: 用于指定幅度到分贝转换的最大分贝值 + return cde.AmplitudeToDBOperation(DE_C_SCALE_TYPE.get(self.stype), self.ref_value, self.amin, self.top_db) + + + +class Angle(AudioTensorOperation): + """ + Calculate the angle of complex number sequence. + 计算复数序列的角度。 + Note: + The dimension of the audio waveform to be processed needs to be (..., complex=2). + The first dimension represents the real part while the second represents the imaginary. + 注:要处理的音频波形的维度需要是(…,复数=2)。 + 第一个维度表示实部,而第二个维度表示虚部。 + Raises: + RuntimeError: If input tensor is not in shape of <..., complex=2>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[1.43, 5.434], [23.54, 89.38]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Angle()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + def parse(self): + return cde.AngleOperation() + + +class BandBiquad(AudioTensorOperation): + """ + Design two-pole band-pass filter for audio waveform. + + The frequency response drops logarithmically around the center frequency. The + bandwidth gives the slope of the drop. The frequencies at band edge will be + half of their original amplitudes. + 设计了用于音频波形的双极带通滤波器。 + 频率响应在中心频率附近呈对数下降。这个带宽给出了下降的斜率。频带边缘的频率为其原始振幅的一半。 + Similar to `SoX `_ implementation. + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + sample_rate (int): Sampling rate (in Hz), which can't be zero. + central_freq (float): Central frequency (in Hz). + Q (float, optional): `Quality factor `_ , + in range of (0, 1]. Default: 0.707. + noise (bool, optional) : If True, uses the alternate mode for un-pitched audio (e.g. percussion). + If False, uses mode oriented to pitched audio, i.e. voice, singing, or instrumental music. Default: False. + + Raises: + TypeError: If `sample_rate` is not of type integer. + ValueError: If `sample_rate` is 0. + TypeError: If `central_freq` is not of type float. + TypeError: If `Q` is not of type float. + ValueError: If `Q` is not in range of (0, 1]. + TypeError: If `noise` is not of type bool. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BandBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_band_biquad + def __init__(self, sample_rate, central_freq, Q=0.707, noise=False): + super().__init__() + self.sample_rate = sample_rate + self.central_freq = central_freq + self.quality_factor = Q + self.noise = noise + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.BandBiquadOperation对象,该对象使用以下参数进行初始化: + # - self.sample_rate: 采样率,用于设置滤波器的采样率 + # - self.central_freq: 中心频率,用于设置滤波器的中心频率 + # - self.quality_factor: 质量因子,用于设置滤波器的质量因子 + # - self.noise: 噪声参数,用于设置滤波器的噪声参数 + return cde.BandBiquadOperation(self.sample_rate, self.central_freq, self.quality_factor, self.noise) + + + +class BandpassBiquad(AudioTensorOperation): + r""" + Design two-pole Butterworth band-pass filter for audio waveform. + + The frequency response of the Butterworth filter is maximally flat (i.e. has no ripples) + in the passband and rolls off towards zero in the stopband. + + The system function of Butterworth band-pass filter is: + 设计了用于音频波形的双极巴特沃斯带通滤波器。 + 巴特沃斯滤波器的频率响应是最大平坦的(即没有波纹)在通带中并且在阻带中滚向零。 + + 巴特沃斯带通滤波器的系统功能是: + .. math:: + H(s) = \begin{cases} + \frac{s}{s^2 + \frac{s}{Q} + 1}, &\text{if const_skirt_gain=True}; \cr + \frac{\frac{s}{Q}}{s^2 + \frac{s}{Q} + 1}, &\text{if const_skirt_gain=False}. + \end{cases} + + Similar to `SoX `_ implementation. + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + sample_rate (int): Sampling rate (in Hz), which can't be zero. + central_freq (float): Central frequency (in Hz). + Q (float, optional): `Quality factor `_ , + in range of (0, 1]. Default: 0.707. + const_skirt_gain (bool, optional) : If True, uses a constant skirt gain (peak gain = Q); + If False, uses a constant 0dB peak gain. Default: False. + + Raises: + TypeError: If `sample_rate` is not of type integer. + ValueError: If `sample_rate` is 0. + TypeError: If `central_freq` is not of type float. + TypeError: If `Q` is not of type float. + ValueError: If `Q` is not in range of (0, 1]. + TypeError: If `const_skirt_gain` is not of type bool. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BandpassBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_bandpass_biquad + def __init__(self, sample_rate, central_freq, Q=0.707, const_skirt_gain=False): + super().__init__() + self.sample_rate = sample_rate + self.central_freq = central_freq + self.quality_factor = Q + self.const_skirt_gain = const_skirt_gain + + def parse(self): + return cde.BandpassBiquadOperation(self.sample_rate, self.central_freq, self.quality_factor, + self.const_skirt_gain) + + +class BandrejectBiquad(AudioTensorOperation): + r""" + Design two-pole Butterworth band-reject filter for audio waveform. + + The frequency response of the Butterworth filter is maximally flat (i.e. has no ripples) + in the passband and rolls off towards zero in the stopband. + + The system function of Butterworth band-reject filter is: + 针对音频波形设计了双极巴特沃斯带阻滤波器。 + 巴特沃斯滤波器的频率响应是最大平坦的(即没有波纹)在通带中并且在阻带中滚向零。 + 巴特沃斯带阻滤波器的系统功能是: + .. math:: + H(s) = \frac{s^2 + 1}{s^2 + \frac{s}{Q} + 1} + + Similar to `SoX `_ implementation. + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + sample_rate (int): Sampling rate (in Hz), which can't be zero. + central_freq (float): Central frequency (in Hz). + Q (float, optional): `Quality factor `_ , + in range of (0, 1]. Default: 0.707. + + Raises: + TypeError: If `sample_rate` is not of type integer. + ValueError: If `sample_rate` is 0. + TypeError: If `central_freq` is not of type float. + TypeError: If `Q` is not of type float. + ValueError: If `Q` is not in range of (0, 1]. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BandrejectBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_bandreject_biquad + def __init__(self, sample_rate, central_freq, Q=0.707): + super().__init__() + self.sample_rate = sample_rate + self.central_freq = central_freq + self.quality_factor = Q + + def parse(self): + return cde.BandrejectBiquadOperation(self.sample_rate, self.central_freq, self.quality_factor) + + +class BassBiquad(AudioTensorOperation): + r""" + Design a bass tone-control effect, also known as two-pole low-shelf filter for audio waveform. + + A low-shelf filter passes all frequencies, but increase or reduces frequencies below the shelf + frequency by specified amount. The system function is: + 设计一种低音控制效果,也称为二极低架滤波器,用于音频波形。 + 低架滤波器通过所有频率,但增加或减少低于架的频率。频率指定的数量。系统功能为: + .. math:: + H(s) = A\frac{s^2 + \frac{\sqrt{A}}{Q}s + A}{As^2 + \frac{\sqrt{A}}{Q}s + 1} + + Similar to `SoX `_ implementation. + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + sample_rate (int): Sampling rate (in Hz), which can't be zero. + gain (float): Desired gain at the boost (or attenuation) in dB. + central_freq (float, optional): Central frequency (in Hz). Default: 100.0. + Q (float, optional): `Quality factor `_ , + in range of (0, 1]. Default: 0.707. + + Raises: + TypeError: If `sample_rate` is not of type integer. + ValueError: If `sample_rate` is 0. + TypeError: If `gain` is not of type float. + TypeError: If `central_freq` is not of type float. + TypeError: If `Q` is not of type float. + ValueError: If `Q` is not in range of (0, 1]. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BassBiquad(44100, 100.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_bass_biquad + def __init__(self, sample_rate, gain, central_freq=100.0, Q=0.707): + super().__init__() + self.sample_rate = sample_rate + self.gain = gain + self.central_freq = central_freq + self.quality_factor = Q + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.BassBiquadOperation对象,该对象使用以下参数进行初始化: + # - self.sample_rate: 采样率,用于设置滤波器的采样率 + # - self.gain: 增益,用于设置滤波器的增益 + # - self.central_freq: 中心频率,用于设置滤波器的中心频率 + # - self.quality_factor: 质量因子,用于设置滤波器的质量因子 + return cde.BassBiquadOperation(self.sample_rate, self.gain, self.central_freq, self.quality_factor) + + + +class Biquad(TensorOperation): + """ + Perform a biquad filter of input audio. + 对输入音频执行双四元滤波器。 + Args: + b0 (float): Numerator coefficient of current input, x[n]. + b1 (float): Numerator coefficient of input one time step ago x[n-1]. + b2 (float): Numerator coefficient of input two time steps ago x[n-2]. + a0 (float): Denominator coefficient of current output y[n], the value can't be zero, typically 1. + a1 (float): Denominator coefficient of current output y[n-1]. + a2 (float): Denominator coefficient of current output y[n-2]. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> biquad_op = audio.Biquad(0.01, 0.02, 0.13, 1, 0.12, 0.3) + >>> waveform_filtered = biquad_op(waveform) + """ + + @check_biquad + def __init__(self, b0, b1, b2, a0, a1, a2): + super().__init__() + self.b0 = b0 + self.b1 = b1 + self.b2 = b2 + self.a0 = a0 + self.a1 = a1 + self.a2 = a2 + + def parse(self): + return cde.BiquadOperation(self.b0, self.b1, self.b2, self.a0, self.a1, self.a2) + + +class ComplexNorm(AudioTensorOperation): + """ + Compute the norm of complex number sequence. + 计算复数序列的范数。 + Note: + The dimension of the audio waveform to be processed needs to be (..., complex=2). + The first dimension represents the real part while the second represents the imaginary. + + Args: + power (float, optional): Power of the norm, which must be non-negative. Default: 1.0. + + Raises: + TypeError: If `power` is not of type float. + ValueError: If `power` is a negative number. + RuntimeError: If input tensor is not in shape of <..., complex=2>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([2, 4, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.ComplexNorm()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_complex_norm + def __init__(self, power=1.0): + super().__init__() + self.power = power + + def parse(self): + return cde.ComplexNormOperation(self.power) + + +DE_C_BORDER_TYPE = { + BorderType.CONSTANT: cde.BorderType.DE_BORDER_CONSTANT, + BorderType.EDGE: cde.BorderType.DE_BORDER_EDGE, + BorderType.REFLECT: cde.BorderType.DE_BORDER_REFLECT, + BorderType.SYMMETRIC: cde.BorderType.DE_BORDER_SYMMETRIC, +} + + +class ComputeDeltas(AudioTensorOperation): + r""" + Compute delta coefficients of a spectrogram. + 计算谱图的delta系数。 + .. math:: + d_{t}=\frac{{\textstyle\sum_{n=1}^{N}}n(c_{t+n}-c_{t-n})}{2{\textstyle\sum_{n=1}^{N}}n^{2}} + + Args: + win_length (int): The window length used for computing delta, must be no less than 3 (default=5). + pad_mode (BorderType): Mode parameter passed to padding (default=BorderType.EDGE).It can be any of + [BorderType.CONSTANT, BorderType.EDGE, BorderType.REFLECT, BordBorderTypeer.SYMMETRIC]. + + - BorderType.CONSTANT, means it fills the border with constant values. + + - BorderType.EDGE, means it pads with the last value on the edge. + + - BorderType.REFLECT, means it reflects the values on the edge omitting the last + value of edge. + + - BorderType.SYMMETRIC, means it reflects the values on the edge repeating the last + value of edge. + + Examples: + >>> import numpy as np + >>> from mindspore.dataset.audio import BorderType + >>> + >>> waveform = np.random.random([1, 400//2+1, 30]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.ComputeDeltas(win_length=7, pad_mode = BorderType.EDGE)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_compute_deltas + def __init__(self, win_length=5, pad_mode=BorderType.EDGE): + super().__init__() + self.win_len = win_length + self.pad_mode = pad_mode + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.ComputeDeltasOperation对象,该对象使用以下参数进行初始化: + # - self.win_len: 窗口长度,用于计算数据的增量 + # - DE_C_BORDER_TYPE.get(self.pad_mode): 使用self.pad_mode作为键从DE_C_BORDER_TYPE字典中获取对应的值, + # 用于指定边界处理模式 + return cde.ComputeDeltasOperation(self.win_len, DE_C_BORDER_TYPE.get(self.pad_mode)) + + + +class Contrast(AudioTensorOperation): + """ + Apply contrast effect for audio waveform. + + Comparable with compression, this effect modifies an audio signal to make it sound louder. + 对音频波形应用对比度效果。 + Similar to `SoX `_ implementation. + + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + enhancement_amount (float, optional): Controls the amount of the enhancement, + in range of [0, 100]. Default: 75.0. Note that `enhancement_amount` equal + to 0 still gives a significant contrast enhancement. + + Raises: + TypeError: If `enhancement_amount` is not of type float. + ValueError: If `enhancement_amount` is not in range [0, 100]. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Contrast()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_contrast + def __init__(self, enhancement_amount=75.0): + super().__init__() + self.enhancement_amount = enhancement_amount + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.ContrastOperation对象,该对象使用以下参数进行初始化: + # - self.enhancement_amount: 对比度增强参数,用于调整对比度的程度 + return cde.ContrastOperation(self.enhancement_amount) + + + +class DBToAmplitude(AudioTensorOperation): + """ + Turn a waveform from the decibel scale to the power/amplitude scale. + 将波形从分贝刻度转换为功率/振幅刻度。 + Args: + ref (float): Reference which the output will be scaled by. + power (float): If power equals 1, will compute DB to power. If 0.5, will compute DB to amplitude. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.DBToAmplitude(0.5, 0.5)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_db_to_amplitude + def __init__(self, ref, power): + super().__init__() + self.ref = ref + self.power = power + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.DBToAmplitudeOperation对象,该对象使用以下参数进行初始化: + # - self.ref: 参考值,用于幅度到分贝的转换 + # - self.power: 幅度到分贝转换的幂次方,通常为10 + return cde.DBToAmplitudeOperation(self.ref, self.power) + + + +class DCShift(AudioTensorOperation): + """ + Apply a DC shift to the audio. + 对音频进行直流转换。 + Args: + shift (float): The amount to shift the audio, the value must be in the range [-2.0, 2.0]. + limiter_gain (float, optional): Used only on peaks to prevent clipping, + the value should be much less than 1, such as 0.05 or 0.02. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([0.60, 0.97, -1.04, -1.26, 0.97, 0.91, 0.48, 0.93]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.DCShift(0.5, 0.02)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_dc_shift + def __init__(self, shift, limiter_gain=None): + super().__init__() + self.shift = shift + self.limiter_gain = limiter_gain if limiter_gain else shift + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.DCShiftOperation对象,该对象使用以下参数进行初始化: + # - self.shift: 直流偏移量,用于对音频信号进行直流偏移调整 + # - self.limiter_gain: 限幅增益,用于限制音频信号的振幅范围 + return cde.DCShiftOperation(self.shift, self.limiter_gain) + + + +class DeemphBiquad(AudioTensorOperation): + """ + Design two-pole deemph filter for audio waveform of dimension of (..., time). + 针对(…,时间)维度的音频波形,设计了双极deemph滤波器。 + Args: + sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), + the value must be 44100 or 48000. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.DeemphBiquad(44100)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_deemph_biquad + def __init__(self, sample_rate): + super().__init__() + self.sample_rate = sample_rate + + def parse(self): + return cde.DeemphBiquadOperation(self.sample_rate) + + +class DetectPitchFrequency(AudioTensorOperation): + """ + Detect pitch frequency. + + It is implemented using normalized cross-correlation function and median smoothing. + 检测音调频率。 + 它使用归一化互相关函数和中值平滑来实现。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + frame_time (float, optional): Duration of a frame, the value must be greater than zero (default=0.01). + win_length (int, optional): The window length for median smoothing (in number of frames), the value must be + greater than zero (default=30). + freq_low (int, optional): Lowest frequency that can be detected (Hz), the value must be greater than zero + (default=85). + freq_high (int, optional): Highest frequency that can be detected (Hz), the value must be greater than zero + (default=3400). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[0.716064e-03, 5.347656e-03, 6.246826e-03, 2.089477e-02, 7.138305e-02], + ... [4.156616e-02, 1.394653e-02, 3.550292e-02, 0.614379e-02, 3.840209e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.DetectPitchFrequency(30, 0.1, 3, 5, 25)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_detect_pitch_frequency + def __init__(self, sample_rate, frame_time=0.01, win_length=30, freq_low=85, freq_high=3400): + super().__init__() + self.sample_rate = sample_rate + self.frame_time = frame_time + self.win_length = win_length + self.freq_low = freq_low + self.freq_high = freq_high + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.DetectPitchFrequencyOperation对象,该对象使用以下参数进行初始化: + # - self.sample_rate: 采样率,用于音高检测操作 + # - self.frame_time: 帧时间,用于音高检测操作 + # - self.win_length: 窗口长度,用于音高检测操作 + # - self.freq_low: 低频阈值,用于音高检测操作 + # - self.freq_high: 高频阈值,用于音高检测操作 + return cde.DetectPitchFrequencyOperation(self.sample_rate, self.frame_time, + self.win_length, self.freq_low, self.freq_high) + +# 定义一个名为DE_C_DENSITY_FUNCTION的字典,其中包含多个键值对 +# 这些键值对用于将DensityFunction枚举类型映射到cde.DensityFunction枚举类型 +DE_C_DENSITY_FUNCTION = {DensityFunction.TPDF: cde.DensityFunction.DE_DENSITY_FUNCTION_TPDF, + DensityFunction.RPDF: cde.DensityFunction.DE_DENSITY_FUNCTION_RPDF, + DensityFunction.GPDF: cde.DensityFunction.DE_DENSITY_FUNCTION_GPDF} + + + +class Dither(AudioTensorOperation): + """ + Dither increases the perceived dynamic range of audio stored at a + particular bit-depth by eliminating nonlinear truncation distortion. + 抖动增加了存储在通过消除非线性截断失真来确定特定的比特深度。 + Args: + density_function (DensityFunction, optional): The density function of a continuous + random variable. Can be one of DensityFunction.TPDF (Triangular Probability Density Function), + DensityFunction.RPDF (Rectangular Probability Density Function) or + DensityFunction.GPDF (Gaussian Probability Density Function) + (default=DensityFunction.TPDF). + noise_shaping (bool, optional): A filtering process that shapes the spectral + energy of quantisation error (default=False). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[1, 2, 3], [4, 5, 6]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Dither()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_dither + def __init__(self, density_function=DensityFunction.TPDF, noise_shaping=False): + super().__init__() + self.density_function = density_function + self.noise_shaping = noise_shaping + + def parse(self): + return cde.DitherOperation(DE_C_DENSITY_FUNCTION.get(self.density_function), self.noise_shaping) + + +class EqualizerBiquad(AudioTensorOperation): + """ + Design biquad equalizer filter and perform filtering. Similar to SoX implementation. + 设计二阶均衡器滤波器并进行滤波。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + center_freq (float): Central frequency (in Hz). + gain (float): Desired gain at the boost (or attenuation) in dB. + Q (float, optional): https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.EqualizerBiquad(44100, 1500, 5.5, 0.7)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_equalizer_biquad + def __init__(self, sample_rate, center_freq, gain, Q=0.707): + super().__init__() + self.sample_rate = sample_rate + self.center_freq = center_freq + self.gain = gain + self.quality_factor = Q + + def parse(self): + return cde.EqualizerBiquadOperation(self.sample_rate, self.center_freq, self.gain, self.quality_factor) + + +DE_C_FADE_SHAPE = {FadeShape.QUARTER_SINE: cde.FadeShape.DE_FADE_SHAPE_QUARTER_SINE, + FadeShape.HALF_SINE: cde.FadeShape.DE_FADE_SHAPE_HALF_SINE, + FadeShape.LINEAR: cde.FadeShape.DE_FADE_SHAPE_LINEAR, + FadeShape.LOGARITHMIC: cde.FadeShape.DE_FADE_SHAPE_LOGARITHMIC, + FadeShape.EXPONENTIAL: cde.FadeShape.DE_FADE_SHAPE_EXPONENTIAL} + + +class Fade(AudioTensorOperation): + """ + Add a fade in and/or fade out to an waveform. + 将淡入和/或淡出添加到波形中。 + Args: + fade_in_len (int, optional): Length of fade-in (time frames), which must be non-negative (default=0). + fade_out_len (int, optional): Length of fade-out (time frames), which must be non-negative (default=0). + fade_shape (FadeShape, optional): Shape of fade (default=FadeShape.LINEAR). Can be one of + FadeShape.QUARTER_SINE, FadeShape.HALF_SINE, FadeShape.LINEAR, FadeShape.LOGARITHMIC or + FadeShape.EXPONENTIAL. + + -FadeShape.QUARTER_SINE, means it tend to 0 in an quarter sin function. + + -FadeShape.HALF_SINE, means it tend to 0 in an half sin function. + + -FadeShape.LINEAR, means it linear to 0. + + -FadeShape.LOGARITHMIC, means it tend to 0 in an logrithmic function. + + -FadeShape.EXPONENTIAL, means it tend to 0 in an exponential function. + + Raises: + RuntimeError: If fade_in_len exceeds waveform length. + RuntimeError: If fade_out_len exceeds waveform length. + + Examples: + >>> import numpy as np + >>> from mindspore.dataset.audio import FadeShape + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03, 9.246826171875e-03, 1.0894775390625e-02]]) + >>> dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Fade(fade_in_len=3, fade_out_len=2, fade_shape=FadeShape.LINEAR)] + >>> dataset = dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_fade + def __init__(self, fade_in_len=0, fade_out_len=0, fade_shape=FadeShape.LINEAR): + super().__init__() + self.fade_in_len = fade_in_len + self.fade_out_len = fade_out_len + self.fade_shape = fade_shape + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.FadeOperation对象,该对象使用以下参数进行初始化: + # - self.fade_in_len: 淡入长度,用于指定淡入效果的持续时间 + # - self.fade_out_len: 淡出长度,用于指定淡出效果的持续时间 + # - DE_C_FADE_SHAPE.get(self.fade_shape): 使用self.fade_shape作为键从DE_C_FADE_SHAPE字典中获取对应的值, + # 用于指定淡入淡出效果的形状 + return cde.FadeOperation(self.fade_in_len, self.fade_out_len, DE_C_FADE_SHAPE.get(self.fade_shape)) + + + +class Filtfilt(AudioTensorOperation): + """ + Apply an IIR filter forward and backward to a waveform. + 将IIR滤波器向前和向后应用于波形。 + Args: + a_coeffs (Sequence): denominator coefficients of difference equation of dimension of (n_order + 1). + Lower delays coefficients are first, e.g. [a0, a1, a2, ...]. + Must be same size as b_coeffs (pad with 0's as necessary). + b_coeffs (Sequence): numerator coefficients of difference equation of dimension of (n_order + 1). + Lower delays coefficients are first, e.g. [b0, b1, b2, ...]. + Must be same size as a_coeffs (pad with 0's as necessary). + clamp (bool, optional): If True, clamp the output signal to be in the range [-1, 1]. Default=True. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> a_coeffs = [0.1, 0.2, 0.3] + >>> b_coeffs = [0.1, 0.2, 0.3] + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Filtfilt(a_coeffs, b_coeffs)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_lfilter + def __init__(self, a_coeffs, b_coeffs, clamp=True): + super().__init__() + self.a_coeffs = a_coeffs + self.b_coeffs = b_coeffs + self.clamp = clamp + + def parse(self): + return cde.FiltfiltOperation(self.a_coeffs, self.b_coeffs, self.clamp) + + +DE_C_MODULATION = {Modulation.SINUSOIDAL: cde.Modulation.DE_MODULATION_SINUSOIDAL, + Modulation.TRIANGULAR: cde.Modulation.DE_MODULATION_TRIANGULAR} + +DE_C_INTERPOLATION = {Interpolation.LINEAR: cde.Interpolation.DE_INTERPOLATION_LINEAR, + Interpolation.QUADRATIC: cde.Interpolation.DE_INTERPOLATION_QUADRATIC} + + +class Flanger(AudioTensorOperation): + """ + Apply a flanger effect to the audio. + 对音频应用翻边效果。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz). + delay (float, optional): Desired delay in milliseconds (ms), range: [0, 30] (default=0.0). + depth (float, optional): Desired delay depth in milliseconds (ms), range: [0, 10] (default=2.0). + regen (float, optional): Desired regen (feedback gain) in dB, range: [-95, 95] (default=0.0). + width (float, optional): Desired width (delay gain) in dB, range: [0, 100] (default=71.0). + speed (float, optional): Modulation speed in Hz, range: [0.1, 10] (default=0.5). + phase (float, optional): Percentage phase-shift for multi-channel, range: [0, 100] (default=25.0). + modulation (Modulation, optional): Modulation of the input tensor (default=Modulation.SINUSOIDAL). + It can be one of Modulation.SINUSOIDAL or Modulation.TRIANGULAR. + interpolation (Interpolation, optional): Interpolation of the input tensor (default=Interpolation.LINEAR). + It can be one of Interpolation.LINEAR or Interpolation.QUADRATIC. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Flanger(44100)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_flanger + def __init__(self, sample_rate, delay=0.0, depth=2.0, regen=0.0, width=71.0, speed=0.5, phase=25.0, + modulation=Modulation.SINUSOIDAL, interpolation=Interpolation.LINEAR): + super().__init__() + self.sample_rate = sample_rate + self.delay = delay + self.depth = depth + self.regen = regen + self.width = width + self.speed = speed + self.phase = phase + self.modulation = modulation + self.interpolation = interpolation + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.FlangerOperation对象,该对象使用以下参数进行初始化: + # - self.sample_rate: 采样率,用于设置混响效果的采样率 + # - self.delay: 延迟,用于控制混响效果的延迟时间 + # - self.depth: 深度,用于控制混响效果的深度程度 + # - self.regen: 再生,用于控制混响效果的再生率 + # - self.width: 宽度,用于控制混响效果的宽度 + # - self.speed: 速度,用于控制混响效果的速度 + # - self.phase: 相位,用于控制混响效果的相位 + # - DE_C_MODULATION.get(self.modulation): 使用self.modulation作为键从DE_C_MODULATION字典中获取对应的值, + # 用于指定混响效果的调制类型 + # - DE_C_INTERPOLATION.get(self.interpolation): 使用self.interpolation作为键从DE_C_INTERPOLATION字典中获取对应的值, + # 用于指定混响效果的插值方法 + return cde.FlangerOperation(self.sample_rate, self.delay, self.depth, self.regen, self.width, self.speed, + self.phase, DE_C_MODULATION.get(self.modulation), + DE_C_INTERPOLATION.get(self.interpolation)) + + + +class FrequencyMasking(AudioTensorOperation): + """ + Apply masking to a spectrogram in the frequency domain. + 对频域中的频谱图应用掩蔽。 + Note: + The dimension of the audio waveform to be processed needs to be (..., freq, time). + 注:要处理的音频波形的维度需要为(…,freq,time)。 + Args: + iid_masks (bool, optional): Whether to apply different masks to each example/channel. Default: False. + freq_mask_param (int, optional): When `iid_masks` is True, length of the mask will be uniformly sampled + from [0, freq_mask_param]; When `iid_masks` is False, directly use it as length of the mask. + The value should be in range of [0, freq_length], where `freq_length` is the length of audio waveform + in frequency domain. Default: 0. + mask_start (int, optional): Starting point to apply mask, only works when `iid_masks` is True. The value should + be in range of [0, freq_length - freq_mask_param], where `freq_length` is the length of audio waveform + in frequency domain. Default: 0. + mask_value (float, optional): Value to assign to the masked columns. Default: 0.0. + + Raises: + TypeError: If `iid_masks` is not of type bool. + TypeError: If `freq_mask_param` is not of type integer. + ValueError: If `freq_mask_param` is greater than the length of audio waveform in frequency domain. + TypeError: If `mask_start` is not of type integer. + ValueError: If `mask_start` is a negative number. + TypeError: If `mask_value` is not of type float. + ValueError: If `mask_value` is a negative number. + RuntimeError: If input tensor is not in shape of <..., freq, time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 3, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.FrequencyMasking(freq_mask_param=1)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + + .. image:: frequency_masking_original.png + + .. image:: frequency_masking.png + """ + + @check_masking + def __init__(self, iid_masks=False, freq_mask_param=0, mask_start=0, mask_value=0.0): + super().__init__() + self.iid_masks = iid_masks + self.frequency_mask_param = freq_mask_param + self.mask_start = mask_start + self.mask_value = mask_value + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.FrequencyMaskingOperation对象,该对象使用以下参数进行初始化: + # - self.iid_masks: 频率掩蔽参数,用于指定哪些频率将被掩蔽 + # - self.frequency_mask_param: 频率掩蔽参数,用于指定掩蔽的频率范围 + # - self.mask_start: 掩蔽开始,用于指定频率掩蔽的起始位置 + # - self.mask_value: 掩蔽值,用于指定掩蔽后的频率值 + return cde.FrequencyMaskingOperation(self.iid_masks, self.frequency_mask_param, self.mask_start, + self.mask_value) + + + +class Gain(AudioTensorOperation): + """ + Apply amplification or attenuation to the whole waveform. + 对整个波形进行放大或衰减。 + Args: + gain_db (float): Gain adjustment in decibels (dB) (default=1.0). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Gain(1.2)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_gain + def __init__(self, gain_db=1.0): + super().__init__() + self.gain_db = gain_db + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.MelScaleOperation对象,该对象使用以下参数进行初始化: + # - self.n_mels: 梅尔滤波器数量,用于指定梅尔频谱的滤波器数量 + # - self.sample_rate: 采样率,用于梅尔频谱计算 + # - self.f_min: 最低频率,用于梅尔频谱计算的频率范围 + # - self.f_max: 最高频率,用于梅尔频谱计算的频率范围 + # - self.n_stft: 短时傅里叶变换的频域点数,用于梅尔频谱计算 + # - DE_C_NORM_TYPE.get(self.norm): 使用self.norm作为键从DE_C_NORM_TYPE字典中获取对应的值, + # 用于指定梅尔频谱的归一化类型 + # - DE_C_MEL_TYPE.get(self.mel_type): 使用self.mel_type作为键从DE_C_MEL_TYPE字典中获取对应的值, + # 用于指定梅尔频谱的类型 + return cde.MelScaleOperation(self.n_mels, self.sample_rate, self.f_min, self.f_max, self.n_stft, + DE_C_NORM_TYPE.get(self.norm), DE_C_MEL_TYPE.get(self.mel_type)) + + + +class GriffinLim(AudioTensorOperation): + r""" + Approximate magnitude spectrogram inversion using the GriffinLim algorithm. + + .. math:: + x(n)=\frac{\sum_{m=-\infty}^{\infty} w(m S-n) y_{w}(m S, n)}{\sum_{m=-\infty}^{\infty} w^{2}(m S-n)} + + where w represents the window function, y represents the reconstructed signal of each frame and x represents the + whole signal. + 使用GriffinLim算法的近似震级谱图反演。 + + 其中w表示窗函数,y表示每帧的重构信号,x表示整个信号。 + Args: + n_fft (int, optional): Size of FFT (default=400). + n_iter (int, optional): Number of iteration for phase recovery (default=32). + win_length (int, optional): Window size for GriffinLim (default=None, will be set to n_fft). + hop_length (int, optional): Length of hop between STFT windows (default=None, will be set to win_length // 2). + window_type (WindowType, optional): Window type for GriffinLim, which can be WindowType.BARTLETT, + WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN or WindowType.KAISER (default=WindowType.HANN). + Currently kaiser window is not supported on macOS. + power (float, optional): Exponent for the magnitude spectrogram (default=2.0). + momentum (float, optional): The momentum for fast Griffin-Lim (default=0.99). + length (int, optional): Length of the expected output waveform (default=None, will be set to the value of last + dimension of the stft matrix). + rand_init (bool, optional): Flag for random phase initialization or all-zero phase initialization + (default=True). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([201, 6]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.GriffinLim(n_fft=400)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_griffin_lim + def __init__(self, n_fft=400, n_iter=32, win_length=None, hop_length=None, window_type=WindowType.HANN, power=2, + momentum=0.99, length=None, rand_init=True): + super().__init__() + self.n_fft = n_fft + self.n_iter = n_iter + self.win_length = win_length if win_length else self.n_fft + self.hop_length = hop_length if hop_length else self.win_length // 2 + self.window_type = window_type + self.power = power + self.momentum = momentum + self.length = length if length else 0 + self.rand_init = rand_init + + def parse(self): + return cde.GriffinLimOperation(self.n_fft, self.n_iter, self.win_length, self.hop_length, + DE_C_WINDOW_TYPE.get(self.window_type), self.power, self.momentum, self.length, + self.rand_init) + + +class HighpassBiquad(AudioTensorOperation): + """ + Design biquad highpass filter and perform filtering. Similar to SoX implementation. + 设计双四阶高通滤波器并进行滤波。类似于SoX实现。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + cutoff_freq (float): Filter cutoff frequency (in Hz). + Q (float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.HighpassBiquad(44100, 1500, 0.7)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_highpass_biquad + def __init__(self, sample_rate, cutoff_freq, Q=0.707): + super().__init__() + self.sample_rate = sample_rate + self.cutoff_freq = cutoff_freq + self.quality_factor = Q + + def parse(self): + return cde.HighpassBiquadOperation(self.sample_rate, self.cutoff_freq, self.quality_factor) + + +class InverseMelScale(AudioTensorOperation): + """ + Solve for a normal STFT form a mel frequency STFT, using a conversion matrix. + 使用转换矩阵从mel频率STFT中求解正常STFT。 + Args: + n_stft (int): Number of bins in STFT. + n_mels (int, optional): Number of mel filterbanks (default=128). + sample_rate (int, optional): Sample rate of audio signal (default=16000). + f_min (float, optional): Minimum frequency (default=0.0). + f_max (float, optional): Maximum frequency (default=None, will be set to sample_rate // 2). + max_iter (int, optional): Maximum number of optimization iterations (default=100000). + tolerance_loss (float, optional): Value of loss to stop optimization at (default=1e-5). + tolerance_change (float, optional): Difference in losses to stop optimization at (default=1e-8). + sgdargs (dict, optional): Arguments for the SGD optimizer (default=None, will be set to + {'sgd_lr': 0.1, 'sgd_momentum': 0.9}). + norm (NormType, optional): Normalization method, can be NormType.SLANEY or NormType.NONE + (default=NormType.NONE). + mel_type (MelType, optional): Mel scale to use, can be MelType.SLANEY or MelType.HTK (default=MelType.HTK). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.randn(2, 2, 3, 2) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.InverseMelScale(20, 3, 16000, 0, 8000, 10)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_inverse_mel_scale + def __init__(self, n_stft, n_mels=128, sample_rate=16000, f_min=0.0, f_max=None, max_iter=100000, + tolerance_loss=1e-5, tolerance_change=1e-8, sgdargs=None, norm=NormType.NONE, mel_type=MelType.HTK): + super().__init__() + self.n_stft = n_stft + self.n_mels = n_mels + self.sample_rate = sample_rate + self.f_min = f_min + self.f_max = f_max if f_max is not None else sample_rate // 2 + self.max_iter = max_iter + self.tolerance_loss = tolerance_loss + self.tolerance_change = tolerance_change + if sgdargs is None: + self.sgdargs = {'sgd_lr': 0.1, 'sgd_momentum': 0.9} + else: + self.sgdargs = sgdargs + self.norm = norm + self.mel_type = mel_type + + def parse(self): + return cde.InverseMelScaleOperation(self.n_stft, self.n_mels, self.sample_rate, self.f_min, self.f_max, + self.max_iter, self.tolerance_loss, self.tolerance_change, self.sgdargs, + DE_C_NORM_TYPE.get(self.norm), DE_C_MEL_TYPE.get(self.mel_type)) + + +class LFilter(AudioTensorOperation): + """ + Design two-pole filter for audio waveform of dimension of (..., time). + 针对(…,时间)维度的音频波形设计双极滤波器。 + Args: + a_coeffs (sequence): denominator coefficients of difference equation of dimension of (n_order + 1). + Lower delays coefficients are first, e.g. [a0, a1, a2, ...]. + Must be same size as b_coeffs (pad with 0's as necessary). + b_coeffs (sequence): numerator coefficients of difference equation of dimension of (n_order + 1). + Lower delays coefficients are first, e.g. [b0, b1, b2, ...]. + Must be same size as a_coeffs (pad with 0's as necessary). + clamp (bool, optional): If True, clamp the output signal to be in the range [-1, 1] (default=True). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> a_coeffs = [0.1, 0.2, 0.3] + >>> b_coeffs = [0.1, 0.2, 0.3] + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.LFilter(a_coeffs, b_coeffs)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_lfilter + def __init__(self, a_coeffs, b_coeffs, clamp=True): + super().__init__() + self.a_coeffs = a_coeffs + self.b_coeffs = b_coeffs + self.clamp = clamp + + def parse(self): + return cde.LFilterOperation(self.a_coeffs, self.b_coeffs, self.clamp) + + +class LowpassBiquad(AudioTensorOperation): + r""" + Design two-pole low-pass filter for audio waveform. + + A low-pass filter passes frequencies lower than a selected cutoff frequency + but attenuates frequencies higher than it. The system function is: + 设计了音频波形的双极低通滤波器。 + 低通滤波器通过低于选定截止频率的频率,但是衰减高于它的频率。系统功能是: + .. math:: + H(s) = \frac{1}{s^2 + \frac{s}{Q} + 1} + + Similar to `SoX `_ implementation. + + Note: + The dimension of the audio waveform to be processed needs to be (..., time). + + Args: + sample_rate (int): Sampling rate (in Hz), which can't be zero. + cutoff_freq (float): Filter cutoff frequency (in Hz). + Q (float, optional): `Quality factor `_ , + in range of (0, 1]. Default: 0.707. + + Raises: + TypeError: If `sample_rate` is not of type integer. + ValueError: If `sample_rate` is 0. + TypeError: If `cutoff_freq` is not of type float. + TypeError: If `Q` is not of type float. + ValueError: If `Q` is not in range of (0, 1]. + RuntimeError: If input tensor is not in shape of <..., time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[0.8236, 0.2049, 0.3335], [0.5933, 0.9911, 0.2482], + ... [0.3007, 0.9054, 0.7598], [0.5394, 0.2842, 0.5634], [0.6363, 0.2226, 0.2288]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.LowpassBiquad(4000, 1500, 0.7)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_lowpass_biquad + def __init__(self, sample_rate, cutoff_freq, Q=0.707): + super().__init__() + self.sample_rate = sample_rate + self.cutoff_freq = cutoff_freq + self.quality_factor = Q + + def parse(self): + return cde.LowpassBiquadOperation(self.sample_rate, self.cutoff_freq, self.quality_factor) + + +class Magphase(AudioTensorOperation): + """ + Separate a complex-valued spectrogram with shape (..., 2) into its magnitude and phase. + 将形状为(…,2)的复值谱图分离为其幅度和相位。 + Args: + power (float): Power of the norm, which must be non-negative (default=1.0). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([2, 4, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Magphase()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_magphase + def __init__(self, power=1.0): + super().__init__() + self.power = power + + def parse(self): + return cde.MagphaseOperation(self.power) + + +class MaskAlongAxis(AudioTensorOperation): + """ + Apply a mask along `axis`. Mask will be applied from indices `[mask_start, mask_start + mask_width)`. + 沿轴应用遮罩。掩码将从索引“[Mask_start,Mask_start+Mask_width)”应用。 + Args: + mask_start (int): Starting position of the mask, which must be non negative. + mask_width (int): The width of the mask, which must be non negative. + mask_value (float): Value to assign to the masked columns. + axis (int): Axis to apply masking on (1 for frequency and 2 for time). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 20, 20]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.MaskAlongAxis(0, 10, 0.5, 1)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_mask_along_axis + def __init__(self, mask_start, mask_width, mask_value, axis): + super().__init__() + self.mask_start = mask_start + self.mask_width = mask_width + self.mask_value = mask_value + self.axis = axis + + def parse(self): + return cde.MaskAlongAxisOperation(self.mask_start, self.mask_width, self.mask_value, self.axis) + + +class MaskAlongAxisIID(AudioTensorOperation): + """ + Apply a mask along `axis`. Mask will be applied from indices `[mask_start, mask_start + mask_width)`, where + `mask_width` is sampled from `uniform[0, mask_param]`, and `mask_start` from `uniform[0, max_length - mask_width]`, + `max_length` is the number of columns of the specified axis of the spectrogram. + 沿“轴”应用遮罩。掩码将从索引“[Mask_start,Mask_start+Mask_width)”应用,其中 + “mask_width”从“uniform[0],mask_param]”采样,“mask_start”从“uniform[0],max_length-mask_width]”采样,“max_length”是声谱图的指定轴的列数。 + Args: + mask_param (int): Number of columns to be masked, will be uniformly sampled from + [0, mask_param], must be non negative. + mask_value (float): Value to assign to the masked columns. + axis (int): Axis to apply masking on (1 for frequency and 2 for time). + + Examples: + >>> import numpy as np + >>> + >>> waveform= np.random.random([1, 20, 20]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.MaskAlongAxisIID(5, 0.5, 2)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_mask_along_axis_iid + def __init__(self, mask_param, mask_value, axis): + super().__init__() + self.mask_param = mask_param + self.mask_value = mask_value + self.axis = axis + + def parse(self): + return cde.MaskAlongAxisIIDOperation(self.mask_param, self.mask_value, self.axis) + + +DE_C_MEL_TYPE = {MelType.SLANEY: cde.MelType.DE_MEL_TYPE_SLANEY, + MelType.HTK: cde.MelType.DE_MEL_TYPE_HTK} + +DE_C_NORM_TYPE = {NormType.NONE: cde.NormType.DE_NORM_TYPE_NONE, + NormType.SLANEY: cde.NormType.DE_NORM_TYPE_SLANEY} + + +class MelScale(AudioTensorOperation): + """ + Convert normal STFT to STFT at the Mel scale. + 将正常STFT转换为梅尔刻度的STFT。 + Args: + n_mels (int, optional): Number of mel filterbanks (default=128). + sample_rate (int, optional): Sample rate of audio signal (default=16000). + f_min (float, optional): Minimum frequency (default=0). + f_max (float, optional): Maximum frequency (default=None, will be set to sample_rate // 2). + n_stft (int, optional): Number of bins in STFT (default=201). + norm (NormType, optional): Type of norm, value should be NormType.SLANEY or NormType::NONE. + If norm is NormType.SLANEY, divide the triangular mel weight by the width of the mel band. + (default=NormType.NONE). + mel_type (MelType, optional): Type to use, value should be MelType.SLANEY or MelType.HTK (default=MelType.HTK). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[0.8236, 0.2049, 0.3335], [0.5933, 0.9911, 0.2482], + ... [0.3007, 0.9054, 0.7598], [0.5394, 0.2842, 0.5634], [0.6363, 0.2226, 0.2288]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.MelScale(4000, 1500, 0.7)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_mel_scale + def __init__(self, n_mels=128, sample_rate=16000, f_min=0, f_max=None, n_stft=201, norm=NormType.NONE, + mel_type=MelType.HTK): + super().__init__() + self.n_mels = n_mels + self.sample_rate = sample_rate + self.f_min = f_min + self.f_max = f_max if f_max is not None else sample_rate // 2 + self.n_stft = n_stft + self.norm = norm + self.mel_type = mel_type + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.MelScaleOperation对象,该对象使用以下参数进行初始化: + # - self.n_mels: 梅尔滤波器数量,用于指定梅尔频谱的滤波器数量 + # - self.sample_rate: 采样率,用于梅尔频谱计算 + # - self.f_min: 最低频率,用于梅尔频谱计算的频率范围 + # - self.f_max: 最高频率,用于梅尔频谱计算的频率范围 + # - self.n_stft: 短时傅里叶变换的频域点数,用于梅尔频谱计算 + # - DE_C_NORM_TYPE.get(self.norm): 使用self.norm作为键从DE_C_NORM_TYPE字典中获取对应的值, + # 用于指定梅尔频谱的归一化类型 + # - DE_C_MEL_TYPE.get(self.mel_type): 使用self.mel_type作为键从DE_C_MEL_TYPE字典中获取对应的值, + # 用于指定梅尔频谱的类型 + return cde.MelScaleOperation(self.n_mels, self.sample_rate, self.f_min, self.f_max, self.n_stft, + DE_C_NORM_TYPE.get(self.norm), DE_C_MEL_TYPE.get(self.mel_type)) + + + +class MuLawDecoding(AudioTensorOperation): + """ + Decode mu-law encoded signal. + 对μ定律编码信号进行解码。 + Args: + quantization_channels (int): Number of channels, which must be positive (Default: 256). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 3, 4]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.MuLawDecoding()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_mu_law_coding + def __init__(self, quantization_channels=256): + super().__init__() + self.quantization_channels = quantization_channels + + def parse(self): + return cde.MuLawDecodingOperation(self.quantization_channels) + + +class MuLawEncoding(AudioTensorOperation): + """ + Encode signal based on mu-law companding. + 基于μ律压缩扩展对信号进行编码。 + Args: + quantization_channels (int): Number of channels, which must be positive (Default: 256). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 3, 4]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.MuLawEncoding()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_mu_law_coding + def __init__(self, quantization_channels=256): + super().__init__() + self.quantization_channels = quantization_channels + + def parse(self): + return cde.MuLawEncodingOperation(self.quantization_channels) + + +class Overdrive(AudioTensorOperation): + """ + Apply overdrive on input audio. + 对输入音频应用驱动。 + Args: + gain (float): Desired gain at the boost (or attenuation) in dB, in range of [0, 100] (default=20.0). + color (float): Controls the amount of even harmonic content in the over-driven output, + in range of [0, 100] (default=20.0). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Overdrive()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_overdrive + def __init__(self, gain=20.0, color=20.0): + super().__init__() + self.gain = gain + self.color = color + + def parse(self): + return cde.OverdriveOperation(self.gain, self.color) + + +class Phaser(AudioTensorOperation): + """ + Apply a phasing effect to the audio. + 对音频应用定相效果。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz). + gain_in (float): Desired input gain at the boost (or attenuation) in dB. + Allowed range of values is [0, 1] (default=0.4). + gain_out (float): Desired output gain at the boost (or attenuation) in dB. + Allowed range of values is [0, 1e9] (default=0.74). + delay_ms (float): Desired delay in milli seconds. Allowed range of values is [0, 5] (default=3.0). + decay (float): Desired decay relative to gain-in. Allowed range of values is [0, 0.99] (default=0.4). + mod_speed (float): Modulation speed in Hz. Allowed range of values is [0.1, 2] (default=0.5). + sinusoidal (bool): If True, use sinusoidal modulation (preferable for multiple instruments). + If False, use triangular modulation (gives single instruments a sharper + phasing effect) (default=True). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Phaser(44100)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_phaser + def __init__(self, sample_rate, gain_in=0.4, gain_out=0.74, delay_ms=3.0, decay=0.4, mod_speed=0.5, + sinusoidal=True): + super().__init__() + self.decay = decay + self.delay_ms = delay_ms + self.gain_in = gain_in + self.gain_out = gain_out + self.mod_speed = mod_speed + self.sample_rate = sample_rate + self.sinusoidal = sinusoidal + + def parse(self): + return cde.PhaserOperation(self.sample_rate, self.gain_in, self.gain_out, + self.delay_ms, self.decay, self.mod_speed, self.sinusoidal) + + +class PhaseVocoder(AudioTensorOperation): + """ + Given a STFT tensor, speed up in time without modifying pitch by a factor of rate. + 给定STFT张量,在不以速率因子修改音高的情况下及时加速。 + Args: + rate (float): Speed-up factor. + phase_advance (numpy.ndarray): Expected phase advance in each bin in shape of (freq, 1). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([2, 44, 10, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> phase_advance = np.random.random([44, 1]) + >>> transforms = [audio.PhaseVocoder(rate=2, phase_advance=phase_advance)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_phase_vocoder + def __init__(self, rate, phase_advance): + super().__init__() + self.rate = rate + self.phase_advance = cde.Tensor(phase_advance) + + def parse(self): + return cde.PhaseVocoderOperation(self.rate, self.phase_advance) + + +DE_C_RESAMPLE_METHOD = {ResampleMethod.SINC_INTERPOLATION: cde.ResampleMethod.DE_RESAMPLE_SINC_INTERPOLATION, + ResampleMethod.KAISER_WINDOW: cde.ResampleMethod.DE_RESAMPLE_KAISER_WINDOW} + + +class Resample(AudioTensorOperation): + """ + Resample a signal from one frequency to another. A resample method can be given. + 将信号从一个频率重新采样到另一个频率。可以给出一种重新采样的方法。 + Args: + orig_freq (float, optional): The original frequency of the signal, which must be positive (default=16000). + new_freq (float, optional): The desired frequency, which must be positive (default=16000). + resample_method (ResampleMethod, optional): The resample method, which can be + ResampleMethod.SINC_INTERPOLATION and ResampleMethod.KAISER_WINDOW + (default=ResampleMethod.SINC_INTERPOLATION). + lowpass_filter_width (int, optional): Controls the shaperness of the filter, more means sharper but less + efficient, which must be positive (default=6). + rolloff (float, optional): The roll-off frequency of the filter, as a fraction of the Nyquist. Lower values + reduce anti-aliasing, but also reduce some of the highest frequencies, range: (0, 1] (default=0.99). + beta (float, optional): The shape parameter used for kaiser window (default=None, will use 14.769656459379492). + + Examples: + >>> import numpy as np + >>> from mindspore.dataset.audio import ResampleMethod + >>> + >>> waveform = np.random.random([1, 30]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Resample(orig_freq=48000, new_freq=16000, + ... resample_method=ResampleMethod.SINC_INTERPOLATION, + ... lowpass_filter_width=6, rolloff=0.99, beta=None)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_resample + def __init__(self, orig_freq=16000, new_freq=16000, resample_method=ResampleMethod.SINC_INTERPOLATION, + lowpass_filter_width=6, rolloff=0.99, beta=None): + super().__init__() + self.orig_freq = orig_freq + self.new_freq = new_freq + self.resample_method = resample_method + self.lowpass_filter_width = lowpass_filter_width + self.rolloff = rolloff + kaiser_beta = 14.769656459379492 + self.beta = beta if beta is not None else kaiser_beta + + def parse(self): + return cde.ResampleOperation(self.orig_freq, self.new_freq, DE_C_RESAMPLE_METHOD.get(self.resample_method), + self.lowpass_filter_width, self.rolloff, self.beta) + + +class RiaaBiquad(AudioTensorOperation): + """ + Apply RIAA vinyl playback equalization. Similar to SoX implementation. + 应用RIAA乙烯基播放均衡。 + Args: + sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), + can only be one of 44100, 48000, 88200, 96000. + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.RiaaBiquad(44100)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_riaa_biquad + def __init__(self, sample_rate): + super().__init__() + self.sample_rate = sample_rate + + def parse(self): + return cde.RiaaBiquadOperation(self.sample_rate) + + +class SlidingWindowCmn(AudioTensorOperation): + """ + Apply sliding-window cepstral mean (and optionally variance) normalization per utterance. + 对每个话语应用滑动窗口倒频谱均值(以及可选的方差)归一化。 + Args: + cmn_window (int, optional): Window in frames for running average CMN computation (default=600). + min_cmn_window (int, optional): Minimum CMN window used at start of decoding (adds latency only at start). + Only applicable if center is False, ignored if center is True (default=100). + center (bool, optional): If True, use a window centered on the current frame. If False, window is + to the left. (default=False). + norm_vars (bool, optional): If True, normalize variance to one. (default=False). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[[1, 2, 3], [4, 5, 6]]], dtype=np.float64) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.SlidingWindowCmn()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_sliding_window_cmn + def __init__(self, cmn_window=600, min_cmn_window=100, center=False, norm_vars=False): + super().__init__() + self.cmn_window = cmn_window + self.min_cmn_window = min_cmn_window + self.center = center + self.norm_vars = norm_vars + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.SlidingWindowCmnOperation对象,该对象使用以下参数进行初始化: + # - self.cmn_window: 滑动窗口大小,用于计算滑动窗口的大小 + # - self.min_cmn_window: 最小滑动窗口大小,用于设置最小的滑动窗口大小 + # - self.center: 中心化,用于控制是否对滑动窗口计算进行中心化处理 + # - self.norm_vars: 归一化方差,用于控制是否对滑动窗口计算进行方差归一化处理 + return cde.SlidingWindowCmnOperation(self.cmn_window, self.min_cmn_window, self.center, self.norm_vars) + +# 定义一个名为DE_C_WINDOW_TYPE的字典,其中包含多个键值对 +# 这些键值对用于将WindowType枚举类型映射到cde.WindowType枚举类型 +DE_C_WINDOW_TYPE = {WindowType.BARTLETT: cde.WindowType.DE_WINDOW_TYPE_BARTLETT, + WindowType.BLACKMAN: cde.WindowType.DE_WINDOW_TYPE_BLACKMAN, + WindowType.HAMMING: cde.WindowType.DE_WINDOW_TYPE_HAMMING, + WindowType.HANN: cde.WindowType.DE_WINDOW_TYPE_HANN, + WindowType.KAISER: cde.WindowType.DE_WINDOW_TYPE_KAISER} + + + +class SpectralCentroid(TensorOperation): + """ + Create a spectral centroid from an audio signal. + 从音频信号创建频谱质心。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz). + n_fft (int, optional): Size of FFT, creates n_fft // 2 + 1 bins (default=400). + win_length (int, optional): Window size (default=None, will use n_fft). + hop_length (int, optional): Length of hop between STFT windows (default=None, will use win_length // 2). + pad (int, optional): Two sided padding of signal (default=0). + window (WindowType, optional): Window function that is applied/multiplied to each frame/window, + which can be WindowType.BARTLETT, WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN + or WindowType.KAISER (default=WindowType.HANN). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([5, 10, 20]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.SpectralCentroid(44100)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_spectral_centroid + def __init__(self, sample_rate, n_fft=400, win_length=None, hop_length=None, pad=0, window=WindowType.HANN): + super().__init__() + self.sample_rate = sample_rate + self.pad = pad + self.window = window + self.n_fft = n_fft + self.win_length = win_length if win_length else n_fft + self.hop_length = hop_length if hop_length else self.win_length // 2 + + def parse(self): + return cde.SpectralCentroidOperation(self.sample_rate, self.n_fft, self.win_length, self.hop_length, + self.pad, DE_C_WINDOW_TYPE.get(self.window)) + + +class Spectrogram(TensorOperation): + """ + Create a spectrogram from an audio signal. + 根据音频信号创建声谱图。 + Args: + n_fft (int, optional): Size of FFT, creates n_fft // 2 + 1 bins (default=400). + win_length (int, optional): Window size (default=None, will use n_fft). + hop_length (int, optional): Length of hop between STFT windows (default=None, will use win_length // 2). + pad (int): Two sided padding of signal (default=0). + window (WindowType, optional): Window function that is applied/multiplied to each frame/window, + which can be WindowType.BARTLETT, WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN + or WindowType.KAISER (default=WindowType.HANN). Currently kaiser window is not supported on macOS. + power (float, optional): Exponent for the magnitude spectrogram, which must be greater + than or equal to 0, e.g., 1 for energy, 2 for power, etc. (default=2.0). + normalized (bool, optional): Whether to normalize by magnitude after stft (default=False). + center (bool, optional): Whether to pad waveform on both sides (default=True). + pad_mode (BorderType, optional): Controls the padding method used when center is True, + which can be BorderType.REFLECT, BorderType.CONSTANT, BorderType.EDGE, BorderType.SYMMETRIC + (default=BorderType.REFLECT). + onesided (bool, optional): Controls whether to return half of results to avoid redundancy (default=True). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([5, 10, 20]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Spectrogram()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_spectrogram + def __init__(self, n_fft=400, win_length=None, hop_length=None, pad=0, window=WindowType.HANN, power=2.0, + normalized=False, center=True, pad_mode=BorderType.REFLECT, onesided=True): + super().__init__() + self.n_fft = n_fft + self.win_length = win_length if win_length else n_fft + self.hop_length = hop_length if hop_length else self.win_length // 2 + self.pad = pad + self.window = window + self.power = power + self.normalized = normalized + self.center = center + self.pad_mode = pad_mode + self.onesided = onesided + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.SpectrogramOperation对象,该对象使用以下参数进行初始化: + # - self.n_fft: FFT大小,用于傅里叶变换的窗口大小 + # - self.win_length: 窗口长度,用于短时傅里叶变换的窗口大小 + # - self.hop_length: 跳跃长度,用于短时傅里叶变换的帧之间的距离 + # - self.pad: 零填充,用于在窗口之外填充零值以增加分析窗口的大小 + # - DE_C_WINDOW_TYPE.get(self.window): 使用self.window作为键从DE_C_WINDOW_TYPE字典中获取对应的值, + # 用于指定窗口函数的类型 + # - self.power: 能量谱的幂次方,通常为2 + # - self.normalized: 是否对能量谱进行归一化 + # - self.center: 是否对窗口函数进行居中处理 + # - DE_C_BORDER_TYPE.get(self.pad_mode): 使用self.pad_mode作为键从DE_C_BORDER_TYPE字典中获取对应的值, + # 用于指定边界处理模式 + # - self.onesided: 是否仅计算单边频谱 + return cde.SpectrogramOperation(self.n_fft, self.win_length, self.hop_length, self.pad, + DE_C_WINDOW_TYPE.get(self.window), self.power, self.normalized, + self.center, DE_C_BORDER_TYPE.get(self.pad_mode), self.onesided) + + + +class TimeMasking(AudioTensorOperation): + """ + Apply masking to a spectrogram in the time domain. + + Note: + The dimension of the audio waveform to be processed needs to be (..., freq, time). + 在时域中对声谱图应用掩蔽。 + 注: + 要处理的音频波形的维度需要为(…,freq,time)。 + + Args: + iid_masks (bool, optional): Whether to apply different masks to each example/channel. Default: False. + time_mask_param (int, optional): When `iid_masks` is True, length of the mask will be uniformly sampled + from [0, time_mask_param]; When `iid_masks` is False, directly use it as length of the mask. + The value should be in range of [0, time_length], where `time_length` is the length of audio waveform + in time domain. Default: 0. + mask_start (int, optional): Starting point to apply mask, only works when `iid_masks` is True. The value should + be in range of [0, time_length - time_mask_param], where `time_length` is the length of audio waveform + in time domain. Default: 0. + mask_value (float, optional): Value to assign to the masked columns. Default: 0.0. + + Raises: + TypeError: If `iid_masks` is not of type bool. + TypeError: If `time_mask_param` is not of type int. + ValueError: If `time_mask_param` is greater than the length of audio waveform in time domain. + TypeError: If `mask_start` is not of type int. + ValueError: If `mask_start` a negative number. + TypeError: If `mask_value` is not of type float. + ValueError: If `mask_value` is a negative number. + RuntimeError: If input tensor is not in shape of <..., freq, time>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([4, 3, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.TimeMasking(time_mask_param=1)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + + .. image:: time_masking_original.png + + .. image:: time_masking.png + """ + + @check_masking + def __init__(self, iid_masks=False, time_mask_param=0, mask_start=0, mask_value=0.0): + super().__init__() + self.iid_masks = iid_masks + self.time_mask_param = time_mask_param + self.mask_start = mask_start + self.mask_value = mask_value + + def parse(self): + return cde.TimeMaskingOperation(self.iid_masks, self.time_mask_param, self.mask_start, self.mask_value) + + +class TimeStretch(AudioTensorOperation): + """ + Stretch Short Time Fourier Transform (STFT) in time without modifying pitch for a given rate. + + Note: + The dimension of the audio waveform to be processed needs to be (..., freq, time, complex=2). + The first dimension represents the real part while the second represents the imaginary. + 在不修改给定速率的音高的情况下,在时间上拉伸短时傅立叶变换。 + + 注: + 要处理的音频波形的维度需要为(…,freq,time,complex=2)。 + 第一个维度表示实部,而第二个维度表示虚部。 + Args: + hop_length (int, optional): Length of hop between STFT windows, i.e. the number of samples + between consecutive frames. Default: None, will use `n_freq - 1`. + n_freq (int, optional): Number of filter banks from STFT. Default: 201. + fixed_rate (float, optional): Rate to speed up or slow down by. Default: None, will keep + the original rate. + + Raises: + TypeError: If `hop_length` is not of type integer. + ValueError: If `hop_length` is not a positive number. + TypeError: If `n_freq` is not of type integer. + ValueError: If `n_freq` is not a positive number. + TypeError: If `fixed_rate` is not of type float. + ValueError: If `fixed_rate` is not a positive number. + RuntimeError: If input tensor is not in shape of <..., freq, num_frame, complex=2>. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([44, 10, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.TimeStretch()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + + .. image:: time_stretch_rate1.5.png + + .. image:: time_stretch_original.png + + .. image:: time_stretch_rate0.8.png + """ + + @check_time_stretch + def __init__(self, hop_length=None, n_freq=201, fixed_rate=None): + super().__init__() + self.n_freq = n_freq + self.fixed_rate = fixed_rate + + n_fft = (n_freq - 1) * 2 + self.hop_length = hop_length if hop_length is not None else n_fft // 2 + self.fixed_rate = fixed_rate if fixed_rate is not None else 1 + + def parse(self): + return cde.TimeStretchOperation(self.hop_length, self.n_freq, self.fixed_rate) + + +class TrebleBiquad(AudioTensorOperation): + """ + Design a treble tone-control effect. Similar to SoX implementation. + 设计高音音调控制效果。 + Args: + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + gain (float): Desired gain at the boost (or attenuation) in dB. + central_freq (float, optional): Central frequency (in Hz) (default=3000). + Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.TrebleBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_treble_biquad + def __init__(self, sample_rate, gain, central_freq=3000, Q=0.707): + super().__init__() + self.sample_rate = sample_rate + self.gain = gain + self.central_freq = central_freq + self.quality_factor = Q + + def parse(self): + return cde.TrebleBiquadOperation(self.sample_rate, self.gain, self.central_freq, self.quality_factor) + + +class Vad(AudioTensorOperation): + """ + Attempt to trim silent background sounds from the end of the voice recording. + 尝试从语音录制结束时修剪无声背景声音。 + Args: + sample_rate (int): Sample rate of audio signal. + trigger_level (float, optional): The measurement level used to trigger activity detection (default=7.0). + trigger_time (float, optional): The time constant (in seconds) used to help ignore short sounds (default=0.25). + search_time (float, optional): The amount of audio (in seconds) to search for quieter/shorter sounds to include + prior to the detected trigger point (default=1.0). + allowed_gap (float, optional): The allowed gap (in seconds) between quiteter/shorter sounds to include prior to + the detected trigger point (default=0.25). + pre_trigger_time (float, optional): The amount of audio (in seconds) to preserve before the trigger point and + any found quieter/shorter bursts (default=0.0). + boot_time (float, optional): The time for the initial noise estimate (default=0.35). + noise_up_time (float, optional): Time constant used by the adaptive noise estimator, when the noise level is + increasing (default=0.1). + noise_down_time (float, optional): Time constant used by the adaptive noise estimator, when the noise level is + decreasing (default=0.01). + noise_reduction_amount (float, optional): The amount of noise reduction used in the detection algorithm + (default=1.35). + measure_freq (float, optional): The frequency of the algorithm’s processing (default=20.0). + measure_duration (float, optional): The duration of measurement (default=None, use twice the measurement + period). + measure_smooth_time (float, optional): The time constant used to smooth spectral measurements (default=0.4). + hp_filter_freq (float, optional): The "Brick-wall" frequency of high-pass filter applied at the input to the + detector algorithm (default=50.0). + lp_filter_freq (float, optional): The "Brick-wall" frequency of low-pass filter applied at the input to the + detector algorithm (default=6000.0). + hp_lifter_freq (float, optional): The "Brick-wall" frequency of high-pass lifter applied at the input to the + detector algorithm (default=150.0). + lp_lifter_freq (float, optional): The "Brick-wall" frequency of low-pass lifter applied at the input to the + detector algorithm (default=2000.0). + + Examples: + >>> import numpy as np + >>> + >>> waveform = np.random.random([2, 1000]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Vad(sample_rate=600)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_vad + def __init__(self, sample_rate, trigger_level=7.0, trigger_time=0.25, search_time=1.0, allowed_gap=0.25, + pre_trigger_time=0.0, boot_time=0.35, noise_up_time=0.1, noise_down_time=0.01, + noise_reduction_amount=1.35, measure_freq=20.0, measure_duration=None, measure_smooth_time=0.4, + hp_filter_freq=50.0, lp_filter_freq=6000.0, hp_lifter_freq=150.0, lp_lifter_freq=2000.0): + super().__init__() + self.sample_rate = sample_rate + self.trigger_level = trigger_level + self.trigger_time = trigger_time + self.search_time = search_time + self.allowed_gap = allowed_gap + self.pre_trigger_time = pre_trigger_time + self.boot_time = boot_time + self.noise_up_time = noise_up_time + self.noise_down_time = noise_down_time + self.noise_reduction_amount = noise_reduction_amount + self.measure_freq = measure_freq + self.measure_duration = measure_duration if measure_duration else 2.0 / measure_freq + self.measure_smooth_time = measure_smooth_time + self.hp_filter_freq = hp_filter_freq + self.lp_filter_freq = lp_filter_freq + self.hp_lifter_freq = hp_lifter_freq + self.lp_lifter_freq = lp_lifter_freq + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.VadOperation对象,该对象使用以下参数进行初始化: + # - self.sample_rate: 采样率,用于声学活动检测操作 + # - self.trigger_level: 触发级别,用于控制何时触发声学活动检测 + # - self.trigger_time: 触发时间,用于设置触发声学活动检测的时间 + # - self.search_time: 搜索时间,用于设置声学活动检测的搜索时间 + # - self.allowed_gap: 允许的间隔时间,用于设置声学活动检测期间允许的静默时间 + # - self.pre_trigger_time: 预触发时间,用于设置声学活动检测前的时间 + # - self.boot_time: 启动时间,用于设置声学活动检测的启动时间 + # - self.noise_up_time: 噪声上升时间,用于设置声学活动检测中噪声上升的时间 + # - self.noise_down_time: 噪声下降时间,用于设置声学活动检测中噪声下降的时间 + # - self.noise_reduction_amount: 噪声降低量,用于控制噪声的降低程度 + # - self.measure_freq: 测量频率,用于设置声学活动检测的测量频率 + # - self.measure_duration: 测量持续时间,用于设置声学活动检测的测量持续时间 + # - self.measure_smooth_time: 测量平滑时间,用于平滑声学活动检测的测量结果 + # - self.hp_filter_freq: 高通滤波器频率,用于设置声学活动检测中的高通滤波器频率 + # - self.lp_filter_freq: 低通滤波器频率,用于设置声学活动检测中的低通滤波器频率 + # - self.hp_lifter_freq: 高通提升器频率,用于设置声学活动检测中的高通提升器频率 + # - self.lp_lifter_freq: 低通提升器频率,用于设置声学活动检测中的低通提升器频率 + return cde.VadOperation(self.sample_rate, self.trigger_level, self.trigger_time, self.search_time, + self.allowed_gap, self.pre_trigger_time, self.boot_time, self.noise_up_time, + self.noise_down_time, self.noise_reduction_amount, self.measure_freq, + self.measure_duration, self.measure_smooth_time, self.hp_filter_freq, + self.lp_filter_freq, self.hp_lifter_freq, self.lp_lifter_freq) + +# 定义一个名为DE_C_GAIN_TYPE的字典,其中包含多个键值对 +# 这些键值对用于将GainType枚举类型映射到cde.GainType枚举类型 +DE_C_GAIN_TYPE = {GainType.AMPLITUDE: cde.GainType.DE_GAIN_TYPE_AMPLITUDE, + GainType.POWER: cde.GainType.DE_GAIN_TYPE_POWER, + GainType.DB: cde.GainType.DE_GAIN_TYPE_DB} + + + +class Vol(AudioTensorOperation): + """ + Apply amplification or attenuation to the whole waveform. + 对整个波形进行放大或衰减。 + Args: + gain (float): Value of gain adjustment. + If gain_type = amplitude, gain stands for nonnegative amplitude ratio. + If gain_type = power, gain stands for power. + If gain_type = db, gain stands for decibels. + gain_type (GainType, optional): Type of gain, contains the following three enumeration values + GainType.AMPLITUDE, GainType.POWER and GainType.DB (default=GainType.AMPLITUDE). + + Examples: + >>> import numpy as np + >>> from mindspore.dataset.audio import GainType + >>> + >>> waveform = np.random.random([20, 30]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Vol(gain=10, gain_type=GainType.DB)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_vol + def __init__(self, gain, gain_type=GainType.AMPLITUDE): + super().__init__() + self.gain = gain + self.gain_type = gain_type + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.VadOperation对象,该对象使用以下参数进行初始化: + # - self.sample_rate: 采样率,用于声学活动检测操作 + # - self.trigger_level: 触发级别,用于控制何时触发声学活动检测 + # - self.trigger_time: 触发时间,用于设置触发声学活动检测的时间 + # - self.search_time: 搜索时间,用于设置声学活动检测的搜索时间 + # - self.allowed_gap: 允许的间隔时间,用于设置声学活动检测期间允许的静默时间 + # - self.pre_trigger_time: 预触发时间,用于设置声学活动检测前的时间 + # - self.boot_time: 启动时间,用于设置声学活动检测的启动时间 + # - self.noise_up_time: 噪声上升时间,用于设置声学活动检测中噪声上升的时间 + # - self.noise_down_time: 噪声下降时间,用于设置声学活动检测中噪声下降的时间 + # - self.noise_reduction_amount: 噪声降低量,用于控制噪声的降低程度 + # - self.measure_freq: 测量频率,用于设置声学活动检测的测量频率 + # - self.measure_duration: 测量持续时间,用于设置声学活动检测的测量持续时间 + # - self.measure_smooth_time: 测量平滑时间,用于平滑声学活动检测的测量结果 + # - self.hp_filter_freq: 高通滤波器频率,用于设置声学活动检测中的高通滤波器频率 + # - self.lp_filter_freq: 低通滤波器频率,用于设置声学活动检测中的低通滤波器频率 + # - self.hp_lifter_freq: 高通提升器频率,用于设置声学活动检测中的高通提升器频率 + # - self.lp_lifter_freq: 低通提升器频率,用于设置声学活动检测中的低通提升器频率 + return cde.VadOperation(self.sample_rate, self.trigger_level, self.trigger_time, self.search_time, + self.allowed_gap, self.pre_trigger_time, self.boot_time, self.noise_up_time, + self.noise_down_time, self.noise_reduction_amount, self.measure_freq, + self.measure_duration, self.measure_smooth_time, self.hp_filter_freq, + self.lp_filter_freq, self.hp_lifter_freq, self.lp_lifter_freq) + +# 定义一个名为DE_C_GAIN_TYPE的字典,其中包含多个键值对 +# 这些键值对用于将GainType枚举类型映射到cde.GainType枚举类型 +DE_C_GAIN_TYPE = {GainType.AMPLITUDE: cde.GainType.DE_GAIN_TYPE_AMPLITUDE, + GainType.POWER: cde.GainType.DE_GAIN_TYPE_POWER, + GainType.DB: cde.GainType.DE_GAIN_TYPE_DB} + -- 2.34.1 From 5b1b14448a88c741c42d6b6ec3517314a9fc8e84 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:21:57 +0800 Subject: [PATCH 66/72] ADD file via upload --- mindspore/ccsrc/transform-update/config.py | 858 +++++++++++++++++++++ 1 file changed, 858 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/config.py diff --git a/mindspore/ccsrc/transform-update/config.py b/mindspore/ccsrc/transform-update/config.py new file mode 100644 index 00000000000..e93b826809c --- /dev/null +++ b/mindspore/ccsrc/transform-update/config.py @@ -0,0 +1,858 @@ +# Copyright 2019-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The configuration module provides various functions to set and get the supported +configuration parameters, and read a configuration file. + +Common imported modules in corresponding API examples are as follows: +配置模块提供了各种功能来设置和获取支持配置参数,并读取配置文件。 +.. code-block:: + + import mindspore.dataset as ds +""" +import os +import platform +import random +import numpy +import mindspore._c_dataengine as cde +from mindspore import log as logger +from .validator_helpers import replace_none + +__all__ = ['set_sending_batches', 'load', '_init_device_info', + 'set_seed', 'get_seed', + 'set_prefetch_size', 'get_prefetch_size', + 'set_num_parallel_workers', 'get_num_parallel_workers', + 'set_numa_enable', 'get_numa_enable', + 'set_monitor_sampling_interval', 'get_monitor_sampling_interval', + 'set_callback_timeout', 'get_callback_timeout', + 'set_auto_num_workers', 'get_auto_num_workers', + 'set_enable_shared_mem', 'get_enable_shared_mem', + 'set_enable_autotune', 'get_enable_autotune', + 'set_autotune_interval', 'get_autotune_interval', + 'set_auto_offload', 'get_auto_offload', + 'set_enable_watchdog', 'get_enable_watchdog', + 'set_multiprocessing_timeout_interval', 'get_multiprocessing_timeout_interval'] + +INT32_MAX = 2147483647 +UINT32_MAX = 4294967295 + +_config = cde.GlobalContext.config_manager() + + +def _init_device_info(): + """ + INTERNAL USE ONLY! + As rank_id need to pass into deep layer for numa and device_queue. + One process work with only one rank_id, In standalone scenario, + rank_id may come from env 'CUDA_VISIBLE_DEVICES', For distribute + scenario, rank_id come from _get_global_rank(). + 由于rank_id需要传递到numa和device_queue的深层。 + 一个进程只使用一个rank_id,在独立场景中,rank_id可能来自env“CUDA_VISIBLE_DEVICES”,用于分发场景中,rank_id来自_get_global_rank()。 + """ + # 导入必要的模块和类 +from mindspore import context +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.parallel._utils import _get_global_rank +import os + + # 默认情况下禁用 NUMA(Non-Uniform Memory Access)支持 + numa_enable = False + + # 从环境变量中获取 NUMA_ENABLE(DATASET_ENABLE_NUMA 和 MS_ENABLE_NUMA 之一)的值,如果为 "True" 则启用 NUMA + numa_enable_env = os.getenv("DATASET_ENABLE_NUMA", None) + if numa_enable_env and numa_enable_env.strip() == 'True': + numa_enable = True + + numa_enable_env = os.getenv("MS_ENABLE_NUMA", None) + if numa_enable_env and numa_enable_env.strip() == 'True': + numa_enable = True + + # 获取当前设备的目标(GPU 或 Ascend) + device_target = context.get_context("device_target") + + # 如果设备目标是 GPU + if device_target == "GPU": + # 获取全局排名(global rank) + rank_id = _get_global_rank() + + # 获取并行模式(parallel mode) + parallel_mode = auto_parallel_context().get_parallel_mode() + + # 如果并行模式为 "stand_alone",则将 rank_id 设置为设备的 ID(device_id) + if parallel_mode == "stand_alone": + rank_id = context.get_context("device_id") + + # 如果启用了 NUMA,则设置 NUMA 支持为 True + if numa_enable: + _config.set_numa_enable(True) + + # 设置 rank_id + _config.set_rank_id(rank_id) + + # 如果设备目标是 Ascend + elif device_target == "Ascend": + # Ascend 是一个特殊情况,最好从环境变量中获取排名信息 + env_rank_size = os.getenv("RANK_SIZE", None) + env_rank_id = os.getenv("RANK_ID", None) + rank_size = 0 + rank_id = 0 + + # 如果环境变量中包含 RANK_SIZE 和 RANK_ID,则解析它们并设置 rank_size 和 rank_id + if env_rank_size and env_rank_id: + try: + rank_size = int(env_rank_size.strip()) + rank_id = int(env_rank_id.strip()) + except ValueError: + raise ValueError("rank_size or rank_id is not int.") + + # 如果排名数量大于 1,且启用了 NUMA,则设置 NUMA 支持为 True + if rank_size > 1: + if numa_enable: + _config.set_numa_enable(True) + + # 设置 rank_id + _config.set_rank_id(rank_id) + + +def set_seed(seed): + """ + Set the seed so the random generated number will be fixed for deterministic results. + 设置种子,使随机生成的数字固定,以获得确定性结果。 + Note: + This set_seed function sets the seed in the Python random library and numpy.random library + for deterministic Python augmentations using randomness. This set_seed function should + be called when iterator is created to reset the random seed. + + Args: + seed(int): Random number seed. It is used to generate deterministic random numbers. + + Raises: + TypeError: If `seed` isn't of type int. + ValueError: If `seed` < 0 or `seed` > UINT32_MAX(4294967295). + + Examples: + >>> # Set a new global configuration value for the seed value. + >>> # Operations with randomness will use the seed value to generate random values. + >>> ds.config.set_seed(1000) + """ + + # 检查输入的种子是否为整数且不是布尔值 + if not isinstance(seed, int) or isinstance(seed, bool): + raise TypeError("seed isn't of type int.") + + # 检查种子是否在合法范围内 [0, UINT32_MAX(4294967295)] + if seed < 0 or seed > UINT32_MAX: + raise ValueError( + "seed given is not within the required range [0, UINT32_MAX(4294967295)].") + + # 使用 Config 设置随机数种子 + _config.set_seed(seed) + + # 使用 random 库设置 Python 的随机数种子 + random.seed(seed) + + # 注意:numpy.random 不是线程安全的,因此在多线程环境中需要设置随机数种子 + numpy.random.seed(seed) + + + +def get_seed(): + """ + Get random number seed. If the seed has been set, then will + return the set value, otherwise it will return the default seed value + which equals to std::mt19937::default_seed. + + Returns: + int, random number seed. + + Examples: + >>> # Get the global configuration of seed. + >>> # If set_seed() is never called before, the default value(std::mt19937::default_seed) will be returned. + >>> seed = ds.config.get_seed() + """ + return _config.get_seed() + + +def set_prefetch_size(size): + """ + Set the queue capacity of the thread in pipeline. + 设置管道中线程的队列容量。 + Args: + size (int): The length of the cache queue. + + Raises: + TypeError: If `size` is not of type int. + ValueError: If `size` <= 0 or `size` > INT32_MAX(2147483647). + + Note: + Since total memory used for prefetch can grow very large with high number of workers, + when the number of workers is greater than 4, the per worker prefetch size will be reduced. + The actual prefetch size at runtime per-worker will be prefetchsize * (4 / num_parallel_workers). + + Examples: + >>> # Set a new global configuration value for the prefetch size. + >>> ds.config.set_prefetch_size(1000) + """ + # 检查输入的大小是否为整数且不是布尔值 + if not isinstance(size, int) or isinstance(size, bool): + raise TypeError("size isn't of type int.") + + # 检查大小是否在合法范围内 (0, INT32_MAX(2147483647)] + if size <= 0 or size > INT32_MAX: + raise ValueError( + "size is not within the required range (0, INT32_MAX(2147483647)].") + + # 使用 Config 设置操作连接器的大小,即设置预取大小 + _config.set_op_connector_size(size) + + + +def get_prefetch_size(): + """ + Get the prefetch size as for number of rows. + If `set_prefetch_size` is never called before, the default value 16 will be returned. + 获取行数的预取大小。 + 如果以前从未调用过“set_prefetch_size”,则将返回默认值16。 + Returns: + int, total number of rows to be prefetched. + + Examples: + >>> # Get the global configuration of prefetch size. + >>> # If set_prefetch_size() is never called before, the default value(16) will be returned. + >>> prefetch_size = ds.config.get_prefetch_size() + """ + return _config.get_op_connector_size() + + +def set_num_parallel_workers(num): + """ + Set a new global configuration default value for the number of parallel workers. + This setting will affect the parallelism of all dataset operation. + 为并行工作线程的数量设置一个新的全局配置默认值。 + 此设置将影响所有数据集操作的并行性。 + Args: + num (int): Number of parallel workers to be used as a default for each operation. + + Raises: + TypeError: If `num` is not of type int. + ValueError: If `num` <= 0 or `num` > INT32_MAX(2147483647). + + Examples: + >>> # Set a new global configuration value for the number of parallel workers. + >>> # Now parallel dataset operators will run with 8 workers. + >>> ds.config.set_num_parallel_workers(8) + """ + + # 检查输入的值是否为整数类型,且不是布尔类型 + if not isinstance(num, int) or isinstance(num, bool): + raise TypeError("num isn't of type int.") + + # 检查输入的值是否在合法范围内 + if num <= 0 or num > INT32_MAX: + raise ValueError("Number of parallel workers given is not within the required range" + " (0, INT32_MAX(2147483647)].") + + # 设置数据集的并行工作线程数 + _config.set_num_parallel_workers(num) + + + +def get_num_parallel_workers(): + """ + Get the global configuration of number of parallel workers. + This is the DEFAULT num_parallel_workers value used for each operation. + 获取并行工作者数量的全局配置。 + 这是用于每个操作的DEFAULT num_paralle_workers值。 + Returns: + int, number of parallel workers to be used as a default for each operation. + + Examples: + >>> # Get the global configuration of parallel workers. + >>> # If set_num_parallel_workers() is never called before, the default value(8) will be returned. + >>> num_parallel_workers = ds.config.get_num_parallel_workers() + """ + return _config.get_num_parallel_workers() + + +def set_numa_enable(numa_enable): + """ + Set the default state of numa enabled. If numa_enable is True, need to ensure numa library is installed. + 设置numa enabled的默认状态。如果numa_enable为True,则需要确保安装了numa库。 + Args: + numa_enable (bool): Whether to use numa bind feature. + + Raises: + TypeError: If `numa_enable` is not a boolean data type. + + Examples: + >>> # Set a new global configuration value for the state of numa enabled. + >>> # Now parallel dataset operators will run with numa bind function + >>> ds.config.set_numa_enable(True) + """ + + if not isinstance(numa_enable, bool): + raise TypeError("numa_enable must be a boolean dtype.")#如果 `numa_enable` 不是布尔类型,将引发一个类型错误(TypeError)异常,其中包含错误消息 "numa_enable must be a boolean dtype." + #如果 `numa_enable` 是布尔类型,调用 `_config.set_numa_enable(numa_enable)` 来设置NUMA支持的状态。这里假设 `_config` 是一个外部配置对象,用于设置NUMA支持的状态。 + _config.set_numa_enable(numa_enable) + + +def get_numa_enable(): + """ + Get the state of numa to indicate enabled/disabled. + This is the DEFAULT numa enabled value used for the all process. + + Returns: + bool, the default state of numa enabled. + + Examples: + >>> # Get the global configuration of numa. + >>> numa_state = ds.config.get_numa_enable() + """ + return _config.get_numa_enable() + + +def set_monitor_sampling_interval(interval): + """ + Set the default interval (in milliseconds) for monitor sampling. + + Args: + interval (int): Interval (in milliseconds) to be used for performance monitor sampling. + + Raises: + TypeError: If `interval` is not type int. + ValueError: If `interval` <= 0 or `interval` > INT32_MAX(2147483647). + + Examples: + >>> # Set a new global configuration value for the monitor sampling interval. + >>> ds.config.set_monitor_sampling_interval(100) + """ + if not isinstance(interval, int) or isinstance(interval, bool): + raise TypeError("interval isn't of type int.") + if interval <= 0 or interval > INT32_MAX: + raise ValueError( + "Interval given is not within the required range (0, INT32_MAX(2147483647)].") + _config.set_monitor_sampling_interval(interval) + + +def get_monitor_sampling_interval(): + """ + Get the global configuration of sampling interval of performance monitor. + If `set_monitor_sampling_interval` is never called before, the default value(1000) will be returned. + + Returns: + int, interval (in milliseconds) for performance monitor sampling. + + Examples: + >>> # Get the global configuration of monitor sampling interval. + >>> # If set_monitor_sampling_interval() is never called before, the default value(1000) will be returned. + >>> sampling_interval = ds.config.get_monitor_sampling_interval() + """ + return _config.get_monitor_sampling_interval() + + +def set_auto_num_workers(enable): + """ + Set num_parallel_workers for each op automatically(This feature is turned off by default). + + If turned on, the num_parallel_workers in each op will be adjusted automatically, possibly overwriting the + num_parallel_workers passed in by user or the default value (if user doesn't pass anything) set by + ds.config.set_num_parallel_workers(). + + For now, this function is only optimized for YoloV3 dataset with per_batch_map (running map in batch). + This feature aims to provide a baseline for optimized num_workers assignment for each operation. + Operation whose num_parallel_workers is adjusted to a new value will be logged. + + Args: + enable (bool): Whether to enable auto num_workers feature or not. + + Raises: + TypeError: If `enable` is not of boolean type. + + Examples: + >>> # Enable auto_num_worker feature, this might override the num_parallel_workers passed in by user + >>> ds.config.set_auto_num_workers(True) + """ + if not isinstance(enable, bool): + raise TypeError("enable must be of type bool.") + _config.set_auto_num_workers(enable) + + +def _set_auto_workers_config(option): + """ + INTERNAL USE ONLY! + Select the weight profile of auto_num_workers. currently these 7 options are supported. + Option #0 leaf_num_workers:batch_num_workers:map_num_workers=1:1:1 + Option #1 leaf_num_workers:batch_num_workers:map_num_workers=2:1:1 + Option #2 leaf_num_workers:batch_num_workers:map_num_workers=1:2:1 + Option #3 leaf_num_workers:batch_num_workers:map_num_workers=1:1:2 + Option #4 leaf_num_workers:batch_num_workers:map_num_workers=2:2:1 + Option #5 leaf_num_workers:batch_num_workers:map_num_workers=2:1:2 + Option #6 leaf_num_workers:batch_num_workers:map_num_workers=1:2:2 + + Args: + option (int): The id of the profile to use. + + Raises: + TypeError: If `option` is not of type int. + ValueError: If `option` is not within the range of [0, 6]. + """ + if not isinstance(option, int) or isinstance(option, bool): + raise TypeError("option isn't of type int.") + if option < 0 or option > 6: + raise ValueError("option isn't within the required range of [0, 6].") + _config.set_auto_worker_config(option) + + +def get_auto_num_workers(): + """ + Get the setting (turned on or off) automatic number of workers. + + Returns: + bool, whether auto number worker feature is turned on. + + Examples: + >>> # Get the global configuration of auto number worker feature. + >>> flag = ds.config.get_auto_num_workers() + """ + return _config.get_auto_num_workers() + + +def set_callback_timeout(timeout): + """ + Set the default timeout (in seconds) for DSWaitedCallback. + + Args: + timeout (int): Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock. + + Raises: + TypeError: If `timeout` is not type int. + ValueError: If `timeout` <= 0 or `timeout` > INT32_MAX(2147483647). + + Examples: + >>> # Set a new global configuration value for the timeout value. + >>> ds.config.set_callback_timeout(100) + """ + if not isinstance(timeout, int) or isinstance(timeout, bool): + raise TypeError("timeout isn't of type int.") + if timeout <= 0 or timeout > INT32_MAX: + raise ValueError("Timeout given is not within the required range.") + _config.set_callback_timeout(timeout) + + +def get_callback_timeout(): + """ + Get the default timeout for WaitedDSCallback. + + Returns: + int, Timeout (in seconds) to be used to end the wait in DSWaitedCallback in case of a deadlock. + + Examples: + >>> # Get the global configuration of callback timeout. + >>> # If set_callback_timeout() is never called before, the default value(60) will be returned. + >>> callback_timeout = ds.config.get_callback_timeout() + """ + return _config.get_callback_timeout() + + +def __str__(): + """ + String representation of the configurations. + + Returns: + str, configurations. + """ + return str(_config) + + +def load(file): + """ + Load the project configuration from the file. + + Args: + file (str): Path of the configuration file to be loaded. + + Raises: + RuntimeError: If `file` is invalid and parsing fails. + + Examples: + >>> # Set new default configuration according to values in the configuration file. + >>> # example config file: + >>> # { + >>> # "logFilePath": "/tmp", + >>> # "numParallelWorkers": 4, + >>> # "seed": 5489, + >>> # "monitorSamplingInterval": 30 + >>> # } + >>> config_file = "/path/to/config/file" + >>> ds.config.load(config_file) + """ + _config.load(file) + + +def set_enable_autotune(enable, filepath_prefix=None): + """ + Set whether to enable AutoTune. AutoTune is disabled by default. + + AutoTune is used to automatically adjust the global configuration of the data pipeline + according to the workload of environmental resources during the training process to + improve the speed of data processing. + + The optimized global configuration can be saved as a JSON file by setting `json_filepath` + for subsequent reuse. + + Args: + enable (bool): Whether to enable AutoTune. + filepath_prefix (str, optional): The prefix filepath to save the optimized global configuration. + The rank id and the json extension will be appended to the filepath_prefix string in multi-device training, + rank id will be set to 0 in standalone training. + For example, if filepath_prefix="/path/to/some/dir/prefixname" and rank_id is 1, then the path + of the generated file will be "/path/to/some/dir/prefixname_1.json" + If the file already exists, it will be automatically overwritten. Default: None, + means not to save the configuration file, but the tuned result still can be checked through INFO log. + + Raises: + TypeError: If `enable` is not of type boolean. + TypeError: If `json_filepath` is not of type str. + RuntimeError: If `json_filepath` is an empty string. + RuntimeError: If `json_filepath` is a directory. + RuntimeError: If `json_filepath` does not exist. + RuntimeError: If `json_filepath` does not have write permission. + + Note: + - When `enable` is False, `json_filepath` will be ignored. + - The JSON file can be loaded by API `mindspore.dataset.deserialize` to build a tuned pipeline. + - In distributed training scenario, set_enable_autotune() must be called after cluster communication has been + initialized (mindspore.communication.management.init()), otherwise the AutoTune file will always suffix with + rank id 0. + + An example of the generated JSON file is as follows. "remark" file will conclude that if the dataset has been + tuned or not. "summary" filed will show the tuned configuration of dataset pipeline. Users can modify scripts + based on the tuned result. + + .. code-block:: + + { + "remark": "The following file has been auto-generated by the Dataset AutoTune.", + "summary": [ + "CifarOp(ID:5) (num_parallel_workers: 2, prefetch_size:64)", + "MapOp(ID:4) (num_parallel_workers: 2, prefetch_size:64)", + "MapOp(ID:3) (num_parallel_workers: 2, prefetch_size:64)", + "BatchOp(ID:2) (num_parallel_workers: 8, prefetch_size:64)" + ], + "tree": { + ... + } + } + + Examples: + >>> # enable AutoTune and save optimized data pipeline configuration + >>> ds.config.set_enable_autotune(True, "/path/to/autotune_out.json") + >>> + >>> # enable AutoTune + >>> ds.config.set_enable_autotune(True) + """ + if not isinstance(enable, bool): + raise TypeError("enable must be of type bool.") + + save_autoconfig = bool(enable and filepath_prefix is not None) + + if filepath_prefix and not isinstance(filepath_prefix, str): + raise TypeError( + "json_filepath must be a str value but was: {}.".format(filepath_prefix)) + + if enable and filepath_prefix == "": + raise RuntimeError( + "The value of json_filepath cannot be the empty string.") + + if not enable and filepath_prefix is not None: + logger.warning( + "The value of json_filepath is ignored when enable is False.") + + if enable and filepath_prefix is None: + logger.warning( + "Dataset AutoTune is enabled but no json path is specified, check INFO log for tuned result.") + + json_filepath = replace_none(filepath_prefix, "") + _config.set_enable_autotune(enable, save_autoconfig, json_filepath) + + +def get_enable_autotune(): + """ + Get whether AutoTune is currently enabled. + + Returns: + bool, whether AutoTune is currently enabled. + + Examples: + >>> # get the state of AutoTune + >>> autotune_flag = ds.config.get_enable_autotune() + """ + return _config.get_enable_autotune() + + +def set_autotune_interval(interval): + """ + Set the configuration adjustment interval (in steps) for AutoTune. + + The default setting is 0, which will adjust the configuration after each epoch. + Otherwise, the configuration will be adjusted every `interval` steps. + + Args: + interval (int): Interval (in steps) to adjust the configuration of the data pipeline. + + Raises: + TypeError: If `interval` is not of type int. + ValueError: If `interval` is not non-negative. + + Examples: + >>> # set a new interval for AutoTune + >>> ds.config.set_autotune_interval(30) + """ + if not isinstance(interval, int) or isinstance(interval, bool): + raise TypeError("interval must be of type int.") + if interval < 0 or interval > INT32_MAX: + raise ValueError( + "Interval given is not within the required range [0, INT32_MAX(2147483647)].") + _config.set_autotune_interval(interval) + + +def get_autotune_interval(): + """ + Get the current configuration adjustment interval (in steps) for AutoTune. + + Returns: + int, the configuration adjustment interval (in steps) for AutoTune. + + Examples: + >>> # get the global configuration of the autotuning interval + >>> autotune_interval = ds.config.get_autotune_interval() + """ + return _config.get_autotune_interval() + + +def get_enable_shared_mem(): + """ + Get the default state of shared mem enabled variable. + + Note: + `get_enable_shared_mem` is not supported on Windows and MacOS platforms yet. + + Returns: + bool, the state of shared mem enabled variable. + + Examples: + >>> # Get the flag of shared memory feature. + >>> shared_mem_flag = ds.config.get_enable_shared_mem() + """ + # For Windows and MacOS we forbid shared mem function temporarily + enable_shared_mem = _config.get_enable_shared_mem() + if enable_shared_mem and platform.system().lower() in {"windows", "darwin"}: + logger.warning( + "For Windows and MacOS we forbid shared mem function temporarily.") + _config.set_enable_shared_mem(False) + return False + return enable_shared_mem + + +def set_enable_shared_mem(enable): + """ + Set the default state of shared memory flag. If shared_mem_enable is True, will use shared memory queues + to pass data to processes that are created for operators that set python_multiprocessing=True. + + Note: + `set_enable_shared_mem` is not supported on Windows and MacOS platforms yet. + + Args: + enable (bool): Whether to use shared memory in operators when python_multiprocessing=True. + + Raises: + TypeError: If `enable` is not a boolean data type. + + Examples: + >>> # Enable shared memory feature to improve the performance of Python multiprocessing. + >>> ds.config.set_enable_shared_mem(True) + """ + if not isinstance(enable, bool): + raise TypeError("enable must be of type bool.") + if enable: + # For Windows and MacOS we forbid shared mem function temporarily + if platform.system().lower() in {"windows", "darwin"}: + logger.warning("For Windows and MacOS we forbid shared mem function temporarily.") + return + logger.warning("The shared memory is on, multiprocessing performance will be improved. " + "Note: the required shared memory can't exceeds 80% of the available shared memory.") + _config.set_enable_shared_mem(enable) + + +def set_sending_batches(batch_num): + """ + Set the default sending batches when training with sink_mode=True in Ascend device. + + Args: + batch_num (int): the total sending batches, when batch_num is set, it will wait unless sending batches + increase, default is 0 which means will send all batches in dataset. + + Raises: + TypeError: If `batch_num` is not of type int. + + Examples: + >>> # Set a new global configuration value for the sending batches + >>> ds.config.set_sending_batches(10) + """ + if not isinstance(batch_num, int) or isinstance(batch_num, bool): + raise TypeError("batch_num must be an int dtype.") + _config.set_sending_batches(batch_num) + + +def set_auto_offload(offload): + """ + Set the automatic offload flag of the dataset. If set_auto_offload is True, + automatically offload as many dataset operations from the CPU to the Device (GPU or Ascend). + + Args: + offload (bool): Whether to use the automatic offload feature. + + Raises: + TypeError: If offload is not a boolean data type. + + Examples: + >>> # Enable automatic offload feature + >>> ds.config.set_auto_offload(True) + """ + if not isinstance(offload, bool): + raise TypeError("offload must be a bool dtype") + _config.set_auto_offload(offload) + + +def get_auto_offload(): + """ + Get the state of the automatic offload flag (True or False) + + Returns: + bool, Whether the automatic offload feature is enabled. + + Example: + >>> # Get the global configuration of the automatic offload feature. + >>> auto_offload = ds.config.get_auto_offload() + """ + return _config.get_auto_offload() + + +def set_enable_watchdog(enable): + """ + Set the default state of watchdog Python thread as enabled, the default state of watchdog Python thread is enabled. + Watchdog is a thread which cleans up hanging subprocesses. + + Args: + enable (bool): Whether to launch a watchdog Python thread. System default: True. + + Raises: + TypeError: If `enable` is not a boolean data type. + + Examples: + >>> # Set a new global configuration value for the state of watchdog Python thread as enabled. + >>> ds.config.set_enable_watchdog(True) + """ + if not isinstance(enable, bool): + raise TypeError("enable must be a boolean dtype.") + _config.set_enable_watchdog(enable) + + +def get_enable_watchdog(): + """ + Get the state of watchdog Python thread to indicate enabled or disabled state. + This is the DEFAULT watchdog Python thread state value used for the all processes. + + Returns: + bool, the default state of watchdog Python thread enabled. + + Examples: + >>> # Get the global configuration of watchdog Python thread. + >>> watchdog_state = ds.config.get_enable_watchdog() + """ + return _config.get_enable_watchdog() + + +def set_multiprocessing_timeout_interval(interval): + """ + Set the default interval (in seconds) for multiprocessing/multithreading timeout when main process/thread gets + data from subprocesses/child threads. + + Args: + interval (int): Interval (in seconds) to be used for multiprocessing/multithreading timeout when main + process/thread gets data from subprocess/child threads. System default: 300s. + + Raises: + TypeError: If `interval` is not of type int. + ValueError: If `interval` <= 0 or `interval` > INT32_MAX(2147483647). + + Examples: + >>> # Set a new global configuration value for multiprocessing/multithreading timeout when getting data. + >>> ds.config.set_multiprocessing_timeout_interval(300) + """ + if not isinstance(interval, int) or isinstance(interval, bool): + raise TypeError("interval isn't of type int.") + if interval <= 0 or interval > INT32_MAX: + raise ValueError( + "Interval given is not within the required range (0, INT32_MAX(2147483647)).") + _config.set_multiprocessing_timeout_interval(interval) + + +def get_multiprocessing_timeout_interval(): + """ + Get the global configuration of multiprocessing/multithreading timeout when main process/thread gets data from + subprocesses/child threads. + + Returns: + int, interval (in seconds) for multiprocessing/multithreading timeout when main process/thread gets data from + subprocesses/child threads (default is 300s). + + Examples: + >>> # Get the global configuration of multiprocessing/multithreading timeout when main process/thread gets data + >>> # from subprocesses/child threads. If set_multiprocessing_timeout_interval() is never called before, the + >>> # default value(300) will be returned. + >>> multiprocessing_timeout_interval = ds.config.get_multiprocessing_timeout_interval() + """ + return _config.get_multiprocessing_timeout_interval() + + +def set_dynamic_shape(is_dynamic): + """ + Set the dynamic shape flag of the dataset. + + Args: + is_dynamic (bool): Whether the dataset is dynamic shape. Default: False + + Raises: + TypeError: If `is_dynamic` is not a boolean data type. + + Examples: + >>> ds.config.set_dynamic_shape(True) + """ + if not isinstance(is_dynamic, bool): + raise TypeError("is_dynamic must be a boolean dtype.") + _config.set_dynamic_shape(is_dynamic) + + +def get_dynamic_shape(): + """ + Get the dynamic shape flag of the dataset + Returns: + bool, whether the dataset is dynamic shape. + + Examples: + >>> is_dynamic_shape = ds.config.get_dynamic_shape() + """ + return _config.get_dynamic_shape() -- 2.34.1 From fef5b1745f55dc417a02fd0565745690ca26b4ce Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:22:15 +0800 Subject: [PATCH 67/72] ADD file via upload --- mindspore/ccsrc/transform-update/datatypes.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/datatypes.py diff --git a/mindspore/ccsrc/transform-update/datatypes.py b/mindspore/ccsrc/transform-update/datatypes.py new file mode 100644 index 00000000000..aae15d42f59 --- /dev/null +++ b/mindspore/ccsrc/transform-update/datatypes.py @@ -0,0 +1,116 @@ +# Copyright 2019-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Define the data types. +定义数据类型 +""" +import numpy as np + +import mindspore._c_dataengine as cde +from mindspore._c_expression import typing +import mindspore.common.dtype as mstype + + +def nptype_to_detype(type_): + """ + Get de data type corresponding to numpy dtype. + 获取numpy dtype对应的de数据类型。 + Args: + type_ (numpy.dtype): Numpy's dtype. + + Returns: + The data type of de. + """ + # 如果传入的 'type_' 不是 NumPy 数据类型对象(np.dtype),则将其转换为 np.dtype 对象 +if not isinstance(type_, np.dtype): + type_ = np.dtype(type_) + +# 创建一个字典,将 NumPy 数据类型映射到 CDE(MindSpore 数据增强库)的数据类型 +# 这个字典用于将 NumPy 数据类型转换为 CDE 数据类型 +return { + np.dtype("bool"): cde.DataType("bool"), + np.dtype("int8"): cde.DataType("int8"), + np.dtype("int16"): cde.DataType("int16"), + np.dtype("int32"): cde.DataType("int32"), + np.dtype("int64"): cde.DataType("int64"), + np.dtype("uint8"): cde.DataType("uint8"), + np.dtype("uint16"): cde.DataType("uint16"), + np.dtype("uint32"): cde.DataType("uint32"), + np.dtype("uint64"): cde.DataType("uint64"), + np.dtype("float16"): cde.DataType("float16"), + np.dtype("float32"): cde.DataType("float32"), + np.dtype("float64"): cde.DataType("float64"), + np.dtype("str"): cde.DataType("string"), +}.get(type_) + + + +def mstype_to_detype(type_): + """ + Get de data type corresponding to mindspore dtype. + 获取mindspore数据类型对应的de数据类型。 + Args: + type_ (mindspore.dtype): MindSpore's dtype. + + Returns: + The data type of de. + """ + # 如果传入的 'type_' 不是 NumPy 数据类型对象(np.dtype),则将其转换为 np.dtype 对象 +if not isinstance(type_, np.dtype): + type_ = np.dtype(type_) + +# 创建一个字典,将 NumPy 数据类型映射到 CDE(MindSpore 数据增强库)的数据类型 +# 这个字典用于将 NumPy 数据类型转换为 CDE 数据类型 +return { + np.dtype("bool"): cde.DataType("bool"), + np.dtype("int8"): cde.DataType("int8"), + np.dtype("int16"): cde.DataType("int16"), + np.dtype("int32"): cde.DataType("int32"), + np.dtype("int64"): cde.DataType("int64"), + np.dtype("uint8"): cde.DataType("uint8"), + np.dtype("uint16"): cde.DataType("uint16"), + np.dtype("uint32"): cde.DataType("uint32"), + np.dtype("uint64"): cde.DataType("uint64"), + np.dtype("float16"): cde.DataType("float16"), + np.dtype("float32"): cde.DataType("float32"), + np.dtype("float64"): cde.DataType("float64"), + np.dtype("str"): cde.DataType("string"), +}.get(type_) + + +def mstypelist_to_detypelist(type_list): + """ + Get list[de type] corresponding to list[mindspore.dtype]. + 获取列表[mindspore.dtype]对应的列表[detype]。 + Args: + type_list (list[mindspore.dtype]): a list of MindSpore's dtype. + + Returns: + The list of de data type. + """ + + # 遍历传入的 type_list 列表 + for index, _ in enumerate(type_list): + # 如果列表中的元素不为 None + if type_list[index] is not None: + # 调用 mstype_to_detype 函数将 MindSpore 数据类型转换为 CDE 数据类型 + type_list[index] = mstype_to_detype(type_list[index]) + else: + # 如果列表中的元素为 None,则将其设置为空字符串的 CDE 数据类型 + type_list[index] = cde.DataType("") + + # 返回转换后的 type_list 列表 + return type_list + -- 2.34.1 From fc4387a315c5c885b55a3afd10bd08d25396d495 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:22:50 +0800 Subject: [PATCH 68/72] ADD file via upload --- .../preprocess_imagenet_validate_dataset.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/preprocess_imagenet_validate_dataset.py diff --git a/mindspore/ccsrc/transform-update/preprocess_imagenet_validate_dataset.py b/mindspore/ccsrc/transform-update/preprocess_imagenet_validate_dataset.py new file mode 100644 index 00000000000..61308bdfa15 --- /dev/null +++ b/mindspore/ccsrc/transform-update/preprocess_imagenet_validate_dataset.py @@ -0,0 +1,74 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Process imagenet validate dataset. +处理imagenet验证数据集 +""" +import os +import stat +from mindspore import log as logger + + +def preprocess_imagenet_validation_dataset(train_dataset_path, validation_dataset_path, image_label_mapping_file): + """ + 在读取imagenet验证数据集之前调用此函数,用于预处理数据集。 + + Args: + train_dataset_path (str): 训练数据集路径 + validation_dataset_path (str): 验证数据集路径 + image_label_mapping_file (str): imagenet_validate_dataset_2012_image_dir_map.txt 文件路径 + """ + # 获取训练数据集的绝对路径 + train_dataset_path = os.path.realpath(train_dataset_path) + + # 获取训练数据集中的子目录列表 + sub_dir = [dir_.name for dir_ in os.scandir(train_dataset_path) if dir_.is_dir()] + + # 遍历子目录并在验证数据集路径下创建对应的子目录 + for sub_dir_name in sub_dir: + validate_sub_dir = os.path.join(validation_dataset_path, sub_dir_name) + validate_sub_dir = os.path.realpath(validate_sub_dir) + + # 如果验证数据集子目录不存在,则创建之 + if not os.path.exists(validate_sub_dir): + os.makedirs(validate_sub_dir, mode=stat.S_IRWXU) + + # 获取映射文件的绝对路径 + real_file_path = os.path.realpath(image_label_mapping_file) + + # 读取映射文件中的映射关系 + mappings = [mapping.strip() for mapping in open(real_file_path).readlines()] + + # 遍历映射关系,将图像从训练目录移动到验证目录中的对应子目录 + for mapping in mappings: + image_dir = mapping.split(':') + old_image_path = os.path.join(validation_dataset_path, image_dir[0]) + old_image_path = os.path.realpath(old_image_path) + + # 如果原始图像路径不存在,则发出警告 + if not os.path.exists(old_image_path): + logger.warning('Image is not existed %s', old_image_path) + + new_image_sub_dir = os.path.join(validation_dataset_path, image_dir[1]) + new_image_sub_dir = os.path.realpath(new_image_sub_dir) + new_image_path = os.path.join(new_image_sub_dir, image_dir[0]) + new_image_path = os.path.realpath(new_image_path) + + # 如果新图像的子目录不存在,则发出警告 + if not os.path.exists(new_image_sub_dir): + logger.warning('Image sub dir is not existed %s', new_image_sub_dir) + + # 将图像从旧路径移到新路径 + os.rename(old_image_path, new_image_path) + -- 2.34.1 From 8d00177b9e4855200549d976b7b750f754af3995 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:23:27 +0800 Subject: [PATCH 69/72] ADD file via upload --- mindspore/ccsrc/transform-update/datasets.py | 4058 ++++++++++++++++++ 1 file changed, 4058 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/datasets.py diff --git a/mindspore/ccsrc/transform-update/datasets.py b/mindspore/ccsrc/transform-update/datasets.py new file mode 100644 index 00000000000..20649425c33 --- /dev/null +++ b/mindspore/ccsrc/transform-update/datasets.py @@ -0,0 +1,4058 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +1. This file is an abstraction of the dataset loading class. It contains +some basic dataset operations(skip, filter, map, batch, ...). +2. Specific dataset loading classes can be found in datasets_vision.py, datasets_text.py, +datasets_audio.py, datasets_standard_format.py and dataets_user_defined.py files. + datasets_vision.py: contains vision dataset loading classes. + datasets_text.py: contains text dataset loading classes. + datasets_audio.py: contains audio dataset loading classes. + datasets_standard_format.py: contains standard format loading classes which + any other kinds of datasets can be converted to. + dataets_user_defined.py: contains basic classes that help users to define + flexible ways to load dataset. +1.该文件是数据集加载类的抽象。它包含一些基本的数据集操作(跳过、筛选、映射、批处理…)。 +2.具体的数据集加载类可以在datasets_,datasets_audio.py、datasets_standard_format.py和dataets_user_defined.py文件。 +datasetsvision.py:包含视觉数据集加载类。 +datasets_text.py:包含文本数据集加载类。 +datasets_audio.py:包含音频数据集加载类。 +datasets_standard_format.py:包含标准格式加载类可以将任何其他类型的数据集转换为。 +dataets_user_defined.py:包含帮助用户定义的基本类加载数据集的灵活方式。 +""" +import atexit +import glob +import json +import os +import signal +import stat + +import gc +import time +import uuid +import multiprocessing +from enum import Enum +from importlib import import_module +import sys +import threading + +import copy +import weakref +import platform +import psutil +import numpy as np + +import mindspore._c_dataengine as cde +from mindspore._c_expression import typing + +from mindspore import log as logger +from mindspore.parallel._ps_context import _is_role_pserver, _is_role_sched, _get_ps_context, _enable_distributed_mindrt +from mindspore.dataset.engine.offload import GetOffloadModel + +import mindspore.dataset.transforms.c_transforms as c_transforms +import mindspore.dataset.transforms.py_transforms as py_transforms +import mindspore.dataset.transforms as transforms +from mindspore.dataset.text.utils import SentencePieceModel, DE_C_INTER_SENTENCEPIECE_MODE +from mindspore.parallel._utils import _get_device_num + +from . import samplers +from .iterators import DictIterator, TupleIterator, DummyIterator, check_iterator_cleanup, _set_iterator_cleanup, \ + ITERATORS_LIST, _unset_iterator_cleanup +from .queue import _SharedQueue, _Queue +from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \ + check_rename, check_device_send, check_take, check_output_shape, check_project, \ + check_sync_wait, check_zip_dataset, check_add_column, check_concat, check_split, check_bucket_batch_by_length, \ + check_save, check_tuple_iterator, check_dict_iterator, check_schema, check_to_device_send, deprecated +from ..core.config import get_callback_timeout, _init_device_info, get_enable_shared_mem, get_num_parallel_workers, \ + get_enable_watchdog +from ..core.datatypes import mstype_to_detype +from ..core.validator_helpers import replace_none +from ..core.py_util_helpers import ExceptionHandler +from ..transforms.py_transforms_util import FuncWrapper, Implementation +from ..vision.transforms import ToNumpy + +try: + context = import_module("mindspore.context") +except ModuleNotFoundError: + context = None + +if platform.system().lower() == "darwin" and multiprocessing.get_start_method() != "fork": + multiprocessing.set_start_method("fork", True) + +OffloadToManualOffloadMode = { + None: cde.ManualOffloadMode.UNSPECIFIED, + False: cde.ManualOffloadMode.DISABLED, + True: cde.ManualOffloadMode.ENABLED +} + +_train_dataset = None + + +def _set_training_dataset(dataset): + """ + Set the dataset to be used when training recovery has occurred. + + Args: + dataset: the training dataset or iterator + """ + global _train_dataset + _train_dataset = dataset + + +def _get_training_dataset(): + """ + Get the dataset to be used when training recovery has occurred. + + Returns: + training dataset/iterator + """ + return _train_dataset + + +def _reset_training_dataset(step): + """ + Reset the training dataset to the given step number. + + Args: + step (int): Global step number. + """ + dataset = _get_training_dataset() + if dataset is not None: + dataset._reset(step) # pylint: disable=W0212 + else: + raise RuntimeError("Training dataset is not set.") + + +class Shuffle(str, Enum): + """Specify the shuffle mode. + + - Shuffle.GLOBAL: Shuffle both the files and samples. + - Shuffle.FILES: Shuffle files only. + - Shuffle.INFILE: Shuffle data within each file. + """ + GLOBAL: str = "global" + FILES: str = "files" + INFILE: str = "infile" + + +ShuffleToShuffleMode = {Shuffle.FILES: cde.ShuffleMode.FILES, + Shuffle.GLOBAL: cde.ShuffleMode.GLOBAL, + Shuffle.INFILE: cde.ShuffleMode.INFILE} + +# 定义一个方法,将数据集的随机打乱参数转换为对应的 ShuffleMode 枚举值 +def shuffle_to_shuffle_mode(shuffle): + """ + Shuffle Enum to Shuffle Mode + 无序排列枚举到无序排列模式 + Args: + shuffle (Shuffle): shuffle flag to shuffle mode in C layer + + Returns: + ShuffleMode, shuffle mode + """ + + # 默认的 shuffle_mode 为全局随机打乱 + shuffle_mode = cde.ShuffleMode.GLOBAL # Global shuffle + + # 检查输入的 shuffle 参数是否为 Shuffle 类型或 None + if not isinstance(shuffle, Shuffle): + # 如果 shuffle 为 None 或 True,设置 shuffle_mode 为全局随机打乱 + if shuffle is None or shuffle: + shuffle_mode = cde.ShuffleMode.GLOBAL # Global shuffle + else: + # 如果 shuffle 为 False,设置 shuffle_mode 为不进行随机打乱 + shuffle_mode = cde.ShuffleMode.FALSE # No shuffle + else: + # 如果 shuffle 参数为 Shuffle 枚举值,则将其转换为对应的 ShuffleMode 枚举值 + shuffle_mode = ShuffleToShuffleMode[shuffle] + + # 返回对应的 shuffle_mode 枚举值,表示随机打乱模式 + return shuffle_mode + + +def shuffle_to_bool(shuffle): + """ + Shuffle Enum to bool + + Args: + shuffle (Shuffle): shuffle flag to bool + + Returns: + bool, True / False + """ + if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)): + raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or " + "'Shuffle.FILES' or 'Shuffle.INFILE'.") + + shuffle_bool = True + if not isinstance(shuffle, Shuffle): + if shuffle is None: + shuffle_bool = None + elif shuffle: + shuffle_bool = True + else: + shuffle_bool = False + else: + shuffle_bool = True + return shuffle_bool + + +@check_zip + # 定义一个方法用于将多个数据集进行 zip 操作,将它们合并为一个 ZipDataset 对象 +def zip(datasets): + """ + Zip the datasets in the input tuple of datasets. + 压缩数据集的输入元组中的数据集。 + Args: + datasets (tuple[Dataset]): A tuple of datasets to be zipped together. + The number of datasets must be more than 1. + + Returns: + Dataset, dataset zipped. + + Raises: + ValueError: If the number of datasets is 1. + TypeError: If datasets is not a tuple. + + Examples: + >>> # Create a dataset which is the combination of dataset_1 and dataset_2 + >>> dataset = ds.zip((dataset_1, dataset_2)) + """ + + # 检查输入的数据集列表长度是否小于等于1,如果是则抛出值错误 + if len(datasets) <= 1: + raise ValueError( + "Can't zip empty or just one dataset!") + + # 遍历数据集列表,检查每个元素是否是 Dataset 对象,如果不是则抛出类型错误 + for dataset in datasets: + if not isinstance(dataset, Dataset): + raise TypeError("Invalid dataset, expected Dataset object, but got %s!" % type(dataset)) + + # 返回一个 ZipDataset 对象,其中包含了合并的数据集 + return ZipDataset(datasets) + + +# 定义一个方法用于获取运算符的处理信息 +def _get_operator_process(): + """ + Inner implemented method, mainly for passing sub-process id in C layer + 内部实现的方法,主要用于在C层传递子进程id + Returns: + dict, mapping dict of operator id and corresponding process id. + """ + + # 获取全局变量 _OP_PROCESS 中的运算符处理信息 + global _OP_PROCESS + process_info = _OP_PROCESS + + # 初始化一个空的运算符处理信息字典 + op_process = dict() + + # 获取 _OP_PROCESS 字典中的所有键 + keys = process_info.keys() + + # 初始化一个标志变量,表示是否成功获取了所有运算符的处理信息 + fetched_all = True + + # 遍历所有键,尝试获取每个运算符的处理信息 + for key in keys: + try: + # 从 _OP_PROCESS 中获取每个运算符的处理信息,并存储在 op_process 字典中 + op_process[key] = list(process_info[key][1]) + + # 检查是否成功获取了所有运算符的处理信息 + item_full = (len(process_info[key][1]) == process_info[key][0]) + + except KeyError as err: + # 如果发生 KeyError 异常,将其抛出 + raise err + + # 更新标志变量,表示是否成功获取了所有运算符的处理信息 + fetched_all = fetched_all and item_full + + # 返回获取到的运算符处理信息字典和是否成功获取了所有运算符的标志变量 + return op_process, fetched_all + + +# 定义一个方法用于设置数据集文件的权限 +def _set_dataset_permissions(file_name, num_files): + """ + set saved dataset files' permissions to 600 + the rule of dataset filenames should be the same as those in C++. + 将保存的数据集文件的权限设置为600 + 数据集文件名的规则应该与C++中的规则相同。 + """ + + + # 计算文件名中数字的位数,以便对文件名进行格式化 + num_digits = len(str(num_files - 1)) + + # 如果只有一个文件,将文件名添加到路径列表中 + if num_files == 1: + paths = [file_name] + else: + # 否则,根据文件数量生成一系列文件路径 + # 格式化文件名,例如:file_name0001,file_name0002,... + paths = ["{}{}".format(file_name, str(x).rjust(num_digits, '0')) for x in range(num_files)] + + # 遍历文件路径列表 + for item in paths: + # 如果文件存在,设置文件的用户读取和写入权限 + if os.path.exists(item): + os.chmod(item, stat.S_IRUSR | stat.S_IWUSR) + # 拼接索引文件的路径 + index_file = item + ".db" + # 如果索引文件存在,也设置其用户读取和写入权限 + if os.path.exists(index_file): + os.chmod(index_file, stat.S_IRUSR | stat.S_IWUSR) + + + +class Dataset: + """ + Abstract class to represent a dataset in DataEngine's data pipeline. + 抽象类来表示DataEngine的数据管道中的数据集。 + This class is the base class of SourceDataset and Dataset, and represents + a node in the data flow graph. + Dataset + ----------------------------------------------------------- + | | | | + VisionBaseDataset TextBaseDataset AudioBaseDataset | + - - - | + | | | | + ---------------------------------------- | + UnionBaseDataset | + | + SourceDataset + - + | + MappableDataset + + DatasetOperator: MapDataset(UnionBaseDataset) + BatchDataset(UnionBaseDataset) + BucketBatchByLengthDataset(UnionBaseDataset) + ShuffleDataset(UnionBaseDataset) + FilterDataset(UnionBaseDataset) + RepeatDataset(UnionBaseDataset) + SkipDataset(UnionBaseDataset) + TakeDataset(UnionBaseDataset) + ZipDataset(UnionBaseDataset) + ConcatDataset(UnionBaseDataset) + RenameDataset(UnionBaseDataset) + ProjectDataset(UnionBaseDataset) + SyncWaitDataset(UnionBaseDataset) + + Impl Dataset - vision: ImageFolderDataset(MappableDataset, VisionBaseDataset) + USPSDataset(SourceDataset, VisionBaseDataset) + Impl Dataset - text: TextFileDataset(SourceDataset, TextBaseDataset) + YahooAnswersDataset(SourceDataset, TextBaseDataset) + Impl Dataset - audio: LJSpeechDataset(MappableDataset, AudioBaseDataset) + TedliumDataset(MappableDataset, AudioBaseDataset) + Impl Dataset - standard: MindDataset(MappableDataset, UnionBaseDataset) + TFRecordDataset(SourceDataset, UnionBaseDataset) + Impl Dataset - user defined: GeneratorDataset(MappableDataset, UnionBaseDataset) + NumpySlicesDataset(GeneratorDataset) + + Args: + num_parallel_workers (int, optional): Number of workers to process the dataset in parallel + (default=None). + """ + + def __init__(self, children=None, num_parallel_workers=None, cache=None): + # Note: children and parent are internal variables, not recommended for external using. + self.children = replace_none(children, []) + if isinstance(self.children, tuple): + self.children = list(self.children) + if not isinstance(self.children, list): + self.children = [self.children] + + self.parent = [] + for child in self.children: + child.parent.append(weakref.ref(self)) + self.num_parallel_workers = num_parallel_workers + self.cache = cache + + self._device_iter = 0 + self._input_indexs = () + self.saved_output_types = None + self.saved_output_shapes = None + self.estimated_output_shapes = None + self.runtime_context = None + self.dynamic_setting = [False, None] + self.saved_min_shapes = None + self.saved_max_shapes = None + self._col_names = None + self.dataset_size = None + self._batch_size = None + self._num_classes = None + self._repeat_count = None + self._class_indexing = None + self._sync = False + + @staticmethod + def _get_operator_id(dataset): + """ + Internal method to iterate the tree and obtain op_id of each operator. + + Returns: + Dataset, the root dataset of the tree. + """ + op_name = dict() + generator_process = dict() + op_name[str(dataset)] = 0 + op_id = 1 + + def process_name(datasets, operator_id): + if not datasets: + return 0 + temp = [] + for item in datasets: + for d in item.children: + temp.append(d) + op_name[str(d)] = operator_id + + from mindspore.dataset.engine.datasets_user_defined import GeneratorDataset + if isinstance(d, GeneratorDataset) and d.sample_fn and d.sample_fn.pids: + generator_process[operator_id] = [d.num_parallel_workers, set(d.sample_fn.pids)] + + operator_id = operator_id + 1 + return process_name(temp, operator_id) + + process_name([dataset], op_id) + if generator_process: + global _OP_PROCESS + _OP_PROCESS.update(generator_process) + return op_name + + def close_pool(self): + """ + Close multiprocessing pool in dataset. If you are familiar with multiprocessing library, you can regard this + as a destructor for a processingPool object. + + Note: + This interface will be deleted or invisible in the future. Please don't use it. + When you find that there are residual processes that do not exit correctly, you can use `kill -9 PROCESS_ID` + to end it, or through www.gitee.com/mindspore/mindspore send us an issue. + """ + logger.warning("This interface will be deleted or invisible in the future. Please don't use it.") + + def create_ir_tree(self): + """ + Internal method to build an IR tree. + 构建IR树的内部方法。 + Returns: + DatasetNode, the root node of the IR tree. + Dataset, the root dataset of the IR tree. + """ +# 这个方法会执行以下操作: +# 1. 保存当前对象的父对象到 parent 变量中 +# 2. 清空当前对象的 parent 属性 +# 3. 创建当前对象的深层副本,以便后续的操作不会影响原始数据集 +# 4. 设置全局变量 _OP_NAME 为当前数据集的运算符标识 +# 5. 调用数据集的 parse_tree() 方法来构建 IR 树 +# 6. 恢复原始的 parent 属性 +# 7. 调用 _init_device_info() 方法来初始化设备信息 +# 8. 返回构建的 IR 树和深层副本的数据集 + + # 保存当前对象的父对象到 parent 变量中 + parent = self.parent + + # 清空当前对象的 parent 属性 + self.parent = [] + + # 创建当前对象的深层副本,以便后续的操作不会影响原始数据集 + dataset = copy.deepcopy(self) + + # 设置全局变量 _OP_NAME 为当前数据集的运算符标识 + global _OP_NAME + _OP_NAME = Dataset._get_operator_id(dataset) + + # 调用数据集的 parse_tree() 方法来构建 IR 树 + ir_tree = dataset.parse_tree() + + # 恢复原始的 parent 属性 + self.parent = parent + + # 调用 _init_device_info() 方法来初始化设备信息 + _init_device_info() + + # 返回构建的 IR 树和深层副本的数据集 + return ir_tree, dataset + + + def parse_tree(self): + """ + Internal method to parse the API tree into an IR tree. + + Returns: + DatasetNode, the root node of the IR tree. + """ + if len(self.parent) > 1: + raise ValueError("The data pipeline is not a tree (i.e., one node has 2 consumers)") + ir_children = [d.parse_tree() for d in self.children] + # Bootstrap can only be performed on a copy of the original dataset node. + # Bootstrap on original dataset node will make all iterators share the same process pool + self.iterator_bootstrap() + ir_node = self.parse(ir_children) + ir_node = self.post_parse(ir_node) + return ir_node + + def __safe_deepcopy__(self, memodict, exclude=()): + if id(self) in memodict: + return memodict[id(self)] + cls = self.__class__ + new_op = cls.__new__(cls) + memodict[id(self)] = new_op + for arg, value in self.__dict__.items(): + if arg in exclude: + setattr(new_op, arg, value) + else: + try: + setattr(new_op, arg, copy.deepcopy(value, memodict)) + except TypeError: + setattr(new_op, arg, value) + return new_op + + @staticmethod + def _noop_mode(): + if _is_role_sched() or (_is_role_pserver() and not _enable_distributed_mindrt()): + return True + return False + + def iterator_bootstrap(self): + pass + + def __add__(self, datasets): + return self.concat(datasets) + + def to_json(self, filename=""): + """ + Serialize a pipeline into JSON string and dump into file if filename is provided. + + Args: + filename (str): filename of JSON file to be saved as (default=""). + + Returns: + str, JSON string of the pipeline. + """ + ir_tree, _ = self.create_ir_tree() + return json.loads(ir_tree.to_json(filename)) + + @check_bucket_batch_by_length + def bucket_batch_by_length(self, column_names, bucket_boundaries, bucket_batch_sizes, element_length_function=None, + pad_info=None, pad_to_bucket_boundary=False, drop_remainder=False): + """ + Bucket elements according to their lengths. Each bucket will be padded and batched when + they are full. + + A length function is called on each row in the dataset. The row is then + bucketed based on its length and bucket boundaries. When a bucket reaches its + corresponding size specified in bucket_batch_sizes, the entire bucket will be + padded according to pad_info, and then form a batch. + Each batch will be full, except one special case: the last batch for each bucket may not be full. + + Args: + column_names (list[str]): Columns passed to element_length_function. + bucket_boundaries (list[int]): A list consisting of the upper boundaries + of the buckets. Must be strictly increasing. If there are n boundaries, + n+1 buckets are created: One bucket for [0, bucket_boundaries[0]), one + bucket for [bucket_boundaries[i], bucket_boundaries[i+1]) for each + 0>> # Create a dataset where certain counts rows are combined into a batch + >>> # and drops the last incomplete batch if there is one. + >>> import numpy as np + >>> def generate_2_columns(n): + ... for i in range(n): + ... yield (np.array([i]), np.array([j for j in range(i + 1)])) + >>> + >>> column_names = ["col1", "col2"] + >>> dataset = ds.GeneratorDataset(generate_2_columns(8), column_names) + >>> bucket_boundaries = [5, 10] + >>> bucket_batch_sizes = [2, 1, 1] + >>> element_length_function = (lambda col1, col2: max(len(col1), len(col2))) + >>> # Will pad col2 to shape [bucket_boundaries[i]] where i is the + >>> # index of the bucket that is currently being batched. + >>> pad_info = {"col2": ([None], -1)} + >>> pad_to_bucket_boundary = True + >>> dataset = dataset.bucket_batch_by_length(column_names, bucket_boundaries, + ... bucket_batch_sizes, + ... element_length_function, pad_info, + ... pad_to_bucket_boundary) + """ + return BucketBatchByLengthDataset(self, column_names, bucket_boundaries, bucket_batch_sizes, + element_length_function, pad_info, pad_to_bucket_boundary, drop_remainder) + + @check_batch + def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None, + input_columns=None, output_columns=None, column_order=None, pad_info=None, + python_multiprocessing=False, max_rowsize=16): + """ + Combine batch_size number of consecutive rows into batches. + + For any child node, a batch is treated as a single row. + For any column, all the elements within that column must have the same shape. + If a per_batch_map callable is provided, it will be applied to the batches of tensors. + + Note: + The order of using repeat and batch reflects the number of batches and per_batch_map. + It is recommended that the repeat operation applied after the batch operation finished. + + Args: + batch_size (int or function): The number of rows each batch is created with. An + int or callable object which takes exactly 1 parameter, BatchInfo. + drop_remainder (bool, optional): Determines whether or not to drop the last block + whose data row number is less than batch size (default=False). If True, and if there are less + than batch_size rows available to make the last batch, then those rows will + be dropped and not propagated to the child node. + num_parallel_workers (int, optional): Number of workers(threads) to process the dataset in parallel + (default=None). + per_batch_map (Callable[[List[numpy.ndarray], ..., List[numpy.ndarray], BatchInfo], (List[numpy.ndarray],\ + ..., List[numpy.ndarray])], optional): Per batch map callable (default=None). A callable + which takes (list[numpy.ndarray], list[numpy.ndarray], ..., BatchInfo) as input parameters. Each + list[numpy.ndarray] represents a batch of numpy.ndarray on a given column. The number of lists should + match with the number of entries in input_columns. The last parameter of the callable should always be + a BatchInfo object. Per_batch_map should return (list[numpy.ndarray], list[numpy.ndarray], ...). The + length of each list in output should be the same as the input. output_columns is required if the number + of output lists is different from input. + input_columns (Union[str, list[str]], optional): List of names of the input columns. The size of the list + should match with signature of per_batch_map callable (default=None). + output_columns (Union[str, list[str]], optional): List of names assigned to the columns + outputted by the last operation. This parameter is mandatory if len(input_columns) != + len(output_columns). The size of this list must match the number of output + columns of the last operation. (default=None, output columns will have the same + name as the input columns, i.e., the columns will be replaced). + column_order (Union[str, list[str]], optional): Specifies the list of all the columns you need in the whole + dataset (default=None). The parameter is required when len(input_column) != len(output_column). + Caution: the list here is not just the columns specified in parameter input_columns and output_columns. + pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)} + would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0 + (default=None). + python_multiprocessing (bool, optional): Parallelize Python function per_batch_map with multi-processing. + This option could be beneficial if the function is computational heavy (default=False). + max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy + data between processes. This is only used if python_multiprocessing is set to True (default=16). + + Returns: + BatchDataset, dataset batched. + + Examples: + >>> # 1) Create a dataset where every 100 rows are combined into a batch + >>> # and drops the last incomplete batch if there is one. + >>> dataset = dataset.batch(100, True) + >>> + >>> # 2)resize image according to its batch number, if it's 5-th batch, resize to (5^2, 5^2) = (25, 25) + >>> def np_resize(col, BatchInfo): + ... output = col.copy() + ... s = (BatchInfo.get_batch_num() + 1) ** 2 + ... index = 0 + ... for c in col: + ... img = Image.fromarray(c.astype('uint8')).convert('RGB') + ... img = img.resize((s, s)) + ... output[index] = np.array(img) + ... index += 1 + ... return (output,) + >>> dataset = dataset.batch(batch_size=8, input_columns=["image"], per_batch_map=np_resize) + >>> + >>> # 3)Create a dataset where its batch size is dynamic + >>> # Define a callable batch size function and let batch size increase 1 each time. + >>> def add_one(BatchInfo): + ... return BatchInfo.get_batch_num() + 1 + >>> dataset = dataset.batch(batch_size=add_one, drop_remainder=True) + >>> + >>> # 4)Create a dataset with batch, then specify the column order. + >>> # Assume that the original coulmn order is ["image", "label"] and change to ["label", "image"]. + >>> dataset = dataset.batch(32, column_order=["label", "image"]) + """ + return BatchDataset(self, batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns, + output_columns, column_order, pad_info, python_multiprocessing, max_rowsize) + + @check_sync_wait + def sync_wait(self, condition_name, num_batch=1, callback=None): + """ + Add a blocking condition to the input Dataset. A synchronize action will be applied. + + Args: + condition_name (str): The condition name that is used to toggle sending next row. + num_batch (int): the number of batches without blocking at the start of each epoch (default=1). + callback (function): The callback function that will be invoked when sync_update is called (default=None). + + Returns: + SyncWaitDataset, dataset added a blocking condition. + + Raises: + RuntimeError: If condition name already exists. + + Examples: + >>> import numpy as np + >>> def gen(): + ... for i in range(100): + ... yield (np.array(i),) + >>> + >>> class Augment: + ... def __init__(self, loss): + ... self.loss = loss + ... + ... def preprocess(self, input_): + ... return input_ + ... + ... def update(self, data): + ... self.loss = data["loss"] + >>> + >>> batch_size = 4 + >>> dataset = ds.GeneratorDataset(gen, column_names=["input"]) + >>> + >>> aug = Augment(0) + >>> dataset = dataset.sync_wait(condition_name="policy", callback=aug.update) + >>> dataset = dataset.map(operations=[aug.preprocess], input_columns=["input"]) + >>> dataset = dataset.batch(batch_size) + >>> count = 0 + >>> for data in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + ... assert data["input"][0] == count + ... count += batch_size + ... data = {"loss": count} + ... dataset.sync_update(condition_name="policy", data=data) + """ + return SyncWaitDataset(self, condition_name, num_batch, callback) + + @check_shuffle + def shuffle(self, buffer_size): + """ + Randomly shuffles the rows of this dataset using the following policy: + + 1. Make a shuffle buffer that contains the first buffer_size rows. + 2. Randomly select an element from the shuffle buffer to be the next row + propagated to the child node. + 3. Get the next row (if any) from the parent node and put it in the shuffle buffer. + 4. Repeat steps 2 and 3 until there are no more rows left in the shuffle buffer. + + A random seed can be provided to be used on the first epoch. In every subsequent + epoch, the seed is changed to a new one, randomly generated value. + + Args: + buffer_size (int): The size of the buffer (must be larger than 1) for + shuffling. Setting buffer_size equal to the number of rows in the entire + dataset will result in a global shuffle. + + Returns: + Dataset, dataset shuffled. + + Raises: + RuntimeError: If exist sync operators before shuffle. + + Examples: + >>> # dataset is an instance object of Dataset + >>> # Optionally set the seed for the first epoch + >>> ds.config.set_seed(58) + >>> # Create a shuffled dataset using a shuffle buffer of size 4 + >>> dataset = dataset.shuffle(4) + """ + return ShuffleDataset(self, buffer_size) + + def flat_map(self, func): + """ + Map `func` to each row in dataset and flatten the result. + + The specified `func` is a function that must take one `numpy.ndarray` as input + and return a `Dataset`. + 将“func”映射到数据集中的每一行并展平结果。 + 指定的“func”是一个必须接受一个“numpy.ndarray”作为输入的函数并返回一个“数据集”。 + Args: + func (function): A function that must take one `numpy.ndarray` as an argument and + return a `Dataset`. + + Returns: + Dataset, dataset applied by the function. + + Examples: + >>> # 1) flat_map on one column dataset + >>> dataset = ds.NumpySlicesDataset([[0, 1], [2, 3]], shuffle=False) + >>> + >>> def repeat(array): + ... # create a NumpySlicesDataset with the array + ... data = ds.NumpySlicesDataset(array, shuffle=False) + ... # repeat the dataset twice + ... data = data.repeat(2) + ... return data + >>> + >>> dataset = dataset.flat_map(repeat) + >>> # [0, 1, 0, 1, 2, 3, 2, 3] + >>> + >>> # 2) flat_map on multi column dataset + >>> dataset = ds.NumpySlicesDataset(([[0, 1], [2, 3]], [[0, -1], [-2, -3]]), shuffle=False) + + >>> def plus_and_minus(col1, col2): + ... # apply different methods on columns + ... data = ds.NumpySlicesDataset((col1 + 1, col2 - 1), shuffle=False) + ... return data + + >>> dataset = dataset.flat_map(plus_and_minus) + >>> # ([1, 2, 3, 4], [-1, -2, -3, -4]) + + Raises: + TypeError: If `func` is not a function. + TypeError: If `func` doesn't return a Dataset. + """ + # 初始化数据集为 None + dataset = None + + # 检查 func 是否是一个可调用函数,否则抛出类型错误 + if not hasattr(func, '__call__'): + logger.critical("func must be a function.") + raise TypeError("func must be a function.") + + # 遍历数据集中的每个元素,使用 func 处理每个元素,并将结果拼接到 dataset 中 + for row_data in self.create_tuple_iterator(num_epochs=1, output_numpy=True): + if dataset is None: + dataset = func(*row_data) + else: + dataset += func(*row_data) + + # 检查 dataset 是否是 Dataset 类型的对象,否则抛出类型错误 + if not isinstance(dataset, Dataset): + logger.critical("flat_map must return a Dataset object.") + raise TypeError("flat_map must return a Dataset object.") + + # 返回拼接后的新数据集 + return dataset + + + @check_map + def map(self, operations, input_columns=None, output_columns=None, column_order=None, + num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, + max_rowsize=16, offload=None): + """ + Apply each operation in operations to this dataset. + + The order of operations is determined by the position of each operation in the operations parameter. + operations[0] will be applied first, then operations[1], then operations[2], etc. + + Each operation will be passed one or more columns from the dataset as input, and zero or + more columns will be outputted. The first operation will be passed the columns specified + in input_columns as input. If there is more than one operator in operations, the outputted + columns of the previous operation are used as the input columns for the next operation. + The columns outputted by the very last operation will be assigned names specified by + output_columns. + + Only the columns specified in column_order will be propagated to the child node. These + columns will be in the same order as specified in column_order. + + Args: + operations (Union[list[TensorOperation], list[functions]]): List of operations to be + applied on the dataset. Operations are applied in the order they appear in this list. + input_columns (Union[str, list[str]], optional): List of the names of the columns that will be passed to + the first operation as input. The size of this list must match the number of + input columns expected by the first operator. (default=None, the first + operation will be passed however many columns that are required, starting from + the first column). + output_columns (Union[str, list[str]], optional): List of names assigned to the columns outputted by + the last operation. This parameter is mandatory if len(input_columns) != + len(output_columns). The size of this list must match the number of output + columns of the last operation. (default=None, output columns will have the same + name as the input columns, i.e., the columns will be replaced). + column_order (list[str], optional): Specifies the list of all the columns you need in the whole + dataset (default=None). The parameter is required when len(input_column) != len(output_column). + Caution: the list here is not just the columns specified in parameter input_columns and output_columns. + num_parallel_workers (int, optional): Number of threads used to process the dataset in + parallel (default=None, the value from the configuration will be used). + python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker processes. This + option could be beneficial if the Python operation is computational heavy (default=False). + cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. + (default=None, which means no cache is used). + callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called (Default=None). + max_rowsize (int, optional): Maximum size of row in MB that is used for shared memory allocation to copy + data between processes. This is only used if python_multiprocessing is set to True (Default=16). + offload (bool, optional): Flag to indicate whether offload is used (Default=None). + + Note: + - Input `operations` accepts TensorOperations defined in mindspore.dataset part, plus user-defined + Python functions (PyFuncs). + - Do not add network computing operators from mindspore.nn and mindspore.ops or others into this + `operations`. + + Returns: + Dataset, dataset after mapping operation. + + Examples: + >>> # dataset is an instance of Dataset which has 2 columns, "image" and "label". + >>> + >>> # Define two operations, where each operation accepts 1 input column and outputs 1 column. + >>> decode_op = c_vision.Decode(rgb=True) + >>> random_jitter_op = c_vision.RandomColorAdjust(brightness=(0.8, 0.8), contrast=(1, 1), + ... saturation=(1, 1), hue=(0, 0)) + >>> + >>> # 1) Simple map example. + >>> + >>> # Apply decode_op on column "image". This column will be replaced by the outputted + >>> # column of decode_op. Since column_order is not provided, both columns "image" + >>> # and "label" will be propagated to the child node in their original order. + >>> dataset = dataset.map(operations=[decode_op], input_columns=["image"]) + >>> + >>> # Decode and rename column "image" to "decoded_image". + >>> dataset = dataset.map(operations=[decode_op], input_columns=["image"], output_columns=["decoded_image"]) + >>> + >>> # Specify the order of the output columns. + >>> dataset = dataset.map(operations=[decode_op], input_columns=["image"], + ... output_columns=None, column_order=["label", "image"]) + >>> + >>> # Rename column "image" to "decoded_image" and also specify the order of the output columns. + >>> dataset = dataset.map(operations=[decode_op], input_columns=["image"], + ... output_columns=["decoded_image"], column_order=["label", "decoded_image"]) + >>> + >>> # Rename column "image" to "decoded_image" and keep only this column. + >>> dataset = dataset.map(operations=[decode_op], input_columns=["image"], + ... output_columns=["decoded_image"], column_order=["decoded_image"]) + >>> + >>> # A simple example for mapping pyfunc. Renaming columns and specifying column order + >>> # work in the same way as the previous examples. + >>> dataset = ds.NumpySlicesDataset(data=[[0, 1, 2]], column_names=["data"]) + >>> dataset = dataset.map(operations=[(lambda x: x + 1)], input_columns=["data"]) + >>> + >>> # 2) Map example with more than one operation. + >>> + >>> # Create a dataset where the images are decoded, then randomly color jittered. + >>> # decode_op takes column "image" as input and outputs one column. The column + >>> # outputted by decode_op is passed as input to random_jitter_op. + >>> # random_jitter_op will output one column. Column "image" will be replaced by + >>> # the column outputted by random_jitter_op (the very last operation). All other + >>> # columns are unchanged. Since column_order is not specified, the order of the + >>> # columns will remain the same. + >>> dataset = dataset.map(operations=[decode_op, random_jitter_op], input_columns=["image"]) + >>> + >>> # Rename the column outputted by random_jitter_op to "image_mapped". + >>> # Specifying column order works in the same way as examples in 1). + >>> dataset = dataset.map(operations=[decode_op, random_jitter_op], input_columns=["image"], + ... output_columns=["image_mapped"]) + >>> + >>> # Map with multiple operations using pyfunc. Renaming columns and specifying column order + >>> # work in the same way as examples in 1). + >>> dataset = ds.NumpySlicesDataset(data=[[0, 1, 2]], column_names=["data"]) + >>> dataset = dataset.map(operations=[(lambda x: x * x), (lambda x: x - 1)], input_columns=["data"], + ... output_columns=["data_mapped"]) + >>> + >>> # 3) Example where number of input columns is not equal to number of output columns. + >>> + >>> # operations[0] is a lambda that takes 2 columns as input and outputs 3 columns. + >>> # operations[1] is a lambda that takes 3 columns as input and outputs 1 column. + >>> # operations[2] is a lambda that takes 1 column as input and outputs 4 columns. + >>> # + >>> # Note: The number of output columns of operation[i] must equal the number of + >>> # input columns of operation[i+1]. Otherwise, this map call will also result + >>> # in an error. + >>> operations = [(lambda x, y: (x, x + y, x + y + 1)), + ... (lambda x, y, z: x * y * z), + ... (lambda x: (x % 2, x % 3, x % 5, x % 7))] + >>> + >>> # Note: Since the number of input columns is not the same as the number of + >>> # output columns, the output_columns and column_order parameters must be + >>> # specified. Otherwise, this map call will also result in an error. + >>> + >>> dataset = ds.NumpySlicesDataset(data=([[0, 1, 2]], [[3, 4, 5]]), column_names=["x", "y"]) + >>> + >>> # Propagate all columns to the child node in this order: + >>> dataset = dataset.map(operations, input_columns=["x", "y"], + ... output_columns=["mod2", "mod3", "mod5", "mod7"], + ... column_order=["mod2", "mod3", "mod5", "mod7"]) + >>> + >>> # Propagate some columns to the child node in this order: + >>> dataset = dataset.map(operations, input_columns=["x", "y"], + ... output_columns=["mod2", "mod3", "mod5", "mod7"], + ... column_order=["mod7", "mod3", "col2"]) + """ + if hasattr(self, 'operator_mixed') and getattr(self, 'operator_mixed') is True: + num_parallel_workers = 1 + logger.warning( + "Input 'operations' of 'map' includes network computing operators like in mindspore.nn, mindspore.ops, " + "mindspore.numpy module and etc, which do not support multi-thread compiling, recommend to replace it " + "with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' into 1.") + + return MapDataset(self, operations, input_columns, output_columns, column_order, num_parallel_workers, + python_multiprocessing, cache, callbacks, max_rowsize, offload) + + @check_filter + def filter(self, predicate, input_columns=None, num_parallel_workers=None): + """ + Filter dataset by prediction. + + Args: + predicate (callable): Python callable which returns a boolean value. If False then filter the element. + input_columns (Union[str, list[str]], optional): List of names of the input columns. If not provided + or provided with None, the predicate will be applied on all columns in the dataset (default=None). + num_parallel_workers (int, optional): Number of workers to process the dataset + in parallel (default=None). + + Returns: + Dataset, dataset filtered. + + Examples: + >>> # generator data(0 ~ 63) + >>> # filter the data that greater than or equal to 11 + >>> dataset = dataset.filter(predicate=lambda data: data < 11, input_columns = ["data"]) + """ + return FilterDataset(self, predicate, input_columns, num_parallel_workers) + + @check_repeat + def repeat(self, count=None): + """ + Repeat this dataset `count` times. Repeat infinitely if the count is None or -1. + + Note: + The order of using repeat and batch reflects the number of batches. It is recommended that + the repeat operation is used after the batch operation. + + Args: + count (int): Number of times the dataset is going to be repeated (default=None). + + Returns: + Dataset, dataset repeated. + + Examples: + >>> # dataset is an instance object of Dataset + >>> + >>> # Create a dataset where the dataset is repeated for 50 epochs + >>> dataset = dataset.repeat(50) + >>> + >>> # Create a dataset where each epoch is shuffled individually + >>> dataset = dataset.shuffle(10) + >>> dataset = dataset.repeat(50) + >>> + >>> # Create a dataset where the dataset is first repeated for + >>> # 50 epochs before shuffling. The shuffle operator will treat + >>> # the entire 50 epochs as one big dataset. + >>> dataset = dataset.repeat(50) + >>> dataset = dataset.shuffle(10) + """ + return RepeatDataset(self, count) + + @check_skip + def skip(self, count): + """ + Skip the first N elements of this dataset. + + Args: + count (int): Number of elements in the dataset to be skipped. + + Returns: + Dataset, dataset that containing rows like origin rows subtract skipped rows. + + Examples: + >>> # dataset is an instance object of Dataset + >>> # Create a dataset which skips first 3 elements from data + >>> dataset = dataset.skip(3) + """ + return SkipDataset(self, count) + + @check_take + def take(self, count=-1): + """ + Takes at most given numbers of elements from the dataset. + + Note: + 1. If count is greater than the number of elements in the dataset or equal to -1, + all the elements in dataset will be taken. + 2. The order of using take and batch matters. If take is before batch operation, + then take the given number of rows; otherwise take the given number of batches. + + Args: + count (int, optional): Number of elements to be taken from the dataset (default=-1). + + Returns: + Dataset, dataset taken. + + Examples: + >>> # dataset is an instance object of Dataset + >>> # Create a dataset where the dataset includes 50 elements. + >>> dataset = dataset.take(50) + """ + return TakeDataset(self, count) +# 定义一个方法用于计算绝对分割大小的函数 + def _get_absolute_split_sizes(self, sizes): + """ + Internal method called by split to calculate absolute split sizes and to + do some error checking after calculating absolute split sizes. + split调用的内部方法,用于计算绝对拆分大小并在计算绝对分割大小后进行一些错误检查。 + Returns: + int, absolute split sizes of the dataset. + """ + # Call get_dataset_size here and check input here because + # don't want to call this once in check_split and another time in + # here again + + # 获取数据集的大小 + dataset_size = self.get_dataset_size() + + # 检查数据集大小是否已知且大于0,否则抛出运行时错误 + if dataset_size is None or dataset_size <= 0: + raise RuntimeError("dataset_size is unknown, unable to split.") + + # 检查输入的分割百分比是否为列表,否则抛出运行时错误 + if not isinstance(sizes, list): + raise RuntimeError("sizes must be a list.") + + # 检查sizes列表中的所有元素是否都为整数 + all_int = all(isinstance(item, int) for item in sizes) + if all_int: + # 如果所有元素都是整数,检查它们的和是否等于数据集大小,否则抛出运行时错误 + sizes_sum = sum(sizes) + if sizes_sum != dataset_size: + raise RuntimeError("Sum of split sizes {} is not equal to dataset size {}." + .format(sizes_sum, dataset_size)) + return sizes + + # 如果sizes列表中包含非整数元素,将百分比转换为绝对大小 + absolute_sizes = [] + for item in sizes: + absolute_size = int(round(item * dataset_size)) + # 检查计算得到的绝对大小是否大于0,否则抛出运行时错误 + if absolute_size == 0: + raise RuntimeError("Split percentage {} is too small.".format(item)) + absolute_sizes.append(absolute_size) + + # 计算绝对分割大小的总和 + absolute_sizes_sum = sum(absolute_sizes) + + # 如果仍然需要更多的行,将它们分配给第一个分割。 + # 如果有太多的行,从第一个具有足够行数的分割中删除多余的行。 + size_difference = int(dataset_size - absolute_sizes_sum) + if size_difference > 0: + absolute_sizes[0] += size_difference + else: + for i, _ in enumerate(absolute_sizes): + if absolute_sizes[i] + size_difference > 0: + absolute_sizes[i] += size_difference + break + + # 检查计算得到的绝对分割大小的总和是否等于数据集大小,否则抛出运行时错误 + if sum(absolute_sizes) != dataset_size: + raise RuntimeError("Sum of calculated split sizes {} is not equal to dataset size {}." + .format(absolute_sizes_sum, dataset_size)) + + # 返回计算得到的绝对分割大小列表 + return absolute_sizes + + @check_split + def split(self, sizes, randomize=True): + """ + Split the dataset into smaller, non-overlapping datasets. + 将数据集拆分为更小、不重叠的数据集。 + This is a general purpose split function which can be called from any operator in the pipeline. + There is another, optimized split function, which will be called automatically if ds.split is + called where ds is a MappableDataset. + + Args: + sizes (Union[list[int], list[float]]): If a list of integers [s1, s2, …, sn] is + provided, the dataset will be split into n datasets of size s1, size s2, …, size sn + respectively. If the sum of all input sizes does not equal the original dataset size, an + error will throw. + If a list of floats [f1, f2, …, fn] is provided, all floats must be between 0 and 1 + and must sum to 1, otherwise an error will throw. The dataset will be split into n + Datasets of size round(f1*K), round(f2*K), …, round(fn*K) where K is the size of the + original dataset. + If after rounding: + + - Any size equals 0, an error will occur. + - The sum of split sizes < K, the difference of K - sigma(round(fi * k)) will be added to the first + split. + - The sum of split sizes > K, the difference of sigma(round(fi * K)) - K will be removed from the first + large enough split such that it will have at least 1 row after removing the difference. + + randomize (bool, optional): Determines whether or not to split the data randomly (default=True). + If True, the data will be randomly split. Otherwise, each split will be created with + consecutive rows from the dataset. + + Note: + 1. Dataset cannot be sharded if split is going to be called. + 2. It is strongly recommended to not shuffle the dataset, but use randomize=True instead. + Shuffling the dataset may not be deterministic, which means the data in each split + will be different in each epoch. + + Raises: + RuntimeError: If get_dataset_size returns None or is not supported for this dataset. + RuntimeError: If `sizes` is list of integers and sum of all elements in sizes does not + equal the dataset size. + RuntimeError: If `sizes` is list of float and there is a split with size 0 after calculations. + RuntimeError: If the dataset is sharded prior to calling split. + ValueError: If `sizes` is list of float and not all floats are between 0 and 1, or if the + floats don't sum to 1. + + Returns: + tuple(Dataset), a tuple of datasets that have been split. + + Examples: + >>> # TextFileDataset is not a mappable dataset, so this non-optimized split will be called. + >>> # Since many datasets have shuffle on by default, set shuffle to False if split will be called! + >>> dataset = ds.TextFileDataset(text_file_dataset_dir, shuffle=False) + >>> train_dataset, test_dataset = dataset.split([0.9, 0.1]) + """ + + if self.is_shuffled(): # 这行代码检查数据集是否已经被洗牌(shuffled),如果是,则发出警告信息,提示数据集已经被洗牌。 + logger.warning("Dataset is shuffled before split.") + + if self.is_sharded(): # 这行代码检查数据集是否已经被分片(sharded),如果是,则引发运行时错误(RuntimeError),表示在分割之前不应该对数据集进行分片。 + raise RuntimeError("Dataset should not be sharded before split.") + # 调用 `_get_absolute_split_sizes` 方法,将相对大小列表 `sizes` 转换为绝对大小列表 `absolute_sizes`,用于指定每个分割的实际大小。 + absolute_sizes = self._get_absolute_split_sizes(sizes) + splits = [] # 创建一个空列表 `splits`,用于存储分割后的数据集。 + rows_to_skip = 0 # 初始化一个变量 `rows_to_skip`,用于跟踪要跳过的行数。 + for size in absolute_sizes: # 遍历绝对大小列表,对每个分割大小执行以下操作: + ds = copy.deepcopy(self) # 创建当前数据集的深度拷贝,以便对其进行操作而不影响原始数据集。 + if randomize: + # want to shuffle the same way every epoch before split + # in alter_tree, shuffle buffer is minimum 10000, so use 10000 here + ds = ds.shuffle(10000) #对数据集进行随机化,这里使用了一个随机缓冲区大小为10000来确保洗牌。 + ds.reshuffle_each_epoch = False #禁用每个epoch重新洗牌,以确保在每个分割中洗牌的方式一致。 + + if rows_to_skip > 0: # 如果需要跳过行数(rows_to_skip大于0),则执行以下操作: + ds = ds.skip(rows_to_skip) # 跳过前面分割已经处理过的行数。 + + ds = ds.take(size) # 从数据集中取出指定数量的行,以创建当前分割。 + splits.append(ds) # 将当前分割添加到 `splits` 列表中。 + + rows_to_skip += size # 更新跳过的行数。 + + return tuple(splits) #返回包含所有分割数据集的元组,每个分割都是一个独立的数据集 + + @check_zip_dataset + def zip(self, datasets): + """ + Zip the datasets in the sense of input tuple of datasets. Columns in the input datasets must have different + name. + + Args: + datasets (Union[tuple, class Dataset]): A tuple of datasets or a single class Dataset + to be zipped together with this dataset. + + Returns: + Dataset, dataset zipped. + + Examples: + >>> # Create a dataset which is the combination of dataset and dataset_1 + >>> dataset = dataset.zip(dataset_1) + """ + if isinstance(datasets, tuple): + datasets = (self, *datasets) + elif isinstance(datasets, Dataset): + datasets = (self, datasets) + else: + raise TypeError("Invalid datasets, expected Dataset object or tuple of Dataset, but got %s!" % datasets) + return ZipDataset(datasets) + + @check_concat + def concat(self, datasets): + """ + Concatenate the dataset objects in the input list. + Performing "+" operation on dataset objects can achieve the same effect. + + Note: + The column name, and rank and type of the column data must be the same in the input datasets. + + Args: + datasets (Union[list, class Dataset]): A list of datasets or a single class Dataset + to be concatenated together with this dataset. + + Returns: + Dataset, dataset concatenated. + + Examples: + >>> # Create a dataset by concatenating dataset_1 and dataset_2 with "+" operator + >>> dataset = dataset_1 + dataset_2 + >>> # Create a dataset by concatenating dataset_1 and dataset_2 with concat operation + >>> dataset = dataset_1.concat(dataset_2) + """ + if isinstance(datasets, Dataset): + datasets = [self] + [datasets] + elif isinstance(datasets, list): + datasets = [self] + datasets + else: + raise TypeError("Invalid datasets, expected Dataset object or list of Dataset, but got %s!" % datasets) + return ConcatDataset(datasets) + + @check_rename + def rename(self, input_columns, output_columns): + """ + Rename the columns in input datasets. + + Args: + input_columns (Union[str, list[str]]): List of names of the input columns. + output_columns (Union[str, list[str]]): List of names of the output columns. + + Returns: + Dataset, dataset renamed. + + Examples: + >>> # dataset is an instance object of Dataset + >>> input_columns = ["input_col1", "input_col2", "input_col3"] + >>> output_columns = ["output_col1", "output_col2", "output_col3"] + >>> + >>> # Create a dataset where input_col1 is renamed to output_col1, and + >>> # input_col2 is renamed to output_col2, and input_col3 is renamed + >>> # to output_col3. + >>> dataset = dataset.rename(input_columns=input_columns, output_columns=output_columns) + """ + + return RenameDataset(self, input_columns, output_columns) + + @check_project + def project(self, columns): + """ + Project certain columns in input dataset. + + The specified columns will be selected from the dataset and passed into + the pipeline with the order specified. The other columns are discarded. + + Args: + columns(Union[str, list[str]]): List of names of the columns to project. + + Returns: + Dataset, dataset projected. + + Examples: + >>> # dataset is an instance object of Dataset + >>> columns_to_project = ["column3", "column1", "column2"] + >>> + >>> # Create a dataset that consists of column3, column1, column2 + >>> # in that order, regardless of the original order of columns. + >>> dataset = dataset.project(columns=columns_to_project) + """ + + return ProjectDataset(self, columns) + + def apply(self, apply_func): + """ + Apply a function in this dataset. + + Args: + apply_func (function): A function that must take one `Dataset` as an argument and + return a preprocessed `Dataset`. + + Returns: + Dataset, dataset applied by the function. + + Examples: + >>> # dataset is an instance object of Dataset + >>> + >>> # Declare an apply_func function which returns a Dataset object + >>> def apply_func(data): + ... data = data.batch(2) + ... return data + >>> + >>> # Use apply to call apply_func + >>> dataset = dataset.apply(apply_func) + + Raises: + TypeError: If apply_func is not a function. + TypeError: If apply_func doesn't return a Dataset. + """ + + if not hasattr(apply_func, '__call__'): + raise TypeError("apply_func must be a function.") + + dataset = apply_func(self) + if not isinstance(dataset, Dataset): + raise TypeError("apply_func must return a dataset.") + return dataset + + @check_device_send + def device_que(self, send_epoch_end=True, create_data_info_queue=False): + """ + Return a transferred Dataset that transfers data through a device. + + Args: + send_epoch_end (bool, optional): Whether to send end of sequence to device or not (default=True). + create_data_info_queue (bool, optional): Whether to create queue which stores + types and shapes of data or not(default=False). + + Note: + If device is Ascend, features of data will be transferred one by one. The limitation + of data transmission per time is 256M. + + Returns: + Dataset, dataset for transferring. + """ + return TransferDataset(self, send_epoch_end, create_data_info_queue) + + @check_device_send + def to_device(self, send_epoch_end=True, create_data_info_queue=False): + """ + Transfer data from CPU to GPU or Ascend or other devices. + + Args: + send_epoch_end (bool, optional): Whether to send the end of sequence to device or not (default=True). + create_data_info_queue (bool, optional): Whether to create queue which stores + types and shapes of data or not(default=False). + + Note: + This interface will be deleted or invisible in the future. + Please use `device_que` to enable dataset sink mode. + If device is Ascend, features of data will be transferred one by one. The limitation + of data transmission per second is 256M. + + Returns: + TransferDataset, dataset for transferring. + + Raises: + RuntimeError: If distribution file path is given but failed to read. + """ + logger.warning("This interface will be deleted or invisible in the future. " + "Please use 'device_que' to enable dataset sink mode.") + + return TransferDataset(self, send_epoch_end, create_data_info_queue) + + @check_save + def save(self, file_name, num_files=1, file_type='mindrecord'): + """ + Save the dynamic data processed by the dataset pipeline in common dataset format. + Supported dataset formats: `mindrecord` only. And you can use `MindDataset` API to read the saved file(s). + 以通用数据集格式保存数据集管道处理的动态数据。 + Implicit type casting exists when saving data as `mindrecord`. The transform table shows how to do type casting. + + .. list-table:: Implicit Type Casting when Saving as `mindrecord` + :widths: 25 25 50 + :header-rows: 1 + + * - Type in `dataset` + - Type in `mindrecord` + - Details + * - bool + - None + - Not supported + * - int8 + - int32 + - + * - uint8 + - bytes(1D uint8) + - Drop dimension + * - int16 + - int32 + - + * - uint16 + - int32 + - + * - int32 + - int32 + - + * - uint32 + - int64 + - + * - int64 + - int64 + - + * - uint64 + - None + - Not supported + * - float16 + - float32 + - + * - float32 + - float32 + - + * - float64 + - float64 + - + * - string + - string + - Multi-dimensional string not supported + + Note: + 1. To save the samples in order, set dataset's shuffle to False and num_files to 1. + 2. Before calling the function, do not use batch operator, repeat operator or data augmentation operators + with random attribute in map operator. + 3. When array dimension is variable, one-dimensional arrays or + multi-dimensional arrays with variable dimension 0 are supported. + 4. Mindrecord does not support uint64, multi-dimensional uint8(drop dimension) nor + multi-dimensional string. + + Args: + file_name (str): Path to dataset file. + num_files (int, optional): Number of dataset files (default=1). + file_type (str, optional): Dataset format (default='mindrecord'). + + """ + ir_tree, api_tree = self.create_ir_tree() #调用 `create_ir_tree()` 方法创建了内部数据结构 `ir_tree` 和 `api_tree`,用于保存数据集的信息。 + + runtime_context = cde.PythonRuntimeContext() #创建了一个Python运行时环境对象 `runtime_context`。 + runtime_context.Init() #初始化运行时环境。 + consumer = cde.PythonSaveToDisk(file_name, num_files, file_type) #创建了一个保存到磁盘的消费者对象 `consumer`,指定了文件名、文件数量和文件类型。 + consumer.Init(ir_tree) #初始化消费者对象,传入 `ir_tree` 数据结构。 + runtime_context.AssignConsumer(consumer) #将消费者对象分配给运行时环境。 + + consumer.Save() #执行保存操作,将数据集保存到磁盘。 + _set_dataset_permissions(file_name, num_files) #调用 `_set_dataset_permissions` 函数,设置数据集文件的权限。 + del api_tree #删除 `api_tree` 对象,释放内存。 + + @check_tuple_iterator + def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True): + """ + Create an iterator over the dataset. The datatype retrieved back will be a list of `numpy.ndarray`. + + To specify which columns to list and the order needed, use columns_list. If columns_list + is not provided, the order of the columns will remain unchanged. + + Args: + columns (list[str], optional): List of columns to be used to specify the order of columns + (default=None, means all columns). + num_epochs (int, optional): Maximum number of epochs that iterator can be iterated. + (default=-1, iterator can be iterated infinite number of epochs) + output_numpy (bool, optional): Whether or not to output NumPy datatype. + If output_numpy=False, iterator will output MSTensor (default=False). + do_copy (bool, optional): when output data type is mindspore.Tensor, + use this param to select the conversion method, only take False for better performance (default=True). + + Returns: + Iterator, tuple iterator over the dataset. + + Examples: + >>> # dataset is an instance object of Dataset + >>> iterator = dataset.create_tuple_iterator() + >>> for item in iterator: + ... # item is a list + ... print(type(item)) + ... break + + """ + if output_numpy is None: + output_numpy = False + + if Dataset._noop_mode(): + return DummyIterator(self, 'tuple', output_numpy) + return TupleIterator(self, columns, num_epochs, output_numpy, do_copy) + + @check_dict_iterator + def create_dict_iterator(self, num_epochs=-1, output_numpy=False): + """ + Create an iterator over the dataset. The data retrieved will be a dictionary datatype. + + The order of the columns in the dictionary may not be the same as the original order. + + Args: + num_epochs (int, optional): Maximum number of epochs that iterator can be iterated + (default=-1, iterator can be iterated infinite number of epochs). + output_numpy (bool, optional): Whether or not to output NumPy datatype, + if output_numpy=False, iterator will output MSTensor (default=False). + + Returns: + Iterator, dictionary iterator over the dataset. + + Examples: + >>> # dataset is an instance object of Dataset + >>> iterator = dataset.create_dict_iterator() + >>> for item in iterator: + ... # item is a dict + ... print(type(item)) + ... break + + """ + if output_numpy is None: + output_numpy = False + + if Dataset._noop_mode(): + return DummyIterator(self, 'dict', output_numpy) + return DictIterator(self, num_epochs, output_numpy) + + def __iter__(self): + """Create an iterator over the dataset.""" + return self.create_tuple_iterator(num_epochs=1) + + @property + def input_indexs(self): + """ + Get the column index, which represents the corresponding relationship between the data column order + and the network when using the sink mode. + + Returns: + int, tuple of the input index information. + + Examples: + >>> # dataset is an instance object of Dataset + >>> # set input_indexs + >>> dataset.input_indexs = 10 + >>> print(dataset.input_indexs) + 10 + """ + if self._input_indexs != (): + return self._input_indexs + + # find input_indexes of children + children_input_index = [child.input_indexs for child in self.children] + + # in case of more than one child, return the first input_indexes + for cix in children_input_index: + if cix != (): + return cix + + # if all children's input_indexes are () or the node is a leaf + return self._input_indexs + + @input_indexs.setter + def input_indexs(self, value): + self._input_indexs = value + + def copy_batch_size(self, value): + self._batch_size = value + + def _init_tree_getters(self): + """ + Get pipeline information. + """ + ir_tree, api_tree = self.create_ir_tree() + + runtime_context = cde.PythonRuntimeContext() + runtime_context.Init() + getter = cde.TreeGetters() + getter.Init(ir_tree) + runtime_context.AssignConsumer(getter) + return getter, runtime_context, api_tree + + def __init_size_getter(self): + """ + Get pipeline information. + """ + ir_tree, api_tree = self.create_ir_tree() + + runtime_context = cde.PythonRuntimeContext() + runtime_context.Init() + getter = cde.DatasetSizeGetters() + getter.Init(ir_tree) + runtime_context.AssignConsumer(getter) + return getter, runtime_context, api_tree + + def get_col_names(self): + """ + Return the names of the columns in dataset. + + Returns: + list, list of column names in the dataset. + + Examples: + >>> # dataset is an instance object of Dataset + >>> col_names = dataset.get_col_names() + """ + if self._col_names is None: + runtime_getter = self._init_tree_getters() + self._col_names = runtime_getter[0].GetColumnNames() + + return self._col_names + + @check_output_shape + def output_shapes(self, estimate=False): + """ + Get the shapes of output data. + + Args: + estimate (bool): If `estimate` is False, will return the shapes of first data row. + Otherwise, will iterate the whole dataset and return the estimated shapes of data row, + where dynamic shape is marked as None (used in dynamic data shapes scenario). Default: False. + + Returns: + list, list of shapes of each column. + + Examples: + >>> import numpy as np + >>> + >>> def generator1(): + ... for i in range(1, 100): + ... yield np.ones((16, i, 83)), np.array(i) + >>> + >>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) + >>> output_shapes = dataset.output_shapes() + """ + # cache single shape + if not estimate and self.saved_output_shapes is not None: + return self.saved_output_shapes + # cache estimate shape + if estimate and self.estimated_output_shapes is not None: + return self.estimated_output_shapes + + # if use set_dynamic_column, the `estimate` does not work, but they get the same result + if self.dynamic_setting[0]: + self.saved_output_shapes, self.saved_min_shapes, self.saved_max_shapes = self._dynamic_output_shapes() + return self.saved_output_shapes + + # We have a hang problem when two-level pipeline with multiprocessing, we need to extend the life cycle + # of runtime_context. We found this hang problem only occur on output_types and output_shapes. + runtime_getter = self._init_tree_getters() + self.runtime_context = runtime_getter[1] + api_tree = runtime_getter[2] + output_shapes = runtime_getter[0].GetOutputShapes(estimate) + del api_tree + del self.runtime_context + + if estimate: + self.estimated_output_shapes = output_shapes + else: + self.saved_output_shapes = output_shapes + return output_shapes + + def output_types(self): + """ + Get the types of output data. + + Returns: + list, list of data types. + + Examples: + >>> # dataset is an instance object of Dataset + >>> output_types = dataset.output_types() + """ + if self.saved_output_types is None: + runtime_getter = self._init_tree_getters() + # We have a hang problem when two-level pipeline with multiprocessing, we need to extend the life cycle + # of runtime_context. We found this hang problem only occur on output_types and output_shapes. + self.runtime_context = runtime_getter[1] + api_tree = runtime_getter[2] + self.saved_output_types = runtime_getter[0].GetOutputTypes() + del api_tree + del self.runtime_context + return self.saved_output_types + + def get_dataset_size(self): + """ + Return the number of batches in an epoch. + + Returns: + int, number of batches. + + Examples: + >>> # dataset is an instance object of Dataset + >>> dataset_size = dataset.get_dataset_size() + """ + if self.dataset_size is None: + runtime_getter = self.__init_size_getter() + self.dataset_size = runtime_getter[0].GetDatasetSize(False) + + return self.dataset_size + + @deprecated("1.5") + def set_dynamic_columns(self, columns=None): + """ + Set dynamic shape information of source data, it should be set after the pipeline is defined. + + Args: + columns (dict): A dict contains shape information of each column in dataset. + The value of shape[i] is :py:obj:`None` indicates that the data length of shape[i] is dynamic. + + Examples: + >>> import numpy as np + >>> + >>> def generator1(): + ... for i in range(1, 100): + ... yield np.ones((16, i, 83)), np.array(i) + >>> + >>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) + >>> dataset.set_dynamic_columns(columns={"data1": [16, None, 83], "data2": []}) + """ + if not isinstance(columns, dict): + raise TypeError("Pass a dict to set dynamic shape, example: {\"data1\": [16, None, 256]}") + self.dynamic_setting[0] = True + self.dynamic_setting[1] = columns + + def dynamic_min_max_shapes(self): + """ + Get minimum and maximum data length of dynamic source data, for dynamic graph compilation. + + Returns: + lists, min_shapes, max_shapes of source data. + + Examples: + >>> import numpy as np + >>> + >>> def generator1(): + ... for i in range(1, 100): + ... yield np.ones((16, i, 83)), np.array(i) + >>> + >>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) + >>> dataset.set_dynamic_columns(columns={"data1": [16, None, 83], "data2": []}) + >>> min_shapes, max_shapes = dataset.dynamic_min_max_shapes() + """ + if self.saved_min_shapes is None or self.saved_max_shapes is None: + self.saved_output_shapes, self.saved_min_shapes, self.saved_max_shapes = self._dynamic_output_shapes() + return self.saved_min_shapes, self.saved_max_shapes + + @staticmethod + def __check_dynamic_column_name(dynamic_columns, dataset_columns): + for column in dynamic_columns: + if column not in dataset_columns: + raise RuntimeError("dynamic column [" + column + "] does not match any column in dataset: " + + str(dataset_columns)) + + @staticmethod + def __check_dynamic_column_shape(data, col, dynamic_columns): + shape_mismatch = "dynamic column [" + col + "] with shape " + str(dynamic_columns[col]) + \ + " does not match dataset column [" + col + "] with shape " + str(list(data[col].shape)) + if data[col].ndim != len(dynamic_columns[col]): + raise RuntimeError(shape_mismatch) + for dim in range(len(dynamic_columns[col])): + if dynamic_columns[col][dim] is not None and dynamic_columns[col][dim] != data[col].shape[dim]: + raise RuntimeError(shape_mismatch) + + def _dynamic_output_shapes(self): + """ + Get dynamic information of source data. + + Returns: + lists, dynamic_shapes, min_shapes, max_shapes of source data. + """ + if not self.dynamic_setting[1]: + raise RuntimeError("dynamic_columns is not set, call set_dynamic_columns() by final Dataset Op.") + + if self.saved_output_shapes is not None and self.saved_min_shapes is not None and \ + self.saved_max_shapes is not None: + return self.saved_output_shapes, self.saved_min_shapes, self.saved_max_shapes + + logger.warning("Calculating dynamic shape of input data, this will take a few minutes...") + # Assume data1 shape is dynamic, data2 shape is fix + dynamic_columns = self.dynamic_setting[1] + # ["data1", "data2"] + dataset_columns = self.get_col_names() + Dataset.__check_dynamic_column_name(dynamic_columns, dataset_columns) + + # Shape[1] of data1 is variable + # {"data1": {(batch_size, 100, feat_len), (16, 200, 83)}, "data2": {(batch_size, feat_len)}} + column_shape_set = {col: set() for col in dataset_columns} + dataset_size_counter = 0 + for data in self.create_dict_iterator(num_epochs=1, output_numpy=True): + dataset_size_counter += 1 + for col in data.keys(): + if col in dynamic_columns: + Dataset.__check_dynamic_column_shape(data, col, dynamic_columns) + column_shape_set[col].add(tuple(data[col].shape)) + + # we get dataset_size after dryrun + self.dataset_size = dataset_size_counter + + min_shapes, max_shapes, dynamic_shapes = list(), list(), list() + for col, shape_set in column_shape_set.items(): + if len(shape_set) > 1: + if col not in dynamic_columns: + raise RuntimeError("column [" + col + "] has dynamic shape but not set by set_dynamic_columns()" + + ", shapes of [" + col + "]: " + str(list(shape_set))) + shape_npy = np.array(list(shape_set)) + max_shape = shape_npy.max(axis=0) + min_shape = shape_npy.min(axis=0) + + # Set min shape to 1 due to unknown shuffle + min_shape = np.where(np.equal(dynamic_columns[col], None), 1, min_shape) + # Set dynamic dim to -1 for ME + dynamic_shape = np.where(np.equal(dynamic_columns[col], None), -1, dynamic_columns[col]) + + max_shapes.append(max_shape.tolist()) + min_shapes.append(min_shape.tolist()) + dynamic_shapes.append(dynamic_shape.tolist()) + else: + # Also append fix shape to keep order of column shape + fix_shape = list(list(shape_set)[0]) + max_shapes.append(fix_shape) + min_shapes.append(fix_shape) + dynamic_shapes.append(fix_shape) + if col in dynamic_columns: + logger.warning("column [" + col + "] has no dynamic shape but set by set_dynamic_columns()") + # Set min shape to 1 due to unknown shuffle + min_shapes[-1] = np.where(np.equal(dynamic_columns[col], None), 1, fix_shape).tolist() + # Set dynamic dim to -1 for ME + dynamic_shapes[-1] = np.where(np.equal(dynamic_columns[col], None), -1, fix_shape).tolist() + return dynamic_shapes, min_shapes, max_shapes + + def num_classes(self): + """ + Get the number of classes in a dataset. + + Returns: + int, number of classes. + + Examples: + >>> # dataset is an instance object of Dataset + >>> num_classes = dataset.num_classes() + """ + if self._num_classes is None: + runtime_getter = self._init_tree_getters() + self._num_classes = runtime_getter[0].GetNumClasses() + + if self._num_classes == -1: + return None + return self._num_classes + + def get_sync_notifiers(self): + if self.children: + return self.children[0].get_sync_notifiers() + return {} + + def disable_sync(self): + if self.children: + return self.children[0].disable_sync() + return {} + + def is_sync(self): + if self.children: + return self.children[0].is_sync() + return False + + def sync_update(self, condition_name, num_batch=None, data=None): + """ + Release a blocking condition and trigger callback with given data. + + Args: + condition_name (str): The condition name that is used to toggle sending next row. + num_batch (Union[int, None]): The number of batches (rows) that are released. + When num_batch is None, it will default to the number specified by the + sync_wait operator (default=None). + data (Any): The data passed to the callback, user defined (default=None). + """ + if (not isinstance(num_batch, int) and num_batch is not None) or \ + (isinstance(num_batch, int) and num_batch <= 0): + # throwing exception, disable all sync_wait in pipeline + self.disable_sync() + raise RuntimeError("Sync_update batch size can only be positive integer, got : {}.".format(num_batch)) + notifiers_dict = self.get_sync_notifiers() + if not isinstance(condition_name, str): + raise TypeError("Argument condition_name with value {} is not of type str, but got {}." + .format(condition_name, type(condition_name))) + if condition_name not in notifiers_dict: + # throwing exception, disable all sync_wait in pipeline + self.disable_sync() + raise RuntimeError("Condition name not found.") + if num_batch is not None: + num_batch *= self.get_batch_size() + notifiers_dict[condition_name](num_batch, data) + + def get_batch_size(self): + """ + Return the size of batch. + + Returns: + int, the number of data in a batch. + + Examples: + >>> # dataset is an instance object of Dataset + >>> batch_size = dataset.get_batch_size() + """ + if self._batch_size is None: + runtime_getter = self._init_tree_getters() + self._batch_size = runtime_getter[0].GetBatchSize() + if self._batch_size is None: + self._batch_size = 1 + return self._batch_size + + def get_repeat_count(self): + """ + Get the replication times in RepeatDataset (default is 1). + + Returns: + int, the count of repeat. + + Examples: + >>> # dataset is an instance object of Dataset + >>> repeat_count = dataset.get_repeat_count() + """ + if self._repeat_count is None: + runtime_getter = self._init_tree_getters() + self._repeat_count = runtime_getter[0].GetRepeatCount() + if self._repeat_count is None: + self._repeat_count = 1 + return self._repeat_count + + def get_class_indexing(self): + """ + Return the class index. + + Returns: + dict, a str-to-int mapping from label name to index. + dict, a str-to-list mapping from label name to index for Coco ONLY. The second number + in the list is used to indicate the super category. + + Examples: + >>> # dataset is an instance object of Dataset + >>> class_indexing = dataset.get_class_indexing() + """ + if self.children: + return self.children[0].get_class_indexing() + return {} + + def reset(self): + """Reset the dataset for next epoch.""" + + def is_shuffled(self): + """Returns True if the dataset or its children is shuffled.""" + for input_dataset in self.children: + if input_dataset.is_shuffled(): + return True + + return False + + def is_sharded(self): + """Returns True if the dataset or its children is sharded.""" + for input_dataset in self.children: + if input_dataset.is_sharded(): + return True + + return False + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + @staticmethod + def _update_data_shard(num_shards, shard_id): + """ + Update the shard number and shard id if necessary. + This is normally used in distributed training mode like Parameter Server training. + """ + # If this is in distributed execution mode, + # the shard number and shard id might need to be updated according to the process's rank or role. + if _is_role_pserver() and _enable_distributed_mindrt(): + num_shards = _get_ps_context("worker_num") + shard_id = 0 + return num_shards, shard_id + + def post_parse(self, ir_node): + if self.cache: + ir_node = ir_node.set_cache_client(self.cache.cache_client) + if self.num_parallel_workers: + ir_node = ir_node.set_num_workers(self.num_parallel_workers) + + return ir_node + + +class VisionBaseDataset(Dataset): + """ + Abstract class to represent a vision source dataset which produces content to the data pipeline. + """ + + def __init__(self, children=None, num_parallel_workers=None, cache=None): + super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache) + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + +class TextBaseDataset(Dataset): + """ + Abstract class to represent a text source dataset which produces content to the data pipeline. + """ + + def __init__(self, children=None, num_parallel_workers=None, cache=None): + super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache) + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + def build_vocab(self, columns, freq_range, top_k, special_tokens, special_first): + """ + Function to create a Vocab from source dataset. + Desired source dataset is a text type dataset. + + Build a vocab from a dataset. This would collect all the unique words in a dataset and return a vocab + which contains top_k most frequent words (if top_k is specified) + + Args: + + columns(Union[str, list[str]]): Column names to get words from. + freq_range(tuple[int]): A tuple of integers (min_frequency, max_frequency). Words within the frequency + range will be stored. + Naturally 0 <= min_frequency <= max_frequency <= total_words. min_frequency/max_frequency + can be set to default, which corresponds to 0/total_words separately. + top_k(int): Number of words to be built into vocab. top_k most frequent words are + taken. The top_k is taken after freq_range. If not enough top_k, all words will be taken + special_tokens(list[str]): A list of strings, each one is a special token. + special_first(bool): Whether special_tokens will be prepended/appended to vocab, If special_tokens + is specified and special_first is set to default, special_tokens will be prepended. + + Returns: + Vocab, vocab built from the dataset. + + Examples: + >>> import numpy as np + >>> + >>> def gen_corpus(): + ... # key: word, value: number of occurrences, reason for using letters is so their order is apparent + ... corpus = {"Z": 4, "Y": 4, "X": 4, "W": 3, "U": 3, "V": 2, "T": 1} + ... for k, v in corpus.items(): + ... yield (np.array([k] * v, dtype='S'),) + >>> column_names = ["column1"] + >>> dataset = ds.GeneratorDataset(gen_corpus, column_names) + >>> dataset = dataset.build_vocab(columns=["column1"], + ... freq_range=(1, 10), top_k=5, + ... special_tokens=["", ""], + ... special_first=True) + + """ + vocab = cde.Vocab() + columns = replace_none(columns, []) + if not isinstance(columns, list): + columns = [columns] + + freq_range = replace_none(freq_range, (0, 9223372036854775807)) + if freq_range[0] is None: + freq_range = (0, freq_range[1]) + if freq_range[1] is None: + freq_range = (freq_range[0], 9223372036854775807) + special_tokens = replace_none(special_tokens, []) + top_k = replace_none(top_k, 9223372036854775807) + + ir_tree, api_tree = self.create_ir_tree() + + # vocab node + vocab_node = cde.BuildVocabNode(ir_tree, vocab, columns, freq_range, top_k, special_tokens, special_first) + + runtime_context = cde.PythonRuntimeContext() + runtime_context.Init() + + # build vocab + consumer = cde.PythonBuildVocabConsumer() + consumer.Init(vocab_node) + runtime_context.AssignConsumer(consumer) + + consumer.Start() + del api_tree + + return vocab + + def build_sentencepiece_vocab(self, columns, vocab_size, character_coverage, model_type, params): + """ + Function to create a SentencePieceVocab from source dataset. + Desired source dataset is a text type dataset. + + Args: + + columns(list[str]): Column names to get words from. + vocab_size(int): Vocabulary size. + character_coverage(float): Percentage of characters covered by the model, must be between + 0.98 and 1.0 Good defaults are: 0.9995 for languages with rich character sets like + Japanese or Chinese character sets, and 1.0 for other languages with small character sets + like English or Latin. + model_type(SentencePieceModel): Model type. Choose from unigram (default), bpe, char, or word. + The input sentence must be pretokenized when using word type. + params(dict): Any extra optional parameters of sentencepiece library according to your raw data + + Returns: + SentencePieceVocab, vocab built from the dataset. + + Examples: + >>> from mindspore.dataset.text import SentencePieceModel + >>> + >>> # You can construct any text dataset as source, take TextFileDataset as example. + >>> dataset = ds.TextFileDataset("/path/to/sentence/piece/vocab/file", shuffle=False) + >>> dataset = dataset.build_sentencepiece_vocab(["text"], 5000, 0.9995, SentencePieceModel.UNIGRAM, {}) + """ + if not isinstance(model_type, SentencePieceModel): + raise TypeError("Argument model_type with value {0} is not of type SentencePieceModel, but got {1}." \ + .format(model_type, type(model_type))) + model_type = DE_C_INTER_SENTENCEPIECE_MODE[model_type] + vocab = cde.SentencePieceVocab() + + ir_tree, api_tree = self.create_ir_tree() + + # vocab node + vocab_node = cde.BuildSentenceVocabNode(ir_tree, vocab, columns, vocab_size, character_coverage, model_type, + params) + + runtime_context = cde.PythonRuntimeContext() + runtime_context.Init() + + # build vocab + consumer = cde.PythonBuildVocabConsumer() + consumer.Init(vocab_node) + runtime_context.AssignConsumer(consumer) + + consumer.Start() + del api_tree + + return vocab + + +class AudioBaseDataset(Dataset): + """ + Abstract class to represent a audio source dataset which produces content to the data pipeline. + """ + + def __init__(self, children=None, num_parallel_workers=None, cache=None): + super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache) + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + +class UnionBaseDataset(VisionBaseDataset, TextBaseDataset, AudioBaseDataset): + """ + Abstract class to represent a union source dataset which produces content to the data pipeline. + """ + + def __init__(self, children=None, num_parallel_workers=None, cache=None): + super().__init__(children=children, num_parallel_workers=num_parallel_workers, cache=cache) + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + +class SourceDataset(Dataset): + """ + Abstract class to represent a source dataset which produces content to the data pipeline. + """ + + def __init__(self, num_parallel_workers=None, num_samples=None, shuffle=True, num_shards=None, shard_id=None, + cache=None): + super().__init__(num_parallel_workers=num_parallel_workers, cache=cache) + self.num_samples = replace_none(num_samples, 0) + self.num_shards = replace_none(num_shards, 1) + self.shard_id = replace_none(shard_id, 0) + + if shuffle is not None and not isinstance(shuffle, (bool, Shuffle)): + raise TypeError("shuffle must be of boolean or enum of 'Shuffle' values like 'Shuffle.GLOBAL' or " + "'Shuffle.FILES' or 'Shuffle.INFILE'.") + + self.shuffle_flag = 2 # Global shuffle + if not isinstance(shuffle, Shuffle): + if shuffle is None or shuffle: + self.shuffle_flag = 2 # Global shuffle + else: + self.shuffle_flag = 0 # No shuffle + else: + if shuffle == Shuffle.GLOBAL: + self.shuffle_flag = 2 # Global shuffle + elif shuffle == Shuffle.FILES: + self.shuffle_flag = 1 # Files shuffle + elif shuffle == Shuffle.INFILE: + self.shuffle_flag = 3 # Infile shuffle + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + @staticmethod + def _find_files(patterns): + """ + Utility function to search for files with the given glob patterns. + + Args: + patterns (Union[str, list[str]]): String or list of patterns to be searched. + + Returns: + list, list of files. + """ + + if not isinstance(patterns, list): + patterns = [patterns] + + file_list = [] + unmatched_patterns = [] + for pattern in patterns: + matches = [match for match in glob.glob(pattern, recursive=True) if os.path.isfile(match)] + + if matches: + file_list.extend(matches) + else: + unmatched_patterns.append(pattern) + + if unmatched_patterns: + raise ValueError("The following patterns did not match any files: {}.".format(unmatched_patterns)) + + if file_list: # not empty + return file_list + raise ValueError("The list of path names matching the patterns is empty.") + + def is_shuffled(self): + return self.shuffle_flag > 0 + + def is_sharded(self): + if self.num_shards is not None: + return self.num_shards > 1 + return False + + +class MappableDataset(SourceDataset): + """ + Abstract class to represent a source dataset which supports use of samplers. + """ + + def parse(self, children=None): + raise NotImplementedError("Dataset has to implement parse method.") + + def __init__(self, num_parallel_workers=None, sampler=None, num_samples=None, shuffle=None, num_shards=None, + shard_id=None, cache=None): + num_shards, shard_id = self._update_data_shard(num_shards, shard_id) + super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle, + num_shards=num_shards, shard_id=shard_id, cache=cache) + self.shuffle_flag = replace_none(shuffle, True) + self.sampler = samplers.select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) + + def add_sampler(self, new_sampler): + """ + Add a child sampler for the current dataset. + + Args: + new_sampler (Sampler): The child sampler to be added. + + Examples: + >>> new_sampler = ds.DistributedSampler(10, 2) + >>> dataset.add_sampler(new_sampler) # dataset is an instance of Dataset + """ + # Note: By adding a sampler, the sampled IDs will flow to the new_sampler + # after first passing through the current samplers attached to this dataset. + self.dataset_size = None + new_sampler.add_child(self.sampler) + self.sampler = new_sampler + + def use_sampler(self, new_sampler): + """ + Replace the last child sampler of the current dataset, remaining the parent sampler unchanged. + + Args: + new_sampler (Sampler): The new sampler to replace with. + + Examples: + >>> # dataset is an instance object of Dataset + >>> # use a DistributedSampler instead + >>> new_sampler = ds.DistributedSampler(10, 2) + >>> dataset.use_sampler(new_sampler) + """ + if new_sampler is None: + raise TypeError("Input sampler can not be None.") + if not isinstance(new_sampler, (samplers.BuiltinSampler, samplers.Sampler)): + raise TypeError("Input sampler is not an instance of a sampler.") + self.dataset_size = None + + self.sampler = self.sampler.child_sampler + self.add_sampler(new_sampler) + + def is_shuffled(self): + return self.sampler.is_shuffled() + + def is_sharded(self): + return self.sampler.is_sharded() + + @check_split + def split(self, sizes, randomize=True): + """ + Split the dataset into smaller, non-overlapping datasets. + + Args: + sizes (Union[list[int], list[float]]): If a list of integers [s1, s2, …, sn] is + provided, the dataset will be split into n datasets of size s1, size s2, …, size sn + respectively. If the sum of all sizes does not equal the original dataset size, an + error will occur. + If a list of floats [f1, f2, …, fn] is provided, all floats must be between 0 and 1 + and must sum to 1, otherwise an error will occur. The dataset will be split into n + Datasets of size round(f1*K), round(f2*K), …, round(fn*K) where K is the size of the + original dataset. + If after rounding: + + - Any size equals 0, an error will occur. + - The sum of split sizes < K, the difference will be added to the first split. + - The sum of split sizes > K, the difference will be removed from the first large + enough split such that it will have at least 1 row after removing the difference. + + randomize (bool, optional): Determines whether or not to split the data randomly (default=True). + If True, the data will be randomly split. Otherwise, each split will be created with + consecutive rows from the dataset. + + Note: + 1. There is an optimized split function, which will be called automatically when the dataset + that calls this function is a MappableDataset. + 2. Dataset should not be sharded if split is going to be called. Instead, create a + DistributedSampler and specify a split to shard after splitting. If the dataset is + sharded after a split, it is strongly recommended setting the same seed in each instance + of execution, otherwise each shard may not be part of the same split (see Examples). + 3. It is strongly recommended to not shuffle the dataset, but use randomize=True instead. + Shuffling the dataset may not be deterministic, which means the data in each split + will be different in each epoch. Furthermore, if sharding occurs after split, each + shard may not be part of the same split. + + Raises: + RuntimeError: If get_dataset_size returns None or is not supported for this dataset. + RuntimeError: If `sizes` is list of integers and sum of all elements in sizes does not + equal the dataset size. + RuntimeError: If `sizes` is list of float and there is a split with size 0 after calculations. + RuntimeError: If the dataset is sharded prior to calling split. + ValueError: If `sizes` is list of float and not all floats are between 0 and 1, or if the + floats don't sum to 1. + + Returns: + tuple(Dataset), a tuple of datasets that have been split. + + Examples: + >>> # Since many datasets have shuffle on by default, set shuffle to False if split will be called! + >>> dataset = ds.ImageFolderDataset(image_folder_dataset_dir, shuffle=False) + >>> + >>> # Set the seed, and tell split to use this seed when randomizing. + >>> # This is needed because sharding will be done later + >>> ds.config.set_seed(58) + >>> train_dataset, test_dataset = dataset.split([0.9, 0.1]) + >>> + >>> # To shard the train dataset, use a DistributedSampler + >>> train_sampler = ds.DistributedSampler(10, 2) + >>> train_dataset.use_sampler(train_sampler) + """ + if self.is_shuffled(): + logger.warning("Dataset is shuffled before split.") + + if self.is_sharded(): + raise RuntimeError("Dataset should not be sharded before split.") + + absolute_sizes = self._get_absolute_split_sizes(sizes) + splits = [] + current_split_start_index = 0 + for size in absolute_sizes: + ds = copy.deepcopy(self) + ds.dataset_size = None + if randomize: + # want to shuffle the same way every epoch before split, we are assuming + # that the user will call set_seed + random_sampler = samplers.RandomSampler() + random_sampler.reshuffle_each_epoch = False + ds.add_sampler(random_sampler) + + subset_sampler = samplers.SequentialSampler(current_split_start_index, size) + ds.add_sampler(subset_sampler) + + # add sequential sampler, so that if user calls use_sampler, we will + # get rid of the sequential sampler instead of something we need + ds.add_sampler(samplers.SequentialSampler()) + + splits.append(ds) + + current_split_start_index += size + + return tuple(splits) + + +class BucketBatchByLengthDataset(UnionBaseDataset): + """ + The result of applying BucketBatchByLength operator to the input dataset. + """ + + def __init__(self, input_dataset, column_names, bucket_boundaries, bucket_batch_sizes, element_length_function, + pad_info, pad_to_bucket_boundary, drop_remainder): + super().__init__(children=input_dataset) + + self.column_names = to_list(column_names) + self.bucket_boundaries = replace_none(bucket_boundaries, []) + self.bucket_batch_sizes = replace_none(bucket_batch_sizes, []) + self.element_length_function = element_length_function + self.pad_info = replace_none(pad_info, {}) + self.pad_to_bucket_boundary = replace_none(pad_to_bucket_boundary, False) + self.drop_remainder = replace_none(drop_remainder, False) + + def parse(self, children=None): + return cde.BucketBatchByLengthNode(children[0], self.column_names, self.bucket_boundaries, + self.bucket_batch_sizes, self.element_length_function, self.pad_info, + self.pad_to_bucket_boundary, self.drop_remainder) + + +def _check_shm_usage(num_worker, queue_size, max_rowsize, num_queues=1): + """ + Check sufficient shared memory is available for shared memory queues + when training in parallel mode. + """ + threshold_ratio = 0.8 + if platform.system().lower() not in {"windows", "darwin"}: + device_num = _get_device_num() + # In the cluster, _get_device_num indicates the number of the entire cluster. The maximum number of cards + # on the ascend server is 8. + if device_num > 1 and context.get_context("device_target") == "Ascend": + device_num = min(device_num, 8) + shm_estimate_usage = device_num * num_worker * num_queues * \ + (queue_size + 2) * max_rowsize * 1024 * 1024 + try: + shm_available = psutil.disk_usage('/dev/shm').free + if shm_estimate_usage >= threshold_ratio * shm_available: + raise RuntimeError( + "Insufficient shared memory available. Required: {}, Available: {}. " + "The required memory can't exceed 80% of the available shared memory, " + "it's recommended to reduce memory usage by following methods:\n" + "1. reduce value of parameter max_rowsize or num_parallel_workers.\n" + "2. reduce prefetch size by set_prefetch_size().\n" + "3. disable shared memory by set_enable_shared_mem().".format(shm_estimate_usage, shm_available)) + except FileNotFoundError: + raise RuntimeError("Expected /dev/shm to exist.") + + +class BatchDataset(UnionBaseDataset): + """ + The result of applying Batch operator to the input dataset. + + Args: + input_dataset (Dataset): Input Dataset to be batched. + batch_size (Union[int, function]): The number of rows each batch is created with. An + int or callable which takes exactly 1 parameter, BatchInfo. + drop_remainder (bool, optional): Determines whether or not to drop the last + possibly incomplete batch (default=False). If True, and if there are less + than batch_size rows available to make the last batch, then those rows will + be dropped and not propagated to the child node. + num_parallel_workers (int, optional): Number of workers to process the dataset in parallel (default=None). + per_batch_map (callable, optional): Per batch map callable. A callable which takes + (list[Tensor], list[Tensor], ..., BatchInfo) as input parameters. Each list[Tensor] represents a batch of + Tensors on a given column. The number of lists should match with number of entries in input_columns. The + last parameter of the callable must always be a BatchInfo object. + input_columns (Union[str, list[str]], optional): List of names of the input columns. The size of the list must + match with signature of per_batch_map callable. + output_columns (Union[str, list[str]], optional): List of names assigned to the columns outputted by + the last operation. This parameter is mandatory if len(input_columns) != + len(output_columns). The size of this list must match the number of output + columns of the last operation. (default=None, output columns will have the same + name as the input columns, i.e., the columns will be replaced). + column_order (Union[str, list[str]], optional): Specifies the list of all the columns you need in the whole + dataset. The parameter is required when len(input_column) != len(output_column). Caution: the list here + is not just the columns specified in parameter input_columns and output_columns. + pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)} + will pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0. + max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy + data between processes. This is only used if python_multiprocessing is set to True (default=16). + + """ + + def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None, + input_columns=None, output_columns=None, column_order=None, pad_info=None, + python_multiprocessing=False, max_rowsize=16): + super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers) + + if BatchDataset._is_ancestor_of_repeat(input_dataset): + logger.warning("Repeat is located before batch, data from two epochs can be batched together.") + + BatchDataset._update_batch_size_for_syncwait(input_dataset, batch_size) + + # if batch_size is callable, set batch_size to 1 and batch_size_func to that callable function + self.batch_size = batch_size if not callable(batch_size) else 1 + self.batch_size_func = None if not callable(batch_size) else batch_size + + self.drop_remainder = replace_none(drop_remainder, False) + + self.per_batch_map = per_batch_map + + self.input_columns = to_list(input_columns) + self.output_columns = to_list(output_columns) + self.column_order = to_list(column_order) + + self.pad = bool(pad_info is not None) + self.pad_info = replace_none(pad_info, dict()) + + self.python_multiprocessing = python_multiprocessing + self.process_pool = None + self.max_rowsize = max_rowsize + + def __del__(self): + if hasattr(self, "process_pool") and self.process_pool is not None: + self.process_pool.terminate() + del self.process_pool + + def parse(self, children=None): + return cde.BatchNode(children[0], self.batch_size, self.drop_remainder, self.pad, self.input_columns, + self.output_columns, self.column_order, self.batch_size_func, self.per_batch_map, + self.pad_info, self.process_pool) + + @staticmethod + def _is_ancestor_of_repeat(dataset): + """ + Utility function to find the case where repeat is used before batch. + + Args: + dataset (Dataset): Dataset to be checked. + + Returns: + bool, whether repeat is used before batch. + """ + if isinstance(dataset, RepeatDataset): + return True + flag = False + for input_dataset in dataset.children: + flag = flag | BatchDataset._is_ancestor_of_repeat(input_dataset) + return flag + + @staticmethod + def _update_batch_size_for_syncwait(dataset, batch_size): + """ + Utility function to notify batch size to sync_wait. + + Args: + dataset (Dataset): Dataset to be checked. + batch_size (int): batch size to notify. + """ + if isinstance(dataset, SyncWaitDataset): + dataset.update_sync_batch_size(batch_size) + for input_dataset in dataset.children: + BatchDataset._update_batch_size_for_syncwait(input_dataset, batch_size) + + def __deepcopy__(self, memodict): + return self.__safe_deepcopy__(memodict, exclude=("per_batch_map", "batch_size_func", "__transfer_dataset__")) + + # Iterator bootstrap will be called on iterator construction. + # A deep copy of Dataset object is created prior of iterator_bootstrap. + # This method will create per iterator process pool and bind pyfunc execution to the pool. + def iterator_bootstrap(self): + """ + Per iterator bootstrap callback. + """ + if self.python_multiprocessing and platform.system().lower() == 'windows': + logger.warning("Python multiprocessing is not supported on Windows platform.") + if self.python_multiprocessing and platform.system().lower() != 'windows': + if self.per_batch_map is None: + logger.warning("per_batch_map is None so python_multiprocessing is ignored for batch.") + return + + # If user didn't specify num_parallel_workers, set it to default + if self.num_parallel_workers is None: + self.num_parallel_workers = get_num_parallel_workers() + + self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, [self.per_batch_map], + self.max_rowsize * self.batch_size) + # Wrap per_batch_map into _PythonCallable + self.per_batch_map = _PythonCallable(self.per_batch_map, 0, self.process_pool) + else: + if self.per_batch_map is not None: + self.per_batch_map = FuncWrapper(self.per_batch_map) + + +class BatchInfo(cde.CBatchInfo): + """ + Only the batch size function and per_batch_map of the batch operator can dynamically adjust parameters + based on the number of batches and epochs during training. + """ + + def get_batch_num(self): + """ + Return the batch number of the current batch. + """ + return + + def get_epoch_num(self): + """ + Return the epoch number of the current batch. + """ + return + + +class BlockReleasePair: + """ + The blocking condition class used by SyncWaitDataset. + + Args: + init_release_rows (int): Number of lines to allow through the pipeline. + callback (function): The callback function that will be called when release is called (default=None). + """ + + def __init__(self, init_release_rows, callback=None): + if isinstance(init_release_rows, int) and init_release_rows <= 0: + raise ValueError("release_rows need to be greater than 0.") + self.row_count = -init_release_rows + self.cv = threading.Condition() + self.callback = callback + self.default_rows = init_release_rows + self.disable = False + + def __deepcopy__(self, memodict): + return self + + def reset(self): + with self.cv: + self.row_count = -self.default_rows + self.cv.notify_all() + + def update_batched_size(self, batch_size): + # sanity check + if isinstance(batch_size, int) and batch_size <= 0: + raise ValueError("batch_size need to be greater than 0.") + + # should only use before the pipeline creates + self.row_count *= batch_size + self.default_rows *= batch_size + + def block_func(self): + """ + Function for handing blocking condition. + + Returns: + bool, True. + """ + with self.cv: + # if disable is true, the always evaluate to true + not_time_out = self.cv.wait_for(lambda: (self.row_count < 0 or self.disable), + timeout=get_callback_timeout()) + # time_out will be False if time out occurs + if not not_time_out: + logger.warning("Timeout happened in sync_wait, maybe dataset.sync_update(condition=...) " + "is not added after dataset.create_dict_iterator(...), now disabling lock.") + self.disable = True + self.row_count += 1 + return True + + def release_func(self, pass_rows=None, data=None): + with self.cv: + if pass_rows is None: + pass_rows = self.default_rows + self.row_count -= pass_rows + if self.callback is not None: + self.callback(data) + self.cv.notify_all() + + def disable_lock(self): + with self.cv: + self.disable = True + self.cv.notify_all() + + +class SyncWaitDataset(UnionBaseDataset): + """ + The result of adding a blocking condition to the input Dataset. + + Args: + input_dataset (Dataset): Input dataset to apply flow control. + num_batch (int): Number of batches without blocking at the start of each epoch. + condition_name (str): Condition name that is used to toggle sending next row. + callback (function): Callback function that will be invoked when sync_update is called (default=None). + + Raises: + RuntimeError: If condition name already exists. + """ + + def __init__(self, input_dataset, condition_name, num_batch, callback=None): + super().__init__(children=input_dataset) + + # set to the default value, waiting for the batch to update it + self._condition_name = condition_name + if isinstance(num_batch, int) and num_batch <= 0: + raise ValueError("num_batch need to be greater than 0.") + + self._pair = BlockReleasePair(num_batch, callback) + if self._condition_name in self.children[0].get_sync_notifiers(): + raise RuntimeError("Condition name is already in use.") + logger.info("Please remember to add dataset.sync_update(condition=%s), otherwise hanging will result. " + "If dataset.sync_update(condition=%s) has already been added, you can ignore the info.", + condition_name, condition_name) + + def parse(self, children=None): + return cde.SyncWaitNode(children[0], self._condition_name, self._pair.block_func) + + def get_sync_notifiers(self): + return {**self.children[0].get_sync_notifiers(), **{self._condition_name: self._pair.release_func}} + + def is_sync(self): + return True + + def update_sync_batch_size(self, batch_size): + if isinstance(batch_size, int) and batch_size <= 0: + raise ValueError("num_batch need to be greater than 0.") + self._pair.update_batched_size(batch_size) + + def disable_sync(self): + logger.info("Disabling Sync") + self._pair.disable_lock() + + @staticmethod + def _is_ancestor_of_batch(dataset): + """ + Utility function to find the case where sync_wait is used before batch. + + Args: + dataset (Dataset): Dataset to be checked. + + Returns: + bool, whether sync_wait is used before batch. + """ + if isinstance(dataset, BatchDataset): + return True + flag = False + for input_dataset in dataset.children: + flag = flag | SyncWaitDataset._is_ancestor_of_batch(input_dataset) + return flag + + def iterator_bootstrap(self): + self._pair.reset() + + +class ShuffleDataset(UnionBaseDataset): + """ + The result of applying Shuffle operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be shuffled. + buffer_size (int): Size of the buffer. + + Raises: + RuntimeError: If exist sync operators before shuffle. + """ + + def __init__(self, input_dataset, buffer_size): + super().__init__(children=input_dataset) + self.buffer_size = buffer_size + self.reshuffle_each_epoch = True + + if self.is_sync(): + raise RuntimeError("No shuffle after sync operators.") + + def parse(self, children=None): + return cde.ShuffleNode(children[0], self.buffer_size, self.reshuffle_each_epoch) + + def is_shuffled(self): + return True + + +# Pyfunc collection for multiprocess pyfunc +# This global variable will only be used within subprocesses +_OP_NAME = dict() +_OP_PROCESS = dict() + + +# PythonCallable wrapper for multiprocess pyfunc +class _PythonCallable: + """ + Internal Python function wrapper for multiprocessing pyfunc. + """ + + def __init__(self, py_callable, idx, pool=None): + # Original Python callable from user. + self.py_callable = py_callable + # Process pool created for current iterator. + self.pool = pool + # Python callable index + self.idx = idx + + def __call__(self, *args): + result = None + if self.pool.is_running() and check_iterator_cleanup() is False: + try: + result = self.pool.execute(self.idx, *args) + except multiprocessing.TimeoutError: + pass + if result is None: + # Invoke original Python callable in master process in case the pool is gone. + result = self.py_callable(*args) + return result + + def to_json(self): + return self.py_callable.to_json() + + +class Pipe: + """ + Class to handle communication between the master process and the worker processes. + """ + + def __init__(self, warning_ctl, shared_memory=False, max_rowsize=16): + self.shared_memory = shared_memory + self.eof = multiprocessing.Event() + if self.shared_memory: + self.in_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize) + self.res_queue = _SharedQueue(1, warning_ctl, max_rowsize=max_rowsize) + else: + self.in_queue = _Queue(1) + self.res_queue = _Queue(1) + self.in_queue._joincancelled = True # pylint: disable=W0212 + self.res_queue._joincancelled = True # pylint: disable=W0212 + + def master_send(self, func_index, data): + self.in_queue.put_nowait((func_index, *data)) + + def master_receive(self): + return self.res_queue.get_until(timeout=1, exit_signal=self.eof) + + def master_close(self): + self.eof.set() + self.res_queue.cancel_join_thread() + self.in_queue.cancel_join_thread() + + def worker_send(self, data): + self.res_queue.put_until(data, timeout=1, exit_signal=self.eof) + + def worker_receive(self): + result = self.in_queue.get_until(timeout=1, exit_signal=self.eof) + if result is None: + return result + if len(result) == 1: + raise RuntimeError(f"Corrupted data. Worker received {len(result)} elements, it should be more than 1.") + func_index, *data = result + return func_index, tuple(data) + + def worker_close(self): + self.res_queue.cancel_join_thread() + self.in_queue.cancel_join_thread() + + +def _main_process_already_exit(): + """ + Judge whether main process already exit. + """ + ppid = os.getppid() + + if (platform.system().lower() != 'windows' and + not _PythonMultiprocessing.is_process_alive(ppid)): + return True + return False + + +def _worker_loop(operations, pipe): + """ + Multiprocess worker process loop. + """ + + def _ignore_sigint(): + """ + We need to ignore sigint signal here so subprocesses can exit normally and clear. + """ + signal.signal(signal.SIGINT, signal.SIG_IGN) + + while not _main_process_already_exit(): + _ignore_sigint() + + result = pipe.worker_receive() + if result is None: + pipe.worker_close() + return + (idx, input_tensors) = result + try: + output_tensors = operations[idx](*input_tensors) + + pipe.worker_send(output_tensors) + except Exception: + pipe.worker_send(ExceptionHandler(where="in map(or batch) worker and execute Python function")) + return + + +def worker_target(operations): + return lambda pipe: _worker_loop(operations, pipe) + + +class _MPWorker(multiprocessing.Process): + """ + Worker process for multiprocessing. + """ + + def __init__(self, operations, warning_ctl, max_rowsize=16): + shared_memory = get_enable_shared_mem() + self.pipe = Pipe(warning_ctl, shared_memory=shared_memory, max_rowsize=max_rowsize) + super().__init__(target=worker_target(operations), args=(self.pipe,), daemon=True) + + def execute(self, idx, *args): + self.pipe.master_send(idx, args) + res = self.pipe.master_receive() + if isinstance(res, ExceptionHandler): + res.reraise() + return res + + def close(self): + try: + if self.is_alive(): + logger.info(f"Closing worker with PID: {self.pid}") + self.pipe.master_close() + super().terminate() + super().join() + super().close() + + except ValueError: + # Process has been closed already + return + return + + def is_alive(self): + try: + return super().is_alive() + except ValueError: + return False + + +class _PythonMultiprocessing(cde.PythonMultiprocessingRuntime): + """ + A wrapper to multiprocessing.pool that performs cleanup and ensure proper termination of forked processes. + """ + + class _ExceptHookHandler: + """ + Internal class ExceptionHandler + """ + + def __init__(self): + sys.excepthook = self.__handler_exception + + @staticmethod + def mp_pool_exit_preprocess(): + if check_iterator_cleanup() is False: + # Set the iterator_cleanup flag to True before exiting, and wait 3s for all apply_async + # applied to the multiprocessing task to prevent multiprocessing from hang when exiting + _set_iterator_cleanup() + time.sleep(3) + + def __handler_exception(self, ex_type, value, tb): + logger.critical("Uncaught exception: ", exc_info=(ex_type, value, tb)) + self.mp_pool_exit_preprocess() + + def __init__(self, op_name, num_parallel_workers, operations, max_row_size=16): + super(_PythonMultiprocessing, self).__init__() + self.op_name = op_name + self.num_parallel_workers = num_parallel_workers + self.operations = operations + self.max_row_size = max_row_size + + self.workers = None + self.pids = None + self.op_id = -1 + + self.queues_map = {} + self.next_queue = 0 + + self.eot = None + self.watch_dog = None + self.ppid = os.getpid() + self.hook = None + self.warning_ctl = None + self.threads_to_workers = {} + + def __del__(self): + try: + self.terminate() + except TypeError: + pass + + # This wait function is for cleaning zombie subprocesses + @staticmethod + def wait_pid(): + """ + This function is used by the main process to release subprocess resources. + """ + try: + while True: + child_pid, _ = os.waitpid(-1, os.WNOHANG) + if child_pid == 0: + break + except OSError: + # waitpid may be failed for some reasons so we ignore this error + pass + + # Dataset need watch_dog thread to monitoring fork multi-processing, + # and thread can't be a member function otherwise python won't collect and release resources. + @staticmethod + def _watch_dog(eot, workers): + """ + This thread is for monitoring subprocesses forked by GeneratorDataset/map/batch + """ + if not isinstance(workers, list): + raise TypeError("[Internal Error] The 2nd parameter of watch dog thread should be list of process, " + "but got {}.".format(type(workers))) + + while not eot.is_set(): + # Monitoring and count how many subprocesses already exit + clear_subprocess_timeout = _PythonMultiprocessing._monitor_subprocess_exit(workers) + # If find subprocess exit, we will wait for 30s and do some waitpid operations + if clear_subprocess_timeout > 0: + start = time.time() + while time.time() - start < clear_subprocess_timeout: + # We need to distinguishing get_dataset_size or train finished normally and hang scenario. + # If get_dataset_size or train finished normally, _stop_subprocess can be execute and + # self.need_abort can be set to True. If main process is hang in get(), self.need_abort + # will never set to True, then we wait for 30s and kill main process + if eot.is_set(): + return + # Sometimes subprocess may be zombie, so in 30s we can wait and do some useful tasks(waitpid). + _PythonMultiprocessing.wait_pid() + # multiprocessing.queue may hang in .get() forever when put() process was killed. + # We have to exit main process otherwise main process will hang. + _PythonMultiprocessing._terminate_processes(workers) + logger.critical("The subprocess of dataset may exit unexpected or be killed, " + "main process will exit. If this is not an artificial operation, you can use " + "ds.config.set_enable_watchdog(False) to block this error.") + os.kill(os.getpid(), signal.SIGTERM) + + @staticmethod + def _terminate_processes(processes): + """Terminate subprocesses""" + + for p in processes: + try: + if p.exitcode is None: + p.terminate() + except Exception: # pylint: disable=broad-except + # process has been closed already + pass + for p in processes: + if p._closed is False: # pylint: disable=W0212 + # We don't use w.join because join can only used in main process or join will raise an error. + p._popen.wait() # pylint: disable=W0212 + + # Monitor the exit number of subprocesses + @staticmethod + def _monitor_subprocess_exit(workers): + """ + To monitor whether process is exit. + + Args: + workers (list of multiprocessing.Process): multiprocessing.Process. + + Returns: + int, the timeout(in seconds) when process exit. + """ + for w in workers: + try: + exit_code = w.exitcode + if exit_code is not None: + # For kill -9, we can exit quickly + if exit_code == -9: + return 1 + # For kill -15, we still exit after 30s + if exit_code == -15: + return 30 + except ValueError: + # process has been closed already + return 0 + return 0 + + @staticmethod + def is_process_alive(pid): + """ + Check if the process is alive or not. + Note: We hit a deadlock when we use psutil or w.exitcode to check whether a process is alive. + Instead we use os.kill(ppid, 0). + + Args: + pid: pid of the process to be checked + + Returns: + True if the process is alive + """ + + try: + os.kill(pid, 0) + except OSError: + return False + return True + + # When main process exit, subprocesses will be terminate + @staticmethod + def _clean_process(ppid, workers): + """ + This is the execute function of clean process, if we found main process exited, we will clean subprocesses. + + Args: + ppid: The process id of main process. + workers: The list of subprocesses. + + """ + signal.signal(signal.SIGINT, signal.SIG_IGN) + while _PythonMultiprocessing.is_process_alive(ppid): + time.sleep(0.1) + + _PythonMultiprocessing._terminate_processes(workers) + os.kill(os.getpid(), signal.SIGTERM) + + def launch(self, op_id=-1): + self.op_id = op_id + logger.info("Launching new Python Multiprocessing pool for Op:" + str(self.op_id)) + self.create_pool() + + def create_pool(self): + """ + + Returns: + + """ + if get_enable_shared_mem(): + self.check_shared_memory() + + if self.workers is not None: + raise Exception("Pool was already created, close it first.") + + # Let gc collect unreferenced memory to avoid child processes in the pool to do it + gc.collect() + + # Construct python worker processes + self.workers = [] + self.warning_ctl = multiprocessing.Value('i', 0) + for _ in range(self.num_parallel_workers): + worker = _MPWorker(self.operations, self.warning_ctl, self.max_row_size) + worker.start() + self.workers.append(worker) + + logger.info("Op: " + str(self.op_id) + " Python multiprocessing pool workers' PIDs: " + str(self.get_pids())) + + self.hook = _PythonMultiprocessing._ExceptHookHandler() + + # The op (Map, Batch, etc) multiprocessing will launch a watch dog thread for monitoring sub processes + self._launch_watch_dog() + + atexit.register(self.terminate) + + def terminate(self): + logger.info("Terminating Python Multiprocessing for Op:" + str(self.op_id)) + self.close_all_workers() + self.abort_watchdog() + + def get_pids(self): + """ + Get list of worker's PIDs + + Returns: + list of strings + """ + if not self.is_mp_enabled: + return [] + if not self.pids: + self.pids = [] + if self.workers: + for w in self.workers: + try: + self.pids.append(w.pid) + except ValueError: + continue + return self.pids + + def add_new_workers(self, num_new_workers): + logger.info( + "Increasing num_parallel_workers of Python Multiprocessing pool for Op:" + str(self.op_id) + + ", old num_workers=" + str(self.num_parallel_workers) + " new num_workers=" + str( + self.num_parallel_workers + + num_new_workers) + ".") + self.terminate() + self.num_parallel_workers += num_new_workers + self.launch(self.op_id) + + def remove_workers(self, num_removed_workers): + logger.info( + "Decreasing num_parallel_workers of Python Multiprocessing pool for Op:" + str(self.op_id) + + ", old num_workers=" + str(self.num_parallel_workers) + " new num_workers=" + str( + self.num_parallel_workers - + num_removed_workers) + ".") + self.terminate() + self.num_parallel_workers -= num_removed_workers + self.launch(self.op_id) + + def is_mp_enabled(self): + return self.workers is not None + + def check_shared_memory(self): + """ + Check if there is enough shared memory in the system. + """ + _check_shm_usage(self.num_parallel_workers, 1, self.max_row_size, 2) + + def execute(self, idx, *args): + """ + Execute + """ + t_id = threading.get_ident() + worker_id = self.threads_to_workers.setdefault(t_id, len(self.threads_to_workers)) + + # todo check_iterator_cleanup + if self.is_running() and check_iterator_cleanup() is False: + return self.workers[worker_id].execute(idx, *args) + + return None + + def _launch_watch_dog(self): + """ + We will launch a watchdog thread and a clean process to cleaning subprocess when there is process was killed. + The watchdog thread will cleanup subprocesses and main process when one of the subprocesses was killed. + The cleaning subprocess will cleanup subprocesses when main process was killed. + """ + if platform.system().lower() != 'windows': + self.cleaning_process = multiprocessing.Process(target=self._clean_process, + args=(self.ppid, self.workers), + daemon=True) + self.cleaning_process.start() + + if get_enable_watchdog(): + self.eot = threading.Event() + self.watch_dog = threading.Thread(target=self._watch_dog, + args=(self.eot, self.workers + [self.cleaning_process]), + daemon=True) + self.watch_dog.start() + + def _abort_watchdog(self): + if not self.eot.is_set(): + self.eot.set() + + def abort_watchdog(self): + if hasattr(self, 'watch_dog') and self.watch_dog is not None and hasattr(self, 'eot') and self.eot is not None: + self._abort_watchdog() + if hasattr(self, 'cleaning_process') and self.cleaning_process is not None: + _PythonMultiprocessing._terminate_processes([self.cleaning_process]) + + def is_running(self): + if hasattr(self, 'workers') and self.workers is not None: + return all([w.is_alive() for w in self.workers]) + return False + + def close_all_workers(self): + if hasattr(self, 'workers') and self.workers is not None: + for w in self.workers: + w.close() + self.workers = None + self.pids = None + + +class MapDataset(UnionBaseDataset): + """ + The result of applying the Map operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be mapped. + operations (Union[list[TensorOperation], list[functions]]): A function mapping a nested structure of tensors + to another nested structure of tensor (default=None). + input_columns (Union[str, list[str]]): List of names of the input columns + (default=None, the operations will be applied on the first columns in the dataset). + The size of the list should match the number of inputs of the first operator. + output_columns (Union[str, list[str]], optional): List of names of the output columns. + The size of the list should match the number of outputs of the last operator + (default=None, output columns will be the input columns, i.e., the columns will + be replaced). + column_order (list[str], optional): Specifies the list of all the columns you need in the whole + dataset. The parameter is required when len(input_column) != len(output_column). Caution: the list here + is not just the columns specified in parameter input_columns and output_columns. + num_parallel_workers (int, optional): Number of workers to process the dataset + in parallel (default=None). + python_multiprocessing (bool, optional): Parallelize Python operations with multiple worker process. This + option could be beneficial if the Python operation is computational heavy (default=False). + cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing. + (default=None, which means no cache is used). + callbacks (DSCallback, list[DSCallback], optional): List of Dataset callbacks to be called (Default=None) + max_rowsize(int, optional): Maximum size of row in MB that is used for shared memory allocation to copy + data between processes. This is only used if python_multiprocessing is set to True (default=16). + offload (bool, optional): Flag to indicate whether offload is used (Default=None). + + Raises: + ValueError: If len(input_columns) != len(output_columns) and column_order is not specified. + """ + + def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None, column_order=None, + num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None, max_rowsize=16, + offload=None): + super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers, cache=cache) + self.operations = to_list(operations) + for op in self.operations: + # user define c_vision.HWC2CHW without parentheses is error + if type(op) == type: # pylint: disable=unidiomatic-typecheck + raise ValueError("Parameter operations's element of method map should be a dataset processing " + "operation instance, but got: {}. It may be missing parentheses for " + "instantiation.".format(op)) + if not isinstance(op, (c_transforms.TensorOperation, py_transforms.PyTensorOperation)) \ + and not callable(op): + raise ValueError("Parameter operations's element of method map should be a python function or " + "class method which should be callable, but got: {}. It doesn't need parentheses " + "for python function or class method.".format(op)) + + self.input_columns = to_list(input_columns) + self.output_columns = to_list(output_columns) + self.column_order = replace_none(column_order, []) + + # If output_columns were not provided then use input_columns + self.output_columns = self.input_columns if not self.output_columns else self.output_columns + + if self.input_columns and self.output_columns \ + and len(self.input_columns) != len(self.output_columns) \ + and not self.column_order: + raise ValueError("When length of input_columns and output_columns are not equal," + " column_order must be specified.") + + self.python_multiprocessing = python_multiprocessing + self.process_pool = None + + self.callbacks = to_list(callbacks) + self.max_rowsize = max_rowsize + self.offload = offload + + def parse(self, children=None): + operations = self.__decompose_callable_operations() + + count_old_transforms, count_new_transforms, count_non_data_vision_transforms = \ + self.__count_transforms(operations) + count_pyfunc = self.__count_pyfuncs(operations) + if count_new_transforms + count_pyfunc == len(operations): + prev_op = None + for op in operations: + if op.implementation is None: + if prev_op and prev_op.implementation == Implementation.PY: + op.implementation = Implementation.PY + else: + op.implementation = Implementation.C + prev_op = op + operations = transforms.transforms.Compose.reduce(operations) + elif count_old_transforms + count_pyfunc + count_non_data_vision_transforms == len(operations): + operations = transforms.py_transforms.Compose.reduce(operations) + else: + raise RuntimeError("Mixing old legacy c/py_transforms and new unified transforms is not allowed.") + + self.operations = self.__process_final_operations(operations) + self.prepare_multiprocessing() + + callbacks = [cb.create_runtime_obj() for cb in self.callbacks] + return cde.MapNode(children[0], self.operations, self.input_columns, self.output_columns, self.column_order, + callbacks, self.max_rowsize, OffloadToManualOffloadMode.get(self.offload), self.process_pool) + + def __deepcopy__(self, memodict): + return self.__safe_deepcopy__(memodict, exclude=("operations", "callbacks", "__transfer_dataset__")) + + def __del__(self): + if hasattr(self, "process_pool") and self.process_pool is not None: + self.process_pool.terminate() + del self.process_pool + + @staticmethod + def __count_pyfuncs(operations): + """ + Count the number of pyfuncs operations + """ + return sum([1 if isinstance(op, FuncWrapper) else 0 for op in operations]) + + @staticmethod + def __count_transforms(operations): + """ + Count the various flavors of transforms operations + """ + # Count the number of old legacy data and vision c_transforms and py_transforms + count_old_transforms = sum( + [1 if "c_transforms" in str(op) + or isinstance(op, (c_transforms.TensorOperation, py_transforms.PyTensorOperation)) + or ("py_transforms" in str(op) and not isinstance(op, FuncWrapper)) + else 0 for op in operations]) + # Count the number of new unified data and vision transforms + count_new_transforms = sum([1 if hasattr(op, "implementation") and not isinstance(op, FuncWrapper) + else 0 for op in operations]) + # Count the number of non-data transforms and non-vision transforms + count_non_data_vision_transforms = sum( + [1 if "text.transforms" in str(op) or "audio.transforms" in str(op) else 0 for op in operations]) + return count_old_transforms, count_new_transforms, count_non_data_vision_transforms + + @staticmethod + def __operation_valid_for_multiprocessing(op): + if callable(op) and str(op).find("c_transform") < 0: + return True + return False + + @staticmethod + def __process_final_operations(operations): + """ + Build final list of operations + """ + operations_fin = [] + for op in operations: + if hasattr(op, "implementation"): + if op.implementation == Implementation.C and not isinstance(op, (FuncWrapper, ToNumpy)): + operations_fin.append(op.parse()) + elif op.implementation == Implementation.PY: + operations_fin.append(op) + elif isinstance(op, (FuncWrapper, ToNumpy)): + operations_fin.append(op) + else: + raise RuntimeError("Wrong implementation") + else: + if op and getattr(op, 'parse', None): + operations_fin.append(op.parse()) + else: + operations_fin.append(op) + return operations_fin + + # Iterator bootstrap will be called on iterator construction. + # A deep copy of Dataset object is created prior of iterator_bootstrap. + # This method will create per iterator process pool and bind pyfunc execution to the pool. + def prepare_multiprocessing(self): + """ + Per iterator bootstrap callback. + """ + if self.python_multiprocessing and platform.system().lower() == 'windows': + logger.warning("Python multiprocessing is not supported on Windows platform.") + return + if self.python_multiprocessing: + iter_specific_operations = [] + callable_list = [] + + # If user didn't specify num_parallel_workers, set it to default + if self.num_parallel_workers is None: + self.num_parallel_workers = get_num_parallel_workers() + + # Pass #1, look for Python callables and build list + for op in self.operations: + # our c transforms is now callable and should not be run in Python multithreading + if MapDataset.__operation_valid_for_multiprocessing(op): + callable_list.append(op) + + if callable_list: + self.process_pool = _PythonMultiprocessing(str(self), self.num_parallel_workers, callable_list, + self.max_rowsize) + # Pass #2 + idx = 0 + for op in self.operations: + # our c transforms is now callable and should not be run in Python multithreading + if MapDataset.__operation_valid_for_multiprocessing(op): + # Wrap Python callable into _PythonCallable + iter_specific_operations.append(_PythonCallable(op, idx, self.process_pool)) + idx += 1 + else: + # CPP ops remain the same + iter_specific_operations.append(op) + self.operations = iter_specific_operations + + def __decompose_callable_operations(self): + """ + Decompose operations and build list of old legacy ops which are callable + """ + decomposed_operations = transforms.transforms.Compose.decompose(self.operations) + operations = [] + for op in decomposed_operations: + if callable(op) and not hasattr(op, "implementation") and str(op).find( + "c_transform") < 0 and not isinstance(op, c_transforms.TensorOperation) and \ + not isinstance(op, py_transforms.PyTensorOperation): + op = transforms.py_transforms_util.FuncWrapper(op) + operations.append(op) + return operations + + +class FilterDataset(UnionBaseDataset): + """ + The result of applying filter predicate to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be mapped. + predicate (callable): Python callable which returns a boolean value. If False then filter the element. + input_columns (Union[str, list[str]], optional): List of names of the input columns + (default=None, the predicate will be applied to all columns in the dataset). + num_parallel_workers (int, optional): Number of workers to process the dataset + in parallel (default=None). + """ + + def __init__(self, input_dataset, predicate, input_columns=None, num_parallel_workers=None): + super().__init__(children=input_dataset, num_parallel_workers=num_parallel_workers) + self.predicate = lambda *args: bool(predicate(*args)) + self.input_columns = to_list(input_columns) + + def parse(self, children=None): + return cde.FilterNode(children[0], self.predicate, self.input_columns) + + +class RepeatDataset(UnionBaseDataset): + """ + The result of applying Repeat operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be repeated. + count (int): Number of times the dataset will be repeated (default=-1, repeat indefinitely). + """ + + def __init__(self, input_dataset, count): + super().__init__(children=input_dataset) + self.count = replace_none(count, -1) + + def parse(self, children=None): + return cde.RepeatNode(children[0], self.count) + + +class SkipDataset(UnionBaseDataset): + """ + The result of applying Skip operator to the input Dataset. + + Args: + input_dataset (Dataset): Input dataset to have elements skipped. + count (int): Number of elements to be skipped in the dataset. + """ + + def __init__(self, input_dataset, count): + super().__init__(input_dataset) + self.count = count + + def parse(self, children=None): + return cde.SkipNode(children[0], self.count) + + +class TakeDataset(UnionBaseDataset): + """ + The result of applying Take operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to have elements taken from. + count (int): Number of elements to be taken from the dataset. + """ + + def __init__(self, input_dataset, count): + super().__init__(children=input_dataset) + self.count = count + + def parse(self, children=None): + return cde.TakeNode(children[0], self.count) + + +class ZipDataset(UnionBaseDataset): + """ + The result of applying Zip operator to the input Dataset. + + Args: + datasets (tuple): A tuple of datasets to be zipped together. + + Raises: + TypeError: If dataset is not an instance of Dataset. + """ + + def __init__(self, datasets): + super().__init__(children=datasets) + + def parse(self, children=None): + return cde.ZipNode(children) + + def is_sync(self): + return any([c.is_sync() for c in self.children]) + + +class ConcatDataset(UnionBaseDataset): + """ + The result of applying concat dataset operator to the input Dataset. + + Args: + datasets (list): A list of datasets to be concatenated together. + + Raises: + TypeError: If dataset is not an instance of Dataset. + ValueError: If there is no samples in the one of the datasets. + """ + + def __init__(self, datasets): + super().__init__(children=datasets) + for dataset in datasets: + if not isinstance(dataset, Dataset): + raise TypeError("Invalid dataset, expected Dataset object, but got %s!" % type(dataset)) + self.datasets = datasets + self._sampler = samplers.SequentialSampler(num_samples=None) + + self.children_sizes_ = [c.get_dataset_size() for c in self.children] + child_index = 0 + for item in self.children_sizes_: + if item == 0: + raise ValueError("There are no samples in the dataset number %d. Please make sure there are " + "valid samples in the dataset." % child_index) + child_index += 1 + + # _children_flag_and_nums: A list of pair.The first element of pair is flag that characterizes + # whether the dataset is mappable. The second element of pair is length of the dataset + self._children_flag_and_nums = [] + + # _children_start_end_index_: A list of pair.The elements of pair are used to characterize + # the valid position of the dataset corresponding to the subscript when sampling + self._children_start_end_index_ = [] + for index, child in enumerate(self.children): + tem_list = [-1, -1] + self._children_start_end_index_.append(tem_list) + dataset_len = self.children_sizes_[index] + + from mindspore.dataset.engine.datasets_user_defined import GeneratorDataset + if isinstance(child, GeneratorDataset) and not hasattr(child.source, "__getitem__"): + dataset_len = 0 + self.children_sizes_[index] = 0 + + if isinstance(child, MappableDataset): + self._children_flag_and_nums.append((0, dataset_len)) + else: + self._children_flag_and_nums.append((1, dataset_len)) + + def parse(self, children=None): + return cde.ConcatNode(children, self._sampler, self._children_flag_and_nums, self._children_start_end_index_) + + def use_sampler(self, sampler): + """ + Set the distributedSampler to concat dataset + + Args: + sampler (Sampler): The sampler to use for the current dataset. + Currently supported: DistributedSampler. + + Raises: + TypeError: If the sampler is not an instance of DistributedSampler + ValueError: If the parameter shuffle of sampler is True + ValueError: If the parameter NumSamples of sampler is not None. + ValueError: If num_shards <=0. + """ + if not isinstance(sampler, samplers.DistributedSampler): + raise TypeError("The parameter %s of concat must be DistributedSampler!" % sampler) + + if sampler.is_shuffled(): + raise ValueError("The parameter shuffle of DistributedSampler must be False!") + + if sampler.num_shards <= 0: + raise ValueError("The parameter num_shards of DistributedSampler must be positive int!") + + if sampler.get_num_samples() is not None: + raise ValueError("The parameter num_samples of DistributedSampler is not support to be set!") + + self.dataset_size = None + + self._sampler = sampler + cumulative_samples_nums = 0 + for index, child in enumerate(self.children): + if hasattr(child, 'sampler') and child.sampler.get_num_samples() is not None: + raise ValueError("The parameter NumSamples of %s is not support to be set!" % child) + + if isinstance(child, BatchDataset): + raise TypeError("The parameter %s of concat must not be BatchDataset!" % child) + + # if child is mappable and the length is greater than 0 + if not self._children_flag_and_nums[index][0] and self._children_flag_and_nums[index][1]: + + tem_value = cumulative_samples_nums + self._children_flag_and_nums[index][1] + + if not self._children_flag_and_nums[index][1] >= sampler.num_shards: + if tem_value < sampler.num_shards: + self._children_start_end_index_[index][0] = cumulative_samples_nums + self._children_start_end_index_[index][1] = tem_value + else: + self._children_start_end_index_[index][0] = cumulative_samples_nums + self._children_start_end_index_[index][1] = tem_value % sampler.num_shards + + tem_sampler = copy.deepcopy(sampler) + tem_sampler.set_offset(cumulative_samples_nums) + child.use_sampler(tem_sampler) + + cumulative_samples_nums += self.children_sizes_[index] + cumulative_samples_nums %= sampler.num_shards + + +class RenameDataset(UnionBaseDataset): + """ + The result of applying Rename operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be Renamed. + input_columns (Union[str, list[str]]): List of names of the input columns. + output_columns (Union[str, list[str]]): List of names of the output columns. + """ + + def __init__(self, input_dataset, input_columns, output_columns): + super().__init__(children=input_dataset) + self.input_column_names = to_list(input_columns) + self.output_column_names = to_list(output_columns) + + def parse(self, children=None): + return cde.RenameNode(children[0], self.input_column_names, self.output_column_names) + + +def to_list(items): + if items is None: + return [] + if isinstance(items, tuple): + return list(items) + if not isinstance(items, list): + return [items] + return items + + +class ProjectDataset(UnionBaseDataset): + """ + The result of applying Project operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be Projected. + columns (Union[str, list[str]]): List of names of the columns to project. + """ + + def __init__(self, input_dataset, columns): + super().__init__(children=input_dataset) + self.columns = to_list(columns) + + def parse(self, children=None): + return cde.ProjectNode(children[0], self.columns) + + +class _ToDevice: + """ + Internal class to handle sending data to device. + """ + + def __init__(self, dataset, num_epochs): + ir_tree, self.api_tree = dataset.create_ir_tree() + + self._runtime_context = cde.PythonRuntimeContext() + self._runtime_context.Init() + self._to_device = cde.ToDevice(num_epochs) + self._to_device.Init(ir_tree) + self._runtime_context.AssignConsumer(self._to_device) + + ITERATORS_LIST.append(weakref.ref(self)) + _unset_iterator_cleanup() + + def send(self): + self._to_device.Send() + + def _reset(self, step): + self._to_device.Reset(step) + + def stop_send(self): + """ + send stop send signal to pipeline, it is used when end of sequence is sent at the epoch end. + """ + self._to_device.StopSend() + + def continue_send(self): + """ + send continue send signal to pipeline, it is used when end of sequence is sent at the epoch end. + """ + self._to_device.ContinueSend() + + def get_data_info(self): + """ + Get type and shape of current batch. + """ + return self._to_device.GetDataInfo() + + def release(self): + """ + Manually terminate Device Queue instead of relying on out of scope destruction. + """ + if hasattr(self, '_runtime_context') and self._runtime_context: + if hasattr(self, '_to_device') and self._to_device: + self._runtime_context.Terminate() + del self._to_device + del self._runtime_context + + def __deepcopy__(self, memodict): + return self + + def get_offload_model(self, col_names): + """ + Get offload model containing removed offload ops from pipeline. + """ + offload_model = GetOffloadModel(self._to_device, col_names) + return offload_model + + +class TransferDataset(Dataset): + """ + The result of applying TDT operator to the input Dataset. + + Args: + input_dataset (Dataset): Input Dataset to be transferred. + send_epoch_end (bool, optional): Whether to send end of sequence to device or not (default=True). + create_data_info_queue (bool, optional): Whether to create queue which stores + types and shapes of data or not (default=False). + + Raises: + TypeError: If device_type is empty. + ValueError: If device_type is not 'Ascend', 'GPU' or 'CPU'. + RuntimeError: If dataset is unknown. + """ + + def __init__(self, input_dataset, send_epoch_end=True, create_data_info_queue=False): + super().__init__(children=input_dataset) + self.queue_name = str(uuid.uuid1()) + self.device_type = context.get_context("device_target") if context else "CPU" + self.device_id = context.get_context("device_id") if context else 0 + + self._send_epoch_end = replace_none(send_epoch_end, True) + self._create_data_info_queue = create_data_info_queue + self._to_device = None + self.column_name = input_dataset.get_col_names() + + def parse(self, children=None): + total_batch = 0 + if hasattr(self.children[0], "__total_batch__"): + total_batch = self.children[0].__total_batch__ + return cde.TransferNode(children[0], self.queue_name, self.device_type, self.device_id, self._send_epoch_end, + total_batch, self._create_data_info_queue) + + def create_dict_iterator(self, num_epochs=-1, output_numpy=False): + raise RuntimeError("TransferDataset is not iterable.") + + def create_tuple_iterator(self, columns=None, num_epochs=-1, output_numpy=False, do_copy=True): + raise RuntimeError("TransferDataset is not iterable.") + + def __iter__(self): + raise RuntimeError("TransferDataset is not iterable.") + + def output_shapes(self): + raise RuntimeError("TransferDataset does not support obtaining output_shapes.") + + def output_types(self): + raise RuntimeError("TransferDataset does not support obtaining output_types.") + + @check_to_device_send + def send(self, num_epochs=-1): + """ + Send to device + """ + if Dataset._noop_mode(): + return + if self._to_device is not None: + del self._to_device + self._to_device = _ToDevice(self, num_epochs) + self._to_device.send() + + def stop_send(self): + if self._to_device is not None: + self._to_device.stop_send() + + def continue_send(self): + if self._to_device is not None: + self._to_device.continue_send() + + def _reset(self, step): + if self._to_device is not None: + logger.info("Reset the dataset pipeline to step " + str(step)) + self._to_device._reset(step) # pylint: disable=W0212 + + def get_data_info(self): + """ + Get type and shape of current batch + """ + if self._to_device is not None: + return self._to_device.get_data_info() + raise RuntimeError("Calling get_data_info with bad state.") + + def get_offload_model(self): + if self._to_device is not None: + return self._to_device.get_offload_model(self.column_name) + + raise RuntimeError("get_offload_model, _to_device is None") + + def release(self): + """ + Manually terminate Device Queue instead of relying on out of scope destruction. + """ + if self._to_device is not None: + self._to_device.release() + + +class Schema: + """ + Class to represent a schema of a dataset. + + Args: + schema_file(str): Path of the schema file (default=None). + + Returns: + Schema object, schema info about dataset. + + Raises: + RuntimeError: If schema file failed to load. + + Examples: + >>> from mindspore import dtype as mstype + >>> + >>> # Create schema; specify column name, mindspore.dtype and shape of the column + >>> schema = ds.Schema() + >>> schema.add_column(name='col1', de_type=mstype.int64, shape=[2]) + """ + + @check_schema + def __init__(self, schema_file=None): + self.schema_file = replace_none(schema_file, "") + self.cpp_schema = cde.SchemaObj(self.schema_file) + + @check_add_column + def add_column(self, name, de_type, shape=None): + """ + Add new column to the schema. + + Args: + name (str): The new name of the column. + de_type (str): Data type of the column. + shape (list[int], optional): Shape of the column + (default=None, [-1] which is an unknown shape of rank 1). + + Raises: + ValueError: If column type is unknown. + """ + if isinstance(de_type, typing.Type): + de_type = mstype_to_detype(de_type) + col_type = str(de_type) + else: + col_type = str(cde.DataType(de_type)) + if shape is None: + self.cpp_schema.add_column(name, col_type) + else: + self.cpp_schema.add_column(name, col_type, shape) + + def parse_columns(self, columns): + """ + Parse the columns and add it to self. + + Args: + columns (Union[dict, list[dict], tuple[dict]]): Dataset attribute information, decoded from schema file. + + - list[dict], `name` and `type` must be in keys, `shape` optional. + + - dict, columns.keys() as name, columns.values() is dict, and `type` inside, `shape` optional. + + Raises: + RuntimeError: If failed to parse columns. + RuntimeError: If column's name field is missing. + RuntimeError: If column's type field is missing. + + Examples: + >>> from mindspore.dataset import Schema + >>> schema = Schema() + >>> columns1 = [{'name': 'image', 'type': 'int8', 'shape': [3, 3]}, + ... {'name': 'label', 'type': 'int8', 'shape': [1]}] + >>> schema.parse_columns(columns1) + >>> columns2 = {'image': {'shape': [3, 3], 'type': 'int8'}, 'label': {'shape': [1], 'type': 'int8'}} + >>> schema.parse_columns(columns2) + """ + self.cpp_schema.parse_columns(json.dumps(columns, indent=2)) + + def to_json(self): + """ + Get a JSON string of the schema. + + Returns: + str, JSON string of the schema. + """ + return self.cpp_schema.to_json() + + def from_json(self, json_obj): + """ + Get schema file from JSON object. + + Args: + json_obj(dictionary): Object of JSON parsed. + + Raises: + RuntimeError: if there is unknown item in the object. + RuntimeError: if dataset type is missing in the object. + RuntimeError: if columns are missing in the object. + """ + self.cpp_schema.from_string(json.dumps(json_obj, indent=2)) + + def __str__(self): + return self.to_json() + + @staticmethod + def get_num_rows(schema): + schema_obj = schema + if not isinstance(schema_obj, Schema): + schema_obj = Schema(schema_obj) + return schema_obj.cpp_schema.get_num_rows() + + +class DeserializedDataset(Dataset): + def __init__(self, input_obj): + super().__init__() + self.input_obj = input_obj + + def parse(self, children=None): + if isinstance(self.input_obj, dict): + json_str = json.dumps(self.input_obj) + return cde.Dataset.from_json_string(json_str) + return cde.Dataset.from_json_file(self.input_obj) -- 2.34.1 From d77a6bc0e3a5a7246a50826bc9e36c24320a695a Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:25:17 +0800 Subject: [PATCH 70/72] ADD file via upload --- .../ccsrc/transform-update/test/transforms.py | 1262 +++++++++++++++++ 1 file changed, 1262 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/test/transforms.py diff --git a/mindspore/ccsrc/transform-update/test/transforms.py b/mindspore/ccsrc/transform-update/test/transforms.py new file mode 100644 index 00000000000..f053caeae29 --- /dev/null +++ b/mindspore/ccsrc/transform-update/test/transforms.py @@ -0,0 +1,1262 @@ +# Copyright 2020-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +The module text.transforms is inherited from _c_dataengine +and is implemented based on ICU4C and cppjieba in C++. +It's a high performance module to process NLP text. +Users can use Vocab to build their own dictionary, +use appropriate tokenizers to split sentences into different tokens, +and use Lookup to find the index of tokens in Vocab. +模块text.transforms继承自_c_dataengine并基于ICU4C和cppjieba在C++中实现。 +它是一个处理NLP文本的高性能模块。 +用户可以使用Vocab构建自己的词典,使用适当的标记器将句子分割成不同的标记,并使用Lookup查找Vocab中的令牌索引。 +.. Note:: + A constructor's arguments for every class in this module must be saved into the + class attributes (self.xxx) to support save() and load(). + +Examples: + >>> text_file_dataset_dir = ["/path/to/text_file_dataset_file"] # contains 1 or multiple text files + >>> # Create a dataset for text sentences saved as line data in a file + >>> text_file_dataset = ds.TextFileDataset(dataset_files=text_file_dataset_dir, shuffle=False) + >>> # Tokenize sentences to unicode characters + >>> tokenizer = text.UnicodeCharTokenizer() + >>> # Load vocabulary from list + >>> vocab = text.Vocab.from_list(word_list=['深', '圳', '欢', '迎', '您']) + >>> # Use Lookup operator to map tokens to ids + >>> lookup = text.Lookup(vocab=vocab) + >>> text_file_dataset = text_file_dataset.map(operations=[tokenizer, lookup]) + >>> # if text line in dataset_file is: + >>> # 深圳欢迎您 + >>> # then the output will be: + >>> # {'text': array([0, 1, 2, 3, 4], dtype=int32)} +""" +import json +import os +import re +import platform +import numpy as np + +import mindspore._c_dataengine as cde +from mindspore.common import dtype as mstype + +from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType, SentencePieceVocab +from .validators import check_lookup, check_jieba_add_dict, check_to_vectors, \ + check_jieba_add_word, check_jieba_init, check_with_offsets, check_unicode_script_tokenizer, \ + check_wordpiece_tokenizer, check_regex_replace, check_regex_tokenizer, check_basic_tokenizer, check_ngram, \ + check_pair_truncate, check_to_number, check_bert_tokenizer, check_python_tokenizer, check_slidingwindow, \ + check_sentence_piece_tokenizer +from ..core.datatypes import mstype_to_detype +from ..core.validator_helpers import replace_none +from ..transforms.py_transforms_util import Implementation +from ..transforms.transforms import TensorOperation +from ..transforms.validators import invalidate_callable + + +class TextTensorOperation(TensorOperation): + """ + Base class of Text Tensor Ops + 文本张量运算的基类 + """ + + def __init__(self): + super().__init__() + self.implementation = Implementation.C + + def parse(self): + raise NotImplementedError("TextTensorOperation has to implement parse() method.") + + +DE_C_INTER_JIEBA_MODE = { + JiebaMode.MIX: cde.JiebaMode.DE_JIEBA_MIX, + JiebaMode.MP: cde.JiebaMode.DE_JIEBA_MP, + JiebaMode.HMM: cde.JiebaMode.DE_JIEBA_HMM +} + +DE_C_INTER_SENTENCEPIECE_LOADTYPE = { + SPieceTokenizerLoadType.FILE: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KFILE, + SPieceTokenizerLoadType.MODEL: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KMODEL +} + +DE_C_INTER_SENTENCEPIECE_OUTTYPE = { + SPieceTokenizerOutType.STRING: cde.SPieceTokenizerOutType.DE_SPIECE_TOKENIZER_OUTTYPE_KString, + SPieceTokenizerOutType.INT: cde.SPieceTokenizerOutType.DE_SPIECE_TOKENIZER_OUTTYPE_KINT +} + + +class JiebaTokenizer(TextTensorOperation): + """ + Tokenize Chinese string into words based on dictionary. + 根据字典将中文字符串标记为单词。 + Note: + The integrity of the HMMSEgment algorithm and MPSegment algorithm files must be confirmed. + + Args: + hmm_path (str): Dictionary file is used by HMMSegment algorithm. + The dictionary can be obtained on the official website of cppjieba. + mp_path (str): Dictionary file is used by MPSegment algorithm. + The dictionary can be obtained on the official website of cppjieba. + mode (JiebaMode, optional): Valid values can be any of [JiebaMode.MP, JiebaMode.HMM, + JiebaMode.MIX](default=JiebaMode.MIX). + + - JiebaMode.MP, tokenize with MPSegment algorithm. + - JiebaMode.HMM, tokenize with Hidden Markov Model Segment algorithm. + - JiebaMode.MIX, tokenize with a mix of MPSegment and HMMSegment algorithm. + with_offsets (bool, optional): Whether or not output offsets of tokens (default=False). + + Raises: + ValueError: If path of HMMSegment dict is not provided. + ValueError: If path of MPSegment dict is not provided. + TypeError: If `hmm_path` or `mp_path` is not of type string. + TypeError: If `with_offsets` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.text import JiebaMode + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> jieba_hmm_file = "/path/to/jieba/hmm/file" + >>> jieba_mp_file = "/path/to/jieba/mp/file" + >>> tokenizer_op = text.JiebaTokenizer(jieba_hmm_file, jieba_mp_file, mode=JiebaMode.MP, with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=False, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.JiebaTokenizer(jieba_hmm_file, jieba_mp_file, mode=JiebaMode.MP, with_offsets=True) + >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", "offsets_limit"], + ... column_order=["token", "offsets_start", "offsets_limit"]) + """ + + @check_jieba_init + def __init__(self, hmm_path, mp_path, mode=JiebaMode.MIX, with_offsets=False): + super().__init__() + if not isinstance(mode, JiebaMode): + raise TypeError("Wrong input type for mode, should be JiebaMode.") + + self.mode = mode + self.__check_path__(hmm_path) + self.hmm_path = hmm_path + self.__check_path__(mp_path) + self.mp_path = mp_path + self.with_offsets = with_offsets + self.words = [] + + # 定义一个名为__check_path__的私有方法 +def __check_path__(self, model_path): + """检查模型路径是否存在""" + # 使用os.path.exists()检查模型路径是否存在 + if not os.path.exists(os.path.realpath(model_path)): + # 如果模型路径不存在,抛出ValueError异常并提供相应错误消息 + raise ValueError(" jieba mode file {} is not exist.".format(model_path)) + +# 定义一个名为parse的方法 +def parse(self): + # 创建一个cde.JiebaTokenizerOperation对象,用于中文分词操作,使用以下参数进行初始化: + # - self.hmm_path: HMM模型文件的路径,用于分词的隐马尔可夫模型 + # - self.mp_path: MP模型文件的路径,用于分词的最大概率模型 + # - DE_C_INTER_JIEBA_MODE.get(self.mode): 使用self.mode作为键从DE_C_INTER_JIEBA_MODE字典中获取对应的值, + # 用于指定jieba分词的模式 + # - self.with_offsets: 是否返回分词的偏移量信息 + jieba_tokenizer = cde.JiebaTokenizerOperation(self.hmm_path, self.mp_path, + DE_C_INTER_JIEBA_MODE.get(self.mode), + self.with_offsets) + + # 遍历self.words中的词语列表,将每个词语添加到分词器中 + for word in self.words: + jieba_tokenizer.add_word(word[0], word[1]) + + # 返回已配置的jieba分词器对象 + return jieba_tokenizer + + + @invalidate_callable + @check_jieba_add_word + def add_word(self, word, freq=None): + """ + Add a user defined word to JiebaTokenizer's dictionary. + 将用户定义的单词添加到JiebaTokenizer的字典中。 + Args: + word (str): The word to be added to the JiebaTokenizer instance. + The added word will not be written into the built-in dictionary on disk. + freq (int, optional): The frequency of the word to be added. The higher the frequency, + the better chance the word will be tokenized (default=None, use default frequency). + + Examples: + >>> from mindspore.dataset.text import JiebaMode + >>> jieba_hmm_file = "/path/to/jieba/hmm/file" + >>> jieba_mp_file = "/path/to/jieba/mp/file" + >>> jieba_op = text.JiebaTokenizer(jieba_hmm_file, jieba_mp_file, mode=JiebaMode.MP) + >>> sentence_piece_vocab_file = "/path/to/sentence/piece/vocab/file" + >>> with open(sentence_piece_vocab_file, 'r') as f: + ... for line in f: + ... word = line.split(',')[0] + ... jieba_op.add_word(word) + >>> text_file_dataset = text_file_dataset.map(operations=jieba_op, input_columns=["text"]) + """ + + if freq is None: + self.words.append((word, 0)) + else: + self.words.append((word, freq)) + + @invalidate_callable + @check_jieba_add_dict + def add_dict(self, user_dict): + """ + Add a user defined word to JiebaTokenizer's dictionary. + 将用户定义的单词添加到JiebaTokenizer的字典中。 + Args: + user_dict (Union[str, dict]): One of the two loading methods is file path(str) loading + (according to the Jieba dictionary format) and the other is Python dictionary(dict) loading, + Python Dict format: {word1:freq1, word2:freq2,...}. + Jieba dictionary format : word(required), freq(optional), such as: + + .. code-block:: + + word1 freq1 + word2 None + word3 freq3 + + Only valid word-freq pairs in user provided file will be added into the dictionary. + Rows containing invalid input will be ignored. No error nor warning Status is returned. + + Examples: + >>> from mindspore.dataset.text import JiebaMode + >>> jieba_hmm_file = "/path/to/jieba/hmm/file" + >>> jieba_mp_file = "/path/to/jieba/mp/file" + >>> user_dict = {"男默女泪": 10} + >>> jieba_op = text.JiebaTokenizer(jieba_hmm_file, jieba_mp_file, mode=JiebaMode.MP) + >>> jieba_op.add_dict(user_dict) + >>> text_file_dataset = text_file_dataset.map(operations=jieba_op, input_columns=["text"]) + """ + + + # 检查user_dict的数据类型 + if isinstance(user_dict, str): + # 如果user_dict是字符串类型,调用__add_dict_py_file方法,该方法用于添加Python词典文件 + self.__add_dict_py_file(user_dict) + elif isinstance(user_dict, dict): + # 如果user_dict是字典类型,遍历字典中的键值对,然后调用add_word方法添加每个词语及其对应的词性 + for k, v in user_dict.items(): + self.add_word(k, v) + else: + # 如果user_dict的数据类型不是字符串或字典,抛出TypeError异常,并提供相应错误消息 + raise TypeError("The type of user_dict must be str or dict.") + + + # 定义一个名为__add_dict_py_file的私有方法,用于从文件中添加用户自定义词典 +def __add_dict_py_file(self, file_path): + """通过文件添加用户自定义词典""" + # 调用__parser_file方法解析词典文件并返回词语列表 + words_list = self.__parser_file(file_path) + + # 遍历词语列表,将每个词语及其频率添加到分词器中 + for data in words_list: + if data[1] is None: + freq = 0 + else: + freq = int(data[1]) + self.add_word(data[0], freq) + +# 定义一个名为__decode的私有方法,用于将词典文件解码为UTF-8格式 +def __decode(self, data): + """将词典文件解码为UTF-8格式""" + try: + data = data.decode('utf-8') + except UnicodeDecodeError: + # 如果解码失败,抛出ValueError异常并提供相应错误消息 + raise ValueError("user dict file must be utf8 format.") + return data.lstrip('\ufeff') + +# 定义一个名为__parser_file的私有方法,用于解析用户自定义词典文件 +def __parser_file(self, file_path): + """解析用户自定义词典文件""" + # 检查词典文件是否存在 + if not os.path.exists(file_path): + # 如果词典文件不存在,抛出ValueError异常并提供相应错误消息 + raise ValueError("user dict file {} is not exist.".format(file_path)) + + # 获取词典文件的绝对路径 + real_file_path = os.path.realpath(file_path) + + # 打开词典文件以进行读取 + file_dict = open(real_file_path) + + # 使用正则表达式定义数据匹配模式,匹配词语和频率 + data_re = re.compile('^\\s*([^\\s*]+?)\\s*([0-9]+)?\\s*$', re.U) + + # 初始化词语列表 + words_list = [] + + # 逐行遍历词典文件 + for item in file_dict: + data = item.strip() + if not isinstance(data, str): + # 解码数据为UTF-8格式 + data = self.__decode(data) + # 使用正则表达式匹配数据 + tmp = data_re.match(data) + if not tmp: + continue + # 获取匹配的词语和频率,添加到词语列表中 + words = tmp.groups() + words_list.append(words) + + # 关闭词典文件 + file_dict.close() + + # 返回词语列表 + return words_list + + + +class Lookup(TextTensorOperation): + """ + Look up a word into an id according to the input vocabulary table. + + Args: + vocab (Vocab): A vocabulary object. + unknown_token (str, optional): Word is used for lookup. In case of the word is out of vocabulary (OOV), + the result of lookup will be replaced with unknown_token. If the unknown_token is not specified or + it is OOV, runtime error will be thrown (default=None, means no unknown_token is specified). + data_type (mindspore.dtype, optional): The data type that lookup operation maps + string to(default=mindspore.int32). + + Raises: + TypeError: If `vocab` is not of type text.Vocab. + TypeError: If `unknown_token` is not of type string. + TypeError: If `data_type` is not of type mindspore.dtype. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Load vocabulary from list + >>> vocab = text.Vocab.from_list(['深', '圳', '欢', '迎', '您']) + >>> # Use Lookup operator to map tokens to ids + >>> lookup = text.Lookup(vocab) + >>> text_file_dataset = text_file_dataset.map(operations=[lookup]) + """ + + @check_lookup + def __init__(self, vocab, unknown_token=None, data_type=mstype.int32): + super().__init__() + # 初始化Lookup对象的属性 + # - vocab: 词汇表对象,用于查找单词的索引 + # - unknown_token: 未知词汇的标记,当查询的单词不在词汇表中时使用 + # - data_type: 数据类型,用于指定查找操作的输出数据类型,默认为32位整数(mstype.int32) + self.vocab = vocab + self.unknown_token = unknown_token + self.data_type = data_type + + def parse(self): + # 返回一个cde.LookupOperation对象,该对象使用以下参数进行初始化: + # - self.vocab.c_vocab: 词汇表对象的C API表示 + # - self.unknown_token: 未知词汇的标记 + # - str(mstype_to_detype(self.data_type)): 输出数据类型的字符串表示 + return cde.LookupOperation(self.vocab.c_vocab, self.unknown_token, str(mstype_to_detype(self.data_type))) + + + +class Ngram(TextTensorOperation): + """ + Generate n-gram from a 1-D string Tensor. + 从一维字符串张量生成n-gram。 + Refer to https://en.wikipedia.org/wiki/N-gram#Examples for an overview of what n-gram is and how it works. + + Args: + n (list[int]): n in n-gram, which is a list of positive integers. For example, if n=[4, 3], then the result + would be a 4-gram followed by a 3-gram in the same tensor. If the number of words is not enough to make up + for a n-gram, an empty string will be returned. For example, 3 grams on ["mindspore", "best"] will result in + an empty string produced. + left_pad (tuple, optional): Padding performed on left side of the sequence shaped like ("pad_token", pad_width). + `pad_width` will be capped at n-1. For example, specifying left_pad=("_", 2) would pad left side of the + sequence with "__" (default=("", 0)). + right_pad (tuple, optional): Padding performed on right side of the sequence shaped like + ("pad_token", pad_width). `pad_width` will be capped at n-1. For example, specifying right_pad=("_", 2) + would pad right side of the sequence with "__" (default=("", 0)). + separator (str, optional): Symbol used to join strings together. For example, if 2-gram is + ["mindspore", "amazing"] with separator="-", the result would be ["mindspore-amazing"] + (default=" ", which will use whitespace as separator). + + Raises: + TypeError: If values of `n` not positive is not of type int. + ValueError: If values of `n` not positive. + ValueError: If `left_pad` is not a tuple of length 2. + ValueError: If `right_pad` is not a tuple of length 2. + TypeError: If `separator` is not of type string. + + Supported Platforms: + ``CPU`` + + Examples: + >>> ngram_op = text.Ngram(3, separator="-") + >>> output = ngram_op(["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"]) + >>> # output + >>> # ["WildRose Country-Canada's Ocean Playground-Land of Living Skies"] + >>> # same ngram_op called through map + >>> text_file_dataset = text_file_dataset.map(operations=ngram_op) + """ + + @check_ngram + def __init__(self, n, left_pad=("", 0), right_pad=("", 0), separator=" "): + super().__init__() + self.ngrams = n + self.left_pad = left_pad + self.right_pad = right_pad + self.separator = separator + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.NgramOperation对象,该对象使用以下参数进行初始化: + # - self.ngrams: N-gram的n值,用于指定要生成的N-gram的长度 + # - self.left_pad: 是否在文本左侧进行填充 + # - self.right_pad: 是否在文本右侧进行填充 + # - self.separator: 用于分隔N-gram中的单词的分隔符 + return cde.NgramOperation(self.ngrams, self.left_pad, self.right_pad, self.separator) + + + +class PythonTokenizer: + """ + Class that applies user-defined string tokenizer into input string. + 该类将用户定义的字符串标记化器应用于输入字符串。 + Args: + tokenizer (Callable): Python function that takes a `str` and returns a list of `str` as tokens. + + Raises: + TypeError: If `tokenizer` is not a callable Python function. + + Supported Platforms: + ``CPU`` + + Examples: + >>> def my_tokenizer(line): + ... return line.split() + >>> text_file_dataset = text_file_dataset.map(operations=text.PythonTokenizer(my_tokenizer)) + """ + + @check_python_tokenizer + def __init__(self, tokenizer): + self.pyfunc = tokenizer + self.tokenizer = np.vectorize(lambda x: np.array(tokenizer(x), dtype='U'), signature='()->(n)') + self.random = False + + # 定义一个名为__call__的方法 +def __call__(self, in_array): + # 检查输入是否为NumPy数组 + if not isinstance(in_array, np.ndarray): + # 如果输入不是NumPy数组,抛出TypeError异常并提供相应错误消息 + raise TypeError("input should be a NumPy array. Got {}.".format(type(in_array))) + + # 如果输入数组的数据类型是np.bytes_,将其转换为字符串 + if in_array.dtype.type is np.bytes_: + in_array = to_str(in_array) + + try: + # 使用Tokenizer对象对输入数组进行处理,将其转换为tokens + tokens = self.tokenizer(in_array) + except Exception as e: + # 如果在处理过程中出现异常,抛出RuntimeError异常并提供相应错误消息 + raise RuntimeError("Error occurred in Pyfunc [" + str(self.pyfunc.__name__) + "], error message: " + str(e)) + + # 返回处理后的tokens + return tokens + +# 定义一个名为to_json的方法,用于将操作信息转换为JSON格式 +def to_json(self): + json_obj = {} + # 存储操作名称 + json_obj["tensor_op_name"] = self.pyfunc.__name__ + # 存储Python模块信息 + json_obj["python_module"] = self.__class__.__module__ + # 将操作信息转换为JSON字符串并返回 + return json.dumps(json_obj) + + + +class SentencePieceTokenizer(TextTensorOperation): + """ + Tokenize scalar token or 1-D tokens to tokens by sentencepiece. + 通过句子片段将标量标记或1-D标记标记为标记。 + Args: + mode (Union[str, SentencePieceVocab]): SentencePiece model. + If the input parameter is a file, it represents the path of SentencePiece mode to be loaded. + If the input parameter is a SentencePieceVocab object, it should be constructed in advanced. + out_type (SPieceTokenizerOutType): The type of output, it can be any of [SPieceTokenizerOutType.STRING, + SPieceTokenizerOutType.INT]. + + - SPieceTokenizerOutType.STRING, means output type of SentencePice Tokenizer is string. + - SPieceTokenizerOutType.INT, means output type of SentencePice Tokenizer is int. + + Raises: + TypeError: If `mode` is not of type string or SentencePieceVocab. + TypeError: If `out_type` is not of type SPieceTokenizerOutType. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.text import SentencePieceModel, SPieceTokenizerOutType + >>> sentence_piece_vocab_file = "/path/to/sentence/piece/vocab/file" + >>> vocab = text.SentencePieceVocab.from_file([sentence_piece_vocab_file], 5000, 0.9995, + ... SentencePieceModel.UNIGRAM, {}) + >>> tokenizer = text.SentencePieceTokenizer(vocab, out_type=SPieceTokenizerOutType.STRING) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer) + """ + + @check_sentence_piece_tokenizer + def __init__(self, mode, out_type): + super().__init__() + self.mode = mode + self.out_type = out_type + + # 定义一个名为parse的方法 +def parse(self): + # 检查self.mode是否是SentencePieceVocab类型的对象 + if isinstance(self.mode, SentencePieceVocab): + # 如果是SentencePieceVocab类型的对象,将self.mode设置为c_sentence_piece_vocab + self.mode = self.mode.c_sentence_piece_vocab + + # 返回一个cde.SentencePieceTokenizerOperation对象,该对象使用以下参数进行初始化: + # - self.mode: SentencePiece模式,用于指定分词的方式 + # - DE_C_INTER_SENTENCEPIECE_OUTTYPE.get(self.out_type): 使用self.out_type作为键从DE_C_INTER_SENTENCEPIECE_OUTTYPE字典中获取对应的值, + # 用于指定输出类型 + return cde.SentencePieceTokenizerOperation(self.mode, DE_C_INTER_SENTENCEPIECE_OUTTYPE.get(self.out_type)) + + + +class SlidingWindow(TextTensorOperation): + """ + Construct a tensor from given data (only support 1-D for now), where each element in the dimension axis + is a slice of data starting at the corresponding position, with a specified width. + 根据给定的数据构造张量(目前仅支持1-D),其中维度轴中的每个元素是从相应位置开始的具有指定宽度的数据切片。 + Args: + width (int): The width of the window. It must be an integer and greater than zero. + axis (int, optional): The axis along which the sliding window is computed (default=0). + + Raises: + TypeError: If `width` is not of type int. + ValueError: If value of `width` is not positive. + TypeError: If `axis` is not of type int. + + Supported Platforms: + ``CPU`` + + Examples: + >>> dataset = ds.NumpySlicesDataset(data=[[1, 2, 3, 4, 5]], column_names="col1") + >>> # Data before + >>> # | col1 | + >>> # +--------------+ + >>> # | [[1, 2, 3, 4, 5]] | + >>> # +--------------+ + >>> dataset = dataset.map(operations=text.SlidingWindow(3, 0)) + >>> # Data after + >>> # | col1 | + >>> # +--------------+ + >>> # | [[1, 2, 3], | + >>> # | [2, 3, 4], | + >>> # | [3, 4, 5]] | + >>> # +--------------+ + """ + + @check_slidingwindow + def __init__(self, width, axis=0): + super().__init__() + self.width = width + self.axis = axis + + def parse(self): + return cde.SlidingWindowOperation(self.width, self.axis) + + +class ToNumber(TextTensorOperation): + """ + Tensor operation to convert every element of a string tensor to a number. + 张量运算,将字符串张量的每个元素转换为一个数字。 + Strings are cast according to the rules specified in the following links, except that any strings which represent + negative numbers cannot be cast to an unsigned integer type, rules links are as follows: + https://en.cppreference.com/w/cpp/string/basic_string/stof, + https://en.cppreference.com/w/cpp/string/basic_string/stoul, + + Args: + data_type (mindspore.dtype): Type to be cast to. Must be a numeric type in mindspore.dtype. + + Raises: + TypeError: If `data_type` is not of type mindspore.dtype. + RuntimeError: If strings are invalid to cast, or are out of range after being cast. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore import dtype as mstype + >>> data = [["1", "2", "3"]] + >>> dataset = ds.NumpySlicesDataset(data) + >>> to_number_op = text.ToNumber(mstype.int8) + >>> dataset = dataset.map(operations=to_number_op) + """ + + @check_to_number + def __init__(self, data_type): + super().__init__() + data_type = mstype_to_detype(data_type) + self.data_type = str(data_type) + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.ToNumberOperation对象,该对象使用以下参数进行初始化: + # - self.data_type: 数据类型,用于指定要将数据转换为的数字类型 + return cde.ToNumberOperation(self.data_type) + + + +class ToVectors(TextTensorOperation): + """ + Look up a token into vectors according to the input vector table. + 根据输入向量表,将令牌查找为向量。 + Args: + vectors (Vectors): A vectors object. + unk_init (sequence, optional): Sequence used to initialize out-of-vectors (OOV) token + (default=None, initialize with zero vectors). + lower_case_backup (bool, optional): Whether to look up the token in the lower case. If False, each token in the + original case will be looked up; if True, each token in the original case will be looked up first, if not + found in the keys of the property stoi, the token in the lower case will be looked up (default=False). + + Raises: + TypeError: If `unk_init` is not of type sequence. + TypeError: If elements of `unk_init` is not of type float or int. + TypeError: If `lower_case_backup` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Load vectors from file + >>> vectors = text.Vectors.from_file("/path/to/vectors/file") + >>> # Use ToVectors operator to map tokens to vectors + >>> to_vectors = text.ToVectors(vectors) + >>> text_file_dataset = text_file_dataset.map(operations=[to_vectors]) + """ + + @check_to_vectors + def __init__(self, vectors, unk_init=None, lower_case_backup=False): + super().__init__() + self.vectors = vectors + self.unk_init = unk_init if unk_init is not None else [] + self.lower_case_backup = lower_case_backup + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.ToVectorsOperation对象,该对象使用以下参数进行初始化: + # - self.vectors: 用于将文本转换为向量的预训练嵌入向量 + # - self.unk_init: 未知单词的初始化方法,用于处理未知的单词 + # - self.lower_case_backup: 是否使用小写备份,用于处理未知单词的情况 + return cde.ToVectorsOperation(self.vectors, self.unk_init, self.lower_case_backup) + + + +class TruncateSequencePair(TextTensorOperation): + """ + Truncate a pair of rank-1 tensors such that the total length is less than max_length. + + This operation takes two input tensors and returns two output Tensors. + 截断一对秩为1的张量,使其总长度小于max_length。 + 此操作获取两个输入张量并返回两个输出张量。 + Args: + max_length (int): Maximum length required. + + Raises: + TypeError: If `max_length` is not of type int. + + Supported Platforms: + ``CPU`` + + Examples: + >>> dataset = ds.NumpySlicesDataset(data={"col1": [[1, 2, 3]], "col2": [[4, 5]]}) + >>> # Data before + >>> # | col1 | col2 | + >>> # +-----------+-----------| + >>> # | [1, 2, 3] | [4, 5] | + >>> # +-----------+-----------+ + >>> truncate_sequence_pair_op = text.TruncateSequencePair(max_length=4) + >>> dataset = dataset.map(operations=truncate_sequence_pair_op) + >>> # Data after + >>> # | col1 | col2 | + >>> # +-----------+-----------+ + >>> # | [1, 2] | [4, 5] | + >>> # +-----------+-----------+ + """ + + @check_pair_truncate + def __init__(self, max_length): + super().__init__() + self.max_length = max_length + + def parse(self): + return cde.TruncateSequencePairOperation(self.max_length) + + +class UnicodeCharTokenizer(TextTensorOperation): + """ + Tokenize a scalar tensor of UTF-8 string to Unicode characters. + 将UTF-8字符串的标量张量标记为Unicode字符。 + Args: + with_offsets (bool, optional): Whether or not output offsets of tokens (default=False). + + Raises: + TypeError: If `with_offsets` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> tokenizer_op = text.UnicodeCharTokenizer(with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.UnicodeCharTokenizer(with_offsets=True) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", "offsets_limit"], + ... column_order=["token", "offsets_start", "offsets_limit"]) + """ + + @check_with_offsets + def __init__(self, with_offsets=False): + super().__init__() + self.with_offsets = with_offsets + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.UnicodeCharTokenizerOperation对象,该对象使用以下参数进行初始化: + # - self.with_offsets: 是否返回字符偏移信息,用于指定是否在分词结果中包含字符的偏移信息 + return cde.UnicodeCharTokenizerOperation(self.with_offsets) + + + +class WordpieceTokenizer(TextTensorOperation): + """ + Tokenize the input text to subword tokens. + 将输入文本标记为子单词标记。 + Args: + vocab (Vocab): Vocabulary used to look up words. + suffix_indicator (str, optional): Prefix flags used to indicate subword suffixes. Default: '##'. + max_bytes_per_token (int, optional): The maximum length of tokenization, words exceeding this length will + not be split. Default: 100. + unknown_token (str, optional): The output for unknown words. When set to an empty string, the corresponding + unknown word will be directly returned as the output. Otherwise, the set string will be returned as the + output. Default: '[UNK]'. + with_offsets (bool, optional): Whether to return the offsets of tokens. Default: False. + + Raises: + TypeError: If `vocab` is not of type :class:`mindspore.dataset.text.Vocab`. + TypeError: If `suffix_indicator` is not of type str. + TypeError: If `max_bytes_per_token` is not of type int. + TypeError: If `unknown_token` is not of type str. + TypeError: If `with_offsets` is not of type bool. + ValueError: If `max_bytes_per_token` is negative. + + Supported Platforms: + ``CPU`` + + Examples: + >>> vocab_list = ["book", "cholera", "era", "favor", "##ite", "my", "is", "love", "dur", "##ing", "the"] + >>> vocab = text.Vocab.from_list(vocab_list) + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token='[UNK]', + ... max_bytes_per_token=100, with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token='[UNK]', + ... max_bytes_per_token=100, with_offsets=True) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", "offsets_limit"], + ... column_order=["token", "offsets_start", "offsets_limit"]) + """ + + @check_wordpiece_tokenizer + def __init__(self, vocab, suffix_indicator='##', max_bytes_per_token=100, unknown_token='[UNK]', + with_offsets=False): + super().__init__() + self.vocab = vocab + self.suffix_indicator = suffix_indicator + self.max_bytes_per_token = max_bytes_per_token + self.unknown_token = unknown_token + self.with_offsets = with_offsets + + # 定义一个名为parse的方法 +def parse(self): + # 返回一个cde.WordpieceTokenizerOperation对象,该对象使用以下参数进行初始化: + # - self.vocab.c_vocab: 词汇表对象的C API表示 + # - self.suffix_indicator: 用于表示子词后缀的标识符 + # - self.max_bytes_per_token: 每个子词的最大字节数 + # - self.unknown_token: 未知子词的标记 + # - self.with_offsets: 是否返回子词的偏移信息,用于指定是否在分词结果中包含子词的偏移信息 + return cde.WordpieceTokenizerOperation(self.vocab.c_vocab, self.suffix_indicator, self.max_bytes_per_token, + self.unknown_token, self.with_offsets) +if platform.system().lower() != 'windows': + DE_C_INTER_NORMALIZE_FORM = { + NormalizeForm.NONE: cde.NormalizeForm.DE_NORMALIZE_NONE, + NormalizeForm.NFC: cde.NormalizeForm.DE_NORMALIZE_NFC, + NormalizeForm.NFKC: cde.NormalizeForm.DE_NORMALIZE_NFKC, + NormalizeForm.NFD: cde.NormalizeForm.DE_NORMALIZE_NFD, + NormalizeForm.NFKD: cde.NormalizeForm.DE_NORMALIZE_NFKD + } + + + + class BasicTokenizer(TextTensorOperation): + """ + Tokenize the input UTF-8 encoded string by specific rules. + + Note: + `BasicTokenizer` is not supported on Windows platform yet. + + Args: + lower_case (bool, optional): Whether to perform lowercase processing on the text. If True, will fold the + text to lower case and strip accented characters. If False, will only perform normalization on the + text, with mode specified by `normalization_form`. Default: False. + keep_whitespace (bool, optional): If True, the whitespace will be kept in the output. Default: False. + normalization_form (NormalizeForm, optional): + `Unicode normalization forms `_, only valid when `lower_case` + is False, can be NormalizeForm.NONE, NormalizeForm.NFC, NormalizeForm.NFKC, NormalizeForm.NFD or + NormalizeForm.NFKD. Default: NormalizeForm.NONE. + + - NormalizeForm.NONE, no normalization. + - NormalizeForm.NFC, Canonical Decomposition, followed by Canonical Composition. + - NormalizeForm.NFKC, Compatibility Decomposition, followed by Canonical Composition. + - NormalizeForm.NFD, Canonical Decomposition. + - NormalizeForm.NFKD, Compatibility Decomposition. + + preserve_unused_token (bool, optional): Whether to preserve special tokens. If True, will not split special + tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]', '[MASK]'. Default: True. + with_offsets (bool, optional): Whether to return the offsets of tokens. Default: False. + + Raises: + TypeError: If `lower_case` is not of type bool. + TypeError: If `keep_whitespace` is not of type bool. + TypeError: If `normalization_form` is not of type :class:`mindspore.dataset.text.NormalizeForm`. + TypeError: If `preserve_unused_token` is not of type bool. + TypeError: If `with_offsets` is not of type bool. + RuntimeError: If dtype of input Tensor is not str. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.text import NormalizeForm + >>> + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> tokenizer_op = text.BasicTokenizer(lower_case=False, + ... keep_whitespace=False, + ... normalization_form=NormalizeForm.NONE, + ... preserve_unused_token=True, + ... with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], + >>> # ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.BasicTokenizer(lower_case=False, + ... keep_whitespace=False, + ... normalization_form=NormalizeForm.NONE, + ... preserve_unused_token=True, + ... with_offsets=True) + >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", + ... "offsets_limit"], + ... column_order=["token", "offsets_start", + ... "offsets_limit"]) + """ + + @check_basic_tokenizer + def __init__(self, lower_case=False, keep_whitespace=False, normalization_form=NormalizeForm.NONE, + preserve_unused_token=True, with_offsets=False): + super().__init__() + if not isinstance(normalization_form, NormalizeForm): + raise TypeError("Wrong input type for normalization_form, should be enum of 'NormalizeForm'.") + + self.lower_case = lower_case + self.keep_whitespace = keep_whitespace + self.normalization_form = DE_C_INTER_NORMALIZE_FORM.get(normalization_form) + self.preserve_unused_token = preserve_unused_token + self.with_offsets = with_offsets + + def parse(self): + return cde.BasicTokenizerOperation(self.lower_case, self.keep_whitespace, self.normalization_form, + self.preserve_unused_token, self.with_offsets) + + + class BertTokenizer(TextTensorOperation): + """ + Tokenizer used for Bert text process. + + Note: + `BertTokenizer` is not supported on Windows platform yet. + + Args: + vocab (Vocab): Vocabulary used to look up words. + suffix_indicator (str, optional): Prefix flags used to indicate subword suffixes. Default: '##'. + max_bytes_per_token (int, optional): The maximum length of tokenization, words exceeding this length will + not be split. Default: 100. + unknown_token (str, optional): The output for unknown words. When set to an empty string, the corresponding + unknown word will be directly returned as the output. Otherwise, the set string will be returned as the + output. Default: '[UNK]'. + lower_case (bool, optional): Whether to perform lowercase processing on the text. If True, will fold the + text to lower case and strip accented characters. If False, will only perform normalization on the + text, with mode specified by `normalization_form`. Default: False. + keep_whitespace (bool, optional): If True, the whitespace will be kept in the output. Default: False. + normalization_form (NormalizeForm, optional): + `Unicode normalization forms `_, only valid when `lower_case` + is False, can be NormalizeForm.NONE, NormalizeForm.NFC, NormalizeForm.NFKC, NormalizeForm.NFD or + NormalizeForm.NFKD. Default: NormalizeForm.NONE. + + - NormalizeForm.NONE, no normalization. + - NormalizeForm.NFC, Canonical Decomposition, followed by Canonical Composition. + - NormalizeForm.NFKC, Compatibility Decomposition, followed by Canonical Composition. + - NormalizeForm.NFD, Canonical Decomposition. + - NormalizeForm.NFKD, Compatibility Decomposition. + + preserve_unused_token (bool, optional): Whether to preserve special tokens. If True, will not split special + tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]', '[MASK]'. Default: True. + with_offsets (bool, optional): Whether to return the offsets of tokens. Default: False. + + Raises: + TypeError: If `vocab` is not of type :class:`mindspore.dataset.text.Vocab`. + TypeError: If `suffix_indicator` is not of type str. + TypeError: If `max_bytes_per_token` is not of type int. + ValueError: If `max_bytes_per_token` is negative. + TypeError: If `unknown_token` is not of type str. + TypeError: If `lower_case` is not of type bool. + TypeError: If `keep_whitespace` is not of type bool. + TypeError: If `normalization_form` is not of type :class:`mindspore.dataset.text.NormalizeForm`. + TypeError: If `preserve_unused_token` is not of type bool. + TypeError: If `with_offsets` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.text import NormalizeForm + >>> + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> vocab_list = ["床", "前", "明", "月", "光", "疑", "是", "地", "上", "霜", "举", "头", "望", "低", + ... "思", "故", "乡","繁", "體", "字", "嘿", "哈", "大", "笑", "嘻", "i", "am", "mak", + ... "make", "small", "mistake", "##s", "during", "work", "##ing", "hour", "😀", "😃", + ... "😄", "😁", "+", "/", "-", "=", "12", "28", "40", "16", " ", "I", "[CLS]", "[SEP]", + ... "[UNK]", "[PAD]", "[MASK]", "[unused1]", "[unused10]"] + >>> vocab = text.Vocab.from_list(vocab_list) + >>> tokenizer_op = text.BertTokenizer(vocab=vocab, suffix_indicator='##', max_bytes_per_token=100, + ... unknown_token='[UNK]', lower_case=False, keep_whitespace=False, + ... normalization_form=NormalizeForm.NONE, preserve_unused_token=True, + ... with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], + >>> # ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.BertTokenizer(vocab=vocab, suffix_indicator='##', max_bytes_per_token=100, + ... unknown_token='[UNK]', lower_case=False, keep_whitespace=False, + ... normalization_form=NormalizeForm.NONE, preserve_unused_token=True, + ... with_offsets=True) + >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", + ... "offsets_limit"], + ... column_order=["token", "offsets_start", + ... "offsets_limit"]) + """ + + @check_bert_tokenizer + def __init__(self, vocab, suffix_indicator='##', max_bytes_per_token=100, unknown_token='[UNK]', + lower_case=False, keep_whitespace=False, normalization_form=NormalizeForm.NONE, + preserve_unused_token=True, with_offsets=False): + super().__init__() + if not isinstance(normalization_form, NormalizeForm): + raise TypeError("Wrong input type for normalization_form, should be enum of 'NormalizeForm'.") + + self.vocab = vocab + self.suffix_indicator = suffix_indicator + self.max_bytes_per_token = max_bytes_per_token + self.unknown_token = unknown_token + self.lower_case = lower_case + self.keep_whitespace = keep_whitespace + self.normalization_form = DE_C_INTER_NORMALIZE_FORM.get(normalization_form) + self.preserve_unused_token = preserve_unused_token + self.with_offsets = with_offsets + + def parse(self): + return cde.BertTokenizerOperation(self.vocab.c_vocab, self.suffix_indicator, self.max_bytes_per_token, + self.unknown_token, self.lower_case, self.keep_whitespace, + self.normalization_form, self.preserve_unused_token, self.with_offsets) + + + class CaseFold(TextTensorOperation): + """ + Apply case fold operation on UTF-8 string tensor, which is aggressive that can convert more characters into + lower case. Supported normalization forms please refer to + `ICU_Normalizer2 `_ . + + Note: + CaseFold is not supported on Windows platform yet. + + Supported Platforms: + ``CPU`` + + Examples: + >>> case_op = text.CaseFold() + >>> text_file_dataset = text_file_dataset.map(operations=case_op) + """ + + def parse(self): + return cde.CaseFoldOperation() + + + class FilterWikipediaXML(TextTensorOperation): + """ + Filter Wikipedia XML dumps to "clean" text consisting only of lowercase letters (a-z, converted from A-Z), + and spaces (never consecutive). + + Note: + FilterWikipediaXML is not supported on Windows platform yet. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import mindspore.dataset.text.transforms as text + >>> + >>> replace_op = text.FilterWikipediaXML() + >>> text_file_dataset = text_file_dataset.map(operations=replace_op) + """ + + def parse(self): + return cde.FilterWikipediaXMLOperation() + + + class NormalizeUTF8(TextTensorOperation): + """ + Apply normalize operation on UTF-8 string tensor. + + Note: + NormalizeUTF8 is not supported on Windows platform yet. + + Args: + normalize_form (NormalizeForm, optional): Valid values can be [NormalizeForm.NONE, NormalizeForm.NFC, + NormalizeForm.NFKC, NormalizeForm.NFD, NormalizeForm.NFKD] any of the four unicode + normalized forms(default=NormalizeForm.NFKC). + See http://unicode.org/reports/tr15/ for details. + + - NormalizeForm.NONE, do nothing for input string tensor. + - NormalizeForm.NFC, normalize with Normalization Form C. + - NormalizeForm.NFKC, normalize with Normalization Form KC. + - NormalizeForm.NFD, normalize with Normalization Form D. + - NormalizeForm.NFKD, normalize with Normalization Form KD. + + Raises: + TypeError: If `normalize_form` is not of type NormalizeForm. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.text import NormalizeForm + >>> normalize_op = text.NormalizeUTF8(normalize_form=NormalizeForm.NFC) + >>> text_file_dataset = text_file_dataset.map(operations=normalize_op) + """ + + def __init__(self, normalize_form=NormalizeForm.NFKC): + super().__init__() + if not isinstance(normalize_form, NormalizeForm): + raise TypeError("Wrong input type for normalization_form, should be enum of 'NormalizeForm'.") + + normalize_form = replace_none(normalize_form, NormalizeForm.NFKC) + self.normalize_form = DE_C_INTER_NORMALIZE_FORM.get(normalize_form) + + def parse(self): + return cde.NormalizeUTF8Operation(self.normalize_form) + + + class RegexReplace(TextTensorOperation): + """ + Replace a part of UTF-8 string tensor with given text according to regular expressions. + + See https://unicode-org.github.io/icu/userguide/strings/regexp.html for supported regex pattern. + + Note: + RegexReplace is not supported on Windows platform yet. + + Args: + pattern (str): the regex expression patterns. + replace (str): the string to replace matched element. + replace_all (bool, optional): If False, only replace first matched element; + if True, replace all matched elements (default=True). + + Raises: + TypeError: If `pattern` is not of type string. + TypeError: If `replace` is not of type string. + TypeError: If `replace_all` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> pattern = 'Canada' + >>> replace = 'China' + >>> replace_op = text.RegexReplace(pattern, replace) + >>> text_file_dataset = text_file_dataset.map(operations=replace_op) + """ + + @check_regex_replace + def __init__(self, pattern, replace, replace_all=True): + super().__init__() + self.pattern = pattern + self.replace = replace + self.replace_all = replace_all + + def parse(self): + return cde.RegexReplaceOperation(self.pattern, self.replace, self.replace_all) + + + class RegexTokenizer(TextTensorOperation): + """ + Tokenize a scalar tensor of UTF-8 string by regex expression pattern. + + See https://unicode-org.github.io/icu/userguide/strings/regexp.html for supported regex pattern. + + Note: + RegexTokenizer is not supported on Windows platform yet. + + Args: + delim_pattern (str): The pattern of regex delimiters. + The original string will be split by matched elements. + keep_delim_pattern (str, optional): The string matched by 'delim_pattern' can be kept as a token + if it can be matched by 'keep_delim_pattern'. The default value is an empty str + which means that delimiters will not be kept as an output token (default=''). + with_offsets (bool, optional): Whether or not output offsets of tokens(default=False). + + Raises: + TypeError: If `delim_pattern` is not of type string. + TypeError: If `keep_delim_pattern` is not of type string. + TypeError: If `with_offsets` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # If with_offsets=False, default output is one column {["text", dtype=str]} + >>> delim_pattern = r"[ |,]" + >>> tokenizer_op = text.RegexTokenizer(delim_pattern, with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], + >>> # ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.RegexTokenizer(delim_pattern, with_offsets=True) + >>> text_file_dataset_1 = text_file_dataset_1.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", + ... "offsets_limit"], + ... column_order=["token", "offsets_start", + ... "offsets_limit"]) + """ + + @check_regex_tokenizer + def __init__(self, delim_pattern, keep_delim_pattern='', with_offsets=False): + super().__init__() + self.delim_pattern = delim_pattern + self.keep_delim_pattern = keep_delim_pattern + self.with_offsets = with_offsets + + def parse(self): + return cde.RegexTokenizerOperation(self.delim_pattern, self.keep_delim_pattern, self.with_offsets) + + + class UnicodeScriptTokenizer(TextTensorOperation): + """ + Tokenize a scalar tensor of UTF-8 string based on Unicode script boundaries. + + Note: + UnicodeScriptTokenizer is not supported on Windows platform yet. + + Args: + keep_whitespace (bool, optional): Whether or not emit whitespace tokens (default=False). + with_offsets (bool, optional): Whether or not output offsets of tokens (default=False). + + Raises: + TypeError: If `keep_whitespace` is not of type bool. + TypeError: If `with_offsets` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> tokenizer_op = text.UnicodeScriptTokenizer(keep_whitespace=True, with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], + >>> # ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.UnicodeScriptTokenizer(keep_whitespace=True, with_offsets=True) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", "offsets_limit"], + ... column_order=["token", "offsets_start", "offsets_limit"]) + + """ + + @check_unicode_script_tokenizer + def __init__(self, keep_whitespace=False, with_offsets=False): + super().__init__() + keep_whitespace = replace_none(keep_whitespace, False) + with_offsets = replace_none(with_offsets, False) + self.keep_whitespace = keep_whitespace + self.with_offsets = with_offsets + + def parse(self): + return cde.UnicodeScriptTokenizerOperation(self.keep_whitespace, self.with_offsets) + + + class WhitespaceTokenizer(TextTensorOperation): + """ + Tokenize a scalar tensor of UTF-8 string on ICU4C defined whitespaces, such as: ' ', '\\\\t', '\\\\r', '\\\\n'. + + Note: + WhitespaceTokenizer is not supported on Windows platform yet. + + Args: + with_offsets (bool, optional): Whether or not output offsets of tokens (default=False). + + Raises: + TypeError: If `with_offsets` is not of type bool. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # If with_offsets=False, default output one column {["text", dtype=str]} + >>> tokenizer_op = text.WhitespaceTokenizer(with_offsets=False) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op) + >>> # If with_offsets=True, then output three columns {["token", dtype=str], + >>> # ["offsets_start", dtype=uint32], + >>> # ["offsets_limit", dtype=uint32]} + >>> tokenizer_op = text.WhitespaceTokenizer(with_offsets=True) + >>> text_file_dataset = text_file_dataset.map(operations=tokenizer_op, input_columns=["text"], + ... output_columns=["token", "offsets_start", "offsets_limit"], + ... column_order=["token", "offsets_start", "offsets_limit"]) + """ + + @check_with_offsets + def __init__(self, with_offsets=False): + super().__init__() + self.with_offsets = with_offsets + + def parse(self): + return cde.WhitespaceTokenizerOperation(self.with_offsets) -- 2.34.1 From 33c6ac4c6d83cb7d585d9519058d878640048586 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:26:08 +0800 Subject: [PATCH 71/72] ADD file via upload --- .../transform-update/transforms/transforms.py | 1052 +++++++++++++++++ 1 file changed, 1052 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/transforms/transforms.py diff --git a/mindspore/ccsrc/transform-update/transforms/transforms.py b/mindspore/ccsrc/transform-update/transforms/transforms.py new file mode 100644 index 00000000000..840932a9381 --- /dev/null +++ b/mindspore/ccsrc/transform-update/transforms/transforms.py @@ -0,0 +1,1052 @@ +# Copyright 2019-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +The module transforms provides common operations, including Compose, OneHot and TypeCast. +提供常见操作,包括Compose、OneHot和TypeCast。 +""" +import json +from abc import ABC + +import sys +from enum import IntEnum +import numpy as np + +import mindspore._c_dataengine as cde +from mindspore._c_expression import typing +from mindspore.common import dtype as mstype +import mindspore.dataset.transforms.c_transforms as c_transforms +import mindspore.dataset.transforms.py_transforms as py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +from . import py_transforms_util as util +from .py_transforms_util import Implementation, FuncWrapper +from .validators import check_fill_value, check_slice_option, check_slice_op, check_one_hot_op, check_compose_call, \ + check_mask_op_new, check_pad_end, check_concat_type, check_random_transform_ops, check_plugin, check_type_cast +from ..core.datatypes import mstype_to_detype, nptype_to_detype +from ..vision.py_transforms_util import is_pil + + +class TensorOperation: + """ + Base class Tensor Ops + """ + + def __init__(self): + super().__init__() + self.implementation = None + self.callable_op_ = None + + def __call__(self, *input_tensor_list): + """ + Call method. + 调用方法。 + """ + # 检查是否使用 Python 实现的操作,或者传入的是 PIL 图像并且有 execute_py 方法 + if (self.implementation == Implementation.PY) or \ + (len(input_tensor_list) == 1 and is_pil(input_tensor_list[0]) and getattr(self, 'execute_py', None)): + # 如果是,调用 execute_py 方法进行操作处理,并返回结果 + return self.execute_py(*input_tensor_list) + + # 如果不是 Python 实现的操作,将输入张量转换为 CDE 的 Tensor 对象 + tensor_row = [] + for tensor in input_tensor_list: + try: + tensor_row.append(cde.Tensor(np.asarray(tensor))) + except (RuntimeError, TypeError): + # 如果转换失败,抛出类型错误异常 + raise TypeError("Invalid user input. Got {}: {}, cannot be converted into tensor." \ + .format(type(tensor), tensor)) + + # 检查是否已经创建了 callable_op_ 对象,如果没有,调用 parse 方法创建它 + if not hasattr(self, 'callable_op_') or self.callable_op_ is None: + self.callable_op_ = cde.Execute(self.parse()) + + # 调用 callable_op_ 对象处理输入张量,并获取输出张量列表 + output_tensor_list = self.callable_op_(tensor_row) + + # 将输出张量列表转换为 NumPy 数组列表 + output_numpy_list = [x.as_decoded_array() for x in output_tensor_list] + + # 如果输出列表中只有一个元素,直接返回该元素,否则返回输出列表的元组 + return output_numpy_list[0] if len(output_numpy_list) == 1 else tuple(output_numpy_list) + + + @staticmethod + def parse(): + """parse function - not yet implemented""" + raise NotImplementedError("TensorOperation has to implement parse() method.") + + +class PyTensorOperation: + """ + Base Python Tensor Operations class + """ + + def __init__(self): + self.transforms = [] + self.output_type = None + + def __call__(self, img): + """ + Call method. + + Args: + img (PIL Image): Image to be augmented. + + Returns: + PIL Image, augmented image. + """ + return self.execute_py(img) + + @classmethod + # 定义一个类方法,用于从 JSON 字符串反序列化创建操作对象 + def from_json(cls, json_string): + """ + Base from_json for Python tensor operations class + Python张量运算类的基础from_json + """ + + # 使用 json.loads 将 JSON 字符串解析为 JSON 对象 + json_obj = json.loads(json_string) + + # 创建一个新的操作对象 + new_op = cls.__new__(cls) + + # 将操作对象的属性字典设置为 JSON 对象的属性字典 + new_op.__dict__ = json_obj + + # 如果 JSON 对象中包含 "transforms" 键 + if "transforms" in json_obj.keys(): + # 对于具有 transforms 作为输入的操作,需要为每个 transform 调用 _from_json() 方法进行反序列化 + transforms = [] + for json_op in json_obj["transforms"]: + # 从操作的 python_module 获取模块,并根据 tensor_op_name 创建一个新的操作对象 + # 并使用 tensor_op_params 初始化新操作的属性 + transforms.append(getattr( + sys.modules.get(json_op.get("python_module")), + json_op["tensor_op_name"]).from_json(json.dumps(json_op["tensor_op_params"]))) + new_op.transforms = transforms + + # 如果 JSON 对象中包含 "output_type" 键 + if "output_type" in json_obj.keys(): + # 将 "output_type" 转换为 NumPy 数据类型,并设置为操作对象的属性 + output_type = np.dtype(json_obj["output_type"]) + new_op.output_type = output_type + + # 返回从 JSON 字符串创建的操作对象 + return new_op + +# 定义一个方法,用于将操作对象序列化为 JSON 格式的字符串 + def to_json(self): + """ + Base to_json for Python tensor operations class + Python张量运算类的基础to_json + """ + + + # 创建一个空的 JSON 对象 + json_obj = {} + json_trans = {} + + # 如果操作对象的属性字典中包含 "transforms" 键 + if "transforms" in self.__dict__.keys(): + # 对于具有 transforms 作为输入的操作,需要调用 _to_json() 方法对每个 transform 进行序列化 + json_list = [] + for transform in self.transforms: + json_list.append(json.loads(transform.to_json())) + json_trans["transforms"] = json_list + + # 移除操作对象属性字典中的 "transforms" 键 + self.__dict__.pop("transforms") + + # 如果操作对象的属性字典中包含 "output_type" 键 + if "output_type" in self.__dict__.keys(): + # 将 "output_type" 转换为 NumPy 数据类型名称,并设置到 json_trans 中 + json_trans["output_type"] = np.dtype( + self.__dict__["output_type"]).name + + # 移除操作对象属性字典中的 "output_type" 键 + self.__dict__.pop("output_type") + + # 将操作对象的属性字典添加到 json_obj 中作为 "tensor_op_params" + json_obj["tensor_op_params"] = self.__dict__ + + # 合并 json_trans 到 "tensor_op_params" 中 + json_obj.get("tensor_op_params").update(json_trans) + + # 添加操作对象的类名和模块名到 json_obj 中 + json_obj["tensor_op_name"] = self.__class__.__name__ + json_obj["python_module"] = self.__class__.__module__ + + # 将 json_obj 转换为 JSON 格式的字符串并返回 + return json.dumps(json_obj) + + + +class CompoundOperation(TensorOperation, PyTensorOperation, ABC): + """ + Compound Tensor Operations class + """ + + def __init__(self, transforms): + super(CompoundOperation, self).__init__() + self.transforms = [] + trans_with_imple = [] + for op in transforms: + if callable(op) and not hasattr(op, "implementation") and \ + not isinstance(op, c_transforms.TensorOperation) and \ + not isinstance(op, py_transforms.PyTensorOperation) and \ + not isinstance(op, c_vision.ImageTensorOperation): + op = util.FuncWrapper(op) + if hasattr(op, "implementation"): + if op.implementation is not None: + trans_with_imple.append(op) + else: + raise RuntimeError("Mixing old legacy c/py_transforms and new unified transforms is not allowed.") + self.transforms.append(op) + + if all([t.implementation == Implementation.PY for t in self.transforms]): + self.implementation = Implementation.PY + elif all([t.implementation is not None for t in self.transforms]): + self.implementation = Implementation.C + elif not trans_with_imple: + self.implementation = None + elif all([t.implementation == Implementation.PY for t in trans_with_imple]): + self.implementation = Implementation.PY + elif all([t.implementation == Implementation.C for t in trans_with_imple]): + self.implementation = Implementation.C + + @staticmethod + def parse(): + """parse function - not yet implemented""" + raise NotImplementedError("CompoundOperation has to implement parse() method.") + + def parse_transforms(self): + operations = [] + for op in self.transforms: + if op and getattr(op, 'parse', None): + operations.append(op.parse()) + else: + operations.append(op) + return operations + + +def not_random(function): + """ + Specify the function as "not random", i.e., it produces deterministic result. + A Python function can only be cached after it is specified as "not random". + """ + function.random = False + return function + + +class Compose(CompoundOperation): + """ + Compose a list of transforms into a single transform. + + .. Note:: + Compose takes a list of transformations either provided in transforms.py or from user-defined implementation; + each can be an initialized transformation class or a lambda function, as long as the output from the last + transformation is a single tensor of type numpy.ndarray. + + Args: + transforms (list): List of transformations to be applied. + + Raises: + TypeError: If `transforms` is not of type list. + ValueError: If `transforms` is empty. + TypeError: If elements of `transforms` are neither Python callable objects nor data + processing operations in transforms.py. + + Supported Platforms: + ``CPU`` + + Examples: + >>> compose = transforms.Compose([vision.Decode(), vision.RandomCrop(512)]) + >>> image_folder_dataset = image_folder_dataset.map(operations=compose) + >>> image_folder_dataset_dir = "/path/to/image_folder_dataset_directory" + >>> + >>> # create a dataset that reads all files in dataset_dir with 8 threads + >>> image_folder_dataset = ds.ImageFolderDataset(image_folder_dataset_dir, num_parallel_workers=8) + >>> # create a list of transformations to be applied to the image data + >>> transform = transforms.Compose([vision.Decode(to_pil=True), + ... vision.RandomHorizontalFlip(0.5), + ... vision.ToTensor(), + ... vision.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262), is_hwc=False), + ... vision.RandomErasing()]) + >>> # apply the transform to the dataset through dataset.map function + >>> image_folder_dataset = image_folder_dataset.map(operations=transform, input_columns=["image"]) + >>> + >>> # Compose is also be invoked implicitly, by just passing in a list of ops + >>> # the above example then becomes: + >>> transforms_list = [vision.Decode(to_pil=True), + ... vision.RandomHorizontalFlip(0.5), + ... vision.ToTensor(), + ... vision.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262), is_hwc=False), + ... vision.RandomErasing()] + >>> + >>> # apply the transform to the dataset through dataset.map() + >>> image_folder_dataset_1 = image_folder_dataset_1.map(operations=transforms_list, input_columns=["image"]) + >>> + >>> # Certain C++ and Python ops can be combined, but not all of them + >>> # An example of combined operations + >>> arr = [0, 1] + >>> dataset = ds.NumpySlicesDataset(arr, column_names=["cols"], shuffle=False) + >>> transformed_list = [transforms.OneHot(2), + ... transforms.Mask(transforms.Relational.EQ, 1)] + >>> dataset = dataset.map(operations=transformed_list, input_columns=["cols"]) + >>> + >>> # Here is an example of mixing vision ops + >>> import numpy as np + >>> op_list=[vision.Decode(), + ... vision.Resize((224, 244)), + ... vision.ToPIL(), + ... np.array, # need to convert PIL image to a NumPy array to pass it to C++ operation + ... vision.Resize((24, 24))] + >>> image_folder_dataset = image_folder_dataset.map(operations=op_list, input_columns=["image"]) + """ + + @check_random_transform_ops + def __init__(self, transforms): + super().__init__(transforms) + self.transforms = Compose.decompose(self.transforms) + if all(hasattr(transform, "random") and not transform.random for transform in self.transforms): + self.random = False + + @staticmethod + # 定义一个方法,用于将复合操作解构为单个操作,并返回解构后的操作列表 + def decompose(operations): + """ + Remove all compose operation from the given list of operations. + 从给定的操作列表中删除所有组合操作。 + Args: + operations: list of transforms + + Returns: + list of operations without compose operations. + """ + + # 创建一个空的新操作列表,用于存储解构后的单个操作 + new_operations = [] + + # 遍历传入的操作列表 + for op in operations: + # 如果当前操作是 Compose 类型的复合操作 + if isinstance(op, Compose): + # 递归调用 decompose 方法将复合操作解构为单个操作,并将其添加到新操作列表中 + new_operations.extend(Compose.decompose(op.transforms)) + else: + # 如果当前操作不是复合操作,直接将其添加到新操作列表中 + new_operations.append(op) + + # 返回解构后的操作列表 + return new_operations + + @staticmethod + # 定义一个方法,用于将连续的 C 实现的操作组合为一个 Compose 操作,返回新的操作列表 + def reduce(operations): + """ + Wraps adjacent Python operations in a Compose to allow mixing of Python and C++ operations. + 在Compose中封装相邻的Python操作,以允许混合Python和C++操作。 + Args: + operations (list): list of tensor operations. + + Returns: + list, the reduced list of operations. + """ + # 创建一个空的新操作列表 new_ops,以及记录起始和结束索引的变量 start_ind 和 end_ind + new_ops, start_ind, end_ind = [], 0, 0 + + # 遍历传入的操作列表 + for i, op in enumerate(operations): + # 如果当前操作是 C 实现的操作且不是 FuncWrapper 类型的操作 + if op.implementation == Implementation.C and not isinstance(op, FuncWrapper): + # 重置起始和结束索引,如果起始和结束索引不相等,则说明存在连续的 C 实现的操作 + if start_ind != end_ind: + # 如果只有一个操作,则直接将该操作添加到新操作列表 + if end_ind == start_ind + 1: + composed_op = operations[start_ind] + else: + # 否则,将连续的操作组合为一个 Compose 操作,实现切换到 Python 实现 + composed_op = Compose(operations[start_ind:end_ind]) + composed_op.implementation = Implementation.PY + # 将组合后的操作添加到新操作列表 + new_ops.append(composed_op) + # 将当前操作添加到新操作列表,并更新起始和结束索引 + new_ops.append(op) + start_ind, end_ind = i + 1, i + 1 + else: + # 如果当前操作不是 C 实现的操作或者是 FuncWrapper 类型的操作,增加结束索引 + end_ind += 1 + + # 额外检查,以防最后一个操作是 Python 实现的操作 + if start_ind != end_ind: + if end_ind == start_ind + 1: + composed_op = operations[start_ind] + else: + composed_op = Compose(operations[start_ind:end_ind]) + composed_op.implementation = Implementation.PY + # 将组合后的操作添加到新操作列表 + new_ops.append(composed_op) + + # 返回新的操作列表,将连续的 C 实现的操作组合为一个 Compose 操作 + return new_ops + + + @check_compose_call + def execute_py(self, *args): + """ + Execute method. + + Returns: + lambda function, Lambda function that takes in an args to apply transformations on. + """ + return util.compose(self.transforms, *args) + + def parse(self): + operations = self.parse_transforms() + return cde.ComposeOperation(operations) + + +class Concatenate(TensorOperation): + """ + Tensor operation that concatenates all columns into a single tensor, only 1D tenspr is supported. + + Args: + axis (int, optional): Concatenate the tensors along given axis (Default=0). + prepend (numpy.array, optional): NumPy array to be prepended to the already concatenated tensors + (Default=None). + append (numpy.array, optional): NumPy array to be appended to the already concatenated tensors (Default=None). + + Raises: + TypeError: If `axis` is not of type int. + TypeError: If `prepend` is not of type numpy.ndarray. + TypeError: If `append` is not of type numpy.ndarray. + + Supported Platforms: + ``CPU`` + + Examples: + >>> import numpy as np + >>> # concatenate string + >>> prepend_tensor = np.array(["dw", "df"], dtype='S') + >>> append_tensor = np.array(["dwsdf", "df"], dtype='S') + >>> concatenate_op = transforms.Concatenate(0, prepend_tensor, append_tensor) + >>> data = [["This","is","a","string"]] + >>> dataset = ds.NumpySlicesDataset(data) + >>> dataset = dataset.map(operations=concatenate_op) + """ + + @check_concat_type + def __init__(self, axis=0, prepend=None, append=None): + super().__init__() + self.axis = axis + self.prepend = cde.Tensor(np.array(prepend)) if prepend is not None else prepend + self.append = cde.Tensor(np.array(append)) if append is not None else append + self.implementation = Implementation.C + + def parse(self): + return cde.ConcatenateOperation(self.axis, self.prepend, self.append) + + +class Duplicate(TensorOperation): + """ + Duplicate the input tensor to output, only support transform one column each time. + + Raises: + RuntimeError: If given tensor has two columns. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Data before + >>> # | x | + >>> # +---------+ + >>> # | [1,2,3] | + >>> # +---------+ + >>> data = [[1,2,3]] + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data, ["x"]) + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms.Duplicate(), + ... input_columns=["x"], + ... output_columns=["x", "y"], + ... column_order=["x", "y"]) + >>> # Data after + >>> # | x | y | + >>> # +---------+---------+ + >>> # | [1,2,3] | [1,2,3] | + >>> # +---------+---------+ + """ + + def __init__(self): + super().__init__() + self.implementation = Implementation.C + + def parse(self): + return cde.DuplicateOperation() + + +class Fill(TensorOperation): + """ + Tensor operation to fill all elements in the tensor with the specified value. + The output tensor will have the same shape and type as the input tensor. + + Args: + fill_value (Union[str, bytes, int, float, bool]) : scalar value + to fill the tensor with. + + Raises: + TypeError: If `fill_value` is not of type str, float, bool, int or bytes. + + Supported Platforms: + ``CPU`` + + + Examples: + >>> import numpy as np + >>> # generate a 1D integer numpy array from 0 to 4 + >>> def generator_1d(): + ... for i in range(5): + ... yield (np.array([i]),) + >>> generator_dataset = ds.GeneratorDataset(generator_1d, column_names="col1") + >>> # [[0], [1], [2], [3], [4]] + >>> fill_op = transforms.Fill(3) + >>> generator_dataset = generator_dataset.map(operations=fill_op) + >>> # [[3], [3], [3], [3], [3]] + """ + + @check_fill_value + def __init__(self, fill_value): + super().__init__() + self.fill_value = cde.Tensor(np.array(fill_value)) + self.implementation = Implementation.C + + def parse(self): + return cde.FillOperation(self.fill_value) + + +class Mask(TensorOperation): + r""" + Mask content of the input tensor with the given predicate. + Any element of the tensor that matches the predicate will be evaluated to True, otherwise False. + + Args: + operator (Relational): relational operators, it can be any of [Relational.EQ, Relational.NE, Relational.LT, + Relational.GT, Relational.LE, Relational.GE], take Relational.EQ as example, EQ refers to equal. + constant (Union[str, int, float, bool]): Constant to be compared to. + dtype (mindspore.dtype, optional): Type of the generated mask. Default: mindspore.dtype.bool\_. + + Raises: + TypeError: `operator` is not of type Relational. + TypeError: `constant` is not of type string int, float or bool. + TypeError: `dtype` is not of type mindspore.dtype. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.transforms import Relational + >>> # Data before + >>> # | col | + >>> # +---------+ + >>> # | [1,2,3] | + >>> # +---------+ + >>> data = [[1, 2, 3]] + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data, ["col"]) + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms.Mask(Relational.EQ, 2)) + >>> # Data after + >>> # | col | + >>> # +--------------------+ + >>> # | [False,True,False] | + >>> # +--------------------+ + """ + + @check_mask_op_new + def __init__(self, operator, constant, dtype=mstype.bool_): + super().__init__() + self.operator = operator + self.dtype = mstype_to_detype(dtype) + self.constant = cde.Tensor(np.array(constant)) + self.implementation = Implementation.C + + def parse(self): + return cde.MaskOperation(DE_C_RELATIONAL.get(self.operator), self.constant, self.dtype) + + +class OneHot(TensorOperation): + """ + Tensor operation to apply one hot encoding. + + Args: + num_classes (int): Number of classes of objects in dataset. + It should be larger than the largest label number in the dataset. + smoothing_rate (float, optional): Adjustable hyperparameter for label smoothing level. + (Default=0.0 means no smoothing is applied.) + + Raises: + TypeError: `num_classes` is not of type int. + TypeError: `smoothing_rate` is not of type float or int. + ValueError: `smoothing_rate` is not in range [0.0, 1.0]. + RuntimeError: Input tensor is not of type int. + RuntimeError: Input tensor is not a 1-D tensor. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Assume that dataset has 10 classes, thus the label ranges from 0 to 9 + >>> onehot_op = transforms.OneHot(num_classes=10) + >>> mnist_dataset = mnist_dataset.map(operations=onehot_op, input_columns=["label"]) + """ + + @check_one_hot_op + def __init__(self, num_classes, smoothing_rate=0.0): + super().__init__() + self.num_classes = num_classes + self.random = False + self.smoothing_rate = smoothing_rate + + def parse(self): + return cde.OneHotOperation(self.num_classes, self.smoothing_rate) + + +class PadEnd(TensorOperation): + """ + Pad input tensor according to pad_shape, input tensor needs to have same rank. + + Args: + pad_shape (list(int)): List of integers representing the shape needed. Dimensions that set to `None` will + not be padded (i.e., original dim will be used). Shorter dimensions will truncate the values. + pad_value (Union[str, bytes, int, float, bool], optional): Value used to pad. Default to 0 or empty + string in case of tensors of strings. + + Raises: + TypeError: If `pad_shape` is not of type list. + TypeError: If `pad_value` is not of type str, float, bool, int or bytes. + TypeError: If elements of `pad_shape` is not of type int. + ValueError: If elements of `pad_shape` is not of positive. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Data before + >>> # | col | + >>> # +---------+ + >>> # | [1,2,3] | + >>> # +---------| + >>> data = [[1, 2, 3]] + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data, ["col"]) + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms.PadEnd(pad_shape=[4], + ... pad_value=10)) + >>> # Data after + >>> # | col | + >>> # +------------+ + >>> # | [1,2,3,10] | + >>> # +------------| + """ + + @check_pad_end + def __init__(self, pad_shape, pad_value=None): + super().__init__() + self.pad_shape = cde.TensorShape(pad_shape) + self.pad_value = cde.Tensor(np.array(pad_value)) if pad_value is not None else pad_value + self.implementation = Implementation.C + + def parse(self): + return cde.PadEndOperation(self.pad_shape, self.pad_value) + + +class Plugin(TensorOperation): + """ + Plugin support for MindData. Use this class to dynamically load a .so file (shared library) and execute its symbols. + + Args: + lib_path (str): Path to .so file which is compiled to support MindData plugin. + func_name (str): Name of the function to load from the .so file. + user_args (str, optional): Serialized args to pass to the plugin. Only needed if "func_name" requires one. + + Raises: + TypeError: If `lib_path` is not of type string. + TypeError: If `func_name` is not of type string. + TypeError: If `user_args` is not of type string. + + Supported Platforms: + ``CPU`` + + Examples: + >>> plugin = transforms.Plugin("pluginlib.so", "PluginDecode") + >>> image_folder_dataset = image_folder_dataset.map(operations=plugin) + """ + + @check_plugin + def __init__(self, lib_path, func_name, user_args=None): + super().__init__() + self.lib_path = lib_path + self.func_name = func_name + self.user_args = str() if (user_args is None) else user_args + self.implementation = Implementation.C + + def parse(self): + return cde.PluginOperation(self.lib_path, self.func_name, self.user_args) + + +class RandomApply(CompoundOperation): + """ + Randomly perform a series of transforms with a given probability. + + Args: + transforms (list): List of transformations to be applied. + prob (float, optional): The probability to apply the transformation list (default=0.5). + + Raises: + TypeError: If `transforms` is not of type list. + ValueError: If `transforms` is empty. + TypeError: If elements of `transforms` are neither Python callable objects nor data + processing operations in transforms.py. + TypeError: If `prob` is not of type float. + ValueError: If `prob` is not in range [0.0, 1.0]. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.transforms import Compose + >>> transforms_list = [vision.RandomHorizontalFlip(0.5), + ... vision.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)), + ... vision.RandomErasing()] + >>> composed_transform = Compose([vision.Decode(to_pil=True), + ... transforms.RandomApply(transforms_list, prob=0.6), + ... vision.ToTensor()]) + >>> image_folder_dataset = image_folder_dataset.map(operations=composed_transform, input_columns=["image"]) + """ + + @check_random_transform_ops + def __init__(self, transforms, prob=0.5): + super().__init__(transforms) + self.prob = prob + + def execute_py(self, img): + """ + Execute method. + + Args: + img (PIL image): Image to be randomly applied a list transformations. + + Returns: + img (PIL image), Transformed image. + """ + return util.random_apply(img, self.transforms, self.prob) + + def parse(self): + operations = self.parse_transforms() + return cde.RandomApplyOperation(self.prob, operations) + + +class RandomChoice(CompoundOperation): + """ + Randomly select one transform from a list of transforms to perform operation. + + Args: + transforms (list): List of transformations to be chosen from to apply. + + Raises: + TypeError: If `transforms` is not of type list. + ValueError: If `transforms` is empty. + TypeError: If elements of `transforms` are neither Python callable objects nor data + processing operations in transforms.py. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.transforms import Compose + >>> transforms_list = [vision.RandomHorizontalFlip(0.5), + ... vision.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)), + ... vision.RandomErasing()] + >>> composed_transform = Compose([vision.Decode(), + ... transforms.RandomChoice(transforms_list), + ... vision.ToTensor()]) + >>> image_folder_dataset = image_folder_dataset.map(operations=composed_transform, input_columns=["image"]) + + """ + + @check_random_transform_ops + def __init__(self, transforms): + super().__init__(transforms) + + def execute_py(self, img): + """ + Execute method. + + Args: + img (PIL image): Image to be applied transformation. + + + Returns: + img (PIL image), Transformed image. + """ + return util.random_choice(img, self.transforms) + + def parse(self): + operations = self.parse_transforms() + return cde.RandomChoiceOperation(operations) + + +class RandomOrder(PyTensorOperation): + """ + Perform a series of transforms to the input image in a random order. + + Args: + transforms (list): List of the transformations to apply. + + Raises: + TypeError: If `transforms` is not of type list. + TypeError: If elements of `transforms` are neither Python callable objects nor data + processing operations in mindspore.dataset.transforms.transforms. + ValueError: If `transforms` is empty. + + Supported Platforms: + ``CPU`` + + Examples: + >>> from mindspore.dataset.transforms import Compose + >>> transforms_list = [vision.RandomHorizontalFlip(0.5), + ... vision.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)), + ... vision.RandomErasing()] + >>> composed_transform = Compose([vision.Decode(to_pil=False), + ... transforms.RandomOrder(transforms_list), + ... vision.ToTensor()]) + >>> image_folder_dataset = image_folder_dataset.map(operations=composed_transform, input_columns=["image"]) + """ + + @check_random_transform_ops + def __init__(self, transforms): + super().__init__() + self.transforms = transforms + self.implementation = Implementation.PY + + def execute_py(self, img): + """ + Execute method. + + Args: + img (PIL image): Image to apply transformations in a random order. + + Returns: + img (PIL image), Transformed image. + """ + return util.random_order(img, self.transforms) + + +class Relational(IntEnum): + """ + Relationship operator. + + Possible enumeration values are: Relational.EQ, Relational.NE, Relational.GT, Relational.GE, Relational.LT, + Relational.LE. + + - Relational.EQ: refers to Equality. + - Relational.NE: refers not equal, or Inequality. + - Relational.GT: refers to Greater than. + - Relational.GE: refers to Greater than or equal to. + - Relational.LT: refers to Less than. + - Relational.LE: refers to Less than or equal to. + """ + EQ = 0 + NE = 1 + GT = 2 + GE = 3 + LT = 4 + LE = 5 + + +DE_C_RELATIONAL = {Relational.EQ: cde.RelationalOp.EQ, + Relational.NE: cde.RelationalOp.NE, + Relational.GT: cde.RelationalOp.GT, + Relational.GE: cde.RelationalOp.GE, + Relational.LT: cde.RelationalOp.LT, + Relational.LE: cde.RelationalOp.LE} + + +class _SliceOption(cde.SliceOption): + """ + Internal class SliceOption to be used with SliceOperation + + Args: + _SliceOption(Union[int, list(int), slice, None, Ellipsis, bool, _SliceOption]): + + 1. :py:obj:`int`: Slice this index only along the dimension. Negative index is supported. + 2. :py:obj:`list(int)`: Slice these indices along the dimension. Negative indices are supported. + 3. :py:obj:`slice`: Slice the generated indices from the slice object along the dimension. + 4. :py:obj:`None`: Slice the whole dimension. Similar to :py:obj:`:` in Python indexing. + 5. :py:obj:`Ellipsis`: Slice the whole dimension. Similar to :py:obj:`:` in Python indexing. + 6. :py:obj:`boolean`: Slice the whole dimension. Similar to :py:obj:`:` in Python indexing. + """ + + @check_slice_option + def __init__(self, slice_option): + if isinstance(slice_option, int) and not isinstance(slice_option, bool): + slice_option = [slice_option] + elif slice_option is Ellipsis: + slice_option = True + elif slice_option is None: + slice_option = True + super().__init__(slice_option) + + +class Slice(TensorOperation): + """ + Slice operation to extract a tensor out using the given n slices. + + The functionality of Slice is similar to NumPy's indexing feature (Currently only rank-1 tensors are supported). + + Args: + slices (Union[int, list[int], slice, None, Ellipsis]): + Maximum `n` number of arguments to slice a tensor of rank `n` . + One object in slices can be one of: + + 1. :py:obj:`int`: Slice this index only along the first dimension. Negative index is supported. + 2. :py:obj:`list(int)`: Slice these indices along the first dimension. Negative indices are supported. + 3. :py:obj:`slice`: Slice the generated indices from the + `slice `_ object along the + first dimension. Similar to start:stop:step. + 4. :py:obj:`None`: Slice the whole dimension. Similar to :py:obj:`[:]` in Python indexing. + 5. :py:obj:`Ellipsis`: Slice the whole dimension, same result with `None`. + + Raises: + TypeError: If `slices` is not of type int, list[int], :py:obj:`slice`, :py:obj:`None` or :py:obj:`Ellipsis`. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Data before + >>> # | col | + >>> # +---------+ + >>> # | [1,2,3] | + >>> # +---------| + >>> data = [[1, 2, 3]] + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data, ["col"]) + >>> # slice indices 1 and 2 only + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms.Slice(slice(1,3))) + >>> # Data after + >>> # | col | + >>> # +---------+ + >>> # | [2,3] | + >>> # +---------| + """ + + @check_slice_op + def __init__(self, *slices): + super().__init__() + slice_input_ = list(slices) + slice_input_ = [_SliceOption(slice_dim) for slice_dim in slice_input_] + self.slice_input_ = slice_input_ + self.implementation = Implementation.C + + def parse(self): + return cde.SliceOperation(self.slice_input_) + + +class TypeCast(TensorOperation): + """ + Tensor operation to cast to a given MindSpore data type or NumPy data type. + + Note: + This operation supports running on Ascend or GPU platforms by Offload. + + Args: + data_type (Union[mindspore.dtype, numpy.dtype]): mindspore.dtype or numpy.dtype (e.g. :class:`numpy.float32`) + to be cast to. + + Raises: + TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type :class:`numpy.dtype`. + + Supported Platforms: + ``CPU`` ``Ascend`` ``GPU`` + + Examples: + >>> import numpy as np + >>> from mindspore import dtype as mstype + >>> + >>> # Generate 1d int numpy array from 0 - 63 + >>> def generator_1d(): + ... for i in range(64): + ... yield (np.array([i]),) + >>> + >>> dataset = ds.GeneratorDataset(generator_1d, column_names='col') + >>> type_cast_op = transforms.TypeCast(mstype.int32) + >>> dataset = dataset.map(operations=type_cast_op) + """ + + @check_type_cast + def __init__(self, data_type): + super().__init__() + if isinstance(data_type, typing.Type): + data_type = mstype_to_detype(data_type) + else: + data_type = nptype_to_detype(data_type) + self.data_type = str(data_type) + self.implementation = Implementation.C + + def parse(self): + return cde.TypeCastOperation(self.data_type) + + +class Unique(TensorOperation): + """ + Perform the unique operation on the input tensor, only support transform one column each time. + + Return 3 tensor: unique output tensor, index tensor, count tensor. + + - Output tensor contains all the unique elements of the input tensor + in the same order that they occur in the input tensor. + - Index tensor that contains the index of each element of the input tensor in the unique output tensor. + - Count tensor that contains the count of each element of the output tensor in the input tensor. + + Note: + Call batch op before calling this function. + + Raises: + RuntimeError: If given Tensor has two columns. + + Supported Platforms: + ``CPU`` + + Examples: + >>> # Data before + >>> # | x | + >>> # +--------------------+ + >>> # | [[0,1,2], [1,2,3]] | + >>> # +--------------------+ + >>> data = [[[0,1,2], [1,2,3]]] + >>> dataset = ds.NumpySlicesDataset(data, ["x"]) + >>> dataset = dataset.map(operations=transforms.Unique(), + ... input_columns=["x"], + ... output_columns=["x", "y", "z"], + ... column_order=["x", "y", "z"]) + >>> # Data after + >>> # | x | y |z | + >>> # +---------+-----------------+---------+ + >>> # | [0,1,2,3] | [0,1,2,1,2,3] | [1,2,2,1] + >>> # +---------+-----------------+---------+ + """ + + def __init__(self): + super().__init__() + self.implementation = Implementation.C + + def parse(self): + return cde.UniqueOperation() -- 2.34.1 From 5b717dbe1e08b1613c75456955aef130c24482b7 Mon Sep 17 00:00:00 2001 From: saltyfish Date: Tue, 3 Oct 2023 09:26:42 +0800 Subject: [PATCH 72/72] ADD file via upload --- .../ccsrc/transform-update/browse_dataset.py | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 mindspore/ccsrc/transform-update/browse_dataset.py diff --git a/mindspore/ccsrc/transform-update/browse_dataset.py b/mindspore/ccsrc/transform-update/browse_dataset.py new file mode 100644 index 00000000000..6de43a86e09 --- /dev/null +++ b/mindspore/ccsrc/transform-update/browse_dataset.py @@ -0,0 +1,197 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Visualization for detection/segmentation dataset. +检测/分割数据集的可视化。 +""" +import os +import sys +import importlib +import numpy as np + +from mindspore import log as logger + + +def imshow_det_bbox(image, bboxes, labels, segm=None, class_names=None, score_threshold=0, bbox_color=(0, 255, 0), + text_color=(203, 192, 255), mask_color=(128, 0, 128), thickness=2, font_size=0.8, show=True, + win_name="win", wait_time=2000, out_file=None): + """Draw an image with given bboxes and class labels (with scores). + 用给定的框和类标签(带分数)绘制一幅图像。 + Args: + image (ndarray): The image to be displayed, shaped (C, H, W) or (H, W, C), formatted RGB. + bboxes (ndarray): Bounding boxes (with scores), shaped (N, 4) or (N, 5), + data should be ordered with (N, x, y, w, h). + labels (ndarray): Labels of bboxes, shaped (N, 1). + segm (ndarray): The segmentation masks of image in M classes, shaped (M, H, W) (Default=None). + class_names (list[str], tuple[str], dict): Names of each class to map label to class name + (Default=None, only display label). + score_threshold (float): Minimum score of bboxes to be shown (Default=0). + bbox_color (tuple(int)): Color of bbox lines. + The tuple of color should be in BGR order (Default=(0, 255 ,0), means 'green'). + text_color (tuple(int)): Color of texts. + The tuple of color should be in BGR order (Default=(203, 192, 255), means 'pink'). + mask_color (tuple(int)): Color of mask. + The tuple of color should be in BGR order (Default=(128, 0, 128), means 'purple'). + thickness (int): Thickness of lines (Default=2). + font_size (int, float): Font size of texts (Default=0.8). + show (bool): Whether to show the image (Default=True). + win_name (str): The window name (Default="win"). + wait_time (int): Value of waitKey param (Default=2000, means display interval is 2000ms). + out_file (str, optional): The filename to write the imagee (Default=None). File extension name + is required to indicate the image compression type, e.g. 'jpg', 'png'. + + Returns: + ndarray: The image with bboxes drawn on it. + + Raises: + ImportError: If opencv-python is not installed. + AssertionError: If `image` is not in (H, W, C) or (C, H, W) format. + AssertionError: If `bboxes` is not in (N, 4) or (N, 5) format. + AssertionError: If `labels` is not in (N, 1) format. + AssertionError: If `segm` is not in (M, H, W) format. + AssertionError: If `class_names` is not of type list, tuple or dict. + AssertionError: If `bbox_color` is not a tuple in format of (B, G, R). + AssertionError: If `text_color` is not a tuple in format of (B, G, R). + AssertionError: If `mask_color` is not a tuple in format of (B, G, R). + + Examples: + >>> import numpy as np + >>> import mindspore.dataset as ds + >>> from mindspore.dataset.utils.browse_dataset import imshow_det_bbox + >>> + >>> # Read Detection dataset, such as VOC2012. + >>> voc_dataset_dir = "/path/to/voc_dataset_directory" + >>> dataset = ds.VOCDataset(voc_dataset_dir, task="Detection", shuffle=False, decode=True, num_samples=5) + >>> dataset_iter = dataset.create_dict_iterator(output_numpy=True, num_epochs=1) + >>> + >>> # draw dataset + >>> for index, data in enumerate(dataset_iter): + ... image = data["image"] + ... bbox = data["bbox"] + ... label = data["label"] + ... # draw image with bboxes + ... imshow_det_bbox(image, bbox, label, + ... class_names=['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', + ... 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', + ... 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'], + ... win_name="my_window", + ... wait_time=5000, + ... show=True, + ... out_file="voc_dataset_{}.jpg".format(str(index))) + + Examples using `imshow_det_bbox` on VOC2012: + + .. image:: browse_dataset.png + + """ + + # 定义一个名为imshow_det_bbox的函数 +def imshow_det_bbox(image, bboxes, labels, segm=None, class_names=None, score_threshold=0, bbox_color=(0, 255, 0), + text_color=(203, 192, 255), mask_color=(128, 0, 128), thickness=2, font_size=0.8, show=True, + win_name="win", wait_time=2000, out_file=None): + try: + # 尝试导入cv2模块(OpenCV库),如果导入失败,抛出ImportError异常 + cv2 = importlib.import_module("cv2") + except ModuleNotFoundError: + raise ImportError("Importing cv2 failed, try to install it by running `pip install opencv-python`.") + + # 数据验证和参数检查 + assert isinstance(image, np.ndarray) and image.ndim == 3 and (image.shape[0] == 3 or image.shape[2] == 3), \ + "image must be a ndarray in (H, W, C) or (C, H, W) format." + if bboxes is not None: + assert isinstance(bboxes, np.ndarray) and bboxes.ndim == 2 and (bboxes.shape[1] == 4 or bboxes.shape[1] == 5), \ + "bboxes must be a ndarray in (N, 4) or (N, 5) format." + assert isinstance(labels, np.ndarray) and labels.ndim == 2 and labels.shape[1] == 1 and \ + labels.shape[0] == bboxes.shape[ + 0], "labels must be a ndarray in (N, 1) format and has same N with bboxes." + if segm is not None: + assert isinstance(segm, np.ndarray) and segm.ndim == 3, "segm must be a ndarray in (M, H, W) format." + H, W = (image.shape[0], image.shape[1]) if image.shape[2] == 3 else (image.shape[1], image.shape[2]) + assert H == segm.shape[1] and W == segm.shape[2], "segm must has same height and width with image." + if bboxes is not None: + assert bboxes.shape[0] <= segm.shape[0], "number of segm masks must not be less than the number of bboxes." + assert isinstance(class_names, (tuple, list, dict)), "class_names must be a list, tuple or dict." + assert isinstance(bbox_color, tuple) and len(bbox_color) == 3, \ + "bbox_color must be a three tuple, formatted (B, G, R)." + assert isinstance(text_color, tuple) and len(text_color) == 3, \ + "text_color must be a three tuple, formatted (B, G, R)." + assert isinstance(mask_color, tuple) and len(mask_color) == 3, \ + "mask_color must be a three tuple, formatted (B, G, R)." + assert isinstance(thickness, int), "thickness must be an int." + assert thickness >= 0, "thickness must be larger than or equal to zero." + assert isinstance(font_size, (int, float)), "font_size must be an int or float." + assert font_size >= 0, "font_size must be larger than or equal to zero." + assert isinstance(show, bool), "show must be a bool." + assert isinstance(win_name, str), "win_name must be a str." + assert isinstance(wait_time, int), "wait_time must be an int." + assert wait_time >= 0, "wait_time must be larger than or equal to zero." + if out_file is not None: + assert isinstance(out_file, str), "out_file must be a str." + + if score_threshold > 0: + assert bboxes.shape[1] == 5 + if not show: + assert out_file is not None + + # 对图像进行处理 + if image.shape[0] == 3: + image = image.transpose((1, 2, 0)) + draw_image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) + + if bboxes is not None: + bbox_num = bboxes.shape[0] + for i in range(bbox_num): + draw_bbox = bboxes[i] + if len(draw_bbox) > 4: + if draw_bbox[4] < score_threshold: + continue + # bbox + x1, y1 = int(draw_bbox[0]), int(draw_bbox[1]) + x2, y2 = int(draw_bbox[0] + draw_bbox[2]), int(draw_bbox[1] + draw_bbox[3]) + cv2.rectangle(draw_image, (x1, y1), (x2, y2), bbox_color, thickness) + # label + try: + draw_label = str(class_names[labels[i][0]]) if class_names is not None else f'class {labels[i][0]}' + except (IndexError, KeyError): + draw_label = f'class {labels[i][0]}' + if len(draw_bbox) > 4: + draw_label += f'|{draw_bbox[-1]:.02f}' + cv2.putText(draw_image, draw_label, (x1, y2), cv2.FONT_HERSHEY_SIMPLEX, font_size, text_color, thickness) + if segm is not None: + mask = segm[i].astype(bool) + draw_image[mask] = draw_image[mask] * 0.5 + np.array(mask_color) * 0.5 + else: + if segm is not None: + segm_num = segm.shape[0] + for i in range(segm_num): + mask = segm[i].astype(bool) + draw_image[mask] = draw_image[mask] * 0.5 + np.array(mask_color) * 0.5 + # 如果需要显示图像,调用cv2.imshow方法显示图像 + if show: + cv2.imshow(win_name, draw_image) + if cv2.waitKey(wait_time) == 27: + sys.exit() + + # 如果需要保存图像到文件,使用cv2.imwrite方法保存图像 + if out_file: + logger.info("Saving image file with name: " + out_file + "...") + cv2.imwrite(out_file, draw_image) + os.chmod(out_file, 0o600) + + # 返回处理后的图像 + return draw_image + + + + -- 2.34.1