From 8c9009861262e9f01e8e0ace9a2f71d0ea735475 Mon Sep 17 00:00:00 2001 From: Lyu Muyang <87132523+senngadaisuki@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:53:04 +0800 Subject: [PATCH] Update documentation (#49) * quickstart snn_simulation-zh * fix a bug in snn_sim * 3 en version --------- Co-authored-by: LyuMuyang <11932083+lyumuyang@user.noreply.gitee.com> --- docs/_static/snn-simulation1.png | Bin 0 -> 89624 bytes docs/quickstart/ann_training-en.ipynb | 374 +++++++++- docs/quickstart/snn_simulation-en.ipynb | 659 ++++++++++++++++- docs/quickstart/snn_simulation-zh.ipynb | 668 +++++++++++++++++- .../artificial_neural_networks-en.ipynb | 514 +++++++++++++- 5 files changed, 2187 insertions(+), 28 deletions(-) create mode 100644 docs/_static/snn-simulation1.png diff --git a/docs/_static/snn-simulation1.png b/docs/_static/snn-simulation1.png new file mode 100644 index 0000000000000000000000000000000000000000..72772d3cee13ab632b1069249a71dffdce903226 GIT binary patch literal 89624 zcmeFYRZtvX_%_%;NP;H>2@oWBa0v_!!6it7+YlUr3=V_Gkq{h$TW}d%XMh9 z0RY?$xr>Lr;@n|51OPaSI?wf#u&4ho{sw+uUk?nljiS+QZPi^13)R&H4MRf(1);Uq zDaCEqp`mRlDSpLhw4dMSRG3@F0_^kW1vfW`xS^2GSy>?=LkUaAs&XbF&-E-q(C7tbHG!31t}p)o!(5}_q#Y>`syj&3elXEsj0Y?x9N_+y!saP>4GxI$13Ih! zbNIhPfF25_@4;pPK)m13+HFXM0~ihxg&yGk&*6U{_`k^lxj&O{h_M;d&J?m85NLb< ze-h?@Cd>chWa&g9%9=?OEFadXbKZaN#?C;C8!UymhEuC+lsBFa-?D*B&~gM^?Y+ZK zr3QM`4{)=gIH6@v`GNmA{og@DET_MlF14YJ;0DKsJJt24B*@u?TL6*>_^dzc+d!Sx zppBYX6T%5y!v}7c>=d32uJJ}6cQE_o48wtTxw^}j3zKkLN6o{3gnXBoflN}THfVrH zi2g~W%`$GUY9d=Mf+zqiSpw~ICR2s(2gfQ(|CoE|?8^IRv=MrqmrMQ6rf5Ty3bYP- zp0=G#a3fGD$mE~AZ*_o5P5C!KQCSJCfiz|NCoL9MEftAsg#NEHxQze$H3a1}Tk-&K z00T?{GXBeJh_bkUf{)aUTQ;K74EO)}DZF8N#2||17y|xz8GdJG2n2y1dJ6uZg1nI{ z_;T0^_vt~1TN4D_`!AkBW`Cmw2FkZXXAJH6Dg?yzVXz}tU<>AdIbKTx%8en zCCD@cwV3taIWaY?&iw$D#nTDgB<}YmtS=4f{|N?aFd`(8+|YlBHc3HkL9bCW^%ehu zSY%q-n1`VHXGFJd-B2W!yT6f(aXk26q7p$}C`?+z^_~mv4SbKrcfDpo5aaU~5C3yX za^&We^*^+dVJ@XU|Ji8=9q&5iTRtRgcjIC!V;V^#j-dD$1czTEys16wJ&BdfmN4^c z`1_vnq;iQLt0y37zG-k>;Ks&WOHSFqWdYRFIHei5aj7vNIkzEQ?Y}t3xRMi3bJEq) z4&MC-U*goDdw)tYk>{&_h_QJz$^S<|YSe!cPfODJL|pl9fX@blRM7W;8`Qlw+5bbF zc6{i6jobRTfyB89OD8Wl3I1>Vix>atG9X!v5d25wmPi7>rujcd^_*k=8>RohG}Lk> z6Rmyv=W`L5kIs|vpCyv1|M(j&a^=cW`^U&|g#IDvf2PX$GNlRD^v zuop1irDU?QXMS4H(I~d|_n!6P8O;qe7g1MY2qcJVSDQ;LzczA&&m#0%3mZ&ZS#(Rn zK3xE&Vn*yUs*@^wPdnL!*3Fg-Y`_zl_j$mL>s#(qVexHv(lKJJJ8Q$$iPAm-@LkrZXewmVRlSn`xD|u2@%1S?7 zeLk_ZUCrM2w_^|zye>PLf=iM%Q#WsIk#9nJ!$ZV#b(pnwCt4G*W`TJZ5ZHtoj zDyCa5VMz6cOa4T+?mSBwB5noV6=IoWtcc^OEtII=4Kka2*AS6r*An2YU27(`K`h{W zGuw`LdHA&-`7|KeE4HePy9>kxgG#M-Ik3|CP@KhI8B;CipVnO8SDmYzx%%FjClZiO zrCyW_@co;rntI#KNJ;*E3{r`j?5z&ye(c>S65d-hHV&5Wm8l$QX>2%YTd%KMHHHSC zH|H7@=Tnxii-UrQPxDcj%j_|&&G~UF9}??<=_8fV2Qp{Myq<%=rx9$)rJ-+YYXJ`^ zzeAPNYOa-@Y+RyVGVoL>(1^kMkLj?HCo&d=((9t>pBRCPqgVA z;tVmbK*DjLFZT8b97>E8i)lGiIR9$AwpnCo@@~(Wl2LFi5HdjXiXWRQE0=FTgm-3m z`qNvkA3i$hYS{7QZ)dyxKJiJb55V!cDT-RVna#hMwog}l z$b_@L$x8VBnr2Z3OtE>(bZ7kMn+Hf4bOEgvRhnDamnZh2pc}_aO!6p6@9`Jr`-O9o zUG4?alMV;PApqH%c-3B$WyH+eS*D+UeYPHBZqr1)NQw3A5$tMgq=i~AwcLikJ>Z57 zs&TSLW6JIM(J0asPgfE69MNCd8xDp1cFjU<9!$VxToL$qm zm@|@fch%9L!z$`m4K17(!^LY;@bY_#{J=NPvj=E@EMId!7P#mV5DG`+4Z|Cu-ZEX-$0FFl(%I4-? z&S_RRLUQgLxWxORHX5NhPLzPs^*>}!18MBnV}YI4V`MRfUkqot{E%^3FAJ@9`uJY! zXmQQu)B7)%_W`jq6Jf#tXFHj_B#l0r^?FYOx!!ywi2s2owliX-@F{_=@}kIT2bL>U z43BVEC)Cv@{#jLhF;=|h_nQ|Lx1}w5UOEZ0mkUSC|8zmSplVm59%%rf2aQfGrd@n_ zwo@$rdI&jazL~j}DCBI_9$4h*awh0zF*bWv|@F6@Vg9n5d&FmJn(|aFCT)Ca;&Ncj>Rky|p^p(GcsmlyLF{kN|T+vTOJAjs0$^xZN&N!6L#}diL-M z$|?CSDrl*z(lF*~j=(7yNN%c(gVRIq8XJ{d-VPMmdruyx+R--dt(`d|I-=GY8zgAj0wYw?+~6x?v9{ORtujNxw!3`DJsuEQ2O=t`f#nBq03_&jRx zIp5i-?>D;v07tQC2dPF^0NYuA%;C{w^w4?W*}jTuSRo?!`EsTCWT>zXuI7yg2lt>c zn3<@!u9Ne1vY}0xAhqBf^Wy;g&7q}A^Vv{gw{r>0!U`-Ux^v#rJpo5{zwNJ~(7}xk zSa>ud&5~aUhm0E9aJLaUo-sl<&JD<2p&xXOG8pEzsZ*f4*A?J_M6xM|gu-SYcABwC zGi$JO1IVJ$!Cci^si0XQ1*F8{>Mo4ezl+9&SPYL%e+QdsJ=5mhnm8tk+scoN(I#d|;H(*g9WdX9+r?1shY-ATFDT2Ia(E!cofj zfY?hGLes8zhDmF3@agg1sb5>5wj#v+c4?(~KQ^+_typYa>0`5&Eh$PUCm{k@V6aT@ zCy0^?uic)vXeEzM4>-L!{Go>_TEyxk}ndje%ji99=yc2ZMQp2 zcWnasAW9AR_goqXTyrL=T(K-%G)TP^y5?L7&YQ=+R8Z31Z>{brO+v?FR@=={(Y~j@ zyeCS1xWCt*)ZdX)N-=XCjc!QdK_0(zlSy7Klmmb(%LLY#NssJ=R1OOU(m8!GsP>Ou zW(!1r6&Q%BpNHk=lzgUuQKl+Mh1Rq4gln4D4YqFMCgkOnznkjB_H;M8G6b`T)**5LAwKSEb-i5S0h;C^ie|7 zr})4(rTQN=b}nNqVQ_0nggFO}mb5Ygi0kWFeL&~aoeOzW%n!)+Ejmt(Q%dOo8ItP0w z>-%zR)kkdBh=AcJOZDvCFL?OlwkwyN=7_Wvxa+aVjnN7I7RMysk@x1PKF_(kKz39* z+cPj3J!dgLOR_5kL%e5Yk70}T7}2YYYUuc1ap?G~=M^T5D0^DcQH4lp&G*xM6MmuMkWz09hN7{c=5>|xg%!HKGuO&19+9khk*{cgPc zEtAURoQzDb!T~~GR#P^Ydg&hg1dFPiHlI$Y#T0a7#STA@fAP%T@#GhgzF<#))_dFDm1TYG#(nAHcLb62aOMAz~uEJV1 zR>8q~w*c0W0$v<2hQ4S=FHqaHQ>$}boG3hG%rw`9qWwa&fs07idbgIYMPJDo#8*Mh zaAwjal~oCZ+6bx>Lq*2vXV;>F^Dtv-=^cgyuU18YSESSO##Fzy%~<_wC5@MzM^;Ry zmxQT+m`CD~k6{Hqoggn3U%pmJ4(Mw`avLW+;D?|nwg0eGBZQG#cs4`oz{NL{+J9|; zh6Rul`brF~8>d*%2Kg)Oji=w5!8DrOVw%7~d#>lc81F$%dq=xtvs@{KPyrlt)aARV zjJAKv2ns2hcdxtKZbb(-m|5QS_&#aZLJd?JK+_(>T75b(i$52cdZW`qBoB+KaRnGz zEUIn!@LZO6Exu!n*K`X8Xn;y~a^*lSwIu=q78#9egx39hVW|== zo}_{0yjpp9z>=-d%6R$-!@SX>Fm#_A_z` z4cw#4E7PxKTKpqie}y9_VD z3wqk+yYh3HeUk_gGIma$t(r&~I@GR#G8e;;2jBo|R_dX_^KX=QS4M}UdYewjgSSe$ ziow7GTz;r;@7($GKls4=2j~}I{Ca_^u;9Sth`=Q$@!tAtuH`=$w?irVDR>Ch%RQ5p zF6WKwqhWW*zi7q_1583`^`8SYPn=0TA7CltS|B6ROGPg7^nrO}lnA55@QC?E%B22^(TM7f*3aqmA>id>x-HEE zVg6J@%kw{AdCt94BEZOCBF^2suk`77qBdw3VuO;?B&e*zk@Hk8>QKbYv6|Ta*@5>y z7nL3aLi1Nv>Gb)E<%To~1?fFRN{IBPWM{GI0O$oCAz{HxTXyHj13r!AOa6WJ$8t(aH3db8H?{Yf-V{BKs zFV%ztypJm%o=jJ^LFF%is6_#X?*hCjyF31{h^ZYg@dmctE0Y^6fQctgK(jDoFSK^P zYouMp zu}Q5M9KgKc`Fat#9te>Jk=COI@fiUdmu2fTX*`TQK1O4<1s|EI&Ef zr66+X%7PAc6*_SEmG@)Ea(r~`*TcXaN!jj0(j$3}ae=osE;OTtPrtkx8Q^wx=IdH7 z+EV{X2?B~7CTP*w$S_};J&ahgF?M=Ku=8{QQ?zT77#;Q0?dV>7~@!ttM_zS?1epy(Do0w&As_9Adu3_#owA} zbm_Icm;3C)ctSpk(4sQbCN0x%VP;mJ)Arn1ZVMl{B^+V4YCd*lSvr2%X0YvL@tgWL z7*2#x$yLD0Xdpz7kaBNn3XpkrJ5P7j5I`uP{Gk+1RJ7C33?uK{t|uM8IG^td$iWoy zZNne%AI&^9#!(ZI-a~;(bjDoQqsEAU6P1ZLeiP(;t{&vbw)raY?Xz9IKv}DxnLpzG zMFBzul{l}&_c%}uq?$TO`HDti#Gp60r;$f3a`kX^v|=})4>tNa_u1D zY}}4oe>JuWbjYMltW`A55Y9n47o0p;Av!^F5dM7RvMDb)(Phn7$XVgCk0x*33RW!j;>i zVVfyks~k-vOD#mToS2klb{5V|99(D`__*T9k+1Ni6e%o?4>yq8ryk)jC4MH{(eHPD zefCj18pJ}Ttw7*Y*aJrAoP2cc9PGkuIj|S?MnDYYa98?-?&~9)Qw^4!_oj(~=6;h} z!w-{(Hgo+jMvmP!Ej~!5P5Puq!e83SY>Zl9#D(RWPRpv@{TL}4a)`R727)s4HxN33 z>q>DH4*#7cLKb4q(D)$ZnwINqU7B)1pg<5-xQu&B35pDT-%Fte-xT3JN^&#Nr#^Kd=XU6J=9j8npd#VV3=GIycif*Ma{U<<0ehtk9kwa($zkvsD5Lj7_%-m_> z)@zAxlQwVuoi5w6&`fwyHcY;7{)R3iODMM|_W*t@?Mj6c9-Ec!H#@8U?MLmH#s^`* zp_nOmiP(d8E9kVpBbE*|($NyQ&eqo6v=)vYi5eJbIhL&(BS6SGIB`4e#`qapwYZP$ z9^Ky0pWLpE#+3B_E(CL><1i^jP>6o=+2H-^7j&mh?_8n5X608@U*5zl{K0k_@5Im0 zDcin3d&(svts(IQnXn=G>!`_ef8M?_Ku%ccG<)X6T~lkdrSO4ouu60U;&DjSh4(?U zo)mwP_o{_x4s}@a3)Vel2h6}wi!N~m-MHICx_^J!5pJmRXCY7o)%HTr^>4nlz#_d3 z>4ezsA`8*BEy=aZoupx+GQ;oed&+j$lKTU?{YSsU$aibr3I`wXvea9#2-0+VZ;)=x zKsKjDfANy*(8qSAknRIDjkwq0M4TW`6i%x*-~O>Gy#KYA4a2+0o!2j&eQj{lbhm)) zJ9$5%az8Mm&-1rUgU4ol<6OGqV;}e)G1#mOj?6#Fp}t!?p@ahGMXWx@mMu4m*9|NW z^o+Lg0TLTtR|;MVsKuAL)kjiXhagB_&-sg6{qQH8m1iCs_x;~^3p`JGCAVZgF)df{ z!t#SZjN?Iq(e@={0A?co=VX`rBf3$CUk@QC9N@1{Q(9>L3VeB0-p$adD?OnbX{1Cm z;D3#YmLaYeP*D1cm~RP`<$p%p8@({&gcG{+k)9c%?O9wHe0|JwR+^=z{orOSl=%ge?-+r`#UBzLJjt#14(cK8eAhxlH)S8QJk4>v z52*ZRk?*o>!l3tHxtiYuYE(&5hxTXkAJn$bf&s*gx{)9q)4^q|I zz(0Iw?VTueO62GsaxA#Q!|Wi->v^Jk-GF>#zcdoc}=8cU}oY0lgHmBCMVNf8ViV7pa#KX%ur5LfK?S*9_I znc0KUdPihDDn(WST_CG^CxX(5{SY^T#1`69I;j3~X@}(qyejRvqLIUz`fw1t9w4Z^ zva|DKyEvn=Fr1Zc zIg%D|j!pDAS40g`>CG2JCAg%xx{@FooPv-dWd18`hq$RehBOx*W8hEwpSF*(L0x#K z3S!$SHdhhY$MjcqWm3#uUQc`<1azfV$eS~Zj4tRkya$VO9h&T4Ew!P)oAFZl)1=}3 z5?oVdX4o%agp$2Qm4nmh{3XHL^y}+tKqU~{Fxg)_+e+(n97x_2q1p!XCn~(abOpv{`hMa5 zOymPJ*!-&Y52*ka@%G`btR;N@s*?L)s&_I$2oUJdzlJ()L$(AA2$&koxHN98#l60K z2UGt&5yyfcF!bqjx@1Moq)QIpMTEY^Q~E5zUB}%(8n(iCi0raC$-W{o`iLg9k%$O)$Nuik*+ z4L*Pf0f54NXg~!N5|r3SMTyweQ!*LXPQI|G6z%`~uyge;Lav03e>QbIHJJlr@y?4P zq%WRG8;6Z9J;Rp&*|R+ORNGROJ&H{e(M`Sm zU7?Beq)naFUBh={>AmshLsI!MCDg#5XDd6>bBRH9T01kGqPnq-`zVv|(WWy}XKe10 zbMLXj66&Z@DA|)*ErbK`$ zcngm6IY1Xv@*DUsMSt#c9qs2lrN8CKjZ%xx8NaxoZ1Bfv_fkYLtkm+jd`T#uu^Jbs zdl8>p65~gJ2+fScyr0)EmPjb5VG1h6{Cc{bQp~EB-hf54CJg@5B&nw!hAL3>F^gPD z;@&4e=W*Hq((#&_8&^aZrvdHpSJ?`^C5DB3CkYoQ?Wai@g zWVY+Pn`g(spynhcjE$&y=Dt79J4DK&TrAaiQX-Ntod(g;^xKn@041D)2i$(h_D`&1 z@^{aV-)n_arm^dR*H2g94ikNIc^_vO9rrYdh`BHG@Z;LmE5V@0YCna5(7`bHZPS>Z z0$4|!OIn)g3YE`b{=1pe+bl?IDg2Pb!v;~D#VzfZOY|O{G`h^+%J7TF0lef^M5bF6 z30tYW9mRZIL~1IlLW23UT?dCiGRP4$xj&B5#|7Q}*DvFGF1zJYtcfB}TKX{IR2+bn zd7pNN-6!2}G#pHg7d}h`AAX26VT}y9SELJX$l2J@JF+|dc{@~55gfnDTD`u#r^HN* zOIGB!kbhG}t%yAf~XO${gz(fa(QWk5SCB)=C$q6V{h#_ZI}fM3dZD6h7G zv;$b_+&Q5v5>BXcxnfDl2vm1TvOc!He701X;c3|z{HPdc?wW78&=tyjh&xZ4eCshA zjFy|ciD+ylPIX+^Y`+{uu$G3CO&L4swqQL3^_Jc!Z+xit9-PgMtsje6NMlRRQb8W} zS5c^fI94~59RO%!X)jgnh=uiwYLwk0LoCw(F3R-bMAXtnf^?1S;Oa=H(yVjqA)dz@Ou!zlt?PQLwR@>STt z2a8PxiflhiQ3`)_XN!`}V|AJ-gqh4Q9v6Up=t=uk%xsZm@vHK;RSa_M8;J_zo3=p_ zwEwW+44l^C?`ZDtHI*!dV-Q|@X|MW&J8owZ`d-`Hy6gwcf`=0C!Hd_Z=N>t#Z923S zc=W9%B6An7jw3FSz&(|fx*y7a^kQ2FuumGVSPpT+l^=z-EXv0glp_~#d3W=Fch+?A zou}a}>VLiubs2yM{Bqm1YVVRqv%d3?vrAvqhvj^8sulNOzduA9nbUdMJt+zliKwrz z9sGFhiJh+|B$}#biL3g3=AB;-UQ3KPjSTo-$uFUVv+)2E7DyL<3bldpmk%gnf#Hwz z;gXgW#t0qSL0oAW^=~W2-&oyAH841G8J3Q^y-iF9KlrMK(~tn3v)$w0B?{X8RY|&hr!iEsZgO$onwZ;NRxOMevLbq)wgpS^-&|4U3)~<1{D~YOy^FdG^j!=@wk2oYJbTOgnIFs z2<;g^+C=a63JOzfHoW&XpLP?wPA3pey1E;b%DwH=M`7r7K3bt~ z_XXzlxEF@j+cjdHVKT&vH4I#v-%HGys=p)T%sBsx=>zaUNY z7^$d31rrZl7@ek=HSGo;oT-$7xl(=GtcKPYxG42O~WY-qvH9jVC?r5;>aWp zhHolW;jliOE;n=aTGUCB!M)E?L5B)sIeJ@BNt!brF0QYpj7hcHIQQXBZyp~MR7&RE zdLSE(1B6KrL`H&`Z&8S2^iyOs7W^B;=hP5< zVjsiaJrFSYq{Pq{yY2w!j2;MfIpxF%3xZm?3{R@$KO#dJD8-MyOdQQvk1zV9MReAs zioIsl8vh>7OcJQ$n+bwuH|QE2*g zPJZh-eI=!gpa1a=cJD}3ivj_p7}|H=*o+Bj7x)vvRM2?ZZfJeSW%h@6^oAOlVRC%$ zZ`532@5M?m+3L8N6HSAEbclrW(wbypwsW~?a?>_hhT@`yFK6jC7rAsEX{mJnG#2K9#OzDNV zh;n$*a*O{{U4iBkF)EM|8%Te`Ie!M@VoFlBkN8~YIPF}D2c6EIYI!5H!09pf-+(u-+GDys@LD@e<|yb%(rjKC8red~uukSf)cUAkl}9bAL!K zv4Mgc`TU#@yC{O)?Bng0ubRow#qhD!21GG?%#;>{6AiEW-DH>3VEA!=Z5$Woa{M&mBeb1zzQY)F>scO|e{4WG~4_knQDzRz`u3!S{5kjZ9>{b0iMl zwR!<#*5%zZGkGdbyhlcL8#t*)SLm$zFXbKC`}TG<_CuHh@>-FLDW}Vc{@ZrWx`F%# z6sJrL{v@6}=AJco0d|8pQ?IL^=~-vIN<;7#zJlue=}wt5{uZVFjc!oSI56SQC*>vV zt56V+F6X$7o;O!HLo~eB5Q&OAxiWdNQu?egA~>au6}yE=@iK}2ESh=Z{=%D77ZhLH z_l+-83;{13%s0nTaPDU!+>eDkk0@5=C4GVq=S-JFN6MYeZ=1_Fy*lp1SSm$PTj{wR zWSp|lthm+E0m?DnMa~At!!2uFh29;)ki)#TfJ|!WhD%G0@7K;_?G~NHk+HZeU-VR- zudd7c@Wlb-CVjJD~=-8jDGJtSn3QWM^C3r7nNftWVq;EuUKVRG+L4? zZo>M$`#}5s)|xE_F6Qj+$i+q+I>kPRuZawJywp`UNTi9n|q$K_0?ue!YzI%8Qa<#_j%f z(uQXL!4h5SyszK~{BYj`f9=wDO2{v_#EdmKn)dI$nfyzM+O&e{YIV0`@M1A>bV{k> zL2zLt#B>0Dl+8mdzE`={qezokI;M$HiwE-O{(tp_|R!crv4lH2Qq{T`?TB5Xy+UV4$bJDxcjq@3l+j%W~^2vdqE5&KG z)K5XaqmL|*=$_Ild+EK)7V>0S-TH|eN@CFxJS~5VAF&33Tzs}3K$ndgPzP2%71R7X z^0?`r^Uc=wyvJyzkt{XqTTj#x3e4!2c(eg5zE>vUy^{$EqSrzpYBy? zkZXh8pAWL^Q-FkYr;SaJ#DZS7ym0|5(IJQ`sFLiPf2BPsbiE_ulhV!6r71q8v7(u+ z*>u2$8ijkztCHNL_g}%Bjt^-}`O;16PAFEBLJ#fa9y-f+`u&mflJm+uKK2baIu3t) z5g9Gjwtdj5#uUK5Vb~Tl+FFQBNNq;u=f>#1z$X<^ihjIp z7fs|<+8*l?Z1@IPR35sdV1_^VpOX*Eg3}hGCVhLH{S)T9K$(P+OS3*ARh-ZAjX6+n zKir-A(412YdIaUyrSgc|&YL?uwl|A4e2KyDxv7c4m(IRWjZlQiF5%rdZtmN0&qm01uP8d}foIo^${w|_Q5j7cgD2`n+Xy<3U zFZ#@T*ttLSYA*@Xc(hDMv1%rtU9P zmwx)I4iOo=XOQ2hUi^HzQsqb}7BnJ?owivg>-Pu|BJ%v9Ue(5HT)aIcBLR)kjOet7frMvsle=X@1M4bT$hb$(4rEVvY+1UnwcwSl@fv+{MR9T`U6uJBwB%1XY;%hl z{>!u~HwhN+r;;mkG;v@ce-9dUuaV>YC_1tDf*0c~`jYpNKQEw4-NJ{%!B<-u$h_5&MN9uKT?N$kI<`EU!qSr8#C2Zw(xn=W?_}m+NZN?76f^OeX>y)=J%cv3}iHFOtar25|O-GQ@&% z)@1voE!cy%g3hCE7RN*H3{g~J0qnZ`B316G>aNez?6KXhyQV%Apu00gN=5mI9LD}9 zSE@aw=IOc$_$=bH8!IO8+`d{9#ed<6UG;2-(5M9eI`Ae~;C0UOTD6>ycWEsQi#+j> zvK>5{6Lj!d7B@;LrFlu?{P7d|XyUMSB8_98b`>#)SvbKP3<-=ouQ**7kUC^9&Lahw zrFF-Aa`HZm&mVsv-m3Kl^|WwlQMy3;9yy#@xEsH%-a~P4aqK=Vdn~ld8cY!D9R|0X z{`27_dL`UYMtJ>7XM@)Wh#~bPAGXCl@gY}N$LCGy`uqQ&OZuU_=oT+~u&?adJ0*OM z>2fu@KK%!v#pf#`L%-^XzoDpEv_(SVvN0ss=JuPJ!P_Z9{Es$Z?F_t2(!(=8*6gQ2 zrk+CG4#?zEw(XWM;@C;K14g!u*+N;B&DX)_`fvSpAF23!x3mrI*aUpPXcJgSFutiD){B?UFZNJ$S4L8dd9KKK?I6cOG^7CuqxA5a~}!TNQ^Kj%lr5&MmXzF6bX zL}VFn2lr^u=+pu2ub}^QSN+kxR&yM*j0Fk)$<4z|=e{Ktd)@wYL6<)qWRfP7?Tiyz z<@`xlDby#$S9^P479JDYRKx(~rXiB()AI`6LOy_wS{}UoIU?yl(QcH}V1pA{%th|D zfjhh?14FgkjvUnl4T%#x!TZVI)LgHPFjjdh1A0}IkC_z8jZS-ND7}dwUQ~Vr=&oX? zkd5%)SM9U=tt8IksinpN)ykeh? z4Yj3i&!(|K2_uDJlBZq==>;PM+P!*Sfm?M>il|y>J=BYr?Cjzr8)}5!y?|PdOON%{ z0{%VbR^Lpabq}Y>@iBSd>P@>Ac~ei1)$Ow&ix=_P3}k`D0@oKu=R?UJ7U+X+5Wo2I zrvHI3gkJP5n z^#xTMCR~Bv-XG`TkPqvt5&U~T%ngIm`X&TMActt#dFN!?ue76Hy|LxJ0+h3<;km?2 zXMi_oyOP5R!D8z@)q+k&1f$lfV^ZIr@dU;p7KL+RHtCW-2_HlJhm#ppsd4ki# zpP^+AL%J(wn3=UKt)D5bgJWLOXu^fs`PsV9A~|z5)8fmj!!PjKr$n2KZnfJy7sX@Q z>cf_`*kI<8u{7)FAnHd9P!gdF=U82xROk+EEOpoqCq6110mCm?R|c93=(MAF_gorH zOzFLwr}++goWA}hnX{cuRdmh0ox$}9`o?eYaN-H%_AA1cK^tN6S5T-mCV6!g-K;r)G(9KDK5+#^>`GSn1g;JwNJ~X8?~i}K1l&_;AMgUMc;yC2pi7B zH@$WaPDhn2EDjLv&xlV;rn(X{@U7SAmEz_hXY=62<%HO^^qd3ao;~VwM<34OwUrC8u^G;d5oHnRDcwb7S8wx6UMazOvzW)P6W*p;os=A#4Wxj~rY~A? zUUBi?pp(uMj~xE>+Vtn>yt$jcgAb)B2pxtV)(mwa;EWocG^>8~W$EwLnx8P_1Gkdg z9CnQ=0feX8%!QR2R#stmD`Gujh=2@5&Kti~PHS`0klQU*k>jxc&f0V~TzDZS42i{x zT~J4sPyqu2W`n6SkK&OJ^|#C^7naTU!E@0(i5ZUoyP!r@zzvluKl+>JQcri<^ zQya*}N0G1eJ+|_tSqf;|E&py<$3@*ON*+LGXd#N*M7Kn>y_#si2-}r3ic3xzV|JZf z5Rx*4vq{mX!X%n-^S zy^0>GAaQCrOZbkf7&>#e3`#a7A(k8yU+|WSo01Gs82-BQ%<%S={^a z8h0C1#4Us)5LOIt8f0_*Pll^Ts!ZhKfH_u65dVx{nlob`Rc!j98aV! zJYx0f-%*#dJgF@0Xi|w^r6+JF9I;0+CnDtn!K2pruMB>0$a>s=l%#4P%tKs1mzX`u z3!y0Y1(ADAd0E#osh6Y;bxry__$KW3t%23D?lXS8?Q3#_2#JHWV&~g>Ng!%o;I7Wp zy_eCuPtoB-17j7p4#_6qu{-G+?OnNXpHRX5kw~x*^-eO|V#uRa&HT9c`hQe~A$FB* zTgG^D@7MU*!f@yTLQ1ymhvSYYS1S{bR|RXma!d5WTCN$N*3u~FR>TjMer<8=r%Za2 zt-6yQWjV(9emqK2N``h^>JQNY%p%&+l)%vsjYi!GBTCj+v-oyJ(8rk5a9US2n0s05UtNYOJD^8??`LGWI} z;3Pa8zTGeViO|E`c@!DYyQ8}hBY7xNm|~t(DX*8(AQwV;uT7JH%M{3du8-?6T4u^t z^^oc|p(5()3#hnGSSiO>VM+Q|3rgE9%iomYsr_SVLM(3V2X<^5)i`p`qA6RR0Asyj z&n>JISD*6N-unVy=YG6+)Ta znSX1~@KHN|cu`eJUr@%1G~0KkH$CTrX@fI~alfa1tVp9i?MM_-xDd|VRvGRd6r;W3KyQ2ycJZl|n>QvcR`lC#oR^z;#%$fPED(Bqv)ZS0$q0 zYDOW?%r<8H9g#w81I**KTp~3j!GGIP2yLMIG~W77sQ5zhA#P=UY6ZB8H`Ns1Xf)Yg z0k5W8$mxlKiHQ;DKD4`ucP4A#{~+lq1ETt#_97`M-Q6t>Qc8DsEFj$_!h#?o9a7RI z-QBQ&bS@1`F1;Y#Ex!Bvzn|`>``n4=&YZbt&fK!g?Kqt?x9Ex0H<0HEB+KcGP!--_ zLsvu#e+&{Qs*!iKJP8L^eT{orY zkF=yisLlui(y;wbEmVB1zJ7CP(`n88Vc7h8zEUplm4Y9a(N-Q}jPw(=^XYdM+5 zrB8Q$gs+pV8=}8w;c0Kf{hewfU?4P;&G|dko#3#Y<^J1C=*E|Wl)UaNt*3I!M8k5$ znJX+6H))pZngkGOEzMsKfKAaj#0T(tDXae7&oXq>rP-wE5urlv{BC}@16`8;*ch$K z7DMf&oNnc>-Osik0#hbBc{3mCk?A-It>S3kze=^MIxMjIjd?5KU<+hCW!|-ISrl~b zth@~PCN~l7CGA3|P0PX7*}+xDn+Nh`yLxq-JsFmW26%0hQNQBEZyAQOrF<|cH28IF zSGhmML7}o3_>U){%(Ld~QX$7<7Qe$p7g)iY1*%jZ0|(2e^{rAPZcVQ@Y192w1Tdv( zN`r*2DUIl&HQN=C`>ffTU%#5c#=Le1jdw9T-`Qygj!vv@^7C5i8{R9SCj^I)S5^?! z2}XXoWV#*|Iv@@3y-QZe3P<{4Im*R!Oo&U8v>b}_digu`@AwbpZBFcuubN5b7}WcgYoroXIsi5%P4 zJCr4_W(c=!o50$vv!~*Eb7~{x^>9-fp=H71H?ZNo6lPN2_hT-4P`I@SS2Nk1`diQ z)RW_w*u|oHta?9rZZmn5Vo1Rv0qN`CKCxR$G8!iQw(cwn9ek~Q8og9Co!M-0SZ05%Ee`G%5o!p&G1nT_?ZGkS)` zdVHg!7kG1!Ab)(F_N7)I39$of;LVGKknE+T3lk24v%>&+a->c5FZu_}amUdR;&m#3 zXpu%XHZBR0XeeGMdz6`mO8t`Cj7Yh6oIcypmbg5t#bc8~Y53X^cfgt}Fh<{@gn5<0 zf;xfT^{a-^)ULi-UvI%u^fYH;_*-J;NsM?9QuZA4+Am~}7^R=g=Cq$l3*7bA_*ak2 ztOu2x=>im9!0K7$+!p(2J>|1%vh}?QC3?#|@_%j5e!=h3ZuW~(Kthpo;O4hgj zW5(EM3?%CM#e2;&1tvh0gKZL?n_qpPTvFN#fGGhd7i?sFw+EohoMR6LSMf zsFXl%-&6L+*5GzeRP*mRss$Z#p1QVj*a^_lsjbux%qPH}+u)B4TMJ)MFA36Yk9#aC$6I=+r*t%uS?xj3M1uc*N6iI>Eh|*Qg{ZE8fXzH$$#t zxE>-Ea2Jh1RJAoazds2CpD@&i4%3`cB*R0_c$}-4W}QY)MbyDcUET8qz|zmfi>a_7oXVehsjU(wZN$f>#M9 zT1pQL+!VN|WR4{G%tjg6L$*l{8?;oXeWWBY3)TG%{bAHBU;@bW&(pt3lVkI z$sd@fyL3+=1!GZJ`w9f7$Ppf`(~60V6`OH}enqiVR>?BDSLmmN8Wp1g!<*Km9!!ft znsE&ci_14y5N++74ud#5qE10a1vJH7?dv|0e8Wh>(kx_vVcCChK06a7g2m;^vp86| zk&)U%6YO6FMvaU%nIj1;8mYYLI!(G|!COAYGh4?&YP#vxKAj}$H$WO!G;mSQV$vH-v_GaT%Q;*`wy%NOHXXeXijS0D{7 zoXaEfj~&1}imPkY)SQHrF@#foKs0&QZVyK0nZ)@u{{R6jl<5bGqlafn2HRyuyMp%#~tI?Hd*;zHa)qH;x%$w_uz zvp33~#)YR=*|B(b!*Aqf7FIYeF3X{^#hlj z8MOnfW0LyTHccL_J0i_N05ya^m>XA1%t@{`HFL>q%dZy$s^vaJR+(529-!lK)$HY6 z{-7KaR#A^%x5dmwMi+2bMT^M(b%2ARAR&n_sxWTZK>oh;@3{T!&Q^it_9uZ`LBO%f z+9{IzVwR&JCxPv%PxrFk^VvAmD69ck4!R1QZR2G(<+yWrML%2}Em*j;4@1s-xHm_a zcYphqG2#{TkdYbDV2@P;xur8HUeQK>x%?bN4FIr4b>~nGFHTmwo_cy1Z4w~AJ8Ks* zps574kBHZ>iMQeU9Z0e*I-N{Xro3}sXk>lOX`pm7YTG*h5sTk}PH^&boddpSGNG_m^RC<6}~RLlS< zgzh-<%c#gndWMbYF$la&ik;?UwiZW$?W5T#KD;yjG|PhgZvRqc1Zi&;oX$XyX+u; zFFfT-{&odF1)<2rDCYlwEy7t7Y zIS&Sb<)cCSL)w~Y&ZJWWc2M{zg|sa4$c-Ohk`qA!Mfp#mvRu(J%-wL3ZSSu zxVa0Ln?ed`s4@l~yFtX<8@yey5pZw;>1TWrdA>=qWF~fXJJCs`;GwNZO>o|i*oT2q z7_@)=)jfuyKa#<8U@BYAZ|nQGKEAG}Z082?lk)4D{PfdA*o|#Zd4=Fvxwe4;he+-a z?pM?Ej61b@_4tw%{;KZW71Fq1i$9XV3us~ z`(0@fTI|TUP;Hw1?z+ZNdC4>IK%{y%(COz-^%@#;t(#dj`Fi!KybTnZ`95C7UOUlO z@o$&0>-aBUKmkoDc&$Y~iK~MGQsXG;q;}*vsJv*IIFIAfV3VudH^%-QC4wd73)TJW zrefMr0xqe<<|#jSqD4zZ4J;(vvb#q!GDp?sVQ@a&~XvGL;FR2}~{ zox~q6D)|lI6%SFeQalkRu)j}vY7yaRyCLRrp2kOynj0+7aGb$lg?Q0wuuCPb`a0)l z_8B$p@Efh`knaoy_fRfJAMYC9FWO%Zx@R>D-R??)z<$`PK7IjWf2xXTxPo#baCQA-?awe%-?rv*W7T|XY zDIm=zx}9`{&2<{C>~76I1)kBT`KBE`ZXR8SVm-w`=i0ayl@TvLBo z7Z~452l(D*4G~l!Q6+@Emtp+GoB&r*V)V?ji><#v8Ga%d#{Im_tN|W;>MJ<=_qB)q zbVpFQo;;nW!~XCtJ7S0Y%LG*4Tqv|iU)W#dOe*wq&aaUOTM^8XtOP2&huy=CVG@7o zBs@2%_@(lQnYQV1m?b6u@56p8tncl>ncE05wsV;E+dHZNN5->6+VJFrmXMqaGR~nS zu?Xbau>t2>HyHc-rFaVIepJ!nv&1^+GHGLWT2sSRHT%m6n0#W}nK_C{8LnE9{#1%6C-a^W16cn}uD zZZXo~X9`uddgKkRykv2N?$E9e)RP56;y+WZ{=mpAWaGgyj6XNpQO~VdVoGD3X;Sc8=0x0m_bn8{fD&RV>g8vYID{QGmj%r_cU$F}k|JzMaq5 znxuzSrCU~U#?f%*+DWbLq)gVj9|h4?#ub`A=Lh_GY(69hc1jrKU0IkuuifoS?4s=_ zjB*Yc*q$Tnv5@Hzl0izX-=ABDg|Gb|cSwO9xfh>W@RT?K6uoBnOiTb*a(y5;_> z?gKYDjRGS+(50$=r05gtc{0ijt4grsN*pPOu-o#0zv#a)yWS9jwh30vKUIvZiW%Q* z+&z${`A5CWa^HVGhxo`W&6+xXNY#8xVoOfx&T+*IY>qNyBh!Ze#n=*uK88%zc4%p# zvi@ya`;fj*@s-QmGe4~>;S=4{q2nH*zj@FtdXwM$hj@XU{!JMyNg7J;?CM;@A2euTBgwy;19RiRhnaP&OR2OjsfNu({e+2T4> ztN3%C*72r#?2Hu4iI6H{Q8FE#(}(6$^tZm#->Qn;$<`F&$9<$DhZylZ3|>BVKR8i^ zdM3Wa+Fv#rC7^NM>=}L8mQC<0uEkxMI%zM=uwn%~>rJznb4gKrr`N})L8rGD&1j(AG4?;9PJd0VCD`Y=MVRJVB|L*R5nlym0l&>5!aWg61NSA4-I6g zUytRhJ{iDpDi-amhR+YYQH!l>ppl(f_T`ve0V^U8!+Oa*6nvv7p})L8l`)=ulP#kQ z*WpI+z-!{nfY3VXn?QL-a$n0vKvWvKP-O}aPP$(hmyKynO{PvLM;WO|h68DE*yI;mvo6~&m%T4)29mu5EJ8Az_&O5Cm9HxSW zQUC2WRJJxcAClH>5>FqSU`p2glLLJ)QassxsZKIfgX@KQlLK9ek^f=2k zm6gWq{WDA7HZ$!pQO09^9n!9OkP&T{8I(L%PR^JOA43L>44{(oUb|i9*NVPH_D!{B z$oV|Mt_KvS>hn54IAToEM1uJDs)Sz&D)t>8T2YF}*6*|St|>LkUx^4E{qzfCu+gol zX`!1htNN?{X=piZk*M5TR{Zpj%3rcdNwVizMt55Nml%BeA0@JJdx!gbW}@{n<02zmQJ}$Q1yhb-#wV;o}RVF*^@D$E`K5ZALE^DAvx6~lGuFGU7t@)fF`0g>TRO%?&^=CV3g`~VidgsFjr<2SKC;Q!NWNs9V9 zS~wouEKwTDSQUzXvhV+QT*^TMiP{OVG!ZRy?2v70qX+f0tpX}f&neE!F;bqTobG0S7CxYs%yCdQ|JKdNkUYcDwL zQB}d0zUYk6rs|HmIEh$)D>u&6+srH3g)~=*q!&V`fjR_RIBifGXIlDkW+OG@%u{xg zKmQA(&86Qmaz`pz)~={3XEd28y8Y3uvLyn;CUALANL(By>ct)uv}QSW@-WSI@st~= z*e)=|K4sJ;>NzbIU#Jyz#NeCaBABkCX$zw<>cA(LE<%{HuJLi11)tDLccoY4@>>)1K zG-zh4K7_j({wyIE&1X)L!q{5(ItR4zk0v3LEVl;Oi0RVdvz3J(Rx;G+OH?-i_IItc1k&&5@psqaOn1LbG&-ePG$L?G(u6RlId!e!r! zo(C3hy(ABZPxmETNX4ycO|pgaUb6_K^7;Cr?-6-Rtc|@LeZ>m=NRqpk+&eYi{8BNM zA#z{%sIda*KAvsU!S_#Fu~YN1enNnsVkWo-WZt7-D0UN6`V+mE2J{uz4wyyQi$u)2 zpF>JL0aiuO=C^wJA}RWMFrVWZ@J*hJNm708^4IsTSI5pRM6Hc%HNZu?GVb)myBn1C zi4g*4zdzG+EnrN{35(nT|410T0g0HE)Efg0iM$;#{09qERM!!2eijMpv@=GIyhWT( zxqA-I1~d3z&Iqsc-zIj$@%xhrTp8=t7F*-?PwylX$y9Vgw+dXvG1S?o+My`q5rk)C zyK>t&=Tqt#`r{cQ=kxFgiYV`fQApn>!l}ul<~J@a>SB*pr^L0K(ITc({Aa>+RgV>B zP6^kBzq(`nn1pH?trW|BzHn?k^9Kb_D0drfa7Pe)!b(k1E+x!YEuL$4BVO8CqeLbY zW`$jNymFQJ3`pm1L#Xzvq|78MjQUbG+4$Ktb2M^5JfVp%CR+3gtxgR*>POR;%9${fdu!(^c;o%-Q+##ob*o62goxiUILpzc zQL>KK)$h=)rye&|t%$98bw+_Rsl9nSkk?hYsvUXX!Od|q`VU0kSx9W!y8%^- zh5$`M{r3*{m=MVyoznrD$z?nb3TJg3p{t-U_c!2~=P#wxS!2X}MbcIx`C)n@kMGES z=#F^(puqgA07S5Zp}9=ybjiFjak)QP9Li+q=LKZEq`n?z8BbW_S_s$QKvo3yt0C5S zb)qv(6P?u^JBAML9YEIgukr=9mNg@RY@R zKg!MVyBDKcts5SBf<6DmR(RfqVs#asK3|MZgf)vkzTsEx=r!fS(_LK1e!KNq70>RV=2~=tx6T(YXcMFszIV}rBer6=G2u|ES!2FdHvb8=EuuiHWDO! z8VFX@lpzX?xcbwB9Bm6YF$qZVERq{NUWyt6D>p>Oox>0lxYo?ivLG|pE_0q&KVm+K zl%V0<0)nN7^P_Qd9Ck2pSzwY*^=Du=VCMr1XKBX0Gdn_#=^YMpq|pGdapzy+O2;TM zIO^kIn9zp|LqDqhPo=UQ4@y0BRemn%69IqbKtJ6wB1{p-cEx*lYBGdcqFlO(+QyeB zXoU!pKtN3!B1&`knvvvSatsA|dOBh@W9tR$M~k&CiE^RuD4 z)^8zDQ)HiWGP^c*gJml?WxI%sC|sGRIL4qvKlwZZ>O+_q(= z3mU;DgUN0*b~;Xc;azsj!;wnNahD7XBc1G-cRn+Ed;#>1s7M9y4MzVP83-$}1)k*= zJ+o)OH##r-TSWDGE%Xk=`t>bW9_+%`7z{gLk6wQ1E!_`#h(!#j$HQcK1zl&JHDbahIVMq(RTlY{W$*#C;VMa-QL3`6lqdA{Y^^Jzjld z6&Yr`GWb64Yn{LsvJ1@JRqNv@%xf6bw6GRtGOT%B6Uke#2yyv>`j=*>`9k;`ugJ_s z|KQ+Vm-Hkr&pDoS?4g;x#2`nDPG%}7Pgh}>)l;x$6kIvP8%k6j{BsBxGW3x@?3<$)G=tylN0QKRulc*EUlSjv*p@Xqf3n} zOb;kA-$D?E$7|C4ZZPaKb7SJ>%HrZCb#3&8-`ywJGA+)cJAGuCEHXKMqo3X;pO=-N zaGqf-q{#6%O67)>%)Jhyr^|G3t)D$ix;%~0z3Ny-N2Bgwu4UrsgPUuKi`&be#EM{1 z^}qf>P%a+ttPSq#)T*VH8zun|`gb zP8_9aDhg_*i%{RKiTlb;HvP&mJJ#NG=2fUq-jE2)fT;B^*qzjWJZveVU2l@6YB=NQ z2d2^5qDxsUh_I@tUFgI=?srKB6tt5G`ucQ_{JnE^I{>U89yXv+SB8-BC?~P`88hy} zrq>liDp4N;i)QZKm6S?`p?*GebhzdBDrW}!&kL!DebERm{cT4`2}h<#?_n#>EF5k< z_XFi)9?0C+>w)Nd+p5rUaf5G((S?}vu5m{ysEZ_*>OqcVI2m!L_B`mq$nNoQb@lojt8`0?uw6Ttl6@}WA|*4DQ+Bq(@9|#F!cg4b_+P9; zs?+=PrQaN()&pVgRz0Nx-R%S)kDEb5JCRAK*Vb38bJ&Cl^&L0yf9QQ0 z)tucKY4YpDOkpT74O}*>xe6_N_y*y2*y;gxP>@F>zT`nZbVoY)-#(4qvCsYNA2t@M zYi;=V&$>j@w%a}nffxZogFLaFC(n7hfAH~_w!e<}#Ry0bc!&@YKX69yjU~9Wp=F@WbQB0-s6z!y zb^eJYN=k_NkTVL5a{5~{7P@CZ#TjWH`^P9b`&~I1Gq;!o0$C88( z{4l#KqJ66(7oJNBo){j1+R9v+d!X)N@AMwH=Y#;am1I)vs)27Q0#W(S*cPfzX+RrE ztBbTDA%72-XQ;GX+lVn@q~d%~X@>)p#>{jm(nx0z$KhAm;FLxTyzu3 z*sgAw0`rg3rf-~V++ro^ELpPlURu+YWIrD`?n$v^q&4Mon|Z-;+|c?&@`jK?RStU1 zH!s@0E8D4pJK6L=2>Ezz;am26#&fqYo_@1Zo$2U@#RSd`jul4}HmyRkLgX@fS;hnv< zKwlOA_kxSqR7jHrmN7jfgEE*hn;*ooL1I@!Q<@l3QSulTU2j~k+QO}Dqy1FHrX;n$ zkN#{Qp+sVqO}oM@JsdWw?%m|CJrXYFoMQS+N9HHgFUj zscsf9Sg`rCRdncm1NLYJ>3cnaGj4m}t)@N(YZ0vi8@V+1`NUQ*zr04**0sH z&U}*d7`cyB9)EIae3F+!>PU#f)jA6;eJd5zW{Xq@+1-XaV|aJ_$3*i` z>-Dl`GAbL}B4BQX(pU&41b@_zgTh8xLT{4_G9Y%fu#A*R6t=32K)K)8LkN@q{ zoXKOcYYJ3~eogX$d_)TL;i86N63nL?c79L^{idbe=&f8w;DSMjs?pWU>D3^>Y$aHg&E{IyTa zULZ!b{{%tD@o{dPx<6FGNHWOKb$?5U3v7Hqb|`8XMaWjpLPr%r4RQx;jR`HFIeK3a zgqo-EIjQIJ;(M}>bR#CE@L$!)re7Sm^o)SDA-CFfuNu;B%1)`d&lgfWY^Y&S%J(f3 z{gn)iF+6j=i<-4PS|uN{3PJ=!c5SYJb5mf3ORZsA+s9#x1o>9 z9!B3>hS`OaLV9urLJClt6)uHo8xV;}fXB4t8`G|C9|}4PSuVE=oooIVLJwWtTxnb?m$pDvjr>?)z`BK63d5vqEX|@!l8& za^PUlV3lBI%DBt^k$+N&ff9$Y>a*=wP)TX|@o#xpc>lDackF;?x`c1Q%zF5OI?~?0 z?qD~-TU(hJa{Ru!!2tR8KIbCDzE?hP?7vhI%JD|(Ne{QlT&uWCHfaXpO?|CFzT+eR zF2FfE1+w1RV>`A6rDDf-bbWC(!R^Xt>IAW00yF&QjPH@GG(j2A=R}JM2i=4Vi_*x^ z_buCwWh!~7;yTm6<_@VBi5FaGarCv04EmUxK5paq%r}npi@#A zgBBw6p|E1%Tp^rz@F|yZ{$1}m`R0V5Ay`nVFFK(~;$+15e1x?D!>YIN)1tS$V8^l) z1up$3l0ba}q(S$C*j2wi1nNXZkIlEIOh%TNn#c(%lOcw*yll*= zr>v~<)e2Tu<<8+vH!DnTtv5N3hO(UZ{h>D43F}y;*<=cUpOOW|NS2?@)q%BZ5D0yv-N?*96$->B+yEcs8n;IeAnoab*=y0uE+M9v_r(u&Tk{Z;M5WPA8F93DCL z-RX8y8-BRrsBvWvaGI z&R3O6$X)MRS+~wy@;34{jq{z3<)*x8 z&e&b&`~C^WuKi5_Ck1|1mgDwo1Af7_@belctB1xiQ=UlQb-#HRcJ@>7oGXQ0C=4qy z0t89jA>vH_d)?`qx&WsAyJ+sQ>{eG|=iI1ucye|B^h_tQc@8}h04&6Sc^U3M(d5=6 z*EMD+48rX)n*@}9EO)7c#iKK;ZTAmL3Vcvc2u!o==|h(zV6UXe>9M@TON}xhov~Y~ zdU?Co{ZoJRqc1B~R-WNiarwElZnZTv8n}2<|6bYy4lK*UGuydi@N&MBE5dNavm9+) z+)6IEmHB%Q2kULaV2q}QB7^5Ow#!94qED0JK!Ws(Yk1Ovcq8YVD$oduw=l}brl}YS6Ok-p;3nG^DQ$ZGPUf9 zq^1;!>#v@*i?UH2S&X{aOb|mhn;Xx$hvE&^cKLZ(jWRrGT2i7l_RF06zaf9mAHY-% zN=xx4y$GqkQqci7KR^^(;`oGS4_RP63~yv=$r0x*@t4ms1JLvu86YT?Fo6ig#cVj_ z;MQT=oZhJ<09<(>Sj~WvHseU-B4%x2LmSTwH!w#WfZ6(O7sf3!J*`KqH2*f-S!n74 z*ZuK0dqV2cPKW_r>HX$a!Znycz0O~|X}8F^_x1`x4n8_H^>iZV%PC3y{_EXCoT}vV zl_q(JL$54cNVL;F_$pzfa@)3J0dq5{s3FDGlBv0=VDFR$G~wre1wzleq?o}#ur~R4 zG1=kA3=0=q@To5%jzzPkofX0z>nEIHQA=+0?-KobTlTelaLF6S2mozc=|`C8!kiGi zwtZ~4K}YOAXtzci{rPPKZ1sG;W}YTYA&H~0-I!Y{ipA*W#v;=Y(AC;C!8ueqS7waK zb(Kl(Ka;t;Xx>Ekma9uZIg(7>C4KW)_gI;qv37-IZDxflGTU6_tsZxtf#mg9a(`Oi ze%WmAtM1bo!Ueg6F5~PDxlxh|L+z>(4vd-bF?JOY{*5NZL1S z?{W)SBQxpORX$9TH%u4M%HF5={{)8wamIoe#zg0>!tJkQDUwdtw?@#gI&}$|C*9fsR9_ zMCZe6+S^a!>lgyT5Ur$>AytM5VD-4)C!(h?5|oy;G)Y{Sx^U5PmG%0eSa3yF#INea z@m>0Z=gc5K;spC$W>)1ZEH!$>NU7E6MY7Nu3xUGTI5)=n7)o}5cWum-XXA`^zo!1m zKX(>94N5)=T6f%JzTpZOv4Po!DH>NJBs@$7;I8^K3d&7IY}Hds1-mnN)o^1x(`Xh4{)$xy`6OuJuYb-$Q=`zKcHfdUXC<`3{*rG}or)PI z^XJUsORwl+&k%7}#e?CY{=v8pYT$D$c=jm!4UixJm6yf@TJL3%~ai01E z_TAk@hCJ}kt*$ahI^Ww|xTjVt6bGe}rXk!{zxV+X3$CdHd{Ae+TXD3E-0!SRmgRe( z$u(P-`WPwPi9+SXY1)rLVeDh={whQWPk9{ugGQRlNIK|1^5orH%;>EbcBw)KUVGWj z>?9Ggbx5vTr(`6UreUOj_Paqb7-{@JcP6vVkun`-GDETbo5ccC!ss|(oQmzJPL3`oxRar9PwK)p<##)7WK_-0@nW=S z)vD3{=J{krT32 zru)yW+2x1&VEpj;!$jW}tHFG;yY|c*4t&nwU)d12T(R-y)ph)+)tI!rI#10sq(HO! zd_LaO6qyL+4w6ZLM!x!kHZ*I{Ol$DjJu0pIZh@qgxZIM4fm~#f=^MGaU&rxwUi9~Z zP0NEo74-$^#E5gP(XnXqs2=e7ZvYk6pUF8J??zW}WdrSDj{UArj1b0W!GfO8mt@OA z8$y#!iT~g(w^(4$evDHzlxLab`G&P>2y%XF0DjJN(85+gv@9A*7<)807JOYzRZ6sS zz>4;;`;eM*haglaDFZHjf$z5gAIgnRjdRHzE!q7<9~wj^@0Y~Ys5#ADf14b;8m&hc z;Cy9CR#n2H5`ro!ae3%phWp{jq3@$GBX1lXN{waf%yY*;mOsK9)<_`@3fG#uRk8oH z#~b>Nj7){DkJfU|!h7lhf>uBDgqz?@idP|zxYIBgBWsnt)^?PBR}(|Rgz>NUU5VPA zBKN*xfLAyRt<#Fav3qLm@Y)eF;PzX4A3us?T$v$L4tMKJsrh8-!kBD@m~7tUiSj)q zdXKnMn!nhtT+sNNwSxvsgXuK?Qf2T^E@H;)HCyqJDp-Hy&B-Xf*PD9@n`UNkqN_;} zl+|93)s2H8*+q^)+`g6|5o3czqg7kBHsb1BN#hjJ$}kIow=g_Z%O=e4rSum$z6Cnu zE3S{KMkgQWOXdhJ_ecJbvp{&h5fXG?oNd$GH*!Sd{YFrbAqiL%N7IVFm3VS{Bic%s z$v;cru0t1Ql4Ws1esJ@NJU-~GDO5_#z6U%1oqscCAyKpL*ABquZtwi=TOVPORhP*K zB&pH}paM{2tAv4JuxMuIvxr3M3FmuAJgOUKQZ7R^q&7t4!1rtYKlRO%R|&!MZUF$NufPu{-GRpacfz}M5_{i@;GK-tj((O(#22IxqKse96sC z+pVmTv1pe+zk2j%>d*OFfL_^0E$#4a`>H6U>REZGdmkXom+u$D4vSsGA`7eVk~@Ba z)51wXENg<}#~_69bd-K!_g-1~F+{|_nb&);8M$(-StD|CB~kf>+_4}XHdk2acP)Q- zaL}OTYL(u>#cC#i7G9o;NlFWC|L2u+r%5)H@N$7>oIi=@_XK&erPz=-Ll8U}gyBSi zugtCWee>?KpN=6%IE)&@KoapkK7->vzGq3hq$j4PI~^%X5ze!VuL1GrAl4uqXcI~ z<trx}kNmwHu0@*Jmsi*h6_ zwuuO+#};P%q6BD7Xsq{+Da!j>cQa-qS0`*U2 zOS!-orlfRIhK%nUyY;QP$x;|-chN%_y|9abh|zC|b%5W9us4EZFi;QE_m+|DVYri8 zIlWgoij*X6sJ+dTb~U{(^v&~mf*J{Z*%3#yEeBe+4lj%T>&N^ZN@NSi3hx}7`t1Hr zNArQB^XVgVgwH5>QE=iSi<%dPB8E>qmSU$n3zd0sdXpk-8Zdk7YbqdJYW1_BWjdow zw#I|v4G^OeqjFij+T9=45`C(KSe}AC-c_QAo?@|oU_Qvwt6xU@42i0!6!+81I1%cv zyH0S_KbuNSbMiKuW<}Wa8P4AEe`O^LC^jo$6&DoC+z?Bprt(iiI)?_2bIB%5M=BxO zcu&x`Xq(D~yF9v-k*$xg1Fd^N; zFs!BPgM_%?NsOO^13BI_X1&+kspGZiqAgqW7 z1w|bvoP7MnP*UTdJxfjMAD8lN;4#XAfkGB9sZF*qWx_%KdhEgE%gYannk$0kEdpd8 z{1DxMlC1V==L!@iX#^^14V|%v!w|tck}h@xCO!hQDxv>(4#uEJz@(pdK2$3v!OTxi zj+aWN;dmNDkGPA^z6DOcOr>nL&)guR80~^9zeUsSwkSkN>W`BWZUbdx@VCO@XCZ_D zB_wtH8y`6&PPB+Ta?(aHnX$8@6Nsv#>bGTTGqw{za&oe3GW`3~6N3uu{tN&1ZSLO^ z;QR;sp#{rAHMGQSTnU5JA0mi1nxZHtIAE_){e#6x-W?NAE2{VgA*hR1Dod8E?!oG% z)KD0Zky{q=PXK(Ji$rL&0%3iDFAczZvLNyZEJxz!bY&nP$zi*Vni&rR5;D`b-$X`+cUY2YO4kcsW2kB{(m`xt zWlCNMU0zY&b1>-1s}}rc$^z#9IVk`0!)xh=5uz?tj1do9I)0M&NmZL}*ZWiX5tG!l zgOhFLA$rCl`sM4$NWbb4DifkEWfX`w5iuv?JB6H0LHIlz$8aV|KT@?OfSeUgc?w}j z`DAGHCn8nrMPmEdaqtYPGL)Htuv*m>Xd4GZbmj9yyJGuQ<>ds2nhTgCV9c2EXZ&d( zMoy&BVpTa7j&cmwTm9pue^VFvP*HXu2Ll6;li!#tE(11{v%~NHi5n2i`~)&s8c7+C zQz*m%tSc01bEbLO*`p?6Qr}}TdI4$;nxpaYl<5#TZctaEb4b^IRzy zVgioI^aWuG<%>gF>k6Q(Fy1c4(0da$tqAv3A>;4(tl`S zGS5S2nKHGmP4K;FYgWw$kLQAConjp5UunBcsFawUY4KV*%b9#OV-$dR>J_%nbZ?o{5l^CvDTS+n)y9vdPFyD3KRmCM|I@2q>}=+b za}Wz@49S~>&ulsdbXLfGN>kByt!Q&YN(SMzbn^$Mp0KJRo?vYfda>KcNLbxy7%&ck z*JV`nsX?q?@GP>|-QXCWaq#QAr+EWZl@|&+3uS;Bd~x`>g!*<2lPL=!n`zs0Wg|0+ z?MIEJRHT=M_&SuJ#sXXGTr*198`F)7g+uLnBZ+E*qiS(%-=P)>oZp6|(b*<^R1HYV zWAu;@vsZ}c&fAzIPU=+jXYB(@{N$D;0890m_L;}bz>a^PO}HZWZ{rWS!Sj1BLXt*l zi=c(2)}0a&(g3xj(A_bFqH}yZ(M_oalrX9t8}ET)!0#XKwc*RB+wJb9;@duoHvtmR zw;iJjy2;<&70X^(EDISU6XgDxTVr6#L&O-7KpR6%=XyasTuNTg2C?nggf=GawOvjYOnfFz@a7*g>Q#BMn~ag2;V0$P*S5@ zABk@V{xHF)s9#WxFE&+W^L8B^9uj8V!0z(_Mpr=oozvZr)!bK=w}$pnRSL3ZG4e!h zR5>iA?*fqSN>ML@o}d^G0>3jT{5-gSmT!Mp6W-f?ft;GEk zivXdgjj#3hldlC$=$e}yG!GDjq{>hh!fm|gi|MTX6v?$988_1@7`l9`CB8=K*3DO& z`H`7MZvfwX$16XuvZasr)yroj=uG_DFW=C293xZrvUbbH8-=e=rQL-xY4Hcylw+Pe zuVhFq+fQ~A&tFh(@6~ZKv;ATvQE;EQ5MKL^_^gb3AN}8fS*<#UEELgw8lTuYOfB=v zTg|~G%nd)23_GJMl4DWNp*Nh?Ol^~@w@)53{TTkF*?xYje1@XUX(NFAjWG+CQIlWPDV4q4Jx#nPCWAIdRH9QijZ-{-CT&U?$WwEFp}9x)8&@@e^h;CL)72% zHcE&z$P&^WODK(W35bYvhje!>A<_-fu?o^1(y^2@>{8OPgs^ljNU8ik-{1Y{egS9B z%yq6aXHLy9oZmqe_S6@#KPi}MwqiWx@8afzR9VX3`_a0w&%@I8&-NEc#y`L4i#}>x z8Zh`sA8^U-UZ5WmkK4|gXv3%`PA=`)7YFe!t!r?he0#>kQJdfc) ze~c+8jSt z_>2B~n5B^~eY}GE zq+dX+Fuy|ektp%#cV+9eetNT;Yt(k#ACf?hkcVD;sec+TSJ~~1)IZBk$;Y^CHz{bQ zM{L7O-033TU*NqS^EA|Jm?AM0KT|`k!QyPV`rQ zUw-*RKIRFK=&GK1*l;6OKT@RB;9`vXXb#v=qL<0ArgkWqffZv@SF2&fl~_)u+j-}~ z{&4$ASU`dx11d9z@U;Qasl_xmX7z|iCgY}sb6W#l`}WsTyi;{0`k*rToI+kX3_=y! z86ARD$KM-mvAD39G5=5nKQBBpUAgLV-*!b|3Wf4~P*M zh9+JP5sejL{Q#EQ&?gJ>Io$N#CjBmg6Vq7npU-HymbR?is^j!)1DFnv1IRBQrGlcH znyDtd(agPqf3^phD%!~lohdPuE$>dm)@WUXCGDTu)Ve@;S3X&fGD-0&{AlE{jwR=K4pH#jOgW-Tg+yNp%lzF(`A7N1 z?G4sbUCZ{*W?oeN(A=4>;9PnCvD$t;Z=vBi6ypvWzP-rO(7PF-ZnpIB*K+zkSN)`p z%8EMM^T+Erpz*$C2Ctluk%a6`tM$oEQG!4@k!$UuXWoL3S`gO!IvKDYq}&pnZx5T_ zK0W@s;g1H-i>){&0H(hU!efySnqwalshX9$Ayv3f_#W2~bwE<(B)QiyG=82C`cIE> zlTfRn*VVli;wehd5HQGlt)K8R`V*)H;_xaJ!+@j0{Zl9%Y2K?K(}@b@N$Mc^lt9Tg z@5HP~>m?>SIbbOl$O|goW0wJ$pGR=Svr>L$rJY$4c#JAtLLmld<0>I4Ev9(t%A*?& z1(O6PQ`Niwl-I?PE;mULWcMO}8+se8lROB%wBHa^sD_h{ zzioW09_nRsNwosU7saFPUEq&-@zt})??9wwJS&lpb#2Jz)}nt-oub>RV4y-hpO(ON z9x18u{j;g-`5tRNalzi-Q4mIy0ks3sBQU;;3o~Svv!-*I6HEtuD5^slZ7{Y;Dc%`|4zLKU)=Dl)>T=M=06f{Wq#n^* zM3r#$y8XtQwy!GvhB)kOb=dIsZ72~zR+x@;1T4OPwQADLjr*w&u^hfV&myp5q!Cw_ z+raNXkbw;Knz+cerq76D;$>{fJbdnt)un_y2V-X!4_#<%b_a zIU$zT8>1Iq=|^{y2!&Fm?5mjoa$!dzs5~5pV|Z|S|G9C z5so&SPEWCa^B(q8WH?zlfBhoi@C z;%ekGJ%?USittOl5o|Gx#H=9E+(0&1Di&PfMD4d8%vpZt{8whc1B5++1V0V@><(oO zi>{YjXg`s$sz!UKeTV=V{$j!D-c`5|Eo07W-{>qgM16{V;Hhgss3l{3YITv<53gg? zDfco8e!IhG)>g#;8X$=*I-q0C)%j$qbP84?sP6Q&aBoCfV$>z2^5&lXT^@_JJ~Y4i z7;w2_M_()m9~-!iLjl9A{1m{rMxpyD+wVZ!GAy~c>A$`eBHOu2WpSXXcbHk}!<-COdxt~3u% z_ypnsAt>PsGs+0NX&e*1Y4Ud*@!gcLf%*xV;(Xal0ERue)Vq{ZQq=hpr}~`ns{;j- z@z?6_(dh~)Ex4)^C7=Cxt|Ol>!1RO~j@TaGUD2Jtlem(8%-48wl&-rP#xO$vL=a3v%?IqKwmNo08Zx z@#--mVd2!d|Lh~OnltK4`j0bs?uSU7u~m85#Ed@>b`tnvud7Szm{Z?TOl{l298esN zmqP$vyJ;s!Porvh=NR7{8Tg6|2eqGkj`ItvZ0$UrcuS3`yy=o)yomA{2PQZ*2E92G zIH%o|v9qT(@2KC3nEfvr;`;5w z8L-fa>(OG-%Gq8_-s&%qWr5 zR*E~bF@64_{?^s*LRX(|2&zW4cN*R+v5<$wVbc|&0ke!32Oybf7c^Bml; zMGp+ST=L0LcyVs|SE|IST?wB`n+ptaLFxv{O%6${=X!gCKP=zsw!9PRJfZF(-jDx| zu#Q!@>D~s3ZMqAFLkUG)i~KSU-vCOC2^X)u(KSFPrOX+Fv^^^OhGbjfLp#H=OC<=r zYdZhIt(5NG$a)@;QXk5{sn^<|G>@ZNb#4)RF+ng-OAONN<<UWU zi)n=@9mYi7^|e}prsPE&@`q@Z%uCNd(#5<(B57>WV?_w=XsDl*rF&z_78T1U@d*Vb z^3NL;%xoS0?%cBu=fijOS)r+->4n!bUsG~v5yH);W`SA2g&YuYd;qF&4o@1SzqJJYRHu)9-suMi_P!#a<*pxI?3e1C;f_6 z{Q2$fx7Mr~ekjvn=`3it#X-Ni4Z0n>jnNN-J|m)L7WSmq1umn1juUdxdlJ$GDQ}6+ z!C9GWUyKvGIzgIAp6-Tm;d zIZ?sY*6u9v^y-#({bZ1BqWxOtt-S4>Zm;Fu;4zjKY={MVcmjE{p`DNMNjSH-+o#~G zev-CWi+Y^b`+`(kMkDX$FIhH6vz94`kP`o#9|Tp81y5y%=?NfY3?}I-DSkMoRrrOy+ibHUH{g|L%GoG2%q)^)+7u;pHs2zllX;oJ)3XB>H_+TfLF$ zWHcB@qrPH}djb8t!_KV5;o6%xIesD;=bw)%QJ4R6>+6PltiH~clq~#!V5di6>S-#v zTpuLrSSQN^UyN3}zc0wmjG_UuKQ>e3a$@yG%9)Ke3ndDm?-UI7yst)hOCPA_JxOc| z=cfwwF}(8^OnL0F9vkwrCP0vs^UCv}j zu$ili((06#XMBj#3I^VGZUrFsRSIh@RJ{hKtuc?jZFWWsf5Nm1Aezby@QZH$RrC1K zE81USX8HeVq^S83rCu*z$Te6%7?(Rtl6(TbZ>c~?E}+LeFrwKIpo1|SpKGFF{&59r z8I#NB#;5if;teKClRDFd)%Vv}l$UouIsOc)N)}6tk^N?X!)AhiP2uR)5T#h@-)2{= z>wG2*mpr9iYqcJA7s?s83o{*8=+ymBlGB1c`rdc_xy7~eBiT;}x$WN;dfF;uykADb z14ppCO!SIFv16pK_+4yQM4E5Mu3(TC!`+a;7_scG33WKpBMWWn>9L=tAeSUpUN-4AyM@RONc-1`Vmw z9M_(%)aZCJ&o`yC`FmKiNeZ~Cl71w6#s_&&X;^R)GSO8WfLA@{18dDQV}W^GN(*?r zr0Q73mlUd#7J>3B!q&%cyTi;?I*Lh=!(|s7HNWa%6nTg_*Sa?pT|Su2@bs4jQaRSCfs4kl`34ae@VI>-@VE z4W5Dud$Nl79KSL8#ooedP!`+x!}q3~S6KF#qv?^rPZ}-BV4)PPb=76+aj#Z)pw_A1 z*~Gxg!t-6|%?zGCD1|*6QpSVb_cn&SVdLVTF95=OUO{{(1Dh9%0L((zy02^nocc_a=`liusz&LKY--PIS0d?&zo2k40bh?s})w2p~(y+nz#O4OeZ|P7u zkP_9gwW<5~W7x?X=wxx8EG^B)?_Q=+_&fI-bh+EvkYIbFur}4=S}toO)^N{Lk)__k z+R0kIOTKf1@H@&OH>&Ld-ikql=DS}fr)CFlzuV6xC$j{dhjp9rg;DxHjCgd9syyz_ z^Ao%Xg{GI+F(1tQN6O>2Z&`PSKBRUs&Dd6jzu``w)8j@RW%()omLznTXOi~>-RZYb zyhHXB#b29va+I?@R<%WHV);s)+Oqnj)qT_b$8?oAoz*cuJ_&P})qT4aX4d@oNuOuk zxP|nGgZcqDcrp_1xcu1d^d7HOD#4;v-z_pZZS;Os=6ZD7Lu@gaNOj_pR_?_D+BHv- zd|DfLJ-(JUD}k0?nJE^(FiCe)RWD9AB>1QlvNI-kkOwiu0*_MAYGm#v-RPJO<(jI+ zah4TT{rV&*rA!LDf~{mJ5$M2fZOMF0Gtjnb=!A|ZX>H&1J;33Z$9NjG6+Z<8>l9Dj zX${TBdr8ab1X-G#DR@-z{s?Nn&dyng&Kx$J1k0Soaf`x^*`}Y;wg|B$Nm7<%LqIJU zeHXn|8yO_iG9)H6+cj2V0yDq+GQ*cs_glVR&!=u@@?wqQHgb&f>sCWe|_Qw`9|rZ;gFS+s}lUsh6C+wxkKB69R?#!2AB%bWM&ikAxnf^|9zp9OjR zv+m1Qk=w_{sYiDKF`$>|yVWml0;d6$k1^hM(Jv(_zSHvYa9NNBX=XLEsd@ofZgJxw zI=`_2_Lb?nau?Feu-q7%(qO$ZmrF5b_I@Q2oPp z>|@KXtXt+~5c;`maDc4oXy$f~71X#IZJ})=xR>u>*Ip5W^EUzacL14qEt8e^3~7GF zN|&ZarjQ1YsQ&AuQ?>P+Yh_d-tPr}}>CH?EJQh`C4!$-~pd5@*U2`tGkX^6QFxo*r z6Z`GVW$A8`dr5pRmC>@=<9myrEL0xP4~KcWeA|rP`MOR!cq}q&&Ew#D`#wG8*U9U3 z!%r4#Uy`oWyvwEjNj7!9lAO$(+6={>pzB`Kd0}v_m2WodBKC1v>&uV6LBk|wVi8?n zNEhCBDz!Li*M*3D;Q&ty)6SWWgY_kc-02)^dAJ6KFI+#bn|dWjp7O|Ayi+cY5h>|Z z=&bASBwN;@k}z^M*W1v0Su*b8rgRd84&zE$BAi zuSyGNyRY$$`~556Vi~0@Jq7%i<-0*Pqq&BsE0O)qgf*^IlCRC~BA!T9_PcFUh%UUp zcGNQMW)nGFtJ^}|DlhF^?*gr}Vh!(kKg_EW2#Dv(7sd%D_0C`+)@A~e* zM(H1ioU<%<*_!<4%2f3K&2W(W7`&v(GFlBH8je-<;i7luKJNKQ*NJxCEisc}S-j_( zGons9=-!DW$kKR;p7AGIB-oY!Yfr#dKrwH26{3>J4xtjWS<={oSG5YLZIrT@Hy0s; zdE67|oBP#M?rZS+24E+#ggvKX&8P(CqUgamymOq-$tL1<=#mcC_3i!^$=sI#3HgJa zNMi+9jG5!4&mLD@P$U|Y_UVdOpoN21=d+KaRW<7^g*Ike+M-}M>|(8GGfUXx{!9Py z5Cv^#rex5IF$>fc1Flx7Ggr*a%RusdBPVy_renztPN`CjXIo`7YG2B})@bAkD-Cas zEa!FpQdNt_?b)7dSqwCu=5R+p_Q3?A5v}TsM|C1QM|!uC(PKjMGH}?eW&E1#M?r}> zcpzrqN`X{imtB;ZG9KNVrJr4946BfqC|}SZris!rd6hw83F5EdX;6#~y5u@)N1V^R zv=*cOUb}-XNDS@x5oAVSi7Ful<{p9;YyEAzE2GMa)s3Om-e0TxxwRP%&8^xf5lBg0 z0Dp<;W$C8>i;53v<`<68Y3oFjtOqTLeYpZQXr-o0W7Yu;<(JEvaZ{_ zbjw!aK~{Uh8m}^?+wXNW7<`O;V6jnBh6}JUQ8^8=o^4R;Ok;r`DG(!SuF+yW!5O{S zQr%0WxDgWXk7IunR>zdv&Hr;8CTbs$kVq%nAV=JFNX)_?+;BZ3+@z9oFxH=v;f*jB zs)#keh`V0~L`8OHRVbr&Mv5E^z=0$IMc5*C_u^*~wS8_kUN|4!7kAS>4_uecfz@PGEA(NdkTjg05pLPa@in{T0YATF`-zW@ z7n0T1tOhP$Pj%NzsyDl)4}kdU(>Hga*VEM-k*zQVo9Ak|+>{@x!gChfUL<}PUOD4S z{emgClKWiw=L!-liEtKR4XD5y`7;NBt3%jD!%>s*y*N)-sKz__$3tX{(b%AYhq&+) zlC{@0nzVm$0wK{9QC91z)|5$f;X(XQ``jYZUbw&z_v6WTB=pnoHPH$tAy=%Ic`Wt{ zGuaSKyQg9A*9h{#fSWTSqpQqmG_C3^s(Sni_To+A;?(_k>K3J67*}MSdPq)I9^{w2 zO`{zVrdCjg8Ia3P_;=}@56X5hd@8W^-&?0TkCMcTj>M|m@C~(?7=Usuo_mFN-uJ&$ z+RhW{gL}rvr<8Iu-0ZL26?vZ)y06j{KL!TQ@KbMk0cV8JX)&m$$a|lR&@J!eo7BsU z?LL*md|7wAU1c3>!s>&L8n(M1A7}xm5qb=gWzsLMQDy6 z?kiiQ7;UrEio*BYEe}LSOzX3HV%<7dkzat9T>3RGa6y!5htgf+!NNC1f&`fgGby!|X($R!2OQ@quQDhgY(>n1p?&b`=0<7>aB(3P%Yf5C8LFa2BL!Rbq1 zu~1L9onb>kbSgdxkB2TP;RKAf<`<>YlT#|BFvDuwAVN8F_jfzl+n1+*#J6*kazvR; z-Ud_^Klqs7nK3X*a`)_uC6z9>OJyn&s*AF}L_1cEn-#JN@kYl0yMjZ;=z?^9e;~H; z@EW=>iP|BL8$I9sT*ve1GlV68rC7-AD_HuIL;V#Vts62U2K=9}r*opiC%q$wZ}I|7 zG8T5U{4!2mEm69ILxU2lV|3NjLLCiFpe{|Zu^`Jhz@uidshEm%@R^~F;-e8|g&sGe1@pg*Dl%skT| zgcl|5VnLt(fN6=)CYB99>=#=lo5n5iuC~RH1NwT!|M6?L1*^=6QnV}rjVTUI>d6^1 z>XEBDDmb1&niBHTldtKT48zq}c)$loXKa=zHb+7cjm+r04S!J;km^6$3y|2hV*e8{ z%N7FRIXj|j?NF8vf#8}!j3a;S8svl(-{Xf{!R0U6LcLFarrc$|65uyq8-T0LUKd`& zFW`PcM^wdl0P2_HFn7siE$hvxGh05snzxdAKP@qS+tMG?PhQ_WK`J)|+L)p(NcH+E z-75a7ca7+azC@E?xL|6bTz|%iS8&fMJ3{vAOyZt%6zOSgGIQSDI29>OL2g_ z{W~R&a|5s6og$}-eKI5g&|_X6Gx>`H2Lx){i}L;a<1Ey^=t^LcOC1Ziy~$qYiDds! zvo2Jhig>ih;(Zn8IJtBQVg7#2f5=W&w%@t?ykis`JZ%a9Q^dPxx^<%p0CRBgoY2lp zUu2!8nCQRxHV%W#b}gk!I;~6vJ|5Os2{a@uHJDSnpfY@ENhav_-aEm;%mxKG`@-7! zF?XG7Ug9IYO-`?amN);ghYzjy-p@`gHL8CWQ3ZFfcq-=lb|*16ApZIMupnAX!|Rxs zG%S)t^CvS#QhJH3gscLSqq<4crzop%@*e9ylq-MFcS5?z(qftb|XZs;$792}I<}3q=qcj&JJ&K zCsu+#-)qtzw0!ZE%8OR3<9P1caWVhuR|?KD{T3}2nhA(LnEQyN|J1V|5HJuCXuZkS z$Ee-Ugz!s#QLb_?yehOMN7pOJ>zN>b3Umn} zhUUC5qbdH+$9V&oGnK2R&huCOZRYJ1d{wpkqF#mDT#o-X{lcoU^8<=aKQkK=o0B!t zMags-Kn5|NC(yzdtPAY0MK-Ms^dX93#!nKY7pe;Mn{Ii1(n2D?W%sBC|2z$u?fcJQ zz-l1}CmZ51=wz?0S>%7DGe&{*#>P^o1kuQEwbp74Wrr_b$Zhk#s{*3!B-%zYyx>hJ z;BI2?mbk5E!m9=BmjzUW;jokaT*$NE=5QFdZfw?jKL@gxXq?^cBXxJr?B0=0?{>`lXH@e)%PxBZ=D> zm^y^p6&dXG_8$AS*&v|cFkC`ftf|s)E&V~YR8#+DEU-#`%E@}123hb;Z5QYmHKnbO zPJK<#H7DrAW2gJtZ0>)J#^Ecv*1)rhe|D#40c4|v7)fap)&gm8OA#qm{(w1&H#CEf zK?A=NH9nI%Iut?{;bJZUvIdm9m@Q=u;(bBVLHVu?f zo&sq_$S%IoDh!0vJ16u}(tA1?R`@jR1v(4s&ug^2Ihqe7^-|v?78A!a z4@*cSRolP~ceLu!|5|xYNED&EJdSh1ZIr%2Hfc;o)X?(AXq2Ns0Y!Az3ll?fpa5T+ zKbUO$dogE&y_uKLX=hEPGg1v&6&IwwKPrKNsA53iBM?xhQF0h&CFL*P)i}AU=M;6T z0H{@36ruYjFa=|Dsubn9wwHU;yt{sws{g%p%Q{%(D(1ip(KaLm*-WC%6srhHWD zZd9cDpJpfjSxqbY7R@|sZ-$MNE1E5wZzUT6^($d7YNkmHJFT3~oHlauu=paZao3!@ zK$zs8d|Hw}tc_GDf*>56kkJJ?R)a(c{9vFt(jKYtlCc5N`97WWd1O~&0mc$os6(K7 z$W%Mws+m3fBza+-2#FKov^)hvH3^^ct2BG_YtVi6~u*ZxETW zhwgz-_9Cb9OJ_&CoWYAEAF4@^!6m(ZA-bH-U7X~#CneEYu!H|~UboZS(5D3!&z*wd z3g_xSLWw8 zfET9d%;MeA=Hd!Z=f*~-hY#MFsKmH|{qXE|_!P{4PxDZgb5x#zFB!Kb+C;*r&wRr(oY81znJ= zuy&80yr5|B$h|@DOy4nHh*A}-pCL_Js7~Am(aGccgouGl>WfK>Gyz=dS$_DL5!}yO z>x%q80Dz6msU^B+)bG_1Sy%S_kFg248q_F$ogPp_Ycoin1K>yp%7^~Qb+2pwx#N6v zXuIy$)&X5Z9v|!HfQ0zi+-n7bnD=O$wARk7pw>i@Y5Y&9M5=LwF`4;%vxRO@R+0em zd=^N!`Cq!@jU7x|j~@}=3cdFG=fzTqSjmFXj8dztkK-pldxcSSeAc{}-Hk#Rbgc%l zcB?Drs0HrsNSE*zT`eAssE97??kcZH?e5M&m8mPzvu!^9gyCUHpzsm#xT0>FW~*gs zf+@B2Wt6>QPhfZgww(~c35|u6JmrNjAg)z|tSD5sWOOIYTekD{yYl^RJgGl!JmjFB zS^9dFAiU1-Qc^-)?)VvdHg#?U41q``r~?%SXl4 ze-paBn)8WuM#jID9sH}j?{KqIr`FLB6HH}JCZ}>XleX^NqYsjF)_a@gcel3|i;01t z@y&#avUJm2msM2)GQ=!z<=nIbPG}?EZMlm1uYGT@OIA?s65=-tI&ni1n;0mwLjC## z-y!#lvc&Z;?2%KAtFVdiW>A;mxes}s`aCa^l=xp}8PN~+pDEJG%2l;F$Ca0zCdH7= zv`@+34H8gFtk72W$>sfRXW?zbc#bYWMt$QN4ER}ILx;|!4ZcvwCpvQ8MTp_%XzzP9 zp1xx=4o>~nX_HGS6`ZJsW-df@TD|xV!TAY|_rUlD=|}jY*|!bnXQqo;X@fHd{Wh1U zW;^!8E+`JkW>_-d~6>Ip6iPnCWBQW0^lOp=HeObA&JK;{lo9!Ok zfv6H@26AQOuIX^=B{mi$P?pc!IoGX{1$|MW#V^AXpD9hjun93$9XbCL#TFk^t`y22 z-z-oii5{AAN2#O{`se5yeCpa9$t1iGc@f^Mh&*f(j0u09djW?D&HW=OyV2RFTW3@; zx*PenY4?N0t@wI|nb{IGmkneHp$KrA*)r;&I1{KF!(M!sV+{2nuOoUj3SQ*`lhkP_gi!jIy^$_Loe2@F`HE=hgyK{D*ScHB#w-Qg*?CLFv^RvP;=at> zSZY&S(|F<^_f!n$Kt`+KD?Dg=cGl$&1zEsuQvC(KtPdT(QA%eg zq!UL&xK}MpD-N$~>BS!)xfthbvc(1n|4Lk4lWjC=k zJ77cC8g=-G?&`}|3fH=_7uGV~&=+Vz!+Q1DIOfZvY9f01>+d*9wr9?%$6?QnD4FZN z1n+VRpre+=u-2JFLX$H(Xu=t-94xgt(PLLMc`U#<#-7FnQm(>jYGjv#mz*NlNC zcM=WGpKQ$j4peTbv8Wa-*q{i%S4`}$8A|H~L}%u-GNwa7pa63oW`JU*TyR)nditvj zN=Uz_5l3VczH+9Pff*xJ31^)Kb9|qM@wy-_wcGQ@Zr7OlX(@6!eyW^kZt$h6ks?6n z+}sF8SMaYlcwNK(X^r80{9V9cm@HQ_P!cvbvD9f4k~RO`#E8GOHNO6QF3X>j2}zD9 zK{On@$Ad_Hi~s<~#jv=Hzk-=A-^9YMeK!UxQ#+xpeJilL*iA&sz)bf;>+3(CMc3u+ zYCgsd$L-OP&>9B#N7%73tv{2Dryxz?Sn8k-mnPqa({yE#1k{n{en5tpc<>k<;2fo^ zcW@Rmfp7mCeT4&jte>VI_1`MSAmIfS77ib%^VXkY6D_U2RisM|(p`;av@`P{CVsPD zpp0yz!@0H>^NKpfA%sA^zXNGAKU)U&n#FPo)}43I{rWq$`{p!zjvsBiahvnrA`*f4 z?|Qhij^t>Cd$7u18|jXd$%px&N^OYgJIXIK>J3#KDk?^=xb9Gz=BU>v?n?HT&#_gM z1-pE)7phBP%}GA?VdHxYz)4~3P$w`*S^Kz`ddinj-Pl*9?ZQ@~u^=$q0!zQo5OpET zFQEm%ffT|rXZAtc=f;ybe1T9#&f9`M_N6toy-@hK;MGzt4nfkhcUU;2XzGPpUh z?5^M@@|LC^L0Z%Dwr~8z)fWK7q1MMmFc~x*ZHuwzQ{<~_l9Ibm>g9S zo>1Rx7#rOxAo}0z%tKAFneR~<;KK(c z15$M~slb#Z)(;s}T1oAY&p1rgf1a!o<{1kDtU7SPdmPC6d%E(APe3|vL55TG3@F!i zX2POn+p zJf?40QX;IW{LsZh)1~|qT`L&loS34o52W4)ue_8Lw?)?9{+`%4As6-hd7qJ5vFr@89Q}h=}@sXyg?o@@h zZ~dnCqL>)CK%1>v3E3eTrFnpuXIyMm1z(>P)7{o?8n)i|6GMI?stEMR~jUe^Q?;tmz~-Ai`!|YmmL#TJPCwiW7_%~DDP{8fYTW(FLR+x>S|~K zUE)0b@Wopgriw08rKuZ%`{lm0ZL*4F^J$0v;Hw26ZGv@|!*-`6GRl8Lf3yKctzP@B z8wKq)mektjK!A53|9J~>T+iPzgPywVX6uDYO&v>DftOqi%|Hs7)hfJ>a`LsS`s><< zN0ZG(11&x6C%hj=Xk3ZH>PY)UMI1>5`33n&uPoEdqniYiOADnur)vT)3mPVQSly>l zx0d&4XRkGnONTW*zKqV{Xzq3&iCL{PYdT@a^3I0)&4!0LXDbq(rkdRyI_^6jbhiNm zaO70zK5};W+1Ivt1Xk!@8(dQIM+EiNCzxM(>*q3 znD%KkQ1Tn39hO)Jb;|oqmOvY$v?`)bfXW#aI;6oJ#QUTraW5vS23qRyyOdgba$V@d z_!$(Tqqv$`3<)-eECBfAMA$$$q4GqVWG|D|ife(FBL3`;@iug8qEf@E_@#1w3-?(8 z*_adjhgco6u+04`a7+989`dZjlToWSDw7)(N+StMZhG`2VDuaOUa$b3Y?}j(*-KWoH9Jla zMB`{4yt#kw28;1ae{QO5d5~YrslhK=F28i?k!pVD@(mPc;^!h`5fcfuYk`EsTM#x+Eb{5pkZ9OMt*Bs zQ1F7O#u7_|Ze+P{%jP8ykhz0GT!NKg6b3WYr1f z09M7nDGtGrZ;i5}df8KX7Bc;Rz6)uXBmpUSR<&M9*0tocj?f&jLvNiYx5aKgRs1B! znjhi)fd&_h5a;c(6;e=VVh9Og%c)#oCceGID?Dua*?MV67$meD&{p?a_RV8(ET$vy#9{1ONdN<}Fc!Iu=tn2G8%pyB!owt0Ud4Hc8 zCL9p>1`1us^~eYCVO>Es3H4)k70Z4UD}ohzKXp67qz_R33~9Zegejqzyn;`Uf1f;t zLihXica!J$c)46$rK9cUfFQ#uIY(bjnl^wA&v zXxN7rkEXMb|5i;R%Ky&Y76R&?ec>p+ADp*kcJvyBSCx3+wm27V#_JgaTZJi_Jt@8W zGqJHRS5i9+yLs|n2>BP!APZD&%QL zefx43F@+4iXscvhplpqD2H8Q}zX|lm_|J0P-;P@ZpwJ4xUmZ4Phva!_5!Mn2Ia}t! z4&|hP{ik7IsQupW!9;{YCSUXx%=_)%P~>j6c|`l_l123?2~Uv0jFp#Qa5h9R6DRdd zvZkCHePXDn=tFxoGfN9}t@E9mmA&QIh;wMVsuH+5x@Yhok$XYG6wY(`2mcPgtvzSi zArmslJsw*qa5`SdwgjBuw)4Qm$DG--c$jCV$biEN;c+V^);SQPU_D(#VP|`JfCpOc z!D~k8#QDXXB7>YSQP?SI;XK^9-n+iSzVKRa6sKw%xkJ~%P3t0XL$=Sp^omz;IXjH8 zw;=R2EA|M&iSk3Xvfri$dHY61AxX|*(FX{bWdB)bFj1LteUo@o}!lSMnwso_!yJei>}&$Z>?YmHJo5 zq<;ZUsf)$&<%)ht9purmCEY*JnC@0ED+Gt`1)BHw;FA`Pr27uX4alFyUSjKwjBuWJ zf78IG2*mvGGH3r)Pq1H(YD4@Q0d^9`J{}M(&b0loF@uctJv^G|YdkK0|Jrm8RZ-Xr zeQoYyV71{(bob5Fd2U8Q4#;mcmIf<9sFj`lN~u`zu2S~98CfGOYbESO!u#UuNR{=fE+XbCoEDLe7W9x zMLUZZ#-jF2+5YD1=Z*Kvl?QPAj)1N%DTkZj`%fo$s_}2rC6!L!>GRGk2rNMDo#Nu5 zs=n_P=2IM5ZmOVOKg>(KC#7q%Lc1hiKR4dHznOX*lK=Bwh74QnS$wj+Sh_eoerNM_ zzOr|~O>kS9ed}-Z_HWSFBm-=ri7gb%9lC9JxVnZFkbFpkbtp1DDZ2hxU6$Jk&{RL1um08lz2D4mOqMZBe=v6~T5DZgI?gcfIHvHPR zT;&oJ6syi#%2eOv5fQb((C}NofU$Yx{_Y;#*6n-qNu=zp0_u#6U)#;ynvp90^E<}2 zhS(@J9G@q?uDQ?3A2kzCHnS}8NrIqG&~Vqs($6z?+Xv&2rxXS)O&%Ovm$j1LHf>ag z^lc{7^f&*`Ca>4opN}|ol`aPLN#wcGLD$hHC9`Vi5Zo;hz1zCCl_W8K={0=CewVwmWt+N^2%YynPBpxUV-N7#+^uVf zDUHOXwtNiann?&Xu2vBuk{g;C=4OpRW<23~_&%PzMppMQ8)%(!LXkYRxw-w9qsQx_ z_p1$#1xNZ&DPzovVn6A#T#p?Iq989X0!Q|M@gF$gS{I_W+le^dRs&7hB+%bx(LYo7 z3NdtL^HZ$mjQ-2#&xR(MI;<&XOzcvwH{E&9zc-t=+M`Fb-17Y^IS1G=7|%jO(^sN* zdo?Xs&JYNMyxUR=CwMh@YZ>(k8ezX|ZbNYlw_v0zij@c~oxEfcl20=* zQ#d2%^0pfH4zR%y759??H7f9Ow}@0ZyS`hJ%ZM&(VG4O9Vkxp9@?@1<@n<*wfH@1f zXD&VXV3EVCGnHy&%ndu`n)PK&^OGpEXFTh-DzLD84U^fLDwf*I*(KUsZjmc+bBL~- z{@|*ff4JKldmzUG%S{*FOBHh~pN+-9RYG%#4PHB!I;H4Cvww8yLVkGS z#aJ>dJee+E@bbXz7CTIz>HMzxs=L}~7M#b8*9iP@Y0Oj16(Q)gNSGd=()cof`Rv(5 z2kF<+N205PrtVLablzvKK0N%nN0m%jYA#il@3epHW#uVC{&fz)=)I|Fwd6V7J!_nq z>-c)QPnsRUKMcL?!c(?yBUxKAG4u>PmzPMq6qJy4|61gw6e0K&S8Sc7IQH`8$<-m* z7R#yAD$SxthJ4*t>W1lX*T(_=zM?~&O3t5q$z!${y?=(LfW0Vp0oc!&VhBmw>aH3` zgC=(6&6uh`jz5FDH5N4^Y-##2+I_xEfJi6WmhpXNFz05ozxv6gWbxmw1@ zSoj1w8&92>~(i*wDJJAs`Fpur#e0*|m zMgHkhQ1pGsg`^mhqg^ZAH}A1mhYy#$jlVe$h~5{&`Iofr@@C4b6=~q*MjW9Zf>BxI z0U6IqdZN=sa3(U;I=IjEriv&#Ph@$=>2`OX1T2_(t;N#rOUWtNwC8u~z^y;oTRg^W z=FkY$^GTb4!tj#fcKjy%U{ z)GnE-%4Yp5Rop_74>!4}+5uk{S!*98hAy4iR8aQ}jQM17le7>AcFy~C4vV{-&rSUK z`p~H5tlssP1L>(HyDl6LW0tSW1PAwzdOup{MxHp_R$?W%4V!aa zR2jIPYxNUeR8eaB=*d^0fnTgMO2PGo0o|>0!^2A8g)9J zz8B4GCcesS4pe|^4`fARXikue{nlZ=Cj%F(t@IDpt~*+ZqIDEH!4ke=yYH{J)zail z2ZfE^sqCr49tHD1x>s|;^AQXW^?Q%d`mf{nfgWC_tnnhZKfk&u{Mxqy)x`A^USbPL z2K2WW&V9e5>{8OeQ?037v>a+pF7VJ?7u)#rlw>C%-0LLOjrVJj?(rt;W{A`M7GVuH z`>BB=Q1`|>*Hq#T-5##gMl!XyI&4+PYh+iP^OAtxU|4qN|9JY!ur`~fZM0C_DNrcx z?(VL|JrsBM0L8sPad&t31b25Qg#v-%PJzptWC+#-4R52i4(~s_Im5l!+EwJ(M%`pMpEplJ#XzW`z%l+d4Kq zevikEEl?5rJ7P!wb>-i(yvj!|hb+Mw68BxNexQkqVMyEp9=2U)Rgq$DpZH;OKN~NvN&2*Bd2|Dh`G0Z_i)-tc-XV3Z?QzKl{NxjQxWp8@}CWEP_{I2l3^}zYdlG6rvYF{FU#`|D520pl*cbrD%)?G?%W4g@u zc*Dd30J%^;UV;)5+y zH2pldtXsC?odF*|8{ZvwhZUp(@U!z`A48glL&fDvl7*}d&*Y=JQGTT~hkX{Zmrx?U zQf;zdJ|l6+h(T>lAn)_Ixg+%9=GM=4*>cSXi~Z{Eq^wG637r`E>(3|NrENN+Z6^^U zi-+ER4syIJxay}VJ~C^7VR5I=2{!G`4ynI=Wct3%f)(pR<$LD)lqh>UFHSlSJ$ct9 z7V4+ozHu=rb?@$-9O>RaM~Ul;(3>DaXP5LOpsnBzge{gjB+Jp*A9fm+6eQPKjAuyIOK38axwMTCQJwE?+`zB&C zoJknXn5jta;qSofBJ@GTD2*_9ttBkVVTY{mw}`iZZ{!-kF?JqtX4@tL$H9~B0ZATl zsSJlTpt<8+;Jabkf~~%kwxKC6^za`{7W9$=!1sufjR8%`V1%B}g2uR}*ueV{!{y?I z-e{sE6aX>*b`99RTm_6gh%LllJc@`lytrk9xqw$&=xnZtm`2QJEDfVQ*ulIf>U{qe z^zo0^J$gdpi52OvNS%lVjCw1oWbVEueV!`uP57-d_s^J=Lmz&0uj)W7v>zo(4=rZ` zv|s-u(j!MIOs3j(5j_~5t=OB_h`G@t%F4CEebKczk+`+_Bjp4#!~-(jO*wZe(COGy2f28o4dSF9tuk490c1RbU!#SNCk&?H*jH#pmqL10Mw7K^uCIg z>Hg$0GA~c{FFNw&CIn96%)pVFsl>{b22@Ac@%}{ZoSOAP^RDqJ7BIRLx#1qaS}@NN zjf_&$NaH*DfqeI$u3%gL7!M-c2cnB|$5@S8K!+}5+HAMXoUxl0D_z(OU#(4&<>>tF zdE2U7DNVL7vfi=FiiCU#*>J!M!D=n7=^`|PT0?Zd;w2+peVsiUwG;vd2cSbSr?@twEmf{fzTwy^p$?8liBXk>!+{zK}D2}uZqi$ z%c^vRPnN}0(JTH7V^~Xm9_r&-Jw(Z$q1NhzSm8=;5MPjZ{ZaZ4tMrt+(0-Ob=EhXg zL9W7CD7Kq)g&EOg{+pj0pDT+!()_Ir=8EMm_Br?eS;4{?8}sWWobt+4X_7z2GiU+H zkBa$z3q7w+=4(cZF7${2m_8k*xD&F_DAwVJcu4;?DlS+a9e-{6Ek)O^OpI9omnLZa*l*PmlwA2sdTmoVS?)?HH9Rz+}gr0TcVxJFZ(UM&w%!x;!-Z$5Bdg-e*k=4{5dSbpgU zG938Wy|&3*ETOM*S>$nVpiSJ0gR3zPcM7Er9f-g}vDDE_!w=1}B(z*W3!%d1yVB8p zdsji1=AUh8?YPU2Du9iRt2+xH;@k=NnvNKeQ0}t+jiLzf(`e{ugY)^~&*~aFP*2Fr z2L%OKR~`GF_k9B9MJ7rRiXtUW87v9Jhjc$$#jtz`r?&obBIVXv0|nP{X4No2L@+;m zQ%|!2-*aNZ5fM)bZyjIVV(n6qGSmAA=yw&kF{!iu5TWRm)=+*`xx^W<Y(tU$YA_U8F~I`s_R;@nO=~TFAc)+0jI*i5dsZIp4re2< zjkUCCvi*oW@!>7%*ovbW*yQ2Ns(^0Wa-CLOfu+xdLJ5mo+QB2Xw6Diwu5Y2Wlj^waO@ZU!0x7S0@jFXWT4I zvqbFhPm~w}k9rMj(Qo|`B-l8*YBC1gnCxLSZ?v2;Ds@2e;E0Y=gk#FF0(*Gw9|q``mS{!U0NE|v^x&F}Cp^vUS5Ak99Zko z`EQ-P0_`nkeMk__+*Dt?Z7dEJeZ|N8HN=!4t8t;p+!5rCTbfsc(+<2)7y;;+q|ds*Qwp*i;q$WO~yWIDvjV#s-IOgSqxVrDk9X|$|?$lT&dqt zybOtwfyPd;REIM66G-p2N5$E?EzXyd{Mi0TrH<}31qUsVvPi4c^`s*lOnRV=X-eT? zHoCk1zAWA-I7Rre#d`BzVUfz&E|TyW8}7r7!**-RH{m1s*&Wl(*Dva29sFk9vLzy? z_N#*!NWP!u7ym~LXzG>vWVH#8?=8<^@=kmAUe-^ywpR!Px|Dj5_80$axSm0B#U=0b zJlL&+(i%m3Y)AU4Wd#cTwz}@GP>G-(Qe}@2vJv*j0CnZaMa;-zwY&~A&8KQoozMH#dAWoq6YKLxp&A z(-tMf$S5BTk;bWh7?4OogLLST<>Y+%TQ6v&M+;$*%m1r>;9PFX=%taat7DC99?mdvaxr#ZEXi}O?8mw9MIzaDAAU5zWkEYXM>=5jJ%TMZHJ zV!n9hHJtJABg0qyiUqP{v$Y>c_}($H)`5l?IPGdnkq`~K*t1OTFOR0nX#*xP?02NE zJCgs&2DTw}k1)v}*D z#eXPhbRGO)m`-@>x~uB-j^QE*}w2xxnI&?8|Ywc_gW9a z!L+-@b--VQiVn7()^2H)+#@3d1_H{sY{_^fCUE_Izz-X$Kqe8{ZtyI!ZP`>+z!PoP zyVlchS{>eD+1!oUT6?OKV_PKszz!(onXbyyUB@JYw7p0$%SJJaNzF!T?z zP8Xj-PX9gTz%g8WSEqKCsqbLrW1bS*_T04|ho8MeBS^d~cWK?^K1JI+H)M9OK%YQ+ z`?);#36As0L%ng^F36Bw74WeG3pF67NmFJxS>@2^oh_Fn!4ic(ak(s@qc@ur?i^Wt zk%~N2==y_7;<_wl=dADDQzD`a&s8aDKyGQZ6G^`HdBP(yNMSBv zIcno-?!-_Ju`|alf}cb?W>)od2SEv90=en%C&gD;6Rgj>^43=_I9 z_vh~|3?xU8cKw-v<(Qy_kq5gS82`w*#M6D?EN5epKq-hE?bX5{hHM(Nh1~6xfRoZ7L!C9U;UPE ziGaJz%u11eLVcjX8T~~~qvMY6sF(rv_BjoN3DkEKIe|r)#*i6NhChQldn%)ArjDiU z8p-lX+Gw%>ema=|a6PX)BDh~9)#L`GXA!w-c(+IqK5W+Nw0*|kk{(1C@6)(Go(stR zJ$GbD=sZQr0n`ZNA*0NJDTQVd)H`Mifi4(tKcF<-$R|JPdsR;;Ylo1wwt0G$uj|Fb z(patT(btEQ|sd6ug8y1Q0fWd@OE45;d z4)-~>bn09@BFv(JhUwrD2kpP1=Zf%bm)Ib)degpAL&NpL53pQSZ8&g?&v{l=`~0tX zMub&Ge=qRkOVH+5M7s>%!!>(9IVn$OS%8fw(!!uL*`lN?u}e9>fLZMgY6+vnT|>27 z9V5mh#~*^3IGRZZ5;cg%0l*iFA|dOv+#O)57NYIJCJTl^)`mWTE}onY;6WedAh_Kn zDso~ithZ~vG7B-Ok# z>;#6uTDD%xr;M)c zjq6?6C>s55voM9$GySpr12*8hx`Uc5vmGs*U(oLR9$deNvkOQko%^Mwsp0eu^iLjW zpOl_ZF`y(;mP(ry0PB*KJ4Q+KPcSs};dtO)RVK+h>sl{7sF^Fy9P@u5j5vZJ<#KNS z{ig{f?+9wE68relztLA+JSvoMoA^1F)PDspPoMp^HQ43C7RpR4$1Kljq6y2n{zAiJp3vq)uG1%*0&%#IIFQ38 z%CSbspOK#vWT>n~n_eMVu6hSy3B0(9HE+AVt2Qv7SN6Q?h1F7bBL^NFK&Ax_NA68L2v_#y+O<**wv)|h=VwAKqQxJSO~r- z=S|DhfhGvkyn_zBj#tc_?M!W3v?J1=p%U-YTvZ4({VpwlO&5^&^XmBG@9Bxqw(zkC_;ihsGm8?lPd_jzT-M4<$}c8Gq08J^*|otiS(UakXvo+@8%%EYz`2SvFaPbKuG%wGpOIGvrhI@|bV>9!B-_Yiez z&`F=na~in3EhtF1E-bVHe)e!O263Jj9@W?4ef`|MT{*DL=iuC-;3gG5F`ePGIQuns zMrK-41b-+3lF+Aa9(*ve;+ame-!g@CvEHieD)W$ItHOrK`N zLgKo9`>8eV^LQKq@Qo=Z@y{Qb#5Pc-c0znFQ$kp2AF5zHe95Qj++52q>VZs<&-xzD zPXZm@_x3FEWVJ7n^|KvWXz)4b(K!_hj+yRmaahmtY5gw3UQac%Ix>$xq2$&XdUyC> z{~PqWDO`m*!pmHidwaDhO+X60ua|(mzb8)_;AFSy=@-HxPCI|C`MWLH;mjePbl|)h zRegl&hakQIn+S4dNZNN;mN+F)-5>!IkgCgpP5hO(SUO~=0IMgJ<7{Gcs{mBE^L%b_ z>ef!6xht{IB6d4}{EfupWj#Ye6U=9EENQt$f`lVt1hu9H^R0f ztVbCD_x*eO$|e(pbSQe@Y0&RV$=Cu7CbbDjm+mdbLz^LA{ypMfU&UeX&aCySCMoB) z6JnyWO7*&TS8(|N{G#ync4*4R%!#?}35*unHlJUeD~NQQXwSd*oQyfp+Kue3g>9T` z&6%vjavL?I6a`3?g+F3X42+Ac!2x>r|8Bv%muyrut{qi>tjrVn+Wk7PHdbCcr13LJ z+uCQ3?Up6^rGhfY6N3k3-g7P37}n<#3G0kTNF^%Th;iB&3LpwV;w6H%mxv!xMF7mG zUFn=vsM+eOv4+@1{f*J&cz1sTlloLpQsrd}>n>SxQa zI`~y6qxdrplb^h&+qjpiF&ojpm`L{dqakws^!4qKAK4w^Ncl>Q`T7 zc61XB2~zqbNDvvi2Df4@;@67$Lr>xY<#*!M`}p-gp2hOT0*Il6)O>1LyF2GAdxq$+ zRSt2Ku99J3B~H|l{7Q6amzq%k&ugPa%7D7Dz~4s%-&++sDAR=AI$3`3WR{o}`$`Nj zsQ#+C=K0N_uV1pziwT>bL6FYf+x?7MFkLhS%JPk|dWwY6V|6H2lTJRLSa8INUyqBj z^Op;{|B&39(K+%LM>4c2Xh65((uM1uS-1}z{l#b{Dc#Qo2H6SB3f-IInib9Idp3dc zb%y2egW;pLj!(qgXAfhiZhcEb$G^cKY$bsWdT@o7Yo>PUBU4`$R zS+zq5ED}ej%&bUo9EA4f2yhq;QUwE^WU$(Ck}8G5I9Fm-7H^8p=_|wBf@2&bT%?PW z55C(8*vwbK4K_^sO7yZ4&I)=nhHeV;>(p}AGzLPS9+=o8%zr13r^1O^D079O2? z_^cg=a=E&SWBCO3V$AndyB1fCsL18SNL`n(}Rl5cM)zC8FDM0-{M_l6>t~<=74%4LRF68N8;3z4Aw#*A6Lv5i{1q z#$j|54;9A?J&Cu>@#lS;d9i-ZwgzY^sjU_mVXBrbus4`|!Qi^ynCKfBx`lJoEy@^2>Y`eRSuv{L`51ALfH3_%7GTsZcxyx;v} zr^M5svJ}h?{@@&YKb^uVn-e}+q_7$_aN0r&P(c~AifpVb=jSqG`eGbrdw=3*fX#aw z7de}E){pK-`{vvc@8FVJ2KdqJkmwF|`06VVtgsv_L)-89wJ$iu4Ph$jmiwNR9vP0= z36L(=x0M5oRtHs02utO|C-rUx;Ox}(3>$&vD<`J$r$ehHISHvFBdK0aKUBg6C*3!I z2ef%yIry3EDE36PRSLoY+hm7=d@YXJJ`V;6yuQg5E(^oS0WR@GVgF4?5aVU~ z8`}UiB`B(49ilU5{Gz=sIHz`18(LG61k~lT2qBZlmRAd2-ya4)g|Azr#`#b-NfUf` z`>1So$t|Q(qw5JR`oy;v>*sXFYz7vazYE{EXmogbna}M@~x^{JbsWs(e!zvo%Wi zt0P?*h@HQaN8!@Y6E?ehaK-@lxn|<^tM^^d+{1V^g6H8cG$XC6>saFSZ{*+Pira;7 zyt4)RE2*@+H)3*r;_L{$>ycS5JUTZZ&wAqxCGZ-m|1+4YDix0}2bO6ZU8v1v0;L2{ z!Ww_M#l3>>^vOR}yKFCQ%P&{|$e*<~wvYood1ZONCOeuML}_hdO;Csm34B=RCO*Hz zp_M7?QX1B{7oH28ty~EB@>s@OT~+at{_!Egw#d|sN2ZZ$$dB$xvLKCU`h&W+#$|_% z26CSsx~I3?*I#-ycJ8~c$eUov_|rJ@EN;@v-H?w%sU1h2V^^|3C3w!)f;s))c)-r< z`O?tvrLq#tjvFLr?s7}ByYb@8+5tWu^iTxkiN48fAfXay2u!xJzI)S4tVc4<7PMv$ z3|M`eIQ{EQUO%_w<5nXbDUNe?&AJtC>BxggYx6r6@=q8~rV(2gk`sI6skKNbK+u8S zHxW;A^qTV{8o&;}sGl}FD`U)P&WY_Whj$fBxPPE{6{+0s5sz)B-!Bd^bxr8a%|Gt^ zq<0Qi9(gbYpZxn-uHXQW6D!%apLJq_3U;)N1KIo#qou0Rn!J0?_snjWh ztcr7y;D$9$XLk1sGhH{iep~@`!sdzyCZs1T?w}^a~Ph`qa|kxctau~;pbHu!MWlz zYRW(>?)M9Ak@WeUR-E?^E;+NarBjD0*e4~RO1;QVFx&l5+pmM+RM}dXfRt*^595UOxjheR;y#5+R23? zdS(h@h+JQD^s^(yzq6g=ekgOoShM;uf3wztbJb!D3qnHbvNLc#WQiS9Kg-)C*l8d% zmq`lwLsH%U9DS49q@;E%DodQ^hCx0mO?pZI?Ea_q2}ZZ@{^!pr!0_YG613JDQ{9M# zGnvEsnn#4A=@YY~)Sk_0J3$}BR;G*Q5@w`Ss-;1sPJ{L~0Yr(`VWVE_L*22RrKR;D zGZ7UPmBjt7!5X=R-rDa=_rVT+M4~IT9Q*>yC`;HMo~UcJ;qVGONUVXzI-7GF`6t=_ z`qn`T`dvR|6_i|>9nit^#+&A>|01gPzUTEh!JgbNoSuT3tS;G(5M%X;o z4e&Pcx-Q*kAkT1Q@pbU8$+c5YvEJ&83-iR2~z@^edV0&CfZ|3N6?l21)DDX=#voN0@p0>dI|yk^IzHb(tA63&T7< zB>^g2xz3rk7KUW&kF44JB&uBu>>dsX#>!|7GI)JJ2s!eob~Z8RWT3Ba&$0bceMRm5 zp{=1?cO_<8Ro()oIuf9Jo@aEcaWs9xY>Fim_8#vMrl>-iHb0nxc4)5u{thSNOI=DK z<)`54V)2BGm*Vmp$Qd@Cd469{%LS3E8Q59h1VmuL;CD?~MJ2D>&alki-hBgg%+8RN ziNc=b5=Ks(3$2tZv~|BVr(ZZdzc$*o331i2>3vNYnO=Eq-dZpfO;zGKO%unwHG;0J zFlg;qmD9oWFYpUZRsgZGdKDcG?Z=TW8jtHT$$#`Z{B$1G;PZF>zBE6msrDNG2>$z3 z9wn(_gw{GRXe7G8)eO8Rqc1OtxD{gOG4NhDJ`%Feev8nW{9R~BsV(--S0M-n#p7TOtog~{_qeP7&KAPEu~kb((;DyZL1P;Jn;d1kNN=xox=sM=T<;Ae#Mynyui|gK z0WdkICNaadju}aw|K?U*^ZS^rt&9fFJfp#oz_JX4Em1RyI%|>F#{+?_l|JDd<1j{H zgEY(H&fZugaU7m2>-f@y@_$PXyKpReRu<(Vp$)(|vqMtw-9r?a?03RYlq- za$B642t}6ns1)QzJr+M#rMBr^ME|{HTwJ~$p&R8n{v9hA;E@bT7*X366@^&ODt<8B z%X=3Md`OMZ%haAkk!u4A?|SzdNPrY-xnFa_r$Fm2U-sDJvpx>)M%mJlG?;5_1q+6~WdF3}GCXcAd|Rb)LOaGqmHGHv81BCUgA- zzuma5Xm|GJvSypU(?8HuA(z>9^sO^GicEkrn%f5AjH*So^lj8r&Kb&oxzL!SIc`G8 z<%<I}$RfV0w$aT~81Rs%V5#X!kl9KRknG!FFce>d5B%_+ogK{WjPY%zsg zLy8ppnPXpYi)6EEo9D*E*t75~x85#>fFE6IlKzy|bYx*FFbyw8&dix?Ssh&PO2T9=GBPsIFkuln2l~m>N%9($PQI2z463$&uHcX&IAgeJS=eT|@%Zk0 zt)`bz?FjbbL?6snyD#?Y3xn5*H(C2w=CPns_ef`g)HHzdvZZosf@^Py7-lxTxnI9~ zN&%dwIwhLyp#{VnstX$bDHiK~g3o~97u>LwB9Uyo z83x#Eck@x{m4D@c>mw~qj4+hkn6x&8dHbUJSvq6VZ5#utr4=MN)Bep=u7;5b5%J95 zw&RQA?_@RUA?6X4+Hm?t*mMpzI*k<6qKhhJ;mo83J8*f*zmqlh3FUcc4NmA*31Oe9 zXBuJkN=Sc+%=SYO!0pWPixxWJ4FKOS&95{HxzdL{x*s>U^37Cv-~#Xrk*4SC4do z%6egxgw#RhaCW~(3%+qZF*Y=k`kt)14sYvb%E%=P`f@+uWH)E|DF`HBvASSss($#7 z>(J=E4ujo+R@hY*8oY6$7m;;wW|~h#g~VpFae&maz5==m&M1s?_8SlVyR4U7|8~Wo z|4A$6$8J*Stlag$j*aW9VmBMMw@9Tmg_i@!Xg%mGH^!G$${MQdVurmX*oz*7ttx$-I8QNs)AyV-*z%&{{RY zEZD9a?@xxXDEPBk1Lz_90XcH+Sa6uO?_hkh4YBFLmL2vycsapQBQYk$7M*iUL50~@ zE`Q*zCF5ZlJ0bDDKyrn#itEM)KeimsO}ABpp3P^umJMENlb)K6t;GJvgIWXoesKvQ zOlFmja9+k5D)d>z!ntprwJ>C6|2EU&_=@XdlI39&E8$p*lyA1;d8cw=>7H`3R8t!; zwz)+g^3F0H{c2$A+x}PV^r8anR|b?S1VfL7&PrXky#9WYA_!6gIc;2!`U#Dly$7bJ z6QU>MvAQ%rTW`?tS!}a?N+PS9h?Z`Y$IP>e@#hVifBB`AiX!dFhpMFtDn$mc z*gQJ&vL_=6E8Y5s;r-mPkShx>`LR3O-4mgbdm$F1>gvKXM2ikCD+y~Fn9`W$0E*%; zOWsSUf_4!(T=9{uw=7)2E%aTZaf&Fvm_;AFgEj6fMlU}J7%XgKf zoHzu@^#2#G|FOb}k1JJHLoY9$eJrp&#x`Cj{g8--n*4CltI%`&N@pr5mM~_+)C!3R zs3b)uoh?u$eFZ9QKzK>v{H-1AKpzQ7(IlcAXB!*msQC%|P%Q237oAWdc z;-Xi~w*hGmCR>2P)_Er%<{TkmmY)Q_P%6>@6J(x1MuCZ6P$kj0F6O=|Ech`f9%TCn z*EJXilhGrO+z+ee?lhu9W9fhq3#VR|yud)MqA`j+pMQO>DxiOZ+_VUb(OS73Xf+^Po9O%Xo6M zp4$^4eXS)XIS7GW0W>cTF` z8&AQy>v?50qoTjF%$ivhnWz#1H(k_?b)T?zzF4#WxA?K0-{dSrRi^Aokl363-oY(8O|=Pe-iZGL3Ntora}xxp zg)m{z3*qZNnGGG@JE48q)0L+UozpU*r!QrcW#PxIX2Bk`pCgC3nEsq_l#u+6k!+}i z<18q9D=ana$xrEs_Ro5zG|0gkO-6A>maz7B6ocgr?K@sF>%BK8Z6>$ubYU%=G-qlp z1B$RpHDASU$BB>r`jkHpbe2_H7F4FX(C@X91s9Cn(w6n0L45I4bO+5!gxvchI;_b< zMRi?zh%={XU){Gt&6eknZ39y&6+Gj30*H4P-a>|HvTMJP-7o%d>34teP8Vd7ywuxr^#A zsv^Q#pxcsRMq*lc;EGXM-gtHqvxX>;e8RuBZr(c~#o|$R_oY3eK1dVi#{0PM6ob*# z5W^^xvHf``h*a#sC3yW$aT8p66QeV1pJ=ldJ2J#f$2OaNoBo^R6B_%xM5w;D6rGA+ z&OP?CwbW2V2ph5MPu=}b=|yM{HFYcaIZ8?E>XA^IpLt6tD$X^HAjQwh1)HIK6s-sH z&Rs}Sf0dKvLsNLex9|O&>>5P2Y=IR38o|#O`Y{uBo|S$X!1a>G zL$I{snVPkgiH__|Ux)CWF3>ujCAZz^?N+H?FQ9WM_{GjLH+hY$W72>%}U^qQdd5-Ex}_VLxdBZ}UMc+iLQ$9(+le#=LX=T5m_mkGSsmPy1+pS(g^7f)@R%IHL8m!WF z7Q-Ep;m-mOYD9t2LU9sz@1krVxr2AVoj)=6=Td&8;-b@1riwTa%(g$=0pZA1)9%LJ zHFT3jYtvnEJ)F)T_fVHaY$_!6{=@1-Iou7N4=z!VndiDBgXw1;UE+aR?Z?t0W=R*s z@;s>7>`4WuFhF#(HpdEIMx6&8mC3MQsN~RcYD#M=#qYak0fvLY*A!yxLk{tZ8P9BA zObhhFYJFdXnj``*TZ`4l{o`7Z`$a(gNo;)F$^00+wk!Lo>(?vN z#e8t>(!xvt-bkgr#r(fO*KB<{u0wT|jy<4afk!f8PJ(6cm(x|UbwXUa1qXCX$E!2p zp`h&eI7Oy=(%v{7*Ld%5ClzBB96VP@4j+K_bn*Mao#%Pq6*2m01#5;rXX9x%LQJae zo$P2!G@aWx3oL^Q5n#fe8rB!<6F+o#L~(g#0ki$b`G5GE^PA)1CW^E^TaZNEl~3G^ z+VCn^6yeQt+ptedb7)d0qyo1;-K&Apk=-f0+5alnUE!v?IXdsmgM4nN}F zfR>rMqVxhud{c*RIKNw+=JUOmGhVNNS#|T2Nq%W-^#$GygD^UvEbBt1-LTv~tI;y# z(=0SI6@_>fD)w@g#m66;txp{9=ZHa zk^CGm`2zJ#rLT__YWS>DR9vmUK{T6)B3COdt8@z;?qm$rU)P)s*W)~5Rp!A?nt2Uv ztQU8leIiCV8>BS;)L$}=tMMtzU!e`J+*K$X2L_QaDN1Yh>E))3QE)~)me0u@_Yd{- zOKWPhL#edvWJ{C#&XDBA*x%eZ_^r==MCC>lFfU$Pi3dFqUxgEc#7<(t0xG&W)=)4X z`yD_bXPFX5t4-cq#9uRIA&kQ``CvBhkSqtd-vW|g>6nLEVQH4_>XX10s$g|x|%PhE`N7Vk^jZ`L=N~!=2k(-#R==`9MM+xI} z7A8}q+Xo+QE5B2=JB68Vs8#o_?zAO0rB|*M5xWgU1md|vOLJ1F)A6YWJyH{+dN{j- z*b#ZnClK9SDJY+F=z1UuGwuUA2ixd}O?|~*R9M!(;=+b&0L+*g;Jz>McdS-L4mvS0 zyshxCICw#;lc`I(ZG%!1|L}!ZM53q%3NI^o2YJ`L$<21RR&?v+ScE5ywsz<=K8dcG zvNZ1Huhk=cXt1UxX8ik0e5;K~74IjAvMkd|NSrlbYA@LxocB^KwYq~Q@9Tyx7hU|c zDB^|OyeXx>iV+_ZXiIl+z!wnIl9-Aw{bwEmer&71u!rj`{YE%1tH)a^d#jis+TtaT z?>WxoY;TT8##rfvf>l1?!W5ZScJlo9s`C2+kj}(n9B|W8_=91C`TV=>n=9XTB7P|O zb_ZF8xAEeU4RhRgAbDy(2Mj}tuEfGv-oTWm<_4fijfLycNC{Zq#@u~w|%b_z)aZ+`x77F~-2mU)Lf z!OL*@R?XH74$YMkLh@Nkv4E5QBVMg@`CqdSecA-xd4rbr-*mKU4wNX?1*d<)61yzP zC^tvKV)4oi6OdlT2YMeNkow7J2ZTG}Ubcc(J3JAerRFvjNubvvDmqzzOQrA*-~1|9 zR+fZooQWOynZ$R+;bbO}&pb_lghABsew&pR`yNmN1A>x7N8ov3A!a5`*78~&T_!zu z&lr$X^sP{mO~Z7Ka37`;iUNt- zq@?lK_QwK3Z4W=iZe+-0h)$U?d%?cZ#|yP&*pV35dR{ft8URci=F`K^+~g0l%6Cgz z1sBXfp}Og(XShTL@0s_Af~Ugc5{?CT?Ed?La1twE^Thoh)?kB%X0-oPSXn+CGn=;Z zV(|>D+XgK^)?xHg(K<3=PI(WKc!<2G-X6JO9rTX8*BqZVt8V%RQ`h`q4`#acr$7 z;P551uK>iR7l2mpu^l-OV^%1oMx&+QeVKqCAk z#4vX1knA_cz&!m6k4O0@U_IN^?LcgEOg21UK~8)oRRL$XokP}GKy|>$|CWV$kD4KQ zsW2L^kNPgvQUQVl#A`}K(Q>uSW26Xv>GG)mjHI%hUFRvce)=&6h#AVaZ>+CNqh(qi>dvc*^)#m2 z@e!j5=CjF7Gb+p0)Zd=mEe@N`(ka=jKaT>fl*FD zoNbgvJJN$P6Z96!2;dz460@Z~{{_Jvsj~jy0@-1OW6X(HMCZT+HAvoY<;qgAyB} z|7vU&)~}rDMGvhW)k6Fy_}S%GML1Wtw@W*)qd5CNR(phCO2S?ukNF!af?+tHOFRX# z-#UnBH#ivueosX7et99LYQhKJ|6Kn%m8rD~@9BCD+P?nZqSdgmkEA@ez<#dUb|wX# z#O=@m25MpuP*#EQuHqEYl^(nOuAr>9m4r>=pabtrse`J@PPQAL>2au@IG|5ofRzlF zR~3vjXp$OZhYIrxIqxrC^iA-Cj)esU>~A0MCk=LY?$Dh(!UH~q9R@{lyQ1HTMRXel z(p_t&p%oE0v#++YmB#$zNSnzCM+E`pdaavbj-p|iomo+tNVWPmXUHxzj6tc>#^_1u&pu0u}2}PLDRs zY3e47*DtfqT__Z=VL=M%3WexuaL4}j;T?E%^WXOC;nq_)Vw4GCJ%D?w! zrAEFu3EC`PwQqm{*Ril0+E-zCP+#7w1mwp0UKqfv&hv2U&^i4?jKd5t7+5qzWn-KeX3u0o2mn^Ff+k$t=>nx7c@w>B@iFXZ19!@OUQE6l^ zk72R(d5nbqos>=Ov>)sk?^g2}&4J}mO7ZcIn(V|r`O%tfil2@;(~JFvCVZZ*=63`d zr(buh1~U~i)L?#bJ~4d>y&*5X;{I+Wm;X7VXA}L8PYWz%Z=33C&# z2zLNUgRW2+jmSNq0VQ>#SCjeng&4lToMD51aZ)zk8Yq zH;6Fy;a~{brfS?e_`TNE%X=hZW>LB}ST750u8RRC@w3hPj0dS1w|ee#%jRg6YvJGL z@p!t$c!|F+uOjlMU?)6Hv+3`o zl}fXJKTzHHz&g%%c0u#Vb$C1MaGm#1N)`iT{&xHFmkS9qA;yY(Cg*~RBQHvq$zG`y z$7gjWDVh2zDHv=BoGYYh?#!+Ag)4E|kj<)e)N$$wH-~n;GWwD7BW6Fr$~T?baPv%9 z33kwt)WCJFn(*f2xMwT8xRQ=h>jE1Q>zo~xfwe2MVZ*~w zbEuepDyTY7DY~&i)^JU-=i+^96hX32CHL1nKVX5)hH@1?ff_mJ|gE0qGQ^ zyGvjfk&u#-t|g=!mJq4u^8Nh_&#UM2yxW~QbLPycId|sXbCTzRKDCL-%tG)Ys*Y&C zpoL~CK|(tEMr`wUo+QesLV>zM}%`=NyCk)mY3sZyz23e(e)DIgBnjyx>1>{|yo2+W?U`2dc7%gaGV8xK zpJD#NOk7v|m`t~Jg6D|8Wi46l;nRu}f_;fVc=A#mV@>Ig_e#HDMl-wg)mB3DnIZGx zoi=s?9Oy7c|sTJy9Yg8UmumYyyXUNoyJs$TUA#GRSU#ZdYSE&chy zV2k)=vPT~$;O%44LeDSAFqmGiHT?3T=^#^&&&vgSAGXBd$hh8@tA28W#8lwXqDsBG zjSLo$#64l3PiL1j1e1P-&e%onzRsYkIU2p_l)6!jW~C$@a-794^mHqzc1Uh=;HNiq zWv`C?90E6t-kT!-urC-Zm)lQ6nj9~Bh-T}Kw5x5XBJylr4=4n)XS&>69Q6LSMg$K1 zp=HT;$lR&Ni#wbuP=RPs=5yuZkNeW~zen-5eOCzOe+y;ow}Q%>MGnV5Et=}Y0Vg7M(5NTpbZ_X$1If}6>4K+mbe!7U{LkGC zf;Y|HY|>8L%->PcO-d{TRq3jGE<0m}MJR(2SsTuJL5wfvcesgK7fS>r{}ER$vuu3s zhWFXy7Tz`aAG4peH{p*9=T^w-ST{{f+CSRKoR5iJJB0XF7404!@QsoXC8+(%uB?M{ zd^%9#K%BG^7 zc?Ys5D;>WiA}Z?zCci67crqy>dSF9S=a%_wnUt5W68zLgnxz)4Hit2Z8F!5(h2qYl zd5@oc-k=GxlO`QF=B=qBBC;`)!gQ*X#!e8CJ36hO%(Nf$`6=b7CH%6&c7d=qhFiul zNePnBodA);3}Si5IkUZRS@t5%zc?~7xgY>5U|DyRIrz$x&FIL3ckiWOkSC#GeC+NT zoq(J44&J9)=GK-z4mbf*!jNpD-bD)eZ!$&F*htoEPOQm=X~CJ;+B&hji%kwD^%cd8 zx6!-?PwA#&i^~%-)$ol^OVu<|l)2`A(if`OZPjLylVQTynV!t3=Pbv;%6s@R3gv%GKz) z<~33p!*BH*{;JPC5!GVmw6Jt8(#bir_)x%NB(;gLIwpGrrSB#(8I0Ky`sg3^!i)?k zZO!w0MX{;TC!z#$WbSQr)Um7^T{3(UI=M#(zld?ta`i+C{S%ia?PYivV#5uv&UFW{ zdkL5P!HYk8(%bUb7SScZH>g$?w#eZzUiZEuOEwH|dK$;>BJ5!`{AevCpL~Yo?L3X| zEc4P>IdCe0(ajmorfu&}$G;kXdcM_K0niZ{5)hP&0Zqx(GQ4l6z=N^_`X{5!Ttk9`R2v^$OB&&7g#n9JQz9_|6>@F( zeotC0YcJxNf|E;vNW-gt4loAIkf&2Tkcj7!6FS~d2w z>c!ep2yY*uo)7~A>C}|O?*OKk$ia+l7ZE+)mO#z$T*i$Vww#mr`?XTnI3tRU)FwgQ zaf82WePq}>r{{EeHAjGJ-M$xaxG~bF6IuOgL1Jj` zZnQXiz6b5w-v(ASVdXjK;AnHWd8HSpjW==^4`VuzJxnN)Mp_@#w(@e?ouK%4zUn*I zIB0HJMoqO>bulVtP0Q?_VbLCz)iJgAz3;(d%v(CWN&kBCB(^!AWKMEFV-cgANU|YI z0@V221;s*c>acLzpJsVSeTk21c#rEcRCyN6B=`2Nf8D=k)0CD!UiM43b4-?iYFxy( zt;^|7O6h;3_-6F@cfHx)lJKyM1KFN0pzLb-lf{3x6V6*O{Sd(wYyjW>?M<)tF+5;W zg~-2dI8MiCUq3D0w@B7WZH*#=hatu?`rxBGT(y=NEr#6RzDGx%_H@xIxsJP9u8cBK z-epWJ^n&4Q;jo4Q z%vMY^J?x`&EI2vO`J=))$Nk~*>5wJ1$_4GOELOq?!dLXq{8e9UI?J};M^2SD&ByAS zluiFTH|*i*F}=UPYUfh0dG=I*x50M!*SSRCi1h*sWbv}?ut*s*ZmHs$HKVKG@sBXF zzDJa?^4;Qt93lM4@3#%a63KGk_+J?!yJkv5y0jYl?${ZyGps*1XN6DMKAy)GHa5ug z>z!2JEtCH|BZwzeS&!YCu%QQ)SDxjrhZ$AQ!c8h#k0#V7XZjdzsrSKk)DB?SwYRc2 z3ah`ZTx`l$D|nx=$0FnZ4HNuL#0;j8471rQYR4eiymjq70I8bZu8y*30N}JE3GCJ0RAg;9Mc-B|zdz zlLZReXzh8N{ANPCo!XsdkXT9 zxpX(sRmAj@Td54Kaa~tLvL>JYRhUX#6X9AFtHE_i>A;#juy}F(b|QOw$e&!n7&fEM zFPxR&XHIb(!UW;M{$f2454Y7K%l~MfDw69*3gtPTQO@W(B3DEQcNY@Wh9%sgJhq`$ zaVGr~fylO!y<$A`>US%WbHyv-RPecIxjIP|_$mXg|84tv##S$bllYd>Has#=&afw2 zR(EYCQIUjf$HG~J=SRKOpa)jWYRg6d);wmh*fcX_a`UQ$@Er=&Wh*V9#?}klptPB(VzM>%q99? z%GiO}@COp*vvwaJTPCC!n7X(@+I1SJozE9YnOo=W4!;azb7V!aPW~n|rDAvdTH-QT zpuFi~DSJ7Vt#nKCz8RO{Qn#<;-ncU~kcVT3<{ZPEvu4_toGk)jaP5cu-ZT%BW~1~M zd6mPrgnW$CVjBf}AnYy2qpzE#?{cVha;j0Ez9X$__omNeyBo`62eJ!__Xa>H*0i;{JcZakO=jtah7`6JZz3jS~htmGP29Yuiby( z^(ZB@TEpKpP{xTv8F5JN>2t0V0XbD94yzOf-diRS>qMC8{~+XH{ad9Jm**>b))81U z{N_L&-g3Bl+TtHkVMFM4;0ur|8LRD7DlI@kcX`FA8wY8B(mHh(nZa3)3 zsnySCNB7{M6b;x*IDYoA8WEAwk7FNX{purunh!drSKN7jQhr}r+syv_S~4oM6$~pS^KW^|J8~rb3E7FaMajVP%|}6TH96v| zjp&bGdspb`hs8V5R& zt2X&x4U#HvHU+JM&ju%n{lGtwCpR>?=7Ae&}nj5U~fEqdVVogRG^()i z?C2>Q^OqbqLaCsZ8Qb%nHS|#913iuw3J;OVB_zJFKyI(BOB!8(ZjweK{AXMor zH@(d_Ad#?~4~Swp!FY^EN(WzVdN3_dP-j%#2QGwEuOQ@S5|t&OOsFZ;x#lBKv2YW*57}ZUEbo ze9EOfugav4pBb9)C)cd}1sP#F(-}jAb26jjISS*=-PL&m1+`2ZSMX^_f4hf7vLEe? zyZiUKSO1$Q9UV(tP3av5biP=!Ru`|e$MGxLg!jyLyC6fsyl8u${TJ|9ioP$-xw5;L z?`vUf{cG%3`O*1)_yweEnNUlm$Q;HjPv-2#pnpr3VSIGLgB`@(vqP7ep3CNchj@F} z&Vl+3S?1RELI1Sl(WZIM>0pnUp!{AFWsp@lhL6L{ z6lnFp8%QG`3%qJN!^qh6j0{Wqk8XtJoUi42&_C0xsV#{Jgla+>7+{}@E z|9*F2U}))y9;D-JQygt#8!9k=Dqn;)8ZrwNK^4;1U-K@Xu-NquEVlolf5-YV%CPdS zzDhme;yn!;ZfmU1HW!cUeLcCK$ej$Cyl3xMkYG97R8l8->Sxzi%BS(Wlp9w1{m*f*){F%djtpbK zq;u)(7YmL?lpKDZb^2M$^XLdg|CuidX_5Vj5lT&H)h6}ol^uUm9;c1@+C#UQ7vCcs zaTuH2;^GN>p!jj|Zu1mnq|I70S~f0l%X&Kslj7ZJS<9d$UcO5pY>U zcZ`@C=;(%VoV*xU8m1)=Tp8|HR}?G-(xJD{m=T>#LIgt~9~$c0>h5j5^_mEjIraZR zJcoSm_+LiBdOL5B%nTk1FF&sIzgAQ^eUDJd{rNQ|t}uGn(=bzcHgMZw_WVbi802=P zE#?`mzG!^InMf_om9Vng&h)^Y$!%?-U1hanz91S%>wgkqm9WgzF4w!iwl6 z*KM}(-braP)QIjZaj4KfC|zp!R~^3OTBgOLlf^u9mFS})GY099nDaOFi(y{_+Fa)w zf)~SAg6Fh^H^(65jAo{nIz7!5O?V)D&9WWdwn9JVJr6oeDb zQRL~+BK3%Ug?Lg*D4P9=TqNwUZRMydy}yD*kRczeMmWivJQh~O?b+1o&_9(tJLQe1 z9to)y<}D4d7xy01LrS$1Yd=NW(#9?%HlrC5S#5-!RSU9`Vdq_UQbONvQ~x`hc62oX zW7QUq4-a&n7p4UMZs*we5WlflBRVUmIdg_cJ8m)*;@ke_DsCXNS%6SAekHO7n((x< zdJ3<8Tu5Fkhn~D>c?Gy+SM(qrC!x!rz*QdskdfnAuxL$an)Tvw-DJCTm2gPthTt!p zMM#;cW#@3QSk^f;u07uwzPQIti?-t?+Ght^D^k?y(!zSfVv^qknRy~_UB@q5H^WD_ z+qqOo9}t??C4v}vikc)c`hu&Ph2I#7Fp1*Lcz6~uRSS(C)((w19r3vtH;D4M`ao7W z+0kqCR8>zZO8Tq4fUw zi*ky?p6lFiBV*Jw*W1^+$U&B~gEYoS(9G>$poEPiu6~awlykc!O65|hHZv5~V0-n( zh#$4)ap&;wJv5jDsUK1?3a4^Ma$+${Egoa!Q{i8%u{SdCy0;;la~^qo64BThYR+ex9l z*S^EdNQ>?fa%CK2r+sz`C4x#k^E;KyZOF)d2U=U!pHGz|Z;lo%%NX)g{8Se-^)Me4yfd8 zCoHnNxp;|HJfg$%5l+{!hK~L077>~!0tkQLtI3u-QWoO}eOBbY%#M^m;7sVuOyEe+ zqOBA$Dje6_uv|G*$TSYNmW6mt%a>zBnh&!zCqv`+e8r}o&iei9IK|0Jw#=xsJQ2cI z|HGy##*R>dJnRnJHG+zG-M*K{7ulmIfvv+FaU7Z&4sFl2?TsToh9e|oSv$znC$#&< ztsq+1XdZ{l$)r0*bjuXaQrRu)O-*t5A|O*-U8mYe1P-T`|LpB3*U^Ju5?D+bNs$!GXH9dU_>ExMobAfy;Lb@aO=cVi=rK5c*&; z<2OUO8T+XE=Y8uuiAR?$d z*hE)fDWH1ISVKpdxZevJ7as_-R>00LUTtBQ+^ZA6^!;iU-kZE7dR*0`;diU|V^cQ9 zyv`F|1ADuy*lH=l^Ux{Mz<16Sl_RTLiN1f z1%a7y+OtFXs#fJGO#>Xs7x`V4>4#t$9d6Y6`Q`_MPLdS4c*E!{wG`|7G2lI(%A zdKWMX?UK-7OJqcxsX30 zlYr8!_9uNUq`kxO>l#P8F=PYt(CAp2VU?tn;VVXl{HQ0FkV4 z-mwf{1JD~$o(X@#5_`{DQO1=z*qvi3Ea)^6tEgyF^NtM1aj{(&Qd&Ega$T z;y>*hwm>0k7t%~ra0{5oo*F4&C*CjTZA|_nI=1wZ3*1-!3qut9h!oK^f=;}Mgh>Td zxmKR+1Y42vkV+(=5}dQ!C^?3w7+RA4?!9mUP_9Q3BVRmrh%}B4hh$IN#qiJU{;J`q z-A})Nv-!o*zqHZll{d1Uh|~ndMdr#gOZWzfD7=N?D4r;vOinS4^Gu6e+mWVbO4e4U zc5hRM(!*aqW6)N?2aT?sfEIU~Ebz6%YgrW6%B@+1MQapAVj7ZS6lyPY6>d09)rtmwWbejQ{Q#K5QyCtLik#a^vxG?e#-(}OR6&)2; zq@cLo{wSH%jZBDsH>Wzw$2rIM*)}WTYSy+xR=snVwosUz!6Elw+t1&hzCGlpiOro} zfPDO6knolDl`oE8Z;I_q4>rum8co1$tWw&_FyX-;=2Q<(!=mUHQ&{<$BDEsv8*@dlFF*7QO28|^&z_W5QeY!3!E1I&?BhS2Z3L`*vqFw~9I z-b^C3xXNjJ$UD0O7j9*`5fSROWZRC$zuDuz9Hh*fqgZ9Wcc1|EtCZd_Bd0UA%~Sgt zTB&L{IRQ*?CxY3&{;8{Ueuk=j-s9oGSifw;B^%h{z-H$mOMS0e3fXI&`s`IJyVd@3 zO`Ou!*`)xgG|N}+1MCqzNV?{~G@Am*L!FnT3FLqUBwDb588VCm?b#UkF_LxtXX(rP z8iPVg!S&cQ(pyTt)K z(iS@D5Z{{g6e+=ByEhf%pZ$~S`0~>S zsQ}>lYsXo<*s*O5d%i!f?BjlX!P9%m9a8Z0`i(o+8Gct_OcP0~Q>)h7MJld713fm{ zgxT$O)(^=!rw(E$7%&_*^l#QFDSn%`6}?;Pcl<{&Y>xgYKq7;E-cJS{otjs$;_V)8 zA_13DVfkxZD+TxJr!z|FPIamIyT%lHaBV=*0s z1m&dU$E)jSEc7jHK5`}&j|PB59pRd~7fxP31#?#yqeRSO8^)V@_h73ZZ1EYmF)Q4A z9KT_U$i$>L%Bmu#@HC~n^h5&HvJ<}0mqC=W9J8q#T%ZB|s^ zg`p6_&^k_e%TNBh2&Hp4^KJu1EAek(g=<080AhE48{|Kv&bj`ci)r2dKkFjnro^N8 z2%T`tV#8gVMs?#D0*Cf0zkS_ZDZwFYSj{27zi z_{|_)=-surTe*f#@~i@7PO%mLP>-pc=Jl^&+HbIaUgYN5%gc*o%XLtcoF#-~L4A4d znM*%DTQYYC&+Jn!rPqaSNjcn($uHsZAEoq8z0z!Gs+S^HQemeKt2s*HL)eHf2Syh` zVtJ4qF78*1ni;;imSK8-+nxaQE=8TcG7%rChHYDXD<%Xb4o1!*kN782ruNFDGR5Nm zW^YL$U0>;6yJye+FlZ%pJ}NF=u__Z$D1Pl)LH8JGoH2kGQ3@@mbg18?7!TTL zi=5*ts}XXW8jPo#u%c~MYj<2&vkzx%!qzuFt0k#z+Z zJulRaHvBuNG&`(Trpz}XxhTqy5WXa;Xepe#4ZOX=nyCN)ZT({+sLuE*l9J4@{WU(m&>?)Yd67h(`(0V)8Is$nHo6yv!}0i8k**qgBW_VLM-k)kd6<}_ zyn_~5$M`R!m_EwkzzP%R+%=L85j5b1G2ho~ZbB$A?b+{F$(f9M-^=>fBBr#01>AgX zV*Jze3FSpkF%5>t3Ie^mlVp;^344WjwKI-8#L#kI1k9}+^JA(E3M-RrRigm1Yid~V z9eilbyq22JzFWp%eMRZhO^uKG?djwzh^NizOXA0zXTkRFe=Y9CkM`K~LGSPl>-WAZ zRA`uNNj&=Zct?Hkub5znA_Rzl(;f&QM39_kId#%5%uX`XL0#-T0FcY?(9H8O(o>rmcxht)*AgjFqRTKy5kwS@8;GQ9%2Z zRKjgc+5haQ+jK6Q{uFgl5C84=j5+SDjm`Mf??xd~y$`bp5afxVOSO4ZG93*IE!BTB zFISzfh*ZyNv`uHpy07*1m2&WMqUifUQJ`ui`J)Hz$&(R`WAxr~(x+8RuPah83&>2A z(1+}+k3u>ws|~{`36m-9xbDBl(?|IEFYmRP>?RjO9v?0FMc-|{j5W0)q_eLv zkj9$O%M;nFHpczZ)dk_bk+LF%4Lst`d47fg3^I&^#@y&7JvD2?T+fPXn3C~B$&}2pAYs%u#u;C@sfiFpy+6O*vm|n)w zIuZK6%USb0^}wr=itg|*e2>8KG#S9wctwKJxe!(I82+w0I~ecz>7j(}b!GEZ;0tFz z42^Yk3=ytZ=MlDYtGG~KYv=cRM$)!(P8gxJpGv$(BcxYK4V`19&#H5VA_SDodWX$V zS7)26o9C$giTQwd=85cGJPkdU@7-)J)waLty#^oxz!`b)whIx|+(jckPT9=72s+L8 zMLu{epimeY;^iX5z%#aNs4b08agJdj_hxbX)^#`CcgCiM68QJ*EVH~+rKIA@U;&oE zb*mRL`7q8#Y3ZDpl!52C(m~6HG_idCP`c{T#>AHlE5ssYz%W3FuETJKK?SW5M%J92 zPuFXf6gCeBQcSz}V=W1sj{F29CzhRE@3qdjzB#sIb#x92IsR&2t)IaKMj6gB9nIri zzi*mbolmxn+Kx9|(U!V7ns`>e)=GO$jApTNXKO~4lJV;`eqj4md%}HV|CdO(EkFU} z&FJQy_l!>6#JDUQ{vBxaI%1xj+Dx@VXW-$jb#QND#ceH~KfE(~+$H1d=tOn)ynQS= z{*n0{u%fgrq^ngxab8Vlbk(Xm+5Ck7Q9REchZhO!B>bs!>i)CTw@3JF<=>yd?I76n z``Axx9+;;Io7FS`EQ3b;2bW2Ds8j@(1$t~{ za~8b)L-i-godROr5EnSMeewaZa(LGKrF!dQV^C3v+4UHZEkRpO=q*Rqoj2_5+Ofq+y%+vF7 z=C5g9mP~Am}TbCU-X9289wG z?NmLN$USYI>Y_Re)-f68qXWm~osbkDzHV@z&p>z+E`NU--jq)|Wd|a`Er)Q^Y1ha5 z+nih19v)14pl-O{ZjAca36deh%5sXz1HO(hN!^HU08UBdZ^kHHWvUmv$y$RNxMc^&Go=2&>ky*kE=jP`7vhS zZfq&w0=$5HO=~^rRhyKIl;lnQZQo3HH`24o=r-|EPV0@XNX>%4CetaZoGI7u4Q&j@ zdZyTK@GEnv`9a0yuBe;mS8W)fSWzsXnHpB^6d5zcaFbjSKu3{3t{jHb)B1593GGVp z{z)l)JP1Ae^Jma0=;qC=V(`p{JP|+yJ9f;2~&iC#k@Ax zOB@E#v$rdD$IoRrYLKH%bb0~_EndtrY#mi778D;K1zZN{Cl{rz9cBcTor9@-JPG6! zu>=lRVto0u1CR8i+7?XwW8o473~tq&<^raysG+Nb&|zV7M|&*q36&%?1Cc$+?O|Ik z3Fe_E{_lw5pkgK@`rA`A@jS~;cTC{0JT{rKC$HBaYD!|4;mNc*-fe+{aMDH zOQmz(QgPEEFjl@td)JTiDR5FfO)s?hY_fM`@YtuCDTyk}*DL}CSWwBkjcvxHwB0m0 z770~6J@GM|c)u_VjFRj$Sn)FO5KfoQ#0EC=EBu(!AzEwrYJJGWnKV;{c8) zwE2MG<*ZJ*aB7N}cWg4=LDr8W`^iP8-&aiC*jGVz6#jvhvoONuk{WA5$L&S77OehxDC zsqgN(@{N~AP1B}WM;S^d)?@Dq(a!!LeCD&9`IBWTaCu*&k^{de-QJcGk&Qgp++VtG z{5@16rMZ$ak{=|}YGC5|57b)=40#@Tt;h7BlOK#0P*Q}@zdmtjqHNZ5ioi2~d=Cj+ zzM6%P!H~CoZmRrF{aGCCr$Q2BXh3?{SPZZAUe1_A*FW|tyx?CZ&reZNz})m{)K~9o znfqx=Kkmht-y^QQ-K*#Z$L~hFm32C_llXv8xhHLQD#5b4Qh0EYtgD#?SR(1Io)VG& zyXLV`vNFm8%7SNP<zis%Q(SO@Ql?=5KcSYl_ptoPz7^^B*76()&0D1?zwB%@F?4S03L zA7!bNRI|1Upp#kWUw0zOyHc<=r$JWDyi@afT9f=bx+GZH#e?y#aFVdKN*(XegJVtR ztg85i>2E3qpg;s3DQ{1`xhulwy3!j4mHu_?@PK)e6P&TmYiyD}6LXyIm$^wF>Gya=AVj^8BWDI%NKL%O>%|mh0BIQYoN)R7&Dgk#_27v?TPEztZ=aL0A%~ z?B7lr+YTgTJ9!K>XFjUKN_JoKiJWpbgByzpMV3IL(kw4d_>#xfzFaHJg!fNh7DvU$ z=T5_bg)~T^d#Dyi^W`(6aN&;xkqcit%+TJ8#aKbI1xM_ z#?ox|Tcmn^O!eL6b5*|;GseoWMo|wOacNgE2WGQ+hXzc?u#~wem!_%3^O>-cNrzD@teMQb(v0;KD%+fRqd>H3m6J_j&*6m zF>xpjnGBduP=P=T%tg z^ZW36^)+-r(n^vm-iq!Vb(Eu-)fwZLQ0j|@HRrajsDG-T{ZN3APfY%kIIvFX1HI5W zklE*}bYp8Y4y{YY|5QKq0>C%&+zl-r-slpTgEDrg(b@#txo3TXoFbRrlxAiR4zSJJ zT>!BVjlr9*061P2O=>13!L^e2mQ9Mz1gEfpP;Q+yZ64kamN6IXon%R#ma|UD@>6;; zWo#XBD8PFmBO$*@oWB+QPRtmcuZbT=rNJZo4oPb^S@i+nZw$+Zo)_TvYsR4lmGcjq z`^B`VXM+f~@T=$LnJ55H%4dV{TidS-9_n-sIj5)*0o5aFu1%kRWEqRvm)D%(_UZwl z9n$?3(no+_`SY6RZhZjG%VAX#W2Kam=0V2ucQuwkrzi7o8ZOib8LdJ^+)&b=#L%Bi z5+ig8vkudX;L}y;RxHy-(-B~A;(C%yUOS&lY6s0OuqoSBGyQ`Od{UPD5iBsaVE536 zn9S~(K=d1m+ea6H^?F%@2(O1jzwh&D`oHsiBbhKQkcp{I`N;LnrS@n>Q5hw`TX-S} zLge;h82G=tWROmIQrA+l{RlBBs6GsYuJ$JFi=aW6{EPetaSWXpo>{t;!rrAJ%J1ua zMJl2yO(;SOUJ%J)vtSDme#o3-Zl@;7j5wpm6MsSf=t*SKqbI!Yz3-6?OF^R98fp^E zce*=BQ^nQ2_KqfCHnsW6)^D_9v!ev|w%@U;WA<9v0o84p4+2Q%8Hp}S(sb6bFU~BU zCzIow6(q&LB`yx6-ifragZ(91qRThlK%c`CbYqsz)e5126T6xgdzR3wm^tWnbD48N*P!4wb9eKQ63;`JQ~$^GfPZ*dFw>qwwf|- zz(E9Ib+K2&6WDJqbM!~5zPN@5pq|+7GZ`&Fp-EZQAbFt~`bXR8d`${ctsz!(uyDrK z^-=(6H`r5tHvq)FSk*!-9+HSEpZmxlHXuva!FDok=8V4t(P~+$-j=8R5spHYQbMDt zC`|g;=Ii5C161?7s7$0PWUDljPo5lN~%;p%+ zfFRSCt*y2zRmKg}O3`Uz)SldQ_Q6rUyG_5Qf!dE(o2`2KG_X~(*%{nG@r$2BRg2g6 zBf+LJ;hi|L8G`?`R@A13-cU;|%;4TUyBLH&9cV}80g80?6yNpLpotoGSFnCIp~8km z!JXC-1;NYLGoyPYgnlTjWwwWoL@FD5RwUb*P8f)v2uZ}zd9e7M$A^yFwxB2>YY-!B;|Av*>Cx(l z7n*?dbJc*8Hx1W2qlrA z;~309U{%%3jXE}-hPiXEe^#&}rO`hper>GYxt3P^N4_Yf$xM1j$V(L6U&q|h>(xrA zSu^(D|LPog8k7JCNoUCQ+n;8Wtug5m0Z|%zz9yTtKxvLGozkH$Tz9C~bO2{x_dsK{ z-M`^JliOS7`ar0e!zq`_4&!V7u92vT;?x%zR{wWFrMb-wqHp1^Cf<1h< zrO~Qj*R6x#f)U+ER;Mgmj1~IHb}f6$&J1=6DGvX4@QU1GBN=?l3l)jhh(Kd~Kkg~|`VmdJ@o9@~ z+t@(xUd`^?kiBA@s^_Zv<`|(e;s+{uSG~Z+;SKkGH7Wo|S5L=T66BoM+jXmw?1?p_ z002yfF~(8^Y+AfKB;KeZ27pk_qx#H%=^?}?x&p%)q!0jDa5cU1n^w=rmQk$hsIvtd zTvU0YrJ&P8BX~7kst~ZD8*QZeTpK&7^}lW;+DNKVT?qN;PjF`p<$6(H`z!jEN{Y$} z0|1%}GgRuXHh^Z=GeUK*PeN$m$*8`%Rg@h^_FO$O%P`-rk`e%BBcWvKk|wLa$ne}O zqhS6Z@wLxXWm5eA0BFc3yhF}*Tz+nsZbb2r`ndt zvj;pIdg@5SU87x_@No^!G@(}j0O+&I?S9b&h0G6)+^AoF1OQ)2^h+S>*=%o5I3LE4 z*72YJjP=&P8G8E=lmUA6;^)$SvqSgU%fFaFnS8a>FF|an69gDhEQLSNq&voN4xJrh zCy!qhCjyDk8xF2~yY=QsOCtAw1AadYz+oKebMzwqk4*nz2!*?CRoyR*xkrN~NdHT% zc(VJD$8S(M?7y0ulV8=hOf>s{IC`MMXIE3Z8*$hc=co6eJTAh)QCWlVA%xHTCecGA z7TQeB>~2#1v(>wr5dubCfGi8HtJGYj2%H`oww&5?TYh^fghZf9`hf9_R=_IO-YKKdm%h|;q@4d zjR4t}*u>IAi36AH^SV#i>KYGCm}eypy5B?(z@nW*ZYhd70st^ZTA&mO=c=10CxU_+ z2He^4su!sFn|?nnfFDHjJ}mz4f*+BkTauYmW-Pc)K)Pd}G=O@S%J08_u2f7JvkRgX zw_*$=56jlw(M%;`ugd&mY^=wXMGyd@JcGXXH|Dwc*K#>G{gJ`Z@Y6%%jqt1dxWE?Z zhEa1LzoTJ!6}ZyB3L&RYyh$+o;F`zH5@z`kldnJuJKHfP1b10!FN15&?`XWOqrQNa z9vY9g6XhVp!ZYte|*1R~^-ObqfG5A$k9!QG|%;rGr> z5-HD)LHYts_J{ME1CNl1qk;CiY-~_s!1COBp!dLVz|+xowFmxnEc@js&wAcH*}Fb< zC5>PRcb<5~3^4r?E^8v;4+Y`xHOw*B%W-5bK`dJE)9!4@9 zvG08cJdWj<~?xf(jbvN zHXTylX>yETd3WF8E*11|i()})Yhz!+D-oI2V4 ziib39bL^*%is8Gr+zj#vbo<=r2)pl;fcW-ij_2)myHkG`a#MVc3+xNOpKZ6A5)PvW zx>JdU9`}6Rd}M<4XeNX3k>l@gdPxkiF%^1+bC*Y++HF0|K3?@+zRQz<$A_mAU;r>f5bB>QfNFsDa&MM|C0iXwCQD%hO!5 z?D{vIx=eG^eI$S(i$n!#vUv7fcE(S1zkj^l+H@mt8!h?mq8^_~Go68gArAt$Wfd8U z6onj=g^Z%W_Ch?bvRgkkyTliT_&4z)&?yLqae$j{E7DfBnMofAKFjF!at`86i_Eo; z=+(GS+(xGG`R_fm(7bUN&>7I}YvNjRYvjsWv(DYg>i_{(7%N83i(jfGb@K~!hCVdB z-;C+kp!{yC;!%!mu_DL=9Lv}`v{JcTxlU_Q9-@8<+e8Pr*_M^oT`1#CRkYgy#(aO2 z(3s(hQcUTyrFFwuYYd8~AWdgXdUaHblOi=%33N{Z3YLeaeLk-Ij_ZqIF@~#9EQ@^* z`P-(8Du&Z92a;|cwCYtjyp);bz_(^2wIRa|>bPJ}h>C8t4$8}Ta8Bv*o8<3xBmO$o zl>=fm4CU+mq@BqR1nY0I6>U$pPME?P7o%RKSaSmpI24dx`n~3-;&-(#Y2V?5f(QVM z=-@g}%(BOiXD)!t}!~HpmccWTxi~l=mR(pFi*f6 zQEqBr=)nL?XHG~k;Q$D}@@Mg<+>I;1sRB;ndGM$AlgNL~N$%5(FV4^EP%=-_PwP?#vsC^E!Y!@x&msB>&`#& zKA<)(iyScYK>i!Wsr~%FpB)&X-isg^aaor%z=b=ATf!A_3Dm|vbh{&S;!!M~itqrn ztD91}Xr1z|5a4SFG8mtQr}Gu~Th;}5Z4(XvO(?P{q^Q6o2B4>p1-zyPbd8?<|IYts znGiTvV>N;Cl^!eK&gv5yV>0Vo=GXq;?-%h{NyAz%GiQxT>*ayadEo0HHVWW~_J7u( zrOm>yf|p^{8Ucm{W()%@xxVz?qdG~355PL&$#dK1W(*I2ss8Nw>UGQqs_gU`1pe-M zUd(Xg+;dijRr9}>!&u?x84{Ek8eToOWj;^_R0h-(@D!-==K0^Y3_8z&ETCX0k^~wT zsLm9k?&f)Gh8&QFtLLlb8Mb`}y5rUJy}S&!f9_#$`1<)D4}|pvxJZ_vCXPYjSLJ^@ zwuY}j=7JxU^4A#|=8J=}Sjx!GV*Q`*q&C!ldn>WEQG=`LLA(J|qu}g_1qlZiJl^~J zd7n%q`-g3-@-7-M11s7Gciw%zmC6VzP0zi3JR|%YE5m|HMvL8Vv;S4Zi8BCIC?xED z*Z=-!>7D#C;P5J_ss$Cez$Q4gtCGtAc~eKBCri_9*6Ca@RCaG4$c1=!%3 zkb7^ZE|8mH_P!X{@^RQ)`Fsu~|LgmZxYUWP-#tA3NTGDFmT;cb_8ZfhJXt~oZv=O z)3OsERs#nx8mihVdNu;>JYYS2b~~^G%CKJMOCB$fUiY=1`S|n&xV}t_t>H-Il?-gGBe_huC9JCqqpg>IO&fmD~FS_9%r+T{j KxvX 0).astype(jnp.float32)\n", + "X_test = (X_test > 0).astype(jnp.float32)\n", + "Y_train = np.array(dataset['train']['label'], dtype=np.int32)\n", + "Y_test = np.array(dataset['test']['label'], dtype=np.int32)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8b886cbc", + "metadata": {}, + "outputs": [], + "source": [ + "class Dataset:\n", + " def __init__(self, X, Y, batch_size, shuffle=True):\n", + " self.X = X\n", + " self.Y = Y\n", + " self.batch_size = batch_size\n", + " self.shuffle = shuffle\n", + " self.indices = np.arange(len(X))\n", + " self.current_index = 0\n", + " if self.shuffle:\n", + " np.random.shuffle(self.indices)\n", + "\n", + " def __iter__(self):\n", + " self.current_index = 0\n", + " if self.shuffle:\n", + " np.random.shuffle(self.indices)\n", + " return self\n", + "\n", + " def __next__(self):\n", + " # Check if all samples have been processed\n", + " if self.current_index >= len(self.X):\n", + " raise StopIteration\n", + "\n", + " # Define the start and end of the current batch\n", + " start = self.current_index\n", + " end = start + self.batch_size\n", + " if end > len(self.X):\n", + " end = len(self.X)\n", + " \n", + " # Update current index\n", + " self.current_index = end\n", + "\n", + " # Select batch samples\n", + " batch_indices = self.indices[start:end]\n", + " batch_X = self.X[batch_indices]\n", + " batch_Y = self.Y[batch_indices]\n", + "\n", + " # Ensure batch has consistent shape\n", + " if batch_X.ndim == 1:\n", + " batch_X = np.expand_dims(batch_X, axis=0)\n", + "\n", + " return batch_X, batch_Y" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6c383f1", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize training and testing datasets\n", + "batch_size = 32\n", + "train_dataset = Dataset(X_train, Y_train, batch_size, shuffle=True)\n", + "test_dataset = Dataset(X_test, Y_test, batch_size, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "8ba1b50a", + "metadata": {}, + "source": [ + "## Defining the Artificial Neural Network\n", + "\n", + "When defining an artificial neural network in brainstate, you need to inherit the base class ``brainstate.nn.Module``. In the class method ``__init__()``, define the layers in the network (make sure to initialize the base class first using ``super().__init__()``). In the class method ``__call__()``, define the forward pass method of the network.\n", + "\n", + "brainstate also supports defining operations for individual layers in the network. For these custom layers, you need to inherit from the base class ``brainstate.nn.Module``, similar to defining a network.\n", + "\n", + "All quantities that need to change in the model should be encapsulated in the ``State`` object. Parameters that need to be updated during training should be encapsulated in a subclass of ``State`` called ``ParamState``. Other quantities that need to be updated during training are encapsulated in another subclass of ``State`` called ``ShortTermState``." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cea8b5d5", + "metadata": {}, + "outputs": [], + "source": [ + "# Define linear layer\n", + "class Linear(bst.nn.Module):\n", + " def __init__(self, din: int, dout: int):\n", + " super().__init__()\n", + " self.w = bst.ParamState(bst.random.rand(din, dout)) # Initialize weight parameters\n", + " self.b = bst.ParamState(jnp.zeros((dout,))) # Initialize bias parameters\n", + "\n", + " def __call__(self, x):\n", + " return x @ self.w.value + self.b.value # Perform linear transformation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e541917", + "metadata": {}, + "outputs": [], + "source": [ + "# Define a short-term state for counting times called\n", + "class Count(bst.ShortTermState):\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f9445bd8", + "metadata": {}, + "outputs": [], + "source": [ + "# Define MLP model\n", + "class MLP(bst.nn.Module):\n", + " def __init__(self, din, dhidden, dout):\n", + " super().__init__()\n", + " self.count = Count(jnp.array(0)) # Count how many times model is called\n", + " self.linear1 = Linear(din, dhidden) # brainstate有常规层的实现,可以直接写 self.linear1 = bst.nn.Linear(din, dhidden)\n", + " self.linear2 = Linear(dhidden, dout)\n", + " self.flatten = bst.nn.Flatten(start_axis=1) # Flatten images to 1D\n", + " self.relu = bst.nn.ReLU() # ReLU activation function\n", + "\n", + " def __call__(self, x):\n", + " self.count.value += 1 # Increment call count\n", + "\n", + " x = self.flatten(x)\n", + " x = self.linear1(x)\n", + " x = self.relu(x) # 也兼容jax函数,可以直接写 x = jax.nn.relu(x)\n", + " x = self.linear2(x)\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97aefeaa", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize model with input, hidden, and output layer sizes\n", + "model = MLP(din=28*28, dhidden=512, dout=10)" + ] + }, + { + "cell_type": "markdown", + "id": "d7edf945", + "metadata": {}, + "source": [ + "## Optimizer Setup\n", + "\n", + "``brainstate.optim`` provides various optimizers to choose from.\n", + "\n", + "After instantiating the optimizer, you need to specify which parameters the optimizer should update by calling ``optimizer.register_trainable_weights()``.\n", + "\n", + "In this case, we use ``brainstate.nn.Module.states()`` to collect all the ``State`` objects of the network nodes and their sub-nodes in the model. We restrict the types of ``State`` collected to ``brainstate.ParamState`` (in this model, ``State`` instances may also have other types like ``Count``, which do not need to be updated by the optimizer, so we apply type restrictions)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8dd079f", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize optimizer and register model parameters\n", + "optimizer = bst.optim.SGD(lr = 1e-3) # Initialize SGD optimizer with learning rate\n", + "optimizer.register_trainable_weights(model.states(bst.ParamState)) # Register parameters for optimization" + ] + }, + { + "cell_type": "markdown", + "id": "0bbbfad2", + "metadata": {}, + "source": [ + "## Model Training\n", + "\n", + "During model training, use the ``brainstate.augment.grad`` function to calculate gradients. This function requires the loss function and the parameters (``State``) for which gradients should be computed.\n", + "\n", + "Then, the gradients are passed to the previously defined optimizer via ``update()`` for the update.\n", + "\n", + "To improve computational efficiency and performance, use the ``brainstate.compile.jit`` function to decorate the training step function, enabling just-in-time compilation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99272b18", + "metadata": {}, + "outputs": [], + "source": [ + "# Training step function\n", + "@bst.compile.jit\n", + "def train_step(batch):\n", + " x, y = batch\n", + " # Define loss function\n", + " def loss_fn():\n", + " return softmax_cross_entropy_with_integer_labels(model(x), y).mean()\n", + " \n", + " # Compute gradients of the loss with respect to model parameters\n", + " grads = bst.augment.grad(loss_fn, model.states(bst.ParamState))()\n", + " optimizer.update(grads) # Update parameters using optimizer" + ] + }, + { + "cell_type": "markdown", + "id": "e8c2dfe0", + "metadata": {}, + "source": [ + "## Model Testing\n", + "\n", + "Similarly, use the ``brainstate.compile.jit`` function to decorate the testing step function, allowing for just-in-time compilation to improve computational efficiency and performance." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7456afa1", + "metadata": {}, + "outputs": [], + "source": [ + "# Testing step function\n", + "@bst.compile.jit\n", + "def test_step(batch):\n", + " x, y = batch\n", + " y_pred = model(x) # Perform forward pass\n", + " loss = softmax_cross_entropy_with_integer_labels(y_pred, y).mean() # Compute loss\n", + " correct = (y_pred.argmax(1) == y).sum() # Count correct predictions\n", + "\n", + " return {'loss': loss, 'correct': correct}" + ] + }, + { + "cell_type": "markdown", + "id": "a550e569", + "metadata": {}, + "source": [ + "## Training Process\n", + "\n", + "This completes the setup and the process for training an artificial neural network with brainstate." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eae5f682", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 0, test loss: 410.2366638183594, test accuracy: 0.24890001118183136\n", + "epoch: 1, test loss: 278.79864501953125, test accuracy: 0.6233000159263611\n", + "epoch: 2, test loss: 75.72823333740234, test accuracy: 0.7638000249862671\n", + "epoch: 3, test loss: 59.49712371826172, test accuracy: 0.7830000519752502\n", + "epoch: 4, test loss: 38.07597351074219, test accuracy: 0.8623000383377075\n", + "epoch: 5, test loss: 54.225074768066406, test accuracy: 0.8329000473022461\n", + "epoch: 6, test loss: 74.46405792236328, test accuracy: 0.7676000595092773\n", + "epoch: 7, test loss: 35.6864128112793, test accuracy: 0.867900013923645\n", + "epoch: 8, test loss: 140.0616912841797, test accuracy: 0.7529000639915466\n", + "epoch: 9, test loss: 42.05353927612305, test accuracy: 0.8574000597000122\n", + "times called: 21880\n" + ] + } + ], + "source": [ + "# Execute training and testing\n", + "total_steps = 20\n", + "for epoch in range(10):\n", + " for step, batch in enumerate(train_dataset):\n", + " train_step(batch) # Perform training step for each batch\n", + "\n", + " # Calculate test loss and accuracy\n", + " test_loss, correct = 0, 0\n", + " for step_, test_ in enumerate(test_dataset):\n", + " logs = test_step(test_)\n", + " test_loss += logs['loss']\n", + " correct += logs['correct']\n", + " test_loss += logs['loss']\n", + " test_loss = test_loss / (step_ + 1)\n", + " test_accuracy = correct / len(X_test)\n", + " print(f\"epoch: {epoch}, test loss: {test_loss}, test accuracy: {test_accuracy}\")\n", + "\n", + "print('times model called:', model.count.value) # Output number of model calls" + ] } ], "metadata": { @@ -20,14 +380,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.11.10" } }, "nbformat": 4, diff --git a/docs/quickstart/snn_simulation-en.ipynb b/docs/quickstart/snn_simulation-en.ipynb index 2aa3e20..e8e2687 100644 --- a/docs/quickstart/snn_simulation-en.ipynb +++ b/docs/quickstart/snn_simulation-en.ipynb @@ -2,13 +2,658 @@ "cells": [ { "cell_type": "markdown", - "source": [ - "# Simulating Spiking Neural Networks" - ], + "id": "b78f6e1b6cb0dada", "metadata": { "collapsed": false }, - "id": "b78f6e1b6cb0dada" + "source": [ + "# Simulating Spiking Neural Networks\n", + "\n", + "Building and simulating brain dynamics models is one of the important methods for studying brain dynamics. In spiking neural network simulations, we specify the model and input parameters, and conduct simulation experiments. During this process, parameter learning and updates (such as synaptic weights) are not involved. The main purpose is to simulate and analyze the designed network.\n", + "\n", + "The spiking neural network models of brain dynamics can be divided into **single neuron models** and **neural network models**. We will demonstrate an example for each of these." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "d16fec85", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ['XLA_PYTHON_CLIENT_PREALLOCATE'] = 'false'" + ] + }, + { + "cell_type": "markdown", + "id": "c5622779", + "metadata": {}, + "source": [ + "## Simulation of a Single Neuron Model\n", + "\n", + "The **Hodgkin-Huxley (HH) model** is a mathematical model proposed in 1952 by neurophysiologists Allen Hodgkin (1914-1998) and Andrew Huxley (1917-2012) to describe the generation and propagation of action potentials in neurons. The HH model is based on the classical electrical circuit model and links the dynamic changes of the neuron membrane potential with the biophysical properties of the membrane ion channels. It is one of the most important theoretical models in neuroscience and earned the two researchers the Nobel Prize in Physiology or Medicine in 1963. The mathematical definition of the HH model is:\n", + "\n", + "\\begin{aligned}C \\frac {dV} {dt} = -(\\bar{g}_{Na} m^3 h (V &-E_{Na})\n", + "+ \\bar{g}_K n^4 (V-E_K) + g_{leak} (V - E_{leak})) + I(t)\\\\\\frac {dx} {dt} &= \\alpha_x (1-x) - \\beta_x, \\quad x\\in {\\rm{\\{m, h, n\\}}}\\\\&\\alpha_m(V) = \\frac {0.1(V+40)}{1-\\exp(\\frac{-(V + 40)} {10})}\\\\&\\beta_m(V) = 4.0 \\exp(\\frac{-(V + 65)} {18})\\\\&\\alpha_h(V) = 0.07 \\exp(\\frac{-(V+65)}{20})\\\\&\\beta_h(V) = \\frac 1 {1 + \\exp(\\frac{-(V + 35)} {10})}\\\\&\\alpha_n(V) = \\frac {0.01(V+55)}{1-\\exp(-(V+55)/10)}\\\\&\\beta_n(V) = 0.125 \\exp(\\frac{-(V + 65)} {80})\\end{aligned}\n", + "\n", + "In this tutorial, we simulate the HH model as an example of a single neuron model.``brainstate`` can run multiple neuron models in parallel, which saves time. We will simulate a group of HH neurons." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9dd07dd9", + "metadata": {}, + "outputs": [], + "source": [ + "import brainunit as u\n", + "import jax.numpy as jnp\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import brainstate as bst" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "051f8f24", + "metadata": {}, + "outputs": [], + "source": [ + "# bst.environ.set(platform='gpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "60058528", + "metadata": {}, + "outputs": [], + "source": [ + "bst.random.seed(100)" + ] + }, + { + "cell_type": "markdown", + "id": "a6ba5685", + "metadata": {}, + "source": [ + "## Defining the Single Neuron Model\n", + "\n", + "We can use ``brainstate`` to define custom neuron models. To define a custom neuron model, we need to inherit the base class ``brainstate.nn.Dynamics``.\n", + "\n", + "1. First, define the initialization method ``__init__()``. This method receives the number of neurons running in parallel, ``in_size``, and other model parameters. The base class is initialized with ``in_size``, and the model parameters are set as class attributes for easy access.\n", + "\n", + "2. Then, we can define some common calculations as class methods for later use. Here, we implement functions related to the calculations of m, h, and n. Note that for the drift term function of an ordinary differential equation, the order of the incoming parameters should be, the dynamic variable, the current moment t and the other parameters.\n", + "\n", + "3. Next, define the state initialization method ``init_state()``. Unlike ``__init__()``, this method initializes the model's state, not the model parameters. The state refers to variables that change during the model's operation. In ``brainstate``, all variables that need to change must be encapsulated in a ``State`` object. The hidden state variables, which change during the model's operation, must be encapsulated in a ``HiddenState`` object (a subclass of ``State``).\n", + "\n", + "4. Then, define the method to calculate dV. Similar to the functions for m, h, and n, this method defines some commonly used computations as class methods for easy access. However, in this case, the calculation of dV involves the current I. In this example, the neurons are not connected, but the same process can be used for defining neurons in a network. Therefore, ``I = self.sum_current_inputs(I, V)`` includes both external input currents and currents from other neurons.\n", + "\n", + "5. Finally, define the ``update()`` method, which receives the input for each time step and updates the model variables. ``bst.environ.get('t')`` is used to get the current time t. The ordinary differential equations are solved, and the current values of each variable are obtained using the exponential Euler method ``brainstate.nn.exp_euler_step()`` (where the first argument is the drift term of the ordinary differential equation, and the other arguments are the parameters the equation requires). For neurons in the network, ``V = self.sum_delta_inputs(init=V)`` allows the model to receive inputs from other neurons through delta synaptic transmission. Then, the updated spike information is computed, and the model variables are updated. The output indicates whether the neurons fired an action potential (1 if fired, 0 if not). When using the model, the ``update()`` method is automatically called when the model instance is invoked with input." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f08c361a", + "metadata": {}, + "outputs": [], + "source": [ + "class HH(bst.nn.Dynamics):\n", + " def __init__(\n", + " self,\n", + " in_size,\n", + " ENa=50. * u.mV, gNa=120. * u.mS / u.cm ** 2,\n", + " EK=-77. * u.mV, gK=36. * u.mS / u.cm ** 2,\n", + " EL=-54.387 * u.mV, gL=0.03 * u.mS / u.cm ** 2,\n", + " V_th=20. * u.mV,\n", + " C=1.0 * u.uF / u.cm ** 2\n", + " ):\n", + " # Initialization of the neuron model parameters\n", + " super().__init__(in_size)\n", + "\n", + " # Set model parameters based on provided values or defaults\n", + " self.ENa = ENa # Sodium reversal potential (mV)\n", + " self.EK = EK # Potassium reversal potential (mV)\n", + " self.EL = EL # Leak reversal potential (mV)\n", + " self.gNa = gNa # Sodium conductance (mS/cm^2)\n", + " self.gK = gK # Potassium conductance (mS/cm^2)\n", + " self.gL = gL # Leak conductance (mS/cm^2)\n", + " self.C = C # Membrane capacitance (uF/cm^2)\n", + " self.V_th = V_th # Threshold for spike (mV)\n", + "\n", + " # m (sodium activation) channel kinetics\n", + " m_alpha = lambda self, V: 1. / u.math.exprel(-(V / u.mV + 40) / 10) # Alpha function for m\n", + " m_beta = lambda self, V: 4.0 * jnp.exp(-(V / u.mV + 65) / 18) # Beta function for m\n", + " m_inf = lambda self, V: self.m_alpha(V) / (self.m_alpha(V) + self.m_beta(V)) # Steady-state value for m\n", + " dm = lambda self, m, t, V: (self.m_alpha(V) * (1 - m) - self.m_beta(V) * m) / u.ms # Rate of change of m\n", + "\n", + " # h (sodium inactivation) channel kinetics\n", + " h_alpha = lambda self, V: 0.07 * jnp.exp(-(V / u.mV + 65) / 20.) # Alpha function for h\n", + " h_beta = lambda self, V: 1 / (1 + jnp.exp(-(V / u.mV + 35) / 10)) # Beta function for h\n", + " h_inf = lambda self, V: self.h_alpha(V) / (self.h_alpha(V) + self.h_beta(V)) # Steady-state value for h\n", + " dh = lambda self, h, t, V: (self.h_alpha(V) * (1 - h) - self.h_beta(V) * h) / u.ms # Rate of change of h\n", + "\n", + " # n (potassium activation) channel kinetics\n", + " n_alpha = lambda self, V: 0.1 / u.math.exprel(-(V / u.mV + 55) / 10) # Alpha function for n\n", + " n_beta = lambda self, V: 0.125 * jnp.exp(-(V / u.mV + 65) / 80) # Beta function for n\n", + " n_inf = lambda self, V: self.n_alpha(V) / (self.n_alpha(V) + self.n_beta(V)) # Steady-state value for n\n", + " dn = lambda self, n, t, V: (self.n_alpha(V) * (1 - n) - self.n_beta(V) * n) / u.ms # Rate of change of n\n", + "\n", + " def init_state(self, batch_size=None):\n", + " # Initialize the state variables for membrane potential (V) and gating variables (m, h, n)\n", + " self.V = bst.HiddenState(jnp.ones(self.varshape, bst.environ.dftype()) * -65. * u.mV) # Resting potential (mV)\n", + " self.m = bst.HiddenState(self.m_inf(self.V.value)) # Sodium activation variable\n", + " self.h = bst.HiddenState(self.h_inf(self.V.value)) # Sodium inactivation variable\n", + " self.n = bst.HiddenState(self.n_inf(self.V.value)) # Potassium activation variable\n", + "\n", + " def dV(self, V, t, m, h, n, I):\n", + " # Compute the derivative of membrane potential (V) based on the currents and model parameters\n", + " I = self.sum_current_inputs(I, V) # Sum of all incoming currents\n", + " I_Na = (self.gNa * m * m * m * h) * (V - self.ENa) # Sodium current (I_Na)\n", + " n2 = n * n # Squared potassium activation variable\n", + " I_K = (self.gK * n2 * n2) * (V - self.EK) # Potassium current (I_K)\n", + " I_leak = self.gL * (V - self.EL) # Leak current (I_leak)\n", + " dVdt = (- I_Na - I_K - I_leak + I) / self.C # Membrane potential change rate (dV/dt)\n", + " return dVdt\n", + "\n", + " def update(self, x=0. * u.mA / u.cm ** 2):\n", + " # Update the state of the neuron based on current inputs and time\n", + " t = bst.environ.get('t') # Retrieve the current time\n", + " V = bst.nn.exp_euler_step(self.dV, self.V.value, t, self.m.value, self.h.value, self.n.value, x) # Update membrane potential\n", + " m = bst.nn.exp_euler_step(self.dm, self.m.value, t, self.V.value) # Update m variable (activation)\n", + " h = bst.nn.exp_euler_step(self.dh, self.h.value, t, self.V.value) # Update h variable (inactivation)\n", + " n = bst.nn.exp_euler_step(self.dn, self.n.value, t, self.V.value) # Update n variable (activation)\n", + " V = self.sum_delta_inputs(init=V) # Sum the inputs for membrane potential\n", + " spike = jnp.logical_and(self.V.value < self.V_th, V >= self.V_th) # Check if a spike occurs\n", + " self.V.value = V # Update membrane potential\n", + " self.m.value = m # Update m variable\n", + " self.h.value = h # Update h variable\n", + " self.n.value = n # Update n variable\n", + " return spike # Return the spike event (True/False)" + ] + }, + { + "cell_type": "markdown", + "id": "4bd51608", + "metadata": {}, + "source": [ + "## Running the Model Simulation\n", + "\n", + "After instantiating the defined model, we need to initialize the instance with ``bst.nn.init_all_states()``.\n", + "\n", + "Define the model’s time step ``dt``." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "85dca7b9", + "metadata": {}, + "outputs": [], + "source": [ + "hh = HH(10)\n", + "bst.nn.init_all_states(hh)\n", + "dt = 0.01 * u.ms" + ] + }, + { + "cell_type": "markdown", + "id": "aab91f0f", + "metadata": {}, + "source": [ + "Define the function ``run()`` for running the model one step at a time.\n", + "\n", + "``with bst.environ.context(t=t, dt=dt):``is used to define environment variables within a code block, and variables can be accessed using ``bst.environ.get()`` (e.g., ``bst.environ.get('t')``). This is necessary because we use ``t = bst.environ.get('t')`` inside the ``update()`` method." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ef0d23e0", + "metadata": {}, + "outputs": [], + "source": [ + "def run(t, inp):\n", + " # Run the simulation for a given time 't' and input current 'inp'\n", + " # `bst.environ.context` sets the environment context for this simulation step\n", + " with bst.environ.context(t=t, dt=dt):\n", + " hh(inp) # Update the Hodgkin-Huxley model using the input current at time 't'\n", + " \n", + " # Return the membrane potential at the current time step\n", + " return hh.V.value" + ] + }, + { + "cell_type": "markdown", + "id": "3602f2a4", + "metadata": {}, + "source": [ + "Use ``bst.compile.for_loop()`` to iterate the function and run the simulation for a period of time. The first argument is the function to iterate, followed by the parameters the function needs. You can also display a progress bar during the iteration.\n", + "\n", + "This completes the simulation of the single neuron model." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "da2ea460", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-12-15 18:46:31.392242: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:31.392310: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:31.392340: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3e6597ac990046f3b6986029f3c87476", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10000 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Define the simulation times, from 0 to 100 ms with a time step of 'dt'\n", + "times = u.math.arange(0. * u.ms, 100. * u.ms, dt)\n", + "\n", + "# Run the simulation using `bst.compile.for_loop`:\n", + "# - `run` function is called iteratively with each time step and random input current\n", + "# - Random input current between 1 and 10 uA/cm² is generated at each time step\n", + "# - `pbar` is used to show a progress bar during the simulation\n", + "vs = bst.compile.for_loop(run,\n", + " times, # Time steps as input\n", + " bst.random.uniform(1., 10., times.shape) * u.uA / u.cm ** 2, # Random input current (1 to 10 uA/cm²)\n", + " pbar=bst.compile.ProgressBar(count=10)) # Show progress bar with 10 steps\n", + "\n", + "# Plot the membrane potential over time\n", + "plt.plot(times, vs)\n", + "plt.show() # Display the plot" + ] + }, + { + "cell_type": "markdown", + "id": "c3f5e7c3", + "metadata": {}, + "source": [ + "# Simulation of Spiking Neural Network Models\n", + "\n", + "One of the goals of neuroscience research is to uncover the possible principles by which the brain encodes information. As a potential encoding rule, we naturally expect neurons to produce the same response to the same stimulus. However, in the 1980s and 1990s, numerous experiments found that when the same external stimulus is presented repeatedly, the spike sequences produced by neurons in the cerebral cortex are different each time, and the spike sequences exhibit highly irregular statistical behaviors. Van Vreeswijk and Haim Sompolinsky proposed the **Excitatory-Inhibitory Balanced Network (E-I balanced network)**. They suggested that there should be both excitatory and inhibitory neurons in the network, and the inputs to both types of neurons must be balanced and counteracting. In this case, the mean input received by the neurons remains very small, and the variance (fluctuation) is large enough to induce irregular firing of neurons. Furthermore, the following conditions must also hold for the network:\n", + "+ Neuron connections are random and sparse, which reduces the statistical correlation between the internal inputs received by different neurons, leading to stronger macroscopic irregularity.\n", + "+ Statistically, the excitatory inputs and inhibitory inputs received by a neuron should approximately cancel each other out, meaning that the excitation and inhibition transmitted within the network are balanced.\n", + "+ The connection strength between neurons within the network is relatively strong, so the activity of the entire network is dominated not by external inputs but by synaptic currents generated by the internal network connections. The random fluctuations in synaptic currents determine the irregular firing of neurons.\n", + "\n", + "
\n", + " \"EI-balance\"\n", + "
\n", + "\n", + "Here, we simulate the Excitatory-Inhibitory Balanced Network model as an example of simulating a spiking neural network model.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "60bc3e45", + "metadata": {}, + "outputs": [], + "source": [ + "import brainunit as u\n", + "import brainstate as bst\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "id": "2b06bb4b", + "metadata": {}, + "source": [ + "## Defining Spiking Neural Network Model\n", + "\n", + "We can use ``brainstate`` to define custom neuron models. To define a custom neuron model, we need to inherit the base class ``brainstate.nn.DynamicsGroup``.\n", + "\n", + "1. First, define the initialization method ``__init__()``, which receives model parameters and initializes the model. Note that we need to first call ``super().__init__()`` to initialize the base class. The model initialization mainly includes initializing neurons and synapses:\n", + " - **Initializing Neurons**: Neurons in the network can either use the pre-defined neuron models in ``brainstate.nn`` or use the custom neurons defined in the **Single Neuron Model Definition** section.\n", + " - **Initializing Synapses**: Here, we use ``brainstate.nn.AlignPostProj``, which is suitable for the align-post projection model. In the align-post projection, the dimensions of the synaptic variables and the postsynaptic neuron group are the same. The update order of align-post projection models is: action potential → synaptic communication → synaptic dynamics → output. The update order of align-pre projection models is: action potential → synaptic dynamics → synaptic communication → output. Several parameters need to be set:\n", + " - ``comm``: Describes the connections between the neuron groups.\n", + " - ``syn``: Specifies which synapse model is used.\n", + " - ``out``: Indicates whether the output is based on conductance or current.\n", + " - ``post``: Specifies the postsynaptic neuron group.\n", + "\n", + "2. Next, define the ``update()`` method, which receives the input for each time step and updates the model's current state. As a neuron network, neurons need to receive inputs not only from external sources but also from other neurons. Therefore, in this model, we first compute the inputs received from other neurons, then calculate the external inputs. Finally, we output the firing state of each neuron in the entire network." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "61efc944", + "metadata": {}, + "outputs": [], + "source": [ + "class EINet(bst.nn.DynamicsGroup):\n", + " def __init__(self, n_exc, n_inh, prob, JE, JI):\n", + " # Initialize the network with the following parameters:\n", + " # - n_exc: number of excitatory neurons\n", + " # - n_inh: number of inhibitory neurons\n", + " # - prob: connection probability between neurons\n", + " # - JE: synaptic weight for excitatory connections\n", + " # - JI: synaptic weight for inhibitory connections\n", + " super().__init__()\n", + "\n", + " self.n_exc = n_exc # Number of excitatory neurons\n", + " self.n_inh = n_inh # Number of inhibitory neurons\n", + " self.num = n_exc + n_inh # Total number of neurons (excitatory + inhibitory)\n", + "\n", + " # Initialize the neurons as LIF (Leaky Integrate-and-Fire) neurons\n", + " self.N = bst.nn.LIF(\n", + " n_exc + n_inh, # Total number of neurons\n", + " V_rest=-52. * u.mV, # Resting potential (mV)\n", + " V_th=-50. * u.mV, # Threshold potential for firing (mV)\n", + " V_reset=-60. * u.mV, # Reset potential after spike (mV)\n", + " tau=10. * u.ms, # Membrane time constant (ms)\n", + " V_initializer=bst.init.Normal(-60., 10., unit=u.mV), # Initialize membrane potential with a normal distribution\n", + " spk_reset='soft' # Soft reset for spiking (reset without forcing a specific value)\n", + " )\n", + "\n", + " # Synapse connections from excitatory neurons to all neurons\n", + " self.E = bst.nn.AlignPostProj(\n", + " comm=bst.event.FixedProb(n_exc, self.num, prob, JE), # Fixed probability of synaptic connection with strength JE\n", + " syn=bst.nn.Expon.desc(self.num, tau=2. * u.ms), # Exponential decay of synaptic weight\n", + " out=bst.nn.CUBA.desc(), # CUBA (Conductance-based) synaptic model\n", + " post=self.N, # Target neurons for these excitatory synapses\n", + " )\n", + "\n", + " # Synapse connections from inhibitory neurons to all neurons\n", + " self.I = bst.nn.AlignPostProj(\n", + " comm=bst.event.FixedProb(n_inh, self.num, prob, JI), # Fixed probability of synaptic connection with strength JI\n", + " syn=bst.nn.Expon.desc(self.num, tau=2. * u.ms), # Exponential decay of synaptic weight\n", + " out=bst.nn.CUBA.desc(), # CUBA (Conductance-based) synaptic model\n", + " post=self.N, # Target neurons for these inhibitory synapses\n", + " )\n", + "\n", + " def update(self, inp):\n", + " # Get the spike states of the neurons\n", + " spks = self.N.get_spike() != 0. # Non-zero spikes (spike detection)\n", + "\n", + " # Update the synaptic currents for excitatory and inhibitory neurons\n", + " self.E(spks[:self.n_exc]) # Apply excitatory synaptic input based on the excitatory neuron spikes\n", + " self.I(spks[self.n_exc:]) # Apply inhibitory synaptic input based on the inhibitory neuron spikes\n", + "\n", + " # Update the neurons with the provided input current (inp)\n", + " self.N(inp)\n", + "\n", + " # Return the spike states of the neurons (whether each neuron spiked)\n", + " return self.N.get_spike()" + ] + }, + { + "cell_type": "markdown", + "id": "3320eacc", + "metadata": {}, + "source": [ + "## Running the Simulation Experiment\n", + "\n", + "Set some model parameters. In this example, we use the sign (positive or negative) of the connection strength to set the excitatory or inhibitory nature of the neurons." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "3e1569f5", + "metadata": {}, + "outputs": [], + "source": [ + "# connectivity\n", + "num_exc = 500\n", + "num_inh = 500\n", + "prob = 0.1\n", + "# external current\n", + "Ib = 3. * u.mA\n", + "# excitatory and inhibitory synaptic weights\n", + "JE = 1 / u.math.sqrt(prob * num_exc) * u.mS\n", + "JI = -1 / u.math.sqrt(prob * num_inh) * u.mS" + ] + }, + { + "cell_type": "markdown", + "id": "f2c2db9e", + "metadata": {}, + "source": [ + "Define the time step ``dt`` for the simulation.\n", + "\n", + "After instantiating the defined model, initialize the instance with ``bst.nn.init_all_states()``." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3aed3747", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "EINet(\n", + " layers_tuple=(),\n", + " layers_dict={},\n", + " n_exc=500,\n", + " n_inh=500,\n", + " num=1000,\n", + " N=LIF(\n", + " in_size=(1000,),\n", + " out_size=(1000,),\n", + " current_inputs={'AlignPostProj0': CUBA(\n", + " scale=volt\n", + " )},\n", + " before_updates={\"(, (1000,), {'tau': 2. * msecond}) // (, (), {})\": _AlignPost(\n", + " syn=Expon(\n", + " in_size=(1000,),\n", + " out_size=(1000,),\n", + " tau=2. * msecond,\n", + " g_initializer=ZeroInit(\n", + " unit=msiemens\n", + " ),\n", + " g=HiddenState(\n", + " value=ArrayImpl([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0.... * msiemens\n", + " )\n", + " ),\n", + " out=CUBA(...)\n", + " )},\n", + " spk_reset='soft',\n", + " spk_fun=ReluGrad(alpha=0.3, width=1.0),\n", + " R=1. * ohm,\n", + " tau=10. * msecond,\n", + " V_th=-50. * mvolt,\n", + " V_rest=-52. * mvolt,\n", + " V_reset=-60. * mvolt,\n", + " V_initializer=Normal(\n", + " scale=10.0,\n", + " mean=-60.0,\n", + " rng=RandomState([2647944946 1939377294]),\n", + " unit=mvolt\n", + " ),\n", + " V=HiddenState(\n", + " value=ArrayImpl([-44.94350815, -49.09746552, -54.77877045, -62.51665115,\n", + " -49.72640991, -53.0278... * mvolt\n", + " )\n", + " ),\n", + " E=AlignPostProj(\n", + " name='AlignPostProj0',\n", + " modules=(),\n", + " merging=True,\n", + " comm=FixedProb(\n", + " in_size=(500,),\n", + " out_size=(1000,),\n", + " n_conn=100,\n", + " float_as_event=True,\n", + " block_size=None,\n", + " indices=Array([[584, 311, 322, ..., 857, 171, 213],\n", + " [502, 87, 501, ..., 176, 239, 808],\n", + " [336, 860, 686, ..., 629, 932, 434],\n", + " ...,\n", + " [838, 631, 745, ..., 767, 427, 536],\n", + " [154, 597, 111, ..., 914, 601, 805],\n", + " [215, 279, 117, ..., 917, 335, 690]], dtype=int32),\n", + " weight=ParamState(\n", + " value=0.14142136 * msiemens\n", + " )\n", + " ),\n", + " syn=Expon(...),\n", + " out=CUBA(...),\n", + " post=LIF(...)\n", + " ),\n", + " I=AlignPostProj(\n", + " name='AlignPostProj1',\n", + " modules=(),\n", + " merging=True,\n", + " comm=FixedProb(\n", + " in_size=(500,),\n", + " out_size=(1000,),\n", + " n_conn=100,\n", + " float_as_event=True,\n", + " block_size=None,\n", + " indices=Array([[257, 901, 935, ..., 722, 965, 139],\n", + " [924, 131, 887, ..., 389, 554, 905],\n", + " [699, 799, 935, ..., 196, 311, 278],\n", + " ...,\n", + " [210, 74, 426, ..., 129, 101, 732],\n", + " [839, 371, 605, ..., 418, 668, 419],\n", + " [924, 822, 688, ..., 137, 877, 855]], dtype=int32),\n", + " weight=ParamState(\n", + " value=-0.14142136 * msiemens\n", + " )\n", + " ),\n", + " syn=Expon(...),\n", + " out=CUBA(...),\n", + " post=LIF(...)\n", + " )\n", + ")" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# network\n", + "bst.environ.set(dt=0.1 * u.ms)\n", + "net = EINet(num_exc, num_inh, prob=prob, JE=JE, JI=JI)\n", + "bst.nn.init_all_states(net)" + ] + }, + { + "cell_type": "markdown", + "id": "0a52aac5", + "metadata": {}, + "source": [ + "The instantiated network model uses the ``update()`` method to input the current for each time step.\n", + "\n", + "Use ``bst.compile.for_loop()`` to iterate the function and run the simulation for a certain period of time. The first argument is the function to iterate, followed by the parameters that the function requires. You can also display a progress bar during the iteration.\n", + "\n", + "This completes the simulation of the spiking neural network model." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "32e8e8ee", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-12-15 18:46:36.710433: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:36.710492: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:36.710519: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "634f59000ce2482bbb528d119d41b4b5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10000 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# visualization\n", + "times = times.to_decimal(u.ms)\n", + "t_indices, n_indices = u.math.where(spikes)\n", + "plt.plot(times[t_indices], n_indices, 'k.', markersize=1)\n", + "plt.xlabel('Time (ms)')\n", + "plt.ylabel('Neuron index')\n", + "plt.show()" + ] } ], "metadata": { @@ -20,14 +665,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.11.10" } }, "nbformat": 4, diff --git a/docs/quickstart/snn_simulation-zh.ipynb b/docs/quickstart/snn_simulation-zh.ipynb index 9ab0228..0f81c6f 100644 --- a/docs/quickstart/snn_simulation-zh.ipynb +++ b/docs/quickstart/snn_simulation-zh.ipynb @@ -2,13 +2,667 @@ "cells": [ { "cell_type": "markdown", - "source": [ - "# 仿真脉冲神经网络" - ], + "id": "b78f6e1b6cb0dada", "metadata": { "collapsed": false }, - "id": "b78f6e1b6cb0dada" + "source": [ + "# 仿真脉冲神经网络\n", + "\n", + "建立脑动力学模型并进行仿真是研究脑动力学的重要方法之一。在进行脉冲神经网络仿真时,我们给定模型和输入的各项参数,进行仿真实验。在这个过程中,不涉及参数(如连接权重)的学习与更新。主要应用于对设计好的网络进行仿真与分析。\n", + "\n", + "脑动力学的脉冲神经网络模型可以分为**单个脉冲神经元模型**和**脉冲神经元网络模型**,我们将分别举一个例子进行演示。" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "d16fec85", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ['XLA_PYTHON_CLIENT_PREALLOCATE'] = 'false'" + ] + }, + { + "cell_type": "markdown", + "id": "c5622779", + "metadata": {}, + "source": [ + "## 单个脉冲神经元模型的仿真\n", + "\n", + "**Hodgkin-Huxley模型(HH模型)** 是由神经生理学家艾伦·霍奇金(Allen Hodgkin,1914-1998)和安德鲁·赫胥黎(Andrew Huxley,1917-2012)于1952年提出的数学模型,用以描述神经元动作电位的产生和传播过程。HH模型以经典电路模型为基础,将神经元膜电位的动态变化与膜离子通道的生物物理特性联系起来,是神经科学中最重要的理论模型之一,曾为二人赢得1963年的诺贝尔生理学或医学奖。HH模型的数学定义是:\n", + "\n", + "\\begin{aligned}C \\frac {dV} {dt} = -(\\bar{g}_{Na} m^3 h (V &-E_{Na})\n", + "+ \\bar{g}_K n^4 (V-E_K) + g_{leak} (V - E_{leak})) + I(t)\\\\\\frac {dx} {dt} &= \\alpha_x (1-x) - \\beta_x, \\quad x\\in {\\rm{\\{m, h, n\\}}}\\\\&\\alpha_m(V) = \\frac {0.1(V+40)}{1-\\exp(\\frac{-(V + 40)} {10})}\\\\&\\beta_m(V) = 4.0 \\exp(\\frac{-(V + 65)} {18})\\\\&\\alpha_h(V) = 0.07 \\exp(\\frac{-(V+65)}{20})\\\\&\\beta_h(V) = \\frac 1 {1 + \\exp(\\frac{-(V + 35)} {10})}\\\\&\\alpha_n(V) = \\frac {0.01(V+55)}{1-\\exp(-(V+55)/10)}\\\\&\\beta_n(V) = 0.125 \\exp(\\frac{-(V + 65)} {80})\\end{aligned}\n", + "\n", + "在这里我们对HH模型进行仿真,作为单个脉冲神经元模型仿真的示例。``brainstate``可以同时运行多个神经元模型,并行运行节省时间。我们对一群HH神经元进行仿真。" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "9dd07dd9", + "metadata": {}, + "outputs": [], + "source": [ + "import brainunit as u\n", + "import jax.numpy as jnp\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import brainstate as bst" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "051f8f24", + "metadata": {}, + "outputs": [], + "source": [ + "# bst.environ.set(platform='gpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "60058528", + "metadata": {}, + "outputs": [], + "source": [ + "bst.random.seed(100)" + ] + }, + { + "cell_type": "markdown", + "id": "a6ba5685", + "metadata": {}, + "source": [ + "## 单神经元模型的定义\n", + "\n", + "我们可以使用``brainstate``自定义神经元模型。自定义神经元模型需要继承模型基类``brainstate.nn.Dynamics``。\n", + "\n", + "1. 首先定义初始化类方法``__init__()``,接收并行运行的神经元个数``in_size``,和其他模型参数。用``in_size``初始化基类,将模型参数设置为模型类属性,便于后续调用。\n", + "\n", + "2. 然后,可以设定一些模型常用计算为类方法,便于后续调用。在这里我们实现了一些计算m、h和n涉及的函数。注意常微分方程的漂移项函数,传入参数的顺序应为,动态变量,当前时刻t和其他参数。\n", + "\n", + "3. 接着,定义模型状态初始化方法``init_state()``。与``__init__()``不同,这里初始化的不是模型参数,而是模型状态,主要是模型运行中会改变的变量的初始化。在``brainstate``中,所有需要改变的量都需封装在 ``State`` 对象中。模型运行时会发生改变的隐变量需封装在``HiddenState``(是``State`` 的子类)对象中。\n", + "\n", + "4. 然后定义dV的计算方法。本质上和上文提到的计算m、h和n涉及的函数一样,都是设定一些模型常用计算为类方法,便于后续调用。但需要注意的地方是,dV的计算涉及到电流I。在这个例子中,我们仿真的神经元是相互之间没有连接的,但这套定义单神经元模型的流程也适用于,定义网络中的神经元。因此,``I = self.sum_current_inputs(I, V)``,I包括外界输入电流和来自其他神经元的电流。\n", + "\n", + "5. 最后定义``update()``方法,接收每个时间步模型的input,把模型中各个变量进行更新。``bst.environ.get('t')``获取当前时刻t。解常微分方程,求得每个变量当前时间步的值,这里使用了指数欧拉法``brainstate.nn.exp_euler_step()``求解方程(接收第一个参数是常微分方程的漂移项,其他参数是方程函数需要接收的参数)。对于网络中的神经元,``V = self.sum_delta_inputs(init=V)``使得模型接收其他神经元通过delta突触传导的输入。然后计算这步更新后哪些神经元产生了动作电位。最后用计算出的值更新模型变量的值。返回值输出神经元是否发放了动作电位,有发放为1,无发放为0。使用时,通过调用模型实例并传入输入,会自动调用``update()``方法。" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "f08c361a", + "metadata": {}, + "outputs": [], + "source": [ + "class HH(bst.nn.Dynamics):\n", + " def __init__(\n", + " self,\n", + " in_size,\n", + " ENa=50. * u.mV, gNa=120. * u.mS / u.cm ** 2,\n", + " EK=-77. * u.mV, gK=36. * u.mS / u.cm ** 2,\n", + " EL=-54.387 * u.mV, gL=0.03 * u.mS / u.cm ** 2,\n", + " V_th=20. * u.mV,\n", + " C=1.0 * u.uF / u.cm ** 2\n", + " ):\n", + " # Initialization of the neuron model parameters\n", + " super().__init__(in_size)\n", + "\n", + " # Set model parameters based on provided values or defaults\n", + " self.ENa = ENa # Sodium reversal potential (mV)\n", + " self.EK = EK # Potassium reversal potential (mV)\n", + " self.EL = EL # Leak reversal potential (mV)\n", + " self.gNa = gNa # Sodium conductance (mS/cm^2)\n", + " self.gK = gK # Potassium conductance (mS/cm^2)\n", + " self.gL = gL # Leak conductance (mS/cm^2)\n", + " self.C = C # Membrane capacitance (uF/cm^2)\n", + " self.V_th = V_th # Threshold for spike (mV)\n", + "\n", + " # m (sodium activation) channel kinetics\n", + " m_alpha = lambda self, V: 1. / u.math.exprel(-(V / u.mV + 40) / 10) # Alpha function for m\n", + " m_beta = lambda self, V: 4.0 * jnp.exp(-(V / u.mV + 65) / 18) # Beta function for m\n", + " m_inf = lambda self, V: self.m_alpha(V) / (self.m_alpha(V) + self.m_beta(V)) # Steady-state value for m\n", + " dm = lambda self, m, t, V: (self.m_alpha(V) * (1 - m) - self.m_beta(V) * m) / u.ms # Rate of change of m\n", + "\n", + " # h (sodium inactivation) channel kinetics\n", + " h_alpha = lambda self, V: 0.07 * jnp.exp(-(V / u.mV + 65) / 20.) # Alpha function for h\n", + " h_beta = lambda self, V: 1 / (1 + jnp.exp(-(V / u.mV + 35) / 10)) # Beta function for h\n", + " h_inf = lambda self, V: self.h_alpha(V) / (self.h_alpha(V) + self.h_beta(V)) # Steady-state value for h\n", + " dh = lambda self, h, t, V: (self.h_alpha(V) * (1 - h) - self.h_beta(V) * h) / u.ms # Rate of change of h\n", + "\n", + " # n (potassium activation) channel kinetics\n", + " n_alpha = lambda self, V: 0.1 / u.math.exprel(-(V / u.mV + 55) / 10) # Alpha function for n\n", + " n_beta = lambda self, V: 0.125 * jnp.exp(-(V / u.mV + 65) / 80) # Beta function for n\n", + " n_inf = lambda self, V: self.n_alpha(V) / (self.n_alpha(V) + self.n_beta(V)) # Steady-state value for n\n", + " dn = lambda self, n, t, V: (self.n_alpha(V) * (1 - n) - self.n_beta(V) * n) / u.ms # Rate of change of n\n", + "\n", + " def init_state(self, batch_size=None):\n", + " # Initialize the state variables for membrane potential (V) and gating variables (m, h, n)\n", + " self.V = bst.HiddenState(jnp.ones(self.varshape, bst.environ.dftype()) * -65. * u.mV) # Resting potential (mV)\n", + " self.m = bst.HiddenState(self.m_inf(self.V.value)) # Sodium activation variable\n", + " self.h = bst.HiddenState(self.h_inf(self.V.value)) # Sodium inactivation variable\n", + " self.n = bst.HiddenState(self.n_inf(self.V.value)) # Potassium activation variable\n", + "\n", + " def dV(self, V, t, m, h, n, I):\n", + " # Compute the derivative of membrane potential (V) based on the currents and model parameters\n", + " I = self.sum_current_inputs(I, V) # Sum of all incoming currents\n", + " I_Na = (self.gNa * m * m * m * h) * (V - self.ENa) # Sodium current (I_Na)\n", + " n2 = n * n # Squared potassium activation variable\n", + " I_K = (self.gK * n2 * n2) * (V - self.EK) # Potassium current (I_K)\n", + " I_leak = self.gL * (V - self.EL) # Leak current (I_leak)\n", + " dVdt = (- I_Na - I_K - I_leak + I) / self.C # Membrane potential change rate (dV/dt)\n", + " return dVdt\n", + "\n", + " def update(self, x=0. * u.mA / u.cm ** 2):\n", + " # Update the state of the neuron based on current inputs and time\n", + " t = bst.environ.get('t') # Retrieve the current time\n", + " V = bst.nn.exp_euler_step(self.dV, self.V.value, t, self.m.value, self.h.value, self.n.value, x) # Update membrane potential\n", + " m = bst.nn.exp_euler_step(self.dm, self.m.value, t, self.V.value) # Update m variable (activation)\n", + " h = bst.nn.exp_euler_step(self.dh, self.h.value, t, self.V.value) # Update h variable (inactivation)\n", + " n = bst.nn.exp_euler_step(self.dn, self.n.value, t, self.V.value) # Update n variable (activation)\n", + " V = self.sum_delta_inputs(init=V) # Sum the inputs for membrane potential\n", + " spike = jnp.logical_and(self.V.value < self.V_th, V >= self.V_th) # Check if a spike occurs\n", + " self.V.value = V # Update membrane potential\n", + " self.m.value = m # Update m variable\n", + " self.h.value = h # Update h variable\n", + " self.n.value = n # Update n variable\n", + " return spike # Return the spike event (True/False)" + ] + }, + { + "cell_type": "markdown", + "id": "4bd51608", + "metadata": {}, + "source": [ + "## 模型仿真实验运行\n", + "\n", + "实例化定义好的模型后,要先``bst.nn.init_all_states()``初始化这个实例。\n", + "\n", + "定义模型``dt``对应的时间。" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "85dca7b9", + "metadata": {}, + "outputs": [], + "source": [ + "hh = HH(10)\n", + "bst.nn.init_all_states(hh)\n", + "dt = 0.01 * u.ms" + ] + }, + { + "cell_type": "markdown", + "id": "aab91f0f", + "metadata": {}, + "source": [ + "定义模型单步运行函数``run()``。\n", + "\n", + "``with bst.environ.context(t=t, dt=dt):``可以定义代码块内的环境变量,代码块内都可以通过``bst.environ.get()``获取变量值(eg. ``bst.environ.get('t')``)。在这里需要用到是因为我们定义的模型``update()``方法中使用了``t = bst.environ.get('t')``。" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ef0d23e0", + "metadata": {}, + "outputs": [], + "source": [ + "def run(t, inp):\n", + " # Run the simulation for a given time 't' and input current 'inp'\n", + " # `bst.environ.context` sets the environment context for this simulation step\n", + " with bst.environ.context(t=t, dt=dt):\n", + " hh(inp) # Update the Hodgkin-Huxley model using the input current at time 't'\n", + " \n", + " # Return the membrane potential at the current time step\n", + " return hh.V.value" + ] + }, + { + "cell_type": "markdown", + "id": "3602f2a4", + "metadata": {}, + "source": [ + "使用``bst.compile.for_loop()``迭代运行函数,进行一段时间的仿真,第一个参数是要迭代的函数,随后是此函数所需要的参数。可以选择绘制迭代进度条。\n", + "\n", + "这样就完成了单个脉冲神经元模型的仿真。" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "da2ea460", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-12-15 18:46:31.392242: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:31.392310: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:31.392340: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3e6597ac990046f3b6986029f3c87476", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10000 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Define the simulation times, from 0 to 100 ms with a time step of 'dt'\n", + "times = u.math.arange(0. * u.ms, 100. * u.ms, dt)\n", + "\n", + "# Run the simulation using `bst.compile.for_loop`:\n", + "# - `run` function is called iteratively with each time step and random input current\n", + "# - Random input current between 1 and 10 uA/cm² is generated at each time step\n", + "# - `pbar` is used to show a progress bar during the simulation\n", + "vs = bst.compile.for_loop(run,\n", + " times, # Time steps as input\n", + " bst.random.uniform(1., 10., times.shape) * u.uA / u.cm ** 2, # Random input current (1 to 10 uA/cm²)\n", + " pbar=bst.compile.ProgressBar(count=10)) # Show progress bar with 10 steps\n", + "\n", + "# Plot the membrane potential over time\n", + "plt.plot(times, vs)\n", + "plt.show() # Display the plot" + ] + }, + { + "cell_type": "markdown", + "id": "f4b57d2a", + "metadata": {}, + "source": [ + "## 脉冲神经元网络模型的仿真" + ] + }, + { + "cell_type": "markdown", + "id": "c3f5e7c3", + "metadata": {}, + "source": [ + "神经科学研究的目的之一是要解开大脑编码信息的可能法则。作为一种编码法则,我们很自然指望神经元在相同刺激下产生相同的反应。但上世纪 80 至 90 年代,大量实验发现,同样的外部刺激重复呈现,大脑皮层中的神经元每次产生的脉冲序列都不同,且单次脉冲序列表现出极不规律的统计行为。范·弗雷斯维克(Van Vreeswijk)和海姆·索姆林斯基(Haim Sompolinsky)提出了**兴奋-抑制平衡网络(E-I balanced network)**。他们提出网络中应同时存在兴奋性神经元和抑制性神经元,且两种神经元的输入必须是平衡的、相互抵消的,此时神经元接收到输入的均值维持在一个很小的值,方差(波动)才足够显著,从而促使神经元无规律发放。除此以外,对网络还有以下要求:\n", + "+ 神经元之间的连接是随机且稀疏的,这使得不同神经元接收到的内部输入的统计相关性很小,宏观上表\n", + "现出更强的不规律性;\n", + "+ 统计上,一个神经元接收到的兴奋性输入和抑制性输入应该能大致抵消,即网络中传递的兴奋和抑制是平衡的;\n", + "+ 网络内部神经元之间的连接强度相对较强,这使得整个网络的活动不是被外部输入而是被网络内部突触\n", + "连接产生的电流主导,突触电流的随机起伏决定了神经元的无规律发放。\n", + "\n", + "
\n", + " \"EI-balance\"\n", + "
\n", + "\n", + "在这里我们对兴奋-抑制平衡网络模型进行仿真,作为脉冲神经元网络模型仿真的示例。" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "60bc3e45", + "metadata": {}, + "outputs": [], + "source": [ + "import brainunit as u\n", + "import brainstate as bst\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "id": "2b06bb4b", + "metadata": {}, + "source": [ + "## 脉冲神经网络模型的定义\n", + "\n", + "我们可以使用``brainstate``自定义脉冲神经网络模型。自定义脉冲神经网络模型需要继承模型基类``brainstate.nn.DynamicsGroup``。\n", + "\n", + "1. 首先定义初始化类方法``__init__()``,接收模型参数,初始化模型。注意要先``super().__init__()``初始化基类。初始化模型主要包括初始化神经元和突触:\n", + " - **初始化神经元**:网络中神经元可以选择``brainstate.nn``中已经实现好的各种神经元,也可以选用我们在**单神经元模型的定义**部分自定义的神经元。\n", + " - **初始化突触**:这里使用了``brainstate.nn.AlignPostProj``,适用于align-post投射模型。align-post投射意味着突触变量和突触后神经元群的维度一致。align-post和align-pre模型的更新顺序不同,align-post投射模型更新顺序是动作电位 -> 突触通讯 -> 突触动力学 -> 输出;align-pre投射模型更新顺序是动作电位 -> 突触动力学 -> 突触通讯 -> 输出。它需要设置的几个参数分别是:\n", + " - ``comm``:神经元群之间的连接是怎样的\n", + " - ``syn``:使用哪种突触模型\n", + " - ``out``:设置输出是基于电导还是基于电流的\n", + " - ``post``:指出突触后神经元群。\n", + "\n", + "\n", + "2. 然后定义``update()``方法,接收每个时间步模型的input,更新模型当前状态。作为神经元群,神经元除了外界输入还要接收其它神经元的输入。因此在这个模型中,先计算神经元群接收神经元的输入,再计算接收外界输入。最后输出整个网络每个神经元的发放情况。" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "61efc944", + "metadata": {}, + "outputs": [], + "source": [ + "class EINet(bst.nn.DynamicsGroup):\n", + " def __init__(self, n_exc, n_inh, prob, JE, JI):\n", + " # Initialize the network with the following parameters:\n", + " # - n_exc: number of excitatory neurons\n", + " # - n_inh: number of inhibitory neurons\n", + " # - prob: connection probability between neurons\n", + " # - JE: synaptic weight for excitatory connections\n", + " # - JI: synaptic weight for inhibitory connections\n", + " super().__init__()\n", + "\n", + " self.n_exc = n_exc # Number of excitatory neurons\n", + " self.n_inh = n_inh # Number of inhibitory neurons\n", + " self.num = n_exc + n_inh # Total number of neurons (excitatory + inhibitory)\n", + "\n", + " # Initialize the neurons as LIF (Leaky Integrate-and-Fire) neurons\n", + " self.N = bst.nn.LIF(\n", + " n_exc + n_inh, # Total number of neurons\n", + " V_rest=-52. * u.mV, # Resting potential (mV)\n", + " V_th=-50. * u.mV, # Threshold potential for firing (mV)\n", + " V_reset=-60. * u.mV, # Reset potential after spike (mV)\n", + " tau=10. * u.ms, # Membrane time constant (ms)\n", + " V_initializer=bst.init.Normal(-60., 10., unit=u.mV), # Initialize membrane potential with a normal distribution\n", + " spk_reset='soft' # Soft reset for spiking (reset without forcing a specific value)\n", + " )\n", + "\n", + " # Synapse connections from excitatory neurons to all neurons\n", + " self.E = bst.nn.AlignPostProj(\n", + " comm=bst.event.FixedProb(n_exc, self.num, prob, JE), # Fixed probability of synaptic connection with strength JE\n", + " syn=bst.nn.Expon.desc(self.num, tau=2. * u.ms), # Exponential decay of synaptic weight\n", + " out=bst.nn.CUBA.desc(), # CUBA (Conductance-based) synaptic model\n", + " post=self.N, # Target neurons for these excitatory synapses\n", + " )\n", + "\n", + " # Synapse connections from inhibitory neurons to all neurons\n", + " self.I = bst.nn.AlignPostProj(\n", + " comm=bst.event.FixedProb(n_inh, self.num, prob, JI), # Fixed probability of synaptic connection with strength JI\n", + " syn=bst.nn.Expon.desc(self.num, tau=2. * u.ms), # Exponential decay of synaptic weight\n", + " out=bst.nn.CUBA.desc(), # CUBA (Conductance-based) synaptic model\n", + " post=self.N, # Target neurons for these inhibitory synapses\n", + " )\n", + "\n", + " def update(self, inp):\n", + " # Get the spike states of the neurons\n", + " spks = self.N.get_spike() != 0. # Non-zero spikes (spike detection)\n", + "\n", + " # Update the synaptic currents for excitatory and inhibitory neurons\n", + " self.E(spks[:self.n_exc]) # Apply excitatory synaptic input based on the excitatory neuron spikes\n", + " self.I(spks[self.n_exc:]) # Apply inhibitory synaptic input based on the inhibitory neuron spikes\n", + "\n", + " # Update the neurons with the provided input current (inp)\n", + " self.N(inp)\n", + "\n", + " # Return the spike states of the neurons (whether each neuron spiked)\n", + " return self.N.get_spike()" + ] + }, + { + "cell_type": "markdown", + "id": "3320eacc", + "metadata": {}, + "source": [ + "## 模型仿真实验运行\n", + "\n", + "设置一些模型参数。在这个例子中,我们用连接强度的正负来设置神经元的兴奋抑制性。" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "3e1569f5", + "metadata": {}, + "outputs": [], + "source": [ + "# connectivity\n", + "num_exc = 500\n", + "num_inh = 500\n", + "prob = 0.1\n", + "# external current\n", + "Ib = 3. * u.mA\n", + "# excitatory and inhibitory synaptic weights\n", + "JE = 1 / u.math.sqrt(prob * num_exc) * u.mS\n", + "JI = -1 / u.math.sqrt(prob * num_inh) * u.mS" + ] + }, + { + "cell_type": "markdown", + "id": "f2c2db9e", + "metadata": {}, + "source": [ + "定义仿真实验中dt对应的时间。\n", + "\n", + "实例化定义好的模型后,要先``bst.nn.init_all_states()``初始化这个实例。" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "3aed3747", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "EINet(\n", + " layers_tuple=(),\n", + " layers_dict={},\n", + " n_exc=500,\n", + " n_inh=500,\n", + " num=1000,\n", + " N=LIF(\n", + " in_size=(1000,),\n", + " out_size=(1000,),\n", + " current_inputs={'AlignPostProj0': CUBA(\n", + " scale=volt\n", + " )},\n", + " before_updates={\"(, (1000,), {'tau': 2. * msecond}) // (, (), {})\": _AlignPost(\n", + " syn=Expon(\n", + " in_size=(1000,),\n", + " out_size=(1000,),\n", + " tau=2. * msecond,\n", + " g_initializer=ZeroInit(\n", + " unit=msiemens\n", + " ),\n", + " g=HiddenState(\n", + " value=ArrayImpl([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0.... * msiemens\n", + " )\n", + " ),\n", + " out=CUBA(...)\n", + " )},\n", + " spk_reset='soft',\n", + " spk_fun=ReluGrad(alpha=0.3, width=1.0),\n", + " R=1. * ohm,\n", + " tau=10. * msecond,\n", + " V_th=-50. * mvolt,\n", + " V_rest=-52. * mvolt,\n", + " V_reset=-60. * mvolt,\n", + " V_initializer=Normal(\n", + " scale=10.0,\n", + " mean=-60.0,\n", + " rng=RandomState([2647944946 1939377294]),\n", + " unit=mvolt\n", + " ),\n", + " V=HiddenState(\n", + " value=ArrayImpl([-44.94350815, -49.09746552, -54.77877045, -62.51665115,\n", + " -49.72640991, -53.0278... * mvolt\n", + " )\n", + " ),\n", + " E=AlignPostProj(\n", + " name='AlignPostProj0',\n", + " modules=(),\n", + " merging=True,\n", + " comm=FixedProb(\n", + " in_size=(500,),\n", + " out_size=(1000,),\n", + " n_conn=100,\n", + " float_as_event=True,\n", + " block_size=None,\n", + " indices=Array([[584, 311, 322, ..., 857, 171, 213],\n", + " [502, 87, 501, ..., 176, 239, 808],\n", + " [336, 860, 686, ..., 629, 932, 434],\n", + " ...,\n", + " [838, 631, 745, ..., 767, 427, 536],\n", + " [154, 597, 111, ..., 914, 601, 805],\n", + " [215, 279, 117, ..., 917, 335, 690]], dtype=int32),\n", + " weight=ParamState(\n", + " value=0.14142136 * msiemens\n", + " )\n", + " ),\n", + " syn=Expon(...),\n", + " out=CUBA(...),\n", + " post=LIF(...)\n", + " ),\n", + " I=AlignPostProj(\n", + " name='AlignPostProj1',\n", + " modules=(),\n", + " merging=True,\n", + " comm=FixedProb(\n", + " in_size=(500,),\n", + " out_size=(1000,),\n", + " n_conn=100,\n", + " float_as_event=True,\n", + " block_size=None,\n", + " indices=Array([[257, 901, 935, ..., 722, 965, 139],\n", + " [924, 131, 887, ..., 389, 554, 905],\n", + " [699, 799, 935, ..., 196, 311, 278],\n", + " ...,\n", + " [210, 74, 426, ..., 129, 101, 732],\n", + " [839, 371, 605, ..., 418, 668, 419],\n", + " [924, 822, 688, ..., 137, 877, 855]], dtype=int32),\n", + " weight=ParamState(\n", + " value=-0.14142136 * msiemens\n", + " )\n", + " ),\n", + " syn=Expon(...),\n", + " out=CUBA(...),\n", + " post=LIF(...)\n", + " )\n", + ")" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# network\n", + "bst.environ.set(dt=0.1 * u.ms)\n", + "net = EINet(num_exc, num_inh, prob=prob, JE=JE, JI=JI)\n", + "bst.nn.init_all_states(net)" + ] + }, + { + "cell_type": "markdown", + "id": "0a52aac5", + "metadata": {}, + "source": [ + "实例化的网络模型使用``update()``方法输入每步的输入电流。\n", + "\n", + "使用``bst.compile.for_loop()``迭代运行函数,进行一段时间的仿真,第一个参数是要迭代的函数,随后是此函数所需要的参数。可以选择绘制迭代进度条。\n", + "\n", + "这样就完成了脉冲神经元网络模型的仿真。" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "32e8e8ee", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-12-15 18:46:36.710433: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:36.710492: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n", + "2024-12-15 18:46:36.710519: W external/xla/xla/service/gpu/ir_emitter_unnested.cc:1171] Unable to parse backend config for custom call: Could not convert JSON string to proto: : Root element must be a message.\n", + "Fall back to parse the raw backend config str.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "634f59000ce2482bbb528d119d41b4b5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/10000 [00:00" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# visualization\n", + "times = times.to_decimal(u.ms)\n", + "t_indices, n_indices = u.math.where(spikes)\n", + "plt.plot(times[t_indices], n_indices, 'k.', markersize=1)\n", + "plt.xlabel('Time (ms)')\n", + "plt.ylabel('Neuron index')\n", + "plt.show()" + ] } ], "metadata": { @@ -20,14 +674,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.11.10" } }, "nbformat": 4, diff --git a/docs/tutorials/artificial_neural_networks-en.ipynb b/docs/tutorials/artificial_neural_networks-en.ipynb index b0f7876..dec3288 100644 --- a/docs/tutorials/artificial_neural_networks-en.ipynb +++ b/docs/tutorials/artificial_neural_networks-en.ipynb @@ -2,13 +2,513 @@ "cells": [ { "cell_type": "markdown", - "source": [ - "# Building Artificial Neural Networks" - ], + "id": "cff08e5afff144b2", "metadata": { "collapsed": false }, - "id": "1a870562d79cf1f1" + "source": [ + "# Building Artificial Neural Networks" + ] + }, + { + "cell_type": "markdown", + "id": "1c216cfe", + "metadata": {}, + "source": [ + "Artificial Neural Networks (ANNs), also known as neural networks (NNs), in the fields of machine learning and cognitive science, are mathematical or computational models that mimic the structure and function of biological neural networks (especially the central nervous system of animals, particularly the brain) for estimating or approximating functions. Similar to how neurons in the human brain are interconnected, neurons in an artificial neural network are also interconnected in various layers of the network.\n", + "\n", + "Compared to spiking neural networks, the neurons in artificial neural networks are simplified and do not have intrinsic dynamics. The information transmitted between neurons is not discrete action potentials (0 or 1), but continuous floating-point numbers, which can be understood as the firing rate of a neuron at a given time step. Although artificial neural networks were originally inspired by biological brains and can exhibit some properties of biological brains, they are primarily used as powerful models to solve specific problems without focusing on how their parts correspond to biological systems.\n", + "\n", + "
\n", + " \"bnn\"\n", + "
\n", + "
\n", + " \"ann\"\n", + "
\n", + "\n", + "## Architecture of Artificial Neural Networks\n", + "\n", + "The neurons in artificial neural networks are distributed across different layers, and information propagates forward layer by layer. These layers can be categorized into three types:\n", + "\n", + "- Input Layer: The first layer of the artificial neural network, which receives input and passes it to the hidden layers.\n", + "- Hidden Layers: These layers perform various computations and feature extraction on the input received from the input layer. Typically, there are multiple hidden layers through which the information flows sequentially.\n", + "- Output Layer: Finally, the output layer receives information from the hidden layers, performs computations, and provides the final result.\n", + "\n", + "
\n", + " \"ann2\"\n", + "
\n", + "\n", + "Depending on the type of computation, there are many different kinds of layers in an artificial neural network. The simplest type of layer is the linear layer. After receiving the input, the linear layer computes the weighted sum of the inputs, adds a bias term, and produces the output. This can be expressed by the formula:\n", + "$$\n", + "\\sum_{\\mathrm{i=1}}^{\\mathrm{n}}\\mathrm{W_i}*\\mathrm{X_i}+\\mathrm{b}。\\tag{1}\n", + "$$\n", + "This dot product and summation are linear operations. If more layers are added but only linear operations are used, adding layers will have no effect, as they can be equivalently reduced to a single linear transformation due to the commutative and associative properties. Therefore, we need to add a nonlinear **activation function** to increase the expressive power of the model. The activation function determines whether a neuron is activated. Only activated neurons will have (non-zero) outputs, which is analogous to biological neurons. There are various types of activation functions to choose from, depending on the task. In the neural network implementation in this tutorial, we will use ReLU (Rectified Linear Unit) and Softmax activation functions, which will be explained in more detail later.\n", + "\n", + "## Workflow of Artificial Neural Networks\n", + "\n", + "Artificial neural networks are data-driven statistical models. We train the model to solve problems not by explicitly writing rules, but by providing training data and allowing the model to learn the solution method.\n", + "\n", + "Specifically, we provide a dataset that specifies the input-output relationship, run the model to obtain its current output, and use the **loss function** to compute the difference between the model’s output and the correct output. Then, using the **backpropagation** method, we compute the partial derivatives of the parameters (mainly weights $W$ and biases $b$) layer by layer to obtain the magnitude and direction of parameter optimization. Finally, the **optimizer** is used to optimize the parameters so that the model’s output is as close as possible to the standard output provided by the dataset." + ] + }, + { + "cell_type": "markdown", + "id": "c7262143", + "metadata": {}, + "source": [ + "# Building Your First Artificial Neural Network\n", + "\n", + "Here, we will use `brainstate` to write code to construct a 3-layer Multilayer Perceptron (MLP) for a handwritten digit recognition (MNIST) task as an example.\n", + "\n", + "We will input handwritten digit images into the constructed MLP and have the network output the digit represented by each image.\n", + "\n", + "
\n", + " \"mnist\n", + "
\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "0825d40e", + "metadata": {}, + "outputs": [], + "source": [ + "import jax.numpy as jnp\n", + "import numpy as np\n", + "from datasets import load_dataset\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import brainstate as bst\n", + "from braintools.metric import softmax_cross_entropy_with_integer_labels" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "8bf694f8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'0.1.0'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bst.__version__" + ] + }, + { + "cell_type": "markdown", + "id": "d67a35ad", + "metadata": {}, + "source": [ + "## Preparing the Dataset\n", + "\n", + "First, prepare the dataset. The dataset provides many corresponding \"input-output\" samples. In this task, the input is the image of the handwritten digit, and the output is the label of the digit.\n", + "\n", + "The dataset can be divided into a training set and a test set (with no overlap between the two). The model is trained on the training set, adjusting the model's parameters, while the test set is used to evaluate the model's performance without updating parameters." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "e37df3d9", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset('mnist')\n", + "X_train = np.array(np.stack(dataset['train']['image']), dtype=np.uint8)\n", + "X_test = np.array(np.stack(dataset['test']['image']), dtype=np.uint8)\n", + "X_train = (X_train > 0).astype(jnp.float32)\n", + "X_test = (X_test > 0).astype(jnp.float32)\n", + "Y_train = np.array(dataset['train']['label'], dtype=np.int32)\n", + "Y_test = np.array(dataset['test']['label'], dtype=np.int32)" + ] + }, + { + "cell_type": "markdown", + "id": "0cc63f9d", + "metadata": {}, + "source": [ + "The MNIST training set contains 60,000 samples, and the test set contains 10,000 samples. Each sample is a $28 \\times 28$ grayscale image, and the output is a single label." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d3150ef6", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAARYAAAEUCAYAAADuhRlEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAASvUlEQVR4nO3dX2hU6RnH8d/ETU51NzPZqMlkMEmz3XaFSi2I2mCRgsE/BWl2vSjbXlgoLu6OC7r0D7nQdKGQ1oVebCvsRalSqFqERtmFCm7USCFJaaqI3SWoK3W2ycSuNGdMNGNI3l7s7rSjMX+f8ZyJ3w88F3POm5nHd5wfZ857ZibinHMCAEMlQTcAYOEhWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmHsq6AYeNDExof7+fpWXlysSiQTdDoDPOOd0584dJRIJlZRMc0ziCuQ3v/mNq6+vd57nuXXr1rmenp4Z/V0qlXKSKIoKaaVSqWlfxwUJluPHj7uysjL3u9/9zv3jH/9wu3btchUVFW5wcHDavx0aGgp84iiKenQNDQ1N+zouSLCsW7fOJZPJ3O3x8XGXSCRcW1vbtH/r+37gE0dR1KPL9/1pX8fmJ2/v37+v3t5eNTU15baVlJSoqalJXV1dD43PZrPKZDJ5BaC4mQfLJ598ovHxcVVXV+dtr66uVjqdfmh8W1ubYrFYrmpra61bAvCYBb7c3NLSIt/3c5VKpYJuCcA8mS83L1u2TIsWLdLg4GDe9sHBQcXj8YfGe54nz/Os2wAQIPMjlrKyMq1Zs0YdHR25bRMTE+ro6FBjY6P1wwEIo3kt/zzC8ePHned57siRI+6DDz5wr7zyiquoqHDpdHrav2VViKLCXTNZFSrIlbff/e539e9//1sHDhxQOp3W17/+dZ0+ffqhE7oAFqaIc+H6Mu1MJqNYLBZ0GwAewfd9RaPRKccEvioEYOEhWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmDMPlp/97GeKRCJ5tXLlSuuHwWPinKNCUsXkqULc6Ve/+lW9//77/3uQpwryMABCqiCv+KeeekrxeLwQdw2gCBTkHMvVq1eVSCT03HPP6fvf/75u3rxZiIcBEFIRZ/zm7c9//rOGh4f1wgsvaGBgQG+++ab+9a9/6cqVKyovL39ofDabVTabzd3OZDKqra21bAnzUGzv7ReySCQSdAuSJN/3FY1Gpx7kCuw///mPi0aj7re//e2k+1tbW50kKqSF8Aj6/8Ln5fv+tL0WfLm5oqJCX/nKV3Tt2rVJ97e0tMj3/VylUqlCtwSgwAoeLMPDw7p+/bpqamom3e95nqLRaF4BKG7mq0I/+tGPtH37dtXX16u/v1+tra1atGiRXn75ZeuHWvAc5zdQpMyD5eOPP9bLL7+s27dva/ny5frmN7+p7u5uLV++3PqhAISU+arQfGUyGcVisaDbCIWQPTUIWDGtCvFZIQDmCBYA5ggWAOYIFgDmCBYA5ggWAOb4opQAsZyM/xeW5WQLHLEAMEewADBHsAAwR7AAMEewADBHsAAwR7AAMMd1LFjwFtL1IcWCIxYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYA5riOJUDTXV/xpHxfC/Ow8HDEAsAcwQLAHMECwBzBAsAcwQLAHMECwBzBAsAc17GE2EK4vsPiu1D4PpXiM+sjlgsXLmj79u1KJBKKRCI6efJk3n7nnA4cOKCamhotXrxYTU1Nunr1qlW/AIrArINlZGREq1ev1qFDhybdf/DgQb399tt655131NPTo6efflpbtmzR6OjovJsFUCTcPEhy7e3tudsTExMuHo+7t956K7dtaGjIeZ7njh07NqP79H3fSaJmUMUg6Dmi7Mv3/Wmfd9OTtzdu3FA6nVZTU1NuWywW0/r169XV1TXp32SzWWUymbwCUNxMgyWdTkuSqqur87ZXV1fn9j2ora1NsVgsV7W1tZYtAQhA4MvNLS0t8n0/V6lUKuiWAMyTabDE43FJ0uDgYN72wcHB3L4HeZ6naDSaVwCKm2mwNDQ0KB6Pq6OjI7ctk8mop6dHjY2Nlg8FIMRmfYHc8PCwrl27lrt948YNXbp0SZWVlaqrq9PevXv185//XF/+8pfV0NCg/fv3K5FIqLm52bJvyObCMVfgi+xmcv9cALcAzXb58Ny5c5MuQe3cudM59+mS8/79+111dbXzPM9t2rTJ9fX1zfj+WW5+vBUGQc8BNbuayXJz5LMnNjQymYxisVjQbTwxwvD0c8RSXHzfn/ZcaOCrQgAWHoIFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOX6w7AkXhh9FK/Rj8Onpx48jFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmuI4FUwrDdS7zNZMeudbFFkcsAMwRLADMESwAzBEsAMwRLADMESwAzBEsAMwRLADMzTpYLly4oO3btyuRSCgSiejkyZN5+3/wgx8oEonk1datW636Rcg8+Fw/WMXCOTdlYXZmHSwjIyNavXq1Dh069MgxW7du1cDAQK6OHTs2ryYBFJdZX9K/bds2bdu2bcoxnucpHo/PuSkAxa0g51jOnz+vqqoqvfDCC3r11Vd1+/btR47NZrPKZDJ5BaC4mQfL1q1b9fvf/14dHR365S9/qc7OTm3btk3j4+OTjm9ra1MsFstVbW2tdUsAHrOIm8eZqUgkovb2djU3Nz9yzEcffaQvfelLev/997Vp06aH9mezWWWz2dztTCZDuCwgC+XEZzGdiC403/cVjUanHFPw5ebnnntOy5Yt07Vr1ybd73meotFoXgEobgUPlo8//li3b99WTU1NoR8KQEjMelVoeHg47+jjxo0bunTpkiorK1VZWak333xTO3bsUDwe1/Xr1/WTn/xEzz//vLZs2WLaOIrDTN5CLJS3S/g/bpbOnTvnJD1UO3fudHfv3nWbN292y5cvd6Wlpa6+vt7t2rXLpdPpGd+/7/uT3j+1cKsYBD1HYSrf96edr3mdvC2ETCajWCwWdBt4jEL2X3BSnLz9n1CcvAXw5CFYAJgjWACYI1gAmCNYAJjjB8sQuPmuuDyOVaXpHoNVo3wcsQAwR7AAMEewADBHsAAwR7AAMEewADBHsAAwx3UsKKhi+OQy7HHEAsAcwQLAHMECwBzBAsAcwQLAHMECwBzBAsAcwQLAHBfIYUpc4PYpvshpdjhiAWCOYAFgjmABYI5gAWCOYAFgjmABYI5gAWBuVsHS1tamtWvXqry8XFVVVWpublZfX1/emNHRUSWTSS1dulTPPPOMduzYocHBQdOmMTPOuXnXkyISiUxZmJ1ZBUtnZ6eSyaS6u7t15swZjY2NafPmzRoZGcmN2bdvn959912dOHFCnZ2d6u/v10svvWTeOIAQc/Nw69YtJ8l1dnY655wbGhpypaWl7sSJE7kxH374oZPkurq6ZnSfvu87SZRBYeaCfq6KqXzfn3Y+53WOxfd9SVJlZaUkqbe3V2NjY2pqasqNWblyperq6tTV1TXpfWSzWWUymbwCUNzmHCwTExPau3evNmzYoFWrVkmS0um0ysrKVFFRkTe2urpa6XR60vtpa2tTLBbLVW1t7VxbAhAScw6WZDKpK1eu6Pjx4/NqoKWlRb7v5yqVSs3r/gAEb06fbt6zZ4/ee+89XbhwQStWrMhtj8fjun//voaGhvKOWgYHBxWPxye9L8/z5HneXNoAEFKzOmJxzmnPnj1qb2/X2bNn1dDQkLd/zZo1Ki0tVUdHR25bX1+fbt68qcbGRpuOAYTerI5Yksmkjh49qlOnTqm8vDx33iQWi2nx4sWKxWL64Q9/qDfeeEOVlZWKRqN6/fXX1djYqG984xsF+QcsZO4Juo6kkLgOJQAWS3KHDx/Ojbl375577bXX3LPPPuuWLFniXnzxRTcwMDDjx2C5+X8FG0E/jwutZrLcHPls4kMjk8koFosF3UYohOypKVocsdjyfV/RaHTKMXxWCIA5ggWAOYIFgDmCBYA5ggWAOX5XqEBY0bHDqk7x4YgFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYA5LpB7BC5ws8HFbU8mjlgAmCNYAJgjWACYI1gAmCNYAJgjWACYI1gAmOM6FkyJ61AwFxyxADBHsAAwR7AAMEewADBHsAAwR7AAMEewADA3q2Bpa2vT2rVrVV5erqqqKjU3N6uvry9vzLe+9S1FIpG82r17t2nTj8OD/4YntYC5mFWwdHZ2KplMqru7W2fOnNHY2Jg2b96skZGRvHG7du3SwMBArg4ePGjaNIBwm9WVt6dPn867feTIEVVVVam3t1cbN27MbV+yZIni8bhNhwCKzrzOsfi+L0mqrKzM2/6HP/xBy5Yt06pVq9TS0qK7d+/O52EAFJk5f1ZoYmJCe/fu1YYNG7Rq1arc9u9973uqr69XIpHQ5cuX9dOf/lR9fX3605/+NOn9ZLNZZbPZ3O1MJjPXlgCEhZuj3bt3u/r6epdKpaYc19HR4SS5a9euTbq/tbXVSaIoqkjK9/1p82FOwZJMJt2KFSvcRx99NO3Y4eFhJ8mdPn160v2jo6PO9/1cpVKpwCeOoqhH10yCZVZvhZxzev3119Xe3q7z58+roaFh2r+5dOmSJKmmpmbS/Z7nyfO82bQBIORmFSzJZFJHjx7VqVOnVF5ernQ6LUmKxWJavHixrl+/rqNHj+rb3/62li5dqsuXL2vfvn3auHGjvva1rxXkHwAghGbzFkiPODQ6fPiwc865mzdvuo0bN7rKykrneZ57/vnn3Y9//OMZHTp9zvf9wA/1KIp6dM3k9Rz5LDBCI5PJKBaLBd0GgEfwfV/RaHTKMXxWCIA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYA5ggWAOYIFgDmCBYC50AVLyD5sDeABM3mNhi5Y7ty5E3QLAKYwk9do6L6PZWJiQv39/SovL1ckElEmk1Ftba1SqdS03wGBqTGXNp7UeXTO6c6dO0okEiopmfqYZM4//1EoJSUlWrFixUPbo9HoE/UkFhJzaeNJnMeZfglb6N4KASh+BAsAc6EPFs/z1Nrayk+EGGAubTCP0wvdyVsAxS/0RywAig/BAsAcwQLAHMECwFzog+XQoUP64he/qC984Qtav369/vrXvwbdUuhduHBB27dvVyKRUCQS0cmTJ/P2O+d04MAB1dTUaPHixWpqatLVq1eDaTbE2tratHbtWpWXl6uqqkrNzc3q6+vLGzM6OqpkMqmlS5fqmWee0Y4dOzQ4OBhQx+ER6mD54x//qDfeeEOtra36+9//rtWrV2vLli26detW0K2F2sjIiFavXq1Dhw5Nuv/gwYN6++239c4776inp0dPP/20tmzZotHR0cfcabh1dnYqmUyqu7tbZ86c0djYmDZv3qyRkZHcmH379undd9/ViRMn1NnZqf7+fr300ksBdh0Ss/lR+Mdt3bp1LplM5m6Pj4+7RCLh2traAuyquEhy7e3tudsTExMuHo+7t956K7dtaGjIeZ7njh07FkCHxePWrVtOkuvs7HTOfTpvpaWl7sSJE7kxH374oZPkurq6gmozFEJ7xHL//n319vaqqakpt62kpERNTU3q6uoKsLPiduPGDaXT6bx5jcViWr9+PfM6Dd/3JUmVlZWSpN7eXo2NjeXN5cqVK1VXV/fEz2Vog+WTTz7R+Pi4qqur87ZXV1crnU4H1FXx+3zumNfZmZiY0N69e7VhwwatWrVK0qdzWVZWpoqKiryxzGUIP90MhFEymdSVK1f0l7/8JehWikJoj1iWLVumRYsWPXSGfXBwUPF4PKCuit/nc8e8ztyePXv03nvv6dy5c3lf6RGPx3X//n0NDQ3ljWcuQxwsZWVlWrNmjTo6OnLbJiYm1NHRocbGxgA7K24NDQ2Kx+N585rJZNTT08O8PsA5pz179qi9vV1nz55VQ0ND3v41a9aotLQ0by77+vp08+ZN5jLos8dTOX78uPM8zx05csR98MEH7pVXXnEVFRUunU4H3Vqo3blzx128eNFdvHjRSXK/+tWv3MWLF90///lP55xzv/jFL1xFRYU7deqUu3z5svvOd77jGhoa3L179wLuPFxeffVVF4vF3Pnz593AwECu7t69mxuze/duV1dX586ePev+9re/ucbGRtfY2Bhg1+EQ6mBxzrlf//rXrq6uzpWVlbl169a57u7uoFsKvXPnzjlJD9XOnTudc58uOe/fv99VV1c7z/Pcpk2bXF9fX7BNh9BkcyjJHT58ODfm3r177rXXXnPPPvusW7JkiXvxxRfdwMBAcE2HBF+bAMBcaM+xACheBAsAcwQLAHMECwBzBAsAcwQLAHMECwBzBAsAcwQLAHMECwBzBAsAcwQLAHP/BdmVef0zDuYfAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5\n" + ] + }, + { + "data": { + "text/plain": [ + "((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plt.figure(figsize=(3, 3))\n", + "plt.imshow(X_train[0], cmap='gray')\n", + "plt.show()\n", + "print(Y_train[0])\n", + "X_train.shape, Y_train.shape, X_test.shape, Y_test.shape" + ] + }, + { + "cell_type": "markdown", + "id": "23747623", + "metadata": {}, + "source": [ + "For convenience, we need to wrap the dataset into a `Dataset` class for unified processing." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5deb09a0", + "metadata": {}, + "outputs": [], + "source": [ + "class Dataset:\n", + " def __init__(self, X, Y, batch_size, shuffle=True):\n", + " self.X = X\n", + " self.Y = Y\n", + " self.batch_size = batch_size\n", + " self.shuffle = shuffle\n", + " self.indices = np.arange(len(X))\n", + " self.current_index = 0\n", + " if self.shuffle:\n", + " np.random.shuffle(self.indices)\n", + "\n", + " def __iter__(self):\n", + " self.current_index = 0\n", + " if self.shuffle:\n", + " np.random.shuffle(self.indices)\n", + " return self\n", + "\n", + " def __next__(self):\n", + " # Check if all samples have been processed\n", + " if self.current_index >= len(self.X):\n", + " raise StopIteration\n", + "\n", + " # Define the start and end of the current batch\n", + " start = self.current_index\n", + " end = start + self.batch_size\n", + " if end > len(self.X):\n", + " end = len(self.X)\n", + " \n", + " # Update current index\n", + " self.current_index = end\n", + "\n", + " # Select batch samples\n", + " batch_indices = self.indices[start:end]\n", + " batch_X = self.X[batch_indices]\n", + " batch_Y = self.Y[batch_indices]\n", + "\n", + " # Ensure batch has consistent shape\n", + " if batch_X.ndim == 1:\n", + " batch_X = np.expand_dims(batch_X, axis=0)\n", + "\n", + " return batch_X, batch_Y" + ] + }, + { + "cell_type": "markdown", + "id": "9c426971", + "metadata": {}, + "source": [ + "During training, the data is generally divided into batches, which are input into the model. The model computes the loss for all samples in a batch and performs gradient backpropagation to optimize the parameters.\n", + "\n", + "Training the entire dataset at once would require excessive GPU memory, and training on just one sample at a time would lead to inefficient parallelization, excessive training time, and each parameter update containing information from only a single sample, which is not ideal for convergence over the entire dataset. Using batches is a good balance. Since the test set does not require parameter updates, a larger batch size can be used, depending on the available GPU memory.\n", + "\n", + "The training set is generally shuffled (`shuffle=True`) to ensure that each iteration of the training process has different sample combinations, which helps the model converge over the entire dataset. Since the test set does not update parameters, shuffling is not required." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "8a44b4d2", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize training and testing datasets\n", + "batch_size = 32\n", + "train_dataset = Dataset(X_train, Y_train, batch_size, shuffle=True)\n", + "test_dataset = Dataset(X_test, Y_test, batch_size, shuffle=False)" + ] + }, + { + "cell_type": "markdown", + "id": "54a36298", + "metadata": {}, + "source": [ + "## Defining the Model Structure\n", + "\n", + "A classic MLP consists of three linear layers: one input layer, one hidden layer, and one output layer.\n", + "\n", + "When defining each linear layer, we need to specify the input and output dimensions of the layer (this is straightforward as the size of $W$ is determined by equation (1)):\n", + "\n", + "- The linear layer only accepts 1D input, so we use the `flatten()` function to convert the 2D image into a 1D vector. Here, $28*28=784$ is the size of the input to the first linear layer.\n", + "- Handwritten digit recognition is a 10-class classification task, so the output of the model should be a 10-dimensional vector, with each dimension representing the probability of each digit. Therefore, the output dimension of the final linear layer is $10$.\n", + "- The hidden layer extracts features from the input, with larger dimensions corresponding to more features and greater expressive power. In this simple task, the hidden layer dimension can be set to a value between $784$ and $10$. If the performance is not satisfactory, the hidden layer dimension can be increased. In more complex tasks, the number of hidden layers can be increased, and the dimensions of hidden layers can exceed the input and output sizes. However, the dimensions of hidden layers generally increase first and then decrease layer by layer.\n", + "- Note that for adjacent layers, the output dimension of the previous layer is the input dimension of the next layer.\n", + "\n", + "
\n", + " \"mnist\n", + "
\n", + "\n", + "As mentioned earlier, activation functions need to be added between linear layers, otherwise the model would be equivalent to a single linear layer. In this example, we use the ReLU (Rectified Linear Unit) activation function, which sets negative values to zero, introducing nonlinearity. The formula for ReLU is:\n", + "$$\n", + "\\text{ReLU}(x) = \\max(0, x)\\tag{2}\n", + "$$\n", + "\n", + "
\n", + " \"relu\"\n", + "
\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "331a3ca7", + "metadata": {}, + "outputs": [], + "source": [ + "# Define MLP model\n", + "class MLP(bst.nn.Module):\n", + " def __init__(self, din, dhidden, dout):\n", + " super().__init__()\n", + " self.linear1 = bst.nn.Linear(din, dhidden) # Define the first linear layer, input dimension is din, output dimension is dhidden \n", + " self.linear2 = bst.nn.Linear(dhidden, dhidden) # Define the second linear layer, input dimension is dhidden, output dimension is dhidden\n", + " self.linear3 = bst.nn.Linear(dhidden, dout) # Define the third linear layer, input dimension is dhidden, output dimension is dout (10 classes for MNIST)\n", + " self.flatten = bst.nn.Flatten(start_axis=1) # Flatten images to 1D\n", + " self.relu = bst.nn.ReLU() # ReLU activation function\n", + "\n", + " def __call__(self, x):\n", + " x = self.flatten(x) # Flatten the input image from 2D to 1D\n", + " x = self.linear1(x) # Pass the flattened input through the first linear layer\n", + " x = self.relu(x) # Alternatively, you can use jax's ReLU function: x = jax.nn.relu(x)\n", + " x = self.linear2(x) # Pass the result through the second linear layer\n", + " x = self.relu(x) # Apply the ReLU activation function\n", + " x = self.linear3(x) # Pass the result through the third linear layer to get the final output\n", + "\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "347dc916", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize model with input, hidden, and output layer sizes\n", + "model = MLP(din=28*28, dhidden=512, dout=10)" + ] + }, + { + "cell_type": "markdown", + "id": "b9c94f0e", + "metadata": {}, + "source": [ + "## Model Optimization\n", + "\n", + "After the artificial neural network receives an image input, it outputs a classification result. We need to compare this result with the ground truth and optimize the parameters to make the predicted class probabilities as close as possible to the true classes.\n", + "\n", + "### Loss Function\n", + "\n", + "During this process, the **loss function** measures the difference between the predicted class probabilities and the true classes. There are many types of loss functions to choose from, depending on the output and task. Here, we use the common cross-entropy loss function for multi-class classification tasks. For a single sample, the formula for the cross-entropy loss is:\n", + "\n", + "$$\n", + "Loss(y_i, \\hat{y_i}) = - \\sum_{i=1}^{N} y_i \\log(\\hat{y_i})\\tag{3}\n", + "$$\n", + "\n", + "where $\\hat{y_i}$ is the predicted probability distribution (the sum of probabilities for all classes is 1), $y_i$ is the true class label (using One-Hot encoding, where only the correct class has a value of 1, and the others are 0), and $N$ is the number of classes. If the model predicts correctly (the probability for the true class is close to 1), the loss will be small; conversely, if the predicted probability for the true class is close to 0, the loss will be large.\n", + "\n", + "Here, the model output provided to the loss function is not directly the probability value (the sum of probabilities is not constrained to 1). This is because the loss function `softmax_cross_entropy_with_integer_labels` in `braintools.metric` automatically applies the softmax activation function to convert the model's output into a probability distribution. The formula is:\n", + "\n", + "$$\n", + "\\sigma(\\mathbf{z})_i = \\frac{e^{z_i}}{\\sum_{j=1}^{K} e^{z_j}}\\tag{4}\n", + "$$\n", + "\n", + "where $\\mathbf{z}$ is the input vector, $z_i$ is the $i$-th element of the input vector, and $K$ is the dimension of the input vector.\n", + "\n", + "At the same time, `softmax_cross_entropy_with_integer_labels` can also automatically convert a 1D true class label into One-Hot encoding.\n", + "\n", + "### Backpropagation Algorithm\n", + "\n", + "**Backpropagation** is the key algorithm used to optimize parameters during neural network training. Its main task is to compute the gradients of each parameter (mainly the weights $W$ and biases $b$) based on the value of the loss function. This algorithm traces the source of the model's prediction error to optimize the parameters. After obtaining the loss value, backpropagation uses the chain rule to compute the partial derivatives of the loss function with respect to each parameter, layer by layer. These partial derivatives (gradients) describe the direction and magnitude of the loss function's change with respect to the parameters and form the basis for optimization.\n", + "\n", + "### Optimizer\n", + "\n", + "The **optimizer** is an algorithm that determines how to update the network's parameters (mainly weights and biases) using the gradients to reduce the loss value. The basic update rule is:\n", + "\n", + "$$\n", + "w=w-\\eta\\cdot\\frac{\\partial L}{\\partial w}\\tag{5}\n", + "$$\n", + "where $w$ is a parameter, $\\eta$ is the learning rate, and $\\frac{\\partial L}{\\partial w}$ is the gradient.\n", + "\n", + "There are many types of optimizers available, and here we choose the commonly used Stochastic Gradient Descent (SGD) optimizer.\n", + "\n", + "Here, we instantiate the model's optimizer and specify which parameters it will update." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "375b3e54", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize optimizer and register model parameters\n", + "optimizer = bst.optim.SGD(lr = 1e-3) # Initialize SGD optimizer with learning rate\n", + "optimizer.register_trainable_weights(model.states(bst.ParamState)) # Register parameters for optimization" + ] + }, + { + "cell_type": "markdown", + "id": "d5ecc46c", + "metadata": {}, + "source": [ + "## Model Training & Testing\n", + "\n", + "During each iteration of training with a batch of data, the training process involves:\n", + "\n", + "- Inputting the data into the model to get the output\n", + "- Computing the loss\n", + "- Computing the gradients\n", + "- Passing the gradients to the optimizer, which updates the parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "68c121df", + "metadata": {}, + "outputs": [], + "source": [ + "# Training step function\n", + "@bst.compile.jit\n", + "def train_step(batch):\n", + " x, y = batch\n", + " # Define loss function\n", + " def loss_fn():\n", + " return softmax_cross_entropy_with_integer_labels(model(x), y).mean()\n", + " \n", + " # Compute gradients of the loss with respect to model parameters\n", + " grads = bst.augment.grad(loss_fn, model.states(bst.ParamState))()\n", + " optimizer.update(grads) # Update parameters using optimizer" + ] + }, + { + "cell_type": "markdown", + "id": "0fe15316", + "metadata": {}, + "source": [ + "During each iteration of testing with a batch of data, the testing process does not require computing gradients or updating parameters, but we may choose to compute the accuracy to reflect the training performance:\n", + "\n", + "- Inputting the data into the model to get the output\n", + "- Computing the loss\n", + "- Computing the accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "9a4df640", + "metadata": {}, + "outputs": [], + "source": [ + "# Testing step function\n", + "@bst.compile.jit\n", + "def test_step(batch):\n", + " x, y = batch\n", + " y_pred = model(x) # Perform forward pass\n", + " loss = softmax_cross_entropy_with_integer_labels(y_pred, y).mean() # Compute loss\n", + " correct = (y_pred.argmax(1) == y).sum() # Count correct predictions\n", + "\n", + " return {'loss': loss, 'correct': correct}" + ] + }, + { + "cell_type": "markdown", + "id": "a0a91dfa", + "metadata": {}, + "source": [ + "The model is typically trained for multiple epochs on the same training set, and after each epoch or several epochs, the performance on the test set is evaluated.\n", + "\n", + "In the following example, as the number of training iterations increases, the training loss decreases, and the test accuracy increases, indicating that we have successfully trained a multilayer perceptron to perform handwritten digit classification." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9964de29", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "epoch: 0, test loss: 1.2870731353759766, test accuracy: 0.8459000587463379\n", + "epoch: 1, test loss: 0.8965848088264465, test accuracy: 0.8851000666618347\n", + "epoch: 2, test loss: 0.7595921754837036, test accuracy: 0.8992000222206116\n", + "epoch: 3, test loss: 0.6877797842025757, test accuracy: 0.9066000580787659\n", + "epoch: 4, test loss: 0.6413018703460693, test accuracy: 0.9117000699043274\n", + "epoch: 5, test loss: 0.6088062524795532, test accuracy: 0.914400041103363\n", + "epoch: 6, test loss: 0.5789325833320618, test accuracy: 0.9187000393867493\n", + "epoch: 7, test loss: 0.5573971271514893, test accuracy: 0.921000063419342\n", + "epoch: 8, test loss: 0.5350563526153564, test accuracy: 0.9230000376701355\n", + "epoch: 9, test loss: 0.5217731595039368, test accuracy: 0.9251000285148621\n" + ] + } + ], + "source": [ + "# Execute training and testing\n", + "total_steps = 20\n", + "for epoch in range(10):\n", + " for step, batch in enumerate(train_dataset):\n", + " train_step(batch) # Perform training step for each batch\n", + "\n", + " # Calculate test loss and accuracy\n", + " test_loss, correct = 0, 0\n", + " for step_, test_ in enumerate(test_dataset):\n", + " logs = test_step(test_)\n", + " test_loss += logs['loss']\n", + " correct += logs['correct']\n", + " test_loss += logs['loss']\n", + " test_loss = test_loss / (step_ + 1)\n", + " test_accuracy = correct / len(X_test)\n", + " print(f\"epoch: {epoch}, test loss: {test_loss}, test accuracy: {test_accuracy}\")" + ] } ], "metadata": { @@ -20,14 +520,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" + "pygments_lexer": "ipython3", + "version": "3.11.10" } }, "nbformat": 4,