From a1b0bd309dc5d047f24dca4c94659d8b716f918b Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 1 Dec 2023 11:28:59 +0800 Subject: [PATCH] [GCP] Support private IPs for GCP (#2819) * Add internal IPs and proxy command * Fix schemas * Add config * vpc_name refactor * format * remove remnant * fix API * filter by proxy command * fix region field * Fix tpu pod * Fix internal IP fetching for TPU VM * remove k80 from the resources unordered tests * Address comments * add comment * Add comment * format * Update docs/source/cloud-setup/cloud-permissions/gcp.rst Co-authored-by: Zongheng Yang * Address comments * improve docs * refactor skypilot_config upload * fix merge issue * fix * Adopt test fixes from #2681 * fix test for quota * longer timeout * Address comments * format --------- Co-authored-by: Zongheng Yang --- .../cloud-setup/cloud-permissions/gcp.rst | 57 ++++++ .../images/screenshots/gcp/cloud-nat.png | Bin 0 -> 64841 bytes docs/source/reference/config.rst | 25 +++ examples/job_queue/job.yaml | 2 +- sky/backends/backend_utils.py | 20 +- sky/backends/cloud_vm_ray_backend.py | 66 ++++--- sky/execution.py | 23 +-- sky/serve/core.py | 23 +-- sky/skylet/events.py | 17 +- sky/skylet/providers/gcp/config.py | 8 +- sky/skypilot_config.py | 26 +-- sky/task.py | 5 +- sky/templates/aws-ray.yml.j2 | 4 +- sky/templates/gcp-ray.yml.j2 | 8 +- sky/templates/sky-serve-controller.yaml.j2 | 4 +- sky/templates/spot-controller.yaml.j2 | 5 +- sky/utils/controller_utils.py | 173 +++++++++++------- sky/utils/remote_cluster_yaml_utils.py | 37 ++++ sky/utils/schemas.py | 73 ++++---- tests/test_config.py | 3 + tests/test_smoke.py | 38 ++-- .../test_multiple_accelerators_unordered.yaml | 2 +- 22 files changed, 381 insertions(+), 238 deletions(-) create mode 100644 docs/source/images/screenshots/gcp/cloud-nat.png create mode 100644 sky/utils/remote_cluster_yaml_utils.py diff --git a/docs/source/cloud-setup/cloud-permissions/gcp.rst b/docs/source/cloud-setup/cloud-permissions/gcp.rst index 379ba7a672a..064aeb7e8d5 100644 --- a/docs/source/cloud-setup/cloud-permissions/gcp.rst +++ b/docs/source/cloud-setup/cloud-permissions/gcp.rst @@ -267,3 +267,60 @@ See details in :ref:`config-yaml`. Example use cases include using a private VP VPC with fine-grained constraints, typically created via Terraform or manually. The custom VPC should contain the :ref:`required firewall rules `. + + +.. _gcp-use-internal-ips: + + +Using Internal IPs +----------------------- +For security reason, users may only want to use internal IPs for SkyPilot instances. +To do so, you can use SkyPilot's global config file ``~/.sky/config.yaml`` to specify the ``gcp.use_internal_ips`` and ``gcp.ssh_proxy_command`` fields (to see the detailed syntax, see :ref:`config-yaml`): + +.. code-block:: yaml + + gcp: + use_internal_ips: true + # VPC with NAT setup, see below + vpc_name: my-vpc-name + ssh_proxy_command: ssh -W %h:%p -o StrictHostKeyChecking=no myself@my.proxy + +The ``gcp.ssh_proxy_command`` field is optional. If SkyPilot is run on a machine that can directly access the internal IPs of the instances, it can be omitted. Otherwise, it should be set to a command that can be used to proxy SSH connections to the internal IPs of the instances. + + +Cloud NAT Setup +~~~~~~~~~~~~~~~~ + +Instances created with internal IPs only on GCP cannot access public internet by default. To make sure SkyPilot can install the dependencies correctly on the instances, +cloud NAT needs to be setup for the VPC (see `GCP's documentation `__ for details). + + +Cloud NAT is a regional resource, so it will need to be created in each region that SkyPilot will be used in. + + +.. image:: ../../images/screenshots/gcp/cloud-nat.png + :width: 80% + :align: center + :alt: GCP Cloud NAT + +To limit SkyPilot to use some specific regions only, you can specify the ``gcp.ssh_proxy_command`` to be a dict mapping from region to the SSH proxy command for that region (see :ref:`config-yaml` for details): + +.. code-block:: yaml + + gcp: + use_internal_ips: true + vpc_name: my-vpc-name + ssh_proxy_command: + us-west1: ssh -W %h:%p -o StrictHostKeyChecking=no myself@my.us-west1.proxy + us-east1: ssh -W %h:%p -o StrictHostKeyChecking=no myself@my.us-west2.proxy + +If proxy is not needed, but the regions need to be limited, you can set the ``gcp.ssh_proxy_command`` to be a dict mapping from region to ``null``: + +.. code-block:: yaml + + gcp: + use_internal_ips: true + vpc_name: my-vpc-name + ssh_proxy_command: + us-west1: null + us-east1: null diff --git a/docs/source/images/screenshots/gcp/cloud-nat.png b/docs/source/images/screenshots/gcp/cloud-nat.png new file mode 100644 index 0000000000000000000000000000000000000000..951056d56598749610e80baeb1ae3cb933dd5e29 GIT binary patch literal 64841 zcmdR#Wn3Ih*64u{GpJK1O7 zyU*_4`|WQiz~Rdv-l{|-}8lEK0t#Xvwnz>@tS`4Iu(St$Yn;!iXb_?^Ny z0Y(G_j4~?;2^Coh2`Uw52XiZ1GX#VWVF^j7YH{=UUrwG1c(uimb3rS}d&sf);+Rxa z63+K zWQr}s!NGBhM(Y_svOy~i6asLbu{wQ!dc~=V7dNP=e!8VkhkQaCA3oVI_&7Ry*7lZG zk0+rMnPlswtSDdhx5l5|Rw-f_uigfOlqzCH)hJb|Pe2oM+U6YhPaI zfM_a_<$FgN3@yTwWI_G!v|hY&9z=c-9cm#QVPs)qfzu96PZz>JIaXYf3XRUJHLInO zNeLD`s*Xy1CYp+m@ZKWdt_Kz2rx4=W(%TG$@4{AOZAyJl2zU(7NXL*H+i{Z-8FP8) z*HD^c!=H&Ugd#f`;;jX|j75Ayg<6bYBTCJHLe`hA)kP_e`uvRK(L=sLMHQF#I^Iu(+iqjr2d73U+-M}mN# zpQ6+wl?KPOE1vR0Nqxu?J6YH9UW$HGag7HIV024XsAl~Fs^QCh)ed?uo{&?hdi9G( zjeeg_F4`;TF0v>4wkQzU)$N+wWhC-tyX-HyqKU{ne z|48xst81id%9PBVf;z2E?EKbqQxeZ-x3~@IKAHE?;#tWm%?xMpOamx#%-cfS=-YAl zpZm4La=J9MOVskFa$P0%YBzWdEkNS&a#tvdVh<`e5jR7=et z@1i8+^9U!SCmO9f_pSG__s93;r;DbPiw5@?8qw1wlnat3VkS?<+J?qbI(|?mSNKUk zM3Qj=tcRvfCvYuaT5(w}%|NsJ$1bN8CYN&Oi*GeCOU$ykl^JDhl4hhzG&rp|m(9=3 zUkwWZE&_Op?Xt*ATXelM-IK0Eqf4XPqBSj=Mm*Cs8eR2Rm(`Y;8!cQzE`NDZc!{?# zds$rRvrUmONekqBbqZiFE-Rt@$vU}Ov|1u!ZD@^a7V9u(csdtr*~g{8v1B}F#Almp z!8L0cq?$hd%;YQkBwsf$)v4f2hTu~iahx07Fx{vkw*r5DL%v&n%n*ZlhPnJu;jn#* z$#BY!)sEn>H(` z3=ez`%nxR(qRe|oSGby7u06-xQ7)i>YA-{Bc@p+^@*(!*q=@7_@lY$|r= z-#(w8?&lu7sz1oISGTva4_VF_WlyI~uJTuEYYGsY^(|{!3S9rTR?#bQ{QIQ;s`%Px z-)y)0Mqrn_kHaFbGP+#fdl4w#Ox7Y&sW;C!e*|El^v|Tpv^U^hWmuJ4rT0Ve>-Tf> z>wJuU9DG=O#)c?{n2L1l?8+yk#B9Mg#_wph7DVN&BB^3FHfb1PDC@YW3$$Nb8g`)! zA`S@^{~;m9K=10ZyU8|L{?@2%sB;7}RfGFPtKJS8ema6?-L(Z#Ep(anP5Z zn|dVyfZ^}vWN$k9+yxv@1(=|?f4s9vx zY_9Pjh2^2Vlon%or!39ZF1P-fx1Haq`o-S8(POad&!!$$O&U_T4lHI}q-}Z+`=sPL z5|&peFD2iUsKunyX3I9qdeiQF>AZ3tO?byRQ@pFwuifB5SpDAVbFX#h^z$Or^ZN_y z8Qj_2l3VUox;Cb#hlS)CyXOMWJ)b{_K%Z1JeyK6MYi=5`TRx^77s2p9)OTp}gWa7- z-Dekd^K@@_8^{CSYWP<^{N5<2z>+0vluwuQuuIMUHMN$l z_NAR?o7>q8<2LqoQypGie{DbGS4QnxGP~9K2OBfF?W6Z6g~8+tf-@)3`na}>9+on_ zt0p&hqLar;nR>{I{!@$MG~M(ztQ=N(BfLo2(sqGxA$lZM64;H}MvhPZeHJvU`^&Ag zM7Y?uz8bqUZW@bR%P|-7Yjix(tLUx;>oR!5WG(q{E+Uh_uhm6p7ubuw6LuHZL+`=$ zku2iJ&wMk*&f#~%=ZW=+D>Lmg(L8mvJqE@8{H}6OYSUI%lX(`Q78oOu>BGW;6D;=r zZY;OD{=FN~ArO<%)kd}x@tf|S5lO7KdIGEc`$FFH59hxbOEt91ILdeoTpDMlw&$jy zMVn<&%Sy|Nv*~4Zm-Uy`S0j7EE}a)W4XFTL*5!?*Mt8w2_mlKrf;Y!3^^9%3Z8!IR zcMrTcm6VZyA#aeU+NI{U(=EXf{d8N((+riz_srvqy#p`Ovr8k9vZw8niPbjjmqIy0 zjFeMk#lH48_fxT}Dx(p%tSI{72GOnW?$AR zG4bnAXRK!~(U+qmLX*O$^xf%Rscp0gXU0X)yr&4zC>iyUI?ngrVE?fIWmA>BUF50_ z0i_mYj})c>U#B7%n`z3LD=H$qhhL*1AR*!-JcC~$!haD$tPqg@agBgL2S39XzTblp zP~m5M_@By8r2k2MR{HbV|6C*fgr^~heUgxsg`Yo}IGdT-yI4B7T4n_&A|N0MTd8Tf zYAPxSm^j$67@0a4o3Q}x9RG+Q2muA)mv&~ZMpQsMTYDD)pfJrpo)Cav|GCXdL-mhG zTy2DDG!<2-BpjU0sCZb|SlDO)7*teLLe8e<0v{!%|0xdtPngEi)zwjemDR(;gT;fB z#lhKvm7SlTpOuY+m4kyB{sgm&r@gBYklEga_ODF-Cm%^O7ZYbIM^`Hcd#XSA8W}sd zxeC+J{3+;v|NK=?GoaPKO0swPXS3i9Wc`!E%Fe>Z`hOC0wKD&|i2X_VOY9%*`l~ph zKbZ-rSOLv!wIr?V;97-O4Zy|0DfEwG{ww8Q75z){vx}Ltgo7Pi&=v46v;0%|?}`5@ z_>Uqr|5YSE7sua={9DT3B>yx)K-t*}UYgM#9Rk>eSpVzZKc5$3{iEQ&Df};M{^Ks( zoB#|V*8kl#02n+t0tyHSq6o5*VroFd!}RBApQXq`3>usA@NjJ_2%5UV$FB}Y?aMJ2 zN25oN#k;`ZSA`kgLY5T$ zw?mE^5qmL?nN{#E*4k4x<138UO!LLGNFgnw2*0 z7wT4}a@(j6aE%Jz=nfn$Fd)%T{rjO(i`KWPT{}`|&G_#LR7fN;qG0Y#r4$UN|4cE9 z_yV$-7g#MU`IoZ((wgaOcE4-sWoa zpXZp60#R{l>sI|10Z;cXWPntP0ZAb9Zq| z;m-W{-d-)<*WnGIa{Rt@tE)+!lbB(;C- zE;^bNyRdDiyg0g1jVMvS!l;#vu&(t?-FF}G!zxj)!DvN@@#Pz4*`29&#LQWe`uPPd zUUpr;^V}46RJVEv#$>&W>djZIS}lk7?z2V6aB7mpP-qD}ZRHs50}whXBA+4nrCE{9 zZ96w+JyYD^mzT&9-2PK6u-Im{bf(_PL~tXN_TKmUxamYYhC=u-P8m>fv+6HGyI+|# z8eWj$H6S97Td2A|a5d~gW34RmP;FN|F74h&9nz(K(Ne6fFU6P^tgQQku{5D*W+Aph z-gZ8md#1sSyGVocz`}9(N6j@P`EA7m*-WV=Gnq!1Z_FwMj~au)NUg1gu+LSde}m8d zv@?vi@UU$(xo(1PRQnsVBJ^Y4N6*EngiQmD>%75P&~(M9Qc0m!&A@@67dW`aW*F1^ z+-|kGu*Ndkjj1+Hga0NhR@s~rQdaFx&*auyYLBI>7Z_A&zcZszD#>C#s8L#xwV8Bh z!d0mAPDe|Jr_dPdpmXD8rxo#+}X^R>rR{5Fr> z2*zTgFILeJ=~0+%z94Vh>rUnc^}2Av%KbsN_Q$Ydx48E``~`<+>UF`JYn7VSt~H~q zwh%9y!+P%D8vSKIxcw6p(;Aipnp||;mK*f;OJzo6Qu99>&u*=5R+h?uq^D3~^bgCz z)F8Y+S{V{uyo)4#)P{v`mg34aZDxuqoHl!I?Fiqim75R$(Apk2D6``kla`+X*o^%Q zm@fQ0cwzvbyowCkT1V;ciG~G>H{&+Pu(v!jhqP;~%_a+!>$V4zARi{Or6OLN4So9+ zs^NHa3A8Ns9|@A6HT5TN?R)Y)?R?#Ib(wg*SsrwKPE>`DG3$5|{BUunpm43QI3Ubg zFrsB8WW#n`5Y1fJbNqZJ2=Z)Db(6wok?Ejeh%?rNNwaccReqfU2WG!iTsU1?@z@Fh zK>Sv{Hc8c44%B1JOSWH7$KKD}nDo;G>quaZQv`N;I!Xqs%@= zub202vJev5yfG4!wqHVZpQ9t7_h`1413^@6Y*>;u1%{Gi63IbrG(|Kf0 zI~8(lYII-8QpC5XcT$AZK=)e}O<+;(`Wsbuk+o60h1O*Ts~u%eTeF_meXvPmt8C2S z^fjNh8#?zQ#@SF={-7JiS@+YZ#$A9(0dpND1<3)K(Y9WWLy1jCdbP4d(^j83fW-5& zqxMaA(V~pU#UNvvW7~b0)AeMzRO^}arbFGZ&3p@__8NRtC3bO?b(6o))WQ>nezNDH z;l-2mdU{C7y*G@N(fIgdU6juTvtu6O?X@DgN%rB*w}`{S0{XceqM90V!IG1oL#%DP zPFA=E7emOaVq!VO2NPMr%aDD|7VB%JFR+yiYXgCpsbxrYWlLS@=A{1VsiuDvl-23! z3Q&?a?dYj{wgs(cHrJ~wJ|dfbg4yOb(eYgDnX_t|S*-9bV9!^SUJ2CXtwN?*E#?`I zS-qAoADl{;VLLAcYi*u2ozE3xGmdoI7!04QB8%ldZh?BFVxp@eqAh<^n$XCsbC`CA z6Ea4ANHUa!RNKx^DDka>l5rm%VJ8n4T)F6(_s3(@eUGkb(t?Px1eqRlW!BS$fO9&p zrE+E;eUD|g>lqz~`S^FAz=d;=yd?d7L2?tuaL;v0dTsAgf|8>&prHLIN^R6Z^Y6{$ zjsS^Q3lW*4)##j+8PvbSNspH;e>4jbY-SQrQ@BzD(wWPsxfgK_A@

5T7Lwt8_h9++ z@nTe@)^?tQjK?;4*>zH}_URGk9Ya9P33}tPKlRQ%5|Zh^a&s<4c5b_ zHii`f1Kl2m*Cs1t-cbC6zWWsoR73aY*%R2(GOympvTTJjBv`4nc<+H|Al> z&UoZOe4hIg?iIyazqH?eZ};s@_1N?8)gp(89r{{hp0MXp-+>D_4}a+$SX|&+3|cNH z3F@bt0=Xy~%A*_@9yf&8LYtcmGL2_5r-a5?&}V}x@xO-FH{PY3U7tR;)5|zG8Q+Zt zdQ$)`3YRxP2JaVf{is))R+Be`v$;%h7P3mqPP${Fs(9N{>f}*>_oDuySiN>I|De0kWr^Qm#(4G{ z+hiu}NMNRXHBGeFkE~(N|D1AGN(S<_jDo-Jb>_hi0nvgiD&nBU>$yGv)G1o{W>29B zcBt<6WC0tkmUesZ$&No*j?9$nth-eJY&xngCipBVqp6iZD}?O*Pgwlkb+5s?vQe}NuxDMO_ zaYhwGYVvGvT9V4|Zm+=aej_V{VxNM>#;3Bvr1y>z$xCij=t3bA+u`-bZFU3;$39UF zV_EYArmG(n+B0Jx7Gqvw8d5}wzPD~-TeQ0T{;D~Ck$Fau97u^3Lwb#_>?1Pet_Q>u z2b{i0u4uSB!&k%(KvLM%0fd%(qfV;LrG>3kC^AOs)E;hJ9_Ot-O^#ZpD9MxOJkENK z$2mRReNb%yp@ZD*{h!Jt1lvcrz{(S<&ufG$F^+!4y&)uIp zB*x0e+e_SjqtE)X`dHJ>YaQb#o}psPr{||oWos0ddvqdzd&)Q2{$#XELkl93Q1N;^ z7_oZ`!NP#h0v%;#ATpYx^u08;X|0P{n^mR1#b{r_aQCcOxV~TXoCPat8td3K1MWSx zw#)!q->?ag0o%Q3CL*4h($~{#w>&d?fuRP?vhhhK30$MM7LsTNV)tziLFL6q+Y~s3t z$1b5Myx!rq?CbpPAhH=6Y7XF_`_YytPZ@x8RzJ}AlQjn_$V8EDQ>N)SwA9?7am*ta#c;`aU+r&uNd1~H^oMW#jJ5C@zs|<9$wr!y zE&>y9%|m4mZ6IMw@U5FMn2?ptgvHX5z&%V=I@){H_vRKo_g?O1HIjtWM+?_qwM=a; z8Z~lrEu~vtuFQx=!d2h*YFeL)MilsgfNs9RM1~b1(4tW}6VSsCUL3sb$_cWRHl#v3 zZAOs*Z78m|PBP&<+f>1gM$_*?zJ~9Tyj7j&&60waoqgO7W`>h8DKy1HcHf91h+NN^ ztYAi2{L%MiOYUVn;Qq98g+Qi{eZ{a#m>hd=hFu1+nuZxkmp2ks&V1cVxi;MrvDAfz z5bV~)4{6Pwq$!iYsvp6v0>T=k8}j#{R9=Xi3&X?b=fza48HBE;&k={+9{Xrxh*x%9 zGyEG8WEL-!5LjoWvEJ)&-VjAPMV09;uK4G%zvPyD870aj(Bl|SKyu>W)|x}0Dp}8Z z-Ez^jj>YE)YkY*sjKIQGdb|^(hjpso>#W%6B~xy8WJ?Q|+XoHa&uZS?ZVgJxKGBpa9NS+adsA@UfMW9AaZ^vAXxSgQ0^z2^@vrO^5lmrM*I|TdQ4=yEJaOcWi$<7@!K@zZnJ(7Z5 z7xOdYlq(~Timxxw`T6*O2eL$y`;kv0 zl`e^e0xP7VmqPygYaSyoGG*(~i~dmc$nu|L_VpbR#nE#xHu8> zkV75$lX&8&;j`TddUxvGcg{7gIdu<4 z(R*du?L*mnB7ggB1zG_VQcwUCzd7~#*o^W~OWJ2Y)?+sIg){v!;#Y;k=hHrIZhMLGGQZH zk+Y9;N#1SFZ#ay;rFO|C-Xfnr4kq2Yji^2|N1&S`To1BfTfvPd)_+Ob)Pehfn7ZP3 zBns~a*2()p;1^t7&`ZuoEaew*SZiW0;?O~1VW%&pqe8em%PIAz*l3-J{qYpZ5=rDk zW#VF2mq_nJW$38aaL$~D*n6QOT?TVW6f%6{@}4~;7NAb3GP1Z(jHH?1?N;0o+%I)~ zlDsVw$Q}c7INyxx1M`O}ol6M_^{)EC>}-kjMZ_TpNCMWO9^!GglA#dw-rZL15i-Qm z0O;wmt4q6}vZhZq$@>Og+)~hXhz)zX4~b2bXT|QEIYacJuk|Rcz>@yH$O#-V+oy?9g!Hkr2(bNmVF4M2aq?NZM~YO^GAn` z&7WqZ3hi*x+a5H{=cO#WqC&R)Dpy@^(vy_SX!0H0P(kJX@IM%W)CKmOL2&e`?a_cEKO;;UqcboqBjzr!|4rMl(1UkMV9^i92? z=Y|Zu?~XmsOUP)>k`E3WpyN*0`X^UyB?`f>(%Ft%>++%fDfft120i!;24in~o4)O` zdZkG&YH8>O$5JoFpCyHn4vB@16ZaC3KAswK^l%FLFlb$y70sXOgR;>T`tFE zU?v>YcqS*fMpI90YloOI7Rt#~xn@;@h_C2P&3SOP{_~K-EqoHa8 zVI5&T{Qdz_P*Wj;`xyD`O=?||gTXE(S2Q;JWVe2%%V=qd5rVvJ-El#(0Rq5p5y*Q5 zvrHXwg6$HwJ-j)%m->PGI}`W#b4kiK90De%`$S!Cl;;H@X%!!y8#M?s@}b0q;BCYh znuDuPiS!Xcyy}U(NIwg<=({sKx)nJonF7FsK~6li74`=Gcfg|UI_K7(g;DH zsoZSF*b+M01$C~GPNA~b%~{5<4CaFrzv ztPYQ8vdsP>BYj8C6X}wlP(1zN;x>^ctvL3cv0|bUZx^SK?izBcr-$n z4O-0`JwH3fn?|fi*&<>Q1q-CCK{Sk>qTr+4jzM|UUbuhQm){b^`HA}=a6RuWathm^ z8aV;AuN5qpGwxd2zU?WmWaaiZ4*6Tn%2@}gi~IQ~UdZkIwEmY_Id}p-0V$R^+xo*&Bo(<3eT-? z0EGnhG9VyFT9dJdSYF7@lUqn2ln!jSM=rU9cRWD(H5P@LnicplFvwB=dfsJJxYPP6 z456n;>X*HM^zPeU<4CIYB(7pSkKd89Sx^%Ns_Y;AcFEQmD4uX<#r>m?-~Czd?@HOm zD6h!_StyI=hTGdG(^n&iE7L$$S69l3bu79|J`jdXolMfFY4#sVgPydantUcvd$DM< zPJIDAs3*;|&F@L=32W=@7H14}bZ;czNis*;nHewv3ezB{!>>Fq z^;lkT{N|VuA-5zeIU8u6Ap3QXxzwa(!zr4pOPMUtW9{`5Mq<@+uEN}f{Sp;c%kEHl z7BbM1L>{_wWB)7YU>ZUmJQqLDcqBuFLK=Ax%l64U$P8sTiI&qiI<<8v+No;7>qN_x zXR+Mjp5BBq$Fs`Ug&rm6gugqr_eHNV-wC-cP&*JLHwSHsZGNZ=VZpe1{-hkF% z2s!|ItmX}la+6SF*TSbR2JL<`*WpzsT)!z!RV3+*!=wj&a`=9^aCr_vYt20d%Rrw%<5n@-zq zZzF@SucMokPMl2wE;b&pV2O#!;IFsL7M5r(ug6xx{PQ`qC?$fwo{_ekMq2m_90gWn z9BJP39Iv}BeggXE+Wd!hBgeMx!w#mt=1@qBXh_Ek!< zQU>+#JSz8D`iIDl?T5+^{AH8$C6`2};+q!kY2}YHGNY*Wp{O*8=&3zTkkaBy{7@lh zd55=?B_z~MkuFp(c&&#_2PIke66ntru2sg zfo#DEs9XfX?9Xhb(A?hICTjV+5M&yO^vs_1O0si3eu_GvO-pfHK8==*7VD$U#%E>I zH*YoybvVb!Sa)RdGsQ*QBhHEgS+-qH$Y)3tqWbfp4!t4cru(rW6C#_<*2Pc31Fp!C zPF6ilgS+Vddb!fgZ8VAjdz5{;k{!isbJDrSm(C(NS_!jc8GNU99oI?GDSVrEHPMPJ zQu=$ro;!j)+#Pq8ANUV8$ta8EXTJ+GGH`CrgsB^f|ALB$2Q|F%Y%3|YL}z<987RC6 z6dQ--$-2jRUy0YeCp^r;Trl;P^{rNm>@C#n2&!mct2S)Dh|?$nwaGsTjwl1b$^Fl2 z2LXvSQJ5`KLvYuWiI8a@uTPIM!&m5rZ*H@D@J*2ZMzqKT_lCW2P99NpR@#}T-o`*A z-rdsVus*CI6&S8zeCjDym;d;XWow~pM#(GBMo@A3Bh4l=eXZLj^FqhlJ-4GT(}cG9 z+I^FgvMo0qCt!Ht$DKbiR~+|qlk>!{YeeUp>9s>_1E5dM5?5mM#%HAfkM15n9|j{Z znv;G39(PX?hwQ9IAd#d7y&l~qZyG&-GD?@;=mlf?vyQK)I20N^wP5~%uhpPun-5s) zLq<=PmtmTDEucPAaXPxZ$Z$1Hwl{&-+R%s*fuTnu;mmOh`}^8v$@_7X@SQeOvgf%_j&?WeVn-w zJ8jg13QL`)vZ2M$j(Z@zMBjUTLDHYZCk!UFdObepEbPrfK+Ln!F)H&H48gp?^OBy) zXL5;aPBg_yD}d|GtcKg2RM=%?f7eAQeWEmL#+2bu_@KhCcnI9#b+Mihis(f`LO`}s zB-fdSYH|6QdaHv^nJ(9e<||tk-g-ZL4o)ARzXh}7Eyt(mOVI^@8T(@ps|c(7PS-jB z@4lvQa#&$kz5T@7+{4s)4UgU4Q0zQhFS0Hkmap^qI1jU-#F}gvMMN8zU5hq5OeLkH zvL~jHB+o87Qmlm%(}PmYK!EJfo9*uqJ&)yWDQrjzM@QP&mg~VTj&}H?3EgrwM$Lx) zdPY`U@BxuBBS9MbB9HJQC_ChIna6Gc>Jnu^nL`i)Lk}3Z>Y7udo6Zglgb|#GNCWw; z>QNfhJ>$n?#nzme$NB@E((zh{Y!^X;y!K0c?kAP$V;;@zVN7MC;f|vd)DoeN%}&1f z`^v~Dn~gcrkhu(&Mw&L&7vs%xoz9Ka?PqRLN((RBn0@GThxk`~=4>^S!ZJfCgfc_R z?}=d>dWz=qOYV_9cKHiPywviF8jrht+X^M}vaB9H-mF@ln0M)A{n4M)X9=SHrY8w} zE)hA8@!dqXP)V7l)o zMp$BNf`kBK&&FnbnVl>xghUmpKC#`si!9pAwrEAnOS()M!~Us-AlefiM}To~hT#(k zQ3C^2iG-5S@OuvMTO*PZg8|vxmICAcM%*p|;HinvHIkVOIvNXpX~q zj3V$L4i@CiH~v7RShNkMZAYd)GbzD1u?$M$tlHessQy@}mYr~fHly&WWC{_0UlPmU zgW1;kJGzTgTfH9ndZ96T`Gn>Tk2Cc_WEEYdu?8!5@6buLH4t}6u}Yr54j-UX{IONgtIQRq%a?(aaRbPwrs=~Wx5Rs z{pSxUOx2uTufb(5f;ez5+K;lt3+XQu*Z0?z1ulB)VmNUg+v=4|$IDCCv)9?Qr*~K6 z;m3vo-_b(bQ3woyFa0aOiy=6?;mp0DCkU7j7tI3aZ7UQ~{1+G_hyI~Hae|=8I_!UV zpnrlvqE^&Y>)P5Qx%<@r3C85b!@-z$!WD&y|6*GHl8S$Yber+Hz8XPASn_8vE<>qS z&>)7v^ryeELqAXdpg2I=gsT4(4i!bjg#$tep#$Q7d;Tx(2rl)Z{uPDJ_J2`2G%5&i z;0O~TyYfHrB{+*DgoIxj)T_=P@gI%*g7y*tj_@FT`Ni~~l#(biMhv`p>lL3z{!>|0 zNY6##KokPid(8i&o^S)ev~YF(zirk)6dA9*9pYBb|DlqSKUL7;bqnZt>?8Tp}+9A=|D0``f%Sch8tsU?4bR> zllI^pNipR6#6SD;->mcJVHC+ zuv$D=NOGV#iAl>C9@o$t;0mg7n#h&C9u;|dGvqf@tmS^%fx5WS9YJ8PTiB%37W>O9I-vmMe;{6i%=6DRIP z%YHM(%YTod$-P!{wwhS-gzckq;!@W1*iu)0SDMmgyIh%ND2g6hjf5e~P|3V}x(Z zWmbtruBJXc(2?$NTTg{5tFNsZcpq`wtTZ4Q3-TW8i^csdR9`_n~7aK510W7T)oea;B= z52`W!8@@?;xd3+oxh(ny);?X$8q8!#ga|>-;3(5V10=)dPi%{*1Wjh33GTsckU7#e zxM>c(cEx}bUZj~Pm1cv^Z~WnYD_?rCU0jS&b-r^S>+$R$;ivnH3K(gYFC0=N*@lOr z6r;TD;PJ5Q(QdVyoh|q_euBP0t4&86PRX6OIMT3qTmN|AH-F&2Jul^Xyg&3E=6mY* z*Q@(w#G5(AaIJ?)6Z;!^U`ucGne#{*e~{M*6z^oE#XH($D2;#eddX!pUO1NDUAyjU zh#&lZoZZUW)#V5_35RLmg(?aS?%Q(f34>bq`LDZ$wu3fFBXyrQ!$$`_4@&A5=reur zVT`upa@!U|$woK;s38yb-FK-kD2_BSN=~P@4>zkve_*e334eH(4K67BSwc0}W6*@w zvVmJt-mse;VVs?Ca$7=4{@fRKeKoWS=RuwXKeQazZ(>(6JMkEIp$Wom?~6({i$QU+ zXx~(#vWR*c&|Y;_H43Qme|nh37P)s6JS^{gfx|dcpiH@n-D&c8z3AYLC|`|v^jd2L zjY=D@Xr1+N_{JvM&N6{9vC1Jhm(mmWdbR*9UJ}XdV?Bk}ZWf?9pG_GJOdRmq;S$Lo zj+iAXu{z$jf+pTzMuaP;W}e&|uEI;Zo&O9A75#{0V;&>l$ZY$8Uh#vDS|+1LDEk3T2b4gI?R$OsiM)kiuHwdvJGn zr2p2U{gsX_9sz?8tL5CfRhJD1@YL`-giVML9#Cw)>08hGsY=X!zG{}FSFG3M4o7v8 zv`(W{{IU(O1$L?0SVzUgnF;1E_a?DeO%`kI?BJ32D&7@fC%t@u-<$j0oEV?ovp<0( zVfB#L&Y=#vAtuxVd;mA$^FCT%{$Hp~UijZC?*3gM83%TNK8JY|Hd?GI$Er|vBL8NCM;B2nZR{o!-K*#!9Qxd>4iq&FtR*r;BOZr zbRJ4X56J2Bj}L5gw6CxI6GQD&znJ|bU_1Zr^KC1115@y{{aFhyZp`~x=WH7y2i6e=dYVcIi@WH`*XKZT>y1b%U81=F48n?dX>|`vG@mSx-VO zqo!?XzZ8_OGHr1D*%m1;#`Fme@gZ0ld@gy**9!`Q=QCk|)hvRy78 zYL}g@HxJ1!6_GiTMN?julK8Y@)a)i1_<2rHUW*HtbFfY$4}AWeV)HQYxg^2n2Ut!t z@DCgw{t#4Qj7BezV%feq4SagIY0*e@Zia)_!KYC8j=iP59=>Du?flWhy4Oj}-&2Yg zzEp11sd=W%w~l=BE^9sb%c39b=mE;og_(_iLMZ;F$D2Xk)}qc4{%#~$&*IW2DU!8# zl@&nTORM`mm|=6}lsGt;CIh?YWZCy-c?A;;09tKCDRFmsz^!Jt)*j#v$IOUFQ9zOc zSF|Es^k#c=6**yh!8w6?L06MXqTT(u-T;(>$|pbd`b%Ri0^Fm=5DAIL|y^=d{3 zK#BaqOfG~HDZOH76aXtGQUpA^S*^C3Z2ls>dq%{K%n8oCf2R?IL3qD|_zVT8Oey|+ zy|?CzszqrCxyRCY5<$h6_B_9|Pso@Q;*lfBLMk9hg2cv)5kYUn9-ecVt;^kU+I2o% zkbwT$)8kE>pDk5Z)A~Im$gc2Qm9C3$TSA4PQI=jQ+s0#WB2WMR)3w?^211zS7YQMe zTGUPLwY%=thie_mQu5MgJ+97nG)rhskk@Y--Np5R^bPcQuu#VU!7{ z?0YZ1klnC*I%97fB-lUZrt)|s=smUVxsSQm9Ci<;>I5zj9_Arl_`V92LEho+<*9@` zO)e!e8Q3`ZrPkQ(g>&;*Zhc%VcKh7)w3J`Ffb}z7z#;)Fvl>jXH$F5n*H$V#;}Ol? z%CoY=tZygs&@{zzsJ@!f3Z$LwsF)dX+yE|OPaif6@>K3%y$XwIyqOGFd5AZc+qRbd zvnPFg0}yBx#y{@6Dp+--POt5;tI2FcTJkbL;f=~0aDud~tB}q8&3EMJ;1?oHQKa1G zErnUC;v{>wX0KleZ{4d*z(|GIxR5N@Dkw!>M<&Rc0eSX(w_{>c3}0tT(~A7KY;HRR zV;X&`9aS^S(gmtpLeDpqOlSpAqA7KIl*WGwz}E^UaWrFdSXhjtdq5n^`f<#bS$*$Y zX&(mUDEDR+_=e~ziqp>rDwY%`DijcISIfzmw4u6Ld&_ku_-fokNL`_k;1T?WnrjZX zdMN@8ONd8-<}2R7KrWZ0`dUJkSOh9I4h;Rj3!IHL>s+R4UdKE*BQRs{ym-p-2;o@X=$e<&Qw z3V`(sqBR`V!3*>gmalj4rILJ+A2n-qfDHb4G!cFl*d?Nee>(@h@Nza|RryWe_61d% z_H~CS?JGQ~&L6Q7T`RleIV}j{q4w|(EY#cl9;%_#t)BVcDRKh*y6|M6jU6;A3}j}a z<-yM1P+HLjqj(njl0rtJ45v~Kg74U`ouQ}et>emoP)$h!3ic6^O*6bIH2nc7)B!k5 zTnt}MJ|>SJyxAJ1F9!0sC^0-k5=nb%E?4{v01<9tdIz4bsIu(&wPy0v}$FT$7={`H)|lgFU(!T@9z%Ej6NO4DdG0KGy$&R zS+C>Z2)136wcZsz2}^$25bh29AcK~;|G3Rd&JrI83&X}2aK_iNzj2sDt?0j?As;L` z{CzHZN|4vCseQcpbPo{$c-F6cJW91BBiEuH2t23VG7_hMbrfdl%{czpP$IEgWB+m9 zeP3<(y*h)1%<;#?umPs5+22xxu(cianF>!coo3r8yOsR#2R>V0dg;adgrnqT8=Zq5 zEpo!w2TOtbS`XTU_fVCdN$d!qQjXcYUDf`~40)Z_LwN11I(PBs962SS*=jFtMj&1k zTb_f{R6HS%*FJ{e)7srky^_>onY`j+o{QT7>;kOmPA|aTB(+$m-sI&8yR4I{bx0y@ zRdBOWD}T_tqYT+*(Zwsl;XAQq<;~{~onbrXcBf(v6%EG!m7KNB5^p@IGsFbC|Fwp` z77{0ohUAdxdwph9eHlEtDGwcMg&iM|{a$W#yNF^Jx3*k5L)Z!7`CRq}i@l37N{+OM z>nB=wR#GaeKolo9!g6VRP7+rJYdy7?>3)pE>UqmKsC(SQ9S?B+QF!>X>mIIbm4|Vcg!x<%Bf9|=C&4bJb~~_A z$n0#f=+LqGTYi0{w5N|mawQebsoL0Bwo?$ASiJEPXDA!!WjZe!E;6!oXBsRD7A@^8 zU}E0D>ak!&-{ud+Q8u_Q{>i0@2UxUEg%XM@T!Mjhmd|iw9B629NT)jlUr(sKB25-s ztH@9p9%E0if5?u~0WpkfJTlPIL`8r3HQq2&__=8JeUV;Mtq70tbJgGvGAidwlE5>c zu8DObW*pdQv$V>`%w|n4%6-DCHK{sk^?KDG)9sZs3-LjAh~Z7-H-P z%PM94x%~H(vjVdQY{-# zx41tO-}T`VHoCgsaVg!+U0c{{{%9c(wwnvyq->FCpbrf-)G0iOu|P;)9*4d2I^1<` zykR{;6mXqjhm-jO8SimQYfMLfsKMSYN+Y9^({$_}=4@staflbJ4?ifdyoI;|2M`U2J5s*~jYkiQSGnf@M*Te#$TikaPi zsUn;nup}mDa_~^zO6)SfL?~}<+-UuI3_XResWwa~AD#BY(r0VVrOPu#J8K8Hym#*c z!qKQaQY|3IxnY+g^`kSPYKbJm1TleuBfkj(=~1;hNrM;pS0dTsR6{Dr*&*oPWsu)7 z@xbB8qoCUQuv-lL($7bx$#7Pxg(@Wm1)e56CkyKdVX;I|gGxgzgr{hqXHpX+90g0z zuy-G&$3_R^363kuA?*3zIz%vqG@A+dS>8PKs7Bv)jSmoiDPhGXsH?NQ(T%gwk-GBF zTI2rM+7@cu$`*hwEU`(Zi^bqOA~Q%KeTd`(V6>MDU^ zhnHf%sWiL$2d?Ce-3`*dd!^g)Fa(SsMx+edoX^aHbyd&y%i;2Tb^wk*sDO{wR%|xK zYr5}F{qz}ot34dzf#?@}#6tUL_9SFjS92TCpwvtQ(E`p3Nieux&ZA|c?-v-GET`u= zJuFWZD5sua1afC+Iv0;53DG_Wi)l4iiYLVzn6u-lG<_ys{(IgPh^|W z`$PlR)auD4xvW&i_W2v7qRA;XAyF@v8nOG5t4Pw_&MRVGxWx@!3X^6czN22P567&Y z{Q0`S#J!_a)&W}l+)l4wZoW~@OBJyt{=jNs;9}|50QoBpQ@o@Jj`yPdsqgi@>*Vvt zvmW)!ibSI^`PBO#iJ%uhxW*{)IdRy1&t#@9rPw1`A)IKx>-=SI#Gj!NITwLjy4n@U?VKM}VPU{PIukmp@0d)=V@@kh zlb3BE$^JA-lFuh_Ik+NiHwOJ#n1cSe5mcFP!yGw8?uYYsa-#C`!`3FS*>Zec8mx+x zt3k^KvU624h4vPokoK?EwW*&eiafuBFD-R1gOteHAxW50!V3WiwdP(I18S*~7 zv`otp(USLtG{esD$Y)=jJ`S}W@+2>!$hc^izR9tGBstlLhkRXy)yxj1KUCWfMv-iP zzKBG$ifM7pWpHeC&9QLOT8FZYb4T)1H~2Yh%wir*SpTHQu8XA}p~>hz?A*aE72(Od zL|JWK_fmY>jqRkj+_l>L;zATQxDb7q@MDHf0-}-ms>2xSZn$4)eBu{6BmnngWwr@m>vSs$}0KT*Jt+X=T~TQ8|Bm|K-%P%BcJ_hrY%?9OZRn1n5S&$% z+uAR3Hc1v*D|yLw|Ne7V%sVPRY?h#lpjcn>7v{xmfusYIQibG-2<#Z>G!O%G=rf92 zcDi%m)elRK7bjckIV2y+GQ!aye`UV8)JDdhtk3{43RXL&S&a_nS$+dfXl@~HA>I7* zANmFlgY+ZQOtg7dSf_}gy#~Br*qzabke|2aO5Nez$mQ~=+nyn9VR#?k3SlX$@eCYh zjc&dqGJ%v?q+sSq6>>&GY@a?3h+TA-`q3^B$t#nr=astnt|LPp638bdkv9mFhDB0N zuDwDyUiIP{yyrjqc|z!q8ySrnoKGrL{|s1|DA{MXKpUW|iaRp-<(espaO4q&Q%UfJ zYbAe~wQ^#NIb5IE5Bv>T&B;!^Mn&}0p&HEXPct-d83;7Nefpwj_JhHPhoUzUWf_Nr zpJ{b&dG@LJA|P#7HYgXQ@k9;}MoXX9Gy@RV5HO?HT<;vbon$dK9UFZ` zJl_8L)O`|G__~y%6e&9XCpeQuGxG@J5euc}5Y42!wU6Sz=$tg&9x#KIx2Wa6## zn>$-)I8Z5B%(3(&)?Co3+t;0M2>!CWImt`?!{hVb4-=HMJyUE`!~7i9K&p7ExvOpl z&zV62?XDmal12#|!C+2B{DuS78Pg#>8-C(6%bM?rbv$GAlPENtG70Rpr45*ytmw3Z z*7N~@9q}I)0DRw%=dn8xbgz)w*}1^Z>$#NB=VcVVbs!=_`|3GacSjltUOQaskAuS8 z=wlRe;?~o?6QQpKiSzG>FZot2^hYtJ?0AgCdILzX_+GG+=ZXiUkO+529_Pvggz0K| z)LLYb#c9YJh1|Kv{#KTzU_F&{EdF4^eY0ETJxMO8iie`c#}1L(6zj>2Fbm04^{Bq8 zFcK!{lV6FvwU9aK5UwW=&Z7ActE)jk6G(_T)Paf~Men0+zNl!Y(Mu_``bfB--ukS# zqW17;f9j=PQ4gI zDA^q`_r93x=w4Dy3y$y{2a13;JBE#%7150o<8s%F7Mj~5nJ6x(EaW$a4TTMzxVuSO zvDY_9e#1SjUZqutO_0U6`aCD?eZnEk2|aQn-V4vr^3Z&#ZX`S2tG2+GFMOm^{dhD& z9CB{BJkr6-!qTQVnownzRhLt7tv~XTOhC;v?CU&lXU=rzGs_X*cvr1V+3=>$M6(7d zLydzciG4Rfp+uhx6N|Fex)@bts;{1b`(ub{-JE0cPG0YjP6Rae7`F$C@qX zsmsaVtjI^+H#qkv(+*#<_7wE)xb>xmyYO65(PIuT8ZYyu+P~d^~g~!oy0&&2POOUB{OrPCfP{0wuAB`>jO6N&u z>pXO%rjEm;$t~w2Mt{?w#hDFk8C6e`;X498m#WwsIa+-fB*08FC(&J^!1SvWhCSe; zcJ@{oYVDP}kb|zP)bCu}iU<0%^eDXWc9q?dl07s{<)Jg8g4Exl(FZB!oJYLX*XvL3 z*T*kI7nrujp4l_zkcSj@Mmr1+ zYfvVn9IEPyh*XP9u|o4gv!O#dge4+UrFzxrnmB%Oq90jiB45*BrIW!0o{h@I{zsW# zjRLClJmmEm4_xHdO3aZ9Bni}G20ts~q_ZZ2ta`gGKOIT{Z~Y%vK3pCy%N?tAfneMIp0z(0pLF~Dm^SMks9 zB)oq9xIvmwcH>X=hfD<84e*=S=fUaAbswUMxQ8d6bq6PnAaOHEU z`O}L+#oX|q>($=)y{3!#Dj;IMnK=TfaCeYNb*puGdb9?Fo(iDLy^&CrPGYFAYq=YL zP)tgsS6i24vz}xoWHA(tK7%1yUMuGtYY957%k0e5l|>Qr^UQd7|CccX=Y`wpnya(!T?a*rWqS|4U1GuDiJZMCxNVM1JF3>D&Lg43R;$e33p-C8lZIdeCDvi z>0bdo{wifT?k+~?p)dgHBoTS;Ry0%SN4%*mKWM%g0)nC-4Czt=@B2I+lU!H3%_GW(+jVz*g`sc<&Cc|I5k zqp~Ez=Lcx~!_;}*R~I`I!xK9|QKqM+Z&DhTNV?>rn%EnrnJAn?kns(HAK^4}NtSb}GsXNp zz)k)47)h1M8bF)?emsc0MGn3PW`< zD0GfOo6KSDQ>ikQC1lnYsuh6|GY}W3`cp>0jNwgk1)dV!1XsXEYR8)bbE&5`!wyew-Jd8ri@N8&^XB7mfHV$L5pK1f7yCYN-_^lJ`Q%4E+=omc>}FyTBsVfUm`CYbEgx*l6pD{lwn2I!sT z1n*7*dq&^?@1F1<3EV0~z63Ap335assX3F=jm7 zQkN(iECNzd@tteVIbO&2?(Clb`;2v!Y%O9u+O|wW;6`Ft!X&Y%Y4npH*uaN!IIlmy z(TAtXSJ)eQF$hL!)l^?(>v{?lGihRrIXc;)1`==bk0JAyXM>xp#^?Pk+;_3V;&qP3 zobyg4mv#pLbIPz%ME8fD=L6M+d-OP*=y<7y3*eDK(4zy&5y3e1x^{~^^Y z6XwUJP*XUjf7;mmShpNdXu^I?JB!k9A+oG9eOkJc0sc!ppr?p7VsG<=jSF`b4FIFh z-#6m$SfOl)c0~CtPXe&5eFhFGS$A^r zU!YB$Y?3OiFV^}savQL1eyfhzT)eTI0;wVCM84sf_v8KlAt>V+q>%YWbse@guN z*TfORfcu>O8WFDq$8S_zD4q@b9$$=^f6A^wHm$1i5LGoi`d76N74JbTH6epOTbQp# zv6EE2F;cjvRAEHlf66CVU8=#m^4%_#3T*WC1^4q@1(RtD1B|l)3CFWBd(*9@e6G1( zW30+o$z41fJMg%Na)j+oA0u*h|MO%=DYF!o*x$YW>kUy-#D2?pt>OK`(Hp3eHU}j; z>O0gb#bO-!QwtbgJ`sXn^^|8J+>+xF^i$tT}<72yd-UJ#YmSG!@wablb znoheI2n8H0YHw`?zKTs7bX=`16v|P}14(l7{bo)IkCCtUX5Dx9gP@T+txioLU zjU{7QLA#lse(fcDm8y0xGww9ZT~b_yIc8MASa&{HG;K$LGlMcLU(Osd{?U`mW!dZN zpdNJk^PeK)e%0a{kklRGz=GzDb5pe%&O&}hpIKkeVsrjH>LD2aX8!y@VOY;fg+95? zc7OhTPk6Yemj*D?nF8lwOO5T^bYHUG0(Vs}bsDVXA^O5Ma{+W2SSex+ww0f?&FtP+ zzzpd9+^EUpDesl0g&~$+GVm}A=;4Br!||5d8~7jX;l~9nf<$0L=slVrSnwxt7}U{j zkaX2O@?i4qD=M8@>(n_An>+t4FSs_Ts!oq)|J8;(|GI{L`rA#(1x}5H=9q`8m8D2G zqf}$*m=5Hy_oj5%=zChRW$*DBSsR1IkBoXZk&RxR2dBFRJT|eC&dj8&>ecnB}#D!8%7`&&b2o)?YPwdh>cEU#k{~KfSpgetIst zuvP2a<ao>?2&dGpRR-*{$^m8p!#yH?*( zbJzqUxi=l`y*|JB$#9^WW3*RiL&Q6b_!b&RGFurociAp_V>N)md$zAAaNw+`6%fK1 z_%AWq+{P53|qO=!@Nh=3-2TZ~5RJ#C=V^ zIiz0gfbVeVLl1MrPMnF=ZoJ;c)R8C0DO3v6P7iEee-d#O6B)c}^2YHG&f=46tBh=k z4Cqx&o07J!=(%b`<}*wKB$zl$-wj>KWmsLu?oBj*wh??O>!ss1^CNR3&@ADo_o9WZ zYonJN+9&(HMmB_i(_;OF=S3rTMRdq}#?@>8jx>UK58)2Mdy~wj`!NR3vxb-Bqb_*X zbIrjn_PPfk)}>$9oR00-a)VRTw83%DM|rLLVHT5!cdQ`@33nO*y4G_AA@?7O)kjgHS{h(50zG8k`mq6*&XgF|W+|M_GGNrY(FIgCu_{hWmC8$i`n_)@cCQPe?! zmDdQ2s|t{Q(Yt?}04#p4M~NCAq-V?2a$CuSI&VU5aX#m!_XQSw689eF z-gx!Ee>>6Q>$~zcZ-aXt@Jx6Cbkj)jt+Tcd@GlCqKN~&=I3YW*=CSxZ`5EJ3Z!O^7 z4XBUPlLg{Y1er6nGqyXZHRdDap1`(X0|0zJzCK78VyKEqK6lqww;J=NX*RnbpS&!8 zflY^JqEl%c_K?w5`}$Hv;*~>S26!sL&(C3WF~B#2m4ViF zeO*C=+f1o5O6_h`wX4JYfb(R2Jat0A@Vc!dE#4iwv!Zj&9go*14ktL&#~D8xct%vl zGI&M3Sn0pHKy+0iNeSi;nh7FfpeygecMxA zeHI*&ias_=aH6gT73dToyzYz98%F$^&W+~jD=-u6e8UM2xDQYsU>Ci6`(Uo|cBojR z%BvZS?^IMpac%PVn0wNhubw}42dB0xM_})a5U{azCtF05;?s-a-G9sU*XDiv5UZEI)}qLlJ$n{hul%iq%3w(_fY$0u7M6k$*{i} z&#*`4iJGqaSHXF(McE*H*bbX~SPVdo<;>$;O>Hw>5RMvOp4|KZ8mLOWVfaCSUad9O-{7GqHF25 zn$nY)6KNs@j)Iq=7r^cpn0Jlkrz8!68GkZ0Rb=49lO8pVZQXv=`kt)o(Yr~6>f3UA zEIq1|icA%_0fW=xp-OvrER{9x^DLx!15@?2t_PH)4X`i-1jSM8#C)vRp(Buz1t@nsG>^I9<;L2R(0mt+hN>Hp+Gp| zG6T${>xxuDt|CL5bUfqZT$Q4U>4JduD|_uxYF-@O1?-|qs3U~3BV{=7saypsfs(X0 zqg$p*bXcePAD(EslM-m1*RO^aZk*2W;$%4_)Mx=U|DBz9WnDUwX586^I*0i_wWY&vTE5 ztMyH3oFhanws(Vo%XYUZfbSy5FxcV7W~R0VvbqbI~;~rA&!9_ONx1=5Nvc0XFl?I z^i;kAO#D}e@D$y&^t#H$Vi7pwMQThBVdg;+35vh-Wo^J|HOgU;Z;>cS(A_Y51v7@x zW$D$T*jE`gv!PRqcHcj#(aPIFJ!uPE(Djie+H@NA?&1DccSC{TUstVWv`>1(WLB_s zL3HCH=_L^l`zy{8c^;ln@@2fkt`j1YPpZ`KDdF%X^}UP8o*~@s&6k%(_ zD%|2qv7xFU375u~93?xdHM;;vYV#G-sL@WbA4yX}a>ienHZtcDRW>!JY+Q>|9S?MV&Uq0;n#q1$c%Th<% z?9tMk9*XY46@o?17cg{2SgAFe?|&e=j-bo#m89`pD;V(bP!DSpH?B_>_$yZuX?-|j zILvyMO93}iY9!0s6=gvU_efE5IH+)Ct1HumY)=%24*-P_u0Z02eO?erS-{70 z6@ikoA1#uJue@0ek~D#X*3Em+Yhj$D;PA`AxJ-Kpe0&KI^i+J}rNlHcuP1DQw7SaB zkypuwGv+k!*vm-MkFE_9qM&HuRc6EIUqqKhZ9wd1$}}Jt5$*>7U{)B5UXZo|Hc7Ao zRdMPNJP$^@Jr48_#PO*6xG^|XJaD6{UYC|Bqf9Y(;4?BkaF2Y2ojEOdbHB$RU}5_C z6Yqo<&c>>}U;NRECy&#ua+L85E2_Z1T-=4;G|iZ{j!%u}K?UgEp!3J0#s@Wq19bCMTH{feoxJt4 zQ5MFI*_(|@y|iN$l|-~(^Z>%yHbeU@ZSbVeWKhhIi*7^kP)E36vci%8qw%F7O^W}7 zOAa9+KF8#`DX14UOzVv><%qCLc^eK^@|lLaE%HuYVh9hUEZ)scF6?`<9~!a6Cw|Kt zPnAQ7l#5(*_#9^M7+En8Dh+=PXaVXM^BqpwPA53rBUc^TNJd3>(L47$d+iZh2yI3Y zOJhX%+T)ATtxV)RxbsFS&&HGAUEMqC?Ii3i^Xm;y?q-{4EmT>G=B)C8Ul~w#F zhsj&en7z9Iv{etk-4@aNj`;TwdvYURQRiE{B~gnSIr#UOew}X$M?Z+bQ1e+bXjz}$ z1;EMEuc*1B63@d^8|Soj7?VOYr&JYcEs8n~JbRBN;A@=7ppo37?4(%!E>77g!`kyh z-+9Zh$zK=*{gO3ac&Pd%tE)WWsf+#RyBP{0}{!TOf&vZc!ZF{_7JdSu}Wx@$}lTu5D~PJKN;9ZgBQTn1Jpb&LWx?V5291(omn)IgCCjAwLRbH| z`WYRzSLjh`+L+@?O_2wgOv8?xwSgMR#H^ljBzivIJDHOPGT~6J&hYS?-~^VDX6FHx z3VCK~BjH6-ES7eoRMFQuNZD-&ywH-sSZdaGLw6jqz(Fuk2QKqAIU(`kN1`qKcC_y{n0liB$-z^Jc%2BzZ$m}u| zHCDO}gpHdfkiWnPL7oTUCvPBP70FrFY^YK<4d!`TA#orzX8T~hDqp`nQukRqhd(3) zab7FBqf!R`J)7TbW3{x?o(dO`KHz9tV{FLj&10-@S6!x-9~V>EzWy#r{0MZFPl^2!zl>ma&(A46^t1QqqegxBSj#WUg zQQ9waXLzQ@h9m2~q^I;2p_f8K((sciK0(BooHJFr7g@Qn&q-)>5eZQ+RgN-0ME~EZ z?E|T{Y8dkK9J1Mq8Rm9(ZI1a_XWxcl$*}POMT^#%XWBAi9{oEGGx{8&cxT zhLIM{Lxby{9a3++z$GT1rKIcapiS3n41&66K}FLu!fyL`{T_EGnBE&ts>JmijoBtx zM*g(KxD8aBjC<%nZoKW}DrSbvExydnO}XVE6Esf>+`QSrejQl6u*6p$Io15D=`*dM z>W5k7#655hAFfSxNE$Ft>jP5Lu@g8Z5#F%5VjLX-xZ)*C9^_9-~2#Roc72gD;u#V(&4 z85SvU)+=%LJ;hB$NHB$4n;p#BPaX9N1|vyb;$x-A)Rj4#>yC$6|=lu`jU+-gY!c_KGgzCM2%%{8m(IvK)7b zEo_EN;x<@OavAMfa!fpKJHxW$&>;>RkB^o^;!#EGSIl(^V5uf_Y{JF9F#myMn7WRe zDZ(@6zoa1&N>)0`BJ7Rs2+oG*!~9T)ftT6$JE@RlrpU)sHVy8I0`N*aJqD4qTWG42 zR%K1=W|-o!YZGq;h8^K&f(dR7GSU`TQvQ(rF%pUCzb}{C;9Eh@_X_iA!O#(Ts>r!U z21k|H8oY2uRbxt*49i@jwN`!JEGv8=^3(o9wWGF2DdycKu4w9zQyBsn`t5}C-|$uQ1{bJd7Nu4ch_`BsE#|4SZ|(zM#O8ocLau%beg?{ zEJ@KRQXgKP%Sy;pGQly>X|9||C*lrIILCtYIR|EUhn8qHtgK`cz}K7IQ-5y6-beAO-(MDn9ICUcjIf7nPDsl}M;1|iad zYBhPvhO3hm)}B8JPM`6guFI;&{otMu^Pe97H7r7J_BDY#b_9GCv42XI_3WjFxd;t| z!vTX!8NVcZYxQQh@^W-1yx;)mz>akaK1AfHmDf}0zf^c}SdYtuj$2JM5pQ?>D-f1GKWKUAZpJ-p2fd%|0Gi7s1|7kwjjlE~;3&KsOlVsU8fPOkW>+wM99-by zv7}jHDCP-b#e1z!P)}zhVu)=(6<-6>`atlubdn6Tr)YTB5|ph$o2|k{^Pg}GKN)fb zO5%w^a};|(u))HKxCO;|W-WkrADrER+^ltXBt9&FOfsas8szX?R1!e%+jRB8^)6+c z`mK)$`5@(U?i(4L-q)wcA+okv3)-E~?fXnTy2y9+$rsC;NW36MOD1iIkFfpN{<0JaLZ&{B`=Ok_p^@ z38Ww3o-%5uLJ(sNK>yQdsSO-xPmDMb6H$D@RXk?{huxs>GuuKP;*B(e)`x`4SPc_u zX0qjY_t+bOgj!Xy(m!L@V#BrIRrNrzizB}Wj1qglD`eDZqSb#|E~T+m|hc) zfD%wcj&O0CDUnH4Z9-j<>k>Fuua67wx~(jdsdFsJWTN-z5Vw60=32Y zy&FYT9ytOzjsdB9Xn0sgA!!QUq(0oUaMEx;;0gU8dN<=AF6ma_JaIqlG~PJyl1*#( zCE>!wNoq!_CuoYzZtSOf&YyID`|w(<01wz!U|q9$$G*n_&rmAC2b}?F?GlA-VX1Kz z0D~d~%K(SX^q=G%kjd}FQ=1OV`J&l2=K->vVF?D)?BLpjoj>1{{rxkhJydl6ON#}Q zj}_C;Sw)-ecaWHs^wsfN(->C2Ro>*x;YxX&VH8RpXwRwET5FVjq{c3YVD019thL5} zMR1nEnT=4O3d@M{ZWrb0}@>IYhZnt0Dj{bR`k@U7L;A)uf zF(y-&Y0T0Waz&NDq#h+$xPbBT&yJ+O=cqTmo7+4>Li02P|c4t9c?*)jh zvR9&nNW-p10tx^N2CdCp!&pyr4%#xK_Sg5=9dOd7hJk^~+5JQ5?(3<#4x%GeSR3$O=pDpIUVyu5&CG&D)djVv+Y#;7IDWc0kJF=@8+*cu5+KG;2 zmHRK(h9TURr2|T7{AHhl6x}>*u6NTX`HzM;hNitaDy7Y#6b5%XO}!fwN~L6vtv6Ic%_1R0>r0rT-Aep*G7ApwTsm&jEB39Ss6S7sMxgC4MRNC&#B|| zZQf{98v9ogev>JW;?TL+n{kWXaVXb2AdgjrgK+*-d0TSooc_Ph#8?2%5nFD<^B6R1 zX)3P`9vjar;lvSq6}9-9QI98~q&bxgwvfX2^f%Ipbcy6&6DvChgM0cbb^bN|xf6wg z>tBR)bYg=WO$5(v%hYM4I-Q5OHv(~gyZv*41AD~VP$3zt2|#y)TuhLT$5G0RR4x(! z5w`i~_VF0~=Lo!EPQCDDAg0_Q;wSr}wJK3~QPLeu2zU#3nr+!r)yj@4CVdGN62$JE zKq4GXz*0+Pko$N^NLi4#GrU|UJ@`Kp%TGp%l)5*SbuF<-kvz5o!C2HWA2cRuG~wtt zBo%cn;we2b)dA5+k36a<{jEce%J&biJ@v{<<^E?iFq9;vp)w0k?cxZ&kS_p+zEE_( z*3kyg-~5nKA^C~iLKULyl0zHh3wFeoQdY!ib-N#6qj)Wsh4b~jO|%fHg65M5dMht8 zs7@t8~YH!M3O#N^5TfYltV7Z8RTu1l? z7^J3gC7Rc(`#uXNDf9+xJ5XE%={Xjb^J%NHr;V6RbygEYJkFc3=4s7&B#%`!^=)pz zcpGD)fdC3L-km_Z-4=kpRD#<;cy;6;AfG=8*eJIcep^b}bYX~(Gf_cvlwEbN5QG|i zyH-1Sq>`v(d8O-gqhJrv?*j%Nps~R0V6aQy1k1rbSVSU9r8anbb#hm!n@UFE;b=36 zeAAJrIvofXaopEjObScgg6&S?{sCSrjH9Itdfdm(l|eOmz7MFmuM_ejAN37``_Z;0 zm{Frsz$G5A8|o0_U$nW7o0~PkGP`|thE1*sYPz#MU}94(-2|f$`gh#&UL>2`s7K2= zn!~H}#QL0pLZoV}NNonljHaM-%t~!rnZx{UG*=>~c=A+cW*tC)Iw3eGf)xM=LkN5h z^0V&)9>4{154>_xg51uV3W6Gm7EB*dq}||@w0{$QqXXsV&tiBmJdIKL4nKl@a4uga z)esn#f>FSATH>GxiokjE-kdXaSRxY70C*+NX384?MQbRDQD^&k*E@}8=6XI?Uwt0Al);0U3Zsy+6R-+Ol?$-75`%m6^l(25(?Yxo z=KK_JXUdtgyx2(1J`zjjc}!gZ%hmPc$EuEo@f*NQQA{}a$iLIw>m}u=%hMaBW4Xo( z)9k`bU_C?(IZUd@(Q^aheX`fwp>LOMc#41C8~0ec8*y4%E9d z;5zta_6Idd)kwelp#!w#_khbmeo^#p)@UF5fkYkMsC?C<{fccR1mJt9Ziy*s4nU#F zsVl>H_z?A{L~CM!_gUAh#CXLzTNx5#J!v+9&jt{4njXBJ%GAvSMi0A^e z_c11Q3Drx?_5d2mT)tcef%TBXfec;i5l_{wEi7D(Fr2{8!dKX1%w9H&!R*(>W zAhMqD1cvi24y!q)8dZ4FMKlEFk4sDAKb*{3?pam9>viuY%>#Ur=5<(cEhqzwamrur zZh>FogE(_Y?Nen5(W8D2d%vqN@a5g@6bcu=-_kQv2exoiN%`rQNCMe0rON0Oz7 z1jimCxAdcjkTk0C&UCt40R_pcH=>vcW1s@L>3!n+wRkkqdo0G8T`gRq{oI(j5*GV^G>hI zbP&0JN(@)W7+8Osdqnk~Sz$@QQ^g&e63JkI6Gq>H)%7CqC10{G`OX7JmfdTUQ8+7J zZ^0I}Z8GFXU17bVfeUJdTJA7EaZWg>+Z5CNZ+cg;ZA{z45Mx1|c*C=4q`{F34pX|^ z=Y(e)q;P?o%wd5WqRcFQL7B`i-sXmq%fKB%MW$9Co=_z$m@%^inIwh1qmC;KF+&|r zA$1O0;K1xb@TT*AUD`X#bx%U25ihh~JY`+;mv>A_p0qri7%Y(Y`U#4}fdjE8<)RKi z_sS>9I@E838K{BidtSk%Zi=3Ok-%6xoW?ag{Nm)JNo??1oVS;m=>zZ$mA3~W-MA&- z3=R93`YV!GJY)Wu&ri#go(7o_k}@TEf%atG-r$Y(Pylv7syXrW$TmuJ3Ah=JB+E!g z_{cDc(297%-%yNbq?Epp;)mO_lX7F>kA6nYURM||m)K?|CGh)9T@6|T9DRXbViIib z`_I1ukjeYF-vGO10?Zg`b$=FL;!5E$pS1L}Zo|%v5+vS#bRkPY@K%p|>~#|!JsW9r zbo{j4w?6|G!4-ZA1w;RLR+}6vZw^5@a9NqUGq zt>aom66o)lQu5mAIv=Ipbbd_X*!fXc-wSH$nNM=bdvz=5!FAtTn($IrXvdYBbm7hDom*i#wM z`;MB7fYW;O#+;!Ic5eFOyeeSiBBo);>o45>@H&$}ZWG?VX?;raAyg^+X_9A`s9w1? zbjvHZZTZr9%(*hCc29Po`|!&rNs#I+%_ZL2rp9m3mY-+zo1g1 z7Ju9o*DAvjkgNZL=~*RmyChuzT!f=AwaGJk2BZ%C9ROxHVY?Whe7<4ujHHMW=hdEw9}!Pb!EU=@N+*42!XnRQLiLEw-s8BJX2<%|HsDgd6aFVyaz@`393&v6>YD9A zHp~Uc0nG!9_Ef330VZ*MS?Q!BZ&_$f{mJb=y?i!y_!2vMLs#Ej;t)=M`EhcWkbj%t z1zf-errQav?m9s`?6w-*WG2TehT9iwO0{`Y*TR4t{Hy4ua^#wx`c; zgDb{^8Wu_3dsdEC0O^*0lUU4BuypRwpz^T+Im$5Z>`FYwq~V)4iaCL5*8V;Wqbv|9 zFyriFQt(!}Ta6JikxvJ+_vb%tBg}Ol5JnKk3OLfglu) zz?>z9XHE>3_IXQj23b(W-?^IVJ3KZPY7agES!}BK}?ZaGk zmez8-3kKC+$6?@6s!@MiG5UczvECN&+*A6M?Si^;%3MtMH*m1?(Q^ziDE3F5{&ArG zM==)t*>pC`U(6}o^j(!&vy(S8#i7)`9^`{Rf_0%W6 zgOqp|%xL%Wj-RtWEO%kh)w|=;s}7Vc5Lwf#6Sf86E?o%uMhX+yM;K%0wsaeQ)?D&} zB*M4bMK2IzN~SZ7{_wAC^Zdow&H&d>dJ|fZ1d6tbWytkmx-o046#4#RRkiwHq~7sM zZv&gIpjS+}lEaKXzLI&~f#~`UUGIk}S#{s@HB!iy27l38V8iuGpU5?3ytf2!P9bE( z1V)`{EoO)QE*s?JTGG<6$B3vBh9>3q=)3_t1gP(x6(i?En3stM^b@}-(ng%&;Oa3r zzFS6{^#e&yG}H1x@_~-zJ%lHg+s4%R6C4ahZCPROm{~LudWha~RJBWur6~8QQm z`5C=dUYra6v?71xt$~!jKl}xkHgUyIQH*<2Cdh8{A9m~ON6eUkK|MQn6xZs}!N2M@81asIKUdxO$3^G* zTBjdNY07Dk`p>10W=|1|QtFYgE8kAo>w>>k9_)Y`W|8;mq5;%V_w$BpUiViUm@n7r z0+!zoyjJCbnkbR0yM{%q-aGJRasX0AT#Ll5se@WRk^11)!;>Z7fCp|UaryRd(|xx0 zxbZU8Pw1;CP2f{z*97$Ux_O_LLRWYOobae{{J4ju&fAiup55cd>SKge(zeSyaumGe zKb}F(EJ|3zA6jLJe|@^7e*UAy&y&l9QX}Le);qeIIfP9N7*F1|=P0EzL!yc2h6W@O@q>Jynm{1S@iAc{5vm7r_@S znglIs^!f$AxUQq@_D6XAk8OWbRxaHe$s!jkWSg8_J$g)yTn>fzSA&0|T+7nifn>$G zFqN#NrsGqkK3ERN^68`K4Uyx^;zrsLu=&5W7feN?=uUjge~pbbnNC};&;f_*31Q{F z1QHbGsqbSDXC=Rv^F2`hq^mB*BTA`?OKEmnf8}!}#UJBNNOr z3JfzA#dpC5@cpzculL`ij-?j1Dh(Y(Nf}Y9i)11`r-&+{c$YbOq+5z=g{-r zlq50oY`^|!c$FA^j{w89jI{|D*%TQ!qoe=oT(ypem0bNugE$zao$wI#XHWigdNLT| zEM&=CE`~oh)l5*`j$q#u>us~dtS5W6Av@@q=b`MxXM(!cpyXLvtSx{+Q;8b?GzFei zeYpZzOiA;H?LBJ8LqR2LO!54Amb~P?8{YOPp3fj{T^t1NQ$mNNbyPft#vxib7MgO0y*Ot#Z0dZrP!6fRr|2F2H~NH3asQQc5>lm%<)Y`*v;oSGB;n(6oy5O?+ZYuNcob{XS(_umiVhEt ziuMY&(yDrVLrVXR_9o4THIf*$8vCzfv<<)9K?U-03*!HoSp!v>HynO`ed&C8&-s4D zN#K5C-uDa-XGFr=kr!!=UJ~O4`SkzpQibCCA%{8Kofe#(PhYAe3EJ86R;ft!$MYuZ z%vYojYF5r=dYPr7H0IW>xz4<-E1s$DF?};{QqtWeEeHbA-QAti-QAt%{>`=4 zf37*_IOoMad%xRb;24gf!0+k%`Cixcxk6QRpbPUY3UKh3FI$|u3hy5n?@SfQj09e6 z94{WSHbu$XPm=%R{K^+1{DeM9Qy@C~?Qmj-&0KW$+)!Sq{n_qDI}Z!KLdW?vvLfql z=5zqZuF?W$joIY_>+bcqS~iMBC9@zNx9^v|%iVnSXq~|Prog59%XNU+Yq>b}m8E6FVTj0+XD&?@cBsnZ zp{`MU_M#?JF1VoYp7HLYBhT?!2T^0%LTM(~{lkgR)RK_z!+ibZLt@iF08T1{VV)yb zn4H8cKOrW}Xh!)c4Z3>RpV!Zt8Psm2d8vQn8;i}5+P)o5dryxPjvWgeI%}W%I`zCL zy|~9P$mk64ShbLFn9FH_b+`Bfr8>qW_!MtM@DE2aD}$@}j}a8*iaZ<|sqLrz8CANi z>gQ|5@nx%wGGy(|q+1L3?3%?{rmpW7lg!uR^W5rpmsCxzSx4bgl4weYYH*8e4w|fu z+8*Hxn_DUx?gvz6>)q10T-qWXZm(m_mw%-cj(e82(6`)Tr1VY?&fo6O3y%~iYKl*5 zbxqKk#P+Q@PD-^m-stLDRWAq-Zooi^QZ{*x*BcpGnf2q~t@|_$%$Ptgs9@?_?y#Jy zKIk4Wdl0I(+YeC}$I>A35kZv@6}w)aD5X%e4VG!Q!6vcU{`3F{(>`WPiEF~RQlJVK z@6Yv=U20(iiAg+_bNcWK49LuTqOJ^YTSzKM8lKPb*yP@`(Kc#|g|i#dOl$9|AE2yl zUn)*HTJ)3lsK9iJr5JU+JTRucyJ~c6TB21wM3>iCq`n$Uy=}V8|GkuFc9)+P-HPOmCMRGU#)x5FG z`d#paa9Y|*Up6J{2p&r?i0&)SO^-;?-7#L)d9C4wauY({RNx>*l62_^`ETN23Uw;| zcy8?zSWA+ZY5_&NYArM_I$_i{6Jtng`ey%TGkRVDc=R~oER1Q+Jfx;{Vb}c&v9n;kgDqP zcJh2oo~(r9G5C|@4z8*CX!T{2swIm9n-dXr?rPL*y!1J#qsIP_5)@=&7z&H1gNS&;v!EjHwvhQLp-yiS)%(u+73`sz!~!gH%pGa@r}qsL6t!|*Sj|a zI={2*g*Tb1#(u?1DszWh4|dBm>baXPBR<(C$CkIuWPXWLH8W_U-b)eJi)HK>H^sa+ zDY+KK_FQu7V(2j9a4Ma@m5xg2_MT-H{PTJ@|m zyxZ^BieHWrIQ^1G9yb}&-5Sf`cCql?ENk?-z$vm`;^{n|SJ;Q*8MJxvI!LuN7X9gg>!gw}9x4^rw z`5w;JYxG49nAo;}#GxsOgwfxuw7n%~v@2IKPQT*$Iv%z&7<`a0hFHHJ%*e+Et zp_&}9Sru|QIWwiLvuNwWy5>kxG-0rK=B66ZB!TphCMqBr2 ze<@eG2#e^vmajQVrJh6v^f~aC5_&ctmQ~HDqD|JU*ElHU!3ymE8tcXTAS&5f>7uEr z!!)<&wqAO@ugcQxe9-x(F%0AJpfU-TwtVt-ytGQ4s~U!JjHj!kyDalJbi6{z3k%p} zr3X)4g{d6(3ba8yS3-ppH`!#2O8!XBiLr|1vpAByu{4&czO1$d8E}$J?58MN*G6aHWATglqH1``%@5kHr zCogujfP{QU-w$ET4=i|9o6j=WZKk^Hfv8>yXjy9o;e5T-u5gz8pJb_!oC-Z%kV7Gu zrM$ruQmeAE-T$2z4+Jm!hkn_gV#jfl-fv<$Sk?wpW+An&d}+!VVZdLC?tXjN+EWe(yTkNA09~d?hc`kRf-W&JD+uFNwYm%trma;s>1hX zPx?<+$2aa@pWF`*ODaxR$+D=Hv^wu0PFATqowTHC=hM-zMay4I$^TaKs2jIntMkI! zywO|RE};Z%NS#Hqfm!>Bh8rAxI>Lv6Iy6i@B-wE#!IFVRhxn6{X~Q|+4tbEv2_MF~ zcnYgZ9;+7o?JBI6wcY9zPgYBU36gf}?;K_|EpWHEbmy)=`8w#b6>N0|qmf$L<$XXe zHXWj}6r2pK8Z5nAVkF4_a1y6wR$Z^FY`Sx^9W#ErjJ$jQI>uJgDZ^911#>uclUvRB zwQ3GQI(771*%td`+^mMmnqpM-Zbe6!pz`kWwZ($nLwha4#WRYk|}8 z%0pGZ<4q_;y0U1xcUX+q~}ZP5PXD24oAbW zfWYKrV7EW>JK)(s5=&7EQ1u-^z|Dr8VB@PjCX4D$j$%>jC2R{(vb>M!sy!=-I&~qL z{Cx7P$yQ>{GFI1@tm>UOE1pg%=j}=r?80&zZd3+)Q^ij1>toGN@FW)@%-St`qd(=0 zqVG0k)lsu8)iBqd4W9JcFdo)c%f}_QoTyYfaX&r;+rod|q7v#x7s`VbbXrFfa10K% zzc*KKt|Kc}NLcUI27M-)C7_7O8uTnSlX&!lKJui!kyLr}qX8!K`lRYacAkY@o$@5lN4E=RoP(I1 zw^ieI*mEq?$YacP=jG8wEcGcj7N6(d9aMUvFxJ@bDm-Ec7(LuwYY5i@)fi$zV^NcA z^G&#FdbsJcOyzc~El?_HMUI>L+S;Q9mldnj2SLC5R@B^IWynq9!$93vBdaIw;ni*g z+y}Sm+_bz`+FB?|WjEmTQB%2DFYf3gLuIhd>#-eGRm!vR9R@Y!+C0^!oudkRkVf8&%pG0QK zQRuen6F8!(ckU)C5(;-IZ)D1opP~%j6BTX7;5Xdp?IwoHKzoA9`>LKDRwZi;^Fmvr|)yi_^E|+cvtM6+&G;$L9sF^iZriCsOk5CB9>S#!uKcK-|23Ns_xJ)?!^xpjwm@Nk@Sa zqhPS`#!p!R=Qr7k!H3f~!kLg6h%nmJD(bIghQ59~jZ4{v{p6qnd#xTC(o&&eeyE!4 z!rYDbwE+YfLpp>$bT0F`9iFexu1NAO9awRMhWNoiYg8B~sX#RHU2m@?Iz?=opn0>-y?@@kKE}t(|DC zgazJ#slm+aY(06q=zGs%uTN=;n6vlsa+Ps3)5NX6UWmP5sJY5c3TW%L>i58uxVSXZ z>|*G|I^vuyXRpE7knGhTg!uaHYs(FX6vEB#O(X}* zTe{@l!&RxXl`%YqSXhEL@uq0zeVe_HhsQS#l{h$2TV>AsL>lCpyQJu{Dzp*(-hatuDV>G5GAl5AL$)dGYL4 zc1X&d87=hqrOtPS<{@)_t7S2_q_wM=@4Vw0se2ALkWyu#d_=#(f#xl?6&bUiXj*u@ zKnE5f8ttXaIG~MX_dQesO53XM16u^3a(|Hw2U`2I@cR@!NDi#mq?qRnPt-gRm1M7a zHSF{rHZTK)lx_$Vusqi1C=?_f>V6FGdy&TYNnyF&V8||mc_)u4ZuTBJ%5^(yNa2ep z$pJdCdh3akL0;6h-OqCQPyQWGPAN7=`^C`X!lmBUSrQ^%S!~ISSS@zAp_m%w*s=cf zImE3xoY`D;`E)67<@s2&AU&(e7?%>UsAoh!>4{NZ1jj5=9!l$0NAUcNd!VD?+V6jwcTnOD@FacMj%D8LfyHS@HcLteyP8kHevCUAGfel_|5xz{OG&k3Yi@P5-2rf zM$`&m@@Q2ScqiPpi$;Tq2e2y?R*=!>7$Ia~LNuus zhE;nCo(r$f>E^;EIrhta?h#d-g=Ic}Jwk}7Pts7@QunQ6-_>;7Ji+M>cq};Yl_%e< zL49sft%TxXKRw@bw6MIjqEUl*;onHGextl1D^2yIhtp8+Ii9J4H6=aWwWQAQyZLXF zaz{Z3H`7r9$gc^!7_Lv(PbQ@Eo5%_eJ(?E$b=R!hTB06(z4n76vH?F;_LS=R4C;PL z&y&uacy2aY>BuuN)^9m0c2`j>jvBN2F7jGdDDcAV7r#$f}wi(wM2;JtPP0hieqfdwzx?Vg&*1}KPXFQ+QOVKoqNit zgZ(GeQQFU3`T}lxr-wppOCX;F46#ZFt5Wr?uOc;_Y?Keff@jUZ6?}tOx1Wextk8br z^=Zg|6sxe@?*RF}9}%-i=p&cF3;en1XZqn*^!TV?1l}h372Bv(;-!5rrA1?rTS=#d z-SMw9qyR?}BA=%$s<3L7?hO;v-WUX`cE51=k?cgN>o51&n_@7bzYBe=N2gcENG7qr zu64Tneq_^rNo#vsETknbU{GWAj4Ee&Wn-Y8QvK?}Xwlk*)ZoH6z8J=Ng1ki4f%yJ= zxI%no-7qTF4<7_2Vr@b%lJIGJsyt$ZR*Eh&S76aV!}H>xwa@{it4nXPl|m`0lFSLv z;1w^>BS5Y@j6oxuLRoL-a5vbDRv1Z~>w=!knc!VRAPn!V*^3995RI>C{M+$tAAtd` zx#9c^%Uhv-3Yr$jx?zMB`9N8LLbkGG0>(%|t+RRiJ7Rn{JnM({&Bx_1=e6D~v}d+z zJhMNwM|*{gv(bkf(IWS$-RpZY@!i~+4Be38CkkmaJR^7r6PN5SGlCcKY(3QDcZr*9 zWfv^ppL1&^AaK#te2I3!{7(EMr801A`%9Kqp zXI*5QD({?6X37lDOMj8{swVA(?ho)h7*@I6%<*LxDl=H)CmunG**wFZ2>e8%tPYIa zZm#TZ0{L;>@Ro{^L;|jlo+m*9EO{b18L8CZ++bM=3R}M`WzoHvWKlD=PHYBG&=x}k9#{5SXLd+x8yG_>8yshbCx`CswKxim6DRu_#qP9tLSZiyULPw%u+>Xa8C$#4C{rO{q zR$T?#&esfV6`Cv8#Jt_Uo+Q^3616Yeb?|TqQ8p{bsEN_UtTZ)_q&U{D+AFW}PU!`jAi+${^dR%yInCT!5Jv5 z-J!4~M@k`ok4D@_9iaz3NnZ+EWjsbogbGVVRzs~)espV%rOVrK!Q0Ip`0N6>e$H>D zge)7*6`DA5ZC?ckR$#4%>e?n2`%zmo0QyUnqD0%ILq~@t@R)LMV?Zk$7~6;{TG2|W z0Ji2THPaj|1D~kO=kG($VqI6V!tM=j9ts2+h$fD|wF4^8c~fSP`bj(IyIkA^FKMct zHFwt^-mx-j8iH;WgPYiI$LJE=qoXYLMQYZe9cMA1Ho zrpoaN&y=I63ml+S^UB#(b8w8a-`i^K*lLTs5`%MUymQpS|Qi9ul9lR%b{Q8orE* z$eXBQw!8Use$UJdE60zIs+E=T!#*vbK_iJ9kLm5AIb|(-kW{z*y8P}(d`)#e=fv$@ z(1L!xkW#&?js2>OAdXjyoRM?f=EqlI<+SB$nL|n*fqCJ?;8e)Gv1w&ivzPwx4!#$~ zS}dJLF0wbv4#!$n37UumKgfY*Zn2noV{Kg}K=6lVU|ePpa6jg1`&7inqcUclHL+N; zQTbLEMNtxycx87xBM%%$dk~|2v^wTGT~NLl`0mFo_*7w#u$?H&eq|b!oGN604T@~qF#Cza(yrNOXIE(q3l9K#u5L> z7$ZoDX72JKEB53+*ULzZLLo;(>>KnYArVuV&7hz8`j@w;U)&I%M@Ow)2(n{N9vU4i zM)sPj>Od{`P@x@)-ysE4q+=?nLuWBr!=JveO1&eO_43w@?v?yFN)qqzo-Fa#QWDuK=cq16N%5l&y!NUuBK#UMU<|(B|hiCABdzAvgdE%*62wDXxwK zW)R1AV`JiAXSDdZX%M!~lsnSV^*F}*7V>@E$f(i5JCYw(zOEAd`OGpb(1Dmyp zB3=dU2E2L)u{-A~Xh^F7nq`kq7O#%)+Efxz*I@h3UBWhJ6S8@7LgJ4~@G146p>}i` zAI0Ss)2lngy&@@1a|8GAK!#a1NH86NHK`?DHp2M z$egdd{x97Vvb%Uip05SjMn!p1fEMP63V1#KT08V%0LT3aF9sOC=0E{7W>v#1>YEzAx!uhEiPD=G^rHkre#M+>c*NXxsJ3t@Sfyg~9 za)^MGJIJ~kBGkcPmxMlv*pb&OKzNGM9T0{jN*OhCT?FpNksmygDt?oSEf`&D%8;l_ zl-XXoCzUw)t$ zdxIwK-l`<`A2L}Y3i(RCuq8<>mX%=mK6rDnZ~6=YlXGiEFbvlioZF?C)GAXTIB5!A zp34A|7*9?DXEeO|{$2b`jk&1u>5RZ=)@XvLgVJ@>b^iD(=Y61NDg}QL*l#B#zkL*n zhs}xlL`r9{!@tY&ATV2RtJ+>Gg2~)6mHQMAau@}Q+_q#4O#sgWoB!wGe{4vEx3qDbBhWa-=|A>`rH&7p8Ps!vrt@~oeZf7yT=Z2m{B`?!Wq zLb3+EgOiktD~@A4P77*}{&@oq$9>7ln~J21so=CiiApsS1UY2aMPDoJMkN$`~Pl-WGkm1oo@Zh3Wb3i;9O*nd!? za%{q4E;_@c^S-u)bSN;NEGx&0%$MQUf>1UNvqeRLT3YFDsGY09zm!A2aM;iCD#ojR zc&d9*js&Gc26qv9(VuREq1_-=1)^KhC}!eBsLo9+gE%+rtGb9@(>3C3W||hS#iT+lf@BJ$Kfu^VTH72*1$b`ZZz0C z!YQ)141Uc(Q3F6uZqb}%Y%BC>kjGU@V9-acG>k_EEb&S>0@vXtzn6F>0PcJgCkGq4 zO#%RK7tr%(za-Dw{hEQSry4&&BI{Oa`7mSVK`Z3Z!SrJ%)Xozk>`i1!Nr&6rEie2j z?T!3f+ACUGkx7n$aqIZ{6#9-I-)$YV3P!W9MO ztm(?Sb~Igq-1U1W8EhF_>VH(UX(~f?^WWcd)a_N+@-!NGiD>FBJEt3pXiMH$xcs+} z)*s~II4karHM8qxGCf;}ujZAoomOV%&y=GugG|Dzlic zRnC>kx@!c4=NSO9#+j|P&vP`~2eWrbpEm?g=t8+tP_|1mQVytI%UO?;t(@iG`=N5x3qB`w)(Wk{#_+ezEd%IXS4*N0;%H*y=E4NwS_XM2j zCu3*S%>QrNnvmz*#D4pz!vN#yK2>&d<6}x$Slw2Ps06hxRvN12B3EG=ce*tFqDN=i zee+_thnl7Cqh&*ZO4F>$IJF5xU!&>3;$fbQE1Wm?7??F!VC+ockuw7As6kmLw?~nQ z7%!$O4C5}f31X5Ojq12gTSQdrzr^pR1y8Nx^{L2CN@iy+AbTiT`p(%~AHYj~2)LnI z;v90T9u>hH*xlXaNT>Q!WYgnFh8t2ZqITfyRYC58DSDeq$+_Aa+imDpD}Q3X@uhW0 z)XiU2<8J%YY#opB?ybN8yA#XutLU!jlvTee9NEU=g?$7ODS#48aQ?+y0+c6I`9+iVs<4nsWrS5`2o z*dUzGq{C;8#|@Hn`#?QdxBI*WH8GvQXn4a8!j7A_$CQQDt%pM|JgS*B{AKm5xsuQ9 zEx`Tu8>$xX`H^bhUjC}cj6d>3S%&Y)6aqn4*U&R0KI3}BE7aMF_k)GIY6~jAd0O35 zOFE>KA`c%H%=ClSPoaa5R4->P$11O$wwgAWUi(i`N*wb`87T@_t!Dgxs#~$cK;6;} zP5#ZSrcj4{L7Qz>D(Iq6(ZVuWe%xePo_kt#tMg+X@eT;-(^lDX$Te9%#qVsR@9E6T zvN9KrPPgikw5+Vd!IWKux1|}H1eNar}wde*0`O?hW z#oMD#nr6Ue!(Dd);4ntSwS?RlB$)8|Ib=VPxy~YhKsiJzk!%DB`2$|_fYo${Mt(k< zcmVn8jr5V@-jsH=1YE1nN2gI19Yp{Ywf=NW02&b#rMypCI4&`E5xq`$fWlR`SK7A+ zNo7C}@E#-e?sBpJN8>VUPx&8>%j{ybUHi2Md&hp`E7AQ*Tgn^w)*E?*8JNlVx3}s9 zY!6jP4r22}?x@Hlwvz{}IBt^WyVJS(NDginy^lAm_DL5rl__wPJZEl2tco+HgI3sk zhxz-v+m}12nmqd^%R2||GR7`jC%@cPw+PTb-dw+t&~}?VZ$tWp!7yhNIe$=Nbs8a{ z5s80k=@wql4ZYFF05gk zb>zd=MCvTIC2?bcF1sQbIsA}xnC=F53rqoP?<-Qj?2s>F!8Bciht3S>!frtGf| zip#vpfA#(YcPqD5+|&G@*Qe~!UeR9Hwz_IZjO+bzr#izvYRd*!d9jpBLp@LSF4>NI z<&X`1oqkwSQ|_b2?t#LOuwT+s4@2Y=%jdOYLN6nEzmI1eMZmyxL}NS0C{tzC1Bq!J zjait z0rvRHgaTlPm7IIkC$h-eNv@0`+VwcpYFN4ZcvD|GMbesQM!nmmsw8QtJW3fXM)R;u zy0YaGZXLJ}wLQ`pl19bZq;C8Uadk7{*iECD=MIlH`eG2rmtR~yWm1eboA8mzBZ*q6 zX6(gUpLvb?`);-^%fLFM8>L>tvrXd2)|KfjdveD;hr^Mmq1ea8)${#5<}9;1+2_Ix zzA_SK<|4Zt>?&nCPdBsB?v+v8xzZPHYaWs|p`#Yn5hm$9)@dM~9`zm8qF zF_6kNdpQLE>?5~UE(sg{J<~lbT1+=shw1}9NSud81WXt=X&!u>9@T(VtLVCwkeC_> zkgq%58g^1WN;+G_Ga`4{lO{*=BY_dJ=oS8rr*h2rO)lfwG?%m?!eOd!R z9&o>f;eonqk$rc!^-Wn&$%ObD`HKQw(6!u{0@mQj^Z7$yHPwScwr7y@r?A9Ps4ZRB zqAU-sE!F1zpcJ1F#V15a7LRE5y7_KZ?Q5Eq!idml@t3!4P$)`l8?x< zlXW0n)r>okK=6CL2yYAr@R|)@n9nC#+|E$la{pRCzEtVCmk$=1IGJJplrECdd~HSg z*LD7ub>u2`n89k*YFF!PC0W#xzs)-W%`G-caczNtD>IqW zpKeV?9Vc?M-DbDctTAacOhVs`2MZ_e9C<_lv}PZ`ro%NQmw_)BL}gTzy!A$5-Pq_dy&03JYk+KSrtbKD!~v`g(0C5%tdRwDe3w^aPUpAFoav0uIs$a{0(VANQ{HXu`A1~>4lXB z-EZSRhKe@wil! zMj|kMR!_6FJWi|{_@fJPlg7_8GY>*2WGndDk}3>J8d%Ov`_kSQak(ha=BZ@VSL7~C zE!?eoILu}b;o0pZl2GE6Q`G@axGiKiXAKcJ^qm^Kk`m>AW=x2EJ~NSItn<*Nw`R-6X%4_#_eKAL?rc#4}yA z30t&af+<9(;o~CSsZp7!#y`dRV&BotNCjO2)*sRCf<1mKYtTk488Z%fO={;HW38E| z`V)jK4__immg5P5SW`xE)=P_FhYUaZBMf14x=D#RGp`(*QKA+lQBa2+2aF`}PZ zS^};ZB~sNWWe((MnSA$3PHbViHnmpaQViDb{os4@T79RqrWl2rx%6~c$={|UoJZUR z1_m#;`QP?zXNHY5Lc`#@xFm30FW4-GN2-PK^O0$Lj@w`kA*id7;!}VW#m<5FUFukh z%dQXkvBug^18m${H0|=J=%RXGO`ife4i8^I zm#9kIx^`*g1;eq>;!R;vKfl9aC`--Hp9^cL&$`CRO=Q(xKi@YTwI9_yrk`qI8qKz} z>17@5_bpo@`C!wk5}(#`PJSS;LfNXrkY|@a#*l7I_N#%1z^ovT=>+~85qSYRcfo=+ zd>8Shr#0&f-Qs=>w`Xqj^m%-DrZ)DOo9FAM`#*jGZU%&r2Y z9V{RH&D(QEDh(Ys?w5TY$0B&4gUK!U<_OiEL8u6a(P1`UF6SL@{oj?5A}ToLEX(*L z(K8-admJbDy}qQ4GNvPZ`%2KJRcjp}LQWY^_@+U%)^IWIyXAe*(|FQ< zwQ3TAkw9Qf7&*gxUtO0@Ja(Zqz@e6!U`-lK%BYBlh=Oid289_V+$!iuX|AjYqjO~ z)%_ifI@??|Lx^eD)fVes+@~b#FSEc)&sxkO(B6(L8l5tR1^otNqTx~FWl$L-c?0R9 zK}ZqT$L}E(eiTd7b>wm#(pdsn+GBOKsH^Q|u#^-Qiq$WpjRLrnY-PjT&Q{3`tDE1R z^&on&(!Ql&Q|=z@KMgQsOkPH(sJ=Q@Ga$H(V4Yn1=+zC=TNY9BWOrrd-flZ9-cKQK zNEl_ymXAbrixi&YB5C_pV1WSC;i<|6-A1Dt`taCTJY=*cP%p@E*3+$QXM0^c|Q7ozz zo*<%`CJ=>oTc?`Hp6BZ0;xVWz`DLr`fL5L$50T*3ZQ~|*yI;@g>uJ`lmUX{%@IApO z&Qo=O4|;z)Xh{G)3Ovm_u>zfCXIN+1XL;!pZF-cV{!nL>Vt&g4ff}CntmDtv@FXIa zKXA+tr)x=Q=bd5o>R4K!eZGea9k;@-{`D<8qS4rxugrdYA}?7#c~~< zYu)y_5wFVZy4Q}!(uC5Ep2m7`pLs-k@R4(Hpt6%K*f2Q>JKQ$m`Rd%BS+ebIsBMXe zHBBYy;IsE#H3{>q@yXuKt*o0ixSIxzt8=5zt#OA8v+p`;;PjKrI&W?Ok>=GV>|5)> zk2qC8!(!dMK;Og+mb9~IWYwQeH?wg#DtkRRNeJD83Ds@PTYO`7M*foihq6x-mlqFeKu=xg|+$E1bUN1mc_5tVS!edml=N1ny%f!WK?vv}6Gf;37xn^_1S&L)uK4#| zKGibFb|NFYW807MG~y1Qf3%HSj>5g=YuYCrU1BHw9LqTmQUYhxbVsM{GqtJ^Z{2vF1!532R~gu!>tgE-NREiPHB; zM!FyseY#HFR>C&nTF$;k=7ht^Pizd*yi7CjSE|0LzkLOZ%^2v7RGP2N=7&^>h5D-7 z1ZHk%vhML`=6v-Nw09r+skf~Jy0#|d4C-Ym+uKItgk<__N(E-Ia$UEWG z?02F3SP+&dyTn`XsrSm}Tf9#c=`i1aHs)U@{_JPgl8bS-WBm*v6AbPigUH$lZ^n78KXn_^^{$ZTOx9?`0o zv(_Iyb<4a^-ht!mn5;O`rRQ6jV6T8s{aX27Y8>wjyzA-;i!{t(1FQ9wG#^%#@aRuh zZHrQ4p6IZCGnheRyNnd6vRtHV^?!?2jhEKyXOteZ+H9Sj28GJ`ygilERWx{){I z1^Zrej;eXOi2^CsVJ!^-UtXY6zmjaHg7%7y-1ExC*mN+ZN*M#VBM&lRCcr|B%^05| za65|(&+dRqPX3OF(uYRwt#2~JyGOB4Qi;LmDaiL$13^HA2nxlDW@DmAljpH1@(s~l znvGfh`c%@-tm_z_+BHH)+cTw}F5WGuq+wX+rxpXGvqBr$4&ieB3=QKpv3(FDE(j(YT7eF4n znW+Mu%4CIu>(dg(j`iXx++z>lZo-E+s~m?kIMN3^!D%^_EGHjo8hBI7x14{Odlr5i#SH!9yx%;n(tUq73C7Yo1mxs@2x#4vIdOthu|k*1yl z3Z6g81X`pdcv?KC5WrKJ^}11XmkkX3Qk_=dE#G#UD+QTC{ltA-2GjHp8p;9;tI!;I z#j+fKIDXiGIvxEk^PVWe9$t&2^Sj`4-@d1IN;JfzL8h6MeMv0gDNe4GrW4ian#|c2 z`|sXiaO}+%lhV_@pM9r5R2o&{KUHdz7eqw-TR8`U@E>15Hl_#D16LJ6kAec# zN>;VW{L_ER^LlTg=u4{oPROJPn^Gp3xXUcI9Nhi{6aPsl7J|_!tp04`Q~5tK@}IvC zMBV9y2hsP}K)oIj3I`O-q3C~a8$Tf>YJhWyse(@?b1<*N+wb|U)9VsD?0^0rLSWs^ zvl^^jtX26(*xm*bwy#nB3HuKw?jVFyW;k!DUhoe!o(-bL&r!>IZ%WnaJ$XX=MVOyY z?oX$~zbp@s-@1}mc2HFaKxhBsg81VvrMOSY3zZMza$6vBDY`gBeCjZ#lKFHRtTO_>sM{_h=F!@g(t_Tz~}^UE{(u6St9<6V!o0#z^0zs1F?A! zcmX`58qnTLltbjslg%NIVeWs>KRxqB00)(BwbW#FY~}!hcExvIdzA+OY39?V9e`Y@ z1<3?fST1rG$m841K}x6~!RgS~^ef-pjHjE4n*kD`#|3=XEL2oI+)vDGiWDHlCP0IwCsw{24{_7>; zY4CzghhP zG32?M%frKhBC|U{x8949l_&uWvKdeyXHb1l1o2*~gKka;4RH!!w++D*5M%&d`VUu! zz`q%q&y>@LOo1xP{CS%`YWUk!=MC&bX0>byJeMU;I5Qw5y8|QCtt%JU6{na>fGSOl z8$Ax!+h(!U?AiEw(s$N=Ng~8I=0cm=ZP^#gH333`Ok_4UseZF))rM4QyD8CxXG(1^ z{eC|i_v17~(k{k#1o%zM9}SLk*8Y~1_PWO%^KY&J;E}5ch~SG9ayeQ>4Ht=!RG~Y8 z2PTc05peNTfRy%LdhVAhK$lajIcABH*Xv>q04$!4E%M?_AaZ zYQh71Qvfht!&;=AKBXy!LcX`K@y5i`T%e?WBhfS;RD3%?`okEtEB1is%>_=liS$o! zMc*KlMSv&Wz$gUFfMH4pvLcVi`zipQ>#IAmp;J)HkFK#2s8577z~6}uwR ziE!A=npaH&f$hGeT6xiFHPoHx$!AA<44rw^AoD(`h$X{Pm#(?wf>J zFr)J;;1VD49t`!mxjk%}h45cZfy@9$UVsY<9!tE#1_F?j2uq;C0TA_{d{IrY49hqYSX#HSsuv5zAM6CBjcTxVVCT6_m?Kf?5zjw_y2Lf{a z?>m57lwjrR$tmu(!CxTg>2ss9umhaWK$LVD8^MT8D zk*M+*K&S(%xf)n>UUlsBAmot?heLRrkh*(F|KtTwEMFgIrlYop+n5nLpKhFs&;~66 zEu(%VbOnp1{m!oziN$2R+bNO??s5yv)hR~aR-gRjcfUVZJUnn2mlhPD!k$6DcXsW{YEI7RJp>{VS6y$k<3Z#ua_Z2o1CPjclr^jt#FxE1dRk*uP%3 zo_fzS>sHWyko(n( z&Hc_v^JUf*xR0Nw0TS)rd}f-<0oyxwECi|vxkB1Rl%bGkmq4zKPqO+T1X=CWt4yNFNn_QuV7cf?Z0} zY`?l@ze=<`%3d~Qw|DULY1JsQerx!tNO>XYR`C@(jDg# zYY^R9i}>sff^s%L$y$CKq0a)a+Avu-Br#X2g-`VpUn}{1lUuQpwsx@=_4*B zc1$$toUimjw2W0WyT zKIgk7ZV_v18ohgg5$NCgj7o`FcEu0J-K_kO%(b3~q`37)R8?w;QD$kVCJ2kxeb_ia zbN!AA3!OqKOLzs{L_GUtZ)7tC8z24Ww_PbJHdoK-*jo6#|C@gWcvNpZ5$UZnzWGuX zwzCZax%Cu&;vUf#Blv1!`1F{Koa9xX!54~xWMmTB#0;!o=rYVdVDcP@vtmyL>frQf zUdG3iJA&NO!g`{jiW$Q6r7ePlrG0DYTOA+`j=Hgl>1NZykj~ zn3GfvZR2YHb;f$afHf$Z)Ry;iJxj=v`tttw?>}DvcpXS14gl|pkxD)3@2l4U-aU9B z>rgJ+6zRXOQTYimz;gH}R9w*C7q(xYx2J-o=oI55_TN{5yf88l3o?aPZqEPxRsVMf z{&`OQubvh*qpN*@-E0ufdD6`%Zw}najgJp5 zj~Uc17kg|uV)@Kg55Qg6wPVNrudbl~ym5nzj98S=vXo@BrWvxpgw%6ZK%l(Lpc66) z;0PYw!8%a|gm*`;L*NvvTrMpjR>Of5j!Ce1Uc%@EMD^LL<5ge&liIqr8Xh+yqjwmn^D z_&N2n{XbrsgW%ictOdxEKqtWdT+JAGLofi#^Z%#4EB~hIfA=NIm}H)tP-da{80$nC z3YjtwCo)f`WFD%cB2wlE$s9*Gh9WZ0QxYLUjxx&-j^RGL@9I;!|G>TL-hODEwU@Qe z*?Yg=`*~ik=XqYQ<>7oOJ&^n5RTa4ERKB2`cPK@1q9bXzWFOfIG}#v7Ov=((Oud*l z5X*9xk?LY}!~QZL0du{7%&tH@$oOIVI7j5egj#`E|B0PZ;SfofqPht-AD4nZf{1A& zz05)Hf;k|pZ^=F1o*u3o2HPAoSlwpn8%mo&U(^hEPT-p&_|D62Lao&f6qxZ*5RG5o zn6kG2kh3uU@&O>HM-;ua%D5>=^5%)XxkeL=BS^d9jJS2{{6GXz8?FVc0f^He$e%Bb zA*_x;!ZWR;9+f@mDyrK+reInXKvNx!_^ z`^-DtTTli~Jl%VHd=0W4^}f43L})+#J`7S~SG}QVFcXUo%LmDbj}WA18U5TXi)O(h z1Q0_@nB2-0Z$uV#vh%;E?iwVl*`4y?$vzko_Vc~YawQ=TQcZNFXkK*xZm_!uX zm&$ROH=dC@0X{^m5HhKmF7>ktBUv2fA=N`O4R$_5o)c@9KJ~e=_Ee<$=7Q;sh-&~M zF^_A6t0{!o#lz3eLI3Ry^diIP)D0rwq>Wjm8PIQQ?HUn8^BqBz2+1%9z!`h`hpXHb zdq6}EbE6Li#Cq@E3VqxUUbKi2&zpj#+A|GdoHK~yOPX4YAtaPD(MC=xHi$=)17P7T z86slqfGFxXRNtX6eMS>$JQ|S2&U<`iehuIa{bbjFzY!vcKB2H7zVW5hk=lok=lSl< zIdr999S2Lh?ya4BNuE+(fA#~;#2(2mnQ9{woWC?aa<3v|_Gqz%wnj&gFBnQ;s~zXn zZ76jen}*I~JoJUL1s}&%0zgrn3_nK%a6UI?l7A2_Hq)Dzq(2WbTobuTfG6C}u5CNd z4LwiepHc|(S6LtlP9ZtMlDwNgCDubggfy1-+304l+EG%i?)Pc8;%u2Bk39?T=NGX` z$9gL1oggsMpV28VE9WykB6{ZBd%udLVdoBKmI;mt>TANEi3U?H*4A?$JBIS4{paK6 z7G=wzu{F!9A2t$5@SW3q#&b`c)EgwHY!zRtv-#oc7T{tIM0n0dv8r%&`j zbQ0&yK{0)kafh3LhsP7?9bWVxYbZPY;J@B;8!s-o4}JnfEYe_Km9Rd|!P=7hACQytV5p z7jKUO+oB}Q)yHz_k8!VeFkw?3#};)xKc8`^F??u4xy}vxTWy^L06niyojDGj;NCdZ zaxkkgH?{LSI#{ra@dZ8RLHa4Cd`ogsjJ1Jn+^!~1&E-^RRg=*T1=R2<6@{V3MXeRz zaUl`IxWxAZ(-I!U4{%PH2Alq*9E`ux4>w+vizW&r2ar%6|Mg!eg-+^dv#~m%_I*3)S~+z6ePeKG;)H26L}UwU$jvA$EuqGxC@Zwa;K$aARYi(<9; zGh5A`idvj?YLwg^A8f15-Z1)vmC(XWnZPYXlvR}*LhPpBXKH0#ZgKR1xra!q?`d2$ zo4?k;&(#oknUtT&_*+7F?O>xrlu4rau((m`eb3xh=UjJU6|LWa+xuwu6R-sZv+zRT z0ud_>j3iYDR#a^lkGb|4o;w;K*l}8a8N|>+f%{3#KHhUayUE~aY2|!1)i+=244$t9 zkuQ-mF`MaL{lJmnGwDObv{>8N77Zw#=%^Sp24mx|ER0}z!(`c#WLP^{C(5mlOl!+zbbKIbNg{=o6Gf}a8v)y1^n0A*z=o0X8>HxbaHC>gU8m3cB8lzG)G_TV7`$4+i zxKGH(UprAeBPx3#SlGI5dA=rwLf~EIS@9oHWO)wDLT_Ujd`2v^fA1o5D|(xi&#w!1 zdsI;RXpvIm%px`Vr*wY(fu3AoFg2s;a%}mLfA`&g?ztfai?KbAbLEcf2_WHt!)lgp zeE)yy zmy!F5c8d`2Tn}6_J-pQJ!@NsrS501F`#Y-`xeXogUCI~zoVABV(e)3mfQg(J zd(|s&fm%otk(B-Pqo~(31WU!N42f%ezi#2oHiYZc=4Cx2M@I0w@hs$w+4BbcRK(zS zfpxLmljTwlWV&Gi_~zPE0}k}@j>QIr+{1gKh0R@rJ~hiR{u}xx)sjc>dIZmdw}Oix ziEe;7A?|&g9-8x|FSX(}A}bY5TYT8d!jxI6hP0V3Yh8CXFJtiB--$bi#=%*O8%w~GYHZren01VEOVx+3nW_hvY?l9!sQ}_ zg%76QGdAGc_vtgh6lu6Il^$Onoqs3oJH|eI52_Y&r#@Zpj+Jt?q{Ktd_~rikTbD`m zbLavzq{kzdA{CpAO@d_RAU9utG&{8yJXZ!vfi6aGtS-2MAy0a5rfB!R9#j4Z00g$~ zjpKgw>jsVdi-usCK9>Cgr^=}AevF*Ut?_V{Mv$y{No>6=I*5=69Ctog%+!N+RSHz5 z7ho@?&un1c?%r3+4a?O0>(YLi>2U8&u)b60f}zNVz~-k0Bsm`Q(cbL@ne6nNA4655 zlhrH0#W!ec$T1GDwcm7$&nyk&kUj|~t5{;|-QJw@S1^%jN4zkdR#!mAHga7KQuyb< zpH)q0d6?)MyDG7rY;*NYC<>D8Go5}drI7%`sSL1M<|*|*6fVw}{E=W#N+c=P-4-aR z{044G2TXrJtX--pzg&F*0$Kc=s@~@dF!8NIsZ4r-&EkRGH22p0U8(PlN6`R~Ua5}2 z*5QCMjR~#P)yLQ-(vi*ItuN%jVqgFGw4f8X<0tW5qG!-FWxS76mjNxFLEZ* zqGfwyLIKmz#wD5>w@V$=qY9&$svcF4`$I|)cIoLqX~ExaRuVsbzFRMJ&lD`IVqx@k zW5(HEZBf4$TyjbQ3v|2R5@HURQ0W?}^x6G(CXAAq7dHXbjK7C$F>tJCk3Nbj~{mJdufs7)!6x=r4Fo1!h9awUdm8N zb2Q##b_@^Yx)Kpu>yE^yBy(71(^z#W0*sM%i#k^y3qira>hTX((4h?9$t$s;*i=fC z{g%9smSTVc{JQSpJ^n{^^y6ycC7G%%x5Or(kkpXoy=$T)kn6OQarQa%*{@$(0_$bj zUJJ_?I!I0xeK36CRH5F)v61^{OrdK_WXPHvFWK}pvVJ@H)+cYDl zV9SN%YD{wwfhm#p#n{2?P-RLCVxmtVcSV4ChWUkE9VCZ!-&U^xACS)Pw54gdVqxQy zugv2C0NS-ni8*cZSd{VlFgfN7)7p3WOLYp}S>$ppcHVm41LEdm=s$&R{4 z5o!>NCT40qf9EtfbZ=MV{NRQ%PqYYcL4C|F1ZJ-+iB}qsepA?9(_qtk>UcF#h)N5z zCfXF7NqBmjH9k<5XuFPYuzff0dKrWhkV&un9TylhljX^t$dx+1tlF!T7pa6T6{1Ax zux=T85Lro_{jTtPg9I4wuk?w=L<4VGdddO~1dX7I%=k_8>*cY=!%}iJqLz7E}Dz-9{)afNS z<>YNRCYYhPT63*EF!@t~8ROPNT3C+;t@E&+dCB-N~))45mP7>POel z8#p~$ob%t7F`=Kh_|=+RG)gWAUyw;JROU#73Nzr0RhB|$E`e>bcKC#^V0&#sN44@> z^aY{7Smg`j#s3K6@9>=o?eWTz`d)M7ypKw!W=&E++hCui8N=0vR4;XG1U5R>UR5%( zE@5lyM`ea*L4tS`dVS0lfY?dFgtd$WdRAT_mT=RbvGF@4SHNkZlW!^^CX8(P%paP@&|s)kBNG^{%5x?h zi!0qHBtTCupT%%llINWE2`j86;h}X+r1p8T?U!CBXTbc8zwdjchBWV@IYpGuN5!-l z%=r41*kI4h1`dq+lkMh-riAgsa8Pdq2c+IJ` z<;w0o7<(SwTPJ4lKDcA?yu!K`+NAX2Z(zQg%zwO__9u!@cX$ zn;#avS5KzpD-}+z4a%MV_udDJ6e}v-5=a${teLqgpGP-en#9K_7EeN{bLs78LusRj zzZajTL7+7^f5KCzBzfO=j@ktib$;4b=zK1ey%?w29|#J8N{Y!J^%>i*S()^O zpEuZjm(u@3t^RMAtG7Qeo3uCTc6FhSc@L9lDXu>$Pe0?=PRVH|VT>Pp&HvnUC%z0I z>7f@b)$8a_{C%dMpPCPdmPQ8>jQ%>D&1deCn^+|G*N=`o3_0kYwvvDOg8yI}9Syi= za`xVrf1$f=>PM?s!4le`F?I|q&8>F6u!&b>_ zO#iNypI;I<36S>1HAju#fbQofD{>a&7R5!OzYhOTy5Vmpd%jLSe<&Xeq;Ty6R`JU> zVZ2!l+_m}i%--;S(jPlGgZk@6o+T05BHF&_%tunMp8yQlo-tst&2aJ1AX*2R+m94_LSZKCJ>JP5xe~ zdMbgzQL=8zI?TmVzi2Y|t(b+N#Qvrpm5oRC)SZwi~!4MlR0b+-8$z8aGQ#M%Xua!VUv zKtSG(w0X8VimW8R-wb}!yMyUd4D@0GB2tJt*T}I*cN+5N6D$WQlc237L2abs6%C_@ z;<6i1SZ6wAJ_qc%Obe7^Sj~PJ!6CTt*ne)=8QpH$BjzvB1URq@5;2}$|1rD4WQs8!Xk#9dBw`YKE5PxdUeyCqW z@x%}j!Oc*b-~c=n0<(!H{kDn26UE~S2rL^*jkwhOMgn?@`h0(U>Rx-uB6ft#;S|&C zkAdN>Q^kOCY@nGa`;b3a4SZwa2BDS%ZlGbYD4|1)sob!d?UVl(%>I0=lUm7kRZ(wa zC7JBAb&tx@vY!&vHS+%+kyI-)R~~U|66xv!rq&bzE?rsL8GSc;O~yb>_C8Sa=@w;B zXfckst{Vfqu&&PCN^gBh zD70CkXBfcuDNYRJ=3xkOaBLZV7`y@vRLqs@J$a*n`?xODJaH~xsK;}o!r)3qBr1OZ zZ*XH{{;mRb5bwb6WeJ}Gy%>>&U@W$Z)C#B$(~(l>t>wB5Z#YJI$hwZ!vdFkQ48l|I zdWD%_*b6jx%@wyV?uBBs~Cto?_*D<+5XIAtYArk*Hd^cp3+wd`&}OE7qlX-aPwJj)(dUJm_`V$_qS2C>2-m46lXCM^e!IL3|u@ z*^$u6L@3*CmQ` z+=N!4S7Y=;h?}^_?5l})v^{+eLwvh`7!9xMAu#5;p+Ud~}Raj5i=tDOTWxi`7X~Q}OaSf`B(nK(F*Xyqjw7WJ;s>6EIe zhW(JfBWS&Up;&io3P^(OID6}==MG)ZH$p)>S9_~FSe{oCBg0-j31z$6N+Gy6EBz83 zQ5IEqh+ny?Wi@Jj!{D>d(3*07+RUF zSoUgTxH8e42pccPukVp`c5Bq;yO`*07FI&*s5+wP^3M?PIh+VS_fogX@M@KIr8OZe z;_~a+Y~#0ghVbR%n)w4G4#gj>ZNqpNt8%m)(^O-T#w&KxdOJx12J{}^AS6<%>ZGB4 zZWIZ};(3Z5s6u)O8WbFBJIaGc`mzucbG)6idzg+$Rz2bLSxcb27R^*cWGhgL!nCS$eovgeOY0+X!RwtPNZB@@qSph?{qOi16#F0Se+Aveo83&^CG5~T#e>)Pw#cboocCNNjYWNe_s`@F zn?ExHiHNt~(dzNvl0RCIE9;yxxR?1-EOHfymONXRl+iv-n{gispNkZc*xp#VjA}zc ze8JQCkMHICwinL|H1I|;QRO8H)xmZdvCgLS_kcde=16s&f(6Q-}*13hqEjI literal 0 HcmV?d00001 diff --git a/docs/source/reference/config.rst b/docs/source/reference/config.rst index 51f8ef92c10..16a22350209 100644 --- a/docs/source/reference/config.rst +++ b/docs/source/reference/config.rst @@ -121,6 +121,31 @@ Available fields and semantics: # will be added. vpc_name: skypilot-vpc + # Should instances be assigned private IPs only? (optional) + # + # Set to true to use private IPs to communicate between the local client and + # any SkyPilot nodes. This requires the networking stack be properly set up. + # + # This flag is typically set together with 'vpc_name' above and + # 'ssh_proxy_command' below. + # + # Default: false. + use_internal_ips: true + # SSH proxy command (optional). + # + # Please refer to the aws.ssh_proxy_command section above for more details. + ### Format 1 ### + # A string; the same proxy command is used for all regions. + ssh_proxy_command: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no gcpuser@ + ### Format 2 ### + # A dict mapping region names to region-specific proxy commands. + # NOTE: This restricts SkyPilot's search space for this cloud to only use + # the specified regions and not any other regions in this cloud. + ssh_proxy_command: + us-central1: ssh -W %h:%p -p 1234 -o StrictHostKeyChecking=no myself@my.us-central1.proxy + us-west1: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no gcpuser@ + + # Reserved capacity (optional). # # The specific reservation to be considered when provisioning clusters on GCP. diff --git a/examples/job_queue/job.yaml b/examples/job_queue/job.yaml index c54e3d9a173..aa9c3502247 100644 --- a/examples/job_queue/job.yaml +++ b/examples/job_queue/job.yaml @@ -17,7 +17,7 @@ setup: | run: | timestamp=$(date +%s) conda env list - for i in {1..120}; do + for i in {1..140}; do echo "$timestamp $i" sleep 1 done diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py index d226828bba8..59de2aba731 100644 --- a/sky/backends/backend_utils.py +++ b/sky/backends/backend_utils.py @@ -48,6 +48,7 @@ from sky.utils import common_utils from sky.utils import controller_utils from sky.utils import env_options +from sky.utils import remote_cluster_yaml_utils from sky.utils import rich_utils from sky.utils import subprocess_utils from sky.utils import timeline @@ -64,7 +65,6 @@ # NOTE: keep in sync with the cluster template 'file_mounts'. SKY_REMOTE_APP_DIR = '~/.sky/sky_app' -SKY_RAY_YAML_REMOTE_PATH = '~/.sky/sky_ray.yml' # Exclude subnet mask from IP address regex. IP_ADDR_REGEX = r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?!/\d{1,2})\b' SKY_REMOTE_PATH = '~/.sky/wheels' @@ -1013,18 +1013,17 @@ def write_cluster_config( # If the current code is run by controller, propagate the real # calling user which should've been passed in as the # SKYPILOT_USER env var (see - # execution.py::_shared_controller_env_vars). + # controller_utils.shared_controller_vars_to_fill(). 'user': get_cleaned_username( os.environ.get(constants.USER_ENV_VAR, '')), - # AWS only: - 'aws_vpc_name': skypilot_config.get_nested(('aws', 'vpc_name'), - None), + # Networking configs 'use_internal_ips': skypilot_config.get_nested( - ('aws', 'use_internal_ips'), False), - # Not exactly AWS only, but we only test it's supported on AWS - # for now: + (str(cloud).lower(), 'use_internal_ips'), False), 'ssh_proxy_command': ssh_proxy_command, + 'vpc_name': skypilot_config.get_nested( + (str(cloud).lower(), 'vpc_name'), None), + # User-supplied instance tags. 'instance_tags': instance_tags, @@ -1033,8 +1032,6 @@ def write_cluster_config( 'resource_group': f'{cluster_name}-{region_name}', # GCP only: - 'gcp_vpc_name': skypilot_config.get_nested(('gcp', 'vpc_name'), - None), 'gcp_project_id': gcp_project_id, 'specific_reservations': filtered_specific_reservations, 'num_specific_reserved_workers': num_specific_reserved_workers, @@ -1057,7 +1054,8 @@ def write_cluster_config( 'sky_remote_path': SKY_REMOTE_PATH, 'sky_local_path': str(local_wheel_path), # Add yaml file path to the template variables. - 'sky_ray_yaml_remote_path': SKY_RAY_YAML_REMOTE_PATH, + 'sky_ray_yaml_remote_path': + remote_cluster_yaml_utils.SKY_CLUSTER_YAML_REMOTE_PATH, 'sky_ray_yaml_local_path': tmp_yaml_path if not isinstance(cloud, clouds.Local) else yaml_path, diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 7fc596d8457..6007bf37726 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -17,7 +17,7 @@ import threading import time import typing -from typing import Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union import colorama import filelock @@ -33,7 +33,6 @@ from sky import resources as resources_lib from sky import serve as serve_lib from sky import sky_logging -from sky import skypilot_config from sky import spot as spot_lib from sky import status_lib from sky import task as task_lib @@ -1460,7 +1459,7 @@ def _retry_zones( cloud_user_identity: Optional[List[str]], prev_cluster_status: Optional[status_lib.ClusterStatus], prev_handle: Optional['CloudVmRayResourceHandle'], - ): + ) -> Dict[str, Any]: """The provision retry loop.""" style = colorama.Style fore = colorama.Fore @@ -2180,7 +2179,7 @@ def provision_with_retries( to_provision_config: ToProvisionConfig, dryrun: bool, stream_logs: bool, - ): + ) -> Dict[str, Any]: """Provision with retries for all launchable resources.""" cluster_name = to_provision_config.cluster_name to_provision = to_provision_config.resources @@ -2218,7 +2217,7 @@ def provision_with_retries( prev_cluster_status=prev_cluster_status, prev_handle=prev_handle) if dryrun: - return + return config_dict except (exceptions.InvalidClusterNameError, exceptions.NotSupportedError, exceptions.CloudUserIdentityError) as e: @@ -2343,8 +2342,9 @@ def __init__(self, self.cluster_name_on_cloud = cluster_name_on_cloud self._cluster_yaml = cluster_yaml.replace(os.path.expanduser('~'), '~', 1) - # List of (internal_ip, external_ip) tuples for all the nodes - # in the cluster, sorted by the external ips. + # List of (internal_ip, feasible_ip) tuples for all the nodes in the + # cluster, sorted by the feasible ips. The feasible ips can be either + # internal or external ips, depending on the use_internal_ips flag. self.stable_internal_external_ips = stable_internal_external_ips self.stable_ssh_ports = stable_ssh_ports self.launched_nodes = launched_nodes @@ -2374,6 +2374,14 @@ def __repr__(self): def get_cluster_name(self): return self.cluster_name + def _use_internal_ips(self): + """Returns whether to use internal IPs for SSH connections.""" + # Directly load the `use_internal_ips` flag from the cluster yaml + # instead of `skypilot_config` as the latter can be changed after the + # cluster is UP. + return common_utils.read_yaml(self.cluster_yaml).get( + 'provider', {}).get('use_internal_ips', False) + def _maybe_make_local_handle(self): """Adds local handle for the local cloud case. @@ -2481,40 +2489,43 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: return (ips is not None and len(ips) == self.num_node_ips and all(ip is not None for ip in ips)) + use_internal_ips = self._use_internal_ips() + + # cluster_feasible_ips is the list of IPs of the nodes in the cluster + # which can be used to connect to the cluster. It is a list of external + # IPs if the cluster is assigned public IPs, otherwise it is a list of + # internal IPs. + cluster_feasible_ips: List[str] if is_provided_ips_valid(external_ips): logger.debug(f'Using provided external IPs: {external_ips}') - cluster_external_ips = typing.cast(List[str], external_ips) + cluster_feasible_ips = typing.cast(List[str], external_ips) else: - cluster_external_ips = backend_utils.get_node_ips( + cluster_feasible_ips = backend_utils.get_node_ips( self.cluster_yaml, self.launched_nodes, handle=self, head_ip_max_attempts=max_attempts, worker_ip_max_attempts=max_attempts, - get_internal_ips=False) + get_internal_ips=use_internal_ips) - if self.cached_external_ips == cluster_external_ips: + if self.cached_external_ips == cluster_feasible_ips: logger.debug('Skipping the fetching of internal IPs as the cached ' 'external IPs matches the newly fetched ones.') # Optimization: If the cached external IPs are the same as the - # retrieved external IPs, then we can skip retrieving internal + # retrieved feasible IPs, then we can skip retrieving internal # IPs since the cached IPs are up-to-date. return logger.debug( 'Cached external IPs do not match with the newly fetched ones: ' - f'cached ({self.cached_external_ips}), new ({cluster_external_ips})' + f'cached ({self.cached_external_ips}), new ({cluster_feasible_ips})' ) - is_cluster_aws = (self.launched_resources is not None and - isinstance(self.launched_resources.cloud, clouds.AWS)) - if is_cluster_aws and skypilot_config.get_nested( - keys=('aws', 'use_internal_ips'), default_value=False): + if use_internal_ips: # Optimization: if we know use_internal_ips is True (currently - # only exposed for AWS), then our AWS NodeProvider is - # guaranteed to pick subnets that will not assign public IPs, - # thus the first list of IPs returned above are already private - # IPs. So skip the second query. - cluster_internal_ips = list(cluster_external_ips) + # only exposed for AWS and GCP), then our provisioner is guaranteed + # to not assign public IPs, thus the first list of IPs returned + # above are already private IPs. So skip the second query. + cluster_internal_ips = list(cluster_feasible_ips) elif is_provided_ips_valid(internal_ips): logger.debug(f'Using provided internal IPs: {internal_ips}') cluster_internal_ips = typing.cast(List[str], internal_ips) @@ -2527,13 +2538,16 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: worker_ip_max_attempts=max_attempts, get_internal_ips=True) - assert len(cluster_external_ips) == len(cluster_internal_ips), ( + assert len(cluster_feasible_ips) == len(cluster_internal_ips), ( f'Cluster {self.cluster_name!r}:' f'Expected same number of internal IPs {cluster_internal_ips}' - f' and external IPs {cluster_external_ips}.') + f' and external IPs {cluster_feasible_ips}.') + # List of (internal_ip, feasible_ip) tuples for all the nodes in the + # cluster, sorted by the feasible ips. The feasible ips can be either + # internal or external ips, depending on the use_internal_ips flag. internal_external_ips: List[Tuple[str, str]] = list( - zip(cluster_internal_ips, cluster_external_ips)) + zip(cluster_internal_ips, cluster_feasible_ips)) # Ensure head node is the first element, then sort based on the # external IPs for stableness @@ -3204,6 +3218,8 @@ def _sync_file_mounts( storage_mounts: Optional[Dict[Path, storage_lib.Storage]], ) -> None: """Mounts all user files to the remote nodes.""" + controller_utils.replace_skypilot_config_path_in_file_mounts( + handle.launched_resources.cloud, all_file_mounts) self._execute_file_mounts(handle, all_file_mounts) self._execute_storage_mounts(handle, storage_mounts) self._set_storage_mounts_metadata(handle.cluster_name, storage_mounts) diff --git a/sky/execution.py b/sky/execution.py index b473df4a95b..9a6f5e98cd3 100644 --- a/sky/execution.py +++ b/sky/execution.py @@ -22,7 +22,6 @@ from sky.backends import backend_utils from sky.skylet import constants from sky.usage import usage_lib -from sky.utils import common_utils from sky.utils import controller_utils from sky.utils import dag_utils from sky.utils import env_options @@ -666,21 +665,10 @@ def spot_launch( prefix = spot.SPOT_TASK_YAML_PREFIX remote_user_yaml_path = f'{prefix}/{dag.name}-{dag_uuid}.yaml' remote_user_config_path = f'{prefix}/{dag.name}-{dag_uuid}.config_yaml' - extra_vars, controller_resources_config = ( - controller_utils.skypilot_config_setup( - controller_type='spot', - controller_resources_config=spot.constants.CONTROLLER_RESOURCES, - remote_user_config_path=remote_user_config_path)) - try: - controller_resources = sky.Resources.from_yaml_config( - controller_resources_config) - except ValueError as e: - with ux_utils.print_exception_no_traceback(): - raise ValueError( - controller_utils.CONTROLLER_RESOURCES_NOT_VALID_MESSAGE. - format(controller_type='spot', - err=common_utils.format_exception( - e, use_bracket=True))) from e + controller_resources = (controller_utils.get_controller_resources( + controller_type='spot', + controller_resources_config=spot.constants.CONTROLLER_RESOURCES)) + vars_to_fill = { 'remote_user_yaml_path': remote_user_yaml_path, 'user_yaml_path': f.name, @@ -688,7 +676,8 @@ def spot_launch( # Note: actual spot cluster name will be - 'dag_name': dag.name, 'retry_until_up': retry_until_up, - **extra_vars, + 'remote_user_config_path': remote_user_config_path, + **controller_utils.shared_controller_vars_to_fill('spot'), } yaml_path = os.path.join(spot.SPOT_CONTROLLER_YAML_PREFIX, diff --git a/sky/serve/core.py b/sky/serve/core.py index d1c9e1914fc..e285492987f 100644 --- a/sky/serve/core.py +++ b/sky/serve/core.py @@ -83,28 +83,17 @@ def up( serve_utils.generate_remote_config_yaml_file_name(service_name)) controller_log_file = ( serve_utils.generate_remote_controller_log_file_name(service_name)) - extra_vars, controller_resources_config = ( - controller_utils.skypilot_config_setup( - controller_type='serve', - controller_resources_config=serve_constants. - CONTROLLER_RESOURCES, - remote_user_config_path=remote_config_yaml_path)) - try: - controller_resources = sky.Resources.from_yaml_config( - controller_resources_config) - except ValueError as e: - with ux_utils.print_exception_no_traceback(): - raise ValueError( - controller_utils.CONTROLLER_RESOURCES_NOT_VALID_MESSAGE. - format(controller_type='serve', - err=common_utils.format_exception( - e, use_bracket=True))) from e + controller_resources = (controller_utils.get_controller_resources( + controller_type='serve', + controller_resources_config=serve_constants.CONTROLLER_RESOURCES)) + vars_to_fill = { 'remote_task_yaml_path': remote_tmp_task_yaml_path, 'local_task_yaml_path': service_file.name, 'service_name': service_name, 'controller_log_file': controller_log_file, - **extra_vars, + 'remote_user_config_path': remote_config_yaml_path, + **controller_utils.shared_controller_vars_to_fill('serve'), } backend_utils.fill_template(serve_constants.CONTROLLER_TEMPLATE, vars_to_fill, diff --git a/sky/skylet/events.py b/sky/skylet/events.py index ade43104048..4287acd394b 100644 --- a/sky/skylet/events.py +++ b/sky/skylet/events.py @@ -11,13 +11,13 @@ import yaml from sky import sky_logging -from sky.backends import backend_utils from sky.backends import cloud_vm_ray_backend from sky.serve import serve_utils from sky.skylet import autostop_lib from sky.skylet import job_lib from sky.spot import spot_utils from sky.utils import common_utils +from sky.utils import remote_cluster_yaml_utils from sky.utils import ux_utils # Seconds of sleep between the processing of skylet events. @@ -104,8 +104,8 @@ class AutostopEvent(SkyletEvent): def __init__(self): super().__init__() autostop_lib.set_last_active_time_to_now() - self._ray_yaml_path = os.path.abspath( - os.path.expanduser(backend_utils.SKY_RAY_YAML_REMOTE_PATH)) + self._ray_yaml_path = ( + remote_cluster_yaml_utils.get_cluster_yaml_absolute_path()) def _run(self): autostop_config = autostop_lib.get_autostop_config() @@ -139,16 +139,9 @@ def _stop_cluster(self, autostop_config): cloud_vm_ray_backend.CloudVmRayBackend.NAME): autostop_lib.set_autostopping_started() - config = common_utils.read_yaml(self._ray_yaml_path) + config = remote_cluster_yaml_utils.load_cluster_yaml() + provider_name = remote_cluster_yaml_utils.get_provider_name(config) - provider_module = config['provider']['module'] - # Examples: - # 'sky.skylet.providers.aws.AWSNodeProviderV2' -> 'aws' - # 'sky.provision.aws' -> 'aws' - provider_search = re.search(r'(?:providers|provision)\.(\w+)\.?', - provider_module) - assert provider_search is not None, config - provider_name = provider_search.group(1).lower() if provider_name in ('aws', 'gcp'): logger.info('Using new provisioner to stop the cluster.') self._stop_cluster_with_new_provisioner(autostop_config, config, diff --git a/sky/skylet/providers/gcp/config.py b/sky/skylet/providers/gcp/config.py index 68c535d73fd..781737c6cfc 100644 --- a/sky/skylet/providers/gcp/config.py +++ b/sky/skylet/providers/gcp/config.py @@ -910,6 +910,9 @@ def _configure_subnet(config, compute): ], } ] + if config["provider"].get("use_internal_ips", False): + # Removing this key means the VM will not be assigned an external IP. + default_interfaces[0].pop("accessConfigs") for node_config in node_configs: # The not applicable key will be removed during node creation @@ -920,7 +923,10 @@ def _configure_subnet(config, compute): # TPU if "networkConfig" not in node_config: node_config["networkConfig"] = copy.deepcopy(default_interfaces)[0] - node_config["networkConfig"].pop("accessConfigs") + # TPU doesn't have accessConfigs + node_config["networkConfig"].pop("accessConfigs", None) + if config["provider"].get("use_internal_ips", False): + node_config["networkConfig"]["enableExternalIps"] = False return config diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py index 8c9c6fd5e5a..78fe4cf6ca5 100644 --- a/sky/skypilot_config.py +++ b/sky/skypilot_config.py @@ -49,7 +49,6 @@ import yaml from sky import sky_logging -from sky.clouds import cloud_registry from sky.utils import common_utils from sky.utils import schemas @@ -74,6 +73,7 @@ # The loaded config. _dict = None +_loaded_config_path = None def get_nested(keys: Iterable[str], default_value: Any) -> Any: @@ -127,27 +127,8 @@ def to_dict() -> Dict[str, Any]: return {} -def _syntax_check_for_ssh_proxy_command(cloud: str) -> None: - ssh_proxy_command_config = get_nested((cloud.lower(), 'ssh_proxy_command'), - None) - if ssh_proxy_command_config is None or isinstance(ssh_proxy_command_config, - str): - return - - if isinstance(ssh_proxy_command_config, dict): - for region, cmd in ssh_proxy_command_config.items(): - if cmd and not isinstance(cmd, str): - raise ValueError( - f'Invalid ssh_proxy_command config for region {region!r} ' - f'(expected a str): {cmd!r}') - return - raise ValueError( - 'Invalid ssh_proxy_command config (expected a str or a dict with ' - f'region names as keys): {ssh_proxy_command_config!r}') - - def _try_load_config() -> None: - global _dict + global _dict, _loaded_config_path config_path_via_env_var = os.environ.get(ENV_VAR_SKYPILOT_CONFIG) if config_path_via_env_var is not None: config_path = config_path_via_env_var @@ -156,6 +137,7 @@ def _try_load_config() -> None: config_path = os.path.expanduser(config_path) if os.path.exists(config_path): logger.debug(f'Using config path: {config_path}') + _loaded_config_path = config_path try: _dict = common_utils.read_yaml(config_path) logger.debug(f'Config loaded:\n{pprint.pformat(_dict)}') @@ -168,8 +150,6 @@ def _try_load_config() -> None: f'Invalid config YAML ({config_path}): ', skip_none=False) - for cloud in cloud_registry.CLOUD_REGISTRY: - _syntax_check_for_ssh_proxy_command(cloud) logger.debug('Config syntax check passed.') diff --git a/sky/task.py b/sky/task.py index b2af3b1ae24..d7e22323aa1 100644 --- a/sky/task.py +++ b/sky/task.py @@ -759,8 +759,9 @@ def set_file_mounts(self, file_mounts: Optional[Dict[str, str]]) -> 'Task': raise ValueError( 'File mount destination paths cannot be cloud storage') if not data_utils.is_cloud_store_url(source): - if not os.path.exists( - os.path.abspath(os.path.expanduser(source))): + if (not os.path.exists( + os.path.abspath(os.path.expanduser(source))) and + not source.startswith('skypilot:')): with ux_utils.print_exception_no_traceback(): raise ValueError( f'File mount source {source!r} does not exist ' diff --git a/sky/templates/aws-ray.yml.j2 b/sky/templates/aws-ray.yml.j2 index 1494a2c7060..efb95799366 100644 --- a/sky/templates/aws-ray.yml.j2 +++ b/sky/templates/aws-ray.yml.j2 @@ -36,9 +36,9 @@ provider: security_group: # AWS config file must include security group name GroupName: {{security_group}} -{% if aws_vpc_name is not none %} +{% if vpc_name is not none %} # NOTE: This is a new field added by SkyPilot to force use a specific VPC. - vpc_name: {{aws_vpc_name}} + vpc_name: {{vpc_name}} {% endif %} {%- if docker_login_config is not none %} # We put docker login config in provider section because ray's schema disabled diff --git a/sky/templates/gcp-ray.yml.j2 b/sky/templates/gcp-ray.yml.j2 index ca73199e317..ab62e4f5413 100644 --- a/sky/templates/gcp-ray.yml.j2 +++ b/sky/templates/gcp-ray.yml.j2 @@ -28,9 +28,9 @@ provider: cache_stopped_nodes: True # The GCP project ID. project_id: {{gcp_project_id}} -{% if gcp_vpc_name is not none %} +{% if vpc_name is not none %} # NOTE: This is a new field added by SkyPilot to force use a specific VPC. - vpc_name: {{gcp_vpc_name}} + vpc_name: {{vpc_name}} {% endif %} # The firewall rule name for customized firewall rules. Only enabled # if we have ports requirement. @@ -46,6 +46,7 @@ provider: password: {{docker_login_config.password}} server: {{docker_login_config.server}} {%- endif %} + use_internal_ips: {{use_internal_ips}} {%- if tpu_vm %} _has_tpus: True {%- endif %} @@ -59,6 +60,9 @@ provider: auth: ssh_user: gcpuser ssh_private_key: {{ssh_private_key}} +{% if ssh_proxy_command is not none %} + ssh_proxy_command: {{ssh_proxy_command}} +{% endif %} available_node_types: ray_head_default: diff --git a/sky/templates/sky-serve-controller.yaml.j2 b/sky/templates/sky-serve-controller.yaml.j2 index 362bd4a93da..d49412fb9cd 100644 --- a/sky/templates/sky-serve-controller.yaml.j2 +++ b/sky/templates/sky-serve-controller.yaml.j2 @@ -16,9 +16,7 @@ setup: | file_mounts: {{remote_task_yaml_path}}: {{local_task_yaml_path}} -{% if user_config_path is not none %} - {{remote_user_config_path}}: {{user_config_path}} -{% endif %} + {{remote_user_config_path}}: skypilot:local_skypilot_config_path run: | # Start sky serve service. diff --git a/sky/templates/spot-controller.yaml.j2 b/sky/templates/spot-controller.yaml.j2 index ee9d863e83f..5181f9d4544 100644 --- a/sky/templates/spot-controller.yaml.j2 +++ b/sky/templates/spot-controller.yaml.j2 @@ -4,9 +4,7 @@ name: {{dag_name}} file_mounts: {{remote_user_yaml_path}}: {{user_yaml_path}} -{% if user_config_path is not none %} - {{remote_user_config_path}}: {{user_config_path}} -{% endif %} + {{remote_user_config_path}}: skypilot:local_skypilot_config_path setup: | {%- for cmd in cloud_dependencies_installation_commands %} @@ -23,6 +21,7 @@ setup: | ((ps aux | grep -v nohup | grep -v grep | grep -q -- "python3 -m sky.spot.dashboard.dashboard") || (nohup python3 -m sky.spot.dashboard.dashboard >> ~/.sky/spot-dashboard.log 2>&1 &)); run: | + # Start the controller for the current spot job. python -u -m sky.spot.controller {{remote_user_yaml_path}} \ --job-id $SKYPILOT_INTERNAL_JOB_ID {% if retry_until_up %}--retry-until-up{% endif %} diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index 00f39e3756a..bdf001965ff 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -6,11 +6,12 @@ import os import tempfile import typing -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional import colorama from sky import exceptions +from sky import resources from sky import sky_logging from sky import skypilot_config from sky.clouds import gcp @@ -24,6 +25,7 @@ from sky.utils import ux_utils if typing.TYPE_CHECKING: + from sky import clouds from sky import task as task_lib from sky.backends import cloud_vm_ray_backend @@ -37,6 +39,9 @@ '{controller_type}.controller.resources is a valid resources spec. ' 'Details:\n {err}') +# The placeholder for the local skypilot config path in file mounts. +LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER = 'skypilot:local_skypilot_config_path' + @dataclasses.dataclass class _ControllerSpec: @@ -220,7 +225,11 @@ def download_and_stream_latest_job_log( return log_file -def _shared_controller_env_vars() -> Dict[str, str]: +def shared_controller_vars_to_fill(controller_type: str) -> Dict[str, str]: + vars_to_fill: Dict[str, Any] = { + 'cloud_dependencies_installation_commands': + _get_cloud_dependencies_installation_commands(controller_type) + } env_vars: Dict[str, str] = { env.value: '1' for env in env_options.Options if env.get() } @@ -232,14 +241,14 @@ def _shared_controller_env_vars() -> Dict[str, str]: # Skip cloud identity check to avoid the overhead. env_options.Options.SKIP_CLOUD_IDENTITY_CHECK.value: '1', }) - return env_vars + vars_to_fill['controller_envs'] = env_vars + return vars_to_fill -def skypilot_config_setup( +def get_controller_resources( controller_type: str, controller_resources_config: Dict[str, Any], - remote_user_config_path: str, -) -> Tuple[Dict[str, Any], Dict[str, Any]]: +) -> resources.Resources: """Read the skypilot config and setup the controller resources. Returns: @@ -248,67 +257,9 @@ def skypilot_config_setup( The controller_resources_config is the resources config that will be used to launch the controller. """ - vars_to_fill: Dict[str, Any] = { - 'cloud_dependencies_installation_commands': - _get_cloud_dependencies_installation_commands(controller_type) - } - controller_envs = _shared_controller_env_vars() controller_resources_config_copied: Dict[str, Any] = copy.copy( controller_resources_config) if skypilot_config.loaded(): - # Look up the contents of the already loaded configs via the - # 'skypilot_config' module. Don't simply read the on-disk file as - # it may have changed since this process started. - # - # Set any proxy command to None, because the controller would've - # been launched behind the proxy, and in general any nodes we - # launch may not have or need the proxy setup. (If the controller - # needs to launch mew clusters in another region/VPC, the user - # should properly set up VPC peering, which will allow the - # cross-region/VPC communication. The proxy command is orthogonal - # to this scenario.) - # - # This file will be uploaded to the controller node and will be - # used throughout the spot job's / service's recovery attempts - # (i.e., if it relaunches due to preemption, we make sure the - # same config is used). - # - # NOTE: suppose that we have a controller in old VPC, then user - # changes 'vpc_name' in the config and does a 'spot launch' / - # 'serve up'. In general, the old controller may not successfully - # launch the job in the new VPC. This happens if the two VPCs don’t - # have peering set up. Like other places in the code, we assume - # properly setting up networking is user's responsibilities. - # TODO(zongheng): consider adding a basic check that checks - # controller VPC (or name) == the spot job's / service's VPC - # (or name). It may not be a sufficient check (as it's always - # possible that peering is not set up), but it may catch some - # obvious errors. - # TODO(zhwu): hacky. We should only set the proxy command of the - # cloud where the controller is launched (currently, only aws user - # uses proxy_command). - proxy_command_key = ('aws', 'ssh_proxy_command') - ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) - config_dict = skypilot_config.to_dict() - if isinstance(ssh_proxy_command, str): - config_dict = skypilot_config.set_nested(proxy_command_key, None) - elif isinstance(ssh_proxy_command, dict): - # Instead of removing the key, we set the value to empty string - # so that the controller will only try the regions specified by - # the keys. - ssh_proxy_command = {k: None for k in ssh_proxy_command} - config_dict = skypilot_config.set_nested(proxy_command_key, - ssh_proxy_command) - - with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmpfile: - common_utils.dump_yaml(tmpfile.name, config_dict) - controller_envs[skypilot_config.ENV_VAR_SKYPILOT_CONFIG] = ( - remote_user_config_path) - vars_to_fill.update({ - 'user_config_path': tmpfile.name, - 'remote_user_config_path': remote_user_config_path, - }) - # Override the controller resources with the ones specified in the # config. custom_controller_resources_config = skypilot_config.get_nested( @@ -316,13 +267,95 @@ def skypilot_config_setup( if custom_controller_resources_config is not None: controller_resources_config_copied.update( custom_controller_resources_config) - else: - # If the user config is not loaded, manually set this to None - # so that the template won't render this. - vars_to_fill['user_config_path'] = None - vars_to_fill['controller_envs'] = controller_envs - return vars_to_fill, controller_resources_config_copied + try: + controller_resources = resources.Resources.from_yaml_config( + controller_resources_config_copied) + except ValueError as e: + with ux_utils.print_exception_no_traceback(): + raise ValueError( + CONTROLLER_RESOURCES_NOT_VALID_MESSAGE.format( + controller_type=controller_type, + err=common_utils.format_exception(e, + use_bracket=True))) from e + + return controller_resources + + +def _setup_proxy_command_on_controller( + controller_launched_cloud: 'clouds.Cloud') -> Dict[str, Any]: + """Sets up proxy command on the controller. + + This function should be called on the controller (remote cluster), which + has the `~/.sky/sky_ray.yaml` file. + """ + # Look up the contents of the already loaded configs via the + # 'skypilot_config' module. Don't simply read the on-disk file as + # it may have changed since this process started. + # + # Set any proxy command to None, because the controller would've + # been launched behind the proxy, and in general any nodes we + # launch may not have or need the proxy setup. (If the controller + # needs to launch mew clusters in another region/VPC, the user + # should properly set up VPC peering, which will allow the + # cross-region/VPC communication. The proxy command is orthogonal + # to this scenario.) + # + # This file will be uploaded to the controller node and will be + # used throughout the spot job's / service's recovery attempts + # (i.e., if it relaunches due to preemption, we make sure the + # same config is used). + # + # NOTE: suppose that we have a controller in old VPC, then user + # changes 'vpc_name' in the config and does a 'spot launch' / + # 'serve up'. In general, the old controller may not successfully + # launch the job in the new VPC. This happens if the two VPCs don’t + # have peering set up. Like other places in the code, we assume + # properly setting up networking is user's responsibilities. + # TODO(zongheng): consider adding a basic check that checks + # controller VPC (or name) == the spot job's / service's VPC + # (or name). It may not be a sufficient check (as it's always + # possible that peering is not set up), but it may catch some + # obvious errors. + proxy_command_key = (str(controller_launched_cloud).lower(), + 'ssh_proxy_command') + ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) + config_dict = skypilot_config.to_dict() + if isinstance(ssh_proxy_command, str): + config_dict = skypilot_config.set_nested(proxy_command_key, None) + elif isinstance(ssh_proxy_command, dict): + # Instead of removing the key, we set the value to empty string + # so that the controller will only try the regions specified by + # the keys. + ssh_proxy_command = {k: None for k in ssh_proxy_command} + config_dict = skypilot_config.set_nested(proxy_command_key, + ssh_proxy_command) + + return config_dict + + +def replace_skypilot_config_path_in_file_mounts( + cloud: 'clouds.Cloud', file_mounts: Optional[Dict[str, str]]): + """Replaces the SkyPilot config path in file mounts with the real path.""" + # TODO(zhwu): This function can be moved to `backend_utils` once we have + # more predefined file mounts that needs to be replaced after the cluster + # is provisioned, e.g., we may need to decide which cloud to create a bucket + # to be mounted to the cluster based on the cloud the cluster is actually + # launched on (after failover). + if file_mounts is None or not skypilot_config.loaded(): + return + replaced = False + with tempfile.NamedTemporaryFile('w', delete=False) as f: + new_skypilot_config = _setup_proxy_command_on_controller(cloud) + if new_skypilot_config is not None: + common_utils.dump_yaml(f.name, new_skypilot_config) + for remote_path, local_path in file_mounts.items(): + if local_path == LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER: + file_mounts[remote_path] = f.name + replaced = True + if replaced: + logger.debug(f'Replaced {LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER} with ' + f'the real path in file mounts: {file_mounts}') def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task', diff --git a/sky/utils/remote_cluster_yaml_utils.py b/sky/utils/remote_cluster_yaml_utils.py new file mode 100644 index 00000000000..6f1a044953e --- /dev/null +++ b/sky/utils/remote_cluster_yaml_utils.py @@ -0,0 +1,37 @@ +"""Utility functions for cluster yaml file on remote cluster. + +This module should only be used on the remote cluster. +""" + +import os +import re + +from sky.utils import common_utils + +# The cluster yaml used to create the current cluster where the module is +# called. +SKY_CLUSTER_YAML_REMOTE_PATH = '~/.sky/sky_ray.yml' + + +def get_cluster_yaml_absolute_path() -> str: + """Return the absolute path of the cluster yaml file.""" + return os.path.abspath(os.path.expanduser(SKY_CLUSTER_YAML_REMOTE_PATH)) + + +def load_cluster_yaml() -> dict: + """Load the cluster yaml file.""" + return common_utils.read_yaml(get_cluster_yaml_absolute_path()) + + +def get_provider_name(config: dict) -> str: + """Return the name of the provider.""" + + provider_module = config['provider']['module'] + # Examples: + # 'sky.skylet.providers.aws.AWSNodeProviderV2' -> 'aws' + # 'sky.provision.aws' -> 'aws' + provider_search = re.search(r'(?:providers|provision)\.(\w+)\.?', + provider_module) + assert provider_search is not None, config + provider_name = provider_search.group(1).lower() + return provider_name diff --git a/sky/utils/schemas.py b/sky/utils/schemas.py index 2fa7614e614..c1646b3fa77 100644 --- a/sky/utils/schemas.py +++ b/sky/utils/schemas.py @@ -458,6 +458,40 @@ def get_cluster_schema(): } +_NETWORK_CONFIG_SCHEMA = { + 'vpc_name': { + 'oneOf': [{ + 'type': 'string', + }, { + 'type': 'null', + }], + }, + 'use_internal_ips': { + 'type': 'boolean', + }, + 'ssh_proxy_command': { + 'oneOf': [{ + 'type': 'string', + }, { + 'type': 'null', + }, { + 'type': 'object', + 'required': [], + 'additionalProperties': { + 'anyOf': [ + { + 'type': 'string' + }, + { + 'type': 'null' + }, + ] + } + }] + }, +} + + def get_config_schema(): # pylint: disable=import-outside-toplevel from sky.utils import kubernetes_enums @@ -505,36 +539,7 @@ def get_config_schema(): 'type': 'string', }, }, - 'vpc_name': { - 'oneOf': [{ - 'type': 'string', - }, { - 'type': 'null', - }], - }, - 'use_internal_ips': { - 'type': 'boolean', - }, - 'ssh_proxy_command': { - 'oneOf': [{ - 'type': 'string', - }, { - 'type': 'null', - }, { - 'type': 'object', - 'required': [], - 'additionalProperties': { - 'anyOf': [ - { - 'type': 'string' - }, - { - 'type': 'null' - }, - ] - } - }] - }, + **_NETWORK_CONFIG_SCHEMA, } }, 'gcp': { @@ -550,13 +555,7 @@ def get_config_schema(): 'minItems': 1, 'maxItems': 1, }, - 'vpc_name': { - 'oneOf': [{ - 'type': 'string', - }, { - 'type': 'null', - }], - }, + **_NETWORK_CONFIG_SCHEMA, } }, 'kubernetes': { diff --git a/tests/test_config.py b/tests/test_config.py index dfe64f77f06..afa85cedf29 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,6 +40,7 @@ def _create_config_file(config_file_path: pathlib.Path) -> None: gcp: vpc_name: {VPC_NAME} + use_internal_ips: true kubernetes: networking: {NODEPORT_MODE_NAME} @@ -195,6 +196,7 @@ def test_config_get_set_nested(monkeypatch, tmp_path) -> None: assert skypilot_config.get_nested( ('aws', 'ssh_proxy_command'), None) is None assert skypilot_config.get_nested(('gcp', 'vpc_name'), None) == VPC_NAME + assert skypilot_config.get_nested(('gcp', 'use_internal_ips'), None) # Check config with only partial keys still works new_config3 = copy.copy(new_config2) @@ -230,3 +232,4 @@ def test_config_with_env(monkeypatch, tmp_path) -> None: assert skypilot_config.get_nested(('aws', 'ssh_proxy_command'), None) == PROXY_COMMAND assert skypilot_config.get_nested(('gcp', 'vpc_name'), None) == VPC_NAME + assert skypilot_config.get_nested(('gcp', 'use_internal_ips'), None) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 0ca34066f72..3c6df59faf1 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -194,12 +194,19 @@ def get_aws_region_for_quota_failover() -> Optional[str]: use_spot=True, region=None, zone=None) + original_resources = sky.Resources(cloud=sky.AWS(), + instance_type='p3.16xlarge', + use_spot=True) + + # Filter the regions with proxy command in ~/.sky/config.yaml. + filtered_regions = original_resources.get_valid_regions_for_launchable() + candidate_regions = [ + region for region in candidate_regions + if region.name in filtered_regions + ] for region in candidate_regions: - resources = sky.Resources(cloud=sky.AWS(), - instance_type='p3.16xlarge', - region=region.name, - use_spot=True) + resources = original_resources.copy(region=region.name) if not AWS.check_quota_available(resources): return region.name @@ -214,12 +221,21 @@ def get_gcp_region_for_quota_failover() -> Optional[str]: region=None, zone=None) + original_resources = sky.Resources(cloud=sky.GCP(), + instance_type='a2-ultragpu-1g', + accelerators={'A100-80GB': 1}, + use_spot=True) + + # Filter the regions with proxy command in ~/.sky/config.yaml. + filtered_regions = original_resources.get_valid_regions_for_launchable() + candidate_regions = [ + region for region in candidate_regions + if region.name in filtered_regions + ] + for region in candidate_regions: if not GCP.check_quota_available( - sky.Resources(cloud=sky.GCP(), - region=region.name, - accelerators={'A100-80GB': 1}, - use_spot=True)): + original_resources.copy(region=region.name)): return region.name return None @@ -1226,7 +1242,7 @@ def test_large_job_queue(generic_cloud: str): ], ], f'sky down -y {name}', - timeout=20 * 60, + timeout=25 * 60, ) run_one_test(test) @@ -1611,7 +1627,7 @@ def test_autostop(generic_cloud: str): f'sky status | grep {name} | grep "1m"', # Ensure the cluster is not stopped early. - 'sleep 45', + 'sleep 30', f's=$(sky status {name} --refresh); echo "$s"; echo; echo; echo "$s" | grep {name} | grep UP', # Ensure the cluster is STOPPED. @@ -1669,7 +1685,7 @@ def test_autodown(generic_cloud: str): # Ensure autostop is set. f'sky status | grep {name} | grep "1m (down)"', # Ensure the cluster is not terminated early. - 'sleep 45', + 'sleep 30', f's=$(sky status {name} --refresh); echo "$s"; echo; echo; echo "$s" | grep {name} | grep UP', # Ensure the cluster is terminated. f'sleep {autodown_timeout}', diff --git a/tests/test_yamls/test_multiple_accelerators_unordered.yaml b/tests/test_yamls/test_multiple_accelerators_unordered.yaml index db0fc9c5f7c..3bb26c197ce 100644 --- a/tests/test_yamls/test_multiple_accelerators_unordered.yaml +++ b/tests/test_yamls/test_multiple_accelerators_unordered.yaml @@ -1,7 +1,7 @@ name: multi-accelerators-unordered resources: - accelerators: {'A100-40GB:1', 'T4:1', 'V100:1', 'K80:1'} + accelerators: {'A100-40GB:1', 'T4:1', 'V100:1'} run: | nvidia-smi