From 760761a32ec07f87e0565d9f82d8db192a372de4 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Mon, 29 Apr 2024 13:43:31 +0000 Subject: [PATCH] feat: add startup taint removal feature fix --- charts/latest/azuredisk-csi-driver-v0.0.0.tgz | Bin 14457 -> 14460 bytes .../templates/rbac-csi-azuredisk-node.yaml | 3 + deploy/rbac-csi-azuredisk-node.yaml | 4 +- pkg/azureconstants/azure_constants.go | 5 +- pkg/azuredisk/azuredisk.go | 124 ++++++++++++++++++ pkg/azuredisk/azuredisk_option.go | 2 + 6 files changed, 135 insertions(+), 3 deletions(-) diff --git a/charts/latest/azuredisk-csi-driver-v0.0.0.tgz b/charts/latest/azuredisk-csi-driver-v0.0.0.tgz index 3a4c20b06a3314adb2b04f2056e987b52b482fd8..8f20de34d2f54d370ea1d834b54f06bcce2e6545 100644 GIT binary patch delta 12970 zcmV;bGF8p_aQtwPOn<10WqF1t92*)*?bc~qQi!r=XIp(dP*sw|s0h>mC`xR%Bj!PN zUv8e{1TJ+)6^j?ivd!9^wpaxuZkdTh;zI)e7IO%Qxbb{JynvGl@U6oU=CV7*PyD) z5YP#gzz5g|^z5*I@;rTi9>zmLHz*!M54jj_W0F>3ySv@p-uTkJ^8Gwc`Jdo02C*^) z&@BHu`}_M9`G0@;Vs|b75Al#O9zt&ep=&JQ0wUH!W1n}&#OF+~f$aKhyn!;QRY?)z z5HUd{UR`};W-P5KkcED!6}pH90XAH(HG>Lvk6gZ z0Jx(RCdAfFd<~p}1fhyhg?dqzDJycar?trJA`hg*{(oqm^03WODC}jc!Y)9N-{vWc zn+&<4cDjG=zL-^1NG3o*h;z=apd_sIU@O>6jsS5WwUG7f)dH z1fAkBAT~gKTMt`lMKo6l##Ir4?cFj>-DwL^6BI{c#N=VAzRCL~tGmIkv0v_g)rjKU zzp;_)uk>?WzbSGdi-SW(2V~d>pMyNj2zvUX(SHO%eQFV5A9_f!peLq6g4Szc$Zn6v zk(?f8d3t8ZHxL4fuS2-xcrYOTJ52FV^;hy$7T%vTE_>+Jt3Ul|Lvwyed@OoscLSj@ zNKSlVQtbYDe|yg@rX#?i9A(K=ULMmk`VI@3FIm5|LwFuy3fZTC2vz5U2__-FCLxj2 zK3d3A7J$eSO~#kg2zsc~*+3|#LjTJ64e-km2ONkV+Syium#Dnz$8AkYh(q=L^78Eo z?Ss##`s(edCbHpMm_XP=FI8vWNsbTrQ1s9*ziuGJB!rh7`-Bcnlb!<`e-Cv=QbxV4 zt&sUR95Eq#doOlgsg_F?0>{d-6{ZFrx`AmAb+mkO7D6}ESY5)l#(Eq&IxRvhQn?xL zq0aFg5mF={U@!n*m3+$j{s@9Ngib5^wmp#=(t9t(TdEk{|~1j6vj6 z`BALIWL(L%o8+pPAop5S!2r3}lKOTwk+VzNU9vQX8m%vEXUW1!X7K&=f8mCn@$Yfc z6LF+;5v za7+Z5ec-TDHV{fzF?HP6#u61-Hk->>p!?ky*641uf(u0-7R;MAHE=g#D6g@$yOd#V<=^^uN6LoYgI_Q~sfv%@0fB}v} zsn;IH$95iHhr~x(8aEN9L7L5(yf3kTV^Tj)`_FXx8_wcL*?P`Oe;xn#N&oWX^bNW= z?w`HCI6OvgF3#Scqh}`rJv)mCd@>*qY^J=Gu9$>mT+Qng$q^u-Rtia>XsAj>SRng5 z^Bg|U_*&R%Njy|-8DrT)Pj6tl^Lm2Aoy{AV?yA3bpBfen0-}ZmhdHj>7yyB>Tc8XZC*&?`(HC|gJ}odFRlB&I$fIgKDUkk7mw)hng|V-dVn5@PLWKNgaW-wWUe zL(nFzvcTfxcRwBs$XyS0&iZ|C1mU=ocx>LV{?X5GQ*B(se*&G(Gs=|G7SA`)(eYp2 zzu815r$3%;qQ4$ooSvM%*+j<|7iSmB>XR~3iu`VBE1_QGe^ZwI+6W1a?+Oat&L@Fz zS@XJ!o}d83m{9>z57h?JvsVbRhqfk|Z-wM~%f2zo9m!b_1#nF;bvDm@+C&2g7{_UC z&VdikB1y*NlRgI_VZk|P0}_Jb1oC}C2j!-HA&D#bzaY9pCZ2#4S(=7bLIYKG9^$&j-2_3<6}v1SOXr}8CT0w3ol zBoB?T^hcAx2OfW$5H4dJdYJ^2=GF(bhig55kH7?9W?`{$OK8Aug-dtE0)Apbo^$4d z5a}*h&bv~t-Mm{u8#Ub1!`{^d^Q(|tU)h&W??C_`NX4n&3EABsdIu#-!_QZ55>|V7}OrSloy$U;6@L4qn6py=v47 zZ%fhhG%+vYWP&9m8#vRgjVDhw^r!PU42?U&xP6}>&syLen^jRURqkwTD1#+rI5-J_ zN-HpaNPvIJlOthj?bc*;0iAwI&n|1V)2>1}`8t<$s1<9=7($z{rzh5vc6zf;=( z-+%S;)q4N`A)d=ovNS)^pq~B-GMZFL$j~kp0*FF3g+Qqkz3FL%qr0(j$&fz+|E7jW z!~%hY#4;9}C<*}!Ks=_s4QurHGDMz76Hn#qPoGft0(81=>c2#vKWh_56Z1S_C&AYp z97TU6V?a@=5J8_mcaZ1ZsyT8EEh;`UwXU4&_%r#)&>0eYs?Xi=T+ z(}Xomflx+WYKN-fNUg@kp@=eLF&knsNZF%oQKHYEt5&AY#4#O0V?LA`qNX=-&Lv=$wC@jT=IP9y-*U6&F_Z9)b3J5Joq~AaQ^t?v=bP zX74nEoehC%eRds_FaTaqXp`;kpDY>>N5sQX#MuObqW-g5;=;myFMG2_ghs9 zIK^Rp+gC4XxAym6b`k^|Vf2!WK}bA~dWgnhnC>9i-5{Gg*LA|c-fhQhhzNf)B5r85 z;p&nZvbHx&PZqZ`ty@LNpB*3Hk_2KK#vF&0+Uaj%#JE8PBu=%7{Xpn24lyrh+dxS8 zEP@_NZSi0OAv+qHikI7!JLiN-;D?Gv=C~`IA2^|bS}lW6Os7}g$(|w+hyxC$dAEgR zY6_Q(1H!gyUF#|NhhX#^%Ta$1bC}-{O>%`|D4hzi}k{ghfyY*?+aAv<@d8{epwnC2Q(saM?!$Er;bE* zy+5gvs&;z*Ys)I1)>o5M4IUp6mP((ILF&XC=#+{Lu`gpBhSQX&9cW=(9xBN-lcST7 z4I3m7lOy0=^cP!~ckla`O8uC+LVzMp7&l0CJS?dlB5eEXIqjO0&ox(S!4o4=?(YFb-mW1B7Y9@A{a6T4jon1~~YPPJpYq z#%DuHeoJXaUe z+JJ0-7y)HN8m2N?O)IKX^2KrgGQ)JK!6^5BX)YJp5U36#bO788qBei@G;cS6?KTU^ zbV|yr@%5O|+7RzY~>QJcJL!18<@L_sfi?eaJLKLA(&2$7p&FEx2 zj-`f$;ekvwCZpY~9Tgh&P0?TL%x_oXQY&lo<{DlyuAM{?I(Jd3wXm(hnyo}?sOGwV z+bYtm!@H(u)rv+b5Fz5RF@3MWZl~4|)tjv8R!_3eTAJt*J)RpqQ-d0!*PtEO<<(?* zxFlJ)C=Kpd#<>^=)iaMfd5r={p4w$f3r4BUTf->0ragtpy#1;S)9RYe)ws+)s197+ z5ZR?=4P7&Cde|5Zr*`ac$rMlw_j9Iy+H!^0kX>>x%}3Kym5+?V*}w&sN!Euw^h>8! z@;krI$XUw@m7FyTv>BTBsbh(#IgL~o`i`WI&YZQrbdk{|D$C*8rA9&+NJL4fgqT=Y zl3Bx<3DoQ?6<4_#7skRZ@CRr}CP0&2ZC(5bPXLYMP?9LjD4Plw3vka4jApriBH+0_ z@vd>Pl=NPzu#8;0!~z$~-7JwXevWKifN-dxOjPA_$ zZ@baY(X(h7Ec29;it15OeWJlja-@uT+Lpz%n$RmDYzHw{g0|WVMD?g%15h-^{H9_< znQABP+%${EY>ae`2bhGK@(?J0Xh)AylFCr*B7J_!LDSmU7IUPJTG?~vGF4b5GYU1S zqEs!EP=3aEau}x-jb)b04d$sykhANJwH|Y><_cw8@^R`g!H)Hzc@7y>_JLXr#KL-7 zrs+EK8sx1SjJ6IG=h{Q4eigCrgAju_M9;3LYBfUwY|S1l=X2APN-=AHm4aL5Il6{{ zVysx0A}S6So3++}Y<9oQ($vklTPSR2L$fZ;d)SC@n`lng+R4B{vvi7x$W(QUOn1A> z7-20k4x(C!vr!MHrb~_MqS3&ag6yKxv&& zn^@Xe)A^X(&DIXWt7de6G&i%)p|XnmX)_o+pETiSIm4gmQoA|tS&f6S>{vu66a`a? z$HYgX>{G6`1^HOOW=@Nzv)X`!p*b37%SWR|n*=uOUvh`HybY0&PA(NVl#;PWNs|5o{NKPyZM5Hzq zsw%}en~Yj<5Ki?>I&#_v&8nM_F})Q2d( zfKMQ>p8|a_Aje6zEt`%8lL5kXT3C*yN4`c7Mrx*%48@T$PD7YLm}5-wjC7&vBAbUh zXX&3SQUUlR041G&+9fSVFwJ>kC805(it|?U5+<4xjXjhbjbezIYb-!ViO||$8tKu~ z9phYj=))%3S>cpD!{Rppkz!V^b%9na%9^Vo%6vcOA{mCFP^a#s_Yq4cCM#2B^y-$S zslgrsUce1C1|ibJ@8zdxHrS*Ezi$8%lQ7+Zr8>9|@Py|lYL?M;oT%6?l}kIzCN->I z)u|_B!y(v%&Ls6ZQ9EXsP$Yl~BFO}kFBE+NW0S2ELjk9g@f0=zF_S12EPv4d_w4GA zzkARB>)Eqkw!J_9`p0KiUH!*@KL4-hzb8Nb`26{^XTSXX?#<=-@vr3h@4wJ^d}Dt7 z{_Gbx{%{z< z__`9-#=6za!QVAB(v1@QE`K4B-#rv^4S{?h2;@qkkE??`t`h3_EkPW=T4>`nNNn(p zPmGr-Z)QM}AOZEIhXwWNkPK3$!(~Mr{7kz)K?mn2dF+=N?t~BrAIL;ad$*WVy{~2u z@oU_fA{{X*>~xqB&rRL7!pxpD6ouW*glcMnDwAKM)C3x1Edoi~@_+R#7DpKUtD)0F zoth&bTM3ye)-$X1|Dz*K*Hb8EHp|~H2RQwPV5X}9mad_s^-$6kfk)SH(Kc|=2LOmZ zE=cIN1^`_a>bWGSb4mE-(m>6x7m~RG@a6I_%cTL8_XwT*B6!7R(TS_TA})_STpDM% zEUIvY7{Yr%3@(cgTz?)7_&s6&R*d$$FMQuOjObe)zqbOk-b%1}OC#~VR(5+#n7UOV z=vIW6TOJ*^JQi*x$hQ^Y+`e&C+e$EO%Olp7$EU3TjkXHx+2cl!&i6aTX7Yuv(WFG;=%ao9&VdA&3;#Gvh)a=!6b9 zO7U{MVtZYm9PNI*fwpXar5icfb$@1dzz)wx-nFWIAj;|$ri*hpiOLqvK~j!CgC=Jbwb@MQM&(2l#T>$)zhQ>&K#) zP_4pSY8jWSv8Zzv(lsTr^_mCujeVJpfM(Xt1*r9W$J~f%F6Qi%4YV{-vbJ%$trKl( zoCX3#qp2W14(ks7GFtEU)OEXoNOsdowJN++L~T`O7hY1>$>Rn$0FB4x4W!&9r`b!> zk*ydqy?cS(G&dB?09I08gu-Tne zn|~s`8Q)N?`A)*CE*x4_GJBovnp$ftk5oD3Es#bOXnQa4Jgaei^nL|P^4cT!hy9ae z-blE}!bdbT)?tWcdVAhO-MTGHx=$Ly1!z0E2m?>6eGMU_L$OS7gZd4t#p~P#yk2&j zCs=mNR_&T}0Rw%7YD0_WXpA`yw`_}+Yk!HZLql^;OKHBYJz3jBCG3|8QGq&)dI1qP z9oM!^deos=J|BS>qP%gi-t$~<+dRT0-i%Y~g+uC@Fc~~)qfe;RS+kFsxE%oX2=*rT zR;3-YU0f~ZrtE2S<+L%R95h$r9COumMeeeByP<^x%|UWSE;R?`_D;4R8BgHDI)7x6 zT7(j#^Br_ke?t|ti(|>gSXweJgz;hjq=BRajsyw7$9ddw(JJoP0*#Oc^*1*8jGO(O zz;SBb_~2B-op@xisRK>8*h_RRZ$%(Flo8TGa*~;{}qrx>}j2N$k{ZqQ+AF+^GgfyJ<64vr*$?UuT zC?!ezf6^}X7U!0V8A?Gd^!b1!#YYl!@)yq*pPI7@v!^^aFR3SqW0Li=yMI^H@R5y! zjWSf*eMA>&-QoGI)U;n4H z&cFZlzkdmtfAjBu{a@>V+wgEG;lf%5IWqfE80>?>*f3%rD+g7BsJ&9>Y)hU3{~es4lwodmD4nat znKQR;srG`-F8k@pz<(6Rvsjh{%uuAhJ4C3EfP)4zi6Wqqa7ahj=q}^I%7r7q8gHeU z2>rkPv(v3NtcIi+FRa4==p%P91}ZnT{xA`ZbmYLk`qCX^N(LZg*OmcX{IdINojIVG z7w+&l3Sl$Zh)YedM23hk-H}q%3eT~ivORzzmIMvI)y-AYqkpm!vGw4EZRvaBE^(Se5XqCm9AloDJuSxoqUtG{oQ8(OJ*bOfl4uCNZuCMCm>g@T=LhK*lYa?2-5* zOowT_!9grz4vU3osCtT%$Kf$9D~>Pbb8Xo;+AdDv)BH|bW8<1)f3gYi34zxSw^Xj ztq~?7&=?1hh0eKWR#qLV!@~Jf&ti5wOat||303M(7wW;7v=ZOx%Mc>P!oz3Zl&MojGToU(dB zvFWwGFm!z)X6_R)D?ADFwV!}lpL}`HlP~KNFOT`e%Q`a4Jw;}*EVEQ>7DHoY(OKp^ zmGl*zMOsHhG0!?|#MgbM=xaVj^o2}+&UN^lmM43bKF70yQ#&g;qcg{eoJKqD8$N)u z^l_Wz57#{Mqcp2KIP>iulUeFO%rZw`zRp7~OCD!=%s1CAdX8b$rxunzqwuYqNO*wr z2g{!}SmoJ*Wls{U>fFHkltB5Ez*6S{R)YLr4&%Qpnty5B{z?$~%V6=BLESHZgRfsB z>E~|T8cDxK(yuY|Ys~x_GynMR|DNq4`)=<5FK#L>gIZr2qrNO6{VT80_|%c7qj2iEzKUjO~+ zlZSu?$ou?xeWn_34I&#L4Wa0u zF8W=2-Kj*oCbehgU`M$okEN+HO7=HA4l@dSlSm>aOvch5z12-i6ey0O*?pf#uceZX zUF-hdH+1GG(mbmDWRRx3W&9>&I5<&6iJW!~Xz}EzKwecOP1+?k?N%(HhdNT(%?0`` z1~6j5+mpH?9U2+qA)Loy*ax43%oSDsQYf!3cd;6k4m2-Dlkp-UE`EXC^~7IrIy2Ej zA))bI;{fF;wI1;rmd#6MOwZ^$EF|z;dR3UmSqQ~DrnbhDKqEjOK#HcdF%mCgD#;j* z%&~#SVUsKbUKXm{%a%gbsL2PDpCcxJVR6o3K<-dyt5cNjt}n=WmP5@QOc2TVG|$mi zUGz{VCy*xpH5PCI5ksFpSH3mT?#)Uq@eqCf+%a7j3TRoM)s`?npR!z3X_ZM1;li^^0xc_!;{?I1sPh zlYAsAfBka1DaCcng?zoUT}T9UQf35umdlJ<8E&S!poi4DNByrWn17Q?GO+=mV=62B zHCAuUc`}u?MA#In$bs|%lJ{BUZ*KhLl_&Osv1k*PWC8t-s8v2YyLmy#>l!R2_D8K1 z&4Eq6axLQ~>=gGj>Eilj>C)P17N?xI)e?z1f7sdGb}591(Fd*6LBd0`bwNRFcefpK zTN2y7FCn(gLc4O2O*pKTyyj!UHHvJC8Av&~pJiQ@_|8#CuXVP$#rpnh(N5gkO7_Eu zZnKe5(JBu&BQ^+-dg^6wG4aX`YAXrjAjog6C&UYrdgb$>ldf}uN>VO98!lPaYK6ne zf3^=HF;6{ny}W05*CR9*;Aup_YrTK*w34B?aJT@UO@JGO{PA6c_1)J$ViLw2v>{uj zkWKQtd*6vPS`L~ii6NN)1tHX8_!`;>W0K=sWmv5|Hju80Nip$)Qn{)U+sI~d;abN- zCZ!&YY^5qH2}V*zUUDY=b;x`ij+l@=YPaQBtoaCVC`adv%VMPs<;m;aN66BJ7F4*} zvMv16MFc3m4&jpH!GQShFvUaQI`BqO9xRWHecl}tpEJP*Qr}gNZ^V}GN9unbjwMr8 zdMsqmI32sYyE{8Of9md*+3h?ILwimgoeWNsjwU<-9+S%^B7aE@LxgPj7A6qZxRw${ z^M_}L{gaQ!r$^^!C#RP#pW7HDC%#Da+;&(0UyWk2sT~Yx$pu6x>GkkLfb=K}pl)?+ zQyxq(3Gp=viJbO9o~j&f(vz_#rxDbUn~X1~QH}G??9(eQ&1bg9!|k@5%WoO7SRXVP zkfA5E@7gO9$$!;ACqLAwjidW?ZpxWu){=aK$I^sJ@?rtp!$Y9b(RP#Oy#RioQn(3a z6g&|R2IS5`C{9T9bqY5j9ER-HgK;FMg&wGLpQz-=D)nL3^g7lM!t=2|0#C}&v+(I! zP7&fzrMhL`)jmGsEx<43*U+=ZEhwdcD2Ye{1cFdGJZt|MC3d?7#o!F2Abcb9CE?=8PJ)so?zeTS@gOn^n2fv-zr) zP;6JRq9D9E``ADF`QzKOHy_^~e>i^IwoJ{_Z&?56=eL=u=<0y{B23M*sB@sQ_9gaj z^nb)#+YP3I$EP1oF3wKh9iLumQ9VDn{Hewb1JchOl-$x)6sjCuaQ7dA(K$x0_1xyH z{MqI4+mHR@iw`G<#~)7*-Z@oFoes^HqyFw21kY&+;Z-T`_`lE3E-sHRKAv5^J)fZg z+Le5Kr{j5ycN-QhQJSns<9}ZEcFUPd4C5rEtLpjXI|!HNZXwE1v&%IT zBHZ$!8uXn54$f#ebxZ)w9G$DI26Cx2lH?3^1`sfgAGWGF;$C!99>j71(GA)1u_t*f z;Gsw(3R=K4f3pfwYuB({se4oU@J+esee>gM=Us@19nFBoS&UtDp_N8@6VcFKla~n7DvW_ z(5)PZ+?yU!Ypsnu2KP&kaPu>l{A(!kEujIs6`oQ|5obOKF+(FX%m2>H?U(x{`G5Ip z|HWGVAL6mhj%k3Nr3W|62qz1U=jd5(PE0<=0)A3cli`5F6zQy`>$y?@_J3#*hB2dk zSm0z)QdA&LAM#*PQta}liV{jIJyK+XdiN#A5{5PewXzxs9lqC}5=&u>@7_~vb|b=K z&niLg7QZKEjRo@O2`+Ln$u)NOdoZb5V(oaW;Dk_))IA*!n(vH_8qvmzuIRjl$=CMu zjxD^tR!bb-$)Dw@le^pjkbk+31*vKsxXy|=s-L9GsP1f|7H4ntWLi3Sv-nYyRUO$$ zKH3(ZMLF{=w;(ZEzq0J*F+C9P1#r{S&gmsOcz^lR*~Q7_->$Cuf9qc!ze_e+3TMx9 zbW~>^HXNK$-@nEU`OI*N9&T^GTIhYa&T0T7orv867b6J6F5EdzcYpYmSEq36ok$oX z{a?KSo_Y|Fr`Eyfuy=Je!TjnfB-dA0O6PXJXQicRv}uIp=zsMeo*iABd^o=NcyavZWB=^%=a1(Hmp{2^7^ZZRIDh;8&B>`I zaddLg;_z7Q{$lIBH;1ug8>yrt5ukecsR0Yx>)RY%;;zjS0^C@GVvjCT_D4HV)2l0I z2|vI8%kjt4v!nX`755gd4k%Q!cbf&1*4CevmfWRtsFjL#iGNs~vTpW4CybFfpdGYl zf_lGd!AL3ExRDNTR!?x?>IdM(kiv!QF>2&yO<2uGS#qKj5- zTC6ci-#RAgK1{m2qkFQmn9cVMqoxjfS2tIaCN3uk#r94){^Q>ElT|J)f4!?Z2)*4d z2}rA*wsjnc1>-=>y%kmt2H{9zj@?xbej3km{QZrU1oPAk8{5X*+?KwJMoST z3$)cqWy}J2?Iu;4jB1Yi2kti?_kFSLlL#+F4luK~Zk#)=E%9W7oer3jS}#j~f$$;{ zfmadd9GR=D?v+Y(mCHs#u=^aPoYf*Uv|y1!bjF$F?E+uZy-Bqj`~zN&Znl$$%bOT-urr^$`LdKYhmlUrWZt80)LO^jAeF8V#} zpHzbOHV>Ynvcw2hvo{Z;O24~*$XHf0&Kl79i%LheoONCY%EF;}W{6i)$d}f#)>_Wy zEUheba|qm=B~_(wWs_T!C7)~2P?_m*~rqJc_n}*JJh~BJ@+WlfT ztL2%e?DTlZ8^ukXRsa=E34MR}h?}N6uMiKYNIWt{)Z${eSCe$uP{5a8nu5H3aud%M*&Yfp-?dMi+J8O198L3&LNWTpfY27{Ym1N6d zCle-Xu=F1RaMY3b(SSy0K@?uj2^wAKYR^SLq7_RlbNg&rx6f9s+vmaV7hMDexwQ}m zvaCPX2*_Er_^uF;76TRmA9yU1WK4e3{PnsbM<(St@WELw-AeC&>RaO~*SN}6;3^Hn zK2B7nsoQK!<>S7JZZ4C5je%TWLbt{}IxDxvJ$|FO$9uXauW9yh)_Z;cWVMI~fIqqm z{bJz9S|2bn?&JvxVVJvCCvDgR+^5&Hz;!MjIg9X68|yvv7ZPY>fu2^JRnU2KZ{Odw zenmn7Oq`hmP#iaZqZVQ+wytBbCDin0a+FUI(8S~K+3zh#`XLrIZBm5v=hZ5h^-YD3 z{HDUPBV_?3WJ|!({C=yVDLn(N)Szq@ui`>AsL?e`cT|dSTdZ2OF1hB4Ep;WmU*>eyLD*6H3B<@^6$Z11k`|9g<9w5Ccg)=ZXCUIqe$C}h(y&;?L{DcR(Y!;F>a zLm~s4DzZFYu!;IgQG^_#koZ`%?&KtcDSB=^V7u|t$1~C!TT!Mi&@H1+CAtPQ#{32X zgoQRp(-RnflO1$;ab%8k#2Knl(N96%5G{5oR%t#E8XVpsG96h6zVei#c0 z{J{`V1fC|>dl*bZdk!X*J-J6 zej(rhxTj^9Fj{wUOetAu7u;pWbXt2_nkRLevlg_{>;Z1on?<+qhUa-@Bw0%4gZA>I zG?WT|bJ&rNR0V@FYjNc|`gS&^s~WWaQu-(8Qce9tpxY6aD1ZT>AdEeb^sq8mFd8v- z(?x$Ja>Qb35`=_;K4WT0{exXANVJT8QbZv{8PZf#y`XAs+ ztBp+k`;!MYI12-|xV~$teaz^L=aWh{#eba_`+KGRpWXd6{_|m;8T&sEs;Tr@_SGt< zrTuO9v_bKO)TZv?`%#$rKzt9PDl=}mKkfMKKW+8@GKTNSJJ1@)l|5rQf{r?Af ztp3L|s6PYnAmD%+CYyggSZrOr-o;<(l6viReORyS>I+=yUfsf#Xze_@S98jxmw#4l zKGh#2SgA2pt6{&S=fYJfgQ-p-)s3^jm&T_&B-@qx<=f#x*MP4L`+I+8*#DM4u;Gsd zk-)S3Kie;NOZNZ%&i>BY{(p$4e3ql+1cA%e#qx?ugR;8{lhx7`VtssH!N{{OvK zyL)T>{}9g?)Bl;jc&+)@nt%O#p=W;mKmSg^4E?|T;zjxV-~KxO&jUU4`2Qbna;Wn| gw3{l{nrr>6pY^kTWwYO0AB8U0{{R3 delta 12996 zcmV;#GCR%uaQSeMOn)d!mgE_paBOHKwOglcNg>Lfoo)B=KvhW+qasiPpd_)~j+h78 zeYts(6S&kJRV-d4%QkCw+F})uxMe01i4O_*X?b5>0bGM9;f`@;4lWUG6m2q z|J(cf`xW`$+kf6)%l|_>WP(S~-$3XJ3%G!Y_0h!V-3jqI6Kp8EKAUWyOlnn9gg8V@ z5XtzokG8J=B({7(whFAIlSw>E)rk5X#bJ2PLgG*R=wx`xpH#yPCy5KtYIe&aR+cX-SqyWyzJd)7{>g-8RYsIMSNq0!Bngj^}8k%53$H z$CK>ED$E?q4FUXy_;BF+ET&Qw?sLEr`bhE^&=d66F;M;YCbDS}Hi-qgC1HrJz-9^1 zka09+aeo&LKqAS|5QhQ~8LF4)7E}E)7NCA}#BOO-<37}77Ez~~7& z#S=hmhz7PEw$h4dt`dx^A_CjHWtzIv7NRC7j>MSB!%}^d_e)lHgI{64-2bW(#kqfD zBiCQ)=emAVA`qPHy{E+xq^wG`+LKBdj z_`;;v`SX5n*Da=Fz@Z#x$y8n*(=+-O3z;ujzqCVm9%2gFr+^4m=YtzeLVQI+BBuk8 zr&ufiktMpBTuvkCqfTc7p_~f+E92L|FUK5kAo^&#rvxuidDV|SO-hJE_5Jel%?TZV zz9ZL21-(Hppdu#fgsXWmJU5BNy*(J#MlAjBkummK?qj!cuF0~>!Ib;eRg{jIH# z`8XUiA^W>8wqL52OBMph%CZ%v1|GVGX&-g8d~p^+H`7>M!nP)Q96CBJLM&3b8SkUc z@f{ITBp+Zn1YebW$_D-zf;fauEBdxQks8u_FU4D`7}b%dAa5DJPPMEY6M+OsN$3dq zEDi&uBP>S-N`+;j#6%*uMiz*l(_gd1x&L(npX*)}n=1`;cg>5fcSjh~&pFV%w&@=u$PI@Aa zl#cu#MzaaJW+Jp-M`i{&Ir@>a$;sQJL8hZ8`0gFWX0{2TkE(jV#YFdFV=);0})20UQhRt0Ihkd6r&FaCj z$x>3kE(Kgw!8J;_qGW&T<>tjw0{WhX@dVN_2(b{gmW9jYyb-?s7C;E90ZAVpsoy4B zggKso1fEulP@lsDsKj9(J?$uo={!YGP%0D1xKsMbeA`4FU5gHSW?rDHDGFeS<523g zhw-tU$5$cok(S0yglUjwb0!~1>|dMI&(r=ho&K7$I8wHrvr>P@|9vvJJUM-hE{+Fh z?=B9H(d&z|cjxHY$xzSEA_AWb2?U!dZ>1|HAsJWmIz@5}NT`)UQYad!QV|x&{?0sy z&ojOjwptR8R9hxk_R-U8m~Owi!QuAiHB5KZUpr3?3kCsE!-B&c*KG^|%a}u^H+~%G z7AJ$JRoI4EVP=184ONk@FGyF9zV&iEQQC}5;t8Vh{D2o(}jACR0zkQ>Nn-nQx$Q-HAu-Y5yNcC;T0$tLdv@PiR( zlU7+^aq@c*4~OKgk2+_Afj5S5(n&lvui4<}=QpV~u3>+HPUjhAN@=B%YJQygvNITg>L7QK)9@V z-9=AOfMLR@0I7#+1L@f-1ldPhH<)jQlG0PpvSsw*(MKEHDd z56>b=CghVq2O)pqIcGx>g5u-9Q%2$Mx#^z{nKA$p=`)UzC+c&;-b03loK4J6GlOWR z2OKD2Ql`U$S@tT$LGo%NnNbLb;3(#Vj_hiN(9y|=vh?-w9r&?k2&<>^C0zoa&;{BrA*$rD=oYFW4RR?&aEAza2d^fC!3&8-h=57&DB z9)StI%)(;hme7FR3YYGR1^mQ>Jm<^@A<|v2oOh*OyLq>SHfp%1hy4#Xn12Y#)d&0X z={*SG1F1OG+|@f~O5O}67Ik5))zD*6LwgH}>h3aXXm*h=nu@${HKYsL&}(|T>uweV z+QQ%1c=CS)UBD1HXk(HgMgcL7CX53Tdi5`?hdve%pxZG}y=qibzg{d0MS{byZ%n%W z)>cuf0_KY?iN&?p^0hA@=HNvv(5ptR@U|2^PZRSZPHwP-WCLfqwejT1hW>OOhoNyt z7`N{e6W3wtMrpld-4P~%|3|NAd@U#$24AL6+jCrk4q4eA-3Afri@gpBNB zA%G}kQwWqw(VL!DIJz4fmkjx1@ULr#L@W?UNGxNqiJ}m&0K{Y3->^oHFGJ*cH1Slv z`t%8PFF>d3rv6Lx`Li~0G%?Q;b`pHm!BKxyG6oc-3K8`Aa|e0et(qg((4yiqL;gRG zA&h{Fu8bCk0?m;B?cKfIlKj8i-|emC{~?}FpSJ!J-H=IN&u?mSQC8xsiRu=A41M&U zTc1B~Z0KaicM+z6p7y9k1?ZVlqeXSPPoF&N{X(6Tv$`sA2YK0d?Yrz6l&Kcb?t6cO zI8!OlZh&(V0Qz06MyQYWU+6!{#Q3>T$IFNT4fNlpf_A+#WE_nloMbjBYN(ybWJuZ- zfimJNswMH=^-;&AV4QiZ0pMLtOSuv>MO{W~B7ORV5?QQvXty?Z3a8X%ntnb=k)1Z1 z>Y=U}Z|UJHB(+WQvNuW2-}NqmQptavKXy>(V@K1f4umr5Qae-)M`|@T4n>q1i`fv9 zVagt5ixPeQT(vTFCXVR{8uOvl5LL}YIev2g>3#nErDgR#o*DYz=Yf~sG+GD~+Nl4x zclTescv;l{JKHZ_?ydFzLp&u3OUKzde6$pLp0|<5iQ|aCodl{q4c+TM8J&N#y>U%w z&_{=Qv*N<4-XqYS55nli1SAfy#Ql=D#q6DCu(J_RtXGtw;M?{X}tFF%`<^5LG0#0$5-}co@+O7Tly-tE)BaB{>2?&WNQ6JGb z4AUJXyBlP4=ekZ9*t_kR4H18VM#PQGHe6jYL)P|&>B-`Drgf_b`LpBW8OOK*)}UrsCyx<<2>w68NE_kvZ-P z=Lb$`pjOKu6w~RIce1BQ1mb{$Y2IxinVP~S6$|u?@nl=|c~XX}tr7TLL$j;wav~&9X?Wo<#AI@69Liby2RiZj z`@=%>h2#cg_VgZD_RqTc=9ss;9Ie�`VpsnFUn*0=z!vllX)b#v zBh4LfE0i{$eJL4j$qG{`dP`%YF$ZxvMrUI!y*YN9j8gON=so~DCLTXeENu`gpBhSQX&9cW=(9xBNdljBkqG*-i1f5{L~5%bIouk&LLNNA5FS96WeMwI-P z($*?L6`DJqls(#xp*S=NSJ54WmGtN9cs2Lp*`-5DU2X0Z=_HJk_!7h9I%~>xMvmFR*_ALR%63 z7<24P;7aysb3d`5kUq}Z~$b;~G%bUh@VxYiKab78HX zZm~DqdO>%HW}@nDa-mx+rzNOEp=J+l{#U?<>2WR2Cfy2Agf2DH5ePM-lgT8O8Wx6! zGS!%jcC&U=Xw)}Ff2}jWU5QJrtj(Kic*(eS5=H3TMXA=pwgzjq5~-n@>wj*mNV5*_ znx0iF8l^ylh$qJMy#l+PT0>NCvZh--$v$goqD%C6ZuCqIYKUHec3hWNlj-4-WZ|MT zxMLaTVjNV@JnrN*3LtrEmnkh6r8aL3qu`qM6ejcbt1?WhYdTlsGW(!9aCJjumzFhj z&A91dV>F!FvBM=(Kr!6UnSW}_6<$Mj$-y)qO;1%mG74uy7g#1)ANJ8Nom$E7{5m6N zEh|)V)-2FwXx^ueC8FjuQeEgfk~%tb*80*#Mwh58hbxyF31KJ^C7}{xVqHmQ4QD1$ zv$IrOY*~&?F8eiNcJssc^9X_w2xEmVYY(p4$`e8W&4R z@1+XM$hAu>aIxIY5((qy$fifNL^X=ppm$uAj3v5FdJhZ1e4?k+0t&$B&V2v28~q$T zijs=cGAvG zvuMo5NY{9XNvJ80fPaE^^e82%48<X1Uy8o|*(XyWUvqG3RQoP{t*nqz)78SRb0_kWpnHsMSC$tfys~t~0Md z-m1Z9>p*d?J%s945&J#}F^og>>}sl3GbF&)?7?zAH%+M&vwv18xMiNBD;O%qiiIhn z;&8E9YYoU|_sc9z-JH9H!ge+^>(ac3jR?1i=5(!{3>`E}r-+D5Rkz4=x4Von)*|B| zs)aZk^>Avs)VMAh4V)>+E;>ECJa(z=ONQ_bBLRw-5JW8(1o$yGh(Qs9*hGF4OFL^i zACtS;+Cg~LjDL>iX7)K$R&hUV27~95CfqD%_!C`fH|IU8aWIx0i|B-+U`p|X_(+s} z%GI_Y9}C#bY4LPc8l-3uQd7}b)qiBQU;qipiDiL^)TTmJr5Ih!~jchb6f-S1Lus-;Ch`VVfPRVr3nP^8wl*D zKpzaqaguGzrlY}Rh%lWNmSgFWuQ7y?nkgkiaiomX5N;sMF{XG%y3loz&BL9u^v@Nk z0DKaFl7CL^l9pqb=De_y&=^p~d8>H|6U~Xn9?FeIF~rOj7NDa4V$Y|?_?HvoxAnC`$*9oz?a!YFe!%jh~zRBV^ZrJZGy8rHAs)DyDN z2<$;;l6sw}9WzWQ5&x@wU&-^|f1&Z@+Wh+c*)MSX zYfB=}pa0h%Dpk4ja?nk+;$z%1a-W`8&W(<1nymWEJl#bbwYiMlouhBpiTOA2RVA#A zb*q_!ziViu8zuN%e?lU^dnn`@0{K7?$dy7LR|k1qCDid-f;fJ)(8eo}*x(zV7%x-a z%zz|80_sZ-3+mG$8Kg{y%ZfPonRb7I4$e>V*e^5O2_X(XkQ+7a-C|DlzM4J6uW@IJ zbi}Bz(_uzDH+9Q;rITM=Gv zd34#mX4fA@pDHVm{3{90336$X2ewa6-0AS+*YJkHg| z8t0w#>3y##9zuuDvuaq?I^kYc2CG;Ld=u9Hqqg_gM;N{moBpe};@j`Ftan$w{O*d; z;SaFAl5-=~Zm^g%v$j}lmFxFdvcG0*t)z8Ya&zUY3y)AbBk!|Gq-NE^W_MO?e~R>G zd_%S7I|;A4aAZ}<>~*$lYOS$6QstDlKpIh??Y+SBtj6`x`xPw7YmeL?4o;GJBjF+o zAJNcQhar~f?RgJ%>$WWEK4}OSpzY`)3_Pv&6@-kA#4^DR>Nl(wuX7vldf9EBVA(BO zwQJG^4D}VN4K13ZG3GejvMpM!eqTT-!G3 zQHN&vd<0&I^2Wh>&vU(P^9YxCGft@&4yk9tWbmYoKA}=)%|2%0b^z2P*qhv2m3GW_ zakZG6vZv9N)5ef;&|Hag%vIMFxy$D5h87Mq2gwz=)Et!CJK26@Jb@4Ee~?LP5lW2C zchF7!4OP%CjwPF5Y00<{#)pHG29gpu5+nd0=W)kHtGHteG(sBG-`M0cZuWBm$EkJW zgHsK6;*rIs4m9CnFAefd&K~DGps-gl4_76VWPKQis=pJ}`8-jb-xD=M)?NNj)ZPc$ zxbSead45G=Pf^*}S`LzFe~1U1t6sVjm+>EZ4e~sUM}#V)^4W8P3?Di}MQd>k>}mA+ z@oDJSg#&~nC%Ou5f@h3pE7RwW3fG7+V!RUePwA3>#6o5f(s0g8Sj(p+v+w?+lqBu{ zNxRfroLeepCN!HKqe_l<)M>Y;N%1~|h z5nZHphv$>Ky4!rWon504vSMdOe97~;In!HrgCbh#MR?2KomS+`mKUbgBsMz3%rS3S zFiXV@O34!?K2d)ckFng2Sq?Jb=%?f?wYL7%h}f8F|Nhti{zyVFQtr=qV{GC@n-&T>y?_JH4z_lpakhvPX#YNE*Ibs$%a@+e-|(;=`W`{%w;Gn;2lsAgcV~tB%s+;aCuR<}y_jO2oP?fQ&fL1C z+6y|n?58I~e^VIGVp$R}Ly`LK5TQZ>4jRlPihxSOAst<#yNm}b7mfgHyp?7m^#2ad zPPbmO8j@zbunq&DkKDlosNB^0!$dUFkplE$5bwLkLD+mtdhogz{zL??zOgN=ns1aLpo|j;LoZqT z-{Pt2e}!b=Y&ciUWh1wyA^y&e&ibZiilIg{iE%X~O81$7U(KckGH%gikHjBiI!faW z4r3W}SS&G@~tiT-Edw&)40ZG#H4-F>!7l_Rzme`-{+wa|VZdpby9N~OFd%P7^cHNr#$ zn&1Gk&^hVyG_cpI0@@9a}qJ=t|&v z+}D`?@jt|Woc`T*T~Yo_ooS&|L6kYZ{pXiGuea^>wjC6Ey}tf`^zM_LA9jC7y}1ml zS^N1-d-t9hv2xBC}YQSt>S*p|P^)EOVYp`ijmX zts|nCXB{@;>poNTHJ>8-LMDIbI($yclRZnH<5|I}ot2!?nd3xGqaF7RAHZ4qxXtp1 zYaaPgn$;bg`F4-VEOj7enWHaX=OLFRkFz}Hn`;+6$FS;C3rn9-_*PCNJiz&b%pnOVTsq+9ULH;j?@n06rzcg-tB?$dxu=vZM?w5bT*RPTEb2n~{ zq+cWH*O>V=W`2#Ce|-0U&vuc0w|9USHx-vbtuKvHUlx)6mEzC8Ikfru_U&&IL4L1y zZ_kMiU*Es|nC{zuJ4Wzfy$s(s<})(a(-&B)yiSJH?Z*Z5h2@ueZOutN-ow zdZmAN_FwMp{;<8f-`m^X-`RW7`=Ph;B$1 zbml11JgWUQG%v=J^CBTGL4n=%#9weaGtoyOq48bg z0OcvQ9`PEM%}ZuX&*)n$B=B5%RhY+F2*o?5wkDH8BS0TWil((O5-(yZ$pntfv4O^6 zlPm;Y7OLFKmO|C2$%m7nBPIo7an4~#lguL-e^DnVkS6~X7H|O(L!UoazBSSA&q^)v z2z~zCFmEo0uF$C z8d{?fMrWtUuG4*0XKOi#D1{E(lVc7z5U<>mek3b@wbyG(aTRkRUv2jaiGbde8Nr_A zGNV?8o2f47Bem{P|LY3oU+0qC*Z|Nml@6+;1_4q}z3eR}Ub#VSC1D%{ z`K|SYcwthnd_Hv2bxu%8%Ef2HCCgf^a5&k2_8}zZsb{X2_YCiPgvJ6qjR<(9_b;AS zG87jM7vQrS;07Uod>3JT_w|pMgfRzg$d)N≪!^cOs3JgJw!%NN#|F5Na`e1#N^e z$#JeStX3WyNLR(An0P^{Tvdr}WV5(%t>Yn+QjbQqQWcd1V<{sqIg|b>WIhhZOvt`! z+j1<{d<;00<8#JkvC@X}&8wY9$kK%tRJhu*E&S6(1Sq}=;gaLwkoa#g#UtQ4@J3M{ zERRfl-klJiGr@*Z-&K#V#g^|!>VF=NB~w;h6@;?K}=cdrlpl z3{R7gCOiQ?lg%a~f14VH2-)Zj+(204T1phnAD$f!PCg!=9-W_^oL;(oZWEB4_#)MF zy{`Vh8pUK&I~dTC3y4tC>){&#(xWVZy4A5wd2oYCh_6UUE`ONlsxZRd>`7J{h>w^YEGV+A>U3+CBf4LgyUdI|jcs};W;7J*J7Cv3eDMB2ow3nA}PUyfwt!v~+GqY8a-o$9h z8oJJT%SpCGe{U(pU#uyU5GwLMp>^G=l3bb;w#q>b1C7DI)?t~vFs7c*!ccoq9lmF! z(YQWC_Xh*<>%VgL(4FM?fR9Agj`{Q{$@)9`XUrs&lQ@-TW!7sw?-do9)B9|bl)e?G z!|Z**;3-yVXqGycV-8}>!k~}#3g4A&_!?vxdZ+C}f0UYv<$4%H&De&&l|MGLysd3X zD3Jt*M-bwv6_P%BQN$KngE#%WUHlvYCoFJhu3yRh&%2A`kB1k>N5`j^CkJmn{&@1{ z*tM?vkuyh_6r)n!;^=3XI?um5FgbdPKIEX$6gd2mgm9RUsd9&Uu3_5OL*V$vB6H?E z7cH*bf5!Zz>|V0Ql-%{Vw(3_M*2bbxR>k^6+PSh(QKwTCG*f|}E-%jqAI~q&{`+t4 z@~bL7N4Je=&ZuFV3eI1@l~j+iS(Q6Io3CmK#dZ}d3c~BNkAtJ1KfXD8{qfE5`{Orl z%hWvmnhlPAev_Gst`5jA!qhyAItME2Kw|$|e^0!%-C!zseER<6;_US8@#&=&)$@bP zpK9DNApP7y$t_()p~}$(cmE+6onz!$&uz}ipIsim`8YVfcz<$u{PFbQty9I+>Ck*R z>hFO;@SK(qUX}8W|NH#x;_~?7(x171eFit|cs-9oIgK%l?7NQ(AyIeCN!Yv=FLEkyx;EaY-#{|&K(YeZMAeUMr zNzPDb2m#~xVXK-W?nT$-K`a*#U6U;zdy>aOZfQHumz5hi=IxjT{7xz3)_774+d2S7 zPJYyDl_SD!#PMPdC*vtgts3hjn$6Aze-U=1L)I`pJ>%v~&PF|pwBTnqgUqaBm0Rku zfb;O|%BoY(n=^^cVANbgUHI?0@c(uuo(KE?!%X~tueZ0mQ^No6?CkEX@&6C;!$e=Wi=VRQfsoGeO;3dHF{9xO_VUH(*2LTRN(icC=N zf#g`i$cCU+RwJRKcluLeDNOL)JBrP2L|E)uCCJ_4_r$ERK>j?zMNTHU#_oO(CRIzU z9gh{95XzCdr{h8Mov~3P+E~#QowqRg+MeFAh1b_=iNibjvmAADmpcG5f7h`fRjmWp zSrJF|lXMx??H+1z_C`;prGqz%A2nIkk)7nDZQ)szGv9Ix5~KAi%U&MS1MyA(H!ba) zUZR6{mp`3doLv6x!-v7&2A9WglZ}?b*|QuS)tQG42WQmxuW>^@Gn}G_+ncWzdLOQ{ z8o)>=Vz7{ag%caGB?e}3iFDcpJ|5++FhS8sr)9t7m6buc>YfB0~N`G*f7x%%)y z>6{SchYyhsn=U?ls9lkK_<+108TZqQG#$^<(rNO097aS)j$1)RRYtSOxk$3DmH}rL zP~{3&mIYBF_fdM8CF|HpEk>N@a*W~whmdUjO5zvAA))d7WS_HMIa z(%Smd(vrJ$4z*Ize=ZTLQ`XHs=!7vc2egCsOi=GvjhuU=6Lp08Gn-G@n+cXZ!uFJ|+7!>Fml z{)g)ilO`@F2t{wZ9RKmf{*zTMEr0zFbr5>HT@sL1J8kPY5DUhEn0qU%91Oyd#2mY; z9Q-t%<@y7953%ia*w@}2r;BIyvZz@3Rc)#?N0TTwb-fw=h_fK3wx9)w1xyJor zEcJ^V9m80KmNXpbg}cn|NFL{Y6S9#&ns(wH85U@(lgfkz@XAf9G#S+#_Xpi?KJNRX z=aUF8L=G~uw{Dy}t}XFogPjhUlUgrJf1&Uq5`k9{=Ny?2AG#k@q7S)jBm}$9QOa2@ zLPHA{DMV+SN!~6{7Ae4HE#-QGwyahl7(!(B9_3Q&g51!D{yAVN~gNe-;_b zYQ|XuI)72=sFt(N>p)pJG|vq2Y6|(%TGm?2*_@@7g>DXko3o^<^sP+tRwQ-vNZc}m zTBJ3f=q)z9MJn^k;GzTVF15io+Eh@h({9;`ui`4S7`v)cCHHi{cx~n?+-Vvt-4hKv z22HiAg!P(Xfi%^#Vvf|6wT7gwf9#nX|2cQ`ubKGIoxQ!6CH&|1%kAwo{_`Q8(z4P- zyt50*RbV)62)&0zaGFAw$8Q=s-ywRlI%@Ze-K>^pp0d;9A#W5nby@*bG$r)?-6L+A z?z}=gpd#_e6j6(d;a*MNC2tis)x94~s?{a;g-Fe+{uuG7HNCM?O!DJIf4jB`7rn;1 zemi%b&9$Ffz3r^o1!bgWjUxRvP^5MD$XAjrhn-BAtijTM1i(>8;zt7-odr?Yn-es; z(AA!cfJ7^nSmyTGvTmQPTDQ-G-7mTb3UX^93}jhjJln99d}72RAW{~80izJzX# zdvsQAjeGn?agXLle-&cwuibpqXvG%zzvQw-FLyufTp++1_kZrc+%4b#_oCNZ-~aa@ zPiak+UaXlcrMwIT2vNwU6QB#A08_HbABPz$(T79^HdSPKykHacm7)kaL?Q99Xx+(4 z22=Fhc))h!r;lf(H@2coU7%Y=pGtHEXoC4Q1PBXlkftXve{QzX;l+_T(h+BSb>V7*FFma$`8}7l7Gj zfahZO72qake~^C^VK=K)TNI3F4UAfNo*%|S0)H@sUT(5q5lpYCh&(RRj-(6h_+>DF z3&=?=E};JA>Y}z@1z2GIvikGpo*P#1&~;iWoL>ky0PblSCXCiy98*eG+68x+F`d?) zmgY&_=Bx#+G<$$s^=8p6yy1CX8A+Csd9S@ZDGjB8b{; zzm)z7x>Qp?5$JY|B?@3jCU3bx6i5#<7ngk)CpwF0EQvYCAijvQ$ zAOUa)P=vWmPLOiT3OaFT4B@0J##@SPr;j2gq(`WbI1D!edrf}5mn(n}G+wV^TwOg; z%%>g8+4UboF!c)+Z5Uj+_If5 z1KR{QlbkhD8Qq66vSBGwJ#=M@nokvp`05|vORJ4c{Rfi=HaH7Iwz$4)seR1oOy-kH zHpPGKofkW${hyuv-a7u{gFG|#e;!m*>9g#sRZh$K+rE%;)BSKiYAqiw??Dt=Hm3Kd z!`9E@Ph0)J%prgo`hWMu_FftP|FXB<|9_Cj>VHgw`ZE9z0uHEQviawO#n#p9UHp|U zsn=fDhxNLyzQC34)h%3!*3P4QHPl>sX~lnrS^YtRm6~a_8um+iE?ku|nCcW#-8c(; zX?)5|3y;`w6wKl4PdHUC=kub(gU%&-6F z-wBwZ|9dZ9l+XX|ujBtb&@+$!|KTQwIzL3asba0U*3bG`KkH}q^Zy3`0RR6c!E}lM G@&N$KduY}G diff --git a/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml b/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml index ae7e080af9..29d99c91fe 100644 --- a/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml +++ b/charts/latest/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml @@ -10,6 +10,9 @@ rules: verbs: ["get"] - apiGroups: [""] resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] verbs: ["get"] --- diff --git a/deploy/rbac-csi-azuredisk-node.yaml b/deploy/rbac-csi-azuredisk-node.yaml index c77181ee14..9225c168d5 100644 --- a/deploy/rbac-csi-azuredisk-node.yaml +++ b/deploy/rbac-csi-azuredisk-node.yaml @@ -16,8 +16,10 @@ rules: verbs: ["get"] - apiGroups: [""] resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] verbs: ["get"] - --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/pkg/azureconstants/azure_constants.go b/pkg/azureconstants/azure_constants.go index 9f5b277328..d01fec8b44 100644 --- a/pkg/azureconstants/azure_constants.go +++ b/pkg/azureconstants/azure_constants.go @@ -96,8 +96,9 @@ const ( BlockDeviceRootPathLinux = "/sys/block" DummyBlockDevicePathLinux = "/sys/block/sda" // define different sleep time when hit throttling - SnapshotOpThrottlingSleepSec = 50 - MaxThrottlingSleepSec = 1200 + SnapshotOpThrottlingSleepSec = 50 + MaxThrottlingSleepSec = 1200 + AgentNotReadyNodeTaintKeySuffix = "/agent-not-ready" ) var ( diff --git a/pkg/azuredisk/azuredisk.go b/pkg/azuredisk/azuredisk.go index 00579d70f9..ce3fd15a4b 100644 --- a/pkg/azuredisk/azuredisk.go +++ b/pkg/azuredisk/azuredisk.go @@ -18,6 +18,7 @@ package azuredisk import ( "context" + "encoding/json" "errors" "fmt" "reflect" @@ -33,8 +34,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" @@ -54,6 +58,17 @@ import ( azure "sigs.k8s.io/cloud-provider-azure/pkg/provider" ) +var ( + // taintRemovalInitialDelay is the initial delay for node taint removal + taintRemovalInitialDelay = 1 * time.Second + // taintRemovalBackoff is the exponential backoff configuration for node taint removal + taintRemovalBackoff = wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 2, + Steps: 10, // Max delay = 0.5 * 2^9 = ~4 minutes + } +) + // CSIDriver defines the interface for a CSI driver. type CSIDriver interface { csi.ControllerServer @@ -106,6 +121,7 @@ type DriverCore struct { forceDetachBackoff bool endpoint string disableAVSetNodes bool + removeNotReadyTaint bool kubeClient kubernetes.Interface // a timed cache storing volume stats volStatsCache azcache.Resource @@ -157,6 +173,7 @@ func newDriverV1(options *DriverOptions) *Driver { driver.forceDetachBackoff = options.ForceDetachBackoff driver.endpoint = options.Endpoint driver.disableAVSetNodes = options.DisableAVSetNodes + driver.removeNotReadyTaint = options.RemoveNotReadyTaint driver.volumeLocks = volumehelper.NewVolumeLocks() driver.ioHandler = azureutils.NewOSIOHandler() driver.hostUtil = hostutil.NewHostUtil() @@ -277,6 +294,14 @@ func newDriverV1(options *DriverOptions) *Driver { csi.NodeServiceCapability_RPC_GET_VOLUME_STATS, csi.NodeServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER, }) + + if kubeClient != nil && driver.removeNotReadyTaint { + // Remove taint from node to indicate driver startup success + // This is done at the last possible moment to prevent race conditions or false positive removals + time.AfterFunc(taintRemovalInitialDelay, func() { + removeTaintInBackground(kubeClient, driver.NodeID, driver.Name, taintRemovalBackoff, removeNotReadyTaint) + }) + } return &driver } @@ -636,3 +661,102 @@ func getVMSSInstanceName(computeName string) (string, error) { } return fmt.Sprintf("%s%06s", names[0], strconv.FormatInt(int64(instanceID), 36)), nil } + +// Struct for JSON patch operations +type JSONPatch struct { + OP string `json:"op,omitempty"` + Path string `json:"path,omitempty"` + Value interface{} `json:"value"` +} + +// removeTaintInBackground is a goroutine that retries removeNotReadyTaint with exponential backoff +func removeTaintInBackground(k8sClient kubernetes.Interface, nodeName, driverName string, backoff wait.Backoff, removalFunc func(kubernetes.Interface, string, string) error) { + backoffErr := wait.ExponentialBackoff(backoff, func() (bool, error) { + err := removalFunc(k8sClient, nodeName, driverName) + if err != nil { + klog.ErrorS(err, "Unexpected failure when attempting to remove node taint(s)") + return false, nil + } + return true, nil + }) + + if backoffErr != nil { + klog.ErrorS(backoffErr, "Retries exhausted, giving up attempting to remove node taint(s)") + } +} + +// removeNotReadyTaint removes the taint disk.csi.azure.com/agent-not-ready from the local node +// This taint can be optionally applied by users to prevent startup race conditions such as +// https://github.com/kubernetes/kubernetes/issues/95911 +func removeNotReadyTaint(clientset kubernetes.Interface, nodeName, driverName string) error { + ctx := context.Background() + node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return err + } + + if err := checkAllocatable(ctx, clientset, nodeName, driverName); err != nil { + return err + } + + taintKeyToRemove := driverName + consts.AgentNotReadyNodeTaintKeySuffix + klog.V(2).Infof("removing taint with key %s from local node %s", taintKeyToRemove, nodeName) + var taintsToKeep []corev1.Taint + for _, taint := range node.Spec.Taints { + klog.V(5).Infof("checking taint key %s, value %s, effect %s", taint.Key, taint.Value, taint.Effect) + if taint.Key != taintKeyToRemove { + taintsToKeep = append(taintsToKeep, taint) + } else { + klog.V(2).Infof("queued taint for removal with key %s, effect %s", taint.Key, taint.Effect) + } + } + + if len(taintsToKeep) == len(node.Spec.Taints) { + klog.V(2).Infof("No taints to remove on node, skipping taint removal") + return nil + } + + patchRemoveTaints := []JSONPatch{ + { + OP: "test", + Path: "/spec/taints", + Value: node.Spec.Taints, + }, + { + OP: "replace", + Path: "/spec/taints", + Value: taintsToKeep, + }, + } + + patch, err := json.Marshal(patchRemoveTaints) + if err != nil { + return err + } + + _, err = clientset.CoreV1().Nodes().Patch(ctx, nodeName, k8stypes.JSONPatchType, patch, metav1.PatchOptions{}) + if err != nil { + return err + } + klog.V(2).Infof("removed taint with key %s from local node %s successfully", taintKeyToRemove, nodeName) + return nil +} + +func checkAllocatable(ctx context.Context, clientset kubernetes.Interface, nodeName, driverName string) error { + csiNode, err := clientset.StorageV1().CSINodes().Get(ctx, nodeName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("isAllocatableSet: failed to get CSINode for %s: %w", nodeName, err) + } + + for _, driver := range csiNode.Spec.Drivers { + if driver.Name == driverName { + if driver.Allocatable != nil && driver.Allocatable.Count != nil { + klog.V(2).Infof("CSINode Allocatable value is set for driver on node %s, count %d", nodeName, *driver.Allocatable.Count) + return nil + } + return fmt.Errorf("isAllocatableSet: allocatable value not set for driver on node %s", nodeName) + } + } + + return fmt.Errorf("isAllocatableSet: driver not found on node %s", nodeName) +} diff --git a/pkg/azuredisk/azuredisk_option.go b/pkg/azuredisk/azuredisk_option.go index ef262db238..ef13816c8e 100644 --- a/pkg/azuredisk/azuredisk_option.go +++ b/pkg/azuredisk/azuredisk_option.go @@ -60,6 +60,7 @@ type DriverOptions struct { Kubeconfig string Endpoint string DisableAVSetNodes bool + RemoveNotReadyTaint bool } func (o *DriverOptions) AddFlags() *flag.FlagSet { @@ -100,6 +101,7 @@ func (o *DriverOptions) AddFlags() *flag.FlagSet { fs.BoolVar(&o.ForceDetachBackoff, "force-detach-backoff", true, "boolean flag to force detach in disk detach backoff") fs.StringVar(&o.Kubeconfig, "kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.") fs.BoolVar(&o.DisableAVSetNodes, "disable-avset-nodes", false, "disable DisableAvailabilitySetNodes in cloud config for controller") + fs.BoolVar(&o.RemoveNotReadyTaint, "remove-not-ready-taint", true, "remove NotReady taint from node when node is ready") fs.StringVar(&o.Endpoint, "endpoint", "unix://tmp/csi.sock", "CSI endpoint") return fs