From b732cf29965688ffae97e9d63b3a2148522675cf Mon Sep 17 00:00:00 2001 From: Akash Mukhopadhyay <43958099+mukhoakash@users.noreply.github.com> Date: Thu, 27 May 2021 09:02:11 +0530 Subject: [PATCH] Removing provisioner's dependency on csi types (#68) * Introducing the v1alpha1 types and first set of changes * Completing the migration to use the v1alpha1 datatypes * Updating the deepcopy * Revert "Updating the deepcopy" This reverts commit 056f5e9b3f85fc2ee4d1b6301b3abc9a482f3d78. * Typo fix * Handled a nil condition and added the metric for list volume and list snapshot * Some minor fixes and review comments addressed * updating generated files * Merge conflicts resolved --- charts/index.yaml | 28 +-- .../azuredisk-csi-driver-v2.0.0-alpha.1.tgz | Bin 14057 -> 14058 bytes pkg/apis/azuredisk/v1alpha1/types.go | 51 ++++++ .../v1alpha1/zz_generated.deepcopy.go | 166 ++++++++++++++++++ pkg/azuredisk/controllerserver_v2.go | 142 +++++++++++++-- pkg/azuredisk/fake_azuredisk_v2.go | 20 ++- pkg/azureutils/azure_snapshot_utils.go | 21 +-- pkg/controller/common.go | 9 +- pkg/provisioner/cloudprovisioner.go | 68 ++++--- 9 files changed, 424 insertions(+), 81 deletions(-) diff --git a/charts/index.yaml b/charts/index.yaml index 24f818c259..3cbcb72984 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -3,16 +3,16 @@ entries: azuredisk-csi-driver: - apiVersion: v1 appVersion: v2.0.0-alpha.1 - created: "2021-05-26T11:19:12.585582735-07:00" + created: "2021-05-26T20:10:28.055171184-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin - digest: 645e680545a1ceee40f6967f8fc3fd3384af4560317578dbd812cfb17e29c951 + digest: fef5f814fe32cff6627644625551e05423aad231f44ee9bdcb32c85ee5593f7a name: azuredisk-csi-driver urls: - https://raw.githubusercontent.com/abhisheksinghbaghel/azuredisk-csi-driver/master/charts/v2.0.0-alpha.1/azuredisk-csi-driver-v2.0.0-alpha.1.tgz version: v2.0.0-alpha.1 - apiVersion: v1 appVersion: latest - created: "2021-05-26T11:19:12.558065554-07:00" + created: "2021-05-26T20:10:28.036147877-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: fa28b8a63ae984366bce7b6afddd184aaed4ab3f36b7c16dab1b95b84af6f1d0 name: azuredisk-csi-driver @@ -21,7 +21,7 @@ entries: version: v1.4.0 - apiVersion: v1 appVersion: v1.3.0 - created: "2021-05-26T11:19:12.582838457-07:00" + created: "2021-05-26T20:10:28.053348041-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 2665483e922a577feb8539ca7f774bc70c945ce490294fd3378f098c2d244dde name: azuredisk-csi-driver @@ -30,7 +30,7 @@ entries: version: v1.3.0 - apiVersion: v1 appVersion: v1.2.0 - created: "2021-05-26T11:19:12.580696274-07:00" + created: "2021-05-26T20:10:28.05207958-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 2bbfe2f9d080f1b3ff10590c7168d05ce026c5a73332b4d48014610a52337808 name: azuredisk-csi-driver @@ -39,7 +39,7 @@ entries: version: v1.2.0 - apiVersion: v1 appVersion: v1.1.1 - created: "2021-05-26T11:19:12.578557691-07:00" + created: "2021-05-26T20:10:28.051029813-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: dd7066be8f499f6c1a396ab27c0013c09f5a8d8319cc04fbdd480d31107bb851 name: azuredisk-csi-driver @@ -48,7 +48,7 @@ entries: version: v1.1.1 - apiVersion: v1 appVersion: v1.1.0 - created: "2021-05-26T11:19:12.575674714-07:00" + created: "2021-05-26T20:10:28.049927548-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 3d2a5189416dd6a43bd3e2097bbe23a8db347b6e1a36c6a43fd59cc9c9633ff3 name: azuredisk-csi-driver @@ -57,7 +57,7 @@ entries: version: v1.1.0 - apiVersion: v1 appVersion: v1.0.0 - created: "2021-05-26T11:19:12.571316548-07:00" + created: "2021-05-26T20:10:28.04888348-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: d179bc6f338518859b6efdc3b3bed8d06513313e8047563eb4b654b2d417c81e name: azuredisk-csi-driver @@ -66,7 +66,7 @@ entries: version: v1.0.0 - apiVersion: v1 appVersion: v0.10.0 - created: "2021-05-26T11:19:12.560179837-07:00" + created: "2021-05-26T20:10:28.038428806-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 3dbbaca098fe8316de079236598253b5831e8e85fd88b390231d828157d62206 name: azuredisk-csi-driver @@ -75,7 +75,7 @@ entries: version: v0.10.0 - apiVersion: v1 appVersion: v0.9.0 - created: "2021-05-26T11:19:12.569007567-07:00" + created: "2021-05-26T20:10:28.047702317-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: a978f3e6ef5d678c3b6512bd8a63277cb4ce40d3f3e34b80370f0c37298824f2 name: azuredisk-csi-driver @@ -84,7 +84,7 @@ entries: version: v0.9.0 - apiVersion: v1 appVersion: v0.8.0 - created: "2021-05-26T11:19:12.566725685-07:00" + created: "2021-05-26T20:10:28.046342759-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 1762b832389b4f7a5eab9748127fa6dbb85131485d67bc3fe485bbe86c468128 name: azuredisk-csi-driver @@ -93,7 +93,7 @@ entries: version: v0.8.0 - apiVersion: v1 appVersion: v0.7.0 - created: "2021-05-26T11:19:12.564396304-07:00" + created: "2021-05-26T20:10:28.042687773-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 29e21f686814f46c1edaaaa95ce2d25579ff1aad270c58b774bdb5a89858b8bf name: azuredisk-csi-driver @@ -102,11 +102,11 @@ entries: version: v0.7.0 - apiVersion: v1 appVersion: v0.6.0 - created: "2021-05-26T11:19:12.562237221-07:00" + created: "2021-05-26T20:10:28.039626469-07:00" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: b11d8dfee371ca7c63a1448ba27c1fd1f032ea33575fefeeb16927fc95d1eeb7 name: azuredisk-csi-driver urls: - https://raw.githubusercontent.com/abhisheksinghbaghel/azuredisk-csi-driver/master/charts/v0.6.0/azuredisk-csi-driver-v0.6.0.tgz version: v0.6.0 -generated: "2021-05-26T11:19:12.555229077-07:00" +generated: "2021-05-26T20:10:28.034706322-07:00" diff --git a/charts/v2.0.0-alpha.1/azuredisk-csi-driver-v2.0.0-alpha.1.tgz b/charts/v2.0.0-alpha.1/azuredisk-csi-driver-v2.0.0-alpha.1.tgz index 7e546a13d3bb464d3be0ae385a6ce0986aad07c2..a7f9afa4c9d9c4b693206d7e058e7af38fc78d3c 100644 GIT binary patch delta 12377 zcmV-fFs9GxZR%~1wSO0bmj^FiemB^AzW@C2J1|&F?+qp40&|TAmy8f2NFXlzme6$L ztd$%rd%YK}V1GCg0tV9u0RT-C1x)Ps1}%EOL>zy7Q0GjyP#F1MbKeo3BZ4@;V0465 z?B2E!%QluBJfR^vQGDkTO)wV>J_HN0HN(cb%(hpe%tmV9!4!o_gxKi^fk@bHTCz1E z7ev|wGEF#E$X>e@vcnOZ;{Y860ZoXIumlW2frdb^1b^-9un`QZPYVk=;pqLo#Q4)_ISVhHyC`C@P=t0d$tc>}qSZH!^W@*2%i zGz8Cgb|mtLuFs#BYgRD~T;IKrlc7hiAO7PX|Msvb2pmNy%pgiRnp#&^ zrmC=$xPKyD@vI%wjGo7X-k^72p43oO0hZ$e=2rsAg44}raTTz*2cnD`G;NkBP*aeq zC`r!{RD?5wM3c~H@fXP`!ojh8vemQy zbj7Lv3p9%(C=lWL5HA$>I2{|ATrCH?=!%jXAM{eKhXdWwuOf1-@}t23aP zAHW<;kPw(mK#1aqE>H*r1q+%m5JZVo1A03<*AxU(6x?|liD}3I7f>X87sL^Q97#Q5 zxPP+)faFGjh(NbX;0*B?2I$qNPoQ^+RK4E0eF{E*mZ0dyx?PSU6bQ;*b(pSM`3ijg z+yUM0Jru!o%hPI0!}`CSq9{g;_e8w9E1+Th?+x~zm)8Hw7l-@X^}mVo=~Mr2V2)=) zz>xrB93ioY(W{w6OE5)4@VEZw&pSJ+T7LvbDB@sD8JMDI1{mTrVFBX0(x*>d5TY?A z2y{}i>gkU6gxT>J-{Zfrf^Vrl1 zrcjy=s*3h(=bBJ2Q`H&7F@GYf z;W6mF*B&>e4_{=Jz~|39;K(IDKVgQo{gohm{hmh-O0y^V2qVlOa9`$ zyMOQ=rv3Te&K)M<5F965(AlMt4Sy$+2V#LK*_j~$Lnxr~HWNYxl#jV_)ubMX?gWvZ z)D1@ojzYv#<7r%I&^s_@JdAM{#xbLF6!tQ^yNBrzbf!YYeAw?#u$U$z?Vl@}=5hp# zz$ow8=}Z3H-#hqc$9mH+CD(X{xPY^G2uKn|JHT+&IgOGTvR~&=$91^NAb)0u$4GhS zCX6QW5csO;W(_tpCf}r;j#=6i48d?A4T%u~ z3D-6SLlPMXKm%CMI79x+DSy!}jGne3dg|+I`i@)Yj-HFl@?)vQ48e%f2tne4P^kcP z48#-xO{w7G{V^CbIs^Apq<{$*1kP{(qi6vHyh8-={9MR#kE2MAM7v%?7vM1pdb)?6 z)TA*NhzLD<#}ULc2}xlvSE};G`{Tk`xL}w}nub!OiepM-XW)K{gMTStDAv9t4j_mb z%Ee2c=yZQmvMwy3C-J0}JRZj*virH5;z%x|fGJ3h1Sx88TShTS1Dl#`I34Fl?n)yeS z7XpPb99LiH0fbN>1AeYKEBksCsZwb5XyR$ih@8C<641EhgDILp>zSr8B1acz?+>mD zk3kWI#p#iFf|3WbCn@PFTLxh|=sr=_tyk;6AOn>M^O<1kVTg$84KF(^S~7>t^a zLc>6fhX6YIFA78tNV8O0c6>^cD3rR&95DeH3h0F3k7?Un4pMfilxIOJrbCcZX{^o_(Ag{@SSQz7XBr*;GYy=tt>@dlvYwR15z>3M5RB~qsS1gs9=h~ z<>}RR>O4@BQO3T)drn|wQy>wB$rv#`iAr9}HW3LGk$-Ao5#fxlq_mohW>}<@F6XKT zj+G*+bdfkz5*(Zna13WCI))rQ`UFT!^R7f}yD6~Bq;Cr3`vUogh+~FA<>!EbKuoT^ zfa?XDF5gsem2fs%7!Kh?L_z89hcw_44FQS;?;8f&*Nh-lviV)vGiLz_lVej3Pzocs5Ifw37fFiHRi0a8XnVoQ(o-;cCgQN~~osm#w6ZdjPpZ#qcz58q$-H z*W)aM9U+JA?VFOCX>nKgEG=8Ss%?uY@P6x=0)OZkPeDw&)ix`e);DzhQn$NPixaD% zHY~d*p3jtm)~rrDM$n{SgR?P^{M=$Y+%-&a43kJy&9|%sN`bd*KpCcZ3@AZB%>#Ee zEd{txL|l{jJlthbC ziwQ@CF{Rp59g8utZH^coFRFY)Orch~te!6wKn0gTY6#YHZ!Oa6Ey*UHQ~z~&rVR^n zEQv<6o-f$+9**CI8(?Cp&$Zy5(J3zaU#X1>QI|s&VK>< z+gu>gGqinW;cS>503%9VdPPW)25^St`Y^nG&zze+af|@ZBB6rV#+y|njS|%!mwBX%(*ntmSAW_pK=2s=c`(Q zDH>cLjsdiAijvB7ZA@9PQ?0Pbx*nIEthkl~PUUb<(k4-AZ^nZc(Ic zMv*XL6mgYM(or6lbjmKZCeQ(s%ZpfucIq^4PLWQChZIUqQ%MmCxg+$RRCNt4BeVm`O@CT7RE`*< ztbx2$9ZS~{QSFDkm`|aVvFfw==i^BcHVz(aFTB<2bb;<&rT;8Gs|gvB2|!_T%8(pxHVUPR?CH zn5gI4n#AuR&!f$4vNf;7MyP8HyDXm8uwJu@Io8t{+;dCRO1RG@aew^s#kHzxB#wgV zR?ypoqmEC!INEG0a6ID0sZurzoH}S7O|Bwu^3ufxrZQ6Qu}7$LqmP+MJ44WMpG#ZU;u$PoO$ zr#FB9z5DE6PoMrW=>GH9zdya{sXzYq>|f7*w?F>=?Ag<&zkmGv=Ev)c(_itk-+v*= z>`wps{pm00^w*Y+JbU)9zg9r`(9NahNNa|&u3oaajYjEYxnx<@Q`A)`+VOs+Rn1)c z#sh3Dn`osOnpo06k`29)SL8a@QS{JNt%M&Wy# zYa<_S6Ruh$Tz?sV;-jZsWv*yn?v<+1kbq^N)r93q!wO7KCKu~i@3B}q5i6HN7vno=%KkU(svVu{w$@%WHZ!$nLPO(xRYHMUe_zfxWDpT4TH2ah!jHxXL5E7&xa0wPJiAJuAR*Z*E31-xgAEnr3(Zc z*?xkCsEm{^+&H+^7piwTh03jpNrhU{sFkB(ly4d`1PCSB&Y(=e*yf0Q%{d~)rC2i; z#fl(o&1zgO)gt3riQ$x*^|K#vrFG481{AS&E|GwkbkvM>6xTtWvf7navgVM}$8TAx z_GfSqPk$F22Qc#49F+K-xeB&B6pLO8RIAP%p(rOP&7GIHwk_@i2}F`vIozkgElH7(=EwZVViSRUv}sM({WwDaO7load{iYS@zhishjo3N|H>_`Ffk!Xr` zn>|~zk)}GNOEm70`VoD*E)BDWOVJh>hy+HULrFKp{I27&dWxCIUSi(54894M!Pe^e z;#NQ2yU%ExP+|c+%vbm?0?T=sq;5g#?kC^YQe~}QtK6IqjfmiT*V{-P~$C70a^$ z*h>Ae^=6UZaRD-U-bN|-Bn6jA`&2NW%*!uVxrn6aodG-=T;Q$ z>NDG09JzJNZOPfyKuC=z+k0EAHG=^V)6fk2UJX|=a4GYk zG4msQhlZe|oMRc7pCt8vc4dgl@>DAp9a*W9slyeWE(20q7z|G)h&hv4fziK^G_#)p z$|ME|%{4*H&gL{XvjX%l`@ec2F5(Md5~2@aUn~BYODsJzE0QmWfPWukPr;OOtJe!W_(Cs9Cex4fn{TED@^q$p;I23q==_Q2wnC4+$14VKBk1 zmhZc>lg$u#f>&CrUgN-btbr$=0=RN2fbSrLZdEOJ_-|cP?jT zyL0(ZwsW~&gPCE1XUS{`21_Y7O;NwPh#&Tc`f{AW#gQ5#d-X{3hI73ucPd4*KHXW) z6AG0hQ_rx#QRLw(`@0E7szFKgTE#OzjD*|52>CP$O@y}td*UK6O7`V5L-`Z~RT4Cv#%Y6oIh zfoUMcL4RxDUuZzu^;S&6oKh~%k+`Squ5buO@gm;y+PUa*>+~TsL{|lLDDGGAj8Z<- zcgCjgOIHKs5uM!xof~1XJw_um1T)A*PS_fjnSK;AOc@r7EJ4DEzOK1pvKufgayvT& zjS$OktkP2C4lRbDqi))j{TBC2^)(QArHsb16@SWxpmX|L0;BvprREJmM`Aty9F4~) z5JS*8r&s3oYWrGYs!9@t+uq!Ckp&0y!^zfI`AU0T^tH+Qwz~WcaJ}MH`n~MFV(|I% zu&K6y6LTqEO0#Uj%WmpUw>w5qBn)*YQbfGErZJ7^WbvwuFjHHRXBbKa;?-ZD9-W+A zo_}6lJ#$o?zuI>^6?ledB3|u1=Z?4fnyPMuV2D^(kvOKrQL$8`t8O_|oXOej2f<)h zXQ6fN%C8I%k$D+AW*>zfWJz5zgw;0}L-#lh@@Fw?ZT*d0*A=AC14$*JK&yPUkr6JGUtf8eIQ zw7rzFuCh=4bXfmcTbgW3evRy~MBJNWWHq#V-2S5q&j`;EL7Yp4vEYdH5}Ezt zLVnOhXT{3xNUf;(`6f=+R8Xf<^MF?>Oc9L4v>Q_j*e~(^@4L&>+vCgAlhgC-v!mCyKb*Zjb;-XU7!^j>dXtgX2(EJCOWZb<6LB;19!o-!Bb}`i_3fNH3yE z2s%d+_$vW}yJPvFQ>MKlGEdLnpIyE^e{*_%t+075Do;eS#89>NJCqW=aG zI6>iAh={;q(Y-=KU@|E)0fu1smP88&6{XyxG`lE-RC+5CMQt}r zQ|3wp<|eMj@mDD=M}J6PApWyd{LkzYUMBYUU1R)jFxcNaEXDu!_n+@?DUfl8`LX|5=x7tmQ6veOP-yB{!RqcIK7$e6w*O4<#<7)Bg@yFv7C zxGc5)ReA5kh_{5PCvv9abCn+zLvtF zR7D+c^Qb$g;(yG&@m3$e3^BvlvBb@==!E+m0$_2lqj9f`r$f)Y=VmJiL)} zp#D>K#ngE;>QLzOZhZBoafUy&=8XTBp$X=K!KElsHh-YCDjJPRvaBZaX{{KRQ?P9`Ag37ne5byHgxOIv=Pb%3$csR6+?@k;{XrGggSyUAin0K7C(1450a zUe|k$bs6Si?Hl<8`25+LhPS4J#f<|Vf5_L(#U2LHQUEx)-dZr?vX$#Vd&~5Gc(>J2 z<$tVoKvvjn84Z7keXZNlt}Gq@TRQsR9RE9beo%`4?Y-Q4zK#EFqLku)Y6s{eME}0Z z$ls=-e(HUj-Mt~<)-Evj(EWfJ7yrg$`jh^|R9rS>xXK3LZRoDl!!~rcS_N+CH-A>;>#lrVX>hyQFl+u1qJB$8Pg+_IMbX<3I;r6n z4})(dC_Kz!EtpE;E+p@&9h*wm$u zNw(;>mSZ;d>*S}1{l6~<2m7V?-+$rr=P$PL zzfF|#{@-lFuQeAP@jdual0!-jmKd@;gf_80+ob+T5z-L+t`>0~ASp@Go_eNox;q(0 z@zf;xXs~}yM#yemV|olp&S{snbY3?Q4Ma4d42$XPs&K(|tnH$d*`#Lmu*4gi#$V>7 zKBUIkePH_dwNrEYn(Z+6w11#FMl`s4tIkqy9&EXG6Yy&#n{AyH>76$_R>G!gxp4n;*Gv43|cuMy<-Dh(7~ zn80Oc7j*6so=kXD%Mw|OqS?Zoz3j4Ul_Zk zB|Y>{rm0TCq6>z`_(Nw1I#VIyVZSdykEB*nIG@uJ<*5~HCd6+1mwgZIb}ZDLF^&Xc z?-3ixzVxr`t(}{X$bZ(zo@BMm@P=S7eWX({ei9<)9dk}X`i{^L9m8OXj$tfo*$V3Y zXN#ICWZ?x3ZQ%ZDG1Gq6e`mNH>;O<*(>A}OazuRVmECel0}$oXHB=IXufgh3@cXS3 zK>R@ZvOo-HEHu^FSo7xYdPZEbn!9afZi6^#*+MmKzym-x5PzAzZ^B5EfJw1Dp>z_V z+pMYnbP*#aCHZi^*V}v1J1Bmc_cxx6&#Ac3w-{w8F9edhN-4;+(cFu!8O#Bc%x0uSduYEb4e@cEz;|9P=J z|8*m!B(w1%NCtn!V6ct0B1Bpr4W_b+jE({Sp$Ehm^(o9gJxoC72s-MT3d9d zLIm22AEN+|vC+bn?&Vv*R*%t)Ya7qm#&ZhT-^O#kns`om`K~jnQ{#1#e67j;r%~#* zY~ZUV{nW+!b5xh-w5@b&)}uDcpXo9Q1roO=iE+WPK5{Ipbe{Es5$_gmq>l{a^TPUSQ<@A#cW&^gQ>1?#r<`ilN(*jB6 zgh;9cS_xxhEefkXE&obHjO7!MooG3gbpQULol0O?r8Rf`lk%^Isy8in!+jB!uadBQ=3_OgG1)d@9Yj93qRkJMmvTLFwESLRAC z6QRWM6h*O=F?R^?gn7d;c5ao!lo$}@bOn@*mE%cy)Ht5u2r~IJK(XN9shIlOoKh}i zOS?vZ(L{w6=QuIbVvI-f=;Evgv@A{8HRs?`%b_5KoNFX@ zySDc(og>C@h`>046PMf3AXVFL_c)3``iat^_jEwqwfIL30Y zuYEBtUCiQPnCLNmwiu9Hm8zfz(6D_O+!}EIK5za4h1?PypH*qaC07I zM+(_k1%z%16}JFJ_i&+c%Nr@*F6W`tUV#FR$%U`QRG_?j;N+*{3lP#7#A<(S+*9&I zDh0icv(bh~u0Fi1deT?R&y@yo>=oc`zV-%+8XNFOp}TqA!tpj6wc}+vCf~ba&TJW2 z=L(OLQH1&Q%W3!Dfc@k%CMs6LHU_pW-*{O){?~5r*LRKazx~7g7v=lE4qhG{ZsUKO zD1VCh-}<9|>QOb;=Z*0B;(C9rqIqkMj(B4tB&)IwfNTRG+W^Q{901AI>hf_9sUvcZ zM>xV_VLn#L2TeJ0#=u1>=&SiSfH4f@nt^|$F`-ns1WkSG zk~MbSl8ZqdqH#M~2p<9q{Tc!O=H1n`%{U$5$waYbNEC1G<&#q@EZ&y47uRQR&yQaB zz>yIbbPNVkilE;T80lRkAR$@6M>nYI`h1nw%GYli6B!f=-1E2B5?q2NP~f?zjrweE z(2&s$at8WJC><#;eT0A1ZVn0m2v17*%rNIc!YcU8Fz0FXOX0I_xlk3r(&#ImqV2)- zarKMTySZ9qPd9kWJ-s)NuiS<+E)5>!AuP1tSfLzd2KAFLgCEpErkC1Nb}aw)1=ig- zh#`YBBnsyp*|HXIVSxlL(%zb8zGl!|pWw6!c+Fu^4*<3~S)+f@q0$F2zXR_s&#Yjn z#PZuaJ8iy&Izw7%ge?V|s*Lj)1@j!AoWgk(2(RMNq;zOS^49GCYqia+lh;%0c|roi zIi3)mOL3$CyVif(TOu0V?dtP{-NBW1dYeIG-B!H{9y2tmD6DEh=P48^-Y4up3l6Kx zcuB}4A&zl`^yzb|lVv5bE=NS~Lbt~8!&Ds5R+UxLc{`3^`5?B2F{|&)J&R`b<@bD> zX{jBtnKB=?z<+YE{Rg}``agE#dTpy5sm-l(ZI(;5LT!IeyOze{m@<<>a{Sgrf)Tjz zZ9B|Ls>nnL-q{3!v~d+SCW0kgpb)4VpKy5tf4N8$3Mya(IR1$8PA3FKI>lE%9ld_7 zIz}G?RKp)+IRroB@9TQ4wYYRy%KGscexPAXEKvGHjt#78K$#|H-hgX`ZgnUcg3HtY z{qF4Y^hAH-xqjPkvkvdx_gfSfbya5WeU~e31NMatT^d-aN~`3W+3a?{3}-#i&@T4@7KwJmRK0JJ-Atj=(6j1#yjxnc#NdA6ldaEm^~ zP-S=B)&o{|0uN{iAAlvPzTGKg{(;qOEo^MQmUS1%-%-JXQAbVpUc%b3ZPSpqW!2J< z|C@i_)T3_!@7kJ6T>#Z^|I^@L?|CWzcmL(f?fE|&DQ)t9P4@4no#I8x**(ho*5x7S zq)Dv$vCWC=4DI#I>T2Jixo{_?CVLsPxe&?L_SaXhKdW17vwK*!oMDC1u>RQy2I@_& z-qN?gHJ$%;__A{U%izUeyZ$#(O6yN6U(CPfT0}8iNfnPe=L6jyX z-&C3-Hc}NPNOX2VC&FC)dk;l0m49e-$iGzh0IBa*=rD*Np0(~(XFa?6GHRJYwX1(; zQzM8V=Pk$Ca!hD0jo9TeXGXINIOcKxqeoob^+Js2#HhU6C^u6^%Lp06U`_EDJbfUiMB1q1$Z zuY(__-PnFjcLmfN=r+cX^9y~;lUPf0j0>E-uTzAK>13P!IJS zYuQmehP3YN_JGHk1r8u(}w z*)*KxMDW*1aIL1S9o2pMWL;xlI&QPBvV8oH&=4&Z{%?x^@9*uG&;QzgvA2!?Z=^KD z|Go+?tF9<~9LD;PxWku>!#&7pxk|vK$$EFi64n>-STTOF8L63)=jMN;rRwr=I`@`HuVc|Z;F|M2-?IsfnZ;mf_P{NF@rkpFomnI7(0Vp1+U zSKZcX3B}S$OTPFBo+~Ff%c~UjC_|q>S}yPyP%_4o8H_iOp_9q$7J_=Ip{EYwre<_1 zIQXS=myA%xMV9I;^H6`cS3B0OKDBi&Re)WoWusH5ybjq*-#l1(9#VKc%GHV}H7Bvl z#h97c3Tm#|#me~pTTOSoLBn6V-CC{pfi15Vpm)ZgQl3$=H=sb{Bzg!|VjAIK;h~uj zbC#%$c%vkoJfN2)9O{zpS90yIltfW7*>=H$vkuqc4R^%jD*b@3 zko8(s%SqqVV#*mPZH$e2|8kAAU&7{U_Ej{gJE=+ZrIoyz(pLYsVaOHI{~PuHz2`40 z=l}1$-0J_EC`SK>Bus69G^YX)IEs+th5h3=qXkJQ>)$cpp}gfa_q9ooHAjNKu|)qb zl&rqZU231Z-;#f{R_?ngVd_zxt}b3+RfwhCsjkKz!>8(HU|)Zv+XZ@rV2BusP#`<1 zXT_UOQ`9A;atyDP(~fOy8Ns$}%eHJwbNT-P00960 Lr#UM}0Hgr`66Fb; delta 12375 zcmV-dFsRS!ZRu^0wSVV(gTsTtcZ0p>`v-^Lfx%jOZzu^DkbO6Ja9`!YeIo_N_&s7A zQ!)hey&V|G+0Xsnpf~8kD4xRJ-cE>kz;GoQ|HHvvW&Iz% z+}p1IO%yzX6Exfb*?N>e0AMJl91F@8Lof?iZ-xU#IUS2$Kxh5CfAD_5alb%4>N^1^ zmDMYs$4L}j&MAvhbKQ*pr%M?~xZ5O93K=p4(Hk64@TDd+vWWP}(&0&&^5gr*y3 zt>j?Y>%C|N!+((wFql3F0BD*hU}C>FXwmy6;`rl(I%m3t!pQfU`;PD&5ybfgqa(Cp z_qL5#wz2Hs2@TPS;yagUg1KPuAy|;D88+5sw!IQ%Hc|r*rYKAz#7;j5M8bB{lC253 zAkrp~X~MBW_S&tG9gf%>2k0mWXhMX9C13~&Gz5YrXn$vijbKoHT3FBtNALGFmhJn1 z62WK`>CrJ1M>v3d2=;dXm?6P%z%M8hL$LqP7lT7tB_VIg8_0!hV+$vx;Hh`tF6C3_W`N@E`y9$3J#7&X1KRDbk>HjZY9vp1-|4o$ZDKf_Vi8AJ|&VXiq z0CO-wLSQlhA&MiqKp_wmENH?&5G7I#=;HC&q8KsW6Y=Va|0c?(PyN4vIi3vx zM*@s-gv267uVxZ0!4wU_-};|F@9d~*5q}(^h=Va@V2Yv{V2IO%1&HfPpFVX#h{l*8 z&`HUvr$6d|?&r@t`fsVhOcBFE>VBY;U%XdZ&y?#|*(B;|sg7OXs)DxnOEoUdV^b@b zLTNgvD%!K1YeKzDU7oo*=9#J{M}|nK#-LFl)il6Sz>v@g39J{i78&xhEB%$mh<~hx z$DsFKd)$;he34ZGpFi)EpAFSV43Y_|{SJUG(5_ZFC}291!*KuE{rve$6XUjQyfpCt zfQ8+>`MZ)&xSs#N*n2*BS>*rwdj~JJ{@;z15(7%-*;wj|Np-v3oqU`K##ni^n~+Mn<3++h+9!EwR`on0E)aDO6sAQqUCof#4^gaRsWGa*z!`IsA5P3nQ@P7vuy z-EfrPC`3#(p2md+y#r&$!x(pA95XscVK1}0dzcPEXDUR@hyDHpi)k{_{<)%QE=Rx! zjPjnHzU0sSy@P*ttTzo)a*bz*3pk61fFx100}NN4(igo~dYFJEJ>E5uR4U{rxldvP$CBoSogx60#5V>AS* z${p?iV34jr>H-*sY8qj5!7ves9m_WC^hsft3lE-=5C@WhNZNwwn59j@5DXX6kQgD5 zaBWjCB$0stG=TMtGvvRV5`XQ&=xH0Gr@p?X@3?jD=()HoKbA_&5R52|5F{=Ll?p(| zKui(PlnO51AA>QYGjKme3Yc&~;0y;aiWWe?J467_&xI`aIEv&*wCgo=0Uo2Er+es0 zO&W87h|sfl96>yjkQ4@Ur7B;%KQ4@g3x>(0X(&aiIHp8)2JWXgn12F>V(m-f0D_pI zT)ZSq2b7X${Yd7nemo;7)oMAa%M4mMuz=3u2nhr@{+LZ%4X~D86lGoEk7a@>KW~CZ z8p_0v=?HR3+rK4<3lxIU!cobG(|NV2Dk}4D*8{R44nYllh$-eSo+At9U_=w*Byf#s z_1f8ke=JK8M9PQd`hO&9KrzKcbTR1)JVOO8qnE3hg@lBNMGM=mB6E|Hz!zMgnSWGy zAy62@arK2BKnMjg;OCmNvaeT>Duq^$CZ5KO$k`hq0gX#On4%f9o@p8*a&&R_{@|+c z7!*-hoF0iMD0wh@l9H~nWe^rq#EbwW6Ui;|h)&zixP_!J2!As6X88CnpbU&K31usS z8n%NzV6NINFJAy+Mq|VTwsKIHobHi(S{mCEIecTYX~Ro14kN`-XfUQ1gAzoB!KnEt zGz`>u2%w|?qCoV3G)tvr$EP%jLaD3F5fgx+fKCYhn6}O3AZ4dYc^0KMmDD6K0xG>? z7eEq%8C(E{WPdLpA-VQU1_pQ|#UdtSIs`eD#_C)Foy`)0#iFm&Cp=07WxOAvIg0um zPr8r=Q!G#*5{CLxVN#$IrFZpa;a^e<{z(DW%2L!tX(cr?AQcl$RQgjriVV?;3Z@8L zo?cz2&I2_WW$Y`w=LA+Z1rlMHj1kk5sN}V56Om96secw05zhEZN~_6ehDA#0a;|#d zSSg}P7l}h9!NC~;$8d(CW605?Pk_WU?@GkBn*ys$`ldj>FOYwTIA$nRehwH2#N^rw zxL&a7@=XO-31_2);Sf$l6qN3MNCPg>5TID_zG1L^%?MH@oBze7m}cb?{gz00dqr%x z`GP`-B7dUt(-K+?rAg%AF-APq@KOYn^f*OVy-K41Tq~l(C{l!uXR|~|I|;zCNI51Y zy=)wnHvB_TZ+#k~t8!_S-fwdQ7e$r8*(d-Pu9obo#9H=p*-GlT2ar2d3{Uf>Aw3Cs zJzA34h7I$^e(z3;?+P0Vi@3)>QfPbFx6vU)kZL_jzeM8qTb-O#YII$XP z!?Jtg`AjKj&FZvc1WgJyI2!}W&n>pYUBd*&Fo{Ife9KCp6nM)9lwpd;fD#1MJaAXj zQh*CZ#1#n!R3bs8W8KZJrS9Bw!#OkrV~8Wy1fI@fv9Q7wa|I8KB6=@57PTT#(n1eh zzkfY>`}6~bWbzw^Ze^-sXtZ3BIBr`m;NM&8a z%^lLRLb_Usua%doM&2}5`a{B)Tw`+7Y-aG17ntSAIe&Y7IyBvZ01`<4ymyz-TBTihmIaQ3#Tl5;fm22oeT^MGs^NJgJsSNwg@n zm~d1WQ>s1Hu^1!U=7{0(qRKbK6l$f*>iJRuRB#EThF~rC)*`*$l5FBR^Glkp`3L=0fL4Y_PCsKT>4z>B^9Dk6% z%>@!YL)%vt&W7m$FrvhzSA-O40B1<94?`>|Rul&+((#*jSJwbW_i(`t^*Ndy;Se(v zXwPKY>2VY!42uQ8oJ$jJ35HhmDK`*uzN!_N zqQM2?7(ffBNZA=8EugZ1a!E`pvVQ`|(QZ!mq*8Q@qe#k8Db<8tCms9Mt%SGZ7DdWt z6bU0n5myN%9p!OJr|eQ|0v#~9yf_9@PwasoBujKD)h?$)9mEd=SAMBLevP@X0P`HE zQngTeiQe_%n6)I1DH9xEr%vPM6zPO`NTK93l@yVXJ3{YCRoBomLOY<`q<>XI<%luL z8pvDKv2-0lE_o)GHhyScg$c_~S30_~l#~UJ0VIopE*w}xVv3@8oJ4XZCvl7zh@^3z zm%C4K5A|Hl!dPX&Y9deTKa)*`D8M0-%g{)4w8(jf1SSar6{nIc_Yf;G)aqJpHgrV9 z+MgpRHBhS8xJ0#_J%t#jR(~vsWqD(BhB#02Z)<1kiK$xf?u1A{99gFhA}Ti+F*}1E z)5bTiauQP#w^GM;JA(@_@>#19olLAcj#C>_E*X=Q0XQNY3p_7qKMu|fnyo|Oou#GV?B+*J-0-yg!^0)$A2$hT&t=^;wYGI z1-(r;>iEQqqs_Jg$0J^xDrK|4se{(htSyFJv+N*i z3CykP$4k^c5_a^^l7FcEBwvuFSMYXy`K-CSypv}P#l>Lr`oXp~NtOO{nVMO}rW9q(6K)y%bT zY@qy;WR!--Y{SN(1U`M<$$U#N2v96gcwWxm>1YbJDLLwAK#9(y;Y0AtuRFS96u!5) zHuB*%;i^T#m4ERkK6=_!=8E>^Ua1-l30MYNO<0~ZtibeSa3B+b9mS`;oWHK#qMOUg!F4BXfa1FI`jD>NKIdT+u}};KqQ%!!+jd;(nF2N zRs^XJJ*XkXbh~3YCT&M!YAB3NAqgYJHYzb+2i5r!LAD82Uu%L@CRmFTk=MF>K2@)` zp_rzViEcHOcWBYo^EiXCHkwO>XT@=;yIezR2Y;Fs>DiV9a6LsF<=GUC;cp3&BIxH# zfnUwQaw=WwMKP-JVB2eJZG8pSmKF;eFt>8@$7^uuI!z{*0lD7hO1pcY)%A^Se_3XC zMe5M30c*Kk77~FI!y=kZPRG6kpNf<+oxFZh8|`kd7cOIvs%;%>uu8rAe2%ZqtaPwc zq<eWh>{6^$i^AJ3A;MXjuaptiKb|` z*|RkpX{tlIMB^T*AJMn#(lBed6m5ZlNMHmylypPP?>a84r}RUG0K zkCY2mua9auuhg<=XyKW1G`WNASOo^C3$;H`<__-jUA5E8VFeH zT#i7{-Wgb{aXYZFJX0|FiwgbDG>~-L70iK`W>Og5D&hPQ34PYORAHY!(+Lw}XG)%0 zT6$b{^HYP@)pp<&4qEVS!}}6D@GLApDMxRMRKMZ}rHx)HtHsv1#bL+2*+3Rg&P_|Q z@ie;DJ#np3lCSPUh<tGJtDU4%8ygN(FoYv9us(-xKXOY`d zyn^K7%v#|V17<3nDrF;&QzKRWlIFgjoQ-v7EIZi(BRv(Vfe$d@0>Q9`g{>|M_SA9q1yhX($&xm5wQz+BDp8G=K);FH@z7oc#OE{ zWv3VZvj3}(aZ2JvAK)yG(0{H?zf4VPyM{2==tx^sTCYP>GZ+9d4b8Cc)o>*Pmog6; zGe5$2Xb3vWIhKL>NmBo3SB9u8Pqkvvk(D}`I$Y7|G9a~u!SG~)m@}Cb82t-LGy55! zOk#l0Toc6XY)*4CD?tCU|Em|`BEA47A^HIJwc?Mt#L_dfBKd*{_YCdBSW9%IS-^v38M z_wcmHlhltIOx1yw#?TojB4Onnqv5|lmKfxpn|k}%u&gHcz>lqS^r~u zIrikrX4{p%Eonw6kM%4j%)tzXnl-E4aF0yN5}|6Je6X;$P;?;)<=^`7kYKSA1{2(B z`Mx_l*$javc%`-KH4c2o8hG+4fGej0_zptoR@HKc|9AO=5`WfmPJ&cgU#I5;Ym8vAopyf#4m%9{_=bauiS!a@`^#0S%q{BIHtE!bH3{?18sL zeZOmSEoM?d`cDW20)PJ>{y*pcF^DRxDwnB@by{^E4vwYTz4yDK`mdu7=u#|Gs6!oi%_+fvjFUJX79H}v~SC2GrIM=&!r&2WQ)1Bo! zp-?$8^$ZIfMQ)xDGdf|2FB4AsGi=k&-wO1lukl#9G=JUy!f`iOCEdla)IBiu_rL5N zz9`@SwSTa;z5jnRMK6I9b-~|ETAjt6Ffs4X_X=BEbSkiqE_U*6UEsUTZ|Bn|fJqP~ zq4TP+9{61aK!^bQvbN1m%x)WsFHGz7m&!7mVuhaX-fIi-O(b-MVxe*rIV>CiTFoRs=gsowj=|?fclwq;R5+sc1>zW%Ty8*)@x3fdg z2(kReDlJ9s&|(NW>ZV=UZ*jj=Ujva>%4jTGp?_=$I;X!SFv`DEYTgiZB-ZoK(Rhpk zF$A4+dSz~}wyzbYsw7dk?afUWS#U5voNSGiue8@iUz@CNtIOX2*DGG7-^=bR2A@9< zn`#R z>3`MLGe^bwt9{2)foF&&;?>@B?s%)Osp>`uhKO|)iDOC}6-za`>Xt*rnVijj5Da#8 z7FyS?{K^0knU}F+_EG3TmeeIfSbcLbbdS>@e-^XW*5Al=T|xRxJ~fv)+k51*qC1t% zA<+$So>D5RkuDLQ;c_nk1aUG1dxJsQ)PICwC~+ zt&?qO{m4{rLFj4Kwp8w<#&Dkz>?SB@G z&DLBu4LSK=17SVs>fH^mirK(yp-Xd_O;c3;_v8@AvfZK?52M!S(S)eba4F=Tl!`bUom;C#IQO7R1fRZtuyn*r0XyM$p z0rE^iTD4(!j3Zac;<3Y!;4*FSkQ3M7JDl3-sUiOYUe$G!jZ z(ss|>)%n@O^a6gf4;j?(&N!`BY~xWJ0OgNMqjffv(Me51bIAG;9)I=kA)Fv0`fnhC z6BM3>hzKke-76#nCX+G~U~t&kE+f_C-JC@z)mnzk+kv`PAZvFE~p8op$Kj{eUSOjcIsB#`G;w(ryUGFyiRj4WfU; zWvTVA0&Mq~g!G;-!3CP)|AT{<<@o=>i*5XWBcBm2J4x4PN3WsJLR4uPQ{i}SHMA2oL{ zlWgax@-#~hJ5h=BrguyxXA?rx$EP1qkci5@x;$*xh|MYra#z#C>4%sh&Wo7K*HSo? zs;J{_9(CtboPW7D-s%IG+^O_xdlqIQfswlMBtkYf&YuUNroLSggm-2iCq2RIa&BqI z9uIGPmaO)cm5dRG`_gi&r99eoAGLOC7NaOlKFYFt+fk+L;67+hknsAGTDxJ6hc|K# z)PKsZm^zO}9SVKkjj!G`&hV$!obmrMG{IajxD+MI27k1+d}%*#OOLKkmvFYbGH8~g zRl}Ilfvap8X~4dc)G~+c<|e|Un;TQ_=0-18esfbb12V$e9NlKk{rv9x)7$g6C#SdP zM{kz!r>ac?9!F|+NQe^QYPw2C%(P!PuDM>tIjDzgC$2W4)p1T@nG9;UaH%~96Stoi z2&k!%e1GL{lWBaN5jUj{t}WIkTf^b2zdkK4trf#^3bu^~

9n+)&(sQ=wUBD9(X4 zw;O#Mg!{&WaNVLUl`rcI#(gWXCPt zh54@9$?|H_pw3?zF2g*meIvgBpFdmE@YZy&xN*Sa5Ba*e*ux-N3IHe9TMI^9wsIY4Z<*c?@3uOs zoPV_r$O@Y+qu~#+uXS76m8Iiz}GE z6lXHRV-zfc^0`1ULsx>vIG3e*v z?N==XIUYs3#a(cnUt761SJ_pUeeOb=yDf(eq@;?mfcnM)a*S_d`}$UiQT5jrp?})$ z3EM`fwh^jrgzAqFp>n+2%r>!YRH}lruV*70M?s=YB1#1MAj(=3p5q8j3dj1Hfoxy{ zT$6g@)BqTb;RNc#AHUDv^5kY~@*92r{2y*FUa{X0&7EI`XpRHaFnYe={ZYdETrP6` zMHxV1&FQXr-=fJt+Tp6v+bQM3jek}7x+`B-8r*I+%$k3MsNa&&la`i4QS>&1PHMQt z!{A#93JjgSzALNWj@z53J3pd=%JArHg)M^ zk}dkJ<(Q5AIyvcAv!iw!%WNo1#s9e36U4gcgFqx~xfi%$|L@De!G0-FJj5s)M@(i#V-aKpUl;vs-5r$Ecj1S3{fU{=ku<& z(xuhNot4h%3FdbaK@kpwv&tJNg6VlKB!i?+llN9RR9p+U9puj)-r)vRf``0HR#FhDxIFHCR0ge!q1B zh#yE_7Kq`Dg{B%CYu?;l&xlJ_bGNO`Z4gH-Td1ZDcmU`IB7f8OO&DnsFe#QNlujaa zn>E#+E@H%_Bp=TAdV4Q=2gNV*{>HQMITaWB7NZR1g+NkQDFvA}ntRbTqj}|9u>G96 zQTT5DDvKGo9g$1UMA7k6Kj)22S8-hp*;RR1L+GyhK{m$po|WfP@^|HqzPzv}f*P5z(#=Y#V3Px~)lyno#Ke>PDXu1ipk(+AuU<)EF- zUlElK%KmZu@H?a21AC-&a(8F#PLXna`>?$5?VU{xcQ*NEOP8hY6e$wwvR8}bzJA~l zGNpSTUMV+2zTP*G23c>-w8CQ1wE08CmKx-LHP(I$Kk8a?<+eKovSI(%!Qt~kCH`}` zJ^ythrL*rMNd|xU%NN`D&qm50JpNN9)%8VzjQC93uEc`M6&w+v3VfBgP`SpYqeJCE zSf8d8k?LViXr%6i#|RhY!PG#t15~Lmh-4dT%fKZhL@ZihGI^i(DQO_zxnC|F+pSR^H&wKYP82XICsR0>gBwxin#+sKWHYVRspUCrnN

0Z9|YxNk-xVG_}Z9J!d{cSwwtBL27m+v~GIyGJ=$=90fe;TE3 z%Lcw`(obEiKSy$PBP_*WO? zgp+;b(E@+$>mpLzFLBt4_}3O@tE_+$zRq!!x`k3|R8BwHU^ZY&n$AW`ZBC(fFfEW| zPKcyRpp`I2)}pZL)AFxG#8^H7*@>1@SqIP{kQNFE)uGa+_kM`vKXL%huwYYM$>3Hh zq;3~>0g#SWT(wA%)2y*NrBNuKz!>*rohKYrXfJ;&Se;Na#)$R6_ec#!w-vxxa%HaM zG7(A~Pf-+08FPmKPnb6xW9L>mOo;(ePFFz5SUH}QM~&kdjv$jy0~8Amo{Fij%_-$V zwzO*m7)?}IagIa801lmxRnJHS&pd735O7UF5J85=3(XuM!!+Rv0emPc{7u;hJ&R{$ zocMo{sU2;3BgS|nk1o!7Ks)0ygDs;)S9suAcdXMIa?E-&Jy?71iET%*;As9vUUULpEwHyj!$hk&h zw`+Uv(m7%bhX{-#IB~fh4N|r3c8{Y7q@O6w2@^1*T$sSUQhQW#;D}2j4$!fuO-L4E zDwokr;&z50!DKQ{qFuA2AwoiTC+99jDXc?jMu92{gN61u7_CI-H>er9wrjvH0bhUc zbqeP?dQ0z#@hN~Gk53g_JtJH|5-3vG1xNqBDm*=Nq=Oh_5&)Fv_s8SY-9jt*g<~uS z``Q=d(#0$u37K9%Z%MIrjg=7!=*(aPq&W3Wy5wVGDut0yoJ>WnI07)@RFadN7ri7> z?5c$c!1vy0ui|)^D@d)Crwtgao+p2?j(3al1c52N_q9nuNHs0aU>svI>18-R`uCM) zdFi~ISRT7T-sPTPp<>$7^i-)QcG>szWOFJx1`I@^(llfhjnfN;=TM*%!t0oy2{-3a zcBGJvRY2&LP;m=jbPpFAx4e<^?Q$MU?G-5Cm|XZ;Oa;oj2Tp!Ez5pSeL9Bn)#yurZ zq*Bo9I2&z<C@{O0()_?y;Wqxa ziSnn2|E)jjryf;feclM4FRp*rDw?e9)95XI#77Y>ktUDSO&1W22#Gem2{Y;}Vg2hpf&y^yKx{Y)NQt+29{d zJOj>Qsih&5mBt+De6?EtOJfy(Dl)Jp2*<)>%WlzEhrXJR0~o_Vt{Hzw8WT#DOVHG} zE?HyOEx8!fAsV-%h43M;(615TZ{A&9+lqZ40E1FzZ5>}mJ3w@ERDY6DcT-P zA6LIfy_>5=_H=`{+|zsW_{wcKPNVjTOpaW>7x~Gx$LrWO}JRWykVwUtryh zgBUV6L!xlrku7WS78Xe0BJHhd=4%Gc^$AX!fY%%r^#EX-lQn+|9V&eg^E>eF^2`dB zN-V#YiG#I@qJJQWNn=T`BeYiU4un^j1IO!0KAk|!iE zoZ|`6xfDkVuxo$4y(OZ--L5`Q*d1JHr?(k2)@{|R;4wp^io&WEbe=+y;(fvnwBWG1 zjF*H=65<#~NS{8ZI$2f{>vBZ&E_7=gKTO2|ZBlSn4<0;GInnNE=sSV1qy+>@d=kV@Ry53p`Zdrfa8xS?{q>?q*HwL)6whK zs$=vaKsEe9mP7C}{=Tl)T8m4UrK}&H;RhPF!~&&H$HtX>2eZNI6x?PCZZie9nS$F)!EL7CHdAnk zum2pGf>X#F{k2A0RCOKBsEhZ@IHG(<8#%k=g~NZ13V9iYQPiq)S%C6Z(3nPa;_!Zf zZOE+A@U)Sxx|qttBR6fl`OTBjrj<6(RNL~#20**>#_A0B#yEkik}FmKnrB-Y1-Ix! z3{`g5Z9QOhC-8uV@Bvto>f4<{<{wzi*22c-Ygu=J{2diM7|7RnmP5!UR{{6I5yhu5_M_J#xJOrII ziB&(gIdPq#y`EWJ?K?CV?xfUYFJm?rBH7yh`s(#(b!%;Q56hM_tWX-(KO4b7y~)*E z`WCpR^S=&XR_=cp91gbYe-ov&{^fl4wCjH=clFz`KGsp4;FI42plZWJOew=3b+j+r z2;jOU+R_mEpcAgUSrCA!TfmmQ_KhVGvPWLU_(w*QSgH%1&aXS5a4Qw~rIQ^*X;Sh{ zr8#0FRbhfeXBTuL%+t1!%v#T$omKjvLdNzMG zf(UZna-1#4gyzzST^@60G`pY^`?_l$R=}}3baR~pdQgJ_r7XlGPx-Ds)a{0kgtYW( z!$#~Gsy83hg0s$!j6vGipSh8*Srw^(R^7Sdf~s))lI+=W_+I{T1>kI6d@~o9)^07w zSOMz3wy!+EJKmC<#rc+C%a6^K8RGb32G{y$X_M;Wvi$o2?wtqqP|vZJ z9mQivyD#q+ablB(ZD+^1cGiE+>K=Xdd^0?*go!V~-ytB2RSWQ379ll@2*(F`XU}H#xFJ_HB<83oV0&bT|Tay%og3B zRW$x7%JTA`2U8R#Qh!o@u&_A~s6qZ8K0hqy|2;o^zPFYCnh^zX$J*7Ww$7yruq(A}bPAQ%A$#eY2P@A*3a>}GS`nq@BzCzN zGZR}u%{9AN8Q*`a>5eyO_)E82tMxvx<<$c8&KOk6GfMUb6lk1855Y=IBOEL|G!tUZ z615R;l!TK9^s6=i;$jxkCDXqyE45{AKz4 z{}(S`ZuS376r=w`5~emlnp1%Y97V|S!v1lb(SjtD_3s$)P~P&I``RSPnj^vASfc+I zN>< z|9OG=JUkLmpX9Utl(*b5AzNP4raWZ*x6K4ty!_}@$6%fUAPXCq~!>px3d*slDq zq%6Pwm-hrV%l{W;`7d8@*Z&5}y8M4d#_>Ro;k9zwv5hSw*p_YCmThS+|33f#|NjFS Jts4NO0RSz`{8In` diff --git a/pkg/apis/azuredisk/v1alpha1/types.go b/pkg/apis/azuredisk/v1alpha1/types.go index b4d70271dd..e5b1db97d1 100644 --- a/pkg/apis/azuredisk/v1alpha1/types.go +++ b/pkg/apis/azuredisk/v1alpha1/types.go @@ -375,3 +375,54 @@ type TopologyRequirement struct { type Topology struct { Segments map[string]string `json:"segments,omitempty"` } + +type Snapshot struct { + SnapshotID string `json:"snapshot_id"` + SourceVolumeID string `json:"source_volume_id"` + CreationTime metav1.Time `json:"creation_time"` + ReadyToUse bool `json:"ready_to_use"` + // +optional + SizeBytes int64 `json:"size_bytes,omitempty"` +} + +type VolumeDetails struct { + VolumeID string `json:"volume_id"` + // +optional + CapacityBytes int64 `json:"capacity_bytes,omitempty"` + // +optional + VolumeContext map[string]string `json:"volume_context,omitempty"` + // +optional + ContentSource *ContentVolumeSource `json:"content_source,omitempty"` + // +optional + AccessibleTopology []Topology `json:"accessible_topology,omitempty"` +} + +type VolumeCondition struct { + Abnormal bool `json:"abnormal"` + Message string `json:"message"` +} + +type VolumeStatus struct { + // +optional + PublishedNodeIds []string `json:"published_node_ids,omitempty"` + // +optional + Condition *VolumeCondition `json:"condition,omitempty"` +} + +type VolumeEntry struct { + Details *VolumeDetails `json:"details,omitempty"` + // +optional + Status *VolumeStatus `json:"status,omitempty"` +} + +type ListVolumesResult struct { + Entries []VolumeEntry `json:"entries"` + // +optional + NextToken string `json:"next_token,omitempty"` +} + +type ListSnapshotsResult struct { + Entries []Snapshot `json:"entries"` + // +optional + NextToken string `json:"next_token,omitempty"` +} diff --git a/pkg/apis/azuredisk/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/azuredisk/v1alpha1/zz_generated.deepcopy.go index 337252c0ad..9fd0871f02 100644 --- a/pkg/apis/azuredisk/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/azuredisk/v1alpha1/zz_generated.deepcopy.go @@ -503,6 +503,69 @@ func (in *ContentVolumeSource) DeepCopy() *ContentVolumeSource { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ListSnapshotsResult) DeepCopyInto(out *ListSnapshotsResult) { + *out = *in + if in.Entries != nil { + in, out := &in.Entries, &out.Entries + *out = make([]Snapshot, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListSnapshotsResult. +func (in *ListSnapshotsResult) DeepCopy() *ListSnapshotsResult { + if in == nil { + return nil + } + out := new(ListSnapshotsResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ListVolumesResult) DeepCopyInto(out *ListVolumesResult) { + *out = *in + if in.Entries != nil { + in, out := &in.Entries, &out.Entries + *out = make([]VolumeEntry, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ListVolumesResult. +func (in *ListVolumesResult) DeepCopy() *ListVolumesResult { + if in == nil { + return nil + } + out := new(ListVolumesResult) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Snapshot) DeepCopyInto(out *Snapshot) { + *out = *in + in.CreationTime.DeepCopyInto(&out.CreationTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Snapshot. +func (in *Snapshot) DeepCopy() *Snapshot { + if in == nil { + return nil + } + out := new(Snapshot) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Topology) DeepCopyInto(out *Topology) { *out = *in @@ -593,3 +656,106 @@ func (in *VolumeCapabilityAccessDetails) DeepCopy() *VolumeCapabilityAccessDetai in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeCondition) DeepCopyInto(out *VolumeCondition) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeCondition. +func (in *VolumeCondition) DeepCopy() *VolumeCondition { + if in == nil { + return nil + } + out := new(VolumeCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeDetails) DeepCopyInto(out *VolumeDetails) { + *out = *in + if in.VolumeContext != nil { + in, out := &in.VolumeContext, &out.VolumeContext + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ContentSource != nil { + in, out := &in.ContentSource, &out.ContentSource + *out = new(ContentVolumeSource) + **out = **in + } + if in.AccessibleTopology != nil { + in, out := &in.AccessibleTopology, &out.AccessibleTopology + *out = make([]Topology, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeDetails. +func (in *VolumeDetails) DeepCopy() *VolumeDetails { + if in == nil { + return nil + } + out := new(VolumeDetails) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeEntry) DeepCopyInto(out *VolumeEntry) { + *out = *in + if in.Details != nil { + in, out := &in.Details, &out.Details + *out = new(VolumeDetails) + (*in).DeepCopyInto(*out) + } + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(VolumeStatus) + (*in).DeepCopyInto(*out) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeEntry. +func (in *VolumeEntry) DeepCopy() *VolumeEntry { + if in == nil { + return nil + } + out := new(VolumeEntry) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VolumeStatus) DeepCopyInto(out *VolumeStatus) { + *out = *in + if in.PublishedNodeIds != nil { + in, out := &in.PublishedNodeIds, &out.PublishedNodeIds + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Condition != nil { + in, out := &in.Condition, &out.Condition + *out = new(VolumeCondition) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeStatus. +func (in *VolumeStatus) DeepCopy() *VolumeStatus { + if in == nil { + return nil + } + out := new(VolumeStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/azuredisk/controllerserver_v2.go b/pkg/azuredisk/controllerserver_v2.go index a1e594a63e..53e4b4f4b2 100644 --- a/pkg/azuredisk/controllerserver_v2.go +++ b/pkg/azuredisk/controllerserver_v2.go @@ -25,6 +25,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/protobuf/ptypes" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" volerr "k8s.io/cloud-provider/volume/errors" @@ -342,16 +343,86 @@ func (d *DriverV2) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) } } - response, err := d.cloudProvisioner.ListVolumes(ctx, req.GetMaxEntries(), startingToken) + mc := metrics.NewMetricContext(d.cloudProvisioner.GetMetricPrefix(), "controller_list_volumes", d.cloudProvisioner.GetCloud().ResourceGroup, d.cloudProvisioner.GetCloud().SubscriptionID, d.Name) + isOperationSucceeded := false + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded) + }() + + result, err := d.cloudProvisioner.ListVolumes(ctx, req.GetMaxEntries(), startingToken) if err != nil { return nil, err } - if response == nil { + if result == nil { return nil, status.Error(codes.Unknown, "Error listing volumes") } + responseEntries := []*csi.ListVolumesResponse_Entry{} + + for _, resultEntry := range result.Entries { + resultVolumeDetail := resultEntry.Details + responseContentSource := &csi.VolumeContentSource{} + + if resultVolumeDetail.ContentSource != nil { + if resultVolumeDetail.ContentSource.ContentSource == v1alpha1.ContentVolumeSourceTypeSnapshot { + responseContentSource.Type = &csi.VolumeContentSource_Snapshot{ + Snapshot: &csi.VolumeContentSource_SnapshotSource{ + SnapshotId: resultVolumeDetail.ContentSource.ContentSourceID, + }, + } + } else { + responseContentSource.Type = &csi.VolumeContentSource_Volume{ + Volume: &csi.VolumeContentSource_VolumeSource{ + VolumeId: resultVolumeDetail.ContentSource.ContentSourceID, + }, + } + } + } + + responseAccessibleTopology := []*csi.Topology{} + for _, t := range resultVolumeDetail.AccessibleTopology { + topology := &csi.Topology{ + Segments: t.Segments, + } + + responseAccessibleTopology = append(responseAccessibleTopology, topology) + } + + responseVolumeStatus := &csi.ListVolumesResponse_VolumeStatus{} + + if resultEntry.Status != nil { + responseVolumeStatus.PublishedNodeIds = resultEntry.Status.PublishedNodeIds + if resultEntry.Status.Condition != nil { + condition := &csi.VolumeCondition{ + Abnormal: resultEntry.Status.Condition.Abnormal, + Message: resultEntry.Status.Condition.Message, + } + + responseVolumeStatus.VolumeCondition = condition + } + } + + responseEntries = append(responseEntries, &csi.ListVolumesResponse_Entry{ + Volume: &csi.Volume{ + VolumeId: resultVolumeDetail.VolumeID, + CapacityBytes: resultVolumeDetail.CapacityBytes, + VolumeContext: resultVolumeDetail.VolumeContext, + ContentSource: responseContentSource, + AccessibleTopology: responseAccessibleTopology, + }, + Status: responseVolumeStatus, + }) + } + + isOperationSucceeded = true + + response := &csi.ListVolumesResponse{ + Entries: responseEntries, + NextToken: result.NextToken, + } + return response, nil } @@ -415,18 +486,31 @@ func (d *DriverV2) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRe mc.ObserveOperationWithResult(isOperationSucceeded) }() - response, err := d.cloudProvisioner.CreateSnapshot(ctx, sourceVolumeID, snapshotName, req.GetSecrets(), req.GetParameters()) + snapshot, err := d.cloudProvisioner.CreateSnapshot(ctx, sourceVolumeID, snapshotName, req.GetSecrets(), req.GetParameters()) if err != nil { return nil, err } - if response == nil { + if snapshot == nil { return nil, status.Error(codes.Unknown, "Error creating snapshot") } + tp, err := ptypes.TimestampProto(snapshot.CreationTime.Time) + if err != nil { + return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + } + isOperationSucceeded = true - return response, nil + return &csi.CreateSnapshotResponse{ + Snapshot: &csi.Snapshot{ + SnapshotId: snapshot.SnapshotID, + SourceVolumeId: snapshot.SourceVolumeID, + CreationTime: tp, + ReadyToUse: snapshot.ReadyToUse, + SizeBytes: snapshot.SizeBytes, + }, + }, nil } // DeleteSnapshot delete a snapshot @@ -442,23 +526,57 @@ func (d *DriverV2) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRe mc.ObserveOperationWithResult(isOperationSucceeded) }() - response, err := d.cloudProvisioner.DeleteSnapshot(ctx, snapshotID, req.GetSecrets()) + err := d.cloudProvisioner.DeleteSnapshot(ctx, snapshotID, req.GetSecrets()) if err != nil { return nil, err } - if response == nil { - return nil, status.Error(codes.Unknown, "Error deleting snapshot") - } - isOperationSucceeded = true - return response, nil + return &csi.DeleteSnapshotResponse{}, nil } // ListSnapshots list all snapshots func (d *DriverV2) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { - return d.cloudProvisioner.ListSnapshots(ctx, req.GetMaxEntries(), req.GetStartingToken(), req.GetSourceVolumeId(), req.GetSnapshotId(), req.GetSecrets()) + mc := metrics.NewMetricContext(d.cloudProvisioner.GetMetricPrefix(), "controller_list_snapshots", d.cloudProvisioner.GetCloud().ResourceGroup, d.cloudProvisioner.GetCloud().SubscriptionID, d.Name) + isOperationSucceeded := false + defer func() { + mc.ObserveOperationWithResult(isOperationSucceeded) + }() + + result, err := d.cloudProvisioner.ListSnapshots(ctx, req.GetMaxEntries(), req.GetStartingToken(), req.GetSourceVolumeId(), req.GetSnapshotId(), req.GetSecrets()) + if err != nil { + return nil, err + } + + if result == nil { + return nil, status.Error(codes.Unknown, "Error listing volumes") + } + + responseEntries := []*csi.ListSnapshotsResponse_Entry{} + + for _, resultEntry := range result.Entries { + tp, err := ptypes.TimestampProto(resultEntry.CreationTime.Time) + if err != nil { + return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + } + responseEntries = append(responseEntries, &csi.ListSnapshotsResponse_Entry{ + Snapshot: &csi.Snapshot{ + SnapshotId: resultEntry.SnapshotID, + SourceVolumeId: resultEntry.SourceVolumeID, + CreationTime: tp, + ReadyToUse: resultEntry.ReadyToUse, + SizeBytes: resultEntry.SizeBytes, + }, + }) + } + + isOperationSucceeded = true + + return &csi.ListSnapshotsResponse{ + Entries: responseEntries, + NextToken: result.NextToken, + }, nil } func generateV1Alpha1VolumeCapability(volumeCapability *csi.VolumeCapability) v1alpha1.VolumeCapability { diff --git a/pkg/azuredisk/fake_azuredisk_v2.go b/pkg/azuredisk/fake_azuredisk_v2.go index c78a6d4287..e082e57f40 100644 --- a/pkg/azuredisk/fake_azuredisk_v2.go +++ b/pkg/azuredisk/fake_azuredisk_v2.go @@ -20,10 +20,12 @@ package azuredisk import ( "context" + "fmt" "testing" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/mock/gomock" + "github.com/golang/protobuf/ptypes" "k8s.io/klog/v2" testingexec "k8s.io/utils/exec/testing" csicommon "sigs.k8s.io/azuredisk-csi-driver/pkg/csi-common" @@ -133,7 +135,23 @@ func (d *fakeDriverV2) getSnapshotInfo(snapshotID string) (string, string, error } func (d *fakeDriverV2) getSnapshotByID(ctx context.Context, resourceGroup string, snapshotName string, sourceVolumeID string) (*csi.Snapshot, error) { - return d.cloudProvisioner.(*provisioner.FakeCloudProvisioner).GetSnapshotByID(ctx, resourceGroup, snapshotName, sourceVolumeID) + snapshot, err := d.cloudProvisioner.(*provisioner.FakeCloudProvisioner).GetSnapshotByID(ctx, resourceGroup, snapshotName, sourceVolumeID) + if err != nil { + return nil, err + } + + tp, err := ptypes.TimestampProto(snapshot.CreationTime.Time) + if err != nil { + return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) + } + + return &csi.Snapshot{ + SnapshotId: snapshot.SnapshotID, + SourceVolumeId: snapshot.SourceVolumeID, + CreationTime: tp, + ReadyToUse: snapshot.ReadyToUse, + SizeBytes: snapshot.SizeBytes, + }, nil } func (d *fakeDriverV2) GetSourceDiskSize(ctx context.Context, resourceGroup, diskName string, curDepth, maxDepth int) (*int32, error) { diff --git a/pkg/azureutils/azure_snapshot_utils.go b/pkg/azureutils/azure_snapshot_utils.go index 6540063166..e021133ced 100644 --- a/pkg/azureutils/azure_snapshot_utils.go +++ b/pkg/azureutils/azure_snapshot_utils.go @@ -21,8 +21,8 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/protobuf/ptypes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/azuredisk-csi-driver/pkg/apis/azuredisk/v1alpha1" volumehelper "sigs.k8s.io/azuredisk-csi-driver/pkg/util" ) @@ -37,16 +37,13 @@ func GetSnapshotAndResourceNameFromSnapshotID(snapshotID string) (snapshotName, return snapshotName, resourceGroup, err } -func GenerateCSISnapshot(sourceVolumeID string, snapshot *compute.Snapshot) (*csi.Snapshot, error) { +func NewAzureDiskSnapshot(sourceVolumeID string, snapshot *compute.Snapshot) (*v1alpha1.Snapshot, error) { if snapshot == nil || snapshot.SnapshotProperties == nil { return nil, fmt.Errorf("snapshot property is nil") } - tp, err := ptypes.TimestampProto(snapshot.SnapshotProperties.TimeCreated.ToTime()) - if err != nil { - return nil, fmt.Errorf("Failed to covert creation timestamp: %v", err) - } - ready, _ := isCSISnapshotReady(*snapshot.SnapshotProperties.ProvisioningState) + tp := metav1.NewTime(snapshot.SnapshotProperties.TimeCreated.ToTime()) + ready, _ := isSnapshotReady(*snapshot.SnapshotProperties.ProvisioningState) if snapshot.SnapshotProperties.DiskSizeGB == nil { return nil, fmt.Errorf("diskSizeGB of snapshot property is nil") @@ -56,10 +53,10 @@ func GenerateCSISnapshot(sourceVolumeID string, snapshot *compute.Snapshot) (*cs sourceVolumeID = GetSnapshotSourceVolumeID(snapshot) } - return &csi.Snapshot{ + return &v1alpha1.Snapshot{ SizeBytes: volumehelper.GiBToBytes(int64(*snapshot.SnapshotProperties.DiskSizeGB)), - SnapshotId: *snapshot.ID, - SourceVolumeId: sourceVolumeID, + SnapshotID: *snapshot.ID, + SourceVolumeID: sourceVolumeID, CreationTime: tp, ReadyToUse: ready, }, nil @@ -75,7 +72,7 @@ func GetSnapshotSourceVolumeID(snapshot *compute.Snapshot) string { return "" } -func isCSISnapshotReady(state string) (bool, error) { +func isSnapshotReady(state string) (bool, error) { switch strings.ToLower(state) { case "succeeded": return true, nil diff --git a/pkg/controller/common.go b/pkg/controller/common.go index a48bac27a0..8073c46c94 100644 --- a/pkg/controller/common.go +++ b/pkg/controller/common.go @@ -16,7 +16,6 @@ package controller import ( "context" - "github.com/container-storage-interface/spec/lib/go/csi" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -44,10 +43,10 @@ type CloudProvisioner interface { PublishVolume(ctx context.Context, volumeID string, nodeID string, volumeContext map[string]string) (map[string]string, error) UnpublishVolume(ctx context.Context, volumeID string, nodeID string) error ExpandVolume(ctx context.Context, volumeID string, capacityRange *v1alpha1.CapacityRange, secrets map[string]string) (*v1alpha1.AzVolumeStatusParams, error) - ListVolumes(ctx context.Context, maxEntries int32, startingToken string) (*csi.ListVolumesResponse, error) - CreateSnapshot(ctx context.Context, sourceVolumeID string, snapshotName string, secrets map[string]string, parameters map[string]string) (*csi.CreateSnapshotResponse, error) - ListSnapshots(ctx context.Context, maxEntries int32, startingToken string, sourceVolumeID string, snapshotID string, secrets map[string]string) (*csi.ListSnapshotsResponse, error) - DeleteSnapshot(ctx context.Context, snapshotID string, secrets map[string]string) (*csi.DeleteSnapshotResponse, error) + ListVolumes(ctx context.Context, maxEntries int32, startingToken string) (*v1alpha1.ListVolumesResult, error) + CreateSnapshot(ctx context.Context, sourceVolumeID string, snapshotName string, secrets map[string]string, parameters map[string]string) (*v1alpha1.Snapshot, error) + ListSnapshots(ctx context.Context, maxEntries int32, startingToken string, sourceVolumeID string, snapshotID string, secrets map[string]string) (*v1alpha1.ListSnapshotsResult, error) + DeleteSnapshot(ctx context.Context, snapshotID string, secrets map[string]string) error CheckDiskExists(ctx context.Context, diskURI string) error GetCloud() *provider.Cloud GetMetricPrefix() string diff --git a/pkg/provisioner/cloudprovisioner.go b/pkg/provisioner/cloudprovisioner.go index 247addfc87..b657be7260 100644 --- a/pkg/provisioner/cloudprovisioner.go +++ b/pkg/provisioner/cloudprovisioner.go @@ -26,7 +26,6 @@ import ( "strings" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-12-01/compute" - "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -57,7 +56,7 @@ type CloudProvisioner struct { type listVolumeStatus struct { numVisited int // the number of iterated azure disks isCompleteRun bool // isCompleteRun is flagged true if the function iterated through all azure disks - entries []*csi.ListVolumesResponse_Entry + entries []v1alpha1.VolumeEntry err error } @@ -318,7 +317,7 @@ func (c *CloudProvisioner) DeleteVolume( func (c *CloudProvisioner) ListVolumes( ctx context.Context, maxEntries int32, - startingToken string) (*csi.ListVolumesResponse, error) { + startingToken string) (*v1alpha1.ListVolumesResult, error) { start, _ := strconv.Atoi(startingToken) kubeClient := c.cloud.KubeClient if kubeClient != nil && kubeClient.CoreV1() != nil && kubeClient.CoreV1().PersistentVolumes() != nil { @@ -465,7 +464,7 @@ func (c *CloudProvisioner) CreateSnapshot( sourceVolumeID string, snapshotName string, secrets map[string]string, - parameters map[string]string) (*csi.CreateSnapshotResponse, error) { + parameters map[string]string) (*v1alpha1.Snapshot, error) { snapshotName = azureutils.GetValidDiskName(snapshotName) var customTags string @@ -534,14 +533,12 @@ func (c *CloudProvisioner) CreateSnapshot( } klog.V(2).Infof("create snapshot(%s) under rg(%s) successfully", snapshotName, resourceGroup) - csiSnapshot, err := c.GetSnapshotByID(ctx, resourceGroup, snapshotName, sourceVolumeID) + snapshotObj, err := c.GetSnapshotByID(ctx, resourceGroup, snapshotName, sourceVolumeID) if err != nil { return nil, err } - return &csi.CreateSnapshotResponse{ - Snapshot: csiSnapshot, - }, nil + return snapshotObj, nil } func (c *CloudProvisioner) ListSnapshots( @@ -550,22 +547,19 @@ func (c *CloudProvisioner) ListSnapshots( startingToken string, sourceVolumeID string, snapshotID string, - secrets map[string]string) (*csi.ListSnapshotsResponse, error) { + secrets map[string]string) (*v1alpha1.ListSnapshotsResult, error) { // SnapshotID is not empty, return snapshot that match the snapshot id. if len(snapshotID) != 0 { snapshot, err := c.GetSnapshotByID(ctx, c.cloud.ResourceGroup, snapshotID, sourceVolumeID) if err != nil { if strings.Contains(err.Error(), azureutils.ResourceNotFound) { - return &csi.ListSnapshotsResponse{}, nil + return &v1alpha1.ListSnapshotsResult{}, nil } return nil, err } - entries := []*csi.ListSnapshotsResponse_Entry{ - { - Snapshot: snapshot, - }, - } - listSnapshotResp := &csi.ListSnapshotsResponse{ + entries := []v1alpha1.Snapshot{*snapshot} + + listSnapshotResp := &v1alpha1.ListSnapshotsResult{ Entries: entries, } return listSnapshotResp, nil @@ -602,14 +596,14 @@ func (c *CloudProvisioner) ListSnapshots( if maxEntries > 0 && int(maxEntries) < maxAvailableEntries { totalEntries = int(maxEntries) } - entries := []*csi.ListSnapshotsResponse_Entry{} + entries := []v1alpha1.Snapshot{} for count := 0; start < len(snapshots) && count < totalEntries; start++ { if (sourceVolumeID != "" && sourceVolumeID == azureutils.GetSnapshotSourceVolumeID(&snapshots[start])) || sourceVolumeID == "" { - csiSnapshot, err := azureutils.GenerateCSISnapshot(sourceVolumeID, &snapshots[start]) + snapshotObj, err := azureutils.NewAzureDiskSnapshot(sourceVolumeID, &snapshots[start]) if err != nil { return nil, fmt.Errorf("failed to generate snapshot entry: %v", err) } - entries = append(entries, &csi.ListSnapshotsResponse_Entry{Snapshot: csiSnapshot}) + entries = append(entries, *snapshotObj) count++ } } @@ -619,7 +613,7 @@ func (c *CloudProvisioner) ListSnapshots( nextToken = start } - listSnapshotResp := &csi.ListSnapshotsResponse{ + listSnapshotResp := &v1alpha1.ListSnapshotsResult{ Entries: entries, NextToken: strconv.Itoa(nextToken), } @@ -630,10 +624,10 @@ func (c *CloudProvisioner) ListSnapshots( func (c *CloudProvisioner) DeleteSnapshot( ctx context.Context, snapshotID string, - secrets map[string]string) (*csi.DeleteSnapshotResponse, error) { + secrets map[string]string) error { snapshotName, resourceGroup, err := c.GetSnapshotAndResourceNameFromSnapshotID(snapshotID) if err != nil { - return nil, err + return err } if snapshotName == "" && resourceGroup == "" { @@ -644,11 +638,11 @@ func (c *CloudProvisioner) DeleteSnapshot( klog.V(2).Infof("begin to delete snapshot(%s) under rg(%s)", snapshotName, resourceGroup) rerr := c.cloud.SnapshotsClient.Delete(ctx, resourceGroup, snapshotName) if rerr != nil { - return nil, status.Error(codes.Internal, fmt.Sprintf("delete snapshot error: %v", rerr.Error())) + return status.Error(codes.Internal, fmt.Sprintf("delete snapshot error: %v", rerr.Error())) } klog.V(2).Infof("delete snapshot(%s) under rg(%s) successfully", snapshotName, resourceGroup) - return &csi.DeleteSnapshotResponse{}, nil + return nil } func (c *CloudProvisioner) CheckDiskExists(ctx context.Context, diskURI string) error { @@ -730,7 +724,7 @@ func (c *CloudProvisioner) GetSnapshotAndResourceNameFromSnapshotID(snapshotID s return snapshotName, resourceGroup, err } -func (c *CloudProvisioner) GetSnapshotByID(ctx context.Context, resourceGroup string, snapshotName string, sourceVolumeID string) (*csi.Snapshot, error) { +func (c *CloudProvisioner) GetSnapshotByID(ctx context.Context, resourceGroup string, snapshotName string, sourceVolumeID string) (*v1alpha1.Snapshot, error) { snapshotNameVal, resourceGroupName, err := c.GetSnapshotAndResourceNameFromSnapshotID(snapshotName) if err != nil { return nil, err @@ -746,7 +740,7 @@ func (c *CloudProvisioner) GetSnapshotByID(ctx context.Context, resourceGroup st return nil, status.Error(codes.Internal, fmt.Sprintf("get snapshot %s from rg(%s) error: %v", snapshotNameVal, resourceGroupName, rerr.Error())) } - return azureutils.GenerateCSISnapshot(sourceVolumeID, &snapshot) + return azureutils.NewAzureDiskSnapshot(sourceVolumeID, &snapshot) } func (c *CloudProvisioner) validateCreateVolumeRequestParams( @@ -803,7 +797,7 @@ func (c *CloudProvisioner) validateCreateVolumeRequestParams( } // listVolumesInCluster is a helper function for ListVolumes used for when there is an available kubeclient -func (c *CloudProvisioner) listVolumesInCluster(ctx context.Context, start, maxEntries int) (*csi.ListVolumesResponse, error) { +func (c *CloudProvisioner) listVolumesInCluster(ctx context.Context, start, maxEntries int) (*v1alpha1.ListVolumesResult, error) { kubeClient := c.cloud.KubeClient pvList, err := kubeClient.CoreV1().PersistentVolumes().List(context.TODO(), metav1.ListOptions{}) if err != nil { @@ -843,7 +837,7 @@ func (c *CloudProvisioner) listVolumesInCluster(ctx context.Context, start, maxE sort.Strings(resourceGroups) // loop through each resourceGroup to get disk lists - entries := []*csi.ListVolumesResponse_Entry{} + entries := []v1alpha1.VolumeEntry{} numVisited := 0 isCompleteRun, startFound := true, false for _, resourceGroup := range resourceGroups { @@ -877,7 +871,7 @@ func (c *CloudProvisioner) listVolumesInCluster(ctx context.Context, start, maxE nextTokenString = strconv.Itoa(start + numVisited) } - listVolumesResp := &csi.ListVolumesResponse{ + listVolumesResp := &v1alpha1.ListVolumesResult{ Entries: entries, NextToken: nextTokenString, } @@ -886,8 +880,8 @@ func (c *CloudProvisioner) listVolumesInCluster(ctx context.Context, start, maxE } // listVolumesInNodeResourceGroup is a helper function for ListVolumes used for when there is no available kubeclient -func (c *CloudProvisioner) listVolumesInNodeResourceGroup(ctx context.Context, start, maxEntries int) (*csi.ListVolumesResponse, error) { - entries := []*csi.ListVolumesResponse_Entry{} +func (c *CloudProvisioner) listVolumesInNodeResourceGroup(ctx context.Context, start, maxEntries int) (*v1alpha1.ListVolumesResult, error) { + entries := []v1alpha1.VolumeEntry{} listStatus := c.listVolumesByResourceGroup(ctx, c.cloud.ResourceGroup, entries, start, maxEntries, nil) if listStatus.err != nil { return nil, listStatus.err @@ -898,7 +892,7 @@ func (c *CloudProvisioner) listVolumesInNodeResourceGroup(ctx context.Context, s nextTokenString = strconv.Itoa(listStatus.numVisited) } - listVolumesResp := &csi.ListVolumesResponse{ + listVolumesResp := &v1alpha1.ListVolumesResult{ Entries: listStatus.entries, NextToken: nextTokenString, } @@ -907,7 +901,7 @@ func (c *CloudProvisioner) listVolumesInNodeResourceGroup(ctx context.Context, s } // listVolumesByResourceGroup is a helper function that updates the ListVolumeResponse_Entry slice and returns number of total visited volumes, number of volumes that needs to be visited and an error if found -func (c *CloudProvisioner) listVolumesByResourceGroup(ctx context.Context, resourceGroup string, entries []*csi.ListVolumesResponse_Entry, start, maxEntries int, volSet map[string]bool) listVolumeStatus { +func (c *CloudProvisioner) listVolumesByResourceGroup(ctx context.Context, resourceGroup string, entries []v1alpha1.VolumeEntry, start, maxEntries int, volSet map[string]bool) listVolumeStatus { disks, derr := c.cloud.DisksClient.ListByResourceGroup(ctx, resourceGroup) if derr != nil { return listVolumeStatus{err: status.Errorf(codes.Internal, "ListVolumes on rg(%s) failed with error: %v", resourceGroup, derr.Error())} @@ -955,11 +949,11 @@ func (c *CloudProvisioner) listVolumesByResourceGroup(ctx context.Context, resou nodeList = append(nodeList, string(attachedNode)) } - entries = append(entries, &csi.ListVolumesResponse_Entry{ - Volume: &csi.Volume{ - VolumeId: *disk.ID, + entries = append(entries, v1alpha1.VolumeEntry{ + Details: &v1alpha1.VolumeDetails{ + VolumeID: *disk.ID, }, - Status: &csi.ListVolumesResponse_VolumeStatus{ + Status: &v1alpha1.VolumeStatus{ PublishedNodeIds: nodeList, }, })