From ae5f461524399f20cf201cda00911ee1e920e108 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Tue, 23 Nov 2021 04:16:11 +0000 Subject: [PATCH 1/2] doc: cut v1.9.0 release --- README.md | 2 +- charts/README.md | 2 +- charts/index.yaml | 49 +- charts/latest/azuredisk-csi-driver-v1.9.0.tgz | Bin 13229 -> 13231 bytes .../latest/azuredisk-csi-driver/values.yaml | 2 +- charts/v1.9.0/azuredisk-csi-driver-v1.9.0.tgz | Bin 0 -> 13229 bytes charts/v1.9.0/azuredisk-csi-driver/Chart.yaml | 5 + .../azuredisk-csi-driver/templates/NOTES.txt | 5 + .../templates/_helpers.tpl | 26 + .../templates/crd-csi-snapshot-ga.yaml | 655 ++++++++++++++++++ .../templates/crd-csi-snapshot.yaml | 504 ++++++++++++++ .../templates/csi-azuredisk-controller.yaml | 226 ++++++ .../templates/csi-azuredisk-driver.yaml | 17 + .../templates/csi-azuredisk-node-windows.yaml | 200 ++++++ .../templates/csi-azuredisk-node.yaml | 209 ++++++ .../templates/csi-snapshot-controller.yaml | 54 ++ .../rbac-csi-azuredisk-controller.yaml | 202 ++++++ .../templates/rbac-csi-azuredisk-node.yaml | 25 + .../rbac-csi-snapshot-controller.yaml | 69 ++ ...rviceaccount-csi-azuredisk-controller.yaml | 8 + .../serviceaccount-csi-azuredisk-node.yaml | 8 + ...erviceaccount-csi-snapshot-controller.yaml | 7 + .../v1.9.0/azuredisk-csi-driver/values.yaml | 240 +++++++ deploy/csi-azuredisk-controller.yaml | 2 +- deploy/csi-azuredisk-node-windows.yaml | 2 +- deploy/csi-azuredisk-node.yaml | 2 +- deploy/v1.9.0/crd-csi-snapshot-ga.yaml | 652 +++++++++++++++++ deploy/v1.9.0/crd-csi-snapshot.yaml | 501 ++++++++++++++ deploy/v1.9.0/csi-azuredisk-controller.yaml | 184 +++++ deploy/v1.9.0/csi-azuredisk-driver.yaml | 11 + deploy/v1.9.0/csi-azuredisk-node-windows.yaml | 194 ++++++ deploy/v1.9.0/csi-azuredisk-node.yaml | 174 +++++ deploy/v1.9.0/csi-snapshot-controller.yaml | 42 ++ .../v1.9.0/rbac-csi-azuredisk-controller.yaml | 200 ++++++ deploy/v1.9.0/rbac-csi-azuredisk-node.yaml | 31 + .../v1.9.0/rbac-csi-snapshot-controller.yaml | 75 ++ docs/install-azuredisk-csi-driver.md | 2 +- docs/install-csi-driver-v1.9.0.md | 36 + 38 files changed, 4596 insertions(+), 27 deletions(-) create mode 100644 charts/v1.9.0/azuredisk-csi-driver-v1.9.0.tgz create mode 100644 charts/v1.9.0/azuredisk-csi-driver/Chart.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/NOTES.txt create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/_helpers.tpl create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot-ga.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-driver.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/csi-snapshot-controller.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-snapshot-controller.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-controller.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-node.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml create mode 100644 charts/v1.9.0/azuredisk-csi-driver/values.yaml create mode 100644 deploy/v1.9.0/crd-csi-snapshot-ga.yaml create mode 100644 deploy/v1.9.0/crd-csi-snapshot.yaml create mode 100644 deploy/v1.9.0/csi-azuredisk-controller.yaml create mode 100644 deploy/v1.9.0/csi-azuredisk-driver.yaml create mode 100644 deploy/v1.9.0/csi-azuredisk-node-windows.yaml create mode 100644 deploy/v1.9.0/csi-azuredisk-node.yaml create mode 100644 deploy/v1.9.0/csi-snapshot-controller.yaml create mode 100644 deploy/v1.9.0/rbac-csi-azuredisk-controller.yaml create mode 100644 deploy/v1.9.0/rbac-csi-azuredisk-node.yaml create mode 100644 deploy/v1.9.0/rbac-csi-snapshot-controller.yaml create mode 100644 docs/install-csi-driver-v1.9.0.md diff --git a/README.md b/README.md index 10c752f975..a02c1ca6e7 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,9 @@ V2: Preview |Driver Version |Image | supported k8s version | |----------------|----------------------------------------------------------|-----------------------| |`master` branch |mcr.microsoft.com/k8s/csi/azuredisk-csi:latest | 1.19+ | +|v1.9.0 |mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.9.0 | 1.19+ | |v1.8.0 |mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.8.0 | 1.18+ | |v1.7.0 |mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.7.0 | 1.18+ | -|v1.6.0 |mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.6.0 | 1.18+ | #### V2 diff --git a/charts/README.md b/charts/README.md index b255204b10..1b87598faa 100644 --- a/charts/README.md +++ b/charts/README.md @@ -72,7 +72,7 @@ helm repo update azuredisk-csi-driver ### install a specific version ```console -helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver --namespace kube-system --version v1.8.0 +helm install azuredisk-csi-driver azuredisk-csi-driver/azuredisk-csi-driver --namespace kube-system --version v1.9.0 ``` ### install on Azure Stack diff --git a/charts/index.yaml b/charts/index.yaml index c5e432aad3..04270c69f8 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -3,7 +3,7 @@ entries: azuredisk-csi-driver: - apiVersion: v1 appVersion: v2.0.0-alpha.1 - created: "2021-11-16T01:12:04.1505436-08:00" + created: "2021-11-23T04:15:02.427011351Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 1fd3187816b46186a2b275a131a83782696758eba6b90134b6ba675e174b3ac4 name: azuredisk-csi-driver @@ -12,16 +12,25 @@ entries: version: v2.0.0-alpha.1 - apiVersion: v1 appVersion: latest - created: "2021-11-16T01:12:04.1218528-08:00" + created: "2021-11-23T04:15:02.422447123Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin - digest: f1a120528a787a944bd9f078fd6ab54f7d76a8f30ba60ab85aac146f519b6e96 + digest: 95abb75d6a4575ed34479261ddd2ac5634ee3ce763a7c9afefb2aaf646834710 + name: azuredisk-csi-driver + urls: + - https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts/v1.9.0/azuredisk-csi-driver-v1.9.0.tgz + version: v1.9.0 + - apiVersion: v1 + appVersion: latest + created: "2021-11-23T04:15:02.387535311Z" + description: Azure disk Container Storage Interface (CSI) Storage Plugin + digest: b14e4eb5b991b03446ddd04d8b2cea57195bd128f11b230d566dacc1c7db1a58 name: azuredisk-csi-driver urls: - https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts/latest/azuredisk-csi-driver-v1.9.0.tgz version: v1.9.0 - apiVersion: v1 appVersion: v1.8.0 - created: "2021-11-16T01:12:04.1481589-08:00" + created: "2021-11-23T04:15:02.42018681Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: a37491e845671e81b9567f2d1a746dbf4f182fde0c8f9e689dfa632c5651c6c9 name: azuredisk-csi-driver @@ -30,7 +39,7 @@ entries: version: v1.8.0 - apiVersion: v1 appVersion: v1.7.0 - created: "2021-11-16T01:12:04.145795-08:00" + created: "2021-11-23T04:15:02.418415599Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 4df7a10466206c60deab00efbca57f67f042afd1ecd5daca738df28526f3ffcc name: azuredisk-csi-driver @@ -39,7 +48,7 @@ entries: version: v1.7.0 - apiVersion: v1 appVersion: v1.6.0 - created: "2021-11-16T01:12:04.1416953-08:00" + created: "2021-11-23T04:15:02.416510287Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: b1a4c384fdfdb6577dd6dfd3709fb746b5c79247846e5c582b93c3a1187f6d88 name: azuredisk-csi-driver @@ -48,7 +57,7 @@ entries: version: v1.6.0 - apiVersion: v1 appVersion: v1.5.1 - created: "2021-11-16T01:12:04.1401381-08:00" + created: "2021-11-23T04:15:02.414573775Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: efb42e1d56cc5596eb4e695b658c2748bc05cc6c47862e3f20ea6b5b199d01d6 name: azuredisk-csi-driver @@ -57,7 +66,7 @@ entries: version: v1.5.1 - apiVersion: v1 appVersion: v1.5.0 - created: "2021-11-16T01:12:04.1382958-08:00" + created: "2021-11-23T04:15:02.412711664Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: c8490efde8d4a43bd25f5c619887498dfb4b41edf3929ac8dbf0a9d3f691e6ae name: azuredisk-csi-driver @@ -66,7 +75,7 @@ entries: version: v1.5.0 - apiVersion: v1 appVersion: v1.4.0 - created: "2021-11-16T01:12:04.1368364-08:00" + created: "2021-11-23T04:15:02.410815053Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 55e5e17aac7c144b21a198867654ef70ad272987b0bb165782905adf91ca3c03 name: azuredisk-csi-driver @@ -75,7 +84,7 @@ entries: version: v1.4.0 - apiVersion: v1 appVersion: v1.3.0 - created: "2021-11-16T01:12:04.1355724-08:00" + created: "2021-11-23T04:15:02.408967341Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 2665483e922a577feb8539ca7f774bc70c945ce490294fd3378f098c2d244dde name: azuredisk-csi-driver @@ -84,7 +93,7 @@ entries: version: v1.3.0 - apiVersion: v1 appVersion: v1.2.0 - created: "2021-11-16T01:12:04.134546-08:00" + created: "2021-11-23T04:15:02.407456032Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 2bbfe2f9d080f1b3ff10590c7168d05ce026c5a73332b4d48014610a52337808 name: azuredisk-csi-driver @@ -93,7 +102,7 @@ entries: version: v1.2.0 - apiVersion: v1 appVersion: v1.1.1 - created: "2021-11-16T01:12:04.1307967-08:00" + created: "2021-11-23T04:15:02.40552082Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: dd7066be8f499f6c1a396ab27c0013c09f5a8d8319cc04fbdd480d31107bb851 name: azuredisk-csi-driver @@ -102,7 +111,7 @@ entries: version: v1.1.1 - apiVersion: v1 appVersion: v1.1.0 - created: "2021-11-16T01:12:04.1298083-08:00" + created: "2021-11-23T04:15:02.404153812Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 3d2a5189416dd6a43bd3e2097bbe23a8db347b6e1a36c6a43fd59cc9c9633ff3 name: azuredisk-csi-driver @@ -111,7 +120,7 @@ entries: version: v1.1.0 - apiVersion: v1 appVersion: v1.0.0 - created: "2021-11-16T01:12:04.1288273-08:00" + created: "2021-11-23T04:15:02.402602303Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: d179bc6f338518859b6efdc3b3bed8d06513313e8047563eb4b654b2d417c81e name: azuredisk-csi-driver @@ -120,7 +129,7 @@ entries: version: v1.0.0 - apiVersion: v1 appVersion: v0.10.0 - created: "2021-11-16T01:12:04.1228895-08:00" + created: "2021-11-23T04:15:02.388706418Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 3dbbaca098fe8316de079236598253b5831e8e85fd88b390231d828157d62206 name: azuredisk-csi-driver @@ -129,7 +138,7 @@ entries: version: v0.10.0 - apiVersion: v1 appVersion: v0.9.0 - created: "2021-11-16T01:12:04.1277246-08:00" + created: "2021-11-23T04:15:02.400896492Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: a978f3e6ef5d678c3b6512bd8a63277cb4ce40d3f3e34b80370f0c37298824f2 name: azuredisk-csi-driver @@ -138,7 +147,7 @@ entries: version: v0.9.0 - apiVersion: v1 appVersion: v0.8.0 - created: "2021-11-16T01:12:04.1258962-08:00" + created: "2021-11-23T04:15:02.399574284Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 1762b832389b4f7a5eab9748127fa6dbb85131485d67bc3fe485bbe86c468128 name: azuredisk-csi-driver @@ -147,7 +156,7 @@ entries: version: v0.8.0 - apiVersion: v1 appVersion: v0.7.0 - created: "2021-11-16T01:12:04.1246119-08:00" + created: "2021-11-23T04:15:02.398120175Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: 29e21f686814f46c1edaaaa95ce2d25579ff1aad270c58b774bdb5a89858b8bf name: azuredisk-csi-driver @@ -156,11 +165,11 @@ entries: version: v0.7.0 - apiVersion: v1 appVersion: v0.6.0 - created: "2021-11-16T01:12:04.1236514-08:00" + created: "2021-11-23T04:15:02.396578366Z" description: Azure disk Container Storage Interface (CSI) Storage Plugin digest: b11d8dfee371ca7c63a1448ba27c1fd1f032ea33575fefeeb16927fc95d1eeb7 name: azuredisk-csi-driver urls: - https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts/v0.6.0/azuredisk-csi-driver-v0.6.0.tgz version: v0.6.0 -generated: "2021-11-16T01:12:04.1117008-08:00" +generated: "2021-11-23T04:15:02.383134484Z" diff --git a/charts/latest/azuredisk-csi-driver-v1.9.0.tgz b/charts/latest/azuredisk-csi-driver-v1.9.0.tgz index 21f8bdbb23eb70d4138f000c32dedd8cd64d3e15..fbe364a1a700f7166d42980a4f2177d1ae23db1d 100644 GIT binary patch literal 13231 zcmb`OQ*>rQwC7`VY}+=zu;Zj-+qUhFI<{@wwmbM@+qQB0&Ye4V);v$Gy=tAacRkcY zom2Z?`&UGfFz6uv*}iE&=nN&5n2jZ6Ib}Qn97b$v%qGelmRiaHPI)yoPFZyuD?>XI zPZdRbeo0dsTab%xZ)ZHN2Cl5$M;e8PoQ5W#LK6L?`D=>xB%QbOhaG7pJ%#p+@v%pL zI%W7GavCIa0`}Bx zG41$_x1=9-crzC4xY_tb=if2ng?IiE5)xF~1cXF{z1Z1&f?keyn<|MbN#t&{@BL@IzP?am< z130JYRSrcfkhVe$%AtN$K2SNK-wLv2;@#Bm)XN9J8w-zx{Bd<9BmbUX-Yb6b))%Wc znx(hh&_aIN#SC@=)Nu_W$|UVjKZLWKIRovgDRM>662S;^TU#YGL28@#JdoH($w%p; z6n$+LV@4Ba;P|=i&ii9P{A7JR4&UiZl$tM{`%p(D@33dl9%cXvpU(mfrYM{tkR2q3 ze<%$EVP}&Zn@nzzbpu2b@@WtuiCI+z3-DwfW9va4?PFFD2r8bile0odZdm4Y62gVw zM$oYNh0Hxb|HuMEF2gRHD%h7{M^w=QDeT(iWE-2U(9k7}rVlL3d&jTw+}y+D>-0-{ zaYY>uw^zr3biTS0T&lWbdpk)$nsnWO8KE3g46nChAh?jPC5j=@h@@j4Rf_QMB`d=* zyDfUscD|_|oJ^W+;0GQmQ6!5}hY4r`ntm1=nN|CVvfg4@+j zAslf4DO0*L$%30z59g04gg=m{{OJ~2h^(y9nhZd@gFisnS1+%d1fWO;Bn3~$&FnEN zR9S@+P)kjFz(K-86vz~Skmi9Jp?|vEa=z-wg24F=p}`De`+gO83fHB*8-$-|wVsNEc)f1h{XeZ;x52i*Wa@!2Bx(JsNjLWq1?PDQxtAB@bUCyj zDwUx!Z02wY#)c}N(|{?oIIDb__wcBD6TOQ!zSk&g1 z-z~VgV^Y+1-BwI}Fk7Ld)WCy0c`$99 zM9;dp$bn#Ks6PTZLp<};q{kni|0s~KJc5p!LRd7Yh&kd<=w3V6L$5;pox3kA*u?gX z(D9=u*Z!an=*<9Ct!&mR+&|dHx^?+wH)?TnwKtF{(uRF;5ga_y3W?J+K=g+#OtsAW z83Ga;GwMfN#xGvU%4FmNYyur)Kjeh*z|Y|{&4fml9|pqAt`QZU&9NoO%rWy@S>U-n zk>r|j+jX%1tJ44*n$p9D98Xlx+#F|43X(F*H!M^1uR~@rbQV3?(qPnf8T^{>pKm>O^QT&*b3Bn`J_*UizT zD@8ZnTM4|zcMJ^O&B08c6=$84IwRZ<4X;1*`81}BJp0zHwLQ5tU`crP2%s=a05EV0i>yww;+x_*(h}!;;(zfaH zTYy)9_VPQc%foNu`g>#4835f`{;p+zrq?AVPDeNIY&Nw3QV_>LqVPj9toxH06ZOTx z{5XNcAMs07mI4J10u^;qw5q7<8^1|B!hlrf=X=SSOw~2GK4fzFpTBxzCV?Tm$zdja~)0UUsNQBb?%PmW0)9GW!!za4*(1DS(62SR+wo1 zCf~pe3c-HUFcARtW9M`lx zX4)PO#FmNX8||um<@+qIO0Hm`C}t%D!IEG0fTc@j=ya`;N$jVLq6PI4wAb4YvCFuw zi`0Y3ZC4N3@`L_oaP}AWSyxpm;q-v=#flQLW){3s(g~SKn}~6C)7Xa3@E)J?wW2q& zm{WM%UXk+EiqLSUkAtuI_q)S<(?`Y}%+KrlJ~~lta}QSyg~BgEpQk+&7mbB)i7+NP zIj>KDB6+ES1W&@XpYnP?b_ExWR^?!D|JEr%^X4319&kxIi0-I?Et16UF8bpkxVAZq zSul&2@|b;W(Vi{mi7*EPvmqt%4Z@pQOOnQYWT>H)(?A#3>`@b$LG%>-mkZJPf0OO) zQtw^h{w`?2FBIBip81nCnwUR>d036e0Vd~|wjBbzy6eB))rW%J3uN|^!Xzfw6Ezmy zXO;nf<)pUnomN!9jw_=08}sKU8MRr~Fc5nYgKbgE3j5brfwrWZvyG3$m_N7Pmt-&v zatO<)VX^yLHn9t4SX%1#6*0LFWciRI8PfJy~SP}S9hq#La4 zfdS!9?Lzb(tEnQnEDcdGIAY4{SeS{GZfX?xK#R2qp3Q5k!2|x}D>g#(49kMPenYMp zsxn*SZO=y|tJjFdfCfw&&pWpmwqo1(n>Z+#BE-N&=1Gt4h3K#nt4z$aKR`@*8uP?V z?xTGZMY5v1AMBDfXD8t)p+na=s*fmxCu~BJN*FBJtU_Ky2GgAKAS3!+7)#*$u!-4* zq#cyEO(NS3AV+IQ&%p#uU?~jsEE1tNKNCf}sR8V#C%$%bE2(H_TqcgZW?R3q_-qy#L z26l{M9GH4!sRL9nc1@lii;ABeTB^w_udM`~K5E;Wq=S+v5wh(jN}@FQR;G#6VIc^> z_Hcd$O3-mY$}j!+awT4?OCx1nrFw!V)2Km*beS^dTZBiefrGA%BBwtapULo`7%b5D z1V;g+;_!vW;UMqMLkIXT$e&X!pYtN|MEDvXNCsjWa}p`(O9C6! zADx>MB#SDu!bK{}XN~V+9?3O-DrQUhK(ZS)ra|=$7NorNXpYr|y=R`)RT zBh^8yU+cJrOR-eZFNJ97Pg&76(zUOmAb;UwdSqP+0@aW0{XN|jUvMzS)%FNTk(bu@ z<&_;5rcsfHF_(%IHk2fEN*0_;UaM)>B%(?$=+kKj`YRVJaZRv6Q;XUnlNxt_cuR+% z@T&&3K;_*YMQ+!D_468nIexL!1JWVXS?T=XDUCY{1{7aFB!LG+Xpp$yn>94bwoUf$Nmm()r_m$jJtngghU~-D`!%89dR3Rq$)_z;(HL&i%cc#ft zQCCDBzjI-wS}u^)(lr#2k~NZ|=EIyIhKXJ>}sNR4QGQNFl*f*IEw#{+Ll0~kKHdl zTvwNb8}5dfgrfGRYDF zVs(h}tJWHvSNXs+i_bigq+=mUgm~dX3Nuo}$^;>2LM4cW=dYLUskrax<2p~{l4=`! z$(p_MAU$67agDu|t*Z6gq0K9Pe--Maq*-}!l|RLV@DQ9aqa1o6Hnfl{#U_G~#)38g za?RWpHV%V{77{=e+Bp}C_+0G9Q`xvgY%;EFo#vj_v1V^wQqwf#`pnHdg^$RadHS}J z-6u9GnV1T6u(XVtj3mrIAxfuRAq7rJvrSJ(i|=kOK%z`h)xN}PaD8X1J@#g)739)`+4iO|y_z$MY3Ie_UBdPY@+%E`wEV?{ zs|$jQ^d4tWq&aF$jqW|;lRuGfaC@G?f+c6FIpp4ywVRL&e*82-Em`MLA;O2=nt+Kw zQX-K|2`dU5!ZvSAPe3jXkvg+dmNt>0M*RsHe}H=j6(zFXQrG2oxzK_^I%Xzo4dojo zmV<6bRI4kg?Y+4JYo=YmP>_Z3cNOCNYIbXeC%^MfszJWqqs+9R8xF4u^?f9#jfCR> zmC}58Ct;MiOj`+LeN$^uRahu#KV241(zTFpM)OqMG`SgfwRy0rL5 zeRnDfc1-1+f^=rA9RJ`~=BqTpui_fpXkx07x+HH5izD#{ywT9N`WkJ;r zr}M)+sRQQC;bvQh^XyiGn*PIvt|YzZuuH2Q@e>JABp?rWReTj4J4;&L)H?E_ACy-4RgqkNvH{|8zmY`sBM` zpi&hwf+TpLI8AHvIq!N+pd5e*$4TkAkcJ>MwDm!BnCdgaIw4fhAC`FY^eIfEN%R%=oonP==tRNO$)Ym;}HOYSL z2d6`PrxY5lGKpqowRz>X-?p%_$$%1}vf(z=&t-fit_DKThd7u#n?PDrUi-R*kYlUt zQ_?eXZLmid-TpZ?=jOh6M|A0nBu0mJc~`e{6V1POoduHsn6XL(%-!#yH>*_qA_IQ+ zAgRO|`>X-kgi+M-*L>wer zF=XcXAZ)>QD&@51!jVaDv4vVcJn{`~YDaL#?djP6E@s;i&=$4or@uHf+bog1=HPO% zZcf~a2xhCKSktvwu2>x|vk>L7TdrS{7;oLtT_z+E?Doi{2ZdeMU=I%Us^R-sW3j;o zwd8)Pk=-TD0$g4}2#cE=C5pVcdM9T-w8O!4276Pc_;w&1^2>xZd6Lt?58xkrt;C-Q zu)`l$sT`fJ{b@p{6-t$ffMhpVfcw*idoaWddcq}bOvLHIZWy%H>IH9J1Se6%md|W3 za%3R$UJun;tBaIo7eTJ-wM8iF<#HrLWIQpN_BHMCn|RUEISEM|6(NIrSv83=hC^Jp9~=nFRz9fMXlxOO*65V5AH&&bq>=ni*~&2K3y7&(XXm1Rrt;C^$; zWAS?tn&1^x=V(sh`S5ou6~slcBL!WT8lk|v3~F6fVrw#2?bh-sGlhf)YJSUDXW3G3 zAwbCs=z%Bp42)_E@G7VOID?E6PU5v9&rNz?z~ zPH&PD{Nvb2AvqbC6WD&Diltpw$=d~EfVXo6&$ z^vn6;kn$KYk->#hy{9cw`N`Z)Ht zY!c;S?n~YbhIFl0s2zrwrlXm*g*J`;-u0#D_Mr(T*2Hs-puSz3zoAQYu|kB8gWuD_ zl|NM&DHS;}z7N{fU-OrY0D!VDamp&AcnLp45MKB z-tBIv^1jtHE+QMOJ*8!Fe=acYsU^23J7dHnB_=w6#QAKAOgIDre}yRdeS{>NtPng( zIg^(+l7mrLp6~6OX)BxHqmO>LULdPN{&yc(*~YSy6{_t?uo~9cltUm*`tLXqZ9IqD z-4pdeQHMw@MVNP~B0k+M)M<(Vc*;*TI{3}vKNPQ)_Acg&!g))oHoA9lI{nxPE>*PF zZ;g9>OyEt{O7aK`g3u-X>?!f;S(<|*&pRfZ^HjRc-ZeTq{Kd0l=!z+HGPX5g4?ar8 zWJEE31bLf#ZOHkc8wF&>=V|R55DJO!5|aZQ#Qm57cEs@AL%MDK z(j1||Nt}a3M-Aww?LNy)uwQxe#k;hVeR~{A72Ic0zCIgw1n|=x5A^xM$y< zcf!AB){KEay3PZBtbH*`6tAH`TO-hh&$})^$}=TIzs$83bM)kmV{!-3V+|0nRg-2< z-|sW=cYa?O;+FNwUIF~uF{_Q{q!j%9-rxK*0F>5JrqzM^smbL~w*fk3&?>2;03-i0I=Tt2$g_JCv~E#?aBFjuZz{Fs-xUh|nL5fz!_^U4ED6cq3h_x+s2^*08Im z*-Z)9%}e>PZQ!ZHBr9Gv5|^vZWv~ZQ?bYrge^*DCdf{5m^UWEd-sWJXw}yjZwU6@rw`UngCD!#yuJV6{Bjx`+*H8Z&phxai<^10-|1;XV zs?oFO%9>ly{|R{(#8bZq{p;(>kmy7EFA4G9#69F~)CrNKhA1wB>(2epUP=Bw3ccXU z78cxSajMbLx86cv*~50?U?Q;b8Zj0rRzFHEhp3zT)y~D`ch`5WSi%C6hQD9s{Ym0=uDWy5L$_sIYby}XlfXUuYljO&U=9ON|T-<*zgIO=T4P#s0+eIWx>dLIf zm!)1@B~mHlFHDL^gHctQaZFB5YEeT)LVJW`b z!e*wi!Dc0sx?EM1yEMq`|7GJQa@!@eu4u(bm!cl>)V^)u;IJcdyr?te8I09!lX*tp z$+&e1RTHoRxtq_J(V-$co8djq*8UIB-!sX*YA zV{tTvR7NsQ4wYqlM<7|H|F|r>ogz(`G;wTZ=Eo1c7$G%jw1o^hU+<cznB=l=Ozz?{G$NTZq`!#!oRJ9tg{(aeK^iT9s{o4$Y^(TsusXDcBQ^P;Uj~A z#Gqjimz!98*}%L`XqGxe*fF-JXVj}w-hyOIf;Qw0f-(yNhAt6x_Bzr-IA94wZM6${u$;%sRgR_jSn|T9VaYSQZIm0hGcB@T=``tWu7e^ z2aCB+3d!1vj2#x$aS;&^c)A0Y@6yscw!E1XEbY!f>f9q%N8=k?f56pC`b!LwR3eE-9^VT$?Ar* zVruu(^9X90U3+KLnJwrwkDL2&w&!wHHNH%9s5h2;QuwUKkmm!sjQl};uCasUU% zQ(K=zB%Dia86GL5uF_)No(mW>&)G%C&H_D?T%r>jetzLZbbLsStC4b5zVad;%v+rz zeG8bZOSO1rAck(^g~2(;O*kwZuht57+X?yvLVlF&rw-PJ22~1PKS8$j-4TeSM6-5~ zsQCY9tq7RNG~8e#%^dtHPJGn3E88Mzs)N>X1<_gc*vlTQX;5$>c>0xU0cx-LME?T^{g=zRrnrgAV@d4 zpwSHAGc87c8*zH<{?P*HyCRJ?8T@5G| zYfqSH91Zf0S)Z&#tT9g-mWy-!EXe4eN$;LXZ?BCZA;5X(F*}zQHy+!ba3 zppkWqu7Xz)()H15Yeb^Ow|1NvRWt)bNoASwAma-6y{{lIs)t8g9tU~VJ_FSiB(>I0 zPKiwBq-p2Ig>&(g?TM?e>8O4W5KdXCxUrjli9_!~-97MEN$h*13g_khk5@kBhVAbV z+F1Ug`n@Sk0Nvqv`+flu&Oy%Bxo4H$O!j89BI%O4kFt32QA(L4R^_1Lv6|hy!=tP8 zvvt+4P{eo|iDd1Lpg~3FXKg?R=QROFZSZdwz0TdTCKr^jd8Z03DoajdEX!6G<*>pd z&Y=e`AJ?!T+)TYAMUkE8nrDZlS7t0fbX0}~f6SzC~)7_6Iv=A6wJMMtu>y2gJ2A?`r zGv`pPu->loZ&v=B;-?+aJjuQN6!_6U{2MC)Rux+(12J!6_KDVfR#ux2PO%wQhnO`V zW#_kVSb+-*ZT*kLj~g9V#U%f}HTCc+p0+RA_C9_IzA$&cxtYp2?tN zyys-f*|RcqNb7H^o_+HE$|bnmU4XSpWbMZHS#Ne6@Mb*qS$|Y}t>NtN1n_=Kob<0t zoYX%xWn_$Q+I?hfUT?S7f7x4tjlKPmCgC9SuDAs<@Ae%@tXCuHMQd2>ZIE&858&a$ zAxOFUR|4?xncMz$B8oLUT}BSEX;;uQxiyg{N321DlTd|kdcHq?U`)eTK4FMv(s#S7 z*V2994-SBYZ!D`y0TnGH8Uf7{nlK(jmM2CBXBX`~*F~TO#?xm1Lg=-ju8dy@Ve)mT* zZ?E|mW`))x9HV3w^ZJj0&Ft_Y4#M34B;y~OKg_c1Yp>3YxEe5>vxO|CgZaQc2VYbP z7@Wi!u9JRK33S~xcY3HD4ey=n*lHpqK{_bz{HR`qQ`Dy%p(bbVk=;&}OgV1-FI*$= z_^3^T$ebh!2)3Rm44F|xB$mZ2cfbnnzhb(1KZQd!b1ZtMOecFmq6st7Ga+koHR{Nf zV`&8@|47nes^)(uJv!mcUL7HybgPsNZ>nA7Ip1eN*=%HrwRm%IR{_90#xFU*L_mZR zJ0f`don@|1jW7TaAH{U9WL<0+1qJY9+m`=N&WGjUQ2J0;d2_#I4s~WWl?hy=L*eJ> zmrd=|hs&ww5u4&x7`De2dR-L2gUyt!tIGrhZq?>hX;ml4%FiRrqs1ik?mUp^w5k1GFzexXw&xqar z{AbaEq3IFknY#0ygs)b`3VTHb_$eg$HR$2Yn%~m|ovde19rj?Z@l%yW4xB!znRR~@w1J-xhJ$J!^ zIeKbstV9zv-f_vRrl*Nz_7b&d?Yg;GWxvib-1U^W>e;cjd-HH^4ZO<8mrV&l)HlJ6?qrADq4SJXPB;91A$~C(3jd$&1Je4G`v*vZLAM_Gmay(H%#J#S1N86cm z+WLptk36ZHMF(%zAECCksrK0^KjGK}A7%8&X13OZE$I8^*V~yMP1oh|Y@lw#;=p*F zt5J~?tn-M2owTTC#b6ScrWc{tlY>GlDlatYyGqz?Du)?a66f8GaHwWueWk^JATu{#bv#aVnhNJzGqf8 zs~BuoR}^)BLgWPR>{^4)719UrK+mN3xJFC;p&}=D@&vt;@l{lA-pW$yN~s0rh&9JM ztGCgZ)QjWDkN-vEsXIxg8+Vn)5?m~nMZdE-%y>i`rCp@bCavLgd1Ge6?BnMHrzJ4v zGWEmk(=SK8ifF*)Ly<2%SW;BXQgO_eZljyS8uNUcZUuup`|wcJ(HC{kKXGA$Fxs_Q#XMbcGSm^9kYRx}M*Ig& z8BGruMCD*@%@jxZfoP79i<-%nkpvKCo@Y<1g$*tv?;VB&#>D^|Ni_y)5>C$t_s(Is z4Flizz3#8SzdpXEO6Q;L^?CuFKJRCv^8Wts`*^7=FYeR$=^*gZU%`d`eRH1d7>8I4 zGsnmt%BUST4F3o$>odSs{aDtP3we$_RKjwvoiS(0ft!zmf_`wC8N664u za!R$(V#65P{nRRx{dOYrcORdmbdk`{ohNN(FAs*qEc9N17w|bL{ z`RtLF5Q4OE5+>b}Bh~Kt-Dok8hi>A27B~*eNpF7&qXz7$O#;X0l2BB@}xfq+;Y9GlnPR#_=_9rIMK!- zUN4KQxtZ;9SEwhgk5&UJdB~I%#bqw67E35`d0~UBT>IGWJd%3d)-QXt&?pJ*=f);! zO;owDmzKlf;by&Aemu1_p@q%CH48oRT^0P@V75oQQhTj+t6vC=Jf8XWQd{_s3*rAd z7s%n;s7&+bN*#FFtCrnbg@K^+GMcU$`JEP#L2%ZU+qD*|2K86h)HDM+Uj2_gO?;d}DR&QAr`{;w zb~GTfKWgK?H{yfNeXEm%-GIvol3|wqJB&$>uX0B)775AwA4FL?WITR=OQ@`43|jDy zs?zJjeVHwq_?fi{&bM$d(-ns=owoWBGvPz#^3>>;Ouq`}^UgL*`GSt=UA5w);geei zVJz4^quz!%u^^V%37M{@T}AwnK$;0Iqb_1e>%`?|Asr`vudDu1YRi}2hrW~MlYi!U z_?th4CzmPp@zNof2Sl*BM^-Tx5ae-QHa^FwVm9yQ(kDMkbgpwFX^y$TDBs)e>Yk~M zVPLAg)7v$8NUhX3TKs4%a?ngLp^5i&j?chL;ID#T`mGNzS|aqIb?ae#S-{7Zgz(wC zWM$YG*}3EXn3>u@3M%SDWOgvAc zN|%IX2=IR`-Tb#leBB*=mOk72y|~<7JrLzi>+62W=f{NB32vk)+ZB>?1k<6^v}|P4 za|Uax|K8y6x~ekm2lF_ssc33iuwl$%w`X)&7g&iP9ItRj*w(GCxcIJOMow1q5k0GJ#D#`Udf=q{mfP~(+J1cO;4>oYF`Br58 zlY*C+hhDM)p^h5b`8emhRP;kLLp5D-^nc)j41VEUyv0`heKUaP&*%&=Tu}d*0%d+s zxCZe8n!WMz_bvX!;(yq^6(Z^~VkiH$K?4I$VRynQQxh$A3FyqId4c4OoWATYj*f;; zdt^Oj9eOg5Mq{*T&JZ&8=z1ydijjlW4$!w-3Hagrse}j|36PPj?C70NTA_JOnr4eX zgjFxDyZUBBU6dBr<`9?H#;X1p+#sxU9wG<+O61xykA#sny2;^z1m4lOn@D*Zp(?F! zNbrdAjdGqxeOZE`rrx*b{-eq|!m_EDI`izRajN{b_@XO5fIQBcpLMpv>h`qRi+81! zJj~XG##%#_u&zMOC7#LEi0b4JJIFxo`r?WQTiLr7a@R`94a!8VDr*j)w!K$ma}{mS zxm;7{V(5yfvPg`qQ0HC&lYYX_fB*H@J7eGm!@wx5t`pPq`LkK^NOnp8NRq{$6^_T6 z-&NN2KQrDmStQQZw8$z~CSGEbf%=y(vF3&Qs9*ax!@Ku{-;PA`u-}G#$=ag+8ck-g zpB8yzo1}shm#PMsC4F4kYJcH@>5BAuq_+Y3gXp7%3I0y}$dRvZ+2qERA~Ju=TA-QB zccZEY>nkbUWcla+4x-~jTu_wDI{CuR=Vp(xhpgeymEtsCUf+$~xa8CqM;RhE6%vfkw~?=2V=-Y0SqUt-C3 z$2*pLn-9MMeor4oS>MJk9vl0DSNk-!Yo+71^bn$x%SY}*#fy~wSpvVM_cM5!cNv^E zN4#6xVi|qQE?&5NUA8@iDs)%i4S@P}Ef`h9xb$vJt#jzb&vv!q<<^z!-Q?}HtzN2A zx{p?oSj#@iiA{&=Hdi(JblbMdiMItV?-GX*9Y*#-lVNbJsju)mGO_RZ6DjLY&fiQNAqE0ykI0msRq%F9}JmaTuY0Z1>i|J zphyCL;i)u-)C9`A1mcYK3*ZKeo_udn7wUf+{we(WK@0)=*2#hU@DEJO*5;UXQ`a6Y zs;)_~g(Y}8cFL+D`HWWW;#<<^A3v z5r^1dp-P#r+~Ov!OR@EL+9(;)h;CB2_z|3>KB{#!+*|ztsQ^*=_HFQNpy0P>R|K;= z?1S#PaO-eu`r=-brmfFi&)+g&neFodwcE5|I zvx*F_ptccl2KMt3RS8F|^R>ZI@T>(Bq2*P_i7^7YW`v>xyjimlpkNZE^1K4C7>#2> zp=d{CC3Trx$!7NLmQ|2+%HrsZmbHu)*Vo&}Yy5`FvY`8IRnK>;7it;i$M){-c$v@l z_PC+{_jWfQ!@K`;N85=D1&HLNe>>O*ieAZ@Frk zRK}HtM-~4*gUS|_YBldMkl0FPsHe<4zf6wn&+GaP-qz%?+e_wL>l|<1uh`e7mG2bG zdc!r2_#K9)?~~Qr$b?USCy%+~?|+Nh=c^IDZaV5yiHEp!%US?@4Igg%Yj@^r=U<%> z?e!WL)-LsX@a{DhOCqfMV|(=8;4U|W(R5;TIp0$7Po5w!U?BeiUNI2l literal 13229 zcmZ9TQ;;aIwx-**ZQHhO+qSvew%xtkwr$(CZENOl@od zuDZRQvD@p}yZkPd|DI(xBrhu@(`=IaOr|)Qn8)p2O;Aa8mY>$Ya_q?@iLii>0f-^3 zUf%wo&^eFlc+3y)Wa!9D@5SDTxgUms{m`BuAUGbs2YLXKulB*qoup6hBhe4sw%kXH z70yQ;8h_tfuu%PSb8&gyZ|~pl-#?~>g^APa|6%9y`SaxSgo|Zc{E|wJhV*zVhP|qK zv!(v97ybpLgCI6~_Td07uJ~adP72YE?0?4@QS|3kD*;;RDNqh0C0$|fOo%Dg+nRZCF}q+Iw5M$t)`I-=9dZQn&+wP%Mp+KSO1U!tezEcZt6E| z7A{WQ^v93SYv=LTm&YW)oMA`MA`moNcxc38|8YO?EgO<8ML3VvO<|;&ic$!Nh%DU1 zhf@q({g)QbM-UFGs3=L2M|wE?gFABYE;?jLKX$(JoHEHA2Snj{9{nh~(Yjy=z(Jf= zCOoLp11LKXB)>u-#Y-`jvhv}>!h?n9LgCn&vYubRn9s)fh{~JIhYiQI>`?Z2l9G4QD>j>Zn0%c+p5JoB!9;Ik5_avA z^LAVLfe;!vd9>X46_5|y0}J)sB7O`G#D#Jc#3zGTBoJ>s3R05_w6NQw8{+t}WU9GM ziI4cAy@U_Vh{SjzsDzx4<47A$VPS6<25zUPEgVPkBTdG{r13}YPjH9|a4rfyLI62o zSI-E^5Hb={dl!J`1Q5dc*zwRNKV~n`AI?*MU`DagNNOZYm5&#nBEC?Y7h)Y@4zXEx zghU#U*c3E!8E#$|&`Cxt3#$W~V8fAI;Q=_bj&|T2G(xoUt?0P|epghe1WFOPnkcx45oaYvjhKrwRmh%Sp9IJ^bU9bRK9tvDPgk?<7~k3Vee zlK#`~ACej|D_)ov?mM~9&lql1GGCB*O);*j*>}@;GGOJFhpMlkNvdA+De+RAsHNE&$oYl?vsC@sym~j2T9PsNI zrcA4P84H)(Pk(5UvK_au|1%2O9&X>Lt`QjtYZuL9j>(Te4p%GRZW|fJK!brw;6iaj zP^&7KVDxMc>oF)^2JQ;G&`fe>gu)-Pe`&npWRWjZsEzFEdUEjJl5_oJ#kVwJRK=isFD}yLF;gRlZ*nr6Kz2rQ7c*2+A@f_a zG%vP<`({M$B=t9s#|6sMkkmrseARrfJ@CsCl=4gv=r7|K8nNbTmQ<4#Ia8WbJ zbb2@vG9m zb(4lQ){7+8v!wuin-vBRW(}i%eD01XXJX}EUf2d$1f1Z!`o(Q3{64X0ixz!1iz7FDj(FD!y@4zwxtL(Pa1nX?}`cG-sm7=MbAgPVCHEHeuFhZ zhca|D0z?4Q)T50gGiAI&9veV0|JC}CH9`XGgFd+RN-hw05aGcQ&DIrGNKQ3ok$^eD zOnCHQdRi~&TF&?CqlA0xWZ2N*67&JD3HxGMXA00+by!@+YHEVV~o5j|C`4v^AQ2zZ~;rl?ZF5?MrL z8syN|&vg49FH?P)=qRwpW^0i{`->KxYsiUTm|xEs%JoY6sY$}e%8Y$_yef5$SqN!N zXmDY3h+-(RPiBn`3L%0B(P-cG=~j=7Dm#wA$@O^*BFm}$H;eQB)J>d#@&ydKY0@K5 zdP91`?U@nC&td|HiZllTQYEvT50X(c8!OFv}v1Eb%O;ba~$B> z(Dt%QuDEKH=sJg?LMvkJXm?M#SUz2rzxy|NHZ}2&^JuMZ@v2K#a>XG*Hn}W|uMR>5 z`vSaihO)?wrbO@wffvW5>`cW%Wed=y&^IHQ8YB456Jwn7O5)g|f z7t(Ufhih}$g~4gBo6UozuX7X#vE~HZak3;5sYBVZYyk`!n~3?yt3U<_$*BgFr)Y`P zRatYCtgDo$+gchU+@L8_{=ggiQYmE8qf^QRfm5qG6Nt(Xjtt)vplCFP&?pk%%W3#H z=N;ikR`q9T9z9(O2cvBefM_;@9YC0e*=QKV=B$@gW?eeijN0<-vJ_21jnfExb&1gH zH1SfB381JW**)r9@RSi20DM~J;PpE|5P12m+#g$7mY<@J{wAJ2B%4m>VN$B$x&{q& zDPP`7%Bh+oGjV}8XN^6_mR}M5yzbBAbqGd3G zy2Io|JE~jdkwKPx^B5*ovV!zWU&p+8>piQOyY=YA-Gv1f>q;w_79HlUg_$Y6wH~tx zP&JyWV?h~b(zQm|GxA~q1TNF{KjYD3BK*(mMu^HpX<#K1Wc4OGI3NzN`vC;Z@Xmg6 zByUV74u6xSw>2tUm-dM~5*b$MoY^>j+~;<~EF z5+iNWmk|)TK<$(^nxT^0Dy zs^3l>K6ZUo2$P{xv8f9`sMOGi^wAw0ior%OAj`=v>>!4UwZ|wUe3jDvRT_$vW1Nax zDLyXg^v#Q!In7{X(z#m0U4u)@C)F7p3rMTm2YoEe99K3am&UJvbXa0u9+30DjaX&@UF(q&_eaxj;Ngz8tVS%_B!lCG6?pK$bn zwTALZ6w_0Tv6N}xhBOL=5?K)Mw$v`?FfK_o6B>HNN9K1M?~2YJ6M%vgmh5_$v-d| z3u^#xUKBE9*1MWq*c)6Yj7Y1}HS2JtAV5SRbHq=Rt6&Y$@*o=RC!p@z~k+47R( z%VVfor!SBg8`55wT6xL{?3CqS+c0FMLbJLNz4#&+VTD<^H>hmTLDuQI)$7b7uy@$> zAMFSur&B3~hfp>dsbrPvaayv~Z<=U2-@p-7uaFB3LCGb!FJ{{iAY2g>DesdpouAM- z#=<68gH4#x=w$XYmY5rNrV)glJ$)Z1HcxTol9f=}$ZWy>GmdJRTbbdT~yqPdQgcvTE|y5>933 z&Nf#E6$mQolds4KTKY&~EL}mF5B#H$ybjF5&Ce~!2RBIg(W!@$of}_*RAgwvZ#pk)I{-Q)urnZbXQ^zNohMkX=_p~w| z-(kiBdi$IOvOh$Ff)BZ`mar}l2sXQBVWI~MTP6+EmxC9dF^oa!iV`!DxcDvD0oE{9 zV~7rOzyW_>0{@qBDecd;GA8hyBtYa#v$yPwVa6kiq>!;dbOE*ui)%V7H+#)6^DHur zKpy&Xc60O-Pdc5GLZj7HIoAZ7unQ;xCD1`;EsHdjtq^*@PxDWtNmC?FqSi9MI891d zYPjqR5#=Z09;?kQC`3Z8wsrPXDVt@#D60PuiA<|&86O8+R|*0OZjnqp4ZI|11k;Qe z^Dv`?9*5Wxf&4ciM%l$@1Q}Z%x zszD3s#=wN;K(<`n4czs26>9n5#6D zL8o?QrW8eZ1An6>V7XoLDJAuGSyiGMaeL|+9c_tCvdfw`>jNh#&nn#2_gsP1Vd=P1 zpa+4Jjdh>cMkHLhk=cZe;xgQ84<0WxH-17D>w6Xz$R< z1z}IxNrC0!y>r5i@@Z3)Hl6nsh_T)X@)YOgl}7`)wdeG|c*sk}fnG8SagY~`g1ZD{ z7C=bY26M=VePflF@gx!sjOX*1F@I(du(_zjhx%uWK9Qe3u5|I-3`@8v!-L=_CWgpfUj{b9<@xJde8NG1aH zsbe>?ux0fCj1AnG!g`QrPVxPIw74en_MT40jmuwi)4!rY4P*{X}1R8v56XK`2$`u^gmFZ)$0PbWgaC#n;z1=Rke%=QLofNnoGf;7)gmow}t_!wqM*M3Mk)K4xOe}=cIqb-XbYEbmfiW)_6 z3ob|f=ZSo-S)7W;8CD1cmBZpnCX?z?yGyR-br`8Yykhce$C?WldJ1tAet#b5L(F_B z_Kj`kHyvjYQ0dKqk_0=9{N_R?l2Hq&mP@+BjO_KwW26!4gk==tA8s@zXrT~izOso4 zi`l*{*R`-B``^OtM%$<5|lQK03*Mw3Y=k$6;xv`)vG3BC^SoOIj5x`Wns30MV77X{r==@ z$eim{H4}L^(#Qxtv9vO4uV?5n7pM=34gg&Hi8DemI|JWv_ntg{C^p4J74v(+JnB@v zXP9lOzy_nKmS|g2--w=8bKkTeO&yBO&L5xl{GX77X18Faqafx@<<8$6oXG5We}}u6 zPGe_EpSRTC1j+)MwDb)P`1AK7DPcaGZ!Uq(;Xi~z zz-^-9Vyomr&yKT7?HWoE_No~_*^qz9UgAzdtOWvT0k#D&ydv5ig3>DB#}zgp{L&{A zOHUuZvtEsrvoc#SWr4C~A`bkJuSsW3iR3~u_(xq*LsK$Gm`>_;1h*cZ0}3@|r&jKs{FB zP>a5GU!<0a;Z`nG4oWZSF7xzAu+E+A&i;xR{zHX+!?P$SYh)P=pQ6ca$n<44#-tTf>beK;C4;%az^(K4=&e*BL~E zC(uWQpK>pjJ|RgA^ONKJKsA6qetL)ea(*V=+9kG2c$*Z$2&9hievXgFuyMT#g;}s- zp8e)$)5xFmon5%CL`3l#<9T0E*vau>N3htrI%M!MOX6UZe>~U|4DSYqLoCtfeQnak zv9U|6-_qxbSTy*Eb^{@OE{CslVP`;OWI!Y3~eHdV9FS*WTVC_e-lQz{<~`W(N%x&{k<{FA8hRN zq4@5zGtBqSo-xcnoO%|0h`Gv-z!1^0X(?b9_?xg9Ak71@Mr+i91*@BbZ{;^utZ<>1 z9!@_x*Vo;{$K&hkcz=I>GHo{|7A~)w=Lat4ret#X|0!otD3K_$d<_$9FL;7Ql1KI< zJCQ1b{FakVvhmCr03pH8rP0UH;CwZ9!h!p}*~B>9x#NTD`ISKf>>|B)a~^r4U4FCp zS}DRZGOiGZilkMooA2OqAcPBnw3^9H$5G7*a2_MBd>~r zU3Y68OUNU4SGgsZ_VA_^#|Wa&^&c2BSE*Xw_L#|ZW`k>}APMT&^+#nVcwXKUE>?aKDVz;TJ`11Mjo{{et$kSpIBpFCy)7oYu+9x+ zg0x#7LjmXPrc2PG#&AmyXQdo|mO*Tm)rT=J9_+&rCs$=vLDoO)RHN2`iy;(AwRjvT z$uyFKTJc`Dv9_~qX;^z;9@UqPVmg!#0{gA{H)_Gw59*27?jW%-mLhRLi=IuXDxI5T zjAWQQ@Ldh?uAtaMGbJh9J+>U!T1##59jxUKx`g6)I%M2X_OX7S!&Lh$0UVXGuIrE! zU(9l!=COfp?!!;#(O0M2uOC{2@*x6mFpw$_QLF)j))c_9$sszLf-1wCrbNoJe!|kN zPkmog++PzVPMSD2Gw|YsUX2i$G}?j&U2b-hrqBEcGi+lMN81@NnO{waK5~!*fQBo8 zv|F{1zVYtrAn0sIR~L_Sk;DS4H`3XioU8dWkKL$m4*1BR!qRFO#OEay+cq$4;+drl z;dP9C=^6E^l&!!U6Cw|J10c=;0HKISU%Zd>;5%Q@hnZXg#szd#?OJj+@9df$uDq`18v}G#gboVT&hHwG|*a#&xXJ74G z!!0(e`>Dy&uBNWAoxy>Yrid33Eve_jEQ2?;daV30lQhqej)z1$Bm!k>g<*ihbUjA} z1YYa|rTD2EN{o%<)r>DS?(PpC#O^{#{lFRSkf;8Sk9TB{w{&tiwy^pZC{ye-6Uk#l z6tVOI`btM`7ZHIlEvP1#yTOq-J?l&T>so{A9HRTmMUv%BdC{=?$Brf7BDGVxdkiGdPeYG$FtL)28(2=g(%Fu~ZcTFB9f-`uY1rC*b zR1pLxWP)S1wzJa&LRh_?BU0!=@=v(z9En zKF!@k7UF?4EdRRw>cci?^71msRj*|i`{`)(VlA2hgBZw0@Y3DQ9{_DrUxZ2sqN}r3 zapd#`#4>f#aj;HCDHm_Sh@Vk78Jpx==d7VvpP{hA2lCmNL{$qQ>sHR28HldZaF2Px z`WOif#A305(Q#!s2~!v=bFYH3uhbulIf$QTc5wnMC|<1)Bp{|Dao`sIZhyah-|p<_ zySwv8clCaKeBX@b|J#TXeNC`F;0%0`B~5 z)W>&s_x_YGBx}=yPb;p#9OfFX~~sdI@! z15CE8AgIB@Ot!LW(yT~gK$lk!F8gb6F7=n}%j3>lZyM!~bDH1V=*36Ul})J~F53~R zToaF4tqDrAClF2D3r)wSysz!gj&l#hG;*Suky+wx8m_h`3(j2@;q{uTi&fJPaJ9)A z&|m+iu>B2>UHE5-Q-d)&>W<=ymQ>G`S##DLYqKkz2AD zug=+>m%xdS+lHUpzg@tMTG)7Hh))C9yL5*&PKCAK6(U&V-(c(rXZLk}VsmAS7Bt^p z|D#Zp&(xiu#bqzqJF^N)=EQvrYG`|cH>KRH5J5ZJmHlpxpDjWJ9gz zTHPH(r5)F=s9;y2nsX*uaY$cRZIf#@%aGr(bq3}s?$|ps3ro&o=&s82pV-<*3QLw` z>0iqz)$H~s;yTc3{SYN&Bd~SgmE@~0^{|wbjr>x^DsnDsk$v#Q zb^I(KrM0aOu#(X!n;k~U>XbaFyDdlb`zJ$=*W!z!Z`g?+mHsK$`GT6TMLL3|?7L_8Y zb%2+=|D+WTq8AP>eAEA9Px78$d$&f7B|cz92({^wQ8jtiks^XF2L}{Ufv>;6IJ=?A z#8JGY4=2-dx@y(PdBYIu{0&c1mU;Xko(DMqSR^cMFb*Jtjr>E;+`B6UNx78Dk;e$Z zaZRL`nhD?SHV6qo^vT@LfTaC?3Tp@qeQ`vuyCvWHQxjvwPVu@Jj$LY$BEYLvI2J@` z7#>rqb;vx>R>#(-w`Xy68+?>HqAnw!5&vukVV-0LwlW7fT1G@~rD(q;SBpZCB`jGMUJ~woDXcvv{ookw^Kqi3NC=^0cuRv?*k&RLQRozQ= zrG-;VThn_=`3(sJPEXo2@XQIL=YckpghCi4ng3N{;nmkgN(TQzF@sj^+XqdW}CQtojyze=H=q7J@u=@C>~)#bA~vDpglc zF)C$de(WkR@Z;BSuDS42I%;IJhj>}beU()e2g(*#aK!5gJfJE3R9&0bb`hbV8E4gr zn4Y$kx{PdAF@|jce-3IfoO%lmzy78dA_Wm2^j`Av8Wqf$$T=8o-`7cwOp-@#UsQ-_nMO$FNZFLMl}D) ztwwT?tyfi=K&p{!%UW5QZ+6lZ*Rgpi>`5QEj;}L>(EGN0Jv!JfZ7sE*+3Dr(cHz`S zBjCIjoj@zh|CwFWt(LVNUsKc(0LcyB+qVW>_y=oo0ne8B_{K+GD9G6jQ z*~?LCO{Q7Q7HG}3Pi~_$XcR`5nw&shYdlSa9Ay?K<((~!KznvLjeCU|ryL~FBC6wZ z`J|&m@8RVFro{GSvG7Ch)v7|d3u;2`~x^}BHfUzV;@Vh=x8C7nQoKYDgb8i>DRX@=l$S~!F7`u2nwa76?+U!@Ylbvfg z=j#tAWEdfh;Lx4o3}uJ(WAM`Z7+VAA*^>}|DmiG33-d9UwesP<}N&tkC z_zleS8=UiGMm|KPo;pKxQ9|mzrBZ9eKA&;+)Q@HFxPau!b0wzr+L^MI9J%>8DAdNO z;Bsf)&h0f@eT5ADq?}X>sy2?npKLDb@8tCIf1h8U7voJguBQDe7wQE)IfE0zJrO{v z#?^Sd*(*X>?(57ixmEP&@4z%Rgw00e4>> zs0V>nb1_&`j8N{cPzR&w&t0u#D$8e8hcqKt)fB=<|7t@1nl2Bi*6HsyAM}b9 zrp@N1-06#p(~I%`T=M1eY*k`-v!f5Z9pu37tV4iMYZ=XyjRd9#X2M={<#cU?s)0@E znp)@20N2TBuKy8^9YwxNSznYX&@H-D9tw1Knb=XTK8;Rz`Fo=xAdY6r2vTu%(yRZ~ zr-_4EEXnRcdU zD5y7af{mf9s0UDlffl1Tfc3FZG9i$=1;n#DFxQiaC7rVI5ijmRBbhi2SW=nKvh2XlW+5FfmQ+jq zv#6FUV*uqK<5cFP*J%(}JaW2dYRiq257#G8Yp=XyE=Rb-W!2CGopRoU3wxJzKgxy7 zvycGVJicgex4nI$2CA`s@j40saukv8-h z1tjb~RB$D$J9z}`{t7CAdrT^~v}PTEUV9W3EiG#nG<_yJdWS8(jcBaNaC^rs-9H>G z=6y7dFvI7|Vs!sdyb&TY4&VCvI2G;d@U*x4+vVeDdQ-l%)=$fjSR~J&W*u;x2*c`H zwo#Gt$F!5e^?rwI$7gXm*a0g57WkAP&9lD1cPaH$K`-$N6v&R6B+nzrbcnHl_vN6! z9$oBw9qmFWMJ!O=Z-e&XITHZZxUNNrd1X*uBRub3*OO2a0UfNrh-vK^Up4c&AESfa z9dNLW{w)jG;H+i`@Toxd*iBd%l?+5lfary2I1T%JaeS2_`A(S<6Kb65 zQJObHbssm#k#ZSZ-@EDRmjY!qCtpG)etc+xGT2C9WdOxyMJc(aUnX(#h z2G{7Ra#bEzRbnw%7aD7=RYJPXXRLBPu17YfWY~d*YS))gz1qe+whXyeI&6_8Y0+4- zu4vnPMHN>~1YOECb*_NziYSZ5$_jQK{NtyP#Fxpn-yVz=w`m7Psbrn#UoTtDiYIa^ z`bUW8_v)>!bU`&&3j$c08w+BWmYc;?>QnCVsSA5I?(t{EMoGU$_a_Gn@qQig)V6>#l;=lR57^_8`xH;UC*XBGxaLjijl1gdCT-h*V9TB76YPSa6w!(4lCGP@ zM^GhfgZ~UByv-Z%Z@8)Fx^M7nk~@uYX~abaCTPZiuXyp4HRIjV`iyfKga$SLg)t>u z1IFCYng_;Q0c65>T#E9S=S7ojAn3Ie@>MmINP2*Id-A2zg^xSiN95%psw@u^N6D)t zXA*&rD2rY{jL;?Qi)-EtQG{$IZHeM&CnKDKYE4;--4dRPB3oWAM{`2k$7Fk5|DyGC z>~4hQifdPToDs4|fri@K$l&z1J`k3s9#X=QF>)k8Da^nzEkl;wGRytm$(P_C8bJB5 zoC$V$mGgaBI+Y6WwMnq&ZIM_v&;sU|cl2|1{cLKBZMCsu3(tMdYEy%@BAFclPO%HG zU9Hku>})s09(J;f&!cTl4YAyw@eYc&EzegyXi?Q9kKkU~q)64ibd9{^vzhS?oAzWu zFY`T?2mcaBx-;3n+}$|%#o_nyW$(r77wm$B(wIb3>w)-v9QCz)Rw$M|zo!N-lGuyQ; zb)@7n@s`d%x19fd2l?D7NfT(@>x~mndCU7yt^M5{R9^U9_JN-kP32tv9Xb;~r3nta zsZ~DIn89q-zHDF1j{ysqu~Y?twV3|>b-w!`CNdE_X!hrSO zBp0oO2^M`2kSJK@RItsn-bKhU0CTR12gO5t-^M6?EWZDB9by(tNl^6s-+K9 zsK70HzDFV+;i9aLsF0)5$z9mJ%#}t{8JE|zlHO$Z^!)xv++JSecXOcW`)mJ9pg{Z9 zJNVDfK79Mp!+n1K-W*%5|8}kpi-u9Zy`-l2w6=8pJt~CTo>WrGDENc;-jP@H;D?Oh zf&JtC)Ijr_*NxMCb^MQ?Rr@ivr18A|$0*J~$T#?>LT!vNJgM-D8NtNDMiN!)wiGx*6Rh3mn&Dzqy*4RT?!H!4V z#M%bvs>jO-i>;9@yYHDo{wcSyd0sx5WYg?D)oO~$%jwIOu!@*OYtHD@eISE0Vi_?V zoIVkKZ16b-%rTzkfe9A9X5Lbh;Y@pU20=`vV`-y5Kp53={G0 zRur>IiEu;ZVK3|pObbF+8>yK`OAmltv${TC+^jb4;CbhBf&lm2VUzWMR5^7|pT?0es+0j#0b zuY8RkggE#(F_U*cUe2py055K1KNI>bKJx&`EP^WD3>fcGp|)?~pPDi?W@rV0uH z{BLAo#$N2Apeg_=WG_A>xaz_Pac&7=%@59qf%~Z70o~Zy_EYi%6Ff-ymcTgLQFN14 z;S!L&7?pH*P?bAKb|6?mrhJU2!iz%fgxS!)j+5z4L-KN3)f{$>)-f6zDwZ?q1|Rs< zDyg;QEst*!Rx0QShZ4+wXbrNhDa9MHL9z596OdY z2gAc|dpQsbbO{($}$`Du+(O!~zO)nIGoBC^pxxnf;)xw`jWZurwg!bL8uM#s$WUQ(eb64B^$ z0_n!g?I&r!IKUjBf@GgJF=7R!4YpNJq{o%w*`oDI25C=)@qrbQIE8qlhWHyQNnG=Z zISy%1=n=yiBS6CaA;J}&7Du{{oLGRV-+s&CqZmECN1Z?q7~TYt6k4q-YB%635(5Os z;|>@&r2!rOAuADcVnuo3-zh_WhKS2?qEVA(bBT47Fl#=o+UhemP5UraRY#LCc(!Zp zLTk4**NMQcbYQ#lQFGsydR3pOZ!cz%F|M!!u(gH3d1@#+kSX5HO-a@6l|b?}n2a;o zG~%%#3eQ}p%yRsdA&l2#jI&7|4RNOKkOfQQwMKiisfpvfO?o6Jr{?a)ei{2x-<$V$ z>-75}5K}dObCL!#w@TD(53Ebf;Dto|mROmNMY<~csVKj6h2715sgGhax_*9+AY_jy zF!~O8a*ZO*MiY|U3+crdq_SK_^C%Pwpx%SbyE6dS!ZI5{qeB=QbOQwt?rnH)x%mi! z{nD2t2irh73DBe=S;WH^3>jX6Bu@LCKcV<%#DUzcXQ&6q7>izNE-up5=!nAcy&>NE zz6t2Z6GFLW#WE%~$xm;1k)i{qXy7vn#twepu)YZy33C_K`0p=2d|4dTf}3q*6n%Ai zN`6bqYy28j!FVHQ`(T=z%4Q%9CZ@Ou>G3SuZCD?+@NYV*%eG z(k~OP)B2L^)G5ys2{3QZ(e>ou{L*v1WQDi1ArysScb*(1$sX)HWMIE+*&h$Qlcy4clTIQDg-7 zE$3ZLk4Mm)VHEa%FKLCMK@72J&>@MNF=(upNKg^;vpD2!_$y3=H`4@F1`IewohXPH zc-hH+N%ea>;5W1*O+M!)W-#s$p_+qA+8tq+e{kz$pgB0kkPtI;91qMUk#T)2YhZ_t z)}k3b?XTvLjW>->_WM>^GU(twadE_bIotrm)rCD@Z}*K^6=UV|@&R^Bd zfX9R5`$u2EDR1r+$C}JGo2Sz20S=?4Mn*1^RVf5wh&6P8B0uNVby!1ue;5~68tl8% zU7aOmggSnFy!xmio(O?HaQK*Ww-F7nUg-FvITF76^E(q}ZV)Y-^bo|^6x#A1rHoVP zuHIfud!@#QKOYY!{@k~sHElf*mfo62fJN*ku)grX+E0waOsDgUQvj1 z$6Ou{mMVWxa#LvK9gxSm7j_sD9NeV%)a4yo5W76dt{&(qG0;<&BV&1-F7`R!#2^^r z-~K5|P|UMF8Au_Ug-e=lj^L>2EfKgLCcF&FnO!5CgDA;aQ6v7BA zg@iieoDkO8gcQM7nI`onv_KGs>V^K;V|UIM1avIamSjHX5kvQfT~PtNTP+ma=$Y_X z^juA$FDNIN5c;-;|1fZ>I+c-R#`IXQBYg;le9bp012pI!__G_2|6&yk~?G-G1LVrf63i>|2N5P za#|%+k%i<30roxpj5u%c(iN8pc6{qD*5-j!KN!)u`s}#*zi-$=-_InU8pI4NOjxJK zc`5TuqR69y{Y&d(6ocV|GAnGq5W@)(O?QP~uMNp4QxgpwonNM)GaXy`*_;mMu4DNX zui;S5)9%62n$wo9F^#}}{w{GSi*o)(DrZ*mK{9M)XCYX%-a_lVTc;xnF@TwXTN~iH zXtrW!P6Fl}++bDLk@ScY+2Jx!enx7T=o?6z%%#ur^#o7=G$uc0K>J*mb(!uID`JJ& zE|DVUV|mt1_$T#_dc@X#hJbhf@p6$)@h*6D59^*q5is6k?;Y3Y(F>@MnM# zpQ=;(h?GiRm$yX8xJZn+uB9=+51B9)3=Xp`RY4~`IwnmLIKHYef+-JUPrIdqMxrxB z$C7|t&HYYt+7o(W+kBDZ)6=GOG5ml4j$}930)oGvUI>HYn)Q^;tWSrS)m@%jmLyE5 zbtHwaDHV*HAzDf@1{QH4z5nw!c*YPD2q8^#@cJDn2(kiC*4Ku{`KS0}@QJ%0$-2*J znS^wtzEK^`(ub$2V!HOoRE+=4Nqx_u6(Fpa*YkONR(QumnOHN-C5&I)HkMa>4dE0CQlLu2N$QNW@F4A> zcl_}gVI6#ANZuGu?DC@}H`L3l7xuroC4>;#`gbf)yX%8{fX3`NT7#4AT?57pNtvd z3cuZecgl1jL_89(90DmCti&e7O_6bOwzYvVbP&GHSPfZ|QP9b!jox{TE&|l@09CTu z5|Bgjz$lyBG=ivOIZA+N`9d5eLe0_$A!kxKfR1C@Q|nyRd-!3Kqj5>8jj43q)?ttu zv%<8>*3w$Z^6l8_9jmVzVM5%rq`1D{l_}t?4$(4?3kqJ&i5iPf;e{y-3NVXa(ZtymCmZH`oSayX1=f> zmopoE=65mW$mJVXZJhJBWlA>CF9XWoBn3zqhM=gUN%Sxw>r3RV84fbk$?iR(Opb8O z>%7YE1`>gpUS@9)_w2UIj*3p~WK)tB)R}`NH;2@tKr~5& zO&Dz;d%<8(UIlXVDwi%b)Ya_78sYrYfTTK2y#`-YM)*A%VM*^L9grPje$|`qRKk)1 zJ%pCglDim37G23At&!MJkNeE{%uR}Khho9SHC=itI4?Wgiy$a{h@ zO2v6Fno85Y@$2^ zxg$cltHUbWXy{1OzYzl_EtGxq+4|}g=?F2Wx6hkPG!1s&N)!Zzfgqfb0Q`-C%rPPy zks;jY7~a?Gsx^?3+?KJqUcJ|RP_EHwDbyK$gFK8*U9lVA_r+s@{h`7CJNrb+gpd5<3OF+-F9#nlY#;Gkhwj&`xVRF6 z+C>m$+2=jcZ^rYGaTlTn>=|AvBnK3RT6g&ss3M2?sTEO4Jszv-P79<(D$6)hG`ylI z8EL7xcgicsY^I$-w@&FndjeF+S+F}RsH-Y|(6XxT#yaz$^G1DyvQwfHL{Uj@5+R3T z*gV8qf@y}RH4-888zSuUL%z_@g*LB?As}xg{{tT!-6SXV6GEB9zvzfXXAsCUIHohR z^DXWt9)aLzkA<3zsR_A|Ov3P%sxf>jW9tUtwLpX;{Mu6~s1hbJXO!u6D|!j?RpbN} zYTr)7iZfOjYu>uXCVXJsp|?Fi1Pe}9F-bp5s5HUlAo1zJ=dJL|kYfRDjDZJ1%20{N z1Qz%VqwChC#KIN^iJcouNEpbGVDbXQ9iiR8hksjXsO$8&T5Lj~8Z{O(1$Oll$U-*9 ztk4!#blKbk*Hg`-$j!i=SO+?}nqQk_&uYCBuau~9Ei=q-2O+A4dmG4X#o;=FCo~w^ zj~HMtQc?O{+f-drt8LOoF?6Can?Tf zMV80pl1jkt`Lown0%EHFj59e>LUt^e{wRv|bomu=tqwo`PDNtJDFSSNzkT2C?C80<@kMv{eSLi2j1~MG z&UyK5fAYMKy?S|db>aQoFCwIw+4+%p_4(!BA54B-zHTGjy{yKjTwQ-Z%cazAd2*O1 zGVWl!Mk|+uJ!qx|Sc(rU5^q#|Pomp*eAFPv14wd}XH_=GgE$OUjRHNT<>R0)7zB7o z%g4Zcg3|M$#jX7X`eI;@drkV#mtyLIE1XvsYXxG5%t z=Ow2@E5e$t*zv5_n!r3NsEkGbZ~1&zj*GiIRKz){h!A(NufW4y876jK7AE#zz|&p%|A7A*ap7@Ow80E=cz`jyg@ri+YVeZ(EcHU>yF4#))V-;s!y~wDbYAXZ)#WY{3dnBvKC9wQ8~VygSBBsk-O6 z(!m_M<$>#=t~gMsgTwbwHKM0U*GytA z_{{E+jF}{H@Sf?zMFA2hCKUlYBY~4tzQUX!I_O!nC5@(?109ar=lpY%{8bRgGR1sk zxWP=E*-ty|Cmq<0RN+VTHgB_8QW5*25I`|#$J=O&Eemm>+&SrF zJz!O_77MV?f0$dY`)CR(7{{2QTF})B0hvMI@| zQu(>HRO+6lwW#cjv4#;j3*fY`6|=TE)KMC~YR(*xz#c6FgkO2;de@0TDAlL2>@x_QaR`*;yYumo;keb_9pEJakS(G?UdPvV!^(M-B&q=6ljq8F| zg1u>ra2L$ACQBFkpO~}JYtEGq@^{qk%MGFinRhJ^tZU0H(T z@vQ7WH=g~^OT`sSPOs%rvn^|Buvn&X()8lcJ+QP$pBz?xbZ=8fTV}pw z!EhaxRdXWdoc21=V|g88_+^n(Ocbcfhy@k!Mpnf(m#htI2oj07g`^kuRp(B$WTQwt z{@j0>th@=<^$mu1Ek|Ks$+ZD;2^M*?t7_;Gy3Byvwh8wllUIx4pgT!ZCepOun4qad zxjij}iU!6tMtYa*XMFUvpR;@G5J&t0WC=Aerd>|*n{Z$NkqN(>Pwfex#LHjN3g$6g zTgMyvbwM-7*2DC z&Q78bN)2Srt$LgBJnJdwq#x)yhPCI*boh%DXGG^9&b<^yVdy;p&jdT~zCYw^!WN3( z5eOgJ4NsM3XQ~Rxf6Q7#T2mfKes*$Sb!1H)3cPPW9*;vm(L`4@f0c|wo|?&B|F}7l zI|_mf4zTUddWY6QrynlPK2pg9t+T@w{$c$ zsUH$P4Hp3b2jAxd2-Aw#mqQgF#8A}!lRh+Mr1PP&<}L+0uxROt6i^(^Ex>4ax!;!Q zP`3AkzYO>ylFO#F20Mh)2a0zsLkhZ38i@I>V{WQ7p_V!$You@!snCl8VpU9J{8X{u zO#EGCE-C^(PWzeDM;RX?nXJ~{aI>dPKZ>hd=Tcz0%TO>g2qzswDrTPJbLA?XkA|A` zAkEv@X@$*!+Ric>o+IRqwwknQQ7FmlV0+T2d9_$)>j!LA5D;OV7(`Of&6&L@BjVoN?T5w(M z>)K&@lY`It12M5dMcWGsJTs7mSdgp92QfOJeWdnI_Cs%f3Yg>=Qq7nQvCY4|?t}qm z*A3?2J1+d-HhyVEir3*mEf6Wg=be`yWoQ$k-shVOn0kMWVX*mAVfJG)R1;)R-|y3M zxBXt|;g)tuU$gtPqf{BriOc!=yqx%|v6EVg8&~@4rY4nx-1%vjKr5#Xk5u5sT)#$e zPCPEqfxknO7MsP6f9jzvBk8XU4KueQEOVeo=+!&hk}+Y>zojQ=3ey>Zc^5S*mtOgO zP}8fe(+LHSqYeo?70FyB7BilthO>-f{5=R8UgX2*Q ze9t@qX3Xe^-~3z}*>i=HGoPih2yRn6&ucOpSsvUdCmUy%bRI_O9ITSB2U~{0-OzD} zh1$H2ZMs+%R;krTnk`hb*HH!UnAFO89nYB5L*zuRKa z+t+eoyYuJvQ;b+S2y8Ca*TL=xbvOIVJl&mPYDF8luXh)?y4yolUhXdcYyZ#+R(n-H z#l74z%JHgp^4a8B=@{euuVocOInL!qw(9@EQF8y*4YU6obcsEyo&LM|KX0$-QcQL2Fw`I ze~RC;pIDK?c^@6TUUY0P7dJ1rFNf#-m-*9I*L>4ZQCiXM-!t1#ZEr&Y27qm|gVynY z4@sP%2&2f59_#{5j#<8YAsDxT8l%Wvo@Ime5GWie{ymV-`@{Ic0N%C>=i5*b%fi5} z53c)nAuX`C)XD2f@~uYg>*7}p&!*&SZ|LP-zq#G}usH9E?L4_zUpBGVK|Yt#_9L1I z3fE+}O_$KknfpI|rR9W5Q({qS=VJHVSq_tYtDGm(*9|nq1Duk5&pRh&DfHWXx2(YGKC*PNpjx zQsb0ZTq=#mi8%mCvDKWD$AG*>Z}k9a%Sa=FfVK@$WOl9l`7Itp#rHRP^nUS2p|kp7 ztemg4Z*xD`jUn)t#bhU@xq(jLR*NIJ;3*raqMXR#tm3oj^BXVaAgkrq{&b6b`w$e# zttpilvZTuk^Pss@rW%a4&rn0ZZd3GTI?#FU9s}tcLkF)P(GySf^LXW3XR5w~LZyWw{Oa89X6DSP7Uibs5 zEQ4Z{q_s5$mV`7)3>K$-g~VB5{J1Q=yTXqcHMD7@5yA+%8lcv#wFdS+T z@svh{U{KMIPK_MsFs7c7)syxuI$rT z&^Sdh4jgG0516I_8zm^B?IP6I=WGWe!AsRptXxY1_uyT@ie~Q~kTU3)Q>giD9t1mz zDGc(EObVhBQ@VsMD0iI+=QSb>Z+rs#Uw%C)q9stU%@IG#;7=oPsZDkAtLw%^ob^?4 z$)xUkOC4mH#w|^uhh|S_?54Jz={B<^rQbtU2?f!UwxG7u&T^ZfPK=1f{(U>}KwEQp z@W`>XK93#A3985fn^GdS2#Wnzic{`Cjx~W8T5IWm5;EK-otwI=qyoN|1oSYbn)WWl z6g=urDM0ePrC+#C&ta1iJ9iacOaWfs$bbFQN7$0dBZFS1Vm-USM_a8IW7Px}`L2^Sy-en-x<**}Bdy$jO4mQwvxvN$D zIFuB0cI-Uh#`o*T&F;4DA0-ss(ewTGeLa*1+@JRJ-F)YM9e(!oY;Pj~-p<3On%epj zdv^K$x;vQsJb&Imx_?@YNxr=Ne3VJ5+4f*Fi+M0bdx;u5t@SxK;!5Eza|NTm_*cRE zQ(?(SPAg#V_z`poYitkXfaCF2;2>Qz4|M z*&|IJ0#u#(j9N2lT_MUXaoAM`5vjI>BI$=AnK)IC4Me#~2*FwUPSn%V3*4W01~AZR$XGZgB*HxCA%!uX&2i(+)oO8x9>N(2 zWinvYrZv}Sw6Ep4tthe;TQ(UOAXv zml=wQjP%B=eJ0GUp=BC6c>9U8%Sp@(g!v+G{dHnEu1>6)3SVOn>*loYU{4vrr10UBrBa;(?&W+ zqrNX_yf zKO*fW%a^v0{+t`SuwQIV+F;i9+MfHys7(ANU_35&lyb9Yx4iMC5?eg!DLP1zkDs+- z!%j)1J+Rl5RkYjhbo#n(k8H^Jw4Cyp- zM{oeSy|Fv9SB+TdZ5yhWktxxepuuM8*}bxj&6LEu59nz9Lb9b>w-7=3I*~o>NTZ_H z#yp#)8?k5Zyl26nEwsBmfKA(PTvx%Y{bR|MX7%q>b(`cmZ3<-%Z0*5C3j21hEMhXt zSUPJ8eP=d~F=8?`S-Lk0N;O-9DR?eCx&XrXOiZ?R+~QoV)gI>J($OEv1Qp0tJ(2}I z5=A2EZ*Zdj9UW&s+Roo4B(yF~Ay(4bwbP?$S?$`#bytJGk@uV);q8#!7tIX}mD1Xss*s-EeQk_6W7of20rw`&-^EzW%gUEoKkSPQ;M zEd7lezu&`CecW6jsBIdd|cA5xq0s3Sg+e-{U$1gr6KxTH;6(Aho0f+a-LJt|t27(_#G_%iLv z)%!bzqyNicKGas+IA~c!n4L|f1rX?v=NtL8uA2IGK5;*0klzZya@+ltDtL%_}x=wJw*#H^ms3I<*cy4`yr=(H>}P z5kwZS#isJHrtx4 zX-<=>Dq~W@H$!H}as4VN*TjC~m#pDiU*QrB`BqxeiE7uxge{BflsXIP8q7BR61=i6 z*94YU8Un3?1c%c(B!_xdRkZuAIA7yXm6Z}ZX9&}`hlFW?60#1Bvr>vK$`RETHN}qC zwpq4v^5-S1wr3Re*smOCbVvc;%kJsCtOYH-9CXx+jxOIV@|4`>=QrE~gqK#6+un{9s2-vcDowiFC6|keew%p_Nrv z8r0pT%+^&Sw2TVQJeTg{$xBTMjMoX}DCwZ^$6Hb8X)LJL4Q-`g7>A8(3QC*LRc3MV zF)&2t$4AV;iy;wB?sKXb?WG%(R3_l7##=I47wDQDwMMk8p9*@?`|#px%wYfS+spC( zW^r?o&FpqJN4GPO8YVvbjqp#@ygXoTZM&MrW_?XThaW6AcyHedc%g{epA&K})!QdV zBAkqv*wF*{LD@$^sbwcyu{*VHK3AkA!AY&1!ni>cLvCUknX~>Zg=)e@3XN~RL|Ul?SV97UWk#i zhhwWOTcqK>^*ZYLnxlcX$fPs_v=Mxo6TIQf8oXL zt*`ebBLrYJg9zE*U?R`{2|=ARuC(y0@c11Y#7s*w!O7#C{BVpi<%;{^Z1 z$EwZrX!2Gp0{q2wf~CIL6b@#$YEEDLTHit=OSvYhhhZJd2L2IytcT1F_F#R<#X5@dWAEl+U!nEuK?gA?vfq{h!@ z4okDM5Dd?vd0b}R@3~l8N3uLo&UlQgmC8bM}lD>4C0d;io_P$o zJDw`~_4DZdRGkL+beH5#m9E`q7-2!hk2DxK1FsH1kg0?nKIiEdlKGj2nP>>kn7J*V zW)n2yyHuMYFK*r?x?m4BU%D}t9Z7F{FufV))JFR zbuD1bYApiwW&BY;4HV;S%6Iknn5pU7+^Phk+5`a{3UdFNKXp^UCpV*5)^} z%A4q}l3D$l(P`fCXqo|M*OjSWl0*uWE>o_N*9|`YTF!Ib&F=d=zJ4!zY~%CpboYCR zlQXNM1C+^&39aGTPEoWeBw`MxK&ffn$f9Bm)=&xDVDh-FH0lF)JF2N@YFe_Q=`!2V zI&bo>MdD6WI>GJg)KuPLKE~k*F}}aeFYF(RH9hs9?5#fwUP) zshn?S9@$7-s3x?_khZ9=S{AnLJfT`?s6eddn7dWMHAIw#VrK=qj)>^>5(iYuwLTn= z6?SL^$EkE4>E5i~&Pc{^$ppnv%p*2fUFd*otQrO|KQ|f1F156Xsy3wD<53m$ZQSFD z6&)h|9NrzDiNpcg|cDe~o`E zoN#u@49OO851sL0^_$YuzP$?k8P%@vFsopT`~z-({l)HYbs!`~H>`*)ZRmiXoS#8A zDNmLeJj4CjE0p3J9#ZkNm<@f7o8x(1v6}MhbBl1-(>9@Ts2Re(@bKs4;>W@P$8LSg zTAt_9&7lT&$vig(I>k1;ZnZ{pwX4Gj>)6raZyrreYKq17tXEK?jYWaVVJnM9Sp-+} zW_haCrc31|ug$Da*^CD>TDi}$9ORc+;+^sK1)jt!Sxb`C2z|YgMbK@}$53w?1XSC``pj)s>lgdaNuRN#DV%Wdb8$P`{u7`7{8x|3>bnfiGAeIS5FPDT1<`U z^&>m;fC9pP_?dCKVKgr+MBhzv^GcXN@hCpAyhTuD>tB|;2wHkjjy18Mc&P7PER&}y zK3In(J>Xz`0lzlUAm8wbLZ>Q-jT_6V^ycN|Mw)^lU7WFX2WfjaN8F* zS&R;&lGcD^Z_tfu`j4;$V@piSGIA1G$7FFgmp|ak6U|*rJ6bmO6OA8l8dfn9m!o=l ze0+&n7aymzxy!ErRXLF5KmZrFHv{D^;MH(x7vM%GH}$jILrcr4GtkyneH7Mm3{H~g za;BS_IEnHcHNN`oybQ?l@87)JElM-__Iq-4Ub_4jG*thLF_(||@4~Pbhfb*KL9M{} zKat1QjPibnfAIh@aW<}O*tCTJZkIA0&jV?+xQEnC81@7g&{^!uPM+B6<_ z0Ju{JX-?`s|J>+H^&5THXeaQ)z?&I`DIxYM8myxw!Cf0-3b=|14y zY3Tf@#eua>0u(p^Ls1I{1mJ literal 0 HcmV?d00001 diff --git a/charts/v1.9.0/azuredisk-csi-driver/Chart.yaml b/charts/v1.9.0/azuredisk-csi-driver/Chart.yaml new file mode 100644 index 0000000000..f1819135b4 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: latest +description: Azure disk Container Storage Interface (CSI) Storage Plugin +name: azuredisk-csi-driver +version: v1.9.0 diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/NOTES.txt b/charts/v1.9.0/azuredisk-csi-driver/templates/NOTES.txt new file mode 100644 index 0000000000..6a9695dad2 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/NOTES.txt @@ -0,0 +1,5 @@ +The Azure Disk CSI Driver is getting deployed to your cluster. + +To check Azure Disk CSI Driver pods status, please run: + + kubectl --namespace={{ .Release.Namespace }} get pods --selector="release={{ .Release.Name }}" --watch diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/_helpers.tpl b/charts/v1.9.0/azuredisk-csi-driver/templates/_helpers.tpl new file mode 100644 index 0000000000..51fcc8a092 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* Expand the name of the chart.*/}} +{{- define "azuredisk.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* labels for helm resources */}} +{{- define "azuredisk.labels" -}} +labels: + app.kubernetes.io/instance: "{{ .Release.Name }}" + app.kubernetes.io/managed-by: "{{ .Release.Service }}" + app.kubernetes.io/name: "{{ template "azuredisk.name" . }}" + app.kubernetes.io/version: "{{ .Chart.AppVersion }}" + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}" +{{- end -}} + +{{/* pull secrets for containers */}} +{{- define "azuredisk.pullSecrets" -}} +{{- if .Values.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot-ga.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot-ga.yaml new file mode 100644 index 0000000000..efbb46b1da --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot-ga.yaml @@ -0,0 +1,655 @@ +{{- if .Values.snapshot.enabled -}} +{{- if eq .Values.snapshot.apiVersion "ga" }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +{{- end -}} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot.yaml new file mode 100644 index 0000000000..cb012847c7 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/crd-csi-snapshot.yaml @@ -0,0 +1,504 @@ +{{- if .Values.snapshot.enabled -}} +{{- if eq .Values.snapshot.apiVersion "beta" }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if a snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Name of the source PVC from where a dynamically taken snapshot + will be created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: Name of the VolumeSnapshotContent which represents a pre-provisioned + snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the complete size of the snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: The name of the VolumeSnapshotContent to which this VolumeSnapshot + is bound. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot is taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. + This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is + no default snapshot class, dynamic snapshot creation will fail. + Empty string is not allowed for this field. TODO(xiangqian): a webhook + validation on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. + NOTE: status can be modified by sources other than system controllers, + and must not be depended upon for accuracy. Controllers should only + use information from the VolumeSnapshotContent object after verifying + that the binding is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: Specified boundVolumeSnapshotContentName alone + does not mean binding is valid. Controllers MUST always verify + bidirectional binding between VolumeSnapshot and VolumeSnapshotContent + to avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in with the "creation_time" + value returned from CSI "CreateSnapshotRequest" gRPC call. For a + pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. If not specified, it indicates that the creation time + of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, + this field will be filled with the "ready_to_use" value returned + from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". If not specified, it + means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if a snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. + For pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system. This + field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in with the "creation_time" + value returned from CSI "CreateSnapshotRequest" gRPC call. For a + pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. If not specified, it indicates the creation time is + unknown. The format of this field is a Unix nanoseconds time encoded + as an int64. On Unix, the command `date +%s%N` returns the current + time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, + this field will be filled with the "ready_to_use" value returned + from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". If not specified, it + means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +{{- end -}} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml new file mode 100644 index 0000000000..f44b5e9480 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-controller.yaml @@ -0,0 +1,226 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.controller.name }} + namespace: {{ .Release.Namespace }} +{{ include "azuredisk.labels" . | indent 2 }} +spec: + replicas: {{ .Values.controller.replicas }} + selector: + matchLabels: + app: {{ .Values.controller.name }} + template: + metadata: +{{ include "azuredisk.labels" . | indent 6 }} + app: {{ .Values.controller.name }} +{{- with .Values.controller.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.controller.hostNetwork }} + serviceAccountName: {{ .Values.serviceAccount.controller }} + nodeSelector: + kubernetes.io/os: linux + {{- if .Values.controller.runOnMaster}} + kubernetes.io/role: master + {{- end}} + priorityClassName: system-cluster-critical +{{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} + {{- include "azuredisk.pullSecrets" . | indent 6 }} + containers: + - name: csi-provisioner +{{- if hasPrefix "/" .Values.image.csiProvisioner.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- else }} + image: "{{ .Values.image.csiProvisioner.repository }}:{{ .Values.image.csiProvisioner.tag }}" +{{- end }} + args: + - "--feature-gates=Topology=true" + - "--csi-address=$(ADDRESS)" + - "--v=2" + - "--timeout=15s" + - "--leader-election" + - "--worker-threads={{ .Values.controller.provisionerWorkerThreads }}" + - "--extra-create-metadata=true" + - "--strict-topology=true" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: {{- toYaml .Values.controller.resources.csiProvisioner | nindent 12 }} + - name: csi-attacher +{{- if hasPrefix "/" .Values.image.csiAttacher.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiAttacher.repository }}:{{ .Values.image.csiAttacher.tag }}" +{{- else }} + image: "{{ .Values.image.csiAttacher.repository }}:{{ .Values.image.csiAttacher.tag }}" +{{- end }} + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=600s" + - "-leader-election" + - "-worker-threads={{ .Values.controller.attacherWorkerThreads }}" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: {{- toYaml .Values.controller.resources.csiAttacher | nindent 12 }} + - name: csi-snapshotter +{{- if eq .Values.snapshot.apiVersion "beta" }} + {{- if hasPrefix "/" .Values.snapshot.image.csiSnapshotter.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.snapshot.image.csiSnapshotter.repository }}:{{ .Values.snapshot.image.csiSnapshotter.tag }}" + {{- else }} + image: "{{ .Values.snapshot.image.csiSnapshotter.repository }}:{{ .Values.snapshot.image.csiSnapshotter.tag }}" + {{- end }} +{{- else }} + {{- if hasPrefix "/" .Values.snapshot.image.csiSnapshotter.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.snapshot.image.csiSnapshotter.repository }}:{{ .Values.snapshot.image.csiSnapshotter.gaTag }}" + {{- else }} + image: "{{ .Values.snapshot.image.csiSnapshotter.repository }}:{{ .Values.snapshot.image.csiSnapshotter.gaTag }}" + {{- end }} +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "-v=2" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiSnapshotter | nindent 12 }} + - name: csi-resizer +{{- if hasPrefix "/" .Values.image.csiResizer.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- else }} + image: "{{ .Values.image.csiResizer.repository }}:{{ .Values.image.csiResizer.tag }}" +{{- end }} + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - '-handle-volume-inuse-error=false' + - "-timeout=60s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.csiResizer | nindent 12 }} + - name: liveness-probe +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.controller.livenessProbe.healthPort }} + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: {{- toYaml .Values.controller.resources.livenessProbe | nindent 12 }} + - name: azuredisk +{{- if hasPrefix "/" .Values.image.azuredisk.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}" +{{- else }} + image: "{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}" +{{- end }} + args: + - "--v={{ .Values.controller.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:{{ .Values.controller.metricsPort }}" + - "--disable-avset-nodes={{ .Values.controller.disableAvailabilitySetNodes }}" + - "--drivername={{ .Values.driver.name }}" + - "--cloud-config-secret-name={{ .Values.controller.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.controller.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.controller.allowEmptyCloudConfig }}" + ports: + - containerPort: {{ .Values.controller.livenessProbe.healthPort }} + name: healthz + protocol: TCP + - containerPort: {{ .Values.controller.metricsPort }} + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.azuredisk.pullPolicy }} + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.controller.resources.azuredisk | nindent 12 }} + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-driver.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-driver.yaml new file mode 100644 index 0000000000..35b0118249 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-driver.yaml @@ -0,0 +1,17 @@ +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: {{ .Values.driver.name }} + annotations: + csiDriver: "{{ .Values.image.azuredisk.tag }}" +{{- if eq .Values.snapshot.apiVersion "beta" }} + snapshot: "{{ .Values.snapshot.image.csiSnapshotter.tag }}" +{{- else }} + snapshot: "{{ .Values.snapshot.image.csiSnapshotter.gaTag }}" +{{- end }} +spec: + attachRequired: true + podInfoOnMount: false + {{- if .Values.feature.enableFSGroupPolicy}} + fsGroupPolicy: File + {{- end}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml new file mode 100644 index 0000000000..d33dc6794b --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node-windows.yaml @@ -0,0 +1,200 @@ +{{- if .Values.windows.enabled}} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .Values.windows.dsName }} + namespace: {{ .Release.Namespace }} +{{ include "azuredisk.labels" . | indent 2 }} +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: {{ .Values.node.maxUnavailable }} + type: RollingUpdate + selector: + matchLabels: + app: {{ .Values.windows.dsName }} + template: + metadata: +{{ include "azuredisk.labels" . | indent 6 }} + app: {{ .Values.windows.dsName }} +{{- with .Values.windows.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ .Values.serviceAccount.node }} +{{- with .Values.windows.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + nodeSelector: + kubernetes.io/os: windows + affinity: + nodeAffinity: +{{ toYaml .Values.windows.nodeAffinity | indent 10 }} + priorityClassName: system-node-critical + {{- include "azuredisk.pullSecrets" . | indent 6 }} + containers: + - name: liveness-probe + volumeMounts: + - mountPath: C:\csi + name: plugin-dir +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - "--csi-address=$(CSI_ENDPOINT)" + - "--probe-timeout=3s" + - "--health-port={{ .Values.node.livenessProbe.healthPort }}" + - "--v=2" + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} + resources: {{- toYaml .Values.windows.resources.livenessProbe | nindent 12 }} + - name: node-driver-registrar +{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- else }} + image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- end }} + args: + - "--v=2" + - "--csi-address=$(CSI_ENDPOINT)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 60 + timeoutSeconds: 30 + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\{{ .Values.driver.name }}\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: {{ .Values.image.nodeDriverRegistrar.pullPolicy }} + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + resources: {{- toYaml .Values.windows.resources.nodeDriverRegistrar | nindent 12 }} + - name: azuredisk +{{- if hasPrefix "/" .Values.image.azuredisk.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}" +{{- else }} + image: "{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}" +{{- end }} + args: + - "--v={{ .Values.node.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}" + - "--drivername={{ .Values.driver.name }}" + - "--volume-attach-limit={{ .Values.driver.volumeAttachLimit }}" + - "--cloud-config-secret-name={{ .Values.node.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.node.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.node.allowEmptyCloudConfig }}" + ports: + - containerPort: {{ .Values.node.livenessProbe.healthPort }} + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path-windows + optional: true + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: C:\k\azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.azuredisk.pullPolicy }} + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: azure-config + mountPath: C:\k + - name: csi-proxy-fs-pipe-v1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-disk-pipe-v1 + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe-v1 + mountPath: \\.\pipe\csi-proxy-volume-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-disk-pipe-v1beta2 + mountPath: \\.\pipe\csi-proxy-disk-v1beta2 + - name: csi-proxy-volume-pipe-v1beta2 + mountPath: \\.\pipe\csi-proxy-volume-v1beta2 + resources: {{- toYaml .Values.windows.resources.azuredisk | nindent 12 }} + volumes: + - name: csi-proxy-fs-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-disk-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-disk-pipe-v1beta2 + hostPath: + path: \\.\pipe\csi-proxy-disk-v1beta2 + - name: csi-proxy-volume-pipe-v1beta2 + hostPath: + path: \\.\pipe\csi-proxy-volume-v1beta2 + - name: registration-dir + hostPath: + path: {{ .Values.windows.kubelet }}\plugins_registry\ + type: Directory + - name: kubelet-dir + hostPath: + path: {{ .Values.windows.kubelet }}\ + type: Directory + - name: plugin-dir + hostPath: + path: {{ .Values.windows.kubelet }}\plugins\{{ .Values.driver.name }}\ + type: DirectoryOrCreate + - name: azure-config + hostPath: + path: C:\k + type: DirectoryOrCreate +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml new file mode 100644 index 0000000000..b0e695268a --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-azuredisk-node.yaml @@ -0,0 +1,209 @@ +{{- if .Values.linux.enabled}} +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: {{ .Values.linux.dsName }} + namespace: {{ .Release.Namespace }} +{{ include "azuredisk.labels" . | indent 2 }} +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: {{ .Values.node.maxUnavailable }} + type: RollingUpdate + selector: + matchLabels: + app: {{ .Values.linux.dsName }} + template: + metadata: +{{ include "azuredisk.labels" . | indent 6 }} + app: {{ .Values.linux.dsName }} +{{- with .Values.linux.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + hostNetwork: {{ .Values.linux.hostNetwork }} + dnsPolicy: Default + serviceAccountName: {{ .Values.serviceAccount.node }} + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: +{{ toYaml .Values.linux.nodeAffinity | indent 10 }} + priorityClassName: system-node-critical +{{- with .Values.linux.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} + {{- include "azuredisk.pullSecrets" . | indent 6 }} + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir +{{- if hasPrefix "/" .Values.image.livenessProbe.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- else }} + image: "{{ .Values.image.livenessProbe.repository }}:{{ .Values.image.livenessProbe.tag }}" +{{- end }} + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port={{ .Values.node.livenessProbe.healthPort }} + - --v=2 + resources: {{- toYaml .Values.linux.resources.livenessProbe | nindent 12 }} + - name: node-driver-registrar +{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- else }} + image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}" +{{- end }} + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }}/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: {{- toYaml .Values.linux.resources.nodeDriverRegistrar | nindent 12 }} + - name: azuredisk +{{- if hasPrefix "/" .Values.image.azuredisk.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}" +{{- else }} + image: "{{ .Values.image.azuredisk.repository }}:{{ .Values.image.azuredisk.tag }}" +{{- end }} + args: + - "--v={{ .Values.node.logLevel }}" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:{{ .Values.node.metricsPort }}" + - "--enable-perf-optimization={{ .Values.linux.enablePerfOptimization }}" + - "--drivername={{ .Values.driver.name }}" + - "--volume-attach-limit={{ .Values.driver.volumeAttachLimit }}" + - "--cloud-config-secret-name={{ .Values.node.cloudConfigSecretName }}" + - "--cloud-config-secret-namespace={{ .Values.node.cloudConfigSecretNamespace }}" + - "--custom-user-agent={{ .Values.driver.customUserAgent }}" + - "--user-agent-suffix={{ .Values.driver.userAgentSuffix }}" + - "--allow-empty-cloud-config={{ .Values.node.allowEmptyCloudConfig }}" + ports: + - containerPort: {{ .Values.node.livenessProbe.healthPort }} + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: AZURE_ENVIRONMENT_FILEPATH + value: /etc/kubernetes/azurestackcloud.json + {{- end }} + imagePullPolicy: {{ .Values.image.azuredisk.pullPolicy }} + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: {{ .Values.linux.kubelet }}/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/scsi_host/ + name: scsi-host-dir + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + mountPath: /etc/ssl/certs + readOnly: true + - name: ssl-pki + mountPath: /etc/pki/ca-trust/extracted + readOnly: true + {{- end }} + resources: {{- toYaml .Values.linux.resources.azuredisk | nindent 12 }} + volumes: + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: {{ .Values.linux.kubelet }}/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: Directory + name: sys-devices-dir + - hostPath: + path: /sys/class/scsi_host/ + type: Directory + name: scsi-host-dir + {{- if eq .Values.cloud "AzureStackCloud" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + {{- end }} + {{- if eq .Values.linux.distro "fedora" }} + - name: ssl + hostPath: + path: /etc/ssl/certs + - name: ssl-pki + hostPath: + path: /etc/pki/ca-trust/extracted + {{- end }} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/csi-snapshot-controller.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-snapshot-controller.yaml new file mode 100644 index 0000000000..ae6f5a49b6 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/csi-snapshot-controller.yaml @@ -0,0 +1,54 @@ +{{- if .Values.snapshot.enabled -}} +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ .Values.snapshot.snapshotController.name }} + namespace: {{ .Release.Namespace }} +{{ include "azuredisk.labels" . | indent 2 }} +spec: + replicas: {{ .Values.snapshot.snapshotController.replicas }} + selector: + matchLabels: + app: {{ .Values.snapshot.snapshotController.name }} + template: + metadata: +{{ include "azuredisk.labels" . | indent 6 }} + app: {{ .Values.snapshot.snapshotController.name }} +{{- with .Values.snapshot.snapshotController.podLabels }} +{{ toYaml . | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ .Values.serviceAccount.snapshotController }} + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical +{{- with .Values.controller.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.controller.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} + {{- include "azuredisk.pullSecrets" . | indent 6 }} + containers: + - name: {{ .Values.snapshot.snapshotController.name }} +{{- if eq .Values.snapshot.apiVersion "beta" }} + {{- if hasPrefix "/" .Values.snapshot.image.csiSnapshotController.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.snapshot.image.csiSnapshotController.repository }}:{{ .Values.snapshot.image.csiSnapshotController.tag }}" + {{- else }} + image: "{{ .Values.snapshot.image.csiSnapshotController.repository }}:{{ .Values.snapshot.image.csiSnapshotController.tag }}" + {{- end }} +{{- else }} + {{- if hasPrefix "/" .Values.snapshot.image.csiSnapshotController.repository }} + image: "{{ .Values.image.baseRepo }}{{ .Values.snapshot.image.csiSnapshotController.repository }}:{{ .Values.snapshot.image.csiSnapshotController.gaTag }}" + {{- else }} + image: "{{ .Values.snapshot.image.csiSnapshotController.repository }}:{{ .Values.snapshot.image.csiSnapshotController.gaTag }}" + {{- end }} +{{- end }} + args: + - "--v=2" + - "--leader-election=true" + resources: {{- toYaml .Values.snapshot.snapshotController.resources | nindent 12 }} + imagePullPolicy: {{ .Values.snapshot.image.csiSnapshotController.pullPolicy }} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml new file mode 100644 index 0000000000..71afa47702 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-controller.yaml @@ -0,0 +1,202 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-provisioner-role +{{ include "azuredisk.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] + +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-provisioner-binding +{{ include "azuredisk.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-attacher-role +{{ include "azuredisk.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-attacher-binding +{{ include "azuredisk.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-snapshotter-role +{{ include "azuredisk.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-snapshotter-binding +{{ include "azuredisk.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-external-resizer-role +{{ include "azuredisk.labels" . | indent 2 }} +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ .Values.rbac.name }}-csi-resizer-role +{{ include "azuredisk.labels" . | indent 2 }} +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ .Values.rbac.name }}-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-controller-secret-binding +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-controller-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml new file mode 100644 index 0000000000..b4a30373ce --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-azuredisk-node.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create -}} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-role +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-{{ .Values.rbac.name }}-node-secret-binding +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-{{ .Values.rbac.name }}-node-secret-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-snapshot-controller.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-snapshot-controller.yaml new file mode 100644 index 0000000000..bbca06976f --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/rbac-csi-snapshot-controller.yaml @@ -0,0 +1,69 @@ +{{- if and .Values.snapshot.enabled .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshotController }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-role +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-binding +subjects: + - kind: ServiceAccount + name: {{ .Values.serviceAccount.snapshotController }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-leaderelection-role + apiGroup: rbac.authorization.k8s.io +{{ end }} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-controller.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-controller.yaml new file mode 100644 index 0000000000..08bd06cda5 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-controller.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.controller }} + namespace: {{ .Release.Namespace }} +{{ include "azuredisk.labels" . | indent 2 }} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-node.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-node.yaml new file mode 100644 index 0000000000..941f3eb92c --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-azuredisk-node.yaml @@ -0,0 +1,8 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.node }} + namespace: {{ .Release.Namespace }} +{{ include "azuredisk.labels" . | indent 2 }} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml b/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml new file mode 100644 index 0000000000..7cdaad0b4f --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/templates/serviceaccount-csi-snapshot-controller.yaml @@ -0,0 +1,7 @@ +{{- if and .Values.snapshot.enabled .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.snapshotController }} + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/charts/v1.9.0/azuredisk-csi-driver/values.yaml b/charts/v1.9.0/azuredisk-csi-driver/values.yaml new file mode 100644 index 0000000000..4c0a096b37 --- /dev/null +++ b/charts/v1.9.0/azuredisk-csi-driver/values.yaml @@ -0,0 +1,240 @@ +image: + baseRepo: mcr.microsoft.com + azuredisk: + repository: /k8s/csi/azuredisk-csi + tag: v1.9.0 + pullPolicy: IfNotPresent + csiProvisioner: + repository: /oss/kubernetes-csi/csi-provisioner + tag: v2.2.2 + pullPolicy: IfNotPresent + csiAttacher: + repository: /oss/kubernetes-csi/csi-attacher + tag: v3.3.0 + pullPolicy: IfNotPresent + csiResizer: + repository: /oss/kubernetes-csi/csi-resizer + tag: v1.3.0 + pullPolicy: IfNotPresent + livenessProbe: + repository: /oss/kubernetes-csi/livenessprobe + tag: v2.5.0 + pullPolicy: IfNotPresent + nodeDriverRegistrar: + repository: /oss/kubernetes-csi/csi-node-driver-registrar + tag: v2.4.0 + pullPolicy: IfNotPresent + +serviceAccount: + create: true # When true, service accounts will be created for you. Set to false if you want to use your own. + controller: csi-azuredisk-controller-sa # Name of Service Account to be created or used + node: csi-azuredisk-node-sa # Name of Service Account to be created or used + snapshotController: csi-snapshot-controller-sa # Name of Service Account to be created or used + +rbac: + create: true + name: azuredisk + +controller: + name: csi-azuredisk-controller + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: false + replicas: 2 + metricsPort: 29604 + livenessProbe: + healthPort: 29602 + runOnMaster: false + disableAvailabilitySetNodes: true + provisionerWorkerThreads: 40 + attacherWorkerThreads: 500 + logLevel: 5 + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + hostNetwork: true # this setting could be disabled if controller does not depend on MSI setting + podLabels: {} + resources: + csiProvisioner: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + csiAttacher: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + csiResizer: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + csiSnapshotter: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + livenessProbe: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + azuredisk: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + +node: + cloudConfigSecretName: azure-cloud-provider + cloudConfigSecretNamespace: kube-system + allowEmptyCloudConfig: true + maxUnavailable: 1 + metricsPort: 29605 + logLevel: 5 + livenessProbe: + healthPort: 29603 + +snapshot: + enabled: false + name: csi-snapshot-controller + apiVersion: beta # available values: beta, ga + image: + csiSnapshotter: + repository: /oss/kubernetes-csi/csi-snapshotter + tag: v3.0.3 + gaTag: v4.2.1 + pullPolicy: IfNotPresent + csiSnapshotController: + repository: /oss/kubernetes-csi/snapshot-controller + tag: v3.0.3 + gaTag: v4.2.1 + pullPolicy: IfNotPresent + snapshotController: + name: csi-snapshot-controller + replicas: 2 + podLabels: {} + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + +feature: + enableFSGroupPolicy: false + +driver: + name: disk.csi.azure.com + # maximum number of attachable volumes per node, + # maximum number is defined according to node instance type by default(-1) + volumeAttachLimit: -1 + customUserAgent: "" + userAgentSuffix: "OSS-helm" + +linux: + enabled: true + dsName: csi-azuredisk-node # daemonset name + kubelet: /var/lib/kubelet + distro: debian # available values: debian, fedora + enablePerfOptimization: true + tolerations: + - operator: "Exists" + hostNetwork: true # this setting could be disabled if perfProfile is `none` + podLabels: {} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + resources: + livenessProbe: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + nodeDriverRegistrar: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + azuredisk: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + +windows: + enabled: true + dsName: csi-azuredisk-node-win # daemonset name + kubelet: 'C:\var\lib\kubelet' + tolerations: + - key: "node.kubernetes.io/os" + operator: "Exists" + effect: "NoSchedule" + podLabels: {} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + resources: + livenessProbe: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + nodeDriverRegistrar: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + azuredisk: + limits: + cpu: 1 + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + +cloud: AzurePublicCloud + +## Reference to one or more secrets to be used when pulling images +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ +## +imagePullSecrets: [] +# - name: "image-pull-secret" diff --git a/deploy/csi-azuredisk-controller.yaml b/deploy/csi-azuredisk-controller.yaml index e411ebe105..aed775bbcd 100644 --- a/deploy/csi-azuredisk-controller.yaml +++ b/deploy/csi-azuredisk-controller.yaml @@ -130,7 +130,7 @@ spec: cpu: 10m memory: 20Mi - name: azuredisk - image: mcr.microsoft.com/k8s/csi/azuredisk-csi:latest + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/csi-azuredisk-node-windows.yaml b/deploy/csi-azuredisk-node-windows.yaml index cfc771ed8d..43f451a2ca 100644 --- a/deploy/csi-azuredisk-node-windows.yaml +++ b/deploy/csi-azuredisk-node-windows.yaml @@ -93,7 +93,7 @@ spec: cpu: 10m memory: 20Mi - name: azuredisk - image: mcr.microsoft.com/k8s/csi/azuredisk-csi:latest + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/csi-azuredisk-node.yaml b/deploy/csi-azuredisk-node.yaml index eefd2d8cb5..0fbe737a91 100644 --- a/deploy/csi-azuredisk-node.yaml +++ b/deploy/csi-azuredisk-node.yaml @@ -84,7 +84,7 @@ spec: cpu: 10m memory: 20Mi - name: azuredisk - image: mcr.microsoft.com/k8s/csi/azuredisk-csi:latest + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/v1.9.0/crd-csi-snapshot-ga.yaml b/deploy/v1.9.0/crd-csi-snapshot-ga.yaml new file mode 100644 index 0000000000..d6553e0c08 --- /dev/null +++ b/deploy/v1.9.0/crd-csi-snapshot-ga.yaml @@ -0,0 +1,652 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshot is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshot" + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.' + properties: + source: + description: source specifies where a snapshot will be created from. This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotClass is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotClass" + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: false + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + # This indicates the v1beta1 version of the custom resource is deprecated. + # API requests to this version receive a warning in the server response. + deprecated: true + # This overrides the default warning returned to clients making v1beta1 API requests. + deprecationWarning: "snapshot.storage.k8s.io/v1beta1 VolumeSnapshotContent is deprecated; use snapshot.storage.k8s.io/v1 VolumeSnapshotContent" + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/v1.9.0/crd-csi-snapshot.yaml b/deploy/v1.9.0/crd-csi-snapshot.yaml new file mode 100644 index 0000000000..e812e99fcf --- /dev/null +++ b/deploy/v1.9.0/crd-csi-snapshot.yaml @@ -0,0 +1,501 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if a snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Name of the source PVC from where a dynamically taken snapshot + will be created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: Name of the VolumeSnapshotContent which represents a pre-provisioned + snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the complete size of the snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: The name of the VolumeSnapshotContent to which this VolumeSnapshot + is bound. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot is taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object in the same namespace as the VolumeSnapshot + object where the snapshot should be dynamically taken from. + This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object. This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: 'volumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. If not specified, the default snapshot + class will be used if one exists. If not specified, and there is + no default snapshot class, dynamic snapshot creation will fail. + Empty string is not allowed for this field. TODO(xiangqian): a webhook + validation on empty string. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshot-classes' + type: string + required: + - source + type: object + status: + description: 'status represents the current information of a snapshot. + NOTE: status can be modified by sources other than system controllers, + and must not be depended upon for accuracy. Controllers should only + use information from the VolumeSnapshotContent object after verifying + that the binding is accurate and complete.' + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName represents the name of + the VolumeSnapshotContent object to which the VolumeSnapshot object + is bound. If not specified, it indicates that the VolumeSnapshot + object has not been successfully bound to a VolumeSnapshotContent + object yet. NOTE: Specified boundVolumeSnapshotContentName alone + does not mean binding is valid. Controllers MUST always verify + bidirectional binding between VolumeSnapshot and VolumeSnapshotContent + to avoid possible security issues.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in with the "creation_time" + value returned from CSI "CreateSnapshotRequest" gRPC call. For a + pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. If not specified, it indicates that the creation time + of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, + this field will be filled with the "ready_to_use" value returned + from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". If not specified, it + means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.3.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/139" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if a snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. In dynamic snapshot creation + case, this field will be filled in with the "DeletionPolicy" field + defined in the VolumeSnapshotClass the VolumeSnapshot refers to. + For pre-existing snapshots, users MUST specify this field when creating + the VolumeSnapshotContent object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. + type: string + source: + description: source specifies from where a snapshot will be created. + This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system. This + field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass to which this snapshot + belongs. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in with the "creation_time" + value returned from CSI "CreateSnapshotRequest" gRPC call. For a + pre-existing snapshot, this field will be filled with the "creation_time" + value returned from the CSI "ListSnapshots" gRPC call if the driver + supports it. If not specified, it indicates the creation time is + unknown. The format of this field is a Unix nanoseconds time encoded + as an int64. On Unix, the command `date +%s%N` returns the current + time in nanoseconds since 1970-01-01 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the latest observed error during snapshot creation, + if any. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in with the "ready_to_use" value returned from CSI + "CreateSnapshotRequest" gRPC call. For a pre-existing snapshot, + this field will be filled with the "ready_to_use" value returned + from the CSI "ListSnapshots" gRPC call if the driver supports it, + otherwise, this field will be set to "True". If not specified, it + means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in with the "size_bytes" value returned from CSI "CreateSnapshotRequest" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "size_bytes" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. When restoring a volume from + this snapshot, the size of the volume MUST NOT be smaller than the + restoreSize if it is specified, otherwise the restoration will fail. + If not specified, it indicates that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/v1.9.0/csi-azuredisk-controller.yaml b/deploy/v1.9.0/csi-azuredisk-controller.yaml new file mode 100644 index 0000000000..aed775bbcd --- /dev/null +++ b/deploy/v1.9.0/csi-azuredisk-controller.yaml @@ -0,0 +1,184 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: csi-azuredisk-controller + template: + metadata: + labels: + app: csi-azuredisk-controller + spec: + hostNetwork: true + serviceAccountName: csi-azuredisk-controller-sa + nodeSelector: + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: mcr.microsoft.com/oss/kubernetes-csi/csi-provisioner:v2.2.2 + args: + - "--feature-gates=Topology=true" + - "--csi-address=$(ADDRESS)" + - "--v=2" + - "--timeout=15s" + - "--leader-election" + - "--worker-threads=40" + - "--extra-create-metadata=true" + - "--strict-topology=true" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-attacher + image: mcr.microsoft.com/oss/kubernetes-csi/csi-attacher:v3.3.0 + args: + - "-v=2" + - "-csi-address=$(ADDRESS)" + - "-timeout=600s" + - "-leader-election" + - "-worker-threads=500" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: mcr.microsoft.com/oss/kubernetes-csi/csi-snapshotter:v3.0.3 + args: + - "-csi-address=$(ADDRESS)" + - "-leader-election" + - "--v=2" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-resizer + image: mcr.microsoft.com/oss/kubernetes-csi/csi-resizer:v1.3.0 + args: + - "-csi-address=$(ADDRESS)" + - "-v=2" + - "-leader-election" + - '-handle-volume-inuse-error=false' + - "-timeout=60s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.5.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29602 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--metrics-address=0.0.0.0:29604" + - "--user-agent-suffix=OSS-kubectl" + - "--disable-avset-nodes=true" + - "--allow-empty-cloud-config=false" + ports: + - containerPort: 29602 + name: healthz + protocol: TCP + - containerPort: 29604 + name: metrics + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + resources: + limits: + cpu: 1 + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + emptyDir: {} + - name: azure-cred + hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate diff --git a/deploy/v1.9.0/csi-azuredisk-driver.yaml b/deploy/v1.9.0/csi-azuredisk-driver.yaml new file mode 100644 index 0000000000..a6fcb90c50 --- /dev/null +++ b/deploy/v1.9.0/csi-azuredisk-driver.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: disk.csi.azure.com + annotations: + csiDriver: v1.9.0 + snapshot: v3.0.3 +spec: + attachRequired: true + podInfoOnMount: false diff --git a/deploy/v1.9.0/csi-azuredisk-node-windows.yaml b/deploy/v1.9.0/csi-azuredisk-node-windows.yaml new file mode 100644 index 0000000000..43f451a2ca --- /dev/null +++ b/deploy/v1.9.0/csi-azuredisk-node-windows.yaml @@ -0,0 +1,194 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-node-win + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azuredisk-node-win + template: + metadata: + labels: + app: csi-azuredisk-node-win + spec: + serviceAccountName: csi-azuredisk-node-sa + tolerations: + - key: "node.kubernetes.io/os" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + kubernetes.io/os: windows + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + priorityClassName: system-node-critical + containers: + - name: liveness-probe + volumeMounts: + - mountPath: C:\csi + name: plugin-dir + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.5.0 + args: + - "--csi-address=$(CSI_ENDPOINT)" + - "--probe-timeout=3s" + - "--health-port=29603" + - "--v=2" + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + resources: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0 + args: + - "--v=2" + - "--csi-address=$(CSI_ENDPOINT)" + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + livenessProbe: + exec: + command: + - /csi-node-driver-registrar.exe + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 60 + timeoutSeconds: 30 + env: + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: DRIVER_REG_SOCK_PATH + value: C:\\var\\lib\\kubelet\\plugins\\disk.csi.azure.com\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: registration-dir + mountPath: C:\registration + resources: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29605" + - "--allow-empty-cloud-config=true" + ports: + - containerPort: 29603 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path-windows + optional: true + - name: CSI_ENDPOINT + value: unix://C:\\csi\\csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - name: kubelet-dir + mountPath: "C:\\var\\lib\\kubelet" + - name: plugin-dir + mountPath: C:\csi + - name: azure-config + mountPath: C:\k + - name: csi-proxy-fs-pipe-v1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-disk-pipe-v1 + mountPath: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe-v1 + mountPath: \\.\pipe\csi-proxy-volume-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + mountPath: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-disk-pipe-v1beta2 + mountPath: \\.\pipe\csi-proxy-disk-v1beta2 + - name: csi-proxy-volume-pipe-v1beta2 + mountPath: \\.\pipe\csi-proxy-volume-v1beta2 + resources: + limits: + cpu: 1 + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: csi-proxy-fs-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1 + - name: csi-proxy-disk-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-disk-v1 + - name: csi-proxy-volume-pipe-v1 + hostPath: + path: \\.\pipe\csi-proxy-volume-v1 + # these paths are still included for compatibility, they're used + # only if the node has still the beta version of the CSI proxy + - name: csi-proxy-fs-pipe-v1beta1 + hostPath: + path: \\.\pipe\csi-proxy-filesystem-v1beta1 + - name: csi-proxy-disk-pipe-v1beta2 + hostPath: + path: \\.\pipe\csi-proxy-disk-v1beta2 + - name: csi-proxy-volume-pipe-v1beta2 + hostPath: + path: \\.\pipe\csi-proxy-volume-v1beta2 + - name: registration-dir + hostPath: + path: C:\var\lib\kubelet\plugins_registry\ + type: Directory + - name: kubelet-dir + hostPath: + path: C:\var\lib\kubelet\ + type: Directory + - name: plugin-dir + hostPath: + path: C:\var\lib\kubelet\plugins\disk.csi.azure.com\ + type: DirectoryOrCreate + - name: azure-config + hostPath: + path: C:\k + type: DirectoryOrCreate diff --git a/deploy/v1.9.0/csi-azuredisk-node.yaml b/deploy/v1.9.0/csi-azuredisk-node.yaml new file mode 100644 index 0000000000..0fbe737a91 --- /dev/null +++ b/deploy/v1.9.0/csi-azuredisk-node.yaml @@ -0,0 +1,174 @@ +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-azuredisk-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-azuredisk-node + template: + metadata: + labels: + app: csi-azuredisk-node + spec: + hostNetwork: true + dnsPolicy: Default + serviceAccountName: csi-azuredisk-node-sa + nodeSelector: + kubernetes.io/os: linux + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: type + operator: NotIn + values: + - virtual-kubelet + priorityClassName: system-node-critical + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + image: mcr.microsoft.com/oss/kubernetes-csi/livenessprobe:v2.5.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29603 + - --v=2 + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: mcr.microsoft.com/oss/kubernetes-csi/csi-node-driver-registrar:v2.4.0 + args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: azuredisk + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 + imagePullPolicy: IfNotPresent + args: + - "--v=5" + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(KUBE_NODE_NAME)" + - "--metrics-address=0.0.0.0:29605" + - "--enable-perf-optimization=true" + - "--allow-empty-cloud-config=true" + ports: + - containerPort: 29603 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + env: + - name: AZURE_CREDENTIAL_FILE + valueFrom: + configMapKeyRef: + name: azure-cred-file + key: path + optional: true + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + securityContext: + privileged: true + volumeMounts: + - mountPath: /csi + name: socket-dir + - mountPath: /var/lib/kubelet/ + mountPropagation: Bidirectional + name: mountpoint-dir + - mountPath: /etc/kubernetes/ + name: azure-cred + - mountPath: /dev + name: device-dir + - mountPath: /sys/bus/scsi/devices + name: sys-devices-dir + - mountPath: /sys/class/scsi_host/ + name: scsi-host-dir + resources: + limits: + cpu: 1 + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - hostPath: + path: /var/lib/kubelet/plugins/disk.csi.azure.com + type: DirectoryOrCreate + name: socket-dir + - hostPath: + path: /var/lib/kubelet/ + type: DirectoryOrCreate + name: mountpoint-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + name: registration-dir + - hostPath: + path: /etc/kubernetes/ + type: DirectoryOrCreate + name: azure-cred + - hostPath: + path: /dev + type: Directory + name: device-dir + - hostPath: + path: /sys/bus/scsi/devices + type: Directory + name: sys-devices-dir + - hostPath: + path: /sys/class/scsi_host/ + type: Directory + name: scsi-host-dir +--- diff --git a/deploy/v1.9.0/csi-snapshot-controller.yaml b/deploy/v1.9.0/csi-snapshot-controller.yaml new file mode 100644 index 0000000000..9d3e387bb8 --- /dev/null +++ b/deploy/v1.9.0/csi-snapshot-controller.yaml @@ -0,0 +1,42 @@ +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-snapshot-controller + namespace: kube-system +spec: + replicas: 2 + selector: + matchLabels: + app: csi-snapshot-controller + template: + metadata: + labels: + app: csi-snapshot-controller + spec: + serviceAccountName: csi-snapshot-controller-sa + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "true" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Equal" + value: "true" + effect: "NoSchedule" + containers: + - name: csi-snapshot-controller + image: mcr.microsoft.com/oss/kubernetes-csi/snapshot-controller:v3.0.3 + args: + - "--v=2" + - "--leader-election=true" + resources: + limits: + cpu: 1 + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi diff --git a/deploy/v1.9.0/rbac-csi-azuredisk-controller.yaml b/deploy/v1.9.0/rbac-csi-azuredisk-controller.yaml new file mode 100644 index 0000000000..b587f0365d --- /dev/null +++ b/deploy/v1.9.0/rbac-csi-azuredisk-controller.yaml @@ -0,0 +1,200 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-controller-sa + namespace: kube-system +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-provisioner-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-attacher-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-attacher-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-attacher-role + apiGroup: rbac.authorization.k8s.io + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "list", "watch", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-snapshotter-role + apiGroup: rbac.authorization.k8s.io +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-external-resizer-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: azuredisk-csi-resizer-role +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: azuredisk-external-resizer-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-controller-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-controller-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/v1.9.0/rbac-csi-azuredisk-node.yaml b/deploy/v1.9.0/rbac-csi-azuredisk-node.yaml new file mode 100644 index 0000000000..eb15f013fc --- /dev/null +++ b/deploy/v1.9.0/rbac-csi-azuredisk-node.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-azuredisk-node-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-role + namespace: kube-system +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-azuredisk-node-secret-binding +subjects: + - kind: ServiceAccount + name: csi-azuredisk-node-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-azuredisk-node-secret-role + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/v1.9.0/rbac-csi-snapshot-controller.yaml b/deploy/v1.9.0/rbac-csi-snapshot-controller.yaml new file mode 100644 index 0000000000..4583e21896 --- /dev/null +++ b/deploy/v1.9.0/rbac-csi-snapshot-controller.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-snapshot-controller-sa + namespace: kube-system + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots/status"] + verbs: ["update"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-binding +subjects: + - kind: ServiceAccount + name: csi-snapshot-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-role + apiGroup: rbac.authorization.k8s.io + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-role +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshot-controller-leaderelection-binding +subjects: + - kind: ServiceAccount + name: csi-snapshot-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshot-controller-leaderelection-role + apiGroup: rbac.authorization.k8s.io diff --git a/docs/install-azuredisk-csi-driver.md b/docs/install-azuredisk-csi-driver.md index 344523de8a..4b2a82d39b 100644 --- a/docs/install-azuredisk-csi-driver.md +++ b/docs/install-azuredisk-csi-driver.md @@ -1,6 +1,6 @@ ## Install azuredisk CSI driver on a Kubernetes cluster - [install CSI driver master version](./install-csi-driver-master.md) + - [install v1.9.0 CSI driver](./install-csi-driver-v1.9.0.md) - [install v1.8.0 CSI driver](./install-csi-driver-v1.8.0.md) - [install v1.7.0 CSI driver](./install-csi-driver-v1.7.0.md) - - [install v1.6.0 CSI driver](./install-csi-driver-v1.6.0.md) diff --git a/docs/install-csi-driver-v1.9.0.md b/docs/install-csi-driver-v1.9.0.md new file mode 100644 index 0000000000..b9b690c71f --- /dev/null +++ b/docs/install-csi-driver-v1.9.0.md @@ -0,0 +1,36 @@ +## Install CSI driver development version on a Kubernetes cluster + +### Install by kubectl +```console +curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/v1.9.0/deploy/install-driver.sh | bash -s v1.9.0 -- +``` + + - Enable snapshot support +> Note: only available from v1.17.0 +```console +curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/v1.9.0/deploy/install-driver.sh | bash -s v1.9.0 snapshot -- +``` + +### Check pods status: + +```console +kubectl -n kube-system get pod -o wide --watch -l app=csi-azuredisk-controller +kubectl -n kube-system get pod -o wide --watch -l app=csi-azuredisk-node +``` + +example output: + +```console +NAME READY STATUS RESTARTS AGE IP NODE +csi-azuredisk-controller-56bfddd689-dh5tk 6/6 Running 0 35s 10.240.0.19 k8s-agentpool-22533604-0 +csi-azuredisk-controller-56bfddd689-7s8yg 6/6 Running 0 35s 10.240.0.29 k8s-agentpool-22533604-1 +csi-snapshot-controller-84db6dbbb-stzwr 6/6 Running 0 41s 10.240.0.17 k8s-agentpool-22533604-0 +csi-azuredisk-node-cvgbs 3/3 Running 0 7m4s 10.240.0.35 k8s-agentpool-22533604-1 +csi-azuredisk-node-dr4s4 3/3 Running 0 7m4s 10.240.0.4 k8s-agentpool-22533604-0 +``` + +### Clean up CSI driver + +```console +curl -skSL https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/v1.9.0/deploy/uninstall-driver.sh | bash -s v1.9.0 -- +``` From 7d3f237b264e2966875052957ac43ed3f6ade4e1 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Tue, 23 Nov 2021 04:16:58 +0000 Subject: [PATCH 2/2] doc: use latest version for master branch --- charts/latest/azuredisk-csi-driver-v1.9.0.tgz | Bin 13231 -> 13232 bytes .../latest/azuredisk-csi-driver/values.yaml | 2 +- deploy/csi-azuredisk-controller.yaml | 2 +- deploy/csi-azuredisk-node-windows.yaml | 2 +- deploy/csi-azuredisk-node.yaml | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/latest/azuredisk-csi-driver-v1.9.0.tgz b/charts/latest/azuredisk-csi-driver-v1.9.0.tgz index fbe364a1a700f7166d42980a4f2177d1ae23db1d..08cf3b5b4e6fa4f4e2d5db2688de6f657434832c 100644 GIT binary patch delta 11628 zcmZ9yb8P2L;4R#CYi(_FYuoPDwryLVa%*k3wryKm+uqu?`@PS5@9*X&Z)P&doHLV5 zGJhS^TGm-6D12v_uzXFiouKn|{cs?wq)61~1ird=Jnv>_OW;J^6FVQh&^LK5D+e@Yu$Yq8TR+1b8gr^8Rj{epPQ@~3_Em= zr9*JDU-{1$+h4TMO*r9~EO8aklTJN?%15vQPI8*k&WR{%oMqrYWjMbmf8kDQ?CYm_ zV0&i3OY#JH2myy2d+<`vQXl8boO|8ds^Y_$sPw~dpBcKI18SBY6h8AkUef#(VBx(w z1PKx~^s(Vwld$(gOp}_JHr>2BiTF)JW{&62VDbG((ox4CMMgyzXYC;<45pZ0ak}L@zl^*A@4`Lc^3l_ngr0C=#LMpu=vVtM= zGn8U|l%JHB4;~wL{~O5iJt=7K_v-6|Ptkg&GvPDgg~+klAtb%k^H^NO2N}E<^J>yH zLBS}NS@L$hHa-uW<6_5#2vYF};S z_geqlA+?GYe`KuzoS#JBe1gAtU@AcWq*~|)O6r8UC=ll$FQ~yYSmRRYi3$)Bq*>XA z^1;MQ&ZOKz%D-5!^1h}}JM+bshzfr%njRgs?6VL)7)+K`|~rC#GQWjo}{0L1o8 zStMmr>4jbPqS=NIaRlBcZcj!TY+d{Evn+=eTlq__PZhPKC2oq|0$~AuxZOt%D$<;U zgwaE&QB|x!X_{13N08YE{L5k1H{`!Kgf!*SP0FWR(TP5|4QugENV_=Qt0K}3DBKzv zS=}RUnRpkbak}peA}>F!3#5T{gI#+Sw{-P5GDed#2r)U#}A^y!G~M~ zEvxJo6ZW^rRYRT+YG~z8tEMu)C;HclhQKY$iLj*$LHwS&4w)KY^JVu}EMMs7zja1j zqi4|@!tR&N-mm5k#~??bWYuZ}WpamB=0!qc&RuD2#?#|BrXnU-g}PWtdLz_WJ#$6C zan=!>bhA>)r}(fpE7Q{);x)-1yft^k%^?-EHOr^UzaIohxf_NSFDf7pc0W!zul#-& zI=ADZIx5f8$Bw4-up!4A4Lmo;nUj*V%=!)69OLVdSptK_K(RCw ztzC|w2J|;n?1t(nbh;{E{N!RQrs3hZHK`K{7pt`u6^69o*5bN3wsf`V#`oWEKJz=m zq%$Z2r5yW$P6t?|)mSZgxo&(>e2K@?v6p?0BVsYuS1Z78{iyVh9xT|r?}ajo~9K0BKWAxkRii@rJzWuUzJjZ)dJgNA-vX-?Mo%SOb?UD4qa)# zJuHbP4&@Hwe7^2Xq-GQ9UtHRUS_d8wJOszDs(~(Tn-UnGUmGn4;jI!}mX$DxNSJ)% z_mv9rMR2XX3BOEGU@8vYZFjOD0|PV_pdaUMT|cR{)4(9`ytK`QzkRlz8kGU^KHXtl z;ZBL^on`r+N93x-ufA1ce#&EO(!pe=>`tfn$6N?cPmiy$ZYR9r9wwD>Mxw0VO}!G48(!_=gpENE(RG{a} z^)1?{FC?y~#j2v3mzZ-7nPrGHNUeQ|8}_K#t9+u4xIxEm=P%l+4~p{yxPVUmU+-oc ze6SUxzn;B!Fbk^Lx_If!ltILTfHyxc{TawIoGCs&sU4ORpysvkbHIGWpC!c^IGKi^ zK(70%gR44jCP?)sgl00(=8`M&Ck_}>S?{@W6j49wtrLc`2a<@gZrmJ+bDE|2^#0ha zW%Qj!I39Q@_vH0}mFMfu$qiEoR{TKMRs z+Xy@SgJU*?FqD4IJ(WK+L|Bm=d4@Y5?Ip5;9N`pGxcJ&i*ss_DBb6qa%v$D^gutn8 zQc93Q^}NH30HsAAf8A)J7pl-wYkI`W$}cYe>+Sa^K>s@}BR%&1Zf5)hw-C;h)adMD zAH!IDm+BlJ3RWByw$ZWR-J>2IQ+AA*hxh#uQl3XQFO%o?*h7k>{0SPNX~HX5c3t+& zBbEgMusS24CC`S1Q_Cn9gk#ap%1E%~yh6}_xkybKs0%xQzStphU*{mm7y~0Tw#Tnx zEbbZs$gZ=QD|{kX54Lw?%@xz+g}On1K=kIHM?r?2Hnq53*e&J)IK266rBBUB-$cRJa#`aVlkG1pERvbl*HVh~KXC$vtGM#U6|6 zLnR$WCVUqYIN`gj=}N|X_-9L@W8UpQ4^i6R>2M)vO^}cHCc!0Qa3n|2z@LsI#&~W> z-!kprOLLhRlX+SEL%<}mm>s~vy(~_}LJ2JesO7cfzFF3u9otYQN^A0GnMajbuzK+mty);Y=bnH)7KC-Rz%zT zWI~$-BdlwneOtgkix6Q&OX)Y5(oxe_O9gf5O=J${`!L2RQevV*z-mbOQOu42JP!+MnOZ;e?R0QUJwYN+b2;||*$pu?5zj?zEy}LGWB+pCkwg`)<`+G6w>Pqk>WWD^ zwqp@b&2WYd%@7zfL6wcI>Q_y-VU`TnuBF>5yKW26wOsCiPaRf!w|7Vb%nKhUALXm7 zZRC5i7;?$e)J<^5=$rpM){1LKco|O)MVesjla$pIb&|qH`_PpNzgIDUH|$9ilN|qb z(tMlS?O9Z?bYI2`rT4Z$x<^*ke7B!kOipRLB7!MNA2haP(WQcY4 zo+HQjBv*yfrReas9@h2-#(s3=b{LUG9-XwVQU{8r$%y|f6aTdu7FhF6**r- z`b#1%arA8w(I%Cr0jVihlJyD_RiT({2rYvr<dg@tpNTU1)apg)<%%&&*ZCiZe>O&Ml}u#xUggc(n39QC0Vo*WTvMme5dDH=tm+PA z3l3L!$E(ckSb28C1I46q&I0)F-H<%urPoZ`adVVnFByqo5^}Hwa?;=dlaYF4Wa5Y> zhV3ESM4i=n@KKqF4D=~X88=B61kI$`A)Mp82q_p$5xM?3>~VhdMna{pD%paR`I4Qb z3})LF;rAC60GL&R+RIOO*94zl)}zG^GM%~aQM|r^7RQ#gOq4i*L<}9#AC3Vly@!V8#Cgpbyh?|RzYkICg=0e_Cobp< z^R7Mc@8HKVtvA_>Nz8f?VZYhHKky*lX=nRRX=^X&fOVq|`qjbH*9e}_L^Q=QB?6j3 z>QGx)*GV#uF_d5Luu^X?@w+__mpO0DV7ZWaUWVZ-2LvhY(n%ET(0_k)^ozTOYd5;O zG30pX7=gcam8Ddf7ajK_CWd6F?<+1jd*Bq%1xmA^thf|QsA!{iz#mmUYEmrH%1EE7 zg9z%?fO{P(&RlHmweR4yyS_}q5f7U>`$a;{5(uOfU=&ow9o1rw{mwhpF?lZ}n89Y= z(s~%?l?UtJ?X1TNue(pZq zu7@=>uOQ_ZwNS2;jcfO3$=6*WUw&4hchKch0Ph&l;W||7opt@R^^O;j_$p$I#x5F& zjNXVL9h6XYTh;3Nlo;YXIcrUUYFy*@S;-D+8rqeK`m%nh8mb;LycgYgpINLqwlGft zUdoTR3uZJT{3z5d?ndOc5~XS^Z00RxHe=n6R_2!U*qlq9*`?Qyaiv;D@`9Hc`2Qx; zfIzusL$bS=tZ|W=ru5!L-O{CzpJZo;NxNNaBahI?B6x9RaYA;=*|E6Ti0i^DW$J3@Tpo%*J^;A zQ554jEI_V)n%|XTo`BHJWXRuE|TMziSXeWRJU2^i8&ET?n0u>sR4_`GydfBR>e-q9c4F7$~!j7#{+ zBSOB9PYj(vyH>OkU1&5wdzV#^ONQwqH;%i?wq7zf9y^E|?^Bg?a1bU(R+Ewf5Dax# zL1uQZ0a%&jF<%RdY?2{(()S`N@|YCKk8hxb^yf3%AO8F&i8=x*l~Oj3($z%eOXSTy zMH=yoZ+Hnh0{j)>Qja%faSe-ug+5+t-{a4AmETTxmH#h6M6~q(i2nXvg z0u4VPpZp^zAWIKF6uvcPrWe`GgC1}>(rA;S)|wFenSL@-J(}hwREmi*Zi}N_^D}5d z1LYG_#j;HlCk@>kG3rSr6RiwudivBjrrfFOI;@N*pc|1F-IjwhZt zME%GibGy%VLkWD%+MQ!Y<4yK#tBhRNY(-P{Z3{QR5+R5w8~k~VOa;Nhn2GD0Q^KdV zJzuvn`$2;uOPgWtO0-jBicwO6yu-OOu31o~2kN@x3HEhtsM8Eix^1|Clzq|laCarn zco0^4u-?-i9(#Av+LIcEl@~peV7_*va#*&rDNRT)k zIuMAP8NzZog16b@Az;)&^IPrI+n4=xE}hLkhKAVmjc*?=SgmwQSrF$ZyVaharg8>% zid|6g=O`47I+flup;$2F&OU!tlNBYh-J>Lc7;J`mi_3+N9<-0x07q3$%H3aVVpxiV z0-c+lg+KMZ7{x?$ienvCp8Bd%nMVyfUCe#19U3awroc@-s48~ReyfBtQ{?A-;X{$Y zG)MLo)M45lOoF8`Ke^hPWT!85u5DbIk+tut_$f4eb#^i-kEH62z9;U?!=yp=A7%q! ztf~a^e!D(F8;v$AT0~t#(ii%uWv;%LaGr0PL(oaXEzh5=Kx zsP1~`k}7Ax0{d20V!zr2T=wM(r@XO{4-Yx|Q)6+cS zj@lv8h~S;s7(@S0NhFKVdcYs(tZT%}!icVK1iU%e;O%Skazgx?EZ2>P#YA&50@SYX z!`X8p}dOY=DdM*iG(Nt z;W6;+j@w&(u$~Zb4)@OCl@#vXJz&37ZWpo(&LBhVOiQEkEC6&YMOhG2t{94kAtZ%V zU7~-K$OX>0rO-YKDCI&?(czawD0%y_i(i*BcIOJCVU54MXa-W|U4vvlR!(~_5~4Tl z-^38?VGGo*b`1^Uy5{n^c*P6nv_MR6uXja)?r~RAu@$4HSPjQHcSi?}{aVM8mZ_&~ zgAeJ(<4o67Yye_FTNm>sZO07Ky_M^rn>=V9Daa$9f9cq3jQ73y*vr~r+x{q@1+Tj4 zq4ko8=seS`;Yw2&(j+|UP1?Li^BWelDO(@6ySPyNuaAET=V(HMs4Gs$Y5)* z($~xVd+pW#;6U`v8PWNsX|`{bYrK1nW9JC(v)1wdAosi>XZ6!t)=U4JzCEp<7Fcc{ z;a!Oqk9WWNpUwX}{oj4gnH7JeH=6xZ|aA<04g{00{- zU096lM|6T+4%7BG+G&YXvT*L?S^zb*h-OOC`5`OI7bIkW|`xAM67BGVQ$sRo){khhzd0zOa7TAz^?)i1T(`V)Q z`d3efBB1P0%Dmwl($)rlA6*K8AW42DD)B<<8ic4!(s?JVg3 z7(m$>?k(@6P@PD&5c53I8#pCAkW&h46@WGFnEMuVlPN__G~&8c^LSzoRe<*84@G}~ zQ5HXhF)w6j#6qib{+~aRd}kJn_A!ODY$}5*dk~sRixsbc@sDc5l|8ggQ|(An##R)G z>D8{smjqn(0Ms8byG3sWZd&_siUD>3O?{A8Cg2~|;~jWb#`?jV%`Q-)$6Qn@ih#ua zw92E!~)HR*+Z%tx2pI2NaH-p zrju=CfEB(-Dbc!05&CQ}ZT+*NS9@Oj7|kqrs7*R(k^yp~Ok zKpB_N70KV?Ntk8{WDq&mqdS7{ymFW3d{$XBul?NAf>>hkOjYh?*wG)ks%c}p&S*~V z^VU$sK=EPBuPJe~0k&9brN}v)Ublkx^mLX64qRI5a=Fo5VGFGZXn)2P!g9kWyXF{r zdj2PdYiyh`!v40(<)mz@szdE0f!&R0q`r!=g$@VOicmgo8WpTD^VlcH&0Itj(LmKT z2w#5nk~L=t$m5i%+fC04F;*GI+BSki)RhbqVZT^82VG&h17byavK4q5SqeMCQrY_0 zSs8~z)+!Fa$lcW1SsG@W73`uo+$u6)M2I<~hBWdbYnH6o>I!UA2<^I|1zC)Zr4aDH zVP}iBlm4kM&pi*uKmY4O-d5-;|BnFdw9LacVH8#tE-P*hbQSRN<>`1`+u2Dh`Oh%m zGvIk&33Pkh4fI}p=6vkGcX$7%FhMV;;S-Imy}!G+g=W2+&A%SsuVB31ZAWCDp1$5n zWi%bS3D~8)*V?kTt)V94f(NmI%3<)!I*lQT-=|fJSc-x{f?n-6_q^Ftz4*f=c7a z6bCYiwEeIQ!pi%4@|>jap_xXm^pkS){4Ikuj^%-QD?j-HCTe1I3`3EeGXEOze$m;B zqGlKOPe;(4)LCWFyHw)0rO8rh-Wb!@ zs&JD{2M{OxykNYM2z8b5EqLgcPN1dL*j{)VpLy7xmspC(O$|rv{Ku^9;N_}l!^1tA*Y*MsjuZ3<&pQr> z%egw?6Mw8-+wD>_{cD=qk-BSYFJ30Ne2ir(+*(qHI)|q; z#MIq}HFdliOdGyb+y97(_r%^MR!Y7))8I;3 zwR&?Pnb=*x07RCMfydQGRFtE;(#>8}GW}<52M3ffzg>QQ@N^?k zjbQl5?*Eq}`c-LQ3xuQ81sY=|FR=!~J`Qq&u}gc~;q1YMngsqj@$9Y?q4X({p-vD( z<{e8ZI~TkA_aG(=+avce3zhQqA+;q;_QoBxE|oMIQcYPbd5lVs3Hb1Gl`n>H(h``H zVWp93;mJG!H)NBoPuo9AjqL`#|5}_*6aQD$;Eo==f$&3CG4p5W#Xa7reD7F|ts2s^ zM9E+yRTTNh9H1^0t(SPgSG@E8RUs}Jb5gwPUwAcjHNKgH>tVEs+CGUcrbe>B!^Ju- zJX8KYd}a7Q9RgsdoriIGL>tz$sqd*sDTwM+3Y9try?wHe*hBl-&vkh8o+t5bQ3l;t z8Y{TI)&)KN-FY+CG9#l;)jz^MeJS^2zW$CtzH$%Uyi1K*Yz@o$A3Jl#&uXnT2@d}D zE}OU^+;Ez9iI5PHMDOq(63aDgnGj|Xiv5YMMICFLLJ=Ttd-55beuseUSC3zLmu-db zriRWQ?%SQYku)~^$!hLpb9yeLz>FW&_&dtCnM{Zj78u78yI-2A^d_X442CqBwn_}u z_E98CSbLR1mpuXZNB?CcLpg;zkCm2jhLqZ1(TA0T*aNqmkOW}QO@Zi=gp_|o!LP>J zUHXUUTJL}{=UE~5S6NlPq!Ja%O_?}>Z=w&s5Az*f#@`-`sn5$GA3EereAkDSD$LYu znOtN36qkHC{z%K6S+)1;$o<1xHMuGJX`}E_rYMVQ2 z#VB%O_BukRUeGVavc3z$l%`(jntq_x{HJQegefSKWQb5nHalk5%=s|7V-AVYr3TjEO-3hd3pGZEZ})AezI=7;~{ps57AKYH&59E1sB2=ySZlZYQ>Wb?eo|2D>PT_P%)o5_skoiXsh~2KMzPzhmK(jJFD$NU5b^a!(eB4qN&2%f5M4 zf-0)UlJI-_Zy!#T5`RWosG~qJo@cUQ^H?ZA_oA8x7;dBYSX}h@JX<3-+aG|{;o(8r zbu-?h|JSGvT~ppNE~DMgf$O=D!uw03<~%KWDqs;?we1@kCsO}^f1ab)_5SemFtzsX zS(;iyG66wB-6K6@!oCP%^Wd_T)#db1!$t|BCiQyJ34Hj0kT>1mNNH$jina#_7pVsr z1oEpJE~-^?m15K$+BR7#+aGLVI>2jJDG#f%-iIR7`y9%+Y|RfTx>=)#@$0u+bPv|c zJdUMAR3c>yPDVpHeV+-G8CjK2&xN{KsR->XsDf7fyFkG97!_iD+Rh1h^8$8SkNo!=rDZ~zYcBLsK7EWP-OVmcUfz2B^@+oax+Py&v z_3}0*DKdTNf=#KbG<$oid*Ovp(AxQG`}gtQE0o*&>E`nFG_|oX;aU5iZ~gqA!QJDI z8v-0pCc^dA9Vl|Y!O&+*sVWD|-T5I0aZ%o6!V2}uE5)9;T1M%)yMI{8Kw#`#x83s4^wYs^sC%v;;Mk(EJW$piB*^?_N%@h= zfv(vf;J*?D!izQN@iG_y1bjXGR`+|a_QwXhquM=PX=0fEtsP{?3fC1-A5GfON0gVX zmP^j=b>7XnKj(>SNFX9`em zX3AJ%dE<^>??d0Sh7wsa+`>!Ao+ofVx6&GMCr1T9m7~2b$N$xQzIl*q%tB1v0WWul z_wZl$Ru2uSpm#4B{uH^IU6vsZERrbW{u8LGpzrb(h<{H7`osYFuStZl`q1>L>#|8M zQA?q7&9UFzeXT;OcEO6q^&?SJ;X}`nGR#j4-aP~EN+paLn+&5(>n2dUsTgYnMjd!E zs)?QTV!FZX8P^?qLiYP(kR@Ov2MFpjw%8Z-i?4p|RgU50i{LNikM>b-lem9%rQ}mQ z@>w%m3cm%gerW;6!BV`9c^=+xQ`Matn-ySm8xWvFLPn{?3<1V!VtlgNOQq@cI>TeD zwXv7NJ?>A?IBMflpKATRdw=OspD@|ZxkIY;E+=&Ah;hk>_U2FV#qi-6K}Erhn!F}X zC3AxnYFqdz{ic83jZ;wE2J%&lGHAharE1km2BC)^OSyo@c2@8E;pJ=TeXEdPhZk@c zuV`7z1S(aM5?dv9n4oD@{*5<~46CVorGSM$P)#Rtg(u*-++YOW<)W#sqhrH~9hb?4 z-C<8`ClPP9)c)(bMq|k{_H`tlG~MsR>h$KeP#bb+B)<8}MfUsG?fv$@0A7F4(Pinv zicobOY5}0gymAp}hKtzzSTa=h2h^aS++pUJZPkNoF~A+$_bVuA)Lgqti@5obCnoja z9^zZ8`fsMLe%8G_tN3m&r;9KapO&x=rRKz)rCg6_mu@QILGG&R*_g*0bgVr~6HNnY ztiNF*%*BWvpHk`;{>;CKru6%m&*;96z;JxZJi&l&E4I&Uq8*#OXri~9m%Q%=EO@_x z7Pz1jiudAIEaOig(^geU2_vr?3`*J%$wo54;vK7-v*b@E<1V@zCA6>Ybquzu8h!z89T%; z@t6qzwkjUOziyNWrDX9^%nr7C{lnK#B)|e!Zhu8iNS0%o=`IOk2alY1-Be`!N4hmQ zos_Dt)VhqQA>^1Nss6Q=WW|sGdt6wE_O*X=_?wXtV!JPinNb!yN85gSrPD zvCj_9#QJYaYqySXCq4JUYMphseFeQn_H>zTU0W#z5ZeK6Be3KPJVoFU@-ac5Lk56 zK{uAbP2pf;0E2}9`(N`y>v;eG delta 11614 zcmb`NRZyP6v*vLL?h@SHU4uh#cemgKK3v|AAPMg7?(Xgy(^ zk)C`+O6OezGBPr>8^k1}BwhF!LZaT^uh$MQpTIZP8#!PtJtr?Sa`q3$`tSOCJ$}?^ zk;AOru%@8!N;D5=?1B&E8W?g9D0C6pnAxrlwbP&2UD@i=)HK+9tAM!wp*|MytdX)Zp-dv*3a*UNArq=}wZ$0HhDme=P8Yi`gqea*cg#)%ESD z>b_@@MtVPWMTIb3A`c28C|3V?K(Arfd-QE46S4lOxHn}TMjS6!!4Qfb@*9zYz&H}z zGY5o08%O_wVmva#oK%a^`iR_S6S5}1N;1E7_Z|dx`Qafo&dGL>Tqc&dRc#@f^KfYT zdo+b0mBfCkC>Be1(7n(t?1C@9dCumHXsek4onL(x{bTG-mo9aSt~%hw{Phc%9YyG( zYbDglTi`y4r@Qh5I95<+Ngl^R5oa|u$>>5mV^X-q^Br{cfYTU?egHyuBF@rcW{aF)5a5iR1lT0EQOxv%xSW9hCi^O`9nGbKcf;*m0eSqD5%DlOT_=sA5FD+3IiZ!FlNFI!y`uSl0mZF3OH}N%a^%hU|~{z=`O|Z8nv1+dsr~auc3N zu*fjE3b_yz*^p+~Z=frlCj&(YB>#RagaLfN_gpXW>f{%bKZkma*xF3Lu}8Yluk$>^ zQD*=alN`708}K2m8L)Smg)e!R2hP1-QoYu+glwBkL@HbIH|LlkC{6vr{U@Oa=&zvS0;HYou6n zQ;oL?{&B2m+Z=_LKjAkPMyGiM>tb}=wI}$^VFgsRlc(B$p7;qx>bm}37QCN%dU7dv z=JdEQf)47dX*`Y+()jd`QsQD^!e~+5KMiGrce&voc+IsL6|Mp7AWP-}l(0 zu-T0iivlrQ6o@K6`U52{1@;3c%5x+SPtU?@Zx49mI$;TLd7Cic$!o8yE*c_=7V|HC z)?)?DubJb{V2M;BdBBFefpDkNaUFLVc2R8*Zjeje_bLSh$jx^M=NByo*!|zpa{d+M|!)b|LVoZ>X0ixd|Szjp3|{WyOcB|RuE<&M)(h5Noz{pR~EjJ8=l`-PX^;mk?ob| zFbI7yx+fXRk(BtwF?}uYIzWc1G2*e%&5sE2R+)}`oEK;PtX#_m3Bz?+JNg6e*=eRn z1xUN~Msh;kqTuvUmALpTQ~34fK_}!fH?}SYMr!ce<}~kw6K?W00nECE?lZ%Za@ori zp+=Tyo+NG=4Dq~wMz0MhEqJ0z%_8beLG>HmKD_tyC#kdO_S2-@u?;s*#gg5g2WYNG zt|>Q7TJe-0C3B^Plnv~kizr4F#_U5!xWp`@YTy2J`j#x`KT}2?Arp2kP*W z*%EAl;0#zdBCE%TAbaY!F5&z(O9j@K4I^ExPSf!k@W~M0sM{ zY?Hs!hvIT3arY`vxuKN4CvOCS&#!x~wv6Erw*%O`<#5TUj3mt^ci0r5pLpn;x+YXr z@T2po|3?1)LP=+pJ^;oC=5RRY^lb6JzUS)8y1Ux@$_xX1M(?tL^yoqCL#72D&lzMO z+>qp?jdL<8U)&!20yyvLU(wX9=dmHq=!Z7_Tl};dP@pbp?e$cBtF|eHJRKJjXmL`L`<^J8XMOFSyGtloOM^7uXP;W zgWq~}5RwI6_@wYvn@66dA)!=ZdQa01I}J}I2h=zeA}0WTDfJ25Lo21%mNg97(vBXe zGmgy7*!$R4L-X)%(o}(vQCV7Xs06E0WeEjbYnt8Eh!=4@k&nH4Hhc0GNWo^A40n_e zD~>NRIxBEx^7I!#&9F~Vc&QNu&lB&X+7g!4jPix(o~XQTm)GV(<+2@|5yM;z!bKcQ zDmx3p+-IAguZ2Ul0?~tED8NJ#q_%Wg=6){>?^WskG19euitbof)7@h&7ft~ILJhjy zE+!P0uE3``)wfnX&4k6LCgL_R(ci`!-)1K9lD=jncZ&U>P{QZ^At6+EOWqq?6{>DRvwZXRMEuAzIPNCX7}MvvOhJ z_D7DroM}Et&v@LHp^mX-2?PC|2-h?HOcbnr-~@QNt3DFojA(5WQ=reU>?o^$pPj%& zAHbc@S6Nk)F({mMEqtnASe6MdI%P^>=h#AC7X;A9$}th(!9yBI zc#;hRfBROmun;@L`HARu)I1VxKf5&}Waj6D6v1w5!i)(m|Fp}l<=94UOI9AID-An% z;l)d`nWd~^tj(pMtfRolQQs`gH!S*&Iu5jNcwWO^;K;4QOTqjj@M9*zRF` zt+Im_RNu8s7qSi`Z=DU7Ae%jv!wu81wLr-n)%e9O@Ymb$NZN1cZbhJOUbC62aM{Tf z=wtd^aZu@G>!4}(yl4MJ)KiW*CTmqvP!3R!iuWU#vnXNb;lqo$QLmwh>CEc$!!BDp zAV%Y`GQjfF2DeQ|p*|G23zXH(lUaw=O%`6;{^wyFKu+jT4~?rX4*mW^_vp z$;Kstoo#Hw$HGW*4oOoO7Ae5v@|+XEsJ!&n+AJ)VoKO<`Gh1;&N>3HD@w}4m-WVZX zu9SqZ8y|ZrrL1P?!j*>s>Cy8X13x5ouLi|eacYi!IHqt~Q`}E=C93wEJE=N!SKdKZ zyM*6r{YuL+ckI4|@x@bps`FN`4xJ0Yx6_bSqk+m?u_Y$7?#O3U1M|A%B^HST87~EB z;aTbQK6}sj)RqvGzXveasPYEI*|HAzva7$_b17u9R1%E%`AO`qqH}61Pa{gSHL3p! zigC4MQ+uGYgPi5zYRsRQE2`)f$4R}%f(XS;F8gj4bL+Xmhetr|=2Vy**qTY8GKJcy zMm9=CKoMcCe9CE7TnTx8uacqw47AsScthP7%&5zgb*qV`m8G5|u;xtelcD&Lz>9wD z2Vd(V26o%rMnKD%H&Fz=)Dy15p@)*v)y1^%Cea0dO%!0CqS8SAc z(Gl)3VoF@FQruk^_=bW<8Xzn=aamBlho3AhUQiE{3;b(2tio9cdKbcB=*+~?EyX`w znk(&MVvw7mzl;*U-Y+`9YO1VSd^4@_1qWtX92BD*5!I`r8dv9gioKrRb|8P@5lpuRv|+#e%I{S!P5U+blE7Xxj{^?%SD9Uc=JEmjLw$*YC2l=B z==dnf6@?8-&`Q(KhoU5h^YJJaBKlppUdm$|+P8 z3#S_RhO_XpGYcP-Rg*gZYlSzCI1u}Sbm;i8x~s^mtDsQQE3Zd^&Rmqjv7f@}nNi6S z`1Dut$OADPzT(ZnY(q?DNyw(nQO@~MANc3;8|Nix(3f%$p%4Fhg9+1=FfKW9^Dp85 z1Pqr)GPj`2!gk65B5w9%nC*z3v^U9LbpB#p?spAh>k#r*sNdMo-bP8b~-{cF6Gd6iB%M7`!-=4DjIra!pR1izLyjLYt^q z)x^fWDV|kpTqxgTcDq5iBU_R8NslA7U35sPfSJbP9$T4E6KDmM)8||oo5wL$Pklx; zWsy(^lVS?W4nuY@@xr=>zxPm7QP`ZTIDm_kYEDB;O}FYpG{dW^GkALF7fHz+WrKIV zR@CfqjGnMsnT+k7Y(*7h$y{Hax2FbL)G?jBuuy4;%-v-%2}wa#oG2C!Ktl4Ls5hUd zhD(ktqXaP-xl4Qj(^w-#3Gd0nB#1;Xn-D^dd ztBDyG%gdzJRfASZ`JXH+`h7awHm$xZvEy@Ywq~V_w8x`j)Tf3O!5%0W#s`2c^zU{! zMBf~bQ%u@C76>5L6T@XfSIBkCdlX|ALOfc|2t*NtQrp}O)0z#fo4bK7J)Xcc@V(WD zfAGaKZd}K^z=s24-!q4%|Jcibv|mN5j4f~+l9<7Vttxie&w9r84c*G82X4iUOy?8U z5M@1KhH~R1(OkgPCjIEQL?MAgQ_Lth8rArJO)4G|Lly9dafARewQr3dO+A9UNH|e~ z_tt{bc;mk!O9yY|KMZo?(WSD&UVTsg^$858s&Qs4MipYy=do_^6`7X<;#)i2 zMQ3tlfpQNPd;wT_!YoK1EOLAf1z0MKr`ms0+AYTEeYgpG(xt000X9= zQ}t}nfh~}6L~%@|4|NAJ?3w0tZRn2qjxB#0&*I#o?B;%Aq5Jo`Eq9C`VYmzhck847 z**1wN-$dA>-=^dc&&$Ui37)@|WbLq`AHEUO=kaQ-J4$`U%>lW#(p3M;?t17&#pD08 zM82TBq4rTOnz9MJk~HGG6ahSTyaWmucyN0;+&_K>I>Se=k3#Cf~#or`(9T#fmWk#Ad4HrnrMLRsxm>>}sbhrY8 zy(@)Ym)Y&{A#M2XD;2jW()rKMVZ^1a&Eh1U-Fy<#Zd;HL+5&xO5&^$fls#dEka{mF zM&w?i1MkI{LlG|IgK~}iljYy_*bHJx3ZbxECbLMtn+bRO*&q*j<;_WXJh@DNtv7ii zTjwLmYUWRiXo^{EFe3T&QO!gj?zuPsbt(r zCH9t1*e1$VvB*b^zwi-q*9(6VL9-Ejp4oGdO39$JOv0nu8UTpfSzF3Rg0X!U3$DlE!?iie=zCs0m@EvS9_=rO$MP6$QHXI*u>z4+Ur8vVsrnymI>itxsHzyU0|TpUFGRE(#Y%X-Y7#F4M^4tB(m(*16w|4pCXc>ChaYh%v>7jN{TQqcETcm+pjr|e=XRz8qVF5CIx|uSJrg4%evrE!&r^yA z!4NHyCcF%iXHe!rhpVRv3WjmBh${;{f3U7+5WjXa4b+IFmn#44hALiNaIwX7I1JRn zJ05oqpa)WZMoZ{_cD~v=)b5jX4#QJLc#+E&GF-=;pzcMcdDCJPxa)LirZfX_Q<9JTtNBF0n}kj>hefd&{^Ty}`s*s|*fxK`}x4eHd?d zO}tx%80=HlM_Y^J_F#%q@ueLSCFW!h7?I%wD3bbFF>1x zGAO_+(58?@P&kRK?(fB$VtU!P7i6Oi_H#^@N&8%A_nhR2Q17!_A~GaDzf>hWB!Bbl z$cYvK{#H=XQD{-d;YyjL9ehuY zAkgFKK-k69p<}Rt@Bf^mM43r(Nc9;bPcvs&XM5%XFhZu<98#BAwl(&B&?q!yR}g+r z*Td62qHQOj%aF-=S$N>VYx6Fw73PvTWg|_!)4FAS^1A8Xo^kDku<{plUdJ95<&#dE zR9^arOX}&LU*`&Llg8Mpr*t=sXQ(ZCI~b1=GreqT*f^NQTy-9VX0^p%K3G9pH^xdH zw)0ECc?k9?PdTFO0h6P7@;*A3pk-O1CDnr4=c&k;W)_Zbq;m{1-l>n44S7O$Z!Js? z$gMw6aY@rmE57!_csiacaE~84sKo*$L2S` znctDH(s+`}W(%G<4x2WO5EOZ;c>+UWHHX0hmqqj^w%I{$h%O7GI0zXV$+E&Y5rXn_ zX>&U-#Rx0KS0S7W+gqrV@$G39M2d8COQagb!o@0u?^=<}@o#sc-Q>YVLF&l+-x7ki z$$&`ydunHDAAkPaBo+heRK-LsiSl+2JDk%1RhW_(=FDc~nNzs{J!L^h#Yz<3d0Zm^ z&WmaJJKXBg@9qum&GLEi(`r1`7YD?&WKP8N1Pai3X$h!SVdpO=AF2BmiCt+EPKq2| zznA`1BR97r>G#uWQ-QuH!)ApC?9J3GP_UMWEyVRq>ZAcJzG7m#TA^@vaOIpKbf@9g^Xin?Yvm3L-Q#Cd1p+gM z$^8@BaM|ZcT+tW(t{3h3ygsoG0!2?J*9!VLyHF7IG1LdV3*C#v4OZ3fm1+7FjHPE2 zL`Y+F(c)5tH6jGXXy&#LW$#vioOOnLG$QT}1uRDsI(|ra>uKnZfTK;QcwZeWxe_yF z+}$S~C${ZNIM$#&5t)YbnNL{JyrHQ3^oppJM2Z!(fw{}vl+WRR$VR8JHiy!OsVT(i z@El-++z-A83_&!)RJc|oGoH3CkT0#ut0r~aKMZ0PJGQiiA3H!^2)KIy0~uZmVn?-WBzGh#T^wn7b2-{)#CJ<*VZ=q zU^b9uSRNf!r{#71Yi20}L>5f6Xk+VN(o}LTI1NOGk^?xb@!%1gGLWy8%)j5>9|^Fw z4<9%sbEh`Fd=+M{iSOS5(>Gg#G!*n7+}!ug-7;ZFpsM1}6tISBbJg1*D0qS6)7H&d zCRU|57fzy_ywQl5ph`D0_3|9`IU$7SYE`C2C`C|(bXow8Vcn_83HRk6L?l7IMf~PN z>`|1QaK$$Ryj30A#Lqp%8FttE5VA52`oAQl|A%Y@{(tC3{?RnkRZfbuzW0K-JDuy| zb@KXZcmp>GgC)=HjK0cR6%g_L*Q9TdCQIvtgQWqcPmR=J0|F@K<5k-8J?n|}`lV)ye z6}w+U?CvKI3)As+`AXm2a`u_R{>KW{{q&Ww4OZwO6Us#QH~x|l8zq6izxc^Ui)C(p zNVd7$;m62?W7v!uG@H!>y^dS9A=|q0Tk)l9O6R?51^S~FI{STs!*+*@q04MzrX|w6 zUvg7>CQ~{lQ(CGb$%zSG1guWvrOoBPg^u4rl5B}H^J7uA4lO~KlQ4EO=<9@G#Wb}Z zo0T?zgDDhf^;_>WjQ&ZRo92JgCRIK>iD{@{-vvie7qv2lSndmu&xkb3RLM-W4laDL79I4(cq-bu?&E*imZ z=K6_OxzS|CT7)VeDr4J;X14F{2T3W!(NiJ z&xjIZ?k2AX;bezFz}EB^4}bdLJa9zSh%<(t51w^1a?*I^v~z{ZN{hk2vVb+|qt*z4 zgsvWHqKHpGT^YY^>H1-Bu&mGy_YU_Cc z`M(>d{Rb!)Xntpi`DcWt$E^5A|80{}4+ps?tjym_(wGg*w_PlGIv4x*7ywJ{jKh~F zUeSe)T)ZVxJ9nYC8mogJ&*tNAHTzW;I<9~Vzt3~r7*HKIW_)DH!V*!xb;r`M(qd=) zxIK>;b@eG(##!M-bsb{H{UeObs8ZG&U@$H4F-g4u_yvRrh!fBMjR-t_r#C)aNTW=T z7SMz2TU3lJt}NuKP%F`pWHgcMA8rnAS(1s=51Auajoq(n^bBu>1Ao9G*AE~9Wjrh!RJl~K4ZqR=Y zcj#5(3Xpxd)gQ`Py28E-AFYlhgji9jJ}uf9i3VrCbq+Kc#~v?A@-4Q-xU z`vco21`fI?@vzRSo1ZF{kW`HUnn5}$u1>{`B-zA+rvK&!0uJ_@bx5tr!@*E1apJJ4 z`J__mY)ZQv(7>YQ+RZot<>Z0nu{xvTDY-7fVCSf!#re>8-b@=?DCIlyMoTT=T7GEM zm8&9DIsQs51KCnP-)pA(8%=|mCEncS?sX|Y?g3HZE-nfxoYX!s@cc8~+L#Wd7bYg0 z^+wIEz%(3=-=A|s`F}wt?6-UJdxq+3JB`zrlT%5o&=RdGUxwc8tHxhJM_&7!s_P;6 zp6l4v;ryPQmYnVF78poN_D^a{2ES}=9MoX~YSE3~Z64-M7A`gn3{Lq!8FdM{yfTbH z=`b0Sh~Pywk#9l(%|)>3Y{zst*q@x0ClZiYj9L?vBY(Y@j2WkHr*b7QY-zci5}@ZM zF`>EjY0p=&3hT;HXD?V{uFUWRt5N6a`J|ZVGjZPm574u<{1M{xY+ncU!7J=D@XDctQVrdmTbo?LZSt z*Li5h&v;xzV;5e>inPB1mStrYG!&j3g!~rOnlOTLCLAw{+a991(@b>wcySh5f+Mo$ z_4lI-T!mT@`qk4>>i%uRge!^B6;s2_H`agnbUv4bJ+8@!a`Z8rO!EprbNW8X?q^TO ztuldT;tdl3v$AEzlGPXegU=fA-p36q-k+Fc-jxKxf01>$8SQVRP3s%)W!woQt>y1N z+r0)mI3zh`Bz{5S6unb0qMTe`7XQZ7J+sondS|(!{MjDvDkK_8(6s^+J=QLpEYL-d zc1j8%mUUtdekI{oaB118E>n9Umwnj)3x_I5_{4JmXavij+$t!F;Ns@_*jzFUPwzta_I1}`z&T~?Wpt*3Ffzc9n!D3ZW{}^AdrP7>* z=S9_6VGC7ukqd_Pqu$5+>V77zZsFi!J=3uDClgY1`XdES7+B?Fa&fJk*)$j{U zX5i))UK*D3!yK(XMI{gDnT-{oUR5<+<9m*LGvhSgu-EfM6BO(*@am~)U@Ym8iqfG`2genkB8qa&U{$t4 zby#YSCAt>>SaHj;t-z5LWDfH3?o{FpHQTaZ!aZ7aF*T4HRbWRnNBzg}m`(JWglFQd zPZosxL+B1tNm?lulm8$|JIR>P3h7%w-`)!fh>ir;QRwtm#vYyYZJ!|UnFhSgOrwTLEb8E_dwkMD2N7(>*>DN6v0qPA;Vh zIxfL*>InEl4=Zy9dqyD5%zhVWFb@A^4Xqc`ZosgYkD}ZwYXLkiOkulJ-T|;U5o2r(nWKR;`wZyEWE!vr>q@m+3I$3 zCby@%-CD`P`246IJ~!_a{NP7<;76_1Hp61oh2E8M9ysh^^6O(&-aj%#`29pAQ)sm; z*@ri&_i?*IaeWB^hSA$>q8u3fnH-u*eB7SZz8tIt_t((UE{6rGTFG$9UqWdR>oQ?! zMxoX)|3tn2XA@|2UDe?rJSH;xT;q!jt|dE6>Fa)%@oTp(5ka1uhbMzeS2%bhfY*RD}4bA5t+gH?YKC(+lq7E3CL+0Y+h8uMg zK2GJ2eo0@07Mk6iY>y_APhNVB9QCCH-PidiFfAeZzHkoEJYwVelrQ?~_+}Nun2UBy zYB!^f&Pv6#!KP^Gmy^B5QlvpEXiL~IxbV7vlaH3ZG1Pu7au7=C#@F=j3I6&0Je3T-h?{I=q^m&DNLjcWNfmd+tMFL2cLg9LwXql*Vgt!$=rtMR z@l5%Fr=qog561iT{#((5lm8>=>im{8Yr@#@T{$N*xLR~IQQa|*iaU@Iqq1=|gNY|l zU;F1OxA%FuWe=3+QDtd;{j5DpI+qg*Xhmc(lw_pT6=lP)qV(b)SVbzq@$?9u-rA9= zLkNu`wR_u31V~YwbdY>#I8jr7apE5Ye=6j1v2*#ucQwe5%LtoG|TB-^dZ z1%x-)>RC{@d+rOK@a@)>7-_p17uAP778H0KzYR&9j&!bF#9&g(8zOt~=y7LmXz0&` zXZn5mo)$6ctoeFH?)Hh(D3PMv%z92qpRArqR)uEtdC~Wrp-^ zNX6WWn}%No+-XrkRVGo5d)v&&2YSnxHyhlQMv8K&CG zs*Hf7&=Ai__`3}hX3|Yd)<1x(H6(+UwJqDRf}q@gT_~dT6&z>79F1U|w}bhK zlRInEGIZ puCszzd`5t7Yquw3Va!E_kIB&|QYGoQ5KeLgIpljU2m~mI{{m)B%s~JE diff --git a/charts/latest/azuredisk-csi-driver/values.yaml b/charts/latest/azuredisk-csi-driver/values.yaml index 4c0a096b37..ad82debcaa 100644 --- a/charts/latest/azuredisk-csi-driver/values.yaml +++ b/charts/latest/azuredisk-csi-driver/values.yaml @@ -2,7 +2,7 @@ image: baseRepo: mcr.microsoft.com azuredisk: repository: /k8s/csi/azuredisk-csi - tag: v1.9.0 + tag: latest pullPolicy: IfNotPresent csiProvisioner: repository: /oss/kubernetes-csi/csi-provisioner diff --git a/deploy/csi-azuredisk-controller.yaml b/deploy/csi-azuredisk-controller.yaml index aed775bbcd..e411ebe105 100644 --- a/deploy/csi-azuredisk-controller.yaml +++ b/deploy/csi-azuredisk-controller.yaml @@ -130,7 +130,7 @@ spec: cpu: 10m memory: 20Mi - name: azuredisk - image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:latest imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/csi-azuredisk-node-windows.yaml b/deploy/csi-azuredisk-node-windows.yaml index 43f451a2ca..cfc771ed8d 100644 --- a/deploy/csi-azuredisk-node-windows.yaml +++ b/deploy/csi-azuredisk-node-windows.yaml @@ -93,7 +93,7 @@ spec: cpu: 10m memory: 20Mi - name: azuredisk - image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:latest imagePullPolicy: IfNotPresent args: - "--v=5" diff --git a/deploy/csi-azuredisk-node.yaml b/deploy/csi-azuredisk-node.yaml index 0fbe737a91..eefd2d8cb5 100644 --- a/deploy/csi-azuredisk-node.yaml +++ b/deploy/csi-azuredisk-node.yaml @@ -84,7 +84,7 @@ spec: cpu: 10m memory: 20Mi - name: azuredisk - image: mcr.microsoft.com/k8s/csi/azuredisk-csi:v1.9.0 + image: mcr.microsoft.com/k8s/csi/azuredisk-csi:latest imagePullPolicy: IfNotPresent args: - "--v=5"