Merge pull request #5129 from consideRatio/pr/upgrade-addons

aws clusters: existing clusters to use eks managed addons and gp3 storage class with volume expansion by default
2i2c-org · Nov 18, 2024 · 7998cad · 7998cad
2 parents f777b56 + f684d07
commit 7998cad
Show file tree

Hide file tree

Showing 21 changed files with 768 additions and 255 deletions.
diff --git a/docs/howto/upgrade-cluster/aws.md b/docs/howto/upgrade-cluster/aws.md
@@ -101,26 +101,20 @@ now upgraded.
 
 #### 4.2. Upgrade EKS add-ons
 
-As documented in `eksctl`'s documentation[^1], we also need to upgrade three EKS
-add-ons managed by `eksctl` (by EKS these are considered self-managed add-ons),
-and one declared in our config (by EKS this is considered a managed add-on).
-
-These upgrades are believed to briefly disrupt networking.
+As documented in `eksctl`'s documentation[^2], we also need to upgrade EKS
+add-ons. This upgrade is believed to very briefly disrupt networking.
 
 ```bash
-# upgrade the kube-proxy daemonset (takes ~5s)
-eksctl utils update-kube-proxy --config-file=$CLUSTER_NAME.eksctl.yaml --approve
-
-# upgrade the aws-node daemonset (takes ~5s)
-eksctl utils update-aws-node --config-file=$CLUSTER_NAME.eksctl.yaml --approve
-
-# upgrade the coredns deployment (takes ~5s)
-eksctl utils update-coredns --config-file=$CLUSTER_NAME.eksctl.yaml --approve
-
-# upgrade the aws-ebs-csi-driver addon's deployment and daemonset (takes ~60s)
+# upgrade all EKS addons (takes up to a few minutes)
 eksctl update addon --config-file=$CLUSTER_NAME.eksctl.yaml
 ```
 
+```{note}
+Since November 2024, the add-ons are systematically installed as EKS managed
+add-ons and not self-managed as they were before when `eksctl` installed them
+without involving EKS.
+```
+
 #### 4.3. Repeat to upgrade multiple minor versions
 
 If you need to upgrade multiple minor versions, repeat the previous steps

diff --git a/docs/hub-deployment-guide/new-cluster/new-cluster.md b/docs/hub-deployment-guide/new-cluster/new-cluster.md
@@ -712,17 +712,3 @@ kubectl get node
 It should show you the provisioned node on the cluster if everything works out ok.
 ````
 `````
-
-## AWS only: Expandable storage class
-
-The default storage class that is created when we deploy a cluster to AWS does permit auto-expansion of persistent volumes.
-This can cause problems when we want to expand the size of a disk, say used by Prometheus to store metrics data.
-We will therefore patch the default storage class to permite auto-expansion.
-
-```bash
-# Gain k8s access to the cluster
-deployer use-cluster-credentials $CLUSTER_NAME
-
-# Patch the storage class
-kubectl patch storageclass gp2 --patch '{\"allowVolumeExpansion\": true}'
-```
diff --git a/eksctl/2i2c-aws-us.jsonnet b/eksctl/2i2c-aws-us.jsonnet
@@ -213,6 +213,10 @@ local daskNodes = [
         name: "2i2c-aws-us",
         region: clusterRegion,
         version: "1.30",
+        tags+: {
+            "ManagedBy": "2i2c",
+            "2i2c.org/cluster-name": $.metadata.name,
+        },
     },
     availabilityZones: masterAzs,
     iam: {
@@ -223,20 +227,48 @@ local daskNodes = [
     //    eksctl create addon --config-file=2i2c-aws-us.eksctl.yaml
     //
     addons: [
-        {
-            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
-            // couple to AWS EBS based storage, without it expect to see pods
-            // mounting a PVC failing to schedule and PVC resources that are
-            // unbound.
-            //
-            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
-            //
-            name: 'aws-ebs-csi-driver',
-            version: "latest",
-            wellKnownPolicies: {
-                ebsCSIController: true,
+        { version: "latest", tags: $.metadata.tags } + addon
+        for addon in
+        [
+            { name: "coredns" },
+            { name: "kube-proxy" },
+            {
+                // vpc-cni is a Amazon maintained container networking interface
+                // (CNI), where a CNI is required for k8s networking. The aws-node
+                // DaemonSet in kube-system stems from installing this.
+                //
+                // Related docs: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/
+                //               https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html
+                //
+                name: "vpc-cni",
+                attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"],
+                # FIXME: enabling network policy enforcement didn't work as of
+                #        August 2024, what's wrong isn't clear.
+                #
+                # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
+                configurationValues: |||
+                    enableNetworkPolicy: "false"
+                |||,
             },
-        },
+            {
+                // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+                // couple to AWS EBS based storage, without it expect to see pods
+                // mounting a PVC failing to schedule and PVC resources that are
+                // unbound.
+                //
+                // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+                //
+                name: "aws-ebs-csi-driver",
+                wellKnownPolicies: {
+                    ebsCSIController: true,
+                },
+                # configurationValues ref: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/HEAD/charts/aws-ebs-csi-driver/values.yaml
+                configurationValues: |||
+                    defaultStorageClass:
+                        enabled: true
+                |||,
+            },
+        ]
     ],
     nodeGroups: [
     n + {clusterName: $.metadata.name} for n in

diff --git a/eksctl/catalystproject-africa.jsonnet b/eksctl/catalystproject-africa.jsonnet
@@ -78,6 +78,10 @@ local daskNodes = [];
         name: "catalystproject-africa",
         region: clusterRegion,
         version: "1.30",
+        tags+: {
+            "ManagedBy": "2i2c",
+            "2i2c.org/cluster-name": $.metadata.name,
+        },
     },
     availabilityZones: masterAzs,
     iam: {
@@ -88,20 +92,48 @@ local daskNodes = [];
     //    eksctl create addon --config-file=catalystproject-africa.eksctl.yaml
     //
     addons: [
-        {
-            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
-            // couple to AWS EBS based storage, without it expect to see pods
-            // mounting a PVC failing to schedule and PVC resources that are
-            // unbound.
-            //
-            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
-            //
-            name: 'aws-ebs-csi-driver',
-            version: "latest",
-            wellKnownPolicies: {
-                ebsCSIController: true,
+        { version: "latest", tags: $.metadata.tags } + addon
+        for addon in
+        [
+            { name: "coredns" },
+            { name: "kube-proxy" },
+            {
+                // vpc-cni is a Amazon maintained container networking interface
+                // (CNI), where a CNI is required for k8s networking. The aws-node
+                // DaemonSet in kube-system stems from installing this.
+                //
+                // Related docs: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/
+                //               https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html
+                //
+                name: "vpc-cni",
+                attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"],
+                # FIXME: enabling network policy enforcement didn't work as of
+                #        August 2024, what's wrong isn't clear.
+                #
+                # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
+                configurationValues: |||
+                    enableNetworkPolicy: "false"
+                |||,
             },
-        },
+            {
+                // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+                // couple to AWS EBS based storage, without it expect to see pods
+                // mounting a PVC failing to schedule and PVC resources that are
+                // unbound.
+                //
+                // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+                //
+                name: "aws-ebs-csi-driver",
+                wellKnownPolicies: {
+                    ebsCSIController: true,
+                },
+                # configurationValues ref: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/HEAD/charts/aws-ebs-csi-driver/values.yaml
+                configurationValues: |||
+                    defaultStorageClass:
+                        enabled: true
+                |||,
+            },
+        ]
     ],
     nodeGroups: [
     n + {clusterName: $.metadata.name} for n in

diff --git a/eksctl/earthscope.jsonnet b/eksctl/earthscope.jsonnet
@@ -117,6 +117,7 @@ local daskNodes = [
         version: "1.30",
         tags+: {
             "ManagedBy": "2i2c",
+            "2i2c.org/cluster-name": $.metadata.name,
             "earthscope:application:name": "geolab",
             "earthscope:application:owner": "research-onramp-to-the-cloud"
         },
@@ -130,20 +131,48 @@ local daskNodes = [
     //    eksctl create addon --config-file=earthscope.eksctl.yaml
     //
     addons: [
-        {
-            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
-            // couple to AWS EBS based storage, without it expect to see pods
-            // mounting a PVC failing to schedule and PVC resources that are
-            // unbound.
-            //
-            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
-            //
-            name: 'aws-ebs-csi-driver',
-            version: "latest",
-            wellKnownPolicies: {
-                ebsCSIController: true,
+        { version: "latest", tags: $.metadata.tags } + addon
+        for addon in
+        [
+            { name: "coredns" },
+            { name: "kube-proxy" },
+            {
+                // vpc-cni is a Amazon maintained container networking interface
+                // (CNI), where a CNI is required for k8s networking. The aws-node
+                // DaemonSet in kube-system stems from installing this.
+                //
+                // Related docs: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/
+                //               https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html
+                //
+                name: "vpc-cni",
+                attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"],
+                # FIXME: enabling network policy enforcement didn't work as of
+                #        August 2024, what's wrong isn't clear.
+                #
+                # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
+                configurationValues: |||
+                    enableNetworkPolicy: "false"
+                |||,
             },
-        },
+            {
+                // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+                // couple to AWS EBS based storage, without it expect to see pods
+                // mounting a PVC failing to schedule and PVC resources that are
+                // unbound.
+                //
+                // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+                //
+                name: "aws-ebs-csi-driver",
+                wellKnownPolicies: {
+                    ebsCSIController: true,
+                },
+                # configurationValues ref: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/HEAD/charts/aws-ebs-csi-driver/values.yaml
+                configurationValues: |||
+                    defaultStorageClass:
+                        enabled: true
+                |||,
+            },
+        ]
     ],
     nodeGroups: [
     n + {clusterName: $.metadata.name} for n in

diff --git a/eksctl/gridsst.jsonnet b/eksctl/gridsst.jsonnet
@@ -74,6 +74,10 @@ local daskNodes = [
         name: "gridsst",
         region: clusterRegion,
         version: "1.29",
+        tags+: {
+            "ManagedBy": "2i2c",
+            "2i2c.org/cluster-name": $.metadata.name,
+        },
     },
     availabilityZones: masterAzs,
     iam: {
@@ -84,20 +88,48 @@ local daskNodes = [
     //    eksctl create addon --config-file=gridsst.eksctl.yaml
     //
     addons: [
-        {
-            // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
-            // couple to AWS EBS based storage, without it expect to see pods
-            // mounting a PVC failing to schedule and PVC resources that are
-            // unbound.
-            //
-            // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
-            //
-            name: 'aws-ebs-csi-driver',
-            version: "latest",
-            wellKnownPolicies: {
-                ebsCSIController: true,
+        { version: "latest", tags: $.metadata.tags } + addon
+        for addon in
+        [
+            { name: "coredns" },
+            { name: "kube-proxy" },
+            {
+                // vpc-cni is a Amazon maintained container networking interface
+                // (CNI), where a CNI is required for k8s networking. The aws-node
+                // DaemonSet in kube-system stems from installing this.
+                //
+                // Related docs: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/
+                //               https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html
+                //
+                name: "vpc-cni",
+                attachPolicyARNs: ["arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"],
+                # FIXME: enabling network policy enforcement didn't work as of
+                #        August 2024, what's wrong isn't clear.
+                #
+                # configurationValues ref: https://github.com/aws/amazon-vpc-cni-k8s/blob/HEAD/charts/aws-vpc-cni/values.yaml
+                configurationValues: |||
+                    enableNetworkPolicy: "false"
+                |||,
             },
-        },
+            {
+                // aws-ebs-csi-driver ensures that our PVCs are bound to PVs that
+                // couple to AWS EBS based storage, without it expect to see pods
+                // mounting a PVC failing to schedule and PVC resources that are
+                // unbound.
+                //
+                // Related docs: https://docs.aws.amazon.com/eks/latest/userguide/managing-ebs-csi.html
+                //
+                name: "aws-ebs-csi-driver",
+                wellKnownPolicies: {
+                    ebsCSIController: true,
+                },
+                # configurationValues ref: https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/HEAD/charts/aws-ebs-csi-driver/values.yaml
+                configurationValues: |||
+                    defaultStorageClass:
+                        enabled: true
+                |||,
+            },
+        ]
     ],
     nodeGroups: [
     n + {clusterName: $.metadata.name} for n in