Merge branch 'main' into feature/add-network-policy-example

.github/scripts/plan-examples.py

@@ -3,28 +3,28 @@
 import re
 
 
-def get_examples():
+def get_patterns():
     """
-    Get all Terraform example root directories using their respective `versions.tf`;
+    Get all pattern root directories using their respective `main.tf`;
     returning a string formatted json array of the example directories minus those that are excluded
     """
     exclude = {
-        'examples/appmesh-mtls',  # excluded until Rout53 is setup
-        'examples/blue-green-upgrade/environment',
-        'examples/blue-green-upgrade/modules/eks_cluster',
-        'examples/istio-multi-cluster/1.cluster1', # relies on remote state
-        'examples/istio-multi-cluster/2.cluster2', # relies on remote state
-        'examples/privatelink-access',
+        'patterns/appmesh-mtls',  # excluded until Rout53 is setup
+        'patterns/blue-green-upgrade/environment',
+        'patterns/blue-green-upgrade/modules/eks_cluster',
+        'patterns/istio-multi-cluster/1.cluster1', # relies on remote state
+        'patterns/istio-multi-cluster/2.cluster2', # relies on remote state
+        'patterns/privatelink-access',
     }
 
     projects = {
-        x.replace('/versions.tf', '')
-        for x in glob.glob('patterns/**/versions.tf', recursive=True)
+        x.replace('/main.tf', '')
+        for x in glob.glob('patterns/**/main.tf', recursive=True)
         if not re.match(r'^.+/_', x)
     }
 
     print(json.dumps(list(projects.difference(exclude))))
 
 
 if __name__ == '__main__':
-    get_examples()
+    get_patterns()

.github/workflows/e2e-parallel-destroy.yml

@@ -48,7 +48,7 @@ jobs:
         run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf
 
       - name: Auth AWS
-        uses: aws-actions/[email protected].0
+        uses: aws-actions/[email protected].1
         with:
           role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
           aws-region: us-west-2

.github/workflows/e2e-parallel-full.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
 
       - name: Auth AWS
-        uses: aws-actions/[email protected].0
+        uses: aws-actions/[email protected].1
         with:
           role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
           aws-region: us-west-2
@@ -81,7 +81,7 @@ jobs:
         run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf
 
       - name: Auth AWS
-        uses: aws-actions/[email protected].0
+        uses: aws-actions/[email protected].1
         with:
           role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
           aws-region: us-west-2
@@ -167,7 +167,7 @@ jobs:
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
 
       - name: Configure AWS credentials from Test account
-        uses: aws-actions/[email protected].0
+        uses: aws-actions/[email protected].1
         with:
           role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
           aws-region: us-west-2

.github/workflows/plan-examples.yml

@@ -88,7 +88,7 @@ jobs:
               - '*.tf'
 
       - name: Configure AWS credentials from Test account
-        uses: aws-actions/[email protected].0
+        uses: aws-actions/[email protected].1
         if: steps.changes.outputs.src== 'true'
         with:
           role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}

.github/workflows/scorecards.yml

@@ -41,7 +41,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@08b4669551908b1024bb425080c797723083c031 # v2.2.0
+        uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0
         with:
           results_file: results.sarif
           results_format: sarif
@@ -71,6 +71,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@ddccb873888234080b77e9bc2d4764d5ccaaccf9 # v2.21.9
+        uses: github/codeql-action/upload-sarif@fdcae64e1484d349b3366718cdfef3d404390e85 # v2.22.1
         with:
           sarif_file: results.sarif

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
       - id: detect-aws-credentials
         args: ['--allow-missing-credentials']
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.83.2
+    rev: v1.83.4
     hooks:
       - id: terraform_fmt
       - id: terraform_docs

patterns/elastic-fabric-adapter/main.tf

@@ -28,26 +28,14 @@ provider "helm" {
   }
 }
 
-provider "kubectl" {
-  apply_retry_count      = 5
-  host                   = module.eks.cluster_endpoint
-  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
-  load_config_file       = false
-
-  exec {
-    api_version = "client.authentication.k8s.io/v1beta1"
-    command     = "aws"
-    # This requires the awscli to be installed locally where Terraform is executed
-    args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]
-  }
-}
-
 data "aws_availability_zones" "available" {}
 
 locals {
   name   = basename(path.cwd)
   region = "us-west-2"
 
+  cluster_version = "1.27"
+
   vpc_cidr = "10.0.0.0/16"
   azs      = slice(data.aws_availability_zones.available.names, 0, 3)
 
@@ -66,7 +54,7 @@ module "eks" {
   version = "~> 19.16"
 
   cluster_name                   = local.name
-  cluster_version                = "1.27"
+  cluster_version                = local.cluster_version
   cluster_endpoint_public_access = true
 
   cluster_addons = {
@@ -98,6 +86,13 @@ module "eks" {
     }
   }
 
+  eks_managed_node_group_defaults = {
+    iam_role_additional_policies = {
+      # Not required, but used in the example to access the nodes to inspect drivers and devices
+      AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+    }
+  }
+
   eks_managed_node_groups = {
     # For running services that do not require GPUs
     default = {
@@ -113,7 +108,7 @@ module "eks" {
       instance_types = ["g5.8xlarge"]
 
       min_size     = 1
-      max_size     = 3
+      max_size     = 1
       desired_size = 1
 
       subnet_ids = slice(module.vpc.private_subnets, 0, 1)
@@ -133,14 +128,26 @@ module "eks" {
       }
 
       pre_bootstrap_user_data = <<-EOT
-        # Install EFA
-        curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz
-        tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer
-        ./efa_installer.sh -y --minimal
-        fi_info -p efa -t FI_EP_RDM
-
-        # Disable ptrace
-        sysctl -w kernel.yama.ptrace_scope=0
+        EFA_BIN='/opt/amazon/efa/bin/'
+
+        # EFA driver is installed by default on EKS GPU AMI starting on EKS 1.28
+        if [ ! -s "$EFA_BIN" ]; then
+
+          # Install EFA
+          # Note: It is recommended to install the EFA driver on a custom AMI and
+          # not rely on dynamic installation during instance provisioning in user data
+          curl -O https://efa-installer.amazonaws.com/aws-efa-installer-latest.tar.gz
+          tar -xf aws-efa-installer-latest.tar.gz && cd aws-efa-installer
+          ./efa_installer.sh -y --minimal
+          cd .. && rm -rf aws-efa-installer*
+
+          # Not required - just displays info on the EFA interfaces
+          $EFA_BIN/fi_info -p efa
+
+          # Disable ptrace
+          sysctl -w kernel.yama.ptrace_scope=0
+
+        fi
       EOT
 
       taints = {
@@ -211,8 +218,20 @@ module "eks_blueprints_addons" {
       repository       = "https://nvidia.github.io/gpu-operator"
       values = [
         <<-EOT
+          dcgmExporter:
+            enabled: false
+          driver:
+            enabled: false
+          toolkit:
+            version: v1.13.5-centos7
           operator:
             defaultRuntime: containerd
+          validator:
+            driver:
+              env:
+                # https://github.com/NVIDIA/gpu-operator/issues/569
+                - name: DISABLE_DEV_CHAR_SYMLINK_CREATION
+                  value: "true"
         EOT
       ]
     }
@@ -225,14 +244,96 @@ module "eks_blueprints_addons" {
 # Amazon Elastic Fabric Adapter (EFA)
 ################################################################################
 
-data "http" "efa_device_plugin_yaml" {
-  url = "https://raw.githubusercontent.com/aws-samples/aws-efa-eks/main/manifest/efa-k8s-device-plugin.yml"
-}
+resource "kubernetes_daemonset" "aws_efa_k8s_device_plugin" {
+  metadata {
+    name      = "aws-efa-k8s-device-plugin-daemonset"
+    namespace = "kube-system"
+  }
 
-resource "kubectl_manifest" "efa_device_plugin" {
-  yaml_body = <<-YAML
-    ${data.http.efa_device_plugin_yaml.response_body}
-  YAML
+  spec {
+    selector {
+      match_labels = {
+        name = "aws-efa-k8s-device-plugin"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          name = "aws-efa-k8s-device-plugin"
+        }
+      }
+
+      spec {
+        volume {
+          name = "device-plugin"
+
+          host_path {
+            path = "/var/lib/kubelet/device-plugins"
+          }
+        }
+
+        container {
+          name  = "aws-efa-k8s-device-plugin"
+          image = "602401143452.dkr.ecr.us-west-2.amazonaws.com/eks/aws-efa-k8s-device-plugin:v0.3.3"
+
+          volume_mount {
+            name       = "device-plugin"
+            mount_path = "/var/lib/kubelet/device-plugins"
+          }
+
+          image_pull_policy = "Always"
+
+          security_context {
+            capabilities {
+              drop = ["ALL"]
+            }
+          }
+        }
+
+        host_network = true
+
+        affinity {
+          node_affinity {
+            required_during_scheduling_ignored_during_execution {
+              node_selector_term {
+                match_expressions {
+                  key      = "beta.kubernetes.io/instance-type"
+                  operator = "In"
+                  values   = ["c5n.18xlarge", "c5n.9xlarge", "c5n.metal", "c6a.48xlarge", "c6a.metal", "c6gn.16xlarge", "c6i.32xlarge", "c6i.metal", "c6id.32xlarge", "c6id.metal", "c6in.32xlarge", "c6in.metal", "c7g.16xlarge", "c7g.metal", "c7gd.16xlarge", "c7gn.16xlarge", "c7i.48xlarge", "dl1.24xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.8xlarge", "g4dn.metal", "g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge", "g5.8xlarge", "hpc7g.16xlarge", "hpc7g.4xlarge", "hpc7g.8xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "i4g.16xlarge", "i4i.32xlarge", "i4i.metal", "im4gn.16xlarge", "inf1.24xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.24xlarge", "m5n.metal", "m5zn.12xlarge", "m5zn.metal", "m6a.48xlarge", "m6a.metal", "m6i.32xlarge", "m6i.metal", "m6id.32xlarge", "m6id.metal", "m6idn.32xlarge", "m6idn.metal", "m6in.32xlarge", "m6in.metal", "m7a.48xlarge", "m7a.metal-48xl", "m7g.16xlarge", "m7g.metal", "m7gd.16xlarge", "m7i.48xlarge", "p3dn.24xlarge", "p4d.24xlarge", "p5.48xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.24xlarge", "r5n.metal", "r6a.48xlarge", "r6a.metal", "r6i.32xlarge", "r6i.metal", "r6id.32xlarge", "r6id.metal", "r6idn.32xlarge", "r6idn.metal", "r6in.32xlarge", "r6in.metal", "r7a.48xlarge", "r7g.16xlarge", "r7g.metal", "r7gd.16xlarge", "r7iz.32xlarge", "trn1.32xlarge", "trn1n.32xlarge", "vt1.24xlarge", "x2idn.32xlarge", "x2idn.metal", "x2iedn.32xlarge", "x2iedn.metal", "x2iezn.12xlarge", "x2iezn.metal"]
+                }
+              }
+
+              node_selector_term {
+                match_expressions {
+                  key      = "node.kubernetes.io/instance-type"
+                  operator = "In"
+                  values   = ["c5n.18xlarge", "c5n.9xlarge", "c5n.metal", "c6a.48xlarge", "c6a.metal", "c6gn.16xlarge", "c6i.32xlarge", "c6i.metal", "c6id.32xlarge", "c6id.metal", "c6in.32xlarge", "c6in.metal", "c7g.16xlarge", "c7g.metal", "c7gd.16xlarge", "c7gn.16xlarge", "c7i.48xlarge", "dl1.24xlarge", "g4dn.12xlarge", "g4dn.16xlarge", "g4dn.8xlarge", "g4dn.metal", "g5.12xlarge", "g5.16xlarge", "g5.24xlarge", "g5.48xlarge", "g5.8xlarge", "hpc7g.16xlarge", "hpc7g.4xlarge", "hpc7g.8xlarge", "i3en.12xlarge", "i3en.24xlarge", "i3en.metal", "i4g.16xlarge", "i4i.32xlarge", "i4i.metal", "im4gn.16xlarge", "inf1.24xlarge", "m5dn.24xlarge", "m5dn.metal", "m5n.24xlarge", "m5n.metal", "m5zn.12xlarge", "m5zn.metal", "m6a.48xlarge", "m6a.metal", "m6i.32xlarge", "m6i.metal", "m6id.32xlarge", "m6id.metal", "m6idn.32xlarge", "m6idn.metal", "m6in.32xlarge", "m6in.metal", "m7a.48xlarge", "m7a.metal-48xl", "m7g.16xlarge", "m7g.metal", "m7gd.16xlarge", "m7i.48xlarge", "p3dn.24xlarge", "p4d.24xlarge", "p5.48xlarge", "r5dn.24xlarge", "r5dn.metal", "r5n.24xlarge", "r5n.metal", "r6a.48xlarge", "r6a.metal", "r6i.32xlarge", "r6i.metal", "r6id.32xlarge", "r6id.metal", "r6idn.32xlarge", "r6idn.metal", "r6in.32xlarge", "r6in.metal", "r7a.48xlarge", "r7g.16xlarge", "r7g.metal", "r7gd.16xlarge", "r7iz.32xlarge", "trn1.32xlarge", "trn1n.32xlarge", "vt1.24xlarge", "x2idn.32xlarge", "x2idn.metal", "x2iedn.32xlarge", "x2iedn.metal", "x2iezn.12xlarge", "x2iezn.metal"]
+                }
+              }
+            }
+          }
+        }
+
+        toleration {
+          key      = "CriticalAddonsOnly"
+          operator = "Exists"
+        }
+
+        toleration {
+          key      = "aws.amazon.com/efa"
+          operator = "Exists"
+          effect   = "NoSchedule"
+        }
+
+        priority_class_name = "system-node-critical"
+      }
+    }
+
+    strategy {
+      type = "RollingUpdate"
+    }
+  }
 }
 
 ################################################################################

patterns/elastic-fabric-adapter/versions.tf

@@ -14,14 +14,6 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = ">= 2.20"
     }
-    kubectl = {
-      source  = "gavinbunney/kubectl"
-      version = ">= 1.14"
-    }
-    http = {
-      source  = "hashicorp/http"
-      version = ">= 3.3"
-    }
   }
 
   # ##  Used for end-to-end testing on project; update to suit your needs