feat: Fix custom AMI bootstrap (#1580)

modules/node_groups/README.md

@@ -18,7 +18,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In
 | additional\_tags | Additional tags to apply to node group | map(string) | Only `var.tags` applied |
 | ami\_release\_version | AMI version of workers | string | Provider default behavior |
 | ami\_type | AMI Type. See Terraform or AWS docs | string | Provider default behavior |
-| ami\_id | ID of custom AMI. If you use a custom AMI, you need to supply bootstrap script via user-data or as AMI built-in. | string | Provider default behavior |
+| ami\_id | ID of custom AMI. If you use a custom AMI, you need to set `ami_is_eks_optimized` | string | Provider default behavior |
+| ami\_is\_eks\_optimized | If the custom AMI is an EKS optimised image, ignored if `ami_id` is not set. If this is `true` then `bootstrap.sh` is called automatically (max pod logic needs to be manually set), if this is `false` you need to provide all the node configuration in `pre_userdata` | bool | `true` |
 | capacity\_type | Type of instance capacity to provision. Options are `ON_DEMAND` and `SPOT` | string | Provider default behavior |
 | create_launch_template | Create and use a default launch template | bool |  `false` |
 | desired\_capacity | Desired number of workers | number | `var.workers_group_defaults[asg_desired_capacity]` |
@@ -83,6 +84,7 @@ No modules.
 |------|------|
 | [aws_eks_node_group.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource |
 | [aws_launch_template.workers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource |
+| [aws_eks_cluster.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source |
 | [cloudinit_config.workers_userdata](https://registry.terraform.io/providers/hashicorp/cloudinit/latest/docs/data-sources/config) | data source |
 
 ## Inputs

modules/node_groups/launch_template.tf

@@ -9,10 +9,15 @@ data "cloudinit_config" "workers_userdata" {
     content_type = "text/x-shellscript"
     content = templatefile("${path.module}/templates/userdata.sh.tpl",
       {
-        pre_userdata         = each.value["pre_userdata"]
         kubelet_extra_args   = each.value["kubelet_extra_args"]
+        pre_userdata         = each.value["pre_userdata"]
+        ami_id               = lookup(each.value, "ami_id", "")
+        ami_is_eks_optimized = each.value["ami_is_eks_optimized"]
         cluster_name         = var.cluster_name
-        run_bootstrap_script = lookup(each.value, "ami_id", null) != null
+        cluster_endpoint     = data.aws_eks_cluster.default.endpoint
+        cluster_ca           = data.aws_eks_cluster.default.certificate_authority[0].data
+        capacity_type        = lookup(each.value, "capacity_type", "ON_DEMAND")
+        append_labels        = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
       }
     )
   }

modules/node_groups/locals.tf

@@ -1,3 +1,7 @@
+data "aws_eks_cluster" "default" {
+  name = var.cluster_name
+}
+
 locals {
   # Merge defaults and per-group values to make code cleaner
   node_groups_expanded = { for k, v in var.node_groups : k => merge(
@@ -32,6 +36,7 @@ locals {
       metadata_http_endpoint               = var.workers_group_defaults["metadata_http_endpoint"]
       metadata_http_tokens                 = var.workers_group_defaults["metadata_http_tokens"]
       metadata_http_put_response_hop_limit = var.workers_group_defaults["metadata_http_put_response_hop_limit"]
+      ami_is_eks_optimized                 = true
     },
     var.node_groups_defaults,
     v,

modules/node_groups/templates/userdata.sh.tpl

@@ -1,9 +1,20 @@
 #!/bin/bash -e
+%{ if length(ami_id) == 0 ~}
 
-# Allow user supplied pre userdata code
+# Set variables directly into bootstrap.sh for default AMI
+sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh
+%{else ~}
+
+# Set variables for custom AMI
+API_SERVER_URL=${cluster_endpoint}
+B64_CLUSTER_CA=${cluster_ca}
+KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}'
+%{endif ~}
+
+# User supplied pre userdata
 ${pre_userdata}
+%{ if length(ami_id) > 0 && ami_is_eks_optimized ~}
 
-sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh
-%{ if run_bootstrap_script }
-    /etc/eks/bootstrap.sh ${cluster_name}
-%{ endif }
+# Call bootstrap for EKS optimised custom AMI
+/etc/eks/bootstrap.sh ${cluster_name} --apiserver-endpoint "$${API_SERVER_URL}" --b64-cluster-ca "$${B64_CLUSTER_CA}" --kubelet-extra-args "$${KUBELET_EXTRA_ARGS}"
+%{ endif ~}