fix: Correct DNS Server of kubelet service when using custom AMI that…

… is EKS optimized and has a custom ipv4 CIDR

The PR (terraform-aws-modules#1580) is passing the "apiserver-endpoint" and "b64-cluster-ca", which causes the SERVICE_IPV4_CIDR empty (https://github.com/awslabs/amazon-eks-ami/blob/v20211206/files/bootstrap.sh#L366). Because of that, the script fallbacks always to 10.100.0.10 or 172.20.0.10.

Defining the ipv4 cidr ensures that the bootstrap script configures the DNS server correctly on the kubelet service, allowing pods to resolve DNS names.

locals.tf

@@ -6,6 +6,7 @@ locals {
   cluster_name                      = coalescelist(aws_eks_cluster.this[*].name, [""])[0]
   cluster_endpoint                  = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0]
   cluster_auth_base64               = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0]
+  cluster_service_ipv4_cidr         = coalescelist(aws_eks_cluster.this[*].kubernetes_network_config[0].service_ipv4_cidr, [""])[0]
   cluster_oidc_issuer_url           = flatten(concat(aws_eks_cluster.this[*].identity[*].oidc[0].issuer, [""]))[0]
   cluster_primary_security_group_id = coalescelist(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, [""])[0]
 

modules/node_groups/README.md

@@ -94,6 +94,7 @@ No modules.
 | <a name="input_cluster_auth_base64"></a> [cluster\_auth\_base64](#input\_cluster\_auth\_base64) | Base64 encoded CA of parent cluster | `string` | `""` | no |
 | <a name="input_cluster_endpoint"></a> [cluster\_endpoint](#input\_cluster\_endpoint) | Endpoint of parent cluster | `string` | `""` | no |
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Name of parent cluster | `string` | `""` | no |
+| <a name="input_cluster_service_ipv4_cidr"></a> [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | service ipv4 cidr for the kubernetes cluster | `string` | `null` | no |
 | <a name="input_create_eks"></a> [create\_eks](#input\_create\_eks) | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no |
 | <a name="input_default_iam_role_arn"></a> [default\_iam\_role\_arn](#input\_default\_iam\_role\_arn) | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | `""` | no |
 | <a name="input_ebs_optimized_not_supported"></a> [ebs\_optimized\_not\_supported](#input\_ebs\_optimized\_not\_supported) | List of instance types that do not support EBS optimization | `list(string)` | `[]` | no |

modules/node_groups/launch_template.tf

@@ -9,16 +9,17 @@ data "cloudinit_config" "workers_userdata" {
     content_type = "text/x-shellscript"
     content = templatefile("${path.module}/templates/userdata.sh.tpl",
       {
-        cluster_name         = var.cluster_name
-        cluster_endpoint     = var.cluster_endpoint
-        cluster_auth_base64  = var.cluster_auth_base64
-        ami_id               = lookup(each.value, "ami_id", "")
-        ami_is_eks_optimized = each.value["ami_is_eks_optimized"]
-        bootstrap_env        = each.value["bootstrap_env"]
-        kubelet_extra_args   = each.value["kubelet_extra_args"]
-        pre_userdata         = each.value["pre_userdata"]
-        capacity_type        = lookup(each.value, "capacity_type", "ON_DEMAND")
-        append_labels        = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
+        cluster_name              = var.cluster_name
+        cluster_endpoint          = var.cluster_endpoint
+        cluster_auth_base64       = var.cluster_auth_base64
+        cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr
+        ami_id                    = lookup(each.value, "ami_id", "")
+        ami_is_eks_optimized      = each.value["ami_is_eks_optimized"]
+        bootstrap_env             = each.value["bootstrap_env"]
+        kubelet_extra_args        = each.value["kubelet_extra_args"]
+        pre_userdata              = each.value["pre_userdata"]
+        capacity_type             = lookup(each.value, "capacity_type", "ON_DEMAND")
+        append_labels             = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : ""
       }
     )
   }

modules/node_groups/templates/userdata.sh.tpl

@@ -29,6 +29,10 @@ KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},ek
 ${pre_userdata}
 %{ if length(ami_id) > 0 && ami_is_eks_optimized ~}
 
+# The bootstrap.sh script doesn't allow to pass service ipv4 cidr as an argument.
+# Therefore the environment variable SERVICE_IPV4_CIDR is exported to be used by the script.
+export SERVICE_IPV4_CIDR=${cluster_service_ipv4_cidr}
+
 # Call bootstrap for EKS optimised custom AMI
 /etc/eks/bootstrap.sh ${cluster_name} --apiserver-endpoint "$${API_SERVER_URL}" --b64-cluster-ca "$${B64_CLUSTER_CA}" --kubelet-extra-args "$${KUBELET_EXTRA_ARGS}"
 %{ endif ~}

modules/node_groups/variables.tf

@@ -22,6 +22,12 @@ variable "cluster_auth_base64" {
   default     = ""
 }
 
+variable "cluster_service_ipv4_cidr" {
+  description = "service ipv4 cidr for the kubernetes cluster"
+  type        = string
+  default     = null
+}
+
 variable "default_iam_role_arn" {
   description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`"
   type        = string

node_groups.tf

@@ -3,9 +3,10 @@ module "node_groups" {
 
   create_eks = var.create_eks
 
-  cluster_name        = local.cluster_name
-  cluster_endpoint    = local.cluster_endpoint
-  cluster_auth_base64 = local.cluster_auth_base64
+  cluster_name              = local.cluster_name
+  cluster_endpoint          = local.cluster_endpoint
+  cluster_auth_base64       = local.cluster_auth_base64
+  cluster_service_ipv4_cidr = local.cluster_service_ipv4_cidr
 
   default_iam_role_arn                 = coalescelist(aws_iam_role.workers[*].arn, [""])[0]
   ebs_optimized_not_supported          = local.ebs_optimized_not_supported