diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index 1474da02dc..98936d6a9b 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -37,7 +37,8 @@ The role ARN specified in `var.default_iam_role_arn` will be used by default. In | instance\_types | Node group's instance type(s). Multiple types can be specified when `capacity_type="SPOT"`. | list | `[var.workers_group_defaults[instance_type]]` | | k8s\_labels | Kubernetes labels | map(string) | No labels applied | | key\_name | Key name for workers. Set to empty string to disable remote access | string | `var.workers_group_defaults[key_name]` | -| kubelet_extra_args | This string is passed directly to kubelet if set. Useful for adding labels or taints. Require `create_launch_template` to be `true`| string | "" | +| bootstrap_env | Provide environment variables to customise [bootstrap.sh](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh). Require `create_launch_template` to be `true` | map(string) | `{}` | +| kubelet_extra_args | Extra arguments for kubelet, this is automatically merged with `labels`. Require `create_launch_template` to be `true` | string | "" | | launch_template_id | The id of a aws_launch_template to use | string | No LT used | | launch\_template_version | The version of the LT to use | string | none | | max\_capacity | Max number of workers | number | `var.workers_group_defaults[asg_max_size]` | diff --git a/modules/node_groups/launch_template.tf b/modules/node_groups/launch_template.tf index fcf65b7c9a..6abe358d5a 100644 --- a/modules/node_groups/launch_template.tf +++ b/modules/node_groups/launch_template.tf @@ -9,13 +9,14 @@ data "cloudinit_config" "workers_userdata" { content_type = "text/x-shellscript" content = templatefile("${path.module}/templates/userdata.sh.tpl", { - kubelet_extra_args = each.value["kubelet_extra_args"] - pre_userdata = each.value["pre_userdata"] - ami_id = lookup(each.value, "ami_id", "") - ami_is_eks_optimized = each.value["ami_is_eks_optimized"] cluster_name = var.cluster_name cluster_endpoint = var.cluster_endpoint cluster_auth_base64 = var.cluster_auth_base64 + ami_id = lookup(each.value, "ami_id", "") + ami_is_eks_optimized = each.value["ami_is_eks_optimized"] + bootstrap_env = each.value["bootstrap_env"] + kubelet_extra_args = each.value["kubelet_extra_args"] + pre_userdata = each.value["pre_userdata"] capacity_type = lookup(each.value, "capacity_type", "ON_DEMAND") append_labels = length(lookup(each.value, "k8s_labels", {})) > 0 ? ",${join(",", [for k, v in lookup(each.value, "k8s_labels", {}) : "${k}=${v}"])}" : "" } diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 0b211721e3..0a6c7cbffb 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -13,6 +13,7 @@ locals { min_capacity = var.workers_group_defaults["asg_min_size"] subnets = var.workers_group_defaults["subnets"] create_launch_template = false + bootstrap_env = {} kubelet_extra_args = var.workers_group_defaults["kubelet_extra_args"] disk_size = var.workers_group_defaults["root_volume_size"] disk_type = var.workers_group_defaults["root_volume_type"] diff --git a/modules/node_groups/templates/userdata.sh.tpl b/modules/node_groups/templates/userdata.sh.tpl index d9a590bcc8..321c17b427 100644 --- a/modules/node_groups/templates/userdata.sh.tpl +++ b/modules/node_groups/templates/userdata.sh.tpl @@ -1,13 +1,27 @@ #!/bin/bash -e %{ if length(ami_id) == 0 ~} -# Set variables directly into bootstrap.sh for default AMI -sed -i '/^KUBELET_EXTRA_ARGS=/a KUBELET_EXTRA_ARGS+=" ${kubelet_extra_args}"' /etc/eks/bootstrap.sh +# Set bootstrap env +printf '#!/bin/bash +%{ for k, v in bootstrap_env ~} +export ${k}="${v}" +%{ endfor ~} +export ADDITIONAL_KUBELET_EXTRA_ARGS="${kubelet_extra_args}" +' > /etc/profile.d/eks-bootstrap-env.sh + +# Source extra environment variables in bootstrap script +sed -i '/^set -o errexit/a\\nsource /etc/profile.d/eks-bootstrap-env.sh' /etc/eks/bootstrap.sh + +# Merge ADDITIONAL_KUBELET_EXTRA_ARGS into KUBELET_EXTRA_ARGS +sed -i 's/^KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-}/KUBELET_EXTRA_ARGS="$${KUBELET_EXTRA_ARGS:-} $${ADDITIONAL_KUBELET_EXTRA_ARGS}/' /etc/eks/bootstrap.sh %{else ~} # Set variables for custom AMI API_SERVER_URL=${cluster_endpoint} B64_CLUSTER_CA=${cluster_auth_base64} +%{ for k, v in bootstrap_env ~} +${k}="${v}" +%{ endfor ~} KUBELET_EXTRA_ARGS='--node-labels=eks.amazonaws.com/nodegroup-image=${ami_id},eks.amazonaws.com/capacityType=${capacity_type}${append_labels} ${kubelet_extra_args}' %{endif ~}