From 87edcca3c2a9905ae41500540271e5189accb49d Mon Sep 17 00:00:00 2001 From: Grzegorz Lisowski Date: Sun, 22 Mar 2020 00:32:32 +0100 Subject: [PATCH] - Worker locals/defaults moved to workers submodule - Create separate defaults for node groups - Workers IAM management left outside of module as both node_group and worker_groups uses them --- README.md | 23 +- aws_auth.tf | 42 +- cluster.tf | 2 +- data.tf | 137 ----- docs/faq.md | 12 +- docs/spot-instances.md | 48 +- examples/basic/main.tf | 10 +- examples/irsa/main.tf | 12 +- examples/launch_templates/main.tf | 21 +- examples/launch_templates/outputs.tf | 5 + examples/secrets_encryption/main.tf | 7 +- examples/spot_instances/main.tf | 7 +- local.tf | 132 +---- modules/node_groups/README.md | 1 + modules/node_groups/locals.tf | 26 +- modules/node_groups/variables.tf | 5 + modules/worker_groups/README.md | 73 +++ modules/worker_groups/data.tf | 60 +++ modules/worker_groups/local.tf | 138 +++++ modules/worker_groups/main.tf | 265 ++++++++++ modules/worker_groups/outputs.tf | 19 + modules/worker_groups/random.tf | 19 + modules/worker_groups/sg.tf | 91 ++++ .../worker_groups/templates}/userdata.sh.tpl | 0 .../templates}/userdata_windows.tpl | 0 modules/worker_groups/variables.tf | 146 ++++++ node_groups.tf | 3 +- outputs.tf | 87 +--- variables.tf | 10 +- worker_groups.tf | 50 ++ workers.tf | 424 --------------- workers_iam.tf | 34 ++ workers_launch_template.tf | 490 ------------------ 33 files changed, 984 insertions(+), 1415 deletions(-) create mode 100644 modules/worker_groups/README.md create mode 100644 modules/worker_groups/data.tf create mode 100644 modules/worker_groups/local.tf create mode 100644 modules/worker_groups/main.tf create mode 100644 modules/worker_groups/outputs.tf create mode 100644 modules/worker_groups/random.tf create mode 100644 modules/worker_groups/sg.tf rename {templates => modules/worker_groups/templates}/userdata.sh.tpl (100%) rename {templates => modules/worker_groups/templates}/userdata_windows.tpl (100%) create mode 100644 modules/worker_groups/variables.tf create mode 100644 worker_groups.tf delete mode 100644 workers.tf create mode 100644 workers_iam.tf delete mode 100644 workers_launch_template.tf diff --git a/README.md b/README.md index 90b5eb44b2d..b9e54ba427e 100644 --- a/README.md +++ b/README.md @@ -49,12 +49,12 @@ module "my-cluster" { subnets = ["subnet-abcde012", "subnet-bcde012a", "subnet-fghi345a"] vpc_id = "vpc-1234556abcdef" - worker_groups = [ - { + worker_groups = { + group = { instance_type = "m4.large" asg_max_size = 5 } - ] + } } ``` ## Conditional creation @@ -150,8 +150,6 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | kubernetes | >= 1.11.1 | | local | >= 1.4 | | null | >= 2.1 | -| random | >= 2.1 | -| template | >= 2.1 | ## Inputs @@ -205,8 +203,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | worker\_create\_cluster\_primary\_security\_group\_rules | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | -| worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | -| worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `[]` | no | +| worker\_groups | A map of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `{}` | no | | worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | | worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | | workers\_additional\_policies | Additional policies to be added to workers | `list(string)` | `[]` | no | @@ -235,17 +232,7 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a | node\_groups | Outputs from EKS node groups. Map of maps, keyed by var.node\_groups keys | | oidc\_provider\_arn | The ARN of the OIDC Provider if `enable_irsa = true`. | | security\_group\_rule\_cluster\_https\_worker\_ingress | Security group rule responsible for allowing pods to communicate with the EKS cluster API. | -| worker\_iam\_instance\_profile\_arns | default IAM instance profile ARN for EKS worker groups | -| worker\_iam\_instance\_profile\_names | default IAM instance profile name for EKS worker groups | -| worker\_iam\_role\_arn | default IAM role ARN for EKS worker groups | -| worker\_iam\_role\_name | default IAM role name for EKS worker groups | +| worker\_groups | Outputs from EKS worker groups. Map of maps, keyed by var.worker\_groups keys | | worker\_security\_group\_id | Security group ID attached to the EKS workers. | -| workers\_asg\_arns | IDs of the autoscaling groups containing workers. | -| workers\_asg\_names | Names of the autoscaling groups containing workers. | -| workers\_default\_ami\_id | ID of the default worker group AMI | -| workers\_launch\_template\_arns | ARNs of the worker launch templates. | -| workers\_launch\_template\_ids | IDs of the worker launch templates. | -| workers\_launch\_template\_latest\_versions | Latest versions of the worker launch templates. | -| workers\_user\_data | User data of worker groups | diff --git a/aws_auth.tf b/aws_auth.tf index b583c069a9c..958f9dc65f1 100644 --- a/aws_auth.tf +++ b/aws_auth.tf @@ -1,48 +1,10 @@ -data "aws_caller_identity" "current" { -} +data "aws_caller_identity" "current" {} locals { - auth_launch_template_worker_roles = [ - for index in range(0, var.create_eks ? local.worker_group_launch_template_count : 0) : { - worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers_launch_template.*.role, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, - [""] - ), - index - )}" - platform = lookup( - var.worker_groups_launch_template[index], - "platform", - local.workers_group_defaults["platform"] - ) - } - ] - - auth_worker_roles = [ - for index in range(0, var.create_eks ? local.worker_group_count : 0) : { - worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${element( - coalescelist( - aws_iam_instance_profile.workers.*.role, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, - [""] - ), - index, - )}" - platform = lookup( - var.worker_groups[index], - "platform", - local.workers_group_defaults["platform"] - ) - } - ] - # Convert to format needed by aws-auth ConfigMap configmap_roles = [ for role in concat( - local.auth_launch_template_worker_roles, - local.auth_worker_roles, + module.worker_groups.aws_auth_roles, module.node_groups.aws_auth_roles, ) : { diff --git a/cluster.tf b/cluster.tf index 1a804702d0c..82a334f31f5 100644 --- a/cluster.tf +++ b/cluster.tf @@ -105,7 +105,7 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id - source_security_group_id = local.worker_security_group_id + source_security_group_id = module.worker_groups.worker_security_group_id from_port = 443 to_port = 443 type = "ingress" diff --git a/data.tf b/data.tf index 59099360312..83e5aa1235c 100644 --- a/data.tf +++ b/data.tf @@ -13,33 +13,6 @@ data "aws_iam_policy_document" "workers_assume_role_policy" { } } -data "aws_ami" "eks_worker" { - filter { - name = "name" - values = [local.worker_ami_name_filter] - } - - most_recent = true - - owners = [var.worker_ami_owner_id] -} - -data "aws_ami" "eks_worker_windows" { - filter { - name = "name" - values = [local.worker_ami_name_filter_windows] - } - - filter { - name = "platform" - values = ["windows"] - } - - most_recent = true - - owners = [var.worker_ami_owner_id_windows] -} - data "aws_iam_policy_document" "cluster_assume_role_policy" { statement { sid = "EKSClusterAssumeRole" @@ -55,119 +28,9 @@ data "aws_iam_policy_document" "cluster_assume_role_policy" { } } -data "template_file" "userdata" { - count = var.create_eks ? local.worker_group_count : 0 - template = lookup( - var.worker_groups[count.index], - "userdata_template_file", - file( - lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ) - ) - - vars = merge({ - platform = lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] - pre_userdata = lookup( - var.worker_groups[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups[count.index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) -} - -data "template_file" "launch_template_userdata" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - template = lookup( - var.worker_groups_launch_template[count.index], - "userdata_template_file", - file( - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" - ? "${path.module}/templates/userdata_windows.tpl" - : "${path.module}/templates/userdata.sh.tpl" - ) - ) - - vars = merge({ - platform = lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) - cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] - endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = coalescelist(aws_eks_cluster.this[*].certificate_authority[0].data, [""])[0] - pre_userdata = lookup( - var.worker_groups_launch_template[count.index], - "pre_userdata", - local.workers_group_defaults["pre_userdata"], - ) - additional_userdata = lookup( - var.worker_groups_launch_template[count.index], - "additional_userdata", - local.workers_group_defaults["additional_userdata"], - ) - bootstrap_extra_args = lookup( - var.worker_groups_launch_template[count.index], - "bootstrap_extra_args", - local.workers_group_defaults["bootstrap_extra_args"], - ) - kubelet_extra_args = lookup( - var.worker_groups_launch_template[count.index], - "kubelet_extra_args", - local.workers_group_defaults["kubelet_extra_args"], - ) - }, - lookup( - var.worker_groups_launch_template[count.index], - "userdata_template_extra_args", - local.workers_group_defaults["userdata_template_extra_args"] - ) - ) -} - data "aws_iam_role" "custom_cluster_iam_role" { count = var.manage_cluster_iam_resources ? 0 : 1 name = var.cluster_iam_role_name } -data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { - count = var.manage_worker_iam_resources ? 0 : local.worker_group_count - name = lookup( - var.worker_groups[count.index], - "iam_instance_profile_name", - local.workers_group_defaults["iam_instance_profile_name"], - ) -} - -data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" { - count = var.manage_worker_iam_resources ? 0 : local.worker_group_launch_template_count - name = lookup( - var.worker_groups_launch_template[count.index], - "iam_instance_profile_name", - local.workers_group_defaults["iam_instance_profile_name"], - ) -} - data "aws_partition" "current" {} diff --git a/docs/faq.md b/docs/faq.md index 043851d6110..ce724d7e702 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -2,7 +2,7 @@ ## How do I customize X on the worker group's settings? -All the options that can be customized for worker groups are listed in [local.tf](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/local.tf) under `workers_group_defaults_defaults`. +All the options that can be customized for worker groups are listed in [local.tf](https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/modules/worker_groups/local.tf) under `workers_group_defaults_defaults`. Please open Issues or PRs if you think something is missing. @@ -61,12 +61,6 @@ You need to add the tags to the VPC and subnets yourself. See the [basic example An alternative is to use the aws provider's [`ignore_tags` variable](https://www.terraform.io/docs/providers/aws/#ignore\_tags-configuration-block). However this can also cause terraform to display a perpetual difference. -## How do I safely remove old worker groups? - -You've added new worker groups. Deleting worker groups from earlier in the list causes Terraform to want to recreate all worker groups. This is a limitation with how Terraform works and the module using `count` to create the ASGs and other resources. - -The safest and easiest option is to set `asg_min_size` and `asg_max_size` to 0 on the worker groups to "remove". - ## Why does changing the worker group's desired count not do anything? The module is configured to ignore this value. Unfortunately Terraform does not support variables within the `lifecycle` block. @@ -77,9 +71,9 @@ You can change the desired count via the CLI or console if you're not using the If you are not using autoscaling and really want to control the number of nodes via terraform then set the `asg_min_size` and `asg_max_size` instead. AWS will remove a random instance when you scale down. You will have to weigh the risks here. -## Why are nodes not recreated when the `launch_configuration`/`launch_template` is recreated? +## Why are nodes not recreated when the `launch_configuration` is recreated? -By default the ASG is not configured to be recreated when the launch configuration or template changes. Terraform spins up new instances and then deletes all the old instances in one go as the AWS provider team have refused to implement rolling updates of autoscaling groups. This is not good for kubernetes stability. +By default the ASG is not configured to be recreated when the launch configuration changes. Terraform spins up new instances and then deletes all the old instances in one go as the AWS provider team have refused to implement rolling updates of autoscaling groups. This is not good for kubernetes stability. You need to use a process to drain and cycle the workers. diff --git a/docs/spot-instances.md b/docs/spot-instances.md index 1eb4da4257f..f6d7c7440b3 100644 --- a/docs/spot-instances.md +++ b/docs/spot-instances.md @@ -22,57 +22,19 @@ Notes: - There is an AWS blog article about this [here](https://aws.amazon.com/blogs/compute/run-your-kubernetes-workloads-on-amazon-ec2-spot-instances-with-amazon-eks/). - Consider using [k8s-spot-rescheduler](https://github.com/pusher/k8s-spot-rescheduler) to move pods from on-demand to spot instances. -## Using Launch Configuration - -Example worker group configuration that uses an ASG with launch configuration for each worker group: - -```hcl - worker_groups = [ - { - name = "on-demand-1" - instance_type = "m4.xlarge" - asg_max_size = 1 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=normal" - suspended_processes = ["AZRebalance"] - }, - { - name = "spot-1" - spot_price = "0.199" - instance_type = "c4.xlarge" - asg_max_size = 20 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - suspended_processes = ["AZRebalance"] - }, - { - name = "spot-2" - spot_price = "0.20" - instance_type = "m4.xlarge" - asg_max_size = 20 - kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" - suspended_processes = ["AZRebalance"] - } - ] -``` - ## Using Launch Templates Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested but it's more suitable for spot instances as it allowed multiple instance types in the same worker group: ```hcl - worker_groups = [ - { - name = "on-demand-1" + worker_groups = { + on-demand-1 = { instance_type = "m4.xlarge" asg_max_size = 10 kubelet_extra_args = "--node-labels=spot=false" suspended_processes = ["AZRebalance"] - } - ] - - - worker_groups_launch_template = [ - { - name = "spot-1" + }, + spot-1 = { override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] spot_instance_pools = 4 asg_max_size = 5 @@ -80,7 +42,7 @@ Launch Template support is a recent addition to both AWS and this module. It mig kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" public_ip = true }, - ] + } ``` ## Important Notes diff --git a/examples/basic/main.tf b/examples/basic/main.tf index 4ce49b95b93..8b1b45337c9 100644 --- a/examples/basic/main.tf +++ b/examples/basic/main.tf @@ -135,22 +135,20 @@ module "eks" { vpc_id = module.vpc.vpc_id - worker_groups = [ - { - name = "worker-group-1" + worker_groups = { + worker-group-1 = { instance_type = "t2.small" additional_userdata = "echo foo bar" asg_desired_capacity = 2 additional_security_group_ids = [aws_security_group.worker_group_mgmt_one.id] }, - { - name = "worker-group-2" + worker-group-2 = { instance_type = "t2.medium" additional_userdata = "echo foo bar" additional_security_group_ids = [aws_security_group.worker_group_mgmt_two.id] asg_desired_capacity = 1 }, - ] + } worker_additional_security_group_ids = [aws_security_group.all_worker_mgmt.id] map_roles = var.map_roles diff --git a/examples/irsa/main.tf b/examples/irsa/main.tf index 849db9c28f1..3a02eff313d 100644 --- a/examples/irsa/main.tf +++ b/examples/irsa/main.tf @@ -48,6 +48,10 @@ module "vpc" { public_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] enable_dns_hostnames = true + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + public_subnet_tags = { "kubernetes.io/cluster/${local.cluster_name}" = "shared" "kubernetes.io/role/elb" = "1" @@ -61,11 +65,11 @@ module "eks" { vpc_id = module.vpc.vpc_id enable_irsa = true - worker_groups = [ - { - name = "worker-group-1" + worker_groups = { + worker-group-1 = { instance_type = "t2.medium" asg_desired_capacity = 1 + tags = [ { "key" = "k8s.io/cluster-autoscaler/enabled" @@ -79,5 +83,5 @@ module "eks" { } ] } - ] + } } diff --git a/examples/launch_templates/main.tf b/examples/launch_templates/main.tf index d76a13c561b..e72a88d550b 100644 --- a/examples/launch_templates/main.tf +++ b/examples/launch_templates/main.tf @@ -39,8 +39,7 @@ provider "kubernetes" { version = "~> 1.11" } -data "aws_availability_zones" "available" { -} +data "aws_availability_zones" "available" {} locals { cluster_name = "test-eks-lt-${random_string.suffix.result}" @@ -60,6 +59,14 @@ module "vpc" { azs = data.aws_availability_zones.available.names public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] enable_dns_hostnames = true + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + } } module "eks" { @@ -68,18 +75,16 @@ module "eks" { subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id - worker_groups_launch_template = [ - { - name = "worker-group-1" + worker_groups = { + worker-group-1 = { instance_type = "t2.small" asg_desired_capacity = 2 public_ip = true }, - { - name = "worker-group-2" + worker-group-2 = { instance_type = "t2.medium" asg_desired_capacity = 1 public_ip = true }, - ] + } } diff --git a/examples/launch_templates/outputs.tf b/examples/launch_templates/outputs.tf index a0788aff1d2..4c8847b03d2 100644 --- a/examples/launch_templates/outputs.tf +++ b/examples/launch_templates/outputs.tf @@ -18,6 +18,11 @@ output "config_map_aws_auth" { value = module.eks.config_map_aws_auth } +output "worker_groups" { + description = "Outputs from EKS worker groups. Map of maps, keyed by var.worker_groups keys" + value = module.eks.worker_groups +} + output "region" { description = "AWS region." value = var.region diff --git a/examples/secrets_encryption/main.tf b/examples/secrets_encryption/main.tf index 1a639e1d20a..088021ab155 100644 --- a/examples/secrets_encryption/main.tf +++ b/examples/secrets_encryption/main.tf @@ -99,14 +99,13 @@ module "eks" { vpc_id = module.vpc.vpc_id - worker_groups = [ - { - name = "worker-group-1" + worker_groups = { + worker-group-1 = { instance_type = "t2.small" additional_userdata = "echo foo bar" asg_desired_capacity = 2 }, - ] + } map_roles = var.map_roles map_users = var.map_users diff --git a/examples/spot_instances/main.tf b/examples/spot_instances/main.tf index 5406f1d234e..46e903164d8 100644 --- a/examples/spot_instances/main.tf +++ b/examples/spot_instances/main.tf @@ -68,9 +68,8 @@ module "eks" { subnets = module.vpc.public_subnets vpc_id = module.vpc.vpc_id - worker_groups_launch_template = [ - { - name = "spot-1" + worker_groups = { + spot-1 = { override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] spot_instance_pools = 4 asg_max_size = 5 @@ -78,5 +77,5 @@ module "eks" { kubelet_extra_args = "--node-labels=node.kubernetes.io/lifecycle=spot" public_ip = true }, - ] + } } diff --git a/local.tf b/local.tf index afc52bee213..225d69bf8e3 100644 --- a/local.tf +++ b/local.tf @@ -1,146 +1,16 @@ locals { - asg_tags = [ - for item in keys(var.tags) : - map( - "key", item, - "value", element(values(var.tags), index(keys(var.tags), item)), - "propagate_at_launch", "true" - ) - if item != "Name" - ] - cluster_security_group_id = var.cluster_create_security_group ? join("", aws_security_group.cluster.*.id) : var.cluster_security_group_id cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) - worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id - default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] - default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) - default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) + default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] kubeconfig_name = var.kubeconfig_name == "" ? "eks_${var.cluster_name}" : var.kubeconfig_name - worker_group_count = length(var.worker_groups) - worker_group_launch_template_count = length(var.worker_groups_launch_template) - - worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" - # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. - worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? - var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" - ) - ec2_principal = "ec2.${data.aws_partition.current.dns_suffix}" policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" - workers_group_defaults_defaults = { - name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. - tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. - ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. - asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. - asg_max_size = "3" # Maximum worker capacity in the autoscaling group. - asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. - asg_force_delete = false # Enable forced deletion for the autoscaling group. - asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. - asg_recreate_on_change = false # Recreate the autoscaling group when the Launch Template or Launch Configuration change. - default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. - health_check_grace_period = null # Time in seconds after instance comes into service before checking health. - instance_type = "m4.large" # Size of the workers instances. - spot_price = "" # Cost of spot instance. - placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". - root_volume_size = "100" # root volume size of workers instances. - root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' - root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". - key_name = "" # The key name that should be used for the instances in the autoscaling group - pre_userdata = "" # userdata to pre-append to the default userdata. - userdata_template_file = "" # alternate template to use for userdata - userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file - bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). - additional_userdata = "" # userdata to append to the default userdata. - ebs_optimized = true # sets whether to use ebs optimization on supported types. - enable_monitoring = true # Enables/disables detailed monitoring. - public_ip = false # Associate a public ip address with a worker - kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. - subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] - additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config - protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. - iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. - iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. - suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] - target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group - enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] - placement_group = null # The name of the placement group into which to launch the instances, if any. - service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS - termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. - platform = "linux" # Platform of workers. either "linux" or "windows" - additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults - # Settings for launch templates - root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. - root_kms_key_id = "" # The KMS key to use when encrypting the root storage device - launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group - launch_template_placement_tenancy = "default" # The placement tenancy for instances - launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. - root_encrypted = false # Whether the volume should be encrypted or not - eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) - cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs - market_type = null - metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. - metadata_http_tokens = "optional" # If session tokens are required: optional, required. - metadata_http_put_response_hop_limit = null # The desired HTTP PUT response hop limit for instance metadata requests. - # Settings for launch templates with mixed instances policy - override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy - on_demand_allocation_strategy = null # Strategy to use when launching on-demand instances. Valid values: prioritized. - on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances - on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity - spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. - spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." - spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price - max_instance_lifetime = 0 # Maximum number of seconds instances can run in the ASG. 0 is unlimited. - } - - workers_group_defaults = merge( - local.workers_group_defaults_defaults, - var.workers_group_defaults, - ) - - ebs_optimized_not_supported = [ - "c1.medium", - "c3.8xlarge", - "c3.large", - "c5d.12xlarge", - "c5d.24xlarge", - "c5d.metal", - "cc2.8xlarge", - "cr1.8xlarge", - "g2.8xlarge", - "g4dn.metal", - "hs1.8xlarge", - "i2.8xlarge", - "m1.medium", - "m1.small", - "m2.xlarge", - "m3.large", - "m3.medium", - "m5ad.16xlarge", - "m5ad.8xlarge", - "m5dn.metal", - "m5n.metal", - "r3.8xlarge", - "r3.large", - "r5ad.16xlarge", - "r5ad.8xlarge", - "r5dn.metal", - "r5n.metal", - "t1.micro", - "t2.2xlarge", - "t2.large", - "t2.medium", - "t2.micro", - "t2.nano", - "t2.small", - "t2.xlarge" - ] kubeconfig = var.create_eks ? templatefile("${path.module}/templates/kubeconfig.tpl", { kubeconfig_name = local.kubeconfig_name diff --git a/modules/node_groups/README.md b/modules/node_groups/README.md index c905c749f27..5c18477bfed 100644 --- a/modules/node_groups/README.md +++ b/modules/node_groups/README.md @@ -55,6 +55,7 @@ No requirements. | ng\_depends\_on | List of references to other resources this submodule depends on | `any` | `null` | no | | node\_groups | Map of maps of `eks_node_groups` to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | `{}` | no | | node\_groups\_defaults | map of maps of node groups to create. See "`node_groups` and `node_groups_defaults` keys" section in README.md for more details | `any` | n/a | yes | +| subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources | `map(string)` | n/a | yes | | workers\_group\_defaults | Workers group defaults from parent | `any` | n/a | yes | diff --git a/modules/node_groups/locals.tf b/modules/node_groups/locals.tf index 43cf672ca02..f7ed28bb7b3 100644 --- a/modules/node_groups/locals.tf +++ b/modules/node_groups/locals.tf @@ -2,15 +2,29 @@ locals { # Merge defaults and per-group values to make code cleaner node_groups_expanded = { for k, v in var.node_groups : k => merge( { - desired_capacity = var.workers_group_defaults["asg_desired_capacity"] + desired_capacity = local.nodes_group_defaults["asg_desired_capacity"] iam_role_arn = var.default_iam_role_arn - instance_type = var.workers_group_defaults["instance_type"] - key_name = var.workers_group_defaults["key_name"] - max_capacity = var.workers_group_defaults["asg_max_size"] - min_capacity = var.workers_group_defaults["asg_min_size"] - subnets = var.workers_group_defaults["subnets"] + instance_type = local.nodes_group_defaults["instance_type"] + key_name = local.nodes_group_defaults["key_name"] + max_capacity = local.nodes_group_defaults["asg_max_size"] + min_capacity = local.nodes_group_defaults["asg_min_size"] + subnets = local.nodes_group_defaults["subnets"] }, var.node_groups_defaults, v, ) if var.create_eks } + + nodes_group_defaults_defaults = { + asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. + asg_max_size = "3" # Maximum worker capacity in the autoscaling group. + asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. + instance_type = "m4.large" # Size of the workers instances. + key_name = "" # The key name that should be used for the instances in the autoscaling group + subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] + } + + nodes_group_defaults = merge( + local.nodes_group_defaults_defaults, + var.workers_group_defaults, + ) } diff --git a/modules/node_groups/variables.tf b/modules/node_groups/variables.tf index fc869d9d99c..e76efbfc72d 100644 --- a/modules/node_groups/variables.tf +++ b/modules/node_groups/variables.tf @@ -24,6 +24,11 @@ variable "tags" { type = map(string) } +variable "subnets" { + description = "A list of subnets to place the EKS cluster and workers within." + type = list(string) +} + variable "node_groups_defaults" { description = "map of maps of node groups to create. See \"`node_groups` and `node_groups_defaults` keys\" section in README.md for more details" type = any diff --git a/modules/worker_groups/README.md b/modules/worker_groups/README.md new file mode 100644 index 00000000000..d0e516f43ac --- /dev/null +++ b/modules/worker_groups/README.md @@ -0,0 +1,73 @@ +# eks `worker_groups` submodule + +Helper submodule to create and manage resources related to `eks_worker_groups`. + +## Assumptions + +* Designed for use by the parent module and not directly by end users + +## Worker Groups' IAM Role + +The role ARN specified in `var.default_iam_role_arn` will be used by default. In a simple configuration this will be the worker role created by the parent module. + +`iam_role_arn` must be specified in either `var.worker_groups_defaults` or `var.worker_groups` if the default parent IAM role is not being created for whatever reason, for example if `manage_worker_iam_resources` is set to false in the parent. + +## `worker_groups` and `worker_groups_defaults` keys + +`worker_groups_defaults` is a map that can take the below keys. Values will be used if not specified in individual worker groups. + +`worker_groups` is a map of maps. Key of first level will be used as unique value for `for_each` resources and in the `aws_autoscaling_group` and `aws_launch_template`. Inner map can take alle the values from `workers_group_defaults_defaults` map. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | +| random | n/a | +| template | n/a | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_auth\_base64 | Cluster auth data | `string` | n/a | yes | +| cluster\_endpoint | Cluster endpojnt | `string` | n/a | yes | +| cluster\_name | Cluster name | `string` | n/a | yes | +| cluster\_primary\_security\_group\_id | EKS cluster primary security group id. | `string` | n/a | yes | +| cluster\_security\_group\_id | EKS cluster security group id. | `string` | n/a | yes | +| cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | +| create\_workers | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | +| default\_iam\_role\_id | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | +| iam\_path | If provided, all IAM roles will be created on this path. | `string` | `"/"` | no | +| manage\_worker\_iam\_resources | Whether to let the module manage worker IAM resources. If set to false, iam\_instance\_profile\_name must be specified for workers. | `bool` | `true` | no | +| ng\_depends\_on | List of references to other resources this submodule depends on | `any` | `null` | no | +| subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | +| tags | A map of tags to add to all resources | `map(string)` | n/a | yes | +| vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | +| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | +| worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | +| worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | +| worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"602401143452"` | no | +| worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"801119661308"` | no | +| worker\_create\_cluster\_primary\_security\_group\_rules | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | +| worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | +| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | +| worker\_groups | A map of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `{}` | no | +| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | +| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | +| workers\_group\_defaults | Workers group defaults from parent | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| aws\_auth\_roles | Roles for use in aws-auth ConfigMap | +| worker\_groups | Outputs from EKS worker groups. Map of maps, keyed by `var.worker_groups` keys. | +| worker\_security\_group\_id | Security group ID attached to the EKS workers. | + + diff --git a/modules/worker_groups/data.tf b/modules/worker_groups/data.tf new file mode 100644 index 00000000000..a3ca373d889 --- /dev/null +++ b/modules/worker_groups/data.tf @@ -0,0 +1,60 @@ +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} + +data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" { + for_each = var.manage_worker_iam_resources ? {} : local.worker_group_configurations + + name = each.value["iam_instance_profile_name"] +} + +data "aws_ami" "eks_worker" { + filter { + name = "name" + values = [local.worker_ami_name_filter] + } + + most_recent = true + + owners = [var.worker_ami_owner_id] +} + +data "aws_ami" "eks_worker_windows" { + filter { + name = "name" + values = [local.worker_ami_name_filter_windows] + } + + filter { + name = "platform" + values = ["windows"] + } + + most_recent = true + + owners = [var.worker_ami_owner_id_windows] +} + +data "template_file" "userdata" { + for_each = local.worker_group_configurations + + template = lookup( + var.worker_groups[each.key], + "userdata_template_file", + file((each.value["platform"] == "windows" ? "${path.module}/templates/userdata_windows.tpl" : "${path.module}/templates/userdata.sh.tpl")) + ) + + vars = merge( + { + platform = each.value["platform"] + cluster_name = var.cluster_name + endpoint = var.cluster_endpoint + cluster_auth_base64 = var.cluster_auth_base64 + + pre_userdata = each.value["pre_userdata"] + additional_userdata = each.value["additional_userdata"] + bootstrap_extra_args = each.value["bootstrap_extra_args"] + kubelet_extra_args = each.value["kubelet_extra_args"] + }, + each.value["userdata_template_extra_args"] + ) +} diff --git a/modules/worker_groups/local.tf b/modules/worker_groups/local.tf new file mode 100644 index 00000000000..dcb804d7d68 --- /dev/null +++ b/modules/worker_groups/local.tf @@ -0,0 +1,138 @@ +locals { + workers_group_defaults_defaults = { + name = "count.index" # Name of the worker group. Literal count.index will never be used but if name is not set, the count.index interpolation will be used. + tags = [] # A list of map defining extra tags to be applied to the worker group autoscaling group. + ami_id = "" # AMI ID for the eks linux based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. + ami_id_windows = "" # AMI ID for the eks windows based workers. If none is provided, Terraform will search for the latest version of their EKS optimized worker AMI based on platform. + asg_desired_capacity = "1" # Desired worker capacity in the autoscaling group and changing its value will not affect the autoscaling group's desired capacity because the cluster-autoscaler manages up and down scaling of the nodes. Cluster-autoscaler add nodes when pods are in pending state and remove the nodes when they are not required by modifying the desirec_capacity of the autoscaling group. Although an issue exists in which if the value of the asg_min_size is changed it modifies the value of asg_desired_capacity. + asg_max_size = "3" # Maximum worker capacity in the autoscaling group. + asg_min_size = "1" # Minimum worker capacity in the autoscaling group. NOTE: Change in this paramater will affect the asg_desired_capacity, like changing its value to 2 will change asg_desired_capacity value to 2 but bringing back it to 1 will not affect the asg_desired_capacity. + asg_force_delete = false # Enable forced deletion for the autoscaling group. + asg_initial_lifecycle_hooks = [] # Initital lifecycle hook for the autoscaling group. + asg_recreate_on_change = false # Recreate the autoscaling group when the Launch Template or Launch Configuration change. + default_cooldown = null # The amount of time, in seconds, after a scaling activity completes before another scaling activity can start. + health_check_grace_period = null # Time in seconds after instance comes into service before checking health. + instance_type = "m4.large" # Size of the workers instances. + spot_price = "" # Cost of spot instance. + placement_tenancy = "" # The tenancy of the instance. Valid values are "default" or "dedicated". + root_volume_size = "100" # root volume size of workers instances. + root_volume_type = "gp2" # root volume type of workers instances, can be 'standard', 'gp2', or 'io1' + root_iops = "0" # The amount of provisioned IOPS. This must be set with a volume_type of "io1". + key_name = "" # The key name that should be used for the instances in the autoscaling group + pre_userdata = "" # userdata to pre-append to the default userdata. + userdata_template_file = "" # alternate template to use for userdata + userdata_template_extra_args = {} # Additional arguments to use when expanding the userdata template file + bootstrap_extra_args = "" # Extra arguments passed to the bootstrap.sh script from the EKS AMI (Amazon Machine Image). + additional_userdata = "" # userdata to append to the default userdata. + ebs_optimized = true # sets whether to use ebs optimization on supported types. + enable_monitoring = true # Enables/disables detailed monitoring. + public_ip = false # Associate a public ip address with a worker + kubelet_extra_args = "" # This string is passed directly to kubelet if set. Useful for adding labels or taints. + subnets = var.subnets # A list of subnets to place the worker nodes in. i.e. ["subnet-123", "subnet-456", "subnet-789"] + additional_security_group_ids = [] # A list of additional security group ids to include in worker launch config + protect_from_scale_in = false # Prevent AWS from scaling in, so that cluster-autoscaler is solely responsible. + iam_instance_profile_name = "" # A custom IAM instance profile name. Used when manage_worker_iam_resources is set to false. Incompatible with iam_role_id. + iam_role_id = "local.default_iam_role_id" # A custom IAM role id. Incompatible with iam_instance_profile_name. Literal local.default_iam_role_id will never be used but if iam_role_id is not set, the local.default_iam_role_id interpolation will be used. + suspended_processes = ["AZRebalance"] # A list of processes to suspend. i.e. ["AZRebalance", "HealthCheck", "ReplaceUnhealthy"] + target_group_arns = null # A list of Application LoadBalancer (ALB) target group ARNs to be associated to the autoscaling group + enabled_metrics = [] # A list of metrics to be collected i.e. ["GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"] + placement_group = null # The name of the placement group into which to launch the instances, if any. + service_linked_role_arn = "" # Arn of custom service linked role that Auto Scaling group will use. Useful when you have encrypted EBS + termination_policies = [] # A list of policies to decide how the instances in the auto scale group should be terminated. + platform = "linux" # Platform of workers. either "linux" or "windows" + additional_ebs_volumes = [] # A list of additional volumes to be attached to the instances on this Auto Scaling group. Each volume should be an object with the following: block_device_name (required), volume_size, volume_type, iops, encrypted, kms_key_id (only on launch-template), delete_on_termination. Optional values are grabbed from root volume or from defaults + # Settings for launch templates + root_block_device_name = data.aws_ami.eks_worker.root_device_name # Root device name for workers. If non is provided, will assume default AMI was used. + root_kms_key_id = "" # The KMS key to use when encrypting the root storage device + launch_template_version = "$Latest" # The lastest version of the launch template to use in the autoscaling group + launch_template_placement_tenancy = "default" # The placement tenancy for instances + launch_template_placement_group = null # The name of the placement group into which to launch the instances, if any. + root_encrypted = false # Whether the volume should be encrypted or not + eni_delete = true # Delete the Elastic Network Interface (ENI) on termination (if set to false you will have to manually delete before destroying) + cpu_credits = "standard" # T2/T3 unlimited mode, can be 'standard' or 'unlimited'. Used 'standard' mode as default to avoid paying higher costs + market_type = null + metadata_http_endpoint = "enabled" # The state of the metadata service: enabled, disabled. + metadata_http_tokens = "optional" # If session tokens are required: optional, required. + metadata_http_put_response_hop_limit = null # The desired HTTP PUT response hop limit for instance metadata requests. + # Settings for launch templates with mixed instances policy + override_instance_types = ["m5.large", "m5a.large", "m5d.large", "m5ad.large"] # A list of override instance types for mixed instances policy + on_demand_allocation_strategy = null # Strategy to use when launching on-demand instances. Valid values: prioritized. + on_demand_base_capacity = "0" # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances + on_demand_percentage_above_base_capacity = "0" # Percentage split between on-demand and Spot instances above the base on-demand capacity + spot_allocation_strategy = "lowest-price" # Valid options are 'lowest-price' and 'capacity-optimized'. If 'lowest-price', the Auto Scaling group launches instances using the Spot pools with the lowest price, and evenly allocates your instances across the number of Spot pools. If 'capacity-optimized', the Auto Scaling group launches instances using Spot pools that are optimally chosen based on the available Spot capacity. + spot_instance_pools = 10 # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify." + spot_max_price = "" # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price + max_instance_lifetime = 0 # Maximum number of seconds instances can run in the ASG. 0 is unlimited. + } + + workers_group_defaults = merge( + local.workers_group_defaults_defaults, + var.workers_group_defaults, + ) + + ebs_optimized_not_supported = [ + "c1.medium", + "c3.8xlarge", + "c3.large", + "c5d.12xlarge", + "c5d.24xlarge", + "c5d.metal", + "cc2.8xlarge", + "cr1.8xlarge", + "g2.8xlarge", + "g4dn.metal", + "hs1.8xlarge", + "i2.8xlarge", + "m1.medium", + "m1.small", + "m2.xlarge", + "m3.large", + "m3.medium", + "m5ad.16xlarge", + "m5ad.8xlarge", + "m5dn.metal", + "m5n.metal", + "r3.8xlarge", + "r3.large", + "r5ad.16xlarge", + "r5ad.8xlarge", + "r5dn.metal", + "r5n.metal", + "t1.micro", + "t2.2xlarge", + "t2.large", + "t2.medium", + "t2.micro", + "t2.nano", + "t2.small", + "t2.xlarge" + ] + + worker_group_configurations = { + for k, v in var.worker_groups : k => merge( + local.workers_group_defaults, + v, + ) if var.create_workers + } + + asg_tags = [ + for item in keys(var.tags) : + map( + "key", item, + "value", element(values(var.tags), index(keys(var.tags), item)), + "propagate_at_launch", "true" + ) + if item != "Name" + ] + + default_ami_id_linux = coalesce(local.workers_group_defaults.ami_id, data.aws_ami.eks_worker.id) + default_ami_id_windows = coalesce(local.workers_group_defaults.ami_id_windows, data.aws_ami.eks_worker_windows.id) + + worker_ami_name_filter = var.worker_ami_name_filter != "" ? var.worker_ami_name_filter : "amazon-eks-node-${var.cluster_version}-v*" + # Windows nodes are available from k8s 1.14. If cluster version is less than 1.14, fix ami filter to some constant to not fail on 'terraform plan'. + worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? + var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" + ) + + worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id +} diff --git a/modules/worker_groups/main.tf b/modules/worker_groups/main.tf new file mode 100644 index 00000000000..a94017c33e4 --- /dev/null +++ b/modules/worker_groups/main.tf @@ -0,0 +1,265 @@ +resource "aws_autoscaling_group" "workers" { + for_each = local.worker_group_configurations + + name_prefix = join( + "-", + compact( + [ + var.cluster_name, + each.key, + each.value["asg_recreate_on_change"] ? random_pet.workers[each.key].id : "" + ] + ) + ) + + desired_capacity = each.value["asg_desired_capacity"] + max_size = each.value["asg_max_size"] + min_size = each.value["asg_min_size"] + force_delete = each.value["asg_force_delete"] + target_group_arns = each.value["target_group_arns"] + service_linked_role_arn = each.value["service_linked_role_arn"] + vpc_zone_identifier = each.value["subnets"] + protect_from_scale_in = each.value["protect_from_scale_in"] + suspended_processes = each.value["suspended_processes"] + + enabled_metrics = each.value["enabled_metrics"] + + placement_group = each.value["placement_group"] + + termination_policies = each.value["termination_policies"] + max_instance_lifetime = each.value["max_instance_lifetime"] + default_cooldown = each.value["default_cooldown"] + health_check_grace_period = each.value["health_check_grace_period"] + + dynamic mixed_instances_policy { + iterator = item + for_each = ((lookup(var.worker_groups[each.key], "override_instance_types", null) != null) || (each.value["on_demand_allocation_strategy"] != null)) ? list(each.value) : [] + + content { + instances_distribution { + on_demand_allocation_strategy = lookup(item.value, "on_demand_allocation_strategy", "prioritized") + on_demand_base_capacity = item.value["on_demand_base_capacity"] + on_demand_percentage_above_base_capacity = item.value["on_demand_percentage_above_base_capacity"] + + spot_allocation_strategy = item.value["spot_allocation_strategy"] + spot_instance_pools = item.value["spot_instance_pools"] + spot_max_price = item.value["spot_max_price"] + } + + launch_template { + launch_template_specification { + launch_template_id = aws_launch_template.workers[each.key].id + version = item.value["launch_template_version"] + } + + dynamic "override" { + for_each = item.value["override_instance_types"] + + content { + instance_type = override.value + } + } + } + } + } + + dynamic launch_template { + iterator = item + for_each = ((lookup(var.worker_groups[each.key], "override_instance_types", null) != null) || (each.value["on_demand_allocation_strategy"] != null)) ? [] : list(each.value) + + content { + id = aws_launch_template.workers[each.key].id + version = item.value["launch_template_version"] + } + } + + dynamic "initial_lifecycle_hook" { + for_each = var.worker_create_initial_lifecycle_hooks ? each.value["asg_initial_lifecycle_hooks"] : [] + + content { + name = initial_lifecycle_hook.value["name"] + lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] + notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) + heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) + notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) + role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) + default_result = lookup(initial_lifecycle_hook.value, "default_result", null) + } + } + + tags = concat( + [ + { + key = "Name" + value = "${var.cluster_name}-${each.key}-eks_asg" + propagate_at_launch = true + }, + { + key = "kubernetes.io/cluster/${var.cluster_name}" + value = "owned" + propagate_at_launch = true + }, + ], + local.asg_tags, + each.value["tags"] + ) + + lifecycle { + create_before_destroy = true + ignore_changes = [desired_capacity] + } +} + +resource "aws_launch_template" "workers" { + for_each = local.worker_group_configurations + + name_prefix = "${var.cluster_name}-${each.key}" + + network_interfaces { + associate_public_ip_address = each.value["public_ip"] + delete_on_termination = each.value["eni_delete"] + + security_groups = flatten([ + local.worker_security_group_id, + var.worker_additional_security_group_ids, + each.value["additional_security_group_ids"] + ]) + } + + iam_instance_profile { + name = var.manage_worker_iam_resources ? aws_iam_instance_profile.workers[each.key].name : data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile[each.key].name + } + + image_id = lookup( + var.worker_groups[each.key], + "ami_id", + each.value["platform"] == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, + ) + + instance_type = each.value["instance_type"] + key_name = each.value["key_name"] + user_data = base64encode(data.template_file.userdata[each.key].rendered) + + ebs_optimized = lookup( + var.worker_groups[each.key], + "ebs_optimized", + ! contains(local.ebs_optimized_not_supported, each.value["instance_type"]) + ) + + metadata_options { + http_endpoint = each.value["metadata_http_endpoint"] + http_tokens = each.value["metadata_http_tokens"] + http_put_response_hop_limit = each.value["metadata_http_put_response_hop_limit"] + } + + credit_specification { + cpu_credits = each.value["cpu_credits"] + } + + monitoring { + enabled = each.value["enable_monitoring"] + } + + dynamic placement { + for_each = each.value["launch_template_placement_group"] != null ? [each.value["launch_template_placement_group"]] : [] + + content { + tenancy = each.value["launch_template_placement_tenancy"] + group_name = placement.value + } + } + + dynamic instance_market_options { + for_each = lookup(var.worker_groups[each.key], "market_type", null) == null ? [] : list(lookup(var.worker_groups[each.key], "market_type", null)) + + content { + market_type = instance_market_options.value + } + } + + block_device_mappings { + device_name = each.value["root_block_device_name"] + + ebs { + volume_size = each.value["root_volume_size"] + volume_type = each.value["root_volume_type"] + iops = each.value["root_iops"] + encrypted = each.value["root_encrypted"] + kms_key_id = each.value["root_kms_key_id"] + + delete_on_termination = true + } + } + + dynamic "block_device_mappings" { + for_each = each.value["additional_ebs_volumes"] + + content { + device_name = block_device_mappings.value.block_device_name + + ebs { + volume_size = lookup(block_device_mappings.value, "volume_size", local.workers_group_defaults["root_volume_size"]) + volume_type = lookup(block_device_mappings.value, "volume_type", local.workers_group_defaults["root_volume_type"]) + iops = lookup(block_device_mappings.value, "iops", local.workers_group_defaults["root_iops"]) + encrypted = lookup(block_device_mappings.value, "encrypted", local.workers_group_defaults["root_encrypted"]) + kms_key_id = lookup(block_device_mappings.value, "kms_key_id", local.workers_group_defaults["root_kms_key_id"]) + + delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) + } + } + } + + tag_specifications { + resource_type = "volume" + + tags = merge( + { + Name = "${var.cluster_name}-${each.key}-eks_asg" + }, + var.tags, + ) + } + + tag_specifications { + resource_type = "instance" + + tags = merge( + { + Name = "${var.cluster_name}-${each.key}-eks_asg" + }, + var.tags, + ) + } + + tags = var.tags + + lifecycle { + create_before_destroy = true + } + + # Prevent premature access of security group roles and policies by pods that + # require permissions on create/destroy that depend on workers. + depends_on = [ + aws_security_group_rule.workers_egress_internet, + aws_security_group_rule.workers_ingress_self, + aws_security_group_rule.workers_ingress_cluster, + aws_security_group_rule.workers_ingress_cluster_kubelet, + aws_security_group_rule.workers_ingress_cluster_https, + aws_security_group_rule.workers_ingress_cluster_primary, + aws_security_group_rule.cluster_primary_ingress_workers, + var.ng_depends_on, + ] +} + +resource "aws_iam_instance_profile" "workers" { + for_each = var.manage_worker_iam_resources ? local.worker_group_configurations : {} + + name_prefix = var.cluster_name + + role = lookup( + var.worker_groups[each.key], + "iam_role_id", + var.default_iam_role_id, + ) + path = var.iam_path +} diff --git a/modules/worker_groups/outputs.tf b/modules/worker_groups/outputs.tf new file mode 100644 index 00000000000..96a4c2a9118 --- /dev/null +++ b/modules/worker_groups/outputs.tf @@ -0,0 +1,19 @@ +output "aws_auth_roles" { + description = "Roles for use in aws-auth ConfigMap" + value = [ + for k, v in local.worker_group_configurations : { + worker_role_arn = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:role/${var.manage_worker_iam_resources ? aws_iam_instance_profile.workers[k].role : data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile[k].role_name}" + platform = v["platform"] + } + ] +} + +output "worker_groups" { + description = "Outputs from EKS worker groups. Map of maps, keyed by `var.worker_groups` keys." + value = aws_autoscaling_group.workers +} + +output "worker_security_group_id" { + description = "Security group ID attached to the EKS workers." + value = local.worker_security_group_id +} diff --git a/modules/worker_groups/random.tf b/modules/worker_groups/random.tf new file mode 100644 index 00000000000..740eee332c4 --- /dev/null +++ b/modules/worker_groups/random.tf @@ -0,0 +1,19 @@ +resource "random_pet" "workers" { + for_each = local.worker_group_configurations + + separator = "-" + length = 2 + + keepers = { + lt_name = join( + "-", + compact( + [ + aws_launch_template.workers[each.key].name, + aws_launch_template.workers[each.key].latest_version + ] + ) + ) + } +} + diff --git a/modules/worker_groups/sg.tf b/modules/worker_groups/sg.tf new file mode 100644 index 00000000000..1aac655b619 --- /dev/null +++ b/modules/worker_groups/sg.tf @@ -0,0 +1,91 @@ +resource "aws_security_group" "workers" { + count = var.worker_create_security_group && var.create_workers ? 1 : 0 + + name_prefix = var.cluster_name + description = "Security group for all nodes in the cluster." + vpc_id = var.vpc_id + tags = merge( + var.tags, + { + "Name" = "${var.cluster_name}-eks_worker_sg" + "kubernetes.io/cluster/${var.cluster_name}" = "owned" + }, + ) +} + +resource "aws_security_group_rule" "workers_egress_internet" { + count = var.worker_create_security_group && var.create_workers ? 1 : 0 + description = "Allow nodes all egress to the Internet." + protocol = "-1" + security_group_id = local.worker_security_group_id + cidr_blocks = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + type = "egress" +} + +resource "aws_security_group_rule" "workers_ingress_self" { + count = var.worker_create_security_group && var.create_workers ? 1 : 0 + description = "Allow node to communicate with each other." + protocol = "-1" + security_group_id = local.worker_security_group_id + source_security_group_id = local.worker_security_group_id + from_port = 0 + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "workers_ingress_cluster" { + count = var.worker_create_security_group && var.create_workers ? 1 : 0 + description = "Allow workers pods to receive communication from the cluster control plane." + protocol = "tcp" + security_group_id = local.worker_security_group_id + source_security_group_id = var.cluster_security_group_id + from_port = var.worker_sg_ingress_from_port + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { + count = var.worker_create_security_group && var.create_workers ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + description = "Allow workers Kubelets to receive communication from the cluster control plane." + protocol = "tcp" + security_group_id = local.worker_security_group_id + source_security_group_id = var.cluster_security_group_id + from_port = 10250 + to_port = 10250 + type = "ingress" +} + +resource "aws_security_group_rule" "workers_ingress_cluster_https" { + count = var.worker_create_security_group && var.create_workers ? 1 : 0 + description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." + protocol = "tcp" + security_group_id = local.worker_security_group_id + source_security_group_id = var.cluster_security_group_id + from_port = 443 + to_port = 443 + type = "ingress" +} + +resource "aws_security_group_rule" "workers_ingress_cluster_primary" { + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_workers ? 1 : 0 + description = "Allow pods running on workers to receive communication from cluster primary security group (e.g. Fargate pods)." + protocol = "all" + security_group_id = local.worker_security_group_id + source_security_group_id = var.cluster_primary_security_group_id + from_port = 0 + to_port = 65535 + type = "ingress" +} + +resource "aws_security_group_rule" "cluster_primary_ingress_workers" { + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_workers ? 1 : 0 + description = "Allow pods running on workers to send communication to cluster primary security group (e.g. Fargate pods)." + protocol = "all" + security_group_id = var.cluster_primary_security_group_id + source_security_group_id = local.worker_security_group_id + from_port = 0 + to_port = 65535 + type = "ingress" +} diff --git a/templates/userdata.sh.tpl b/modules/worker_groups/templates/userdata.sh.tpl similarity index 100% rename from templates/userdata.sh.tpl rename to modules/worker_groups/templates/userdata.sh.tpl diff --git a/templates/userdata_windows.tpl b/modules/worker_groups/templates/userdata_windows.tpl similarity index 100% rename from templates/userdata_windows.tpl rename to modules/worker_groups/templates/userdata_windows.tpl diff --git a/modules/worker_groups/variables.tf b/modules/worker_groups/variables.tf new file mode 100644 index 00000000000..ff626eb28d1 --- /dev/null +++ b/modules/worker_groups/variables.tf @@ -0,0 +1,146 @@ +variable "create_workers" { + description = "Controls if EKS resources should be created (it affects almost all resources)" + type = bool + default = true +} + +variable "cluster_version" { + description = "Kubernetes version to use for the EKS cluster." + type = string +} + +variable "cluster_name" { + description = "Cluster name" + type = string +} + +variable "cluster_endpoint" { + description = "Cluster endpojnt" + type = string +} + +variable "cluster_auth_base64" { + description = "Cluster auth data" + type = string +} + +variable "default_iam_role_id" { + description = "ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults`" + type = string +} + +variable "workers_group_defaults" { + description = "Workers group defaults from parent" + type = any +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) +} + +variable "worker_groups" { + description = "A map of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys." + type = any + default = {} +} + +variable "worker_create_initial_lifecycle_hooks" { + description = "Whether to create initial lifecycle hooks provided in worker groups." + type = bool + default = false +} + +variable "iam_path" { + description = "If provided, all IAM roles will be created on this path." + type = string + default = "/" +} + +variable "manage_worker_iam_resources" { + description = "Whether to let the module manage worker IAM resources. If set to false, iam_instance_profile_name must be specified for workers." + type = bool + default = true +} + +variable "vpc_id" { + description = "VPC where the cluster and workers will be deployed." + type = string +} + +variable "subnets" { + description = "A list of subnets to place the EKS cluster and workers within." + type = list(string) +} + +variable "cluster_security_group_id" { + description = "EKS cluster security group id." + type = string +} + +variable "cluster_primary_security_group_id" { + description = "EKS cluster primary security group id." + type = string +} + +variable "worker_create_cluster_primary_security_group_rules" { + description = "Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group." + type = bool + default = false +} + +variable "worker_sg_ingress_from_port" { + description = "Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443)." + type = number + default = 1025 +} + +variable "worker_create_security_group" { + description = "Whether to create a security group for the workers or attach the workers to `worker_security_group_id`." + type = bool + default = true +} + +variable "worker_security_group_id" { + description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." + type = string + default = "" +} + +variable "worker_additional_security_group_ids" { + description = "A list of additional security group ids to attach to worker instances" + type = list(string) + default = [] +} + +variable "worker_ami_name_filter" { + description = "Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used." + type = string + default = "" +} + +variable "worker_ami_name_filter_windows" { + description = "Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster_version' is used." + type = string + default = "" +} + +variable "worker_ami_owner_id" { + description = "The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." + type = string + default = "602401143452" // The ID of the owner of the official AWS EKS AMIs. +} + +variable "worker_ami_owner_id_windows" { + description = "The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft')." + type = string + default = "801119661308" // The ID of the owner of the official AWS EKS Windows AMIs. +} + +# Hack for a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 +# Will be removed in Terraform 0.13 with the support of module's `depends_on` https://github.com/hashicorp/terraform/issues/10462 +variable "ng_depends_on" { + description = "List of references to other resources this submodule depends on" + type = any + default = null +} diff --git a/node_groups.tf b/node_groups.tf index 6721f51aa27..0d24b95b473 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -3,10 +3,11 @@ module "node_groups" { create_eks = var.create_eks cluster_name = coalescelist(aws_eks_cluster.this[*].name, [""])[0] default_iam_role_arn = coalescelist(aws_iam_role.workers[*].arn, [""])[0] - workers_group_defaults = local.workers_group_defaults + workers_group_defaults = var.workers_group_defaults tags = var.tags node_groups_defaults = var.node_groups_defaults node_groups = var.node_groups + subnets = var.subnets # Hack to ensure ordering of resource creation. # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 diff --git a/outputs.tf b/outputs.tf index 0e00989a838..6a85ff326c5 100644 --- a/outputs.tf +++ b/outputs.tf @@ -76,89 +76,9 @@ output "oidc_provider_arn" { value = var.enable_irsa ? concat(aws_iam_openid_connect_provider.oidc_provider[*].arn, [""])[0] : null } -output "workers_asg_arns" { - description = "IDs of the autoscaling groups containing workers." - value = concat( - aws_autoscaling_group.workers.*.arn, - aws_autoscaling_group.workers_launch_template.*.arn, - ) -} - -output "workers_asg_names" { - description = "Names of the autoscaling groups containing workers." - value = concat( - aws_autoscaling_group.workers.*.id, - aws_autoscaling_group.workers_launch_template.*.id, - ) -} - -output "workers_user_data" { - description = "User data of worker groups" - value = concat( - data.template_file.userdata.*.rendered, - data.template_file.launch_template_userdata.*.rendered, - ) -} - -output "workers_default_ami_id" { - description = "ID of the default worker group AMI" - value = data.aws_ami.eks_worker.id -} - -output "workers_launch_template_ids" { - description = "IDs of the worker launch templates." - value = aws_launch_template.workers_launch_template.*.id -} - -output "workers_launch_template_arns" { - description = "ARNs of the worker launch templates." - value = aws_launch_template.workers_launch_template.*.arn -} - -output "workers_launch_template_latest_versions" { - description = "Latest versions of the worker launch templates." - value = aws_launch_template.workers_launch_template.*.latest_version -} - output "worker_security_group_id" { description = "Security group ID attached to the EKS workers." - value = local.worker_security_group_id -} - -output "worker_iam_instance_profile_arns" { - description = "default IAM instance profile ARN for EKS worker groups" - value = concat( - aws_iam_instance_profile.workers.*.arn, - aws_iam_instance_profile.workers_launch_template.*.arn - ) -} - -output "worker_iam_instance_profile_names" { - description = "default IAM instance profile name for EKS worker groups" - value = concat( - aws_iam_instance_profile.workers.*.name, - aws_iam_instance_profile.workers_launch_template.*.name - ) -} - -output "worker_iam_role_name" { - description = "default IAM role name for EKS worker groups" - value = coalescelist( - aws_iam_role.workers.*.name, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_name, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_name, - [""] - )[0] -} - -output "worker_iam_role_arn" { - description = "default IAM role ARN for EKS worker groups" - value = coalescelist( - aws_iam_role.workers.*.arn, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.role_arn, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.role_arn, - [""] - )[0] + value = module.worker_groups.worker_security_group_id } output "node_groups" { @@ -170,3 +90,8 @@ output "security_group_rule_cluster_https_worker_ingress" { description = "Security group rule responsible for allowing pods to communicate with the EKS cluster API." value = aws_security_group_rule.cluster_https_worker_ingress } + +output "worker_groups" { + description = "Outputs from EKS worker groups. Map of maps, keyed by var.worker_groups keys" + value = module.worker_groups.worker_groups +} diff --git a/variables.tf b/variables.tf index 460bdc687f1..f1c68fe59aa 100644 --- a/variables.tf +++ b/variables.tf @@ -91,9 +91,9 @@ variable "vpc_id" { } variable "worker_groups" { - description = "A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys." + description = "A map of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys." type = any - default = [] + default = {} } variable "workers_group_defaults" { @@ -102,12 +102,6 @@ variable "workers_group_defaults" { default = {} } -variable "worker_groups_launch_template" { - description = "A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys." - type = any - default = [] -} - variable "worker_security_group_id" { description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." type = string diff --git a/worker_groups.tf b/worker_groups.tf new file mode 100644 index 00000000000..8d8db0bb1ec --- /dev/null +++ b/worker_groups.tf @@ -0,0 +1,50 @@ +module "worker_groups" { + source = "./modules/worker_groups" + + create_workers = var.create_eks + + cluster_version = var.cluster_version + cluster_name = var.cluster_name + cluster_endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] + cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + + default_iam_role_id = coalescelist(aws_iam_role.workers[*].id, [""])[0] + + vpc_id = var.vpc_id + subnets = var.subnets + + iam_path = var.iam_path + manage_worker_iam_resources = var.manage_worker_iam_resources + worker_create_initial_lifecycle_hooks = var.worker_create_initial_lifecycle_hooks + + workers_group_defaults = var.workers_group_defaults + worker_groups = var.worker_groups + + worker_ami_name_filter = var.worker_ami_name_filter + worker_ami_name_filter_windows = var.worker_ami_name_filter_windows + worker_ami_owner_id = var.worker_ami_owner_id + worker_ami_owner_id_windows = var.worker_ami_owner_id_windows + + cluster_security_group_id = local.cluster_security_group_id + cluster_primary_security_group_id = local.cluster_primary_security_group_id + worker_create_security_group = var.worker_create_security_group + worker_security_group_id = var.worker_security_group_id + worker_sg_ingress_from_port = var.worker_sg_ingress_from_port + worker_additional_security_group_ids = var.worker_additional_security_group_ids + + tags = var.tags + + # Hack to ensure ordering of resource creation. + # This is a homemade `depends_on` https://discuss.hashicorp.com/t/tips-howto-implement-module-depends-on-emulation/2305/2 + # Do not create node_groups before other resources are ready and removes race conditions + # Ensure these resources are created before "unlocking" the data source. + # Will be removed in Terraform 0.13 + ng_depends_on = [ + aws_eks_cluster.this, + kubernetes_config_map.aws_auth, + aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, + aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, + aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, + aws_iam_role_policy_attachment.workers_additional_policies, + ] +} diff --git a/workers.tf b/workers.tf deleted file mode 100644 index 6c9d924f0e3..00000000000 --- a/workers.tf +++ /dev/null @@ -1,424 +0,0 @@ -# Worker Groups using Launch Configurations - -resource "aws_autoscaling_group" "workers" { - count = var.create_eks ? local.worker_group_count : 0 - name_prefix = join( - "-", - compact( - [ - aws_eks_cluster.this[0].name, - lookup(var.worker_groups[count.index], "name", count.index), - lookup(var.worker_groups[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers[count.index].id : "" - ] - ) - ) - desired_capacity = lookup( - var.worker_groups[count.index], - "asg_desired_capacity", - local.workers_group_defaults["asg_desired_capacity"], - ) - max_size = lookup( - var.worker_groups[count.index], - "asg_max_size", - local.workers_group_defaults["asg_max_size"], - ) - min_size = lookup( - var.worker_groups[count.index], - "asg_min_size", - local.workers_group_defaults["asg_min_size"], - ) - force_delete = lookup( - var.worker_groups[count.index], - "asg_force_delete", - local.workers_group_defaults["asg_force_delete"], - ) - target_group_arns = lookup( - var.worker_groups[count.index], - "target_group_arns", - local.workers_group_defaults["target_group_arns"] - ) - service_linked_role_arn = lookup( - var.worker_groups[count.index], - "service_linked_role_arn", - local.workers_group_defaults["service_linked_role_arn"], - ) - launch_configuration = aws_launch_configuration.workers.*.id[count.index] - vpc_zone_identifier = lookup( - var.worker_groups[count.index], - "subnets", - local.workers_group_defaults["subnets"] - ) - protect_from_scale_in = lookup( - var.worker_groups[count.index], - "protect_from_scale_in", - local.workers_group_defaults["protect_from_scale_in"], - ) - suspended_processes = lookup( - var.worker_groups[count.index], - "suspended_processes", - local.workers_group_defaults["suspended_processes"] - ) - enabled_metrics = lookup( - var.worker_groups[count.index], - "enabled_metrics", - local.workers_group_defaults["enabled_metrics"] - ) - placement_group = lookup( - var.worker_groups[count.index], - "placement_group", - local.workers_group_defaults["placement_group"], - ) - termination_policies = lookup( - var.worker_groups[count.index], - "termination_policies", - local.workers_group_defaults["termination_policies"] - ) - max_instance_lifetime = lookup( - var.worker_groups[count.index], - "max_instance_lifetime", - local.workers_group_defaults["max_instance_lifetime"], - ) - default_cooldown = lookup( - var.worker_groups[count.index], - "default_cooldown", - local.workers_group_defaults["default_cooldown"] - ) - health_check_grace_period = lookup( - var.worker_groups[count.index], - "health_check_grace_period", - local.workers_group_defaults["health_check_grace_period"] - ) - - dynamic "initial_lifecycle_hook" { - for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] - content { - name = initial_lifecycle_hook.value["name"] - lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] - notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) - heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) - notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) - role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) - default_result = lookup(initial_lifecycle_hook.value, "default_result", null) - } - } - - tags = concat( - [ - { - "key" = "Name" - "value" = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" - "value" = "owned" - "propagate_at_launch" = true - }, - { - "key" = "k8s.io/cluster/${aws_eks_cluster.this[0].name}" - "value" = "owned" - "propagate_at_launch" = true - }, - ], - local.asg_tags, - lookup( - var.worker_groups[count.index], - "tags", - local.workers_group_defaults["tags"] - ) - ) - - lifecycle { - create_before_destroy = true - ignore_changes = [desired_capacity] - } -} - -resource "aws_launch_configuration" "workers" { - count = var.create_eks ? local.worker_group_count : 0 - name_prefix = "${aws_eks_cluster.this[0].name}-${lookup(var.worker_groups[count.index], "name", count.index)}" - associate_public_ip_address = lookup( - var.worker_groups[count.index], - "public_ip", - local.workers_group_defaults["public_ip"], - ) - security_groups = flatten([ - local.worker_security_group_id, - var.worker_additional_security_group_ids, - lookup( - var.worker_groups[count.index], - "additional_security_group_ids", - local.workers_group_defaults["additional_security_group_ids"] - ) - ]) - iam_instance_profile = coalescelist( - aws_iam_instance_profile.workers.*.id, - data.aws_iam_instance_profile.custom_worker_group_iam_instance_profile.*.name, - )[count.index] - image_id = lookup( - var.worker_groups[count.index], - "ami_id", - lookup(var.worker_groups[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, - ) - instance_type = lookup( - var.worker_groups[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - key_name = lookup( - var.worker_groups[count.index], - "key_name", - local.workers_group_defaults["key_name"], - ) - user_data_base64 = base64encode(data.template_file.userdata.*.rendered[count.index]) - ebs_optimized = lookup( - var.worker_groups[count.index], - "ebs_optimized", - ! contains( - local.ebs_optimized_not_supported, - lookup( - var.worker_groups[count.index], - "instance_type", - local.workers_group_defaults["instance_type"] - ) - ) - ) - enable_monitoring = lookup( - var.worker_groups[count.index], - "enable_monitoring", - local.workers_group_defaults["enable_monitoring"], - ) - spot_price = lookup( - var.worker_groups[count.index], - "spot_price", - local.workers_group_defaults["spot_price"], - ) - placement_tenancy = lookup( - var.worker_groups[count.index], - "placement_tenancy", - local.workers_group_defaults["placement_tenancy"], - ) - - root_block_device { - encrypted = lookup( - var.worker_groups[count.index], - "root_encrypted", - local.workers_group_defaults["root_encrypted"], - ) - volume_size = lookup( - var.worker_groups[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - var.worker_groups[count.index], - "root_volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - var.worker_groups[count.index], - "root_iops", - local.workers_group_defaults["root_iops"], - ) - delete_on_termination = true - } - - dynamic "ebs_block_device" { - for_each = lookup(var.worker_groups[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) - - content { - device_name = ebs_block_device.value.block_device_name - volume_size = lookup( - ebs_block_device.value, - "volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - ebs_block_device.value, - "volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - ebs_block_device.value, - "iops", - local.workers_group_defaults["root_iops"], - ) - encrypted = lookup( - ebs_block_device.value, - "encrypted", - local.workers_group_defaults["root_encrypted"], - ) - delete_on_termination = lookup(ebs_block_device.value, "delete_on_termination", true) - } - - } - - lifecycle { - create_before_destroy = true - } - - # Prevent premature access of security group roles and policies by pods that - # require permissions on create/destroy that depend on workers. - depends_on = [ - aws_security_group_rule.workers_egress_internet, - aws_security_group_rule.workers_ingress_self, - aws_security_group_rule.workers_ingress_cluster, - aws_security_group_rule.workers_ingress_cluster_kubelet, - aws_security_group_rule.workers_ingress_cluster_https, - aws_security_group_rule.workers_ingress_cluster_primary, - aws_security_group_rule.cluster_primary_ingress_workers, - aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, - aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, - aws_iam_role_policy_attachment.workers_additional_policies - ] -} - -resource "random_pet" "workers" { - count = var.create_eks ? local.worker_group_count : 0 - - separator = "-" - length = 2 - - keepers = { - lc_name = aws_launch_configuration.workers[count.index].name - } -} - -resource "aws_security_group" "workers" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - name_prefix = var.cluster_name - description = "Security group for all nodes in the cluster." - vpc_id = var.vpc_id - tags = merge( - var.tags, - { - "Name" = "${var.cluster_name}-eks_worker_sg" - "kubernetes.io/cluster/${var.cluster_name}" = "owned" - }, - ) -} - -resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - description = "Allow nodes all egress to the Internet." - protocol = "-1" - security_group_id = local.worker_security_group_id - cidr_blocks = ["0.0.0.0/0"] - from_port = 0 - to_port = 0 - type = "egress" -} - -resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - description = "Allow node to communicate with each other." - protocol = "-1" - security_group_id = local.worker_security_group_id - source_security_group_id = local.worker_security_group_id - from_port = 0 - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - description = "Allow workers pods to receive communication from the cluster control plane." - protocol = "tcp" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_security_group_id - from_port = var.worker_sg_ingress_from_port - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 - description = "Allow workers Kubelets to receive communication from the cluster control plane." - protocol = "tcp" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_security_group_id - from_port = 10250 - to_port = 10250 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_create_security_group && var.create_eks ? 1 : 0 - description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." - protocol = "tcp" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_security_group_id - from_port = 443 - to_port = 443 - type = "ingress" -} - -resource "aws_security_group_rule" "workers_ingress_cluster_primary" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 - description = "Allow pods running on workers to receive communication from cluster primary security group (e.g. Fargate pods)." - protocol = "all" - security_group_id = local.worker_security_group_id - source_security_group_id = local.cluster_primary_security_group_id - from_port = 0 - to_port = 65535 - type = "ingress" -} - -resource "aws_security_group_rule" "cluster_primary_ingress_workers" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 - description = "Allow pods running on workers to send communication to cluster primary security group (e.g. Fargate pods)." - protocol = "all" - security_group_id = local.cluster_primary_security_group_id - source_security_group_id = local.worker_security_group_id - from_port = 0 - to_port = 65535 - type = "ingress" -} - -resource "aws_iam_role" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name - name = var.workers_role_name != "" ? var.workers_role_name : null - assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json - permissions_boundary = var.permissions_boundary - path = var.iam_path - force_detach_policies = true - tags = var.tags -} - -resource "aws_iam_instance_profile" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_count : 0 - name_prefix = aws_eks_cluster.this[0].name - role = lookup( - var.worker_groups[count.index], - "iam_role_id", - local.default_iam_role_id, - ) - - path = var.iam_path -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 - role = aws_iam_role.workers[0].name - policy_arn = var.workers_additional_policies[count.index] -} diff --git a/workers_iam.tf b/workers_iam.tf new file mode 100644 index 00000000000..cd40988a645 --- /dev/null +++ b/workers_iam.tf @@ -0,0 +1,34 @@ +resource "aws_iam_role" "workers" { + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name + name = var.workers_role_name != "" ? var.workers_role_name : null + assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json + permissions_boundary = var.permissions_boundary + path = var.iam_path + force_detach_policies = true + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.workers[0].name +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { + count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" + role = aws_iam_role.workers[0].name +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.workers[0].name +} + +resource "aws_iam_role_policy_attachment" "workers_additional_policies" { + count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 + role = aws_iam_role.workers[0].name + policy_arn = var.workers_additional_policies[count.index] +} diff --git a/workers_launch_template.tf b/workers_launch_template.tf deleted file mode 100644 index eaf734b41f4..00000000000 --- a/workers_launch_template.tf +++ /dev/null @@ -1,490 +0,0 @@ -# Worker Groups using Launch Templates - -resource "aws_autoscaling_group" "workers_launch_template" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - name_prefix = join( - "-", - compact( - [ - aws_eks_cluster.this[0].name, - lookup(var.worker_groups_launch_template[count.index], "name", count.index), - lookup(var.worker_groups_launch_template[count.index], "asg_recreate_on_change", local.workers_group_defaults["asg_recreate_on_change"]) ? random_pet.workers_launch_template[count.index].id : "" - ] - ) - ) - desired_capacity = lookup( - var.worker_groups_launch_template[count.index], - "asg_desired_capacity", - local.workers_group_defaults["asg_desired_capacity"], - ) - max_size = lookup( - var.worker_groups_launch_template[count.index], - "asg_max_size", - local.workers_group_defaults["asg_max_size"], - ) - min_size = lookup( - var.worker_groups_launch_template[count.index], - "asg_min_size", - local.workers_group_defaults["asg_min_size"], - ) - force_delete = lookup( - var.worker_groups_launch_template[count.index], - "asg_force_delete", - local.workers_group_defaults["asg_force_delete"], - ) - target_group_arns = lookup( - var.worker_groups_launch_template[count.index], - "target_group_arns", - local.workers_group_defaults["target_group_arns"] - ) - service_linked_role_arn = lookup( - var.worker_groups_launch_template[count.index], - "service_linked_role_arn", - local.workers_group_defaults["service_linked_role_arn"], - ) - vpc_zone_identifier = lookup( - var.worker_groups_launch_template[count.index], - "subnets", - local.workers_group_defaults["subnets"] - ) - protect_from_scale_in = lookup( - var.worker_groups_launch_template[count.index], - "protect_from_scale_in", - local.workers_group_defaults["protect_from_scale_in"], - ) - suspended_processes = lookup( - var.worker_groups_launch_template[count.index], - "suspended_processes", - local.workers_group_defaults["suspended_processes"] - ) - enabled_metrics = lookup( - var.worker_groups_launch_template[count.index], - "enabled_metrics", - local.workers_group_defaults["enabled_metrics"] - ) - placement_group = lookup( - var.worker_groups_launch_template[count.index], - "placement_group", - local.workers_group_defaults["placement_group"], - ) - termination_policies = lookup( - var.worker_groups_launch_template[count.index], - "termination_policies", - local.workers_group_defaults["termination_policies"] - ) - max_instance_lifetime = lookup( - var.worker_groups_launch_template[count.index], - "max_instance_lifetime", - local.workers_group_defaults["max_instance_lifetime"], - ) - default_cooldown = lookup( - var.worker_groups_launch_template[count.index], - "default_cooldown", - local.workers_group_defaults["default_cooldown"] - ) - health_check_grace_period = lookup( - var.worker_groups_launch_template[count.index], - "health_check_grace_period", - local.workers_group_defaults["health_check_grace_period"] - ) - - dynamic mixed_instances_policy { - iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? list(var.worker_groups_launch_template[count.index]) : [] - - content { - instances_distribution { - on_demand_allocation_strategy = lookup( - item.value, - "on_demand_allocation_strategy", - "prioritized", - ) - on_demand_base_capacity = lookup( - item.value, - "on_demand_base_capacity", - local.workers_group_defaults["on_demand_base_capacity"], - ) - on_demand_percentage_above_base_capacity = lookup( - item.value, - "on_demand_percentage_above_base_capacity", - local.workers_group_defaults["on_demand_percentage_above_base_capacity"], - ) - spot_allocation_strategy = lookup( - item.value, - "spot_allocation_strategy", - local.workers_group_defaults["spot_allocation_strategy"], - ) - spot_instance_pools = lookup( - item.value, - "spot_instance_pools", - local.workers_group_defaults["spot_instance_pools"], - ) - spot_max_price = lookup( - item.value, - "spot_max_price", - local.workers_group_defaults["spot_max_price"], - ) - } - - launch_template { - launch_template_specification { - launch_template_id = aws_launch_template.workers_launch_template.*.id[count.index] - version = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - local.workers_group_defaults["launch_template_version"], - ) - } - - dynamic "override" { - for_each = lookup( - var.worker_groups_launch_template[count.index], - "override_instance_types", - local.workers_group_defaults["override_instance_types"] - ) - - content { - instance_type = override.value - } - } - - } - } - } - dynamic launch_template { - iterator = item - for_each = (lookup(var.worker_groups_launch_template[count.index], "override_instance_types", null) != null) || (lookup(var.worker_groups_launch_template[count.index], "on_demand_allocation_strategy", local.workers_group_defaults["on_demand_allocation_strategy"]) != null) ? [] : list(var.worker_groups_launch_template[count.index]) - - content { - id = aws_launch_template.workers_launch_template.*.id[count.index] - version = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_version", - local.workers_group_defaults["launch_template_version"], - ) - } - } - - dynamic "initial_lifecycle_hook" { - for_each = var.worker_create_initial_lifecycle_hooks ? lookup(var.worker_groups_launch_template[count.index], "asg_initial_lifecycle_hooks", local.workers_group_defaults["asg_initial_lifecycle_hooks"]) : [] - content { - name = initial_lifecycle_hook.value["name"] - lifecycle_transition = initial_lifecycle_hook.value["lifecycle_transition"] - notification_metadata = lookup(initial_lifecycle_hook.value, "notification_metadata", null) - heartbeat_timeout = lookup(initial_lifecycle_hook.value, "heartbeat_timeout", null) - notification_target_arn = lookup(initial_lifecycle_hook.value, "notification_target_arn", null) - role_arn = lookup(initial_lifecycle_hook.value, "role_arn", null) - default_result = lookup(initial_lifecycle_hook.value, "default_result", null) - } - } - - tags = concat( - [ - { - "key" = "Name" - "value" = "${aws_eks_cluster.this[0].name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - "propagate_at_launch" = true - }, - { - "key" = "kubernetes.io/cluster/${aws_eks_cluster.this[0].name}" - "value" = "owned" - "propagate_at_launch" = true - }, - ], - local.asg_tags, - lookup( - var.worker_groups_launch_template[count.index], - "tags", - local.workers_group_defaults["tags"] - ) - ) - - lifecycle { - create_before_destroy = true - ignore_changes = [desired_capacity] - } -} - -resource "aws_launch_template" "workers_launch_template" { - count = var.create_eks ? (local.worker_group_launch_template_count) : 0 - name_prefix = "${aws_eks_cluster.this[0].name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}" - - network_interfaces { - associate_public_ip_address = lookup( - var.worker_groups_launch_template[count.index], - "public_ip", - local.workers_group_defaults["public_ip"], - ) - delete_on_termination = lookup( - var.worker_groups_launch_template[count.index], - "eni_delete", - local.workers_group_defaults["eni_delete"], - ) - security_groups = flatten([ - local.worker_security_group_id, - var.worker_additional_security_group_ids, - lookup( - var.worker_groups_launch_template[count.index], - "additional_security_group_ids", - local.workers_group_defaults["additional_security_group_ids"], - ), - ]) - } - - iam_instance_profile { - name = coalescelist( - aws_iam_instance_profile.workers_launch_template.*.name, - data.aws_iam_instance_profile.custom_worker_group_launch_template_iam_instance_profile.*.name, - )[count.index] - } - - image_id = lookup( - var.worker_groups_launch_template[count.index], - "ami_id", - lookup(var.worker_groups_launch_template[count.index], "platform", local.workers_group_defaults["platform"]) == "windows" ? local.default_ami_id_windows : local.default_ami_id_linux, - ) - instance_type = lookup( - var.worker_groups_launch_template[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - key_name = lookup( - var.worker_groups_launch_template[count.index], - "key_name", - local.workers_group_defaults["key_name"], - ) - user_data = base64encode( - data.template_file.launch_template_userdata.*.rendered[count.index], - ) - - ebs_optimized = lookup( - var.worker_groups_launch_template[count.index], - "ebs_optimized", - ! contains( - local.ebs_optimized_not_supported, - lookup( - var.worker_groups_launch_template[count.index], - "instance_type", - local.workers_group_defaults["instance_type"], - ) - ) - ) - - metadata_options { - http_endpoint = lookup( - var.worker_groups_launch_template[count.index], - "metadata_http_endpoint", - local.workers_group_defaults["metadata_http_endpoint"], - ) - http_tokens = lookup( - var.worker_groups_launch_template[count.index], - "metadata_http_tokens", - local.workers_group_defaults["metadata_http_tokens"], - ) - http_put_response_hop_limit = lookup( - var.worker_groups_launch_template[count.index], - "metadata_http_put_response_hop_limit", - local.workers_group_defaults["metadata_http_put_response_hop_limit"], - ) - } - - credit_specification { - cpu_credits = lookup( - var.worker_groups_launch_template[count.index], - "cpu_credits", - local.workers_group_defaults["cpu_credits"] - ) - } - - monitoring { - enabled = lookup( - var.worker_groups_launch_template[count.index], - "enable_monitoring", - local.workers_group_defaults["enable_monitoring"], - ) - } - - dynamic placement { - for_each = lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"]) != null ? [lookup(var.worker_groups_launch_template[count.index], "launch_template_placement_group", local.workers_group_defaults["launch_template_placement_group"])] : [] - - content { - tenancy = lookup( - var.worker_groups_launch_template[count.index], - "launch_template_placement_tenancy", - local.workers_group_defaults["launch_template_placement_tenancy"], - ) - group_name = placement.value - } - } - - dynamic instance_market_options { - for_each = lookup(var.worker_groups_launch_template[count.index], "market_type", null) == null ? [] : list(lookup(var.worker_groups_launch_template[count.index], "market_type", null)) - content { - market_type = instance_market_options.value - } - } - - block_device_mappings { - device_name = lookup( - var.worker_groups_launch_template[count.index], - "root_block_device_name", - local.workers_group_defaults["root_block_device_name"], - ) - - ebs { - volume_size = lookup( - var.worker_groups_launch_template[count.index], - "root_volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - var.worker_groups_launch_template[count.index], - "root_volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - var.worker_groups_launch_template[count.index], - "root_iops", - local.workers_group_defaults["root_iops"], - ) - encrypted = lookup( - var.worker_groups_launch_template[count.index], - "root_encrypted", - local.workers_group_defaults["root_encrypted"], - ) - kms_key_id = lookup( - var.worker_groups_launch_template[count.index], - "root_kms_key_id", - local.workers_group_defaults["root_kms_key_id"], - ) - delete_on_termination = true - } - } - - dynamic "block_device_mappings" { - for_each = lookup(var.worker_groups_launch_template[count.index], "additional_ebs_volumes", local.workers_group_defaults["additional_ebs_volumes"]) - content { - device_name = block_device_mappings.value.block_device_name - - ebs { - volume_size = lookup( - block_device_mappings.value, - "volume_size", - local.workers_group_defaults["root_volume_size"], - ) - volume_type = lookup( - block_device_mappings.value, - "volume_type", - local.workers_group_defaults["root_volume_type"], - ) - iops = lookup( - block_device_mappings.value, - "iops", - local.workers_group_defaults["root_iops"], - ) - encrypted = lookup( - block_device_mappings.value, - "encrypted", - local.workers_group_defaults["root_encrypted"], - ) - kms_key_id = lookup( - block_device_mappings.value, - "kms_key_id", - local.workers_group_defaults["root_kms_key_id"], - ) - delete_on_termination = lookup(block_device_mappings.value, "delete_on_termination", true) - } - } - - } - - tag_specifications { - resource_type = "volume" - - tags = merge( - { - "Name" = "${aws_eks_cluster.this[0].name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - }, - var.tags, - ) - } - - tag_specifications { - resource_type = "instance" - - tags = merge( - { - "Name" = "${aws_eks_cluster.this[0].name}-${lookup( - var.worker_groups_launch_template[count.index], - "name", - count.index, - )}-eks_asg" - }, - var.tags, - ) - } - - tags = var.tags - - lifecycle { - create_before_destroy = true - } - - # Prevent premature access of security group roles and policies by pods that - # require permissions on create/destroy that depend on workers. - depends_on = [ - aws_security_group_rule.workers_egress_internet, - aws_security_group_rule.workers_ingress_self, - aws_security_group_rule.workers_ingress_cluster, - aws_security_group_rule.workers_ingress_cluster_kubelet, - aws_security_group_rule.workers_ingress_cluster_https, - aws_security_group_rule.workers_ingress_cluster_primary, - aws_security_group_rule.cluster_primary_ingress_workers, - aws_iam_role_policy_attachment.workers_AmazonEKSWorkerNodePolicy, - aws_iam_role_policy_attachment.workers_AmazonEKS_CNI_Policy, - aws_iam_role_policy_attachment.workers_AmazonEC2ContainerRegistryReadOnly, - aws_iam_role_policy_attachment.workers_additional_policies - ] -} - -resource "random_pet" "workers_launch_template" { - count = var.create_eks ? local.worker_group_launch_template_count : 0 - - separator = "-" - length = 2 - - keepers = { - lt_name = join( - "-", - compact( - [ - aws_launch_template.workers_launch_template[count.index].name, - aws_launch_template.workers_launch_template[count.index].latest_version - ] - ) - ) - } -} - -resource "aws_iam_instance_profile" "workers_launch_template" { - count = var.manage_worker_iam_resources && var.create_eks ? local.worker_group_launch_template_count : 0 - name_prefix = aws_eks_cluster.this[0].name - role = lookup( - var.worker_groups_launch_template[count.index], - "iam_role_id", - local.default_iam_role_id, - ) - path = var.iam_path -}