diff --git a/cluster.tf b/cluster.tf index bbd1ff9f66..a6a4aec33d 100644 --- a/cluster.tf +++ b/cluster.tf @@ -81,6 +81,7 @@ resource "aws_security_group" "cluster" { name_prefix = var.cluster_name description = "EKS cluster security group." vpc_id = var.vpc_id + tags = merge( var.tags, { @@ -105,7 +106,7 @@ resource "aws_security_group_rule" "cluster_https_worker_ingress" { description = "Allow pods to communicate with the EKS cluster API." protocol = "tcp" security_group_id = local.cluster_security_group_id - source_security_group_id = module.worker_groups.worker_security_group_id + source_security_group_id = local.worker_security_group_id from_port = 443 to_port = 443 type = "ingress" diff --git a/local.tf b/local.tf index 225d69bf8e..c0c37e368b 100644 --- a/local.tf +++ b/local.tf @@ -3,6 +3,7 @@ locals { cluster_primary_security_group_id = var.cluster_version >= 1.14 ? element(concat(aws_eks_cluster.this[*].vpc_config[0].cluster_security_group_id, list("")), 0) : null cluster_iam_role_name = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.name) : var.cluster_iam_role_name cluster_iam_role_arn = var.manage_cluster_iam_resources ? join("", aws_iam_role.cluster.*.arn) : join("", data.aws_iam_role.custom_cluster_iam_role.*.arn) + worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id default_iam_role_id = concat(aws_iam_role.workers.*.id, [""])[0] diff --git a/modules/worker_groups/README.md b/modules/worker_groups/README.md index d0e516f43a..ed8fdd7a48 100644 --- a/modules/worker_groups/README.md +++ b/modules/worker_groups/README.md @@ -38,8 +38,6 @@ No requirements. | cluster\_auth\_base64 | Cluster auth data | `string` | n/a | yes | | cluster\_endpoint | Cluster endpojnt | `string` | n/a | yes | | cluster\_name | Cluster name | `string` | n/a | yes | -| cluster\_primary\_security\_group\_id | EKS cluster primary security group id. | `string` | n/a | yes | -| cluster\_security\_group\_id | EKS cluster security group id. | `string` | n/a | yes | | cluster\_version | Kubernetes version to use for the EKS cluster. | `string` | n/a | yes | | create\_workers | Controls if EKS resources should be created (it affects almost all resources) | `bool` | `true` | no | | default\_iam\_role\_id | ARN of the default IAM worker role to use if one is not specified in `var.node_groups` or `var.node_groups_defaults` | `string` | n/a | yes | @@ -49,17 +47,13 @@ No requirements. | subnets | A list of subnets to place the EKS cluster and workers within. | `list(string)` | n/a | yes | | tags | A map of tags to add to all resources | `map(string)` | n/a | yes | | vpc\_id | VPC where the cluster and workers will be deployed. | `string` | n/a | yes | -| worker\_additional\_security\_group\_ids | A list of additional security group ids to attach to worker instances | `list(string)` | `[]` | no | | worker\_ami\_name\_filter | Name filter for AWS EKS worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_name\_filter\_windows | Name filter for AWS EKS Windows worker AMI. If not provided, the latest official AMI for the specified 'cluster\_version' is used. | `string` | `""` | no | | worker\_ami\_owner\_id | The ID of the owner for the AMI to use for the AWS EKS workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"602401143452"` | no | | worker\_ami\_owner\_id\_windows | The ID of the owner for the AMI to use for the AWS EKS Windows workers. Valid values are an AWS account ID, 'self' (the current account), or an AWS owner alias (e.g. 'amazon', 'aws-marketplace', 'microsoft'). | `string` | `"801119661308"` | no | -| worker\_create\_cluster\_primary\_security\_group\_rules | Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group. | `bool` | `false` | no | | worker\_create\_initial\_lifecycle\_hooks | Whether to create initial lifecycle hooks provided in worker groups. | `bool` | `false` | no | -| worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | `bool` | `true` | no | | worker\_groups | A map of maps defining worker group configurations to be defined using AWS Launch Templates. See workers\_group\_defaults for valid keys. | `any` | `{}` | no | -| worker\_security\_group\_id | If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster. | `string` | `""` | no | -| worker\_sg\_ingress\_from\_port | Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443). | `number` | `1025` | no | +| worker\_security\_group\_ids | A list of security group ids to attach to worker instances | `list(string)` | `[]` | no | | workers\_group\_defaults | Workers group defaults from parent | `any` | n/a | yes | ## Outputs @@ -68,6 +62,5 @@ No requirements. |------|-------------| | aws\_auth\_roles | Roles for use in aws-auth ConfigMap | | worker\_groups | Outputs from EKS worker groups. Map of maps, keyed by `var.worker_groups` keys. | -| worker\_security\_group\_id | Security group ID attached to the EKS workers. | diff --git a/modules/worker_groups/local.tf b/modules/worker_groups/local.tf index 2633cc4730..483bb50797 100644 --- a/modules/worker_groups/local.tf +++ b/modules/worker_groups/local.tf @@ -133,6 +133,4 @@ locals { worker_ami_name_filter_windows = (var.worker_ami_name_filter_windows != "" ? var.worker_ami_name_filter_windows : "Windows_Server-2019-English-Core-EKS_Optimized-${tonumber(var.cluster_version) >= 1.14 ? var.cluster_version : 1.14}-*" ) - - worker_security_group_id = var.worker_create_security_group ? join("", aws_security_group.workers.*.id) : var.worker_security_group_id } diff --git a/modules/worker_groups/main.tf b/modules/worker_groups/main.tf index a94017c33e..d2c5ceb45d 100644 --- a/modules/worker_groups/main.tf +++ b/modules/worker_groups/main.tf @@ -120,8 +120,7 @@ resource "aws_launch_template" "workers" { delete_on_termination = each.value["eni_delete"] security_groups = flatten([ - local.worker_security_group_id, - var.worker_additional_security_group_ids, + var.worker_security_group_ids, each.value["additional_security_group_ids"] ]) } @@ -237,16 +236,7 @@ resource "aws_launch_template" "workers" { create_before_destroy = true } - # Prevent premature access of security group roles and policies by pods that - # require permissions on create/destroy that depend on workers. depends_on = [ - aws_security_group_rule.workers_egress_internet, - aws_security_group_rule.workers_ingress_self, - aws_security_group_rule.workers_ingress_cluster, - aws_security_group_rule.workers_ingress_cluster_kubelet, - aws_security_group_rule.workers_ingress_cluster_https, - aws_security_group_rule.workers_ingress_cluster_primary, - aws_security_group_rule.cluster_primary_ingress_workers, var.ng_depends_on, ] } diff --git a/modules/worker_groups/outputs.tf b/modules/worker_groups/outputs.tf index 96a4c2a911..aa466ee257 100644 --- a/modules/worker_groups/outputs.tf +++ b/modules/worker_groups/outputs.tf @@ -12,8 +12,3 @@ output "worker_groups" { description = "Outputs from EKS worker groups. Map of maps, keyed by `var.worker_groups` keys." value = aws_autoscaling_group.workers } - -output "worker_security_group_id" { - description = "Security group ID attached to the EKS workers." - value = local.worker_security_group_id -} diff --git a/modules/worker_groups/variables.tf b/modules/worker_groups/variables.tf index ff626eb28d..6154246e6f 100644 --- a/modules/worker_groups/variables.tf +++ b/modules/worker_groups/variables.tf @@ -73,42 +73,8 @@ variable "subnets" { type = list(string) } -variable "cluster_security_group_id" { - description = "EKS cluster security group id." - type = string -} - -variable "cluster_primary_security_group_id" { - description = "EKS cluster primary security group id." - type = string -} - -variable "worker_create_cluster_primary_security_group_rules" { - description = "Whether to create security group rules to allow communication between pods on workers and pods using the primary cluster security group." - type = bool - default = false -} - -variable "worker_sg_ingress_from_port" { - description = "Minimum port number from which pods will accept communication. Must be changed to a lower value if some pods in your cluster will expose a port lower than 1025 (e.g. 22, 80, or 443)." - type = number - default = 1025 -} - -variable "worker_create_security_group" { - description = "Whether to create a security group for the workers or attach the workers to `worker_security_group_id`." - type = bool - default = true -} - -variable "worker_security_group_id" { - description = "If provided, all workers will be attached to this security group. If not given, a security group will be created with necessary ingress/egress to work with the EKS cluster." - type = string - default = "" -} - -variable "worker_additional_security_group_ids" { - description = "A list of additional security group ids to attach to worker instances" +variable "worker_security_group_ids" { + description = "A list of security group ids to attach to worker instances" type = list(string) default = [] } diff --git a/outputs.tf b/outputs.tf index 86e1fc5a1f..ce188b9a70 100644 --- a/outputs.tf +++ b/outputs.tf @@ -78,7 +78,7 @@ output "oidc_provider_arn" { output "worker_security_group_id" { description = "Security group ID attached to the EKS workers." - value = module.worker_groups.worker_security_group_id + value = local.worker_security_group_id } output "node_groups" { diff --git a/worker_groups.tf b/worker_groups.tf index 8d8db0bb1e..66dfa75e17 100644 --- a/worker_groups.tf +++ b/worker_groups.tf @@ -6,7 +6,7 @@ module "worker_groups" { cluster_version = var.cluster_version cluster_name = var.cluster_name cluster_endpoint = coalescelist(aws_eks_cluster.this[*].endpoint, [""])[0] - cluster_auth_base64 = aws_eks_cluster.this[0].certificate_authority[0].data + cluster_auth_base64 = flatten(concat(aws_eks_cluster.this[*].certificate_authority[*].data, [""]))[0] default_iam_role_id = coalescelist(aws_iam_role.workers[*].id, [""])[0] @@ -25,12 +25,10 @@ module "worker_groups" { worker_ami_owner_id = var.worker_ami_owner_id worker_ami_owner_id_windows = var.worker_ami_owner_id_windows - cluster_security_group_id = local.cluster_security_group_id - cluster_primary_security_group_id = local.cluster_primary_security_group_id - worker_create_security_group = var.worker_create_security_group - worker_security_group_id = var.worker_security_group_id - worker_sg_ingress_from_port = var.worker_sg_ingress_from_port - worker_additional_security_group_ids = var.worker_additional_security_group_ids + worker_security_group_ids = flatten([ + local.worker_security_group_id, + var.worker_additional_security_group_ids + ]) tags = var.tags diff --git a/modules/worker_groups/sg.tf b/worker_groups_support.tf similarity index 61% rename from modules/worker_groups/sg.tf rename to worker_groups_support.tf index 1aac655b61..da8b25d414 100644 --- a/modules/worker_groups/sg.tf +++ b/worker_groups_support.tf @@ -1,9 +1,45 @@ +resource "aws_iam_role" "workers" { + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name + name = var.workers_role_name != "" ? var.workers_role_name : null + assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json + permissions_boundary = var.permissions_boundary + path = var.iam_path + force_detach_policies = true + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" + role = aws_iam_role.workers[0].name +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { + count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" + role = aws_iam_role.workers[0].name +} + +resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { + count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 + policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" + role = aws_iam_role.workers[0].name +} + +resource "aws_iam_role_policy_attachment" "workers_additional_policies" { + count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 + role = aws_iam_role.workers[0].name + policy_arn = var.workers_additional_policies[count.index] +} + resource "aws_security_group" "workers" { - count = var.worker_create_security_group && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 name_prefix = var.cluster_name description = "Security group for all nodes in the cluster." vpc_id = var.vpc_id + tags = merge( var.tags, { @@ -14,7 +50,7 @@ resource "aws_security_group" "workers" { } resource "aws_security_group_rule" "workers_egress_internet" { - count = var.worker_create_security_group && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow nodes all egress to the Internet." protocol = "-1" security_group_id = local.worker_security_group_id @@ -25,7 +61,7 @@ resource "aws_security_group_rule" "workers_egress_internet" { } resource "aws_security_group_rule" "workers_ingress_self" { - count = var.worker_create_security_group && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow node to communicate with each other." protocol = "-1" security_group_id = local.worker_security_group_id @@ -36,54 +72,54 @@ resource "aws_security_group_rule" "workers_ingress_self" { } resource "aws_security_group_rule" "workers_ingress_cluster" { - count = var.worker_create_security_group && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow workers pods to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id - source_security_group_id = var.cluster_security_group_id + source_security_group_id = local.cluster_security_group_id from_port = var.worker_sg_ingress_from_port to_port = 65535 type = "ingress" } resource "aws_security_group_rule" "workers_ingress_cluster_kubelet" { - count = var.worker_create_security_group && var.create_workers ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 + count = var.worker_create_security_group && var.create_eks ? var.worker_sg_ingress_from_port > 10250 ? 1 : 0 : 0 description = "Allow workers Kubelets to receive communication from the cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id - source_security_group_id = var.cluster_security_group_id + source_security_group_id = local.cluster_security_group_id from_port = 10250 to_port = 10250 type = "ingress" } resource "aws_security_group_rule" "workers_ingress_cluster_https" { - count = var.worker_create_security_group && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.create_eks ? 1 : 0 description = "Allow pods running extension API servers on port 443 to receive communication from cluster control plane." protocol = "tcp" security_group_id = local.worker_security_group_id - source_security_group_id = var.cluster_security_group_id + source_security_group_id = local.cluster_security_group_id from_port = 443 to_port = 443 type = "ingress" } resource "aws_security_group_rule" "workers_ingress_cluster_primary" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 description = "Allow pods running on workers to receive communication from cluster primary security group (e.g. Fargate pods)." protocol = "all" security_group_id = local.worker_security_group_id - source_security_group_id = var.cluster_primary_security_group_id + source_security_group_id = local.cluster_primary_security_group_id from_port = 0 to_port = 65535 type = "ingress" } resource "aws_security_group_rule" "cluster_primary_ingress_workers" { - count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_workers ? 1 : 0 + count = var.worker_create_security_group && var.worker_create_cluster_primary_security_group_rules && var.cluster_version >= 1.14 && var.create_eks ? 1 : 0 description = "Allow pods running on workers to send communication to cluster primary security group (e.g. Fargate pods)." protocol = "all" - security_group_id = var.cluster_primary_security_group_id + security_group_id = local.cluster_primary_security_group_id source_security_group_id = local.worker_security_group_id from_port = 0 to_port = 65535 diff --git a/workers_iam.tf b/workers_iam.tf deleted file mode 100644 index cd40988a64..0000000000 --- a/workers_iam.tf +++ /dev/null @@ -1,34 +0,0 @@ -resource "aws_iam_role" "workers" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - name_prefix = var.workers_role_name != "" ? null : aws_eks_cluster.this[0].name - name = var.workers_role_name != "" ? var.workers_role_name : null - assume_role_policy = data.aws_iam_policy_document.workers_assume_role_policy.json - permissions_boundary = var.permissions_boundary - path = var.iam_path - force_detach_policies = true - tags = var.tags -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEKSWorkerNodePolicy" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEKSWorkerNodePolicy" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEKS_CNI_Policy" { - count = var.manage_worker_iam_resources && var.attach_worker_cni_policy && var.create_eks ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEKS_CNI_Policy" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_AmazonEC2ContainerRegistryReadOnly" { - count = var.manage_worker_iam_resources && var.create_eks ? 1 : 0 - policy_arn = "${local.policy_arn_prefix}/AmazonEC2ContainerRegistryReadOnly" - role = aws_iam_role.workers[0].name -} - -resource "aws_iam_role_policy_attachment" "workers_additional_policies" { - count = var.manage_worker_iam_resources && var.create_eks ? length(var.workers_additional_policies) : 0 - role = aws_iam_role.workers[0].name - policy_arn = var.workers_additional_policies[count.index] -}