From 78027f37e43c79748cd7528d3803122cb8072ed7 Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Fri, 17 Feb 2023 07:28:03 -0500 Subject: [PATCH] feat: Add support for enabling addons before data plane compute is created (#2478) --- README.md | 5 ++++ examples/eks_managed_node_group/main.tf | 1 + main.tf | 27 ++++++++++++++++-- node_groups.tf | 37 +++++++++++++++++++------ outputs.tf | 4 +-- variables.tf | 6 ++++ versions.tf | 4 +++ 7 files changed, 71 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 423546a973..f265acf94a 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [terraform](#requirement\_terraform) | >= 1.0 | | [aws](#requirement\_aws) | >= 4.47 | | [kubernetes](#requirement\_kubernetes) | >= 2.10 | +| [time](#requirement\_time) | >= 0.9 | | [tls](#requirement\_tls) | >= 3.0 | ## Providers @@ -233,6 +234,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|---------| | [aws](#provider\_aws) | >= 4.47 | | [kubernetes](#provider\_kubernetes) | >= 2.10 | +| [time](#provider\_time) | >= 0.9 | | [tls](#provider\_tls) | >= 3.0 | ## Modules @@ -250,6 +252,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple |------|------| | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_ec2_tag.cluster_primary_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource | +| [aws_eks_addon.before_compute](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | @@ -266,6 +269,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource | | [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource | +| [time_sleep.this](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -326,6 +330,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple | [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no | | [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no | | [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no | +| [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s)) | `string` | `"30s"` | no | | [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no | | [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no | diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf index 2373d88d6f..75210c00ec 100644 --- a/examples/eks_managed_node_group/main.tf +++ b/examples/eks_managed_node_group/main.tf @@ -63,6 +63,7 @@ module "eks" { } vpc-cni = { most_recent = true + before_compute = true service_account_role_arn = module.vpc_cni_irsa.iam_role_arn configuration_values = jsonencode({ env = { diff --git a/main.tf b/main.tf index 4f0ffb6ab3..604f90ce86 100644 --- a/main.tf +++ b/main.tf @@ -85,7 +85,8 @@ resource "aws_eks_cluster" "this" { aws_iam_role_policy_attachment.this, aws_security_group_rule.cluster, aws_security_group_rule.node, - aws_cloudwatch_log_group.this + aws_cloudwatch_log_group.this, + aws_iam_policy.cni_ipv6_policy, ] } @@ -377,7 +378,7 @@ resource "aws_iam_policy" "cluster_encryption" { resource "aws_eks_addon" "this" { # Not supported on outposts - for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } + for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } cluster_name = aws_eks_cluster.this[0].name addon_name = try(each.value.name, each.key) @@ -403,6 +404,28 @@ resource "aws_eks_addon" "this" { tags = var.tags } +resource "aws_eks_addon" "before_compute" { + # Not supported on outposts + for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster } + + cluster_name = aws_eks_cluster.this[0].name + addon_name = try(each.value.name, each.key) + + addon_version = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version) + configuration_values = try(each.value.configuration_values, null) + preserve = try(each.value.preserve, null) + resolve_conflicts = try(each.value.resolve_conflicts, "OVERWRITE") + service_account_role_arn = try(each.value.service_account_role_arn, null) + + timeouts { + create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null) + update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null) + delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null) + } + + tags = var.tags +} + data "aws_eks_addon_version" "this" { for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster } diff --git a/node_groups.tf b/node_groups.tf index 85c61387c4..0dacc1ed23 100644 --- a/node_groups.tf +++ b/node_groups.tf @@ -19,6 +19,25 @@ locals { } } +# This sleep resource is used to provide a timed gap between the cluster creation and the downstream dependencies +# that consume the outputs from here. Any of the values that are used as triggers can be used in dependencies +# to ensure that the downstream resources are created after both the cluster is ready and the sleep time has passed. +# This was primarily added to give addons that need to be configured BEFORE data plane compute resources +# enough time to create and configure themselves before the data plane compute resources are created. +resource "time_sleep" "this" { + count = var.create ? 1 : 0 + + create_duration = var.dataplane_wait_duration + + triggers = { + cluster_name = aws_eks_cluster.this[0].name + cluster_endpoint = aws_eks_cluster.this[0].endpoint + cluster_version = aws_eks_cluster.this[0].version + + cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data + } +} + ################################################################################ # EKS IPV6 CNI Policy # TODO - hopefully AWS releases a managed policy which can replace this @@ -220,7 +239,7 @@ module "fargate_profile" { create = try(each.value.create, true) # Fargate Profile - cluster_name = aws_eks_cluster.this[0].name + cluster_name = time_sleep.this[0].triggers["cluster_name"] cluster_ip_family = var.cluster_ip_family name = try(each.value.name, each.key) subnet_ids = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids) @@ -255,8 +274,8 @@ module "eks_managed_node_group" { create = try(each.value.create, true) - cluster_name = aws_eks_cluster.this[0].name - cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) + cluster_name = time_sleep.this[0].triggers["cluster_name"] + cluster_version = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) cluster_ip_family = var.cluster_ip_family # EKS Managed Node Group @@ -286,8 +305,8 @@ module "eks_managed_node_group" { # User data platform = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux") - cluster_endpoint = try(aws_eks_cluster.this[0].endpoint, "") - cluster_auth_base64 = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") cluster_service_ipv4_cidr = var.cluster_service_ipv4_cidr enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false) pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "") @@ -362,7 +381,7 @@ module "self_managed_node_group" { create = try(each.value.create, true) - cluster_name = aws_eks_cluster.this[0].name + cluster_name = time_sleep.this[0].triggers["cluster_name"] cluster_ip_family = var.cluster_ip_family # Autoscaling Group @@ -415,8 +434,8 @@ module "self_managed_node_group" { # User data platform = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux") - cluster_endpoint = try(aws_eks_cluster.this[0].endpoint, "") - cluster_auth_base64 = try(aws_eks_cluster.this[0].certificate_authority[0].data, "") + cluster_endpoint = try(time_sleep.this[0].triggers["cluster_endpoint"], "") + cluster_auth_base64 = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "") pre_bootstrap_user_data = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "") post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "") bootstrap_extra_args = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "") @@ -436,7 +455,7 @@ module "self_managed_node_group" { ebs_optimized = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null) ami_id = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "") - cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version) + cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"]) instance_type = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large") key_name = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null) diff --git a/outputs.tf b/outputs.tf index a0795ec736..f0cb797e14 100644 --- a/outputs.tf +++ b/outputs.tf @@ -19,12 +19,12 @@ output "cluster_endpoint" { output "cluster_id" { description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts" - value = try(aws_eks_cluster.this[0].cluster_id, null) + value = try(aws_eks_cluster.this[0].cluster_id, "") } output "cluster_name" { description = "The name of the EKS cluster" - value = try(aws_eks_cluster.this[0].name, null) + value = try(aws_eks_cluster.this[0].name, "") } output "cluster_oidc_issuer_url" { diff --git a/variables.tf b/variables.tf index d0047ff1c0..dc96661810 100644 --- a/variables.tf +++ b/variables.tf @@ -460,6 +460,12 @@ variable "cluster_encryption_policy_tags" { default = {} } +variable "dataplane_wait_duration" { + description = "Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s))" + type = string + default = "30s" +} + ################################################################################ # EKS Addons ################################################################################ diff --git a/versions.tf b/versions.tf index 5da3d23ea7..dfe867e876 100644 --- a/versions.tf +++ b/versions.tf @@ -14,5 +14,9 @@ terraform { source = "hashicorp/kubernetes" version = ">= 2.10" } + time = { + source = "hashicorp/time" + version = ">= 0.9" + } } }