feat: Add support for enabling addons before data plane compute is cr…

…eated (#2478)

README.md

@@ -225,6 +225,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.47 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.10 |
+| <a name="requirement_time"></a> [time](#requirement\_time) | >= 0.9 |
 | <a name="requirement_tls"></a> [tls](#requirement\_tls) | >= 3.0 |
 
 ## Providers
@@ -233,6 +234,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 |------|---------|
 | <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.47 |
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | >= 2.10 |
+| <a name="provider_time"></a> [time](#provider\_time) | >= 0.9 |
 | <a name="provider_tls"></a> [tls](#provider\_tls) | >= 3.0 |
 
 ## Modules
@@ -250,6 +252,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 |------|------|
 | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource |
 | [aws_ec2_tag.cluster_primary_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ec2_tag) | resource |
+| [aws_eks_addon.before_compute](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource |
 | [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource |
 | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource |
 | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource |
@@ -266,6 +269,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [kubernetes_config_map.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map) | resource |
 | [kubernetes_config_map_v1_data.aws_auth](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1_data) | resource |
+| [time_sleep.this](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source |
 | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
@@ -326,6 +330,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | <a name="input_create_kms_key"></a> [create\_kms\_key](#input\_create\_kms\_key) | Controls if a KMS key for cluster encryption should be created | `bool` | `true` | no |
 | <a name="input_create_node_security_group"></a> [create\_node\_security\_group](#input\_create\_node\_security\_group) | Determines whether to create a security group for the node groups or use the existing `node_security_group_id` | `bool` | `true` | no |
 | <a name="input_custom_oidc_thumbprints"></a> [custom\_oidc\_thumbprints](#input\_custom\_oidc\_thumbprints) | Additional list of server certificate thumbprints for the OpenID Connect (OIDC) identity provider's server certificate(s) | `list(string)` | `[]` | no |
+| <a name="input_dataplane_wait_duration"></a> [dataplane\_wait\_duration](#input\_dataplane\_wait\_duration) | Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s)) | `string` | `"30s"` | no |
 | <a name="input_eks_managed_node_group_defaults"></a> [eks\_managed\_node\_group\_defaults](#input\_eks\_managed\_node\_group\_defaults) | Map of EKS managed node group default configurations | `any` | `{}` | no |
 | <a name="input_eks_managed_node_groups"></a> [eks\_managed\_node\_groups](#input\_eks\_managed\_node\_groups) | Map of EKS managed node group definitions to create | `any` | `{}` | no |
 | <a name="input_enable_irsa"></a> [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no |

examples/eks_managed_node_group/main.tf

@@ -63,6 +63,7 @@ module "eks" {
     }
     vpc-cni = {
       most_recent              = true
+      before_compute           = true
       service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
       configuration_values = jsonencode({
         env = {

main.tf

@@ -85,7 +85,8 @@ resource "aws_eks_cluster" "this" {
     aws_iam_role_policy_attachment.this,
     aws_security_group_rule.cluster,
     aws_security_group_rule.node,
-    aws_cloudwatch_log_group.this
+    aws_cloudwatch_log_group.this,
+    aws_iam_policy.cni_ipv6_policy,
   ]
 }
 
@@ -377,7 +378,7 @@ resource "aws_iam_policy" "cluster_encryption" {
 
 resource "aws_eks_addon" "this" {
   # Not supported on outposts
-  for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster }
+  for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster }
 
   cluster_name = aws_eks_cluster.this[0].name
   addon_name   = try(each.value.name, each.key)
@@ -403,6 +404,28 @@ resource "aws_eks_addon" "this" {
   tags = var.tags
 }
 
+resource "aws_eks_addon" "before_compute" {
+  # Not supported on outposts
+  for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster }
+
+  cluster_name = aws_eks_cluster.this[0].name
+  addon_name   = try(each.value.name, each.key)
+
+  addon_version            = try(each.value.addon_version, data.aws_eks_addon_version.this[each.key].version)
+  configuration_values     = try(each.value.configuration_values, null)
+  preserve                 = try(each.value.preserve, null)
+  resolve_conflicts        = try(each.value.resolve_conflicts, "OVERWRITE")
+  service_account_role_arn = try(each.value.service_account_role_arn, null)
+
+  timeouts {
+    create = try(each.value.timeouts.create, var.cluster_addons_timeouts.create, null)
+    update = try(each.value.timeouts.update, var.cluster_addons_timeouts.update, null)
+    delete = try(each.value.timeouts.delete, var.cluster_addons_timeouts.delete, null)
+  }
+
+  tags = var.tags
+}
+
 data "aws_eks_addon_version" "this" {
   for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster }
 

node_groups.tf

@@ -19,6 +19,25 @@ locals {
   }
 }
 
+# This sleep resource is used to provide a timed gap between the cluster creation and the downstream dependencies
+# that consume the outputs from here. Any of the values that are used as triggers can be used in dependencies
+# to ensure that the downstream resources are created after both the cluster is ready and the sleep time has passed.
+# This was primarily added to give addons that need to be configured BEFORE data plane compute resources
+# enough time to create and configure themselves before the data plane compute resources are created.
+resource "time_sleep" "this" {
+  count = var.create ? 1 : 0
+
+  create_duration = var.dataplane_wait_duration
+
+  triggers = {
+    cluster_name     = aws_eks_cluster.this[0].name
+    cluster_endpoint = aws_eks_cluster.this[0].endpoint
+    cluster_version  = aws_eks_cluster.this[0].version
+
+    cluster_certificate_authority_data = aws_eks_cluster.this[0].certificate_authority[0].data
+  }
+}
+
 ################################################################################
 # EKS IPV6 CNI Policy
 # TODO - hopefully AWS releases a managed policy which can replace this
@@ -220,7 +239,7 @@ module "fargate_profile" {
   create = try(each.value.create, true)
 
   # Fargate Profile
-  cluster_name      = aws_eks_cluster.this[0].name
+  cluster_name      = time_sleep.this[0].triggers["cluster_name"]
   cluster_ip_family = var.cluster_ip_family
   name              = try(each.value.name, each.key)
   subnet_ids        = try(each.value.subnet_ids, var.fargate_profile_defaults.subnet_ids, var.subnet_ids)
@@ -255,8 +274,8 @@ module "eks_managed_node_group" {
 
   create = try(each.value.create, true)
 
-  cluster_name      = aws_eks_cluster.this[0].name
-  cluster_version   = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version)
+  cluster_name      = time_sleep.this[0].triggers["cluster_name"]
+  cluster_version   = try(each.value.cluster_version, var.eks_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"])
   cluster_ip_family = var.cluster_ip_family
 
   # EKS Managed Node Group
@@ -286,8 +305,8 @@ module "eks_managed_node_group" {
 
   # User data
   platform                   = try(each.value.platform, var.eks_managed_node_group_defaults.platform, "linux")
-  cluster_endpoint           = try(aws_eks_cluster.this[0].endpoint, "")
-  cluster_auth_base64        = try(aws_eks_cluster.this[0].certificate_authority[0].data, "")
+  cluster_endpoint           = try(time_sleep.this[0].triggers["cluster_endpoint"], "")
+  cluster_auth_base64        = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "")
   cluster_service_ipv4_cidr  = var.cluster_service_ipv4_cidr
   enable_bootstrap_user_data = try(each.value.enable_bootstrap_user_data, var.eks_managed_node_group_defaults.enable_bootstrap_user_data, false)
   pre_bootstrap_user_data    = try(each.value.pre_bootstrap_user_data, var.eks_managed_node_group_defaults.pre_bootstrap_user_data, "")
@@ -362,7 +381,7 @@ module "self_managed_node_group" {
 
   create = try(each.value.create, true)
 
-  cluster_name      = aws_eks_cluster.this[0].name
+  cluster_name      = time_sleep.this[0].triggers["cluster_name"]
   cluster_ip_family = var.cluster_ip_family
 
   # Autoscaling Group
@@ -415,8 +434,8 @@ module "self_managed_node_group" {
 
   # User data
   platform                 = try(each.value.platform, var.self_managed_node_group_defaults.platform, "linux")
-  cluster_endpoint         = try(aws_eks_cluster.this[0].endpoint, "")
-  cluster_auth_base64      = try(aws_eks_cluster.this[0].certificate_authority[0].data, "")
+  cluster_endpoint         = try(time_sleep.this[0].triggers["cluster_endpoint"], "")
+  cluster_auth_base64      = try(time_sleep.this[0].triggers["cluster_certificate_authority_data"], "")
   pre_bootstrap_user_data  = try(each.value.pre_bootstrap_user_data, var.self_managed_node_group_defaults.pre_bootstrap_user_data, "")
   post_bootstrap_user_data = try(each.value.post_bootstrap_user_data, var.self_managed_node_group_defaults.post_bootstrap_user_data, "")
   bootstrap_extra_args     = try(each.value.bootstrap_extra_args, var.self_managed_node_group_defaults.bootstrap_extra_args, "")
@@ -436,7 +455,7 @@ module "self_managed_node_group" {
 
   ebs_optimized   = try(each.value.ebs_optimized, var.self_managed_node_group_defaults.ebs_optimized, null)
   ami_id          = try(each.value.ami_id, var.self_managed_node_group_defaults.ami_id, "")
-  cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, aws_eks_cluster.this[0].version)
+  cluster_version = try(each.value.cluster_version, var.self_managed_node_group_defaults.cluster_version, time_sleep.this[0].triggers["cluster_version"])
   instance_type   = try(each.value.instance_type, var.self_managed_node_group_defaults.instance_type, "m6i.large")
   key_name        = try(each.value.key_name, var.self_managed_node_group_defaults.key_name, null)
 

outputs.tf

@@ -19,12 +19,12 @@ output "cluster_endpoint" {
 
 output "cluster_id" {
   description = "The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts"
-  value       = try(aws_eks_cluster.this[0].cluster_id, null)
+  value       = try(aws_eks_cluster.this[0].cluster_id, "")
 }
 
 output "cluster_name" {
   description = "The name of the EKS cluster"
-  value       = try(aws_eks_cluster.this[0].name, null)
+  value       = try(aws_eks_cluster.this[0].name, "")
 }
 
 output "cluster_oidc_issuer_url" {

variables.tf

@@ -460,6 +460,12 @@ variable "cluster_encryption_policy_tags" {
   default     = {}
 }
 
+variable "dataplane_wait_duration" {
+  description = "Duration to wait after the EKS cluster has become active before creating the dataplane components (EKS managed nodegroup(s), self-managed nodegroup(s), Fargate profile(s))"
+  type        = string
+  default     = "30s"
+}
+
 ################################################################################
 # EKS Addons
 ################################################################################

versions.tf

@@ -14,5 +14,9 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = ">= 2.10"
     }
+    time = {
+      source  = "hashicorp/time"
+      version = ">= 0.9"
+    }
   }
 }