From f77e5b06606db97533a52417a9e7aa1bfc15c456 Mon Sep 17 00:00:00 2001 From: Steven Davidovitz Date: Tue, 19 Mar 2024 16:48:30 -0700 Subject: [PATCH] add a default EFS CSI driver taint to node pools https://github.com/kubernetes-sigs/aws-efs-csi-driver?tab=readme-ov-file#configure-node-startup-taint --- modules/nodes/README.md | 4 ++-- modules/nodes/variables.tf | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/nodes/README.md b/modules/nodes/README.md index 64f68105..71fd739e 100644 --- a/modules/nodes/README.md +++ b/modules/nodes/README.md @@ -43,8 +43,8 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                        = optional(string, null)
    bootstrap_extra_args       = optional(string, "")
    instance_types             = list(string)
    spot                       = optional(bool, false)
    min_per_az                 = number
    max_per_az                 = number
    max_unavailable_percentage = optional(number, 50)
    max_unavailable            = optional(number)
    desired_per_az             = number
    availability_zone_ids      = list(string)
    labels                     = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
      })), [
      {
        key    = "ebs.csi.aws.com/agent-not-ready",
        value  = "true",
        effect = "NO_EXECUTE"
      }
    ])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size       = string
      type       = string
      iops       = optional(number)
      throughput = optional(number)
    })
  }))
| `{}` | no | -| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                        = optional(string, null)
          bootstrap_extra_args       = optional(string, "")
          instance_types             = optional(list(string), ["m5.2xlarge"])
          spot                       = optional(bool, false)
          min_per_az                 = optional(number, 0)
          max_per_az                 = optional(number, 10)
          max_unavailable_percentage = optional(number, 50)
          max_unavailable            = optional(number, null)
          desired_per_az             = optional(number, 0)
          availability_zone_ids      = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [
            {
              key    = "ebs.csi.aws.com/agent-not-ready",
              value  = "true",
              effect = "NO_EXECUTE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size       = optional(number, 1000)
            type       = optional(string, "gp3")
            iops       = optional(number)
            throughput = optional(number)
            }), {
            size       = 1000
            type       = "gp3"
            iops       = null
            throughput = null
            }
          )
      }),
      platform = object(
        {
          ami                        = optional(string, null)
          bootstrap_extra_args       = optional(string, "")
          instance_types             = optional(list(string), ["m5.2xlarge"])
          spot                       = optional(bool, false)
          min_per_az                 = optional(number, 1)
          max_per_az                 = optional(number, 10)
          max_unavailable_percentage = optional(number, null)
          max_unavailable            = optional(number, 1)
          desired_per_az             = optional(number, 1)
          availability_zone_ids      = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), []
          )
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size       = optional(number, 100)
            type       = optional(string, "gp3")
            iops       = optional(number)
            throughput = optional(number)
            }), {
            size       = 100
            type       = "gp3"
            iops       = null
            throughput = null
            }
          )
      }),
      gpu = object(
        {
          ami                        = optional(string, null)
          bootstrap_extra_args       = optional(string, "")
          instance_types             = optional(list(string), ["g5.xlarge"])
          spot                       = optional(bool, false)
          min_per_az                 = optional(number, 0)
          max_per_az                 = optional(number, 10)
          max_unavailable_percentage = optional(number, 50)
          max_unavailable            = optional(number, null)
          desired_per_az             = optional(number, 0)
          availability_zone_ids      = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            },
            {
              key    = "ebs.csi.aws.com/agent-not-ready",
              value  = "true",
              effect = "NO_EXECUTE"
            }

          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size       = optional(number, 1000)
            type       = optional(string, "gp3")
            iops       = optional(number)
            throughput = optional(number)
            }), {
            size       = 1000
            type       = "gp3"
            iops       = null
            throughput = null
            }
          )
      })
  })
| n/a | yes | +| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                        = optional(string, null)
    bootstrap_extra_args       = optional(string, "")
    instance_types             = list(string)
    spot                       = optional(bool, false)
    min_per_az                 = number
    max_per_az                 = number
    max_unavailable_percentage = optional(number, 50)
    max_unavailable            = optional(number)
    desired_per_az             = number
    availability_zone_ids      = list(string)
    labels                     = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
      })), [
      {
        key    = "ebs.csi.aws.com/agent-not-ready",
        value  = "true",
        effect = "NO_EXECUTE"
      },
      {
        key    = "efs.csi.aws.com/agent-not-ready",
        value  = "true",
        effect = "NO_EXECUTE"
      }

    ])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size       = string
      type       = string
      iops       = optional(number)
      throughput = optional(number)
    })
  }))
| `{}` | no | +| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                        = optional(string, null)
          bootstrap_extra_args       = optional(string, "")
          instance_types             = optional(list(string), ["m5.2xlarge"])
          spot                       = optional(bool, false)
          min_per_az                 = optional(number, 0)
          max_per_az                 = optional(number, 10)
          max_unavailable_percentage = optional(number, 50)
          max_unavailable            = optional(number, null)
          desired_per_az             = optional(number, 0)
          availability_zone_ids      = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [
            {
              key    = "ebs.csi.aws.com/agent-not-ready",
              value  = "true",
              effect = "NO_EXECUTE"
            },
            {
              key    = "efs.csi.aws.com/agent-not-ready",
              value  = "true",
              effect = "NO_EXECUTE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size       = optional(number, 1000)
            type       = optional(string, "gp3")
            iops       = optional(number)
            throughput = optional(number)
            }), {
            size       = 1000
            type       = "gp3"
            iops       = null
            throughput = null
            }
          )
      }),
      platform = object(
        {
          ami                        = optional(string, null)
          bootstrap_extra_args       = optional(string, "")
          instance_types             = optional(list(string), ["m5.2xlarge"])
          spot                       = optional(bool, false)
          min_per_az                 = optional(number, 1)
          max_per_az                 = optional(number, 10)
          max_unavailable_percentage = optional(number, null)
          max_unavailable            = optional(number, 1)
          desired_per_az             = optional(number, 1)
          availability_zone_ids      = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), []
          )
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size       = optional(number, 100)
            type       = optional(string, "gp3")
            iops       = optional(number)
            throughput = optional(number)
            }), {
            size       = 100
            type       = "gp3"
            iops       = null
            throughput = null
            }
          )
      }),
      gpu = object(
        {
          ami                        = optional(string, null)
          bootstrap_extra_args       = optional(string, "")
          instance_types             = optional(list(string), ["g5.xlarge"])
          spot                       = optional(bool, false)
          min_per_az                 = optional(number, 0)
          max_per_az                 = optional(number, 10)
          max_unavailable_percentage = optional(number, 50)
          max_unavailable            = optional(number, null)
          desired_per_az             = optional(number, 0)
          availability_zone_ids      = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            },
            {
              key    = "ebs.csi.aws.com/agent-not-ready",
              value  = "true",
              effect = "NO_EXECUTE"
            },
            {
              key    = "efs.csi.aws.com/agent-not-ready",
              value  = "true",
              effect = "NO_EXECUTE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size       = optional(number, 1000)
            type       = optional(string, "gp3")
            iops       = optional(number)
            throughput = optional(number)
            }), {
            size       = 1000
            type       = "gp3"
            iops       = null
            throughput = null
            }
          )
      })
  })
| n/a | yes | | [eks\_info](#input\_eks\_info) | cluster = {
addons = List of addons
specs = Cluster spes. {
name = Cluster name.
endpoint = Cluster endpont.
kubernetes\_network\_config = Cluster k8s nw config.
}
version = K8s version.
arn = EKS Cluster arn.
security\_group\_id = EKS Cluster security group id.
endpoint = EKS Cluster API endpoint.
roles = Default IAM Roles associated with the EKS cluster. {
name = string
arn = string
}
custom\_roles = Custom IAM Roles associated with the EKS cluster. {
rolearn = string
username = string
groups = list(string)
}
oidc = {
arn = OIDC provider ARN.
url = OIDC provider url.
}
}
nodes = {
security\_group\_id = EKS Nodes security group id.
roles = IAM Roles associated with the EKS Nodes.{
name = string
arn = string
}
}
kubeconfig = Kubeconfig details.{
path = string
extra\_args = string
} | 
object({
    k8s_pre_setup_sh_file = string
    cluster = object({
      addons = optional(list(string), ["kube-proxy", "coredns", "vpc-cni"])
      vpc_cni = optional(object({
        prefix_delegation = optional(bool, false)
        annotate_pod_ip   = optional(bool, true)
      }))
      specs = object({
        name                      = string
        endpoint                  = string
        kubernetes_network_config = list(map(any))
        certificate_authority     = list(map(any))
      })
      version           = string
      arn               = string
      security_group_id = string
      endpoint          = string
      roles = list(object({
        name = string
        arn  = string
      }))
      custom_roles = list(object({
        rolearn  = string
        username = string
        groups   = list(string)
      }))
      oidc = object({
        arn = string
        url = string
      })
    })
    nodes = object({
      security_group_id = string
      roles = list(object({
        name = string
        arn  = string
      }))
    })
    kubeconfig = object({
      path       = string
      extra_args = string
    })
  })
| n/a | yes | | [ignore\_tags](#input\_ignore\_tags) | Tag keys to be ignored by the aws provider. | `list(string)` | `[]` | no | | [kms\_info](#input\_kms\_info) | key\_id = KMS key id.
key\_arn = KMS key arn.
enabled = KMS key is enabled | 
object({
    key_id  = string
    key_arn = string
    enabled = bool
  })
| n/a | yes | diff --git a/modules/nodes/variables.tf b/modules/nodes/variables.tf index 013255f4..d789e7e6 100644 --- a/modules/nodes/variables.tf +++ b/modules/nodes/variables.tf @@ -174,6 +174,11 @@ variable "default_node_groups" { key = "ebs.csi.aws.com/agent-not-ready", value = "true", effect = "NO_EXECUTE" + }, + { + key = "efs.csi.aws.com/agent-not-ready", + value = "true", + effect = "NO_EXECUTE" } ]) tags = optional(map(string), {}) @@ -256,8 +261,12 @@ variable "default_node_groups" { key = "ebs.csi.aws.com/agent-not-ready", value = "true", effect = "NO_EXECUTE" + }, + { + key = "efs.csi.aws.com/agent-not-ready", + value = "true", + effect = "NO_EXECUTE" } - ]) tags = optional(map(string), {}) gpu = optional(bool, null) @@ -300,7 +309,13 @@ variable "additional_node_groups" { key = "ebs.csi.aws.com/agent-not-ready", value = "true", effect = "NO_EXECUTE" + }, + { + key = "efs.csi.aws.com/agent-not-ready", + value = "true", + effect = "NO_EXECUTE" } + ]) tags = optional(map(string), {}) gpu = optional(bool, null)