diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 15d3d3efd7..145baf94e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.4 + rev: v1.83.5 hooks: - id: terraform_fmt - id: terraform_validate diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 6549a3e8d2..06b29e7bb4 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -73,7 +73,7 @@ Note that this example may create resources which cost money. Run `terraform des |------|--------|---------| | [eks](#module\_eks) | ../.. | n/a | | [karpenter](#module\_karpenter) | ../../modules/karpenter | n/a | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 4.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | ## Resources @@ -81,8 +81,8 @@ Note that this example may create resources which cost money. Run `terraform des |------|------| | [helm_release.karpenter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubectl_manifest.karpenter_example_deployment](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_node_template](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | -| [kubectl_manifest.karpenter_provisioner](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_class](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | +| [kubectl_manifest.karpenter_node_pool](https://registry.terraform.io/providers/gavinbunney/kubectl/latest/docs/resources/manifest) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index b925de7855..b8d7be97ff 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -160,10 +160,9 @@ module "karpenter" { cluster_name = module.eks.cluster_name irsa_oidc_provider_arn = module.eks.oidc_provider_arn - # Used to attach additional IAM policies to the Karpenter controller IRSA role - # policies = { - # "xxx" = "yyy" - # } + # In v0.32.0/v1beta1, Karpenter now creates the IAM instance profile + # so we disable the Terraform creation and add the necessary permissions for Karpenter IRSA + enable_karpenter_instance_profile_creation = true # Used to attach additional IAM policies to the Karpenter node IAM role iam_role_additional_policies = { @@ -182,51 +181,38 @@ resource "helm_release" "karpenter" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "v0.29.0" - - set { - name = "settings.aws.clusterName" - value = module.eks.cluster_name - } - - set { - name = "settings.aws.clusterEndpoint" - value = module.eks.cluster_endpoint - } - - set { - name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" - value = module.karpenter.irsa_arn - } - - set { - name = "settings.aws.defaultInstanceProfile" - value = module.karpenter.instance_profile_name - } - - set { - name = "settings.aws.interruptionQueueName" - value = module.karpenter.queue_name - } + version = "v0.32.1" + + values = [ + <<-EOT + settings: + clusterName: ${module.eks.cluster_name} + clusterEndpoint: ${module.eks.cluster_endpoint} + interruptionQueueName: ${module.karpenter.queue_name} + serviceAccount: + annotations: + eks.amazonaws.com/role-arn: ${module.karpenter.irsa_arn} + EOT + ] } -resource "kubectl_manifest" "karpenter_provisioner" { +resource "kubectl_manifest" "karpenter_node_class" { yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass metadata: name: default spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - providerRef: - name: default - ttlSecondsAfterEmpty: 30 + amiFamily: AL2 + role: ${module.karpenter.role_name} + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ${module.eks.cluster_name} + tags: + karpenter.sh/discovery: ${module.eks.cluster_name} YAML depends_on = [ @@ -234,23 +220,39 @@ resource "kubectl_manifest" "karpenter_provisioner" { ] } -resource "kubectl_manifest" "karpenter_node_template" { +resource "kubectl_manifest" "karpenter_node_pool" { yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1alpha1 - kind: AWSNodeTemplate + apiVersion: karpenter.sh/v1beta1 + kind: NodePool metadata: name: default spec: - subnetSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s YAML depends_on = [ - helm_release.karpenter + kubectl_manifest.karpenter_node_class ] } @@ -292,7 +294,7 @@ resource "kubectl_manifest" "karpenter_example_deployment" { module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "~> 4.0" + version = "~> 5.0" name = local.name cidr = local.vpc_cidr diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 4a7f9c7f74..7b813471ad 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -150,6 +150,7 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `true` | no | | [create\_irsa](#input\_create\_irsa) | Determines whether an IAM role for service accounts is created | `bool` | `true` | no | +| [enable\_karpenter\_instance\_profile\_creation](#input\_enable\_karpenter\_instance\_profile\_creation) | Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1) | `bool` | `false` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [iam\_role\_arn](#input\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 223c400b5d..5cf4d39fd3 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -160,6 +160,24 @@ data "aws_iam_policy_document" "irsa" { resources = [aws_sqs_queue.this[0].arn] } } + + # TODO - this will be replaced in v20.0 with the scoped policy provided by Karpenter + # https://github.com/aws/karpenter/blob/main/website/content/en/docs/upgrading/v1beta1-controller-policy.json + dynamic "statement" { + for_each = var.enable_karpenter_instance_profile_creation ? [1] : [] + + content { + actions = [ + "iam:AddRoleToInstanceProfile", + "iam:CreateInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:GetInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:TagInstanceProfile", + ] + resources = ["*"] + } + } } resource "aws_iam_policy" "irsa" { @@ -368,7 +386,7 @@ locals { } resource "aws_iam_instance_profile" "this" { - count = var.create && var.create_instance_profile ? 1 : 0 + count = var.create && var.create_instance_profile && !var.enable_karpenter_instance_profile_creation ? 1 : 0 name = var.iam_role_use_name_prefix ? null : local.iam_role_name name_prefix = var.iam_role_use_name_prefix ? "${local.iam_role_name}-" : null diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index c025237b36..4a8389671b 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -123,6 +123,12 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } +variable "enable_karpenter_instance_profile_creation" { + description = "Determines whether Karpenter will be allowed to create the IAM instance profile (v1beta1) or if Terraform will (v1alpha1)" + type = bool + default = false +} + ################################################################################ # Node Termination Queue ################################################################################