From 472fc2b8637171b54292dc99cfff550bee1bd185 Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 16 Nov 2022 20:34:40 -0600 Subject: [PATCH 01/23] add helm login for example --- examples/complete/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/complete/README.md b/examples/complete/README.md index 58f698c..16e8aab 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -24,8 +24,9 @@ Ensure that you have the following tools installed locally: To provision this example: ```sh +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws terraform init -terraform apply +terraform apply -var aws_region= # defaults to us-west-2 ``` Enter `yes` at command prompt to apply From 9e9e4822e3f506d6c04455b056a302039ffc198c Mon Sep 17 00:00:00 2001 From: Gu Date: Thu, 17 Nov 2022 08:49:38 -0600 Subject: [PATCH 02/23] add ecr token --- examples/complete/README.md | 1 - examples/complete/main.tf | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/complete/README.md b/examples/complete/README.md index 16e8aab..7256353 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -24,7 +24,6 @@ Ensure that you have the following tools installed locally: To provision this example: ```sh -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws terraform init terraform apply -var aws_region= # defaults to us-west-2 ``` diff --git a/examples/complete/main.tf b/examples/complete/main.tf index e1e568d..9576cc8 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -20,6 +20,8 @@ data "aws_eks_cluster_auth" "this" { name = module.eks_blueprints.eks_cluster_id } +data "aws_ecr_authorization_token" "token" {} + data "aws_availability_zones" "available" {} data "aws_caller_identity" "current" {} data "aws_partition" "current" {} From e884919c009a9f26e4ca6775c6ee68d9141009c5 Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 15:23:32 -0600 Subject: [PATCH 03/23] add emr container ack --- README.md | 3 ++ main.tf | 144 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/README.md b/README.md index 8b4f829..400861a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ module "eks_ack_addons" { enable_s3 = true enable_rds = true enable_amp = true + enable_emrcontainers = true tags = { Environment = "dev" @@ -51,6 +52,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [amp](#module\_amp) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | | [api\_gatewayv2](#module\_api\_gatewayv2) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | | [dynamodb](#module\_dynamodb) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | +| [emrcontainers](#module\_emrcontainers) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | | [rds](#module\_rds) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | | [s3](#module\_s3) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | @@ -65,6 +67,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.api_gatewayv2_admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.api_gatewayv2_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.dynamodb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | diff --git a/main.tf b/main.tf index 41a199e..81411cd 100644 --- a/main.tf +++ b/main.tf @@ -372,3 +372,147 @@ data "aws_iam_policy" "amp" { name = "AmazonPrometheusFullAccess" } + +################################################################################ +# EMR Containers +################################################################################ + +locals { + emr_name = "ack-emrcontainers" +} + +module "emrcontainers" { + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.0" + + count = var.enable_emrcontainers ? 1 : 0 + + helm_config = merge( + { + name = local.emr_name + chart = "emrcontainers-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0-stable" + namespace = local.emr_name + description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" + values = [ + # shortens pod name from `ack-emrcontainers-emrcontainers-chart-xxxxxxxxxxxxx` to `ack-emrcontainers-xxxxxxxxxxxxx` + <<-EOT + nameOverride: ack-emrcontainers + EOT + ] + }, + var.emrcontainers_helm_config + ) + + set_values = [ + { + name = "serviceAccount.name" + value = local.emr_name + }, + { + name = "serviceAccount.create" + value = false + }, + { + name = "aws.region" + value = local.region + } + ] + + irsa_config = { + create_kubernetes_namespace = true + kubernetes_namespace = try(var.emrcontainers_helm_config.namespace, local.emr_name) + + create_kubernetes_service_account = true + kubernetes_service_account = local.emr_name + + irsa_iam_policies = [data.aws_iam_policy.emrcontainers[0].arn] + } + + addon_context = local.addon_context +} + +data "aws_iam_policy" "emrcontainers" { + count = var.enable_emrcontainers ? 1 : 0 + + name = format("%s-%s", local.emr_name, "controller-iam-policies") + description = "IAM policy for EMRcontainers controller" + path = "/" + policy = data.aws_iam_policy_document.emrcontainers.json +} + +data "aws_iam_policy_document" "emrcontainers" { + Statement { + effect = "Allow" + actions = [ + "iam:CreateServiceLinkedRole" + ] + resources = "*" + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["emr-containers.amazonaws.com"] + } + } + + Statement { + effect = "Allow" + actions = [ + "emr-containers:CreateVirtualCluster", + "emr-containers:ListVirtualClusters", + "emr-containers:DescribeVirtualCluster", + "emr-containers:DeleteVirtualCluster" + ] + resources = "*" + } + + Statement { + effect = "Allow" + actions = [ + "emr-containers:StartJobRun", + "emr-containers:ListJobRuns", + "emr-containers:DescribeJobRun", + "emr-containers:CancelJobRun" + ] + + resources = "*" + } + + Statement { + effect = "Allow" + actions = [ + "emr-containers:DescribeJobRun", + "emr-containers:TagResource", + "elasticmapreduce:CreatePersistentAppUI", + "elasticmapreduce:DescribePersistentAppUI", + "elasticmapreduce:GetPersistentAppUIPresignedURL" + ] + + resources = "*" + } + + Statement { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:ListBucket" + ] + + resources = "*" + } + + Statement { + effect = "Allow" + actions = [ + "logs:Get*", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams" + ] + resources = "*" + } + +} + + + From d85232f56ec5ac69e577e962ad37cd0ea21b8bf9 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 21:39:30 +0000 Subject: [PATCH 04/23] add emr container --- main.tf | 140 +++++++++++++++++++++++++-------------------------- variables.tf | 16 ++++++ 2 files changed, 85 insertions(+), 71 deletions(-) diff --git a/main.tf b/main.tf index 81411cd..eed691b 100644 --- a/main.tf +++ b/main.tf @@ -441,78 +441,76 @@ data "aws_iam_policy" "emrcontainers" { policy = data.aws_iam_policy_document.emrcontainers.json } +// inline policy providered by ack https://raw.githubusercontent.com/aws-controllers-k8s/emrcontainers-controller/main/config/iam/recommended-inline-policy data "aws_iam_policy_document" "emrcontainers" { - Statement { - effect = "Allow" - actions = [ - "iam:CreateServiceLinkedRole" - ] - resources = "*" - - condition { - test = "StringLike" - variable = "iam:AWSServiceName" - values = ["emr-containers.amazonaws.com"] - } - } - - Statement { - effect = "Allow" - actions = [ - "emr-containers:CreateVirtualCluster", - "emr-containers:ListVirtualClusters", - "emr-containers:DescribeVirtualCluster", - "emr-containers:DeleteVirtualCluster" - ] - resources = "*" - } - - Statement { - effect = "Allow" - actions = [ - "emr-containers:StartJobRun", - "emr-containers:ListJobRuns", - "emr-containers:DescribeJobRun", - "emr-containers:CancelJobRun" - ] - - resources = "*" - } - - Statement { - effect = "Allow" - actions = [ - "emr-containers:DescribeJobRun", - "emr-containers:TagResource", - "elasticmapreduce:CreatePersistentAppUI", - "elasticmapreduce:DescribePersistentAppUI", - "elasticmapreduce:GetPersistentAppUIPresignedURL" - ] - - resources = "*" - } - - Statement { - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket" - ] - - resources = "*" - } - - Statement { - effect = "Allow" - actions = [ - "logs:Get*", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams" - ] - resources = "*" - } - -} + statement { + effect = "Allow" + actions = [ + "iam:CreateServiceLinkedRole" + ] + resources = "*" + + condition { + test = "StringLike" + variable = "iam:AWSServiceName" + values = ["emr-containers.amazonaws.com"] + } + } + + statement { + effect = "Allow" + actions = [ + "emr-containers:CreateVirtualCluster", + "emr-containers:ListVirtualClusters", + "emr-containers:DescribeVirtualCluster", + "emr-containers:DeleteVirtualCluster" + ] + resources = "*" + } + + statement { + effect = "Allow" + actions = [ + "emr-containers:StartJobRun", + "emr-containers:ListJobRuns", + "emr-containers:DescribeJobRun", + "emr-containers:CancelJobRun" + ] + + resources = "*" + } + + statement { + effect = "Allow" + actions = [ + "emr-containers:DescribeJobRun", + "emr-containers:TagResource", + "elasticmapreduce:CreatePersistentAppUI", + "elasticmapreduce:DescribePersistentAppUI", + "elasticmapreduce:GetPersistentAppUIPresignedURL" + ] + resources = "*" + } + statement { + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:ListBucket" + ] + resources = "*" + } + + statement { + effect = "Allow" + actions = [ + "logs:Get*", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams" + ] + resources = "*" + } + +} diff --git a/variables.tf b/variables.tf index b60e031..9559be6 100644 --- a/variables.tf +++ b/variables.tf @@ -91,6 +91,22 @@ variable "rds_helm_config" { default = {} } +################################################################################ +# EMR Containers +################################################################################ + +variable "enable_emrcontainers" { + description = "Enable ACK EMR container add-on" + type = bool + default = false +} + +variable "emrcontainers_helm_config" { + description = "ACK EMR container Helm Chart config" + type = any + default = {} +} + ################################################################################ # AMP ################################################################################ From 337aaf4c2d15f9aa6d52c8f2256c01bba22552c6 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 21:50:23 +0000 Subject: [PATCH 05/23] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 400861a..eb4cadd 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.api_gatewayv2_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.dynamodb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy.rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | @@ -92,7 +93,8 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [rds\_helm\_config](#input\_rds\_helm\_config) | ACK rds Helm Chart config | `any` | `{}` | no | | [s3\_helm\_config](#input\_s3\_helm\_config) | ACK s3 Helm Chart config | `any` | `{}` | no | | [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no | - +| [emrcontainers\_helm\_config](#input\_emrcontainers\_helm\_config) | ACK EMR container Helm Chart config | `any` | `{}` | no | +| [enable\_emrcontainers](#input\_enable\_emrcontainers) | Enable ACK EMR container add-on | `bool` | `false` | no | ## Outputs No outputs. From d2ff61e2554b8a1b649aa57d0f0795f20bb035f7 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 21:57:57 +0000 Subject: [PATCH 06/23] update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index eb4cadd..10de0a5 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,9 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.api_gatewayv2_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.dynamodb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | -| [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy.rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | @@ -83,9 +83,11 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [cluster\_id](#input\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes | | [data\_plane\_wait\_arn](#input\_data\_plane\_wait\_arn) | Addon deployment will not proceed until this value is known. Set to node group/Fargate profile ARN to wait for data plane to be ready before provisioning addons | `string` | `""` | no | | [dynamodb\_helm\_config](#input\_dynamodb\_helm\_config) | ACK dynamodb Helm Chart config | `any` | `{}` | no | +| [emrcontainers\_helm\_config](#input\_emrcontainers\_helm\_config) | ACK EMR container Helm Chart config | `any` | `{}` | no | | [enable\_amp](#input\_enable\_amp) | Enable ACK amp add-on | `bool` | `false` | no | | [enable\_api\_gatewayv2](#input\_enable\_api\_gatewayv2) | Enable ACK API gateway v2 add-on | `bool` | `false` | no | | [enable\_dynamodb](#input\_enable\_dynamodb) | Enable ACK dynamodb add-on | `bool` | `false` | no | +| [enable\_emrcontainers](#input\_enable\_emrcontainers) | Enable ACK EMR container add-on | `bool` | `false` | no | | [enable\_rds](#input\_enable\_rds) | Enable ACK rds add-on | `bool` | `false` | no | | [enable\_s3](#input\_enable\_s3) | Enable ACK s3 add-on | `bool` | `false` | no | | [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `""` | no | @@ -93,8 +95,6 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [rds\_helm\_config](#input\_rds\_helm\_config) | ACK rds Helm Chart config | `any` | `{}` | no | | [s3\_helm\_config](#input\_s3\_helm\_config) | ACK s3 Helm Chart config | `any` | `{}` | no | | [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no | -| [emrcontainers\_helm\_config](#input\_emrcontainers\_helm\_config) | ACK EMR container Helm Chart config | `any` | `{}` | no | -| [enable\_emrcontainers](#input\_enable\_emrcontainers) | Enable ACK EMR container add-on | `bool` | `false` | no | ## Outputs No outputs. From b4fb8486b237d20b242a43f77bbb5c0e74adef46 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 22:02:05 +0000 Subject: [PATCH 07/23] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 10de0a5..19bee71 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [rds\_helm\_config](#input\_rds\_helm\_config) | ACK rds Helm Chart config | `any` | `{}` | no | | [s3\_helm\_config](#input\_s3\_helm\_config) | ACK s3 Helm Chart config | `any` | `{}` | no | | [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no | + ## Outputs No outputs. From be36045a56fd5cba0b4eabc031a00c896317100d Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 22:19:41 +0000 Subject: [PATCH 08/23] change emr policy --- main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/main.tf b/main.tf index eed691b..4ebdc76 100644 --- a/main.tf +++ b/main.tf @@ -448,7 +448,7 @@ data "aws_iam_policy_document" "emrcontainers" { actions = [ "iam:CreateServiceLinkedRole" ] - resources = "*" + resources = ["*"] condition { test = "StringLike" @@ -465,7 +465,7 @@ data "aws_iam_policy_document" "emrcontainers" { "emr-containers:DescribeVirtualCluster", "emr-containers:DeleteVirtualCluster" ] - resources = "*" + resources = ["*"] } statement { @@ -477,7 +477,7 @@ data "aws_iam_policy_document" "emrcontainers" { "emr-containers:CancelJobRun" ] - resources = "*" + resources = ["*"] } statement { @@ -490,7 +490,7 @@ data "aws_iam_policy_document" "emrcontainers" { "elasticmapreduce:GetPersistentAppUIPresignedURL" ] - resources = "*" + resources = ["*"] } statement { @@ -500,7 +500,7 @@ data "aws_iam_policy_document" "emrcontainers" { "s3:ListBucket" ] - resources = "*" + resources = ["*"] } statement { @@ -510,7 +510,7 @@ data "aws_iam_policy_document" "emrcontainers" { "logs:DescribeLogGroups", "logs:DescribeLogStreams" ] - resources = "*" + resources = ["*"] } } From a9244a540da4073f366ef05fd0ec17ad15ee617a Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 16:28:28 -0600 Subject: [PATCH 09/23] change emr policy --- README.md | 2 +- main.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 19bee71..f8c2fc1 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.api_gatewayv2_admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.api_gatewayv2_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.dynamodb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | -| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | resource | | [aws_iam_policy.rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | diff --git a/main.tf b/main.tf index 4ebdc76..e4396ba 100644 --- a/main.tf +++ b/main.tf @@ -426,13 +426,13 @@ module "emrcontainers" { create_kubernetes_service_account = true kubernetes_service_account = local.emr_name - irsa_iam_policies = [data.aws_iam_policy.emrcontainers[0].arn] + irsa_iam_policies = [data.aws_iam_policy.emrcontainers.arn] } addon_context = local.addon_context } -data "aws_iam_policy" "emrcontainers" { +resource "aws_iam_policy" "emrcontainers" { count = var.enable_emrcontainers ? 1 : 0 name = format("%s-%s", local.emr_name, "controller-iam-policies") From d67681dcaf251738151afe3d33b1f4e4ed3a6738 Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 16:30:17 -0600 Subject: [PATCH 10/23] change emr policy --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index e4396ba..700e41a 100644 --- a/main.tf +++ b/main.tf @@ -426,7 +426,7 @@ module "emrcontainers" { create_kubernetes_service_account = true kubernetes_service_account = local.emr_name - irsa_iam_policies = [data.aws_iam_policy.emrcontainers.arn] + irsa_iam_policies = [aws_iam_policy.emrcontainers.arn] } addon_context = local.addon_context From f934cdbfa0e0e0dd3707cc2e0df9593eff2c77c4 Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 16:32:01 -0600 Subject: [PATCH 11/23] change emr policy --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 700e41a..d2a5d07 100644 --- a/main.tf +++ b/main.tf @@ -426,7 +426,7 @@ module "emrcontainers" { create_kubernetes_service_account = true kubernetes_service_account = local.emr_name - irsa_iam_policies = [aws_iam_policy.emrcontainers.arn] + irsa_iam_policies = [aws_iam_policy.emrcontainers[0].arn] } addon_context = local.addon_context From 07d30bff5d30dd0ad3d2d1c822feed8dec2bee52 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 22:35:50 +0000 Subject: [PATCH 12/23] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f8c2fc1..32f901b 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | Name | Type | |------|------| +| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | resource | | [time_sleep.dataplane](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | @@ -67,7 +68,6 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.api_gatewayv2_admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.api_gatewayv2_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.dynamodb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | -| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | resource | | [aws_iam_policy.rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | From 6aade8f6472c16c9e44154b973c7cae07c56d543 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Fri, 9 Dec 2022 22:39:04 +0000 Subject: [PATCH 13/23] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32f901b..991d8a8 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | Name | Type | |------|------| -| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | resource | +| [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [time_sleep.dataplane](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | From f0016e344a304b301c7f3c8f1f5d8933a7f1a11e Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 17:21:42 -0600 Subject: [PATCH 14/23] change emr ack version --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index d2a5d07..ed45d7e 100644 --- a/main.tf +++ b/main.tf @@ -391,7 +391,7 @@ module "emrcontainers" { name = local.emr_name chart = "emrcontainers-chart" repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0-stable" + version = "v0.1.0" namespace = local.emr_name description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" values = [ From 9510d9a5f700578c9aa3584360abf271f184d112 Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 18:37:47 -0600 Subject: [PATCH 15/23] change emr version back --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index ed45d7e..d2a5d07 100644 --- a/main.tf +++ b/main.tf @@ -391,7 +391,7 @@ module "emrcontainers" { name = local.emr_name chart = "emrcontainers-chart" repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0.1.0" + version = "v0-stable" namespace = local.emr_name description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" values = [ From 03ba0342279b96695177d70748ef7d124ce06920 Mon Sep 17 00:00:00 2001 From: Gu Date: Fri, 9 Dec 2022 18:58:02 -0600 Subject: [PATCH 16/23] add emr into example --- examples/complete/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index ab6d7f5..d39ed45 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -103,7 +103,8 @@ module "eks_ack_addons" { enable_s3 = true enable_rds = true enable_amp = true - + enable_emrcontainers = true + tags = local.tags } From 2986ec6969503b9588ce5569040d2d1063292136 Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 14 Dec 2022 12:55:55 -0600 Subject: [PATCH 17/23] add ecr login --- examples/complete/main.tf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index d39ed45..37d02c0 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -20,7 +20,18 @@ data "aws_eks_cluster_auth" "this" { name = module.eks_blueprints.eks_cluster_id } +# Equivalent of aws ecr get-login data "aws_ecr_authorization_token" "token" {} +# Multiple docker push commands can be run against a single token +resource "null_resource" "renew_ecr_token" { + triggers = { + token_expired = data.aws_ecr_authorization_token.ecr_token.expires_at + } + + provisioner "local-exec" { + command = "echo ${data.aws_ecr_authorization_token.ecr_token.password} | docker login --username ${data.aws_ecr_authorization_token.ecr_token.user_name} --password-stdin ${data.aws_ecr_authorization_token.ecr_token.proxy_endpoint}" + } +} data "aws_availability_zones" "available" {} data "aws_caller_identity" "current" {} From 2fcd39a9dfff4cee338ccd79cbe4f48c39bf4d86 Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 14 Dec 2022 13:00:23 -0600 Subject: [PATCH 18/23] add ecr login --- examples/complete/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 37d02c0..ea207ba 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -21,7 +21,7 @@ data "aws_eks_cluster_auth" "this" { } # Equivalent of aws ecr get-login -data "aws_ecr_authorization_token" "token" {} +data "aws_ecr_authorization_token" "ecr_token" {} # Multiple docker push commands can be run against a single token resource "null_resource" "renew_ecr_token" { triggers = { From 2837db7780ad5517814ba4a4c17583bdc8bdce4b Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 14 Dec 2022 21:07:50 -0600 Subject: [PATCH 19/23] put ecr token into helm --- examples/complete/main.tf | 13 ------------- main.tf | 5 +++++ 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index ea207ba..69ec9ef 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -20,19 +20,6 @@ data "aws_eks_cluster_auth" "this" { name = module.eks_blueprints.eks_cluster_id } -# Equivalent of aws ecr get-login -data "aws_ecr_authorization_token" "ecr_token" {} -# Multiple docker push commands can be run against a single token -resource "null_resource" "renew_ecr_token" { - triggers = { - token_expired = data.aws_ecr_authorization_token.ecr_token.expires_at - } - - provisioner "local-exec" { - command = "echo ${data.aws_ecr_authorization_token.ecr_token.password} | docker login --username ${data.aws_ecr_authorization_token.ecr_token.user_name} --password-stdin ${data.aws_ecr_authorization_token.ecr_token.proxy_endpoint}" - } -} - data "aws_availability_zones" "available" {} data "aws_caller_identity" "current" {} data "aws_partition" "current" {} diff --git a/main.tf b/main.tf index d2a5d07..39aadeb 100644 --- a/main.tf +++ b/main.tf @@ -6,6 +6,9 @@ data "aws_eks_cluster" "this" { name = local.cluster_id } +# Equivalent of aws ecr get-login +data "aws_ecrpublic_authorization_token" "token" {} + locals { # this makes downstream resources wait for data plane to be ready cluster_id = time_sleep.dataplane.triggers["cluster_id"] @@ -393,6 +396,8 @@ module "emrcontainers" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0-stable" namespace = local.emr_name + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" values = [ # shortens pod name from `ack-emrcontainers-emrcontainers-chart-xxxxxxxxxxxxx` to `ack-emrcontainers-xxxxxxxxxxxxx` From d9f9a9f394e4a8646ac05bb117c9d877a94c49f6 Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 14 Dec 2022 21:12:05 -0600 Subject: [PATCH 20/23] add ecr login --- main.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/main.tf b/main.tf index 39aadeb..903e697 100644 --- a/main.tf +++ b/main.tf @@ -60,6 +60,8 @@ module "api_gatewayv2" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.4" namespace = local.api_gatewayv2_name + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password description = "ACK API Gateway Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-api-gatewayv2-apigatewayv2-chart-xxxxxxxxxxxxx` to `ack-api-gatewayv2-xxxxxxxxxxxxx` @@ -134,6 +136,8 @@ module "dynamodb" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0-stable" namespace = local.dynamodb_name + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password description = "ACK DynamoDB Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-dynamodb-dynamodb-chart-xxxxxxxxxxxxx` to `ack-dynamodb-xxxxxxxxxxxxx` @@ -199,6 +203,8 @@ module "s3" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.5" namespace = local.s3_name + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password description = "ACK S3 Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-s3-s3-chart-xxxxxxxxxxxxx` to `ack-s3-xxxxxxxxxxxxx` @@ -264,6 +270,8 @@ module "rds" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.1" namespace = local.rds_name + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password create_namespace = true description = "ACK RDS Controller v2 Helm chart deployment configuration" values = [ @@ -330,6 +338,8 @@ module "amp" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.1" namespace = local.amp_name + repository_username = data.aws_ecrpublic_authorization_token.token.user_name + repository_password = data.aws_ecrpublic_authorization_token.token.password create_namespace = true description = "ACK amp Controller v2 Helm chart deployment configuration" values = [ From 6643881bea684882340be5b1dade3c6eed75492f Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Thu, 15 Dec 2022 03:21:17 +0000 Subject: [PATCH 21/23] update format --- examples/complete/main.tf | 2 +- main.tf | 76 +++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 69ec9ef..db12d8d 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -102,7 +102,7 @@ module "eks_ack_addons" { enable_rds = true enable_amp = true enable_emrcontainers = true - + tags = local.tags } diff --git a/main.tf b/main.tf index 903e697..86bdc1d 100644 --- a/main.tf +++ b/main.tf @@ -55,14 +55,14 @@ module "api_gatewayv2" { helm_config = merge( { - name = local.api_gatewayv2_name - chart = "apigatewayv2-chart" - repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0.1.4" - namespace = local.api_gatewayv2_name + name = local.api_gatewayv2_name + chart = "apigatewayv2-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0.1.4" + namespace = local.api_gatewayv2_name repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password - description = "ACK API Gateway Controller v2 Helm chart deployment configuration" + description = "ACK API Gateway Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-api-gatewayv2-apigatewayv2-chart-xxxxxxxxxxxxx` to `ack-api-gatewayv2-xxxxxxxxxxxxx` <<-EOT @@ -131,14 +131,14 @@ module "dynamodb" { helm_config = merge( { - name = local.dynamodb_name - chart = "dynamodb-chart" - repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0-stable" - namespace = local.dynamodb_name + name = local.dynamodb_name + chart = "dynamodb-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0-stable" + namespace = local.dynamodb_name repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password - description = "ACK DynamoDB Controller v2 Helm chart deployment configuration" + description = "ACK DynamoDB Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-dynamodb-dynamodb-chart-xxxxxxxxxxxxx` to `ack-dynamodb-xxxxxxxxxxxxx` <<-EOT @@ -198,14 +198,14 @@ module "s3" { helm_config = merge( { - name = local.s3_name - chart = "s3-chart" - repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0.1.5" - namespace = local.s3_name + name = local.s3_name + chart = "s3-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0.1.5" + namespace = local.s3_name repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password - description = "ACK S3 Controller v2 Helm chart deployment configuration" + description = "ACK S3 Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-s3-s3-chart-xxxxxxxxxxxxx` to `ack-s3-xxxxxxxxxxxxx` <<-EOT @@ -265,15 +265,15 @@ module "rds" { helm_config = merge( { - name = local.rds_name - chart = "rds-chart" - repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0.1.1" - namespace = local.rds_name + name = local.rds_name + chart = "rds-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0.1.1" + namespace = local.rds_name repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password - create_namespace = true - description = "ACK RDS Controller v2 Helm chart deployment configuration" + create_namespace = true + description = "ACK RDS Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-rds-rds-chart-xxxxxxxxxxxxx` to `ack-rds-xxxxxxxxxxxxx` <<-EOT @@ -333,15 +333,15 @@ module "amp" { helm_config = merge( { - name = local.amp_name - chart = "prometheusservice-chart" - repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0.1.1" - namespace = local.amp_name + name = local.amp_name + chart = "prometheusservice-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0.1.1" + namespace = local.amp_name repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password - create_namespace = true - description = "ACK amp Controller v2 Helm chart deployment configuration" + create_namespace = true + description = "ACK amp Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-amp-amp-chart-xxxxxxxxxxxxx` to `ack-amp-xxxxxxxxxxxxx` <<-EOT @@ -401,14 +401,14 @@ module "emrcontainers" { helm_config = merge( { - name = local.emr_name - chart = "emrcontainers-chart" - repository = "oci://public.ecr.aws/aws-controllers-k8s" - version = "v0-stable" - namespace = local.emr_name + name = local.emr_name + chart = "emrcontainers-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0-stable" + namespace = local.emr_name repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password - description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" + description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" values = [ # shortens pod name from `ack-emrcontainers-emrcontainers-chart-xxxxxxxxxxxxx` to `ack-emrcontainers-xxxxxxxxxxxxx` <<-EOT From 1b27ab032d3f928edc25bdcda0be535b75c1d7e1 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Thu, 15 Dec 2022 03:27:11 +0000 Subject: [PATCH 22/23] update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 991d8a8..11a2b2f 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [time_sleep.dataplane](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [aws_iam_policy.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.api_gatewayv2_admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | From 1e0a4518ec72221a552e44ad70ad9b81af364fc2 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Mon, 19 Dec 2022 19:49:15 +0000 Subject: [PATCH 23/23] pre-commit fix --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 86bdc1d..3c6299c 100644 --- a/main.tf +++ b/main.tf @@ -456,7 +456,7 @@ resource "aws_iam_policy" "emrcontainers" { policy = data.aws_iam_policy_document.emrcontainers.json } -// inline policy providered by ack https://raw.githubusercontent.com/aws-controllers-k8s/emrcontainers-controller/main/config/iam/recommended-inline-policy +# inline policy providered by ack https://raw.githubusercontent.com/aws-controllers-k8s/emrcontainers-controller/main/config/iam/recommended-inline-policy data "aws_iam_policy_document" "emrcontainers" { statement { effect = "Allow"