From 93b2aa3dbaca742e6ed627256f747c9b3326d43d Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 19 Dec 2022 15:35:19 -0600 Subject: [PATCH 01/26] add ecr user and token as variables --- README.md | 2 ++ examples/complete/main.tf | 17 +++++++++++++++++ main.tf | 30 +++++++++++++++--------------- variables.tf | 10 ++++++++++ 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 11a2b2f..cb9a5df 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ module "eks_ack_addons" { source = "aws-ia/eks-ack-addons/aws" cluster_id = "example-ack" + ecrpublic_username = local.ecrpublic_username + ecrpublic_token = local.ecrpublic_token enable_api_gatewayv2 = true enable_dynamodb = true diff --git a/examples/complete/main.tf b/examples/complete/main.tf index db12d8d..9a4d106 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -2,6 +2,17 @@ provider "aws" { region = var.aws_region } +# This provider is required for ECR to autheticate with public repos. Please note ECR authetication requires us-east-1 as region hence its hardcoded below. +# If your region is same as us-east-1 then you can just use one aws provider +provider "aws" { + alias = ecr + region = "us-east-1" +} + +data "aws_ecrpublic_authorization_token" "token" { + provider = aws.ecr +} + provider "kubernetes" { host = module.eks_blueprints.eks_cluster_endpoint cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) @@ -26,6 +37,9 @@ data "aws_partition" "current" {} locals { name = "ack-eks-${basename(path.cwd)}" + ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name + ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password + vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -92,6 +106,9 @@ module "eks_ack_addons" { source = "../../" cluster_id = module.eks_blueprints.eks_cluster_id + ecrpublic_username = local.ecrpublic_username + ecrpublic_token = local.ecrpublic_token + # Wait for data plane to be ready data_plane_wait_arn = module.eks_blueprints.managed_node_group_arn[0] diff --git a/main.tf b/main.tf index 3c6299c..b6be4d0 100644 --- a/main.tf +++ b/main.tf @@ -6,15 +6,15 @@ data "aws_eks_cluster" "this" { name = local.cluster_id } -# Equivalent of aws ecr get-login -data "aws_ecrpublic_authorization_token" "token" {} - locals { # this makes downstream resources wait for data plane to be ready cluster_id = time_sleep.dataplane.triggers["cluster_id"] region = data.aws_region.current.name eks_oidc_issuer_url = replace(data.aws_eks_cluster.this.identity[0].oidc[0].issuer, "https://", "") + + ecrpublic_username = var.ecrpublic_username + ecrpublic_token = var.ecrpublic_token addon_context = { aws_caller_identity_account_id = data.aws_caller_identity.current.account_id @@ -60,8 +60,8 @@ module "api_gatewayv2" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.4" namespace = local.api_gatewayv2_name - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + repository_username = local.ecrpublic_username + repository_password = local.ecrpublic_token description = "ACK API Gateway Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-api-gatewayv2-apigatewayv2-chart-xxxxxxxxxxxxx` to `ack-api-gatewayv2-xxxxxxxxxxxxx` @@ -136,8 +136,8 @@ module "dynamodb" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0-stable" namespace = local.dynamodb_name - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + repository_username = local.ecrpublic_username + repository_password = local.ecrpublic_token description = "ACK DynamoDB Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-dynamodb-dynamodb-chart-xxxxxxxxxxxxx` to `ack-dynamodb-xxxxxxxxxxxxx` @@ -203,8 +203,8 @@ module "s3" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.5" namespace = local.s3_name - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + repository_username = local.ecrpublic_username + repository_password = local.ecrpublic_token description = "ACK S3 Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-s3-s3-chart-xxxxxxxxxxxxx` to `ack-s3-xxxxxxxxxxxxx` @@ -270,8 +270,8 @@ module "rds" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.1" namespace = local.rds_name - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + repository_username = local.ecrpublic_username + repository_password = local.ecrpublic_token create_namespace = true description = "ACK RDS Controller v2 Helm chart deployment configuration" values = [ @@ -338,8 +338,8 @@ module "amp" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.1" namespace = local.amp_name - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + repository_username = local.ecrpublic_username + repository_password = local.ecrpublic_token create_namespace = true description = "ACK amp Controller v2 Helm chart deployment configuration" values = [ @@ -406,8 +406,8 @@ module "emrcontainers" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0-stable" namespace = local.emr_name - repository_username = data.aws_ecrpublic_authorization_token.token.user_name - repository_password = data.aws_ecrpublic_authorization_token.token.password + repository_username = local.ecrpublic_username + repository_password = local.ecrpublic_token description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" values = [ # shortens pod name from `ack-emrcontainers-emrcontainers-chart-xxxxxxxxxxxxx` to `ack-emrcontainers-xxxxxxxxxxxxx` diff --git a/variables.tf b/variables.tf index 9559be6..6bd8f94 100644 --- a/variables.tf +++ b/variables.tf @@ -3,6 +3,16 @@ variable "cluster_id" { type = string } +variable "ecrpublic_username" { + description = "EKS Cluster Id" + type = string +} + +variable "ecrpublic_token" { + description = "EKS Cluster Id" + type = string +} + variable "data_plane_wait_arn" { description = "Addon deployment will not proceed until this value is known. Set to node group/Fargate profile ARN to wait for data plane to be ready before provisioning addons" type = string From a0695b43c1b8b894301673ad501fa184e4b7424a Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 19 Dec 2022 15:41:41 -0600 Subject: [PATCH 02/26] add ecr user and token as variables --- examples/complete/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 9a4d106..be860e5 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -5,7 +5,7 @@ provider "aws" { # This provider is required for ECR to autheticate with public repos. Please note ECR authetication requires us-east-1 as region hence its hardcoded below. # If your region is same as us-east-1 then you can just use one aws provider provider "aws" { - alias = ecr + alias = "ecr" region = "us-east-1" } From 50cceba199b7b9c318505d7ef2a0aa30bd002af3 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 19 Dec 2022 21:45:46 +0000 Subject: [PATCH 03/26] precommit fix --- examples/complete/main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index be860e5..814de8f 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -9,7 +9,7 @@ provider "aws" { region = "us-east-1" } -data "aws_ecrpublic_authorization_token" "token" { +data "aws_ecrpublic_authorization_token" "token" { provider = aws.ecr } @@ -36,9 +36,9 @@ data "aws_caller_identity" "current" {} data "aws_partition" "current" {} locals { - name = "ack-eks-${basename(path.cwd)}" + name = "ack-eks-${basename(path.cwd)}" ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name - ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password + ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password vpc_cidr = "10.0.0.0/16" @@ -105,9 +105,9 @@ module "eks_blueprints_kubernetes_addons" { module "eks_ack_addons" { source = "../../" - cluster_id = module.eks_blueprints.eks_cluster_id + cluster_id = module.eks_blueprints.eks_cluster_id ecrpublic_username = local.ecrpublic_username - ecrpublic_token = local.ecrpublic_token + ecrpublic_token = local.ecrpublic_token # Wait for data plane to be ready From b1c23539fc0cc3fe1adf3ca3ed77ec98885bab94 Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 19 Dec 2022 21:47:29 +0000 Subject: [PATCH 04/26] precommit fix --- README.md | 3 ++- main.tf | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cb9a5df..8a7320e 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,6 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [time_sleep.dataplane](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | | [aws_iam_policy.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.api_gatewayv2_admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | @@ -86,6 +85,8 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [cluster\_id](#input\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes | | [data\_plane\_wait\_arn](#input\_data\_plane\_wait\_arn) | Addon deployment will not proceed until this value is known. Set to node group/Fargate profile ARN to wait for data plane to be ready before provisioning addons | `string` | `""` | no | | [dynamodb\_helm\_config](#input\_dynamodb\_helm\_config) | ACK dynamodb Helm Chart config | `any` | `{}` | no | +| [ecrpublic\_token](#input\_ecrpublic\_token) | EKS Cluster Id | `string` | n/a | yes | +| [ecrpublic\_username](#input\_ecrpublic\_username) | EKS Cluster Id | `string` | n/a | yes | | [emrcontainers\_helm\_config](#input\_emrcontainers\_helm\_config) | ACK EMR container Helm Chart config | `any` | `{}` | no | | [enable\_amp](#input\_enable\_amp) | Enable ACK amp add-on | `bool` | `false` | no | | [enable\_api\_gatewayv2](#input\_enable\_api\_gatewayv2) | Enable ACK API gateway v2 add-on | `bool` | `false` | no | diff --git a/main.tf b/main.tf index b6be4d0..0940a4a 100644 --- a/main.tf +++ b/main.tf @@ -12,7 +12,7 @@ locals { region = data.aws_region.current.name eks_oidc_issuer_url = replace(data.aws_eks_cluster.this.identity[0].oidc[0].issuer, "https://", "") - + ecrpublic_username = var.ecrpublic_username ecrpublic_token = var.ecrpublic_token From eaaf065313fea78f237d3ea6f56e2b929aee991b Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 19 Dec 2022 16:09:28 -0600 Subject: [PATCH 05/26] use var instead of local --- examples/complete/main.tf | 7 ++----- main.tf | 27 ++++++++++++--------------- variables.tf | 4 ++-- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 814de8f..1432faf 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -37,9 +37,6 @@ data "aws_partition" "current" {} locals { name = "ack-eks-${basename(path.cwd)}" - ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name - ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password - vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -106,8 +103,8 @@ module "eks_ack_addons" { source = "../../" cluster_id = module.eks_blueprints.eks_cluster_id - ecrpublic_username = local.ecrpublic_username - ecrpublic_token = local.ecrpublic_token + ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name + ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password # Wait for data plane to be ready diff --git a/main.tf b/main.tf index 0940a4a..82d3c20 100644 --- a/main.tf +++ b/main.tf @@ -13,9 +13,6 @@ locals { eks_oidc_issuer_url = replace(data.aws_eks_cluster.this.identity[0].oidc[0].issuer, "https://", "") - ecrpublic_username = var.ecrpublic_username - ecrpublic_token = var.ecrpublic_token - addon_context = { aws_caller_identity_account_id = data.aws_caller_identity.current.account_id aws_caller_identity_arn = data.aws_caller_identity.current.arn @@ -60,8 +57,8 @@ module "api_gatewayv2" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.4" namespace = local.api_gatewayv2_name - repository_username = local.ecrpublic_username - repository_password = local.ecrpublic_token + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token description = "ACK API Gateway Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-api-gatewayv2-apigatewayv2-chart-xxxxxxxxxxxxx` to `ack-api-gatewayv2-xxxxxxxxxxxxx` @@ -136,8 +133,8 @@ module "dynamodb" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0-stable" namespace = local.dynamodb_name - repository_username = local.ecrpublic_username - repository_password = local.ecrpublic_token + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token description = "ACK DynamoDB Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-dynamodb-dynamodb-chart-xxxxxxxxxxxxx` to `ack-dynamodb-xxxxxxxxxxxxx` @@ -203,8 +200,8 @@ module "s3" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.5" namespace = local.s3_name - repository_username = local.ecrpublic_username - repository_password = local.ecrpublic_token + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token description = "ACK S3 Controller v2 Helm chart deployment configuration" values = [ # shortens pod name from `ack-s3-s3-chart-xxxxxxxxxxxxx` to `ack-s3-xxxxxxxxxxxxx` @@ -270,8 +267,8 @@ module "rds" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.1" namespace = local.rds_name - repository_username = local.ecrpublic_username - repository_password = local.ecrpublic_token + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token create_namespace = true description = "ACK RDS Controller v2 Helm chart deployment configuration" values = [ @@ -338,8 +335,8 @@ module "amp" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0.1.1" namespace = local.amp_name - repository_username = local.ecrpublic_username - repository_password = local.ecrpublic_token + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token create_namespace = true description = "ACK amp Controller v2 Helm chart deployment configuration" values = [ @@ -406,8 +403,8 @@ module "emrcontainers" { repository = "oci://public.ecr.aws/aws-controllers-k8s" version = "v0-stable" namespace = local.emr_name - repository_username = local.ecrpublic_username - repository_password = local.ecrpublic_token + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token description = "Helm Charts for the emrcontainers controller for AWS Controllers for Kubernetes (ACK)" values = [ # shortens pod name from `ack-emrcontainers-emrcontainers-chart-xxxxxxxxxxxxx` to `ack-emrcontainers-xxxxxxxxxxxxx` diff --git a/variables.tf b/variables.tf index 6bd8f94..3c92bcf 100644 --- a/variables.tf +++ b/variables.tf @@ -4,12 +4,12 @@ variable "cluster_id" { } variable "ecrpublic_username" { - description = "EKS Cluster Id" + description = "User name decoded from the authorization token for accessing public ECR" type = string } variable "ecrpublic_token" { - description = "EKS Cluster Id" + description = "Password decoded from the authorization token for accessing public ECR" type = string } From 93d69aada56408d928b54751c64e10e637b356af Mon Sep 17 00:00:00 2001 From: EC2 Default User Date: Mon, 19 Dec 2022 22:10:18 +0000 Subject: [PATCH 06/26] precommit fix --- README.md | 4 ++-- examples/complete/main.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8a7320e..500cb51 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [cluster\_id](#input\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes | | [data\_plane\_wait\_arn](#input\_data\_plane\_wait\_arn) | Addon deployment will not proceed until this value is known. Set to node group/Fargate profile ARN to wait for data plane to be ready before provisioning addons | `string` | `""` | no | | [dynamodb\_helm\_config](#input\_dynamodb\_helm\_config) | ACK dynamodb Helm Chart config | `any` | `{}` | no | -| [ecrpublic\_token](#input\_ecrpublic\_token) | EKS Cluster Id | `string` | n/a | yes | -| [ecrpublic\_username](#input\_ecrpublic\_username) | EKS Cluster Id | `string` | n/a | yes | +| [ecrpublic\_token](#input\_ecrpublic\_token) | Password decoded from the authorization token for accessing public ECR | `string` | n/a | yes | +| [ecrpublic\_username](#input\_ecrpublic\_username) | User name decoded from the authorization token for accessing public ECR | `string` | n/a | yes | | [emrcontainers\_helm\_config](#input\_emrcontainers\_helm\_config) | ACK EMR container Helm Chart config | `any` | `{}` | no | | [enable\_amp](#input\_enable\_amp) | Enable ACK amp add-on | `bool` | `false` | no | | [enable\_api\_gatewayv2](#input\_enable\_api\_gatewayv2) | Enable ACK API gateway v2 add-on | `bool` | `false` | no | diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 1432faf..9cbd5a1 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -36,7 +36,7 @@ data "aws_caller_identity" "current" {} data "aws_partition" "current" {} locals { - name = "ack-eks-${basename(path.cwd)}" + name = "ack-eks-${basename(path.cwd)}" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) From d220eaccb0b2a343fd3d0d73c7e9d78b43ab7fee Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 9 Jan 2023 11:59:44 -0600 Subject: [PATCH 07/26] avoid duplicate emrcontainer policy --- main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/main.tf b/main.tf index 82d3c20..9cd3909 100644 --- a/main.tf +++ b/main.tf @@ -447,7 +447,6 @@ module "emrcontainers" { resource "aws_iam_policy" "emrcontainers" { count = var.enable_emrcontainers ? 1 : 0 - name = format("%s-%s", local.emr_name, "controller-iam-policies") description = "IAM policy for EMRcontainers controller" path = "/" policy = data.aws_iam_policy_document.emrcontainers.json From 25fd0a5dc1016995d7053cb5d41b60213a7c1b70 Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 9 Jan 2023 12:05:45 -0600 Subject: [PATCH 08/26] avoid duplicate emrcontainer policy --- main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/main.tf b/main.tf index 9cd3909..abe45fe 100644 --- a/main.tf +++ b/main.tf @@ -447,6 +447,7 @@ module "emrcontainers" { resource "aws_iam_policy" "emrcontainers" { count = var.enable_emrcontainers ? 1 : 0 + name_prefix = format("%s-%s", local.emr_name, "controller-iam-policies") description = "IAM policy for EMRcontainers controller" path = "/" policy = data.aws_iam_policy_document.emrcontainers.json From fdd171f1f615727bf07e0bb1fc386296af8d58d7 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Mon, 9 Jan 2023 18:12:41 +0000 Subject: [PATCH 09/26] formatting --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index abe45fe..aa8d994 100644 --- a/main.tf +++ b/main.tf @@ -447,7 +447,7 @@ module "emrcontainers" { resource "aws_iam_policy" "emrcontainers" { count = var.enable_emrcontainers ? 1 : 0 - name_prefix = format("%s-%s", local.emr_name, "controller-iam-policies") + name_prefix = format("%s-%s", local.emr_name, "controller-iam-policies") description = "IAM policy for EMRcontainers controller" path = "/" policy = data.aws_iam_policy_document.emrcontainers.json From 59bbd364c28da3e9a51b6b925ae1ffebc6889cc2 Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 6 Feb 2023 13:39:07 -0600 Subject: [PATCH 10/26] add sfn addon --- examples/complete/main.tf | 3 +- main.tf | 82 +++++++++++++++++++++++++++++++++++++++ variables.tf | 32 +++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 9cbd5a1..9785d58 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -116,7 +116,8 @@ module "eks_ack_addons" { enable_rds = true enable_amp = true enable_emrcontainers = true - + enable_sfn = true + tags = local.tags } diff --git a/main.tf b/main.tf index abe45fe..9b8b019 100644 --- a/main.tf +++ b/main.tf @@ -526,3 +526,85 @@ data "aws_iam_policy_document" "emrcontainers" { } } + +################################################################################ +# Step Functions +################################################################################ + +locals { + sfn_name = "ack-sfn" +} + +module "sfn" { + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.0" + + count = var.enable_sfn ? 1 : 0 + + helm_config = merge( + { + name = local.sfn_name + chart = "sfn-chart" + repository = "oci://public.ecr.aws/aws-controllers-k8s" + version = "v0.1.3" + namespace = local.sfn_name + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token + create_namespace = true + description = "ACK SFN Controller v2 Helm chart deployment configuration" + values = [ + # shortens pod name from `ack-sfn-sfn-chart-xxxxxxxxxxxxx` to `ack-sfn-xxxxxxxxxxxxx` + <<-EOT + nameOverride: ack-sfn + EOT + ] + }, + var.sfn_helm_config + ) + + set_values = [ + { + name = "serviceAccount.name" + value = local.sfn_name + }, + { + name = "serviceAccount.create" + value = false + }, + { + name = "aws.region" + value = local.region + } + ] + + irsa_config = { + create_kubernetes_namespace = true + kubernetes_namespace = try(var.sfn_helm_config.namespace, local.sfn_name) + + create_kubernetes_service_account = true + kubernetes_service_account = local.sfn_name + + irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn] + } + + addon_context = local.addon_context +} + +data "aws_iam_policy" "sfn" { + count = var.enable_sfn ? 1 : 0 + + name = "AWSStepFunctionsFullAccess" +} + +################################################################################ +# Event Bridge +################################################################################ + +locals { + sfn_name = "ack-eb" +} + +data "aws_iam_policy" "eb" { + count = var.enable_eb ? 1 : 0 + + name = "AmazonEventBridgeFullAccess" +} \ No newline at end of file diff --git a/variables.tf b/variables.tf index 3c92bcf..107c874 100644 --- a/variables.tf +++ b/variables.tf @@ -132,3 +132,35 @@ variable "amp_helm_config" { type = any default = {} } + +################################################################################ +# Step Functions +################################################################################ + +variable "enable_sfn" { + description = "Enable ACK step functions add-on" + type = bool + default = false +} + +variable "sfn_helm_config" { + description = "ACK step functions Helm Chart config" + type = any + default = {} +} + +################################################################################ +# Event Bridge +################################################################################ + +variable "enable_eb" { + description = "Enable ACK Event Bridge add-on" + type = bool + default = false +} + +variable "eb_helm_config" { + description = "ACK Event Bridge Helm Chart config" + type = any + default = {} +} \ No newline at end of file From c324226a8e43fc2736acc5335f3cf6208d70706b Mon Sep 17 00:00:00 2001 From: Gu Date: Mon, 6 Feb 2023 13:42:13 -0600 Subject: [PATCH 11/26] add sfn addon --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 3432486..2b867ee 100644 --- a/main.tf +++ b/main.tf @@ -600,7 +600,7 @@ data "aws_iam_policy" "sfn" { ################################################################################ locals { - sfn_name = "ack-eb" + eb_name = "ack-eb" } data "aws_iam_policy" "eb" { From 8257de264e49af37fcaeeebb46cf5a729b3ae6c8 Mon Sep 17 00:00:00 2001 From: Gu Date: Tue, 7 Feb 2023 13:25:54 -0600 Subject: [PATCH 12/26] add event pipeline example --- examples/event-driven-pipeline/README.md | 78 +++++ examples/event-driven-pipeline/addons.tf | 105 ++++++ examples/event-driven-pipeline/data.tf | 299 ++++++++++++++++++ .../examples/ack/datapipeline.yaml | 37 +++ .../examples/ack/jobrun.yaml | 25 ++ .../examples/ack/sfn.yaml | 8 + .../spark-s3-podTemplate/basic-pyspark-job.sh | 53 ++++ .../emr-eks-spark-amp-amg.sh | 118 +++++++ .../execute_emr_eks_job.sh | 125 ++++++++ .../spark-s3-podTemplate/prepare-taxi-trip.sh | 32 ++ .../pod-templates/driver-pod-template.yaml | 28 ++ .../pod-templates/executor-pod-template.yaml | 29 ++ .../scripts/pyspark-taxi-trip.py | 66 ++++ .../aws-cloudwatch-metrics-valyes.yaml | 9 + .../helm-values/aws-for-fluentbit-values.yaml | 70 ++++ .../cluster-autoscaler-values.yaml | 52 +++ .../helm-values/crossplane-values.yaml | 3 + .../helm-values/metrics-server-values.yaml | 56 ++++ .../helm-values/prometheus-values.yaml | 65 ++++ .../helm-values/vpa-values.yaml | 20 ++ examples/event-driven-pipeline/locals.tf | 13 + examples/event-driven-pipeline/main.tf | 244 ++++++++++++++ examples/event-driven-pipeline/outputs.tf | 25 ++ examples/event-driven-pipeline/providers.tf | 24 ++ examples/event-driven-pipeline/variables.tf | 29 ++ examples/event-driven-pipeline/versions.tf | 18 ++ examples/event-driven-pipeline/vpc.tf | 172 ++++++++++ 27 files changed, 1803 insertions(+) create mode 100644 examples/event-driven-pipeline/README.md create mode 100644 examples/event-driven-pipeline/addons.tf create mode 100644 examples/event-driven-pipeline/data.tf create mode 100644 examples/event-driven-pipeline/examples/ack/datapipeline.yaml create mode 100644 examples/event-driven-pipeline/examples/ack/jobrun.yaml create mode 100644 examples/event-driven-pipeline/examples/ack/sfn.yaml create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/prepare-taxi-trip.sh create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/driver-pod-template.yaml create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/executor-pod-template.yaml create mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/scripts/pyspark-taxi-trip.py create mode 100644 examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml create mode 100644 examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml create mode 100644 examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml create mode 100644 examples/event-driven-pipeline/helm-values/crossplane-values.yaml create mode 100644 examples/event-driven-pipeline/helm-values/metrics-server-values.yaml create mode 100644 examples/event-driven-pipeline/helm-values/prometheus-values.yaml create mode 100644 examples/event-driven-pipeline/helm-values/vpa-values.yaml create mode 100644 examples/event-driven-pipeline/locals.tf create mode 100644 examples/event-driven-pipeline/main.tf create mode 100644 examples/event-driven-pipeline/outputs.tf create mode 100644 examples/event-driven-pipeline/providers.tf create mode 100644 examples/event-driven-pipeline/variables.tf create mode 100644 examples/event-driven-pipeline/versions.tf create mode 100644 examples/event-driven-pipeline/vpc.tf diff --git a/examples/event-driven-pipeline/README.md b/examples/event-driven-pipeline/README.md new file mode 100644 index 0000000..4cccfb7 --- /dev/null +++ b/examples/event-driven-pipeline/README.md @@ -0,0 +1,78 @@ +# Event driven data pipeline example +This pattern is used to deploy the EKS Cluster with ACK Controllers for building an event driven data pipeline. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [aws](#requirement\_aws) | >= 3.72 | +| [helm](#requirement\_helm) | >= 2.4.1 | +| [kubernetes](#requirement\_kubernetes) | >= 2.10 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 3.72 | +| [aws.ecr](#provider\_aws.ecr) | >= 3.72 | +| [random](#provider\_random) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks\_ack\_addons](#module\_eks\_ack\_addons) | github.com/aws-ia/terraform-aws-eks-ack-addons | n/a | +| [eks\_blueprints](#module\_eks\_blueprints) | github.com/aws-ia/terraform-aws-eks-blueprints | v4.18.1 | +| [eks\_blueprints\_kubernetes\_addons](#module\_eks\_blueprints\_kubernetes\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons | v4.18.1 | +| [emrstudio\_engine\_sg](#module\_emrstudio\_engine\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | +| [emrstudio\_workspace\_sg](#module\_emrstudio\_workspace\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | +| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | +| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 3.0 | +| [vpc\_endpoints\_sg](#module\_vpc\_endpoints\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | + +## Resources + +| Name | Type | +|------|------| +| [aws_emr_studio.example](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/emr_studio) | resource | +| [aws_emrcontainers_virtual_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/emrcontainers_virtual_cluster) | resource | +| [aws_iam_policy.emr_on_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.emr_on_eks_emrstudio](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.emr_studio_service_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_prometheus_workspace.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource | +| [random_id.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | +| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | +| [aws_iam_policy_document.emr_on_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.emr_on_eks_emrstudio](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.emr_studio_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_role.emr-studio-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.22"` | no | +| [enable\_ack](#input\_enable\_ack) | Enable ACK Controller for EMR on EKS | `bool` | `true` | no | +| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"emr-eks-emrstudio"` | no | +| [region](#input\_region) | region | `string` | `"us-west-2"` | no | +| [tags](#input\_tags) | Default tags | `map(string)` | `{}` | no | +| [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR | `string` | `"10.1.0.0/16"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | +| [emr\_on\_eks\_role\_arn](#output\_emr\_on\_eks\_role\_arn) | IAM execution role arn for EMR on EKS | +| [emr\_on\_eks\_role\_id](#output\_emr\_on\_eks\_role\_id) | IAM execution role ID for EMR on EKS | +| [emr\_studio\_S3\_bucket\_id](#output\_emr\_studio\_S3\_bucket\_id) | EMR studio s3 bucket id | +| [emr\_studio\_service\_role\_arn](#output\_emr\_studio\_service\_role\_arn) | EMR studio service role ARN | + diff --git a/examples/event-driven-pipeline/addons.tf b/examples/event-driven-pipeline/addons.tf new file mode 100644 index 0000000..7c16e2d --- /dev/null +++ b/examples/event-driven-pipeline/addons.tf @@ -0,0 +1,105 @@ +module "eks_blueprints_kubernetes_addons" { + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.18.1" + + eks_cluster_id = module.eks_blueprints.eks_cluster_id + eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + eks_oidc_provider = module.eks_blueprints.oidc_provider + eks_cluster_version = module.eks_blueprints.eks_cluster_version + + #--------------------------------------- + # Amazon EKS Managed Add-ons + #--------------------------------------- + enable_amazon_eks_vpc_cni = true + enable_amazon_eks_coredns = true + enable_amazon_eks_kube_proxy = true + enable_amazon_eks_aws_ebs_csi_driver = true + + enable_aws_load_balancer_controller = true + enable_cert_manager = true + #--------------------------------------- + # Metrics Server + #--------------------------------------- + enable_metrics_server = true + metrics_server_helm_config = { + name = "metrics-server" + repository = "https://kubernetes-sigs.github.io/metrics-server/" # (Optional) Repository URL where to locate the requested chart. + chart = "metrics-server" + version = "3.8.2" + namespace = "kube-system" + timeout = "300" + values = [templatefile("${path.module}/helm-values/metrics-server-values.yaml", { + operating_system = "linux" + })] + } + + #--------------------------------------- + # Cluster Autoscaler + #--------------------------------------- + enable_cluster_autoscaler = true + cluster_autoscaler_helm_config = { + name = "cluster-autoscaler" + repository = "https://kubernetes.github.io/autoscaler" # (Optional) Repository URL where to locate the requested chart. + chart = "cluster-autoscaler" + version = "9.15.0" + namespace = "kube-system" + timeout = "300" + values = [templatefile("${path.module}/helm-values/cluster-autoscaler-values.yaml", { + aws_region = var.region, + eks_cluster_id = local.name, + operating_system = "linux" + })] + } + + #--------------------------------------- + # CloudWatch metrics for EKS + #--------------------------------------- + enable_aws_cloudwatch_metrics = true + aws_cloudwatch_metrics_helm_config = { + name = "aws-cloudwatch-metrics" + chart = "aws-cloudwatch-metrics" + repository = "https://aws.github.io/eks-charts" + version = "0.0.7" + namespace = "amazon-cloudwatch" + values = [templatefile("${path.module}/helm-values/aws-cloudwatch-metrics-valyes.yaml", { + eks_cluster_id = var.name + })] + } + + #--------------------------------------- + # AWS for FluentBit - DaemonSet + #--------------------------------------- + enable_aws_for_fluentbit = true + aws_for_fluentbit_helm_config = { + name = "aws-for-fluent-bit" + chart = "aws-for-fluent-bit" + repository = "https://aws.github.io/eks-charts" + version = "0.1.21" + namespace = "aws-for-fluent-bit" + aws_for_fluent_bit_cw_log_group = "/${var.name}/worker-fluentbit-logs" # Optional + aws_for_fluentbit_cwlog_retention_in_days = 90 + values = [templatefile("${path.module}/helm-values/aws-for-fluentbit-values.yaml", { + region = var.region, + aws_for_fluent_bit_cw_log = "/${var.name}/worker-fluentbit-logs" + })] + } + + tags = local.tags +} + +################################################################################ +# ACK Addons +################################################################################ +module "eks_ack_addons" { + + source = "github.com/aws-ia/terraform-aws-eks-ack-addons" + + cluster_id = module.eks_blueprints.eks_cluster_id + data_plane_wait_arn = module.eks_blueprints.managed_node_group_arn[0] # Wait for data plane to be ready + + ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name + ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password + + enable_sfn = true + + tags = local.tags +} diff --git a/examples/event-driven-pipeline/data.tf b/examples/event-driven-pipeline/data.tf new file mode 100644 index 0000000..6fda1ac --- /dev/null +++ b/examples/event-driven-pipeline/data.tf @@ -0,0 +1,299 @@ +data "aws_eks_cluster_auth" "this" { + name = module.eks_blueprints.eks_cluster_id +} + +data "aws_ecrpublic_authorization_token" "token" { + provider = aws.ecr +} + +data "aws_availability_zones" "available" {} + +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} + +data "aws_partition" "current" {} + +data "aws_iam_policy_document" "emr_on_eks" { + statement { + sid = "" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:s3:::*"] + + actions = [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:ListBucket", + "s3:PutObject", + ] + } + + statement { + sid = "" + effect = "Allow" + resources = ["arn:${data.aws_partition.current.partition}:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:*"] + + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:DescribeLogGroups", + "logs:DescribeLogStreams", + "logs:PutLogEvents", + ] + } +} + +# emr-studio virtual cluster execution role +data "aws_iam_role" "emr-studio-role" { + name = format("%s-%s", var.name, "emr-eks-studio") + + depends_on = [ + module.eks_blueprints + ] +} + +# emr studio role, based on https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-studio-service-role.html +data "aws_iam_policy_document" "emr_on_eks_emrstudio" { + statement { + sid = "AllowEMRReadOnlyActions" + effect = "Allow" + + actions = [ + "elasticmapreduce:ListInstances", + "elasticmapreduce:DescribeCluster", + "elasticmapreduce:ListSteps" + ] + + resources = ["*"] + } + + statement { + sid = "AllowEC2ENIActionsWithEMRTags" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterfacePermission", + "ec2:DeleteNetworkInterface" + ] + + resources = [ + "arn:aws:ec2:*:*:network-interface/*" + ] + + } + + statement { + sid = "AllowEC2ENIAttributeAction" + effect = "Allow" + + actions = [ + "ec2:ModifyNetworkInterfaceAttribute" + ] + + resources = ["arn:aws:ec2:*:*:instance/*", + "arn:aws:ec2:*:*:network-interface/*", + "arn:aws:ec2:*:*:security-group/*"] + } + + statement { + sid = "AllowEC2SecurityGroupActionsWithEMRTags" + effect = "Allow" + + actions = [ + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupEgress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteNetworkInterfacePermission" + ] + + resources = [ + "*" + ] + + } + + + statement { + sid = "AllowDefaultEC2SecurityGroupsCreationWithEMRTags" + effect = "Allow" + + actions = [ + "ec2:CreateSecurityGroup" + ] + + resources = [ + "arn:aws:ec2:*:*:security-group/*" + ] + + } + + statement { + sid = "AllowDefaultEC2SecurityGroupsCreationInVPCWithEMRTags" + effect = "Allow" + + actions = [ + "ec2:CreateSecurityGroup" + ] + + resources = [ + "arn:aws:ec2:*:*:vpc/*" + ] + + } + + statement { + sid = "AllowAddingEMRTagsDuringDefaultSecurityGroupCreation" + effect = "Allow" + + actions = [ + "ec2:CreateTags" + ] + + resources = [ + "arn:aws:ec2:*:*:security-group/*" + ] + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = ["CreateSecurityGroup"] + } + } + + statement { + sid = "AllowEC2ENICreationWithEMRTags" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterface" + ] + + resources = [ + "arn:aws:ec2:*:*:network-interface/*" + ] + + } + + statement { + sid = "AllowEC2ENICreationInSubnetAndSecurityGroupWithEMRTags" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterface" + ] + + resources = [ + "arn:aws:ec2:*:*:subnet/*", + "arn:aws:ec2:*:*:security-group/*" + ] + + } + + statement { + sid = "AllowAddingTagsDuringEC2ENICreation" + effect = "Allow" + + actions = [ + "ec2:CreateTags" + ] + + resources = [ + "arn:aws:ec2:*:*:network-interface/*" + ] + + condition { + test = "StringEquals" + variable = "ec2:CreateAction" + values = ["CreateNetworkInterface"] + } + } + + statement { + sid = "AllowSecretsManagerReadOnlyActionsWithEMRTags" + effect = "Allow" + + actions = [ + "secretsmanager:GetSecretValue" + ] + + resources = [ + "arn:aws:secretsmanager:*:*:secret:*" + ] + + } + + statement { + sid = "AllowEC2ReadOnlyActions" + effect = "Allow" + + actions = [ + "ec2:DescribeSecurityGroups", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:DescribeInstances", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs" + ] + + resources = ["*"] + } + + statement { + sid = "AllowWorkspaceCollaboration" + effect = "Allow" + + actions = [ + "iam:GetUser", + "iam:GetRole", + "iam:ListUsers", + "iam:ListRoles", + "sso:GetManagedApplicationInstance", + "sso-directory:SearchUsers" + ] + + resources = ["*"] + } + + statement { + sid = "AllowAccessToS3BucketEncryptionKey" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + "kms:ReEncrypt", + "kms:DescribeKey" + ] + + resources = ["*"] + } + + statement { + sid = "AllowEMRStudioAccesstoS3BucketforStudioWorkspaces" + effect = "Allow" + + actions = [ + "s3:PutObject", + "s3:GetObject", + "s3:GetEncryptionConfiguration", + "s3:ListBucket", + "s3:DeleteObject" + ] + + resources = ["*"] + } +} + +data "aws_iam_policy_document" "emr_studio_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["elasticmapreduce.amazonaws.com"] + } + + } +} diff --git a/examples/event-driven-pipeline/examples/ack/datapipeline.yaml b/examples/event-driven-pipeline/examples/ack/datapipeline.yaml new file mode 100644 index 0000000..082ca9f --- /dev/null +++ b/examples/event-driven-pipeline/examples/ack/datapipeline.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: emrcontainers.services.k8s.aws/v1alpha1 +kind: VirtualCluster +metadata: + name: my-ack-vc +spec: + name: my-ack-vc + containerProvider: + id: emr-eks-ack # your eks cluster name + type_: EKS + info: + eksInfo: + namespace: emr-data-team-a +--- +# S3 bucket for spark job scripts and data input/output + +--- +# steps functions to orchestrate spark jobs in EMR on EKS +apiVersion: sfn.services.k8s.aws/v1alpha1 +kind: StateMachine +definition: string +loggingConfiguration: + destinations: + cloudWatchLogsLogGroup: + logGroupARN: string + includeExecutionData: boolean + level: string +name: string +roleARN: string +tags: +- key: string + value: string +tracingConfiguration: + enabled: boolean +type_: string +--- +# Event Bridge to capture the new data upload and trigger step functions diff --git a/examples/event-driven-pipeline/examples/ack/jobrun.yaml b/examples/event-driven-pipeline/examples/ack/jobrun.yaml new file mode 100644 index 0000000..e4d8466 --- /dev/null +++ b/examples/event-driven-pipeline/examples/ack/jobrun.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: emrcontainers.services.k8s.aws/v1alpha1 +kind: JobRun +metadata: + name: my-ack-jobrun-myp7a5be +spec: + name: my-ack-jobrun-myp7a5be + virtualClusterRef: + from: + name: my-ack-vc + executionRoleARN: "{emr_on_eks_role_arn}" # emr_on_eks_role_arn for team a from terraform output + releaseLabel: "emr-6.7.0-latest" + jobDriver: + sparkSubmitJobDriver: + entryPoint: "local:///usr/lib/spark/examples/src/main/python/pi.py" + entryPointArguments: + sparkSubmitParameters: "--conf spark.executor.instances=2 --conf spark.executor.memory=1G --conf spark.executor.cores=1 --conf spark.driver.cores=1" + configurationOverrides: | + ApplicationConfiguration: null + MonitoringConfiguration: + CloudWatchMonitoringConfiguration: + LogGroupName: /emr-on-eks-logs/emr-eks-ack + LogStreamNamePrefix: pi-job + S3MonitoringConfiguration: + LogUri: s3://emr-eks-ack-rapthg5f diff --git a/examples/event-driven-pipeline/examples/ack/sfn.yaml b/examples/event-driven-pipeline/examples/ack/sfn.yaml new file mode 100644 index 0000000..2b64d11 --- /dev/null +++ b/examples/event-driven-pipeline/examples/ack/sfn.yaml @@ -0,0 +1,8 @@ +apiVersion: sfn.services.k8s.aws/v1alpha1 +kind: StateMachine +name: run-spark-job-ack +roleARN: "arn:aws:iam::349361870252:role/service-role/StepFunctions-run-emreks-job-role-624b324e" # need to pass. need permission to access emr virtual cluster +tags: +- key: owner + value: sfn-ack +definition: "{\"Comment\":\"Adescriptionofmystatemachine\",\"StartAt\":\"input-output-s3\",\"States\":{\"input-output-s3\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"s3://step-functions-test2/scripts/pyspark-taxi-trip.py\",\"EntryPointArguments\":[\"s3://step-functions-test2/input/\",\"s3://step-functions-test2/output/\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=10\"}},\"ConfigurationOverrides\":{\"ApplicationConfiguration\":[{\"Classification\":\"spark-defaults\",\"Properties\":{\"spark.driver.cores\":\"1\",\"spark.executor.cores\":\"1\",\"spark.driver.memory\":\"10g\",\"spark.executor.memory\":\"10g\",\"spark.kubernetes.driver.podTemplateFile\":\"s3://step-functions-test2/scripts/driver-pod-template.yaml\",\"spark.kubernetes.executor.podTemplateFile\":\"s3://step-functions-test2/scripts/executor-pod-template.yaml\",\"spark.local.dir\":\"/data1,/data2\"}}],\"MonitoringConfiguration\":{\"PersistentAppUI\":\"ENABLED\",\"CloudWatchMonitoringConfiguration\":{\"LogGroupName\":\"emr-on-eks\",\"LogStreamNamePrefix\":\"test1\"}}}},\"Next\":\"local-data\"},\"local-data\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"local:///usr/lib/spark/examples/src/main/python/pi.py\",\"EntryPointArguments\":[\"60\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=2--confspark.executor.memory=1G--confspark.executor.cores=1--confspark.driver.cores=1\"}}},\"End\":true}}}" diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh new file mode 100644 index 0000000..6193553 --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +if [ $# -eq 0 ]; +then + echo "$0: Missing arguments ENTER_EMR_EMR_VIRTUAL_CLUSTER_ID and EMR_JOB_EXECUTION_ROLE_ARN" + echo "USAGE: ./basic-pyspark-job '' ''" + exit 1 +elif [ $# -gt 3 ]; +then + echo "$0: Too many arguments: $@" + echo "Usage example-> ./basic-pyspark-job '' ''" + exit 1 +else + echo "We got some argument(s)" + echo "===========================" + echo "Number of arguments.: $#" + echo "List of arguments...: $@" + echo "Arg #1..............: $1" + echo "Arg #2..............: $2" + echo "===========================" +fi + +#-------------------------------------------- +# INPUT VARIABLES +#-------------------------------------------- +EMR_EMR_VIRTUAL_CLUSTER_ID=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` +EMR_JOB_EXECUTION_ROLE_ARN=$2 # Terraform output variable is emr_on_eks_role_arn +JOB_NAME='pi' +EMR_EKS_RELEASE_LABEL='emr-6.5.0-latest' + +#-------------------------------------------- +# DERIVED VARIABLES +#-------------------------------------------- +EMR_VIRTUAL_CLUSTER_NAME=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].name" --output text) + +# Execute Spark job +if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then + echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." + aws emr-containers start-job-run \ + --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ + --name $JOB_NAME \ + --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ + --release-label $EMR_EKS_RELEASE_LABEL \ + --job-driver '{ + "sparkSubmitJobDriver": { + "entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py", + "sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1" + } + }' + +else + echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" +fi diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh new file mode 100644 index 0000000..d0d3f89 --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +if [ $# -ne 3 ]; +then + echo "$0: Missing arguments ENTER_EMR_VIRTUAL_CLUSTER_ID, S3_BUCKET_NAME and EMR_JOB_EXECUTION_ROLE_ARN" + echo "USAGE: ./emr-eks-spark-amp-amg.sh '' '' ''" + exit 1 +else + echo "We got some argument(s)" + echo "===========================" + echo "Number of arguments.: $#" + echo "List of arguments...: $@" + echo "Arg #1..............: $1" + echo "Arg #2..............: $2" + echo "Arg #3..............: $3" + echo "===========================" +fi + +#-------------------------------------------- +# INPUT VARIABLES +#-------------------------------------------- +EMR_VIRTUAL_CLUSTER_ID=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` +S3_BUCKET=$2 # This script requires s3 bucket as input parameter e.g., s3:// +EMR_JOB_EXECUTION_ROLE_ARN=$3 # Terraform output variable is emr_on_eks_role_arn + +#-------------------------------------------- +# DERIVED VARIABLES +#-------------------------------------------- +EMR_VIRTUAL_CLUSTER_NAMESPACE=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].containerProvider.info.eksInfo" --output text) +EMR_VIRTUAL_CLUSTER_NAME=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].name" --output text) + +#-------------------------------------------- +# DEFAULT VARIABLES CAN BE MODIFIED +#-------------------------------------------- +JOB_NAME='taxidata' +EMR_EKS_RELEASE_LABEL="emr-6.5.0-latest" +SPARK_JOB_S3_PATH="${S3_BUCKET}/emr_virtual_cluster_name=${EMR_VIRTUAL_CLUSTER_NAME}/namespace=${EMR_VIRTUAL_CLUSTER_NAMESPACE}/job_name=${JOB_NAME}" + +#-------------------------------------------- +# CLOUDWATCH LOG GROUP NAME +#-------------------------------------------- +CW_LOG_GROUP="/emr-on-eks-logs/${EMR_VIRTUAL_CLUSTER_NAME}/${EMR_VIRTUAL_CLUSTER_NAMESPACE}" # Create CW Log group if not exist + +#-------------------------------------------- +# Download sample input data from https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page +#-------------------------------------------- +Create folder locally to store the input data +mkdir -p "spark-scripts/input" + +# Download the input data from public data set to local folders +max=40 +for (( i=1; i <= $max; ++i )) +do + wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "spark-scripts/input/yellow_tripdata_2022-${i}.parquet" +done + +#-------------------------------------------- +# This command copies PySpark script and the Pod templates to S3 bucket +#-------------------------------------------- +aws s3 sync ./spark-scripts/ "${SPARK_JOB_S3_PATH}/" + +#-------------------------------------------- +# Execute Spark job +#-------------------------------------------- + +if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then + echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." + aws emr-containers start-job-run \ + --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ + --name $JOB_NAME \ + --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ + --release-label $EMR_EKS_RELEASE_LABEL \ + --job-driver '{ + "sparkSubmitJobDriver": { + "entryPoint": "'"$SPARK_JOB_S3_PATH"'/scripts/sample-spark-taxi-trip.py", + "entryPointArguments": ["'"$SPARK_JOB_S3_PATH"'/input/", + "'"$SPARK_JOB_S3_PATH"'/output/" + ], + "sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.cores=1 --conf spark.driver.cores=1" + } + }' \ + --configuration-overrides '{ + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": { + "spark.kubernetes.driver.podTemplateFile":"'"$SPARK_JOB_S3_PATH"'/pod-templates/nvme-spark-driver.yaml", + "spark.kubernetes.executor.podTemplateFile":"'"$SPARK_JOB_S3_PATH"'/pod-templates/nvme-spark-executor.yaml", + "spark.driver.memory":"2g", + "spark.executor.memory":"4g", + "spark.local.dir" : "/data1,/data2", + "spark.kubernetes.executor.podNamePrefix":"'"$JOB_NAME"'", + "spark.ui.prometheus.enabled":"true", + "spark.executor.processTreeMetrics.enabled":"true", + "spark.kubernetes.driver.annotation.prometheus.io/scrape":"true", + "spark.kubernetes.driver.annotation.prometheus.io/path":"/metrics/executors/prometheus/", + "spark.kubernetes.driver.annotation.prometheus.io/port":"4040", + "spark.kubernetes.driver.service.annotation.prometheus.io/scrape":"true", + "spark.kubernetes.driver.service.annotation.prometheus.io/path":"/metrics/driver/prometheus/", + "spark.kubernetes.driver.service.annotation.prometheus.io/port":"4040", + "spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet", + "spark.metrics.conf.*.sink.prometheusServlet.path":"/metrics/driver/prometheus/", + "spark.metrics.conf.master.sink.prometheusServlet.path":"/metrics/master/prometheus/", + "spark.metrics.conf.applications.sink.prometheusServlet.path":"/metrics/applications/prometheus/" + } + } + ], + "monitoringConfiguration": { + "persistentAppUI":"ENABLED", + "cloudWatchMonitoringConfiguration": { + "logGroupName":"'"$CW_LOG_GROUP"'", + "logStreamNamePrefix":"'"$JOB_NAME"'" + } + } + }' +else + echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" +fi diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh new file mode 100644 index 0000000..c8cc30f --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +if [ $# -ne 4 ]; +then + echo "$0: Missing arguments EMR_VIRTUAL_CLUSTER_NAME, S3_BUCKET_NAME and EMR_JOB_EXECUTION_ROLE_ARN" + echo "USAGE: ./execute_emr_eks_job.sh '' '' ''" + exit 1 +else + echo "We got some argument(s)" + echo "===========================" + echo "Number of arguments.: $#" + echo "List of arguments...: $@" + echo "Arg #1..............: $1" + echo "Arg #2..............: $2" + echo "Arg #3..............: $3" + echo "===========================" +fi + +#-------------------------------------------- +# INPUT VARIABLES +#-------------------------------------------- +EMR_VIRTUAL_CLUSTER_NAME=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` +S3_BUCKET=$2 # This script requires s3 bucket as input parameter e.g., s3:// +EMR_JOB_EXECUTION_ROLE_ARN=$3 # Terraform output variable is emr_on_eks_role_arn + +#-------------------------------------------- +# DERIVED VARIABLES +#-------------------------------------------- +EMR_VIRTUAL_CLUSTER_ID=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?name == '$EMR_VIRTUAL_CLUSTER_NAME' && state == 'RUNNING'].id" --output text) + +#-------------------------------------------- +# DEFAULT VARIABLES CAN BE MODIFIED +#-------------------------------------------- +JOB_NAME='taxidata' +EMR_EKS_RELEASE_LABEL="emr-6.7.0-latest" # Spark 3.2.1 +CW_LOG_GROUP="/emr-on-eks-logs/${EMR_VIRTUAL_CLUSTER_NAME}" # Create CW Log group if not exist + +SPARK_JOB_S3_PATH="${S3_BUCKET}/${EMR_VIRTUAL_CLUSTER_NAME}/${JOB_NAME}" +SCRIPTS_S3_PATH="${SPARK_JOB_S3_PATH}/scripts" +INPUT_DATA_S3_PATH="${SPARK_JOB_S3_PATH}/input" +OUTPUT_DATA_S3_PATH="${SPARK_JOB_S3_PATH}/output" + +#-------------------------------------------- +# Copy PySpark Scripts, Pod Templates and Input data to S3 bucket +#-------------------------------------------- +aws s3 sync "./" ${SCRIPTS_S3_PATH} + +#-------------------------------------------- +# NOTE: This section downloads the test data from AWS Public Dataset. You can comment this section and bring your own input data required for sample PySpark test +# https://registry.opendata.aws/nyc-tlc-trip-records-pds/ +#-------------------------------------------- + +mkdir -p "../input" +# Download the input data from public data set to local folders +wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "../input/yellow_tripdata_2022-0.parquet" + +# Making duplicate copies to increase the size of the data. +max=20 +for (( i=1; i <= $max; ++i )) +do + cp -rf "../input/yellow_tripdata_2022-0.parquet" "../input/yellow_tripdata_2022-${i}.parquet" +done + +aws s3 sync "../input" ${INPUT_DATA_S3_PATH} # Sync from local folder to S3 path + +#-------------------------------------------- +# Execute Spark job +#-------------------------------------------- + +if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then + echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." + aws emr-containers start-job-run \ + --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ + --name $JOB_NAME \ + --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ + --release-label $EMR_EKS_RELEASE_LABEL \ + --job-driver '{ + "sparkSubmitJobDriver": { + "entryPoint": "'"$SCRIPTS_S3_PATH"'/pyspark-taxi-trip.py", + "entryPointArguments": ["'"$INPUT_DATA_S3_PATH"'", + "'"$OUTPUT_DATA_S3_PATH"'" + ], + "sparkSubmitParameters": "--conf spark.executor.instances=10" + } + }' \ + --configuration-overrides '{ + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": { + "spark.driver.cores":"1", + "spark.executor.cores":"1", + "spark.driver.memory": "10g", + "spark.executor.memory": "10g", + "spark.kubernetes.driver.podTemplateFile":"'"$SCRIPTS_S3_PATH"'/driver-pod-template.yaml", + "spark.kubernetes.executor.podTemplateFile":"'"$SCRIPTS_S3_PATH"'/executor-pod-template.yaml", + "spark.local.dir" : "/data1,/data2", + + "spark.kubernetes.executor.podNamePrefix":"'"$JOB_NAME"'", + "spark.ui.prometheus.enabled":"true", + "spark.executor.processTreeMetrics.enabled":"true", + "spark.kubernetes.driver.annotation.prometheus.io/scrape":"true", + "spark.kubernetes.driver.annotation.prometheus.io/path":"/metrics/executors/prometheus/", + "spark.kubernetes.driver.annotation.prometheus.io/port":"4040", + "spark.kubernetes.driver.service.annotation.prometheus.io/scrape":"true", + "spark.kubernetes.driver.service.annotation.prometheus.io/path":"/metrics/driver/prometheus/", + "spark.kubernetes.driver.service.annotation.prometheus.io/port":"4040", + "spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet", + "spark.metrics.conf.*.sink.prometheusServlet.path":"/metrics/driver/prometheus/", + "spark.metrics.conf.master.sink.prometheusServlet.path":"/metrics/master/prometheus/", + "spark.metrics.conf.applications.sink.prometheusServlet.path":"/metrics/applications/prometheus/" + } + } + ], + "monitoringConfiguration": { + "persistentAppUI":"ENABLED", + "cloudWatchMonitoringConfiguration": { + "logGroupName":"'"$CW_LOG_GROUP"'", + "logStreamNamePrefix":"'"$JOB_NAME"'" + } + } + }' +else + echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" +fi diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/prepare-taxi-trip.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/prepare-taxi-trip.sh new file mode 100644 index 0000000..037a184 --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/prepare-taxi-trip.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# This job copies Sample PySpark script and some test data to your S3 bucket which will enable you to run the following Spark Operator script + +# Prerequisites for running this shell script +# 1/ Enter your S3 bucket below () that "spark-team-a" service account IRSA can access. +# 2/ Enter region below (). Same as the EKS Cluster region +# 3/ Change according to your needs +# 4/ Ensure and is replaced in "ebs-storage-dynamic-pvc.yaml" file +# 5/ Execute the shell script which creates the input data in your S3 bucket +# 6/ Run `kubectl apply -f ebs-storage-dynamic-pvc.yaml` to trigger the Spark job +# 7/ Monitor the Spark job using "kubectl get pods -n spark-team-a -w" + +S3_BUCKET="" +REGION="" # Enter region +JOB_NAME="ebs-taxi-trip" # Change job name according to your needs + +INPUT_DATA_S3_PATH="s3://${S3_BUCKET}/${JOB_NAME}/input/" + +# Copy PySpark Script to S3 bucket +aws s3 cp pyspark-taxi-trip.py s3://${S3_BUCKET}/${JOB_NAME}/scripts/ --region ${REGION} + +# Copy Test Input data to S3 bucket +wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "input/yellow_tripdata_2022-0.parquet" + +# Making duplicate copies to increase the size of the data. +max=20 +for (( i=1; i <= $max; ++i )) +do + cp -rf "input/yellow_tripdata_2022-0.parquet" "input/yellow_tripdata_2022-${i}.parquet" +done + +aws s3 sync "input/" ${INPUT_DATA_S3_PATH} diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/driver-pod-template.yaml b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/driver-pod-template.yaml new file mode 100644 index 0000000..4766305 --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/driver-pod-template.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: ny-taxi-driver + namespace: emr-data-team-a +spec: + volumes: + - name: spark-local-dir-1 + hostPath: + path: /local1 + type: Directory + - name: spark-local-dir-2 + hostPath: + path: /local2 + type: Directory + + nodeSelector: + "NodeGroupType": "spark" + + containers: + - name: spark-kubernetes-driver # Don't change this name. EMR on EKS looking for this name + volumeMounts: + - name: spark-local-dir-1 + mountPath: /data1 + readOnly: false + - name: spark-local-dir-2 + mountPath: /data2 + readOnly: false diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/executor-pod-template.yaml b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/executor-pod-template.yaml new file mode 100644 index 0000000..3eeef2c --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/executor-pod-template.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Pod +metadata: + name: ny-taxi-exec + namespace: emr-data-team-a + +spec: + volumes: + - name: spark-local-dir-1 + hostPath: + path: /local1 + type: Directory #cannot create here. needs to be created when provision node for proper permission + - name: spark-local-dir-2 + hostPath: + path: /local2 + type: Directory + + nodeSelector: + "NodeGroupType": "spark" + + containers: + - name: spark-kubernetes-executor # Don't change this name. EMR on EKS looking for this name + volumeMounts: + - name: spark-local-dir-1 + mountPath: /data1 + readOnly: false + - name: spark-local-dir-2 + mountPath: /data2 + readOnly: false diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/scripts/pyspark-taxi-trip.py b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/scripts/pyspark-taxi-trip.py new file mode 100644 index 0000000..4a0cf0e --- /dev/null +++ b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/scripts/pyspark-taxi-trip.py @@ -0,0 +1,66 @@ +import logging +import sys +from datetime import datetime + +from pyspark.sql import SparkSession +from pyspark.sql.functions import * +from pyspark.sql import functions as f + +# Logging configuration +formatter = logging.Formatter('[%(asctime)s] %(levelname)s @ line %(lineno)d: %(message)s') +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.INFO) +handler.setFormatter(formatter) +logger = logging.getLogger() +logger.setLevel(logging.INFO) +logger.addHandler(handler) + +dt_string = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") +AppName = "NewYorkTaxiData" + + +def main(args): + + raw_input_folder = args[1] + transform_output_folder = args[2] + + # Create Spark Session + spark = SparkSession \ + .builder \ + .appName(AppName + "_" + str(dt_string)) \ + .getOrCreate() + + spark.sparkContext.setLogLevel("INFO") + logger.info("Starting spark application") + + logger.info("Reading Parquet file from S3") + ny_taxi_df = spark.read.parquet(raw_input_folder) + + # Add additional columns to the DF + final_ny_taxi_df = ny_taxi_df.withColumn("current_date", f.lit(datetime.now())) + + logger.info("NewYork Taxi data schema preview") + final_ny_taxi_df.printSchema() + + logger.info("Previewing New York Taxi data sample") + final_ny_taxi_df.show(20, truncate=False) + + logger.info("Total number of records: " + str(final_ny_taxi_df.count())) + + logger.info("Write New York Taxi data to S3 transform table") + final_ny_taxi_df.repartition(2).write.mode("overwrite").parquet(transform_output_folder) + + logger.info("Ending spark application") + # end spark code + spark.stop() + + return None + + +if __name__ == "__main__": + print(len(sys.argv)) + if len(sys.argv) != 3: + print("Usage: spark-etl [input-folder] [output-folder]") + sys.exit(0) + + main(sys.argv) diff --git a/examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml b/examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml new file mode 100644 index 0000000..0af891e --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml @@ -0,0 +1,9 @@ +clusterName: ${eks_cluster_id} + +resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 200m + memory: 1Gi diff --git a/examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml b/examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml new file mode 100644 index 0000000..697b992 --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml @@ -0,0 +1,70 @@ +global: + +#hostNetwork and dnsPolicy are critical for enabling large clusters to avoid making calls to API server +# see this link https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#optional-feature-using-kubelet-to-get-metadata +hostNetwork: true +dnsPolicy: ClusterFirstWithHostNet + +# NOTE: extraFilters config for using Kubelet to get the Metadata instead of talking to API server for large clusters +filter: + name: "kubernetes" + match: "kube.*" + kubeURL: "https://kubernetes.default.svc.cluster.local:443" + mergeLog: "On" + mergeLogKey: "log_processed" + keepLog: "On" + k8sLoggingParser: "On" + k8sLoggingExclude: "Off" + bufferSize: "0" + extraFilters: | + Kube_Tag_Prefix application.var.log.containers. + Labels Off + Annotations Off + Use_Kubelet true + Kubelet_Port 10250 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + +cloudWatch: + enabled: true + match: "*" + region: ${region} + logGroupName: ${aws_for_fluent_bit_cw_log} + logStreamName: + logStreamPrefix: "fluentbit-" + logKey: + logFormat: + roleArn: + autoCreateGroup: false + endpoint: + credentialsEndpoint: {} + +firehose: + enabled: false + +kinesis: + enabled: false + +elasticsearch: + enabled: false + +serviceAccount: + create: true + +# Resource config for large clusters +resources: + limits: + cpu: 1000m + memory: 1500Mi + requests: + cpu: 500m + memory: 500Mi + +## Assign a PriorityClassName to pods if set +priorityClassName: system-node-critical + +updateStrategy: + type: RollingUpdate + +nodeSelector: + kubernetes.io/os: linux diff --git a/examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml b/examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml new file mode 100644 index 0000000..680aff7 --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml @@ -0,0 +1,52 @@ +autoDiscovery: + clusterName: ${eks_cluster_id} + tags: + - k8s.io/cluster-autoscaler/enabled + - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} + + roles: + - worker + +awsRegion: ${aws_region} + +cloudProvider: aws + +extraArgs: + logtostderr: true + stderrthreshold: info + v: 4 + aws-use-static-instance-list: true + +nodeSelector: + NodeGroupType: core + kubernetes.io/os: ${operating_system} + +podDisruptionBudget: + maxUnavailable: 1 + +priorityClassName: "system-cluster-critical" + +rbac: + create: true + pspEnabled: false + serviceAccount: + create: false + +# replicaCount -- Desired number of pods +replicaCount: 1 + +# Best practice to update the resource requests and limits for each add-on +resources: + limits: + cpu: 1000m + memory: 1G + requests: + cpu: 200m + memory: 512Mi + +# Best practice to updateStrategy for each add-on +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 diff --git a/examples/event-driven-pipeline/helm-values/crossplane-values.yaml b/examples/event-driven-pipeline/helm-values/crossplane-values.yaml new file mode 100644 index 0000000..6b0d50c --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/crossplane-values.yaml @@ -0,0 +1,3 @@ +nodeSelector: + kubernetes.io/os: ${operating-system} + NodeGroupType: core diff --git a/examples/event-driven-pipeline/helm-values/metrics-server-values.yaml b/examples/event-driven-pipeline/helm-values/metrics-server-values.yaml new file mode 100644 index 0000000..65f071d --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/metrics-server-values.yaml @@ -0,0 +1,56 @@ +# HA config for metrics-server +image: + repository: k8s.gcr.io/metrics-server/metrics-server + pullPolicy: IfNotPresent + +serviceAccount: + create: true + name: metrics-server + +rbac: + create: true + pspEnabled: false + +apiService: + create: true + +podLabels: + k8s-app: metrics-server + +# HA enabled by enabling replicas to 2, updateStrategy and podDisruptionBudget to true +replicas: 2 + +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + +podDisruptionBudget: + enabled: true + minAvailable: 1 + +defaultArgs: + - --cert-dir=/tmp + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + +resources: + requests: + cpu: 200m + memory: 512Mi + +nodeSelector: + NodeGroupType: core + kubernetes.io/os: ${operating_system} + +affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + k8s-app: metrics-server + namespaces: + - kube-system + topologyKey: kubernetes.io/hostname diff --git a/examples/event-driven-pipeline/helm-values/prometheus-values.yaml b/examples/event-driven-pipeline/helm-values/prometheus-values.yaml new file mode 100644 index 0000000..600952d --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/prometheus-values.yaml @@ -0,0 +1,65 @@ +server: + retention: 1h + remoteWrite: + - queue_config: + max_samples_per_send: 1000 + max_shards: 200 + capacity: 2500 + global: + evaluation_interval: 30s + scrape_interval: 30s + scrape_timeout: 10s + + resources: + requests: + cpu: 500m + memory: 1Gi + + verticalAutoscaler: + enabled: true + updateMode: "Auto" + containerPolicies: + - containerName: "prometheus-server" + minAllowed: + cpu: 500m + memory: 1Gi + maxAllowed: + cpu: 1000m + memory: 4Gi + + nodeSelector: + kubernetes.io/os: ${operating_system} + NodeGroupType: core + persistentVolume: + accessModes: + - ReadWriteOnce + enabled: true + mountPath: /data + size: 20Gi + storageClass: gp2 + +alertmanager: + nodeSelector: + kubernetes.io/os: ${operating_system} + NodeGroupType: core + +kube-state-metrics: + nodeSelector: + kubernetes.io/os: ${operating_system} + NodeGroupType: core + +pushgateway: + nodeSelector: + kubernetes.io/os: ${operating_system} + NodeGroupType: core + +nodeExporter: + nodeSelector: + kubernetes.io/os: ${operating_system} + + # Additional scrape config for Apache YuniKorn Scheduler metrics + # - job_name: 'yunikorn' + # metrics_path: '/ws/v1/metrics' + # static_configs: + # - targets: + # - yunikorn-service.yunikorn.svc.cluster.local:9080 diff --git a/examples/event-driven-pipeline/helm-values/vpa-values.yaml b/examples/event-driven-pipeline/helm-values/vpa-values.yaml new file mode 100644 index 0000000..948fa6d --- /dev/null +++ b/examples/event-driven-pipeline/helm-values/vpa-values.yaml @@ -0,0 +1,20 @@ +# Default values for vertical-pod-autoscaler. +serviceAccount: + name: vpa + +recommender: + enabled: true + extraArgs: + v: "4" + pod-recommendation-min-cpu-millicores: 15 + pod-recommendation-min-memory-mb: 100 + # Best practise to use Prometheus as a history provider for the VPA recommender. Not required for this example +# storage: prometheus +# prometheus-address: http://prometheus-server.prometheus.svc.cluster.local:9090 + nodeSelector: + NodeGroupType: core + kubernetes.io/os: ${operating_system} +updater: + enabled: true + nodeSelector: + kubernetes.io/os: ${operating_system} diff --git a/examples/event-driven-pipeline/locals.tf b/examples/event-driven-pipeline/locals.tf new file mode 100644 index 0000000..42cfeb4 --- /dev/null +++ b/examples/event-driven-pipeline/locals.tf @@ -0,0 +1,13 @@ +locals { + name = var.name + region = var.region + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + vpc_endpoints = ["autoscaling", "ecr.api", "ecr.dkr", "ec2", "ec2messages", "elasticloadbalancing", "sts", "kms", "logs", "ssm", "ssmmessages"] + + tags = merge(var.tags, { + Blueprint = local.name + GithubRepo = "aws-ia/terraform-aws-eks-ack-addons" + }) +} diff --git a/examples/event-driven-pipeline/main.tf b/examples/event-driven-pipeline/main.tf new file mode 100644 index 0000000..7a4998b --- /dev/null +++ b/examples/event-driven-pipeline/main.tf @@ -0,0 +1,244 @@ +#--------------------------------------------------------------- +# EKS Blueprints +#--------------------------------------------------------------- +module "eks_blueprints" { + source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.18.1" + cluster_name = local.name + cluster_version = var.eks_cluster_version + + vpc_id = module.vpc.vpc_id + private_subnet_ids = module.vpc.private_subnets + + create_node_security_group = false + + cluster_endpoint_private_access = true # if true, Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint + cluster_endpoint_public_access = true # if true, Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. + + #--------------------------------------- + # Note: This can further restricted to specific required for each Add-on and your application + #--------------------------------------- + node_security_group_additional_rules = { + # Extend node-to-node security group rules. Recommended and required for the Add-ons + ingress_self_all = { + description = "Node to node all ports/protocols" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + self = true + } + # Recommended outbound traffic for Node groups + egress_all = { + description = "Node all egress" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "egress" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + # Allows Control Plane Nodes to talk to Worker nodes on all ports. Added this to simplify the example and further avoid issues with Add-ons communication with Control plane. + # This can be restricted further to specific port based on the requirement for each Add-on e.g., metrics-server 4443, analytics-operator 8080, karpenter 8443 etc. + # Change this according to your security requirements if needed + ingress_cluster_to_node_all_traffic = { + description = "Cluster API to Nodegroup all traffic" + protocol = "-1" + from_port = 0 + to_port = 0 + type = "ingress" + source_cluster_security_group = true + } + } + + managed_node_groups = { + # Core node group for deploying all the critical add-ons + mng1 = { + node_group_name = "core-node-grp" + subnet_ids = module.vpc.private_subnets + + instance_types = ["m5.xlarge"] + ami_type = "AL2_x86_64" + capacity_type = "ON_DEMAND" + + disk_size = 100 + disk_type = "gp3" + + max_size = 9 + min_size = 3 + desired_size = 3 + create_launch_template = true + launch_template_os = "amazonlinux2eks" + + update_config = [{ + max_unavailable_percentage = 50 + }] + + k8s_labels = { + Environment = "preprod" + Zone = "test" + WorkerType = "ON_DEMAND" + NodeGroupType = "core" + } + + additional_tags = { + Name = "core-node-grp" + subnet_type = "private" + "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" + "k8s.io/cluster-autoscaler/node-template/label/noderole" = "core" + "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" + "k8s.io/cluster-autoscaler/experiments" = "owned" + "k8s.io/cluster-autoscaler/enabled" = "true" + } + }, + #--------------------------------------- + # Note: This example only uses ON_DEMAND node group for both Spark Driver and Executors. + # If you want to leverage SPOT nodes for Spark executors then create ON_DEMAND node group for placing your driver pods and SPOT nodegroup for executors. + # Use NodeSelectors to place your driver/executor pods with the help of Pod Templates. + #--------------------------------------- + mng2 = { + node_group_name = "spark-node-grp" + subnet_ids = [module.vpc.private_subnets[0]] + instance_types = ["r5d.4xlarge"] + ami_type = "AL2_x86_64" + capacity_type = "SPOT" + + # Enable this option only when you are using NVMe disks + format_mount_nvme_disk = true # Mounts NVMe disks to /local1, /local2 etc. for multiple NVMe disks + + # RAID0 configuration is recommended for better performance when you use larger instances with multiple NVMe disks e.g., r5d.24xlarge + # Permissions for hadoop user runs the analytics job. user > hadoop:x:999:1000::/home/hadoop:/bin/bash + post_userdata = <<-EOT + #!/bin/bash + set -ex + mkdir local1 + mkdir local2 + /usr/bin/chown -hR +999:+1000 /local* + EOT + + disk_size = 100 + disk_type = "gp3" + + max_size = 9 # Managed node group soft limit is 450; request AWS for limit increase + min_size = 3 + desired_size = 3 + + create_launch_template = true + launch_template_os = "amazonlinux2eks" + + update_config = [{ + max_unavailable_percentage = 50 + }] + + additional_iam_policies = [] + k8s_taints = [] + + k8s_labels = { + Environment = "preprod" + Zone = "test" + WorkerType = "ON_DEMAND" + NodeGroupType = "spark" + } + + additional_tags = { + Name = "spark-node-grp" + subnet_type = "private" + "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" + "k8s.io/cluster-autoscaler/node-template/label/noderole" = "spark" + "k8s.io/cluster-autoscaler/node-template/label/disk" = "nvme" + "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" + "k8s.io/cluster-autoscaler/experiments" = "owned" + "k8s.io/cluster-autoscaler/enabled" = "true" + } + }, + } + + #--------------------------------------- + # ENABLE EMR ON EKS + # 1. Creates namespace + # 2. k8s role and role binding(emr-containers user) for the above namespace + # 3. IAM role for the team execution role + # 4. Update AWS_AUTH config map with emr-containers user and AWSServiceRoleForAmazonEMRContainers role + # 5. Create a trust relationship between the job execution role and the identity of the EMR managed service account + #--------------------------------------- + enable_emr_on_eks = true + emr_on_eks_teams = { + emr-data-team-a = { + namespace = "emr-data-team-a" + job_execution_role = "emr-eks-data-team-a" + additional_iam_policies = [aws_iam_policy.emr_on_eks.arn] + } + emr-studio = { + namespace = "emr-studio" + job_execution_role = "emr-eks-studio" #role for managed endpoint + additional_iam_policies = [aws_iam_policy.emr_on_eks.arn] + } + } + tags = local.tags +} + +#--------------------------------------------------------------- +# Example IAM policies for EMR job execution +#--------------------------------------------------------------- +resource "aws_iam_policy" "emr_on_eks" { + name = format("%s-%s", local.name, "emr-job-iam-policies") + description = "IAM policy for EMR on EKS Job execution" + path = "/" + policy = data.aws_iam_policy_document.emr_on_eks.json +} + + + +#--------------------------------------------------------------- +# Create EMR on EKS Virtual Cluster +#--------------------------------------------------------------- +resource "aws_emrcontainers_virtual_cluster" "this" { + name = format("%s-%s", module.eks_blueprints.eks_cluster_id, "emr-studio") + + container_provider { + id = module.eks_blueprints.eks_cluster_id + type = "EKS" + + info { + eks_info { + namespace = "emr-studio" + } + } + } +} + +module "s3_bucket" { + source = "terraform-aws-modules/s3-bucket/aws" + version = "~> 3.0" + + bucket = "emr-studio-${random_id.this.hex}" + acl = "private" + + # For example only - please evaluate for your environment + force_destroy = true + + attach_deny_insecure_transport_policy = true + attach_require_latest_tls_policy = true + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true + + server_side_encryption_configuration = { + rule = { + apply_server_side_encryption_by_default = { + sse_algorithm = "AES256" + } + } + } + + tags = local.tags +} + +resource "random_id" "this" { + byte_length = "2" +} + + diff --git a/examples/event-driven-pipeline/outputs.tf b/examples/event-driven-pipeline/outputs.tf new file mode 100644 index 0000000..dd8702f --- /dev/null +++ b/examples/event-driven-pipeline/outputs.tf @@ -0,0 +1,25 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = module.eks_blueprints.configure_kubectl +} + + +output "emr_on_eks_role_id" { + description = "IAM execution role ID for EMR on EKS" + value = module.eks_blueprints.emr_on_eks_role_id +} + +output "emr_on_eks_role_arn" { + description = "IAM execution role arn for EMR on EKS" + value = module.eks_blueprints.emr_on_eks_role_arn +} + +output "emr_studio_service_role_arn" { + description = "EMR studio service role ARN" + value = aws_iam_role.emr_studio_service_role.arn +} + +output "emr_studio_S3_bucket_id" { + description = "EMR studio s3 bucket id" + value = module.s3_bucket.s3_bucket_id +} diff --git a/examples/event-driven-pipeline/providers.tf b/examples/event-driven-pipeline/providers.tf new file mode 100644 index 0000000..d56f0b9 --- /dev/null +++ b/examples/event-driven-pipeline/providers.tf @@ -0,0 +1,24 @@ +provider "aws" { + region = local.region +} + +# ECR always authenticates with `us-east-1` region +# Docs -> https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html +provider "aws" { + alias = "ecr" + region = "us-east-1" +} + +provider "kubernetes" { + host = module.eks_blueprints.eks_cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token +} + +provider "helm" { + kubernetes { + host = module.eks_blueprints.eks_cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) + token = data.aws_eks_cluster_auth.this.token + } +} diff --git a/examples/event-driven-pipeline/variables.tf b/examples/event-driven-pipeline/variables.tf new file mode 100644 index 0000000..f4a779e --- /dev/null +++ b/examples/event-driven-pipeline/variables.tf @@ -0,0 +1,29 @@ +variable "name" { + description = "Name of the VPC and EKS Cluster" + default = "event-driven-pipeline-demo" + type = string +} + +variable "region" { + description = "region" + type = string + default = "us-west-2" +} + +variable "eks_cluster_version" { + description = "EKS Cluster version" + default = "1.24" # EMR studio managed endpoint only supports up to 1.22 for now + type = string +} + +variable "tags" { + description = "Default tags" + default = {} + type = map(string) +} + +variable "vpc_cidr" { + description = "VPC CIDR" + default = "10.1.0.0/16" + type = string +} diff --git a/examples/event-driven-pipeline/versions.tf b/examples/event-driven-pipeline/versions.tf new file mode 100644 index 0000000..9ac1742 --- /dev/null +++ b/examples/event-driven-pipeline/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.72" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.4.1" + } + } +} diff --git a/examples/event-driven-pipeline/vpc.tf b/examples/event-driven-pipeline/vpc.tf new file mode 100644 index 0000000..05995cd --- /dev/null +++ b/examples/event-driven-pipeline/vpc.tf @@ -0,0 +1,172 @@ +#--------------------------------------------------------------- +# VPC and Subnets +#--------------------------------------------------------------- +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 3.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 10)] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + # Manage so we can name + manage_default_network_acl = true + default_network_acl_tags = { Name = "${local.name}-default" } + manage_default_route_table = true + default_route_table_tags = { Name = "${local.name}-default" } + manage_default_security_group = true + default_security_group_tags = { Name = "${local.name}-default" } + + public_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + default_security_group_name = "${local.name}-endpoint-secgrp" + + default_security_group_ingress = [ + { + protocol = -1 + from_port = 0 + to_port = 0 + cidr_blocks = local.vpc_cidr + }] + default_security_group_egress = [ + { + from_port = 0 + to_port = 0 + protocol = -1 + cidr_blocks = "0.0.0.0/0" + }] + + tags = local.tags +} + +module "vpc_endpoints_sg" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 4.0" + + name = "${local.name}-vpc-endpoints" + description = "Security group for VPC endpoint access" + vpc_id = module.vpc.vpc_id + + ingress_with_cidr_blocks = [ + { + rule = "https-443-tcp" + description = "VPC CIDR HTTPS" + cidr_blocks = join(",", module.vpc.private_subnets_cidr_blocks) + }, + ] + + egress_with_cidr_blocks = [ + { + rule = "https-443-tcp" + description = "All egress HTTPS" + cidr_blocks = "0.0.0.0/0" + }, + ] + + tags = local.tags +} + +module "vpc_endpoints" { + source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" + version = "~> 3.0" + + vpc_id = module.vpc.vpc_id + security_group_ids = [module.vpc_endpoints_sg.security_group_id] + + endpoints = merge({ + s3 = { + service = "s3" + service_type = "Gateway" + route_table_ids = module.vpc.private_route_table_ids + tags = { + Name = "${local.name}-s3" + } + } + }, + { for service in toset(local.vpc_endpoints) : + replace(service, ".", "_") => + { + service = service + subnet_ids = module.vpc.private_subnets + private_dns_enabled = true + tags = { Name = "${local.name}-${service}" } + } + }) + + tags = local.tags +} + +#--------------------------------------------------------------- +# security groups for EMR Studio +#--------------------------------------------------------------- +# engine_security_group +module "emrstudio_engine_sg" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 4.0" + + name = "${local.name}-engine-security-group" + description = "engineSecurityGroup for EMR studio" + vpc_id = module.vpc.vpc_id + + ingress_with_cidr_blocks = [ + { + from_port = 18888 + to_port = 18888 + protocol = "tcp" + description = "VPC CIDR HTTPS" + cidr_blocks = module.vpc.vpc_cidr_block + }, + ] + + egress_with_cidr_blocks = [ + { + from_port = 86 + to_port = 86 + protocol = "252" + + description = "Disallow all traffic" + cidr_blocks = "255.255.255.255/32" + }, + ] + + tags = local.tags +} + + + +# workspace_security_group +module "emrstudio_workspace_sg" { + source = "terraform-aws-modules/security-group/aws" + version = "~> 4.0" + + name = "${local.name}-workspace-security-group" + description = "workspace SecurityGroup for EMR studio" + vpc_id = module.vpc.vpc_id + + egress_with_cidr_blocks = [ + { + from_port = 18888 + to_port = 18888 + protocol = "tcp" + description = "VPC CIDR HTTPS" + cidr_blocks = module.vpc.vpc_cidr_block + }, + ] + + tags = local.tags +} From aeab04e2ca7ec8029e485484d65bfa5305b1f04e Mon Sep 17 00:00:00 2001 From: Gu Date: Tue, 7 Feb 2023 13:47:17 -0600 Subject: [PATCH 13/26] add additional policy --- .../event-driven-pipeline/examples/ack/sfn.yaml | 16 ++++++++++------ main.tf | 13 ++++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/examples/event-driven-pipeline/examples/ack/sfn.yaml b/examples/event-driven-pipeline/examples/ack/sfn.yaml index 2b64d11..55f5db3 100644 --- a/examples/event-driven-pipeline/examples/ack/sfn.yaml +++ b/examples/event-driven-pipeline/examples/ack/sfn.yaml @@ -1,8 +1,12 @@ apiVersion: sfn.services.k8s.aws/v1alpha1 kind: StateMachine -name: run-spark-job-ack -roleARN: "arn:aws:iam::349361870252:role/service-role/StepFunctions-run-emreks-job-role-624b324e" # need to pass. need permission to access emr virtual cluster -tags: -- key: owner - value: sfn-ack -definition: "{\"Comment\":\"Adescriptionofmystatemachine\",\"StartAt\":\"input-output-s3\",\"States\":{\"input-output-s3\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"s3://step-functions-test2/scripts/pyspark-taxi-trip.py\",\"EntryPointArguments\":[\"s3://step-functions-test2/input/\",\"s3://step-functions-test2/output/\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=10\"}},\"ConfigurationOverrides\":{\"ApplicationConfiguration\":[{\"Classification\":\"spark-defaults\",\"Properties\":{\"spark.driver.cores\":\"1\",\"spark.executor.cores\":\"1\",\"spark.driver.memory\":\"10g\",\"spark.executor.memory\":\"10g\",\"spark.kubernetes.driver.podTemplateFile\":\"s3://step-functions-test2/scripts/driver-pod-template.yaml\",\"spark.kubernetes.executor.podTemplateFile\":\"s3://step-functions-test2/scripts/executor-pod-template.yaml\",\"spark.local.dir\":\"/data1,/data2\"}}],\"MonitoringConfiguration\":{\"PersistentAppUI\":\"ENABLED\",\"CloudWatchMonitoringConfiguration\":{\"LogGroupName\":\"emr-on-eks\",\"LogStreamNamePrefix\":\"test1\"}}}},\"Next\":\"local-data\"},\"local-data\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"local:///usr/lib/spark/examples/src/main/python/pi.py\",\"EntryPointArguments\":[\"60\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=2--confspark.executor.memory=1G--confspark.executor.cores=1--confspark.driver.cores=1\"}}},\"End\":true}}}" +metadata: + name: run-spark-job-ack + namespace: ack-demo +spec: + name: run-spark-job-ack + roleARN: "arn:aws:iam::349361870252:role/service-role/StepFunctions-run-emreks-job-role-624b324e" # need to pass. need permission to access emr virtual cluster + tags: + - key: owner + value: sfn-ack + definition: "{\"Comment\":\"Adescriptionofmystatemachine\",\"StartAt\":\"input-output-s3\",\"States\":{\"input-output-s3\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"s3://step-functions-test2/scripts/pyspark-taxi-trip.py\",\"EntryPointArguments\":[\"s3://step-functions-test2/input/\",\"s3://step-functions-test2/output/\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=10\"}},\"ConfigurationOverrides\":{\"ApplicationConfiguration\":[{\"Classification\":\"spark-defaults\",\"Properties\":{\"spark.driver.cores\":\"1\",\"spark.executor.cores\":\"1\",\"spark.driver.memory\":\"10g\",\"spark.executor.memory\":\"10g\",\"spark.kubernetes.driver.podTemplateFile\":\"s3://step-functions-test2/scripts/driver-pod-template.yaml\",\"spark.kubernetes.executor.podTemplateFile\":\"s3://step-functions-test2/scripts/executor-pod-template.yaml\",\"spark.local.dir\":\"/data1,/data2\"}}],\"MonitoringConfiguration\":{\"PersistentAppUI\":\"ENABLED\",\"CloudWatchMonitoringConfiguration\":{\"LogGroupName\":\"emr-on-eks\",\"LogStreamNamePrefix\":\"test1\"}}}},\"Next\":\"local-data\"},\"local-data\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"local:///usr/lib/spark/examples/src/main/python/pi.py\",\"EntryPointArguments\":[\"60\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=2--confspark.executor.memory=1G--confspark.executor.cores=1--confspark.driver.cores=1\"}}},\"End\":true}}}" diff --git a/main.tf b/main.tf index 2b867ee..528ad5e 100644 --- a/main.tf +++ b/main.tf @@ -583,7 +583,7 @@ module "sfn" { create_kubernetes_service_account = true kubernetes_service_account = local.sfn_name - irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn] + irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn,data.aws_iam_policy.sfn_additional[0].arn] } addon_context = local.addon_context @@ -594,6 +594,17 @@ data "aws_iam_policy" "sfn" { name = "AWSStepFunctionsFullAccess" } +data "aws_iam_policy_document" "sfn_additional" { + statement { + + effect = "Allow" + resources = ["*"] + + actions = [ + "iam:PassRole" + ] + } +} ################################################################################ # Event Bridge From 4db48fdd61bcbdb20f42060f84bb75dea3b250c0 Mon Sep 17 00:00:00 2001 From: Gu Date: Tue, 7 Feb 2023 13:56:30 -0600 Subject: [PATCH 14/26] add additional policy --- main.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 528ad5e..4d1d95e 100644 --- a/main.tf +++ b/main.tf @@ -583,7 +583,7 @@ module "sfn" { create_kubernetes_service_account = true kubernetes_service_account = local.sfn_name - irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn,data.aws_iam_policy.sfn_additional[0].arn] + irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn, aws_iam_policy.sfn_additional[0].arn] } addon_context = local.addon_context @@ -594,6 +594,16 @@ data "aws_iam_policy" "sfn" { name = "AWSStepFunctionsFullAccess" } + +resource "aws_iam_policy" "sfn_additional" { + count = var.enable_sfn ? 1 : 0 + + name_prefix = format("%s-%s", local.sfn_name, "controller-iam-policies") + description = "additional IAM policy for sfn controller" + path = "/" + policy = data.aws_iam_policy_document.sfn_additional.json +} + data "aws_iam_policy_document" "sfn_additional" { statement { From 758316f6ef90d2bc725c449b189c576136d232cf Mon Sep 17 00:00:00 2001 From: Gu Date: Tue, 7 Feb 2023 16:57:04 -0600 Subject: [PATCH 15/26] add eb chart --- examples/complete/main.tf | 3 +- .../examples/ack/sfn.yaml | 2 +- main.tf | 76 ++++++++++++++----- 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 9785d58..11199fe 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -117,7 +117,8 @@ module "eks_ack_addons" { enable_amp = true enable_emrcontainers = true enable_sfn = true - + enable_eb = true + tags = local.tags } diff --git a/examples/event-driven-pipeline/examples/ack/sfn.yaml b/examples/event-driven-pipeline/examples/ack/sfn.yaml index 55f5db3..0adff86 100644 --- a/examples/event-driven-pipeline/examples/ack/sfn.yaml +++ b/examples/event-driven-pipeline/examples/ack/sfn.yaml @@ -5,7 +5,7 @@ metadata: namespace: ack-demo spec: name: run-spark-job-ack - roleARN: "arn:aws:iam::349361870252:role/service-role/StepFunctions-run-emreks-job-role-624b324e" # need to pass. need permission to access emr virtual cluster + roleARN: "arn:aws:iam::349361870252:role/service-role/StepFunctions-run-emreks-job-role-624b324e" # need to pass. need permission to access emr virtual cluster. need PassRole tags: - key: owner value: sfn-ack diff --git a/main.tf b/main.tf index 4d1d95e..9c25470 100644 --- a/main.tf +++ b/main.tf @@ -583,7 +583,7 @@ module "sfn" { create_kubernetes_service_account = true kubernetes_service_account = local.sfn_name - irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn, aws_iam_policy.sfn_additional[0].arn] + irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn] } addon_context = local.addon_context @@ -595,26 +595,6 @@ data "aws_iam_policy" "sfn" { name = "AWSStepFunctionsFullAccess" } -resource "aws_iam_policy" "sfn_additional" { - count = var.enable_sfn ? 1 : 0 - - name_prefix = format("%s-%s", local.sfn_name, "controller-iam-policies") - description = "additional IAM policy for sfn controller" - path = "/" - policy = data.aws_iam_policy_document.sfn_additional.json -} - -data "aws_iam_policy_document" "sfn_additional" { - statement { - - effect = "Allow" - resources = ["*"] - - actions = [ - "iam:PassRole" - ] - } -} ################################################################################ # Event Bridge @@ -624,6 +604,60 @@ locals { eb_name = "ack-eb" } +module "eventbridge" { + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.0" + + count = var.enable_eb ? 1 : 0 + + helm_config = merge( + { + name = local.eb_name + chart = "eb-ack-chart" + repository = "oci://public.ecr.aws/k4r0k1t7" + version = "v0.0.2" + namespace = local.eb_name + repository_username = var.ecrpublic_username + repository_password = var.ecrpublic_token + create_namespace = true + description = "ACK eventbridge Controller v2 Helm chart deployment configuration" + values = [ + # shortens pod name from `ack-eb-eb-chart-xxxxxxxxxxxxx` to `ack-eb-xxxxxxxxxxxxx` + <<-EOT + nameOverride: ack-eb + EOT + ] + }, + var.eb_helm_config + ) + + set_values = [ + { + name = "serviceAccount.name" + value = local.eb_name + }, + { + name = "serviceAccount.create" + value = false + }, + { + name = "aws.region" + value = local.region + } + ] + + irsa_config = { + create_kubernetes_namespace = true + kubernetes_namespace = try(var.eb_helm_config.namespace, local.eb_name) + + create_kubernetes_service_account = true + kubernetes_service_account = local.eb_name + + irsa_iam_policies = [data.aws_iam_policy.eb[0].arn] + } + + addon_context = local.addon_context +} + data "aws_iam_policy" "eb" { count = var.enable_eb ? 1 : 0 From bb9717dcbae3282510641295347871b56e5e408c Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 8 Feb 2023 16:09:39 -0600 Subject: [PATCH 16/26] add exe roles --- .../ack-yamls/emr-virtualcluster.yaml | 14 + .../ack-yamls/eventbridge.yaml | 49 ++++ .../event-driven-pipeline/ack-yamls/s3.yaml | 18 ++ .../{examples/ack => ack-yamls}/sfn.yaml | 0 examples/event-driven-pipeline/addons.tf | 22 +- examples/event-driven-pipeline/data.tf | 251 ---------------- .../examples/ack/datapipeline.yaml | 37 --- .../examples/ack/jobrun.yaml | 25 -- .../spark-s3-podTemplate/basic-pyspark-job.sh | 53 ---- .../emr-eks-spark-amp-amg.sh | 118 -------- .../execute_emr_eks_job.sh | 125 -------- .../aws-cloudwatch-metrics-valyes.yaml | 9 - .../helm-values/crossplane-values.yaml | 3 - .../helm-values/prometheus-values.yaml | 65 ----- .../helm-values/vpa-values.yaml | 20 -- examples/event-driven-pipeline/main.tf | 275 ++++++++---------- examples/event-driven-pipeline/outputs.tf | 14 +- .../pod-templates/driver-pod-template.yaml | 0 .../pod-templates/executor-pod-template.yaml | 0 .../prepare-taxi-trip.sh | 0 .../pyspark-taxi-trip.py | 0 examples/event-driven-pipeline/vpc.tf | 60 ---- 22 files changed, 216 insertions(+), 942 deletions(-) create mode 100644 examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml create mode 100644 examples/event-driven-pipeline/ack-yamls/eventbridge.yaml create mode 100644 examples/event-driven-pipeline/ack-yamls/s3.yaml rename examples/event-driven-pipeline/{examples/ack => ack-yamls}/sfn.yaml (100%) delete mode 100644 examples/event-driven-pipeline/examples/ack/datapipeline.yaml delete mode 100644 examples/event-driven-pipeline/examples/ack/jobrun.yaml delete mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh delete mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh delete mode 100644 examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh delete mode 100644 examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml delete mode 100644 examples/event-driven-pipeline/helm-values/crossplane-values.yaml delete mode 100644 examples/event-driven-pipeline/helm-values/prometheus-values.yaml delete mode 100644 examples/event-driven-pipeline/helm-values/vpa-values.yaml rename examples/event-driven-pipeline/{examples/spark-s3-podTemplate/spark-scripts => spark-scripts-data}/pod-templates/driver-pod-template.yaml (100%) rename examples/event-driven-pipeline/{examples/spark-s3-podTemplate/spark-scripts => spark-scripts-data}/pod-templates/executor-pod-template.yaml (100%) rename examples/event-driven-pipeline/{examples/spark-s3-podTemplate => spark-scripts-data}/prepare-taxi-trip.sh (100%) rename examples/event-driven-pipeline/{examples/spark-s3-podTemplate/spark-scripts/scripts => spark-scripts-data}/pyspark-taxi-trip.py (100%) diff --git a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml new file mode 100644 index 0000000..c30bf5d --- /dev/null +++ b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: emrcontainers.services.k8s.aws/v1alpha1 +kind: VirtualCluster +metadata: + name: my-ack-vc +spec: + name: my-ack-vc + containerProvider: + id: emr-eks-ack # your eks cluster name + type_: EKS + info: + eksInfo: + namespace: emr-data-team-a + diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml new file mode 100644 index 0000000..5f5a586 --- /dev/null +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -0,0 +1,49 @@ +# apiVersion: eventbridge.services.k8s.aws/v1alpha1 +# kind: EventBus +# metadata: +# name: eventbus-ack +# annotations: +# services.k8s.aws/region: us-east-1 +# spec: +# name: eventbus-ack +# tags: +# - key: tag-01 +# value: value-01 +# --- +apiVersion: eventbridge.services.k8s.aws/v1alpha1 +kind: Rule +metadata: + name: eb-rule-ack + annotations: + services.k8s.aws/region: us-east-1 +spec: + name: eb-rule-ack + description: "ACK EventBridge Filter Rule to sfn using event bus reference" + #use default one if don't set + # eventBusRef: + # from: + # name: eventbus-ack + eventPattern: | # replace s3 bucket name and dir + { + "source": ["aws.s3"], + "detail-type": ["Object Created"], + "detail": { + "bucket": { + "name": ["sparkjobbucket"] + }, + "object": { + "key": [{ + "prefix": "input/" + }] + } + } + } + targets: + - arn: arn:aws:states:us-east-1:349361870252:stateMachine:run-spark-job-ack # replace with your sfn arn + id: sfn-run-spark-job-target + roleARN: arn:aws:iam::349361870252:role/service-role/Amazon_EventBridge_Invoke_Step_Functions_1004390267 # replace your EventBridge_Invoke_Step_Functions role arn + retryPolicy: + maximumRetryAttempts: 0 # no retries + tags: + - key: tag-01 + value: value-01 \ No newline at end of file diff --git a/examples/event-driven-pipeline/ack-yamls/s3.yaml b/examples/event-driven-pipeline/ack-yamls/s3.yaml new file mode 100644 index 0000000..81851cb --- /dev/null +++ b/examples/event-driven-pipeline/ack-yamls/s3.yaml @@ -0,0 +1,18 @@ +apiVersion: s3.services.k8s.aws/v1alpha1 +kind: Bucket +metadata: + name: sparkjobbucket +spec: + name: sparkjobbucket + notification: + EventBridgeConfiguration: + + events: + - string + filter: + key: + filterRules: + - name: string + value: string + id: string + topicARN: string diff --git a/examples/event-driven-pipeline/examples/ack/sfn.yaml b/examples/event-driven-pipeline/ack-yamls/sfn.yaml similarity index 100% rename from examples/event-driven-pipeline/examples/ack/sfn.yaml rename to examples/event-driven-pipeline/ack-yamls/sfn.yaml diff --git a/examples/event-driven-pipeline/addons.tf b/examples/event-driven-pipeline/addons.tf index 7c16e2d..1f7024c 100644 --- a/examples/event-driven-pipeline/addons.tf +++ b/examples/event-driven-pipeline/addons.tf @@ -14,8 +14,6 @@ module "eks_blueprints_kubernetes_addons" { enable_amazon_eks_kube_proxy = true enable_amazon_eks_aws_ebs_csi_driver = true - enable_aws_load_balancer_controller = true - enable_cert_manager = true #--------------------------------------- # Metrics Server #--------------------------------------- @@ -50,21 +48,6 @@ module "eks_blueprints_kubernetes_addons" { })] } - #--------------------------------------- - # CloudWatch metrics for EKS - #--------------------------------------- - enable_aws_cloudwatch_metrics = true - aws_cloudwatch_metrics_helm_config = { - name = "aws-cloudwatch-metrics" - chart = "aws-cloudwatch-metrics" - repository = "https://aws.github.io/eks-charts" - version = "0.0.7" - namespace = "amazon-cloudwatch" - values = [templatefile("${path.module}/helm-values/aws-cloudwatch-metrics-valyes.yaml", { - eks_cluster_id = var.name - })] - } - #--------------------------------------- # AWS for FluentBit - DaemonSet #--------------------------------------- @@ -100,6 +83,9 @@ module "eks_ack_addons" { ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password enable_sfn = true - + enable_s3 = true + enable_emrcontainers = true + enable_eb = true + tags = local.tags } diff --git a/examples/event-driven-pipeline/data.tf b/examples/event-driven-pipeline/data.tf index 6fda1ac..20d7a02 100644 --- a/examples/event-driven-pipeline/data.tf +++ b/examples/event-driven-pipeline/data.tf @@ -44,256 +44,5 @@ data "aws_iam_policy_document" "emr_on_eks" { } } -# emr-studio virtual cluster execution role -data "aws_iam_role" "emr-studio-role" { - name = format("%s-%s", var.name, "emr-eks-studio") - depends_on = [ - module.eks_blueprints - ] -} - -# emr studio role, based on https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-studio-service-role.html -data "aws_iam_policy_document" "emr_on_eks_emrstudio" { - statement { - sid = "AllowEMRReadOnlyActions" - effect = "Allow" - - actions = [ - "elasticmapreduce:ListInstances", - "elasticmapreduce:DescribeCluster", - "elasticmapreduce:ListSteps" - ] - - resources = ["*"] - } - - statement { - sid = "AllowEC2ENIActionsWithEMRTags" - effect = "Allow" - - actions = [ - "ec2:CreateNetworkInterfacePermission", - "ec2:DeleteNetworkInterface" - ] - - resources = [ - "arn:aws:ec2:*:*:network-interface/*" - ] - - } - - statement { - sid = "AllowEC2ENIAttributeAction" - effect = "Allow" - - actions = [ - "ec2:ModifyNetworkInterfaceAttribute" - ] - - resources = ["arn:aws:ec2:*:*:instance/*", - "arn:aws:ec2:*:*:network-interface/*", - "arn:aws:ec2:*:*:security-group/*"] - } - - statement { - sid = "AllowEC2SecurityGroupActionsWithEMRTags" - effect = "Allow" - - actions = [ - "ec2:AuthorizeSecurityGroupEgress", - "ec2:AuthorizeSecurityGroupIngress", - "ec2:RevokeSecurityGroupEgress", - "ec2:RevokeSecurityGroupIngress", - "ec2:DeleteNetworkInterfacePermission" - ] - - resources = [ - "*" - ] - - } - - - statement { - sid = "AllowDefaultEC2SecurityGroupsCreationWithEMRTags" - effect = "Allow" - - actions = [ - "ec2:CreateSecurityGroup" - ] - - resources = [ - "arn:aws:ec2:*:*:security-group/*" - ] - - } - - statement { - sid = "AllowDefaultEC2SecurityGroupsCreationInVPCWithEMRTags" - effect = "Allow" - - actions = [ - "ec2:CreateSecurityGroup" - ] - - resources = [ - "arn:aws:ec2:*:*:vpc/*" - ] - - } - - statement { - sid = "AllowAddingEMRTagsDuringDefaultSecurityGroupCreation" - effect = "Allow" - - actions = [ - "ec2:CreateTags" - ] - - resources = [ - "arn:aws:ec2:*:*:security-group/*" - ] - - condition { - test = "StringEquals" - variable = "ec2:CreateAction" - values = ["CreateSecurityGroup"] - } - } - - statement { - sid = "AllowEC2ENICreationWithEMRTags" - effect = "Allow" - - actions = [ - "ec2:CreateNetworkInterface" - ] - - resources = [ - "arn:aws:ec2:*:*:network-interface/*" - ] - - } - - statement { - sid = "AllowEC2ENICreationInSubnetAndSecurityGroupWithEMRTags" - effect = "Allow" - - actions = [ - "ec2:CreateNetworkInterface" - ] - - resources = [ - "arn:aws:ec2:*:*:subnet/*", - "arn:aws:ec2:*:*:security-group/*" - ] - - } - - statement { - sid = "AllowAddingTagsDuringEC2ENICreation" - effect = "Allow" - - actions = [ - "ec2:CreateTags" - ] - resources = [ - "arn:aws:ec2:*:*:network-interface/*" - ] - - condition { - test = "StringEquals" - variable = "ec2:CreateAction" - values = ["CreateNetworkInterface"] - } - } - - statement { - sid = "AllowSecretsManagerReadOnlyActionsWithEMRTags" - effect = "Allow" - - actions = [ - "secretsmanager:GetSecretValue" - ] - - resources = [ - "arn:aws:secretsmanager:*:*:secret:*" - ] - - } - - statement { - sid = "AllowEC2ReadOnlyActions" - effect = "Allow" - - actions = [ - "ec2:DescribeSecurityGroups", - "ec2:DescribeNetworkInterfaces", - "ec2:DescribeTags", - "ec2:DescribeInstances", - "ec2:DescribeSubnets", - "ec2:DescribeVpcs" - ] - - resources = ["*"] - } - - statement { - sid = "AllowWorkspaceCollaboration" - effect = "Allow" - - actions = [ - "iam:GetUser", - "iam:GetRole", - "iam:ListUsers", - "iam:ListRoles", - "sso:GetManagedApplicationInstance", - "sso-directory:SearchUsers" - ] - - resources = ["*"] - } - - statement { - sid = "AllowAccessToS3BucketEncryptionKey" - effect = "Allow" - - actions = [ - "kms:Decrypt", - "kms:GenerateDataKey", - "kms:ReEncrypt", - "kms:DescribeKey" - ] - - resources = ["*"] - } - - statement { - sid = "AllowEMRStudioAccesstoS3BucketforStudioWorkspaces" - effect = "Allow" - - actions = [ - "s3:PutObject", - "s3:GetObject", - "s3:GetEncryptionConfiguration", - "s3:ListBucket", - "s3:DeleteObject" - ] - - resources = ["*"] - } -} - -data "aws_iam_policy_document" "emr_studio_assume_role_policy" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["elasticmapreduce.amazonaws.com"] - } - - } -} diff --git a/examples/event-driven-pipeline/examples/ack/datapipeline.yaml b/examples/event-driven-pipeline/examples/ack/datapipeline.yaml deleted file mode 100644 index 082ca9f..0000000 --- a/examples/event-driven-pipeline/examples/ack/datapipeline.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -apiVersion: emrcontainers.services.k8s.aws/v1alpha1 -kind: VirtualCluster -metadata: - name: my-ack-vc -spec: - name: my-ack-vc - containerProvider: - id: emr-eks-ack # your eks cluster name - type_: EKS - info: - eksInfo: - namespace: emr-data-team-a ---- -# S3 bucket for spark job scripts and data input/output - ---- -# steps functions to orchestrate spark jobs in EMR on EKS -apiVersion: sfn.services.k8s.aws/v1alpha1 -kind: StateMachine -definition: string -loggingConfiguration: - destinations: - cloudWatchLogsLogGroup: - logGroupARN: string - includeExecutionData: boolean - level: string -name: string -roleARN: string -tags: -- key: string - value: string -tracingConfiguration: - enabled: boolean -type_: string ---- -# Event Bridge to capture the new data upload and trigger step functions diff --git a/examples/event-driven-pipeline/examples/ack/jobrun.yaml b/examples/event-driven-pipeline/examples/ack/jobrun.yaml deleted file mode 100644 index e4d8466..0000000 --- a/examples/event-driven-pipeline/examples/ack/jobrun.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -apiVersion: emrcontainers.services.k8s.aws/v1alpha1 -kind: JobRun -metadata: - name: my-ack-jobrun-myp7a5be -spec: - name: my-ack-jobrun-myp7a5be - virtualClusterRef: - from: - name: my-ack-vc - executionRoleARN: "{emr_on_eks_role_arn}" # emr_on_eks_role_arn for team a from terraform output - releaseLabel: "emr-6.7.0-latest" - jobDriver: - sparkSubmitJobDriver: - entryPoint: "local:///usr/lib/spark/examples/src/main/python/pi.py" - entryPointArguments: - sparkSubmitParameters: "--conf spark.executor.instances=2 --conf spark.executor.memory=1G --conf spark.executor.cores=1 --conf spark.driver.cores=1" - configurationOverrides: | - ApplicationConfiguration: null - MonitoringConfiguration: - CloudWatchMonitoringConfiguration: - LogGroupName: /emr-on-eks-logs/emr-eks-ack - LogStreamNamePrefix: pi-job - S3MonitoringConfiguration: - LogUri: s3://emr-eks-ack-rapthg5f diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh deleted file mode 100644 index 6193553..0000000 --- a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/basic-pyspark-job.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -if [ $# -eq 0 ]; -then - echo "$0: Missing arguments ENTER_EMR_EMR_VIRTUAL_CLUSTER_ID and EMR_JOB_EXECUTION_ROLE_ARN" - echo "USAGE: ./basic-pyspark-job '' ''" - exit 1 -elif [ $# -gt 3 ]; -then - echo "$0: Too many arguments: $@" - echo "Usage example-> ./basic-pyspark-job '' ''" - exit 1 -else - echo "We got some argument(s)" - echo "===========================" - echo "Number of arguments.: $#" - echo "List of arguments...: $@" - echo "Arg #1..............: $1" - echo "Arg #2..............: $2" - echo "===========================" -fi - -#-------------------------------------------- -# INPUT VARIABLES -#-------------------------------------------- -EMR_EMR_VIRTUAL_CLUSTER_ID=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` -EMR_JOB_EXECUTION_ROLE_ARN=$2 # Terraform output variable is emr_on_eks_role_arn -JOB_NAME='pi' -EMR_EKS_RELEASE_LABEL='emr-6.5.0-latest' - -#-------------------------------------------- -# DERIVED VARIABLES -#-------------------------------------------- -EMR_VIRTUAL_CLUSTER_NAME=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].name" --output text) - -# Execute Spark job -if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then - echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." - aws emr-containers start-job-run \ - --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ - --name $JOB_NAME \ - --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ - --release-label $EMR_EKS_RELEASE_LABEL \ - --job-driver '{ - "sparkSubmitJobDriver": { - "entryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py", - "sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1" - } - }' - -else - echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" -fi diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh deleted file mode 100644 index d0d3f89..0000000 --- a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/emr-eks-spark-amp-amg.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash - -if [ $# -ne 3 ]; -then - echo "$0: Missing arguments ENTER_EMR_VIRTUAL_CLUSTER_ID, S3_BUCKET_NAME and EMR_JOB_EXECUTION_ROLE_ARN" - echo "USAGE: ./emr-eks-spark-amp-amg.sh '' '' ''" - exit 1 -else - echo "We got some argument(s)" - echo "===========================" - echo "Number of arguments.: $#" - echo "List of arguments...: $@" - echo "Arg #1..............: $1" - echo "Arg #2..............: $2" - echo "Arg #3..............: $3" - echo "===========================" -fi - -#-------------------------------------------- -# INPUT VARIABLES -#-------------------------------------------- -EMR_VIRTUAL_CLUSTER_ID=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` -S3_BUCKET=$2 # This script requires s3 bucket as input parameter e.g., s3:// -EMR_JOB_EXECUTION_ROLE_ARN=$3 # Terraform output variable is emr_on_eks_role_arn - -#-------------------------------------------- -# DERIVED VARIABLES -#-------------------------------------------- -EMR_VIRTUAL_CLUSTER_NAMESPACE=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].containerProvider.info.eksInfo" --output text) -EMR_VIRTUAL_CLUSTER_NAME=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?id=='${EMR_VIRTUAL_CLUSTER_ID}' && state=='RUNNING'].name" --output text) - -#-------------------------------------------- -# DEFAULT VARIABLES CAN BE MODIFIED -#-------------------------------------------- -JOB_NAME='taxidata' -EMR_EKS_RELEASE_LABEL="emr-6.5.0-latest" -SPARK_JOB_S3_PATH="${S3_BUCKET}/emr_virtual_cluster_name=${EMR_VIRTUAL_CLUSTER_NAME}/namespace=${EMR_VIRTUAL_CLUSTER_NAMESPACE}/job_name=${JOB_NAME}" - -#-------------------------------------------- -# CLOUDWATCH LOG GROUP NAME -#-------------------------------------------- -CW_LOG_GROUP="/emr-on-eks-logs/${EMR_VIRTUAL_CLUSTER_NAME}/${EMR_VIRTUAL_CLUSTER_NAMESPACE}" # Create CW Log group if not exist - -#-------------------------------------------- -# Download sample input data from https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page -#-------------------------------------------- -Create folder locally to store the input data -mkdir -p "spark-scripts/input" - -# Download the input data from public data set to local folders -max=40 -for (( i=1; i <= $max; ++i )) -do - wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "spark-scripts/input/yellow_tripdata_2022-${i}.parquet" -done - -#-------------------------------------------- -# This command copies PySpark script and the Pod templates to S3 bucket -#-------------------------------------------- -aws s3 sync ./spark-scripts/ "${SPARK_JOB_S3_PATH}/" - -#-------------------------------------------- -# Execute Spark job -#-------------------------------------------- - -if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then - echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." - aws emr-containers start-job-run \ - --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ - --name $JOB_NAME \ - --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ - --release-label $EMR_EKS_RELEASE_LABEL \ - --job-driver '{ - "sparkSubmitJobDriver": { - "entryPoint": "'"$SPARK_JOB_S3_PATH"'/scripts/sample-spark-taxi-trip.py", - "entryPointArguments": ["'"$SPARK_JOB_S3_PATH"'/input/", - "'"$SPARK_JOB_S3_PATH"'/output/" - ], - "sparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.cores=1 --conf spark.driver.cores=1" - } - }' \ - --configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.kubernetes.driver.podTemplateFile":"'"$SPARK_JOB_S3_PATH"'/pod-templates/nvme-spark-driver.yaml", - "spark.kubernetes.executor.podTemplateFile":"'"$SPARK_JOB_S3_PATH"'/pod-templates/nvme-spark-executor.yaml", - "spark.driver.memory":"2g", - "spark.executor.memory":"4g", - "spark.local.dir" : "/data1,/data2", - "spark.kubernetes.executor.podNamePrefix":"'"$JOB_NAME"'", - "spark.ui.prometheus.enabled":"true", - "spark.executor.processTreeMetrics.enabled":"true", - "spark.kubernetes.driver.annotation.prometheus.io/scrape":"true", - "spark.kubernetes.driver.annotation.prometheus.io/path":"/metrics/executors/prometheus/", - "spark.kubernetes.driver.annotation.prometheus.io/port":"4040", - "spark.kubernetes.driver.service.annotation.prometheus.io/scrape":"true", - "spark.kubernetes.driver.service.annotation.prometheus.io/path":"/metrics/driver/prometheus/", - "spark.kubernetes.driver.service.annotation.prometheus.io/port":"4040", - "spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet", - "spark.metrics.conf.*.sink.prometheusServlet.path":"/metrics/driver/prometheus/", - "spark.metrics.conf.master.sink.prometheusServlet.path":"/metrics/master/prometheus/", - "spark.metrics.conf.applications.sink.prometheusServlet.path":"/metrics/applications/prometheus/" - } - } - ], - "monitoringConfiguration": { - "persistentAppUI":"ENABLED", - "cloudWatchMonitoringConfiguration": { - "logGroupName":"'"$CW_LOG_GROUP"'", - "logStreamNamePrefix":"'"$JOB_NAME"'" - } - } - }' -else - echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" -fi diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh b/examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh deleted file mode 100644 index c8cc30f..0000000 --- a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/execute_emr_eks_job.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -if [ $# -ne 4 ]; -then - echo "$0: Missing arguments EMR_VIRTUAL_CLUSTER_NAME, S3_BUCKET_NAME and EMR_JOB_EXECUTION_ROLE_ARN" - echo "USAGE: ./execute_emr_eks_job.sh '' '' ''" - exit 1 -else - echo "We got some argument(s)" - echo "===========================" - echo "Number of arguments.: $#" - echo "List of arguments...: $@" - echo "Arg #1..............: $1" - echo "Arg #2..............: $2" - echo "Arg #3..............: $3" - echo "===========================" -fi - -#-------------------------------------------- -# INPUT VARIABLES -#-------------------------------------------- -EMR_VIRTUAL_CLUSTER_NAME=$1 # Terraform output variable is `emrcontainers_virtual_cluster_id` -S3_BUCKET=$2 # This script requires s3 bucket as input parameter e.g., s3:// -EMR_JOB_EXECUTION_ROLE_ARN=$3 # Terraform output variable is emr_on_eks_role_arn - -#-------------------------------------------- -# DERIVED VARIABLES -#-------------------------------------------- -EMR_VIRTUAL_CLUSTER_ID=$(aws emr-containers list-virtual-clusters --query "virtualClusters[?name == '$EMR_VIRTUAL_CLUSTER_NAME' && state == 'RUNNING'].id" --output text) - -#-------------------------------------------- -# DEFAULT VARIABLES CAN BE MODIFIED -#-------------------------------------------- -JOB_NAME='taxidata' -EMR_EKS_RELEASE_LABEL="emr-6.7.0-latest" # Spark 3.2.1 -CW_LOG_GROUP="/emr-on-eks-logs/${EMR_VIRTUAL_CLUSTER_NAME}" # Create CW Log group if not exist - -SPARK_JOB_S3_PATH="${S3_BUCKET}/${EMR_VIRTUAL_CLUSTER_NAME}/${JOB_NAME}" -SCRIPTS_S3_PATH="${SPARK_JOB_S3_PATH}/scripts" -INPUT_DATA_S3_PATH="${SPARK_JOB_S3_PATH}/input" -OUTPUT_DATA_S3_PATH="${SPARK_JOB_S3_PATH}/output" - -#-------------------------------------------- -# Copy PySpark Scripts, Pod Templates and Input data to S3 bucket -#-------------------------------------------- -aws s3 sync "./" ${SCRIPTS_S3_PATH} - -#-------------------------------------------- -# NOTE: This section downloads the test data from AWS Public Dataset. You can comment this section and bring your own input data required for sample PySpark test -# https://registry.opendata.aws/nyc-tlc-trip-records-pds/ -#-------------------------------------------- - -mkdir -p "../input" -# Download the input data from public data set to local folders -wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "../input/yellow_tripdata_2022-0.parquet" - -# Making duplicate copies to increase the size of the data. -max=20 -for (( i=1; i <= $max; ++i )) -do - cp -rf "../input/yellow_tripdata_2022-0.parquet" "../input/yellow_tripdata_2022-${i}.parquet" -done - -aws s3 sync "../input" ${INPUT_DATA_S3_PATH} # Sync from local folder to S3 path - -#-------------------------------------------- -# Execute Spark job -#-------------------------------------------- - -if [[ $EMR_VIRTUAL_CLUSTER_ID != "" ]]; then - echo "Found Cluster $EMR_VIRTUAL_CLUSTER_NAME; Executing the Spark job now..." - aws emr-containers start-job-run \ - --virtual-cluster-id $EMR_VIRTUAL_CLUSTER_ID \ - --name $JOB_NAME \ - --execution-role-arn $EMR_JOB_EXECUTION_ROLE_ARN \ - --release-label $EMR_EKS_RELEASE_LABEL \ - --job-driver '{ - "sparkSubmitJobDriver": { - "entryPoint": "'"$SCRIPTS_S3_PATH"'/pyspark-taxi-trip.py", - "entryPointArguments": ["'"$INPUT_DATA_S3_PATH"'", - "'"$OUTPUT_DATA_S3_PATH"'" - ], - "sparkSubmitParameters": "--conf spark.executor.instances=10" - } - }' \ - --configuration-overrides '{ - "applicationConfiguration": [ - { - "classification": "spark-defaults", - "properties": { - "spark.driver.cores":"1", - "spark.executor.cores":"1", - "spark.driver.memory": "10g", - "spark.executor.memory": "10g", - "spark.kubernetes.driver.podTemplateFile":"'"$SCRIPTS_S3_PATH"'/driver-pod-template.yaml", - "spark.kubernetes.executor.podTemplateFile":"'"$SCRIPTS_S3_PATH"'/executor-pod-template.yaml", - "spark.local.dir" : "/data1,/data2", - - "spark.kubernetes.executor.podNamePrefix":"'"$JOB_NAME"'", - "spark.ui.prometheus.enabled":"true", - "spark.executor.processTreeMetrics.enabled":"true", - "spark.kubernetes.driver.annotation.prometheus.io/scrape":"true", - "spark.kubernetes.driver.annotation.prometheus.io/path":"/metrics/executors/prometheus/", - "spark.kubernetes.driver.annotation.prometheus.io/port":"4040", - "spark.kubernetes.driver.service.annotation.prometheus.io/scrape":"true", - "spark.kubernetes.driver.service.annotation.prometheus.io/path":"/metrics/driver/prometheus/", - "spark.kubernetes.driver.service.annotation.prometheus.io/port":"4040", - "spark.metrics.conf.*.sink.prometheusServlet.class":"org.apache.spark.metrics.sink.PrometheusServlet", - "spark.metrics.conf.*.sink.prometheusServlet.path":"/metrics/driver/prometheus/", - "spark.metrics.conf.master.sink.prometheusServlet.path":"/metrics/master/prometheus/", - "spark.metrics.conf.applications.sink.prometheusServlet.path":"/metrics/applications/prometheus/" - } - } - ], - "monitoringConfiguration": { - "persistentAppUI":"ENABLED", - "cloudWatchMonitoringConfiguration": { - "logGroupName":"'"$CW_LOG_GROUP"'", - "logStreamNamePrefix":"'"$JOB_NAME"'" - } - } - }' -else - echo "Cluster is not in running state $EMR_VIRTUAL_CLUSTER_NAME" -fi diff --git a/examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml b/examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml deleted file mode 100644 index 0af891e..0000000 --- a/examples/event-driven-pipeline/helm-values/aws-cloudwatch-metrics-valyes.yaml +++ /dev/null @@ -1,9 +0,0 @@ -clusterName: ${eks_cluster_id} - -resources: - limits: - cpu: 500m - memory: 2Gi - requests: - cpu: 200m - memory: 1Gi diff --git a/examples/event-driven-pipeline/helm-values/crossplane-values.yaml b/examples/event-driven-pipeline/helm-values/crossplane-values.yaml deleted file mode 100644 index 6b0d50c..0000000 --- a/examples/event-driven-pipeline/helm-values/crossplane-values.yaml +++ /dev/null @@ -1,3 +0,0 @@ -nodeSelector: - kubernetes.io/os: ${operating-system} - NodeGroupType: core diff --git a/examples/event-driven-pipeline/helm-values/prometheus-values.yaml b/examples/event-driven-pipeline/helm-values/prometheus-values.yaml deleted file mode 100644 index 600952d..0000000 --- a/examples/event-driven-pipeline/helm-values/prometheus-values.yaml +++ /dev/null @@ -1,65 +0,0 @@ -server: - retention: 1h - remoteWrite: - - queue_config: - max_samples_per_send: 1000 - max_shards: 200 - capacity: 2500 - global: - evaluation_interval: 30s - scrape_interval: 30s - scrape_timeout: 10s - - resources: - requests: - cpu: 500m - memory: 1Gi - - verticalAutoscaler: - enabled: true - updateMode: "Auto" - containerPolicies: - - containerName: "prometheus-server" - minAllowed: - cpu: 500m - memory: 1Gi - maxAllowed: - cpu: 1000m - memory: 4Gi - - nodeSelector: - kubernetes.io/os: ${operating_system} - NodeGroupType: core - persistentVolume: - accessModes: - - ReadWriteOnce - enabled: true - mountPath: /data - size: 20Gi - storageClass: gp2 - -alertmanager: - nodeSelector: - kubernetes.io/os: ${operating_system} - NodeGroupType: core - -kube-state-metrics: - nodeSelector: - kubernetes.io/os: ${operating_system} - NodeGroupType: core - -pushgateway: - nodeSelector: - kubernetes.io/os: ${operating_system} - NodeGroupType: core - -nodeExporter: - nodeSelector: - kubernetes.io/os: ${operating_system} - - # Additional scrape config for Apache YuniKorn Scheduler metrics - # - job_name: 'yunikorn' - # metrics_path: '/ws/v1/metrics' - # static_configs: - # - targets: - # - yunikorn-service.yunikorn.svc.cluster.local:9080 diff --git a/examples/event-driven-pipeline/helm-values/vpa-values.yaml b/examples/event-driven-pipeline/helm-values/vpa-values.yaml deleted file mode 100644 index 948fa6d..0000000 --- a/examples/event-driven-pipeline/helm-values/vpa-values.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Default values for vertical-pod-autoscaler. -serviceAccount: - name: vpa - -recommender: - enabled: true - extraArgs: - v: "4" - pod-recommendation-min-cpu-millicores: 15 - pod-recommendation-min-memory-mb: 100 - # Best practise to use Prometheus as a history provider for the VPA recommender. Not required for this example -# storage: prometheus -# prometheus-address: http://prometheus-server.prometheus.svc.cluster.local:9090 - nodeSelector: - NodeGroupType: core - kubernetes.io/os: ${operating_system} -updater: - enabled: true - nodeSelector: - kubernetes.io/os: ${operating_system} diff --git a/examples/event-driven-pipeline/main.tf b/examples/event-driven-pipeline/main.tf index 7a4998b..6d93ad9 100644 --- a/examples/event-driven-pipeline/main.tf +++ b/examples/event-driven-pipeline/main.tf @@ -51,109 +51,15 @@ module "eks_blueprints" { } managed_node_groups = { - # Core node group for deploying all the critical add-ons - mng1 = { - node_group_name = "core-node-grp" + example = { + node_group_name = "example" + instance_types = ["m5.large"] + min_size = 3 subnet_ids = module.vpc.private_subnets - - instance_types = ["m5.xlarge"] - ami_type = "AL2_x86_64" - capacity_type = "ON_DEMAND" - - disk_size = 100 - disk_type = "gp3" - - max_size = 9 - min_size = 3 - desired_size = 3 - create_launch_template = true - launch_template_os = "amazonlinux2eks" - - update_config = [{ - max_unavailable_percentage = 50 - }] - - k8s_labels = { - Environment = "preprod" - Zone = "test" - WorkerType = "ON_DEMAND" - NodeGroupType = "core" - } - - additional_tags = { - Name = "core-node-grp" - subnet_type = "private" - "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" - "k8s.io/cluster-autoscaler/node-template/label/noderole" = "core" - "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" - "k8s.io/cluster-autoscaler/experiments" = "owned" - "k8s.io/cluster-autoscaler/enabled" = "true" - } - }, - #--------------------------------------- - # Note: This example only uses ON_DEMAND node group for both Spark Driver and Executors. - # If you want to leverage SPOT nodes for Spark executors then create ON_DEMAND node group for placing your driver pods and SPOT nodegroup for executors. - # Use NodeSelectors to place your driver/executor pods with the help of Pod Templates. - #--------------------------------------- - mng2 = { - node_group_name = "spark-node-grp" - subnet_ids = [module.vpc.private_subnets[0]] - instance_types = ["r5d.4xlarge"] - ami_type = "AL2_x86_64" - capacity_type = "SPOT" - - # Enable this option only when you are using NVMe disks - format_mount_nvme_disk = true # Mounts NVMe disks to /local1, /local2 etc. for multiple NVMe disks - - # RAID0 configuration is recommended for better performance when you use larger instances with multiple NVMe disks e.g., r5d.24xlarge - # Permissions for hadoop user runs the analytics job. user > hadoop:x:999:1000::/home/hadoop:/bin/bash - post_userdata = <<-EOT - #!/bin/bash - set -ex - mkdir local1 - mkdir local2 - /usr/bin/chown -hR +999:+1000 /local* - EOT - - disk_size = 100 - disk_type = "gp3" - - max_size = 9 # Managed node group soft limit is 450; request AWS for limit increase - min_size = 3 - desired_size = 3 - - create_launch_template = true - launch_template_os = "amazonlinux2eks" - - update_config = [{ - max_unavailable_percentage = 50 - }] - - additional_iam_policies = [] - k8s_taints = [] - - k8s_labels = { - Environment = "preprod" - Zone = "test" - WorkerType = "ON_DEMAND" - NodeGroupType = "spark" - } - - additional_tags = { - Name = "spark-node-grp" - subnet_type = "private" - "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" - "k8s.io/cluster-autoscaler/node-template/label/noderole" = "spark" - "k8s.io/cluster-autoscaler/node-template/label/disk" = "nvme" - "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" - "k8s.io/cluster-autoscaler/experiments" = "owned" - "k8s.io/cluster-autoscaler/enabled" = "true" - } - }, + } } + #--------------------------------------- # ENABLE EMR ON EKS # 1. Creates namespace @@ -169,11 +75,6 @@ module "eks_blueprints" { job_execution_role = "emr-eks-data-team-a" additional_iam_policies = [aws_iam_policy.emr_on_eks.arn] } - emr-studio = { - namespace = "emr-studio" - job_execution_role = "emr-eks-studio" #role for managed endpoint - additional_iam_policies = [aws_iam_policy.emr_on_eks.arn] - } } tags = local.tags } @@ -188,57 +89,129 @@ resource "aws_iam_policy" "emr_on_eks" { policy = data.aws_iam_policy_document.emr_on_eks.json } - - #--------------------------------------------------------------- -# Create EMR on EKS Virtual Cluster +# IAM execution role for Step functions #--------------------------------------------------------------- -resource "aws_emrcontainers_virtual_cluster" "this" { - name = format("%s-%s", module.eks_blueprints.eks_cluster_id, "emr-studio") - - container_provider { - id = module.eks_blueprints.eks_cluster_id - type = "EKS" - - info { - eks_info { - namespace = "emr-studio" - } - } +resource "aws_iam_role" "sfn_execution_role" { + name = "${local.cluster_name}-efn-execution-role" + + # Terraform's "jsonencode" function converts a + # Terraform expression result to valid JSON syntax. + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "states.amazonaws.com" + } + + }, + ] + }) + + inline_policy { + name = "my_inline_policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + "Effect": "Allow", + "Action": "emr-containers:StartJobRun", + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "emr-containers:DescribeJobRun", + "emr-containers:CancelJobRun" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogDelivery", + "logs:GetLogDelivery", + "logs:UpdateLogDelivery", + "logs:DeleteLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:DescribeResourcePolicies", + "logs:DescribeLogGroups" + ], + "Resource": "*" + }, + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": "*", + "Sid": "" + }, + { + "Effect": "Allow", + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets" + ], + "Resource": [ + "*" + ] + } + ] + }) } } -module "s3_bucket" { - source = "terraform-aws-modules/s3-bucket/aws" - version = "~> 3.0" - - bucket = "emr-studio-${random_id.this.hex}" - acl = "private" - - # For example only - please evaluate for your environment - force_destroy = true - - attach_deny_insecure_transport_policy = true - attach_require_latest_tls_policy = true - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true - server_side_encryption_configuration = { - rule = { - apply_server_side_encryption_by_default = { - sse_algorithm = "AES256" - } - } +#--------------------------------------------------------------- +# IAM execution role for EventBridge +#--------------------------------------------------------------- +resource "aws_iam_role" "eb_execution_role" { + name = "${local.cluster_name}-eb-execution-role" + + # Terraform's "jsonencode" function converts a + # Terraform expression result to valid JSON syntax. + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Sid = "" + Principal = { + Service = "events.amazonaws.com" + } + + }, + ] + }) + + inline_policy { + name = "my_inline_policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + "Effect": "Allow", + "Action": [ + "states:StartExecution" + ], + "Resource": [ + "*" + ] + } + ] + }) } - - tags = local.tags } - -resource "random_id" "this" { - byte_length = "2" -} - - diff --git a/examples/event-driven-pipeline/outputs.tf b/examples/event-driven-pipeline/outputs.tf index dd8702f..cefb113 100644 --- a/examples/event-driven-pipeline/outputs.tf +++ b/examples/event-driven-pipeline/outputs.tf @@ -14,12 +14,12 @@ output "emr_on_eks_role_arn" { value = module.eks_blueprints.emr_on_eks_role_arn } -output "emr_studio_service_role_arn" { - description = "EMR studio service role ARN" - value = aws_iam_role.emr_studio_service_role.arn +output "stepfunctions_role_arn" { + description = "IAM execution role arn for step functions" + value = aws_iam_role.sfn_execution_role.arn } -output "emr_studio_S3_bucket_id" { - description = "EMR studio s3 bucket id" - value = module.s3_bucket.s3_bucket_id -} +output "eventbridge_role_arn" { + description = "IAM execution role arn for eventbridge" + value = aws_iam_role.eb_execution_role.arn +} \ No newline at end of file diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/driver-pod-template.yaml b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml similarity index 100% rename from examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/driver-pod-template.yaml rename to examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/executor-pod-template.yaml b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml similarity index 100% rename from examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/pod-templates/executor-pod-template.yaml rename to examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/prepare-taxi-trip.sh b/examples/event-driven-pipeline/spark-scripts-data/prepare-taxi-trip.sh similarity index 100% rename from examples/event-driven-pipeline/examples/spark-s3-podTemplate/prepare-taxi-trip.sh rename to examples/event-driven-pipeline/spark-scripts-data/prepare-taxi-trip.sh diff --git a/examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/scripts/pyspark-taxi-trip.py b/examples/event-driven-pipeline/spark-scripts-data/pyspark-taxi-trip.py similarity index 100% rename from examples/event-driven-pipeline/examples/spark-s3-podTemplate/spark-scripts/scripts/pyspark-taxi-trip.py rename to examples/event-driven-pipeline/spark-scripts-data/pyspark-taxi-trip.py diff --git a/examples/event-driven-pipeline/vpc.tf b/examples/event-driven-pipeline/vpc.tf index 05995cd..447b668 100644 --- a/examples/event-driven-pipeline/vpc.tf +++ b/examples/event-driven-pipeline/vpc.tf @@ -110,63 +110,3 @@ module "vpc_endpoints" { tags = local.tags } - -#--------------------------------------------------------------- -# security groups for EMR Studio -#--------------------------------------------------------------- -# engine_security_group -module "emrstudio_engine_sg" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 4.0" - - name = "${local.name}-engine-security-group" - description = "engineSecurityGroup for EMR studio" - vpc_id = module.vpc.vpc_id - - ingress_with_cidr_blocks = [ - { - from_port = 18888 - to_port = 18888 - protocol = "tcp" - description = "VPC CIDR HTTPS" - cidr_blocks = module.vpc.vpc_cidr_block - }, - ] - - egress_with_cidr_blocks = [ - { - from_port = 86 - to_port = 86 - protocol = "252" - - description = "Disallow all traffic" - cidr_blocks = "255.255.255.255/32" - }, - ] - - tags = local.tags -} - - - -# workspace_security_group -module "emrstudio_workspace_sg" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 4.0" - - name = "${local.name}-workspace-security-group" - description = "workspace SecurityGroup for EMR studio" - vpc_id = module.vpc.vpc_id - - egress_with_cidr_blocks = [ - { - from_port = 18888 - to_port = 18888 - protocol = "tcp" - description = "VPC CIDR HTTPS" - cidr_blocks = module.vpc.vpc_cidr_block - }, - ] - - tags = local.tags -} From 682ee0f3363e6bb78b8a69df3672760dfb70aad7 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Thu, 9 Feb 2023 01:55:51 +0000 Subject: [PATCH 17/26] clean up --- README.md | 8 + examples/event-driven-pipeline/README.md | 26 +-- .../ack-yamls/emr-virtualcluster.yaml | 1 - .../ack-yamls/eventbridge.yaml | 2 +- .../event-driven-pipeline/ack-yamls/sfn.yaml | 70 +++++++- examples/event-driven-pipeline/addons.tf | 21 +-- examples/event-driven-pipeline/data.tf | 3 - examples/event-driven-pipeline/main.tf | 164 ++++++++++++------ examples/event-driven-pipeline/outputs.tf | 2 +- main.tf | 2 +- variables.tf | 2 +- 11 files changed, 202 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 500cb51..795d38d 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,10 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [api\_gatewayv2](#module\_api\_gatewayv2) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | | [dynamodb](#module\_dynamodb) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | | [emrcontainers](#module\_emrcontainers) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | +| [eventbridge](#module\_eventbridge) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | | [rds](#module\_rds) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | | [s3](#module\_s3) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | +| [sfn](#module\_sfn) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | ## Resources @@ -70,8 +72,10 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [aws_iam_policy.api_gatewayv2_admin](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.api_gatewayv2_invoke](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.dynamodb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy.eb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.rds](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | +| [aws_iam_policy.sfn](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy) | data source | | [aws_iam_policy_document.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | @@ -85,19 +89,23 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | [cluster\_id](#input\_cluster\_id) | EKS Cluster Id | `string` | n/a | yes | | [data\_plane\_wait\_arn](#input\_data\_plane\_wait\_arn) | Addon deployment will not proceed until this value is known. Set to node group/Fargate profile ARN to wait for data plane to be ready before provisioning addons | `string` | `""` | no | | [dynamodb\_helm\_config](#input\_dynamodb\_helm\_config) | ACK dynamodb Helm Chart config | `any` | `{}` | no | +| [eb\_helm\_config](#input\_eb\_helm\_config) | ACK Event Bridge Helm Chart config | `any` | `{}` | no | | [ecrpublic\_token](#input\_ecrpublic\_token) | Password decoded from the authorization token for accessing public ECR | `string` | n/a | yes | | [ecrpublic\_username](#input\_ecrpublic\_username) | User name decoded from the authorization token for accessing public ECR | `string` | n/a | yes | | [emrcontainers\_helm\_config](#input\_emrcontainers\_helm\_config) | ACK EMR container Helm Chart config | `any` | `{}` | no | | [enable\_amp](#input\_enable\_amp) | Enable ACK amp add-on | `bool` | `false` | no | | [enable\_api\_gatewayv2](#input\_enable\_api\_gatewayv2) | Enable ACK API gateway v2 add-on | `bool` | `false` | no | | [enable\_dynamodb](#input\_enable\_dynamodb) | Enable ACK dynamodb add-on | `bool` | `false` | no | +| [enable\_eb](#input\_enable\_eb) | Enable ACK Event Bridge add-on | `bool` | `false` | no | | [enable\_emrcontainers](#input\_enable\_emrcontainers) | Enable ACK EMR container add-on | `bool` | `false` | no | | [enable\_rds](#input\_enable\_rds) | Enable ACK rds add-on | `bool` | `false` | no | | [enable\_s3](#input\_enable\_s3) | Enable ACK s3 add-on | `bool` | `false` | no | +| [enable\_sfn](#input\_enable\_sfn) | Enable ACK step functions add-on | `bool` | `false` | no | | [irsa\_iam\_permissions\_boundary](#input\_irsa\_iam\_permissions\_boundary) | IAM permissions boundary for IRSA roles | `string` | `""` | no | | [irsa\_iam\_role\_path](#input\_irsa\_iam\_role\_path) | IAM role path for IRSA roles | `string` | `"/"` | no | | [rds\_helm\_config](#input\_rds\_helm\_config) | ACK rds Helm Chart config | `any` | `{}` | no | | [s3\_helm\_config](#input\_s3\_helm\_config) | ACK s3 Helm Chart config | `any` | `{}` | no | +| [sfn\_helm\_config](#input\_sfn\_helm\_config) | ACK step functions Helm Chart config | `any` | `{}` | no | | [tags](#input\_tags) | Additional tags (e.g. `map('BusinessUnit`,`XYZ`) | `map(string)` | `{}` | no | ## Outputs diff --git a/examples/event-driven-pipeline/README.md b/examples/event-driven-pipeline/README.md index 4cccfb7..d636763 100644 --- a/examples/event-driven-pipeline/README.md +++ b/examples/event-driven-pipeline/README.md @@ -17,18 +17,14 @@ This pattern is used to deploy the EKS Cluster with ACK Controllers for building |------|---------| | [aws](#provider\_aws) | >= 3.72 | | [aws.ecr](#provider\_aws.ecr) | >= 3.72 | -| [random](#provider\_random) | n/a | ## Modules | Name | Source | Version | |------|--------|---------| -| [eks\_ack\_addons](#module\_eks\_ack\_addons) | github.com/aws-ia/terraform-aws-eks-ack-addons | n/a | +| [eks\_ack\_addons](#module\_eks\_ack\_addons) | ../../ | n/a | | [eks\_blueprints](#module\_eks\_blueprints) | github.com/aws-ia/terraform-aws-eks-blueprints | v4.18.1 | | [eks\_blueprints\_kubernetes\_addons](#module\_eks\_blueprints\_kubernetes\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons | v4.18.1 | -| [emrstudio\_engine\_sg](#module\_emrstudio\_engine\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | -| [emrstudio\_workspace\_sg](#module\_emrstudio\_workspace\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | -| [s3\_bucket](#module\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 3.0 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | | [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 3.0 | | [vpc\_endpoints\_sg](#module\_vpc\_endpoints\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | @@ -37,21 +33,14 @@ This pattern is used to deploy the EKS Cluster with ACK Controllers for building | Name | Type | |------|------| -| [aws_emr_studio.example](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/emr_studio) | resource | -| [aws_emrcontainers_virtual_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/emrcontainers_virtual_cluster) | resource | | [aws_iam_policy.emr_on_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.emr_on_eks_emrstudio](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.emr_studio_service_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_prometheus_workspace.amp](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/prometheus_workspace) | resource | -| [random_id.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [aws_iam_role.eb_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.sfn_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | | [aws_iam_policy_document.emr_on_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.emr_on_eks_emrstudio](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.emr_studio_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_role.emr-studio-role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | @@ -59,9 +48,8 @@ This pattern is used to deploy the EKS Cluster with ACK Controllers for building | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.22"` | no | -| [enable\_ack](#input\_enable\_ack) | Enable ACK Controller for EMR on EKS | `bool` | `true` | no | -| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"emr-eks-emrstudio"` | no | +| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.24"` | no | +| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"event-driven-pipeline-demo"` | no | | [region](#input\_region) | region | `string` | `"us-west-2"` | no | | [tags](#input\_tags) | Default tags | `map(string)` | `{}` | no | | [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR | `string` | `"10.1.0.0/16"` | no | @@ -73,6 +61,6 @@ This pattern is used to deploy the EKS Cluster with ACK Controllers for building | [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | | [emr\_on\_eks\_role\_arn](#output\_emr\_on\_eks\_role\_arn) | IAM execution role arn for EMR on EKS | | [emr\_on\_eks\_role\_id](#output\_emr\_on\_eks\_role\_id) | IAM execution role ID for EMR on EKS | -| [emr\_studio\_S3\_bucket\_id](#output\_emr\_studio\_S3\_bucket\_id) | EMR studio s3 bucket id | -| [emr\_studio\_service\_role\_arn](#output\_emr\_studio\_service\_role\_arn) | EMR studio service role ARN | +| [eventbridge\_role\_arn](#output\_eventbridge\_role\_arn) | IAM execution role arn for eventbridge | +| [stepfunctions\_role\_arn](#output\_stepfunctions\_role\_arn) | IAM execution role arn for step functions | diff --git a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml index c30bf5d..d46bde0 100644 --- a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml +++ b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml @@ -11,4 +11,3 @@ spec: info: eksInfo: namespace: emr-data-team-a - diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml index 5f5a586..ff25475 100644 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -46,4 +46,4 @@ spec: maximumRetryAttempts: 0 # no retries tags: - key: tag-01 - value: value-01 \ No newline at end of file + value: value-01 diff --git a/examples/event-driven-pipeline/ack-yamls/sfn.yaml b/examples/event-driven-pipeline/ack-yamls/sfn.yaml index 0adff86..4bc2ee2 100644 --- a/examples/event-driven-pipeline/ack-yamls/sfn.yaml +++ b/examples/event-driven-pipeline/ack-yamls/sfn.yaml @@ -9,4 +9,72 @@ spec: tags: - key: owner value: sfn-ack - definition: "{\"Comment\":\"Adescriptionofmystatemachine\",\"StartAt\":\"input-output-s3\",\"States\":{\"input-output-s3\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"s3://step-functions-test2/scripts/pyspark-taxi-trip.py\",\"EntryPointArguments\":[\"s3://step-functions-test2/input/\",\"s3://step-functions-test2/output/\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=10\"}},\"ConfigurationOverrides\":{\"ApplicationConfiguration\":[{\"Classification\":\"spark-defaults\",\"Properties\":{\"spark.driver.cores\":\"1\",\"spark.executor.cores\":\"1\",\"spark.driver.memory\":\"10g\",\"spark.executor.memory\":\"10g\",\"spark.kubernetes.driver.podTemplateFile\":\"s3://step-functions-test2/scripts/driver-pod-template.yaml\",\"spark.kubernetes.executor.podTemplateFile\":\"s3://step-functions-test2/scripts/executor-pod-template.yaml\",\"spark.local.dir\":\"/data1,/data2\"}}],\"MonitoringConfiguration\":{\"PersistentAppUI\":\"ENABLED\",\"CloudWatchMonitoringConfiguration\":{\"LogGroupName\":\"emr-on-eks\",\"LogStreamNamePrefix\":\"test1\"}}}},\"Next\":\"local-data\"},\"local-data\":{\"Type\":\"Task\",\"Resource\":\"arn:aws:states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"yapyjlu010v917k3jxmx76fq1\",\"ExecutionRoleArn\":\"arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a\",\"ReleaseLabel\":\"emr-6.7.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"local:///usr/lib/spark/examples/src/main/python/pi.py\",\"EntryPointArguments\":[\"60\"],\"SparkSubmitParameters\":\"--confspark.executor.instances=2--confspark.executor.memory=1G--confspark.executor.cores=1--confspark.driver.cores=1\"}}},\"End\":true}}}" + definition: | + { + "Comment": "A description of my state machine", + "StartAt": "input-output-s3", + "States": { + "input-output-s3": { + "Type": "Task", + "Resource": "arn:aws:states:::emr-containers:startJobRun.sync", + "Parameters": { + "VirtualClusterId": "yapyjlu010v917k3jxmx76fq1", + "ExecutionRoleArn": "arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a", + "ReleaseLabel": "emr-6.7.0-latest", + "JobDriver": { + "SparkSubmitJobDriver": { + "EntryPoint": "s3://step-functions-test2/scripts/pyspark-taxi-trip.py", + "EntryPointArguments": [ + "s3://step-functions-test2/input/", + "s3://step-functions-test2/output/" + ], + "SparkSubmitParameters": "--conf spark.executor.instances=10" + } + }, + "ConfigurationOverrides": { + "ApplicationConfiguration": [ + { + "Classification": "spark-defaults", + "Properties": { + "spark.driver.cores":"1", + "spark.executor.cores":"1", + "spark.driver.memory": "10g", + "spark.executor.memory": "10g", + "spark.kubernetes.driver.podTemplateFile":"s3://step-functions-test2/scripts/driver-pod-template.yaml", + "spark.kubernetes.executor.podTemplateFile":"s3://step-functions-test2/scripts/executor-pod-template.yaml", + "spark.local.dir" : "/data1,/data2" + } + } + ], + "MonitoringConfiguration": { + "PersistentAppUI":"ENABLED", + "CloudWatchMonitoringConfiguration": { + "LogGroupName":"emr-on-eks", + "LogStreamNamePrefix":"test1" + } + } + } + }, + "Next": "local-data" + }, + "local-data": { + "Type": "Task", + "Resource": "arn:aws:states:::emr-containers:startJobRun.sync", + "Parameters": { + "VirtualClusterId": "yapyjlu010v917k3jxmx76fq1", + "ExecutionRoleArn": "arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a", + "ReleaseLabel": "emr-6.7.0-latest", + "JobDriver": { + "SparkSubmitJobDriver": { + "EntryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py", + "EntryPointArguments": [ + "60" + ], + "SparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=1G --conf spark.executor.cores=1 --conf spark.driver.cores=1" + } + } + }, + "End": true + } + } + } diff --git a/examples/event-driven-pipeline/addons.tf b/examples/event-driven-pipeline/addons.tf index 1f7024c..abcd10f 100644 --- a/examples/event-driven-pipeline/addons.tf +++ b/examples/event-driven-pipeline/addons.tf @@ -14,21 +14,6 @@ module "eks_blueprints_kubernetes_addons" { enable_amazon_eks_kube_proxy = true enable_amazon_eks_aws_ebs_csi_driver = true - #--------------------------------------- - # Metrics Server - #--------------------------------------- - enable_metrics_server = true - metrics_server_helm_config = { - name = "metrics-server" - repository = "https://kubernetes-sigs.github.io/metrics-server/" # (Optional) Repository URL where to locate the requested chart. - chart = "metrics-server" - version = "3.8.2" - namespace = "kube-system" - timeout = "300" - values = [templatefile("${path.module}/helm-values/metrics-server-values.yaml", { - operating_system = "linux" - })] - } #--------------------------------------- # Cluster Autoscaler @@ -74,7 +59,7 @@ module "eks_blueprints_kubernetes_addons" { ################################################################################ module "eks_ack_addons" { - source = "github.com/aws-ia/terraform-aws-eks-ack-addons" + source = "../../" cluster_id = module.eks_blueprints.eks_cluster_id data_plane_wait_arn = module.eks_blueprints.managed_node_group_arn[0] # Wait for data plane to be ready @@ -82,10 +67,10 @@ module "eks_ack_addons" { ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password - enable_sfn = true + enable_sfn = true enable_s3 = true enable_emrcontainers = true enable_eb = true - + tags = local.tags } diff --git a/examples/event-driven-pipeline/data.tf b/examples/event-driven-pipeline/data.tf index 20d7a02..15b7f69 100644 --- a/examples/event-driven-pipeline/data.tf +++ b/examples/event-driven-pipeline/data.tf @@ -43,6 +43,3 @@ data "aws_iam_policy_document" "emr_on_eks" { ] } } - - - diff --git a/examples/event-driven-pipeline/main.tf b/examples/event-driven-pipeline/main.tf index 6d93ad9..edc0476 100644 --- a/examples/event-driven-pipeline/main.tf +++ b/examples/event-driven-pipeline/main.tf @@ -9,7 +9,6 @@ module "eks_blueprints" { vpc_id = module.vpc.vpc_id private_subnet_ids = module.vpc.private_subnets - create_node_security_group = false cluster_endpoint_private_access = true # if true, Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint cluster_endpoint_public_access = true # if true, Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. @@ -56,10 +55,69 @@ module "eks_blueprints" { instance_types = ["m5.large"] min_size = 3 subnet_ids = module.vpc.private_subnets + }, + + mng2 = { + node_group_name = "spark-node-grp" + subnet_ids = [module.vpc.private_subnets[0]] + instance_types = ["r5d.4xlarge"] + ami_type = "AL2_x86_64" + capacity_type = "SPOT" + + # Enable this option only when you are using NVMe disks + format_mount_nvme_disk = true # Mounts NVMe disks to /local1, /local2 etc. for multiple NVMe disks + + # RAID0 configuration is recommended for better performance when you use larger instances with multiple NVMe disks e.g., r5d.24xlarge + # Permissions for hadoop user runs the analytics job. user > hadoop:x:999:1000::/home/hadoop:/bin/bash + post_userdata = <<-EOT + #!/bin/bash + set -ex + mkdir local1 + mkdir local2 + /usr/bin/chown -hR +999:+1000 /local* + EOT + + disk_size = 100 + disk_type = "gp3" + + max_size = 9 # Managed node group soft limit is 450; request AWS for limit increase + min_size = 3 + desired_size = 3 + + create_launch_template = true + launch_template_os = "amazonlinux2eks" + + update_config = [{ + max_unavailable_percentage = 50 + }] + + additional_iam_policies = [] + k8s_taints = [] + + k8s_labels = { + Environment = "preprod" + Zone = "test" + WorkerType = "ON_DEMAND" + NodeGroupType = "spark" + } + + additional_tags = { + Name = "spark-node-grp" + subnet_type = "private" + "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" + "k8s.io/cluster-autoscaler/node-template/label/noderole" = "spark" + "k8s.io/cluster-autoscaler/node-template/label/disk" = "nvme" + "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" + "k8s.io/cluster-autoscaler/experiments" = "owned" + "k8s.io/cluster-autoscaler/enabled" = "true" + } } } + + #--------------------------------------- # ENABLE EMR ON EKS # 1. Creates namespace @@ -93,7 +151,7 @@ resource "aws_iam_policy" "emr_on_eks" { # IAM execution role for Step functions #--------------------------------------------------------------- resource "aws_iam_role" "sfn_execution_role" { - name = "${local.cluster_name}-efn-execution-role" + name = "${local.name}-sfn-execution-role" # Terraform's "jsonencode" function converts a # Terraform expression result to valid JSON syntax. @@ -107,11 +165,11 @@ resource "aws_iam_role" "sfn_execution_role" { Principal = { Service = "states.amazonaws.com" } - + }, ] }) - + inline_policy { name = "my_inline_policy" @@ -119,53 +177,53 @@ resource "aws_iam_role" "sfn_execution_role" { Version = "2012-10-17" Statement = [ { - "Effect": "Allow", - "Action": "emr-containers:StartJobRun", - "Resource": [ - "*" - ] + "Effect" : "Allow", + "Action" : "emr-containers:StartJobRun", + "Resource" : [ + "*" + ] }, { - "Effect": "Allow", - "Action": [ - "emr-containers:DescribeJobRun", - "emr-containers:CancelJobRun" - ], - "Resource": [ - "*" - ] + "Effect" : "Allow", + "Action" : [ + "emr-containers:DescribeJobRun", + "emr-containers:CancelJobRun" + ], + "Resource" : [ + "*" + ] }, { - "Effect": "Allow", - "Action": [ - "logs:CreateLogDelivery", - "logs:GetLogDelivery", - "logs:UpdateLogDelivery", - "logs:DeleteLogDelivery", - "logs:ListLogDeliveries", - "logs:PutResourcePolicy", - "logs:DescribeResourcePolicies", - "logs:DescribeLogGroups" - ], - "Resource": "*" + "Effect" : "Allow", + "Action" : [ + "logs:CreateLogDelivery", + "logs:GetLogDelivery", + "logs:UpdateLogDelivery", + "logs:DeleteLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:DescribeResourcePolicies", + "logs:DescribeLogGroups" + ], + "Resource" : "*" }, { - "Action": "iam:PassRole", - "Effect": "Allow", - "Resource": "*", - "Sid": "" + "Action" : "iam:PassRole", + "Effect" : "Allow", + "Resource" : "*", + "Sid" : "" }, { - "Effect": "Allow", - "Action": [ - "xray:PutTraceSegments", - "xray:PutTelemetryRecords", - "xray:GetSamplingRules", - "xray:GetSamplingTargets" - ], - "Resource": [ - "*" - ] + "Effect" : "Allow", + "Action" : [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + "xray:GetSamplingRules", + "xray:GetSamplingTargets" + ], + "Resource" : [ + "*" + ] } ] }) @@ -177,7 +235,7 @@ resource "aws_iam_role" "sfn_execution_role" { # IAM execution role for EventBridge #--------------------------------------------------------------- resource "aws_iam_role" "eb_execution_role" { - name = "${local.cluster_name}-eb-execution-role" + name = "${local.name}-eb-execution-role" # Terraform's "jsonencode" function converts a # Terraform expression result to valid JSON syntax. @@ -191,11 +249,11 @@ resource "aws_iam_role" "eb_execution_role" { Principal = { Service = "events.amazonaws.com" } - + }, ] }) - + inline_policy { name = "my_inline_policy" @@ -203,13 +261,13 @@ resource "aws_iam_role" "eb_execution_role" { Version = "2012-10-17" Statement = [ { - "Effect": "Allow", - "Action": [ - "states:StartExecution" - ], - "Resource": [ - "*" - ] + "Effect" : "Allow", + "Action" : [ + "states:StartExecution" + ], + "Resource" : [ + "*" + ] } ] }) diff --git a/examples/event-driven-pipeline/outputs.tf b/examples/event-driven-pipeline/outputs.tf index cefb113..3204b64 100644 --- a/examples/event-driven-pipeline/outputs.tf +++ b/examples/event-driven-pipeline/outputs.tf @@ -22,4 +22,4 @@ output "stepfunctions_role_arn" { output "eventbridge_role_arn" { description = "IAM execution role arn for eventbridge" value = aws_iam_role.eb_execution_role.arn -} \ No newline at end of file +} diff --git a/main.tf b/main.tf index 9c25470..e881d18 100644 --- a/main.tf +++ b/main.tf @@ -662,4 +662,4 @@ data "aws_iam_policy" "eb" { count = var.enable_eb ? 1 : 0 name = "AmazonEventBridgeFullAccess" -} \ No newline at end of file +} diff --git a/variables.tf b/variables.tf index 107c874..ae27c41 100644 --- a/variables.tf +++ b/variables.tf @@ -163,4 +163,4 @@ variable "eb_helm_config" { description = "ACK Event Bridge Helm Chart config" type = any default = {} -} \ No newline at end of file +} From 2653ec6f2747433817d46f736f4b33d848784e24 Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Thu, 9 Feb 2023 17:37:03 +0000 Subject: [PATCH 18/26] clean up --- .gitignore | 2 + README.md | 1 + .../ack-yamls/emr-virtualcluster.yaml | 4 +- .../ack-yamls/eventbridge.yaml | 10 ++-- .../event-driven-pipeline/ack-yamls/s3.yaml | 16 +----- .../event-driven-pipeline/ack-yamls/sfn.yaml | 49 +++++-------------- examples/event-driven-pipeline/main.tf | 43 ++++++++++++++-- .../pod-templates/driver-pod-template.yaml | 9 ++++ .../pod-templates/executor-pod-template.yaml | 9 ++++ .../spark-scripts-data/prepare-taxi-trip.sh | 32 ------------ .../spark-scripts-data/upload-inputdata.sh | 28 +++++++++++ .../upload-spark-scripts.sh | 9 ++++ main.tf | 27 +++++++++- 13 files changed, 142 insertions(+), 97 deletions(-) delete mode 100644 examples/event-driven-pipeline/spark-scripts-data/prepare-taxi-trip.sh create mode 100644 examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh create mode 100644 examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh diff --git a/.gitignore b/.gitignore index 627f068..b5748ae 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc + +**/examples/event-driven-pipeline/input/* diff --git a/README.md b/README.md index 795d38d..78ae987 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | Name | Type | |------|------| | [aws_iam_policy.emrcontainers](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.sfnpasspolicy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [time_sleep.dataplane](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster) | data source | diff --git a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml index d46bde0..7bbef31 100644 --- a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml +++ b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml @@ -6,8 +6,8 @@ metadata: spec: name: my-ack-vc containerProvider: - id: emr-eks-ack # your eks cluster name + id: event-driven-pipeline-demo # your eks cluster name type_: EKS info: eksInfo: - namespace: emr-data-team-a + namespace: emr-data-team-a # namespace binding with EMR virtual cluster diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml index ff25475..6d3ebaf 100644 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -15,7 +15,7 @@ kind: Rule metadata: name: eb-rule-ack annotations: - services.k8s.aws/region: us-east-1 + services.k8s.aws/region: us-west-2 spec: name: eb-rule-ack description: "ACK EventBridge Filter Rule to sfn using event bus reference" @@ -29,19 +29,19 @@ spec: "detail-type": ["Object Created"], "detail": { "bucket": { - "name": ["sparkjobbucket"] + "name": ["sparkjob-demo-bucket"] }, "object": { "key": [{ - "prefix": "input/" + "prefix": "scripts/" }] } } } targets: - - arn: arn:aws:states:us-east-1:349361870252:stateMachine:run-spark-job-ack # replace with your sfn arn + - arn: arn:aws:states:us-west-2:349361870252:stateMachine:run-spark-job-ack # replace with your sfn arn id: sfn-run-spark-job-target - roleARN: arn:aws:iam::349361870252:role/service-role/Amazon_EventBridge_Invoke_Step_Functions_1004390267 # replace your EventBridge_Invoke_Step_Functions role arn + roleARN: arn:aws:iam::349361870252:role/event-driven-pipeline-demo-eb-execution-role # replace your eventbridge_role_arn retryPolicy: maximumRetryAttempts: 0 # no retries tags: diff --git a/examples/event-driven-pipeline/ack-yamls/s3.yaml b/examples/event-driven-pipeline/ack-yamls/s3.yaml index 81851cb..4d78ca4 100644 --- a/examples/event-driven-pipeline/ack-yamls/s3.yaml +++ b/examples/event-driven-pipeline/ack-yamls/s3.yaml @@ -1,18 +1,6 @@ apiVersion: s3.services.k8s.aws/v1alpha1 kind: Bucket metadata: - name: sparkjobbucket + name: sparkjob-demo-bucket spec: - name: sparkjobbucket - notification: - EventBridgeConfiguration: - - events: - - string - filter: - key: - filterRules: - - name: string - value: string - id: string - topicARN: string + name: sparkjob-demo-bucket diff --git a/examples/event-driven-pipeline/ack-yamls/sfn.yaml b/examples/event-driven-pipeline/ack-yamls/sfn.yaml index 4bc2ee2..dc024bd 100644 --- a/examples/event-driven-pipeline/ack-yamls/sfn.yaml +++ b/examples/event-driven-pipeline/ack-yamls/sfn.yaml @@ -2,15 +2,14 @@ apiVersion: sfn.services.k8s.aws/v1alpha1 kind: StateMachine metadata: name: run-spark-job-ack - namespace: ack-demo spec: name: run-spark-job-ack - roleARN: "arn:aws:iam::349361870252:role/service-role/StepFunctions-run-emreks-job-role-624b324e" # need to pass. need permission to access emr virtual cluster. need PassRole + roleARN: "arn:aws:iam::349361870252:role/event-driven-pipeline-demo-sfn-execution-role" # replace with your stepfunctions_role_arn tags: - key: owner value: sfn-ack definition: | - { + { "Comment": "A description of my state machine", "StartAt": "input-output-s3", "States": { @@ -18,15 +17,15 @@ spec: "Type": "Task", "Resource": "arn:aws:states:::emr-containers:startJobRun.sync", "Parameters": { - "VirtualClusterId": "yapyjlu010v917k3jxmx76fq1", - "ExecutionRoleArn": "arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a", + "VirtualClusterId": "f0u3vt3y4q2r1ot11m7v809y6", + "ExecutionRoleArn": "arn:aws:iam::349361870252:role/event-driven-pipeline-demo-emr-eks-data-team-a", "ReleaseLabel": "emr-6.7.0-latest", "JobDriver": { "SparkSubmitJobDriver": { - "EntryPoint": "s3://step-functions-test2/scripts/pyspark-taxi-trip.py", + "EntryPoint": "s3://sparkjob-demo-bucket/scripts/pyspark-taxi-trip.py", "EntryPointArguments": [ - "s3://step-functions-test2/input/", - "s3://step-functions-test2/output/" + "s3://sparkjob-demo-bucket/input/", + "s3://sparkjob-demo-bucket/output/" ], "SparkSubmitParameters": "--conf spark.executor.instances=10" } @@ -40,41 +39,15 @@ spec: "spark.executor.cores":"1", "spark.driver.memory": "10g", "spark.executor.memory": "10g", - "spark.kubernetes.driver.podTemplateFile":"s3://step-functions-test2/scripts/driver-pod-template.yaml", - "spark.kubernetes.executor.podTemplateFile":"s3://step-functions-test2/scripts/executor-pod-template.yaml", + "spark.kubernetes.driver.podTemplateFile":"s3://sparkjob-demo-bucket/scripts/driver-pod-template.yaml", + "spark.kubernetes.executor.podTemplateFile":"s3://sparkjob-demo-bucket/scripts/executor-pod-template.yaml", "spark.local.dir" : "/data1,/data2" } } - ], - "MonitoringConfiguration": { - "PersistentAppUI":"ENABLED", - "CloudWatchMonitoringConfiguration": { - "LogGroupName":"emr-on-eks", - "LogStreamNamePrefix":"test1" - } - } - } - }, - "Next": "local-data" - }, - "local-data": { - "Type": "Task", - "Resource": "arn:aws:states:::emr-containers:startJobRun.sync", - "Parameters": { - "VirtualClusterId": "yapyjlu010v917k3jxmx76fq1", - "ExecutionRoleArn": "arn:aws:iam::349361870252:role/emr-eks-emrstudio-emr-eks-data-team-a", - "ReleaseLabel": "emr-6.7.0-latest", - "JobDriver": { - "SparkSubmitJobDriver": { - "EntryPoint": "local:///usr/lib/spark/examples/src/main/python/pi.py", - "EntryPointArguments": [ - "60" - ], - "SparkSubmitParameters": "--conf spark.executor.instances=2 --conf spark.executor.memory=1G --conf spark.executor.cores=1 --conf spark.driver.cores=1" - } + ] } }, - "End": true + "End": true } } } diff --git a/examples/event-driven-pipeline/main.tf b/examples/event-driven-pipeline/main.tf index edc0476..1eb7768 100644 --- a/examples/event-driven-pipeline/main.tf +++ b/examples/event-driven-pipeline/main.tf @@ -50,13 +50,46 @@ module "eks_blueprints" { } managed_node_groups = { - example = { - node_group_name = "example" - instance_types = ["m5.large"] - min_size = 3 + # Core node group for deploying all the admin add-ons + mng1 = { + node_group_name = "core-node-grp" subnet_ids = module.vpc.private_subnets - }, + instance_types = ["m5.xlarge"] + ami_type = "AL2_x86_64" + capacity_type = "ON_DEMAND" + + disk_size = 100 + disk_type = "gp3" + + max_size = 9 + min_size = 3 + desired_size = 3 + create_launch_template = true + launch_template_os = "amazonlinux2eks" + + update_config = [{ + max_unavailable_percentage = 50 + }] + + k8s_labels = { + Environment = "preprod" + Zone = "test" + WorkerType = "ON_DEMAND" + NodeGroupType = "core" + } + + additional_tags = { + Name = "core-node-grp" + subnet_type = "private" + "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" + "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" + "k8s.io/cluster-autoscaler/node-template/label/noderole" = "core" + "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" + "k8s.io/cluster-autoscaler/experiments" = "owned" + "k8s.io/cluster-autoscaler/enabled" = "true" + } + }, mng2 = { node_group_name = "spark-node-grp" subnet_ids = [module.vpc.private_subnets[0]] diff --git a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml index 4766305..510d18f 100644 --- a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml +++ b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml @@ -26,3 +26,12 @@ spec: - name: spark-local-dir-2 mountPath: /data2 readOnly: false + initContainers: + - name: volume-permissions + image: public.ecr.aws/y4g4v0z7/busybox + command: [ 'sh', '-c', 'chown 999 /local*' ] + volumeMounts: + - mountPath: "/local1" + name: "spark-local-dir-1" + - mountPath: "/local2" + name: "spark-local-dir-2" diff --git a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml index 3eeef2c..6f9be14 100644 --- a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml +++ b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml @@ -27,3 +27,12 @@ spec: - name: spark-local-dir-2 mountPath: /data2 readOnly: false + initContainers: + - name: volume-permissions + image: public.ecr.aws/y4g4v0z7/busybox + command: [ 'sh', '-c', 'chown 999 /local1' ] + volumeMounts: + - mountPath: "/local1" + name: "spark-local-dir-1" + - mountPath: "/local2" + name: "spark-local-dir-2" diff --git a/examples/event-driven-pipeline/spark-scripts-data/prepare-taxi-trip.sh b/examples/event-driven-pipeline/spark-scripts-data/prepare-taxi-trip.sh deleted file mode 100644 index 037a184..0000000 --- a/examples/event-driven-pipeline/spark-scripts-data/prepare-taxi-trip.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# This job copies Sample PySpark script and some test data to your S3 bucket which will enable you to run the following Spark Operator script - -# Prerequisites for running this shell script -# 1/ Enter your S3 bucket below () that "spark-team-a" service account IRSA can access. -# 2/ Enter region below (). Same as the EKS Cluster region -# 3/ Change according to your needs -# 4/ Ensure and is replaced in "ebs-storage-dynamic-pvc.yaml" file -# 5/ Execute the shell script which creates the input data in your S3 bucket -# 6/ Run `kubectl apply -f ebs-storage-dynamic-pvc.yaml` to trigger the Spark job -# 7/ Monitor the Spark job using "kubectl get pods -n spark-team-a -w" - -S3_BUCKET="" -REGION="" # Enter region -JOB_NAME="ebs-taxi-trip" # Change job name according to your needs - -INPUT_DATA_S3_PATH="s3://${S3_BUCKET}/${JOB_NAME}/input/" - -# Copy PySpark Script to S3 bucket -aws s3 cp pyspark-taxi-trip.py s3://${S3_BUCKET}/${JOB_NAME}/scripts/ --region ${REGION} - -# Copy Test Input data to S3 bucket -wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "input/yellow_tripdata_2022-0.parquet" - -# Making duplicate copies to increase the size of the data. -max=20 -for (( i=1; i <= $max; ++i )) -do - cp -rf "input/yellow_tripdata_2022-0.parquet" "input/yellow_tripdata_2022-${i}.parquet" -done - -aws s3 sync "input/" ${INPUT_DATA_S3_PATH} diff --git a/examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh b/examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh new file mode 100644 index 0000000..dd22306 --- /dev/null +++ b/examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# This job copies Sample data and pod templates to your S3 bucket which will enable you to run the Spark script + +# Prerequisites for running this shell script +# 1/ Enter your S3 bucket below () that "emr-data-team-a" service account IRSA can access. +# 2/ Enter region below (). Same as the EKS Cluster region +# 3/ Execute the shell script which creates the input data in your S3 bucket + +S3_BUCKET="sparkjob-demo-bucket" +REGION="us-west-2" # Enter region + + +INPUT_DATA_S3_PATH="s3://${S3_BUCKET}/input/" +aws s3 cp spark-scripts-data/pod-templates/driver-pod-template.yaml s3://${S3_BUCKET}/scripts/ --region ${REGION} +aws s3 cp spark-scripts-data/pod-templates/executor-pod-template.yaml s3://${S3_BUCKET}/scripts/ --region ${REGION} + +# Copy Test Input data to S3 bucket +mkdir input +wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "input/yellow_tripdata_2022-0.parquet" + +# Making duplicate copies to increase the size of the data. +max=20 +for (( i=1; i <= $max; ++i )) +do + cp -rf "input/yellow_tripdata_2022-0.parquet" "input/yellow_tripdata_2022-${i}.parquet" +done + +aws s3 sync "input/" ${INPUT_DATA_S3_PATH} diff --git a/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh b/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh new file mode 100644 index 0000000..4b01e6a --- /dev/null +++ b/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh @@ -0,0 +1,9 @@ + + +S3_BUCKET="sparkjob-demo-bucket" +REGION="us-west-2" # Enter region + + +# Copy PySpark Script to S3 bucket +aws s3 cp spark-scripts-data/pyspark-taxi-trip.py s3://${S3_BUCKET}/scripts/ --region ${REGION} + diff --git a/main.tf b/main.tf index e881d18..effd285 100644 --- a/main.tf +++ b/main.tf @@ -583,7 +583,7 @@ module "sfn" { create_kubernetes_service_account = true kubernetes_service_account = local.sfn_name - irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn] + irsa_iam_policies = [data.aws_iam_policy.sfn[0].arn, aws_iam_policy.sfnpasspolicy[0].arn] } addon_context = local.addon_context @@ -596,6 +596,31 @@ data "aws_iam_policy" "sfn" { } +resource "aws_iam_policy" "sfnpasspolicy" { + count = var.enable_sfn ? 1 : 0 + + name_prefix = format("%s-%s", local.sfn_name, "controller-iam-policies") + + path = "/" + description = "passrole policy" + + # Terraform's "jsonencode" function converts a + # Terraform expression result to valid JSON syntax. + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "iam:PassRole", + ] + Effect = "Allow" + Resource = "*" + }, + ] + }) +} + + ################################################################################ # Event Bridge ################################################################################ From 1ff97812e00d08382ed6b8fc85855684ddfc809b Mon Sep 17 00:00:00 2001 From: Gu Date: Thu, 9 Feb 2023 11:41:40 -0600 Subject: [PATCH 19/26] fix format --- .../spark-scripts-data/upload-spark-scripts.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh b/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh index 4b01e6a..f839464 100644 --- a/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh +++ b/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh @@ -6,4 +6,3 @@ REGION="us-west-2" # Enter region # Copy PySpark Script to S3 bucket aws s3 cp spark-scripts-data/pyspark-taxi-trip.py s3://${S3_BUCKET}/scripts/ --region ${REGION} - From df7b5cf7f3a844b44f4c1fd72d86474d2d00845f Mon Sep 17 00:00:00 2001 From: Gu Date: Thu, 9 Feb 2023 12:24:19 -0600 Subject: [PATCH 20/26] remove account info --- README.md | 4 +++- examples/complete/main.tf | 6 ++--- .../ack-yamls/eventbridge.yaml | 22 +++---------------- .../event-driven-pipeline/ack-yamls/sfn.yaml | 4 ++-- main.tf | 16 +++++++------- 5 files changed, 19 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 78ae987..c6992f2 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,9 @@ module "eks_ack_addons" { enable_rds = true enable_amp = true enable_emrcontainers = true - + enable_sfn = true + enable_eb = true + tags = { Environment = "dev" } diff --git a/examples/complete/main.tf b/examples/complete/main.tf index 11199fe..fac77b1 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -52,7 +52,7 @@ locals { ################################################################################ module "eks_blueprints" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.18.1" cluster_name = local.name cluster_version = "1.23" @@ -77,7 +77,7 @@ module "eks_blueprints" { ################################################################################ module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.18.1" eks_cluster_id = module.eks_blueprints.eks_cluster_id eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint @@ -199,7 +199,7 @@ resource "aws_iam_policy" "dynamodb_access" { } module "irsa" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa?ref=v4.18.1" create_kubernetes_namespace = true kubernetes_namespace = "ack-demo" diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml index 6d3ebaf..6458f01 100644 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -1,15 +1,3 @@ -# apiVersion: eventbridge.services.k8s.aws/v1alpha1 -# kind: EventBus -# metadata: -# name: eventbus-ack -# annotations: -# services.k8s.aws/region: us-east-1 -# spec: -# name: eventbus-ack -# tags: -# - key: tag-01 -# value: value-01 -# --- apiVersion: eventbridge.services.k8s.aws/v1alpha1 kind: Rule metadata: @@ -19,11 +7,7 @@ metadata: spec: name: eb-rule-ack description: "ACK EventBridge Filter Rule to sfn using event bus reference" - #use default one if don't set - # eventBusRef: - # from: - # name: eventbus-ack - eventPattern: | # replace s3 bucket name and dir + eventPattern: | { "source": ["aws.s3"], "detail-type": ["Object Created"], @@ -39,9 +23,9 @@ spec: } } targets: - - arn: arn:aws:states:us-west-2:349361870252:stateMachine:run-spark-job-ack # replace with your sfn arn + - arn: arn:aws:states:us-west-2:xxxxxxx:stateMachine:run-spark-job-ack # replace with your sfn arn id: sfn-run-spark-job-target - roleARN: arn:aws:iam::349361870252:role/event-driven-pipeline-demo-eb-execution-role # replace your eventbridge_role_arn + roleARN: arn:aws:iam::xxxxxxx:role/event-driven-pipeline-demo-eb-execution-role # replace your eventbridge_role_arn retryPolicy: maximumRetryAttempts: 0 # no retries tags: diff --git a/examples/event-driven-pipeline/ack-yamls/sfn.yaml b/examples/event-driven-pipeline/ack-yamls/sfn.yaml index dc024bd..0a8300b 100644 --- a/examples/event-driven-pipeline/ack-yamls/sfn.yaml +++ b/examples/event-driven-pipeline/ack-yamls/sfn.yaml @@ -4,7 +4,7 @@ metadata: name: run-spark-job-ack spec: name: run-spark-job-ack - roleARN: "arn:aws:iam::349361870252:role/event-driven-pipeline-demo-sfn-execution-role" # replace with your stepfunctions_role_arn + roleARN: "arn:aws:iam::xxxxxxxx:role/event-driven-pipeline-demo-sfn-execution-role" # replace with your stepfunctions_role_arn tags: - key: owner value: sfn-ack @@ -18,7 +18,7 @@ spec: "Resource": "arn:aws:states:::emr-containers:startJobRun.sync", "Parameters": { "VirtualClusterId": "f0u3vt3y4q2r1ot11m7v809y6", - "ExecutionRoleArn": "arn:aws:iam::349361870252:role/event-driven-pipeline-demo-emr-eks-data-team-a", + "ExecutionRoleArn": "arn:aws:iam::xxxxxx:role/event-driven-pipeline-demo-emr-eks-data-team-a", "ReleaseLabel": "emr-6.7.0-latest", "JobDriver": { "SparkSubmitJobDriver": { diff --git a/main.tf b/main.tf index effd285..d45faea 100644 --- a/main.tf +++ b/main.tf @@ -46,7 +46,7 @@ locals { } module "api_gatewayv2" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_api_gatewayv2 ? 1 : 0 @@ -122,7 +122,7 @@ locals { } module "dynamodb" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_dynamodb ? 1 : 0 @@ -189,7 +189,7 @@ locals { } module "s3" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_s3 ? 1 : 0 @@ -256,7 +256,7 @@ locals { } module "rds" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_rds ? 1 : 0 @@ -324,7 +324,7 @@ locals { } module "amp" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.12.2" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_amp ? 1 : 0 @@ -392,7 +392,7 @@ locals { } module "emrcontainers" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.0" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_emrcontainers ? 1 : 0 @@ -536,7 +536,7 @@ locals { } module "sfn" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.0" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_sfn ? 1 : 0 @@ -630,7 +630,7 @@ locals { } module "eventbridge" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.0" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" count = var.enable_eb ? 1 : 0 From 4299571cc4fe9f478403c3f0c2668f17a5244dce Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Thu, 9 Feb 2023 18:28:03 +0000 Subject: [PATCH 21/26] update version --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c6992f2..6174f29 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,14 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | Name | Source | Version | |------|--------|---------| -| [amp](#module\_amp) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | -| [api\_gatewayv2](#module\_api\_gatewayv2) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | -| [dynamodb](#module\_dynamodb) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | -| [emrcontainers](#module\_emrcontainers) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | -| [eventbridge](#module\_eventbridge) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | -| [rds](#module\_rds) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | -| [s3](#module\_s3) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.12.2 | -| [sfn](#module\_sfn) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.0 | +| [amp](#module\_amp) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [api\_gatewayv2](#module\_api\_gatewayv2) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [dynamodb](#module\_dynamodb) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [emrcontainers](#module\_emrcontainers) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [eventbridge](#module\_eventbridge) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [rds](#module\_rds) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [s3](#module\_s3) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [sfn](#module\_sfn) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | ## Resources From bde9a64bb6b4ae106a0630cf66ecd9820b1bbabb Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Wed, 15 Feb 2023 09:00:05 -0600 Subject: [PATCH 22/26] Update eventbridge.yaml --- examples/event-driven-pipeline/ack-yamls/eventbridge.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml index 6458f01..67e1ebf 100644 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -2,8 +2,6 @@ apiVersion: eventbridge.services.k8s.aws/v1alpha1 kind: Rule metadata: name: eb-rule-ack - annotations: - services.k8s.aws/region: us-west-2 spec: name: eb-rule-ack description: "ACK EventBridge Filter Rule to sfn using event bus reference" From ae94fc18103201c1d4b3e0e92647e25109f2367f Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Wed, 15 Feb 2023 09:33:35 -0600 Subject: [PATCH 23/26] Update eventbridge.yaml --- examples/event-driven-pipeline/ack-yamls/eventbridge.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml index 67e1ebf..1b23719 100644 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -27,5 +27,5 @@ spec: retryPolicy: maximumRetryAttempts: 0 # no retries tags: - - key: tag-01 - value: value-01 + - key: env + value: dev From da3d31bde1b18fdea00583a47c93c95706678afa Mon Sep 17 00:00:00 2001 From: Victor Gu Date: Wed, 15 Feb 2023 10:19:18 -0600 Subject: [PATCH 24/26] Update eventbridge.yaml --- examples/event-driven-pipeline/ack-yamls/eventbridge.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml index 1b23719..d7a2f9c 100644 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml @@ -27,5 +27,5 @@ spec: retryPolicy: maximumRetryAttempts: 0 # no retries tags: - - key: env - value: dev + - key: owner + value: eb-ack From 6d572a6dc2ab5edb13798d662c60b80477a51861 Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 15 Feb 2023 10:54:59 -0600 Subject: [PATCH 25/26] remove event pipeline code --- examples/event-driven-pipeline/README.md | 66 ---- .../ack-yamls/emr-virtualcluster.yaml | 13 - .../ack-yamls/eventbridge.yaml | 31 -- .../event-driven-pipeline/ack-yamls/s3.yaml | 6 - .../event-driven-pipeline/ack-yamls/sfn.yaml | 53 --- examples/event-driven-pipeline/addons.tf | 76 ----- examples/event-driven-pipeline/data.tf | 45 --- .../helm-values/aws-for-fluentbit-values.yaml | 70 ---- .../cluster-autoscaler-values.yaml | 52 --- .../helm-values/metrics-server-values.yaml | 56 ---- examples/event-driven-pipeline/locals.tf | 13 - examples/event-driven-pipeline/main.tf | 308 ------------------ examples/event-driven-pipeline/outputs.tf | 25 -- examples/event-driven-pipeline/providers.tf | 24 -- .../pod-templates/driver-pod-template.yaml | 37 --- .../pod-templates/executor-pod-template.yaml | 38 --- .../spark-scripts-data/pyspark-taxi-trip.py | 66 ---- .../spark-scripts-data/upload-inputdata.sh | 28 -- .../upload-spark-scripts.sh | 8 - examples/event-driven-pipeline/variables.tf | 29 -- examples/event-driven-pipeline/versions.tf | 18 - examples/event-driven-pipeline/vpc.tf | 112 ------- 22 files changed, 1174 deletions(-) delete mode 100644 examples/event-driven-pipeline/README.md delete mode 100644 examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml delete mode 100644 examples/event-driven-pipeline/ack-yamls/eventbridge.yaml delete mode 100644 examples/event-driven-pipeline/ack-yamls/s3.yaml delete mode 100644 examples/event-driven-pipeline/ack-yamls/sfn.yaml delete mode 100644 examples/event-driven-pipeline/addons.tf delete mode 100644 examples/event-driven-pipeline/data.tf delete mode 100644 examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml delete mode 100644 examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml delete mode 100644 examples/event-driven-pipeline/helm-values/metrics-server-values.yaml delete mode 100644 examples/event-driven-pipeline/locals.tf delete mode 100644 examples/event-driven-pipeline/main.tf delete mode 100644 examples/event-driven-pipeline/outputs.tf delete mode 100644 examples/event-driven-pipeline/providers.tf delete mode 100644 examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml delete mode 100644 examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml delete mode 100644 examples/event-driven-pipeline/spark-scripts-data/pyspark-taxi-trip.py delete mode 100644 examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh delete mode 100644 examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh delete mode 100644 examples/event-driven-pipeline/variables.tf delete mode 100644 examples/event-driven-pipeline/versions.tf delete mode 100644 examples/event-driven-pipeline/vpc.tf diff --git a/examples/event-driven-pipeline/README.md b/examples/event-driven-pipeline/README.md deleted file mode 100644 index d636763..0000000 --- a/examples/event-driven-pipeline/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Event driven data pipeline example -This pattern is used to deploy the EKS Cluster with ACK Controllers for building an event driven data pipeline. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.0.0 | -| [aws](#requirement\_aws) | >= 3.72 | -| [helm](#requirement\_helm) | >= 2.4.1 | -| [kubernetes](#requirement\_kubernetes) | >= 2.10 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 3.72 | -| [aws.ecr](#provider\_aws.ecr) | >= 3.72 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eks\_ack\_addons](#module\_eks\_ack\_addons) | ../../ | n/a | -| [eks\_blueprints](#module\_eks\_blueprints) | github.com/aws-ia/terraform-aws-eks-blueprints | v4.18.1 | -| [eks\_blueprints\_kubernetes\_addons](#module\_eks\_blueprints\_kubernetes\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons | v4.18.1 | -| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 | -| [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 3.0 | -| [vpc\_endpoints\_sg](#module\_vpc\_endpoints\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.emr_on_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.eb_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.sfn_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_ecrpublic_authorization_token.token](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ecrpublic_authorization_token) | data source | -| [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source | -| [aws_iam_policy_document.emr_on_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.24"` | no | -| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"event-driven-pipeline-demo"` | no | -| [region](#input\_region) | region | `string` | `"us-west-2"` | no | -| [tags](#input\_tags) | Default tags | `map(string)` | `{}` | no | -| [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR | `string` | `"10.1.0.0/16"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | -| [emr\_on\_eks\_role\_arn](#output\_emr\_on\_eks\_role\_arn) | IAM execution role arn for EMR on EKS | -| [emr\_on\_eks\_role\_id](#output\_emr\_on\_eks\_role\_id) | IAM execution role ID for EMR on EKS | -| [eventbridge\_role\_arn](#output\_eventbridge\_role\_arn) | IAM execution role arn for eventbridge | -| [stepfunctions\_role\_arn](#output\_stepfunctions\_role\_arn) | IAM execution role arn for step functions | - diff --git a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml b/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml deleted file mode 100644 index 7bbef31..0000000 --- a/examples/event-driven-pipeline/ack-yamls/emr-virtualcluster.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: emrcontainers.services.k8s.aws/v1alpha1 -kind: VirtualCluster -metadata: - name: my-ack-vc -spec: - name: my-ack-vc - containerProvider: - id: event-driven-pipeline-demo # your eks cluster name - type_: EKS - info: - eksInfo: - namespace: emr-data-team-a # namespace binding with EMR virtual cluster diff --git a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml b/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml deleted file mode 100644 index d7a2f9c..0000000 --- a/examples/event-driven-pipeline/ack-yamls/eventbridge.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: eventbridge.services.k8s.aws/v1alpha1 -kind: Rule -metadata: - name: eb-rule-ack -spec: - name: eb-rule-ack - description: "ACK EventBridge Filter Rule to sfn using event bus reference" - eventPattern: | - { - "source": ["aws.s3"], - "detail-type": ["Object Created"], - "detail": { - "bucket": { - "name": ["sparkjob-demo-bucket"] - }, - "object": { - "key": [{ - "prefix": "scripts/" - }] - } - } - } - targets: - - arn: arn:aws:states:us-west-2:xxxxxxx:stateMachine:run-spark-job-ack # replace with your sfn arn - id: sfn-run-spark-job-target - roleARN: arn:aws:iam::xxxxxxx:role/event-driven-pipeline-demo-eb-execution-role # replace your eventbridge_role_arn - retryPolicy: - maximumRetryAttempts: 0 # no retries - tags: - - key: owner - value: eb-ack diff --git a/examples/event-driven-pipeline/ack-yamls/s3.yaml b/examples/event-driven-pipeline/ack-yamls/s3.yaml deleted file mode 100644 index 4d78ca4..0000000 --- a/examples/event-driven-pipeline/ack-yamls/s3.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: s3.services.k8s.aws/v1alpha1 -kind: Bucket -metadata: - name: sparkjob-demo-bucket -spec: - name: sparkjob-demo-bucket diff --git a/examples/event-driven-pipeline/ack-yamls/sfn.yaml b/examples/event-driven-pipeline/ack-yamls/sfn.yaml deleted file mode 100644 index 0a8300b..0000000 --- a/examples/event-driven-pipeline/ack-yamls/sfn.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: sfn.services.k8s.aws/v1alpha1 -kind: StateMachine -metadata: - name: run-spark-job-ack -spec: - name: run-spark-job-ack - roleARN: "arn:aws:iam::xxxxxxxx:role/event-driven-pipeline-demo-sfn-execution-role" # replace with your stepfunctions_role_arn - tags: - - key: owner - value: sfn-ack - definition: | - { - "Comment": "A description of my state machine", - "StartAt": "input-output-s3", - "States": { - "input-output-s3": { - "Type": "Task", - "Resource": "arn:aws:states:::emr-containers:startJobRun.sync", - "Parameters": { - "VirtualClusterId": "f0u3vt3y4q2r1ot11m7v809y6", - "ExecutionRoleArn": "arn:aws:iam::xxxxxx:role/event-driven-pipeline-demo-emr-eks-data-team-a", - "ReleaseLabel": "emr-6.7.0-latest", - "JobDriver": { - "SparkSubmitJobDriver": { - "EntryPoint": "s3://sparkjob-demo-bucket/scripts/pyspark-taxi-trip.py", - "EntryPointArguments": [ - "s3://sparkjob-demo-bucket/input/", - "s3://sparkjob-demo-bucket/output/" - ], - "SparkSubmitParameters": "--conf spark.executor.instances=10" - } - }, - "ConfigurationOverrides": { - "ApplicationConfiguration": [ - { - "Classification": "spark-defaults", - "Properties": { - "spark.driver.cores":"1", - "spark.executor.cores":"1", - "spark.driver.memory": "10g", - "spark.executor.memory": "10g", - "spark.kubernetes.driver.podTemplateFile":"s3://sparkjob-demo-bucket/scripts/driver-pod-template.yaml", - "spark.kubernetes.executor.podTemplateFile":"s3://sparkjob-demo-bucket/scripts/executor-pod-template.yaml", - "spark.local.dir" : "/data1,/data2" - } - } - ] - } - }, - "End": true - } - } - } diff --git a/examples/event-driven-pipeline/addons.tf b/examples/event-driven-pipeline/addons.tf deleted file mode 100644 index abcd10f..0000000 --- a/examples/event-driven-pipeline/addons.tf +++ /dev/null @@ -1,76 +0,0 @@ -module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.18.1" - - eks_cluster_id = module.eks_blueprints.eks_cluster_id - eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint - eks_oidc_provider = module.eks_blueprints.oidc_provider - eks_cluster_version = module.eks_blueprints.eks_cluster_version - - #--------------------------------------- - # Amazon EKS Managed Add-ons - #--------------------------------------- - enable_amazon_eks_vpc_cni = true - enable_amazon_eks_coredns = true - enable_amazon_eks_kube_proxy = true - enable_amazon_eks_aws_ebs_csi_driver = true - - - #--------------------------------------- - # Cluster Autoscaler - #--------------------------------------- - enable_cluster_autoscaler = true - cluster_autoscaler_helm_config = { - name = "cluster-autoscaler" - repository = "https://kubernetes.github.io/autoscaler" # (Optional) Repository URL where to locate the requested chart. - chart = "cluster-autoscaler" - version = "9.15.0" - namespace = "kube-system" - timeout = "300" - values = [templatefile("${path.module}/helm-values/cluster-autoscaler-values.yaml", { - aws_region = var.region, - eks_cluster_id = local.name, - operating_system = "linux" - })] - } - - #--------------------------------------- - # AWS for FluentBit - DaemonSet - #--------------------------------------- - enable_aws_for_fluentbit = true - aws_for_fluentbit_helm_config = { - name = "aws-for-fluent-bit" - chart = "aws-for-fluent-bit" - repository = "https://aws.github.io/eks-charts" - version = "0.1.21" - namespace = "aws-for-fluent-bit" - aws_for_fluent_bit_cw_log_group = "/${var.name}/worker-fluentbit-logs" # Optional - aws_for_fluentbit_cwlog_retention_in_days = 90 - values = [templatefile("${path.module}/helm-values/aws-for-fluentbit-values.yaml", { - region = var.region, - aws_for_fluent_bit_cw_log = "/${var.name}/worker-fluentbit-logs" - })] - } - - tags = local.tags -} - -################################################################################ -# ACK Addons -################################################################################ -module "eks_ack_addons" { - - source = "../../" - - cluster_id = module.eks_blueprints.eks_cluster_id - data_plane_wait_arn = module.eks_blueprints.managed_node_group_arn[0] # Wait for data plane to be ready - - ecrpublic_username = data.aws_ecrpublic_authorization_token.token.user_name - ecrpublic_token = data.aws_ecrpublic_authorization_token.token.password - - enable_sfn = true - enable_s3 = true - enable_emrcontainers = true - enable_eb = true - - tags = local.tags -} diff --git a/examples/event-driven-pipeline/data.tf b/examples/event-driven-pipeline/data.tf deleted file mode 100644 index 15b7f69..0000000 --- a/examples/event-driven-pipeline/data.tf +++ /dev/null @@ -1,45 +0,0 @@ -data "aws_eks_cluster_auth" "this" { - name = module.eks_blueprints.eks_cluster_id -} - -data "aws_ecrpublic_authorization_token" "token" { - provider = aws.ecr -} - -data "aws_availability_zones" "available" {} - -data "aws_region" "current" {} - -data "aws_caller_identity" "current" {} - -data "aws_partition" "current" {} - -data "aws_iam_policy_document" "emr_on_eks" { - statement { - sid = "" - effect = "Allow" - resources = ["arn:${data.aws_partition.current.partition}:s3:::*"] - - actions = [ - "s3:DeleteObject", - "s3:DeleteObjectVersion", - "s3:GetObject", - "s3:ListBucket", - "s3:PutObject", - ] - } - - statement { - sid = "" - effect = "Allow" - resources = ["arn:${data.aws_partition.current.partition}:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:*"] - - actions = [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogGroups", - "logs:DescribeLogStreams", - "logs:PutLogEvents", - ] - } -} diff --git a/examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml b/examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml deleted file mode 100644 index 697b992..0000000 --- a/examples/event-driven-pipeline/helm-values/aws-for-fluentbit-values.yaml +++ /dev/null @@ -1,70 +0,0 @@ -global: - -#hostNetwork and dnsPolicy are critical for enabling large clusters to avoid making calls to API server -# see this link https://docs.fluentbit.io/manual/pipeline/filters/kubernetes#optional-feature-using-kubelet-to-get-metadata -hostNetwork: true -dnsPolicy: ClusterFirstWithHostNet - -# NOTE: extraFilters config for using Kubelet to get the Metadata instead of talking to API server for large clusters -filter: - name: "kubernetes" - match: "kube.*" - kubeURL: "https://kubernetes.default.svc.cluster.local:443" - mergeLog: "On" - mergeLogKey: "log_processed" - keepLog: "On" - k8sLoggingParser: "On" - k8sLoggingExclude: "Off" - bufferSize: "0" - extraFilters: | - Kube_Tag_Prefix application.var.log.containers. - Labels Off - Annotations Off - Use_Kubelet true - Kubelet_Port 10250 - Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token - -cloudWatch: - enabled: true - match: "*" - region: ${region} - logGroupName: ${aws_for_fluent_bit_cw_log} - logStreamName: - logStreamPrefix: "fluentbit-" - logKey: - logFormat: - roleArn: - autoCreateGroup: false - endpoint: - credentialsEndpoint: {} - -firehose: - enabled: false - -kinesis: - enabled: false - -elasticsearch: - enabled: false - -serviceAccount: - create: true - -# Resource config for large clusters -resources: - limits: - cpu: 1000m - memory: 1500Mi - requests: - cpu: 500m - memory: 500Mi - -## Assign a PriorityClassName to pods if set -priorityClassName: system-node-critical - -updateStrategy: - type: RollingUpdate - -nodeSelector: - kubernetes.io/os: linux diff --git a/examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml b/examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml deleted file mode 100644 index 680aff7..0000000 --- a/examples/event-driven-pipeline/helm-values/cluster-autoscaler-values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -autoDiscovery: - clusterName: ${eks_cluster_id} - tags: - - k8s.io/cluster-autoscaler/enabled - - k8s.io/cluster-autoscaler/{{ .Values.autoDiscovery.clusterName }} - - roles: - - worker - -awsRegion: ${aws_region} - -cloudProvider: aws - -extraArgs: - logtostderr: true - stderrthreshold: info - v: 4 - aws-use-static-instance-list: true - -nodeSelector: - NodeGroupType: core - kubernetes.io/os: ${operating_system} - -podDisruptionBudget: - maxUnavailable: 1 - -priorityClassName: "system-cluster-critical" - -rbac: - create: true - pspEnabled: false - serviceAccount: - create: false - -# replicaCount -- Desired number of pods -replicaCount: 1 - -# Best practice to update the resource requests and limits for each add-on -resources: - limits: - cpu: 1000m - memory: 1G - requests: - cpu: 200m - memory: 512Mi - -# Best practice to updateStrategy for each add-on -updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 diff --git a/examples/event-driven-pipeline/helm-values/metrics-server-values.yaml b/examples/event-driven-pipeline/helm-values/metrics-server-values.yaml deleted file mode 100644 index 65f071d..0000000 --- a/examples/event-driven-pipeline/helm-values/metrics-server-values.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# HA config for metrics-server -image: - repository: k8s.gcr.io/metrics-server/metrics-server - pullPolicy: IfNotPresent - -serviceAccount: - create: true - name: metrics-server - -rbac: - create: true - pspEnabled: false - -apiService: - create: true - -podLabels: - k8s-app: metrics-server - -# HA enabled by enabling replicas to 2, updateStrategy and podDisruptionBudget to true -replicas: 2 - -updateStrategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - -podDisruptionBudget: - enabled: true - minAvailable: 1 - -defaultArgs: - - --cert-dir=/tmp - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - -resources: - requests: - cpu: 200m - memory: 512Mi - -nodeSelector: - NodeGroupType: core - kubernetes.io/os: ${operating_system} - -affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - k8s-app: metrics-server - namespaces: - - kube-system - topologyKey: kubernetes.io/hostname diff --git a/examples/event-driven-pipeline/locals.tf b/examples/event-driven-pipeline/locals.tf deleted file mode 100644 index 42cfeb4..0000000 --- a/examples/event-driven-pipeline/locals.tf +++ /dev/null @@ -1,13 +0,0 @@ -locals { - name = var.name - region = var.region - - vpc_cidr = var.vpc_cidr - azs = slice(data.aws_availability_zones.available.names, 0, 3) - vpc_endpoints = ["autoscaling", "ecr.api", "ecr.dkr", "ec2", "ec2messages", "elasticloadbalancing", "sts", "kms", "logs", "ssm", "ssmmessages"] - - tags = merge(var.tags, { - Blueprint = local.name - GithubRepo = "aws-ia/terraform-aws-eks-ack-addons" - }) -} diff --git a/examples/event-driven-pipeline/main.tf b/examples/event-driven-pipeline/main.tf deleted file mode 100644 index 1eb7768..0000000 --- a/examples/event-driven-pipeline/main.tf +++ /dev/null @@ -1,308 +0,0 @@ -#--------------------------------------------------------------- -# EKS Blueprints -#--------------------------------------------------------------- -module "eks_blueprints" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.18.1" - cluster_name = local.name - cluster_version = var.eks_cluster_version - - vpc_id = module.vpc.vpc_id - private_subnet_ids = module.vpc.private_subnets - - - cluster_endpoint_private_access = true # if true, Kubernetes API requests within your cluster's VPC (such as node to control plane communication) use the private VPC endpoint - cluster_endpoint_public_access = true # if true, Your cluster API server is accessible from the internet. You can, optionally, limit the CIDR blocks that can access the public endpoint. - - #--------------------------------------- - # Note: This can further restricted to specific required for each Add-on and your application - #--------------------------------------- - node_security_group_additional_rules = { - # Extend node-to-node security group rules. Recommended and required for the Add-ons - ingress_self_all = { - description = "Node to node all ports/protocols" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - self = true - } - # Recommended outbound traffic for Node groups - egress_all = { - description = "Node all egress" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "egress" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } - # Allows Control Plane Nodes to talk to Worker nodes on all ports. Added this to simplify the example and further avoid issues with Add-ons communication with Control plane. - # This can be restricted further to specific port based on the requirement for each Add-on e.g., metrics-server 4443, analytics-operator 8080, karpenter 8443 etc. - # Change this according to your security requirements if needed - ingress_cluster_to_node_all_traffic = { - description = "Cluster API to Nodegroup all traffic" - protocol = "-1" - from_port = 0 - to_port = 0 - type = "ingress" - source_cluster_security_group = true - } - } - - managed_node_groups = { - # Core node group for deploying all the admin add-ons - mng1 = { - node_group_name = "core-node-grp" - subnet_ids = module.vpc.private_subnets - - instance_types = ["m5.xlarge"] - ami_type = "AL2_x86_64" - capacity_type = "ON_DEMAND" - - disk_size = 100 - disk_type = "gp3" - - max_size = 9 - min_size = 3 - desired_size = 3 - create_launch_template = true - launch_template_os = "amazonlinux2eks" - - update_config = [{ - max_unavailable_percentage = 50 - }] - - k8s_labels = { - Environment = "preprod" - Zone = "test" - WorkerType = "ON_DEMAND" - NodeGroupType = "core" - } - - additional_tags = { - Name = "core-node-grp" - subnet_type = "private" - "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" - "k8s.io/cluster-autoscaler/node-template/label/noderole" = "core" - "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" - "k8s.io/cluster-autoscaler/experiments" = "owned" - "k8s.io/cluster-autoscaler/enabled" = "true" - } - }, - mng2 = { - node_group_name = "spark-node-grp" - subnet_ids = [module.vpc.private_subnets[0]] - instance_types = ["r5d.4xlarge"] - ami_type = "AL2_x86_64" - capacity_type = "SPOT" - - # Enable this option only when you are using NVMe disks - format_mount_nvme_disk = true # Mounts NVMe disks to /local1, /local2 etc. for multiple NVMe disks - - # RAID0 configuration is recommended for better performance when you use larger instances with multiple NVMe disks e.g., r5d.24xlarge - # Permissions for hadoop user runs the analytics job. user > hadoop:x:999:1000::/home/hadoop:/bin/bash - post_userdata = <<-EOT - #!/bin/bash - set -ex - mkdir local1 - mkdir local2 - /usr/bin/chown -hR +999:+1000 /local* - EOT - - disk_size = 100 - disk_type = "gp3" - - max_size = 9 # Managed node group soft limit is 450; request AWS for limit increase - min_size = 3 - desired_size = 3 - - create_launch_template = true - launch_template_os = "amazonlinux2eks" - - update_config = [{ - max_unavailable_percentage = 50 - }] - - additional_iam_policies = [] - k8s_taints = [] - - k8s_labels = { - Environment = "preprod" - Zone = "test" - WorkerType = "ON_DEMAND" - NodeGroupType = "spark" - } - - additional_tags = { - Name = "spark-node-grp" - subnet_type = "private" - "k8s.io/cluster-autoscaler/node-template/label/arch" = "x86" - "k8s.io/cluster-autoscaler/node-template/label/kubernetes.io/os" = "linux" - "k8s.io/cluster-autoscaler/node-template/label/noderole" = "spark" - "k8s.io/cluster-autoscaler/node-template/label/disk" = "nvme" - "k8s.io/cluster-autoscaler/node-template/label/node-lifecycle" = "on-demand" - "k8s.io/cluster-autoscaler/experiments" = "owned" - "k8s.io/cluster-autoscaler/enabled" = "true" - } - } - } - - - - - #--------------------------------------- - # ENABLE EMR ON EKS - # 1. Creates namespace - # 2. k8s role and role binding(emr-containers user) for the above namespace - # 3. IAM role for the team execution role - # 4. Update AWS_AUTH config map with emr-containers user and AWSServiceRoleForAmazonEMRContainers role - # 5. Create a trust relationship between the job execution role and the identity of the EMR managed service account - #--------------------------------------- - enable_emr_on_eks = true - emr_on_eks_teams = { - emr-data-team-a = { - namespace = "emr-data-team-a" - job_execution_role = "emr-eks-data-team-a" - additional_iam_policies = [aws_iam_policy.emr_on_eks.arn] - } - } - tags = local.tags -} - -#--------------------------------------------------------------- -# Example IAM policies for EMR job execution -#--------------------------------------------------------------- -resource "aws_iam_policy" "emr_on_eks" { - name = format("%s-%s", local.name, "emr-job-iam-policies") - description = "IAM policy for EMR on EKS Job execution" - path = "/" - policy = data.aws_iam_policy_document.emr_on_eks.json -} - -#--------------------------------------------------------------- -# IAM execution role for Step functions -#--------------------------------------------------------------- -resource "aws_iam_role" "sfn_execution_role" { - name = "${local.name}-sfn-execution-role" - - # Terraform's "jsonencode" function converts a - # Terraform expression result to valid JSON syntax. - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = "sts:AssumeRole" - Effect = "Allow" - Sid = "" - Principal = { - Service = "states.amazonaws.com" - } - - }, - ] - }) - - inline_policy { - name = "my_inline_policy" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - "Effect" : "Allow", - "Action" : "emr-containers:StartJobRun", - "Resource" : [ - "*" - ] - }, - { - "Effect" : "Allow", - "Action" : [ - "emr-containers:DescribeJobRun", - "emr-containers:CancelJobRun" - ], - "Resource" : [ - "*" - ] - }, - { - "Effect" : "Allow", - "Action" : [ - "logs:CreateLogDelivery", - "logs:GetLogDelivery", - "logs:UpdateLogDelivery", - "logs:DeleteLogDelivery", - "logs:ListLogDeliveries", - "logs:PutResourcePolicy", - "logs:DescribeResourcePolicies", - "logs:DescribeLogGroups" - ], - "Resource" : "*" - }, - { - "Action" : "iam:PassRole", - "Effect" : "Allow", - "Resource" : "*", - "Sid" : "" - }, - { - "Effect" : "Allow", - "Action" : [ - "xray:PutTraceSegments", - "xray:PutTelemetryRecords", - "xray:GetSamplingRules", - "xray:GetSamplingTargets" - ], - "Resource" : [ - "*" - ] - } - ] - }) - } -} - - -#--------------------------------------------------------------- -# IAM execution role for EventBridge -#--------------------------------------------------------------- -resource "aws_iam_role" "eb_execution_role" { - name = "${local.name}-eb-execution-role" - - # Terraform's "jsonencode" function converts a - # Terraform expression result to valid JSON syntax. - assume_role_policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = "sts:AssumeRole" - Effect = "Allow" - Sid = "" - Principal = { - Service = "events.amazonaws.com" - } - - }, - ] - }) - - inline_policy { - name = "my_inline_policy" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - "Effect" : "Allow", - "Action" : [ - "states:StartExecution" - ], - "Resource" : [ - "*" - ] - } - ] - }) - } -} diff --git a/examples/event-driven-pipeline/outputs.tf b/examples/event-driven-pipeline/outputs.tf deleted file mode 100644 index 3204b64..0000000 --- a/examples/event-driven-pipeline/outputs.tf +++ /dev/null @@ -1,25 +0,0 @@ -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = module.eks_blueprints.configure_kubectl -} - - -output "emr_on_eks_role_id" { - description = "IAM execution role ID for EMR on EKS" - value = module.eks_blueprints.emr_on_eks_role_id -} - -output "emr_on_eks_role_arn" { - description = "IAM execution role arn for EMR on EKS" - value = module.eks_blueprints.emr_on_eks_role_arn -} - -output "stepfunctions_role_arn" { - description = "IAM execution role arn for step functions" - value = aws_iam_role.sfn_execution_role.arn -} - -output "eventbridge_role_arn" { - description = "IAM execution role arn for eventbridge" - value = aws_iam_role.eb_execution_role.arn -} diff --git a/examples/event-driven-pipeline/providers.tf b/examples/event-driven-pipeline/providers.tf deleted file mode 100644 index d56f0b9..0000000 --- a/examples/event-driven-pipeline/providers.tf +++ /dev/null @@ -1,24 +0,0 @@ -provider "aws" { - region = local.region -} - -# ECR always authenticates with `us-east-1` region -# Docs -> https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html -provider "aws" { - alias = "ecr" - region = "us-east-1" -} - -provider "kubernetes" { - host = module.eks_blueprints.eks_cluster_endpoint - cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.this.token -} - -provider "helm" { - kubernetes { - host = module.eks_blueprints.eks_cluster_endpoint - cluster_ca_certificate = base64decode(module.eks_blueprints.eks_cluster_certificate_authority_data) - token = data.aws_eks_cluster_auth.this.token - } -} diff --git a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml deleted file mode 100644 index 510d18f..0000000 --- a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/driver-pod-template.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: ny-taxi-driver - namespace: emr-data-team-a -spec: - volumes: - - name: spark-local-dir-1 - hostPath: - path: /local1 - type: Directory - - name: spark-local-dir-2 - hostPath: - path: /local2 - type: Directory - - nodeSelector: - "NodeGroupType": "spark" - - containers: - - name: spark-kubernetes-driver # Don't change this name. EMR on EKS looking for this name - volumeMounts: - - name: spark-local-dir-1 - mountPath: /data1 - readOnly: false - - name: spark-local-dir-2 - mountPath: /data2 - readOnly: false - initContainers: - - name: volume-permissions - image: public.ecr.aws/y4g4v0z7/busybox - command: [ 'sh', '-c', 'chown 999 /local*' ] - volumeMounts: - - mountPath: "/local1" - name: "spark-local-dir-1" - - mountPath: "/local2" - name: "spark-local-dir-2" diff --git a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml b/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml deleted file mode 100644 index 6f9be14..0000000 --- a/examples/event-driven-pipeline/spark-scripts-data/pod-templates/executor-pod-template.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: ny-taxi-exec - namespace: emr-data-team-a - -spec: - volumes: - - name: spark-local-dir-1 - hostPath: - path: /local1 - type: Directory #cannot create here. needs to be created when provision node for proper permission - - name: spark-local-dir-2 - hostPath: - path: /local2 - type: Directory - - nodeSelector: - "NodeGroupType": "spark" - - containers: - - name: spark-kubernetes-executor # Don't change this name. EMR on EKS looking for this name - volumeMounts: - - name: spark-local-dir-1 - mountPath: /data1 - readOnly: false - - name: spark-local-dir-2 - mountPath: /data2 - readOnly: false - initContainers: - - name: volume-permissions - image: public.ecr.aws/y4g4v0z7/busybox - command: [ 'sh', '-c', 'chown 999 /local1' ] - volumeMounts: - - mountPath: "/local1" - name: "spark-local-dir-1" - - mountPath: "/local2" - name: "spark-local-dir-2" diff --git a/examples/event-driven-pipeline/spark-scripts-data/pyspark-taxi-trip.py b/examples/event-driven-pipeline/spark-scripts-data/pyspark-taxi-trip.py deleted file mode 100644 index 4a0cf0e..0000000 --- a/examples/event-driven-pipeline/spark-scripts-data/pyspark-taxi-trip.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -import sys -from datetime import datetime - -from pyspark.sql import SparkSession -from pyspark.sql.functions import * -from pyspark.sql import functions as f - -# Logging configuration -formatter = logging.Formatter('[%(asctime)s] %(levelname)s @ line %(lineno)d: %(message)s') -handler = logging.StreamHandler(sys.stdout) -handler.setLevel(logging.INFO) -handler.setFormatter(formatter) -logger = logging.getLogger() -logger.setLevel(logging.INFO) -logger.addHandler(handler) - -dt_string = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") -AppName = "NewYorkTaxiData" - - -def main(args): - - raw_input_folder = args[1] - transform_output_folder = args[2] - - # Create Spark Session - spark = SparkSession \ - .builder \ - .appName(AppName + "_" + str(dt_string)) \ - .getOrCreate() - - spark.sparkContext.setLogLevel("INFO") - logger.info("Starting spark application") - - logger.info("Reading Parquet file from S3") - ny_taxi_df = spark.read.parquet(raw_input_folder) - - # Add additional columns to the DF - final_ny_taxi_df = ny_taxi_df.withColumn("current_date", f.lit(datetime.now())) - - logger.info("NewYork Taxi data schema preview") - final_ny_taxi_df.printSchema() - - logger.info("Previewing New York Taxi data sample") - final_ny_taxi_df.show(20, truncate=False) - - logger.info("Total number of records: " + str(final_ny_taxi_df.count())) - - logger.info("Write New York Taxi data to S3 transform table") - final_ny_taxi_df.repartition(2).write.mode("overwrite").parquet(transform_output_folder) - - logger.info("Ending spark application") - # end spark code - spark.stop() - - return None - - -if __name__ == "__main__": - print(len(sys.argv)) - if len(sys.argv) != 3: - print("Usage: spark-etl [input-folder] [output-folder]") - sys.exit(0) - - main(sys.argv) diff --git a/examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh b/examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh deleted file mode 100644 index dd22306..0000000 --- a/examples/event-driven-pipeline/spark-scripts-data/upload-inputdata.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# This job copies Sample data and pod templates to your S3 bucket which will enable you to run the Spark script - -# Prerequisites for running this shell script -# 1/ Enter your S3 bucket below () that "emr-data-team-a" service account IRSA can access. -# 2/ Enter region below (). Same as the EKS Cluster region -# 3/ Execute the shell script which creates the input data in your S3 bucket - -S3_BUCKET="sparkjob-demo-bucket" -REGION="us-west-2" # Enter region - - -INPUT_DATA_S3_PATH="s3://${S3_BUCKET}/input/" -aws s3 cp spark-scripts-data/pod-templates/driver-pod-template.yaml s3://${S3_BUCKET}/scripts/ --region ${REGION} -aws s3 cp spark-scripts-data/pod-templates/executor-pod-template.yaml s3://${S3_BUCKET}/scripts/ --region ${REGION} - -# Copy Test Input data to S3 bucket -mkdir input -wget https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2022-01.parquet -O "input/yellow_tripdata_2022-0.parquet" - -# Making duplicate copies to increase the size of the data. -max=20 -for (( i=1; i <= $max; ++i )) -do - cp -rf "input/yellow_tripdata_2022-0.parquet" "input/yellow_tripdata_2022-${i}.parquet" -done - -aws s3 sync "input/" ${INPUT_DATA_S3_PATH} diff --git a/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh b/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh deleted file mode 100644 index f839464..0000000 --- a/examples/event-driven-pipeline/spark-scripts-data/upload-spark-scripts.sh +++ /dev/null @@ -1,8 +0,0 @@ - - -S3_BUCKET="sparkjob-demo-bucket" -REGION="us-west-2" # Enter region - - -# Copy PySpark Script to S3 bucket -aws s3 cp spark-scripts-data/pyspark-taxi-trip.py s3://${S3_BUCKET}/scripts/ --region ${REGION} diff --git a/examples/event-driven-pipeline/variables.tf b/examples/event-driven-pipeline/variables.tf deleted file mode 100644 index f4a779e..0000000 --- a/examples/event-driven-pipeline/variables.tf +++ /dev/null @@ -1,29 +0,0 @@ -variable "name" { - description = "Name of the VPC and EKS Cluster" - default = "event-driven-pipeline-demo" - type = string -} - -variable "region" { - description = "region" - type = string - default = "us-west-2" -} - -variable "eks_cluster_version" { - description = "EKS Cluster version" - default = "1.24" # EMR studio managed endpoint only supports up to 1.22 for now - type = string -} - -variable "tags" { - description = "Default tags" - default = {} - type = map(string) -} - -variable "vpc_cidr" { - description = "VPC CIDR" - default = "10.1.0.0/16" - type = string -} diff --git a/examples/event-driven-pipeline/versions.tf b/examples/event-driven-pipeline/versions.tf deleted file mode 100644 index 9ac1742..0000000 --- a/examples/event-driven-pipeline/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 3.72" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = ">= 2.10" - } - helm = { - source = "hashicorp/helm" - version = ">= 2.4.1" - } - } -} diff --git a/examples/event-driven-pipeline/vpc.tf b/examples/event-driven-pipeline/vpc.tf deleted file mode 100644 index 447b668..0000000 --- a/examples/event-driven-pipeline/vpc.tf +++ /dev/null @@ -1,112 +0,0 @@ -#--------------------------------------------------------------- -# VPC and Subnets -#--------------------------------------------------------------- -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 3.0" - - name = local.name - cidr = local.vpc_cidr - - azs = local.azs - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 10)] - - enable_nat_gateway = true - single_nat_gateway = true - enable_dns_hostnames = true - - # Manage so we can name - manage_default_network_acl = true - default_network_acl_tags = { Name = "${local.name}-default" } - manage_default_route_table = true - default_route_table_tags = { Name = "${local.name}-default" } - manage_default_security_group = true - default_security_group_tags = { Name = "${local.name}-default" } - - public_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/cluster/${local.name}" = "shared" - "kubernetes.io/role/internal-elb" = 1 - } - - default_security_group_name = "${local.name}-endpoint-secgrp" - - default_security_group_ingress = [ - { - protocol = -1 - from_port = 0 - to_port = 0 - cidr_blocks = local.vpc_cidr - }] - default_security_group_egress = [ - { - from_port = 0 - to_port = 0 - protocol = -1 - cidr_blocks = "0.0.0.0/0" - }] - - tags = local.tags -} - -module "vpc_endpoints_sg" { - source = "terraform-aws-modules/security-group/aws" - version = "~> 4.0" - - name = "${local.name}-vpc-endpoints" - description = "Security group for VPC endpoint access" - vpc_id = module.vpc.vpc_id - - ingress_with_cidr_blocks = [ - { - rule = "https-443-tcp" - description = "VPC CIDR HTTPS" - cidr_blocks = join(",", module.vpc.private_subnets_cidr_blocks) - }, - ] - - egress_with_cidr_blocks = [ - { - rule = "https-443-tcp" - description = "All egress HTTPS" - cidr_blocks = "0.0.0.0/0" - }, - ] - - tags = local.tags -} - -module "vpc_endpoints" { - source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" - version = "~> 3.0" - - vpc_id = module.vpc.vpc_id - security_group_ids = [module.vpc_endpoints_sg.security_group_id] - - endpoints = merge({ - s3 = { - service = "s3" - service_type = "Gateway" - route_table_ids = module.vpc.private_route_table_ids - tags = { - Name = "${local.name}-s3" - } - } - }, - { for service in toset(local.vpc_endpoints) : - replace(service, ".", "_") => - { - service = service - subnet_ids = module.vpc.private_subnets - private_dns_enabled = true - tags = { Name = "${local.name}-${service}" } - } - }) - - tags = local.tags -} From ec03cf15746570083e4b12015b5b5e42aca3b6dd Mon Sep 17 00:00:00 2001 From: Gu Date: Wed, 15 Feb 2023 17:01:57 -0600 Subject: [PATCH 26/26] upgrade --- README.md | 16 ++++++++-------- examples/complete/main.tf | 6 +++--- main.tf | 16 ++++++++-------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6174f29..0885f87 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,14 @@ Examples codified under the [`examples`](https://github.com/aws-ia/terraform-aws | Name | Source | Version | |------|--------|---------| -| [amp](#module\_amp) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [api\_gatewayv2](#module\_api\_gatewayv2) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [dynamodb](#module\_dynamodb) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [emrcontainers](#module\_emrcontainers) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [eventbridge](#module\_eventbridge) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [rds](#module\_rds) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [s3](#module\_s3) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | -| [sfn](#module\_sfn) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.18.1 | +| [amp](#module\_amp) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [api\_gatewayv2](#module\_api\_gatewayv2) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [dynamodb](#module\_dynamodb) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [emrcontainers](#module\_emrcontainers) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [eventbridge](#module\_eventbridge) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [rds](#module\_rds) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [s3](#module\_s3) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | +| [sfn](#module\_sfn) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon | v4.24.0 | ## Resources diff --git a/examples/complete/main.tf b/examples/complete/main.tf index fac77b1..c3d041d 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -52,7 +52,7 @@ locals { ################################################################################ module "eks_blueprints" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints?ref=v4.24.0" cluster_name = local.name cluster_version = "1.23" @@ -77,7 +77,7 @@ module "eks_blueprints" { ################################################################################ module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.24.0" eks_cluster_id = module.eks_blueprints.eks_cluster_id eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint @@ -199,7 +199,7 @@ resource "aws_iam_policy" "dynamodb_access" { } module "irsa" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/irsa?ref=v4.24.0" create_kubernetes_namespace = true kubernetes_namespace = "ack-demo" diff --git a/main.tf b/main.tf index d45faea..dd6bbb5 100644 --- a/main.tf +++ b/main.tf @@ -46,7 +46,7 @@ locals { } module "api_gatewayv2" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_api_gatewayv2 ? 1 : 0 @@ -122,7 +122,7 @@ locals { } module "dynamodb" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_dynamodb ? 1 : 0 @@ -189,7 +189,7 @@ locals { } module "s3" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_s3 ? 1 : 0 @@ -256,7 +256,7 @@ locals { } module "rds" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_rds ? 1 : 0 @@ -324,7 +324,7 @@ locals { } module "amp" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_amp ? 1 : 0 @@ -392,7 +392,7 @@ locals { } module "emrcontainers" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_emrcontainers ? 1 : 0 @@ -536,7 +536,7 @@ locals { } module "sfn" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_sfn ? 1 : 0 @@ -630,7 +630,7 @@ locals { } module "eventbridge" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.18.1" + source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons/helm-addon?ref=v4.24.0" count = var.enable_eb ? 1 : 0