From 181bda53d8922705dfb08085b89fec889d738f06 Mon Sep 17 00:00:00 2001 From: ananth102 Date: Mon, 28 Aug 2023 12:41:11 -0700 Subject: [PATCH] Migrate to v5 blueprints (#779) **Which issue is resolved by this Pull Request:** Resolves #775 **Description of your changes:** Upgrade to v5 blueprints for the eks addons Major changes: 1. v5 does not have an option to enable the ebs csi driver, will need to do with the help of another module 2. v5 does not have an option for enabling the nvidia plugin, an operator is used instead. 3. V5/V4 parameters are different. **Testing:** - [ ] Unit tests pass - [x] e2e tests pass - Cognito, rds-s3-static, rds/s3-irsa passes, efs/fsx look fine manually. Need to test nvidia. - Details about new tests (If this PR adds a new feature) - Details about any manual tests performed - GPU testing By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --- deployments/cognito-rds-s3/terraform/main.tf | 84 +++++++++++++------ deployments/cognito/terraform/main.tf | 72 +++++++++++----- deployments/rds-s3/terraform/main.tf | 83 ++++++++++++------ deployments/vanilla/terraform/main.tf | 74 +++++++++++----- .../aws-infra/ebs-csi-driver-irsa/main.tf | 17 ++++ .../aws-infra/ebs-csi-driver-irsa/outputs.tf | 9 ++ .../ebs-csi-driver-irsa/variables.tf | 12 +++ 7 files changed, 256 insertions(+), 95 deletions(-) create mode 100644 iaac/terraform/aws-infra/ebs-csi-driver-irsa/main.tf create mode 100644 iaac/terraform/aws-infra/ebs-csi-driver-irsa/outputs.tf create mode 100644 iaac/terraform/aws-infra/ebs-csi-driver-irsa/variables.tf diff --git a/deployments/cognito-rds-s3/terraform/main.tf b/deployments/cognito-rds-s3/terraform/main.tf index dbb5beffd..29c576494 100644 --- a/deployments/cognito-rds-s3/terraform/main.tf +++ b/deployments/cognito-rds-s3/terraform/main.tf @@ -128,43 +128,65 @@ module "eks_blueprints" { tags = local.tags } +module "ebs_csi_driver_irsa" { + source = "../../../iaac/terraform/aws-infra/ebs-csi-driver-irsa" + cluster_name = local.cluster_name + cluster_region = local.region + tags = local.tags + eks_oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn +} + module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1" + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" #ensure to update this to the latest/desired version - eks_cluster_id = module.eks_blueprints.eks_cluster_id - eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint - eks_oidc_provider = module.eks_blueprints.oidc_provider - eks_cluster_version = module.eks_blueprints.eks_cluster_version + cluster_name = local.cluster_name + cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + cluster_version = module.eks_blueprints.eks_cluster_version + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn - # EKS Managed Add-ons - enable_amazon_eks_vpc_cni = true - enable_amazon_eks_coredns = true - enable_amazon_eks_kube_proxy = true - enable_amazon_eks_aws_ebs_csi_driver = true + depends_on = [module.ebs_csi_driver_irsa, module.eks_data_addons] + + eks_addons = { + aws-ebs-csi-driver = { + most_recent = true + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } + coredns = { + most_recent = true + } + vpc-cni = { + most_recent = true + } + kube-proxy = { + most_recent = true + } + } - # EKS Blueprints Add-ons - enable_cert_manager = true enable_aws_load_balancer_controller = true + enable_cert_manager = true - aws_efs_csi_driver_helm_config = { - namespace = "kube-system" - version = "2.4.1" + cert_manager = { + chart_version = "v1.10.0" } enable_aws_efs_csi_driver = true + enable_aws_fsx_csi_driver = true - aws_fsx_csi_driver_helm_config = { - namespace = "kube-system" - version = "1.5.1" - } - enable_aws_fsx_csi_driver = true + aws_efs_csi_driver = { + namespace = "kube-system" + chart_version = "2.4.1" + } - enable_nvidia_device_plugin = local.using_gpu + aws_fsx_csi_driver = { + namespace = "kube-system" + chart_version = "1.5.1" + } - secrets_store_csi_driver_helm_config = { - namespace = "kube-system" - version = "1.3.2" + secrets_store_csi_driver = { + namespace = "kube-system" + chart_version = "1.3.2" set = [ { name = "syncSecret.enabled", @@ -172,10 +194,10 @@ module "eks_blueprints_kubernetes_addons" { } ] } - enable_secrets_store_csi_driver = true + enable_secrets_store_csi_driver = true - csi_secrets_store_provider_aws_helm_config = { + secrets_store_csi_driver_provider_aws = { namespace = "kube-system" set = [ { @@ -184,10 +206,19 @@ module "eks_blueprints_kubernetes_addons" { } ] } + enable_secrets_store_csi_driver_provider_aws = true tags = local.tags +} +module "eks_data_addons" { + source = "aws-ia/eks-data-addons/aws" + version = "~> 1.0" # ensure to update this to the latest/desired version + + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn + + enable_nvidia_gpu_operator = local.using_gpu } # todo: update the blueprints repo code to export the desired values as outputs @@ -251,6 +282,7 @@ module "kubeflow_components" { tags = local.tags + providers = { aws = aws aws.virginia = aws.virginia diff --git a/deployments/cognito/terraform/main.tf b/deployments/cognito/terraform/main.tf index 266a711be..6376088ed 100644 --- a/deployments/cognito/terraform/main.tf +++ b/deployments/cognito/terraform/main.tf @@ -129,42 +129,72 @@ module "eks_blueprints" { tags = local.tags } +module "ebs_csi_driver_irsa" { + source = "../../../iaac/terraform/aws-infra/ebs-csi-driver-irsa" + cluster_name = local.cluster_name + cluster_region = local.region + tags = local.tags + eks_oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn +} + module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1" + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" #ensure to update this to the latest/desired version - eks_cluster_id = module.eks_blueprints.eks_cluster_id - eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint - eks_oidc_provider = module.eks_blueprints.oidc_provider - eks_cluster_version = module.eks_blueprints.eks_cluster_version + cluster_name = local.cluster_name + cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + cluster_version = module.eks_blueprints.eks_cluster_version + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn - # EKS Managed Add-ons - enable_amazon_eks_vpc_cni = true - enable_amazon_eks_coredns = true - enable_amazon_eks_kube_proxy = true - enable_amazon_eks_aws_ebs_csi_driver = true + depends_on = [module.ebs_csi_driver_irsa, module.eks_data_addons] + + eks_addons = { + aws-ebs-csi-driver = { + most_recent = true + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } + coredns = { + most_recent = true + } + vpc-cni = { + most_recent = true + } + kube-proxy = { + most_recent = true + } + } - # EKS Blueprints Add-ons - enable_cert_manager = true enable_aws_load_balancer_controller = true + enable_cert_manager = true - aws_efs_csi_driver_helm_config = { - namespace = "kube-system" - version = "2.4.1" + cert_manager = { + chart_version = "v1.10.0" } enable_aws_efs_csi_driver = true + enable_aws_fsx_csi_driver = true - aws_fsx_csi_driver_helm_config = { - namespace = "kube-system" - version = "1.5.1" - } - enable_aws_fsx_csi_driver = true + aws_efs_csi_driver = { + namespace = "kube-system" + chart_version = "2.4.1" + } - enable_nvidia_device_plugin = local.using_gpu + aws_fsx_csi_driver = { + namespace = "kube-system" + chart_version = "1.5.1" + } tags = local.tags +} + +module "eks_data_addons" { + source = "aws-ia/eks-data-addons/aws" + version = "~> 1.0" # ensure to update this to the latest/desired version + + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn + enable_nvidia_gpu_operator = local.using_gpu } # todo: update the blueprints repo code to export the desired values as outputs diff --git a/deployments/rds-s3/terraform/main.tf b/deployments/rds-s3/terraform/main.tf index d07faf8ae..a448e81d1 100644 --- a/deployments/rds-s3/terraform/main.tf +++ b/deployments/rds-s3/terraform/main.tf @@ -121,43 +121,65 @@ module "eks_blueprints" { tags = local.tags } +module "ebs_csi_driver_irsa" { + source = "../../../iaac/terraform/aws-infra/ebs-csi-driver-irsa" + cluster_name = local.cluster_name + cluster_region = local.region + tags = local.tags + eks_oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn +} + module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1" + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" #ensure to update this to the latest/desired version - eks_cluster_id = module.eks_blueprints.eks_cluster_id - eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint - eks_oidc_provider = module.eks_blueprints.oidc_provider - eks_cluster_version = module.eks_blueprints.eks_cluster_version + cluster_name = local.cluster_name + cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + cluster_version = module.eks_blueprints.eks_cluster_version + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn - # EKS Managed Add-ons - enable_amazon_eks_vpc_cni = true - enable_amazon_eks_coredns = true - enable_amazon_eks_kube_proxy = true - enable_amazon_eks_aws_ebs_csi_driver = true + depends_on = [module.ebs_csi_driver_irsa, module.eks_data_addons] + + eks_addons = { + aws-ebs-csi-driver = { + most_recent = true + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } + coredns = { + most_recent = true + } + vpc-cni = { + most_recent = true + } + kube-proxy = { + most_recent = true + } + } - # EKS Blueprints Add-ons - enable_cert_manager = true enable_aws_load_balancer_controller = true + enable_cert_manager = true - aws_efs_csi_driver_helm_config = { - namespace = "kube-system" - version = "2.4.1" + cert_manager = { + chart_version = "v1.10.0" } enable_aws_efs_csi_driver = true + enable_aws_fsx_csi_driver = true - aws_fsx_csi_driver_helm_config = { - namespace = "kube-system" - version = "1.5.1" - } - enable_aws_fsx_csi_driver = true + aws_efs_csi_driver = { + namespace = "kube-system" + chart_version = "2.4.1" + } - enable_nvidia_device_plugin = local.using_gpu + aws_fsx_csi_driver = { + namespace = "kube-system" + chart_version = "1.5.1" + } - secrets_store_csi_driver_helm_config = { - namespace = "kube-system" - version = "1.3.2" + secrets_store_csi_driver = { + namespace = "kube-system" + chart_version = "1.3.2" set = [ { name = "syncSecret.enabled", @@ -165,10 +187,10 @@ module "eks_blueprints_kubernetes_addons" { } ] } - enable_secrets_store_csi_driver = true + enable_secrets_store_csi_driver = true - csi_secrets_store_provider_aws_helm_config = { + secrets_store_csi_driver_provider_aws = { namespace = "kube-system" set = [ { @@ -177,10 +199,19 @@ module "eks_blueprints_kubernetes_addons" { } ] } + enable_secrets_store_csi_driver_provider_aws = true tags = local.tags +} + +module "eks_data_addons" { + source = "aws-ia/eks-data-addons/aws" + version = "~> 1.0" # ensure to update this to the latest/desired version + + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn + enable_nvidia_gpu_operator = local.using_gpu } # todo: update the blueprints repo code to export the desired values as outputs diff --git a/deployments/vanilla/terraform/main.tf b/deployments/vanilla/terraform/main.tf index 76b34d583..6af1ae75e 100644 --- a/deployments/vanilla/terraform/main.tf +++ b/deployments/vanilla/terraform/main.tf @@ -120,44 +120,73 @@ module "eks_blueprints" { tags = local.tags } +module "ebs_csi_driver_irsa" { + source = "../../../iaac/terraform/aws-infra/ebs-csi-driver-irsa" + cluster_name = local.cluster_name + cluster_region = local.region + tags = local.tags + eks_oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn +} + module "eks_blueprints_kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1" + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" #ensure to update this to the latest/desired version - eks_cluster_id = module.eks_blueprints.eks_cluster_id - eks_cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint - eks_oidc_provider = module.eks_blueprints.oidc_provider - eks_cluster_version = module.eks_blueprints.eks_cluster_version + cluster_name = local.cluster_name + cluster_endpoint = module.eks_blueprints.eks_cluster_endpoint + cluster_version = module.eks_blueprints.eks_cluster_version + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn - # EKS Managed Add-ons - enable_amazon_eks_vpc_cni = true - enable_amazon_eks_coredns = true - enable_amazon_eks_kube_proxy = true - enable_amazon_eks_aws_ebs_csi_driver = true + depends_on = [module.ebs_csi_driver_irsa, module.eks_data_addons] + + eks_addons = { + aws-ebs-csi-driver = { + most_recent = true + service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + } + coredns = { + most_recent = true + } + vpc-cni = { + most_recent = true + } + kube-proxy = { + most_recent = true + } + } - # EKS Blueprints Add-ons - enable_cert_manager = true enable_aws_load_balancer_controller = true + enable_cert_manager = true - aws_efs_csi_driver_helm_config = { - namespace = "kube-system" - version = "2.4.1" + cert_manager = { + chart_version = "v1.10.0" } enable_aws_efs_csi_driver = true + enable_aws_fsx_csi_driver = true - aws_fsx_csi_driver_helm_config = { - namespace = "kube-system" - version = "1.5.1" - } - enable_aws_fsx_csi_driver = true + aws_efs_csi_driver = { + namespace = "kube-system" + chart_version = "2.4.1" + } - enable_nvidia_device_plugin = local.using_gpu + aws_fsx_csi_driver = { + namespace = "kube-system" + chart_version = "1.5.1" + } tags = local.tags - } +module "eks_data_addons" { + source = "aws-ia/eks-data-addons/aws" + version = "~> 1.0" # ensure to update this to the latest/desired version + + oidc_provider_arn = module.eks_blueprints.eks_oidc_provider_arn + + enable_nvidia_gpu_operator = local.using_gpu +} # todo: update the blueprints repo code to export the desired values as outputs module "eks_blueprints_outputs" { @@ -181,6 +210,7 @@ module "kubeflow_components" { notebook_cull_idle_time = var.notebook_cull_idle_time notebook_idleness_check_period = var.notebook_idleness_check_period + tags = local.tags } diff --git a/iaac/terraform/aws-infra/ebs-csi-driver-irsa/main.tf b/iaac/terraform/aws-infra/ebs-csi-driver-irsa/main.tf new file mode 100644 index 000000000..b0f5b0cd1 --- /dev/null +++ b/iaac/terraform/aws-infra/ebs-csi-driver-irsa/main.tf @@ -0,0 +1,17 @@ +module "ebs_csi_driver_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.20" + + role_name_prefix = "${var.cluster_name}-${var.cluster_region}-ebs-csi-driver-" + + attach_ebs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = var.eks_oidc_provider_arn + namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"] + } + } + + tags = var.tags +} \ No newline at end of file diff --git a/iaac/terraform/aws-infra/ebs-csi-driver-irsa/outputs.tf b/iaac/terraform/aws-infra/ebs-csi-driver-irsa/outputs.tf new file mode 100644 index 000000000..4c0d2d612 --- /dev/null +++ b/iaac/terraform/aws-infra/ebs-csi-driver-irsa/outputs.tf @@ -0,0 +1,9 @@ +output "iam_role_arn" { + description = "ARN of IAM role" + value = module.ebs_csi_driver_irsa.iam_role_arn +} + +output "iam_role_name" { + description = "Name of IAM role" + value = module.ebs_csi_driver_irsa.iam_role_name +} \ No newline at end of file diff --git a/iaac/terraform/aws-infra/ebs-csi-driver-irsa/variables.tf b/iaac/terraform/aws-infra/ebs-csi-driver-irsa/variables.tf new file mode 100644 index 000000000..6852cc6bd --- /dev/null +++ b/iaac/terraform/aws-infra/ebs-csi-driver-irsa/variables.tf @@ -0,0 +1,12 @@ +variable "cluster_name" { + type = string +} +variable "cluster_region" { + type = string +} +variable "eks_oidc_provider_arn" { + type = string +} +variable "tags" { + type = any +} \ No newline at end of file