feat: Update gitops multi-cluster pattern to use the new eks pod identity and CAM

patterns/gitops/multi-cluster-hub-spoke-argocd/README.md

@@ -110,24 +110,12 @@ echo "ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-
 echo "ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')"
 ```
 
-## Verify that ArgoCD Service Accounts has the annotation for IRSA
-
-```shell
-kubectl --context hub get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"'
-kubectl --context hub get sa -n argocd argocd-server  -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"'
-```
-
-The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters
-
-```text
-arn:aws:iam::0123456789:role/argocd-hub-0123abc..
-arn:aws:iam::0123456789:role/argocd-hub-0123abc..
-```
-
 ## Deploy the Spoke EKS Cluster
 
 Use the `deploy.sh` script to create terraform workspace, initialize Terraform, and deploy the EKS clusters:
 
+***You may want to create few spoke environments to validate multi-cluster hub spoke to avoid quota limits***
+
 ```shell
 cd ../spokes
 ./deploy.sh dev

patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf

@@ -45,6 +45,8 @@ locals {
   gitops_addons_path     = var.gitops_addons_path
   gitops_addons_revision = var.gitops_addons_revision
 
+  authentication_mode = var.authentication_mode
+
   argocd_namespace = "argocd"
 
   aws_addons = {
@@ -107,7 +109,6 @@ locals {
       aws_vpc_id       = module.vpc.vpc_id
     },
     {
-      argocd_iam_role_arn = module.argocd_irsa.iam_role_arn
       argocd_namespace    = local.argocd_namespace
     },
     {
@@ -145,40 +146,51 @@ module "gitops_bridge_bootstrap" {
 ################################################################################
 # ArgoCD EKS Access
 ################################################################################
-module "argocd_irsa" {
-  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "~> 5.20"
-
-  role_name_prefix           = "argocd-hub-"
-  assume_role_condition_test = "StringLike"
-  role_policy_arns = {
-    ArgoCD_EKS_Policy = aws_iam_policy.irsa_policy.arn
-  }
-  oidc_providers = {
-    main = {
-      provider_arn               = module.eks.oidc_provider_arn
-      namespace_service_accounts = ["${local.argocd_namespace}:argocd-*"]
+data "aws_iam_policy_document" "eks_assume" {
+  statement {
+    effect = "Allow"
+    principals {
+      type        = "Service"
+      identifiers = ["pods.eks.amazonaws.com"]
     }
+    actions = ["sts:AssumeRole","sts:TagSession"]
   }
-
-  tags = local.tags
 }
-
-
-resource "aws_iam_policy" "irsa_policy" {
-  name        = "${module.eks.cluster_name}-argocd-irsa"
-  description = "IAM Policy for ArgoCD Hub"
-  policy      = data.aws_iam_policy_document.irsa_policy.json
-  tags        = local.tags
+resource "aws_iam_role" "argocd_hub" {
+  name               = "${module.eks.cluster_name}-argocd-hub"
+  assume_role_policy = data.aws_iam_policy_document.eks_assume.json
 }
-
-data "aws_iam_policy_document" "irsa_policy" {
+data "aws_iam_policy_document" "aws_assume_policy" {
   statement {
     effect    = "Allow"
     resources = ["*"]
-    actions   = ["sts:AssumeRole"]
+    actions   = ["sts:AssumeRole","sts:TagSession"]
   }
 }
+resource "aws_iam_policy" "aws_assume_policy" {
+  name        = "${module.eks.cluster_name}-argocd-aws-assume"
+  description = "IAM Policy for ArgoCD Hub"
+  policy      = data.aws_iam_policy_document.aws_assume_policy.json
+  tags        = local.tags
+}
+resource "aws_iam_role_policy_attachment" "aws_assume_policy" {
+  role       = aws_iam_role.argocd_hub.name
+  policy_arn = aws_iam_policy.aws_assume_policy.arn
+}
+resource "aws_eks_pod_identity_association" "argocd_app_controller" {
+  cluster_name    = module.eks.cluster_name
+  namespace       = "argocd"
+  service_account = "argocd-application-controller"
+  role_arn        = aws_iam_role.argocd_hub.arn
+}
+resource "aws_eks_pod_identity_association" "argocd_api_server" {
+  cluster_name    = module.eks.cluster_name
+  namespace       = "argocd"
+  service_account = "argocd-server"
+  role_arn        = aws_iam_role.argocd_hub.arn
+}
+
+
 
 ################################################################################
 # EKS Blueprints Addons
@@ -221,7 +233,7 @@ module "eks_blueprints_addons" {
 
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "~> 19.13"
+  version = "~> 20.8"
 
   cluster_name                   = local.name
   cluster_version                = local.cluster_version
@@ -231,6 +243,10 @@ module "eks" {
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.private_subnets
 
+  authentication_mode = local.authentication_mode
+
+  enable_cluster_creator_admin_permissions = true
+
   eks_managed_node_groups = {
     initial = {
       instance_types = ["t3.medium"]
@@ -242,6 +258,9 @@ module "eks" {
   }
   # EKS Addons
   cluster_addons = {
+    eks-pod-identity-agent = {
+      most_recent = true
+    }
     vpc-cni = {
       # Specify the VPC CNI addon should be deployed before compute to ensure
       # the addon is configured before data plane compute resources are created

patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf

@@ -1,7 +1,7 @@
 output "configure_kubectl" {
   description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig"
   value       = <<-EOT
-    export KUBECONFIG="/tmp/hup-spoke"
+    export KUBECONFIG="/tmp/hub-spoke"
     aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub
   EOT
 }
@@ -10,7 +10,7 @@ output "configure_kubectl" {
 output "configure_argocd" {
   description = "Terminal Setup"
   value       = <<-EOT
-    export KUBECONFIG="/tmp/hup-spoke"
+    export KUBECONFIG="/tmp/hub-spoke"
     aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub
     export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web"
     kubectl --context hub config set-context --current --namespace argocd
@@ -25,7 +25,7 @@ output "configure_argocd" {
 output "access_argocd" {
   description = "ArgoCD Access"
   value       = <<-EOT
-    export KUBECONFIG="/tmp/hup-spoke"
+    export KUBECONFIG="/tmp/hub-spoke"
     aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub
     echo "ArgoCD Username: admin"
     echo "ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")"
@@ -34,11 +34,6 @@ output "access_argocd" {
 }
 
 
-output "argocd_iam_role_arn" {
-  description = "IAM Role for ArgoCD Cluster Hub, use to connect to spoke clusters"
-  value       = module.argocd_irsa.iam_role_arn
-}
-
 output "cluster_name" {
   description = "Cluster Hub name"
   value       = module.eks.cluster_name

patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf

@@ -11,20 +11,23 @@ variable "region" {
 variable "kubernetes_version" {
   description = "Kubernetes version"
   type        = string
-  default     = "1.28"
+  default     = "1.29"
 }
 variable "addons" {
   description = "Kubernetes addons"
   type        = any
   default = {
     enable_aws_load_balancer_controller = true
     enable_metrics_server               = true
-    # Enable argocd with IRSA
-    enable_aws_argocd = true
-    # Disable argocd without IRSA
-    enable_argocd = false
+    enable_argocd = true
   }
 }
+
+output "argocd_iam_role_arn" {
+  description = "IAM Role for ArgoCD Cluster Hub, use to connect to spoke clusters"
+  value       = aws_iam_role.argocd_hub.arn
+}
+
 # Addons Git
 variable "gitops_addons_org" {
   description = "Git repository org/user contains for addons"
@@ -51,3 +54,9 @@ variable "gitops_addons_path" {
   type        = string
   default     = "bootstrap/control-plane/addons"
 }
+
+variable "authentication_mode" {
+  description = "The authentication mode for the cluster. Valid values are CONFIG_MAP, API or API_AND_CONFIG_MAP"
+  type        = string
+  default     = "API"
+}

patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf

@@ -70,6 +70,8 @@ locals {
   gitops_workload_revision = var.gitops_workload_revision
   gitops_workload_url      = "${local.gitops_workload_org}/${local.gitops_workload_repo}"
 
+  authentication_mode = var.authentication_mode  
+
   aws_addons = {
     enable_cert_manager                          = try(var.addons.enable_cert_manager, false)
     enable_aws_efs_csi_driver                    = try(var.addons.enable_aws_efs_csi_driver, false)
@@ -186,13 +188,13 @@ module "gitops_bridge_bootstrap_hub" {
 # ArgoCD EKS Access
 ################################################################################
 resource "aws_iam_role" "spoke" {
-  name               = "${module.eks.cluster_name}-argocd-spoke"
+  name               = "${local.name}-argocd-spoke"
   assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json
 }
 
 data "aws_iam_policy_document" "assume_role_policy" {
   statement {
-    actions = ["sts:AssumeRole"]
+    actions = ["sts:AssumeRole","sts:TagSession"]
     principals {
       type        = "AWS"
       identifiers = [data.terraform_remote_state.cluster_hub.outputs.argocd_iam_role_arn]
@@ -243,7 +245,7 @@ module "eks_blueprints_addons" {
 
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
-  version = "~> 19.13"
+  version = "~> 20.8.4"
 
   cluster_name                   = local.name
   cluster_version                = local.cluster_version
@@ -252,18 +254,28 @@ module "eks" {
 
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.private_subnets
-
-  manage_aws_auth_configmap = true
-  aws_auth_roles = [
-    # Granting access to ArgoCD from hub cluster
-    {
-      rolearn  = aws_iam_role.spoke.arn
-      username = "gitops-role"
-      groups = [
-        "system:masters"
-      ]
-    },
-  ]
+
+  authentication_mode = local.authentication_mode
+
+  # Cluster access entry
+  # To add the current caller identity as an administrator
+  enable_cluster_creator_admin_permissions = true
+
+  access_entries = {
+    # One access entry with a policy associated
+    example = {
+      principal_arn     = aws_iam_role.spoke.arn
+
+      policy_associations = {
+        argocd = {
+          policy_arn = "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"
+          access_scope = {
+            type       = "cluster"
+          }
+        }
+      }
+    }
+  }
 
   eks_managed_node_groups = {
     initial = {
@@ -276,6 +288,9 @@ module "eks" {
   }
   # EKS Addons
   cluster_addons = {
+    eks-pod-identity-agent = {
+      most_recent = true
+    }  
     vpc-cni = {
       # Specify the VPC CNI addon should be deployed before compute to ensure
       # the addon is configured before data plane compute resources are created

patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf

@@ -1,7 +1,7 @@
 output "configure_kubectl" {
   description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig"
   value       = <<-EOT
-    export KUBECONFIG="/tmp/hup-spoke"
+    export KUBECONFIG="/tmp/hub-spoke"
     aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias ${local.environment}
   EOT
 }

patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf

@@ -67,3 +67,9 @@ variable "gitops_workload_path" {
   type        = string
   default     = "helm-guestbook"
 }
+
+variable "authentication_mode" {
+  description = "The authentication mode for the cluster. Valid values are CONFIG_MAP, API or API_AND_CONFIG_MAP"
+  type        = string
+  default     = "API"
+}

patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars

@@ -1,10 +1,11 @@
 vpc_cidr           = "10.1.0.0/16"
 region             = "us-west-2"
-kubernetes_version = "1.28"
+kubernetes_version = "1.29"
 addons = {
   enable_aws_load_balancer_controller = true
   enable_metrics_server               = true
   # Disable argocd on spoke clusters
   enable_aws_argocd = false
   enable_argocd     = false
 }
+authentication_mode = "API"

patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars

@@ -1,10 +1,11 @@
 vpc_cidr           = "10.3.0.0/16"
 region             = "us-west-2"
-kubernetes_version = "1.28"
+kubernetes_version = "1.29"
 addons = {
   enable_aws_load_balancer_controller = true
   enable_metrics_server               = true
   # Disable argocd on spoke clusters
   enable_aws_argocd = false
   enable_argocd     = false
 }
+authentication_mode = "API"

patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars

@@ -1,10 +1,11 @@
 vpc_cidr           = "10.2.0.0/16"
 region             = "us-west-2"
-kubernetes_version = "1.28"
+kubernetes_version = "1.29"
 addons = {
   enable_aws_load_balancer_controller = true
   enable_metrics_server               = true
   # Disable argocd on spoke clusters
   enable_aws_argocd = false
   enable_argocd     = false
 }
+authentication_mode = "API"