From b1c7279db55495a315727f4ff02877260765e9b9 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Wed, 1 Nov 2023 06:05:41 -0400 Subject: [PATCH 1/8] add multi-cluster hub-spoke Signed-off-by: Carlos Santana update docs Signed-off-by: Carlos Santana --- .../gitops-multi-cluster-hub-spoke-argocd.md | 7 + .../multi-cluster-hub-spoke-argocd/README.md | 159 +++++++++ .../hub/bootstrap/addons.yaml | 32 ++ .../hub/bootstrap/workloads.yaml | 34 ++ .../hub/destroy.sh | 25 ++ .../hub/main.tf | 297 ++++++++++++++++ .../hub/outputs.tf | 57 ++++ .../hub/variables.tf | 53 +++ .../hub/versions.tf | 25 ++ .../spokes/deploy.sh | 17 + .../spokes/destroy.sh | 24 ++ .../spokes/main.tf | 323 ++++++++++++++++++ .../spokes/outputs.tf | 7 + .../spokes/variables.tf | 69 ++++ .../spokes/versions.tf | 21 ++ ...tops-bridge-multi-cluster-hup-spoke.drawio | 1 + ...-bridge-multi-cluster-hup-spoke.drawio.png | Bin 0 -> 33075 bytes 17 files changed, 1151 insertions(+) create mode 100644 docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/README.md create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml create mode 100755 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf create mode 100755 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh create mode 100755 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio.png diff --git a/docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md b/docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md new file mode 100644 index 0000000000..0ef45bb13a --- /dev/null +++ b/docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md @@ -0,0 +1,7 @@ +--- +title: GitOps Multi-Cluster Hub-Spoke Topology (ArgoCD) +--- + +{% + include-markdown "../../patterns/gitops/multi-cluster-hub-spoke-argocd/README.md" +%} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md new file mode 100644 index 0000000000..21a8a35b3e --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -0,0 +1,159 @@ +# Multi-Cluster centralized hub-spoke topology + +This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD in a Multi-Cluster Hub-Spoke topoloy, employing the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev). + + + + +This example deploys ArgoCD on the Hub cluster (ie. management/control-plane cluster). +The spoke clusters are registered as remote clusters in the Hub Cluster's ArgoCD +The ArgoCD on the Hub Cluster deploy addons and workloads to the spoke clusters + +Each spoke cluster gets deployed an app of apps ArgoCD Application with the name `workloads-${env}` + +## Prerequisites +Before you begin, make sure you have the following command line tools installed: +- git +- terraform +- kubectl +- argocd + +## Fork the Git Repositories + +### Fork the Addon GitOps Repo +1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). +2. Update the following environment variables to point to your fork by changing the default values: +```shell +export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev +export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template +``` + +## Deploy the Hub EKS Cluster +Change Director to `hub` +```shell +cd hub +``` +Initialize Terraform and deploy the EKS cluster: +```shell +terraform init +terraform apply -auto-approve +``` +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +### Monitor GitOps Progress for Addons +Wait until **all** the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +```shell +watch kubectl get applications -n argocd +``` + +## Access ArgoCD on Hub Cluster +Access ArgoCD's UI, run the command from the output: +```shell +terraform output -raw access_argocd +``` + +## Verify that ArgoCD Service Accouts has the annotation for IRSA +```shell +kubectl get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +kubectl get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +``` +The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters +```text +"arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub" +``` + +## Deploy the Spoke EKS Cluster +Initialize Terraform and deploy the EKS clusters: +```shell +cd ../spokes +./deploy.sh dev +./deploy.sh staging +./deploy.sh prod +``` +Each environment uses a Terraform workspace + +To access Terraform output run the following commands for the particular environment +```shell +terraform workspace select dev +terraform output +``` +```shell +terraform workspace select staging +terraform output +``` +```shell +terraform workspace select prod +terraform output +``` + +Retrieve `kubectl` config, then execute the output command: +```shell +terraform output -raw configure_kubectl +``` + +### Verify ArgoCD Cluster Secret for Spoke has the correct IAM Role to be assume by Hub Cluster +```shell +kubectl get secret -n argocd hub-spoke-dev --template='{{index .data.config | base64decode}}' +``` +Do the same for the other cluster replaced `dev` in `hub-spoke-dev` +The output have a section `awsAuthConfig` with the `clusterName` and the `roleARN` that has write access to the spoke cluster +```json +{ + "tlsClientConfig": { + "insecure": false, + "caData" : "LS0tL...." + }, + "awsAuthConfig" : { + "clusterName": "hub-spoke-dev", + "roleARN": "arn:aws:iam::0123456789:role/hub-spoke-dev-argocd-spoke" + } +} +``` + + +### Verify the Addons on Spoke Clusters +Verify that the addons are ready: +```shell +kubectl get deployment -n kube-system \ + metrics-server +``` + + +### Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context) +Watch until **all* the Workloads ArgoCD Applications are `Healthy` +```shell +watch kubectl get -n argocd applications +``` +Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command + + +### Verify the Application +Verify that the application configuration is present and the pod is running: +```shell +kubectl get all -n workload +``` + +### Container Metrics +Check the application's CPU and memory metrics: +```shell +kubectl top pods -n workload +``` + +## Destroy the Spoke EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: +```shell +./destroy.sh dev +./destroy.sh staging +./destroy.sh prod +``` + +## Destroy the Hub EKS Clusters +To tear down all the resources and the EKS cluster, run the following command: +Destroy Hub Clusters +```shell +cd ../hub +./destroy.sh +``` diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml new file mode 100644 index 0000000000..89f3602844 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml @@ -0,0 +1,32 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + template: + metadata: + name: cluster-addons + spec: + project: default + source: + repoURL: '{{metadata.annotations.addons_repo_url}}' + path: '{{metadata.annotations.addons_repo_basepath}}{{metadata.annotations.addons_repo_path}}' + targetRevision: '{{metadata.annotations.addons_repo_revision}}' + directory: + recurse: true + exclude: exclude/* + destination: + namespace: 'argocd' + name: '{{name}}' + syncPolicy: + automated: {} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml new file mode 100644 index 0000000000..c399039367 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml @@ -0,0 +1,34 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: workloads + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: false + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + - key: environment + operator: NotIn + values: [control-plane] + template: + metadata: + name: 'workload-{{metadata.labels.environment}}' + spec: + project: default + source: + repoURL: '{{metadata.annotations.workload_repo_url}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' + targetRevision: '{{metadata.annotations.workload_repo_revision}}' + destination: + namespace: 'workload' + name: '{{name}}' + syncPolicy: + automated: {} + syncOptions: + - CreateNamespace=true diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh new file mode 100755 index 0000000000..8ad29aaebb --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + +# Delete the Ingress/SVC before removing the addons +TMPFILE=$(mktemp) +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + kubectl delete -n argocd svc argo-cd-argocd-server +fi + +terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -target="module.eks" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf new file mode 100644 index 0000000000..60710e5dbb --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf @@ -0,0 +1,297 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + +locals { + name = "hub-spoke-${local.environment}" + environment = "control-plane" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + argocd_namespace = "argocd" + + aws_addons = { + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) + } + oss_addons = { + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) + } + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + argocd_iam_role_arn = module.argocd_irsa.iam_role_arn + argocd_namespace = local.argocd_namespace + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + } + ) + + argocd_apps = { + addons = file("${path.module}/bootstrap/addons.yaml") + workloads = file("${path.module}/bootstrap/workloads.yaml") + } + + tags = { + Blueprint = local.name + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap +################################################################################ +module "gitops_bridge_bootstrap" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + cluster = { + cluster_name = module.eks.cluster_name + environment = local.environment + metadata = local.addons_metadata + addons = local.addons + } + apps = local.argocd_apps + argocd = { + namespace = local.argocd_namespace + } +} + +################################################################################ +# ArgoCD EKS Access +################################################################################ +module "argocd_irsa" { + source = "aws-ia/eks-blueprints-addon/aws" + + create_release = false + create_role = true + role_name_use_prefix = false + role_name = "${module.eks.cluster_name}-argocd-hub" + assume_role_condition_test = "StringLike" + create_policy = false + role_policies = { + ArgoCD_EKS_Policy = aws_iam_policy.irsa_policy.arn + } + oidc_providers = { + this = { + provider_arn = module.eks.oidc_provider_arn + namespace = local.argocd_namespace + service_account = "argocd-*" + } + } + tags = local.tags + +} + +resource "aws_iam_policy" "irsa_policy" { + name = "${module.eks.cluster_name}-argocd-irsa" + description = "IAM Policy for ArgoCD Hub" + policy = data.aws_iam_policy_document.irsa_policy.json + tags = local.tags +} + +data "aws_iam_policy_document" "irsa_policy" { + statement { + effect = "Allow" + resources = ["*"] + actions = ["sts:AssumeRole"] + } +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 3 + max_size = 10 + desired_size = 3 + } + } + # EKS Addons + cluster_addons = { + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf new file mode 100644 index 0000000000..c5b089f69a --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf @@ -0,0 +1,57 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + EOT +} + + +output "configure_argocd" { + description = "Terminal Setup" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" + kubectl config set-context --current --namespace argocd + argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo Port Forward: http://localhost:8080 + kubectl port-forward -n argocd svc/argo-cd-argocd-server 8080:80 + EOT +} + +output "access_argocd" { + description = "ArgoCD Access" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" + EOT +} + + +output "argocd_iam_role_arn" { + description = "IAM Role for ArgoCD Cluster Hub, use to connect to spoke clusters" + value = module.argocd_irsa.iam_role_arn +} + +output "cluster_name" { + description = "Cluster Hub name" + value = module.eks.cluster_name +} +output "cluster_endpoint" { + description = "Cluster Hub endpoint" + value = module.eks.cluster_endpoint +} +output "cluster_certificate_authority_data" { + description = "Cluster Hub certificate_authority_data" + value = module.eks.cluster_certificate_authority_data +} +output "cluster_region" { + description = "Cluster Hub region" + value = local.region +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf new file mode 100644 index 0000000000..2e2c8462ee --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf @@ -0,0 +1,53 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd with IRSA + enable_aws_argocd = true + # Disable argocd without IRSA + enable_argocd = false + } +} +# Addons Git +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + type = string + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_addons_repo" { + description = "Git repository contains for addons" + type = string + default = "gitops-bridge-argocd-control-plane-template" +} +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} +variable "gitops_addons_basepath" { + description = "Git repository base path for addons" + type = string + default = "" +} +variable "gitops_addons_path" { + description = "Git repository path for addons" + type = string + default = "bootstrap/control-plane/addons" +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf new file mode 100644 index 0000000000..2de60d58ee --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.67.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.10.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/ipv4-prefix-delegation/terraform.tfstate" + # } +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh new file mode 100755 index 0000000000..0dad2a7030 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [[ $# -eq 0 ]] ; then + echo "No arguments supplied" + echo "Usage: deploy.sh " + echo "Example: deploy.sh dev" + exit 1 +fi +env=$1 +echo "Deploying $env with "workspaces/${env}.tfvars" ..." + +set -x + +terraform workspace new $env +terraform workspace select $env +terraform init +terraform apply -var-file="workspaces/${env}.tfvars" diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh new file mode 100755 index 0000000000..cc2a333fdd --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + + +if [[ $# -eq 0 ]] ; then + echo "No arguments supplied" + echo "Usage: destroy.sh " + echo "Example: destroy.sh dev" + exit 1 +fi +env=$1 +echo "Destroying $env ..." +terraform workspace select $env + +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.vpc" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf new file mode 100644 index 0000000000..28b328e5a3 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf @@ -0,0 +1,323 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + + +data "terraform_remote_state" "cluster_hub" { + backend = "local" + + config = { + path = "${path.module}/../hub/terraform.tfstate" + } +} + +################################################################################ +# Kubernetes Access for Hub Cluster +################################################################################ + +provider "kubernetes" { + host = data.terraform_remote_state.cluster_hub.outputs.cluster_endpoint + cluster_ca_certificate = base64decode(data.terraform_remote_state.cluster_hub.outputs.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", data.terraform_remote_state.cluster_hub.outputs.cluster_name, "--region", data.terraform_remote_state.cluster_hub.outputs.cluster_region] + } + alias = "hub" +} + +################################################################################ +# Kubernetes Access for Spoke Cluster +################################################################################ + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + + + +locals { + name = "hub-spoke-${terraform.workspace}" + environment = terraform.workspace + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + gitops_workload_org = var.gitops_workload_org + gitops_workload_repo = var.gitops_workload_repo + gitops_workload_basepath = var.gitops_workload_basepath + gitops_workload_path = var.gitops_workload_path + gitops_workload_revision = var.gitops_workload_revision + gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" + + aws_addons = { + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) + } + oss_addons = { + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) + } + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + }, + { + workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath + workload_repo_path = local.gitops_workload_path + workload_repo_revision = local.gitops_workload_revision + } + ) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap for Hub Cluster +################################################################################ +module "gitops_bridge_bootstrap_hub" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + # The ArgoCD remote cluster secret is deploy on hub cluster not on spoke clusters + providers = { + kubernetes = kubernetes.hub + } + + install = false # We are not installing argocd via helm on hub cluster + cluster = { + cluster_name = module.eks.cluster_name + environment = local.environment + metadata = local.addons_metadata + addons = local.addons + server = module.eks.cluster_endpoint + config = <<-EOT + { + "tlsClientConfig": { + "insecure": false, + "caData" : "${module.eks.cluster_certificate_authority_data}" + }, + "awsAuthConfig" : { + "clusterName": "${module.eks.cluster_name}", + "roleARN": "${aws_iam_role.spoke.arn}" + } + } + EOT + } +} + +################################################################################ +# ArgoCD EKS Access +################################################################################ +resource "aws_iam_role" "spoke" { + name = "${module.eks.cluster_name}-argocd-spoke" + assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json +} + +data "aws_iam_policy_document" "assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = [data.terraform_remote_state.cluster_hub.outputs.argocd_iam_role_arn] + } + } +} + + + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # Granting access to ArgoCD from hub cluster + { + rolearn = aws_iam_role.spoke.arn + username = "gitops-role" + groups = [ + "system:masters" + ] + }, + ] + + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 3 + max_size = 10 + desired_size = 3 + } + } + # EKS Addons + cluster_addons = { + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf new file mode 100644 index 0000000000..e398229ac0 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf @@ -0,0 +1,7 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + EOT +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf new file mode 100644 index 0000000000..570325119c --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf @@ -0,0 +1,69 @@ +variable "region" { + description = "AWS region" + type = string +} +variable "vpc_cidr" { + description = "VPC CIDR" + type = string +} +variable "kubernetes_version" { + description = "EKS version" + type = string +} +variable "addons" { + description = "Kubernetes addons" + type = any +} +# Addons Git +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + type = string + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_addons_repo" { + description = "Git repository contains for addons" + type = string + default = "gitops-bridge-argocd-control-plane-template" +} +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} +variable "gitops_addons_basepath" { + description = "Git repository base path for addons" + type = string + default = "" +} +variable "gitops_addons_path" { + description = "Git repository path for addons" + type = string + default = "bootstrap/control-plane/addons" +} + +# Workloads Git +variable "gitops_workload_org" { + description = "Git repository org/user contains for workload" + type = string + default = "https://github.com/argoproj" +} +variable "gitops_workload_repo" { + description = "Git repository contains for workload" + type = string + default = "argocd-example-apps" +} +variable "gitops_workload_revision" { + description = "Git repository revision/branch/ref for workload" + type = string + default = "master" +} +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" + type = string + default = "" +} +variable "gitops_workload_path" { + description = "Git repository path for workload" + type = string + default = "helm-guestbook" +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf new file mode 100644 index 0000000000..6ff7c991ec --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.67.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/ipv4-prefix-delegation/terraform.tfstate" + # } +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio new file mode 100644 index 0000000000..083d637330 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio @@ -0,0 +1 @@ +7Vvbcps6FP0aPyaDwPjyGGPnnE7bmc7JQ+NHxcigRkZUyDHu1x8JhEESvSWxIRMSZ4yWLoi1tLe2duyRF+zyfxhM4880RGTkOmE+8pYj13Wd8Vi8SeRYIgDM/BKJGA4VVgN3+AdSoKPQPQ5RpjXklBKOUx3c0CRBG65hkDF60JttKdHvmsIIWcDdBhIb/YpDHpfozHdq/F+Eo7i6M3BUzQ5WjRWQxTCkhwbkrUZewCjl5dUuDxCR7FW8lP1uf1J7mhhDCf+TDph9/b7+L/h8e3+TTO7Xa/QD7K68cpQnSPbqgdVk+bFigNF9EiI5iDPyFocYc3SXwo2sPQjRBRbzHRElIC7VcIhxlP90nuD09GLdILpDnB1FE9VhqvhSK2ZcrYRDTT+YKSxuUj9WIFSSR6eha1bEhSLmL0ga/56k7BHxTaw4SilOeDEJfyFeYlpB+eeLpoFErl2/BWzDpjYI7GbiDbTdwQTbsKkNAruZLFWz1sE2bOrbMzZ7g5bewOgtXt6C7jnBCQpOFi453tKEB5RQVvDvid9bqegiYjDESKu7nc5WzrhRt8RMDIRpIuoTyuSqWmwxIY0+S8cPwFTgGWf0ETVqtsWPqAlhFp/MQq54LLzGJ/iAyBeaYTX8A+Wc7hoNbgiOZAWn0nKgKm3ErBDTTUk+ofKHwK3KasXJW8IsLenY4lzOYyEcTCord3kknfE1PGTja4Yyumcb9GEj57MQxfJKb4Ues4sZr99iu/65TNe3TBeGIU2yzr0ccA2mpjZT8xamvHMxNbGYOlD2SCgM+0eW37KsLkrWtGVHmBCurFQja/J9T6uKq6yw5xvRwAVpXpBT1YurSL6vPt5VY4mplcOVNZYMglCuc617q4QmyHBtCrL8jumedjgM5W1axdXlfw19PcMYXFtf75L6VgM3yEahiAtVUe4ZNKIJJKsaNWip23yi0tcX+nxDnB+VU4d7TnX1BFvseC/7i21RFddquKKwzLXSsbEXqkHdYpwc88YworSu7iCu60Fk4aipKB/y1xoKTor95Bfkzct2HLII8d/FnvaaYIhAjp/0eby+wuAtK9yBWmDcqVx2yP2G5HojBgnmL5RYdf0izz+1d/fmunf3pobbLh9A9TIWymkaz18785bNuuOAxnN0SlzH3vAufMitQqrhlDuccodT7svN96LHXNCWxxsOJM8XeGII3HLgvOiBBPxBDrLrTawHmVpg53uGTWzYxIZN7Jnme9lNzE5B9iRZ6/kGVV0na4GdgOxPttZkq/NsLZgN0dE5o6PO07Wgh0d838xpdx8dVTIN0dEQHQ3R0cvN96LRUfUI/YuO/FnPoqPKrfUyOjLZ6jw6cofc0atGR/68Z9GRa+eO4v1DqZxwoYJlcpUSKLjspyblPatPX552EOvfoa96GgYtTqw1XDufbHYuK0sFEaVwIXp6z3IZnzPzJ52rZZ/IG2plHEY4id6zYqaB9UAyOy3QkCxlNHzPepkBcA/0sk/5NyyiwbLzmM7MeLnni4BFsf7KQPkJifqbF97qfw== \ No newline at end of file diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio.png b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio.png new file mode 100644 index 0000000000000000000000000000000000000000..7ac6de28b85b9366f1d874f53a43a875600a6fa5 GIT binary patch literal 33075 zcmeFZcT|(j+Bd3*3KoP=1f>}Qs7UX)>+?Q-|LESXYMK2TyxFU<~I;@wN)=)xOU;xsZ*EL zkZ}D|r^pDWPMt2HI0s7n$(A$0uhU-os!FFyJDKNBouZKOMwocJ;A|b7v8OnNmH&R? z6cVs^^YZ2thI0xDA)K++-kxr*;1QI&xY;<^I$&-7ekUX#B+M%)$SWXdC@8=wtSBH% z`XR(8AR=ME!^J^@nx; z9GukH8}E+&yVM5j>tGEi5#kg^0G8W0SlM}6xqvDT|KO^)J4VFN&(+b=Mby{Z1L0-q zq-*zg@2*y03V*uR1&rkt{JRrzV_%H5yO)@Xu7H&;-U{xg>xfrX)pd16h+|x=#ntsV zg_N|BIBN$xeJ4{RJRIpGh?7vaM<5J*4Ny2WeJul15j;xM%|Xyx6DwrngV$F9E&TDe zID0J-2?+sxRYNBo7X&Dia7Q}|3ZrZ!j2$Elu=;2k9hV3aX2{*xSREZ9!{8xVt@AVHJ!x_yR;l#a;qSqR`M#N8H2;eAk1E zYlul$yX(0c=wdwVY>b`2Ta1c>owkZ3!a+^c0j`TzakEBS3mcgDxe0mc>N)Bw>0|tn z`Uqz=WpQI`HG7=1uB3pg4_rV}RntaI(;e=rB&dRrG`9EfwKvcZGPU+Z>%$EVLBD7X z1DubMqa;dA#lS#c3~O&Ch(XvJXu4<$*pY}46>}0-LgEaq{6q{KoOQ5pgt{$W1&ele z2m3?`i`8^@Q$ct-17T88Lixkp9nosysu%}2#?Z!3&k*CK;iRqT>0_#?g7O#iLI`1W zJTY4Srs8&@7_6U;owtd%n6kK|tCF&wyN-&3h^Vl)20}?xU(eVV?(1M9iSfYNIiMZk z2m=itaaU_ml)tCFzPE(F7F(oe=B0^sMT!9%nYeQu>Pa9z;Cs$j9 zpsuQ+qm>96@8e>PkyJ%Eii&IL;cV44Fd{}+FA*VrQx+p38 zc&g|*k`_o^(ooDt!$S=3CyMjc_fs@>H*r@o6h!-Y8fc37>UxVBqeWcYovn3@wN24l zcFxK^)?$L<9wK@!c%+iIvZtnmqmK*L#?eko6eVbdvoUeD^%IoTwbnwLsCe6oi)xA) z+i3~o5mxpNir&UXSP@fg4K;Oj4KFKgl!LpFFy2AH+ZrfGC087fGVt5jNW@0m(A7jx zNyXO-3lyP>xPy?OzKV^FhmVGapTDMwffoj0;^Bm}LA(1|OGu#o+&m3@#2u7c3O~MAeQE~OPvO=rbD>{CB!8}G1@*pI*K?~Z2<#kf26RlqK~bXgSVcCr@b@W$J*M>#!%eL zS6fm{6KSHYjKbl(bkwyZOu$;I2_rS_e03a@lyq@Qa8FH7uxpSm#_o2a!ak~oD1S6g z%~V@l-PHvv>8$OK5LJ?N&^HBEL&Zte*jT{TN<`JhP6wy!?QQBrQZ?FMnx=Nz0yYRy zH!)ou0dbU_y?~8`umsNC1}9--Y^X+hr-#%N))%n;Lyx|Zt&tjNi1yI|KUG|H-8^(P zJZ+8eI*yvAVvc@d)^4`e0#@Ft`rx&ggsG&eK9C?Ktd5AUAl$s8>@k=p9APt|)Ylq)ZM0_6jK=*TSqzC!FLZw0IIIYE~s{<>#;Dhw2OTp|TrT znlEf<4sh!eHyW0eehmImT%>$->c2h`6pIRpZ8-MRkbixmxIsz!zrKP8Bs(a1_U|Rn zkT#e8y_-C)_bJ35u_oc<$k2a$*5gkhFeCMu=aT<5oPr}69pN=o#kv0)e~RUFG_lc3 z@-o-IC&&1*f`#4K54%q zf5@)N;`4a1VWCvgMLOr2*&F-BVv`CbD!JXU%!AohwvfkVP5bJ5b|*(WUEIZQQpl0- ztwFD>XKXiMW1?Z&SyJr_WzBqLjqA~yv)vp8!*|_3oLz6@NN7vek-KG=ANfGk`b49igHRE%=U0wzN9u3FlSra`Bz%R^f+ht!(-xCa;d`Z&?QKp>ateWAWX$#v_3nTId?*C(Qn{&pmg?BQ^5XQ*!NZ54xUvksyC;0%pxJ z5VSMg`!4_TH4)_4d}`VTF)0bMw>{)$bYNENYE!*Ny9#8&b*$j0S{ zg{hYruKJi(^h=!_K5E=eVY^PllStkB?w+Yk!nVkr<>CCGgYEUt*WHLw@LwAb59Zi2 zGpCo|uA~B%-TCe2GWaI(!NsdzdQq0VfahKhvIm-i-o`QtD#b}R#azx08Sa=oix60@mYK=)g6uy$ z)iZI?S|O)PW+S$6r-Q%w=J>3yj087JBE|TFifa3ArB3;iKqhHl=dZg`e#^y+E%V&! zc&Nw62udM`p1cTLf56JSl;Nl_Z~R=Cg1KejTFd(A$+5VtGo$cw+^}2o(d|v?D?s2P z!huXm91``C6zTy}e$Z=6&B3-#eXi<~D>A0nTh&!jR8;&7)vJ}ot={8&WD&fF-73HI zr5Ih6r;*0jy_U}bDXaf|t$sSy>_!*ke=j zCwmEBv!qeIEf?mp1D2UMAZvb6AA^tfU-y?oXY}y{Z+U6~9#7 zXP9CG^%#8P&{JinEZMprqzz^LvGEy=b7D4Ri%<6*{6U`QeIY|$^o1qPQ{+*XMdwB=KAXumouF+<-``x zlTui2T$lWb%ziXI#Tx@1StH8uT_ZK;c1#P6SwjCc-m+PGVdjtt0{7HnRad;R>}0I) z!Asw}%Op~6ao z^{iQizmv#-EWtGJIUjNHviL9R;on+zG7|Czi4*}dU!Tw}UA@m}HhI%Dlbe=s<~)^Q zQ61&U=hD1sP>pErb+g`Kqs;jUUC&WNB(c*ux&tK0{ur zKO-3k`3vf?yE=ZKSHH)mOS2_e6c0i z33No^XoXPikn7vmCj}5fzeNeysD|eA0c69mcC9Ran$!g9N|Qt{h9BX(KGAl&d=~aj zA0%zRGJfDa*A9moZ>|Pw{2C&pSl%rkyfC7Dkd-@lnZNiYrs$}mmDPZ!p)+^#BAd25 zKc_S1#%TxzKhVId`|%vyrx6M<#7I_uq^$kKoq|ly=mxjNri0~V4ZeEVWD3#d{#fm3 zaLfg2864z#N}WBd-{fz6WF9b$FEF6WL0SZ5F$ zwha2=%m zxp>yavki5&Hv^;uKQ_P=$UZ=TOr^>PaEBFBPN~B%0atg*IcW*~*FFc~JIPMxTFPX# z7t&a8h8q=ZbV~C#?B@-&8sD+lyWjU-fqWjd9tf;<570E}9tzYfe!o2C-TJjZOIk$3 zXVh~t*6v-Gvw%%}NZT#=OVIY$=!`Slj=7`Ro9?<+>fCEkuL&Z7a_F1=b)Fadr?1Lj zQw#2u+HPVBWML$#>Exxk!xky? zfRb{&uDOW2y|ptMk}$_?c{P$dCYeCAQKGu?_;4fV#Bk_&yK$pF`X%DlXEMcEIy&41 zCLSwBo@KA_Rb{oBd^zkFx-Igunp-U0FKXL(23_MV)gnT{&Sms_d8KqL>+zZn^}V-HeH8_gg?)3}!Ss%JsY4<;3F{8E3fLI09{Uo{rv8o#+@igOXyY7_#t=JN}!L@$Q4nyi&yVtv|8#&89rATl3m zV_a(f+TvwgU&EN3d}A&*ZDLfwLb+^o-YVNK5b#^2y>VApQ*vV6H7G9AZ*Rp^538xJnv= z#8n-RN!iCHdo4jPZkoupB#+-*oBUPdl7Ttfeeq=M!LK6Y5`E0~Jt1$Isfb>MA6j++ ztfNBVj7kS%XU8d<$FowTO2;l9mD3nJ2G*hxSc^{!((#$(bf+mO8XJgv-)>*~=1`FJ zlFMzT$dujPor;*kXA<{GHWIhQ8C*)LcO3Hxu(rYoB$hVOdFwZ|IqP_*y5nSa1le0&%%rke?+(HWYLH1$5e z|ABrcxd#^lHJLjg_gG;a+3;(jN4*MR)ClyDX3Z=>+^pGHD-oop57L2>^D|L z^S7%%7FvEvcFtURx0Aij8Ij3XmqWc1IuVqHTRnSL$7M47J_ErOKi=*eIC;T1E$i%# z98`OiKIjhE;Nu`@U<_vKA%nC@X;sOL&3y{Pbxf+j( zk}E2}jyzi^h_Sk2QQG9cc8uVKhuXuqPH=x(KYhxKUqTgM1YhKTsQ^I~Yv=q>&`rpO>;?M}M&JZL{Gh>XJwnSk zP+b|j*l5js{@qIxR@RMpWp;ho?;r^Sm=KU>7|HvYxJf14Oc}eVk5(~(LW&qXI0c(u zph;bwvJbH*b@kbZ*43OU2XvKF2uoTh9rMg^f#e1$;r{s3)SKAxP zy-li_Y%+Bf7p6k8(~)`~=b{oCrHq|5M?YlC8RE|5SsKx1r&#kk__$W-@lwgf4L->E85s#4aDgp}zXeZoLZpp4DNl1!4aMqDP-Aon z#Iw|d$==bT8m9cIvZ$>*{z)0k zc;rk`B*$6#+Hd@+B^w>a=NDr8Ax9aIhc2pL8Ymy=jqJPaDj(L*N##FVLPit4pQBB3 zpxsIZJun_hU6|>d@ipxAFsrOLNKH51vUOIM-e`8>c#iy6gNfBHw%@IdZWYUICG#t0 zGxQpYZxaiT2Pf`F^!Z+WbDftOZ+1gyX56A>({dqhy5lPWo@`THy0d7L4v=K3wCR%=ed ziXcwtP@Lhr7ssji$-4>Xmn6G%E}r5sLjVcA?bpK?Ln3?8{Hg7gFe{d#Hr+^MY%zB# zL1JdbXxpCqY-=3d;V-hao!`4X0Zhq#W#axC4(~8N7FKT)EB(!_>X(%B7?{P?d%t5m zHZrJV+K2ef++w4MA19$R6;mYF?Qlz$_sUi9;Jz30;+p~4Cn@8U8%sP*+!g9ozWRT; zMvTp{X;k=2C?07V&EI{Cu4zOUA1l5eADv$)Dv;7*>D%()Grn!GRj}~ZF*Bu@6KH}G zCx>(_5J6tAo8KyuPZ`rhzA4i$_2?Z^C9=Q=Q^kh~$|5hj!rO*qYE3A=4xWe`P^ug* zznm4*74O9}a?Dr#E2_Uj+VZQr(H!5b$&{rHQ*N}5XA_C72uB6>8nzo|zqKD}dBxn7 z)z+^^n{oau6mjASxs%1ez$$mcKil<6op@IkyV-AsxBgr4a&AYTxZAe#HiJd&q-Aq20PE9)hUUJs;PIVn%~c^qhCuCfvVYZ&8PR zz8ck9_-*L5?+rS=E-eQMUxNo*P6{yJsJm+B^`7~GAbxgzn)dOdlEY9ewUfhS%gT35ELLHt>I-(err9!L%t4zEqN zcFcCa$oE?rSwGz9;?6t)p)m5T@6y!Ul_rB$QrHeM8U`mPAQv(PDx~dg(4G~FXA9cw zUf-Qe&;V4M$Hk=dkcedE|H&2h3}~!#i0q>zg=^&J*a{Gs#fQQc4hB_|NY2L8goj8? znfQWuC-2y(s-@8`6x8)#2losiA7_4iK*dq^^_Dc7^254cn^MLMv|Vl@uGSA)1<%Fi z%Ot)+QFbVQ&r_qt|9)bpA5FxCN!WpN!XK}brB*<%Wou+6kom%JA(z5Hq~e0 z=WC-D5Cx)oheA${rb}u^lB9g*^Tux(`;grG%bY)NgpGlHb#CQMd?>j-gfY-ul<8MZ zp?}x&CLXz=;l6rz~uHaJC{dr%5Rq+HA2UwL&^t68MS)NZannK z4RZToZDtz^;(L<^XDN?FuQe-r%Xe;1bm?$Gu8TUtnoo`c)EOF-@H~VyGLnnm&Fh%C zU|xR+IGV!f%|N02Zuk(StG<9>amHV>coro2a;3G45oC&_sB!SjVh*`EHRG?+ru9|nOD++w0(mPjvx~KA zYbxlV5b=TIqx%utPwCtqt#>L*GunOAZbw19Za}igXo@009dQg_}F{- z7n()KxWRiacNx|A`>7KtmE$>n!&r@E6{Oynq>bLoLj#HvGBn>Ce7H{NT>N+ZU;az1~ z2{QI-lw0H4Ly%^Z`tTcMGb}#GGI5+|3oLKk841uYF|Ce}0y&lkfqTUjs{2L)5`P2eSSFAdesDBtFr5I`A$Od}#`=}~Ac(0%WwJASIMbCa_}%5JHfuPIGd$49PS z<7=iW%6e0_pQddPV(H_S5HY#mzrDinN=)qQlUK(^hEKIk%LWHiZ8&LKZ#ya{^bRy_ zuqw*DGuI1yFFh7|#*pM*630HrU4Nk?IN6V~4B|XktkUUuxjLWG6m;e+Vgez^$csX> z6JK=!@ox&O->3H(2|jSyTK;4+z0!O_%JH=UCHIomW}w_kS*iMl^24?WO4>aAeD&84 zTP7WVo2W_4Sd^@w0GM@#tjkr6orUbo5M7&SE0?93A+%ycU=NF2M^)$15qO7QJ$Fu* zeQoYB!@R&JUK|reWZ}sHqHeyfV-PTX(#EbRlCm=dBjn()Hclg5u<@D(?0Ml!uP>KmeUj0HfU!AJ~Gz zLGEYGeylVdl!~|v8q&xDil$!|Wb2=VfNWcyb!)s{kBOpr1?Is%%aLL+G0mvR_;moo zXvj&>7HbR0M;H7@r51{8+)uxsS627>JGSn}vXEx*HA8U{(W*5WX`^v&qJ*FydEL|0 z1c{%@x@D#vVud3$oH$R(J&hW_w;ox=A@oYrMcx%lOTLqXJ=XoM+A?gDVE!ht@9O;< z(BcgsWEcRs&L2O8BiOSIKVN`dSiT#vsvJf^3cHdkLze4jrZ&Ippjv~Ewon5=y&3FB z(f%no@=h5ESA8a5q8Y|4kOmBOGEZm*Gz7k7A; z+24-7F|Of5vv+8O=xkejmdZa;gt;F0HaXv|6u8acA$Q#5YV4U92)x^>&*%(N^%|GxZ&8`$zhL3 zlrVhhRc#$jW<TM4QTU`j&pV-F2gog<9%p z?$qdJ;`p4Lht2SvsE=*$Lf(!x{#<)&p>{`l6rX)uW8!Ah4_T}9A4^i2sTM5!5Dw9d&6r{azPG{_FfRFX&!fD4 zc0PqU*?pGV4YQ_x)G*t$COvP>+};`Ub45ziJ-$l9GHmkPz)P)+JpZlvZqyjDjBZ{` z%x&ZjP`XJiF;|P^$HyOo^gn9iB%b^E+BKHOb&`TM9({8^X+9{%_4600c3?(@gplma zU>koL=Z+WZqBQr>d~WVzcow=A@w{@gFe^#l(X*w)@qzo&*LKR>EB7zwcHKJsi(>u- zjN&8h!@}m)+I%8LvNbbYDGQevh3fWR$=-Dln65XEfj4H`PPHX^&c80WuzW+)n1R=n zE(l9SWVd)8em~M+)q4||_NSXNzY?NKX4fs+vS!B9tM}*#Xbzh*u(6A$qv4hURkV<{ z`1eMtSHFaM!Y8zcp=3)3r`r)w|DQf@-kS+?Y!z8;Moo6LjCNbX&H3nt8RqF1(J&`l zO>OF|2Hwm_|Cf(bcu<9X%lBfK_y<1sty)!A0H^NdrS2~t{|G2UoAm8M^NQmRzphZxz6-EMRuBeI}1DBZ=X?)*ZO{*vOlmd zaPNZN>Mx4h4?rwE?rRRR$VIpUwl;9K_`|6001x=yUX>rj9dYly-UH9+#0T-H&e`wk zc@{fX>btyimi%*TZAJE9<|lXjhdYQ+^0E23WDTCj)sgbfd$wVR{$^zn8(VCUp$`|Z zn2_#$>es^@&=k%PFv9P$6$gABDbVb&WOm7hFuEK1)UQ(%AnydoTa8<|INW-+Em>pPY>(<-W5xI~6*)POC$EX$uhv=4`jT;wGI$r!# z7k6$a`O7}M5Uj*pnk`nZlt`f$y?-3iNKuwd(*$xOJrqKSMDZUr$h?y) zXx=Ve^y6d*fE%B=v2Cp7HR6?SBERl~pRPrXtlfVXlNY@6T))mQ(HY&SYpj)X$Lwlz z%Fo^P{w%zk6gX3jJasxE0_b}rbBon)@vvi6!?fc_>$d}UUypjd`oT{hQ^d|-*}cQu zAl;m63eI1e^3$9xVwbRfXRSZ({U_o|iV6_Vg1g$Hp~0p8)91EAj)Y4ds`I>H8Ri;n z&*Qbb{vV3AGXleeZv+6O{AzM-Yy^)H4gc%HlK9mSJZS`^Sh;IBdHm&~AGR0j8{c z;9-3Is%l07`97P4@;*!Ih&o2*c2w0lZYtvti@kL^^c)Ba%Z*11kk>Yu@9fs^Ej*}P zPq{U3CwBJ80A{QFSG?CP$kHlDNqG+NB#z`>K474(%k{xS1%q7#NBV(l6(HF%ak+9L z|CS)PBAu8_g(h<;t{&=x+0@jMm%Mc&x8 z{9EdG#1=(OjxO^&;*BFuJYESi$ydX|XEwXClxaN0ncDaMMeG!tB$Jbobi|E**KN-% z{vn@_KW4gOiQW)Gz^I`YS3LF>|sACLxDD%x)W48g5|yW!Ao*OMNp05JOCq=qR* z@U(UzgS7Y+Eo+&X=iePXZvqj0*yF)v^Z?L+P$x-CvKl zzcwj(B}+S5Sq*4}$F7slHsVfk_Xcj%4Q~r&FKzReRDob6SGFmSlYw?D&#}U5<1dlV zUaV4CT#-p&tKS^{tT=NW%y93w$J-^W%+QjhsU65Rzyku+H12S{SL?;h{C@5bJ0s&Q zrA5UFR$8|qF1%A5%y8q|r0-A%u4btDO(!dmpY7eGR1?k6$dUzya%qi|w<7%%w^2-1M&wJu?2W(XL?j=B7s_*D zR8pf9?)u-v6NX&)CkOubEy%r&ybIDMhc|Tf^(S;xnKm&RGIsZ*bZ&&{0)e`e@>Zfh zkQ3B#jA41%`g|@^i!EpFczPH4o*zssIMG%f_UPmYDv-{gO{nRK1#(2K6JjWhQhz^;E4 z?t*tM3|1G(?2d)^ym2ttTM0Sg0cWM^nCdt6dzYhe%uVx89&NCyDjVi**a7Z*BU^ttzcPoeR51RJSa>(6qcf*-~Tpn@m7J#E%Do()8X;P8#V%J^19vuXjm$rI7=iz%@4>u6L}1R~^R( zD>+5uFqc;T&L`Kb_IHJzkCuFi&elOiRKJg5dgaV}s ze~e|OaP^1(o&33p;{!Ct#i3lq>{D*o3FwwNS>+A zTfm(+Fk4hbk>8$*8bH@-hFp_JgAcTPe|f0ZgEb%lijtg4>s4B`LDDWC02w^Np|?h@ z>#)ty{?`2iN7IzF@Y9=a%pSzW~3xFGKlL_*r zn6p=E@Aa0eCa`v=s4%19lySMn!oR@BF*d!=I7z zHPg82G2cxEETmHV^1H z@AjYQ7_&;LcIx}$)g^m1E)p9Kgo@Mb6aL~Q)I`Sz1YGg`KrqFB@J3rJ8 z!fzDMqgTQt@WsYgVI!8PF5ylIW!b;hr;>HPd^pc`2V_E;Oyh{kLe!!DZ<%%^bMI|Y z?2^S9v92jifP!v$R}nk<^Va6F8FG<@_qX+YJs30zl%L0(yJx6 zz8C1AbU0Q(f~%(p#lPiBEWi34oU1*Q&6PL~1P2i5W!VM8&!PAB^HvyM^7__Qi@);! zV<8mx;*OF+9f164ZBVo-ll^kFy=0^I-kZi2$ON(LyW0C}US;crzX3noSIF(L;y!tPM~F;1!srJK`+e+ID3Bx;j9KHvk4!Kf1dwvC@Q zqeQZ*<|c)Qo0-`tcS%P={5^}20RlZB(l^4Wfg^S)ELj2$0k-i0 z%bjED_kPA1rp^RAgFR1k*drb=yLxE5FY>)I@MB~nI$j2GzsGH}NJ_)PVu6ydjfmq; z3~dG;%YH}3JG@5d+9I!IjW$_5u)66yNzRevGBDs7Dc3m<`jN%jjV4wplTN1*7?pk; zgmPB{_(08#0~xjCj9nw^*oFTY@PAnPKSJ|=!nQb-eETA0Km1u<9+At-{vxFb2+F$E zYkO`2^rI$)`vV?YN0~@~4>u-RIR8{%fZWa_rF2|uWWqo(1{5<$r@bTnhSbBO-4FEq;jfH;-T~0YpI+ESPnW`8+7hpU=6lMaJFwpgWe&F}A1Pb_ zG+d(j|I~Qs8wmrWP6m6wWcxR*bzt_h*XVfwBl@*Nc?dND>DwfRlQL>7ro|e7;i7=y zMZ+4CB!-h9@>~MO+<@U}fZ^|&^09!dIbe`AbC|jS-Q|x#aejP08Wb~;k`NTvb)cX) z5fsN+x$p^T0<2q@`2cVcYJC6kv*OMgTVze{3An~5WALF`271F|z%u&2T zB@>XmRnYQM^A(5FfQ8Gzy#0>nGy@b;NCG4MNczzyFt@f_;5l=`#v1f!MT+HYS3DL; zy^}=6Dc3mxV0GS*jPimMRVEne3>b-30hc1H@@66VTL|*x>YthhbTwf`M{ZEfhb;?Xry^Tc!};ANE+!WkBR!HN5+# z{h8lsf)Vn)9obinoUkfsh`S=FNtUDv&K70KG78b>f^I9xXp6%9Kq~xJ~x8 z!!V*zT03!;W$1#L?O*4~vxk!u&XT96UAh7hNI112B#{G7x{29}Y}7bkjlZRtg8p=8 z+K2iWs380HrTx(NRc2tNzy%bAR}}6|mpbp#5l&aw{oU0m1Qdv?5+Za1>%N?eyc8uH zF2}yQm`d6yXJFK~$ujxFy{;lK3X4per^D=O0aw}>*+CXzERZSlEjZi5bUnL5A@dfb zP)M65BKp)!$Jj4KA|rLcxgdqH0bY<*0-K24kX*v@+zekdF|=QJ_RoPJ*nj|@@f`Yo zPe{T~?|-K7|7R6~F!FxO<_-X%{-$>K85@XMW0{0M7j&0j2U>;to`nz<^1TcYIB<9$@*n-Z$FZc@&BVLTS4j0qs_mpEdfl%U2&g z`+^o!AcBsBV1E-JOd_C)ZL@&GuGSo}IY7A>Pd))Q2@?c%kM;C#?5f;z{VbHQ8=!ZSzkC015CIiD#U>%o;z2KX>U_Bd1gR8?ZwD%P1N0HTMTx+iI^2VM zo%(BllQNGTG>rO3!+p?@{Wwt4{*Oo+Du7n2|7f+Lb|1;?qm)0Ul02CmV|SD_$uMBf zNZDVW-_>}@ddlM~J&6(YY?=Ju)>U<15V|_@pdN;4>Hq*%8NNvUZ=gcwJ@N~QZH-@7 zOlsQwf9In&cxE&*DtWxjlBTh_$VF2%6>pDHt_jXA9HsZ6jx4os0!q^MjiDsK#cjA( zOlmrDy|A~K4-}R@`I5`Mz5l-P4M$mJ^~aIMOLoCR{8e%G(w%;s5LMrjI~>x=uvG5j zk83-(TjO()Z#Z15DnF*SN8z6sF|OKt0?hdKR8`XQS}ObL%dJ^R5(LqEL~shC_gJCW zl$b&c#e7S5EqM}~p7xnYcRooLaDM%${#hQ!Ovj@2;6n!U7~*(qz{aY2{_s${rF>^h zWkl3alY1t&a}h&aE^;%%x$auEgAm5zA;?XGO z<{%HK&h5URGjugR&E?`}wQ4bwx$F7xNlW=}1|c2T*XI)ZAMz0DZLWO=L1nCSyT+7^ zC+R#vfWKo$ex~EFGmgE)kDmljY?m-9uVu<7-XK1M5H7MqT8o-dF2d?Zr6Ch5^|$S5 zgUSJ>tcZ)_H#Ycu72>BbKW$d~UQh0+E3a>r!Y5WIc{*0LQ>p;y&2&g8v1V_LZ@o2P zBd>F{M4jC=L*8O@&4>14-g`Y1Ti5o;B?|cF2LKTTB&DbAv9#8pyBcHPaVKu%yU?&| z2bf0_ZwnQ_@Zzj!RNWVi=$}m4u`e=Sy}i<{UGoRDSP>VCnvMFwRDqx*mTbgud@Nr= zqQ#%H`-D(pBFnirZ%h!_hL4%_R9TV!rnp% zpQNx{5J5^SviDu`A?XENkV9`Swc$jw$E!n9WhNH})vcQ;C{alQ_3{fKlf5HwOnAz^fYd7BXUm=S_XiV`SoHJpcBJCoxro) zr0h8GXfC1QsTa$S@J~kfUX|4H^M9u$Wb!{b&joP@OJ}IIAWLvuw-*Rks(qjI#DivyFAT*c0c50^&7D1s_fJW?rCHJW`826#hKD}0TArMcNZ(?p z@{l!W#i4#aS{m?3X>}tZ-$XX^S@6r-TcyKy%tqjzbUddcLyF4Evgbv7%==pFOC7Oqdtn7{GkKnfWy$H3LD z@uP2vv!i33HNzu2vJqa&y^w=%&m=!>w$by1Z9d#=F7Wk6W$xhzmYwNuX>0h`Gh>Du zZh;o_d-|%+>uD-6nwL>*@ZwBF z9|hXOmt?h#^HOOT~~IgOgTvKf6Bp|AcgJ7gT8anzIrJ^a?HMTCEJY8rGAGxQLp>KnfFU(?z* zgncYN|F#%}$ zSg0=31gOewaCm^cUUd)x2F(DqSd`u^;B$~V8mT-RH7E<*O5M1sR%3#tibIXRjzF>te# zbb`{?bjyuX(1N`)L!7?>5%)QZCoL!BxE>PTNoMt`HUj`=^(0psH_Q_V^usjec0gl& z4#>pM@txN!ZfU$jBb(#d6bruzykglm5zJ#lts0zG7-SU7VZiu zqwDvN+l-57@Re6k*PRMJ!@ z4H}bBrqGH2NFrl^W^MIaH9_qx^qRepRd{hz^IPW=K97!Tt+%C6wJQ&hu{lvAxXH^7 zz#+WZ%ocW|kcTd~CrEGBvQfJR|LM>3 z2Vda1a`DzTN%+ocjUj+el($prLHRyg?xf{>aT@%sp}+qw0157iLFBq#0o1ZVD}Jth z@@h)LfzvTnHvJD5ChniT_?SY|*tRp1DJXQh{1a6&xJ|2eu)VbRC_173-ZvY=r)ljL z&!2Bl0u_i0@A;+7*TPoTXV>sWp85M;T;rescV%Tx@w=Dtjj{5j8LBrSF_lTg^iw{|3&^~R#Qy2P8a8_(S8e!MS9o1C2>_l1@q zaaxBTWyrotB{6o&2vKIDaLM&oo!~y5aD32KFN$=z9OiN%VSQ(52=!7s>zWty4jhxg zbfqrSchF%zMe@!F$++GwOck;8Ewb%4emS zj3;!BW^tFG>f=Jz_F_jpj_z6b(=J#!_P%VK3IxI=q&a{(Hn~e6hPf0HsP=bXI{L+% zkA*N@qam#2e~{-$?*0-0g?6Lei%nzN0?yyqPwFMe+oiQH7|DVgl&v0@XuJ0Zuuw3= zNg^(bGYd6Fx;=ltSDua+;$udQ2S=``=G;zjy|`o~D7~0;d0xo+^CkRZMeC2a`s-rO z$X8-EZPZT$tnM2Y8n(wHh1)LtQRA>z>1pRUQ4Gpf7SZlanwQu+)6g&@Ds7)zkc(4K zt!Y=Xluy)QzN}o$m(-36Qy(jiR>lcge_B+Vj(*E{1NW&R3Ta-+{^9Vhs|m^qfU3bo zx*;Ms%5!;uT<9xfqj%9xjQS5iRK9(1`M>ulvQFOPXEAymTwzw}s^1=T!Y=k#+#0&~ zV3VR>m`RX-9RC~ZzvkxX@;m=xVjF2&c2O>>lLY2TY*Ns$P8Od@)^#rVE2YVsxECEz z9s~eT@WVasgpE#Ah<6@EajcuUIi83xWc^=&Qg=4Js{{cijV1{h_ z*>%_u8#QB%j>hBgOiePjB(?(eYx=u^fq-H&7-I~0;%oq|=n&X*&>aTyeHVCn9V@=U zMQT%rDrwF7{>72cvLVraJC9v7R;MZrMpB0N-wwq&VULQa@0;Yuq$Fco%ZD>@!C8FN zgin~jXSEq#TiL|kB2-84!BRm@b~hJ*qN<<&U!bU;KY=4XZqgqKklF@E;a@tXSDa0M zMfl5zX{jVOY6d=riSg4WwEo5}uB~tAmyK`-#||xh_TgLCFDq`%1~HzG)U@;J_HQgY zc+Mo@D{xp~@sczfp!$!Ed|$)>z5>q_Amh;#E8E%l6>T5ca03Nho439=HE*5dCf+wE zWlpUrPo>#3{Oc77(aN)yP29BxkXG$Kkk;fMy^noDs0l+&S!MGJ_czvde5#U0lcNY# z+9)>!^z(f;lMeyZdK1kxW#Gd8{EajKK775y$;H#Tv38jkKohBa9z7Am{gG#kjnSsh zldl@zg#>+NzG~2Prd3#ab?gc4W|Aa&dj3T+pa-BuDa+W(UT1h`)On2d^nQHn&&BL! z=)bu0*1yoF*)gDo=EvwhZv+MtbhJlt_(uS~~N+0OL z*2G0U+W#@hjBFd2Y7G{quN}XBBdJRS`4QhjZnpoH?@Y_&*J|%zQD4nwP_5rjSE|3Q z4V(WYokdOKEN{-j1KC>RI)W;m9As7GUJ3c$nmWuxLzsO-tyD3}qrO{`#Y7Wgx~K?u zJrMKkf~pVGKBR=-z5xD$L0FEHZK|xmgbZ7&M|@S@hPQC%&>M$j!!irmeB`f2eAS!( ztGu@k$}0N1#T8LNkdl;=ltw_hrAq`PBpw>1LApCdkWK*sk?weCk&s4Gy1P>v?mp-{ z^Zm`-`Mr1UU+>KQi$BF{W+O6qEU<#wmKHkj8WgwQkSM%d*aG(f2K8GQO*e}PY~hj@cgd(s zZSFaXWoNQ_2Ob~yOB zfR4xrCV^?&C`U@)(;Y*%Bz^ycV!z`@)(P?GCg+68!^`JM;LJi^d~#iz=E zQ*E~rvTKwhVK0~$xT{8C`%iBxEva1WCI(4u`;Xrp^Jv^DN)_aBEjk>Fv&QQvu0E!( zo(`RmcEt8yef3XT3rb{GmP4*U%Ze12#Hsug7R#_ddt*yVRI2d|5jLT%Wha8TJ1SH_ zGfQQjJ^R#1pJl(nBC=nqRxCV!ZdXw33Tt)qMwEErglU$MQQ*tMpCoi=#fK+mlug(9 zYU@0mo0%R*ouKRkC73;ork(HL3nPoIo234A+iK=@1MGx+07JRdrRMdhTpDAC0@wBz zFD(Tq`VFK& zPKi3G74bW+lQt?I{)j3nV*kuX{Wn?Y-d(HX7X@R_R&??XyWcZVpm%q(OnWuM#Fd(U zVjed?I)j@{7>?)p8}Xb)V1TRC3UXPSC}&_nm9dVHNxA7tX6f#ZIK3QR{n;O%+b1bV z(SPJ#V>a=qY=l*ZMoWlWCy+U)o}k1$a_nYrX|lIZM5Kh=Tk+|i=RDApV~#qX4tnTm z1?hkF91Kcf4vbBbtEcq@_>*Wzcax!-j$1!GQ%)Z6erIeBF(rYsh*Kf0lzH8_%wid& z?pLXzBQgjN1q%yYshyg4EiP6yT|15O7oPJLk&1o=DqpzzLA*By1f;$E#$I+?c6I|5 zF+LaB47ZL3K`UMo$Y{t*x7HS5I;AiqMF1*J?XM(bO@$4L1?#qKVl1@1g$wd%0kMuo^1Ydq?E zc89-Fjg}ddhx;uao_~|-2}~zhoq(XqcCeJjUC@mv99Kr>=T{Sno$=tzz{G)mrl`Pr;0 z{2e?uIuDHAoz&Jhu$)rsNQ-m&IyZFXW1o;5EOC8$xlyuesOHlkD{wJMlQ)qeWar(uW;evsnp%ggWE%Czme=>44KjKmay$Xk z-lsA*>}d~nyF}^i;UHlP`-Aj!^>Q6isG=wF-8SZ$&9Z(meR1M`7$AP zlrijnAonzmv3cY9xP&^@vb4qerW}0!nks1>KfFs6p-QV2d?>0`zK)4?djO@_mzFQC zQlQriNmAtg9neENpL0p0E6YErA2fSy;$;kaTQ7!Kh+~5gEoy?nhe1WgL*x{<2S^6L z3=%#51lo#0HsckE{|-o0{GoUTL|D#MZRkx`{ntI9sr4>fhnpS)W$ZmxbsIW{c5k+* z*zEz~m;vZP8;}JAW4_E3+{Nty?3w*<5o(CbHQBr&#V%Qy=4wK`-jRll z0jB`$z^V?6%$aAR1b#Da^fkN9_I*}N;K@|~d7#pNt9>e*eHWyuRDeHZjbt^9SKSf$r zR}#>*Hk5ceFUn9Da4{fc>restvF2$-3lrTa>ZT!%L_aG_%Ub1JyM=4L4K*u#tfXr7 z`2!Dv&w9Kj)pB5r6gG&AutUlzyG^dY_jKGsq=rpc_ZI8<6@}^f>C}&Jf1kj!%;&D* zE`$=r_qmLR76TYUn%e?5&VV8^2C#?E!WMvn5ARaek#Uav+m>LMNZKInJ%)t7aVECM z+q$sYW7y6{A^}pOx+EXlmz5pjm9KoSY6`?76J(Q2tByep)hU8RLe&ZDW_Q?l;{xh# zNP4SKJ~Ee|>>2sO!`(_i3f!wqG3gioW8imvXRqWDAeyb^N7Z9ao7$NZI zleXiy_-LW#H&yJW%LCc<$?9UL+%5wi#V)4mDGG6M@dF@s?Ud)5rUGeR-cum+ZV=@5 z)c~Sy8ORdjfWN&1B=kLl%EuS)ehJgaCs6@iX9m(J4X5cu8?~oD7;A;m(2$9A?dc|o z+R@089H^TJGua(89xT(j)`bPKu$;}w&=4+jo1}UCi3DWF z3$(M*48`?I8r4=zen=Q`%iZz$s9YP`uIpdf9Jkb=usAd#PC3_;9VWo=W^L81rkZKG z0xd5aP!XT~Zn~xhg?gbou;dTc*qNwcP+(`+fN-oBm&29Lk6bgfb}K!Rzz+@FzS)pi zxXNmpek4z&d$4x++l7Ozv_RAn2Y8b->;CID_T(kjUB<#Ddb*D;zt^Mbz?7)6@fT4l z$xmyYcA4So^`{FTARRBDoHb+S>HurG3{yuB(~?rEH#{*aAq9$};Xvl0i$gZ70mLYVy2Vf?V*D zq%LBGFPOF9uhBt1U@p+*#*KazoW6Y6m?*TEEZ#Uy{qFu>>26FOFfNHx7 zir)#>quzu3^+nosU6$3;G=TYZsYv6roXBy7@?~z7)qtkWH^_;wd{=P-NRRHfCZ9p0 zuH6Q_4&6Yg8u`^|=M~rAw^A`%%D!9rw*f5BnoZ{SPnzM?V`>4G=PWJp@xYhc<}9}j z5%}AB1#@~T*82->P_zPVWbg$BK9dF&OZ}cN?i07qoe-qj_3^NQ76bL=-t#Wtkfs6jeuc$kmHbFc-cwIW zK1hfSiSfTY;ycw>akX=m!a_yUNIV1!KZVWa{YmXeu|rk+j#=`~}4ol_neKE4yB zG_3n*zozmd_=TxA$XQm5PE{GF+{RLpdSp);b-`$?jld%_JCCf#baxJQ1`&Q;6a|P2 zAFQ5%-aYb=!AwaEI$GM-3Zjc4Mi#h>sba{D z{}SjZj&)@~t^s7;$4)vIlsIw&p<)-pJSif(s0qYQ3zGb+&K=Uphb`zU)Q5u+>Jnt0 zt>t7@r+P?-gO|?p3^x2-oECR~=c_504d;AGrEeDkM$R((btURM$uCT?hxle_B9>2p zJyoLZb5V)owuVC*`3{Ty#U2W&u80og?OoiA{>z=$ zZ-49YyZGtTb&M4w_(9ZUFLQM8NYvxK!-c324Cu{G{<~%c;xf0FzqK{{Ak>qQ#QkK5 z$Bug72>Ids=?{V<^41>9!son9-&xsNu6#f@vl_?ZzZMsEU=NSfvu)BtJAYo=0+z%? zU^dZo^UE{2wZe>A|68pd3IRdb|Jk}zDGMR9hGw`U6vEEScb$n@X%Xos@HnO3|KX?fP2JCvepSv=_S(8aww4~9PISaOV!bHd< zOrXU|S#!(A-1_yg9Hd2_I)}yxSe38#+GEAqtyB@^ZU=Vtp6TC3ypuL86}*81=FK0+ zkDTXa^leJ@Ec>DRe6OhHBN5{Zn?P z5Ul+V&qmlAqlbp|U6pE=jiX7dER7qsfFz-OadpLFAXH5i`%BVHGhnwR7s(pYnIgXK7R`dan0|c4CT^?@% z5!XB5xzYxDa|k;6iO3*m&6}Wo@=gTmrQRb{$z}4pnys-+j?QaloFo(oRjuVrj{dav z+;1=d>U2epmrs`dd+G#2YT{VX@`t{BZ}MiVUk<=EjDOo zuE?4&jf;;ARaC|6DEfa#XVdNIymxnKu?_rKA1VHVBn}q3{~jYz?^Y$t!$u!5%h^rZ z5%>vKBVL;Jg7Hr`_P-tf|3~8jRSnzk!y|Jybc0_(1BBUjj@`rHC!8R9BXBE@ulCIR zi`IndZK?lzzzUBd(6Ug!y|}Qx=by_pv%%9Hok`zFgp6c0$3%bMn)CUO3?X6D1;`11 zvDp03-}kw5?}k(s}JiOSdaf5f)V>9?GuLctMGcEIXYAnTey(1HjOxJ>i1TjbgZ z@Y~DQIPP?kL}TB#tE?>D{jx<==LGp6VKTbbVY3UEBaA>QXy~7?JB3PLL*4&%V3rKU zfa4?%3ScG=B94mw(`DZJ7ddYdLzCJ0aYFn$d-or-cDpJ>JGaIV0F72ge_ClQ%4879 zkd6+BG?|Ui=c*OMe5HVKh{Xu>f_poWX|ZecBcp}8Y$<6JK%Gy_SLeGT7v?8d@Qoz# zwVf0Cs;pDW`?o;+cP|O}TgtSY=L6mkN!*<2X4(Jot4{z& zbhd^(UIx*!k&s~sU@3;Swl=6c&a;wWumrw3p2)gh5n4w;x35x{?MLVWFm@P$%%k^U zO^^l4nD~d*c$lknAO;z;(A7~igg${&4SgIS#gVmverP3B_t+Z943bM^6Q@_nqXNJ$ z5sl!vu;=xe3D0Ih`O^7jaeRc}xm4k8h9;@|p2#?$ZJ!)=L{g48DlmusofAiqDk$S{ z4$BrgfQ(p}FQ*_C zsSApzg-k~XJ5ZAHF*AG)0VJ7(O-K}};X`R(fZ(5$Whmo!eh4tGP8TUB^Kzw9GpV1V@C-_-UmuG z$LHX6oo-KwKYowf4+|fVpR6hKb1~I{9i+D65FP<@4PEKU@oyd3#rBUQp)q;e*`4Ha=oD<;WI|H#$h9JDU%((Z-2$?DxId*AM@cwOwUcc?3>92GjvlGxZ zRi?0oa9TT|Ne9$O0=mFxmd4YUPdF`d{7soNXYyAEas=`mP{Br6hsL(K20tcC` z;Ph5BK=m>~WX&`(Cukp78N0!Sl2T%mOF4}j;n$q4B&$>c=#W#mb_4v8P~lRk64qG4 z8{&&Ahy0t!3NC{3Hm2$NTvlKlEz%h>IgPYIu==y#h+KhlMz2;3s5F_k{?t7J%k*+2 z178Awrd9@h-L5sECM-YKvbVLyo9ioBSZboyDEw@6$Vh2@_G%&hq$<1E!p+s}O=qG> zDjf;0{imT%Q$^43v%pl$gI29->Vewez0#b`mI8PbWMYQC!cRSV}K zz}V!AKO^FS1q@!o?+~k{t)+Al^Y#&hndm1Xr9xfadLDrYuk45jm-RRx7$#B5{wbh_ zJ+z;lxf6^&wGx?S&n_P%l?>}HWv*n&*7UQY!hKN!mS#Wp;LiGpBBKL+Pu zLok6TTf*LVQV+PEx`BSV(fQH3Oz^_pI8ak-?L9|TWlQF=W(G;Xbx>Yf9_fxYF^HU+ zFir*CSuOhk@r^II`~?V++(53H2&z5THPD^r-5|D5KG~fW5`9woEr4~+P9NU?5dt>h zS`*FnSyJ{yPOy<~BYrmy39jvxLG+cc+Af{3V)%qv)nF8#8dT<;fCzL^lxpwqf)n~koi^9{K zfCpi1>s?}|^r`K^g2Sju4|%vo1+7gqW1KDx;tr403)J_<~iA1Z@)?CwRuK@1bYj33(D^L zp0MQKCW=(9^4u7c|0=OSwA)2e`1_GFE8rBDIC!NY6fS zJ!P7$@_!1EwKNTo_LomO0`W011m-2{((8CB^1!^9cw9#Ly}lY@1H9FL@u|%w z*JwFN`pkgB8>Er{GL{eMP;3IyfZ+Z==~67U|2tgzKY_BH)1XS?aSsngWTVtQAljMM z@Lw__2VQ8-@iTRL2g$JFy^9u@i*9GXQF@rv%qp;Q|<(my)d;BcT+$BrfY6OB0LFh-FmX z>Q`XJg(Jg^1uz`b0JLK>Ndot)3f2m-sK>YDs=)wIH&{dBG>wu=?q$L** z*cZg_N#Ld@9f*j_GpUZf9KSmt{t>WJX!xkW+kOpu_U^klN zlW;9`A=&22MH@{JIR~h~ZU`hnV67T7p9EBE}-JV$i)nAxEoI-2@?3Chh zQL=^;0wNabz~&OPm^!7(*g3(%1S>z&`K2NWa9q+Z9*JUum*Ni>Nz{mQ)&CkKat1j}}A% zX8F}Le(;prza30u;Kzml{sQ7l3u2xO^4Df`wK4@SLAabe{5N4uzzt}BYl6L0&f61S z+gMHj2K9bTnEAtl5|ChU0LELcqs`TK4r8D)j3aRzBliiJy#7HperDpMRuks3?sapK z=6=5Z&6gMu^d4C`IkfYtsE=2E+H>5oB02-r!7%%S5>y^^5h&~6E9>j2je8`vr@|-T z=-kc62YZ0}j8lWT1WfkAvZ>N_0A@J=xgFH@b}f&%IzX7)@O>-_Bz#5W6hLXzS+4t1R2O0kSx=~HNxDCg ziyC&t(9xG1XDwNfN&9~XOy1;jC_~ZS;IhS}0YdiD8@9+`)DuVs!fxF2I>U7SN5B9o zC-YpeTQIE&6ad=ggiNbF*qIASj6pKoSO2*3;XD;tQ2z0N-{k{8*q>)3HRIG5S1^Wa z##wY435|T6u|)4+sq=mFef-wTD{$^8P#y8jKe>NP73W>|@oq9YFb!bjueo7^jV$hS z>Nrh1Z1_>Yxl!K}dhmXjyyiz2;7l_>fUU6kuYw|}sOX~47sDh$cCy-4yk(#Vt>0sKH z25|de`m;a9R9xv#D~@SG$Mhf-+)i;`2sr@MXP$R^>)?OwZG!9mk{jfc47Ylx@6I+g zL4hxYH2h#@%0q7*12u2{HCY;>dZk#D@KcDGm#4uDriG*2K%Dsv8IUKRw?=gCfKj62 z)#409X7#kC_B1al06}k(V>jRh-h!AMyU=&5ehgMWGaf(KUQnh~6?8ZQS-K-;j1PA| z9MJyquo#U=3B?alujF2iQ8ong0R6Mo-ei335W9AP*2f1#wh=+F*8ovdf@0EIg6BRK zH=vRo1iX>K9v?;LM-iO}MqqlAxttfBgEo4(b_6A)444S-`=Zqmb>PdPW5Q1{gPTOw zGPH5r(}(%U-hBi4zZ<(H_wu9FdD@Vqw5Ni_wDBK5ag-_X4G~n_yIBn9dvzuB`Pv;# z)G_uR0&JO0H4ju$@$bU)NiTt%?so5)P$r!E&%(v(1wFs(1 z)%*-v4XSLuI4Dx4lz_`BMI-muea7#Ik0-$^yc_v#yg&z0u9FUwI5I?ny`VVUG~>F% zV0T{0jTtdkCEnO3gtFaZ>8?BUIC(M!Q0vM^v8o;T7UY{5ZJDsQ-)Ex*;2Fi#aQXYi zwZHD?XJ7^rRL_Wf@4~1mBj??3J^}$SA4Y$2yeb~7!wu=fNP!Cq6D-`zh3zX;3pMF~ zx>^eK1{0TdMXvIM7{S0&Q9+34;};J7%!D$gn;}7M4>72XV_rfSu+zkBHF$J;_v1{vmR{ zADC_oju0X)KJH`ip+Wg8uRzYS?KTu56mx3VNQ!E?=!g@nz;tdSFt97*7d%YHJiu8M zXRC>Xm_-yBjDqqT(WF`Cnc$M%hfRn9D&Cg))?_2;c>QtagO&u^`o#&E^W0sBy7+S4 znbyk?UksWK$=jvDTbXL3jndMD_xuhSqXQ|Q+Ma`Ef--Bh5|wLA8kC-dtyMesBlzN` zKppEb;{7jR<#e;O`tuOa*_zzVq=Jt0tU~PFt&>&dEGRljRg1aAH4R#J{OUd^` z&A{Wz{E6Qn5XkgY&V3!HK?C_qaL9{B-6%idVG7piQtoPkM$n7pSO*3Ow~Ytxd#(PR z&lrdH6A~Tdp{Msbrq*#gPn9`k8QOH~gmf|Pu}6Nu;px%(kP7prc7T1Z0mC-nx`I;! zaj~@wN}}MPsnzyEZHB>SPw!!M*_=Qq98ZrQGwA$U+Jj_Za%FkiE9?TNG<05qieh#b z%zX*_ddEolPtJMct-RLD46bxhoFlv8v9c`fx<66Lv{LS$>}@Qn=M1ofv?E15T?LD& zH(p4>KnD>88N88+gzF%5W+T7aGHix5iXQvqjj`Z#O z#K+%6UX;1Nj-;lQA|yU9-}(3^s*yh5)96|*ve^`SWx+vS$BZ&6on z%d?*Q2xwS%{^Hi53i%aM?z5>ah%@xK{&k0>xCL)@u39lYUu02=;o`yvWMU zh-i;0`vk*Bo79?gw`^|+!?V#Nph z9s^3kQousx`=cV&OPKQ15M=n%3sRim2?0&ytkN#&d7*0=EnjHA>5EM+6$wMmK>wPS6 za!Usc`ss*CY#aCE#XjX$3p2QHtG(8Wk0Fd_Rml66pX?VA2KiOfqQfv z+tML92UDr?rL|xbL=KR=a)CX*YoW04;Lp2XhZO2BoR7@%@5Sb;*SbEa;uUu><&k{w zLgWOHLki2ZFViolttfLYpQHgF15(!QYWj!6JOTduQTk;K}c7M%+hsBsY}OS)Ca=E82t(^7Rin{jOQ4 z6uFav22!$>gw4onnLpd6S~4ksN6_0=7~$rMSOxY3_`vu!l`PhzVE`eU$8KUTlS`|=0a5~81d!RDbFK!{cq zK!0fS^Wq)FiH|+PnZ^?n{|aLlihhuxeKnihQAejJjw)Th1WQSiE7a0K?zf&!6+&<; z%gyBt87Xp^>mc*DGxh((l$Xw9D% zbAPmwh66X45IQ*X zNjtsW0ofcfOzugu5)(;aPPH$EnL4?iCHRE|^R9dFR|{P~5=ET6N@3atpHF5{oWnA% z!BkzU*%z{ODQk!Zx_yh&bF6Q?6bUp$`LTXgP&;5h7JSc4f{3xMZcsQ1E|TTE@a^!m zrM{Hoszh;3T?09STWM}kBOF_sQQ`iD*bg>SzmU`j^zp!cbZt>MPKl10GM>f{I}Nd9 zeyqEPcMlR=?%7r%zk7>D+@0S)r1V(F(6?5{P~q!ckHl|dd3z=PbsL1ZFS))&77SC& zn@5VB)GxVm5eItY;$n_<#hFhFNvj$@Hq^pqrZytHA)Ix4OBVm$+m33WzQms_;e!o& zZyKlj%l(zY3YriGw9{N>sxLOb!*y_N3SQIp`)a?!Ej-BCpBTw5N?(dUHx*p^nI-6H z8*mp6J4SdYG>IEF!B1`D-(F$Jcmsx8j4m6E>l(gd?|w~_Ax!i6(T%!t*=kD=h8J;m zOvD>^J#URF-SvjGHwisI+6*X?huIzZ(4prsJWFhzuHo-IS&>v4hM9zFCQ20$u%2V) zxPYCUK{>da@Ohm#a}^k@$-W^=Ad^IpD?D4sEAe{r{#~PpF;fX=)Ji(e@8!E33e<{$ zS>hkX?9eb?twf!-;d%sf{t}=V8&Ff@&(I{5b?a@Iv6VRU;O~s9vS4DniO|uS7g^0> z#Tx>z5cBe#GpSZl9YW0?7;{fiRE_+>3CwGI^FQvssc8@h9|T*RI%WNFb~4%N?0ayC zFG!A~l1km?_?gp~bKolRX#LJMDBxa4x2IVvPjH|JL{yG3`*c4oNB$*85zLS0Kwg*C zpJMPHvk%XA)cR^_?oE^COgBgWz^Wh#LjOQ^n3F5z`$w}(cq_G>Z`+8Tvd>38)6~=C zl(z59ydEgs5h299!N;ik7%To$i=tL{zLQuVeZfHl(cDtmW1n+RW9j7GHX z#+iwe*(-U!Uh`cmb-ld{n(!0Cqrcu(lEu6r*n66ihX8P zJ@_@FP6?hyd~FhP^t|z+{DpiOB0M>KPWkfvD+-(n*JU31U7TR;v*}Zv23LoJVPx`XXJo!- zwgDcF9lL@-s~i;>QR!Zb64|er;~E@e<#@t3p{Kfj-#4EsBe@2H~j+P zqwwt?N%3^)1;tqlA+`VfRPE_6s{Mz1=fe`|Snm*@_}u;p4;7AX&1@X^Ec(w+i-f@m zaOaM5|DEvm4>7I}O^BpN`IXjxeyZgIPQdVeQ<>=R32)EKO^+at9dT}aQSZvVg$ ce7x??(8hn5XX~N`&cr*?;;+PtUh4Y(FYCNmRR910 literal 0 HcmV?d00001 From d88b32b4aaee223f227ef680bbcf99708a08ab04 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Thu, 2 Nov 2023 11:05:55 -0400 Subject: [PATCH 2/8] add missing tfvars Signed-off-by: Carlos Santana --- .../multi-cluster-hub-spoke-argocd/spokes/.gitignore | 4 ++++ .../spokes/workspaces/dev.tfvars | 10 ++++++++++ .../spokes/workspaces/prod.tfvars | 10 ++++++++++ .../spokes/workspaces/staging.tfvars | 10 ++++++++++ 4 files changed, 34 insertions(+) create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars create mode 100644 patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore new file mode 100644 index 0000000000..e0cc55d252 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore @@ -0,0 +1,4 @@ +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf +!workspaces/*.tfvars diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars new file mode 100644 index 0000000000..706f9b8432 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars @@ -0,0 +1,10 @@ +vpc_cidr = "10.1.0.0/16" +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} \ No newline at end of file diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars new file mode 100644 index 0000000000..f1fdaa1ce9 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars @@ -0,0 +1,10 @@ +vpc_cidr = "10.3.0.0/16" +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} \ No newline at end of file diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars new file mode 100644 index 0000000000..5fc3593f28 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars @@ -0,0 +1,10 @@ +vpc_cidr = "10.2.0.0/16" +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} \ No newline at end of file From 5027b7cb22070bdf64c1dd4836ad723dcaecc789 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 3 Nov 2023 00:30:38 -0400 Subject: [PATCH 3/8] update readme based on feedback Signed-off-by: Carlos Santana --- .../multi-cluster-hub-spoke-argocd/README.md | 50 ++++++++++--------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md index 21a8a35b3e..302db6d9d2 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -5,9 +5,9 @@ This tutorial guides you through deploying an Amazon EKS cluster with addons con -This example deploys ArgoCD on the Hub cluster (ie. management/control-plane cluster). +This example deploys ArgoCD on the Hub cluster (i.e. management/control-plane cluster). The spoke clusters are registered as remote clusters in the Hub Cluster's ArgoCD -The ArgoCD on the Hub Cluster deploy addons and workloads to the spoke clusters +The ArgoCD on the Hub Cluster deploys addons and workloads to the spoke clusters Each spoke cluster gets deployed an app of apps ArgoCD Application with the name `workloads-${env}` @@ -18,18 +18,11 @@ Before you begin, make sure you have the following command line tools installed: - kubectl - argocd -## Fork the Git Repositories - -### Fork the Addon GitOps Repo -1. Fork the git repository for addons [here](https://github.com/gitops-bridge-dev/gitops-bridge-argocd-control-plane-template). -2. Update the following environment variables to point to your fork by changing the default values: -```shell -export TF_VAR_gitops_addons_org=https://github.com/gitops-bridge-dev -export TF_VAR_gitops_addons_repo=gitops-bridge-argocd-control-plane-template -``` +## (Optional) Fork the GitOps git repositories +See the appendix section [Fork GitOps Repositories](#fork-gitops-repositories) for more info on the terraform variables to override. ## Deploy the Hub EKS Cluster -Change Director to `hub` +Change directory to `hub` ```shell cd hub ``` @@ -38,7 +31,7 @@ Initialize Terraform and deploy the EKS cluster: terraform init terraform apply -auto-approve ``` -Retrieve `kubectl` config, then execute the output command: +To retrieve `kubectl` config, execute the terraform output command: ```shell terraform output -raw configure_kubectl ``` @@ -46,7 +39,7 @@ terraform output -raw configure_kubectl ### Monitor GitOps Progress for Addons Wait until **all** the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command ```shell -watch kubectl get applications -n argocd +kubectl get applications -n argocd -w ``` ## Access ArgoCD on Hub Cluster @@ -66,7 +59,7 @@ The output should match the `arn` for the IAM Role that will assume the IAM Role ``` ## Deploy the Spoke EKS Cluster -Initialize Terraform and deploy the EKS clusters: +Use the `deploy.sh` script to create terraform workspace, initialize Terraform, and deploy the EKS clusters: ```shell cd ../spokes ./deploy.sh dev @@ -94,11 +87,10 @@ Retrieve `kubectl` config, then execute the output command: terraform output -raw configure_kubectl ``` -### Verify ArgoCD Cluster Secret for Spoke has the correct IAM Role to be assume by Hub Cluster +### Verify ArgoCD Cluster Secret for Spokes have the correct IAM Role to be assume by Hub Cluster ```shell -kubectl get secret -n argocd hub-spoke-dev --template='{{index .data.config | base64decode}}' +for i in dev staging prod ; do echo $i && kubectl --context hub get secret -n argocd spoke-$i --template='{{index .data.config | base64decode}}' ; done ``` -Do the same for the other cluster replaced `dev` in `hub-spoke-dev` The output have a section `awsAuthConfig` with the `clusterName` and the `roleARN` that has write access to the spoke cluster ```json { @@ -117,15 +109,14 @@ The output have a section `awsAuthConfig` with the `clusterName` and the `roleAR ### Verify the Addons on Spoke Clusters Verify that the addons are ready: ```shell -kubectl get deployment -n kube-system \ - metrics-server +for i in dev staging prod ; do echo $i && kubectl --context spoke-$i get deployment -n kube-system metrics-server ; done ``` ### Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context) Watch until **all* the Workloads ArgoCD Applications are `Healthy` ```shell -watch kubectl get -n argocd applications +kubectl --context hub get -n argocd applications -w ``` Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command @@ -133,13 +124,13 @@ Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit t ### Verify the Application Verify that the application configuration is present and the pod is running: ```shell -kubectl get all -n workload +for i in dev staging prod ; do echo $i && kubectl --context spoke-$i get all -n workload ; done ``` ### Container Metrics Check the application's CPU and memory metrics: ```shell -kubectl top pods -n workload +for i in dev staging prod ; do echo $i && kubectl --context spoke-$i top pods -n workload ; done ``` ## Destroy the Spoke EKS Clusters @@ -157,3 +148,16 @@ Destroy Hub Clusters cd ../hub ./destroy.sh ``` + +## Appendix + +## Fork GitOps Repositories +To modify the `values.yaml` file or the helm chart version for addons, you'll need to fork tthe repository [aws-samples/eks-blueprints-add-ons](https://github.com/aws-samples/eks-blueprints-add-ons). + +After forking, update the following environment variables to point to your forks, replacing the default values. +```shell +export TF_VAR_gitops_addons_org=https://github.com/aws-samples +export TF_VAR_gitops_addons_repo=eks-blueprints-add-ons +export TF_VAR_gitops_addons_revision=main + +``` From 8048c03d8bbdc84e693b4b571927c972a25b4dc1 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 3 Nov 2023 00:43:12 -0400 Subject: [PATCH 4/8] address context changes Signed-off-by: Carlos Santana --- patterns/gitops/multi-cluster-hub-spoke-argocd/README.md | 6 +++--- patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf | 2 +- .../gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf | 6 +++--- .../multi-cluster-hub-spoke-argocd/spokes/destroy.sh | 7 ++++++- .../gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf | 2 +- .../multi-cluster-hub-spoke-argocd/spokes/outputs.tf | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md index 302db6d9d2..f9bc96c19c 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -109,7 +109,7 @@ The output have a section `awsAuthConfig` with the `clusterName` and the `roleAR ### Verify the Addons on Spoke Clusters Verify that the addons are ready: ```shell -for i in dev staging prod ; do echo $i && kubectl --context spoke-$i get deployment -n kube-system metrics-server ; done +for i in dev staging prod ; do echo $i && kubectl --context $i get deployment -n kube-system metrics-server ; done ``` @@ -124,13 +124,13 @@ Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit t ### Verify the Application Verify that the application configuration is present and the pod is running: ```shell -for i in dev staging prod ; do echo $i && kubectl --context spoke-$i get all -n workload ; done +for i in dev staging prod ; do echo $i && kubectl --context $i get all -n workload ; done ``` ### Container Metrics Check the application's CPU and memory metrics: ```shell -for i in dev staging prod ; do echo $i && kubectl --context spoke-$i top pods -n workload ; done +for i in dev staging prod ; do echo $i && kubectl --context $i top pods -n workload ; done ``` ## Destroy the Spoke EKS Clusters diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf index 60710e5dbb..2bd7ba8ccf 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf @@ -31,7 +31,7 @@ provider "kubernetes" { } locals { - name = "hub-spoke-${local.environment}" + name = "hub-${local.environment}" environment = "control-plane" region = var.region diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf index c5b089f69a..f573f6b585 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf @@ -2,7 +2,7 @@ output "configure_kubectl" { description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub EOT } @@ -11,7 +11,7 @@ output "configure_argocd" { description = "Terminal Setup" value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" kubectl config set-context --current --namespace argocd argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) @@ -26,7 +26,7 @@ output "access_argocd" { description = "ArgoCD Access" value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub echo "ArgoCD Username: admin" echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh index cc2a333fdd..5b99b400bd 100755 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh @@ -15,7 +15,12 @@ if [[ $# -eq 0 ]] ; then fi env=$1 echo "Destroying $env ..." -terraform workspace select $env +if terraform workspace list | grep -q $env; then + echo "Workspace $env already exists." + terraform workspace select $env +else + terraform workspace new $env +fi terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf index 28b328e5a3..617769acb7 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf @@ -49,7 +49,7 @@ provider "kubernetes" { locals { - name = "hub-spoke-${terraform.workspace}" + name = "spoke-${terraform.workspace}" environment = terraform.workspace region = var.region diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf index e398229ac0..18bd9c0dc9 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf @@ -2,6 +2,6 @@ output "configure_kubectl" { description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" value = <<-EOT export KUBECONFIG="/tmp/${module.eks.cluster_name}" - aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias ${local.environment} EOT } From daba75cbcbcab9f7f2c6407dc831cb49092b0c31 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 3 Nov 2023 00:59:43 -0400 Subject: [PATCH 5/8] switch irsa creation to module Signed-off-by: Carlos Santana --- .../hub/main.tf | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf index 2bd7ba8ccf..254499b4ec 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf @@ -151,28 +151,25 @@ module "gitops_bridge_bootstrap" { # ArgoCD EKS Access ################################################################################ module "argocd_irsa" { - source = "aws-ia/eks-blueprints-addon/aws" + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.20" - create_release = false - create_role = true - role_name_use_prefix = false - role_name = "${module.eks.cluster_name}-argocd-hub" + role_name_prefix = "argocd-hub-" assume_role_condition_test = "StringLike" - create_policy = false - role_policies = { + role_policy_arns = { ArgoCD_EKS_Policy = aws_iam_policy.irsa_policy.arn } oidc_providers = { - this = { - provider_arn = module.eks.oidc_provider_arn - namespace = local.argocd_namespace - service_account = "argocd-*" + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["${local.argocd_namespace}:argocd-*"] } } - tags = local.tags + tags = local.tags } + resource "aws_iam_policy" "irsa_policy" { name = "${module.eks.cluster_name}-argocd-irsa" description = "IAM Policy for ArgoCD Hub" From d3d98e71c2814c80ee8b14348583800ab3489055 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 3 Nov 2023 01:04:40 -0400 Subject: [PATCH 6/8] update readme Signed-off-by: Carlos Santana --- .../gitops/getting-started-argocd/README.md | 4 +-- .../multi-cluster-hub-spoke-argocd/README.md | 31 ++++++++++++++++--- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/patterns/gitops/getting-started-argocd/README.md b/patterns/gitops/getting-started-argocd/README.md index 80c0e06663..1ba14d81b1 100644 --- a/patterns/gitops/getting-started-argocd/README.md +++ b/patterns/gitops/getting-started-argocd/README.md @@ -38,7 +38,7 @@ terraform apply -target="module.vpc" -auto-approve terraform apply -target="module.eks" -auto-approve terraform apply -auto-approve ``` -Retrieve `kubectl` config, then execute the output command: +To retrieve `kubectl` config, execute the terraform output command: ```shell terraform output -raw configure_kubectl ``` @@ -110,7 +110,7 @@ Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use `Ctrl+C` or `Cmd+C` to exit the `watch` command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status. ```shell -watch kubectl get applications -n argocd +kubectl get applications -n argocd -w ``` The expected output should look like the following: ```text diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md index f9bc96c19c..8c6f81203a 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -29,24 +29,47 @@ cd hub Initialize Terraform and deploy the EKS cluster: ```shell terraform init +terraform apply -target="module.vpc" -auto-approve +terraform apply -target="module.eks" -auto-approve terraform apply -auto-approve ``` To retrieve `kubectl` config, execute the terraform output command: ```shell terraform output -raw configure_kubectl ``` +The expected output will have two lines you run in your terminal +```text +export KUBECONFIG="/tmp/getting-started-gitops" +aws eks --region us-west-2 update-kubeconfig --name getting-started-gitops +``` +>The first line sets the `KUBECONFIG` environment variable to a temporary file +that includes the cluster name. The second line uses the `aws` CLI to populate +that temporary file with the `kubectl` configuration. This approach offers the +advantage of not altering your existing `kubectl` context, allowing you to work +in other terminal windows without interference. ### Monitor GitOps Progress for Addons -Wait until **all** the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use Crl+C to exit the `watch` command +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. +Use `Ctrl+C` or `Cmd+C` to exit the `watch` command. ArgoCD Applications +can take a couple of minutes in order to achieve the Healthy status. ```shell kubectl get applications -n argocd -w ``` -## Access ArgoCD on Hub Cluster -Access ArgoCD's UI, run the command from the output: +## (Optional) Access ArgoCD +Access to the ArgoCD's UI is completely optional, if you want to do it, +run the commands shown in the Terraform output as the example below: ```shell terraform output -raw access_argocd ``` +The expected output should contain the `kubectl` config followed by `kubectl` command to retrieve +the URL, username, password to login into ArgoCD UI or CLI. +```text +echo "ArgoCD Username: admin" +echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" +echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +``` + ## Verify that ArgoCD Service Accouts has the annotation for IRSA ```shell @@ -55,7 +78,7 @@ kubectl get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks ``` The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters ```text -"arn:aws:iam::0123456789:role/hub-spoke-control-plane-argocd-hub" +"arn:aws:iam::0123456789:role/argocd-hub-0123abc.." ``` ## Deploy the Spoke EKS Cluster From 10402b91458869cb668f976c34e8df270d389546 Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 3 Nov 2023 01:12:24 -0400 Subject: [PATCH 7/8] fix images fo rwebsite Signed-off-by: Carlos Santana --- patterns/gitops/getting-started-argocd/README.md | 2 +- .../multi-cluster-hub-spoke-argocd/README.md | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/patterns/gitops/getting-started-argocd/README.md b/patterns/gitops/getting-started-argocd/README.md index 1ba14d81b1..3cdf8e6d16 100644 --- a/patterns/gitops/getting-started-argocd/README.md +++ b/patterns/gitops/getting-started-argocd/README.md @@ -2,7 +2,7 @@ This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD, employing the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev). - + The [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev) enables Kubernetes administrators to utilize Infrastructure as Code (IaC) and GitOps tools for deploying Kubernetes Addons and Workloads. Addons often depend on Cloud resources that are external to the cluster. The configuration metadata for these external resources is required by the Addons' Helm charts. While IaC is used to create these cloud resources, it is not used to install the Helm charts. Instead, the IaC tool stores this metadata either within GitOps resources in the cluster or in a Git repository. The GitOps tool then extracts these metadata values and passes them to the Helm chart during the Addon installation process. This mechanism forms the bridge between IaC and GitOps, hence the term "GitOps Bridge." diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md index 8c6f81203a..c5fce6586a 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -2,7 +2,7 @@ This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD in a Multi-Cluster Hub-Spoke topoloy, employing the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev). - + This example deploys ArgoCD on the Hub cluster (i.e. management/control-plane cluster). @@ -40,7 +40,7 @@ terraform output -raw configure_kubectl The expected output will have two lines you run in your terminal ```text export KUBECONFIG="/tmp/getting-started-gitops" -aws eks --region us-west-2 update-kubeconfig --name getting-started-gitops +aws eks --region us-west-2 update-kubeconfig --name getting-started-gitops --alias hub ``` >The first line sets the `KUBECONFIG` environment variable to a temporary file that includes the cluster name. The second line uses the `aws` CLI to populate @@ -53,7 +53,7 @@ Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. Use `Ctrl+C` or `Cmd+C` to exit the `watch` command. ArgoCD Applications can take a couple of minutes in order to achieve the Healthy status. ```shell -kubectl get applications -n argocd -w +kubectl --context hub get applications -n argocd -w ``` ## (Optional) Access ArgoCD @@ -66,15 +66,15 @@ The expected output should contain the `kubectl` config followed by `kubectl` co the URL, username, password to login into ArgoCD UI or CLI. ```text echo "ArgoCD Username: admin" -echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" -echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +echo "ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" +echo "ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" ``` ## Verify that ArgoCD Service Accouts has the annotation for IRSA ```shell -kubectl get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' -kubectl get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +kubectl --context hub get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +kubectl --context hub get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' ``` The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters ```text From bbb6424fdf9a4cd46b0cb485f9cf38a37d0f5eec Mon Sep 17 00:00:00 2001 From: Carlos Santana Date: Fri, 3 Nov 2023 01:48:10 -0400 Subject: [PATCH 8/8] fix context Signed-off-by: Carlos Santana --- .../multi-cluster-hub-spoke-argocd/README.md | 27 +++++++++++-------- .../hub/outputs.tf | 12 ++++----- .../spokes/deploy.sh | 18 ++++++++++--- .../spokes/destroy.sh | 7 +---- .../spokes/outputs.tf | 2 +- 5 files changed, 38 insertions(+), 28 deletions(-) diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md index c5fce6586a..c61d2d2197 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -39,7 +39,7 @@ terraform output -raw configure_kubectl ``` The expected output will have two lines you run in your terminal ```text -export KUBECONFIG="/tmp/getting-started-gitops" +export KUBECONFIG="/tmp/hub-spoke" aws eks --region us-west-2 update-kubeconfig --name getting-started-gitops --alias hub ``` >The first line sets the `KUBECONFIG` environment variable to a temporary file @@ -55,6 +55,14 @@ can take a couple of minutes in order to achieve the Healthy status. ```shell kubectl --context hub get applications -n argocd -w ``` +The expected output should look like the following: +```text +NAME SYNC STATUS HEALTH STATUS +addon-in-cluster-argo-cd Synced Healthy +addon-in-cluster-aws-load-balancer-controller Synced Healthy +addon-in-cluster-metrics-server Synced Healthy +cluster-addons Synced Healthy +``` ## (Optional) Access ArgoCD Access to the ArgoCD's UI is completely optional, if you want to do it, @@ -78,7 +86,8 @@ kubectl --context hub get sa -n argocd argocd-server -o json | jq '.metadata.an ``` The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters ```text -"arn:aws:iam::0123456789:role/argocd-hub-0123abc.." +arn:aws:iam::0123456789:role/argocd-hub-0123abc.. +arn:aws:iam::0123456789:role/argocd-hub-0123abc.. ``` ## Deploy the Spoke EKS Cluster @@ -91,25 +100,21 @@ cd ../spokes ``` Each environment uses a Terraform workspace -To access Terraform output run the following commands for the particular environment +To retrieve `kubectl` config, execute the terraform output command: ```shell terraform workspace select dev -terraform output +terraform output -raw configure_kubectl ``` ```shell terraform workspace select staging -terraform output +terraform output -raw configure_kubectl ``` ```shell terraform workspace select prod -terraform output -``` - -Retrieve `kubectl` config, then execute the output command: -```shell terraform output -raw configure_kubectl ``` + ### Verify ArgoCD Cluster Secret for Spokes have the correct IAM Role to be assume by Hub Cluster ```shell for i in dev staging prod ; do echo $i && kubectl --context hub get secret -n argocd spoke-$i --template='{{index .data.config | base64decode}}' ; done @@ -132,7 +137,7 @@ The output have a section `awsAuthConfig` with the `clusterName` and the `roleAR ### Verify the Addons on Spoke Clusters Verify that the addons are ready: ```shell -for i in dev staging prod ; do echo $i && kubectl --context $i get deployment -n kube-system metrics-server ; done +for i in dev staging prod ; do echo $i && kubectl --context $i get deployment -n kube-system ; done ``` diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf index f573f6b585..406672e261 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf @@ -1,7 +1,7 @@ output "configure_kubectl" { description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" + export KUBECONFIG="/tmp/hup-spoke" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub EOT } @@ -10,10 +10,10 @@ output "configure_kubectl" { output "configure_argocd" { description = "Terminal Setup" value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" + export KUBECONFIG="/tmp/hup-spoke" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" - kubectl config set-context --current --namespace argocd + kubectl --context hub config set-context --current --namespace argocd argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) echo "ArgoCD Username: admin" echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" @@ -25,11 +25,11 @@ output "configure_argocd" { output "access_argocd" { description = "ArgoCD Access" value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" + export KUBECONFIG="/tmp/hup-spoke" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub echo "ArgoCD Username: admin" - echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" - echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" + echo "ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" EOT } diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh index 0dad2a7030..766cffd8d8 100755 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh @@ -1,17 +1,27 @@ #!/bin/bash +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + if [[ $# -eq 0 ]] ; then echo "No arguments supplied" - echo "Usage: deploy.sh " - echo "Example: deploy.sh dev" + echo "Usage: destroy.sh " + echo "Example: destroy.sh dev" exit 1 fi env=$1 echo "Deploying $env with "workspaces/${env}.tfvars" ..." -set -x -terraform workspace new $env +if terraform workspace list | grep -q $env; then + echo "Workspace $env already exists." +else + terraform workspace new $env +fi + terraform workspace select $env terraform init terraform apply -var-file="workspaces/${env}.tfvars" diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh index 5b99b400bd..cf44b4d9a2 100755 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh @@ -15,13 +15,8 @@ if [[ $# -eq 0 ]] ; then fi env=$1 echo "Destroying $env ..." -if terraform workspace list | grep -q $env; then - echo "Workspace $env already exists." - terraform workspace select $env -else - terraform workspace new $env -fi +terraform workspace select $env terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf index 18bd9c0dc9..18b5a74f23 100644 --- a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf @@ -1,7 +1,7 @@ output "configure_kubectl" { description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" value = <<-EOT - export KUBECONFIG="/tmp/${module.eks.cluster_name}" + export KUBECONFIG="/tmp/hup-spoke" aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias ${local.environment} EOT }