diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 75c8b7801c..7e7129fe19 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -24,4 +24,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.5.4 - name: 'Dependency Review' - uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034 # v3.1.0 + uses: actions/dependency-review-action@01bc87099ba56df1e897b6874784491ea6309bc4 # v3.1.4 diff --git a/.github/workflows/e2e-parallel-destroy.yml b/.github/workflows/e2e-parallel-destroy.yml index 708e8dabe6..85b70c810e 100644 --- a/.github/workflows/e2e-parallel-destroy.yml +++ b/.github/workflows/e2e-parallel-destroy.yml @@ -28,7 +28,7 @@ jobs: include: - example_path: patterns/agones-game-controller - example_path: patterns/fargate-serverless - - example_path: patterns/argocd + - example_path: patterns/gitops/getting-started-argocd - example_path: patterns/ipv6-eks-cluster - example_path: patterns/karpenter - example_path: patterns/multi-tenancy-with-teams @@ -56,7 +56,7 @@ jobs: role-session-name: GithubActions-Session - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 + uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.0.0 @@ -65,6 +65,7 @@ jobs: run: | terraform init -upgrade=true terraform destroy -target=module.eks_blueprints_kubernetes_addons -no-color -input=false -auto-approve - terraform destroy -target=module.eks_blueprints -no-color -input=false -auto-approve + terraform destroy -target=module.eks_blueprints_addons -no-color -input=false -auto-approve + terraform destroy -target=module.eks_blueprints -no-color -input=false -auto-approve terraform destroy -target=module.eks -no-color -input=false -auto-approve terraform destroy -no-color -input=false -auto-approve diff --git a/.github/workflows/e2e-parallel-full.yml b/.github/workflows/e2e-parallel-full.yml index d1692e282e..45d7f623af 100644 --- a/.github/workflows/e2e-parallel-full.yml +++ b/.github/workflows/e2e-parallel-full.yml @@ -62,7 +62,7 @@ jobs: include: - example_path: patterns/agones-game-controller - example_path: patterns/fargate-serverless - - example_path: patterns/argocd + - example_path: patterns/gitops/getting-started-argocd - example_path: patterns/ipv6-eks-cluster - example_path: patterns/karpenter - example_path: patterns/multi-tenancy-with-teams @@ -99,7 +99,7 @@ jobs: echo "iamlive_pid=$IAMLIVE_PID" >> $GITHUB_ENV - name: Setup Terraform - uses: hashicorp/setup-terraform@v2 + uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.0.0 @@ -115,6 +115,7 @@ jobs: terraform apply -target=module.eks_blueprints -no-color -input=false -auto-approve terraform apply -target=module.eks -no-color -input=false -auto-approve terraform apply -target=module.eks_blueprints_kubernetes_addons -no-color -input=false -auto-approve + terraform apply -target=module.eks_blueprints_addons -no-color -input=false -auto-approve terraform apply -no-color -input=false -auto-approve - name: Terraform Destroy @@ -125,9 +126,10 @@ jobs: export AWS_CSM_ENABLED=true export AWS_CSM_PORT=31000 export AWS_CSM_HOST=127.0.0.1 - terraform destroy -target=module.eks_blueprints_kubernetes_addons -no-color -input=false -auto-approve - terraform destroy -target=module.eks_blueprints -no-color -input=false -auto-approve - terraform destroy -target=module.eks -no-color -input=false -auto-approve + terraform destroy -target=module.eks_blueprints_kubernetes_addons -no-color -input=false -auto-approve + terraform destroy -target=module.eks_blueprints_addons -no-color -input=false -auto-approve + terraform destroy -target=module.eks_blueprints -no-color -input=false -auto-approve + terraform destroy -target=module.eks -no-color -input=false -auto-approve terraform destroy -no-color -input=false -auto-approve - name: Fail if TF apply failed diff --git a/.github/workflows/markdown-link-check.yml b/.github/workflows/markdown-link-check.yml index eb608ff513..f1415a4284 100644 --- a/.github/workflows/markdown-link-check.yml +++ b/.github/workflows/markdown-link-check.yml @@ -26,7 +26,7 @@ jobs: egress-policy: audit - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: '16.x' - name: install markdown-link-check diff --git a/.github/workflows/plan-examples.yml b/.github/workflows/plan-examples.yml index 531528a1cf..04f2ed18c9 100644 --- a/.github/workflows/plan-examples.yml +++ b/.github/workflows/plan-examples.yml @@ -1,10 +1,6 @@ name: plan-examples on: - # Review https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ and better understand the risks of using pull_request_target before making major changes to this workflow. - pull_request_target: - branches: - - main workflow_dispatch: concurrency: @@ -97,7 +93,7 @@ jobs: role-session-name: GithubActions-Session - name: Terraform Job - uses: hashicorp/setup-terraform@v2 + uses: hashicorp/setup-terraform@v3 if: steps.changes.outputs.src== 'true' with: terraform_version: 1.0.0 diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index ae9100b017..13c0b75b76 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -23,7 +23,7 @@ jobs: with: egress-policy: audit - - uses: amannn/action-semantic-pull-request@v5.3.0 + - uses: amannn/action-semantic-pull-request@v5.4.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index 768a8e081b..b4dc12dd44 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -28,7 +28,7 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index e60f25f57c..975049c1be 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -41,7 +41,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@483ef80eb98fb506c348f7d62e28055e49fe2398 # v2.3.0 + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 with: results_file: results.sarif results_format: sarif @@ -63,7 +63,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: SARIF file path: results.sarif @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@49abf0ba24d0b7953cb586944e918a0b92074c80 # v2.22.4 + uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 with: sarif_file: results.sarif diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 69734a100a..2235d24cc7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace args: ['--markdown-linebreak-ext=md'] @@ -10,7 +10,7 @@ repos: - id: detect-aws-credentials args: ['--allow-missing-credentials'] - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.4 + rev: v1.83.5 hooks: - id: terraform_fmt - id: terraform_docs diff --git a/docs/patterns/argocd.md b/docs/patterns/argocd.md deleted file mode 100644 index d9fd12c2e7..0000000000 --- a/docs/patterns/argocd.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: ArgoCD ---- - -{% - include-markdown "../../patterns/argocd/README.md" -%} diff --git a/docs/patterns/gitops-getting-started-argocd.md b/docs/patterns/gitops-getting-started-argocd.md new file mode 100644 index 0000000000..1492cc377a --- /dev/null +++ b/docs/patterns/gitops-getting-started-argocd.md @@ -0,0 +1,7 @@ +--- +title: GitOps Getting Started (ArgoCD) +--- + +{% + include-markdown "../../patterns/gitops/getting-started-argocd/README.md" +%} diff --git a/docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md b/docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md new file mode 100644 index 0000000000..0ef45bb13a --- /dev/null +++ b/docs/patterns/gitops-multi-cluster-hub-spoke-argocd.md @@ -0,0 +1,7 @@ +--- +title: GitOps Multi-Cluster Hub-Spoke Topology (ArgoCD) +--- + +{% + include-markdown "../../patterns/gitops/multi-cluster-hub-spoke-argocd/README.md" +%} diff --git a/patterns/argocd/README.md b/patterns/argocd/README.md deleted file mode 100644 index 507c96553f..0000000000 --- a/patterns/argocd/README.md +++ /dev/null @@ -1,117 +0,0 @@ -# Amazon EKS Cluster w/ ArgoCD - -This pattern demonstrates an EKS cluster that uses ArgoCD for application deployments. - -- [Documentation](https://argo-cd.readthedocs.io/en/stable/) -- [EKS Blueprints Add-ons Repo](https://github.com/aws-samples/eks-blueprints-add-ons) -- [EKS Blueprints Workloads Repo](https://github.com/aws-samples/eks-blueprints-workloads) - -## Deploy - -See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and steps to deploy this pattern. - -## Validate - -1. List out the pods running currently: - - ```sh - kubectl get pods -A - ``` - - ```text - NAMESPACE NAME READY STATUS RESTARTS AGE - argo-rollouts argo-rollouts-5d47ccb8d4-854s6 1/1 Running 0 23h - argo-rollouts argo-rollouts-5d47ccb8d4-srjk9 1/1 Running 0 23h - argocd argo-cd-argocd-application-controller-0 1/1 Running 0 24h - argocd argo-cd-argocd-applicationset-controller-547f9cfd68-kp89p 1/1 Running 0 24h - argocd argo-cd-argocd-dex-server-55765f7cd7-t8r2f 1/1 Running 0 24h - argocd argo-cd-argocd-notifications-controller-657df4dbcb-p596r 1/1 Running 0 24h - argocd argo-cd-argocd-repo-server-7d4dddf886-2vmgt 1/1 Running 0 24h - argocd argo-cd-argocd-repo-server-7d4dddf886-bm7tz 1/1 Running 0 24h - argocd argo-cd-argocd-server-775ddf74b8-8jzvc 1/1 Running 0 24h - argocd argo-cd-argocd-server-775ddf74b8-z6lz6 1/1 Running 0 24h - argocd argo-cd-redis-ha-haproxy-6d7b7d4656-b8bt8 1/1 Running 0 24h - argocd argo-cd-redis-ha-haproxy-6d7b7d4656-mgjx5 1/1 Running 0 24h - argocd argo-cd-redis-ha-haproxy-6d7b7d4656-qsbgw 1/1 Running 0 24h - argocd argo-cd-redis-ha-server-0 4/4 Running 0 24h - argocd argo-cd-redis-ha-server-1 4/4 Running 0 24h - argocd argo-cd-redis-ha-server-2 4/4 Running 0 24h - cert-manager cert-manager-586ccb6656-2v8mf 1/1 Running 0 23h - cert-manager cert-manager-cainjector-99d64d795-2gwnj 1/1 Running 0 23h - cert-manager cert-manager-webhook-8d87786cb-24kww 1/1 Running 0 23h - geolocationapi geolocationapi-85599c5c74-rqqqs 2/2 Running 0 25m - geolocationapi geolocationapi-85599c5c74-whsp6 2/2 Running 0 25m - geordie downstream0-7f6ff946b6-r8sxc 1/1 Running 0 25m - geordie downstream1-64c7db6f9-rsbk5 1/1 Running 0 25m - geordie frontend-646bfb947c-wshpb 1/1 Running 0 25m - geordie redis-server-6bd7885d5d-s7rqw 1/1 Running 0 25m - geordie yelb-appserver-5d89946ffd-vkxt9 1/1 Running 0 25m - geordie yelb-db-697bd9f9d9-2t4b6 1/1 Running 0 25m - geordie yelb-ui-75ff8b96ff-fh6bw 1/1 Running 0 25m - karpenter karpenter-7b99fb785d-87k6h 1/1 Running 0 106m - karpenter karpenter-7b99fb785d-lkq9l 1/1 Running 0 106m - kube-system aws-load-balancer-controller-6cf9bdbfdf-h7bzb 1/1 Running 0 20m - kube-system aws-load-balancer-controller-6cf9bdbfdf-vfbrj 1/1 Running 0 20m - kube-system aws-node-cvjmq 1/1 Running 0 24h - kube-system aws-node-fw7zc 1/1 Running 0 24h - kube-system aws-node-l7589 1/1 Running 0 24h - kube-system aws-node-nll82 1/1 Running 0 24h - kube-system aws-node-zhz8l 1/1 Running 0 24h - kube-system coredns-7975d6fb9b-5sf7r 1/1 Running 0 24h - kube-system coredns-7975d6fb9b-k78dz 1/1 Running 0 24h - kube-system ebs-csi-controller-5cd4944c94-7jwlb 6/6 Running 0 24h - kube-system ebs-csi-controller-5cd4944c94-8tcsg 6/6 Running 0 24h - kube-system ebs-csi-node-66jmx 3/3 Running 0 24h - kube-system ebs-csi-node-b2pw4 3/3 Running 0 24h - kube-system ebs-csi-node-g4v9z 3/3 Running 0 24h - kube-system ebs-csi-node-k7nvp 3/3 Running 0 24h - kube-system ebs-csi-node-tfq9q 3/3 Running 0 24h - kube-system kube-proxy-4x8vm 1/1 Running 0 24h - kube-system kube-proxy-gtlpm 1/1 Running 0 24h - kube-system kube-proxy-vfnbf 1/1 Running 0 24h - kube-system kube-proxy-z9wdh 1/1 Running 0 24h - kube-system kube-proxy-zzx9m 1/1 Running 0 24h - kube-system metrics-server-7f4db5fd87-9n6dv 1/1 Running 0 23h - kube-system metrics-server-7f4db5fd87-t8wxg 1/1 Running 0 23h - kube-system metrics-server-7f4db5fd87-xcxlv 1/1 Running 0 23h - team-burnham burnham-66fccc4fb5-k4qtm 1/1 Running 0 25m - team-burnham burnham-66fccc4fb5-rrf4j 1/1 Running 0 25m - team-burnham burnham-66fccc4fb5-s9kbr 1/1 Running 0 25m - team-burnham nginx-7d47cfdff7-lzdjb 1/1 Running 0 25m - team-riker deployment-2048-6f7c78f959-h76rx 1/1 Running 0 25m - team-riker deployment-2048-6f7c78f959-skmrr 1/1 Running 0 25m - team-riker deployment-2048-6f7c78f959-tn9dw 1/1 Running 0 25m - team-riker guestbook-ui-c86c478bd-zg2z4 1/1 Running 0 25m - ``` - -2. Access the ArgoCD UI by running the following command: - - ```sh - kubectl port-forward svc/argo-cd-argocd-server 8080:443 -n argocd - ``` - - Then, open your browser and navigate to `https://localhost:8080/` - Username should be `admin`. - - The password will be the generated password by `random_password` resource, stored in AWS Secrets Manager. - You can easily retrieve the password by running the following command: - - ```sh - aws secretsmanager get-secret-value --secret-id --region - ``` - - Replace `` with the name of the secret name, if you haven't changed it then it should be `argocd`, also, make sure to replace `` with the region you are using. - - Pickup the the secret from the `SecretString`. - -## Destroy - -First, we need to ensure that the ArgoCD applications are properly cleaned up from the cluster, this can be achieved in multiple ways: - -- Disabling the `argocd_applications` configuration and running `terraform apply` again -- Deleting the apps using `argocd` [cli](https://argo-cd.readthedocs.io/en/stable/user-guide/app_deletion/#deletion-using-argocd) -- Deleting the apps using `kubectl` following [ArgoCD guidance](https://argo-cd.readthedocs.io/en/stable/user-guide/app_deletion/#deletion-using-kubectl) - -{% - include-markdown "../../docs/_partials/destroy.md" -%} diff --git a/patterns/argocd/main.tf b/patterns/argocd/main.tf deleted file mode 100644 index 20438794d4..0000000000 --- a/patterns/argocd/main.tf +++ /dev/null @@ -1,188 +0,0 @@ -provider "aws" { - region = local.region -} - -provider "kubernetes" { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - -provider "helm" { - kubernetes { - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } - } -} - -provider "bcrypt" {} - -data "aws_availability_zones" "available" {} - -locals { - name = basename(path.cwd) - region = "us-west-2" - - vpc_cidr = "10.0.0.0/16" - azs = slice(data.aws_availability_zones.available.names, 0, 3) - - tags = { - Blueprint = local.name - GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" - } -} - -################################################################################ -# Cluster -################################################################################ - -module "eks" { - source = "terraform-aws-modules/eks/aws" - version = "~> 19.16" - - cluster_name = local.name - cluster_version = "1.27" - cluster_endpoint_public_access = true - - # EKS Addons - cluster_addons = { - coredns = {} - kube-proxy = {} - vpc-cni = {} - } - - vpc_id = module.vpc.vpc_id - subnet_ids = module.vpc.private_subnets - - eks_managed_node_groups = { - initial = { - instance_types = ["m5.large"] - - min_size = 3 - max_size = 10 - desired_size = 5 - } - } - - tags = local.tags -} - -################################################################################ -# EKS Blueprints Addons -################################################################################ - -module "eks_blueprints_addons" { - # Users should pin the version to the latest available release - # tflint-ignore: terraform_module_pinned_source - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1" - - eks_cluster_id = module.eks.cluster_name - eks_cluster_endpoint = module.eks.cluster_endpoint - eks_cluster_version = module.eks.cluster_version - eks_oidc_provider = module.eks.oidc_provider - eks_oidc_provider_arn = module.eks.oidc_provider_arn - - enable_argocd = true - # This example shows how to set default ArgoCD Admin Password using SecretsManager with Helm Chart set_sensitive values. - argocd_helm_config = { - set_sensitive = [ - { - name = "configs.secret.argocdServerAdminPassword" - value = bcrypt_hash.argo.id - } - ] - } - - argocd_manage_add_ons = true # Indicates that ArgoCD is responsible for managing/deploying add-ons - argocd_applications = { - addons = { - path = "chart" - repo_url = "https://github.com/aws-samples/eks-blueprints-add-ons.git" - add_on_application = true - } - workloads = { - path = "envs/dev" - repo_url = "https://github.com/aws-samples/eks-blueprints-workloads.git" - add_on_application = false - } - } - - # Add-ons - enable_amazon_eks_aws_ebs_csi_driver = true - enable_aws_load_balancer_controller = true - enable_cert_manager = true - enable_karpenter = true - enable_metrics_server = true - enable_argo_rollouts = true - - tags = local.tags -} - -#--------------------------------------------------------------- -# ArgoCD Admin Password credentials with Secrets Manager -# Login to AWS Secrets manager with the same role as Terraform to extract the ArgoCD admin password with the secret name as "argocd" -#--------------------------------------------------------------- -resource "random_password" "argocd" { - length = 16 - special = true - override_special = "!#$%&*()-_=+[]{}<>:?" -} - -# Argo requires the password to be bcrypt, we use custom provider of bcrypt, -# as the default bcrypt function generates diff for each terraform plan -resource "bcrypt_hash" "argo" { - cleartext = random_password.argocd.result -} - -#tfsec:ignore:aws-ssm-secret-use-customer-key -resource "aws_secretsmanager_secret" "argocd" { - name = "argocd" - recovery_window_in_days = 0 # Set to zero for this example to force delete during Terraform destroy -} - -resource "aws_secretsmanager_secret_version" "argocd" { - secret_id = aws_secretsmanager_secret.argocd.id - secret_string = random_password.argocd.result -} - -################################################################################ -# Supporting Resources -################################################################################ - -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "~> 5.0" - - name = local.name - cidr = local.vpc_cidr - - azs = local.azs - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] - public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] - - enable_nat_gateway = true - single_nat_gateway = true - - public_subnet_tags = { - "kubernetes.io/role/elb" = 1 - } - - private_subnet_tags = { - "kubernetes.io/role/internal-elb" = 1 - } - - tags = local.tags -} diff --git a/patterns/argocd/outputs.tf b/patterns/argocd/outputs.tf deleted file mode 100644 index d79912bf44..0000000000 --- a/patterns/argocd/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "configure_kubectl" { - description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" - value = "aws eks update-kubeconfig --name ${module.eks.cluster_name} --alias ${module.eks.cluster_name}" -} diff --git a/patterns/argocd/variables.tf b/patterns/argocd/variables.tf deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml index 7a8d00dd4e..7e6d29f9b3 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v2 name: demo-application description: A Helm chart to deploy the demo-application type: application -version: 1.0.0 \ No newline at end of file +version: 1.0.0 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml index e3c3c1d465..766e0b573a 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: backend + name: backend namespace: stars spec: replicas: 1 @@ -11,10 +11,10 @@ spec: template: metadata: labels: - role: backend + role: backend spec: containers: - - name: backend + - name: backend image: calico/star-probe:v0.1.0 imagePullPolicy: Always command: @@ -22,4 +22,4 @@ spec: - --http-port=6379 - --urls=http://frontend.stars:80/status,http://backend.stars:6379/status,http://client.client:9000/status ports: - - containerPort: 6379 \ No newline at end of file + - containerPort: 6379 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml index 5a579569ad..1298c3b74e 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/backend-svc.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: backend + name: backend namespace: stars spec: ports: - port: 6379 - targetPort: 6379 + targetPort: 6379 selector: - role: backend \ No newline at end of file + role: backend diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml index 8d763dcede..35c2d7caab 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: client + name: client namespace: client spec: replicas: 1 @@ -11,14 +11,14 @@ spec: template: metadata: labels: - role: client + role: client spec: containers: - - name: client + - name: client image: calico/star-probe:v0.1.0 imagePullPolicy: Always command: - probe - --urls=http://frontend.stars:80/status,http://backend.stars:6379/status ports: - - containerPort: 9000 \ No newline at end of file + - containerPort: 9000 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml index 91f714e9cb..d8e2c02b5a 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-ns.yaml @@ -3,4 +3,4 @@ kind: Namespace metadata: name: client labels: - role: client \ No newline at end of file + role: client diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml index cfd20be303..a91ded3d1c 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/client-svc.yaml @@ -5,7 +5,7 @@ metadata: namespace: client spec: ports: - - port: 9000 + - port: 9000 targetPort: 9000 selector: - role: client \ No newline at end of file + role: client diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml index c1b0762c32..5c0352c0d4 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-deploy.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: frontend + name: frontend namespace: stars spec: replicas: 1 @@ -11,10 +11,10 @@ spec: template: metadata: labels: - role: frontend + role: frontend spec: containers: - - name: frontend + - name: frontend image: calico/star-probe:v0.1.0 imagePullPolicy: Always command: @@ -22,4 +22,4 @@ spec: - --http-port=80 - --urls=http://frontend.stars:80/status,http://backend.stars:6379/status,http://client.client:9000/status ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml index 08fafd487b..51426395b7 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/frontend-svc.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: Service metadata: - name: frontend + name: frontend namespace: stars spec: ports: - - port: 80 - targetPort: 80 + - port: 80 + targetPort: 80 selector: - role: frontend \ No newline at end of file + role: frontend diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml index 555f338290..391119d3e6 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-deploy.yaml @@ -1,8 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: management-ui - namespace: management-ui + name: management-ui + namespace: management-ui spec: replicas: 1 selector: @@ -11,11 +11,11 @@ spec: template: metadata: labels: - role: management-ui + role: management-ui spec: containers: - - name: management-ui + - name: management-ui image: calico/star-collect:v0.1.0 imagePullPolicy: Always ports: - - containerPort: 9001 \ No newline at end of file + - containerPort: 9001 diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml index ef0a8ec158..afc419b80d 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-ns.yaml @@ -1,6 +1,6 @@ apiVersion: v1 kind: Namespace metadata: - name: management-ui + name: management-ui labels: - role: management-ui \ No newline at end of file + role: management-ui diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml index 09b850d8d2..9c0e64337d 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/management-ui-svc.yaml @@ -1,12 +1,12 @@ apiVersion: v1 kind: Service metadata: - name: management-ui - namespace: management-ui + name: management-ui + namespace: management-ui spec: type: LoadBalancer ports: - - port: 80 + - port: 80 targetPort: 9001 selector: - role: management-ui \ No newline at end of file + role: management-ui diff --git a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml index de71efa857..67c4219fe9 100644 --- a/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml +++ b/patterns/aws-vpc-cni-network-policy/charts/demo-application/templates/stars-ns.yaml @@ -1,4 +1,4 @@ apiVersion: v1 kind: Namespace metadata: - name: stars \ No newline at end of file + name: stars diff --git a/patterns/blue-green-upgrade/environment/versions.tf b/patterns/blue-green-upgrade/environment/versions.tf index 967337612f..6227221a4a 100644 --- a/patterns/blue-green-upgrade/environment/versions.tf +++ b/patterns/blue-green-upgrade/environment/versions.tf @@ -10,5 +10,6 @@ terraform { source = "hashicorp/random" version = ">= 3.0" } + } } diff --git a/patterns/blue-green-upgrade/modules/eks_cluster/outputs.tf b/patterns/blue-green-upgrade/modules/eks_cluster/outputs.tf index 8c8c5f5a90..3df93a5810 100644 --- a/patterns/blue-green-upgrade/modules/eks_cluster/outputs.tf +++ b/patterns/blue-green-upgrade/modules/eks_cluster/outputs.tf @@ -49,10 +49,3 @@ output "gitops_metadata" { value = local.addons_metadata sensitive = true } - -# output "debug" { -# description = "debug output" -# #value = data.template_file.addons_template.rendered -# value = data.template_file.workloads_template.rendered -# #value = file("${path.module}/../../bootstrap/addons.yaml") -# } diff --git a/patterns/blue-green-upgrade/modules/eks_cluster/versions.tf b/patterns/blue-green-upgrade/modules/eks_cluster/versions.tf index eda2864d1e..e3ec65ce24 100644 --- a/patterns/blue-green-upgrade/modules/eks_cluster/versions.tf +++ b/patterns/blue-green-upgrade/modules/eks_cluster/versions.tf @@ -11,9 +11,5 @@ terraform { source = "hashicorp/kubernetes" version = "2.22.0" } - template = { - source = "hashicorp/template" - version = ">= 2.2.0" - } } } diff --git a/patterns/blue-green-upgrade/terraform.tfvars.example b/patterns/blue-green-upgrade/terraform.tfvars.example index 4687c83db5..0d70039604 100644 --- a/patterns/blue-green-upgrade/terraform.tfvars.example +++ b/patterns/blue-green-upgrade/terraform.tfvars.example @@ -5,12 +5,17 @@ environment_name = "eks-blueprint" hosted_zone_name = "eks.mydomain.org" # your Existing Hosted Zone eks_admin_role_name = "Admin" # Additional role admin in the cluster (usually the role I use in the AWS console) -# EKS Blueprint AddOns ArgoCD App of App repository -gitops_bridge_repo_url = "git@github.com:gitops-bridge-dev/gitops-bridge-argocd-control-plane-template" -gitops_bridge_repo_revision = "HEAD" +#gitops_addons_org = "git@github.com:aws-samples" +#gitops_addons_repo = "eks-blueprints-add-ons" +#gitops_addons_path = "argocd/bootstrap/control-plane/addons" +#gitops_addons_basepath = "argocd/" # EKS Blueprint Workloads ArgoCD App of App repository -workload_repo_url = "git@github.com:aws-samples/eks-blueprints-workloads.git" -workload_repo_revision = "main" -workload_repo_path = "envs/dev" -workload_repo_secret = "github-blueprint-ssh-key" +gitops_workloads_org = "git@github.com:aws-samples" +gitops_workloads_repo = "eks-blueprints-workloads" +gitops_workloads_revision = "main" +gitops_workloads_path = "envs/dev" + + +#Secret manager secret for github ssk jey +aws_secret_manager_git_private_ssh_key_name = "github-blueprint-ssh-key" diff --git a/patterns/gitops/getting-started-argocd/README.md b/patterns/gitops/getting-started-argocd/README.md new file mode 100644 index 0000000000..292978fa6d --- /dev/null +++ b/patterns/gitops/getting-started-argocd/README.md @@ -0,0 +1,345 @@ +# ArgoCD on Amazon EKS + +This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD, employing the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev). + + + +The [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev) enables Kubernetes administrators to utilize Infrastructure as Code (IaC) and GitOps tools for deploying Kubernetes Addons and Workloads. Addons often depend on Cloud resources that are external to the cluster. The configuration metadata for these external resources is required by the Addons' Helm charts. While IaC is used to create these cloud resources, it is not used to install the Helm charts. Instead, the IaC tool stores this metadata either within GitOps resources in the cluster or in a Git repository. The GitOps tool then extracts these metadata values and passes them to the Helm chart during the Addon installation process. This mechanism forms the bridge between IaC and GitOps, hence the term "GitOps Bridge." + +Additional examples available on the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev): + +- [argocd-ingress](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/argocd-ingress) +- [aws-secrets-manager](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/aws-secrets-manager) +- [crossplane](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/crossplane) +- [external-secrets](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/external-secrets) +- [multi-cluster/distributed](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/distributed) +- [multi-cluster/hub-spoke](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke) +- [multi-cluster/hub-spoke-shared](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/multi-cluster/hub-spoke-shared) +- [private-git](https://github.com/gitops-bridge-dev/gitops-bridge/tree/main/argocd/iac/terraform/examples/eks/private-git) + +## Prerequisites + +Before you begin, make sure you have the following command line tools installed: + +- git +- terraform +- kubectl +- argocd + +## (Optional) Fork the GitOps git repositories + +See the appendix section [Fork GitOps Repositories](#fork-gitops-repositories) for more info on the terraform variables to override. + +## Deploy the EKS Cluster + +Initialize Terraform and deploy the EKS cluster: + +```shell +terraform init +terraform apply -target="module.vpc" -auto-approve +terraform apply -target="module.eks" -auto-approve +terraform apply -auto-approve +``` + +To retrieve `kubectl` config, execute the terraform output command: + +```shell +terraform output -raw configure_kubectl +``` + +The expected output will have two lines you run in your terminal + +```text +export KUBECONFIG="/tmp/getting-started-gitops" +aws eks --region us-west-2 update-kubeconfig --name getting-started-gitops +``` + +>The first line sets the `KUBECONFIG` environment variable to a temporary file +that includes the cluster name. The second line uses the `aws` CLI to populate +that temporary file with the `kubectl` configuration. This approach offers the +advantage of not altering your existing `kubectl` context, allowing you to work +in other terminal windows without interference. + +Terraform will add GitOps Bridge Metadata to the ArgoCD secret. +The annotations contain metadata for the addons' Helm charts and ArgoCD ApplicationSets. + +```shell +kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.annotations' +``` + +The output looks like the following: + +```json +{ + "addons_repo_basepath": "argocd/", + "addons_repo_path": "bootstrap/control-plane/addons", + "addons_repo_revision": "main", + "addons_repo_url": "https://github.com/aws-samples/eks-blueprints-add-ons", + "aws_account_id": "0123456789", + "aws_cluster_name": "getting-started-gitops", + "aws_load_balancer_controller_iam_role_arn": "arn:aws:iam::0123456789:role/alb-controller", + "aws_load_balancer_controller_namespace": "kube-system", + "aws_load_balancer_controller_service_account": "aws-load-balancer-controller-sa", + "aws_region": "us-west-2", + "aws_vpc_id": "vpc-001d3f00151bbb731", + "cluster_name": "in-cluster", + "environment": "dev", + "workload_repo_basepath": "patterns/gitops/", + "workload_repo_path": "getting-started-argocd/k8s", + "workload_repo_revision": "main", + "workload_repo_url": "https://github.com/csantanapr/terraform-aws-eks-blueprints" +} +``` + +The labels offer a straightforward way to enable or disable an addon in ArgoCD for the cluster. + +```shell +kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster -o json | jq '.items[0].metadata.labels' | grep -v false | jq . +``` + +The output looks like the following: + +```json +{ + "argocd.argoproj.io/secret-type": "cluster", + "aws_cluster_name": "getting-started-gitops", + "cluster_name": "in-cluster", + "enable_argocd": "true", + "enable_aws_load_balancer_controller": "true", + "enable_metrics_server": "true", + "environment": "dev", + "kubernetes_version": "1.28" +} +``` + +## Deploy the Addons + +Bootstrap the addons using ArgoCD: + +```shell +kubectl apply -f bootstrap/addons.yaml +``` + +### Monitor GitOps Progress for Addons + +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. +Use `Ctrl+C` or `Cmd+C` to exit the `watch` command. ArgoCD Applications +can take a couple of minutes in order to achieve the Healthy status. + +```shell +kubectl get applications -n argocd -w +``` + +The expected output should look like the following: + +```text +NAME SYNC STATUS HEALTH STATUS +addon-in-cluster-argo-cd Synced Healthy +addon-in-cluster-aws-load-balancer-controller Synced Healthy +addon-in-cluster-metrics-server Synced Healthy +cluster-addons Synced Healthy +``` + +### Verify the Addons + +Verify that the addons are ready: + +```shell +kubectl get deployment -n kube-system \ + aws-load-balancer-controller \ + metrics-server +kubectl get deploy -n argocd \ + argo-cd-argocd-applicationset-controller \ + argo-cd-argocd-repo-server \ + argo-cd-argocd-server +``` + +The expected output should look like the following: + +```text +NAME READY UP-TO-DATE AVAILABLE AGE +aws-load-balancer-controller 2/2 2 2 7m21s +metrics-server 1/1 1 1 7m41s +argo-cd-argocd-applicationset-controller 1/1 1 1 109m +argo-cd-argocd-repo-server 1/1 1 1 109m +argo-cd-argocd-server 1/1 1 1 109m +``` + +## (Optional) Access ArgoCD + +Access to the ArgoCD's UI is completely optional, if you want to do it, +run the commands shown in the Terraform output as the example below: + +```shell +terraform output -raw access_argocd +``` + +The expected output should contain the `kubectl` config followed by `kubectl` command to retrieve +the URL, username, password to login into ArgoCD UI or CLI. + +```text +echo "ArgoCD Username: admin" +echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" +echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +``` + +## Deploy the Workloads + +Deploy a sample application located in [k8s/game-2048.yaml](k8s/game-2048.yaml) using ArgoCD: + +```shell +kubectl apply -f bootstrap/workloads.yaml +``` + +### Monitor GitOps Progress for Workloads + +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. +Use `Ctrl+C` or `Cmd+C` to exit the `watch` command. ArgoCD Applications +can take a couple of minutes in order to achieve the Healthy status. + +```shell +watch kubectl get -n argocd applications workloads +``` + +The expected output should look like the following: + +```text +NAME SYNC STATUS HEALTH STATUS +workloads Synced Healthy +``` + +### Verify the Application + +Verify that the application configuration is present and the pod is running: + +```shell +kubectl get -n game-2048 deployments,service,ep,ingress +``` + +The expected output should look like the following: + +```text +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/game-2048 1/1 1 1 7h59m + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/game-2048 ClusterIP 172.20.155.47 80/TCP 7h59m + +NAME ENDPOINTS AGE +endpoints/game-2048 10.0.13.64:80 7h59m + +NAME CLASS HOSTS ADDRESS PORTS AGE +ingress/game-2048 alb * k8s-<>.us-west-2.elb.amazonaws.com 80 7h59m +``` + +AWS Load Balancer can take a couple of minutes in order to be created. + +Run the following command and wait until and event for ingress `game-2048` contains `Successfully reconciled`. +Use `Ctrl+C` or `Cmd+C`to exit the `watch` command. + +```shell +kubectl events -n game-2048 --for ingress/game-2048 --watch +``` + +The expected output should look like the following: + +```text +LAST SEEN TYPE REASON OBJECT MESSAGE +11m Normal SuccessfullyReconciled Ingress/game-2048 Successfully reconciled +``` + +### Access the Application using AWS Load Balancer + +Verify the application endpoint health using `wget`: + +```shell +kubectl exec -n game-2048 deploy/game-2048 -- \ +wget -S --spider $(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') +``` + +The expected output should look like the following: + +```text + HTTP/1.1 200 OK + Date: Wed, 01 Nov 2023 22:44:57 GMT + Content-Type: text/html + Content-Length: 3988 +``` + +>A success response should contain `HTTP/1.1 200 OK`. + +Retrieve the ingress URL to access the application in your local web browser. + +```shell +echo "Application URL: http://$(kubectl get -n game-2048 ingress game-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +``` + +### Container Metrics + +Check the application's CPU and memory metrics: + +```shell +kubectl top pods -n game-2048 +``` + +The expected output should look like the following: + +```text +NAME CPU(cores) MEMORY(bytes) +game-2048-66fb78b995-hqbjv 1m 2Mi +``` + +Check the CPU and memory metrics for all pods for Addons and Workloads: + +```shell +kubectl top pods -A +``` + +The expected output should look like the following: + +```text +NAMESPACE NAME CPU(cores) MEMORY(bytes) +argocd argo-cd-argocd-application-controller-0 43m 138Mi +argocd argo-cd-argocd-applicationset-controller-5db688844c-79skp 1m 25Mi +argocd argo-cd-argocd-dex-server-cd48d7bc-x7flf 1m 16Mi +argocd argo-cd-argocd-notifications-controller-7d7ccc6b9d-dg9r6 1m 17Mi +argocd argo-cd-argocd-redis-7f89c69877-6mmcj 2m 3Mi +argocd argo-cd-argocd-repo-server-644b9b5668-m9ddg 8m 62Mi +argocd argo-cd-argocd-server-57cbbd6f94-lp4wx 2m 26Mi +game-2048 game-2048-66fb78b995-hqbjv 1m 2Mi +kube-system aws-load-balancer-controller-8488df87c-4nxv6 2m 26Mi +kube-system aws-load-balancer-controller-8488df87c-zs4p6 1m 19Mi +kube-system aws-node-ck6vq 3m 57Mi +kube-system aws-node-fvvsg 3m 56Mi +kube-system coredns-59754897cf-5rlxp 1m 13Mi +kube-system coredns-59754897cf-fn7jb 1m 13Mi +kube-system kube-proxy-lzbdc 1m 11Mi +kube-system kube-proxy-pdvlm 1m 12Mi +kube-system metrics-server-5b76987ff-5gzsv 4m 17Mi +``` + +## Destroy the EKS Cluster + +To tear down all the resources and the EKS cluster, run the following command: + +```shell +./destroy.sh +``` + +## Appendix + +## Fork GitOps Repositories + +To modify the `values.yaml` file for addons or the workload manifest files (.ie yaml), you'll need to fork two repositories: [aws-samples/eks-blueprints-add-ons](https://github.com/aws-samples/eks-blueprints-add-ons) for addons and [github.com/aws-ia/terraform-aws-eks-blueprints](https://github.com/aws-ia/terraform-aws-eks-blueprints) for workloads located in this pattern directory. + +After forking, update the following environment variables to point to your forks, replacing the default values. + +```shell +export TF_VAR_gitops_addons_org=https://github.com/aws-samples +export TF_VAR_gitops_addons_repo=eks-blueprints-add-ons +export TF_VAR_gitops_addons_revision=main + +export TF_VAR_gitops_workload_org=https://github.com/aws-ia +export TF_VAR_gitops_workload_repo=terraform-aws-eks-blueprints +export TF_VAR_gitops_workload_revision=main +``` diff --git a/patterns/gitops/getting-started-argocd/bootstrap/addons.yaml b/patterns/gitops/getting-started-argocd/bootstrap/addons.yaml new file mode 100644 index 0000000000..e867a1c878 --- /dev/null +++ b/patterns/gitops/getting-started-argocd/bootstrap/addons.yaml @@ -0,0 +1,26 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: {} + template: + metadata: + name: cluster-addons + spec: + project: default + source: + repoURL: '{{metadata.annotations.addons_repo_url}}' + path: '{{metadata.annotations.addons_repo_basepath}}{{metadata.annotations.addons_repo_path}}' + targetRevision: '{{metadata.annotations.addons_repo_revision}}' + directory: + recurse: true + destination: + namespace: argocd + name: '{{name}}' + syncPolicy: + automated: {} diff --git a/patterns/gitops/getting-started-argocd/bootstrap/workloads.yaml b/patterns/gitops/getting-started-argocd/bootstrap/workloads.yaml new file mode 100644 index 0000000000..abebd17f9e --- /dev/null +++ b/patterns/gitops/getting-started-argocd/bootstrap/workloads.yaml @@ -0,0 +1,32 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: workloads + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: {} + template: + metadata: + name: workloads + finalizers: + # This finalizer is for demo purposes, in production remove apps using argocd CLI "argocd app delete workload --cascade" + # When you invoke argocd app delete with --cascade, the finalizer is added automatically. + - resources-finalizer.argocd.argoproj.io + spec: + project: default + source: + repoURL: '{{metadata.annotations.workload_repo_url}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' + targetRevision: '{{metadata.annotations.workload_repo_revision}}' + destination: + name: '{{name}}' + syncPolicy: + automated: + allowEmpty: true + syncOptions: + - CreateNamespace=true + retry: + limit: 60 diff --git a/patterns/gitops/getting-started-argocd/destroy.sh b/patterns/gitops/getting-started-argocd/destroy.sh new file mode 100755 index 0000000000..06f44bf77b --- /dev/null +++ b/patterns/gitops/getting-started-argocd/destroy.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + +# Delete the Ingress/SVC before removing the addons +TMPFILE=$(mktemp) +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + echo "Deleting ingress/svc for game-2048, takes a few minutes for Load Balancer to be deleted" + kubectl delete -n game-2048 ing game-2048 + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + echo "Deleting ingress/svc for argo-cd-argocd-server, takes a few minutes for Load Balancer to be deleted" + kubectl delete -n argocd svc argo-cd-argocd-server +fi + +terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -target="module.eks" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve diff --git a/patterns/gitops/getting-started-argocd/k8s/game-2048.yaml b/patterns/gitops/getting-started-argocd/k8s/game-2048.yaml new file mode 100644 index 0000000000..c261a166dc --- /dev/null +++ b/patterns/gitops/getting-started-argocd/k8s/game-2048.yaml @@ -0,0 +1,62 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: game-2048 +spec: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + namespace: game-2048 + name: game-2048 +spec: + selector: + matchLabels: + app.kubernetes.io/name: game-2048 + template: + metadata: + labels: + app.kubernetes.io/name: game-2048 + spec: + containers: + - image: public.ecr.aws/l6m2t8p7/docker-2048 + name: game-2048 + ports: + - containerPort: 80 + name: http +--- +apiVersion: v1 +kind: Service +metadata: + namespace: game-2048 + name: game-2048 +spec: + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP + type: ClusterIP + selector: + app.kubernetes.io/name: game-2048 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + namespace: game-2048 + name: game-2048 + annotations: + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: ip +spec: + ingressClassName: alb + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: game-2048 + port: + name: http diff --git a/patterns/gitops/getting-started-argocd/main.tf b/patterns/gitops/getting-started-argocd/main.tf new file mode 100644 index 0000000000..a4a0413d89 --- /dev/null +++ b/patterns/gitops/getting-started-argocd/main.tf @@ -0,0 +1,261 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + +locals { + name = "getting-started-gitops" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + gitops_workload_url = "${var.gitops_workload_org}/${var.gitops_workload_repo}" + gitops_workload_basepath = var.gitops_workload_basepath + gitops_workload_path = var.gitops_workload_path + gitops_workload_revision = var.gitops_workload_revision + + aws_addons = { + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + } + oss_addons = { + enable_argocd = try(var.addons.enable_argocd, true) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) + } + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + }, + { + workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath + workload_repo_path = local.gitops_workload_path + workload_repo_revision = local.gitops_workload_revision + } + ) + + argocd_app_of_appsets_addons = var.enable_gitops_auto_addons ? { + addons = file("${path.module}/bootstrap/addons.yaml") + } : {} + argocd_app_of_appsets_workloads = var.enable_gitops_auto_workloads ? { + workloads = file("${path.module}/bootstrap/workloads.yaml") + } : {} + + argocd_apps = merge(local.argocd_app_of_appsets_addons, local.argocd_app_of_appsets_workloads) + + + tags = { + Blueprint = local.name + GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap +################################################################################ +module "gitops_bridge_bootstrap" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + cluster = { + metadata = local.addons_metadata + addons = local.addons + } + apps = local.argocd_apps +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["m5.large"] + + min_size = 1 + max_size = 3 + desired_size = 2 + } + } + # EKS Addons + cluster_addons = { + coredns = {} + kube-proxy = {} + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/patterns/gitops/getting-started-argocd/outputs.tf b/patterns/gitops/getting-started-argocd/outputs.tf new file mode 100644 index 0000000000..d4ecfbf1fe --- /dev/null +++ b/patterns/gitops/getting-started-argocd/outputs.tf @@ -0,0 +1,33 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + EOT +} + +output "configure_argocd" { + description = "Terminal Setup" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" + kubectl config set-context --current --namespace argocd + argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo Port Forward: http://localhost:8080 + kubectl port-forward -n argocd svc/argo-cd-argocd-server 8080:80 + EOT +} + +output "access_argocd" { + description = "ArgoCD Access" + value = <<-EOT + export KUBECONFIG="/tmp/${module.eks.cluster_name}" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" + EOT +} diff --git a/patterns/gitops/getting-started-argocd/static/gitops-bridge.drawio b/patterns/gitops/getting-started-argocd/static/gitops-bridge.drawio new file mode 100644 index 0000000000..9d1da3c514 --- /dev/null +++ b/patterns/gitops/getting-started-argocd/static/gitops-bridge.drawio @@ -0,0 +1 @@ +7b3XkuNIsi36NW127sNugxaPAKEIQQhCEHjZBg2Q0JLk198IZmZ1V2W1mJkWM2M7SyShAx7uy9fyCIA/4If2Lk/xUBl9ljc/YEh2/wEXfsAwlMEZ8AuuebytIWn8bUU51dn7Tj+tONfP/H0l8r52rbN8/mrHpe+bpR6+Xpn2XZeny1fr4mnq9693K/rm66sOcZl/WnFO4+bz2qDOluptLUMiP61X8rqsPq6MIu9b2vhj5/cVcxVn/f6zVbj4A36Y+n55+9TeD3kDjfdhl7fjpF/Y+qVhU94tv+cAj+gvwv/SCb1Nzjp1bMdP0/+g76fZ4mZ9v+P31i6PDxNM/dplOTwL8gPO71W95OchTuHWHXQ6WFctbQOWUPDx/XT5tOT3X2wo+uX2gd/kfZsv0wPs8n7Ah23fPQZj35f3n+yP0+/rqp/Z/suO8Xufl19O/ZNZwId3y/wDVqI+GUnUzp/sNN/yJa3ezTT0dbe82kHy4C9o2eHtHwl2PcA1P2Lkd1Z+bx39eSX6eTfwC/3eFb5d+b119OeV6Ofd4NJHq79e+b11NPm5xd8ejX7naPSbo8FfnO/Xpam7/PAlyqGNi75bDn3TTy/74+CPBDuVL6c4q/Ovtkk0IyLEz7YJ9QROVPcd2N71E3Qsvqib5mfHCAh5QGmwfl6m/pb/bEvx+gFbsniuvkQGdPoaIIceJ3lj9XP9fvqkX5a+/dkOXFOXcMPSw+CJ35dS0Kp8+jqa4B2+YyKKfSy/exy8ZDwPb+Yo6jtsBw9AZoAb23sJ8fjHeJ+JH6d87tcpzY8pbA8PFt8+fb1Xfpv/svglvxO+5J8VvR+Q9u+Ecf+Dsl9bCcfoT1b6ss9XKIdgf5ad8E92OsaHT6YCN718bY+v46Pru/ybYHpf9cnTvw2Its4yeJnvdsDXXfSH9AH+TR/gn/sA/04X4H+apxKfegBG5S87K/rbzvrJ6H+I5Sj8a8sx7GfvJb5nuj/LeQnmk5nyDDC590WI8H3Zd3Ej/rT2G5f6aR+9h8j8Mt81X5bHOwTH69J/bdz8Xi+X98Ph5xB+BvnsbUm4/2yT8PhY6MD9Xn7aES6GP9/202GvpY/j3u4P3tSv9xqwwQvtf83P3gnNEk9lvvyaVanv+8GUN/FSb1+35I8Ph8+8C4TD/yZgaZggvfrfOMv67j8hQAic/JsD5CO//MyYZQ0YCLAnEF5l/r8JUCIAyYF5/gPMyaB/tznZT+Z0pX9HWsF8MtRfSr7IvweXf8JYBPkaZX8ErvTrQPtasvKpBhaAHvwFwn8G2h9Y/6tI/wci9gfl+E3EJvG/E7HJz1FxmPJ4yX/AqAayxmQCn0r4ifs+dENrvQTUb6DKL1JHoG7qZ5x8kUhfC3Hh1+LsvYb0fvAPXyo3P++wX3HyX45KoHtRlvoqMN9x6Xd3y/vJLXg3P9ulL4oZ+MO3/falDf98V3543M+6kpvK/iD87SD3pV8+kgHymbb/pRj3kZz+/bnnH4lIH9XY30IkHPl+b/41iER9llROHmdgDTcMDUAQWKQ55wAhPiHUB3L9fM/v7LbXwOkwBDQ3zuIl/u9ANOoX0sj76UFaRUn0qyj8H/RfQ7QvDIb5+pA/EeH+DWvfBP3b/O27ZSGUZX7Z1P9SBH1Q6J+ZSa4Xc/icu/9rKkMk8duFIeyvLAzhn0tzH/zpWzgCou4VhUP/K878N+k44lt1wn4uG31vZOeDVv7xdv1cYQC+razJfwQQ/KUkB/9cPwj66db0cQa8EPl3djvyG1sSJPXZ7bAfye9Y808DVeaTNY/dvMTwyG/5Q5dxcCAbGqWJ57lOv7bY1575wQo/GOI/VpH8Wil/Ec5/QUXy3bv+aTb5k9Si8I9a8MeUg38ocP5F/oJRX/saiXyDXW+meD/qJy/6dCIc/dZpvznRm6l+5UR/AYP6LPf5vl/Ob1XET4r/K8L9yc+/Gj7/mX9/Ncb6Qo/5a/T4ZtgWY2hSxD6Rj58P2347OP8qEsFRZ0h7vxoYfx+1/mYt+9213xu6Z9lfODH9OvpHlv0do/MQ+r/a90X6f3O4+TdJ1dv487dDxrc1yacuB2b+sX4bMB6m9/HidMr+ADhmvnFshvhdAUr8WVhMfNYBTg5uOK2b79Wtft2L/zm0fnOUnyP2W5XyH4Zs+rcwG57l29rmH4jjH4jwB+A4QhPEV46C/ZU4/j/fjA2TKAVQ4qcf7J9DdfQbZUGw5I8swdAo/fqf+RgB/t0g/436/vMxn/gsB895Ov0foP8nAvr8s577lzCdJr9y6y/z8/42TP8smH9pIOIXvPcPoN2/E8H/qCGqv4OOv6HBr8I4xWBfQ97fCuMIDcLup59v/O/3wjiGflvIoD/k478RPSd+qfb9bQj8VLv+dkvcdf3yUfn+96I62N9MdT4S4R9AdQAN+KaYjv+dQUKw/yS7Yd8m+H6VCLAfYZnvF+Lt3yFIyN+dKQ7NOkNX+vep0GCvWcF/a6YgfmcUoL9OIMC9YPg3vvPfFAP/KK//8x2fYj878l85M+dvqzUSv3f24xsL+rtGronPYxM/jUgH51c73u4V1t+7PIf9Ap9wmmBr3keHkP93dM4c3LdvAP//8cf/D27s4I75fehneLKlylv4q//hV0ax/2vG9v4H/3Y04IOf/nyI9XujUH/a6B75y6N7/yep/9Mk9dBnf4CX4t8+moD+zYKa/Kwlfjb8939u+n9u+uaSn7H0L3XTj3nhfy2peXPPj+eDf5oB8IfQFZL8F1nIL020+qaGQX6b336Bo/5R/JP5nPWAb08xIDDtdwoRw9A8vtu3/3Fz65jfUkKfSmb/6ty6P19MfDT430NMsBT1z8jivyCM/wWh/NeoDuZznodZJF2a/+6YxH5dNEBeQbNfU7J/95gkP1e2PjO1d8YQpwukVn/FI0rfyK8vcy1/Rhm+9yQX9Wepr48T/3c+ofRRZfw3ekKJ+FP4DMZ+XXUjUPrrU/zuiVfE5woeifzeCt4fVp37PCnll8rSv6zD/iOhmPrt4ZL/uIep6N/xqMEHFNft65VEvy0KG7iBj9Nb+UKjD3Wc5UW8vlzkF9438u07Q14X5D7WIh9r4KlgJRDn3hYxaehKIFtrnzedHdHksufAz+nsVaJXgk8G/E84HrgjXF+b104Fv8er2Ii27xCd+chy3/F8OgHnGqPbeJgOtl3Hju3wN5G/22Lg2CLW3Qc3qJGQdKslvhVel2TjDxh/2fTL3LjRtsXuk5n6Z9bOFH4OW6c1U1nTz6NwtY2+M5jn4+KJnu9nNlCZkh8/QaYGfsrPGJ2XY3+gaqUHiwV96APwO89T8D/F4s9JwLc1fTzPydMjwqtmVhbY0muhShqzzu3N8SBVsSjqh91QA74KzyV/2Bfh4FehLE7cPqs2V4VBVWn7LBy88hhIvW2/Dg1lqdI4cCgCDpXK8Ax22BsOnhAcCk94fh0KTgi2+GUYiD0HDrXBoeBS2m7oB4+DJwSH/m1t6QwHxfggdNVIOIaCBaMhtx/JwApyqbChzidodrnPuT6KkWvHiMueR8cVI0GLbNFC2cIH2/12RtvFb1c0GC1nPEn9VQyvqiY4sXW4OGAddXGBBpSIlOYH8NtyEdIkO41kDlgGlgHS8WtpWNWqt6QdNjtYp0rQRUMQjrRaVFp3oRDJUAdzWKYYW/Uq7H3Bpi62XLKKg8SKE+tOPHlUZ9xLsN9pyBTHm9jCjYWz+7DPXU/OwtM5Umf+HGHBLncXmbxwsWDHhYsFp93k73iGZ8UFxDbvPlK3So/OIQzuM308SgfyahSgQVQo27sM78G8MaWtHRjndhrWk9OeTPdGJgG5pPpSTOwZWYAP+4fRuZ6GDc2v2H1DrPMxGPp+Y50L0hmc6x97oUFuhl/b2daSaxgo/HOmzX0WufpQEtZlQhnr6dTKTilEc90PXelUtm1cG2abqLVH5Bt6DanTUZWOY3PRyE3DluTiIJLVtygH7qQedvqSNKjK8HnjaAjwHJI7RYJZC1qDBFfZ8Pla6AuBvGhC2avpxUdYS3jU/agThU5dGjQDHlv3zN71J569hs7N4PtGRJYuwrJrqKmWal57Clu2pPH6hpU38Y7tu6xWoB9Qs8ISA3jXvfV2udxFdZUeqzc2SNbFZn2sytmvS7uMBc5Wa2NoGAskwpcRqj5VlPsjHbVyiB2jVEjueIrk07VHtCrUpdDiCStvKttSwVWoe4lzSaZGlurow232q/m0bPoD29RbfVnplTz0LjSI1RC9pKvgBMcnWHSu91a5PA1FnqaRCs7jRbqnHnpyvevxYeynmypEjTpxucYZ0QHsAk4Fun898PYG8cjaQ4GL9ftFxbKVXugUVx/2zRKeLEbzZRw/jHJhnyluJi05u+L9lBXFHX+oh4tLXRyYmTDeMIqpeYGa1A11KPPpJXra8RJYPXsZJbuRfD/2bU075Dof6b1/KGXbMlTgu1d/XxMMXU5RY+g1sQKvACfqBirX0HgOdL60vepY6znOYtaluiL5ftUiVwB9DLxzHM+3QQMHbBNdEzkIJAnYEOGpsnQPINHcRO76uknCPyAgoI4Od1xaOls7obLbrIvusb4EAs+NGoeBwEwWijl4+AXEjWAO+Ao82eCo6C6yfH8SqOKJcTtXHg9dt62mHQtlQjxk7nG6hsgIfGdtYOwxuS6vU0206pxI9wDglcjN0gpiqLYfsWF659YHLQW4g2ZLcsaSp01oh76PQ5ZicpeAfTwk547EjbiVhiWbKnBrGLslbybeqLRTH8AwMrjkDeISsLUQUcUjcOqZ7ewI9gNgmzy8ZeiJFJuu7vGmivbYJDSKNrNcPkQQ30h5lIXUWzgEMcZ+l8NDeJu8rFTn03Vu1TKKhxX2O20+K4TOxnM7lIQhnB50BlgRP/hWgkPnOO2MKczHuUmlchdAdKSTg/jy4EtDFKTdlfHv2SW3amLQzrfyKHA0vJ0mpRt3o5B9PxMeaoLobbzhmXbaOFKLdKxEI6YBLF4U4KuI2nmEdeDq47Un81XhQW6o+lDmSsYtFmqdRog6jTc6fCqeZz4Pr7zRpgcuvobhnR+20+XKyZlmnOsYPg+Awe4mwc2N4zyUhnkFveNQN6a9AchDgxbNLw3J26HcDVhWhuahdaP5wO0WC93p3g6YBNYDfcXKAChBb3bATisAFLzQARI9MRsmgQvwr3su7TswS8ISDMD8HkRy7vsXKRJ0nF6wpFN3Q2D2hwsMeuyLBgY1dtnOlGA8QSwVPNI1iZAkEfHwNTQ3r8c9tS4jsgHTt0Ge39Kt2RKMLQlT0IZAc8dcKFIHpCMXGhln0QxcriHyg92oMRov5wEJz/XlhImv6JVudainx1ipiKBmgouOMqt+uHsab+/FNnRgjzTcZe4OsuStFDlb5uuwGx7gbAlAIc0/5/kznGNq1evYfQTxU2XGCDVKNuhrubgtpTOC7EeD3u7nbbtWON08Fr9ZkgDfJnTE2gFwrvMBZLlU6Pk8SaHbGlhhXnSSMc9cGZ4umu+sOebytDzxin54+OfFQ50mCvxuod/o1LXXFD3cz5U2sanCc69A2gbZ8YIlmPxxD4+SIYYyQfvmgBmP1gdN9ttjFgXX++LPp81SSiEaUuWwQ5gE2Mv0wMk2mi0mP04AoOzZkeMnuiUh/9MD0sOS03yR7zUZL/LV7ITuuizM814bLjpfj2clg/PPpaIoUDa7nDH3ER9UoxpPFk9ZzyBMBC6Mr2nUzmy+BHT21h84CfI4iKlncT4+5ciVuSifHmLknG4P7sKdBHjOKwjeRZWdW6I95p55O7Ip61sCQFSu0WAoLzKBywA5KoTKrAsNbOnayEm4kabrof6Zs63DpCDe4xIDREXlAfVQCzphYXXyvY8rcPPI4QHaGT+Dwx2iFeAWzKYDgJbM541yziqgu8LVSTiaunbycu6ZTRVS5+DGie7YmvQgU8qcxsdaPH2KzfEMECcWz/RIOhJa3kOw3Jq0a0CyAlTsgvJ2KsenYMaSbLoBNoExFA7Cdpgi2b2BWKFRisxi4NUSsUk6H+r6KbUPPGeLPHcWBy1oyiSgHaNzNxcFTnCjtK6GvG0f3MrC2gNHW883ELWJ1IwBI9x8ugAnpKE6h6kMeCWwDXXp6abzwY/r4Sc23y7NA02WTrnexwMgGc28BXGvAt8PfDLvhvUNmnmQSUCm4QLidPVCQSphpxTvORLELYjf5OWV4yG349QQOKvt6AG9mrzU6lirQyWS0FEV01QFyGjWqc/2qXZX0rRvEjfBMcZsGOMWRgd6uvilj8hOWQIkh3CGgni4qI+0SRaMyuYs1tNWGdhiBFzoWMB2PO79qB70IRcMx4oThym6h683td2qNZmtjd1L/LHQPbt4nqqZhlxXm9Hey2+kx6FRkQpkPIbPZtk1PjwFox8BDwTgpwCwzItixZ05V5ZkyNTDuy17yrqSoEMB7g/tM9u6a7YpywVvbmol2w8Zthsi6pMBCuEw+6URtsdHG4xoOxdXwK1mFjgNBCF8hnntkMj3NGyWfo6tCa/FuMMy+9QK4JCIY7GTWbuS7fpocQG3FAJ3t+tjfdRh7zLz9M5RGMBXUwQF3BO0cIF9hIP4zDBZcFbQzmPP2qRzrB9XxK+gE5y9Tl+wa+Ccj67heXt7Y4qr7lPMCtiqG/tnGDUTFTSyb6XqwaadIVxb5WhLq7TO7bHVwmfhTuET47Owea7ZgwPaFAOpm8dbsgoj+XwLanql1rVC1hvVeY/Su6cXiX1rcKE/bsPZa1SgbY+pEmFdR0n6RWZmBUgIT+J3B7AyQWQs7nHV+MpZAQqoh7RL2AeR5sqhurw5IUNCtgDR4rREre8GVogf/eTBJSnIr9Q2xYBAxdIeCU62vR0jbXNzi9tl0kAbONdIjKSH+mGR+pNRilHszS2rIOx4vh/8h9SQxUUHAlciz7QVgSwXZOtFa6I4kLqDieQdAnEVYN0KZDSinUFYPHcGN0VBR1LtgdkP34+CCHAz6R4Gj9V/HJQKOMENZBcK9Ci0wW3yKRLyb9mT8J1qAOhEVMQ66uLUj0qWVVsQ7ujoO3GArpM33hC5R88jUEKJ6uqggSibvN0blIvgX6VtDrV58ZULcD5whKvvg8uAaOJLZC8ZmcvBH2c0T6Cxfaa/KOrWPyOG2oESy6+w+yFHmiggNZWSQLnznDkGHW5q59J4UmFLMy8D3A2CHGKCdISfHilkolcPOWlOMD5EhE7ToqAe83R6MHlxsort6FI0rkSKHM7W9JbnYKg82Gzd3J3wLEQXbqnbOjfSiE8ym6+XdOsYohvsSxQYk9PN8tOtNd/TZpHNoL/Fp9zVkmAYAX2KkbVBtMHTsDLaxSNmXBWOUPTrFSTinl/vgiFyIrNnDCDs3l3z3BRodAA2IETcU3QFt8yc6rqvj4CX8dFTiz78NcOle+nUp/oy5JqCU5fzeAebYteJOYjV4DPLSiBSZhBXNIDXCaEs4T4bwy0UxAiHmTDtOvx+9tEcLz46i6/s+eJOJ7viTHHIigmKVJckGCyb4nGxht40nnKP83v6zm3yxkKpe2TAElpunY4Dwqvhhud9Q8tnvn+ijphX7iF8ytzzlaoRCDfxNX/n2sV2eqYotRUUTdOE4J75i5I4Fd8bfH4V91Q7SeaBI8jTIPCokmoZoNBZejjkXDFzxgT4qOqcAD0MFbMK6H59SxPg7w0pb/ao2A9m1p/izCglWJmWB75x2Bv4eFby8AFpBoA7X3pxGF2pnwNCA99P7VjeeduTQN48rXgB7Ldgk8eVrUI7yP0SQNx7y+QTSrHaIB1PQMA8RZFQDGr049lUrnsIPADQsqY80cdVl+v72SJOj6db9is90yd3yDiQoZ5vlgd/5ROdP2cxMtT5YAznkQ1uD0k/KoR37ImxeihO7x/rOx1D6suHjUwBai4Ivc5VVllfe9qEmiE1DQXk/BujDJbKWoBe0PCpdj4dQXa8zU1DbSp6UYmEDyfBn9D6DtSa++76MAGDj0o2L5DfouwF33c7xZPlTrCZfzk+jvXQNqeumFRA8tYhvjSs2blOkWtwIo+8ycSqX/dVuL+rKb7stYP6vF2shz1hT1hgi1wt0oZkgK0CPCZWngRjgqjWeHMgzAIW51xZ4y2f3z28B87H0NKzPS8IAiLgamyNUDIWyrpeJGedCfTXCrRppvgoUzTM4g+LGhbDlTmU7EXFM7YYMp5LBA/d4qAZYPVGKh6P90wFYIgEDMpcL8zwKOebV3studBsPvg3rxSPiC2Xd/EsTirMsSCZJi/5DOuJHcY+6auB3qhTqkPJv9F3JlX0+9upL0NxIdl4utA7wYnxXAmJ5Nv2mN3vV3TOF60914BbrtHRsM0ZDeWYu48hIt+wDqFxHO9WyFfhmeCrB6Vc4d3HTQ1UbGkLzJGOIl9GK0YnC7viCUTTWbivqSLsPZxsw68dTrPgFCtl1v26Dw9VGw7a+pxfap3JTakK/XiKEjWSgR67sZC4SYA+mqCz91XaQbeVZWopzXXb14cIVM/1mLCpPhRqZAI6AtlIsKxJH4a27sdCaOSonDsX4fzEXObE3Nf+6AxeGQQmOBZfdDs18cg48jyR49s71i5d1L4sal25u4Hl0CO0zMsc2DKFeHhAiLPJUu+b/nZAfQ8XIOwBgnhx41WeLwG0FyqCyuRrNIOMu+oTSMzAeyUn8Adwo4lW9zdtQ0G2TDTLkflbtF6KDR/PN9WqfOgz+pBNDKPOQqNUAdV4GGiW6XL7TQVyE1ztBD5z0qlpjwtvE5eAHiUMVkLpuONfNcijKCjuSp91rttT9jywxkmA8lSPdK0dxuQBWP1bEIKVLHt/Hu7hfBPHZjbcI4wRDLnfgZazHoDrgi7F2GIDkbherA5kK41UNBJkx2GiILPIN7e/nU9KU98jYu0GM+6mLjzvAhcJSUuvC5xsxptP0OZDxKuDCYgnHqMo9KIX8BeQ7ZXGZDya2q+KTeYeLWDqzzTQJdjIzIRlW9tfoc1V4iAm641jmEOmP3RyQW3N2tJgfpx8j/SO876GkS7f97SwAOSBDqkATjKWsN/E8gY0b24MnQENcC4JDHQYdECpnP0G8C/REe+whEl/wcB1A8qHAL2e3wPruUI8Z1qQx4AjXu3z9jirRajFIAEx1rZte3oRQnCs0rNSX3RQNlrwLgeXZFm26c6Lo3TeKYxYRpkyYYSKym983/M116WiMWmJdpUhk5YTd92q1JlpwIJsbZ6lfgWNB9ihDJBA8xvIlm+t3Mt5q45n0eXdLUHe1Y471bT5SEHvrXSOs0+3KHwpu+yMVsH8AWAgPrEkATg+DOjjKdsrTXV49i2LKiABRYoTm85uivHVALKPXPzLBcUekdF6KuWL5uNcemMbgw3PFLdeFGmh827i1GNy9qyCUKiTbQdnC1b6AOcYP5TYPB0fsP6qH/b5+LAEPMMyTWMGl2BCwUiAhpwFbjdAjrsTRKpJt0h0Um4A/HRCCUMWsjaq7XvY3qiW2SyA1fkJ6En8KFZlJBfd9E5j43q4BM8q1O0RLHEjfoHVlxQorBebG4GOUWrAgMDCWJDIbN4ubQtzxONAoIXtXe9356ZGvnYA0EMvQM7CBGt2kTf5TKLckUxyUvGYqBB7HUAjnvAhagpQ1c7qnrNy5A86RxtneVYGKt9oDNYtJCCqvejqJYHMxQLwUeWOTxdhqZjgOmawCjFNMUDUo9GadXUpYeQ0TxMceO8TRJWfsY6F+XLwxMOaOLHgALESLJDF8zZUgt2dKaTz6Dgrodga0XGEdYUCFNK864kyFdkYmnSqiEB+YvkTXg8IuUtF+IC4ux17E9K9YV81zcJK14sB2ZGNEgnCy22JxzqlOfplGU3hLvT2JZN7gxz5uml5STR5SzfiaxyjQAdOcMhsg32pS/UeKg6WXXvPLQ9LNgeq5NwSub36Ey2AW3OqSeAWBkhBNVwCOvpC1WlI90Abh+EwrsCRV3bzsQkFesEfj7sXq056BiCa66ajcOLaJrX/uBLX1fVn2JGXNN/Da2MjjJ0cwmt9D7jnySItlTa3CSJXAvKn3lQCkkO6KOwMrEuIjY/0MWiyScE6wAb3i2NEq5gTWEEdnmcgEyCWQLULo/u52wh6EgqHcg+JiwyOdosnjrTvyUET61AgpMt1l0tGgVXR9GJVzNbnSgXM7iM8x0nTrZYG+2RiUHJ559HTKk+7InmFmE2tV60tTg4+rs4e8m0pDUJjt6JdQ1z8YiiYmGFVsuu6mzyd01Kd4qfVWESYlTY6a9IBRInlDrASx9saV4TYdLVDVPZQx+NV/QxLPB4QAZk5IKd57x+Lezxmz4egcUf3QECIPNg3kIH7I6GIZB9rNb4881sIqBMsBZyWNIvkqBXK3ehFzAPEJubLVAlBjn3Mg1Z7zkE31vs49D0s3ztRxWJkK97b4g3FpNY9dVOD0UsHpznxAIPZO9A6Z0A8EkvsugOfJmKlw/GvaGKx8+lJ1Dw3AkkL3O3wFIPKE4VdRGE0WggOYEUMTVbOji0tQxx2bxLfx3AwRKnSC1wDWFYL4hmkY02T7PqgHlTUoyrGUAXuLioUxChx7lmFgKizpxJ/Awn+pTAPJWU+N6cpsf6ZGEbZcZ7HIVi743qQou0MuP7aXe8EADJbAWHZ3kHWzEEWuebRBEcL+codDeE2SXp/9RetLFx/OEkDgOol8Jc4M/EchGbfYi2I/tWwHjiShjF3wvog0vknzm6xWUNme7ynG06wiXHEQPLruLU978FhDiZPyzjW4R5eo8KKhgMSITg5nXFXm1yWt8xBHr1doIrC9U78cALygoeM9Cxej9cKocXKcPsAoMTSnzTkGW9y45JXgfbPo++/YkOo7FumVIxUEoAR+IdHwdXpyoou7IRGFo96qZS9KfTAlpCph+3pOrDx6N5vVJfSVyYCiHXSzwgU+c2QBOQat8BSD6gdHqIp2j5tJlH6IEIbIvrJ08Z1GsFt5ODe6904cO1TZrLRIAbmeIccui0NBIPSHNaa/MMgFVdkVOpp9FpRrvQQjhqSYSDBERYg0KJZtEtR4egOySmbOx91GbOEqsRA3wMi0KBTIj/mMb45NXM5drIYQSlb2kcFtKenEEUT+9tNh1e2IM82BzSYJq6o3EvfraEsspCFwHoY1GPsyiuW60i07To2B84wPa4gcaSAnDyzDa92V0ROjzTdz51RAChcL5fxxeMhIL+XLqI5v9zI6nA9ILelfuvHoLdb+XywGkEMRhRruX0+dTMIwQAa8xRswVXYxE4T703lGfztBDJXMHhUNsZtX868gc47lcaOgdOPFGgskL8ajMpMWK8qiteoYYGRF5OeA0gPJf0KhKLda2an32qAhFzXoBmeCyGkkxUcNc5WtaSOEsuh3oi+DxoyHOBQ6iNu25H05ELbmLt6wLa0z3TgICWQGA8XUiZwLXbVFc33TmZ3dL3QkB3vyAvi3XBuAGRk52aT0UE4uPAuy+OBO7EsM79V6TOtOaPA322gZGH10Of2AjU4o7lX3tlmG/G4SIA5oVDD8HQGtAWuo+3jfoAlrod30+uHUwVGQqJ+VsCtAhtTgUzVNM6XqpOEnQR8ddFd5J4uwex2DOx2UoIJREuFOoWwQZhCyW4JRoOfadTqsHMQ9CA5q+GS0tzk7vPRe4eqXPcsDImqffpmnt7QODDOE5o3CJZ10VstvRtsLz9mLTLaaqrldxo09+ZGDWlxBxACi+IgwRurgnZLgPYvnsx2jOX7UhHCteH46yRa2DOfsHwPOOQkOMhrrEFwZ3xCEntkSqVjtNZj48DiTVkYM6FZaDJP2kdrQ7/mRngFSYyx3ZmapErpKQKCIF4vzhxIbdSwrKJIQoovDPDyR9qoEfLym2mDdRyGTlRWjcPbPoBkvCJjs8IaxYviARe7G2exKWe03NUDOTawVoifnulzJs1FnpGJfyh5mnrMZLtCnz9zC8Syv+JTs+J0CYegczji5hy1Csg55VQTaX6Cl920C7Ko57HCnw6UDS6cQRHAelmy3IzqJh5vO7YzN1Z+1DQcGJ203ShkqWf25/5Eudhl9CqFfARPekRTSkI+aPVQ+anepVBNHJHcvOjNg8xAR9F8ojCtfkox2g0fV1X2GrFvzE4o+ZE6tzNvPQcJoaLpRAJFbJ9LO3VS9ZHhRbFAzynW7jLt+gVnTWGiQCfTKGm5MO7cAYECgi28aiKHK31ub2QulEKjneA8EFi6AEosQo+RclWKPE8Lc9ABmK8PxJHG9Vq7TWoah8i8voVWEGAQmwEjpVe9JDRIb5QDUF8Kv6e6aql8Gcq1gAhogVzZ417fwqsaAYeq4OOZ0Mt0/hKI6MnNv/gdTl7DV9aFUb5F1anUjKo2gKS8KE8SSHtYD7YDc14qg6+gb3SQJrPByHqZ4qC9SojRrPOcqDSux0LeCQw867NlrBjHIbIN6O0e3Z4RIMEy1eTPUDX6E+cxr9o+xLfNutYQDKPiSZqnltyS0jYUkHBTHSCZY3lMbVfikKqa/Vzvx/MjctU4afGtffJNyGFuN7SKy+VhyfCxS5ruDY4XuZfwRA/GqWPglVKT40AW0EA654Vk7PUK3i8dmajJVvkAFPoNqJZChZmKlvnzDjyYKMIGaEeospdhguqzURPCNPSOQd9qp3zg5Ve8uBO3IHLFCD0Reu09Iv9mYmsqh0EK5V9v8UeeOwLgwrAMozPs5LvhWa+Zzezc5aQu1pua90Hmv04zQWB8EkSaR/FhRZMAGdfAb6OVDufQRJbzWa6Oq/4g5m2brx3zClerRLZIffCNM2EqSQcooJ+8CNQ+4N8wT2JJ81TR2LcKntjMvChw+gKyhIlHRXDMqn6O4bCOTp8zRC4ROMFNwnHFsmgg9p+sNlhCSQTSyUXYJODXk8L3hiM7LZmZzbnewn4hVJDdsXFshzW5xXTKeTI/J+JdqeicYXfX5syKNEkrsgTCgcWM9AI0mOJid6UwDi1Qq4K9NuTq3ArlfIbI5fjKpXkMF0DE2GLD6SxdNxHomjbJVoyO8jfvNX1xRWqGmfupAsSAkk9YtkXIk48h/FOkerKgzISDXmcfnPUeaHfAs7t2P4rCa4SrgwDjEckpeokFmAQZbKx2Ni+ON2xL+pt7Sl5pE/S/CAyyl/KSwKS+TRjNshvpIoJwZz7K/c/na/SKmbe3xDl+KYcDRy8wLOp1QOWodjhcMHG538rnQLDm00Yq4X1onP5yBGJLZ/DjoyRJFgCu7kaDmBtqEdQdDhpONE09XgNisMx51AfbxloandAHlblbsSwsW4duW50PMJta8LxvdYcclhaVMzPrJ2jqwl21ITkf/HFjnwH0bDgjCUbN5XKB/jVS3PuYtX5gNv7OdmIVwiFmQEEGODmGXR2SYOKTdF4CiKvqmViTRvNblwRpftObGkcBClrK9Qln0Zn03ZTsVb6xLbVbl7oz28ECkvs1+UfXIY99aAtFsfHCWg6So2fJVC4jqktA199zK268dtDO4ZiPnCodG3WM6/pYRHcgvvSqe97Jt3l1CqxEn8GVFKdH0m5x2uEaIQvUBtu2rebd1Ui0Kjk1h6N1zi2UmXYfpWNt36pTpcIpdnBiyVbQFEW2UQswUQeesohUYXX4iB5apHjYRnifY7mv6FI95c9b9OgPZLENzzDsUVpBSYIIL9u7N+S54cJiHm+Ve2glS4vp1xCDM+OAfaDpcIo9nSyIvGfxEdxUfn8Vtk91lvReX6iRPsD8gmVTKEw2i5hX5GngAEHWwDUZdD7ccWk3T8pmXYKAsuP7nMhZ56OBn8y43jwH1VGu1eOeu7cjAuvMljHLcDYDrByGuVxNMbJIdQuZbx9cCihOgRwp5olAeEMFSNCxojdeZTq7td75cNalGjufD27/UHcyJUdAPXJtdweCMQ+q+ChsB5brMF33gH525ScJTX2R0OyMDNpRpWfGVAQ4sEudtpmWn0QEO+5QOaIjegDL1y2YkoVkHvFdnS8SyEuB7oy4ejsJNi3f5+I1aSOvjhfpmZZ7ahFQ1gOflA9lqSOSY1dSvz7nSdXOt0kMr/LlAT17Z8TKdCdwywtQoSDFAQHleLEUjBTQSOkcwhUpuCD0Rl/YUrVMpY1sB4ZBcJw8jDBOZEVBB/WCgqt4vQXJACcAAKB1S57GsR9ntmUAp5GpZ3GbkiI3hu006ESoJMVhgK9k5c/XsZlGwtzaYU4Odm4BivjQ0GR53AsDdBLJ24HUJRt+xfWnmMlOQ1LU2ZuQvjWvNdFr6nW8XlJmRpZQp3FE3KQWjicLgC4GLpT+jMbYGzmas0HlljdT+AsxEhppw0kOSUEg3gaNJAbuXmuq9gRMNEuwKBCfQQTYzVkF7Og8Js6zVl6eNuDdrBjoNgfG4SQ/86OCKYKSr52nn6HE2AA6MWcLKjVH3o6r8cjMZ7rqQL/jydafFR9lk3KY0DfQvG4UjvONPbDWggNQHcZQpiMFoeK8wCfYvR16teHIBJyRojq6ugTbBc4ARfyCfh/F3OiVyiWbujsUPheE5C1olueBR60hnoS1PAvwEQ+JGcxC1FmQyuDw6sA4lwuO+fkpOQ/eVplaJ3Gw9Xt6pxXYxVL7VlXl79sgwKGrFhWwSKsjKBEoxMu3J8EEXWoQ6b1+hsR8ei559vBHWAhmmNfsvvD6DumKqfjH6GYNV5hhL+XtLGJS42bEVgiHwyOzOjgTxvboh/W6LUimiZuqNxRTm2oV6jF/lJFrarm95zEHWGxkm7Mft7Ezi+XaEK04ZP7YOMaTDFB8kq0ubJezMc+T3T333c5B0IPwS+0WBfrgklQ7VxAdVF6TiyzaetHbgzG8JCP4R9gl/EV27xYYqMEziFWMkgZnAcljDuZY309D6V6ca8tJs18ZMCdfV/pKrvMKZXj3JNE5x2nAP8Am8SqPmpOvmFb3w6l9Jvdch96WAjx9Ei32lgDtMzTeTGSRetipzqOp2KM1WJM8hWcrHtepp3KGqP1LXY8+yxTWg7gzrxHnnvNHOGVqYFm2apk5ehyxAOa4NLdk586yi1xYSslYV+h3sLh6+OFjkPeoj/fiisE7JEHkLbIi8EDK1n2v9Ysh59ZruAWf4Y2YJL5RZq2ODu+DPNgF/ulsAtI1qFmStmc1N9tOLtvzczSJzkrxDNPrfgRId3tM603XDvNk4kJpKwOsZ9cP1VfJbG9GNr9e7iGcCYiSuVK3J/sA1MqTGwRaYW8HbTrx42ko5MBRXuMTc4d3quwYAalhpnIWF41pT9fyCfg2yRiWhd235JVV4WMMIPtfivo1Y1W5uBvFfmEaUg6/pSDKn/eu9jxWh51/ibFmOj82+Vhz9eJfFyefpNC1dAhzlmwViFu4c5GGQpyc4aCGebAZCyoEXLwjQYeQImUqGZ7RDqunXSryfWy2RAsuROuuEvWeqUynfdp8GJckQ2WQtd+86JG+avVOtt0tRG19YvYB7KkeRHxwgZ1IO/k6MkwaJs1EvGY1y69xAwwWJ3WYExQOKAgXzZ1XBXjw4+A2MGm5FZus1/ExgRXOWiY1RyRNYROvG8x+5KC5N+4oP+arTjRzc1NrrwFKeBgosL/nXK6IXR+zBuhJzAqW5gbnZOgCWsMMAUt0d/kMznsB8qwNmqfVSPP8SES58W7gBgb98fI3S6W7W92vOkqhiYNMLZ2jJ1YaYvlWF0URRV6nvMWbAblozd1CgUtmErEenLPT2fx8DJVShAzfzrA+E4EMeTajLD0+Wgb/UlGGv6yc3MKTXDvn5TIB9hj2ge8/fDPwK6h9WZJEcbzbNivW6Ck57KkEM2OCVMeztYfi7tEX0MrGvwarmuAukBGSk15K4vjOZWwXqXXtetwNU0rHJoUVgTSDvWnBubrk3XgWUGgyLNHf1IPdelbUqAa/SpEbJ9c7Qkcbg1mp/So9rdIpEvG2cqXUSmbMaxZZPHfI260gw2v2vN5maei3PCzbGE7ACq9qQYwPnA35wlu72M6Cs8bP5X3m9XdO3W6aMs+3VrscYA6GrVOtIlrNJ1BVRki1Z3DsfKzv/s2ed8zQDvvGs9jxNTetU/f0gc3VIdhQavGDcgU0TIqAGcMB+jvQUyyVH6BnGodKwJTxKZGV7Qsehr3LgkhmIOeBELLNBmHxNv+YsJqZZ7aBBG9zBmj7gCOYKHvklwfsSRpb0DiArHul1go6T6FUe6oI6eEhgjR9ew3+5EZuKlFCVsdcgcNBsOATlQZfv19a1bW7GMBZlxccR4HPbWtqhhdHvhGUfqFo8znDWUKMX8+eXcMxAXsU9+NZrIyn4Qb54LAWHHABKtOJCR9SNcW5RbZVAYvnJhx80MZpZGD1R1EfsnZHRzKTgU43WvL0lKMnYDHm2WtVwBYPfo7v+4TD0suGJ5P/uMY0nBfjnE5k2h0fdwG/oEvUAuu+qpLtmSPNqENC+nCDkwjgfMo+FvkJyGiAqNq1B+e+jY9iPRrMvTxSe1MeW5Hd0x3w+6St7W+fVBIOtDQYTGyfRZ7n2FNlANJGqEepq5xK2yWk+uqpp/DchZsAYtQR8RP0wftrBCw6lFZnIzxP+DwPpKe8X2414IkhduoGMk+xAuotHlC7leBmILhZ5vHC4bdeKWZfcnzE7PABGS/HO3UPXmNuE4XcBshyCDdw3admYc2NFNErhcTgVoFuzOd7a+DyU75xpajuTJ6p82k2R/YKwON0mk8Q8BjyyXQDfPTlApRz441NBGcYBCOebx1CG6mMJdkiO0hxiiazGYdH2imvOjWsREhUsxxMj5Xq/cJ5gGYil+v13qOWtXW3DOYMGvffq8JVQVCjaVm9Xu4ZnePd6806pxMkTekit25kk28RvMG5OFLxZOBM5bQ7PWFDQWYnG4OvVpde2idJpEVxrSjWrABSGwkcvYHx/CVLbTj8KhvpjfLDBA6RkjmqVAwAWLrK5+clvQKIlYfmubt2OMrWbA0bODE7GgH1qmi+4UEVY/ZUgmV+7CR7mc9tire6v2K7EGrIIYHz3KxABpqhHOf2yG/HyOzqMe/v537QQrHsJLNzO5FQTlpwDoEy6radOEq2xu2WQKc5OzWvqk4EkJZuYK7OhWMhSBLK3IrbalihfUJYQU4gW+lbkGpehQf0kR9boMeA5PrhbchgoxE2TUAMtHYLSxXKQOc7IttoFDQzgiez9jAoNBhRbQhPSKGIrH5GTsNz2EnuzUq3ay+b7ykFmgDiO8CQJkXlrlafHDY25rOngsMz2DqcRjiixujs5Uce6Acljil2wRJVmnzIZpnUu5w6UzifNWXfYQ8I6oBWPVVBOj/rh3KC4z3XFiQWVVMISU3uLi1bR2/xa5ArBE5wZhAeiiTtyEm4cDeIM3o2vagQQJF5fKz27TCZbLJ1O7lIcLQVVgMbIMiJPVY2koxyVWf8Pb3IdwODE4XrXBziSbSPg9irj1x5jefAeH0mDGHyZeeRXWRnD8Am8d1OoR3MxirQnnLg1OPOQVgOeN+RCzgp8vxTlV7frCcyd1s90MgFTwvFedhnxQXGOO2LFMh3lC6nBs6KxeCs1UKAGvpL9QamyZKQBUTvn89XYMEQs+rbMCag2a54n+M1APqZyXXpzhvvPl5d5V5kX6NFdjYEzypO67UWaROFk3xkKxN69m1eAGi33GxCsV0W/EGGSwwfODu2ACIBMNaNanulx+9UpjVgsYezTEy8WMVDrsJaJhwEVGnacM8apFBWsBW7R07NiFJb13VPlmZXACDJhuQbYsHMdefGGwsfshjjeoIvGpGQPj+PYhU1ZDdqqyiY7n/Hk6r/altgoF52juOst06FzxBf/AuTTdwRPufMcWfPNx2NPITHI3yAG+fTph6s+PUOi7qb8+X/wUfQAW4gxNsvSCVev+Gbwcj/7495nQbKfvnioo+H4b/zqjgU+c6XG1HEj8if9RoZ+vO3RX1+O8FPr819f1XkL78z9598a+g/8629X78FCPuDX5eL/c63ZNC/0O1/zdtw0W/fhot98/aV3/19F9982cuXl2/+Ra/RoD9/z9Yf44c/fv3Olt/ph//IO5//VD/8eCHFb7+8/G/1w/9hvnkRy8e30P3j37vyCSR/ZCmMpCiGIFiU/Xh/2b/slmBx6vvl57vDVzEafQbfwiL+/w== diff --git a/patterns/gitops/getting-started-argocd/static/gitops-bridge.drawio.png b/patterns/gitops/getting-started-argocd/static/gitops-bridge.drawio.png new file mode 100644 index 0000000000..603b7e3e33 Binary files /dev/null and b/patterns/gitops/getting-started-argocd/static/gitops-bridge.drawio.png differ diff --git a/patterns/gitops/getting-started-argocd/variables.tf b/patterns/gitops/getting-started-argocd/variables.tf new file mode 100644 index 0000000000..b4c7511302 --- /dev/null +++ b/patterns/gitops/getting-started-argocd/variables.tf @@ -0,0 +1,88 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + } +} +# Addons Git +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + type = string + default = "https://github.com/aws-samples" +} +variable "gitops_addons_repo" { + description = "Git repository contains for addons" + type = string + default = "eks-blueprints-add-ons" +} +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} +variable "gitops_addons_basepath" { + description = "Git repository base path for addons" + type = string + default = "argocd/" +} +variable "gitops_addons_path" { + description = "Git repository path for addons" + type = string + default = "bootstrap/control-plane/addons" +} + +# Workloads Git +variable "gitops_workload_org" { + description = "Git repository org/user contains for workload" + type = string + default = "https://github.com/aws-ia" +} +variable "gitops_workload_repo" { + description = "Git repository contains for workload" + type = string + default = "terraform-aws-eks-blueprints" +} +variable "gitops_workload_revision" { + description = "Git repository revision/branch/ref for workload" + type = string + default = "main" +} +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" + type = string + default = "patterns/gitops/" +} +variable "gitops_workload_path" { + description = "Git repository path for workload" + type = string + default = "getting-started-argocd/k8s" +} + +variable "enable_gitops_auto_addons" { + description = "Automatically deploy addons" + type = bool + default = false +} + +variable "enable_gitops_auto_workloads" { + description = "Automatically deploy addons" + type = bool + default = false +} diff --git a/patterns/argocd/versions.tf b/patterns/gitops/getting-started-argocd/versions.tf similarity index 58% rename from patterns/argocd/versions.tf rename to patterns/gitops/getting-started-argocd/versions.tf index aa00573a68..c3fb7ee058 100644 --- a/patterns/argocd/versions.tf +++ b/patterns/gitops/getting-started-argocd/versions.tf @@ -4,23 +4,15 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.47" + version = ">= 4.67.0" } helm = { source = "hashicorp/helm" - version = ">= 2.9" + version = ">= 2.10.1" } kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.20" - } - random = { - source = "hashicorp/random" - version = ">= 3.5" - } - bcrypt = { - source = "viktorradnai/bcrypt" - version = ">= 0.1.2" + version = "2.22.0" } } @@ -28,6 +20,6 @@ terraform { # backend "s3" { # bucket = "terraform-ssp-github-actions-state" # region = "us-west-2" - # key = "e2e/argocd/terraform.tfstate" + # key = "e2e/getting-started-argocd/terraform.tfstate" # } } diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md new file mode 100644 index 0000000000..72cd798fc8 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/README.md @@ -0,0 +1,232 @@ +# Multi-Cluster centralized hub-spoke topology + +This tutorial guides you through deploying an Amazon EKS cluster with addons configured via ArgoCD in a Multi-Cluster Hub-Spoke topoloy, employing the [GitOps Bridge Pattern](https://github.com/gitops-bridge-dev). + + + +This example deploys ArgoCD on the Hub cluster (i.e. management/control-plane cluster). +The spoke clusters are registered as remote clusters in the Hub Cluster's ArgoCD +The ArgoCD on the Hub Cluster deploys addons and workloads to the spoke clusters + +Each spoke cluster gets deployed an app of apps ArgoCD Application with the name `workloads-${env}` + +## Prerequisites + +Before you begin, make sure you have the following command line tools installed: + +- git +- terraform +- kubectl +- argocd + +## (Optional) Fork the GitOps git repositories + +See the appendix section [Fork GitOps Repositories](#fork-gitops-repositories) for more info on the terraform variables to override. + +## Deploy the Hub EKS Cluster + +Change directory to `hub` + +```shell +cd hub +``` + +Initialize Terraform and deploy the EKS cluster: + +```shell +terraform init +terraform apply -target="module.vpc" -auto-approve +terraform apply -target="module.eks" -auto-approve +terraform apply -auto-approve +``` + +To retrieve `kubectl` config, execute the terraform output command: + +```shell +terraform output -raw configure_kubectl +``` + +The expected output will have two lines you run in your terminal + +```text +export KUBECONFIG="/tmp/hub-spoke" +aws eks --region us-west-2 update-kubeconfig --name getting-started-gitops --alias hub +``` + +>The first line sets the `KUBECONFIG` environment variable to a temporary file +that includes the cluster name. The second line uses the `aws` CLI to populate +that temporary file with the `kubectl` configuration. This approach offers the +advantage of not altering your existing `kubectl` context, allowing you to work +in other terminal windows without interference. + +### Monitor GitOps Progress for Addons + +Wait until all the ArgoCD applications' `HEALTH STATUS` is `Healthy`. +Use `Ctrl+C` or `Cmd+C` to exit the `watch` command. ArgoCD Applications +can take a couple of minutes in order to achieve the Healthy status. + +```shell +kubectl --context hub get applications -n argocd -w +``` + +The expected output should look like the following: + +```text +NAME SYNC STATUS HEALTH STATUS +addon-in-cluster-argo-cd Synced Healthy +addon-in-cluster-aws-load-balancer-controller Synced Healthy +addon-in-cluster-metrics-server Synced Healthy +cluster-addons Synced Healthy +``` + +## (Optional) Access ArgoCD + +Access to the ArgoCD's UI is completely optional, if you want to do it, +run the commands shown in the Terraform output as the example below: + +```shell +terraform output -raw access_argocd +``` + +The expected output should contain the `kubectl` config followed by `kubectl` command to retrieve +the URL, username, password to login into ArgoCD UI or CLI. + +```text +echo "ArgoCD Username: admin" +echo "ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" +echo "ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" +``` + +## Verify that ArgoCD Service Accouts has the annotation for IRSA + +```shell +kubectl --context hub get sa -n argocd argocd-application-controller -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +kubectl --context hub get sa -n argocd argocd-server -o json | jq '.metadata.annotations."eks.amazonaws.com/role-arn"' +``` + +The output should match the `arn` for the IAM Role that will assume the IAM Role in spoke/remote clusters + +```text +arn:aws:iam::0123456789:role/argocd-hub-0123abc.. +arn:aws:iam::0123456789:role/argocd-hub-0123abc.. +``` + +## Deploy the Spoke EKS Cluster + +Use the `deploy.sh` script to create terraform workspace, initialize Terraform, and deploy the EKS clusters: + +```shell +cd ../spokes +./deploy.sh dev +./deploy.sh staging +./deploy.sh prod +``` + +Each environment uses a Terraform workspace + +To retrieve `kubectl` config, execute the terraform output command: + +```shell +terraform workspace select dev +terraform output -raw configure_kubectl +``` + +```shell +terraform workspace select staging +terraform output -raw configure_kubectl +``` + +```shell +terraform workspace select prod +terraform output -raw configure_kubectl +``` + +### Verify ArgoCD Cluster Secret for Spokes have the correct IAM Role to be assume by Hub Cluster + +```shell +for i in dev staging prod ; do echo $i && kubectl --context hub get secret -n argocd spoke-$i --template='{{index .data.config | base64decode}}' ; done +``` + +The output have a section `awsAuthConfig` with the `clusterName` and the `roleARN` that has write access to the spoke cluster + +```json +{ + "tlsClientConfig": { + "insecure": false, + "caData" : "LS0tL...." + }, + "awsAuthConfig" : { + "clusterName": "hub-spoke-dev", + "roleARN": "arn:aws:iam::0123456789:role/hub-spoke-dev-argocd-spoke" + } +} +``` + +### Verify the Addons on Spoke Clusters + +Verify that the addons are ready: + +```shell +for i in dev staging prod ; do echo $i && kubectl --context $i get deployment -n kube-system ; done +``` + +### Monitor GitOps Progress for Workloads from Hub Cluster (run on Hub Cluster context) + +Watch until **all* the Workloads ArgoCD Applications are `Healthy` + +```shell +kubectl --context hub get -n argocd applications -w +``` + +Wait until the ArgoCD Applications `HEALTH STATUS` is `Healthy`. Crl+C to exit the `watch` command + +### Verify the Application + +Verify that the application configuration is present and the pod is running: + +```shell +for i in dev staging prod ; do echo $i && kubectl --context $i get all -n workload ; done +``` + +### Container Metrics + +Check the application's CPU and memory metrics: + +```shell +for i in dev staging prod ; do echo $i && kubectl --context $i top pods -n workload ; done +``` + +## Destroy the Spoke EKS Clusters + +To tear down all the resources and the EKS cluster, run the following command: + +```shell +./destroy.sh dev +./destroy.sh staging +./destroy.sh prod +``` + +## Destroy the Hub EKS Clusters + +To tear down all the resources and the EKS cluster, run the following command: +Destroy Hub Clusters + +```shell +cd ../hub +./destroy.sh +``` + +## Appendix + +## Fork GitOps Repositories + +To modify the `values.yaml` file or the helm chart version for addons, you'll need to fork tthe repository [aws-samples/eks-blueprints-add-ons](https://github.com/aws-samples/eks-blueprints-add-ons). + +After forking, update the following environment variables to point to your forks, replacing the default values. + +```shell +export TF_VAR_gitops_addons_org=https://github.com/aws-samples +export TF_VAR_gitops_addons_repo=eks-blueprints-add-ons +export TF_VAR_gitops_addons_revision=main + +``` diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml new file mode 100644 index 0000000000..89f3602844 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/addons.yaml @@ -0,0 +1,32 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-addons + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: true + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + template: + metadata: + name: cluster-addons + spec: + project: default + source: + repoURL: '{{metadata.annotations.addons_repo_url}}' + path: '{{metadata.annotations.addons_repo_basepath}}{{metadata.annotations.addons_repo_path}}' + targetRevision: '{{metadata.annotations.addons_repo_revision}}' + directory: + recurse: true + exclude: exclude/* + destination: + namespace: 'argocd' + name: '{{name}}' + syncPolicy: + automated: {} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml new file mode 100644 index 0000000000..c399039367 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/bootstrap/workloads.yaml @@ -0,0 +1,34 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: workloads + namespace: argocd +spec: + syncPolicy: + preserveResourcesOnDeletion: false + generators: + - clusters: + selector: + matchExpressions: + - key: akuity.io/argo-cd-cluster-name + operator: NotIn + values: [in-cluster] + - key: environment + operator: NotIn + values: [control-plane] + template: + metadata: + name: 'workload-{{metadata.labels.environment}}' + spec: + project: default + source: + repoURL: '{{metadata.annotations.workload_repo_url}}' + path: '{{metadata.annotations.workload_repo_basepath}}{{metadata.annotations.workload_repo_path}}' + targetRevision: '{{metadata.annotations.workload_repo_revision}}' + destination: + namespace: 'workload' + name: '{{name}}' + syncPolicy: + automated: {} + syncOptions: + - CreateNamespace=true diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh new file mode 100755 index 0000000000..8ad29aaebb --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/destroy.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + +# Delete the Ingress/SVC before removing the addons +TMPFILE=$(mktemp) +terraform -chdir=$SCRIPTDIR output -raw configure_kubectl > "$TMPFILE" +# check if TMPFILE contains the string "No outputs found" +if [[ ! $(cat $TMPFILE) == *"No outputs found"* ]]; then + source "$TMPFILE" + kubectl delete -n argocd applicationset workloads + kubectl delete -n argocd applicationset cluster-addons + kubectl delete -n argocd applicationset addons-argocd + kubectl delete -n argocd svc argo-cd-argocd-server +fi + +terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -target="module.eks" -auto-approve +terraform destroy -target="module.vpc" -auto-approve +terraform destroy -auto-approve diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf new file mode 100644 index 0000000000..254499b4ec --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/main.tf @@ -0,0 +1,294 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } + } +} + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + +locals { + name = "hub-${local.environment}" + environment = "control-plane" + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + argocd_namespace = "argocd" + + aws_addons = { + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) + } + oss_addons = { + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) + } + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + argocd_iam_role_arn = module.argocd_irsa.iam_role_arn + argocd_namespace = local.argocd_namespace + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + } + ) + + argocd_apps = { + addons = file("${path.module}/bootstrap/addons.yaml") + workloads = file("${path.module}/bootstrap/workloads.yaml") + } + + tags = { + Blueprint = local.name + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap +################################################################################ +module "gitops_bridge_bootstrap" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + cluster = { + cluster_name = module.eks.cluster_name + environment = local.environment + metadata = local.addons_metadata + addons = local.addons + } + apps = local.argocd_apps + argocd = { + namespace = local.argocd_namespace + } +} + +################################################################################ +# ArgoCD EKS Access +################################################################################ +module "argocd_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.20" + + role_name_prefix = "argocd-hub-" + assume_role_condition_test = "StringLike" + role_policy_arns = { + ArgoCD_EKS_Policy = aws_iam_policy.irsa_policy.arn + } + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["${local.argocd_namespace}:argocd-*"] + } + } + + tags = local.tags +} + + +resource "aws_iam_policy" "irsa_policy" { + name = "${module.eks.cluster_name}-argocd-irsa" + description = "IAM Policy for ArgoCD Hub" + policy = data.aws_iam_policy_document.irsa_policy.json + tags = local.tags +} + +data "aws_iam_policy_document" "irsa_policy" { + statement { + effect = "Allow" + resources = ["*"] + actions = ["sts:AssumeRole"] + } +} + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 3 + max_size = 10 + desired_size = 3 + } + } + # EKS Addons + cluster_addons = { + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf new file mode 100644 index 0000000000..406672e261 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/outputs.tf @@ -0,0 +1,57 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/hup-spoke" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub + EOT +} + + +output "configure_argocd" { + description = "Terminal Setup" + value = <<-EOT + export KUBECONFIG="/tmp/hup-spoke" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub + export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" + kubectl --context hub config set-context --current --namespace argocd + argocd login --port-forward --username admin --password $(argocd admin initial-password | head -1) + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo Port Forward: http://localhost:8080 + kubectl port-forward -n argocd svc/argo-cd-argocd-server 8080:80 + EOT +} + +output "access_argocd" { + description = "ArgoCD Access" + value = <<-EOT + export KUBECONFIG="/tmp/hup-spoke" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias hub + echo "ArgoCD Username: admin" + echo "ArgoCD Password: $(kubectl --context hub get secrets argocd-initial-admin-secret -n argocd --template="{{index .data.password | base64decode}}")" + echo "ArgoCD URL: https://$(kubectl --context hub get svc -n argocd argo-cd-argocd-server -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" + EOT +} + + +output "argocd_iam_role_arn" { + description = "IAM Role for ArgoCD Cluster Hub, use to connect to spoke clusters" + value = module.argocd_irsa.iam_role_arn +} + +output "cluster_name" { + description = "Cluster Hub name" + value = module.eks.cluster_name +} +output "cluster_endpoint" { + description = "Cluster Hub endpoint" + value = module.eks.cluster_endpoint +} +output "cluster_certificate_authority_data" { + description = "Cluster Hub certificate_authority_data" + value = module.eks.cluster_certificate_authority_data +} +output "cluster_region" { + description = "Cluster Hub region" + value = local.region +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf new file mode 100644 index 0000000000..2e2c8462ee --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/variables.tf @@ -0,0 +1,53 @@ +variable "vpc_cidr" { + description = "VPC CIDR" + type = string + default = "10.0.0.0/16" +} +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} +variable "kubernetes_version" { + description = "Kubernetes version" + type = string + default = "1.28" +} +variable "addons" { + description = "Kubernetes addons" + type = any + default = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Enable argocd with IRSA + enable_aws_argocd = true + # Disable argocd without IRSA + enable_argocd = false + } +} +# Addons Git +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + type = string + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_addons_repo" { + description = "Git repository contains for addons" + type = string + default = "gitops-bridge-argocd-control-plane-template" +} +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} +variable "gitops_addons_basepath" { + description = "Git repository base path for addons" + type = string + default = "" +} +variable "gitops_addons_path" { + description = "Git repository path for addons" + type = string + default = "bootstrap/control-plane/addons" +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf new file mode 100644 index 0000000000..2de60d58ee --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/hub/versions.tf @@ -0,0 +1,25 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.67.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.10.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/ipv4-prefix-delegation/terraform.tfstate" + # } +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore new file mode 100644 index 0000000000..e0cc55d252 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/.gitignore @@ -0,0 +1,4 @@ +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf +!workspaces/*.tfvars diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh new file mode 100755 index 0000000000..766cffd8d8 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/deploy.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + + +if [[ $# -eq 0 ]] ; then + echo "No arguments supplied" + echo "Usage: destroy.sh " + echo "Example: destroy.sh dev" + exit 1 +fi +env=$1 +echo "Deploying $env with "workspaces/${env}.tfvars" ..." + + +if terraform workspace list | grep -q $env; then + echo "Workspace $env already exists." +else + terraform workspace new $env +fi + +terraform workspace select $env +terraform init +terraform apply -var-file="workspaces/${env}.tfvars" diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh new file mode 100755 index 0000000000..cf44b4d9a2 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/destroy.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -uo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOTDIR="$(cd ${SCRIPTDIR}/../..; pwd )" +[[ -n "${DEBUG:-}" ]] && set -x + + +if [[ $# -eq 0 ]] ; then + echo "No arguments supplied" + echo "Usage: destroy.sh " + echo "Example: destroy.sh dev" + exit 1 +fi +env=$1 +echo "Destroying $env ..." + +terraform workspace select $env +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.gitops_bridge_bootstrap" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks_blueprints_addons" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.eks" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" -target="module.vpc" -auto-approve +terraform destroy -auto-approve -var-file="workspaces/${env}.tfvars" diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf new file mode 100644 index 0000000000..617769acb7 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/main.tf @@ -0,0 +1,323 @@ +provider "aws" { + region = local.region +} +data "aws_caller_identity" "current" {} +data "aws_availability_zones" "available" {} + + +data "terraform_remote_state" "cluster_hub" { + backend = "local" + + config = { + path = "${path.module}/../hub/terraform.tfstate" + } +} + +################################################################################ +# Kubernetes Access for Hub Cluster +################################################################################ + +provider "kubernetes" { + host = data.terraform_remote_state.cluster_hub.outputs.cluster_endpoint + cluster_ca_certificate = base64decode(data.terraform_remote_state.cluster_hub.outputs.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", data.terraform_remote_state.cluster_hub.outputs.cluster_name, "--region", data.terraform_remote_state.cluster_hub.outputs.cluster_region] + } + alias = "hub" +} + +################################################################################ +# Kubernetes Access for Spoke Cluster +################################################################################ + +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name, "--region", local.region] + } +} + + + +locals { + name = "spoke-${terraform.workspace}" + environment = terraform.workspace + region = var.region + + cluster_version = var.kubernetes_version + + vpc_cidr = var.vpc_cidr + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision + + gitops_workload_org = var.gitops_workload_org + gitops_workload_repo = var.gitops_workload_repo + gitops_workload_basepath = var.gitops_workload_basepath + gitops_workload_path = var.gitops_workload_path + gitops_workload_revision = var.gitops_workload_revision + gitops_workload_url = "${local.gitops_workload_org}/${local.gitops_workload_repo}" + + aws_addons = { + enable_cert_manager = try(var.addons.enable_cert_manager, false) + enable_aws_efs_csi_driver = try(var.addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(var.addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(var.addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(var.addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(var.addons.enable_cluster_autoscaler, false) + enable_external_dns = try(var.addons.enable_external_dns, false) + enable_external_secrets = try(var.addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(var.addons.enable_aws_load_balancer_controller, false) + enable_fargate_fluentbit = try(var.addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(var.addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(var.addons.enable_aws_node_termination_handler, false) + enable_karpenter = try(var.addons.enable_karpenter, false) + enable_velero = try(var.addons.enable_velero, false) + enable_aws_gateway_api_controller = try(var.addons.enable_aws_gateway_api_controller, false) + enable_aws_ebs_csi_resources = try(var.addons.enable_aws_ebs_csi_resources, false) + enable_aws_secrets_store_csi_driver_provider = try(var.addons.enable_aws_secrets_store_csi_driver_provider, false) + enable_ack_apigatewayv2 = try(var.addons.enable_ack_apigatewayv2, false) + enable_ack_dynamodb = try(var.addons.enable_ack_dynamodb, false) + enable_ack_s3 = try(var.addons.enable_ack_s3, false) + enable_ack_rds = try(var.addons.enable_ack_rds, false) + enable_ack_prometheusservice = try(var.addons.enable_ack_prometheusservice, false) + enable_ack_emrcontainers = try(var.addons.enable_ack_emrcontainers, false) + enable_ack_sfn = try(var.addons.enable_ack_sfn, false) + enable_ack_eventbridge = try(var.addons.enable_ack_eventbridge, false) + enable_aws_argocd = try(var.addons.enable_aws_argocd, false) + } + oss_addons = { + enable_argocd = try(var.addons.enable_argocd, false) + enable_argo_rollouts = try(var.addons.enable_argo_rollouts, false) + enable_argo_events = try(var.addons.enable_argo_events, false) + enable_argo_workflows = try(var.addons.enable_argo_workflows, false) + enable_cluster_proportional_autoscaler = try(var.addons.enable_cluster_proportional_autoscaler, false) + enable_gatekeeper = try(var.addons.enable_gatekeeper, false) + enable_gpu_operator = try(var.addons.enable_gpu_operator, false) + enable_ingress_nginx = try(var.addons.enable_ingress_nginx, false) + enable_kyverno = try(var.addons.enable_kyverno, false) + enable_kube_prometheus_stack = try(var.addons.enable_kube_prometheus_stack, false) + enable_metrics_server = try(var.addons.enable_metrics_server, false) + enable_prometheus_adapter = try(var.addons.enable_prometheus_adapter, false) + enable_secrets_store_csi_driver = try(var.addons.enable_secrets_store_csi_driver, false) + enable_vpa = try(var.addons.enable_vpa, false) + } + addons = merge( + local.aws_addons, + local.oss_addons, + { kubernetes_version = local.cluster_version }, + { aws_cluster_name = module.eks.cluster_name } + ) + + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = module.vpc.vpc_id + }, + { + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + }, + { + workload_repo_url = local.gitops_workload_url + workload_repo_basepath = local.gitops_workload_basepath + workload_repo_path = local.gitops_workload_path + workload_repo_revision = local.gitops_workload_revision + } + ) + + tags = { + Blueprint = local.name + GithubRepo = "github.com/gitops-bridge-dev/gitops-bridge" + } +} + +################################################################################ +# GitOps Bridge: Bootstrap for Hub Cluster +################################################################################ +module "gitops_bridge_bootstrap_hub" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v2.0.0" + + # The ArgoCD remote cluster secret is deploy on hub cluster not on spoke clusters + providers = { + kubernetes = kubernetes.hub + } + + install = false # We are not installing argocd via helm on hub cluster + cluster = { + cluster_name = module.eks.cluster_name + environment = local.environment + metadata = local.addons_metadata + addons = local.addons + server = module.eks.cluster_endpoint + config = <<-EOT + { + "tlsClientConfig": { + "insecure": false, + "caData" : "${module.eks.cluster_certificate_authority_data}" + }, + "awsAuthConfig" : { + "clusterName": "${module.eks.cluster_name}", + "roleARN": "${aws_iam_role.spoke.arn}" + } + } + EOT + } +} + +################################################################################ +# ArgoCD EKS Access +################################################################################ +resource "aws_iam_role" "spoke" { + name = "${module.eks.cluster_name}-argocd-spoke" + assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json +} + +data "aws_iam_policy_document" "assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + principals { + type = "AWS" + identifiers = [data.terraform_remote_state.cluster_hub.outputs.argocd_iam_role_arn] + } + } +} + + + +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" + version = "~> 1.0" + + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn + + # Using GitOps Bridge + create_kubernetes_resources = false + + # EKS Blueprints Addons + enable_cert_manager = local.aws_addons.enable_cert_manager + enable_aws_efs_csi_driver = local.aws_addons.enable_aws_efs_csi_driver + enable_aws_fsx_csi_driver = local.aws_addons.enable_aws_fsx_csi_driver + enable_aws_cloudwatch_metrics = local.aws_addons.enable_aws_cloudwatch_metrics + enable_aws_privateca_issuer = local.aws_addons.enable_aws_privateca_issuer + enable_cluster_autoscaler = local.aws_addons.enable_cluster_autoscaler + enable_external_dns = local.aws_addons.enable_external_dns + enable_external_secrets = local.aws_addons.enable_external_secrets + enable_aws_load_balancer_controller = local.aws_addons.enable_aws_load_balancer_controller + enable_fargate_fluentbit = local.aws_addons.enable_fargate_fluentbit + enable_aws_for_fluentbit = local.aws_addons.enable_aws_for_fluentbit + enable_aws_node_termination_handler = local.aws_addons.enable_aws_node_termination_handler + enable_karpenter = local.aws_addons.enable_karpenter + enable_velero = local.aws_addons.enable_velero + enable_aws_gateway_api_controller = local.aws_addons.enable_aws_gateway_api_controller + + tags = local.tags +} + +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "~> 19.13" + + cluster_name = local.name + cluster_version = local.cluster_version + cluster_endpoint_public_access = true + + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + + manage_aws_auth_configmap = true + aws_auth_roles = [ + # Granting access to ArgoCD from hub cluster + { + rolearn = aws_iam_role.spoke.arn + username = "gitops-role" + groups = [ + "system:masters" + ] + }, + ] + + eks_managed_node_groups = { + initial = { + instance_types = ["t3.medium"] + + min_size = 3 + max_size = 10 + desired_size = 3 + } + } + # EKS Addons + cluster_addons = { + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + before_compute = true + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + } + tags = local.tags +} + +################################################################################ +# Supporting Resources +################################################################################ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "~> 5.0" + + name = local.name + cidr = local.vpc_cidr + + azs = local.azs + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)] + public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)] + + enable_nat_gateway = true + single_nat_gateway = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/role/internal-elb" = 1 + } + + tags = local.tags +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf new file mode 100644 index 0000000000..18b5a74f23 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/outputs.tf @@ -0,0 +1,7 @@ +output "configure_kubectl" { + description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig" + value = <<-EOT + export KUBECONFIG="/tmp/hup-spoke" + aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name} --alias ${local.environment} + EOT +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf new file mode 100644 index 0000000000..570325119c --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/variables.tf @@ -0,0 +1,69 @@ +variable "region" { + description = "AWS region" + type = string +} +variable "vpc_cidr" { + description = "VPC CIDR" + type = string +} +variable "kubernetes_version" { + description = "EKS version" + type = string +} +variable "addons" { + description = "Kubernetes addons" + type = any +} +# Addons Git +variable "gitops_addons_org" { + description = "Git repository org/user contains for addons" + type = string + default = "https://github.com/gitops-bridge-dev" +} +variable "gitops_addons_repo" { + description = "Git repository contains for addons" + type = string + default = "gitops-bridge-argocd-control-plane-template" +} +variable "gitops_addons_revision" { + description = "Git repository revision/branch/ref for addons" + type = string + default = "main" +} +variable "gitops_addons_basepath" { + description = "Git repository base path for addons" + type = string + default = "" +} +variable "gitops_addons_path" { + description = "Git repository path for addons" + type = string + default = "bootstrap/control-plane/addons" +} + +# Workloads Git +variable "gitops_workload_org" { + description = "Git repository org/user contains for workload" + type = string + default = "https://github.com/argoproj" +} +variable "gitops_workload_repo" { + description = "Git repository contains for workload" + type = string + default = "argocd-example-apps" +} +variable "gitops_workload_revision" { + description = "Git repository revision/branch/ref for workload" + type = string + default = "master" +} +variable "gitops_workload_basepath" { + description = "Git repository base path for workload" + type = string + default = "" +} +variable "gitops_workload_path" { + description = "Git repository path for workload" + type = string + default = "helm-guestbook" +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf new file mode 100644 index 0000000000..6ff7c991ec --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/versions.tf @@ -0,0 +1,21 @@ +terraform { + required_version = ">= 1.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.67.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + } + + # ## Used for end-to-end testing on project; update to suit your needs + # backend "s3" { + # bucket = "terraform-ssp-github-actions-state" + # region = "us-west-2" + # key = "e2e/ipv4-prefix-delegation/terraform.tfstate" + # } +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars new file mode 100644 index 0000000000..755615e6ea --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/dev.tfvars @@ -0,0 +1,10 @@ +vpc_cidr = "10.1.0.0/16" +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars new file mode 100644 index 0000000000..22789479cf --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/prod.tfvars @@ -0,0 +1,10 @@ +vpc_cidr = "10.3.0.0/16" +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars new file mode 100644 index 0000000000..18a50cfbf3 --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/spokes/workspaces/staging.tfvars @@ -0,0 +1,10 @@ +vpc_cidr = "10.2.0.0/16" +region = "us-west-2" +kubernetes_version = "1.28" +addons = { + enable_aws_load_balancer_controller = true + enable_metrics_server = true + # Disable argocd on spoke clusters + enable_aws_argocd = false + enable_argocd = false +} diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio new file mode 100644 index 0000000000..c1cdad1fed --- /dev/null +++ b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio @@ -0,0 +1 @@ +7Vvbcps6FP0aPyaDwPjyGGPnnE7bmc7JQ+NHxcigRkZUyDHu1x8JhEESvSWxIRMSZ4yWLoi1tLe2duyRF+zyfxhM4880RGTkOmE+8pYj13Wd8Vi8SeRYIgDM/BKJGA4VVgN3+AdSoKPQPQ5RpjXklBKOUx3c0CRBG65hkDF60JttKdHvmsIIWcDdBhIb/YpDHpfozHdq/F+Eo7i6M3BUzQ5WjRWQxTCkhwbkrUZewCjl5dUuDxCR7FW8lP1uf1J7mhhDCf+TDph9/b7+L/h8e3+TTO7Xa/QD7K68cpQnSPbqgdVk+bFigNF9EiI5iDPyFocYc3SXwo2sPQjRBRbzHRElIC7VcIhxlP90nuD09GLdILpDnB1FE9VhqvhSK2ZcrYRDTT+YKSxuUj9WIFSSR6eha1bEhSLmL0ga/56k7BHxTaw4SilOeDEJfyFeYlpB+eeLpoFErl2/BWzDpjYI7GbiDbTdwQTbsKkNAruZLFWz1sE2bOrbMzZ7g5bewOgtXt6C7jnBCQpOFi453tKEB5RQVvDvid9bqegiYjDESKu7nc5WzrhRt8RMDIRpIuoTyuSqWmwxIY0+S8cPwFTgGWf0ETVqtsWPqAlhFp/MQq54LLzGJ/iAyBeaYTX8A+Wc7hoNbgiOZAWn0nKgKm3ErBDTTUk+ofKHwK3KasXJW8IsLenY4lzOYyEcTCord3kknfE1PGTja4Yyumcb9GEj57MQxfJKb4Ues4sZr99iu/65TNe3TBeGIU2yzr0ccA2mpjZT8xamvHMxNbGYOlD2SCgM+0eW37KsLkrWtGVHmBCurFQja/J9T6uKq6yw5xvRwAVpXpBT1YurSL6vPt5VY4mplcOVNZYMglCuc617q4QmyHBtCrL8jumedjgM5W1axdXlfw19PcMYXFtf75L6VgM3yEahiAtVUe4ZNKIJJKsaNWip23yi0tcX+nxDnB+VU4d7TnX1BFvseC/7i21RFddquKKwzLXSsbEXqkHdYpwc88YworSu7iCu60Fk4aipKB/y1xoKTor95Bfkzct2HLII8d/FnvaaYIhAjp/0eby+wuAtK9yBWmDcqVx2yP2G5HojBgnmL5RYdf0izz+1d/fmunf3pobbLh9A9TIWymkaz18785bNuuOAxnN0SlzH3vAufMitQqrhlDuccodT7svN96LHXNCWxxsOJM8XeGII3HLgvOiBBPxBDrLrTawHmVpg53uGTWzYxIZN7Jnme9lNzE5B9iRZ6/kGVV0na4GdgOxPttZkq/NsLZgN0dE5o6PO07Wgh0d838xpdx8dVTIN0dEQHQ3R0cvN96LRUfUI/YuO/FnPoqPKrfUyOjLZ6jw6cofc0atGR/68Z9GRa+eO4v1DqZxwoYJlcpUSKLjspyblPatPX552EOvfoa96GgYtTqw1XDufbHYuK0sFEaVwIXp6z3IZnzPzJ52rZZ/IG2plHEY4id6zYqaB9UAyOy3QkCxlNHzPepkBcA/0sk/5NyyiwbLzmM7MeLnni4BFsf7KQPkJifqbF97qfw== diff --git a/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio.png b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio.png new file mode 100644 index 0000000000..7ac6de28b8 Binary files /dev/null and b/patterns/gitops/multi-cluster-hub-spoke-argocd/static/gitops-bridge-multi-cluster-hup-spoke.drawio.png differ diff --git a/patterns/istio/README.md b/patterns/istio/README.md index 6d69ad81be..22855de34e 100644 --- a/patterns/istio/README.md +++ b/patterns/istio/README.md @@ -14,7 +14,12 @@ concepts. ## Deploy -See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and steps to deploy this pattern. +See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and run the following command to deploy this pattern. + +```sh +terraform init +terraform apply --auto-approve +``` Once the resources have been provisioned, you will need to replace the `istio-ingress` pods due to a [`istiod` dependency issue](https://github.com/istio/istio/issues/35789). Use the following command to perform a rolling restart of the `istio-ingress` pods: @@ -290,6 +295,25 @@ kubectl port-forward svc/jaeger 16686:16686 -n istio-system ## Destroy +The AWS Load Balancer Controller add-on asynchronously reconciles resource deletions. +During stack destruction, the istio ingress resource and the load balancer controller +add-on are deleted in quick succession, preventing the removal of some of the AWS +resources associated with the ingress gateway load balancer like, the frontend and the +backend security groups. +This causes the final `terraform destroy -auto-approve` command to timeout and fail with VPC dependency errors like below: + +```text +│ Error: deleting EC2 VPC (vpc-XXXX): operation error EC2: DeleteVpc, https response error StatusCode: 400, RequestID: XXXXX-XXXX-XXXX-XXXX-XXXXXX, api error DependencyViolation: The vpc 'vpc-XXXX' has dependencies and cannot be deleted. +``` + +A possible workaround is to manually uninstall the `istio-ingress` helm chart. + +```sh +terraform destroy -target='module.eks_blueprints_addons.helm_release.this["istio-ingress"]' -auto-approve +``` + +Once the chart is uninstalled move on to destroy the stack. + {% include-markdown "../../docs/_partials/destroy.md" %} diff --git a/patterns/istio/main.tf b/patterns/istio/main.tf index 76a38c8ac9..bf0edfc6d8 100644 --- a/patterns/istio/main.tf +++ b/patterns/istio/main.tf @@ -127,19 +127,19 @@ module "eks_blueprints_addons" { helm_releases = { istio-base = { - chart = "base" - version = local.istio_chart_version - repository = local.istio_chart_url - name = "istio-base" - namespace = kubernetes_namespace_v1.istio_system.metadata[0].name + chart = "base" + chart_version = local.istio_chart_version + repository = local.istio_chart_url + name = "istio-base" + namespace = kubernetes_namespace_v1.istio_system.metadata[0].name } istiod = { - chart = "istiod" - version = local.istio_chart_version - repository = local.istio_chart_url - name = "istiod" - namespace = kubernetes_namespace_v1.istio_system.metadata[0].name + chart = "istiod" + chart_version = local.istio_chart_version + repository = local.istio_chart_url + name = "istiod" + namespace = kubernetes_namespace_v1.istio_system.metadata[0].name set = [ { @@ -151,7 +151,7 @@ module "eks_blueprints_addons" { istio-ingress = { chart = "gateway" - version = local.istio_chart_version + chart_version = local.istio_chart_version repository = local.istio_chart_url name = "istio-ingress" namespace = "istio-ingress" # per https://github.com/istio/istio/blob/master/manifests/charts/gateways/istio-ingress/values.yaml#L2 @@ -165,9 +165,10 @@ module "eks_blueprints_addons" { } service = { annotations = { - "service.beta.kubernetes.io/aws-load-balancer-type" = "nlb" - "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" - "service.beta.kubernetes.io/aws-load-balancer-attributes" = "load_balancing.cross_zone.enabled=true" + "service.beta.kubernetes.io/aws-load-balancer-type" = "external" + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "ip" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-attributes" = "load_balancing.cross_zone.enabled=true" } } } diff --git a/patterns/karpenter/README.md b/patterns/karpenter/README.md index 9efc642074..7b03643de0 100644 --- a/patterns/karpenter/README.md +++ b/patterns/karpenter/README.md @@ -8,15 +8,55 @@ See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started ## Validate -!!! danger "TODO" - Add in validation steps +1. Test by listing the nodes in the cluster. You should see four Fargate nodes in the cluster: + + ```sh + kubectl get nodes + + NAME STATUS ROLES AGE VERSION + fargate-ip-10-0-11-195.us-west-2.compute.internal Ready 5m20s v1.28.2-eks-f8587cb + fargate-ip-10-0-27-183.us-west-2.compute.internal Ready 5m2s v1.28.2-eks-f8587cb + fargate-ip-10-0-4-169.us-west-2.compute.internal Ready 5m3s v1.28.2-eks-f8587cb + fargate-ip-10-0-44-106.us-west-2.compute.internal Ready 5m12s v1.28.2-eks-f8587cb + ``` + +2. Provision the Karpenter `EC2NodeClass` and `NodePool` resources which provide Karpenter the necessary configurations to provision EC2 resources: + + ```sh + kubectl apply -f karpenter.yaml + ``` + +3. Once the Karpenter resources are in place, Karpenter will provision the necessary EC2 resources to satisfy any pending pods in the scheduler's queue. You can demonstrate this with the example deployment provided. First deploy the example deployment which has the initial number replicas set to 0: + + ```sh + kubectl apply -f example.yaml + ``` + +4. When you scale the example deployment, you should see Karpenter respond by quickly provisioning EC2 resources to satisfy those pending pod requests: + + ```sh + kubectl scale deployment inflate --replicas=3 + ``` + +5. Listing the nodes should now show some EC2 compute that Karpenter has created for the example deployment: + + ```sh + kubectl get nodes + + NAME STATUS ROLES AGE VERSION + fargate-ip-10-0-11-195.us-west-2.compute.internal Ready 13m v1.28.2-eks-f8587cb + fargate-ip-10-0-27-183.us-west-2.compute.internal Ready 12m v1.28.2-eks-f8587cb + fargate-ip-10-0-4-169.us-west-2.compute.internal Ready 12m v1.28.2-eks-f8587cb + fargate-ip-10-0-44-106.us-west-2.compute.internal Ready 13m v1.28.2-eks-f8587cb + ip-10-0-32-199.us-west-2.compute.internal Ready 29s v1.28.2-eks-a5df82a # <== EC2 created by Karpenter + ``` ## Destroy Scale down the deployment to de-provision Karpenter created resources first: ```sh -kubectl delete deployment inflate +kubectl delete -f example.yaml ``` {% diff --git a/patterns/karpenter/example.yaml b/patterns/karpenter/example.yaml new file mode 100644 index 0000000000..25ee0fef3c --- /dev/null +++ b/patterns/karpenter/example.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 0 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + terminationGracePeriodSeconds: 0 + containers: + - name: inflate + image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 + resources: + requests: + cpu: 1 diff --git a/patterns/karpenter/karpenter.yaml b/patterns/karpenter/karpenter.yaml new file mode 100644 index 0000000000..c07b61db57 --- /dev/null +++ b/patterns/karpenter/karpenter.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + role: karpenter-ex-karpenter + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: ex-karpenter + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: ex-karpenter + tags: + karpenter.sh/discovery: ex-karpenter +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + name: default + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + limits: + cpu: 1000 + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 30s diff --git a/patterns/karpenter/main.tf b/patterns/karpenter/main.tf index 5b9d732940..7055526364 100644 --- a/patterns/karpenter/main.tf +++ b/patterns/karpenter/main.tf @@ -34,20 +34,6 @@ provider "helm" { } } -provider "kubectl" { - apply_retry_count = 5 - host = module.eks.cluster_endpoint - cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) - load_config_file = false - - exec { - api_version = "client.authentication.k8s.io/v1beta1" - command = "aws" - # This requires the awscli to be installed locally where Terraform is executed - args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] - } -} - data "aws_ecrpublic_authorization_token" "token" { provider = aws.virginia } @@ -55,7 +41,7 @@ data "aws_ecrpublic_authorization_token" "token" { data "aws_availability_zones" "available" {} locals { - name = basename(path.cwd) + name = "ex-${basename(path.cwd)}" region = "us-west-2" vpc_cidr = "10.0.0.0/16" @@ -73,10 +59,10 @@ locals { module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 19.16" + version = "~> 19.18" cluster_name = local.name - cluster_version = "1.27" + cluster_version = "1.28" cluster_endpoint_public_access = true vpc_id = module.vpc.vpc_id @@ -127,7 +113,7 @@ module "eks" { module "eks_blueprints_addons" { source = "aws-ia/eks-blueprints-addons/aws" - version = "~> 1.0" + version = "~> 1.11" cluster_name = module.eks.cluster_name cluster_endpoint = module.eks.cluster_endpoint @@ -172,107 +158,14 @@ module "eks_blueprints_addons" { repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password } + karpenter_node = { + # Use static name so that it matches what is defined in `karpenter.yaml` example manifest + iam_role_use_name_prefix = false + } tags = local.tags } -################################################################################ -# Karpenter -################################################################################ - -resource "kubectl_manifest" "karpenter_provisioner" { - yaml_body = <<-YAML - apiVersion: karpenter.sh/v1alpha5 - kind: Provisioner - metadata: - name: default - spec: - requirements: - - key: "karpenter.k8s.aws/instance-category" - operator: In - values: ["c", "m"] - - key: "karpenter.k8s.aws/instance-cpu" - operator: In - values: ["8", "16", "32"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] - - key: "topology.kubernetes.io/zone" - operator: In - values: ${jsonencode(local.azs)} - - key: "kubernetes.io/arch" - operator: In - values: ["arm64", "amd64"] - - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand - operator: In - values: ["spot", "on-demand"] - kubeletConfiguration: - containerRuntime: containerd - maxPods: 110 - limits: - resources: - cpu: 1000 - consolidation: - enabled: true - providerRef: - name: default - ttlSecondsUntilExpired: 604800 # 7 Days = 7 * 24 * 60 * 60 Seconds - YAML - - depends_on = [ - module.eks_blueprints_addons - ] -} - -resource "kubectl_manifest" "karpenter_node_template" { - yaml_body = <<-YAML - apiVersion: karpenter.k8s.aws/v1alpha1 - kind: AWSNodeTemplate - metadata: - name: default - spec: - subnetSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - securityGroupSelector: - karpenter.sh/discovery: ${module.eks.cluster_name} - instanceProfile: ${module.eks_blueprints_addons.karpenter.node_instance_profile_name} - tags: - karpenter.sh/discovery: ${module.eks.cluster_name} - YAML -} - -# Example deployment using the [pause image](https://www.ianlewis.org/en/almighty-pause-container) -# and starts with zero replicas -resource "kubectl_manifest" "karpenter_example_deployment" { - yaml_body = <<-YAML - apiVersion: apps/v1 - kind: Deployment - metadata: - name: inflate - spec: - replicas: 0 - selector: - matchLabels: - app: inflate - template: - metadata: - labels: - app: inflate - spec: - terminationGracePeriodSeconds: 0 - containers: - - name: inflate - image: public.ecr.aws/eks-distro/kubernetes/pause:3.7 - resources: - requests: - cpu: 1 - YAML - - depends_on = [ - kubectl_manifest.karpenter_node_template - ] -} - ################################################################################ # Supporting Resources ################################################################################ diff --git a/patterns/karpenter/versions.tf b/patterns/karpenter/versions.tf index 24d5350878..2c63637eba 100644 --- a/patterns/karpenter/versions.tf +++ b/patterns/karpenter/versions.tf @@ -14,10 +14,6 @@ terraform { source = "hashicorp/kubernetes" version = ">= 2.20" } - kubectl = { - source = "gavinbunney/kubectl" - version = ">= 1.14" - } } # ## Used for end-to-end testing on project; update to suit your needs diff --git a/patterns/private-public-ingress/main.tf b/patterns/private-public-ingress/main.tf index 1a77cf8fdd..fd81fc2772 100644 --- a/patterns/private-public-ingress/main.tf +++ b/patterns/private-public-ingress/main.tf @@ -243,7 +243,7 @@ module "ingres_nginx_internal" { } } -module "eks_blueprints_kubernetes_addons" { +module "eks_blueprints_addons" { source = "aws-ia/eks-blueprints-addons/aws" version = "~> 1.0" diff --git a/patterns/privatelink-access/README.md b/patterns/privatelink-access/README.md index f7c6e514d8..1124f2012d 100644 --- a/patterns/privatelink-access/README.md +++ b/patterns/privatelink-access/README.md @@ -7,17 +7,37 @@ for further details on `AWS PrivateLink`. ## Deploy -See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and steps to deploy this pattern. +See [here](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/#prerequisites) for the prerequisites and follow the steps below to deploy this pattern. -## Validate +```sh +terraform init +terraform apply -target=module.eventbridge -target=module.nlb --auto-approve +terraform apply --auto-approve +``` -### Network Connectivity +Once the pattern has successfully deployed, you will be provided with multiple +output values. -An output `ssm_test` has been provided to aid in quickly testing the -connectivity from the client EC2 instance to the private EKS cluster via AWS -PrivateLink. Copy the output value and paste it into your terminal to execute -and check the connectivity. If configured correctly, the value returned should -be `ok`. +Review the output value for `cluster_endpoint_private`, it should look similar +to snippet below: + +```sh +aws eks update-cluster-config \ +--region us-west-2 \ +--name privatelink-access \ +--resources-vpc-config endpointPublicAccess=false,endpointPrivateAccess=true +``` + +Copy the command and run it in a terminal session to take cluster API +endpoint private. + +## Test access to EKS Kubernetes API server endpoint + +Of the other output values, the value `ssm_test` is provided to aid in quickly +testing the connectivity from the client EC2 instance to the private EKS cluster +via AWS PrivateLink. Copy the output value, which looks like the snippet shown +below (as an example) and paste it into your terminal to execute and check the +connectivity. If configured correctly, the value returned should be `ok`. ```sh COMMAND="curl -ks https://9A85B21811733524E3ABCDFEA8714642.gr7.us-west-2.eks.amazonaws.com/readyz" @@ -36,78 +56,62 @@ aws ssm get-command-invocation --region us-west-2 \ --output text ``` -### Cluster Access - -To test access to the cluster, you will need to execute Kubernetes API calls -from within the private network to access the cluster. An EC2 instance has been -deployed into a "client" VPC to simulate this scenario. However, since the EKS -cluster was created with your local IAM identity, the `aws-auth` ConfigMap will -only have your local identity that is permitted to access the cluster. Since -cluster's API endpoint is private, we cannot use Terraform to reach it to -add additional entries to the ConfigMap; we can only access the cluster from -within the private network of the cluster's VPC or from the client VPC using AWS -PrivateLink access. - -!!! info - The "client" EC2 instance provided and copying of AWS credentials to - that instance are merely for demonstration purposes only. Please consider - alternate methods of network access such as AWS Client VPN to provide more - secure access. +## Test access to EKS Kubernetes API with `kubectl` Perform the following steps to access the cluster with `kubectl` from the -provided "client" EC2 instance. - -1. Execute the command below on your local machine to get temporary credentials -that will be used on the "client" EC2 instance: +provided Client EC2 instance. - ```sh - aws sts get-session-token --duration-seconds 3600 --output yaml - ``` +### Log into the Client EC2 instance +Start a new SSM session on the Client EC2 instance using the provided +`ssm_start_session` output value. It should look similar to the snippet +shown below. Copy the output value and paste it into your terminal to execute. +Your terminal will now be connected to the Client EC2 instance. -2. Start a new SSM session on the "client" EC2 instance using the provided -`ssm_start_session` output value. Copy the output value and paste it into your -terminal to execute. Your terminal will now be connected to the "client" EC2 -instance. +```sh +aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44 +``` - ```sh - aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44 - ``` +### Update Kubeconfig +On the Client EC2 machine, run the following command to update the local +`~/.kube/config` file to enable access to the cluster: -3. Once logged in, export the following environment variables from the output -of step #1: +```sh +aws eks update-kubeconfig --region us-west-2 --name privatelink-access +``` - !!! warning - The session credentials are only valid for 1 hour; you can - adjust the session duration in the command provided in step #1 +### Test complete access with `kubectl` +Test access by listing the pods running on the cluster: - ```sh - export AWS_ACCESS_KEY_ID=XXXX - export AWS_SECRET_ACCESS_KEY=YYYY - export AWS_SESSION_TOKEN=ZZZZ - ``` +```sh +kubectl get pods -A +``` -4. Run the following command to update the local `~/.kube/config` file to enable -access to the cluster: +```text +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system aws-node-4f8g8 1/1 Running 0 1m +kube-system coredns-6ff9c46cd8-59sqp 1/1 Running 0 1m +kube-system coredns-6ff9c46cd8-svnpb 1/1 Running 0 2m +kube-system kube-proxy-mm2zc 1/1 Running 0 1m +``` - ```sh - aws eks update-kubeconfig --region us-west-2 --name privatelink-access - ``` +## Destroy -5. Test access by listing the pods running on the cluster: +Before we could destroy/teardown all the resources created, we need to ensure +that the cluster state is restored for the Terraform to do a complete cleanup. +This would mean that we make cluster API endpoint public again. - ```sh - kubectl get pods -A - ``` +Review the output value for `cluster_endpoint_public`, it should look similar +to snippet below: - ```text - NAMESPACE NAME READY STATUS RESTARTS AGE - kube-system aws-node-4f8g8 1/1 Running 0 1m - kube-system coredns-6ff9c46cd8-59sqp 1/1 Running 0 1m - kube-system coredns-6ff9c46cd8-svnpb 1/1 Running 0 2m - kube-system kube-proxy-mm2zc 1/1 Running 0 1m - ``` +```sh +aws eks update-cluster-config \ +--region us-west-2 \ +--name privatelink-access \ +--resources-vpc-config endpointPublicAccess=true,endpointPrivateAccess=true +``` -## Destroy +Copy the command and run it in a terminal session to take cluster API +endpoint public. {% include-markdown "../../docs/_partials/destroy.md" diff --git a/patterns/privatelink-access/client.tf b/patterns/privatelink-access/client.tf index 0793c4804f..2852f18f87 100644 --- a/patterns/privatelink-access/client.tf +++ b/patterns/privatelink-access/client.tf @@ -30,7 +30,9 @@ module "client_vpc" { manage_default_security_group = true default_security_group_tags = { Name = "${local.client_name}-default" } - tags = local.tags + tags = merge(local.tags, { + Name = local.client_name + }) } ################################################################################ @@ -44,6 +46,7 @@ module "client_ec2_instance" { create_iam_instance_profile = true iam_role_policies = { + EKSFullAccess = aws_iam_policy.eks_full_access_policy.arn AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } @@ -64,7 +67,29 @@ module "client_ec2_instance" { ./aws/install EOT - tags = local.tags + tags = merge(local.tags, { + Name = local.client_name + }) +} + +resource "aws_iam_policy" "eks_full_access_policy" { + name = "EKSFullAccess" + path = "/" + description = "EKS full-access policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "eks:DescribeCluster", + "eks:ListClusters" + ] + Effect = "Allow" + Resource = module.eks.cluster_arn + }, + ] + }) } module "client_security_group" { @@ -84,5 +109,7 @@ module "client_security_group" { }, ] - tags = local.tags + tags = merge(local.tags, { + Name = local.client_name + }) } diff --git a/patterns/privatelink-access/eks.tf b/patterns/privatelink-access/eks.tf index e698ed35a0..0ac647e023 100644 --- a/patterns/privatelink-access/eks.tf +++ b/patterns/privatelink-access/eks.tf @@ -2,6 +2,18 @@ # EKS Cluster ################################################################################ +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + } +} + module "eks" { source = "terraform-aws-modules/eks/aws" version = "~> 19.16" @@ -9,6 +21,15 @@ module "eks" { cluster_name = local.name cluster_version = "1.27" + cluster_endpoint_public_access = true + manage_aws_auth_configmap = true + + aws_auth_roles = [{ + rolearn = module.client_ec2_instance.iam_role_arn + username = "ec2-client" + groups = ["system:masters"] + }] + cluster_addons = { coredns = {} kube-proxy = {} diff --git a/patterns/privatelink-access/outputs.tf b/patterns/privatelink-access/outputs.tf index 3130bf693f..50f5ac870d 100644 --- a/patterns/privatelink-access/outputs.tf +++ b/patterns/privatelink-access/outputs.tf @@ -6,6 +6,7 @@ output "ssm_start_session" { output "ssm_test" { description = "SSM commands to test connectivity from client EC2 instance to the private EKS cluster" value = <<-EOT + COMMAND="curl -ks ${module.eks.cluster_endpoint}/readyz" COMMAND_ID=$(aws ssm send-command --region ${local.region} \ @@ -22,3 +23,25 @@ output "ssm_test" { --output text EOT } + +output "cluster_endpoint_private" { + description = "Command to set the EKS API server endpoint access private" + value = <<-EOT + + aws eks update-cluster-config \ + --region ${local.region} \ + --name ${module.eks.cluster_name} \ + --resources-vpc-config endpointPublicAccess=false,endpointPrivateAccess=true + EOT +} + +output "cluster_endpoint_public" { + description = "Command to set the EKS API server endpoint access private" + value = <<-EOT + + aws eks update-cluster-config \ + --region ${local.region} \ + --name ${module.eks.cluster_name} \ + --resources-vpc-config endpointPublicAccess=true,endpointPrivateAccess=true + EOT +} diff --git a/patterns/privatelink-access/versions.tf b/patterns/privatelink-access/versions.tf index 4cbe90687e..fb2fa577b1 100644 --- a/patterns/privatelink-access/versions.tf +++ b/patterns/privatelink-access/versions.tf @@ -10,6 +10,10 @@ terraform { source = "hashicorp/dns" version = ">= 3.0" } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.20" + } } # ## Used for end-to-end testing on project; update to suit your needs