diff --git a/patterns/blue-green-upgrade/bootstrap/addons.yaml b/patterns/blue-green-upgrade/bootstrap/addons.yaml index 1c010e4a0c..f9415677a6 100644 --- a/patterns/blue-green-upgrade/bootstrap/addons.yaml +++ b/patterns/blue-green-upgrade/bootstrap/addons.yaml @@ -1,4 +1,3 @@ ---- apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: diff --git a/patterns/blue-green-upgrade/bootstrap/workloads.yaml b/patterns/blue-green-upgrade/bootstrap/workloads.yaml index 8552278f9a..73f2567cc5 100644 --- a/patterns/blue-green-upgrade/bootstrap/workloads.yaml +++ b/patterns/blue-green-upgrade/bootstrap/workloads.yaml @@ -1,4 +1,3 @@ ---- apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: @@ -25,7 +24,7 @@ spec: - path: '{{.metadata.annotations.gitops_workloads_path}}/*' template: metadata: - name: 'bootstrap-workload-{{.name}}' + name: 'bootstrap-workloads-{{.name}}' spec: project: default sources: @@ -34,7 +33,7 @@ spec: ref: values path: '{{.metadata.annotations.gitops_workloads_path}}' helm: - releaseName: 'bootstrap-workload-{{.name}}' + releaseName: 'bootstrap-workloads-{{.name}}' ignoreMissingValueFiles: true values: | "account": "{{.metadata.annotations.aws_account_id}}" diff --git a/patterns/blue-green-upgrade/eks-blue/variables.tf b/patterns/blue-green-upgrade/eks-blue/variables.tf index e95e6a4683..77416b8ed8 100644 --- a/patterns/blue-green-upgrade/eks-blue/variables.tf +++ b/patterns/blue-green-upgrade/eks-blue/variables.tf @@ -5,7 +5,7 @@ variable "aws_region" { } variable "environment_name" { - description = "The name of Environment Infrastructure stack name, feel free to rename it. Used for cluster and VPC names." + description = "The name of Environment Infrastructure stack, feel free to rename it. Used for cluster and VPC names." type = string default = "eks-blueprint" } @@ -40,51 +40,52 @@ variable "argocd_secret_manager_name_suffix" { default = "argocd-admin-secret" } -variable "gitops_workloads_org" { - type = string - description = "Git repository org/user contains for workloads" - default = "https://github.com/aws-samples" -} - -variable "gitops_workloads_repo" { - type = string - description = "Git repository contains for workloads" - default = "eks-blueprints-workloads" -} - -variable "gitops_workloads_revision" { - type = string - description = "Git repo revision in workload_repo_url for the ArgoCD workload deployment" - default = "main" -} - -variable "gitops_workloads_path" { - type = string - description = "Git repo path in workload_repo_url for the ArgoCD workload deployment" - default = "envs/dev" -} variable "gitops_addons_org" { type = string description = "Git repository org/user contains for addons" - default = "https://github.com/gitops-bridge-dev" + default = "git@github.com:aws-samples" } variable "gitops_addons_repo" { type = string description = "Git repository contains for addons" - default = "gitops-bridge-argocd-control-plane-template" + default = "eks-blueprints-add-ons" } variable "gitops_addons_basepath" { type = string description = "Git repository base path for addons" - default = "" + default = "argocd/" } variable "gitops_addons_path" { type = string description = "Git repository path for addons" - default = "bootstrap/control-plane/addons" + default = "argocd/bootstrap/control-plane/addons" } variable "gitops_addons_revision" { type = string description = "Git repository revision/branch/ref for addons" default = "HEAD" } + +variable "gitops_workloads_org" { + type = string + description = "Git repository org/user contains for workloads" + default = "git@github.com:aws-samples" +} + +variable "gitops_workloads_repo" { + type = string + description = "Git repository contains for workloads" + default = "eks-blueprints-workloads" +} + +variable "gitops_workloads_path" { + type = string + description = "Git repo path in workload_repo_url for the ArgoCD workload deployment" + default = "envs/dev" +} + +variable "gitops_workloads_revision" { + type = string + description = "Git repo revision in workload_repo_url for the ArgoCD workload deployment" + default = "main" +} diff --git a/patterns/blue-green-upgrade/eks-green/variables.tf b/patterns/blue-green-upgrade/eks-green/variables.tf index d7b1f6125a..d40215df04 100644 --- a/patterns/blue-green-upgrade/eks-green/variables.tf +++ b/patterns/blue-green-upgrade/eks-green/variables.tf @@ -10,6 +10,12 @@ variable "environment_name" { default = "eks-blueprint" } +variable "ingress_type" { + type = string + description = "Type of ingress to uses (alb | nginx | ...). this parameter will be sent to arocd via gitops bridge" + default = "alb" +} + variable "hosted_zone_name" { type = string description = "Route53 domain for the cluster." @@ -34,29 +40,6 @@ variable "argocd_secret_manager_name_suffix" { default = "argocd-admin-secret" } -variable "gitops_workloads_org" { - type = string - description = "Git repository org/user contains for workloads" - default = "https://github.com/aws-samples" -} - -variable "gitops_workloads_repo" { - type = string - description = "Git repository contains for workloads" - default = "eks-blueprints-workloads" -} - -variable "gitops_workloads_revision" { - type = string - description = "Git repo revision in workload_repo_url for the ArgoCD workload deployment" - default = "main" -} - -variable "gitops_workloads_path" { - type = string - description = "Git repo path in workload_repo_url for the ArgoCD workload deployment" - default = "envs/dev" -} variable "gitops_addons_org" { type = string description = "Git repository org/user contains for addons" @@ -82,3 +65,27 @@ variable "gitops_addons_revision" { description = "Git repository revision/branch/ref for addons" default = "HEAD" } + +variable "gitops_workloads_org" { + type = string + description = "Git repository org/user contains for workloads" + default = "https://github.com/aws-samples" +} + +variable "gitops_workloads_repo" { + type = string + description = "Git repository contains for workloads" + default = "eks-blueprints-workloads" +} + +variable "gitops_workloads_path" { + type = string + description = "Git repo path in workload_repo_url for the ArgoCD workload deployment" + default = "envs/dev" +} + +variable "gitops_workloads_revision" { + type = string + description = "Git repo revision in workload_repo_url for the ArgoCD workload deployment" + default = "main" +} diff --git a/patterns/blue-green-upgrade/modules/eks_cluster/main.tf b/patterns/blue-green-upgrade/modules/eks_cluster/main.tf index 6bc975ff7a..b96f0a0450 100644 --- a/patterns/blue-green-upgrade/modules/eks_cluster/main.tf +++ b/patterns/blue-green-upgrade/modules/eks_cluster/main.tf @@ -7,33 +7,27 @@ provider "aws" { locals { environment = var.environment_name service = var.service_name + region = var.aws_region - env = local.environment + env = local.service name = "${local.environment}-${local.service}" - # Mapping - hosted_zone_name = var.hosted_zone_name - addons_repo_url = var.addons_repo_url - workload_repo_secret = var.workload_repo_secret - cluster_version = var.cluster_version - argocd_secret_manager_name = var.argocd_secret_manager_name_suffix - workload_repo_path = var.workload_repo_path - workload_repo_url = var.workload_repo_url - workload_repo_revision = var.workload_repo_revision - eks_admin_role_name = var.eks_admin_role_name - iam_platform_user = var.iam_platform_user - - metrics_server = true - aws_load_balancer_controller = true - karpenter = true - aws_for_fluentbit = true - cert_manager = true - cloudwatch_metrics = true - external_dns = true - vpa = true - kubecost = true - argo_rollouts = true + hosted_zone_name = var.hosted_zone_name + ingress_type = var.ingress_type + aws_secret_manager_git_private_ssh_key_name = var.aws_secret_manager_git_private_ssh_key_name + cluster_version = var.cluster_version + argocd_secret_manager_name = var.argocd_secret_manager_name_suffix + eks_admin_role_name = var.eks_admin_role_name + + gitops_workloads_url = "${var.gitops_workloads_org}/${var.gitops_workloads_repo}" + gitops_workloads_path = var.gitops_workloads_path + gitops_workloads_revision = var.gitops_workloads_revision + + gitops_addons_url = "${var.gitops_addons_org}/${var.gitops_addons_repo}" + gitops_addons_basepath = var.gitops_addons_basepath + gitops_addons_path = var.gitops_addons_path + gitops_addons_revision = var.gitops_addons_revision # Route 53 Ingress Weights argocd_route53_weight = var.argocd_route53_weight @@ -48,212 +42,100 @@ locals { node_group_name = "managed-ondemand" - #--------------------------------------------------------------- # ARGOCD ADD-ON APPLICATION #--------------------------------------------------------------- - #At this time (with new v5 addon repository), the Addons need to be managed by Terrform and not ArgoCD - addons_application = { - path = "chart" - repo_url = local.addons_repo_url - ssh_key_secret_name = local.workload_repo_secret - add_on_application = true + aws_addons = { + enable_cert_manager = true + enable_aws_ebs_csi_resources = true # generate gp2 and gp3 storage classes for ebs-csi + #enable_aws_efs_csi_driver = true + #enable_aws_fsx_csi_driver = true + enable_aws_cloudwatch_metrics = true + #enable_aws_privateca_issuer = true + #enable_cluster_autoscaler = true + enable_external_dns = true + enable_external_secrets = true + enable_aws_load_balancer_controller = true + #enable_fargate_fluentbit = true + enable_aws_for_fluentbit = true + #enable_aws_node_termination_handler = true + enable_karpenter = true + #enable_velero = true + #enable_aws_gateway_api_controller = true + #enable_aws_secrets_store_csi_driver_provider = true } + oss_addons = { + #enable_argo_rollouts = true + #enable_argo_workflows = true + #enable_cluster_proportional_autoscaler = true + #enable_gatekeeper = true + #enable_gpu_operator = true + enable_ingress_nginx = true + enable_kyverno = true + #enable_kube_prometheus_stack = true + enable_metrics_server = true + #enable_prometheus_adapter = true + #enable_secrets_store_csi_driver = true + #enable_vpa = true + #enable_foo = true # you can add any addon here, make sure to update the gitops repo with the corresponding application set + } + addons = merge(local.aws_addons, local.oss_addons, { kubernetes_version = local.cluster_version }) - #--------------------------------------------------------------- - # ARGOCD WORKLOAD APPLICATION - #--------------------------------------------------------------- + #---------------------------------------------------------------- + # GitOps Bridge, define metadatas to pass from Terraform to ArgoCD + #---------------------------------------------------------------- - workload_application = { - path = local.workload_repo_path # <-- we could also to blue/green on the workload repo path like: envs/dev-blue / envs/dev-green - repo_url = local.workload_repo_url - target_revision = local.workload_repo_revision - ssh_key_secret_name = local.workload_repo_secret - add_on_application = false - values = { - labels = { - env = local.env - myapp = "myvalue" - } - spec = { - source = { - repoURL = local.workload_repo_url - targetRevision = local.workload_repo_revision - } - blueprint = "terraform" - clusterName = local.name - karpenterInstanceProfile = module.karpenter.instance_profile_name - env = local.env - ingress = { - type = "alb" - host = local.eks_cluster_domain - route53_weight = local.route53_weight # <-- You can control the weight of the route53 weighted records between clusters - argocd_route53_weight = local.argocd_route53_weight - } - } + addons_metadata = merge( + module.eks_blueprints_addons.gitops_metadata, # eks blueprints addons automatically expose metadatas + { + aws_cluster_name = module.eks.cluster_name + aws_region = local.region + aws_account_id = data.aws_caller_identity.current.account_id + aws_vpc_id = data.aws_vpc.vpc.id + cluster_endpoint = module.eks.cluster_endpoint + env = local.env + }, + { + argocd_password = bcrypt(data.aws_secretsmanager_secret_version.admin_password_version.secret_string) + aws_secret_manager_git_private_ssh_key_name = local.aws_secret_manager_git_private_ssh_key_name + + gitops_workloads_url = local.gitops_workloads_url + gitops_workloads_path = local.gitops_workloads_path + gitops_workloads_revision = local.gitops_workloads_revision + + addons_repo_url = local.gitops_addons_url + addons_repo_basepath = local.gitops_addons_basepath + addons_repo_path = local.gitops_addons_path + addons_repo_revision = local.gitops_addons_revision + }, + { + eks_cluster_domain = local.eks_cluster_domain + external_dns_policy = "sync" + ingress_type = local.ingress_type + argocd_route53_weight = local.argocd_route53_weight + route53_weight = local.route53_weight + ecsfrontend_route53_weight = local.ecsfrontend_route53_weight + #target_group_arn = local.service == "blue" ? data.aws_lb_target_group.tg_blue.arn : data.aws_lb_target_group.tg_green.arn # <-- Add this line + # external_lb_dns = data.aws_lb.alb.dns_name } - } + ) #--------------------------------------------------------------- - # ARGOCD ECSDEMO APPLICATION + # Manifests for bootstraping the cluster for addons & workloads #--------------------------------------------------------------- - ecsdemo_application = { - path = "multi-repo/argo-app-of-apps/dev" - repo_url = local.workload_repo_url - target_revision = local.workload_repo_revision - ssh_key_secret_name = local.workload_repo_secret - add_on_application = false - values = { - spec = { - blueprint = "terraform" - clusterName = local.name - karpenterInstanceProfile = module.karpenter.instance_profile_name - - apps = { - ecsdemoNodejs = { - - helm = { - replicaCount = "9" - nodeSelector = { - "karpenter.sh/provisioner-name" = "default" - } - tolerations = [ - { - key = "karpenter" - operator = "Exists" - effect = "NoSchedule" - } - ] - topologyAwareHints = "true" - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "topology.kubernetes.io/zone" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - "app.kubernetes.io/name" = "ecsdemo-nodejs" - } - } - } - ] - } - } - - ecsdemoCrystal = { - - helm = { - replicaCount = "9" - nodeSelector = { - "karpenter.sh/provisioner-name" = "default" - } - tolerations = [ - { - key = "karpenter" - operator = "Exists" - effect = "NoSchedule" - } - ] - topologyAwareHints = "true" - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "topology.kubernetes.io/zone" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - "app.kubernetes.io/name" = "ecsdemo-crystal" - } - } - } - ] - } - } - - - ecsdemoFrontend = { - repoURL = "https://github.com/allamand/ecsdemo-frontend" - targetRevision = "main" - helm = { - image = { - repository = "public.ecr.aws/seb-demo/ecsdemo-frontend" - tag = "latest" - } - ingress = { - enabled = "true" - className = "alb" - annotations = { - "alb.ingress.kubernetes.io/scheme" = "internet-facing" - "alb.ingress.kubernetes.io/group.name" = "ecsdemo" - "alb.ingress.kubernetes.io/listen-ports" = "[{\\\"HTTPS\\\": 443}]" - "alb.ingress.kubernetes.io/ssl-redirect" = "443" - "alb.ingress.kubernetes.io/target-type" = "ip" - "external-dns.alpha.kubernetes.io/set-identifier" = local.name - "external-dns.alpha.kubernetes.io/aws-weight" = local.ecsfrontend_route53_weight - } - hosts = [ - { - host = "frontend.${local.eks_cluster_domain}" - paths = [ - { - path = "/" - pathType = "Prefix" - } - ] - } - ] - } - resources = { - requests = { - cpu = "1" - memory = "256Mi" - } - limits = { - cpu = "1" - memory = "512Mi" - } - } - autoscaling = { - enabled = "true" - minReplicas = "9" - maxReplicas = "100" - targetCPUUtilizationPercentage = "60" - } - nodeSelector = { - "karpenter.sh/provisioner-name" = "default" - } - tolerations = [ - { - key = "karpenter" - operator = "Exists" - effect = "NoSchedule" - } - ] - topologySpreadConstraints = [ - { - maxSkew = 1 - topologyKey = "topology.kubernetes.io/zone" - whenUnsatisfiable = "DoNotSchedule" - labelSelector = { - matchLabels = { - "app.kubernetes.io/name" = "ecsdemo-frontend" - } - } - } - ] - } - } - } - } - } + argocd_bootstrap_app_of_apps = { + addons = file("${path.module}/../../bootstrap/addons.yaml") + workloads = file("${path.module}/../../bootstrap/workloads.yaml") } + tags = { Blueprint = local.name GithubRepo = "github.com/aws-ia/terraform-aws-eks-blueprints" } + } # Find the user currently in use by AWS @@ -296,11 +178,14 @@ resource "aws_ec2_tag" "public_subnets" { value = "shared" } -# Create Sub HostedZone four our deployment +# Get HostedZone four our deployment data "aws_route53_zone" "sub" { name = "${local.environment}.${local.hosted_zone_name}" } +################################################################################ +# AWS Secret Manager for argocd password +################################################################################ data "aws_secretsmanager_secret" "argocd" { name = "${local.argocd_secret_manager_name}.${local.environment}" @@ -310,9 +195,13 @@ data "aws_secretsmanager_secret_version" "admin_password_version" { secret_id = data.aws_secretsmanager_secret.argocd.id } +################################################################################ +# EKS Cluster +################################################################################ +#tfsec:ignore:aws-eks-enable-control-plane-logging module "eks" { source = "terraform-aws-modules/eks/aws" - version = "~> 19.16" + version = "~> 19.15.2" cluster_name = local.name cluster_version = local.cluster_version @@ -341,7 +230,7 @@ module "eks" { [ module.eks_blueprints_platform_teams.aws_auth_configmap_role, { - rolearn = module.karpenter.role_arn + rolearn = module.eks_blueprints_addons.karpenter.node_iam_role_arn username = "system:node:{{EC2PrivateDNSName}}" groups = [ "system:bootstrappers", @@ -364,16 +253,14 @@ module "eks" { }) } -data "aws_iam_user" "platform_user" { - count = local.iam_platform_user != "" ? 1 : 0 - user_name = local.iam_platform_user -} - data "aws_iam_role" "eks_admin_role_name" { count = local.eks_admin_role_name != "" ? 1 : 0 name = local.eks_admin_role_name } +################################################################################ +# EKS Blueprints Teams +################################################################################ module "eks_blueprints_platform_teams" { source = "aws-ia/eks-blueprints-teams/aws" version = "~> 1.0" @@ -386,7 +273,6 @@ module "eks_blueprints_platform_teams" { # Define who can impersonate the team-platform Role users = [ data.aws_caller_identity.current.arn, - try(data.aws_iam_user.platform_user[0].arn, data.aws_caller_identity.current.arn), try(data.aws_iam_role.eks_admin_role_name[0].arn, data.aws_caller_identity.current.arn), ] cluster_arn = module.eks.cluster_arn @@ -438,6 +324,7 @@ module "eks_blueprints_platform_teams" { } ] } + } } @@ -619,109 +506,171 @@ module "eks_blueprints_ecsdemo_teams" { tags = local.tags } -module "kubernetes_addons" { - source = "github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons?ref=v4.32.1" +################################################################################ +# GitOps Bridge: Private ssh keys for git +################################################################################ +data "aws_secretsmanager_secret" "workload_repo_secret" { + name = local.aws_secret_manager_git_private_ssh_key_name +} - eks_cluster_id = module.eks.cluster_name - eks_cluster_domain = local.eks_cluster_domain +data "aws_secretsmanager_secret_version" "workload_repo_secret" { + secret_id = data.aws_secretsmanager_secret.workload_repo_secret.id +} - #--------------------------------------------------------------- - # ARGO CD ADD-ON - #--------------------------------------------------------------- +resource "kubernetes_namespace" "argocd" { + depends_on = [module.eks_blueprints_addons] + metadata { + name = "argocd" + } +} - enable_argocd = true - argocd_manage_add_ons = true # Indicates that ArgoCD is responsible for managing/deploying Add-ons. +resource "kubernetes_secret" "git_secrets" { - argocd_applications = { - addons = local.addons_application - workloads = local.workload_application - ecsdemo = local.ecsdemo_application + for_each = { + git-addons = { + type = "git" + url = local.gitops_addons_url + # comment if you want to uses public repo wigh syntax "https://github.com/xxx" syntax, uncomment when using syntax "git@github.com:xxx" + sshPrivateKey = data.aws_secretsmanager_secret_version.workload_repo_secret.secret_string + } + git-workloads = { + type = "git" + url = local.gitops_workloads_url + # comment if you want to uses public repo wigh syntax "https://github.com/xxx" syntax, uncomment when using syntax "git@github.com:xxx" + #sshPrivateKey = data.aws_secretsmanager_secret_version.workload_repo_secret.secret_string + } + } + metadata { + name = each.key + namespace = kubernetes_namespace.argocd.metadata[0].name + labels = { + "argocd.argoproj.io/secret-type" = "repo-creds" + } } + data = each.value +} - # This example shows how to set default ArgoCD Admin Password using SecretsManager with Helm Chart set_sensitive values. - argocd_helm_config = { - set_sensitive = [ - { - name = "configs.secret.argocdServerAdminPassword" - value = bcrypt(data.aws_secretsmanager_secret_version.admin_password_version.secret_string) - } - ] +################################################################################ +# GitOps Bridge: Metadata +################################################################################ +module "gitops_bridge_metadata" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-metadata-terraform?ref=v1.0.0" + + cluster_name = module.eks.cluster_name + metadata = local.addons_metadata + environment = local.environment + addons = local.addons +} + +################################################################################ +# GitOps Bridge: Bootstrap +################################################################################ + +module "gitops_bridge_bootstrap" { + source = "github.com/gitops-bridge-dev/gitops-bridge-argocd-bootstrap-terraform?ref=v1.0.0" + + argocd_cluster = module.gitops_bridge_metadata.argocd + argocd_bootstrap_app_of_apps = local.argocd_bootstrap_app_of_apps + #argocd = { create_namespace = false } + argocd = { + create_namespace = false set = [ { name = "server.service.type" value = "LoadBalancer" } ] + set_sensitive = [ + { + name = "configs.secret.argocdServerAdminPassword" + value = bcrypt(data.aws_secretsmanager_secret_version.admin_password_version.secret_string) + } + ] } + depends_on = [kubernetes_secret.git_secrets] +} - #--------------------------------------------------------------- - # EKS Managed AddOns - # https://aws-ia.github.io/terraform-aws-eks-blueprints/add-ons/ - #--------------------------------------------------------------- +################################################################################ +# EKS Blueprints Addons +################################################################################ +module "eks_blueprints_addons" { + source = "aws-ia/eks-blueprints-addons/aws" - enable_amazon_eks_coredns = true - amazon_eks_coredns_config = { - most_recent = true - kubernetes_version = local.cluster_version - resolve_conflicts = "OVERWRITE" - } + cluster_name = module.eks.cluster_name + cluster_endpoint = module.eks.cluster_endpoint + cluster_version = module.eks.cluster_version + oidc_provider_arn = module.eks.oidc_provider_arn - enable_amazon_eks_aws_ebs_csi_driver = true - amazon_eks_aws_ebs_csi_driver_config = { - most_recent = true - kubernetes_version = local.cluster_version - resolve_conflicts = "OVERWRITE" - } + # Using GitOps Bridge + create_kubernetes_resources = false - enable_amazon_eks_kube_proxy = true - amazon_eks_kube_proxy_config = { - most_recent = true - kubernetes_version = local.cluster_version - resolve_conflicts = "OVERWRITE" - } + eks_addons = { - enable_amazon_eks_vpc_cni = true - amazon_eks_vpc_cni_config = { - most_recent = true - kubernetes_version = local.cluster_version - resolve_conflicts = "OVERWRITE" + # Remove for workshop as ebs-csi is long to provision (15mn) + # aws-ebs-csi-driver = { + # most_recent = true + # service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + # } + coredns = { + most_recent = true + } + vpc-cni = { + # Specify the VPC CNI addon should be deployed before compute to ensure + # the addon is configured before data plane compute resources are created + # See README for further details + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + before_compute = true + #addon_version = "v1.12.2-eksbuild.1" + most_recent = true # To ensure access to the latest settings provided + configuration_values = jsonencode({ + env = { + # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html + ENABLE_PREFIX_DELEGATION = "true" + WARM_PREFIX_TARGET = "1" + } + }) + } + kube-proxy = { + most_recent = true + } } - #--------------------------------------------------------------- - # ADD-ONS - You can add additional addons here - # https://aws-ia.github.io/terraform-aws-eks-blueprints/add-ons/ - #--------------------------------------------------------------- - - enable_metrics_server = local.metrics_server - enable_vpa = local.vpa - enable_aws_load_balancer_controller = local.aws_load_balancer_controller - aws_load_balancer_controller_helm_config = { - service_account = "aws-lb-sa" - } - enable_karpenter = local.karpenter - enable_aws_for_fluentbit = local.aws_for_fluentbit - enable_aws_cloudwatch_metrics = local.cloudwatch_metrics - - #to view the result : terraform state show 'module.kubernetes_addons.module.external_dns[0].module.helm_addon.helm_release.addon[0]' - enable_external_dns = local.external_dns - - external_dns_helm_config = { - txtOwnerId = local.name - zoneIdFilter = data.aws_route53_zone.sub.zone_id # Note: this uses GitOpsBridge - policy = "sync" - logLevel = "debug" + # EKS Blueprints Addons + enable_cert_manager = try(local.aws_addons.enable_cert_manager, false) + #enable_aws_ebs_csi_resources = try(local.aws_addons.enable_aws_ebs_csi_resources, false) + enable_aws_efs_csi_driver = try(local.aws_addons.enable_aws_efs_csi_driver, false) + enable_aws_fsx_csi_driver = try(local.aws_addons.enable_aws_fsx_csi_driver, false) + enable_aws_cloudwatch_metrics = try(local.aws_addons.enable_aws_cloudwatch_metrics, false) + enable_aws_privateca_issuer = try(local.aws_addons.enable_aws_privateca_issuer, false) + enable_cluster_autoscaler = try(local.aws_addons.enable_cluster_autoscaler, false) + enable_external_dns = try(local.aws_addons.enable_external_dns, false) + external_dns_route53_zone_arns = [data.aws_route53_zone.sub.arn] + enable_external_secrets = try(local.aws_addons.enable_external_secrets, false) + enable_aws_load_balancer_controller = try(local.aws_addons.enable_aws_load_balancer_controller, false) + aws_load_balancer_controller = { + service_account_name = "aws-lb-sa" } - enable_kubecost = local.kubecost - enable_cert_manager = local.cert_manager - enable_argo_rollouts = local.argo_rollouts + enable_fargate_fluentbit = try(local.aws_addons.enable_fargate_fluentbit, false) + enable_aws_for_fluentbit = try(local.aws_addons.enable_aws_for_fluentbit, false) + enable_aws_node_termination_handler = try(local.aws_addons.enable_aws_node_termination_handler, false) + aws_node_termination_handler_asg_arns = [for asg in module.eks.self_managed_node_groups : asg.autoscaling_group_arn] + enable_karpenter = try(local.aws_addons.enable_karpenter, false) + enable_velero = try(local.aws_addons.enable_velero, false) + #velero = { + # s3_backup_location = "${module.velero_backup_s3_bucket.s3_bucket_arn}/backups" + #} + enable_aws_gateway_api_controller = try(local.aws_addons.enable_aws_gateway_api_controller, false) + #enable_aws_secrets_store_csi_driver_provider = try(local.enable_aws_secrets_store_csi_driver_provider, false) + tags = local.tags } + module "ebs_csi_driver_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" version = "~> 5.20" - role_name_prefix = "${module.eks.cluster_name}-ebs-csi-driver-" + role_name_prefix = "${module.eks.cluster_name}-ebs-csi-" attach_ebs_csi_policy = true @@ -753,18 +702,3 @@ module "vpc_cni_irsa" { tags = local.tags } -################################################################################ -# Karpenter -################################################################################ - -# Creates Karpenter native node termination handler resources and IAM instance profile -module "karpenter" { - source = "terraform-aws-modules/eks/aws//modules/karpenter" - version = "~> 19.15.2" - - cluster_name = module.eks.cluster_name - irsa_oidc_provider_arn = module.eks.oidc_provider_arn - create_irsa = false # IRSA will be created by the kubernetes-addons module - - tags = local.tags -} diff --git a/patterns/blue-green-upgrade/modules/eks_cluster/variables.tf b/patterns/blue-green-upgrade/modules/eks_cluster/variables.tf index b23b3ca254..087f7fd1b3 100644 --- a/patterns/blue-green-upgrade/modules/eks_cluster/variables.tf +++ b/patterns/blue-green-upgrade/modules/eks_cluster/variables.tf @@ -3,43 +3,62 @@ variable "aws_region" { type = string default = "us-west-2" } + variable "environment_name" { description = "The name of Environment Infrastructure stack, feel free to rename it. Used for cluster and VPC names." type = string default = "eks-blueprint" } -variable "service_name" { - description = "The name of the Suffix for the stack name" +variable "ingress_type" { type = string - default = "blue" + description = "Type of ingress to uses (alb | nginx | ...). this parameter will be sent to arocd via gitops bridge" + default = "alb" } -variable "cluster_version" { - description = "The Version of Kubernetes to deploy" +variable "hosted_zone_name" { type = string - default = "1.25" + description = "Route53 domain for the cluster." + default = "" +} + +variable "eks_admin_role_name" { + type = string + description = "Additional IAM role to be admin in the cluster" + default = "" +} + +variable "aws_secret_manager_git_private_ssh_key_name" { + type = string + description = "Secret Manager secret name for hosting Github SSH-Key to Access private repository" + default = "github-blueprint-ssh-key" +} + +variable "argocd_secret_manager_name_suffix" { + type = string + description = "Name of secret manager secret for ArgoCD Admin UI Password" + default = "argocd-admin-secret" } variable "gitops_addons_org" { type = string description = "Git repository org/user contains for addons" - default = "https://github.com/gitops-bridge-dev" + default = "git@github.com:aws-samples" } variable "gitops_addons_repo" { type = string description = "Git repository contains for addons" - default = "gitops-bridge-argocd-control-plane-template" + default = "eks-blueprints-add-ons" } variable "gitops_addons_basepath" { type = string description = "Git repository base path for addons" - default = "" + default = "argocd/" } variable "gitops_addons_path" { type = string description = "Git repository path for addons" - default = "bootstrap/control-plane/addons" + default = "argocd/bootstrap/control-plane/addons" } variable "gitops_addons_revision" { type = string @@ -47,34 +66,10 @@ variable "gitops_addons_revision" { default = "HEAD" } -variable "hosted_zone_name" { - type = string - description = "Route53 domain for the cluster." - default = "" -} - -variable "ingress_type" { - type = string - description = "Type of ingress to uses (alb | nginx | ...). this parameter will be sent to arocd via gitops bridge" - default = "alb" -} - -variable "eks_admin_role_name" { - type = string - description = "Additional IAM role to be admin in the cluster" - default = "" -} - -variable "aws_secret_manager_git_private_ssh_key_name" { - type = string - description = "Secret Manager secret name for hosting Github SSH-Key to Access private repository" - default = "github-blueprint-ssh-key" -} - variable "gitops_workloads_org" { type = string description = "Git repository org/user contains for workloads" - default = "https://github.com/aws-samples" + default = "git@github.com:aws-samples" } variable "gitops_workloads_repo" { @@ -95,26 +90,32 @@ variable "gitops_workloads_revision" { default = "main" } -variable "argocd_secret_manager_name_suffix" { +variable "service_name" { + description = "The name of the Suffix for the stack name" type = string - description = "Name of secret manager secret for ArgoCD Admin UI Password" - default = "argocd-admin-secret" + default = "blue" +} + +variable "cluster_version" { + description = "The Version of Kubernetes to deploy" + type = string + default = "1.25" } variable "argocd_route53_weight" { description = "The Route53 weighted records weight for argocd application" type = string - default = "0" + default = "100" } variable "ecsfrontend_route53_weight" { description = "The Route53 weighted records weight for ecsdeo-frontend application" type = string - default = "0" + default = "100" } variable "route53_weight" { description = "The Route53 weighted records weight for others application" type = string - default = "0" + default = "100" } diff --git a/patterns/blue-green-upgrade/tear-down-applications.sh b/patterns/blue-green-upgrade/tear-down-applications.sh index 0cdf0fe830..4e8090e010 100755 --- a/patterns/blue-green-upgrade/tear-down-applications.sh +++ b/patterns/blue-green-upgrade/tear-down-applications.sh @@ -1,21 +1,118 @@ #!/bin/bash -set -e +#set -e +#set -x -# First tear down Applications -kubectl delete provisioners.karpenter.sh --all # this is ok if no addons are deployed on Karpenter. -kubectl delete application bootstrap-workloads -n argocd || (echo "error deleting bootstrap-workloads application" && exit -1) -kubectl delete application -l argocd.argoproj.io/application-set-name=eks-blueprints-workloads -n argocd || (echo "error deleting workloads application"; exit -1) +#export ARGOCD_PWD=$(aws secretsmanager get-secret-value --secret-id argocd-admin-secret.eks-blueprint --query SecretString --output text --region eu-west-3) +#export ARGOCD_OPTS="--port-forward --port-forward-namespace argocd --grpc-web" +#argocd login --port-forward --username admin --password $ARGOCD_PWD --insecure -#kubectl delete application ecsdemo -n argocd || (echo "error deleting ecsdemo application" && exit -1) -#namespace geordie was stuck -#kubectl get applications -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | xargs -I {} kubectl patch application {} --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' +function delete_argocd_appset_except_pattern() { + # List all your app to destroy + # Get the list of ArgoCD applications and store them in an array + #applicationsets=($(kubectl get applicationset -A -o json | jq -r '.items[] | .metadata.namespace + "/" + .metadata.name')) + applicationsets=($(kubectl get applicationset -A -o json | jq -r '.items[] | .metadata.name')) -for x in $(kubectl get namespaces -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}') ; do - echo $x - kubectl get -n $x ingress -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | xargs -I {} kubectl patch -n $x ingress {} --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' + # Iterate over the applications and delete them + for app in "${applicationsets[@]}"; do + if [[ ! "$app" =~ $1 ]]; then + echo "Deleting applicationset: $app" + kubectl delete ApplicationSet -n argocd $app --cascade=orphan + else + echo "Skipping deletion of applicationset: $app (contain '$1')" + fi + done + + #Wait for everything to delete + continue_process=true + while $continue_process; do + # Get the list of ArgoCD applications and store them in an array + applicationsets=($(kubectl get applicationset -A -o json | jq -r '.items[] | .metadata.name')) + + still_have_application=false + # Iterate over the applications and delete them + for app in "${applicationsets[@]}"; do + if [[ ! "$app" =~ $1 ]]; then + echo "applicationset $app still exists" + still_have_application=true + fi + done + sleep 5 + continue_process=$still_have_application + done + echo "No more applicationsets except $1" +} + +function delete_argocd_app_except_pattern() { + # List all your app to destroy + # Get the list of ArgoCD applications and store them in an array + #applications=($(argocd app list -o name)) + applications=($(kubectl get application -A -o json | jq -r '.items[] | .metadata.name')) + + # Iterate over the applications and delete them + for app in "${applications[@]}"; do + if [[ ! "$app" =~ $1 ]]; then + echo "Deleting application: $app" + kubectl -n argocd patch app $app -p '{"metadata": {"finalizers": ["resources-finalizer.argocd.argoproj.io"]}}' --type merge + kubectl -n argocd delete app $app + else + echo "Skipping deletion of application: $app (contain '$1')" + fi + done + + # Wait for everything to delete + continue_process=true + while $continue_process; do + # Get the list of ArgoCD applications and store them in an array + #applications=($(argocd app list -o name)) + applications=($(kubectl get application -A -o json | jq -r '.items[] | .metadata.name')) + + still_have_application=false + # Iterate over the applications and delete them + for app in "${applications[@]}"; do + if [[ ! "$app" =~ $1 ]]; then + echo "application $app still exists" + still_have_application=true + fi + done + sleep 5 + continue_process=$still_have_application + done + echo "No more applications except $1" +} + +function wait_for_deletion() { + # Loop until all Ingress resources are deleted + while true; do + # Get the list of Ingress resources in the specified namespace + ingress_list=$(kubectl get ingress -A -o json) + + # Check if there are no Ingress resources left + if [[ "$(echo "$ingress_list" | jq -r '.items | length')" -eq 0 ]]; then + echo "All Ingress resources have been deleted." + break + fi + echo "waiting for deletion" + # Wait for a while before checking again (adjust the sleep duration as needed) + sleep 5 done +} + +echo "#1. First, we deactivate application sets" +delete_argocd_appset_except_pattern "^nomatch" + +echo "#2. No we delete all app except addons" +delete_argocd_app_except_pattern "^.*addon-|^.*argo-cd|^bootstrap-addons|^team-platform" + +echo "#3. Wait for objects to be deleted" +wait_for_deletion -#Error from server (InternalError): Internal error occurred: failed calling webhook "vingress.elbv2.k8s.aws": failed to call webhook: Post "https://aws-load-balancer-webhook-service.kube-system.svc:443/validate-networking-v1-ingress?timeout=10s": no endpoints available for service "aws-load-balancer-webhook-service" + +echo "#4. Then we delete all addons except LBC and external-dns" +delete_argocd_app_except_pattern "^.*load-balancer|^.*external-dns|^.*argo-cd|^bootstrap-addons" + +#delete_argocd_app_except_pattern "^.*load-balancer" echo "Tear Down Applications OK" + +set +x diff --git a/patterns/blue-green-upgrade/tear-down.sh b/patterns/blue-green-upgrade/tear-down.sh index 826a5d3165..0fd665adeb 100755 --- a/patterns/blue-green-upgrade/tear-down.sh +++ b/patterns/blue-green-upgrade/tear-down.sh @@ -1,5 +1,6 @@ #!/bin/bash -set -e +#set -e +set -x # Get the directory of the currently executing script (shell1.sh) SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -15,18 +16,25 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" esac } -kubectl delete svc -n argocd argo-cd-argocd-server - -# terraform destroy -target="module.gitops_bridge_bootstrap" -auto-approve -# terraform destroy -target="module.eks_blueprints_addons" -auto-approve -# terraform destroy -target="module.eks" -auto-approve -# terraform destroy -target="module.vpc" -auto-approve -# terraform destroy -auto-approve +#terraform destroy -target="module.eks_cluster.module.gitops_bridge_bootstrap" -auto-approve # Then Tear down the cluster -terraform apply -destroy -target="module.eks_cluster.module.kubernetes_addons" -auto-approve || (echo "error deleting module.eks_cluster.module.kubernetes_addons" && exit -1) -terraform apply -destroy -target="module.eks_cluster.module.eks" -auto-approve || (echo "error deleting module.eks_cluster.module.eks" && exit -1) -terraform apply -destroy -auto-approve || (echo "error deleting terraform" && exit -1) +terraform destroy -target="module.eks_cluster.module.kubernetes_addons" -auto-approve || (echo "error deleting module.eks_cluster.module.kubernetes_addons" && exit -1) +terraform destroy -target="module.eks_cluster.module.eks_blueprints_platform_teams" -auto-approve || (echo "error deleting module.eks_cluster.module.eks_blueprints_platform_teams" && exit -1) +terraform destroy -target="module.eks_cluster.module.eks_blueprints_dev_teams" -auto-approve || (echo "error deleting module.eks_cluster.module.eks_blueprints_dev_teams" && exit -1) +terraform destroy -target="module.eks_cluster.module.eks_blueprints_ecsdemo_teams" -auto-approve || (echo "error deleting module.eks_cluster.module.eks_blueprints_ecsdemo_teams" && exit -1) + +terraform destroy -target="module.eks_cluster.module.gitops_bridge_bootstrap" -auto-approve || (echo "error deleting module.eks_cluster.module.gitops_bridge_bootstrap" && exit -1) +terraform destroy -target="module.eks_cluster.module.gitops_bridge_metadata" -auto-approve || (echo "error deleting module.eks_cluster.module.gitops_bridge_metadata" && exit -1) + +terraform destroy -target="module.eks_cluster.module.eks_blueprints_addons" -auto-approve || (echo "error deleting module.eks_cluster.module.eks" && exit -1) + +terraform destroy -target="module.eks_cluster.module.ebs_csi_driver_irsa" --auto-approve +terraform destroy -target="module.eks_cluster.module.vpc_cni_irsa" --auto-approve +terraform destroy -target="module.eks_cluster.module.eks" -auto-approve || (echo "error deleting module.eks_cluster.module.eks" && exit -1) + +terraform destroy -auto-approve || (echo "error deleting terraform" && exit -1) echo "Tear Down OK" +set +x