Limit external-dns to exactly one zone

.github/workflows/test.yml

@@ -40,10 +40,11 @@ jobs:
       # Docs for this method:
       # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
       run: |
-        INSTANCE_NAME="$(echo ci-${{ github.event.pull_request.number }}-${GITHUB_RUN_ID})"
+        INSTANCE_NAME="$(echo ci-${{ github.event.pull_request.number }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT})"
         if [[ "$(echo -n ${INSTANCE_NAME} | wc -c)" -lt  "64" ]]; then
-          INSTANCE_NAME=${INSTANCE_NAME}-special-extra-long-padding-for-real-test-case-because-ssb-needs-it
-          INSTANCE_NAME=$(head -c 64 <<< $INSTANCE_NAME)
+          INSTANCE_NAME="${INSTANCE_NAME}-a-long-suffix-to-ensure-we-are-generating-ids-that-are-short-enough-for-underlying-identifiers"
+          INSTANCE_NAME="${INSTANCE_NAME//-}"
+          INSTANCE_NAME=$(head -c 64 <<< "$INSTANCE_NAME")
         fi
         echo "INSTANCE_NAME=${INSTANCE_NAME}" | tee -a $GITHUB_ENV
 

.gitignore

@@ -54,3 +54,5 @@ kubeconfig-*
 *.swp
 terraform.log
 *.binding.json
+terraform/modules/provision-aws/provider-*.tf
+terraform/modules/provision-aws/k8s-*.tf

eks-service-definition.yml

@@ -117,23 +117,41 @@ provision:
     details: The cluster token for authentication of the eks cluster user
 
   template_refs: 
-    admin-account: terraform/modules/provision/admin-account.tf
-    crds: terraform/modules/provision/crds.tf
-    clusterdns: terraform/modules/provision/cluster-dns.tf
-    data: terraform/modules/provision/data.tf
-    eks: terraform/modules/provision/eks.tf
-    external-dns: terraform/modules/provision/external-dns.tf
-    ingress: terraform/modules/provision/ingress.tf
-    logging: terraform/modules/provision/logging.tf
-    main: terraform/modules/provision/main.tf
-    outputs: terraform/modules/provision/outputs.tf
-    providers-aws: terraform/modules/provision/providers/aws.tf
-    providers-kubernetes: terraform/modules/provision/providers/kubernetes.tf
-    providers-helm: terraform/modules/provision/providers/helm.tf
-    rbac: terraform/modules/provision/rbac.tf
-    variables: terraform/modules/provision/variables.tf
-    vpc: terraform/modules/provision/vpc.tf
-    persistent-storage: terraform/modules/provision/persistent-storage.tf
+
+    # These files make up the provision module (AWS resources)
+    cluster-dns:        terraform/modules/provision-aws/cluster-dns.tf
+    crds:               terraform/modules/provision-aws/crds.tf
+    data:               terraform/modules/provision-aws/data.tf
+    eks:                terraform/modules/provision-aws/eks.tf
+    external-dns:       terraform/modules/provision-aws/external-dns.tf
+    ingress:            terraform/modules/provision-aws/ingress.tf
+    main:               terraform/modules/provision-aws/main.tf
+    outputs:            terraform/modules/provision-aws/outputs.tf
+    persistent-storage: terraform/modules/provision-aws/persistent-storage.tf
+    variables:          terraform/modules/provision-aws/variables.tf
+    versions:           terraform/modules/provision-aws/versions.tf
+    vpc:                terraform/modules/provision-aws/vpc.tf
+
+    # Since these modules are being used as a root module in the brokerpak,
+    # these files add the necessary provider configuration.
+    provider-aws:         terraform/modules/provision-aws/providers/provider-aws.tf
+    provider-kubernetes:  terraform/modules/provision-aws/providers/provider-kubernetes.tf
+    provider-helm:        terraform/modules/provision-aws/providers/provider-helm.tf
+
+    # This file sets provision-k8s locals based on internal/output values from provision-aws
+    k8s-locals: terraform/modules/provision-aws/locals/k8s-locals.tf
+
+    # These files make up the provision module (k8s/helm resources)
+    k8s-admin-account:      terraform/modules/provision-k8s/k8s-admin-account.tf
+    k8s-external-dns:       terraform/modules/provision-k8s/k8s-external-dns.tf
+    k8s-logging:            terraform/modules/provision-k8s/k8s-logging.tf
+    k8s-outputs:            terraform/modules/provision-k8s/k8s-outputs.tf
+    k8s-persistent-storage: terraform/modules/provision-k8s/k8s-persistent-storage.tf
+
+    # EXPLICITLY OMITTED; these are only useful for submodule usage
+    # terraform/modules/provision-aws/standalone-outputs.tf
+    # terraform/modules/provision-k8s/standalone-variables.tf
+    # terraform/modules/provision-k8s/standalone-versions.tf
 
 bind:
   plan_inputs: []
@@ -181,11 +199,15 @@ bind:
     type: string
     details: The CA data for the kubernetes endpoint
   template_refs: 
-    main: terraform/modules/bind/main.tf
-    data: terraform/modules/bind/data.tf
-    outputs: terraform/modules/bind/outputs.tf
-    providers: terraform/modules/bind/providers/kubernetes.tf
-    variables: terraform/modules/bind/variables.tf
+    data:       terraform/modules/bind/data.tf
+    main:       terraform/modules/bind/main.tf
+    outputs:    terraform/modules/bind/outputs.tf
+    variables:  terraform/modules/bind/variables.tf
+    versions:   terraform/modules/bind/versions.tf
+
+    # Since this module is being used as a root module in the brokerpak, this file adds the necessary provider
+    provider-kubernetes: terraform/modules/bind/providers/provider-kubernetes.tf
+
 examples:
 - name: Demonstrate provisioning and binding the "raw" plan
   description: |

terraform/bind-providers.tf

@@ -1,6 +1,6 @@
 provider "kubernetes" {
   alias                  = "bind"
-  host                   = module.provision.server
-  cluster_ca_certificate = base64decode(module.provision.certificate_authority_data)
-  token                  = module.provision.token
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
+  token                  = module.provision-k8s.token
 }

terraform/main.tf

@@ -6,6 +6,8 @@ terraform {
     # configuration pointing specifically at the us-east-1 region. We are
     # required to set up KMS keys in that region in order for them to be usable
     # for setting up a DNSSEC KSK in Route53.
+    # Documentation on using aliased providers in a module:
+    # https://www.terraform.io/language/modules/develop/providers#provider-aliases-within-modules
     aws = {
       source                = "hashicorp/aws"
       version               = "~> 3.63"
@@ -15,7 +17,7 @@ terraform {
 }
 
 output "domain_name" {
-  value = module.provision.domain_name
+  value = module.provision-aws.domain_name
 }
 
 output "certificate_authority_data" {
@@ -87,13 +89,11 @@ variable "write_kubeconfig" {
   default = false
 }
 
-module "provision" {
-  source = "./modules/provision"
+module "provision-aws" {
+  source = "./modules/provision-aws"
   providers = {
     aws                     = aws
     aws.dnssec-key-provider = aws.dnssec-key-provider
-    kubernetes              = kubernetes.provision
-    helm                    = helm.provision
   }
   instance_name        = var.instance_name
   labels               = var.labels
@@ -107,13 +107,30 @@ module "provision" {
   zone                 = var.zone
 }
 
+module "provision-k8s" {
+  source = "./modules/provision-k8s"
+  providers = {
+    aws        = aws
+    kubernetes = kubernetes.provision
+    helm       = helm.provision
+  }
+  certificate_authority_data = module.provision-aws.certificate_authority_data
+  domain                     = module.provision-aws.domain_name
+  instance_name              = var.instance_name
+  persistent_storage_key_id  = module.provision-aws.persistent_storage_key_id
+  region                     = var.region
+  server                     = module.provision-aws.server
+  zone_id                    = module.provision-aws.zone_id
+  zone_role_arn              = module.provision-aws.zone_role_arn
+}
+
 module "bind" {
   source = "./modules/bind"
   providers = {
     kubernetes = kubernetes.bind
   }
   instance_name = var.instance_name
   depends_on = [
-    module.provision
+    module.provision-k8s
   ]
 }

terraform/modules/bind/data.tf

@@ -6,11 +6,6 @@ locals {
 data "aws_eks_cluster" "main" {
   name = local.cluster_name
 }
-
-data "aws_eks_cluster_auth" "main" {
-  name = local.cluster_name
-}
-
 # Read in the generated secret for the service account
 data "kubernetes_secret" "secret" {
   metadata {

terraform/modules/bind/main.tf

@@ -11,7 +11,8 @@ resource "kubernetes_service_account" "account" {
   }
 }
 
-# Bind the namespace-admin role to the service account
+# Make the service account an admin within the namespace. See
+# https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles.
 resource "kubernetes_role_binding" "binding" {
   metadata {
     name      = "${kubernetes_service_account.account.metadata[0].name}-namespace-admin-role-binding"
@@ -20,8 +21,8 @@ resource "kubernetes_role_binding" "binding" {
 
   role_ref {
     api_group = "rbac.authorization.k8s.io"
-    kind      = "Role"
-    name      = "namespace-admin"
+    kind      = "ClusterRole"
+    name      = "admin"
   }
 
   subject {

terraform/modules/provision/README.md → terraform/modules/provision-aws/README.md

@@ -22,12 +22,18 @@ the broker context here.
     docker build -t eks-provision:latest .
     ```
 
+1. Symlink the various files that make this module self-contained into this directory:
+
+    ```bash
+    ln -s providers/* locals/* ../provision-k8s/k8s-* .
+    ```
+
 1. Then, start a shell inside a container based on this image. The parameters
    here carry some of your environment variables into that shell, and ensure
    that you'll have permission to remove any files that get created.
 
     ```bash
-    $ docker run -v `pwd`:`pwd` -w `pwd` -e HOME=`pwd` --user $(id -u):$(id -g) -e TERM -it --rm -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION eks-provision:latest
+    $ docker run -v `pwd`/..:`pwd`/.. -w `pwd` -e HOME=`pwd` --user $(id -u):$(id -g) -e TERM -it --rm -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e AWS_DEFAULT_REGION eks-provision:latest
 
     [within the container]
     terraform init

terraform/modules/provision/eks.tf → terraform/modules/provision-aws/eks.tf

@@ -1,5 +1,4 @@
 locals {
-  # Prevent provisioning if the necessary CLI binaries aren't present
   cluster_name    = "k8s-${substr(sha256(var.instance_name), 0, 16)}"
   cluster_version = "1.21"
   kubeconfig      = "kubeconfig-${local.cluster_name}"
@@ -99,6 +98,18 @@ module "eks" {
       max_size     = var.mng_max_capacity
       min_size     = var.mng_min_capacity
 
+      block_device_mappings = {
+        xvda = {
+          device_name = "/dev/xvda"
+          ebs = {
+            volume_size           = 20
+            encrypted             = true
+            kms_key_id            = aws_kms_key.ebs-key.arn
+            delete_on_termination = true
+          }
+        }
+      }
+
       instance_types = var.mng_instance_types
       capacity_type  = "ON_DEMAND"
     }
@@ -132,6 +143,28 @@ resource "aws_iam_role_policy_attachment" "ebs-usage" {
   role       = each.value.iam_role_name
 }
 
+# ---------------------------------------------
+# Logging Policy for the pod execution IAM role
+# ---------------------------------------------
+resource "aws_iam_policy" "pod-logging" {
+  name   = "${local.cluster_name}-pod-logging"
+  policy = <<-EOF
+  {
+    "Version": "2012-10-17",
+    "Statement": [{
+      "Effect": "Allow",
+      "Action": [
+        "logs:CreateLogStream",
+        "logs:CreateLogGroup",
+        "logs:DescribeLogStreams",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "*"
+    }]
+  }
+  EOF
+}
+
 
 # Generate a kubeconfig file for use in provisioners
 data "template_file" "kubeconfig" {

terraform/modules/provision-aws/external-dns.tf

@@ -0,0 +1,59 @@
+# Modeled after an example here:
+# https://tech.polyconseil.fr/external-dns-helm-terraform.html
+
+data "aws_caller_identity" "current" {}
+
+resource "aws_iam_role" "external_dns" {
+  name               = "${local.cluster_name}-external-dns"
+  tags               = var.labels
+  assume_role_policy = <<-EOF
+    {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+        "Action": "sts:AssumeRoleWithWebIdentity",
+        "Effect": "Allow",
+        "Principal": {
+            "Federated": "arn:aws:iam::${data.aws_caller_identity.current.account_id}:oidc-provider/${module.eks.oidc_provider}"
+        },
+        "Condition": {
+            "StringEquals": {
+            "${module.eks.oidc_provider}:sub": "system:serviceaccount:kube-system:external-dns"
+            }
+        }
+        }
+    ]
+    }
+    EOF
+}
+
+resource "aws_iam_role_policy" "external_dns" {
+  name_prefix = "${local.cluster_name}-external-dns"
+  role        = aws_iam_role.external_dns.name
+  policy      = <<-EOF
+    {
+        "Version": "2012-10-17",
+        "Statement": [
+            {
+                "Effect": "Allow",
+                "Action": [
+                    "route53:ChangeResourceRecordSets"
+                ],
+                "Resource": [
+                    "arn:aws:route53:::hostedzone/${aws_route53_zone.cluster.zone_id}"
+                ]
+            },
+            {
+                "Effect": "Allow",
+                "Action": [
+                    "route53:ListHostedZones",
+                    "route53:ListResourceRecordSets"
+                ],
+                "Resource": [
+                    "*"
+                ]
+            }
+        ]
+    }
+    EOF
+}

terraform/modules/provision/ingress.tf → terraform/modules/provision-aws/ingress.tf

@@ -47,6 +47,8 @@ resource "helm_release" "ingress_nginx" {
 
   dynamic "set" {
     for_each = {
+      "aws_iam_role_arn"                                                                                   = module.aws_load_balancer_controller.aws_iam_role_arn
+      "controller.ingressClassResource.default"                                                            = true
       "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-scheme"           = "internet-facing",
       "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-ports"        = "https",
       "controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-cert"         = aws_acm_certificate.cert.arn,
@@ -87,7 +89,6 @@ resource "helm_release" "ingress_nginx" {
       "clusterName"                                  = module.eks.cluster_id,
       "region"                                       = local.region,
       "vpcId"                                        = module.vpc.vpc_id,
-      "aws_iam_role_arn"                             = module.aws_load_balancer_controller.aws_iam_role_arn
     }
     content {
       name  = set.key

terraform/modules/provision-aws/locals/k8s-locals.tf

@@ -0,0 +1,7 @@
+locals {
+  certificate_authority_data = data.aws_eks_cluster.main.certificate_authority[0].data
+  persistent_storage_key_id  = aws_kms_key.ebs-key.key_id
+  server                     = data.aws_eks_cluster.main.endpoint
+  zone_id                    = aws_route53_zone.cluster.zone_id
+  zone_role_arn              = aws_iam_role.external_dns.arn
+}

terraform/modules/provision/main.tf → terraform/modules/provision-aws/main.tf

@@ -16,7 +16,7 @@
 # can to avoid that. 
 resource "null_resource" "prerequisite_binaries_present" {
   provisioner "local-exec" {
-    interpreter = ["/bin/sh", "-c"]
-    command     = "type -p aws-iam-authenticator git helm kubectl"
+    interpreter = ["/bin/bash", "-c"]
+    command     = "type -a aws-iam-authenticator git helm kubectl"
   }
 }

terraform/modules/provision-aws/outputs.tf

@@ -0,0 +1,4 @@
+output "domain_name" { value = local.domain }
+output "server" { value = data.aws_eks_cluster.main.endpoint }
+output "certificate_authority_data" { value = data.aws_eks_cluster.main.certificate_authority[0].data }
+output "cluster-id" { value = data.aws_eks_cluster.main.id }