Move all CRD-creating resources to cluster-mid module for better laye…

…ring Didn't solve #6 but it's still better IMHO
rkwaysltd · Apr 11, 2021 · c31db53 · c31db53
1 parent a4667d5
commit c31db53
Show file tree

Hide file tree

Showing 12 changed files with 142 additions and 81 deletions.
diff --git a/main.tf b/main.tf
@@ -17,6 +17,15 @@ module "cluster-core" {
   initial_node_count = var.initial_node_count
 }
 
+module "cluster-mid" {
+  source = "./modules/cluster-mid"
+
+  project_id                    = var.project_id
+  load_balancing_gfe_proxy_cidr = var.load_balancing_gfe_proxy_cidr
+
+  depends_on = [module.cluster-core]
+}
+
 module "cluster-late" {
   source = "./modules/cluster-late"
 
@@ -25,19 +34,20 @@ module "cluster-late" {
   zones                                       = var.zones
   location                                    = module.cluster-core.location
   disk_encryption_key                         = module.cluster-core.storageclass_cmek_disk_encryption_key
+  cert_manager_namespace                      = module.cluster-mid.cert_manager_namespace
+  nginx_ingress_namespace                     = module.cluster-mid.nginx_ingress_namespace
   cloudflare_api_token                        = var.cloudflare_api_token
   letsencrypt_email                           = var.letsencrypt_email
   cloudflare_api_email                        = var.cloudflare_api_email
   cloudflare_domain_list                      = var.cloudflare_domain_list
   logs_retention_days                         = var.logs_retention_days
   load_balancing_network_tier                 = var.load_balancing_network_tier
-  load_balancing_gfe_proxy_cidr               = var.load_balancing_gfe_proxy_cidr
   load_balancing_health_check_cidr            = var.load_balancing_health_check_cidr
   load_balancing_max_connections_per_endpoint = var.load_balancing_max_connections_per_endpoint
   cloudflare_domain_ingress_rr                = var.cloudflare_domain_ingress_rr
   ingress_rr_name                             = var.ingress_rr_name
   cloudflare_domain_ingress_proxied           = var.cloudflare_domain_ingress_proxied
   ingress_default_wildcard_certificate        = var.ingress_default_wildcard_certificate
 
-  depends_on = [module.cluster-core]
+  depends_on = [module.cluster-core, module.cluster-mid]
 }
diff --git a/modules/cluster-late/certman.tf b/modules/cluster-late/certman.tf
@@ -1,39 +1,7 @@
-resource "kubernetes_namespace" "cert_manager" {
-  metadata {
-    annotations = {
-      name = "cert-manager"
-    }
-
-    labels = {
-      name = "cert-manager"
-    }
-
-    name = "cert-manager"
-  }
-}
-
-resource "helm_release" "cert_manager" {
-  name       = "cert-manager"
-  repository = "https://charts.jetstack.io"
-  chart      = "cert-manager"
-  version    = "v1.1.0"
-  namespace  = kubernetes_namespace.cert_manager.metadata[0].name
-  skip_crds  = false
-
-  set {
-    name  = "installCRDs"
-    value = "true"
-  }
-
-  values = [
-    file("${path.module}/chart-values/certman-values.yaml")
-  ]
-}
-
 resource "kubernetes_secret" "cert_manager_cf" {
   metadata {
     name      = "cloudflare-api-token-secret"
-    namespace = kubernetes_namespace.cert_manager.metadata[0].name
+    namespace = var.cert_manager_namespace
   }
 
   data = {
@@ -80,5 +48,4 @@ resource "kubernetes_manifest" "cert_manager_cf_issuer" {
   }
 
   count      = (var.cloudflare_api_email == "" || var.letsencrypt_email == "" || var.cloudflare_domain_list == "" ? 0 : 1)
-  depends_on = [helm_release.cert_manager]
 }
diff --git a/modules/cluster-late/logging.tf b/modules/cluster-late/logging.tf
@@ -17,12 +17,12 @@ resource "google_logging_project_bucket_config" "cert_manager" {
 resource "google_logging_project_sink" "cert_manager" {
   name                   = "cert-manager"
   destination            = "logging.googleapis.com/${google_logging_project_bucket_config.cert_manager.id}"
-  filter                 = "resource.type = k8s_container resource.labels.namespace_name=\"${kubernetes_namespace.cert_manager.metadata[0].name}\" "
+  filter                 = "resource.type = k8s_container resource.labels.namespace_name=\"${var.cert_manager_namespace}\" "
   unique_writer_identity = true
 }
 
 resource "google_logging_project_exclusion" "cert_manager" {
   name        = "cert-manager"
   description = "Exclude cert-manager namespace logs. Stored elsewhere."
-  filter      = "resource.type = k8s_container resource.labels.namespace_name=\"${kubernetes_namespace.cert_manager.metadata[0].name}\" "
+  filter      = "resource.type = k8s_container resource.labels.namespace_name=\"${var.cert_manager_namespace}\" "
 }
diff --git a/modules/cluster-late/nginx-ingress.tf b/modules/cluster-late/nginx-ingress.tf
@@ -23,53 +23,32 @@ resource "google_compute_address" "nginx_ingress_ip" {
   }
 }
 
-resource "kubernetes_namespace" "nginx_ingress" {
-  metadata {
-    annotations = {
-      name = "nginx-ingress"
-    }
-
-    labels = {
-      name = "nginx-ingress"
-    }
-
-    name = "nginx-ingress"
-  }
-}
-
-resource "helm_release" "nginx_ingress" {
-  name       = "nginx-ingress"
-  repository = "https://kubernetes.github.io/ingress-nginx"
-  chart      = "ingress-nginx"
-  version    = "3.24.0"
-  namespace  = kubernetes_namespace.nginx_ingress.metadata[0].name
-  skip_crds  = false
-
-  values = [
-    templatefile(
-      "${path.module}/chart-values/nginx-ingress-values.yaml.tmpl",
-      {
-        project_id               = var.project_id
-        gfe_proxy_cird           = var.load_balancing_gfe_proxy_cidr
-        controller_namespace     = kubernetes_namespace.nginx_ingress.metadata[0].name
-        default_certificate_name = "nginx-ingress-certificate"
-      }
-    )
-  ]
-}
+# FIXME: check if really needed
+#data "kubernetes_service" "nginx_ingress" {
+#  metadata {
+#    name = "nginx-ingress-ingress-nginx-controller"
+#    namespace = var.nginx_ingress_namespace
+#  }
+#}
 
 data "google_compute_network_endpoint_group" "nginx_ingress_80" {
   for_each = toset(var.zones)
 
   name = "${var.project_id}-nginx-ingress-80"
   zone = each.value
+
+  # FIXME: check if really needed
+  #depends_on = [data.kubernetes_service.nginx_ingress]
 }
 
 data "google_compute_network_endpoint_group" "nginx_ingress_443" {
   for_each = toset(var.zones)
 
   name = "${var.project_id}-nginx-ingress-443"
   zone = each.value
+
+  # FIXME: check if really needed
+  #depends_on = [data.kubernetes_service.nginx_ingress]
 }
 
 resource "google_compute_health_check" "nginx_ingress_443_health_check" {
@@ -215,7 +194,7 @@ resource "kubernetes_manifest" "nginx_ingress_certificate" {
     kind       = "Certificate"
     metadata = {
       name      = "nginx-ingress-certificate"
-      namespace = kubernetes_namespace.nginx_ingress.metadata[0].name
+      namespace = var.nginx_ingress_namespace
     }
     spec = {
       secretName = "nginx-ingress-certificate"
@@ -227,6 +206,4 @@ resource "kubernetes_manifest" "nginx_ingress_certificate" {
       }
     }
   }
-
-  depends_on = [helm_release.cert_manager]
 }
diff --git a/modules/cluster-late/variables.tf b/modules/cluster-late/variables.tf
@@ -23,6 +23,16 @@ variable "disk_encryption_key" {
   description = "The KMS key to encrypt PVs in all StorageClasses (as google_kms_crypto_key.x.self_link)."
 }
 
+variable "cert_manager_namespace" {
+  type        = string
+  description = "The name of Namespace with Cert Manager."
+}
+
+variable "nginx_ingress_namespace" {
+  type        = string
+  description = "The name of Namespace with Nginx Ingress Controller."
+}
+
 variable "cloudflare_api_token" {
   type        = string
   sensitive   = true
@@ -64,12 +74,6 @@ variable "load_balancing_network_tier" {
   }
 }
 
-# https://cloud.google.com/load-balancing/docs/tcp#firewall_rules
-variable "load_balancing_gfe_proxy_cidr" {
-  description = "Configuration for GKE/Nginx load balancing: source IPs for Google Front End (GFE) proxies"
-  type        = list(string)
-}
-
 # https://cloud.google.com/load-balancing/docs/tcp#firewall_rules
 variable "load_balancing_health_check_cidr" {
   description = "Configuration for GKE/Nginx load balancing: source IPs for health checks"

diff --git a/modules/cluster-mid/certman.tf b/modules/cluster-mid/certman.tf
@@ -0,0 +1,31 @@
+resource "kubernetes_namespace" "cert_manager" {
+  metadata {
+    annotations = {
+      name = "cert-manager"
+    }
+
+    labels = {
+      name = "cert-manager"
+    }
+
+    name = "cert-manager"
+  }
+}
+
+resource "helm_release" "cert_manager" {
+  name       = "cert-manager"
+  repository = "https://charts.jetstack.io"
+  chart      = "cert-manager"
+  version    = "v1.1.0"
+  namespace  = kubernetes_namespace.cert_manager.metadata[0].name
+  skip_crds  = false
+
+  set {
+    name  = "installCRDs"
+    value = "true"
+  }
+
+  values = [
+    file("${path.module}/chart-values/certman-values.yaml")
+  ]
+}
diff --git a/...ter-late/chart-values/certman-values.yaml → ...ster-mid/chart-values/certman-values.yaml b/...ter-late/chart-values/certman-values.yaml → ...ster-mid/chart-values/certman-values.yaml
diff --git a/...art-values/nginx-ingress-values.yaml.tmpl → ...art-values/nginx-ingress-values.yaml.tmpl b/...art-values/nginx-ingress-values.yaml.tmpl → ...art-values/nginx-ingress-values.yaml.tmpl
diff --git a/modules/cluster-mid/main.tf b/modules/cluster-mid/main.tf
@@ -0,0 +1,18 @@
+// Cluster-mid module should install all CRDs on the cluster
+//
+terraform {
+  required_providers {
+
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+      # FIXME: see https://github.com/rkwaysltd/gke-infra/issues/15
+      version = ">= 1.13.3"
+    }
+
+    helm = {
+      source  = "hashicorp/helm"
+      version = ">= 2.1.0"
+    }
+
+  }
+}
diff --git a/modules/cluster-mid/nginx-ingress.tf b/modules/cluster-mid/nginx-ingress.tf
@@ -0,0 +1,34 @@
+resource "kubernetes_namespace" "nginx_ingress" {
+  metadata {
+    annotations = {
+      name = "nginx-ingress"
+    }
+
+    labels = {
+      name = "nginx-ingress"
+    }
+
+    name = "nginx-ingress"
+  }
+}
+
+resource "helm_release" "nginx_ingress" {
+  name       = "nginx-ingress"
+  repository = "https://kubernetes.github.io/ingress-nginx"
+  chart      = "ingress-nginx"
+  version    = "3.24.0"
+  namespace  = kubernetes_namespace.nginx_ingress.metadata[0].name
+  skip_crds  = false
+
+  values = [
+    templatefile(
+      "${path.module}/chart-values/nginx-ingress-values.yaml.tmpl",
+      {
+        project_id               = var.project_id
+        gfe_proxy_cird           = var.load_balancing_gfe_proxy_cidr
+        controller_namespace     = kubernetes_namespace.nginx_ingress.metadata[0].name
+        default_certificate_name = "nginx-ingress-certificate"
+      }
+    )
+  ]
+}
diff --git a/modules/cluster-mid/outputs.tf b/modules/cluster-mid/outputs.tf
@@ -0,0 +1,9 @@
+output "cert_manager_namespace" {
+  description = "Certificate Manager namespace."
+  value       = kubernetes_namespace.cert_manager.metadata[0].name
+}
+
+output "nginx_ingress_namespace" {
+  description = "Nginx Ingress Controller namespace."
+  value       = kubernetes_namespace.nginx_ingress.metadata[0].name
+}
diff --git a/modules/cluster-mid/variables.tf b/modules/cluster-mid/variables.tf
@@ -0,0 +1,11 @@
+variable "project_id" {
+  type        = string
+  description = "The project ID to host the cluster in."
+}
+
+# https://cloud.google.com/load-balancing/docs/tcp#firewall_rules
+variable "load_balancing_gfe_proxy_cidr" {
+  description = "Configuration for GKE/Nginx load balancing: source IPs for Google Front End (GFE) proxies"
+  type        = list(string)
+}
+