diff --git a/main.tf b/main.tf index 971f814..0d9c439 100644 --- a/main.tf +++ b/main.tf @@ -17,6 +17,15 @@ module "cluster-core" { initial_node_count = var.initial_node_count } +module "cluster-mid" { + source = "./modules/cluster-mid" + + project_id = var.project_id + load_balancing_gfe_proxy_cidr = var.load_balancing_gfe_proxy_cidr + + depends_on = [module.cluster-core] +} + module "cluster-late" { source = "./modules/cluster-late" @@ -25,13 +34,14 @@ module "cluster-late" { zones = var.zones location = module.cluster-core.location disk_encryption_key = module.cluster-core.storageclass_cmek_disk_encryption_key + cert_manager_namespace = module.cluster-mid.cert_manager_namespace + nginx_ingress_namespace = module.cluster-mid.nginx_ingress_namespace cloudflare_api_token = var.cloudflare_api_token letsencrypt_email = var.letsencrypt_email cloudflare_api_email = var.cloudflare_api_email cloudflare_domain_list = var.cloudflare_domain_list logs_retention_days = var.logs_retention_days load_balancing_network_tier = var.load_balancing_network_tier - load_balancing_gfe_proxy_cidr = var.load_balancing_gfe_proxy_cidr load_balancing_health_check_cidr = var.load_balancing_health_check_cidr load_balancing_max_connections_per_endpoint = var.load_balancing_max_connections_per_endpoint cloudflare_domain_ingress_rr = var.cloudflare_domain_ingress_rr @@ -39,5 +49,5 @@ module "cluster-late" { cloudflare_domain_ingress_proxied = var.cloudflare_domain_ingress_proxied ingress_default_wildcard_certificate = var.ingress_default_wildcard_certificate - depends_on = [module.cluster-core] + depends_on = [module.cluster-core, module.cluster-mid] } diff --git a/modules/cluster-late/certman.tf b/modules/cluster-late/certman.tf index 81e62fd..edd55f6 100644 --- a/modules/cluster-late/certman.tf +++ b/modules/cluster-late/certman.tf @@ -1,39 +1,7 @@ -resource "kubernetes_namespace" "cert_manager" { - metadata { - annotations = { - name = "cert-manager" - } - - labels = { - name = "cert-manager" - } - - name = "cert-manager" - } -} - -resource "helm_release" "cert_manager" { - name = "cert-manager" - repository = "https://charts.jetstack.io" - chart = "cert-manager" - version = "v1.1.0" - namespace = kubernetes_namespace.cert_manager.metadata[0].name - skip_crds = false - - set { - name = "installCRDs" - value = "true" - } - - values = [ - file("${path.module}/chart-values/certman-values.yaml") - ] -} - resource "kubernetes_secret" "cert_manager_cf" { metadata { name = "cloudflare-api-token-secret" - namespace = kubernetes_namespace.cert_manager.metadata[0].name + namespace = var.cert_manager_namespace } data = { @@ -80,5 +48,4 @@ resource "kubernetes_manifest" "cert_manager_cf_issuer" { } count = (var.cloudflare_api_email == "" || var.letsencrypt_email == "" || var.cloudflare_domain_list == "" ? 0 : 1) - depends_on = [helm_release.cert_manager] } diff --git a/modules/cluster-late/logging.tf b/modules/cluster-late/logging.tf index f5a79fe..b71bf0f 100644 --- a/modules/cluster-late/logging.tf +++ b/modules/cluster-late/logging.tf @@ -17,12 +17,12 @@ resource "google_logging_project_bucket_config" "cert_manager" { resource "google_logging_project_sink" "cert_manager" { name = "cert-manager" destination = "logging.googleapis.com/${google_logging_project_bucket_config.cert_manager.id}" - filter = "resource.type = k8s_container resource.labels.namespace_name=\"${kubernetes_namespace.cert_manager.metadata[0].name}\" " + filter = "resource.type = k8s_container resource.labels.namespace_name=\"${var.cert_manager_namespace}\" " unique_writer_identity = true } resource "google_logging_project_exclusion" "cert_manager" { name = "cert-manager" description = "Exclude cert-manager namespace logs. Stored elsewhere." - filter = "resource.type = k8s_container resource.labels.namespace_name=\"${kubernetes_namespace.cert_manager.metadata[0].name}\" " + filter = "resource.type = k8s_container resource.labels.namespace_name=\"${var.cert_manager_namespace}\" " } diff --git a/modules/cluster-late/nginx-ingress.tf b/modules/cluster-late/nginx-ingress.tf index 92de31a..e0237cf 100644 --- a/modules/cluster-late/nginx-ingress.tf +++ b/modules/cluster-late/nginx-ingress.tf @@ -23,46 +23,22 @@ resource "google_compute_address" "nginx_ingress_ip" { } } -resource "kubernetes_namespace" "nginx_ingress" { - metadata { - annotations = { - name = "nginx-ingress" - } - - labels = { - name = "nginx-ingress" - } - - name = "nginx-ingress" - } -} - -resource "helm_release" "nginx_ingress" { - name = "nginx-ingress" - repository = "https://kubernetes.github.io/ingress-nginx" - chart = "ingress-nginx" - version = "3.24.0" - namespace = kubernetes_namespace.nginx_ingress.metadata[0].name - skip_crds = false - - values = [ - templatefile( - "${path.module}/chart-values/nginx-ingress-values.yaml.tmpl", - { - project_id = var.project_id - gfe_proxy_cird = var.load_balancing_gfe_proxy_cidr - controller_namespace = kubernetes_namespace.nginx_ingress.metadata[0].name - default_certificate_name = "nginx-ingress-certificate" - } - ) - ] -} +# FIXME: check if really needed +#data "kubernetes_service" "nginx_ingress" { +# metadata { +# name = "nginx-ingress-ingress-nginx-controller" +# namespace = var.nginx_ingress_namespace +# } +#} data "google_compute_network_endpoint_group" "nginx_ingress_80" { for_each = toset(var.zones) name = "${var.project_id}-nginx-ingress-80" zone = each.value + + # FIXME: check if really needed + #depends_on = [data.kubernetes_service.nginx_ingress] } data "google_compute_network_endpoint_group" "nginx_ingress_443" { @@ -70,6 +46,9 @@ data "google_compute_network_endpoint_group" "nginx_ingress_443" { name = "${var.project_id}-nginx-ingress-443" zone = each.value + + # FIXME: check if really needed + #depends_on = [data.kubernetes_service.nginx_ingress] } resource "google_compute_health_check" "nginx_ingress_443_health_check" { @@ -215,7 +194,7 @@ resource "kubernetes_manifest" "nginx_ingress_certificate" { kind = "Certificate" metadata = { name = "nginx-ingress-certificate" - namespace = kubernetes_namespace.nginx_ingress.metadata[0].name + namespace = var.nginx_ingress_namespace } spec = { secretName = "nginx-ingress-certificate" @@ -227,6 +206,4 @@ resource "kubernetes_manifest" "nginx_ingress_certificate" { } } } - - depends_on = [helm_release.cert_manager] } diff --git a/modules/cluster-late/variables.tf b/modules/cluster-late/variables.tf index 8ed75a4..56f558f 100644 --- a/modules/cluster-late/variables.tf +++ b/modules/cluster-late/variables.tf @@ -23,6 +23,16 @@ variable "disk_encryption_key" { description = "The KMS key to encrypt PVs in all StorageClasses (as google_kms_crypto_key.x.self_link)." } +variable "cert_manager_namespace" { + type = string + description = "The name of Namespace with Cert Manager." +} + +variable "nginx_ingress_namespace" { + type = string + description = "The name of Namespace with Nginx Ingress Controller." +} + variable "cloudflare_api_token" { type = string sensitive = true @@ -64,12 +74,6 @@ variable "load_balancing_network_tier" { } } -# https://cloud.google.com/load-balancing/docs/tcp#firewall_rules -variable "load_balancing_gfe_proxy_cidr" { - description = "Configuration for GKE/Nginx load balancing: source IPs for Google Front End (GFE) proxies" - type = list(string) -} - # https://cloud.google.com/load-balancing/docs/tcp#firewall_rules variable "load_balancing_health_check_cidr" { description = "Configuration for GKE/Nginx load balancing: source IPs for health checks" diff --git a/modules/cluster-mid/certman.tf b/modules/cluster-mid/certman.tf new file mode 100644 index 0000000..1f3a498 --- /dev/null +++ b/modules/cluster-mid/certman.tf @@ -0,0 +1,31 @@ +resource "kubernetes_namespace" "cert_manager" { + metadata { + annotations = { + name = "cert-manager" + } + + labels = { + name = "cert-manager" + } + + name = "cert-manager" + } +} + +resource "helm_release" "cert_manager" { + name = "cert-manager" + repository = "https://charts.jetstack.io" + chart = "cert-manager" + version = "v1.1.0" + namespace = kubernetes_namespace.cert_manager.metadata[0].name + skip_crds = false + + set { + name = "installCRDs" + value = "true" + } + + values = [ + file("${path.module}/chart-values/certman-values.yaml") + ] +} diff --git a/modules/cluster-late/chart-values/certman-values.yaml b/modules/cluster-mid/chart-values/certman-values.yaml similarity index 100% rename from modules/cluster-late/chart-values/certman-values.yaml rename to modules/cluster-mid/chart-values/certman-values.yaml diff --git a/modules/cluster-late/chart-values/nginx-ingress-values.yaml.tmpl b/modules/cluster-mid/chart-values/nginx-ingress-values.yaml.tmpl similarity index 100% rename from modules/cluster-late/chart-values/nginx-ingress-values.yaml.tmpl rename to modules/cluster-mid/chart-values/nginx-ingress-values.yaml.tmpl diff --git a/modules/cluster-mid/main.tf b/modules/cluster-mid/main.tf new file mode 100644 index 0000000..1656953 --- /dev/null +++ b/modules/cluster-mid/main.tf @@ -0,0 +1,18 @@ +// Cluster-mid module should install all CRDs on the cluster +// +terraform { + required_providers { + + kubernetes = { + source = "hashicorp/kubernetes" + # FIXME: see https://github.com/rkwaysltd/gke-infra/issues/15 + version = ">= 1.13.3" + } + + helm = { + source = "hashicorp/helm" + version = ">= 2.1.0" + } + + } +} diff --git a/modules/cluster-mid/nginx-ingress.tf b/modules/cluster-mid/nginx-ingress.tf new file mode 100644 index 0000000..e25a3ac --- /dev/null +++ b/modules/cluster-mid/nginx-ingress.tf @@ -0,0 +1,34 @@ +resource "kubernetes_namespace" "nginx_ingress" { + metadata { + annotations = { + name = "nginx-ingress" + } + + labels = { + name = "nginx-ingress" + } + + name = "nginx-ingress" + } +} + +resource "helm_release" "nginx_ingress" { + name = "nginx-ingress" + repository = "https://kubernetes.github.io/ingress-nginx" + chart = "ingress-nginx" + version = "3.24.0" + namespace = kubernetes_namespace.nginx_ingress.metadata[0].name + skip_crds = false + + values = [ + templatefile( + "${path.module}/chart-values/nginx-ingress-values.yaml.tmpl", + { + project_id = var.project_id + gfe_proxy_cird = var.load_balancing_gfe_proxy_cidr + controller_namespace = kubernetes_namespace.nginx_ingress.metadata[0].name + default_certificate_name = "nginx-ingress-certificate" + } + ) + ] +} diff --git a/modules/cluster-mid/outputs.tf b/modules/cluster-mid/outputs.tf new file mode 100644 index 0000000..b96ea4a --- /dev/null +++ b/modules/cluster-mid/outputs.tf @@ -0,0 +1,9 @@ +output "cert_manager_namespace" { + description = "Certificate Manager namespace." + value = kubernetes_namespace.cert_manager.metadata[0].name +} + +output "nginx_ingress_namespace" { + description = "Nginx Ingress Controller namespace." + value = kubernetes_namespace.nginx_ingress.metadata[0].name +} diff --git a/modules/cluster-mid/variables.tf b/modules/cluster-mid/variables.tf new file mode 100644 index 0000000..f6ddc94 --- /dev/null +++ b/modules/cluster-mid/variables.tf @@ -0,0 +1,11 @@ +variable "project_id" { + type = string + description = "The project ID to host the cluster in." +} + +# https://cloud.google.com/load-balancing/docs/tcp#firewall_rules +variable "load_balancing_gfe_proxy_cidr" { + description = "Configuration for GKE/Nginx load balancing: source IPs for Google Front End (GFE) proxies" + type = list(string) +} +