Support for pinning the IP address of the load balancer via terraform overrides

docs/source/installation/configuration.md

@@ -690,6 +690,36 @@ jupyterhub:
       users: true
 ```
 
+## Terraform Overrides
+
+The QHub configuration file provides a huge number of configuration options for customizing your
+QHub Infrastructure, while these options are sufficient for an average user, but aren't
+exhaustive by any means. There are still a plenty of things you might want to achieve which
+cannot be configured directly by the above mentioned options, hence we've introduced a
+new option called terraform overrides (`terraform_overrides`), which lets you override
+the values of terraform variables in specific modules/resource. This is a relatively
+advance feature and must be used with utmost care and you should really know, what
+you're doing.
+
+Here we describe the overrides supported via QHub config file:
+
+### Ingress
+
+You can configure the IP of the load balancer and add annotations for the same via `ingress`'s
+terraform overrides, one such example for GCP is:
+
+
+```yaml
+ingress:
+  terraform_overrides:
+      load_balancer_annotations:
+          "networking.gke.io/load-balancer-type": "Internal"
+          "networking.gke.io/internal-load-balancer-subnet": "pre-existing-subnet"
+      load_balancer_ip: "1.2.3.4"
+```
+
+This is quite useful for pinning the IP Address of the load balancer.
+
 # Full configuration example
 
 ```yaml

qhub/schema.py

@@ -382,6 +382,15 @@ class QHubExtension(Base):
     envs: typing.Optional[typing.List[QHubExtensionEnv]]
 
 
+class IngressTerraformOverrides(Base):
+    load_balancer_annotations: typing.Optional[typing.Dict]
+    load_balancer_ip: str
+
+
+class Ingress(Base):
+    terraform_overrides: IngressTerraformOverrides
+
+
 # ======== External Container Registry ========
 
 # This allows the user to set a private AWS ECR as a replacement for
@@ -455,6 +464,7 @@ class Main(Base):
     prevent_deploy: bool = (
         False  # Optional, but will be given default value if not present
     )
+    ingress: typing.Optional[Ingress]
 
     # If the qhub_version in the schema is old
     # we must tell the user to first run qhub upgrade

qhub/stages/input_vars.py

@@ -142,6 +142,9 @@ def _calculate_note_groups(config):
 
 
 def stage_04_kubernetes_ingress(stage_outputs, config):
+    ingress_terraform_overrides = config.get("ingress", {}).get(
+        "terraform_overrides", {}
+    )
     return {
         "name": config["project_name"],
         "environment": config["namespace"],
@@ -152,6 +155,10 @@ def stage_04_kubernetes_ingress(stage_outputs, config):
         "certificate-secret-name": config["certificate"]["secret_name"]
         if config["certificate"]["type"] == "existing"
         else None,
+        "load-balancer-annotations": ingress_terraform_overrides.get(
+            "load_balancer_annotations", {}
+        ),
+        "load-balancer-ip": ingress_terraform_overrides.get("load-balancer-ip"),
     }
 
 

qhub/template/stages/04-kubernetes-ingress/main.tf

@@ -5,8 +5,10 @@ module "kubernetes-ingress" {
 
   node-group = var.node_groups.general
 
-  enable-certificates     = var.enable-certificates
-  acme-email              = var.acme-email
-  acme-server             = var.acme-server
-  certificate-secret-name = var.certificate-secret-name
+  enable-certificates       = var.enable-certificates
+  acme-email                = var.acme-email
+  acme-server               = var.acme-server
+  certificate-secret-name   = var.certificate-secret-name
+  load-balancer-annotations = var.load-balancer-annotations
+  load-balancer-ip          = var.load-balancer-ip
 }

qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf

@@ -64,9 +64,9 @@ resource "kubernetes_service" "main" {
   }
 
   spec {
-    selector = {
+    selector = merge({
       "app.kubernetes.io/component" = "traefik-ingress"
-    }
+    }, var.load-balancer-annotations)
 
     port {
       name        = "http"
@@ -111,6 +111,7 @@ resource "kubernetes_service" "main" {
     }
 
     type = "LoadBalancer"
+    load_balancer_ip = var.load-balancer-ip
   }
 }
 

qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf

@@ -58,3 +58,18 @@ variable "certificate-secret-name" {
   type        = string
   default     = null
 }
+
+variable "load-balancer-ip" {
+  description = "IP Address of the load balancer"
+  type = string
+  default = null
+}
+
+variable "load-balancer-annotations" {
+  description = "Annotations for the load balancer"
+  type    = map(object({
+    key   = string
+    value = string
+  }))
+  default = null
+}

qhub/template/stages/04-kubernetes-ingress/variables.tf

@@ -38,3 +38,20 @@ variable "certificate-secret-name" {
   description = "Kubernetes secret used for certificate"
   default     = ""
 }
+
+
+variable "load-balancer-ip" {
+  description = "IP Address of the load balancer"
+  type = string
+  default = null
+}
+
+
+variable "load-balancer-annotations" {
+  description = "Annotations for the load balancer"
+  type    = map(object({
+    key   = string
+    value = string
+  }))
+  default = null
+}