From 927d443e89dcb4a1202451db4fb55a90273761ef Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Mon, 11 Apr 2022 19:39:01 +0530 Subject: [PATCH] Support for pinning the IP address of the load balancer via terraform overrides (#1235) * Suport adding load balancer annotations and ip via terraform overrides * add documentation for terraform overrides * make terraform overrides being able to override any variable --- docs/source/installation/configuration.md | 30 +++++++++++++++++++ qhub/schema.py | 5 ++++ qhub/stages/input_vars.py | 1 + .../stages/04-kubernetes-ingress/main.tf | 10 ++++--- .../modules/kubernetes/ingress/main.tf | 5 ++-- .../modules/kubernetes/ingress/variables.tf | 15 ++++++++++ .../stages/04-kubernetes-ingress/variables.tf | 17 +++++++++++ 7 files changed, 77 insertions(+), 6 deletions(-) diff --git a/docs/source/installation/configuration.md b/docs/source/installation/configuration.md index 9668fa36a..6ca8d1b86 100644 --- a/docs/source/installation/configuration.md +++ b/docs/source/installation/configuration.md @@ -690,6 +690,36 @@ jupyterhub: users: true ``` +## Terraform Overrides + +The QHub configuration file provides a huge number of configuration options for customizing your +QHub Infrastructure, while these options are sufficient for an average user, but aren't +exhaustive by any means. There are still a plenty of things you might want to achieve which +cannot be configured directly by the above mentioned options, hence we've introduced a +new option called terraform overrides (`terraform_overrides`), which lets you override +the values of terraform variables in specific modules/resource. This is a relatively +advance feature and must be used with utmost care and you should really know, what +you're doing. + +Here we describe the overrides supported via QHub config file: + +### Ingress + +You can configure the IP of the load balancer and add annotations for the same via `ingress`'s +terraform overrides, one such example for GCP is: + + +```yaml +ingress: + terraform_overrides: + load-balancer-annotations: + "networking.gke.io/load-balancer-type": "Internal" + "networking.gke.io/internal-load-balancer-subnet": "pre-existing-subnet" + load-balancer-ip: "1.2.3.4" +``` + +This is quite useful for pinning the IP Address of the load balancer. + # Full configuration example ```yaml diff --git a/qhub/schema.py b/qhub/schema.py index 10b5918f1..498e5eddf 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -382,6 +382,10 @@ class QHubExtension(Base): envs: typing.Optional[typing.List[QHubExtensionEnv]] +class Ingress(Base): + terraform_overrides: typing.Any + + # ======== External Container Registry ======== # This allows the user to set a private AWS ECR as a replacement for @@ -455,6 +459,7 @@ class Main(Base): prevent_deploy: bool = ( False # Optional, but will be given default value if not present ) + ingress: typing.Optional[Ingress] # If the qhub_version in the schema is old # we must tell the user to first run qhub upgrade diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index b68319f33..68e718caa 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -152,6 +152,7 @@ def stage_04_kubernetes_ingress(stage_outputs, config): "certificate-secret-name": config["certificate"]["secret_name"] if config["certificate"]["type"] == "existing" else None, + **config.get("ingress", {}).get("terraform_overrides", {}), } diff --git a/qhub/template/stages/04-kubernetes-ingress/main.tf b/qhub/template/stages/04-kubernetes-ingress/main.tf index 44c8387c5..ec2348f9e 100644 --- a/qhub/template/stages/04-kubernetes-ingress/main.tf +++ b/qhub/template/stages/04-kubernetes-ingress/main.tf @@ -5,8 +5,10 @@ module "kubernetes-ingress" { node-group = var.node_groups.general - enable-certificates = var.enable-certificates - acme-email = var.acme-email - acme-server = var.acme-server - certificate-secret-name = var.certificate-secret-name + enable-certificates = var.enable-certificates + acme-email = var.acme-email + acme-server = var.acme-server + certificate-secret-name = var.certificate-secret-name + load-balancer-annotations = var.load-balancer-annotations + load-balancer-ip = var.load-balancer-ip } diff --git a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf index 99a2ff71e..32a284899 100644 --- a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf +++ b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/main.tf @@ -64,9 +64,9 @@ resource "kubernetes_service" "main" { } spec { - selector = { + selector = merge({ "app.kubernetes.io/component" = "traefik-ingress" - } + }, var.load-balancer-annotations) port { name = "http" @@ -111,6 +111,7 @@ resource "kubernetes_service" "main" { } type = "LoadBalancer" + load_balancer_ip = var.load-balancer-ip } } diff --git a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf index 6d5eac16e..d30bb9334 100644 --- a/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf +++ b/qhub/template/stages/04-kubernetes-ingress/modules/kubernetes/ingress/variables.tf @@ -58,3 +58,18 @@ variable "certificate-secret-name" { type = string default = null } + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(object({ + key = string + value = string + })) + default = null +} diff --git a/qhub/template/stages/04-kubernetes-ingress/variables.tf b/qhub/template/stages/04-kubernetes-ingress/variables.tf index c3dc006ba..85929c4fb 100644 --- a/qhub/template/stages/04-kubernetes-ingress/variables.tf +++ b/qhub/template/stages/04-kubernetes-ingress/variables.tf @@ -38,3 +38,20 @@ variable "certificate-secret-name" { description = "Kubernetes secret used for certificate" default = "" } + + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(object({ + key = string + value = string + })) + default = null +}