Terraform GCP HTTP(S) Load Balancing

Terraform module for provisioning of GCP LB on top of precreated named NEGs, Cloud Run services and GCS buckets passed as parameter to this module.

Usage

HTTPS Load-balancer with self-signed certificate and Cloudflare DNS record creation:

data "cloudflare_zones" "ackee_cz" {
  filter {
    name = "ackee.cz"
  }
}

resource "google_storage_bucket" "test" {
  name                        = "test-randompostfix-98582341"
  location                    = var.region
  storage_class               = "STANDARD"
  uniform_bucket_level_access = true
  website {
    main_page_suffix = "index.html"
  }
}

module "api_unicorn" {
  source          = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name            = "main-${var.project}-${var.namespace}"
  project         = var.project
  region          = var.region
  self_signed_tls = true

  services = [
    {
      type = "neg"
      name = "ackee-api-unicorn"
      zone = var.zone
    },
    {
      type        = "bucket"
      bucket_name = "${google_storage_bucket.test.name}"
    },
    {
      type         = "cloudrun"
      service_name = cloud-run-service
    }
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz", "api-unicorn2.ackee.cz"]
      path_rules = [
        {
          paths = ["/api/v1/*"]
          service = {
            type = "neg"
            name = "ackee-api-unicorn"
            zone = var.zone
          }
        },
      ]
    }
    matcher2 = {
      hostnames  = ["api-unicorn.ackee.cz", "api-unicorn2.ackee.cz"]
      path_rules = [
        {
          paths   = ["/*"]
          service = "${google_storage_bucket.test.name}"
        },
      ]
    }
    matcher3 = {
      hostnames  = ["cloud-run-service.ackee.cz"]
      path_rules = [
        {
          paths   = ["/*"]
          service = "cloud-run-service"
        },
      ]
    }
  }
}

resource "cloudflare_record" "api" {
  zone_id = data.cloudflare_zones.ackee_cz.zones[0].id
  name    = "api-unicorn"
  value   = module.api_unicorn.ip_address
  type    = "A"
  ttl     = 1
  proxied = true
}

If NEG named ackee-api-unicorn exists and CF is set to "SSL:Full" you should have working app now on https://api-unicorn.ackee.cz and https://api-unicorn2.ackee.cz

HTTPS Load-balancer with Google-managed certificate and Cloudflare DNS record creation:

data "cloudflare_zones" "ackee_cz" {
  filter {
    name = "ackee.cz"
  }
}

module "api_unicorn" {
  source             = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name               = "main-${var.project}-${var.namespace}"
  project            = var.project
  region             = var.region
  google_managed_tls = true

  services = [
    {
      type = "neg"
      name = "ackee-api-unicorn"
      zone = var.zone
    },
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz", "api-unicorn2.ackee.cz"]
      path_rules = [
        {
          paths = ["/api/v1/*"]
          service = "ackee-api-unicorn"
        },
      ]
    }
  }
}

resource "cloudflare_record" "api" {
  zone_id = data.cloudflare_zones.ackee_cz.zones[0].id
  name    = "api-unicorn"
  value   = module.api_unicorn.ip_address
  type    = "A"
  ttl     = 1
  proxied = false
}

If NEG named ackee-api-unicorn exists you should have working app now on https://api-unicorn.ackee.cz Beware: If you use more then one hostname with Google-managed certificate, only one certificate, with first hostname in list, will be created.

HTTPS Load-balancer with preexisting NEG, Google-managed certificate and Cloudflare DNS record creation:

data "google_compute_network_endpoint_group" "old_neg" {
  name  = "k8s1-aab5af95-production-ackee-unicorn-80-bafd3c69"
  zone  = "europe-west3-c"
  count = 1
}

data "cloudflare_zones" "ackee_cz" {
  filter {
    name = "ackee.cz"
  }
}

module "api_unicorn" {
  source             = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name               = "main-${var.project}-${var.namespace}"
  project            = var.project
  region             = var.region
  google_managed_tls = true

  services = [
    {
      type                  = "neg"
      name                  = "ackee-api-unicorn"
      zone                  = var.zone
      additional_negs       = [data.google_compute_network_endpoint_group.old_neg]
      http_backend_protocol = "HTTP"
    },
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz""]
      path_rules = [
        {
          paths   = ["/*"]
          service = "ackee-api-unicorn"
        },
      ]
    }
  }
}

resource "cloudflare_record" "api" {
  zone_id = data.cloudflare_zones.ackee_cz.zones[0].id
  name    = "api-unicorn"
  value   = module.api_unicorn.ip_address
  type    = "A"
  ttl     = 1
  proxied = false
}

If we pass data.google_compute_network_endpoint_group resource as value for additional_negs parameter, then our new load-balancer gets created from new named NEG's auto discovered by name in neg_name parameter and from NEG's from additional_negs parameter - this should be used when migrating from old setup, so we balance to both new and old application.

HTTPS Load-balancer with pre-existing certificate (signed by external CA):

module "api_unicorn" {
  source             = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name               = "main-${var.project}-${var.namespace}"
  project            = var.project
  region             = var.region

  services = [
    {
      type                  = "neg"
      name                  = "ackee-api-unicorn"
      zone                  = var.zone
      additional_negs       = [data.google_compute_network_endpoint_group.old_neg]
      http_backend_protocol = "HTTP"
    },
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz""]
      path_rules = [
        {
          paths   = ["/*"]
          service = "ackee-api-unicorn"
        },
      ]
    }
  }

  certificate = file("${path.root}/tls/certificate_chain.crt")
  private_key = file("${path.root}/tls/private.key")
}

It is recommended to use some secure storage (eg. Vault) and pass value from here, rather then saving plaintext private key into git repo

Pitfalls

Error: Error creating SslCertificate: googleapi: Error 409: The resource ... already exists, alreadyExists

This might show once you are adding new hostname to the load balancer and SSL certificate web_lb_cert needs to add the hostname into dns_names. Terraform is trying to update the certificate in-place or creates a certificate with the same name. For that, you might want to do these few steps manually:

Get certificates from the state file:

CERT=`mktemp`
CERT_KEY=`mktemp`

terraform show -json | jq -r --arg MODULE "$MODULE" '.values.root_module.child_modules[] | select (.address=="module.lb_72541") | .resources[] | select(.address=="module.lb_72541.google_compute_ssl_certificate.gcs_certs[0]") | .values.private_key' > $CERT_KEY
terraform show -json | jq -r --arg MODULE "$MODULE" '.values.root_module.child_modules[] | select (.address=="module.lb_72541") | .resources[] | select(.address=="module.lb_72541.google_compute_ssl_certificate.gcs_certs[0]") | .values.certificate' > $CERT

where module.lb_72541 is the name of the module used in your Terraform.

Create new temporary certificate:

gcloud compute ssl-certificates create tmp --certificate=$CERT --private-key=$CERT_KEY
gcloud compute target-https-proxies update "NAME_OF_PROXY" --ssl-certificates "tmp"

The name of the https proxy can be found in the state file:

terraform state show 'module.lb_72541.google_compute_target_https_proxy.self_signed[0]'

Remove the old certificate:

Get the name from the error output. Let's say you have this error:

Error: Error creating SslCertificate: googleapi: Error 409: The resource 'projects/awesome-project/global/sslCertificates/main-awesome-project-development-72541-cert-self-signed' already exists, alreadyExists

Then command will look like this:

gcloud compute ssl-certificates delete main-awesome-project-development-72541-cert-self-signed

Run terraform apply to create a certificate from Terraform and once done delete the temporary certificate:

gcloud compute ssl-certificates delete tmp

Creation of NEG's is not automatic!

BEWARE: Network Endpoint Groups REFERENCED BY THIS MODULE MUST EXIST BEFORE YOU USE THIS MODULE, OTHERWISE IT WILL FAIL WITH ERROR SIMILIAR TO:

Error: Required attribute is not set

  on ../load-balancer.tf line 68, in resource "google_compute_backend_service" "cn_lb":
  68: resource "google_compute_backend_service" "cn_lb" {

Also note, that purging app (typically with helm delete) does not automatically cleanup existing NEGs

Using example

Because of chicken-egg mentioned in previous section, it is also not so easy to use provider example code. Running just terraform apply will fail on similiar error. Workaround is to create testing NEG first and load balancer above it in next step.

terraform apply -target=google_compute_network_endpoint_group.neg_one -target=google_compute_network_endpoint_group.neg_two
terraform apply

Before you do anything in this module

Install pre-commit hooks by running following commands:

brew install pre-commit
pre-commit install

Requirements

Name	Version
terraform	>= 1.0

Providers

Name	Version
google	5.8.0
google-beta	5.8.0
random	3.6.0
tls	4.0.5

Modules

No modules.

Resources

Name	Type
google-beta_google_compute_backend_service.app_backend	resource
google-beta_google_compute_backend_service.cloudrun	resource
google-beta_google_compute_global_forwarding_rule.external_signed	resource
google-beta_google_compute_global_forwarding_rule.google_managed	resource
google-beta_google_compute_global_forwarding_rule.self_signed	resource
google_compute_backend_bucket.bucket	resource
google_compute_backend_bucket.cn_lb	resource
google_compute_firewall.gcp_hc_ip_allow	resource
google_compute_global_address.gca	resource
google_compute_global_forwarding_rule.non_tls	resource
google_compute_health_check.cn_lb	resource
google_compute_managed_ssl_certificate.gcs_certs	resource
google_compute_region_network_endpoint_group.cloudrun_neg	resource
google_compute_ssl_certificate.external_certs	resource
google_compute_ssl_certificate.gcs_certs	resource
google_compute_target_http_proxy.non_tls	resource
google_compute_target_https_proxy.external_signed	resource
google_compute_target_https_proxy.google_managed	resource
google_compute_target_https_proxy.self_signed	resource
google_compute_url_map.cn_lb	resource
google_logging_project_sink.log_archive_sink	resource
google_storage_bucket.cn_lb	resource
google_storage_bucket.log_archive_sink	resource
google_storage_bucket_iam_binding.log_archive_sink_writer	resource
random_id.external_certificate	resource
random_string.random_suffix	resource
tls_private_key.web_lb_key	resource
tls_self_signed_cert.web_lb_cert	resource
google_cloud_run_service.cloud_run_service	data source
google_compute_network_endpoint_group.cn_lb	data source
google_compute_zones.available	data source

Inputs

Name	Description	Type	Default	Required
allow_non_tls_frontend	If true, enables port 80 frontend - creates non-TLS (http://) variant of LB	string	false	no
backend_bucket_location	GCS location(https://cloud.google.com/storage/docs/locations) of bucket where invalid requests are routed.	string	"EUROPE-WEST3"	no
certificate	The certificate in PEM format. The certificate chain must be no greater than 5 certs long. The chain must include at least one intermediate cert. Note: This property is sensitive and will not be displayed in the plan.	string	null	no
check_interval_sec	How often (in seconds) to send a health check. The default value is 5 seconds.	number	5	no
create_logging_sink_bucket	If true, creates bucket and set up logging sink	bool	false	no
custom_health_check_ports	Custom ports for GCE health checks, not needed unless your services are not in 30000-32767 or 3000, 5000	list(string)	[]	no
custom_target_http_proxy_name	Custom name for HTTP proxy name used instead of non-tls-proxy-	string	""	no
custom_url_map_name	Custom name for URL map name used instead of lb-var.name	string	""	no
default_iap_setup	In case you use the same IAP setup for all backends	object({ oauth2_client_id = string oauth2_client_secret = string })	null	no
default_network_name	Default firewall network name, used to place a default fw allowing google's default health checks. Leave blank if you use GKE ingress-provisioned LB (now deprecated)	string	"default"	no
dont_use_dns_names_in_certificate	Due to backward compatibility, TLS setup can omit setup of dns_names in self signed certificate	bool	false	no
google_managed_tls	If true, creates Google-managed TLS cert	bool	false	no
health_check_request_path	Health checked path (URN)	string	"/healthz"	no
healthy_threshold	A so-far unhealthy instance will be marked healthy after this many consecutive successes. The default value is 2.	number	2	no
http_backend_protocol	HTTP backend protocol, one of: HTTP/HTTP2	string	"HTTP"	no
http_backend_timeout	Time of http request timeout (in seconds)	string	"30"	no
iap_setup	Service setup for IAP, overwrites default_iap_setup if used	map(object({ oauth2_client_id = string oauth2_client_secret = string }))	{}	no
keys_alg	Algorithm used for private keys	string	"RSA"	no
keys_valid_period	Validation period of the self signed key	number	29200	no
log_config_sample_rate	The value of the field must be in [0, 1]. This configures the sampling rate of requests to the load balancer where 1.0 means all logged requests are reported and 0.0 means no logged requests are reported. The default value is 1.0.	string	"1.0"	no
logging_sink_bucket_retency	Number of days after which log files are deleted from bucket	number	730	no
managed_certificate_name	Name of Google-managed certificate. Useful when migrating from Ingress-provisioned load balancer	string	null	no
mask_metrics_endpoint	If set, requests /metrics will be sent to default backend	bool	false	no
name	Instance name	string	"default_value"	no
non_tls_global_forwarding_rule_name	Global non tls forwarding rule name, if set, changes name of non-tls forwarding rule	string	""	no
private_key	The write-only private key in PEM format. Note: This property is sensitive and will not be displayed in the plan.	string	null	no
project	Project ID	string	n/a	yes
random_suffix_size	Size of random suffix	number	8	no
region	GCP region where we will look for NEGs	string	n/a	yes
self_signed_tls	If true, creates self-signed TLS cert	bool	false	no
services	List of services: cloudrun, neg, bucket, ... to be used in the map	list(object({ name = string type = string bucket_name = optional(string) location = optional(string) zone = optional(string) additional_negs = optional(list(string)) timeout_sec = optional(number) check_interval_sec = optional(number) healthy_threshold = optional(number) unhealthy_threshold = optional(number) http_backend_protocol = optional(string) http_backend_timeout = optional(string) health_check_request_path = optional(string) enable_cdn = optional(bool) }))	n/a	yes
timeout_sec	How long (in seconds) to wait before claiming failure. The default value is 5 seconds. It is invalid for timeout_sec to have greater value than check_interval_sec.	number	5	no
unhealthy_threshold	A so-far healthy instance will be marked unhealthy after this many consecutive failures. The default value is 2.	number	2	no
url_map	Url map setup	map(object({ hostnames = list(string) default_service = string path_rules = optional(list(object({ paths = list(string) service = string }))) }))	n/a	yes
use_random_suffix_for_network_endpoint_group	If true, uses random suffix for NEG name	bool	true	no
zone	GCP zone where we will look for NEGs - optional parameter, if not set, the we will automatically search in all zones in region	string	null	no

Outputs

Name	Description
ip_address	IP address