Skip to content

Latest commit

 

History

History
420 lines (356 loc) · 23.5 KB

README.md

File metadata and controls

420 lines (356 loc) · 23.5 KB
Raw

Terraform GCP HTTP(S) Load Balancing

Terraform module for provisioning of GCP LB on top of precreated named NEGs, Cloud Run services and GCS buckets passed as parameter to this module.

Usage

HTTPS Load-balancer with self-signed certificate and Cloudflare DNS record creation:

data "cloudflare_zones" "ackee_cz" {
  filter {
    name = "ackee.cz"
  }
}

resource "google_storage_bucket" "test" {
  name                        = "test-randompostfix-98582341"
  location                    = var.region
  storage_class               = "STANDARD"
  uniform_bucket_level_access = true
  website {
    main_page_suffix = "index.html"
  }
}

module "api_unicorn" {
  source          = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name            = "main-${var.project}-${var.namespace}"
  project         = var.project
  region          = var.region
  self_signed_tls = true

  services = [
    {
      type = "neg"
      name = "ackee-api-unicorn"
      zone = var.zone
    },
    {
      type        = "bucket"
      bucket_name = "${google_storage_bucket.test.name}"
    },
    {
      type         = "cloudrun"
      service_name = cloud-run-service
    }
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz", "api-unicorn2.ackee.cz"]
      path_rules = [
        {
          paths = ["/api/v1/*"]
          service = {
            type = "neg"
            name = "ackee-api-unicorn"
            zone = var.zone
          }
        },
      ]
    }
    matcher2 = {
      hostnames  = ["api-unicorn.ackee.cz", "api-unicorn2.ackee.cz"]
      path_rules = [
        {
          paths   = ["/*"]
          service = "${google_storage_bucket.test.name}"
        },
      ]
    }
    matcher3 = {
      hostnames  = ["cloud-run-service.ackee.cz"]
      path_rules = [
        {
          paths   = ["/*"]
          service = "cloud-run-service"
        },
      ]
    }
  }
}

resource "cloudflare_record" "api" {
  zone_id = data.cloudflare_zones.ackee_cz.zones[0].id
  name    = "api-unicorn"
  value   = module.api_unicorn.ip_address
  type    = "A"
  ttl     = 1
  proxied = true
}

If NEG named ackee-api-unicorn exists and CF is set to "SSL:Full" you should have working app now on https://api-unicorn.ackee.cz and https://api-unicorn2.ackee.cz

HTTPS Load-balancer with Google-managed certificate and Cloudflare DNS record creation:

data "cloudflare_zones" "ackee_cz" {
  filter {
    name = "ackee.cz"
  }
}

module "api_unicorn" {
  source             = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name               = "main-${var.project}-${var.namespace}"
  project            = var.project
  region             = var.region
  google_managed_tls = true

  services = [
    {
      type = "neg"
      name = "ackee-api-unicorn"
      zone = var.zone
    },
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz", "api-unicorn2.ackee.cz"]
      path_rules = [
        {
          paths = ["/api/v1/*"]
          service = "ackee-api-unicorn"
        },
      ]
    }
  }
}

resource "cloudflare_record" "api" {
  zone_id = data.cloudflare_zones.ackee_cz.zones[0].id
  name    = "api-unicorn"
  value   = module.api_unicorn.ip_address
  type    = "A"
  ttl     = 1
  proxied = false
}

If NEG named ackee-api-unicorn exists you should have working app now on https://api-unicorn.ackee.cz Beware: If you use more then one hostname with Google-managed certificate, only one certificate, with first hostname in list, will be created.

HTTPS Load-balancer with preexisting NEG, Google-managed certificate and Cloudflare DNS record creation:

data "google_compute_network_endpoint_group" "old_neg" {
  name  = "k8s1-aab5af95-production-ackee-unicorn-80-bafd3c69"
  zone  = "europe-west3-c"
  count = 1
}

data "cloudflare_zones" "ackee_cz" {
  filter {
    name = "ackee.cz"
  }
}

module "api_unicorn" {
  source             = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name               = "main-${var.project}-${var.namespace}"
  project            = var.project
  region             = var.region
  google_managed_tls = true

  services = [
    {
      type                  = "neg"
      name                  = "ackee-api-unicorn"
      zone                  = var.zone
      additional_negs       = [data.google_compute_network_endpoint_group.old_neg]
      http_backend_protocol = "HTTP"
    },
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz""]
      path_rules = [
        {
          paths   = ["/*"]
          service = "ackee-api-unicorn"
        },
      ]
    }
  }
}

resource "cloudflare_record" "api" {
  zone_id = data.cloudflare_zones.ackee_cz.zones[0].id
  name    = "api-unicorn"
  value   = module.api_unicorn.ip_address
  type    = "A"
  ttl     = 1
  proxied = false
}

If we pass data.google_compute_network_endpoint_group resource as value for additional_negs parameter, then our new load-balancer gets created from new named NEG's auto discovered by name in neg_name parameter and from NEG's from additional_negs parameter - this should be used when migrating from old setup, so we balance to both new and old application.

HTTPS Load-balancer with pre-existing certificate (signed by external CA):

module "api_unicorn" {
  source             = "git::ssh://[email protected]/Infra/tf-module/terraform-gcp-lb.git?ref=master"
  name               = "main-${var.project}-${var.namespace}"
  project            = var.project
  region             = var.region

  services = [
    {
      type                  = "neg"
      name                  = "ackee-api-unicorn"
      zone                  = var.zone
      additional_negs       = [data.google_compute_network_endpoint_group.old_neg]
      http_backend_protocol = "HTTP"
    },
  ]

  url_map = {
    matcher1 = {
      hostnames  = ["api-unicorn.ackee.cz""]
      path_rules = [
        {
          paths   = ["/*"]
          service = "ackee-api-unicorn"
        },
      ]
    }
  }

  certificate = file("${path.root}/tls/certificate_chain.crt")
  private_key = file("${path.root}/tls/private.key")
}

It is recommended to use some secure storage (eg. Vault) and pass value from here, rather then saving plaintext private key into git repo

Pitfalls

Error: Error creating SslCertificate: googleapi: Error 409: The resource ... already exists, alreadyExists

This might show once you are adding new hostname to the load balancer and SSL certificate web_lb_cert needs to add the hostname into dns_names. Terraform is trying to update the certificate in-place or creates a certificate with the same name. For that, you might want to do these few steps manually:

Get certificates from the state file:

CERT=`mktemp`
CERT_KEY=`mktemp`

terraform show -json | jq -r --arg MODULE "$MODULE" '.values.root_module.child_modules[] | select (.address=="module.lb_72541") | .resources[] | select(.address=="module.lb_72541.google_compute_ssl_certificate.gcs_certs[0]") | .values.private_key' > $CERT_KEY
terraform show -json | jq -r --arg MODULE "$MODULE" '.values.root_module.child_modules[] | select (.address=="module.lb_72541") | .resources[] | select(.address=="module.lb_72541.google_compute_ssl_certificate.gcs_certs[0]") | .values.certificate' > $CERT

where module.lb_72541 is the name of the module used in your Terraform.

Create new temporary certificate:

gcloud compute ssl-certificates create tmp --certificate=$CERT --private-key=$CERT_KEY
gcloud compute target-https-proxies update "NAME_OF_PROXY" --ssl-certificates "tmp"

The name of the https proxy can be found in the state file:

terraform state show 'module.lb_72541.google_compute_target_https_proxy.self_signed[0]'

Remove the old certificate:

Get the name from the error output. Let's say you have this error:

Error: Error creating SslCertificate: googleapi: Error 409: The resource 'projects/awesome-project/global/sslCertificates/main-awesome-project-development-72541-cert-self-signed' already exists, alreadyExists

Then command will look like this:

gcloud compute ssl-certificates delete main-awesome-project-development-72541-cert-self-signed

Run terraform apply to create a certificate from Terraform and once done delete the temporary certificate:

gcloud compute ssl-certificates delete tmp

Creation of NEG's is not automatic!

BEWARE: Network Endpoint Groups REFERENCED BY THIS MODULE MUST EXIST BEFORE YOU USE THIS MODULE, OTHERWISE IT WILL FAIL WITH ERROR SIMILIAR TO:

Error: Required attribute is not set

  on ../load-balancer.tf line 68, in resource "google_compute_backend_service" "cn_lb":
  68: resource "google_compute_backend_service" "cn_lb" {

Also note, that purging app (typically with helm delete) does not automatically cleanup existing NEGs

Using example

Because of chicken-egg mentioned in previous section, it is also not so easy to use provider example code. Running just terraform apply will fail on similiar error. Workaround is to create testing NEG first and load balancer above it in next step.

terraform apply -target=google_compute_network_endpoint_group.neg_one -target=google_compute_network_endpoint_group.neg_two
terraform apply

Before you do anything in this module

Install pre-commit hooks by running following commands:

brew install pre-commit
pre-commit install

Requirements

Name Version
terraform >= 1.0

Providers

Name Version
google 5.8.0
google-beta 5.8.0
random 3.6.0
tls 4.0.5

Modules

No modules.

Resources

Name Type
google-beta_google_compute_backend_service.app_backend resource
google-beta_google_compute_backend_service.cloudrun resource
google-beta_google_compute_global_forwarding_rule.external_signed resource
google-beta_google_compute_global_forwarding_rule.google_managed resource
google-beta_google_compute_global_forwarding_rule.self_signed resource
google_compute_backend_bucket.bucket resource
google_compute_backend_bucket.cn_lb resource
google_compute_firewall.gcp_hc_ip_allow resource
google_compute_global_address.gca resource
google_compute_global_forwarding_rule.non_tls resource
google_compute_health_check.cn_lb resource
google_compute_managed_ssl_certificate.gcs_certs resource
google_compute_region_network_endpoint_group.cloudrun_neg resource
google_compute_ssl_certificate.external_certs resource
google_compute_ssl_certificate.gcs_certs resource
google_compute_target_http_proxy.non_tls resource
google_compute_target_https_proxy.external_signed resource
google_compute_target_https_proxy.google_managed resource
google_compute_target_https_proxy.self_signed resource
google_compute_url_map.cn_lb resource
google_logging_project_sink.log_archive_sink resource
google_storage_bucket.cn_lb resource
google_storage_bucket.log_archive_sink resource
google_storage_bucket_iam_binding.log_archive_sink_writer resource
random_id.external_certificate resource
random_string.random_suffix resource
tls_private_key.web_lb_key resource
tls_self_signed_cert.web_lb_cert resource
google_cloud_run_service.cloud_run_service data source
google_compute_network_endpoint_group.cn_lb data source
google_compute_zones.available data source

Inputs

Name Description Type Default Required
allow_non_tls_frontend If true, enables port 80 frontend - creates non-TLS (http://) variant of LB string false no
backend_bucket_location GCS location(https://cloud.google.com/storage/docs/locations) of bucket where invalid requests are routed. string "EUROPE-WEST3" no
certificate The certificate in PEM format. The certificate chain must be no greater than 5 certs long. The chain must include at least one intermediate cert. Note: This property is sensitive and will not be displayed in the plan. string null no
check_interval_sec How often (in seconds) to send a health check. The default value is 5 seconds. number 5 no
create_logging_sink_bucket If true, creates bucket and set up logging sink bool false no
custom_health_check_ports Custom ports for GCE health checks, not needed unless your services are not in 30000-32767 or 3000, 5000 list(string) [] no
custom_target_http_proxy_name Custom name for HTTP proxy name used instead of non-tls-proxy- string "" no
custom_url_map_name Custom name for URL map name used instead of lb-var.name string "" no
default_iap_setup In case you use the same IAP setup for all backends 
object({
    oauth2_client_id     = string
    oauth2_client_secret = string
  })
 null no
default_network_name Default firewall network name, used to place a default fw allowing google's default health checks. Leave blank if you use GKE ingress-provisioned LB (now deprecated) string "default" no
dont_use_dns_names_in_certificate Due to backward compatibility, TLS setup can omit setup of dns_names in self signed certificate bool false no
google_managed_tls If true, creates Google-managed TLS cert bool false no
health_check_request_path Health checked path (URN) string "/healthz" no
healthy_threshold A so-far unhealthy instance will be marked healthy after this many consecutive successes. The default value is 2. number 2 no
http_backend_protocol HTTP backend protocol, one of: HTTP/HTTP2 string "HTTP" no
http_backend_timeout Time of http request timeout (in seconds) string "30" no
iap_setup Service setup for IAP, overwrites default_iap_setup if used 
map(object({
    oauth2_client_id     = string
    oauth2_client_secret = string
  }))
 {} no
keys_alg Algorithm used for private keys string "RSA" no
keys_valid_period Validation period of the self signed key number 29200 no
log_config_sample_rate The value of the field must be in [0, 1]. This configures the sampling rate of requests to the load balancer where 1.0 means all logged requests are reported and 0.0 means no logged requests are reported. The default value is 1.0. string "1.0" no
logging_sink_bucket_retency Number of days after which log files are deleted from bucket number 730 no
managed_certificate_name Name of Google-managed certificate. Useful when migrating from Ingress-provisioned load balancer string null no
mask_metrics_endpoint If set, requests /metrics will be sent to default backend bool false no
name Instance name string "default_value" no
non_tls_global_forwarding_rule_name Global non tls forwarding rule name, if set, changes name of non-tls forwarding rule string "" no
private_key The write-only private key in PEM format. Note: This property is sensitive and will not be displayed in the plan. string null no
project Project ID string n/a yes
random_suffix_size Size of random suffix number 8 no
region GCP region where we will look for NEGs string n/a yes
self_signed_tls If true, creates self-signed TLS cert bool false no
services List of services: cloudrun, neg, bucket, ... to be used in the map 
list(object({
    name                      = string
    type                      = string
    bucket_name               = optional(string)
    location                  = optional(string)
    zone                      = optional(string)
    additional_negs           = optional(list(string))
    timeout_sec               = optional(number)
    check_interval_sec        = optional(number)
    healthy_threshold         = optional(number)
    unhealthy_threshold       = optional(number)
    http_backend_protocol     = optional(string)
    http_backend_timeout      = optional(string)
    health_check_request_path = optional(string)
    enable_cdn                = optional(bool)
  }))
 n/a yes
timeout_sec How long (in seconds) to wait before claiming failure. The default value is 5 seconds. It is invalid for timeout_sec to have greater value than check_interval_sec. number 5 no
unhealthy_threshold A so-far healthy instance will be marked unhealthy after this many consecutive failures. The default value is 2. number 2 no
url_map Url map setup 
map(object({
    hostnames       = list(string)
    default_service = string
    path_rules = optional(list(object({
      paths   = list(string)
      service = string
    })))
  }))
 n/a yes
use_random_suffix_for_network_endpoint_group If true, uses random suffix for NEG name bool true no
zone GCP zone where we will look for NEGs - optional parameter, if not set, the we will automatically search in all zones in region string null no

Outputs

Name Description
ip_address IP address