diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf index 32904d1184..5cf759d857 100644 --- a/terraform/gcp/buckets.tf +++ b/terraform/gcp/buckets.tf @@ -7,6 +7,9 @@ resource "google_storage_bucket" "user_buckets" { name = "${var.prefix}-${each.key}" location = var.region project = var.project_id + + // Set these values explicitly so they don't "change outside terraform" + labels = {} } resource "google_storage_bucket_iam_member" "member" { diff --git a/terraform/gcp/cluster.tf b/terraform/gcp/cluster.tf index fa8534f3b3..dd3cba4908 100644 --- a/terraform/gcp/cluster.tf +++ b/terraform/gcp/cluster.tf @@ -9,6 +9,32 @@ resource "google_container_cluster" "cluster" { initial_node_count = 1 remove_default_node_pool = true + // For private clusters, pass the name of the network and subnetwork created + // by the VPC + network = var.enable_private_cluster ? data.google_compute_network.default_network.name : null + subnetwork = var.enable_private_cluster ? data.google_compute_subnetwork.default_subnetwork.name : null + + // Dynamically provision the private cluster config when deploying a + // private cluster + dynamic "private_cluster_config" { + for_each = var.enable_private_cluster == "" ? [] : [1] + + content { + // Decide if this CIDR block is sensible or not + master_ipv4_cidr_block = "172.16.0.0/28" + enable_private_nodes = true + enable_private_endpoint = false + } + } + + // Dynamically provision the IP allocation policy when deploying a + // private cluster. This allows for IP aliasing and makes the cluster + // VPC-native + dynamic "ip_allocation_policy" { + for_each = var.enable_private_cluster ? [1] : [] + content {} + } + addons_config { http_load_balancing { // FIXME: This used to not work well with websockets, and @@ -58,6 +84,9 @@ resource "google_container_cluster" "cluster" { # DO NOT TOUCH THIS BLOCK, IT REPLACES ENTIRE CLUSTER LOL service_account = google_service_account.cluster_sa.email } + + // Set these values explicitly so they don't "change outside terraform" + resource_labels = {} } resource "google_container_node_pool" "core" { @@ -97,6 +126,9 @@ resource "google_container_node_pool" "core" { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] + + // Set these values explicitly so they don't "change outside terraform" + tags = [] } } @@ -155,6 +187,9 @@ resource "google_container_node_pool" "notebook" { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] + + // Set these values explicitly so they don't "change outside terraform" + tags = [] } } @@ -218,6 +253,8 @@ resource "google_container_node_pool" "dask_worker" { oauth_scopes = [ "https://www.googleapis.com/auth/cloud-platform" ] + + // Set these values explicitly so they don't "change outside terraform" + tags = [] } } - diff --git a/terraform/gcp/network.tf b/terraform/gcp/network.tf new file mode 100644 index 0000000000..84e561cfd6 --- /dev/null +++ b/terraform/gcp/network.tf @@ -0,0 +1,58 @@ +/** +* Networking to support private clusters +* +* This config is only deployed when the enable_private_cluster variable is set +* to true +*/ + +data "google_compute_network" "default_network" { + name = "default" + project = var.project_id +} + +data "google_compute_subnetwork" "default_subnetwork" { + name = "default" + project = var.project_id + region = var.region +} + +resource "google_compute_firewall" "iap_ssh_ingress" { + count = var.enable_private_cluster ? 1 : 0 + + name = "allow-ssh" + project = var.project_id + network = data.google_compute_network.default_network.name + + allow { + protocol = "tcp" + ports = ["22"] + } + + // This range contains all IP addresses that IAP uses for TCP forwarding. + // https://cloud.google.com/iap/docs/using-tcp-forwarding + source_ranges = ["35.235.240.0/20"] +} + +resource "google_compute_router" "router" { + count = var.enable_private_cluster ? 1 : 0 + + name = "${var.prefix}-router" + project = var.project_id + region = var.region + network = data.google_compute_network.default_network.id +} + +resource "google_compute_router_nat" "nat" { + count = var.enable_private_cluster ? 1 : 0 + + name = "${var.prefix}-router-nat" + project = var.project_id + region = var.region + router = google_compute_router.router[0].name + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" + + // Set these values explicitly so they don't "change outside terraform" + nat_ips = [] + drain_nat_ips = [] +} diff --git a/terraform/gcp/registry.tf b/terraform/gcp/registry.tf index 29c8e3975a..24a8ccab8d 100644 --- a/terraform/gcp/registry.tf +++ b/terraform/gcp/registry.tf @@ -10,4 +10,7 @@ resource "google_artifact_registry_repository" "registry" { repository_id = "${var.prefix}-registry" format = "DOCKER" project = var.project_id + + // Set these values explicitly so they don't "change outside terraform" + labels = {} } diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 2d8704e04b..0de5d42dff 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -171,3 +171,19 @@ variable "user_buckets" { default = [] description = "Buckets to create for the project, they will be prefixed with {var.prefix}-" } + +variable "enable_private_cluster" { + type = bool + default = false + description = <<-EOT + Deploy the kubernetes cluster into a private subnet + + By default, GKE gives each of your nodes a public IP & puts them in a public + subnet. When this variable is set to `true`, the nodes will be in a private subnet + and not have public IPs. A cloud NAT will provide outbound internet access from + these nodes. The kubernetes API will still be exposed publicly, so we can access + it from our laptops & CD. + + This is often required by institutional controls banning VMs from having public IPs. + EOT +}