Enabling private nodes for GKE clusters

terraform/gcp/buckets.tf

@@ -7,6 +7,9 @@ resource "google_storage_bucket" "user_buckets" {
   name     = "${var.prefix}-${each.key}"
   location = var.region
   project  = var.project_id
+
+  // Set these values explicitly so they don't "change outside terraform"
+  labels = {}
 }
 
 resource "google_storage_bucket_iam_member" "member" {

terraform/gcp/cluster.tf

@@ -9,6 +9,32 @@ resource "google_container_cluster" "cluster" {
   initial_node_count       = 1
   remove_default_node_pool = true
 
+  // For private clusters, pass the name of the network and subnetwork created
+  // by the VPC
+  network    = var.enable_private_cluster ? google_compute_network.vpc_network[0].self_link : null
+  subnetwork = var.enable_private_cluster ? google_compute_subnetwork.subnetwork[0].self_link : null
+
+  // Dynamically provision the private cluster config when deploying a
+  // private cluster
+  dynamic "private_cluster_config" {
+    for_each = var.enable_private_cluster == "" ? [] : [1]
+
+    content {
+      // Decide if this CIDR block is sensible or not
+      master_ipv4_cidr_block  = "172.16.0.0/28"
+      enable_private_nodes    = true
+      enable_private_endpoint = false
+    }
+  }
+
+  // Dynamically provision the IP allocation policy when deploying a
+  // private cluster. This allows for IP aliasing and makes the cluster
+  // VPC-native
+  dynamic "ip_allocation_policy" {
+    for_each = var.enable_private_cluster ? [1] : []
+    content {}
+  }
+
   addons_config {
     http_load_balancing {
       // FIXME: This used to not work well with websockets, and
@@ -58,6 +84,9 @@ resource "google_container_cluster" "cluster" {
     # DO NOT TOUCH THIS BLOCK, IT REPLACES ENTIRE CLUSTER LOL
     service_account = google_service_account.cluster_sa.email
   }
+
+  // Set these values explicitly so they don't "change outside terraform"
+  resource_labels = {}
 }
 
 resource "google_container_node_pool" "core" {
@@ -97,6 +126,9 @@ resource "google_container_node_pool" "core" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+
+    // Set these values explicitly so they don't "change outside terraform"
+    tags = []
   }
 }
 
@@ -155,6 +187,9 @@ resource "google_container_node_pool" "notebook" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+
+    // Set these values explicitly so they don't "change outside terraform"
+    tags = []
   }
 }
 
@@ -218,6 +253,8 @@ resource "google_container_node_pool" "dask_worker" {
     oauth_scopes = [
       "https://www.googleapis.com/auth/cloud-platform"
     ]
+
+    // Set these values explicitly so they don't "change outside terraform"
+    tags = []
   }
 }
-

terraform/gcp/network.tf

@@ -0,0 +1,62 @@
+/**
+* Networking to support private clusters
+*
+* This config is only deployed when the enable_private_cluster variable is set
+* to true
+*/
+
+resource "google_compute_network" "vpc_network" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name                    = "${var.prefix}-vpc-network"
+  project                 = var.project_id
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "subnetwork" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name                     = "${var.prefix}-subnetwork"
+  project                  = var.project_id
+  region                   = var.region
+  network                  = google_compute_network.vpc_network[0].id
+  private_ip_google_access = true
+
+  // Decide if this is a sensible IP CIDR range or not
+  ip_cidr_range = "10.2.0.0/16"
+}
+
+resource "google_compute_firewall" "firewall_rules" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name    = "allow-ssh"
+  project = var.project_id
+  network = google_compute_network.vpc_network[0].name
+
+  allow {
+    protocol = "tcp"
+    ports    = ["22"]
+  }
+
+  source_ranges = ["35.235.240.0/20"]
+}
+
+resource "google_compute_router" "router" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name    = "${var.prefix}-router"
+  project = var.project_id
+  region  = var.region
+  network = google_compute_network.vpc_network[0].id
+}
+
+resource "google_compute_router_nat" "nat" {
+  count = var.enable_private_cluster ? 1 : 0
+
+  name                               = "${var.prefix}-router-nat"
+  project                            = var.project_id
+  region                             = var.region
+  router                             = google_compute_router.router[0].name
+  nat_ip_allocate_option             = "AUTO_ONLY"
+  source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES"
+}

terraform/gcp/registry.tf

@@ -10,4 +10,7 @@ resource "google_artifact_registry_repository" "registry" {
   repository_id = "${var.prefix}-registry"
   format        = "DOCKER"
   project       = var.project_id
+
+  // Set these values explicitly so they don't "change outside terraform"
+  labels = {}
 }

terraform/gcp/variables.tf

@@ -171,3 +171,16 @@ variable "user_buckets" {
   default     = []
   description = "Buckets to create for the project, they will be prefixed with {var.prefix}-"
 }
+
+variable "enable_private_cluster" {
+  type        = bool
+  default     = false
+  description = <<-EOT
+  Enable deployment of GKE into a private cluster.
+
+  For projects that are managed by universities and such, we may find that they
+  have enabled certain constraints and controls that mean our usual method of
+  deployment fails. Enabling a private cluster tends to satisfy many of these
+  controls.
+  EOT
+}