Add ability to give users / groups write access to buckets + multi-zone GPUs

terraform/gcp/buckets.tf

@@ -28,23 +28,55 @@ resource "google_storage_bucket" "user_buckets" {
 
 locals {
   # Nested for loop, thanks to https://www.daveperrett.com/articles/2021/08/19/nested-for-each-with-terraform/
-  bucket_permissions = distinct(flatten([
+  bucket_admin_permissions = distinct(flatten([
     for hub_name, permissions in var.hub_cloud_permissions : [
       for bucket_name in permissions.bucket_admin_access : {
         hub_name    = hub_name
         bucket_name = bucket_name
       }
     ]
   ]))
+
+  bucket_readonly_permissions = distinct(flatten([
+    for hub_name, permissions in var.hub_cloud_permissions : [
+      for bucket_name in permissions.bucket_readonly_access : {
+        hub_name    = hub_name
+        bucket_name = bucket_name
+      }
+    ]
+  ]))
+
+  bucket_extra_admin_members = distinct(flatten([
+    for bucket_name, properties in var.user_buckets : [
+      for extra_member in properties.extra_admin_members : {
+        bucket_name = bucket_name
+        member = extra_member
+      }
+    ]
+  ]))
 }
 
 resource "google_storage_bucket_iam_member" "member" {
-  for_each = { for bp in local.bucket_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp }
+  for_each = { for bp in local.bucket_admin_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp }
   bucket   = google_storage_bucket.user_buckets[each.value.bucket_name].name
   role     = "roles/storage.admin"
   member   = "serviceAccount:${google_service_account.workload_sa[each.value.hub_name].email}"
 }
 
+resource "google_storage_bucket_iam_member" "member_readonly" {
+  for_each = { for bp in local.bucket_readonly_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp }
+  bucket   = google_storage_bucket.user_buckets[each.value.bucket_name].name
+  role     = "roles/storage.objectViewer"
+  member   = "serviceAccount:${google_service_account.workload_sa[each.value.hub_name].email}"
+}
+
+resource "google_storage_bucket_iam_member" "extra_admin_members" {
+  for_each = { for bm in local.bucket_extra_admin_members : "${bm.bucket_name}.${bm.member}" => bm }
+  bucket   = google_storage_bucket.user_buckets[each.value.bucket_name].name
+  role     = "roles/storage.admin"
+  member   = each.value.member
+}
+
 resource "google_storage_default_object_access_control" "public_rule" {
   for_each = toset(var.bucket_public_access)
   bucket   = google_storage_bucket.user_buckets[each.key].name

terraform/gcp/cluster.tf

@@ -236,13 +236,15 @@ resource "google_container_node_pool" "core" {
 
 # resource ref: https://registry.terraform.io/providers/hashicorp/google-beta/latest/docs/resources/container_node_pool
 resource "google_container_node_pool" "notebook" {
+  for_each = var.notebook_nodes
+
   name     = "nb-${each.key}"
   cluster  = google_container_cluster.cluster.name
   project  = google_container_cluster.cluster.project
-  location = google_container_cluster.cluster.location
   version  = var.k8s_versions.notebook_nodes_version
 
-  for_each = var.notebook_nodes
+  node_locations = each.value.zones == null ? google_container_cluster.cluster.node_locations : each.value.zones
+
 
   initial_node_count = each.value.min
   autoscaling {
@@ -335,9 +337,9 @@ resource "google_container_node_pool" "dask_worker" {
   name     = "dask-${each.key}"
   cluster  = google_container_cluster.cluster.name
   project  = google_container_cluster.cluster.project
-  location = google_container_cluster.cluster.location
   version  = var.k8s_versions.dask_nodes_version
 
+  node_locations = each.value.zones == null ? google_container_cluster.cluster.node_locations : each.value.zones
   # Default to same config as notebook nodepools config
   for_each = var.dask_nodes
 

terraform/gcp/projects/leap.tfvars

@@ -25,29 +25,44 @@ filestore_capacity_gb = 2048
 
 user_buckets = {
   "scratch-staging" : {
-    "delete_after" : 7
+    "delete_after" : 7,
+    "extra_admin_members": []
   },
   "scratch" : {
-    "delete_after" : 7
+    "delete_after" : 7,
+    "extra_admin_members": []
   }
   # For https://github.com/2i2c-org/infrastructure/issues/1230#issuecomment-1278183441
   "persistent" : {
-    "delete_after" : null
+    "delete_after" : null,
+    "extra_admin_members": ["group:[email protected]"]
   },
   "persistent-staging" : {
-    "delete_after" : null
+    "delete_after" : null,
+    "extra_admin_members": ["group:[email protected]"]
+  }
+  # For https://github.com/2i2c-org/infrastructure/issues/1230#issuecomment-1278183441
+  "persistent-ro" : {
+    "delete_after" : null,
+    "extra_admin_members": ["group:[email protected]"]
+  },
+  "persistent-ro-staging" : {
+    "delete_after" : null,
+    "extra_admin_members": ["group:[email protected]"]
   }
 }
 
 hub_cloud_permissions = {
   "staging" : {
     requestor_pays : true,
     bucket_admin_access : ["scratch-staging", "persistent-staging"],
+    bucket_readonly_access: ["persistent-ro-staging"],
     hub_namespace : "staging"
   },
   "prod" : {
     requestor_pays : true,
     bucket_admin_access : ["scratch", "persistent"],
+    bucket_readonly_access: ["persistent-ro"],
     hub_namespace : "prod"
   }
 }
@@ -76,7 +91,15 @@ notebook_nodes = {
       enabled : true,
       type : "nvidia-tesla-t4",
       count : 1
-    }
+    },
+    zones: [
+      # Get GPUs wherever they are available, as sometimes a single
+      # zone might be out of GPUs.
+      "us-central1-a",
+      "us-central1-b",
+      "us-central1-c",
+      "us-central1-f"
+    ]
   },
 }
 

terraform/gcp/projects/m2lines.tfvars

@@ -22,13 +22,16 @@ user_buckets = {
   },
   # For https://2i2c.freshdesk.com/a/tickets/218
   "persistent": {
-    "delete_after": null
+    "delete_after": null,
+    "extra_admin_members": ["group:[email protected]"]
   },
   "persistent-staging": {
-    "delete_after": null
+    "delete_after": null,
+    "extra_admin_members": ["group:[email protected]"]
   },
   "public-persistent": {
-    "delete_after": null
+    "delete_after": null,
+    "extra_admin_members": ["group:[email protected]"]
   },
 
 }

terraform/gcp/variables.tf

@@ -55,7 +55,20 @@ variable "k8s_versions" {
 }
 
 variable "notebook_nodes" {
-  type        = map(object({ min : number, max : number, machine_type : string, labels : map(string), gpu : object({ enabled : bool, type : string, count : number }) }))
+  type = map(object({
+    min : number,
+    max : number,
+    machine_type : string,
+    labels : map(string),
+    taints : optional(list(object({
+      key : string,
+      value : string,
+      effect : string
+    })), [])
+    gpu : object({ enabled : bool, type : string, count : number }),
+    resource_labels : optional(map(string), {}),
+    zones : optional(list(string), null)
+  }))
   description = "Notebook node pools to create"
   default     = {}
 }
@@ -67,7 +80,14 @@ variable "dask_nodes" {
     preemptible: optional(bool, true),
     machine_type : string,
     labels : map(string),
-    gpu : object({ enabled : bool, type : string, count : number })
+    taints : optional(list(object({
+      key : string,
+      value : string,
+      effect : string
+    })), [])
+    gpu : object({ enabled : bool, type : string, count : number }),
+    resource_labels : optional(map(string), {}),
+    zones : optional(list(string), null)
   }))
   description = "Dask node pools to create. Defaults to notebook_nodes"
   default     = {}
@@ -198,17 +218,24 @@ variable "enable_network_policy" {
 }
 
 variable "user_buckets" {
-  type        = map(object({ delete_after : number }))
+  type        = map(object({ delete_after : number, extra_admin_members: optional(list(string), []) }))
   default     = {}
   description = <<-EOT
   GCS Buckets to be created.
 
   The key for each entry will be prefixed with {var.prefix}- to form
   the name of the bucket.
 
-  The value is a map, with 'delete_after' the only accepted key in that
-  map - it lists the number of days after which any content in the
-  bucket will be deleted. Set to null to not delete data.
+  The value is a map, accepting the following keys:
+
+  'delete_after' specifies the number of days after which any content
+  in the bucket will be deleted. Set to null to not delete data.
+
+  'extra_admin_members' describes extra identies (user groups, user accounts,
+  service accounts, etc) that will have *full* access to this bucket. This
+  is primarily useful for moving data into and out of buckets from outside
+  the cloud. See https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_iam#member/members
+  for the format this would be specified in.
   EOT
 }
 
@@ -316,7 +343,14 @@ variable "max_cpu" {
 }
 
 variable "hub_cloud_permissions" {
-  type        = map(object({ requestor_pays : bool, bucket_admin_access : set(string), hub_namespace : string }))
+  type        = map(
+    object({
+      requestor_pays : bool,
+      bucket_admin_access : set(string),
+      bucket_readonly_access: optional(set(string), []),
+      hub_namespace : string
+    })
+  )
   default     = {}
   description = <<-EOT
   Map of cloud permissions given to a particular hub