leap, m2lines: Bucket public access update

docs/howto/features/buckets.md

@@ -28,20 +28,12 @@ on why users want this!
    very helpful for 'scratch' buckets that are temporary. Set to
    `null` to prevent this cleaning up process from happening, e.g., if users want a persistent bucket.
 
-2. Enable access to these buckets from the hub by [editing `hub_cloud_permissions`](howto:features:cloud-access:access-perms)
+2. Enable access to these buckets from the hub or make them publicly accessible from outside
+   by [editing `hub_cloud_permissions`](howto:features:cloud-access:access-perms)
    in the same `.tfvars` file. Follow all the steps listed there - this
    should create the storage buckets and provide all users access to them!
 
-3. (If requested) Enable public read access to these buckets by editing the
-   `bucket_public_access` list in the same `.tfvars`:
-
-   ```terraform
-   bucket_public_access = [
-      "public-persistent"
-   ]
-   ```
-
-4. You can set the `SCRATCH_BUCKET` (and the deprecated `PANGEO_SCRATCH`)
+3. You can set the `SCRATCH_BUCKET` (and the deprecated `PANGEO_SCRATCH`)
    env vars on all user pods so users can use the created bucket without
    having to hard-code the bucket name in their code. In the hub-specific
    `.values.yaml` file in `config/clusters/<cluster-name>`,
@@ -79,7 +71,7 @@ on why users want this!
 
    You can also add other env vars pointing to other buckets users requested.
 
-5. Get this change deployed, and users should now be able to use the buckets!
+4. Get this change deployed, and users should now be able to use the buckets!
    Currently running users might have to restart their pods for the change to take effect.
 
 

docs/howto/features/cloud-access.md

@@ -44,6 +44,7 @@ This AWS IAM Role is managed via terraform.
        "<hub-name-slug>": {
            requestor_pays : true,
            bucket_admin_access : ["bucket-1", "bucket-2"]
+           bucket_public_access : ["bucket-1"]
            hub_namespace : "<hub-name>"
        }
    }
@@ -63,7 +64,9 @@ This AWS IAM Role is managed via terraform.
       access to. Used along with the [user_buckets](howto:features:cloud-access:storage-buckets)
       terraform variable to enable the [scratch buckets](topic:features:cloud:scratch-buckets)
       feature.
-   4. (GCP only) `hub_namespace` is the full name of the hub, as hubs are put in Kubernetes
+   4. `bucket_public_access` lists bucket names (as specified in `user_buckets`
+      terraform variable) that should be publicly accessible.
+   5. (GCP only) `hub_namespace` is the full name of the hub, as hubs are put in Kubernetes
       Namespaces that are the same as their names. This is explicitly specified here
       because `<hub-name-slug>` could possibly be truncated on GCP.
 

terraform/gcp/buckets.tf

@@ -54,6 +54,15 @@ locals {
       }
     ]
   ]))
+
+  bucket_public_permissions = distinct(flatten([
+    for hub_name, permissions in var.hub_cloud_permissions : [
+      for bucket_name in permissions.bucket_public_access : {
+        hub_name    = hub_name
+        bucket_name = bucket_name
+      }
+    ]
+  ]))
 }
 
 resource "google_storage_bucket_iam_member" "member" {
@@ -77,9 +86,9 @@ resource "google_storage_bucket_iam_member" "extra_admin_members" {
   member   = each.value.member
 }
 
-resource "google_storage_default_object_access_control" "public_rule" {
-  for_each = toset(var.bucket_public_access)
-  bucket   = google_storage_bucket.user_buckets[each.key].name
+resource "google_storage_bucket_access_control" "public_rule" {
+  for_each = { for bp in local.bucket_public_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp }
+  bucket   = google_storage_bucket.user_buckets[each.value.bucket_name].name
   role     = "READER"
   entity   = "allUsers"
 }

terraform/gcp/projects/leap.tfvars

@@ -63,14 +63,11 @@ hub_cloud_permissions = {
     requestor_pays : true,
     bucket_admin_access : ["scratch", "persistent"],
     bucket_readonly_access : ["persistent-ro"],
+    bucket_public_access : ["persistent-ro"],
     hub_namespace : "prod"
   }
 }
 
-bucket_public_access = [
-  "persistent-ro"
-]
-
 # Setup notebook node pools
 notebook_nodes = {
   "medium" : {

terraform/gcp/projects/m2lines.tfvars

@@ -102,10 +102,7 @@ hub_cloud_permissions = {
   "prod" : {
     requestor_pays : true,
     bucket_admin_access : ["scratch", "persistent", "public-persistent"],
+    bucket_public_access : ["public-persistent"],
     hub_namespace : "prod"
   },
 }
-
-bucket_public_access = [
-  "public-persistent"
-]

terraform/gcp/variables.tf

@@ -355,6 +355,7 @@ variable "hub_cloud_permissions" {
       requestor_pays : bool,
       bucket_admin_access : set(string),
       bucket_readonly_access : optional(set(string), []),
+      bucket_public_access : optional(set(string), []),
       hub_namespace : string
     })
   )
@@ -373,15 +374,6 @@ variable "hub_cloud_permissions" {
   EOT
 }
 
-variable "bucket_public_access" {
-  type        = list(any)
-  default     = []
-  description = <<-EOT
-  A list of GCS storage buckets defined in user_buckets that should be granted public read access.
-
-  EOT
-}
-
 variable "container_repos" {
   type        = list(any)
   default     = []