diff --git a/docs/howto/features/buckets.md b/docs/howto/features/buckets.md index dd8efa0617..4293d46541 100644 --- a/docs/howto/features/buckets.md +++ b/docs/howto/features/buckets.md @@ -28,20 +28,12 @@ on why users want this! very helpful for 'scratch' buckets that are temporary. Set to `null` to prevent this cleaning up process from happening, e.g., if users want a persistent bucket. -2. Enable access to these buckets from the hub by [editing `hub_cloud_permissions`](howto:features:cloud-access:access-perms) +2. Enable access to these buckets from the hub or make them publicly accessible from outside + by [editing `hub_cloud_permissions`](howto:features:cloud-access:access-perms) in the same `.tfvars` file. Follow all the steps listed there - this should create the storage buckets and provide all users access to them! -3. (If requested) Enable public read access to these buckets by editing the - `bucket_public_access` list in the same `.tfvars`: - - ```terraform - bucket_public_access = [ - "public-persistent" - ] - ``` - -4. You can set the `SCRATCH_BUCKET` (and the deprecated `PANGEO_SCRATCH`) +3. You can set the `SCRATCH_BUCKET` (and the deprecated `PANGEO_SCRATCH`) env vars on all user pods so users can use the created bucket without having to hard-code the bucket name in their code. In the hub-specific `.values.yaml` file in `config/clusters/`, @@ -79,7 +71,7 @@ on why users want this! You can also add other env vars pointing to other buckets users requested. -5. Get this change deployed, and users should now be able to use the buckets! +4. Get this change deployed, and users should now be able to use the buckets! Currently running users might have to restart their pods for the change to take effect. diff --git a/docs/howto/features/cloud-access.md b/docs/howto/features/cloud-access.md index ac2b5f400b..74fec1847e 100644 --- a/docs/howto/features/cloud-access.md +++ b/docs/howto/features/cloud-access.md @@ -44,6 +44,7 @@ This AWS IAM Role is managed via terraform. "": { requestor_pays : true, bucket_admin_access : ["bucket-1", "bucket-2"] + bucket_public_access : ["bucket-1"] hub_namespace : "" } } @@ -63,7 +64,9 @@ This AWS IAM Role is managed via terraform. access to. Used along with the [user_buckets](howto:features:cloud-access:storage-buckets) terraform variable to enable the [scratch buckets](topic:features:cloud:scratch-buckets) feature. - 4. (GCP only) `hub_namespace` is the full name of the hub, as hubs are put in Kubernetes + 4. `bucket_public_access` lists bucket names (as specified in `user_buckets` + terraform variable) that should be publicly accessible. + 5. (GCP only) `hub_namespace` is the full name of the hub, as hubs are put in Kubernetes Namespaces that are the same as their names. This is explicitly specified here because `` could possibly be truncated on GCP. diff --git a/terraform/gcp/buckets.tf b/terraform/gcp/buckets.tf index 59044a42bc..cd541761a8 100644 --- a/terraform/gcp/buckets.tf +++ b/terraform/gcp/buckets.tf @@ -54,6 +54,15 @@ locals { } ] ])) + + bucket_public_permissions = distinct(flatten([ + for hub_name, permissions in var.hub_cloud_permissions : [ + for bucket_name in permissions.bucket_public_access : { + hub_name = hub_name + bucket_name = bucket_name + } + ] + ])) } resource "google_storage_bucket_iam_member" "member" { @@ -77,9 +86,9 @@ resource "google_storage_bucket_iam_member" "extra_admin_members" { member = each.value.member } -resource "google_storage_default_object_access_control" "public_rule" { - for_each = toset(var.bucket_public_access) - bucket = google_storage_bucket.user_buckets[each.key].name +resource "google_storage_bucket_access_control" "public_rule" { + for_each = { for bp in local.bucket_public_permissions : "${bp.hub_name}.${bp.bucket_name}" => bp } + bucket = google_storage_bucket.user_buckets[each.value.bucket_name].name role = "READER" entity = "allUsers" } diff --git a/terraform/gcp/projects/leap.tfvars b/terraform/gcp/projects/leap.tfvars index dbd1bb621a..28ef2b800a 100644 --- a/terraform/gcp/projects/leap.tfvars +++ b/terraform/gcp/projects/leap.tfvars @@ -63,14 +63,11 @@ hub_cloud_permissions = { requestor_pays : true, bucket_admin_access : ["scratch", "persistent"], bucket_readonly_access : ["persistent-ro"], + bucket_public_access : ["persistent-ro"], hub_namespace : "prod" } } -bucket_public_access = [ - "persistent-ro" -] - # Setup notebook node pools notebook_nodes = { "medium" : { diff --git a/terraform/gcp/projects/m2lines.tfvars b/terraform/gcp/projects/m2lines.tfvars index bf4c67d9ff..edb0d1fed8 100644 --- a/terraform/gcp/projects/m2lines.tfvars +++ b/terraform/gcp/projects/m2lines.tfvars @@ -102,10 +102,7 @@ hub_cloud_permissions = { "prod" : { requestor_pays : true, bucket_admin_access : ["scratch", "persistent", "public-persistent"], + bucket_public_access : ["public-persistent"], hub_namespace : "prod" }, } - -bucket_public_access = [ - "public-persistent" -] diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 80d674d1db..c24c52e090 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -355,6 +355,7 @@ variable "hub_cloud_permissions" { requestor_pays : bool, bucket_admin_access : set(string), bucket_readonly_access : optional(set(string), []), + bucket_public_access : optional(set(string), []), hub_namespace : string }) ) @@ -373,15 +374,6 @@ variable "hub_cloud_permissions" { EOT } -variable "bucket_public_access" { - type = list(any) - default = [] - description = <<-EOT - A list of GCS storage buckets defined in user_buckets that should be granted public read access. - - EOT -} - variable "container_repos" { type = list(any) default = []