aws hubs: consistent setup of cloud permissions and bucket envs

consideRatio · Jul 5, 2024 · 735caa3 · 735caa3
1 parent 4c3a5be
commit 735caa3
Show file tree

Hide file tree

Showing 25 changed files with 96 additions and 10 deletions.
diff --git a/config/clusters/2i2c-aws-us/dask-staging.values.yaml b/config/clusters/2i2c-aws-us/dask-staging.values.yaml
@@ -31,6 +31,8 @@ basehub:
       image:
         name: pangeo/pangeo-notebook
         tag: "latest"
+      extraEnv:
+        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-dask-staging/$(JUPYTERHUB_USER)
     hub:
       config:
         JupyterHub:

diff --git a/config/clusters/2i2c-aws-us/itcoocean.values.yaml b/config/clusters/2i2c-aws-us/itcoocean.values.yaml
@@ -81,8 +81,7 @@ jupyterhub:
             mountPath: /home/jovyan/shared-public
             subPath: _shared-public
     extraEnv:
-      SCRATCH_BUCKET: s3://scratch-itcoocean/$(JUPYTERHUB_USER)
-      PANGEO_SCRATCH: s3://scratch-itcoocean/$(JUPYTERHUB_USER)
+      SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-itcoocean/$(JUPYTERHUB_USER)
     profileList:
       # NOTE: About node sharing
       #

diff --git a/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml b/config/clusters/2i2c-aws-us/ncar-cisl.values.yaml
@@ -52,6 +52,8 @@ basehub:
         # pangeo/pangeo-notebook is maintained at: https://github.com/pangeo-data/pangeo-docker-images
         name: pangeo/pangeo-notebook
         tag: "2023.05.18"
+      extraEnv:
+        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-ncar-cisl/$(JUPYTERHUB_USER)
       profileList:
         # NOTE: About node sharing
         #

diff --git a/config/clusters/2i2c-aws-us/showcase.values.yaml b/config/clusters/2i2c-aws-us/showcase.values.yaml
@@ -50,8 +50,7 @@ basehub:
           enable_auth_state: true
     singleuser:
       extraEnv:
-        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-researchdelight/$(JUPYTERHUB_USER)
-        PANGEO_SCRATCH: s3://2i2c-aws-us-scratch-researchdelight/$(JUPYTERHUB_USER)
+        SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-showcase/$(JUPYTERHUB_USER)
         PERSISTENT_BUCKET: s3://2i2c-aws-us-persistent-showcase/$(JUPYTERHUB_USER)
         GH_SCOPED_CREDS_CLIENT_ID: Iv1.f9261c4c78b4dfdd
         GH_SCOPED_CREDS_APP_URL: https://github.com/apps/2i2c-community-showcase-hub

diff --git a/config/clusters/2i2c-aws-us/staging.values.yaml b/config/clusters/2i2c-aws-us/staging.values.yaml
@@ -32,3 +32,6 @@ jupyterhub:
         authenticator_class: "github"
       GitHubOAuthenticator:
         oauth_callback_url: "https://staging.aws.2i2c.cloud/hub/oauth_callback"
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://2i2c-aws-us-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/gridsst/prod.values.yaml b/config/clusters/gridsst/prod.values.yaml
@@ -12,3 +12,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://gridsst.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://gridsst-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/gridsst/staging.values.yaml b/config/clusters/gridsst/staging.values.yaml
@@ -12,3 +12,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.gridsst.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://gridsst-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/jupyter-health/prod.values.yaml b/config/clusters/jupyter-health/prod.values.yaml
@@ -11,3 +11,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://jupyter-health.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://jupyter-health-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/jupyter-health/staging.values.yaml b/config/clusters/jupyter-health/staging.values.yaml
@@ -11,3 +11,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://staging.jupyter-health.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://jupyter-health-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/jupyter-meets-the-earth/staging.values.yaml b/config/clusters/jupyter-meets-the-earth/staging.values.yaml
@@ -12,9 +12,6 @@ basehub:
       tls:
         - hosts: [staging.jmte.2i2c.cloud]
           secretName: https-auto-tls
-
     singleuser:
       extraEnv:
-        # This bucket is created via terraform.
-        SCRATCH_BUCKET: s3://jupyter-meets-the-earth-staging-scratch/$(JUPYTERHUB_USER)
-        PANGEO_SCRATCH: s3://jupyter-meets-the-earth-staging-scratch/$(JUPYTERHUB_USER)
+        SCRATCH_BUCKET: s3://jupyter-meets-the-earth-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/kitware/prod.values.yaml b/config/clusters/kitware/prod.values.yaml
@@ -12,3 +12,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://kitware.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://kitware-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/kitware/staging.values.yaml b/config/clusters/kitware/staging.values.yaml
@@ -12,3 +12,6 @@ jupyterhub:
     config:
       GitHubOAuthenticator:
         oauth_callback_url: https://staging.kitware.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://kitware-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-ghg/prod.values.yaml b/config/clusters/nasa-ghg/prod.values.yaml
@@ -16,3 +16,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://hub.ghg.center/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-ghg-hub-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-ghg/staging.values.yaml b/config/clusters/nasa-ghg/staging.values.yaml
@@ -16,3 +16,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.ghg.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-ghg-hub-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-veda/prod.values.yaml b/config/clusters/nasa-veda/prod.values.yaml
@@ -16,3 +16,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://hub.openveda.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-veda-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/nasa-veda/staging.values.yaml b/config/clusters/nasa-veda/staging.values.yaml
@@ -4,6 +4,8 @@ basehub:
       eks.amazonaws.com/role-arn: arn:aws:iam::444055461661:role/nasa-veda-staging
   jupyterhub:
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://nasa-veda-scratch-staging/$(JUPYTERHUB_USER)
       initContainers:
         - &volume_ownership_fix_initcontainer
           name: volume-mount-ownership-fix

diff --git a/config/clusters/opensci/staging.values.yaml b/config/clusters/opensci/staging.values.yaml
@@ -1,3 +1,8 @@
+userServiceAccount:
+  enabled: true
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::211125293633:role/opensci-staging
+
 jupyterhub:
   ingress:
     hosts:
@@ -28,6 +33,8 @@ jupyterhub:
           name: ""
           url: ""
   singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://opensci-scratch-staging/$(JUPYTERHUB_USER)
     profileList:
       - display_name: "Only Profile Available, this info is not shown in the UI"
         slug: only-choice

diff --git a/config/clusters/smithsonian/prod.values.yaml b/config/clusters/smithsonian/prod.values.yaml
@@ -1,7 +1,19 @@
 basehub:
+  userServiceAccount:
+    enabled: true
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::969396938818:role/smithsonian-prod
+
   jupyterhub:
     ingress:
       hosts: [smithsonian.2i2c.cloud]
       tls:
         - hosts: [smithsonian.2i2c.cloud]
           secretName: https-auto-tls
+    hub:
+      config:
+        GitHubOAuthenticator:
+          oauth_callback_url: https://smithsonian.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://smithsonian-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/smithsonian/staging.values.yaml b/config/clusters/smithsonian/staging.values.yaml
@@ -1,7 +1,19 @@
 basehub:
+  userServiceAccount:
+    enabled: true
+    annotations:
+      eks.amazonaws.com/role-arn: arn:aws:iam::969396938818:role/smithsonian-staging
+
   jupyterhub:
     ingress:
       hosts: [staging.smithsonian.2i2c.cloud]
       tls:
         - hosts: [staging.smithsonian.2i2c.cloud]
           secretName: https-auto-tls
+    hub:
+      config:
+        GitHubOAuthenticator:
+          oauth_callback_url: https://staging.smithsonian.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://smithsonian-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/ubc-eoas/prod.values.yaml b/config/clusters/ubc-eoas/prod.values.yaml
@@ -1,3 +1,8 @@
+userServiceAccount:
+  enabled: true
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::259060176665:role/ubc-eoas-prod
+
 jupyterhub:
   ingress:
     hosts: [ubc-eoas.2i2c.cloud]
@@ -8,3 +13,6 @@ jupyterhub:
     config:
       CILogonOAuthenticator:
         oauth_callback_url: https://ubc-eoas.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://ubc-eoas-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/ubc-eoas/staging.values.yaml b/config/clusters/ubc-eoas/staging.values.yaml
@@ -1,3 +1,8 @@
+userServiceAccount:
+  enabled: true
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::259060176665:role/ubc-eoas-staging
+
 jupyterhub:
   ingress:
     hosts: [staging.ubc-eoas.2i2c.cloud]
@@ -8,3 +13,6 @@ jupyterhub:
     config:
       CILogonOAuthenticator:
         oauth_callback_url: https://staging.ubc-eoas.2i2c.cloud/hub/oauth_callback
+  singleuser:
+    extraEnv:
+      SCRATCH_BUCKET: s3://ubc-eoas-scratch-staging/$(JUPYTERHUB_USER)
diff --git a/config/clusters/victor/prod.values.yaml b/config/clusters/victor/prod.values.yaml
@@ -13,3 +13,6 @@ basehub:
       config:
         GitHubOAuthenticator:
           oauth_callback_url: https://victor.2i2c.cloud/hub/oauth_callback
+    singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://victor-scratch/$(JUPYTERHUB_USER)
diff --git a/config/clusters/victor/staging.values.yaml b/config/clusters/victor/staging.values.yaml
@@ -14,6 +14,8 @@ basehub:
         GitHubOAuthenticator:
           oauth_callback_url: https://staging.victor.2i2c.cloud/hub/oauth_callback
     singleuser:
+      extraEnv:
+        SCRATCH_BUCKET: s3://victor-scratch-staging/$(JUPYTERHUB_USER)
       profileList:
         # Create a small instance that can launch a custom image
         - display_name: "Bring your own image - Small: m5.large"

diff --git a/terraform/aws/projects/2i2c-aws-us.tfvars b/terraform/aws/projects/2i2c-aws-us.tfvars
@@ -14,7 +14,7 @@ user_buckets = {
   "scratch-dask-staging" : {
     "delete_after" : 7
   },
-  "scratch-researchdelight" : {
+  "scratch-showcase" : {
     "delete_after" : 7
   },
   "persistent-showcase" : {
@@ -46,7 +46,7 @@ hub_cloud_permissions = {
   "showcase" : {
     "user-sa" : {
       bucket_admin_access : [
-        "scratch-researchdelight",
+        "scratch-showcase",
         "persistent-showcase",
       ],
     },

diff --git a/terraform/aws/projects/jupyter-meets-the-earth.tfvars b/terraform/aws/projects/jupyter-meets-the-earth.tfvars
@@ -11,6 +11,9 @@ user_buckets = {
   "scratch-staging" : {
     "delete_after" : 7
   },
+  // IMPORTANT: This bucket isn't used, they are instead using s3://jmte-scratch
+  //            that doesn't have a delete_after policy setup etc, but maybe
+  //            they want to have.
   "scratch" : {
     "delete_after" : 7
   },