From a34aa776b566b38bb2d8cd95dc8708e4976109c9 Mon Sep 17 00:00:00 2001
From: yuvipanda <yuvipanda@gmail.com>
Date: Sun, 12 May 2019 12:01:53 -0700
Subject: [PATCH 1/2] w261: Create GCP resources for users

When a user pod starts, we ensure the following GCP
resources:

1. A GCS bucket
2. A Service Account with admin rights on the bucket
3. A Service Account Key for the Service Account

We then populate two environment variables in the
spawned pod

1. SPARK_GCS_BUCKET - name of GCS bucket for this user
2. SPARK_GCS_KEY - JSON Private key for service account

This is all done through a custom Spawner that inherits
from KubeSpawner. This should operate at the level of
a 'Spark+GCS on GCP', so nothing too UCB specific.

In addition, there's a custom role created in the
ucb-datahub-2018 project - projects/ucb-datahub-2018/roles/w261_hub_resource_creator
that has the appropriate permissions needed by the *hub*
to create the resources it needs.
---
 deployments/w261/config/common.yaml           |  11 ++
 deployments/w261/secrets/staging.yaml         | Bin 524 -> 3061 bytes
 images/hub/Dockerfile                         |  17 +++
 images/hub/sparklyspawner/setup.py            |  13 ++
 .../sparklyspawner/sparklyspawner/__init__.py | 124 ++++++++++++++++++
 5 files changed, 165 insertions(+)
 create mode 100644 images/hub/sparklyspawner/setup.py
 create mode 100644 images/hub/sparklyspawner/sparklyspawner/__init__.py

diff --git a/deployments/w261/config/common.yaml b/deployments/w261/config/common.yaml
index af5bcd694..fcd8037a4 100644
--- a/deployments/w261/config/common.yaml
+++ b/deployments/w261/config/common.yaml
@@ -1,4 +1,15 @@
 jupyterhub:
+  hub:
+    extraConfig:
+      90-sparklyspawner: |
+        import z2jh
+        c.JupyterHub.spawner_class = 'sparklyspawner.SparklySpawner'
+
+        sparkly_config = z2jh.get_config('custom.sparkly', {})
+        c.SparklySpawner.gcp_service_key = sparkly_config.get('gcp_service_key', None)
+        c.SparklySpawner.storage_bucket_template = 'ucb-datahub-2018-w261-test-user-{username}'
+        c.SparklySpawner.service_account_template = 'w261-s-{username}'
+        c.SparklySpawner.gcp_project = 'ucb-datahub-2018'
   auth:
     type: google
     admin:
diff --git a/deployments/w261/secrets/staging.yaml b/deployments/w261/secrets/staging.yaml
index 1d3870a2900cc57b28b99ea18f5f8dba1ffdcc96..3718fb6d670068b4ecff437ff233623fedd01644 100644
GIT binary patch
literal 3061
zcmV<R3kviAM@dveQdv+`0HZ$Y&Dnp*z6$uP@k}!%23UA5ay$wWy!H^0Ymjbs>_+|@
z1?|jqn|`-=j+s@3@`dQoTyc&I;R`mTmr5K40FT92xUz}UjIjijeRJzX1%SYlv}ill
zOH>+Vp_}KTS%_Twh>yO~L$mGECfpOv(#ld(UHZ#1$+xRVuh*6uostLX%i8(ft>V<a
zAAFsmSi$VQ4$p%#$rK`-*ht$^LFb*LsFKyjGc(Q&au2+}1Ah3IgysMk={u_f{mQWp
z6bR*yi!>gd7~KKIqEVqC3S0rbarD$vw1L>3bteqMu<z{tuNww3Hew#&jNSU2JO5OH
z=5Ng?WqS%*m3|v0>50eqSH3m2DGmR&E^16CnvUhW%HVC`yjydyWEt&4qQY#5#vy&r
zSgr!3F<%`v!6O=Lx*CmzosdEn4}6#*RcH(F6_#a6hR}Ldwv7OW^H5!X_bNm>Pscu@
z!x(8&66Kka7y-M9qS+r&kVq!-jU$Kr>#(o*Bl=!ySkWP$OWa~g(0T}#xI4lJ(!#Ry
zC+F!x%v(FjEOp-;sO>+$X^`OBUZa*qr#iQ~6v08ps`A*KeVsBGqt13jv&;J!73wm1
zRbZuE=49sFa~x}b!Ngxr(&v#`PM1fC&twz$S9Y2fbCpw#GU!v*&tObhK(B@f8_D3~
zN`oTrQ9WQWA>zR;T%%OHa943M>zK|IF(g164IeQk!grF>kaDzH&9I`_ywXfMvz{+@
zAoK8~?f;^jFK%wUGJYbZ!P@U7BQ{ugZ#zh-;tuzTL|y+Be=lM_w?GBx{L2->qx)u~
zv}5TtwckjgeSPiAdW(L6D^2V}1`9@QOWd@E@Kj)-Qdq>I`;e+6#>3u{PttYBdfA8A
zXnY+2Y}Bb@9lCE^mSkei=-bCftu3;2X<&~$X6&9b4O}3G&roFR1fQ`hVu9-{dhrtJ
z@(g|F+Qy^a%0yK_{nfqr|JQl3c9UtvnwHpOzpZK*OflXpIQ7?##aA-mSe4=45ep@K
zmWfTJXu@-M@FB^8EXU2p7PMsTK2PZ5_N%`r>%+^8eHvsDjh&`)&{o97=CJh*M1ku}
zfnTewx+Fr~$rW8@STmp~JTROJ(VBh^fg1>vkW=Bu!$Ip@Hs}dPaA(KKSC`Da)^#|=
zFRr!olap@l7-<K6)acz96`lQc(U6<p237_cP(KaGm@55i%<pMar@&!j0E}i$Jc1(@
zvRwz~R{k7fI2@I7IxCl)x(%c|ufU|B&WPQ|CmA#MlFgiZ$)j?zc|Pg@)tRXtAIK)S
zJ}QI7&*kX*AKJhRL^ogef7@*Qs};7%;!^1=`4@JII>30>P!g7W=1Mhm7}17B6xIwO
z;nTaejsj3#vOtvz4nexSICIDO(p~0i92r0xz!l62U^$B&r`IN=$}bI@?wQ)C-FS)G
z@#GC!9KQ2w){Oh#==R@ihdwq6N=Sv}5hm8GX4McmarMO@aS1LyALc6|YZt!UO3f+=
z<EhMIXD5(R{|^3Ch;3EIYTJ*{Xd<uq38>0K-jKa|#ND7O<yZ}sV9Cae;DuW~W_zVA
zAs}rthAe)usPxrPAownrlMcePfE~|wt26`k&seF^W^br8`38nL|7|Qq--B>Ii^;4)
zE6%;<Z_#pY9+8kAk8JW1tc*K~$p|R$$|vTSTp4SMT)V+{A=_AcFfQq|XQmDG$MG(;
zl{!nWM?C0wMxpIaS#ilVb(7DcFYnUn%Iom4%HMz=Q|U4F>AnMy3c`JEl`fRMh>w0K
z*w<rix%9G&m)(h9kAL)XG`sSfQE<#<@OZXLNlY=lwB3Y4AHLU=a9|DFwY<JWREMW%
z9)n<kUcz2!aRSBXgyv?5oIglnTq8<c+{v<K0_y&M{VEB6!bomAw^ijThe02{Q#qwj
zjMOov0jO83F-UyN4AAdII~P`t%$AqQKQCRIjE!?)L&r&xHByyhjCj&$1+Y0S4}L+z
zJS@1MSd|!eq~AEXHw(EK^RaSNn5wBtZx0{c0%3H|l-Dn|#Ced<O->9?O#JJOJIBWo
z#a0W?NeRXbKe#{`DE!D8=InL(2~P7`#&k;TC$08(A1B-<7f(iz%O~8nx0huo7wuR(
zR7b(<rZ}MgcmM!9yI@R(`#}GN?9(Q3%I*}|WCMZ6S*EPFr<aG^#u8c5{`eeYR#U8^
zc+}}X{4;kA-3-1opX!uPwt&QOW!*N;g*b{bK-}PbNArMe4ZQRAr}$cWk7bnL51-w%
zOQVNuE2KjA*HO|TAxJU2kyFSO=+S*b53e5l73sv<tM?EIs!iEPJ&i|T<n&;oZ@P|z
zDjwP>kIM-Kdt;<NV9_Y%h`U1`XGn#>WjR1`c#3DvQpV^cU>$D8t3UfhKR)dOJ#$&}
zy@`!Pbhg-93ys@2m7O&a_Gvhobht}F4Ef}N5*@NJ1gD?CksOtGR-^=={+yB_nNz<X
zT#eRvnh%*%@<37^i+|ZA>t64PYHseLFu$A&K-IlX0sO$QR_*Zb-!A{;7JRJ|SLJ{1
zH$A!$@q0#S+6uL{T6ruu{cX3dK@f?|4HQ&y$%U(o<NR+h6X(=RYvLgn+jtd95&0O}
z466f70#_OgB=E{sfKrh_0h^A`bQ;y}OYw$S|9BNPQ~^gL!f`+^KnHOAMnEcv>_7ng
zDon-j?V30!EOp6*6TMqA1T<urgs}7a%FMBC(f2A+RP3r-S+I7Aq_Cc)WWVxwsW5&I
z9!s0hum4##78j)u<gZ$lksbIPAIE8w8OO6q%es%T9sja<A@~Cs_ZT1S(rdKceFHBd
zttd-_MkMO)({KMG4QJ{s^(5A%U<XN_(S5^Wgt&-I_j?m?hn+tYZl<v@^VYWbsO<q;
zXZ(o|4>{jwJx>wmYWe(#NbQK?x_HB-<7iYl5kiL`O`28EU{Qr42XwiI*?bQjs?@1R
z?{}a-kvSw7sNn@RYXODU%SxSo)7sHbgRTB2i-A;4O`@~$E)r;A{(MV*B^}~HSO{u;
z1{@Os+tN1u*!objJO4RaNNHtZg#B86bi|$@{t?+PTG27&w~`>))PP;>)5%C3JvMVp
z?e8nBLMh#kcd)YiNV+7)hupH<Sf_R7A1se5H3{3x-R!g8Ftcb++Sij*UeXneRV|Rn
zB065v|BpD<loJHo`I-fqeETj^W@lQ=Y1isIDm>zGuP{(Q{n}**V9wX7x`%WLcXG%4
z22FPdo#*$5D^IJDlzkZUZoMy?uM9l4N9GEDr0TI_fnwoLO^y4)Q*D=2kK&XlBAyOC
zw1xF2icrCYL16v!>%ruhsIaS#RDHmd1eA!aailzq3XFiWHZhG{f@-j^`?(rSXSWaa
zo$2mhm~3-?k)Ru30E;dV86))SBLEE^QQ=OopB1C@VJg7Zr#n6^$pbQ?qFDh1i*uCp
zN7xuO-^*;4Ign7Mls^fDy80>)9elhH%vLNFcq`VQOrmaZ8N6<4e?E!GDE0f?3fOr&
zNEDO3WrYJc^2g4fQZ>v;#SL^)z|0k+Ck%}Vq;I-n0pU&xAc>|?a?xjqN$a6e^@C?a
z$g7~j^e16)=UtOl?b^&ZnF{w>$x^ajQ=e~um{PTc>iJU=*5;84%e03=-+*uqKs5v@
z6_>XoqJp$_)_A=_dVMJ#aJDsY{z7KX8h1;cS~~zStYrvasiR4ds^jp}CKUYOofMe$
zfx0ecysF_dvv&lYL-mp$sE-r_f~$fJtb`oSwHEe7J`#w3tC;{VAmx+li45{Rc%Sta
zt&=a7>p-xD#zGLyVxTCX;mkz;OdqVFPxu3(qx_(=>Ra34E$5J7b*b6LmpGr`u^(UH
zecVVZ<;&?f5d%x-O63cCB@kPHEWgZaClR_YbphuIR(L}sG$W`RFecM646v6Y=56lA
zb=8Mgy!$#gPst8MGQ80Fl>IMWew;W8_cNH$>b#vLNAgdmxosa>nHbC+OMqXwSwJCd
zkYUpPc#z?{tJVBfjeBzbJ(j*n^xwP<ubc<>NDPe;nw|SEN7;r=(t2brNLf6ZLy3$N
DgU<Y5

literal 524
zcmV+n0`vU<M@dveQdv+`03PCO748!V<KMQx&L}aCA!O*FPFK+HpQ)KO<XNfV>`dRf
zI0e>S$*SB7h>S`1*tq5%qk?X6)m4-wP{rb=HZe;L+3zNq*t!n^=~L|O2T-;PG~Pe2
z26P3#X9iSEHRVy^S@*tGMrhFsI(P^%(!J;d3mk#eVXl2IF_dM5SzvbZv8oTTWA0+G
zL92;B-WZSwvV~Rrz(;YyfLqdRtGOXuJNp!5U!(ZA^?=e3ZxH1~jMGkIe<Q#)7<!&4
z5vS%e+%pgP?)vN}mBi*gDiYseybR7C|Fi^1Y&2Zm>h>|$SmVg7cZ%eC!a!pO&&{&o
zICb{b)IMM{Hq0_hq))${Vp;tT@<$Ts$+3^j;}pPn_(Pm<2O%s70Cz&q|A5YTRgRjr
zRaOD1Z|$0miU>_7_-0hqkI(SozAJN>oLxf~MQ{oi_r@}0HLsEBSMA$*5b5pDhC;8|
zTeGxp)2dpex6+j&V?g;_-#8yhiRUJJJ2o~*-gJI-A{Cuh<>h95apQS(;cxw^E*6f5
zaBidx7yvSmwiG)b6J7xCE7-d`-+XRXUU}$wIx|2Pa|Al8hLSm;0iHz9gnFodQ9E89
zF|RnOnj9ffEzxPxe;#5?@<G;;y(7dJD?8Rps0t$Inr_Ne-JbyhZ@u}4vk)oUrxg?I
OL;cb0^QT#AfB){cX8{iY

diff --git a/images/hub/Dockerfile b/images/hub/Dockerfile
index 9d8cf8d61..3340bce66 100644
--- a/images/hub/Dockerfile
+++ b/images/hub/Dockerfile
@@ -1,3 +1,20 @@
 # Should match the hub image used by version of chart in hub/requirements.yaml
 # If that changes, this should be changed too!
 FROM jupyterhub/k8s-hub:7b33351
+
+USER root
+RUN apt update && apt install --yes curl python
+
+RUN curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-245.0.0-linux-x86_64.tar.gz && \
+    tar xzf google-cloud-sdk-245.0.0-linux-x86_64.tar.gz && \
+    mv google-cloud-sdk /usr/local/google-cloud-sdk && \
+    rm google-cloud-sdk-245.0.0-linux-x86_64.tar.gz 
+
+ENV PATH /usr/local/google-cloud-sdk/bin:${PATH}
+RUN gcloud components install kubectl
+
+COPY sparklyspawner /srv/sparklyspawner
+
+RUN python3 -m pip install --no-cache /srv/sparklyspawner
+
+USER ${NB_USER}
\ No newline at end of file
diff --git a/images/hub/sparklyspawner/setup.py b/images/hub/sparklyspawner/setup.py
new file mode 100644
index 000000000..85f0724e5
--- /dev/null
+++ b/images/hub/sparklyspawner/setup.py
@@ -0,0 +1,13 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='jupyterhub-sparklykubespawner',
+    version='0.1dev',
+    python_requires='>=3.5',
+    packages=find_packages(),
+    install_requires=[
+        'google-cloud-storage',
+        'google-cloud-iam',
+        'google-api-python-client',
+    ]
+)
diff --git a/images/hub/sparklyspawner/sparklyspawner/__init__.py b/images/hub/sparklyspawner/sparklyspawner/__init__.py
new file mode 100644
index 000000000..14471800c
--- /dev/null
+++ b/images/hub/sparklyspawner/sparklyspawner/__init__.py
@@ -0,0 +1,124 @@
+from kubespawner import KubeSpawner
+from traitlets import Unicode
+from google.cloud import storage
+from google.api_core.exceptions import Conflict
+from google.api_core.iam import Policy
+from googleapiclient.errors import HttpError
+import tempfile
+from google.oauth2 import service_account
+import googleapiclient.discovery
+import json
+import base64
+
+class SparklySpawner(KubeSpawner):
+    gcp_service_key = Unicode(
+        None,
+        allow_none=True,
+        config=True,
+        help="""
+        Google Service Account JSON key for authenticating to GCP.
+
+        *Must* be set for SparklySpawner to work
+        """
+    )
+
+    gcp_project = Unicode(
+        None,
+        allow_none=True,
+        config=True,
+        help="""
+        Name of Google Cloud Project this hub is running in.
+
+        *Must* be set.
+        """
+    )
+
+    storage_bucket_template = Unicode(
+        None,
+        allow_none=True,
+        config=True,
+        help="""
+        Template to use when creating storage buckets for users.
+
+        {username} is expanded.
+        """
+    )
+
+    service_account_template = Unicode(
+        None,
+        allow_none=True,
+        config=True,
+        help="""
+        Template to use when creating service accounts for users.
+
+        {username} is expanded.
+        """
+    )
+
+    async def ensure_gcp_resources(self):
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.gcp_service_key.encode())
+            f.flush()
+            storage_client = storage.Client.from_service_account_json(f.name)
+
+            credentials = service_account.Credentials.from_service_account_file(
+                f.name, scopes=['https://www.googleapis.com/auth/cloud-platform'])
+            service = googleapiclient.discovery.build('iam', 'v1', credentials=credentials)
+
+
+        # Create bucket if it doesn't exist
+        # FIXME: Don't use a private method?
+        bucket_name = self._expand_all(self.storage_bucket_template)
+
+        bucket = storage.Bucket(storage_client, bucket_name)
+
+        try:
+            bucket.create()
+            self.log.info(f'Creating {bucket_name}')
+        except Conflict as e:
+            # Bucket already exists
+            self.log.info(f'Not creating {bucket_name}, it already exists')
+        self.environment['SPARK_GCS_BUCKET'] = bucket_name
+
+        # This is how service account emails are formatted
+        # FIXME: Clip this to 30char
+        sa_name = self._expand_all(self.service_account_template)
+        sa_email = f'{sa_name}@{self.gcp_project}.iam.gserviceaccount.com'
+        try:
+            sa = service.projects().serviceAccounts().create(
+                name=f'projects/{self.gcp_project}',
+                body={
+                    'accountId': sa_name,
+                    'serviceAccount': {'displayName': sa_name}
+            }).execute()
+            # We assume this create call will create a service account with email sa_email
+            assert sa_email == sa['email']            
+            self.log.info(f'Created service account {sa["email"]}')
+        except HttpError as e:
+            if e.resp.status == 409:
+                self.log.info(f'Created service account {sa_email}')
+            else:
+                raise
+
+        # Grant SA access to bucket if it isn't present
+        policy = bucket.get_iam_policy()
+        role = 'roles/storage.objectAdmin'
+        if Policy.service_account(sa_email) not in policy.get(role, set()):
+            policy[role].add(Policy.service_account(sa_email))
+            bucket.set_iam_policy(policy)
+
+        # Check if _key exists in bucket. This is where we store private key
+        key_blob = bucket.blob('__key__.json')
+        if not key_blob.exists():
+            key = service.projects().serviceAccounts().keys().create(
+                name=f'projects/{self.gcp_project}/serviceAccounts/{sa_email}', body={}
+            ).execute()
+        else:
+            key = json.loads(key_blob.download_as_string())
+
+        self.environment['SPARK_GCS_KEY'] = base64.b64decode(key['privateKeyData']).decode()
+
+    async def start(self):
+        self.log.info('Testing Sparkly spawner')
+        await self.ensure_gcp_resources()
+        return await super().start()
\ No newline at end of file

From 49f8b9df5ff997a2f37ac517d83596a38829bf44 Mon Sep 17 00:00:00 2001
From: yuvipanda <yuvipanda@gmail.com>
Date: Mon, 13 May 2019 21:20:01 -0700
Subject: [PATCH 2/2] w261: Update hub image tag for previous commit

---
 hub/Chart.yaml  | 2 +-
 hub/values.yaml | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/hub/Chart.yaml b/hub/Chart.yaml
index 2b081ba58..3c328fba1 100644
--- a/hub/Chart.yaml
+++ b/hub/Chart.yaml
@@ -2,4 +2,4 @@ apiVersion: v1
 appVersion: '1.0'
 description: Deployment Chart for JupyterHub
 name: hub
-version: 0.1.0-facec6f
+version: 0.1.0-a34aa77
diff --git a/hub/values.yaml b/hub/values.yaml
index 530beb026..292330843 100644
--- a/hub/values.yaml
+++ b/hub/values.yaml
@@ -35,7 +35,7 @@ jupyterhub:
     # Generated by chartpress
     image:
       name: gcr.io/ucb-datahub-2018/jupyterhub-hub
-      tag: '0.1.0-401ee46'
+      tag: '0.1.0-a34aa77'
     networkPolicy:
       enabled: true
     extraConfig:
@@ -47,7 +47,7 @@ jupyterhub:
 
         hosted_domain = 'berkeley.edu'
         course_profile_tmpl = '/srv/jupyterhub/profiles.d/{}-{}.txt'
-    
+
         def memparse(val):
           '''Parse memory for relative comparisons.'''
           if type(val) != str or len(val) == 0: return val
@@ -55,7 +55,7 @@ jupyterhub:
           unit = val[-1]
           n = {'B':0, 'K':1, 'M':2, 'G':3}[unit]
           return mem * 1024**n
-    
+
         def mem_cmp(a, b):
           '''Compare memory values.'''
           x = memparse(a) ; y = memparse(b)