Update spark to 3.0.0 & add notebook image option

bin/cowait

@@ -219,11 +219,15 @@ def agent(ctx, cluster: str, detach: bool, upstream: str):
               type=bool, is_flag=True,
               help='build and push first',
               default=False)
+@click.option('-i', '--image',
+              type=str,
+              default=None,
+              help='default image')
 @click.pass_context
-def notebook(ctx, cluster, build):
+def notebook(ctx, cluster, build, image):
     if cluster is not None:
         ctx.obj.default_cluster = cluster
-    cowait.cli.notebook(ctx.obj, build)
+    cowait.cli.notebook(ctx.obj, build, image)
 
 
 #

cowait/cli/commands/notebook.py

@@ -1,10 +1,13 @@
 from .run import run as run_cmd
 
 
-def notebook(config, build):
+def notebook(config, build, image = None):
+    task = 'cowait.notebook'
+    if image is not None:
+        task = f'{image}/{task}'
     return run_cmd(
         config=config,
-        task='cowait.notebook',
+        task=task,
         build=build,
         routes={
             '/': '8888',

cowait/tasks/spark/worker.py

@@ -2,7 +2,7 @@
 
 
 class SparkWorker(ShellTask):
-    async def run(self, master: str, cores: int = 2) -> dict:
+    async def run(self, master: str, cores: int = 2, memory = '4G') -> dict:
         return await super().run(
             command=' '.join([
                 'spark-class',
@@ -11,5 +11,6 @@ async def run(self, master: str, cores: int = 2) -> dict:
             ]),
             env={
                 'SPARK_WORKER_CORES': str(cores),
+                'SPARK_WORKER_MEMORY': memory,
             },
         )

images/docker-compose.yml

@@ -2,4 +2,4 @@ version: '2'
 services:
   spark:
     build: './spark'
-    image: 'cowait/task-spark'
+    image: 'cowait/spark'

images/spark/Dockerfile

@@ -6,9 +6,9 @@ RUN apt-get update && apt-get install -y wget default-jre-headless
 ENV JAVA_HOME=/usr/lib/jvm/default-java
 
 # install spark
-ENV SPARK_VERSION=3.0.0-preview2
+ENV SPARK_VERSION=3.0.0
 ENV HADOOP_VERSION=3.2
-ENV PY4J_VERSION=0.10.8.1
+ENV PY4J_VERSION=0.10.9
 
 RUN wget \
         -O spark.tgz https://www-eu.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz \