From 4bba1ad78dbc6b260a8f8614fd3d6075e422065a Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Wed, 15 Jul 2020 00:27:58 -0400
Subject: [PATCH 01/11] Begin to support Julia

---
 caliban/docker/build.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 0740853..ac6b013 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -212,12 +212,14 @@ def _dependency_entries(workdir: str,
                         user_group: int,
                         requirements_path: Optional[str] = None,
                         conda_env_path: Optional[str] = None,
-                        setup_extras: Optional[List[str]] = None) -> str:
+                        setup_extras: Optional[List[str]] = None,
+                        julia_url: Optional[str] = None) -> str:
   """Returns the Dockerfile entries required to install dependencies from either:
 
   - a requirements.txt file, path supplied by requirements_path
   - a conda environment.yml file, path supplied by conda_env_path.
   - a setup.py file, if some sequence of dependencies is supplied.
+  - a Julia version, specified as url
 
   An empty list for setup_extras means, run `pip install -c .` with no extras.
   None for this argument means do nothing. If a list of strings is supplied,
@@ -246,6 +248,25 @@ def _dependency_entries(workdir: str,
 RUN /bin/bash -c "pip install --no-cache-dir -r {requirements_path}"
 """
 
+  julia_url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-rc1-linux-x86_64.tar.gz"
+  if julia_url is not None:
+    # e.g. "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-rc1-linux-x86_64.tar.gz"
+    ret += f"""
+COPY --chown={user_id}:{user_group} *.toml {workdir}/
+COPY --chown={user_id}:{user_group} src {workdir}/src
+# TODO: Use /usr/local instead of /tmp
+ENV JULIA_LOC /tmp/julia
+ENV JULIA_DEPOT_PATH /tmp/var/julia_depot
+ENV JULIA_PROJECT {workdir}
+ENV PATH $PATH:$JULIA_LOC/bin
+RUN /bin/bash -c "mkdir -p $JULIA_LOC && \
+                  wget -nv {julia_url} && \
+                  tar xzf $(basename {julia_url}) -C $JULIA_LOC --strip-components 1 && \
+                  rm -f $(basename {julia_url}) && \
+                  mkdir -p $JULIA_DEPOT_PATH && \
+                  julia --eval 'using Pkg; Pkg.instantiate()'"
+"""
+
   return ret
 
 
@@ -463,6 +484,7 @@ def _dockerfile_template(
     requirements_path: Optional[str] = None,
     conda_env_path: Optional[str] = None,
     setup_extras: Optional[List[str]] = None,
+    julia_url: Optional[str] = None,
     adc_path: Optional[str] = None,
     credentials_path: Optional[str] = None,
     jupyter_version: Optional[str] = None,
@@ -545,7 +567,8 @@ def _dockerfile_template(
                                     gid,
                                     requirements_path=requirements_path,
                                     conda_env_path=conda_env_path,
-                                    setup_extras=setup_extras)
+                                    setup_extras=setup_extras,
+                                    julia_url=julia_url)
 
   if inject_notebook.value != 'none':
     install_lab = inject_notebook == NotebookInstall.lab

From a0916969569b36c1f88208d9ea52148f443b7a90 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Fri, 17 Jul 2020 11:00:00 -0400
Subject: [PATCH 02/11] Thread Julia options through configuration file

---
 caliban/cli.py             |  1 +
 caliban/config/__init__.py |  3 ++-
 caliban/docker/build.py    | 18 ++++++++++--------
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/caliban/cli.py b/caliban/cli.py
index 443e9f0..9d93ad7 100644
--- a/caliban/cli.py
+++ b/caliban/cli.py
@@ -555,6 +555,7 @@ def generate_docker_args(job_mode: conf.JobMode,
   docker_args = {
       "extra_dirs": args.get("dir"),
       "requirements_path": reqs if os.path.exists(reqs) else None,
+      "julia_url": conf.caliban_config().get("julia_url", None),
       "conda_env_path": conda_env if os.path.exists(conda_env) else None,
       "caliban_config": conf.caliban_config(),
       "credentials_path": creds_path,
diff --git a/caliban/config/__init__.py b/caliban/config/__init__.py
index 7bb7a60..424d607 100644
--- a/caliban/config/__init__.py
+++ b/caliban/config/__init__.py
@@ -130,7 +130,8 @@ def expand_image(image: str) -> str:
     s.Optional("project_id"): s.And(str, len),
     s.Optional("cloud_key"): s.And(str, len),
     s.Optional("base_image", default=None): BaseImage,
-    s.Optional("apt_packages", default=AptPackages.validate({})): AptPackages
+    s.Optional("apt_packages", default=AptPackages.validate({})): AptPackages,
+    s.Optional("julia_url", default=None): str
 })
 
 # Accessors
diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index cf43a92..5b195f7 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -22,6 +22,7 @@
 
 import json
 import os
+import os.path
 import subprocess
 from enum import Enum
 from pathlib import Path
@@ -247,17 +248,18 @@ def _dependency_entries(workdir: str,
 RUN /bin/bash -c "pip install --no-cache-dir -r {requirements_path}"
 """
 
-  julia_url = "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-rc1-linux-x86_64.tar.gz"
   if julia_url is not None:
-    # e.g. "https://julialang-s3.julialang.org/bin/linux/x64/1.5/julia-1.5.0-rc1-linux-x86_64.tar.gz"
-    ret += f"""
+    if os.path.exists("Project.toml"):
+      ret += f"""
 COPY --chown={user_id}:{user_group} *.toml {workdir}/
+"""
+    # TODO: Use /usr/local instead of /tmp for Julia directories
+    ret += f"""
 COPY --chown={user_id}:{user_group} src {workdir}/src
-# TODO: Use /usr/local instead of /tmp
-ENV JULIA_LOC /tmp/julia
-ENV JULIA_DEPOT_PATH /tmp/var/julia_depot
-ENV JULIA_PROJECT {workdir}
-ENV PATH $PATH:$JULIA_LOC/bin
+ENV JULIA_LOC=/tmp/julia
+ENV JULIA_DEPOT_PATH=/tmp/var/julia_depot
+ENV JULIA_PROJECT={workdir}
+ENV PATH=$PATH:$JULIA_LOC/bin
 RUN /bin/bash -c "mkdir -p $JULIA_LOC && \
                   wget -nv {julia_url} && \
                   tar xzf $(basename {julia_url}) -C $JULIA_LOC --strip-components 1 && \

From 486ce0c414851d8b90c364472b1ffbe78e82ee70 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Thu, 23 Jul 2020 16:39:10 -0400
Subject: [PATCH 03/11] Julia: Disable telemetry by default

---
 caliban/docker/build.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 5b195f7..8f7d161 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -265,6 +265,8 @@ def _dependency_entries(workdir: str,
                   tar xzf $(basename {julia_url}) -C $JULIA_LOC --strip-components 1 && \
                   rm -f $(basename {julia_url}) && \
                   mkdir -p $JULIA_DEPOT_PATH && \
+                  mkdir -p $JULIA_DEPOT_PATH/servers && \
+                  echo 'telemetry = false' > $JULIA_DEPOT_PATH/servers/telemetry.toml && \
                   julia --eval 'using Pkg; Pkg.instantiate()'"
 """
 

From 66da797f12ffdec0e9272c4c9e5306888fcb40af Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Thu, 30 Jul 2020 14:42:00 -0400
Subject: [PATCH 04/11] Update Julia install statements in Dockerfile

---
 caliban/docker/build.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 8f7d161..a90ca1e 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -249,25 +249,34 @@ def _dependency_entries(workdir: str,
 """
 
   if julia_url is not None:
-    if os.path.exists("Project.toml"):
-      ret += f"""
-COPY --chown={user_id}:{user_group} *.toml {workdir}/
-"""
-    # TODO: Use /usr/local instead of /tmp for Julia directories
+    # Install Julia
     ret += f"""
-COPY --chown={user_id}:{user_group} src {workdir}/src
+# Where Julia is installed
+# TODO: Use /usr/local instead of /tmp for Julia directories
 ENV JULIA_LOC=/tmp/julia
-ENV JULIA_DEPOT_PATH=/tmp/var/julia_depot
-ENV JULIA_PROJECT={workdir}
 ENV PATH=$PATH:$JULIA_LOC/bin
+# Where Julia installs packages
+ENV JULIA_DEPOT_PATH=/tmp/var/julia_depot
 RUN /bin/bash -c "mkdir -p $JULIA_LOC && \
                   wget -nv {julia_url} && \
                   tar xzf $(basename {julia_url}) -C $JULIA_LOC --strip-components 1 && \
                   rm -f $(basename {julia_url}) && \
+                  echo 'ENV["PYTHON"] = ""' >> $JULIA_LOC/etc/julia/startup.jl && \
                   mkdir -p $JULIA_DEPOT_PATH && \
                   mkdir -p $JULIA_DEPOT_PATH/servers && \
-                  echo 'telemetry = false' > $JULIA_DEPOT_PATH/servers/telemetry.toml && \
-                  julia --eval 'using Pkg; Pkg.instantiate()'"
+                  echo 'telemetry = false' > $JULIA_DEPOT_PATH/servers/telemetry.toml"
+"""
+
+    # Install dependencies for the current Julia project
+    if os.path.exists("Project.toml"):
+      ret += f"""
+# The current Julia project
+ENV JULIA_PROJECT={workdir}
+COPY --chown={user_id}:{user_group} *.toml {workdir}/
+COPY --chown={user_id}:{user_group} src {workdir}/src/
+RUN /bin/bash -c "cd {workdir} && \
+                  julia --eval 'using Pkg; Pkg.instantiate()' && \
+                  rm -rf *"
 """
 
   return ret

From 8b77859c685a0930874277e0ccbfe58c61ff2dcb Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Thu, 30 Jul 2020 15:13:22 -0400
Subject: [PATCH 05/11] Use julia_version instead of julia_url configuration
 key

---
 caliban/cli.py          |  2 +-
 caliban/docker/build.py | 15 ++++++++++-----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/caliban/cli.py b/caliban/cli.py
index abff91d..9637ece 100644
--- a/caliban/cli.py
+++ b/caliban/cli.py
@@ -555,7 +555,7 @@ def generate_docker_args(job_mode: conf.JobMode,
   docker_args = {
       "extra_dirs": args.get("dir"),
       "requirements_path": reqs if os.path.exists(reqs) else None,
-      "julia_url": conf.caliban_config().get("julia_url", None),
+      "julia_version": conf.caliban_config().get("julia_version", None),
       "conda_env_path": conda_env if os.path.exists(conda_env) else None,
       "caliban_config": conf.caliban_config(),
       "credentials_path": creds_path,
diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index fa2adcc..b1cafa9 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -23,6 +23,7 @@
 import json
 import os
 import os.path
+import re
 import subprocess
 from enum import Enum
 from pathlib import Path
@@ -237,7 +238,7 @@ def _dependency_entries(workdir: str,
                         requirements_path: Optional[str] = None,
                         conda_env_path: Optional[str] = None,
                         setup_extras: Optional[List[str]] = None,
-                        julia_url: Optional[str] = None) -> str:
+                        julia_version: Optional[str] = None) -> str:
   """Returns the Dockerfile entries required to install dependencies from either:
 
   - a requirements.txt file, path supplied by requirements_path
@@ -275,8 +276,12 @@ def copy(from_path, to_path):
 RUN /bin/bash -c "pip install --no-cache-dir -r {requirements_path}"
 """
 
-  if julia_url is not None:
+  if julia_version is not None:
     # Install Julia
+    # Extract major and minor version numbers
+    m = re.match(r"(\d+[.]\d+)", julia_version)
+    julia_version2 = m[1]
+    julia_url = f"https://julialang-s3.julialang.org/bin/linux/x64/{julia_version2}/julia-{julia_version}-linux-x86_64.tar.gz"
     ret += f"""
 # Where Julia is installed
 # TODO: Use /usr/local instead of /tmp for Julia directories
@@ -288,7 +293,7 @@ def copy(from_path, to_path):
                   wget -nv {julia_url} && \
                   tar xzf $(basename {julia_url}) -C $JULIA_LOC --strip-components 1 && \
                   rm -f $(basename {julia_url}) && \
-                  echo 'ENV["PYTHON"] = ""' >> $JULIA_LOC/etc/julia/startup.jl && \
+                  echo 'ENV[\\"PYTHON\\"] = \\"\\"' >> $JULIA_LOC/etc/julia/startup.jl && \
                   mkdir -p $JULIA_DEPOT_PATH && \
                   mkdir -p $JULIA_DEPOT_PATH/servers && \
                   echo 'telemetry = false' > $JULIA_DEPOT_PATH/servers/telemetry.toml"
@@ -540,7 +545,7 @@ def _dockerfile_template(
     requirements_path: Optional[str] = None,
     conda_env_path: Optional[str] = None,
     setup_extras: Optional[List[str]] = None,
-    julia_url: Optional[str] = None,
+    julia_version: Optional[str] = None,
     adc_path: Optional[str] = None,
     credentials_path: Optional[str] = None,
     jupyter_version: Optional[str] = None,
@@ -610,7 +615,7 @@ def _dockerfile_template(
                                     requirements_path=requirements_path,
                                     conda_env_path=conda_env_path,
                                     setup_extras=setup_extras,
-                                    julia_url=julia_url)
+                                    julia_version=julia_version)
 
   if inject_notebook.value != 'none':
     install_lab = inject_notebook == NotebookInstall.lab

From 4aa039d405ce4c9ddee41cdf3afd79eb18d1ce32 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Thu, 30 Jul 2020 15:28:54 -0400
Subject: [PATCH 06/11] Precompile Julia dependencies

---
 caliban/docker/build.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index b1cafa9..ca2f77b 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -307,7 +307,7 @@ def copy(from_path, to_path):
 COPY --chown={user_id}:{user_group} *.toml {workdir}/
 COPY --chown={user_id}:{user_group} src {workdir}/src/
 RUN /bin/bash -c "cd {workdir} && \
-                  julia --eval 'using Pkg; Pkg.instantiate()' && \
+                  julia --eval 'using Pkg; Pkg.instantiate(); Pkg.precompile()' && \
                   rm -rf *"
 """
 

From a2b53649d025c93268786149391b6b22f1c5e14a Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Wed, 26 Aug 2020 15:17:57 -0400
Subject: [PATCH 07/11] Julia: import re

---
 caliban/docker/build.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 3732350..553723d 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -22,6 +22,7 @@
 
 import json
 import os
+import re
 import subprocess
 from enum import Enum
 from pathlib import Path

From ce023d77f73def082c21c479b62d7512ce4ee87c Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Wed, 23 Sep 2020 09:18:13 -0400
Subject: [PATCH 08/11] Allow using nightly Julia builds, not just released
 versions

---
 caliban/docker/build.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 553723d..2309c0a 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -24,6 +24,7 @@
 import os
 import re
 import subprocess
+from datetime import date
 from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, NewType, Optional, Union
@@ -280,9 +281,17 @@ def copy(from_path, to_path):
   if julia_version is not None:
     # Install Julia
     # Extract major and minor version numbers
-    m = re.match(r"(\d+[.]\d+)", julia_version)
-    julia_version2 = m[1]
-    julia_url = f"https://julialang-s3.julialang.org/bin/linux/x64/{julia_version2}/julia-{julia_version}-linux-x86_64.tar.gz"
+    if julia_version == "nightly":
+      julia_url = f"https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz"
+      today = date.today()
+      # A changing tag forces a new Julia download
+      julia_tag = f"{today.year}-{today.month}-{today.day}"
+    else:
+      m = re.match(r"(\d+[.]\d+)", julia_version)
+      julia_version2 = m[1]
+      julia_url = f"https://julialang-s3.julialang.org/bin/linux/x64/{julia_version2}/julia-{julia_version}-linux-x86_64.tar.gz"
+      julia_tag = julia_version
+
     ret += f"""
 # Where Julia is installed
 # TODO: Use /usr/local instead of /tmp for Julia directories
@@ -291,6 +300,7 @@ def copy(from_path, to_path):
 # Where Julia installs packages
 ENV JULIA_DEPOT_PATH=/tmp/var/julia_depot
 RUN /bin/bash -c "mkdir -p $JULIA_LOC && \
+                  echo 'Downloading Julia {julia_tag}' && \
                   wget -nv {julia_url} && \
                   tar xzf $(basename {julia_url}) -C $JULIA_LOC --strip-components 1 && \
                   rm -f $(basename {julia_url}) && \

From d450421f1f097eaf964cb812ec3b6516eacda158 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Wed, 23 Sep 2020 09:18:24 -0400
Subject: [PATCH 09/11] Precompiling a Julia package may fail

---
 caliban/docker/build.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 2309c0a..ba70755 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -318,7 +318,12 @@ def copy(from_path, to_path):
 COPY --chown={user_id}:{user_group} *.toml {workdir}/
 COPY --chown={user_id}:{user_group} src {workdir}/src/
 RUN /bin/bash -c "cd {workdir} && \
-                  julia --eval 'using Pkg; Pkg.instantiate(); Pkg.precompile()' && \
+                  julia --eval 'using Pkg; \
+                                Pkg.instantiate(); \
+                                try \
+                                    Pkg.precompile() \
+                                catch \
+                                end' && \
                   rm -rf *"
 """
 

From 776d906921c9beac9f02e4c48c08d767ff1b8dd8 Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Tue, 3 Nov 2020 13:57:10 -0500
Subject: [PATCH 10/11] Output debug message indicating whether Julia is
 provided

---
 caliban/docker/build.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index ba70755..3894e5a 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -278,6 +278,11 @@ def copy(from_path, to_path):
 RUN /bin/bash -c "pip install --no-cache-dir -r {requirements_path}"
 """
 
+  if julia_version is not None:
+    print(f"Providing Julia version {julia_version}")
+  else:
+    print("Not providing Julia")
+
   if julia_version is not None:
     # Install Julia
     # Extract major and minor version numbers

From c5eb7e9c17c5ef3653a3296104a7be8bcf2f9acf Mon Sep 17 00:00:00 2001
From: Erik Schnetter <schnetter@gmail.com>
Date: Tue, 3 Nov 2020 14:16:53 -0500
Subject: [PATCH 11/11] Improve outputting Julia version number

---
 caliban/docker/build.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/caliban/docker/build.py b/caliban/docker/build.py
index 3894e5a..c565a95 100644
--- a/caliban/docker/build.py
+++ b/caliban/docker/build.py
@@ -278,13 +278,9 @@ def copy(from_path, to_path):
 RUN /bin/bash -c "pip install --no-cache-dir -r {requirements_path}"
 """
 
-  if julia_version is not None:
-    print(f"Providing Julia version {julia_version}")
-  else:
-    print("Not providing Julia")
-
   if julia_version is not None:
     # Install Julia
+    print(f"Providing Julia version {julia_version}")
     # Extract major and minor version numbers
     if julia_version == "nightly":
       julia_url = f"https://julialangnightlies-s3.julialang.org/bin/linux/x64/julia-latest-linux64.tar.gz"