cleanup

.github/workflows/ci.yml

@@ -18,11 +18,11 @@ jobs:
     strategy:
       matrix:
         image_base_name:
-          # - minimal
-          # - datascience
+          - minimal
+          - datascience
           - llamaindex
-          # - nlp
-          # - audio
+          - nlp
+          - audio
 
     name: notebook-${{ matrix.image_base_name }}
     runs-on: ubuntu-latest

audio/Dockerfile

@@ -1,5 +1,5 @@
 ARG BASE_IMAGE_TAG=latest
-FROM b08x/notebook-minimal:$BASE_IMAGE_TAG
+FROM b08x/notebook-datascience:$BASE_IMAGE_TAG
 
 LABEL maintainer="Robert Pannick <[email protected]>"
 
@@ -48,18 +48,14 @@ RUN git clone https://github.com/mike-bourgeous/mb-sound.git
 #RUN conda update jupyterlab -y && conda install pip -y
 # RUN pip install 'jupyter_ai>=1.0,<2.0'
 
-# List of txtai components to install
-ARG COMPONENTS=[all]
 #
 # # Locale environment variables
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 #
 RUN \
     pip install --no-cache-dir -U pip wheel setuptools && \
-    pip install --no-cache-dir torch==2.1.2+cpu torchvision==0.16.2+cpu torchaudio -f https://download.pytorch.org/whl/torch_stable.html && \
-    pip install --no-cache-dir txtai${COMPONENTS} && \
-    python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')"
+    pip install --no-cache-dir torch==2.1.2+cpu torchvision==0.16.2+cpu torchaudio -f https://download.pytorch.org/whl/torch_stable.html
 
 
 # NOTE: DO NOT CHANGE the version in the path of gem's bin directory

audio/Gemfile

@@ -79,7 +79,7 @@ gem "nmatrix", git: "https://github.com/b08x/nmatrix.git", branch: "development"
 gem "nmatrix-lapacke", git: "https://github.com/b08x/nmatrix.git", branch: "development"
 gem "nmatrix-fftw", git: "https://github.com/b08x/nmatrix.git", branch: "development"
 
-gem 'torch-rb'
+gem 'torch-rb', '0.14.1'
 gem 'torchaudio'
 gem 'matplotlib', '>= 1.2.0'
 gem 'gnuplot'

datascience/Dockerfile

@@ -5,8 +5,6 @@ LABEL maintainer="Robert Pannick <[email protected]>"
 
 USER root
 
-ENV USER jovyan
-
 RUN apt-get update && \
     apt-get install -y -V --no-install-recommends gpg-agent
 
@@ -30,47 +28,47 @@ RUN curl -sfSL -o /tmp/apache-arrow-apt-source-latest.deb \
   apt install -y \
   libgirepository1.0-dev
 
-RUN mkdir -pv /home/$USER/packages
+RUN mkdir -pv /home/$NB_USER/packages
 
 COPY packages packages/
 
 RUN apt install -y --no-install-recommends \
-    /home/$USER/packages/gir1.2-arrow-1.0_11.0.0-1_amd64.deb \
-    /home/$USER/packages/gir1.2-arrow-cuda-1.0_11.0.0-1_amd64.deb \
-    /home/$USER/packages/gir1.2-gandiva-1.0_11.0.0-1_amd64.deb \
-    /home/$USER/packages/gir1.2-parquet-1.0_11.0.0-1_amd64.deb \
-    /home/$USER/packages/gir1.2-plasma-1.0_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-cuda-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-cuda-glib-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-cuda-glib1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-cuda1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-dataset-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-dataset1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-flight-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-flight1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-glib-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libarrow-glib1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libgandiva-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libgandiva-glib-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libgandiva-glib1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libgandiva1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libparquet-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libparquet-glib-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libparquet-glib1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libparquet1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libplasma-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libplasma-glib-dev_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libplasma-glib1100_11.0.0-1_amd64.deb \
-    /home/$USER/packages/libplasma1100_11.0.0-1_amd64.deb && \
+    /home/$NB_USER/packages/gir1.2-arrow-1.0_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/gir1.2-arrow-cuda-1.0_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/gir1.2-gandiva-1.0_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/gir1.2-parquet-1.0_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/gir1.2-plasma-1.0_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-cuda-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-cuda-glib-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-cuda-glib1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-cuda1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-dataset-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-dataset1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-flight-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-flight1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-glib-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libarrow-glib1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libgandiva-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libgandiva-glib-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libgandiva-glib1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libgandiva1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libparquet-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libparquet-glib-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libparquet-glib1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libparquet1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libplasma-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libplasma-glib-dev_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libplasma-glib1100_11.0.0-1_amd64.deb \
+    /home/$NB_USER/packages/libplasma1100_11.0.0-1_amd64.deb && \
     apt-get autoremove -y && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /home/$USER
+WORKDIR /home/$NB_USER
 
 COPY datascience/Gemfile .
 
-RUN chown -R $USER:$GROUP_ID /home/$USER
+RUN chown -R $NB_USER:$GROUP_ID /home/$NB_USER
 
 USER $NB_UID
 

llamaindex/Dockerfile

@@ -3,18 +3,12 @@ FROM b08x/notebook-datascience:$BASE_IMAGE_TAG
 
 LABEL maintainer="Robert Pannick <[email protected]>"
 
-USER root
-
-ENV USER jovyan
-
-RUN apt-get update && \
-    apt-get install -y -V --no-install-recommends gpg-agent && \
-    apt-get autoremove -y && \
-    rm -rf /var/lib/apt/lists/*
-
 USER $NB_UID
 
-WORKDIR /home/$USER
+WORKDIR /home/$NB_USER
+
+# List of txtai components to install
+ARG COMPONENTS=[all]
 
 RUN \
     pip --no-cache-dir install spacy && \

minimal/Dockerfile

@@ -5,15 +5,13 @@ LABEL maintainer="Kenta Murata <[email protected]>"
 
 USER root
 
-ENV USER jovyan
-
 ARG GROUP_ID=1041
 
 RUN groupadd -g $GROUP_ID devops
 
-RUN usermod -a -G sudo $USER && usermod -a -G $GROUP_ID $USER
+RUN usermod -a -G sudo $NB_USER && usermod -a -G $GROUP_ID $NB_USER
 
-RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USER && chmod 0440 /etc/sudoers.d/99-$USER
+RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$NB_USER && chmod 0440 /etc/sudoers.d/99-$NB_USER
 
 # Pre-requisites
 RUN apt-get update && apt-get install -y -V --no-install-recommends \
@@ -96,17 +94,17 @@ COPY --from=rubylang/ruby:3.1.3-focal \
   /usr/local/share/man/man1/ruby.1 \
   /usr/local/share/man/man1/
 
-WORKDIR /home/$USER
+WORKDIR /home/$NB_USER
 
 COPY minimal/Gemfile .
 
 RUN mkdir -pv output && chmod 2775 output
 
-RUN echo "[[ -f ~/.env ]] && source ~/.env" >> /home/$USER/.bashrc
+RUN echo "[[ -f ~/.env ]] && source ~/.env" >> /home/$NB_USER/.bashrc
 
-RUN chown -R $USER:$GROUP_ID /home/$USER
+RUN chown -R $NB_USER:$GROUP_ID /home/$NB_USER
 
-USER $NB_UID
+NB_USER $NB_UID
 
 RUN echo "gem: --user-install" >> $HOME/.gemrc
 
@@ -117,4 +115,6 @@ ENV BUNDLE_PATH $HOME/.local/share/gem
 
 RUN bundle install
 
+RUN conda update jupyterlab -y && conda install pip -y
+
 RUN iruby register --force

nlp/Dockerfile

@@ -1,12 +1,10 @@
 ARG BASE_IMAGE_TAG=latest
-FROM b08x/notebook-datascience:$BASE_IMAGE_TAG
+FROM b08x/notebook-llamaindex:$BASE_IMAGE_TAG
 
 LABEL maintainer="Robert Pannick <[email protected]>"
 
 USER root
 
-ENV USER jovyan
-
 RUN apt update -qq && \
     apt install -y -V --no-install-recommends  \
       libcairo2-dev \
@@ -26,31 +24,15 @@ RUN apt update -qq && \
     apt-get autoremove -y && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
-RUN echo "gem: --user-install" >> /home/$USER/.gemrc && \
+RUN echo "gem: --user-install" >> /home/$NB_USER/.gemrc && \
     echo "gem: --user-install" >> /etc/gemrc && \
     echo "gem: --no-user-install --no-document" >> /root/.gemrc && \
-    gem update --system 3.5.5 && chown -R $USER:$GROUP_ID /home/$USER
-
-USER $USER
-WORKDIR /home/$USER
+    gem update --system 3.5.5 && chown -R $NB_USER:$GROUP_ID /home/$NB_USER
 
-# List of txtai components to install
-ARG COMPONENTS=[all]
+USER $NB_USER
+WORKDIR /home/$NB_USER
 # Locale environment variables
 ENV LC_ALL=C.UTF-8
-ENV LANG=C.UTF-8
-#RUN conda update jupyterlab -y && conda install pip -y
-# RUN pip install 'jupyter_ai>=1.0,<2.0'
-RUN \
-    pip --no-cache-dir install spacy && \
-    python3 -m spacy download en_core_web_sm && \
-    python3 -m spacy download en_core_web_lg && \
-    pip --no-cache-dir install prompttools && \
-    pip install --no-cache-dir -U pip wheel setuptools && \
-    pip install --no-cache-dir tensorflow torch==2.1.2+cpu torchaudio torchvision==0.16.2+cpu -f https://download.pytorch.org/whl/torch_stable.html && \
-    pip install --no-cache-dir prompttools && \
-    pip install --no-cache-dir txtai${COMPONENTS} && \
-    python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')"
 
 COPY nlp/Gemfile .
 #TODO: Create gemspec for ferret