diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69a7d7b..cd683a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,11 +18,11 @@ jobs: strategy: matrix: image_base_name: - # - minimal - # - datascience + - minimal + - datascience - llamaindex - # - nlp - # - audio + - nlp + - audio name: notebook-${{ matrix.image_base_name }} runs-on: ubuntu-latest diff --git a/audio/Dockerfile b/audio/Dockerfile index 19225c9..3346a02 100644 --- a/audio/Dockerfile +++ b/audio/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMAGE_TAG=latest -FROM b08x/notebook-minimal:$BASE_IMAGE_TAG +FROM b08x/notebook-datascience:$BASE_IMAGE_TAG LABEL maintainer="Robert Pannick " @@ -48,8 +48,6 @@ RUN git clone https://github.com/mike-bourgeous/mb-sound.git #RUN conda update jupyterlab -y && conda install pip -y # RUN pip install 'jupyter_ai>=1.0,<2.0' -# List of txtai components to install -ARG COMPONENTS=[all] # # # Locale environment variables ENV LC_ALL=C.UTF-8 @@ -57,9 +55,7 @@ ENV LANG=C.UTF-8 # RUN \ pip install --no-cache-dir -U pip wheel setuptools && \ - pip install --no-cache-dir torch==2.1.2+cpu torchvision==0.16.2+cpu torchaudio -f https://download.pytorch.org/whl/torch_stable.html && \ - pip install --no-cache-dir txtai${COMPONENTS} && \ - python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')" + pip install --no-cache-dir torch==2.1.2+cpu torchvision==0.16.2+cpu torchaudio -f https://download.pytorch.org/whl/torch_stable.html # NOTE: DO NOT CHANGE the version in the path of gem's bin directory diff --git a/audio/Gemfile b/audio/Gemfile index 93f293b..d242c64 100644 --- a/audio/Gemfile +++ b/audio/Gemfile @@ -79,7 +79,7 @@ gem "nmatrix", git: "https://github.com/b08x/nmatrix.git", branch: "development" gem "nmatrix-lapacke", git: "https://github.com/b08x/nmatrix.git", branch: "development" gem "nmatrix-fftw", git: "https://github.com/b08x/nmatrix.git", branch: "development" -gem 'torch-rb' +gem 'torch-rb', '0.14.1' gem 'torchaudio' gem 'matplotlib', '>= 1.2.0' gem 'gnuplot' diff --git a/datascience/Dockerfile b/datascience/Dockerfile index 984cb0e..a7ff12d 100644 --- a/datascience/Dockerfile +++ b/datascience/Dockerfile @@ -5,8 +5,6 @@ LABEL maintainer="Robert Pannick " USER root -ENV USER jovyan - RUN apt-get update && \ apt-get install -y -V --no-install-recommends gpg-agent @@ -30,47 +28,47 @@ RUN curl -sfSL -o /tmp/apache-arrow-apt-source-latest.deb \ apt install -y \ libgirepository1.0-dev -RUN mkdir -pv /home/$USER/packages +RUN mkdir -pv /home/$NB_USER/packages COPY packages packages/ RUN apt install -y --no-install-recommends \ - /home/$USER/packages/gir1.2-arrow-1.0_11.0.0-1_amd64.deb \ - /home/$USER/packages/gir1.2-arrow-cuda-1.0_11.0.0-1_amd64.deb \ - /home/$USER/packages/gir1.2-gandiva-1.0_11.0.0-1_amd64.deb \ - /home/$USER/packages/gir1.2-parquet-1.0_11.0.0-1_amd64.deb \ - /home/$USER/packages/gir1.2-plasma-1.0_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-cuda-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-cuda-glib-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-cuda-glib1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-cuda1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-dataset-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-dataset1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-flight-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-flight1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-glib-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libarrow-glib1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libgandiva-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libgandiva-glib-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libgandiva-glib1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libgandiva1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libparquet-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libparquet-glib-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libparquet-glib1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libparquet1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libplasma-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libplasma-glib-dev_11.0.0-1_amd64.deb \ - /home/$USER/packages/libplasma-glib1100_11.0.0-1_amd64.deb \ - /home/$USER/packages/libplasma1100_11.0.0-1_amd64.deb && \ + /home/$NB_USER/packages/gir1.2-arrow-1.0_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/gir1.2-arrow-cuda-1.0_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/gir1.2-gandiva-1.0_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/gir1.2-parquet-1.0_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/gir1.2-plasma-1.0_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-cuda-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-cuda-glib-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-cuda-glib1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-cuda1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-dataset-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-dataset1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-flight-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-flight1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-glib-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libarrow-glib1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libgandiva-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libgandiva-glib-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libgandiva-glib1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libgandiva1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libparquet-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libparquet-glib-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libparquet-glib1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libparquet1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libplasma-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libplasma-glib-dev_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libplasma-glib1100_11.0.0-1_amd64.deb \ + /home/$NB_USER/packages/libplasma1100_11.0.0-1_amd64.deb && \ apt-get autoremove -y && \ apt-get clean && rm -rf /var/lib/apt/lists/* -WORKDIR /home/$USER +WORKDIR /home/$NB_USER COPY datascience/Gemfile . -RUN chown -R $USER:$GROUP_ID /home/$USER +RUN chown -R $NB_USER:$GROUP_ID /home/$NB_USER USER $NB_UID diff --git a/llamaindex/Dockerfile b/llamaindex/Dockerfile index 6fe5c6d..9767608 100644 --- a/llamaindex/Dockerfile +++ b/llamaindex/Dockerfile @@ -3,18 +3,12 @@ FROM b08x/notebook-datascience:$BASE_IMAGE_TAG LABEL maintainer="Robert Pannick " -USER root - -ENV USER jovyan - -RUN apt-get update && \ - apt-get install -y -V --no-install-recommends gpg-agent && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - USER $NB_UID -WORKDIR /home/$USER +WORKDIR /home/$NB_USER + +# List of txtai components to install +ARG COMPONENTS=[all] RUN \ pip --no-cache-dir install spacy && \ diff --git a/minimal/Dockerfile b/minimal/Dockerfile index bc37b64..e0eed00 100644 --- a/minimal/Dockerfile +++ b/minimal/Dockerfile @@ -5,15 +5,13 @@ LABEL maintainer="Kenta Murata " USER root -ENV USER jovyan - ARG GROUP_ID=1041 RUN groupadd -g $GROUP_ID devops -RUN usermod -a -G sudo $USER && usermod -a -G $GROUP_ID $USER +RUN usermod -a -G sudo $NB_USER && usermod -a -G $GROUP_ID $NB_USER -RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USER && chmod 0440 /etc/sudoers.d/99-$USER +RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$NB_USER && chmod 0440 /etc/sudoers.d/99-$NB_USER # Pre-requisites RUN apt-get update && apt-get install -y -V --no-install-recommends \ @@ -96,17 +94,17 @@ COPY --from=rubylang/ruby:3.1.3-focal \ /usr/local/share/man/man1/ruby.1 \ /usr/local/share/man/man1/ -WORKDIR /home/$USER +WORKDIR /home/$NB_USER COPY minimal/Gemfile . RUN mkdir -pv output && chmod 2775 output -RUN echo "[[ -f ~/.env ]] && source ~/.env" >> /home/$USER/.bashrc +RUN echo "[[ -f ~/.env ]] && source ~/.env" >> /home/$NB_USER/.bashrc -RUN chown -R $USER:$GROUP_ID /home/$USER +RUN chown -R $NB_USER:$GROUP_ID /home/$NB_USER -USER $NB_UID +NB_USER $NB_UID RUN echo "gem: --user-install" >> $HOME/.gemrc @@ -117,4 +115,6 @@ ENV BUNDLE_PATH $HOME/.local/share/gem RUN bundle install +RUN conda update jupyterlab -y && conda install pip -y + RUN iruby register --force diff --git a/nlp/Dockerfile b/nlp/Dockerfile index c81d172..5ab2726 100644 --- a/nlp/Dockerfile +++ b/nlp/Dockerfile @@ -1,12 +1,10 @@ ARG BASE_IMAGE_TAG=latest -FROM b08x/notebook-datascience:$BASE_IMAGE_TAG +FROM b08x/notebook-llamaindex:$BASE_IMAGE_TAG LABEL maintainer="Robert Pannick " USER root -ENV USER jovyan - RUN apt update -qq && \ apt install -y -V --no-install-recommends \ libcairo2-dev \ @@ -26,31 +24,15 @@ RUN apt update -qq && \ apt-get autoremove -y && \ apt-get clean && rm -rf /var/lib/apt/lists/* -RUN echo "gem: --user-install" >> /home/$USER/.gemrc && \ +RUN echo "gem: --user-install" >> /home/$NB_USER/.gemrc && \ echo "gem: --user-install" >> /etc/gemrc && \ echo "gem: --no-user-install --no-document" >> /root/.gemrc && \ - gem update --system 3.5.5 && chown -R $USER:$GROUP_ID /home/$USER - -USER $USER -WORKDIR /home/$USER + gem update --system 3.5.5 && chown -R $NB_USER:$GROUP_ID /home/$NB_USER -# List of txtai components to install -ARG COMPONENTS=[all] +USER $NB_USER +WORKDIR /home/$NB_USER # Locale environment variables ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 -#RUN conda update jupyterlab -y && conda install pip -y -# RUN pip install 'jupyter_ai>=1.0,<2.0' -RUN \ - pip --no-cache-dir install spacy && \ - python3 -m spacy download en_core_web_sm && \ - python3 -m spacy download en_core_web_lg && \ - pip --no-cache-dir install prompttools && \ - pip install --no-cache-dir -U pip wheel setuptools && \ - pip install --no-cache-dir tensorflow torch==2.1.2+cpu torchaudio torchvision==0.16.2+cpu -f https://download.pytorch.org/whl/torch_stable.html && \ - pip install --no-cache-dir prompttools && \ - pip install --no-cache-dir txtai${COMPONENTS} && \ - python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')" COPY nlp/Gemfile . #TODO: Create gemspec for ferret