consoldating

.github/workflows/ci.yml

@@ -18,11 +18,12 @@ jobs:
     strategy:
       matrix:
         image_base_name:
-          - minimal
+          - full
+          # - minimal
           # - datascience
-          - llamaindex
-          - nlp
-          - audio
+          # - llamaindex
+          # - nlp
+          # - audio
 
     name: notebook-${{ matrix.image_base_name }}
     runs-on: ubuntu-latest

Rakefile

@@ -1,12 +1,9 @@
 OWNER = 'b08x'.freeze
 ALL_IMAGES = %w[
-  minimal
-  datascience
-  llamaindex
-  nlp
-  audio
+  full
 ].each(&:freeze).freeze
 
+
 BASE_IMAGES = ALL_IMAGES.map do |name|
   base_image_name, base_image_tag = nil
   IO.foreach("#{name}/Dockerfile") do |line|

full/Dockerfile

@@ -0,0 +1,158 @@
+ARG BASE_IMAGE_TAG=f3079808ca8c
+FROM jupyter/scipy-notebook:$BASE_IMAGE_TAG
+
+LABEL maintainer="Robert Pannick <[email protected]>"
+LABEL contributor="Kenta Murata <[email protected]>"
+
+USER root
+
+ARG GROUP=1041
+
+RUN groupadd -g $GROUP notebook && \
+    usermod -a -G sudo $NB_USER && usermod -a -G $GROUP $NB_USER && \
+    echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$NB_USER && \
+    chmod 0440 /etc/sudoers.d/99-$NB_USER
+
+# Pre-requisites
+RUN apt-get update && apt-get install -y -V --no-install-recommends dialog gnupg gpg-agent \
+    apt-transport-https gnupg lsb-release apt-utils apt-transport-https ca-certificates && \
+    echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal main' >> /etc/apt/sources.list && \
+    echo 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-11 main' >> /etc/apt/sources.list && \
+    wget -O llvm-snapshot.gpg.key http://llvm.org/apt/llvm-snapshot.gpg.key && \
+    apt-key add llvm-snapshot.gpg.key && \
+    apt-get update -qq && apt-get autoremove -y && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get update -qq && apt-get install -y -V --no-install-recommends \
+      cmake \
+      curl \
+      exiftool \
+      gcc \
+      ghostscript \
+      git \
+      git-lfs \
+      gnuplot \
+      graphviz \ 
+      jruby \
+      libcairo2-dev \
+      libczmq-dev \
+      libffi-dev \
+      libfftw3-dev \
+      libgdbm-dev \
+      libgirepository1.0-dev \
+      libglib2.0-dev \
+      libgmp-dev \
+      libgsl-dev \
+      liblapacke-dev \
+      libmagick++-dev \
+      libmysqlclient-dev \
+      libncurses5-dev \
+      libopenblas-dev \
+      libplot2c2 \
+      libpoppler-glib-dev \
+      libpq-dev \
+      libreadline-dev \
+      libreoffice \
+      libsqlite3-dev \
+      libssl-dev \
+      libtamuanova-0.2 \
+      libxml2-dev \
+      libxslt1-dev \
+      libyaml-dev \
+      libzmq3-dev \
+      minisat \
+      mysql-client \
+      neovim \
+      neovim \
+      openjdk-21-jdk-headless \
+      openssl \
+      pandoc \
+      pdftk \
+      pkg-config \
+      plotutils \
+      poppler-utils \
+      postgresql-client \
+      ruby-psych \
+      rsync \
+      software-properties-common \
+      sqlite3 \
+      tesseract-ocr \
+      tidy \
+      tzdata \
+      zip \     
+      zlib1g-dev \
+      wget && \
+   apt-get clean
+
+# Copy Ruby 3.1.3 from rubylang/ruby
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/bin/bundle \
+  /usr/local/bin/bundler \
+  /usr/local/bin/erb \
+  /usr/local/bin/gem \
+  /usr/local/bin/irb \
+  /usr/local/bin/racc \
+  /usr/local/bin/rake \
+  /usr/local/bin/rdoc \
+  /usr/local/bin/ri \
+  /usr/local/bin/ruby \
+  /usr/local/bin/
+
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/etc/gemrc \
+  /usr/local/etc/
+
+# NOTE: DO NOT CHANGE the version in the path of include directory
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/include/ruby-3.1.0/ \
+  /usr/local/include/ruby-3.1.0/
+
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/lib/libruby.so \
+  /usr/local/lib/libruby.so.* \
+  /usr/local/lib/
+
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/lib/pkgconfig/ \
+  /usr/local/lib/pkgconfig/
+
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/lib/ruby/ \
+  /usr/local/lib/ruby/
+
+COPY --from=rubylang/ruby:3.1.3-focal \
+  /usr/local/share/man/man1/erb.1 \
+  /usr/local/share/man/man1/irb.1 \
+  /usr/local/share/man/man1/ri.1 \
+  /usr/local/share/man/man1/ruby.1 \
+  /usr/local/share/man/man1/
+
+WORKDIR /home/$NB_USER
+
+RUN echo "[[ -f ~/.env ]] && source ~/.env" >> /home/$NB_USER/.bashrc && \
+    echo "gem: --user-install--no-document" >> /home/$NB_USER/.gemrc && \
+    echo "gem: --user-install --no-document" >> /etc/gemrc && \
+    echo "gem: --no-user-install --no-document" >> /root/.gemrc && \
+    gem update --system --platform x86_64-linux && gem clean
+
+COPY nlp/Gemfile .
+COPY gems/ferret-0.11.9.0.gem .
+
+RUN chown -R $NB_UID:100 /home/$NB_USER
+USER $NB_USER
+
+# NOTE: DO NOT CHANGE the version in the path of gem's bin directory
+ENV PATH $HOME/.local/share/gem/ruby/3.1.0/bin:$PATH
+ENV BUNDLE_PATH $HOME/.local/share/gem
+
+
+RUN pip install spacy && \
+    python3 -m spacy download en_core_web_trf && \
+    python3 -m spacy download en_core_web_lg && \
+    python -c "import sys, importlib.util as util; 1 if util.find_spec('nltk') else sys.exit(); import nltk; nltk.download('punkt')"
+
+RUN gem install ferret-0.11.9.0.gem && \
+    bundle lock --add-platform x86_64-linux && \
+    bundle config build.redic --with-cxx="clang++" --with-cflags="-std=c++0x" && \
+    bundle install && gem clean
+
+RUN iruby register --force

full/Gemfile

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Install basic gems
+gem 'irb', '>= 1.3.8.pre.9'
+gem 'iruby', '>= 0.7.4'
+gem 'pycall', '>= 1.5.1'
+
+gem 'activesupport'
+gem 'awesome_print'
+gem 'bcrypt_pbkdf'
+gem 'bundler'
+gem 'childprocess'
+gem 'colorize'
+gem 'coltrane'
+gem 'debug'
+gem 'dry-initializer'
+gem 'dry-monads'
+gem 'ed25519'
+gem 'eventmachine'
+gem 'ffi'
+gem 'fractional'
+gem 'geo_coord'
+gem 'hashie'
+gem 'highline'
+gem 'i18n'
+gem 'json'
+gem 'jsonl'
+gem 'kramdown'
+gem 'logging'
+gem 'minitest'
+gem 'mocha'
+gem 'multi_json'
+gem 'mysql2'
+gem 'net-ssh'
+gem 'open3'
+gem 'open4'
+gem 'parallel'
+gem 'pastel'
+gem 'pg'
+gem 'pry'
+gem 'pry-doc'
+gem 'psych'
+gem 'rake'
+gem 'rdoc'
+gem 'redcarpet'
+gem 'rexml'
+gem 'rouge'
+gem 'rugged'
+gem 'ruport'
+gem 'sequel'
+gem 'sqlite3'
+gem 'sync'
+gem 'sys-proctable'
+gem 'treetop'
+gem 'tty-box'
+gem 'tty-command'
+gem 'tty-config'
+gem 'tty-markdown'
+gem 'tty-prompt'
+gem 'tty-screen'
+gem 'yajl-ruby', require: 'yajl'
+gem 'yaml'
+
+gem 'charty', '>= 0.2.12'
+gem 'matplotlib', '>= 1.2.0'
+gem 'numpy', '>= 0.4.0'
+gem 'pandas', '>= 0.3.8'
+gem 'unicode_plot', '>= 0.0.5'
+
+gem 'daru'
+gem 'daru-view'
+gem 'enumerable-statistics'
+gem 'ffi-rzmq'
+gem 'numo-linalg'
+gem 'numo-narray'
+gem 'rumale'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gem 'nmatrix', git: 'https://github.com/b08x/nmatrix.git', branch: 'development'
+gem 'nmatrix-fftw', git: 'https://github.com/b08x/nmatrix.git', branch: 'development'
+gem 'nmatrix-lapacke', git: 'https://github.com/b08x/nmatrix.git', branch: 'development'
+gem 'rbplotly', git: 'https://github.com/b08x/rbplotly.git', branch: 'development'
+
+gem 'beckett'
+gem 'chroma-db'
+gem 'cli-ui'
+gem 'cohere'
+gem 'cohere-ruby'
+gem 'composable_operations'
+gem 'debug_me'
+gem 'decisiontree'
+gem 'docsplit'
+gem 'docx', '~> 0.8.0'
+gem 'dotenv'
+gem 'engtagger'
+gem 'epitome'
+gem 'fasttext'
+gem 'fuzzy_match'
+gem 'fuzzy-string-match'
+gem 'fuzzy_tools'
+gem 'gnuplot'
+gem 'google-cloud-ai_platform-v1'
+gem 'google_drive'
+gem 'google_search_results'
+gem 'graphr'
+gem 'hexapdf'
+gem 'hugging-face'
+gem 'jongleur'
+gem 'langchainrb'
+gem 'lemmatizer'
+gem 'lingua'
+gem 'loofah'
+# gem 'llm_memory'
+gem 'mimemagic'
+gem 'networkx'
+gem 'nokogiri'
+gem 'ohm'
+gem 'ohm-contrib'
+gem 'oj'
+gem 'ollama-ai'
+gem 'openssl'
+gem 'open-uri'
+gem 'optimist'
+gem 'osc-ruby'
+gem 'pdf_paradise'
+gem 'pdf-reader'
+gem 'pgvector'
+gem 'poppler'
+gem 'pragmatic_segmenter'
+gem 'pragmatic_tokenizer'
+gem 'rdatasets'
+gem 'redic'
+gem 'redis'
+gem 'roo'
+gem 'rover-df'
+gem 'rspec'
+gem 'rubydown'
+gem 'ruby-openai'
+gem 'ruby-spacy'
+gem 'sad_panda'
+gem 'safe_ruby'
+gem 'scalpel'
+gem 'socrates'
+gem 'standard'
+gem 'stream_lines'
+gem 'syntax_tree'
+gem 'tomoto'
+gem 'vega'
+gem 'verbal_expressions'
+gem 'wikipedia-client'
+gem 'wordnet'
+gem 'wordnet-defaultdb'
+
+gem 'distribution'
+gem 'sciruby'
+gem 'treat'
+gem 'zip'
+
+gem 'linguistics'
+gem 'pdf-textstream'
+
+#TODO:
+# source "https://rubygems.pkg.github.com/b08x" do
+#   gem "ferret"
+#   gem "nmatrix"
+#   gem 'nmatrix-lapacke'
+#   gem 'rbplotly'
+# end
+
+
+# gem "betty", git: "https://github.com/b08x/betty.git", branch: "main"
+# gem 'aoororachain'
+# gem 'google-cloud'
+# gem 'google_palm_api', '>= 0.1.3'
+# gem 'red-arrow', '11.0.0'
+# gem 'red-arrow-numo-narray'
+# gem 'red-chainer'
+# gem 'red-datasets', '>= 0.1.4'
+# gem 'red-datasets-arrow'
+# gem 'red-datasets-daru'
+# gem 'red-datasets-pandas'
+# gem 'red-gandiva', '11.0.0'
+# gem 'red-parquet', '11.0.0'
+# gem 'red-plasma'
+# gem 'red_amber', '0.4.2'
+# gem 'summarize'