Adding classical-ml and data-analytics presets (#255)

Signed-off-by: Tyler Titsworth <[email protected]> Signed-off-by: tylertitsworth <[email protected]> Co-authored-by: Tyler Titsworth <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jafraustro <[email protected]>
intel · Aug 12, 2024 · 67f6d76 · 67f6d76
1 parent f9cad1c
commit 67f6d76
Show file tree

Hide file tree

Showing 18 changed files with 709 additions and 781 deletions.
diff --git a/.github/workflows/container-ci.yaml b/.github/workflows/container-ci.yaml
@@ -63,7 +63,7 @@ jobs:
   setup-build:
     outputs:
       matrix: ${{ steps.build-matrix.outputs.matrix }}
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@0d381219ddf674d61a7572ddd19d7941e271515c # v2.9.0
@@ -79,13 +79,14 @@ jobs:
   build-containers:
     needs: [setup-build]
     env: ${{ matrix }}
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
     strategy:
       matrix: ${{ fromJson(needs.setup-build.outputs.matrix) }}
       fail-fast: false
     outputs:
       group: ${{ steps.build-group.outputs.container-group }}
     steps:
+    - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
       if: ${{ !inputs.no_build }}
     - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
@@ -111,7 +112,7 @@ jobs:
   setup-scan:
     needs: [build-containers]
     if: ${{ github.event_name == 'pull_request' }}
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
     outputs:
       matrix: ${{ steps.scan-matrix.outputs.matrix }}
     steps:
@@ -164,7 +165,7 @@ jobs:
       ####################################################################################################
   setup-test:
     needs: [build-containers]
-    runs-on: ubuntu-latest # ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
     outputs:
       matrix: ${{ steps.test-matrix.outputs.matrix }}
     steps:

diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml
@@ -113,7 +113,7 @@ jobs:
         path: output.txt
         recreate: true
   status-check:
-    needs: [group-diff, pipeline-ci]
+    needs: [group-diff, pipeline-ci, merge-logs]
     runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
     if: always()
     steps:

diff --git a/preset/classical-ml/Dockerfile b/preset/classical-ml/Dockerfile
@@ -12,40 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 ARG BASE_IMAGE="ubuntu"
 ARG BASE_TAG="22.04"
 
-FROM ${BASE_IMAGE}:${BASE_TAG} as classical-ml-base
+FROM ${BASE_IMAGE}:${BASE_TAG} as classical-ml
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-# See http://bugs.python.org/issue19846
-
 ENV LANG=C.UTF-8
 
 SHELL ["/bin/bash", "-c"]
 
 RUN apt-get update -y && \
     apt-get install -y --no-install-recommends --fix-missing \
-    bzip2 \
-    ca-certificates \
-    diffutils \
-    gcc \
-    git \
-    gzip \
-    make \
-    patch \
-    rsync \
-    unzip \
-    wget \
-    xz-utils && \
+        bzip2 \
+        ca-certificates \
+        diffutils \
+        gcc \
+        git \
+        gzip \
+        make \
+        patch \
+        rsync \
+        unzip \
+        wget \
+        xz-utils && \
     rm -rf /var/lib/apt/lists/*
 
-FROM classical-ml-base as classical-ml-python
-
-# Setting up non-root directories
 RUN useradd --uid 1000 -d /home/dev -s /bin/bash -m dev
-# Set a password for the user (Optional)
 RUN echo 'dev:password' | chpasswd
 USER dev
 WORKDIR /home/dev
@@ -56,77 +51,76 @@ ARG PYTHON_VERSION
 ARG IDP_VERSION
 ARG INTEL_CHANNEL
 
-RUN wget --progress=dot:giga --no-check-certificate https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-${MINIFORGE_VERSION}.sh -O miniforge.sh && \
+RUN wget --progress=dot:giga --no-check-certificate https://github.com/conda-forge/miniforge/releases/latest/download/${MINIFORGE_VERSION}.sh -O miniforge.sh && \
     chmod +x miniforge.sh && \
     ./miniforge.sh -b -p "${CONDA_ROOT}" && \
     rm ./miniforge.sh && \
-    ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge3"  && \
+    ln -s "${CONDA_ROOT}" "${CONDA_ROOT}/../miniforge" && \
     export PATH="${CONDA_ROOT}/bin/:${PATH}" && \
-    conda update -y conda && \
-    conda config --add channels conda-forge && \
-    conda config --add channels https://software.repos.intel.com/python/conda/ && \
     conda init --all && \
     conda install -y \
-        'jupyterlab>=4.1.8' \
-        'notebook>=7.1.3' \
-        'jupyterhub>=4.1.5' \
-        'jupyter-server-proxy>=4.1.2' \
-        'mako>=1.2.2' \
-        'pyjwt>=2.4.0' \
-        'cryptography>=42.0.5' \
-        'nodejs>=20.12.2' \
-        'aiohttp>=3.9.4' \
+        'colorama==0.4.6' \
+        'conda==24.5.0' \
+        'jupyterhub==5.1.0' \
+        'jupyter-server-proxy==4.3.0' \
+        'mamba==1.5.8' \
+        'networkx==3.3' \
+        'notebook==7.2.1' \
+        'pip==24.0' \
+        'python==3.10.14' \
         'idna>=3.7' \
-        'oauthlib>=3.2.2' \
-        && \
-    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
-    conda clean -y --all
+        'requests>=2.32.0' \
+        'setuptools>=70.0.0' \
+        'tqdm>=4.66.3' \
+        'urllib3>=2.2.2' \
+        'nodejs==22.5.1' \
+    && \
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \
+    && \
+    conda clean -y --all \
+    && \
+    conda config --add channels ${INTEL_CHANNEL}
 
 ENV PATH ${CONDA_ROOT}/condabin:${CONDA_ROOT}/bin/:${PATH}
+RUN conda config --set pip_interop_enabled True
 
 ARG IDP_VERSION
+ARG DAAL4PY_VERSION
 ARG DPNP_VERSION
 ARG XGBOOST_VERSION
 ARG MODIN_VERSION
 ARG NUMPY_VERSION
 ARG SKLEARNEX_VERSION
 
-# Conda packages
-RUN conda create -yn classical-ml -c ${INTEL_CHANNEL} -c conda-forge \
-        dpnp=${DPNP_VERSION} \
-        numpy=${NUMPY_VERSION} \
-        python=${PYTHON_VERSION} \
-        scikit-learn-intelex==${SKLEARNEX_VERSION} \
-        xgboost=${XGBOOST_VERSION} \
-        modin-ray=${MODIN_VERSION} \
-        'python-dotenv>=1.0.1' \
-        'tqdm>=4.66.2' \
-        'matplotlib-base>=3.4.3' \
-        'threadpoolctl>=3.3.0' \
-        'ipython>=8.18.1' \
-        'ipykernel>=6.29.3' \
-        'kernda>=0.3.0' \
-        'protobuf>=4.24' \
-        'pillow>=10.2.0' \
-        'tornado>=6.3.3' && \
+RUN conda create -yn classical-ml \
+        "python=${PYTHON_VERSION}" \
+        "daal4py=${DAAL4PY_VERSION}" \
+        "dpnp=${DPNP_VERSION}" \
+        'ipykernel==6.29.5' \
+        'kernda==0.3.0' \
+        'matplotlib-base==3.8.4' \
+        "modin-ray=${MODIN_VERSION}" \
+        'python-dotenv==1.0.1' \
+        "scikit-learn-intelex=${SKLEARNEX_VERSION}" \
+        'tqdm==4.66.4' \
+        "xgboost=${XGBOOST_VERSION}" \
+        'idna>=3.7' \
+        'requests>=2.32.0' \
+        'setuptools>=70.0.0' \
+        'tqdm>=4.66.3' \
+        'urllib3>=2.2.2' \
+        && \
     conda clean -y --all
 
-
-
-# PyPI packages
 RUN conda run -n classical-ml python -m pip install --no-deps --no-cache-dir \
-    'dataset-librarian==1.0.4' \
-    'cloud-data-connector==1.0.3'
-
+    'dataset-librarian==1.0.4'
 
 ENV PYTHONSTARTUP=~/.patch_sklearn.py
 COPY base/.patch_sklearn.py ~/.patch_sklearn.py
 
 ENV PYTHONSTARTUP=/home/dev/.patch_sklearn.py
 COPY base/.patch_sklearn.py /home/dev/.patch_sklearn.py
 
-FROM classical-ml-python as classical-ml-jupyter
-
 EXPOSE 8888
 
 RUN mkdir -p ~/jupyter/ && chmod -R a+rwx ~/jupyter/ && \
@@ -136,10 +130,10 @@ WORKDIR /home/dev
 COPY --chown=dev notebooks /home/dev/jupyter
 COPY --chown=dev tests /home/dev/sample-tests
 
-RUN "${CONDA_ROOT}/envs/classical-ml/bin/python" -m ipykernel install --user --name classical-ml --display-name "Classical ML" && \
-    "${CONDA_ROOT}/envs/classical-ml/bin/kernda" -o -y "$HOME/.local/share/jupyter/kernels/$(echo classical-ml | sed -e 's/\(.*\)/\L\1/')/kernel.json" && \
-    "${CONDA_ROOT}/envs/classical-ml/bin/python" -m ipykernel.kernelspec --user && \
-    conda clean -y --all
+RUN KERNEL_DIR="${CONDA_ROOT}/share/jupyter/kernels/classical-ml" && \
+    conda run -n classical-ml python -m ipykernel install --prefix "$CONDA_ROOT" --name classical-ml --display-name "Classical ML" && \
+    conda run -n classical-ml kernda -o -y "$KERNEL_DIR/kernel.json" && \
+    conda run -n base jupyter kernelspec list
 
 CMD ["bash", "-c", "source activate classical-ml && jupyter lab --notebook-dir=~/jupyter --port 8888 --ip 0.0.0.0 --no-browser --allow-root"]
 

diff --git a/preset/classical-ml/docker-compose.yaml b/preset/classical-ml/docker-compose.yaml
@@ -15,35 +15,38 @@
 # -*- coding: utf-8 -*-
 #
 
+
 version: '3'
 services:
   classical-ml:
     build:
       args:
         BASE_IMAGE: ${BASE_IMAGE:-ubuntu}
         BASE_TAG: ${BASE_TAG:-22.04}
-        DPNP_VERSION: ${NUMBA_DPEX_VERSION:-0.14.0}
-        IDP_VERSION: ${IDP_VERSION:-2024.1.0}
+        DAAL4PY_VERSION: ${DAAL4PY_VERSION:-2024.5.0}
+        DPNP_VERSION: ${DPNP_VERSION:-0.15.0}
+        IDP_VERSION: ${IDP_VERSION:-2024.2}
         INTEL_CHANNEL: ${INTEL_CHANNEL:-https://software.repos.intel.com/python/conda/}
-        MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Linux-x86_64}
-        MODIN_VERSION: ${MODIN_VERSION:-0.26.1}
-        MPI_VERSION: ${MPI_VERSION:-2021.12.0}
-        NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.22.1}
+        MINIFORGE_VERSION: ${MINIFORGE_VERSION:-Miniforge3-Linux-x86_64}
+        MODIN_VERSION: ${MODIN_VERSION:-0.30.0}
+        MPI_VERSION: ${MPI_VERSION:-2021.13}
+        NUMBA_DPEX_VERSION: ${NUMBA_DPEX_VERSION:-0.23.0}
         NUMPY_VERSION: ${NUMPY_VERSION:-1.26.4}
-        PYTHON_VERSION: ${PYTHON_VERSION:-3.10}
-        SKLEARNEX_VERSION: ${SKLEARNEX_VERSION:-2024.2.0}
+        PYTHON_VERSION: ${PYTHON_VERSION:-3.9}
+        SKLEARNEX_VERSION: ${SKLEARNEX_VERSION:-2024.5.0}
         XGBOOST_VERSION: ${XGBOOST_VERSION:-2.0.3}
         http_proxy: ${http_proxy}
         https_proxy: ${https_proxy}
         no_proxy: ''
       context: .
+      target: classical-ml
       labels:
         docs: classical_ml
         org.opencontainers.image.title: "Intel® AI Tools Selector Preset Containers - Classical ML"
         org.opencontainers.base.name: "ubuntu:22.04"
         org.opencontainers.image.name: "intel/classical-ml"
-        org.opencontainers.image.version: 2024.1.0-py${PYTHON_VERSION:-3.10}
-        dependency.python: ${PYTHON_VERSION:-3.10}
+        org.opencontainers.image.version: 2024.2.0-py${PYTHON_VERSION:-3.9}
+        dependency.python: ${PYTHON_VERSION:-3.9}
         dependency.python.pip: requirements.txt
         dependency.apt.bzip2: true
         dependency.apt.ca-certificates: true
@@ -57,43 +60,36 @@ services:
         dependency.apt.unzip: true
         dependency.apt.wget: true
         dependency.apt.xz-utils: true
-        dependency.conda.jupyterlab: '>=4.1.8'
-        dependency.conda.notebook: '>=7.1.3'
-        dependency.conda.jupyterhub: '>=4.1.5'
-        dependency.conda.jupyter-server-proxy: '>=4.1.2'
-        dependency.conda.mako: '>=1.2.2'
-        dependency.conda.pyjwt: '>=2.4.0'
-        dependency.conda.cryptography: '>=42.0.5'
-        dependency.conda.nodejs: '>=20.12.2'
-        dependency.conda.aiohttp: '>=3.9.4'
-        dependency.conda.idna: '>=3.7'
-        dependency.conda.oauthlib: '>=3.2.2'
-        dependency.conda.dpnp: '>=0.14.0'
-        dependency.conda.numpy: '>=1.26.4'
-        dependency.conda.python: "=${PYTHON_VERSION:-3.10}"
-        dependency.conda.scikit-learn-intelex: '>=2024.2.0'
-        dependency.conda.xgboost: '>=2.0.3'
-        dependency.conda.modin-ray: '>=0.26.1'
-        dependency.conda.python-dotenv: '>=1.0.1'
-        dependency.conda.tqdm: '>=4.66.2'
-        dependency.conda.matplotlib-base: '>=3.4.3'
-        dependency.conda.dataset_librarian: '>=1.0.4'
-        dependency.conda.threadpoolctl: '>=3.3.0'
-        dependency.conda.ipython: '>=8.18.1'
-        dependency.conda.ipykernel: '>=6.29.3'
-        dependency.conda.kernda: '>=0.3.0'
-        dependency.conda.protobuf: '>=4.24'
-        dependency.conda.pillow: '>=10.2.0'
-        dependency.conda.tornado: '>=6.3.3'
-      target: classical-ml-jupyter
-    command: |
-      bash -c "conda run -n classical-ml python -c 'import sklearn; import xgboost; print(\"SciKit:\", sklearn.__version__, \" XGBoost:\",xgboost.__version__)' && \
-      conda run -n classical-ml python -c 'import modin.pandas as pd, modin.config as cfg; cfg.Engine.put(\"Ray\"); df = pd.DataFrame([1]);print(df+1)'"
-    image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-classical-ml-2024.1.0-py${PYTHON_VERSION:-3.10}
+        dependency.conda.colorama: '==0.4.6'
+        dependency.conda.conda: '==24.5.0'
+        dependency.conda.daal4py: '=2024.5.0'
+        dependency.conda.dpnp: '=0.15.0'
+        dependency.conda.ipykernel: '==6.29.5'
+        dependency.conda.jupyterhub: '==5.1.0'
+        dependency.conda.jupyter-server-proxy: '==4.3.0'
+        dependency.conda.kernda: '==0.3.0'
+        dependency.conda.mamba: '==1.5.8'
+        dependency.conda.matplotlib-base: '==3.8.4'
+        dependency.conda.modin-ray: '=0.30.0'
+        dependency.conda.networkx: '==3.3'
+        dependency.conda.notebook: '==7.2.1'
+        dependency.conda.pip: '==24.0'
+        dependency.conda.python: '==3.10.14'
+        dependency.conda.python-dotenv: '==1.0.1'
+        dependency.conda.scikit-learn-intelex: '=2024.5.0'
+        dependency.conda.tqdm: '==4.66.4'
+        dependency.conda.xgboost: '=2.0.3'
+    image: ${REGISTRY}/${REPO}:b-${GITHUB_RUN_NUMBER:-0}-classical-ml-${RELEASE:-2024.2.0}-py${PYTHON_VERSION:-3.9}
     environment:
       http_proxy: ${http_proxy}
       https_proxy: ${https_proxy}
     network_mode: host
     shm_size: 12GB
     volumes:
       - /dev/dri/by-path:/dev/dri/by-path
+    command: >
+      bash -c " conda run -n classical-ml python -c 'import sklearn;import xgboost;print(\"SciKit:\",
+      sklearn.__version__, \" XGBoost:\", xgboost.__version__)' &&
+
+      conda run -n classical-ml python -c 'import modin.pandas as pd;import modin.config
+      as cfg;cfg.Engine.put(\"Ray\");df = pd.DataFrame([1]);print(df+1)' "
diff --git a/preset/classical-ml/requirements.txt b/preset/classical-ml/requirements.txt
@@ -1 +1 @@
-cloud-data-connector==1.0.3
+dataset-librarian==1.0.4
diff --git a/preset/classical-ml/tests/scikit/kmeans.py b/preset/classical-ml/tests/scikit/kmeans.py
@@ -62,6 +62,7 @@
 
 data, labels = load_digits(return_X_y=True)
 (n_samples, n_features), n_digits = data.shape, np.unique(labels).size
+data = np.array(data, dtype=np.float64)
 
 print(f"# digits: {n_digits}; # samples: {n_samples}; # features {n_features}")
 

diff --git a/preset/classical-ml/tests/scikit/test_scikit.sh b/preset/classical-ml/tests/scikit/test_scikit.sh
@@ -14,8 +14,8 @@
 # limitations under the License.
 
 set -xe
+
 SCRIPT_DIR=$(dirname "$0")
 
 python "${SCRIPT_DIR}/kmeans.py"
-
-python "${SCRIPT_DIR}/kmeans.py" true
+python "${SCRIPT_DIR}/kmeans.py" true # Enable intel opt
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		cloud-data-connector==1.0.3
		dataset-librarian==1.0.4