From 354655279223029472af6ac213ed1499e5137a32 Mon Sep 17 00:00:00 2001 From: Tim Liu Date: Tue, 14 May 2024 09:51:50 +0800 Subject: [PATCH] Drop Support for CentOS 7 (#10805) To fix: https://github.com/NVIDIA/spark-rapids/issues/10496 1, remove the CentOS Dockerfile. 2, merge the arm64 and amd64 Dockerfiles into one. Signed-off-by: Tim Liu --- jenkins/Dockerfile-blossom.integration.centos | 67 ----------------- jenkins/Dockerfile-blossom.multi | 73 ------------------- jenkins/Dockerfile-blossom.ubuntu | 46 ++++++------ 3 files changed, 22 insertions(+), 164 deletions(-) delete mode 100644 jenkins/Dockerfile-blossom.integration.centos delete mode 100644 jenkins/Dockerfile-blossom.multi diff --git a/jenkins/Dockerfile-blossom.integration.centos b/jenkins/Dockerfile-blossom.integration.centos deleted file mode 100644 index f1bb9dc16da..00000000000 --- a/jenkins/Dockerfile-blossom.integration.centos +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -### DEPRECATED: https://github.com/NVIDIA/spark-rapids/issues/8789 -# -# Arguments: -# CUDA_VER=11.0+ -# CENTOS_VER=7 or 8 -# CUDF_VER= -# URM_URL= -### - -ARG CUDA_VER=11.0 -ARG CENTOS_VER=7 -FROM nvidia/cuda:${CUDA_VER}-runtime-centos${CENTOS_VER} -ARG CENTOS_VER -ARG CUDA_VER -ARG CUDF_VER -ARG URM_URL - -# centOS 8 went EOL, use vault.centos.org to avoid "No URLs in mirrorlist" -RUN if [ "$CENTOS_VER" == "8" ]; then\ - sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* &&\ - sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-*;\ - fi -# Install jdk-8, jdk-11, maven, docker image -RUN yum update -y && \ - yum install epel-release -y && \ - yum install -y java-1.8.0-openjdk-devel java-11-openjdk-devel wget expect rsync zip unzip - -# The default mvn verision is 3.0.5 on centos7 docker container. -# The plugin: net.alchim31.maven requires a higher mvn version. -ENV MAVEN_HOME "/usr/local/apache-maven-3.6.3" -RUN wget ${URM_URL}/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.tar.gz -P /usr/local && \ - tar xzvf $MAVEN_HOME-bin.tar.gz -C /usr/local && \ - rm -f $MAVEN_HOME-bin.tar.gz - -RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ - /bin/bash ~/miniconda.sh -b -p /opt/conda && \ - rm -f ~/miniconda.sh -ENV PATH="/opt/conda/bin:$MAVEN_HOME/bin:${PATH}" -# 'pyarrow' and 'pandas' will be installed as the dependencies of cudf below -RUN export CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f 1,2` && \ - conda install -y -c conda-forge mamba=1.4.9 libarchive && \ - mamba install -y -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge -c defaults cudf=${CUDF_VER} python=3.9 cudatoolkit=${CUDA_VER} && \ - mamba install -y spacy && python -m spacy download en_core_web_sm && \ - mamba install -y -c anaconda pytest requests && \ - mamba install -y -c conda-forge sre_yield && \ - conda clean -ay -# install pytest plugins for xdist parallel run -RUN python -m pip install findspark pytest-xdist pytest-order - -# Set default java as 1.8.0 -ENV JAVA_HOME "/usr/lib/jvm/java-1.8.0-openjdk" diff --git a/jenkins/Dockerfile-blossom.multi b/jenkins/Dockerfile-blossom.multi deleted file mode 100644 index 3884c437ff7..00000000000 --- a/jenkins/Dockerfile-blossom.multi +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -### -# -# Dockerfile for multi-platform build -# -# Arguments Example: -# CUDA_VER=11.8.0 -# UBUNTU_VER=20.04 -# UCX_VER=1.15.0-rc6 -### - -ARG CUDA_VER=11.8.0 -ARG UBUNTU_VER=20.04 -ARG UCX_VER=1.16.0 -# multi-platform build with: docker buildx build --platform linux/arm64,linux/amd64 on either amd64 or arm64 host -# check available official arm-based docker images at https://hub.docker.com/r/nvidia/cuda/tags (OS/ARCH) -FROM --platform=$TARGETPLATFORM nvidia/cuda:${CUDA_VER}-runtime-ubuntu${UBUNTU_VER} -ARG CUDA_VER -ARG UBUNTU_VER -ARG UCX_VER -ARG ARCH=arm64 -ARG UCX_ARCH=aarch64 - -# Install jdk-8, jdk-11, maven, docker image -RUN apt-get update -y && \ - apt-get install -y software-properties-common rsync - -RUN add-apt-repository ppa:deadsnakes/ppa && \ - apt-get update -y && \ - DEBIAN_FRONTEND="noninteractive" apt-get install -y maven \ - openjdk-8-jdk openjdk-11-jdk python3.9 python3.9-distutils python3-setuptools tzdata git zip unzip wget - -# apt python3-pip would install pip for OS default python3 version only -# like for ubuntu 18.04, it would only install pip for python3.6 -# so we install pip for specific python version explicitly -RUN wget https://bootstrap.pypa.io/get-pip.py && python3.9 get-pip.py - -# Set default jdk as 1.8.0 -RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-${ARCH} -# export JAVA_HOME for mvn option 'source-javadoc' -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${ARCH} - -RUN ln -sfn /usr/bin/python3.9 /usr/bin/python -RUN ln -sfn /usr/bin/python3.9 /usr/bin/python3 -RUN python -m pip install pytest sre_yield requests pandas pyarrow findspark pytest-xdist pre-commit pytest-order - -# libnuma1 and libgomp1 are required by ucx packaging -RUN apt install -y inetutils-ping expect wget libnuma1 libgomp1 - -RUN UCX_CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f1` && \ - mkdir -p /tmp/ucx && cd /tmp/ucx && \ - wget https://github.com/openucx/ucx/releases/download/v${UCX_VER}/ucx-${UCX_VER}-ubuntu${UBUNTU_VER}-mofed5-cuda${UCX_CUDA_VER}-${UCX_ARCH}.tar.bz2 && \ - tar -xvf *.bz2 && dpkg -i *.deb && \ - rm -rf /tmp/ucx - -# install locale and generate UTF-8, but don't set it as default, the test cases should specify it somehow -RUN apt-get install -y locales -RUN locale-gen en_US.UTF-8 diff --git a/jenkins/Dockerfile-blossom.ubuntu b/jenkins/Dockerfile-blossom.ubuntu index 64b0f4f26a1..c2c2dabf293 100644 --- a/jenkins/Dockerfile-blossom.ubuntu +++ b/jenkins/Dockerfile-blossom.ubuntu @@ -19,52 +19,52 @@ # Build the image for rapids-plugin development environment # # Arguments: -# CUDA_VER=11.X.Y -# UBUNTU_VER=20.04 -# UCX_CUDA_VER=11 (major CUDA version) -# UCX_VER=1.13.1 +# CUDA_VER=[11.X.Y,12.X.Y] +# UBUNTU_VER=[20.04,22.0.4] +# UCX_VER=1.16.0 +# TARGETPLATFORM=[linux/amd64,linux/arm64] +# ARCH=[amd64,arm64] +# UCX_ARCH=[x86_64,aarch64] ### -ARG CUDA_VER=11.0.3 +ARG CUDA_VER=11.8.0 ARG UBUNTU_VER=20.04 ARG UCX_VER=1.16.0 -ARG UCX_CUDA_VER=11 -FROM nvidia/cuda:${CUDA_VER}-runtime-ubuntu${UBUNTU_VER} +ARG TARGETPLATFORM=linux/amd64 +# multi-platform build with: docker buildx build --platform linux/arm64,linux/amd64 on either amd64 or arm64 host +# check available official arm-based docker images at https://hub.docker.com/r/nvidia/cuda/tags (OS/ARCH) +FROM --platform=$TARGETPLATFORM nvidia/cuda:${CUDA_VER}-runtime-ubuntu${UBUNTU_VER} ARG CUDA_VER ARG UBUNTU_VER ARG UCX_VER -ARG UCX_CUDA_VER +ARG ARCH=amd64 ARG UCX_ARCH=x86_64 -# https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212771 -RUN UB_VER=$(echo ${UBUNTU_VER} | tr -d '.') && \ - apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${UB_VER}/x86_64/3bf863cc.pub || true - # Install jdk-8, jdk-11, maven, docker image RUN apt-get update -y && \ apt-get install -y software-properties-common rsync +# libnuma1 and libgomp1 are required by ucx packaging RUN add-apt-repository ppa:deadsnakes/ppa && \ apt-get update -y && \ DEBIAN_FRONTEND="noninteractive" apt-get install -y maven \ - openjdk-8-jdk openjdk-11-jdk python3.8 python3.8-distutils python3-setuptools tzdata git zip unzip wget + openjdk-8-jdk openjdk-11-jdk python3.9 python3.9-distutils python3-setuptools tzdata git zip unzip wget \ + inetutils-ping expect wget libnuma1 libgomp1 locales # apt python3-pip would install pip for OS default python3 version only # like for ubuntu 18.04, it would only install pip for python3.6 # so we install pip for specific python version explicitly -RUN wget https://bootstrap.pypa.io/get-pip.py && python3.8 get-pip.py +RUN wget https://bootstrap.pypa.io/get-pip.py && python3.9 get-pip.py # Set default jdk as 1.8.0 -RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64 +RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-${ARCH} -RUN ln -sfn /usr/bin/python3.8 /usr/bin/python -RUN ln -sfn /usr/bin/python3.8 /usr/bin/python3 +RUN ln -sfn /usr/bin/python3.9 /usr/bin/python +RUN ln -sfn /usr/bin/python3.9 /usr/bin/python3 RUN python -m pip install pytest sre_yield requests pandas pyarrow findspark pytest-xdist pre-commit pytest-order fastparquet==0.8.3 -# libnuma1 and libgomp1 are required by ucx packaging -RUN apt install -y inetutils-ping expect wget libnuma1 libgomp1 - -RUN mkdir -p /tmp/ucx && \ +RUN UCX_CUDA_VER=`echo ${CUDA_VER} | cut -d '.' -f1` && \ + mkdir -p /tmp/ucx && \ cd /tmp/ucx && \ wget https://github.com/openucx/ucx/releases/download/v${UCX_VER}/ucx-${UCX_VER}-ubuntu${UBUNTU_VER}-mofed5-cuda${UCX_CUDA_VER}-${UCX_ARCH}.tar.bz2 && \ tar -xvf *.bz2 && \ @@ -72,8 +72,6 @@ RUN mkdir -p /tmp/ucx && \ rm -rf /tmp/ucx # export JAVA_HOME for mvn option 'source-javadoc' -ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 - +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-${ARCH} # install locale and generate UTF-8, but don't set it as default, the test cases should specify it somehow -RUN apt-get install -y locales RUN locale-gen en_US.UTF-8