diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index eef8a04067d..00000000000 --- a/Dockerfile +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. - -# An integration test & dev container which builds and installs cuDF from main -ARG CUDA_VERSION=11.0 -ARG CUDA_SHORT_VERSION=${CUDA_VERSION} -ARG LINUX_VERSION=ubuntu18.04 -FROM nvidia/cuda:${CUDA_VERSION}-devel-${LINUX_VERSION} -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/lib -ENV DEBIAN_FRONTEND=noninteractive - -ARG CC=9 -ARG CXX=9 -RUN apt update -y --fix-missing && \ - apt upgrade -y && \ - apt install -y --no-install-recommends software-properties-common && \ - add-apt-repository ppa:ubuntu-toolchain-r/test && \ - apt update -y --fix-missing && \ - apt install -y --no-install-recommends \ - git \ - gcc-${CC} \ - g++-${CXX} \ - tzdata && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install conda -ADD https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh /miniconda.sh -RUN sh /miniconda.sh -b -p /conda && /conda/bin/conda update -n base conda -ENV PATH=${PATH}:/conda/bin -# Enables "source activate conda" -SHELL ["/bin/bash", "-c"] - -# Build cuDF conda env -ARG CUDA_SHORT_VERSION -ARG PYTHON_VERSION -ENV PYTHON_VERSION=$PYTHON_VERSION -ARG NUMBA_VERSION -ENV NUMBA_VERSION=$NUMBA_VERSION -ARG NUMPY_VERSION -ENV NUMPY_VERSION=$NUMPY_VERSION -ARG PANDAS_VERSION -ENV PANDAS_VERSION=$PANDAS_VERSION -ARG PYARROW_VERSION -ENV PYARROW_VERSION=$PYARROW_VERSION -ARG CYTHON_VERSION -ENV CYTHON_VERSION=$CYTHON_VERSION -ARG CMAKE_VERSION -ENV CMAKE_VERSION=$CMAKE_VERSION -ARG CUDF_REPO=https://github.com/rapidsai/cudf -ENV CUDF_REPO=$CUDF_REPO -ARG CUDF_BRANCH=main -ENV CUDF_BRANCH=$CUDF_BRANCH - -# Add everything from the local build context -ADD . /cudf/ - -# Checks if local build context has the source, if not clone it then run a bash script to modify -# the environment file based on versions set in build args -RUN ls -la /cudf -RUN if [ -f /cudf/docker/package_versions.sh ]; \ - then /cudf/docker/package_versions.sh /cudf/conda/environments/cudf_dev_cuda${CUDA_SHORT_VERSION}.yml && \ - conda env create --name cudf --file /cudf/conda/environments/cudf_dev_cuda${CUDA_SHORT_VERSION}.yml ; \ - else rm -rf /cudf && \ - git clone --recurse-submodules -b ${CUDF_BRANCH} ${CUDF_REPO} /cudf && \ - /cudf/docker/package_versions.sh /cudf/conda/environments/cudf_dev_cuda${CUDA_SHORT_VERSION}.yml && \ - conda env create --name cudf --file /cudf/conda/environments/cudf_dev_cuda${CUDA_SHORT_VERSION}.yml ; \ - fi - -ENV CC=/opts/conda/envs/rapids/bin/gcc-${CC} -ENV CXX=/opts/conda/envs/rapids/bin/g++-${CXX} - -# libcudf & cudf build/install -RUN source activate cudf && \ - cd /cudf/ && \ - ./build.sh libcudf cudf diff --git a/docker/package_versions.sh b/docker/package_versions.sh deleted file mode 100755 index c558f66d511..00000000000 --- a/docker/package_versions.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Usage: -# "./package_versions.sh /cudf/conda/environments/cudf_dev.yml" - Updates package versions in file based on Docker build-args - -FILENAME=$1 - -set_version() { - sed -i "s/\- $1\([<>=][^a-zA-Z]*\)\?$/\- $1=$2/" $FILENAME -} - -replace_text() { - sed -i "s/$1/$2/" $FILENAME -} - -add_package() { - sed -i "s/\- $1\([<>=][^a-zA-Z]*\)\?$/a \- $2=$3/" $FILENAME -} - -if [ "$PYTHON_VERSION" ]; then - PACKAGE_NAME="python" - set_version "$PACKAGE_NAME" "$PYTHON_VERSION" -fi - -if [ "$NUMBA_VERSION" ]; then - PACKAGE_NAME="numba" - set_version "$PACKAGE_NAME" "$NUMBA_VERSION" -fi - -if [ "$PANDAS_VERSION" ]; then - PACKAGE_NAME="pandas" - set_version "$PACKAGE_NAME" "$PANDAS_VERSION" -fi - -if [ "$PYARROW_VERSION" ]; then - PACKAGE_NAME="pyarrow" - set_version "$PACKAGE_NAME" "$PYARROW_VERSION" -fi - -if [ "$CYTHON_VERSION" ]; then - PACKAGE_NAME="cython" - set_version "$PACKAGE_NAME" "$CYTHON_VERSION" -fi - -if [ "$CMAKE_VERSION" ]; then - PACKAGE_NAME="cmake" - set_version "$PACKAGE_NAME" "$CMAKE_VERSION" -fi - -if [ "$NUMPY_VERSION" ]; then - ABOVE_PACKAGE="pandas" - PACKAGE_NAME="numpy" - add_package "$ABOVE_PACKAGE" "$PACKAGE_NAME" "$NUMPY_VERSION" -fi diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile deleted file mode 100644 index 696a6969778..00000000000 --- a/docker_build/Dockerfile +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. - -ARG CUDA_VERSION=11.2.2 -FROM nvidia/cuda:${CUDA_VERSION}-devel -ENV CUDA_SHORT_VERSION=11.2 - -SHELL ["/bin/bash", "-c"] -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib - -ENV DEBIAN_FRONTEND=noninteractive - -ENV CUDA_HOME=/usr/local/cuda -ENV CUDA_PATH=$CUDA_HOME -ENV PATH=${CUDA_HOME}/lib64/:${PATH}:${CUDA_HOME}/bin - -# Build env variables for arrow -ENV CMAKE_BUILD_TYPE=release -ENV PYARROW_WITH_PARQUET=1 -ENV PYARROW_WITH_CUDA=1 -ENV PYARROW_WITH_ORC=1 -ENV PYARROW_WITH_DATASET=1 - -ENV ARROW_HOME=/repos/dist - -# Build env variables for rmm -ENV INSTALL_PREFIX=/usr - - -RUN apt update -y --fix-missing && \ - apt upgrade -y && \ - apt install -y --no-install-recommends software-properties-common && \ - add-apt-repository ppa:deadsnakes/ppa && \ - apt update -y --fix-missing - -RUN apt install -y --no-install-recommends \ - git \ - python3.8-dev \ - build-essential \ - autoconf \ - bison \ - flex \ - libjemalloc-dev \ - wget \ - libssl-dev \ - protobuf-compiler && \ - apt-get autoremove -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \ - wget https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py - -# Install cmake -RUN version=3.18 && build=5 && mkdir ~/temp && cd ~/temp && wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz && \ - tar -xzvf cmake-$version.$build.tar.gz && cd cmake-$version.$build/ && ./bootstrap && make -j$(nproc) && make install - -# Install arrow from source -RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/dist/ && cd /repos/arrow && git checkout apache-arrow-1.0.1 && git submodule init && \ - git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ - cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ - cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=OFF -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ - make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release bdist_wheel && pip install /repos/arrow/python/dist/*.whl - - -# Install rmm from source -RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd /repos/rmm/ && ./build.sh librmm && pip install /repos/rmm/python/. - -ADD . /repos/cudf/ - -# Build env for CUDF build -ENV CUDF_HOME=/repos/cudf/ -ENV CUDF_ROOT=/repos/cudf/cpp/build/ - -# Install cudf from source -RUN cd /repos/cudf/ && git submodule update --init --recursive && ./build.sh libcudf && \ - pip install /repos/cudf/python/cudf/. -