From f4d5bde7f4b3f6d412d952728d9c90b76f562e94 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 1 Apr 2021 12:32:41 -0500 Subject: [PATCH] Fix issues with building cudf in a non-conda environment (#7647) Extension to https://github.com/rapidsai/rmm/pull/733 Also resolves: https://github.com/rapidsai/cudf/issues/7578 This PR introduces fixes to `setup.py` in cudf build and also adds a DockerFile with which one can create a container with cudf fully built from source. Arrow & rmm will also be built from the source. This PR also contains a `requirements.txt` which has packages needed by cudf and that are available to install via pip. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Devavret Makkar (https://github.com/devavret) - Keith Kraus (https://github.com/kkraus14) - Christopher Harris (https://github.com/cwharris) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/7647 --- CONTRIBUTING.md | 2 +- conda/environments/cudf_dev_cuda10.1.yml | 3 +- conda/environments/cudf_dev_cuda10.2.yml | 3 +- conda/environments/cudf_dev_cuda11.0.yml | 3 +- conda/environments/cudf_dev_cuda11.1.yml | 67 +++++++++++++++ conda/environments/cudf_dev_cuda11.2.yml | 67 +++++++++++++++ cpp/cmake/Modules/StringifyJITHeaders.cmake | 4 + cpp/src/binaryop/binaryop.cpp | 4 +- docker_build/Dockerfile | 80 ++++++++++++++++++ python/cudf/pyproject.toml | 29 +++++++ .../cuda-10.1/dev_requirements.txt | 41 +++++++++ .../cuda-10.2/dev_requirements.txt | 41 +++++++++ .../cuda-11.0/dev_requirements.txt | 41 +++++++++ .../cuda-11.1/dev_requirements.txt | 41 +++++++++ .../cuda-11.2/dev_requirements.txt | 41 +++++++++ python/cudf/setup.cfg | 2 +- python/cudf/setup.py | 83 +++++++++++++++++-- python/cudf_kafka/dev_requirements.txt | 11 +++ python/cudf_kafka/pyproject.toml | 30 +++++++ python/cudf_kafka/setup.py | 22 +++-- python/custreamz/dev_requirements.txt | 12 +++ python/custreamz/pyproject.toml | 28 +++++++ python/custreamz/setup.py | 3 +- python/dask_cudf/dev_requirements.txt | 14 ++++ python/dask_cudf/pyproject.toml | 28 +++++++ python/dask_cudf/setup.py | 75 ++++++++++++++++- 26 files changed, 747 insertions(+), 28 deletions(-) create mode 100644 conda/environments/cudf_dev_cuda11.1.yml create mode 100644 conda/environments/cudf_dev_cuda11.2.yml create mode 100644 docker_build/Dockerfile create mode 100644 python/cudf/pyproject.toml create mode 100644 python/cudf/requirements/cuda-10.1/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-10.2/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-11.0/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-11.1/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-11.2/dev_requirements.txt create mode 100644 python/cudf_kafka/dev_requirements.txt create mode 100644 python/cudf_kafka/pyproject.toml create mode 100644 python/custreamz/dev_requirements.txt create mode 100644 python/custreamz/pyproject.toml create mode 100644 python/dask_cudf/dev_requirements.txt create mode 100644 python/dask_cudf/pyproject.toml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c332539ec7..98ecbcd3295 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -320,7 +320,7 @@ flag. Below is a list of the available arguments and their purpose: | `PYARROW_VERSION` | 1.0.1 | Not supported | set pyarrow version | | `CMAKE_VERSION` | newest | >=3.14 | set cmake version | | `CYTHON_VERSION` | 0.29 | Not supported | set Cython version | -| `PYTHON_VERSION` | 3.6 | 3.7 | set python version | +| `PYTHON_VERSION` | 3.7 | 3.8 | set python version | --- diff --git a/conda/environments/cudf_dev_cuda10.1.yml b/conda/environments/cudf_dev_cuda10.1.yml index fa0b1126190..2ec8af368f2 100644 --- a/conda/environments/cudf_dev_cuda10.1.yml +++ b/conda/environments/cudf_dev_cuda10.1.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49.0,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=10.1 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda10.2.yml b/conda/environments/cudf_dev_cuda10.2.yml index 52d82c4f4ef..64dcc458196 100644 --- a/conda/environments/cudf_dev_cuda10.2.yml +++ b/conda/environments/cudf_dev_cuda10.2.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=10.2 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 2e64365bdf6..f464738eaa8 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=11.0 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda11.1.yml b/conda/environments/cudf_dev_cuda11.1.yml new file mode 100644 index 00000000000..fae94974502 --- /dev/null +++ b/conda/environments/cudf_dev_cuda11.1.yml @@ -0,0 +1,67 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +name: cudf_dev +channels: + - rapidsai + - nvidia + - rapidsai-nightly + - conda-forge + - defaults +dependencies: + - clang=8.0.1 + - clang-tools=8.0.1 + - cupy>7.1.0,<9.0.0a0 + - rmm=0.19.* + - cmake>=3.14 + - cmake_setuptools>=0.1.3 + - python>=3.7,<3.9 + - numba>=0.49,!=0.51.0 + - numpy + - pandas>=1.0,<1.3.0dev0 + - pyarrow=1.0.1 + - fastavro>=0.22.9 + - notebook>=0.5.0 + - cython>=0.29,<0.30 + - fsspec>=0.6.0 + - pytest + - pytest-benchmark + - pytest-xdist + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-websupport + - nbsphinx + - numpydoc + - ipython + - recommonmark + - pandoc=<2.0.0 + - cudatoolkit=11.1 + - pip + - flake8=3.8.3 + - black=19.10 + - isort=5.0.7 + - mypy=0.782 + - typing_extensions + - pre_commit + - dask>=2021.3.1 + - distributed>=2.22.0 + - streamz + - dlpack + - arrow-cpp=1.0.1 + - arrow-cpp-proc * cuda + - boost-cpp>=1.72.0 + - double-conversion + - rapidjson + - flatbuffers + - hypothesis + - sphinx-markdown-tables + - sphinx-copybutton + - mimesis + - packaging + - protobuf + - nvtx>=0.2.1 + - cachetools + - pip: + - git+https://github.com/dask/dask.git@main + - git+https://github.com/dask/distributed.git@main + - git+https://github.com/python-streamz/streamz.git + - pyorc diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml new file mode 100644 index 00000000000..fa0a69ad1ea --- /dev/null +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -0,0 +1,67 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +name: cudf_dev +channels: + - rapidsai + - nvidia + - rapidsai-nightly + - conda-forge + - defaults +dependencies: + - clang=8.0.1 + - clang-tools=8.0.1 + - cupy>7.1.0,<9.0.0a0 + - rmm=0.19.* + - cmake>=3.14 + - cmake_setuptools>=0.1.3 + - python>=3.7,<3.9 + - numba>=0.49,!=0.51.0 + - numpy + - pandas>=1.0,<1.3.0dev0 + - pyarrow=1.0.1 + - fastavro>=0.22.9 + - notebook>=0.5.0 + - cython>=0.29,<0.30 + - fsspec>=0.6.0 + - pytest + - pytest-benchmark + - pytest-xdist + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-websupport + - nbsphinx + - numpydoc + - ipython + - recommonmark + - pandoc=<2.0.0 + - cudatoolkit=11.2 + - pip + - flake8=3.8.3 + - black=19.10 + - isort=5.0.7 + - mypy=0.782 + - typing_extensions + - pre_commit + - dask>=2021.3.1 + - distributed>=2.22.0 + - streamz + - dlpack + - arrow-cpp=1.0.1 + - arrow-cpp-proc * cuda + - boost-cpp>=1.72.0 + - double-conversion + - rapidjson + - flatbuffers + - hypothesis + - sphinx-markdown-tables + - sphinx-copybutton + - mimesis + - packaging + - protobuf + - nvtx>=0.2.1 + - cachetools + - pip: + - git+https://github.com/dask/dask.git@main + - git+https://github.com/dask/distributed.git@main + - git+https://github.com/python-streamz/streamz.git + - pyorc diff --git a/cpp/cmake/Modules/StringifyJITHeaders.cmake b/cpp/cmake/Modules/StringifyJITHeaders.cmake index 0bfb37773dc..d67c546cd30 100644 --- a/cpp/cmake/Modules/StringifyJITHeaders.cmake +++ b/cpp/cmake/Modules/StringifyJITHeaders.cmake @@ -34,6 +34,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit + ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/assert.cuh.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit @@ -65,6 +66,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include ${CUDF_SOURCE_DIR}/include/cudf/wrappers/timestamps.hpp ${CUDF_SOURCE_DIR}/include/cudf/fixed_point/fixed_point.hpp ${CUDF_SOURCE_DIR}/include/cudf/wrappers/durations.hpp + ${CUDF_SOURCE_DIR}/include/cudf/detail/utilities/assert.cuh ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/chrono ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/climits ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/cstddef @@ -99,6 +101,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include COMMAND ${CUDF_BINARY_DIR}/stringify cudf/wrappers/timestamps.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit COMMAND ${CUDF_BINARY_DIR}/stringify cudf/fixed_point/fixed_point.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit COMMAND ${CUDF_BINARY_DIR}/stringify cudf/wrappers/durations.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit + COMMAND ${CUDF_BINARY_DIR}/stringify cudf/detail/utilities/assert.cuh > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/assert.cuh.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/chrono cuda_std_chrono > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/climits cuda_std_climits > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/cstddef cuda_std_cstddef > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit @@ -133,6 +136,7 @@ add_custom_target(stringify_run DEPENDS ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit + ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/assert.cuh.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 6b5afa69300..9557670e967 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -86,7 +87,8 @@ const std::vector header_names{"operation.h", cudf_utilities_bit_hpp, cudf_wrappers_timestamps_hpp, cudf_wrappers_durations_hpp, - cudf_fixed_point_fixed_point_hpp}; + cudf_fixed_point_fixed_point_hpp, + cudf_detail_utilities_assert_cuh}; std::istream* headers_code(std::string filename, std::iostream& stream) { diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile new file mode 100644 index 00000000000..9216f15b419 --- /dev/null +++ b/docker_build/Dockerfile @@ -0,0 +1,80 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +ARG CUDA_VERSION=11.2.2 +FROM nvidia/cuda:${CUDA_VERSION}-devel +ENV CUDA_SHORT_VERSION=11.2 + +SHELL ["/bin/bash", "-c"] +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib + +ENV DEBIAN_FRONTEND=noninteractive + +ENV CUDA_HOME=/usr/local/cuda +ENV CUDA_PATH=$CUDA_HOME +ENV PATH=${CUDA_HOME}/lib64/:${PATH}:${CUDA_HOME}/bin + +# Build env variables for arrow +ENV CMAKE_BUILD_TYPE=release +ENV PYARROW_WITH_PARQUET=1 +ENV PYARROW_WITH_CUDA=1 +ENV PYARROW_WITH_ORC=1 +ENV PYARROW_WITH_DATASET=1 + +ENV ARROW_HOME=/repos/dist + +# Build env variables for rmm +ENV INSTALL_PREFIX=/usr + + +RUN apt update -y --fix-missing && \ + apt upgrade -y && \ + apt install -y --no-install-recommends software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt update -y --fix-missing + +RUN apt install -y --no-install-recommends \ + git \ + libboost-all-dev \ + python3.8-dev \ + build-essential \ + autoconf \ + bison \ + flex \ + libboost-filesystem-dev \ + libboost-system-dev \ + libboost-regex-dev \ + libjemalloc-dev \ + wget \ + libssl-dev \ + protobuf-compiler && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py + +# Install cmake +RUN version=3.18 && build=5 && mkdir ~/temp && cd ~/temp && wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz && \ + tar -xzvf cmake-$version.$build.tar.gz && cd cmake-$version.$build/ && ./bootstrap && make -j$(nproc) && make install + +# Install arrow from source +RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/dist/ && cd /repos/arrow && git checkout apache-arrow-1.0.1 && git submodule init && \ + git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ + cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ + cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=OFF -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ + make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release bdist_wheel && pip install /repos/arrow/python/dist/*.whl + + +# Install rmm from source +RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd /repos/rmm/ && ./build.sh librmm && pip install /repos/rmm/python/. + +ADD . /repos/cudf/ + +# Build env for CUDF build +ENV CUDF_ROOT=/repos/cudf/cpp/build/ + +# Install cudf from source +RUN cd /repos/cudf/ && git submodule update --init --recursive && ./build.sh libcudf && \ + pip install /repos/cudf/python/cudf/. + diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml new file mode 100644 index 00000000000..630efd5b9ec --- /dev/null +++ b/python/cudf/pyproject.toml @@ -0,0 +1,29 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", + "cython>=0.29,<0.30", +] + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/cudf/requirements/cuda-10.1/dev_requirements.txt b/python/cudf/requirements/cuda-10.1/dev_requirements.txt new file mode 100644 index 00000000000..967974d38b5 --- /dev/null +++ b/python/cudf/requirements/cuda-10.1/dev_requirements.txt @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + +cupy-cuda101 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.2/dev_requirements.txt b/python/cudf/requirements/cuda-10.2/dev_requirements.txt new file mode 100644 index 00000000000..34450456b5a --- /dev/null +++ b/python/cudf/requirements/cuda-10.2/dev_requirements.txt @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + +cupy-cuda102 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/dev_requirements.txt b/python/cudf/requirements/cuda-11.0/dev_requirements.txt new file mode 100644 index 00000000000..278b1a6bf61 --- /dev/null +++ b/python/cudf/requirements/cuda-11.0/dev_requirements.txt @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + +cupy-cuda110 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/dev_requirements.txt b/python/cudf/requirements/cuda-11.1/dev_requirements.txt new file mode 100644 index 00000000000..fafdc7d7d4f --- /dev/null +++ b/python/cudf/requirements/cuda-11.1/dev_requirements.txt @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + +cupy-cuda111 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/dev_requirements.txt b/python/cudf/requirements/cuda-11.2/dev_requirements.txt new file mode 100644 index 00000000000..db434b7c8ec --- /dev/null +++ b/python/cudf/requirements/cuda-11.2/dev_requirements.txt @@ -0,0 +1,41 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + +cupy-cuda112 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index 3067d2daafd..697a689801f 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2020, NVIDIA CORPORATION. +# Copyright (c) 2018-2021, NVIDIA CORPORATION. # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the diff --git a/python/cudf/setup.py b/python/cudf/setup.py index fdef940bc88..8e691b99d25 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -1,5 +1,7 @@ -# Copyright (c) 2018-2020, NVIDIA CORPORATION. +# Copyright (c) 2018-2021, NVIDIA CORPORATION. + import os +import re import shutil import subprocess import sys @@ -16,10 +18,56 @@ import versioneer -install_requires = ["numba", "cython"] +install_requires = [ + "numba>=0.49.0,!=0.51.0", + "Cython>=0.29,<0.30", + "fastavro>=0.22.9", + "fsspec>=0.6.0", + "numpy", + "pandas>=1.0,<1.3.0dev0", + "typing_extensions", + "protobuf", + "nvtx>=0.2.1", + "cachetools", + "packaging", +] + +extras_require = { + "test": [ + "pytest", + "pytest-benchmark", + "pytest-xdist", + "hypothesis" "mimesis", + "pyorc", + "msgpack", + ] +} cython_files = ["cudf/**/*.pyx"] + +def get_cuda_version_from_header(cuda_include_dir, delimeter=""): + + cuda_version = None + + with open( + os.path.join(cuda_include_dir, "cuda.h"), "r", encoding="utf-8" + ) as f: + for line in f.readlines(): + if re.search(r"#define CUDA_VERSION ", line) is not None: + cuda_version = line + break + + if cuda_version is None: + raise TypeError("CUDA_VERSION not found in cuda.h") + cuda_version = int(cuda_version.split()[2]) + return "%d%s%d" % ( + cuda_version // 1000, + delimeter, + (cuda_version % 1000) // 10, + ) + + CUDA_HOME = os.environ.get("CUDA_HOME", False) if not CUDA_HOME: path_to_cuda_gdb = shutil.which("cuda-gdb") @@ -36,8 +84,19 @@ raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") cuda_include_dir = os.path.join(CUDA_HOME, "include") +cuda_lib_dir = os.path.join(CUDA_HOME, "lib64") +install_requires.append( + "cupy-cuda" + get_cuda_version_from_header(cuda_include_dir) +) -CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") +CUDF_ROOT = os.environ.get( + "CUDF_ROOT", + os.path.abspath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + ) + ), +) try: nthreads = int(os.environ.get("PARALLEL_LEVEL", "0") or "0") @@ -102,10 +161,11 @@ def run(self): "*", sources=cython_files, include_dirs=[ - "../../cpp/include/cudf", - "../../cpp/include", - os.path.join(CUDF_ROOT, "include"), + os.path.abspath(os.path.join(CUDF_ROOT, "../include/cudf")), + os.path.abspath(os.path.join(CUDF_ROOT, "../include")), + os.path.abspath(os.path.join(CUDF_ROOT, "include")), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), + os.path.join(CUDF_ROOT, "_deps/dlpack-src/include"), os.path.join( os.path.dirname(sysconfig.get_path("include")), "libcudf/libcudacxx", @@ -117,9 +177,13 @@ def run(self): ], library_dirs=( pa.get_library_dirs() - + [get_python_lib(), os.path.join(os.sys.prefix, "lib")] + + [ + get_python_lib(), + os.path.join(os.sys.prefix, "lib"), + cuda_lib_dir, + ] ), - libraries=["cudf"] + pa.get_libraries() + ["arrow_cuda"], + libraries=["cudart", "cudf"] + pa.get_libraries() + ["arrow_cuda"], language="c++", extra_compile_args=["-std=c++14"], ) @@ -138,8 +202,8 @@ def run(self): "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], # Include the separately-compiled shared library setup_requires=["cython", "protobuf"], @@ -157,4 +221,5 @@ def run(self): cmdclass=cmdclass, install_requires=install_requires, zip_safe=False, + extras_require=extras_require, ) diff --git a/python/cudf_kafka/dev_requirements.txt b/python/cudf_kafka/dev_requirements.txt new file mode 100644 index 00000000000..e7f0d8edf99 --- /dev/null +++ b/python/cudf_kafka/dev_requirements.txt @@ -0,0 +1,11 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +python-confluent-kafka +pytest +setuptools +wheel +cython>=0.29,<0.30 +python-confluent-kafka \ No newline at end of file diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml new file mode 100644 index 00000000000..9855188ac6c --- /dev/null +++ b/python/cudf_kafka/pyproject.toml @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", + "Cython>=0.29,<0.30", +] + + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 290dcc036af..f7523dda503 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -32,7 +32,14 @@ cuda_include_dir = os.path.join(CUDA_HOME, "include") -CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") +CUDF_ROOT = os.environ.get( + "CUDF_ROOT", + os.path.abspath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + ) + ), +) CUDF_KAFKA_ROOT = os.environ.get( "CUDF_KAFKA_ROOT", "../../libcudf_kafka/build" ) @@ -47,9 +54,11 @@ "*", sources=cython_files, include_dirs=[ - "../../cpp/include/cudf", - "../../cpp/include", - "../../cpp/libcudf_kafka/include/cudf_kafka", + os.path.abspath(os.path.join(CUDF_ROOT, "../include/cudf")), + os.path.abspath(os.path.join(CUDF_ROOT, "../include")), + os.path.abspath( + os.path.join(CUDF_ROOT, "../libcudf_kafka/include/cudf_kafka") + ), os.path.join(CUDF_ROOT, "include"), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), os.path.join( @@ -81,11 +90,11 @@ "Topic :: Apache Kafka", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], # Include the separately-compiled shared library - setup_requires=["cython"], + setup_requires=["Cython>=0.29,<0.30"], ext_modules=cythonize( extensions, nthreads=nthreads, @@ -99,5 +108,6 @@ ), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, + extras_require={"test": ["pytest", "pytest-xdist"]}, zip_safe=False, ) diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt new file mode 100644 index 00000000000..7e300376f4e --- /dev/null +++ b/python/custreamz/dev_requirements.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +dask>=2021.3.1 +distributed>=2.22.0 +streamz +python-confluent-kafka +pytest +setuptools +wheel diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml new file mode 100644 index 00000000000..dfe475a2e46 --- /dev/null +++ b/python/custreamz/pyproject.toml @@ -0,0 +1,28 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", +] + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py index 976412f6e18..07a6b92f65d 100644 --- a/python/custreamz/setup.py +++ b/python/custreamz/setup.py @@ -20,11 +20,12 @@ "Topic :: Apache Kafka", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], packages=find_packages(include=["custreamz", "custreamz.*"]), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, zip_safe=False, + extras_require={"test": ["pytest", "pytest-xdist"]}, ) diff --git a/python/dask_cudf/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt new file mode 100644 index 00000000000..39da0da3012 --- /dev/null +++ b/python/dask_cudf/dev_requirements.txt @@ -0,0 +1,14 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +dask>=2021.3.1 +distributed>=2.22.0 +fsspec>=0.6.0 +numba>=0.49.0,!=0.51.0 +numpy +pandas>=1.0,<1.3.0dev0 +pytest +setuptools +wheel +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 \ No newline at end of file diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml new file mode 100644 index 00000000000..dfe475a2e46 --- /dev/null +++ b/python/dask_cudf/pyproject.toml @@ -0,0 +1,28 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", +] + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index d4809ff8f34..1318701b845 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,9 +1,77 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2019-2021, NVIDIA CORPORATION. + +import os +import re +import shutil + from setuptools import find_packages, setup import versioneer -install_requires = ["cudf", "dask", "distributed"] +install_requires = [ + "cudf", + "dask>=2021.3.1", + "distributed>=2.22.0", + "fsspec>=0.6.0", + "numpy", + "pandas>=1.0,<1.3.0dev0", +] + +extras_require = { + "test": [ + "numpy", + "pandas>=1.0,<1.3.0dev0", + "pytest", + "numba>=0.49.0,!=0.51.0", + "dask>=2021.3.1", + "distributed>=2.22.0", + ] +} + + +def get_cuda_version_from_header(cuda_include_dir, delimeter=""): + + cuda_version = None + + with open( + os.path.join(cuda_include_dir, "cuda.h"), "r", encoding="utf-8" + ) as f: + for line in f.readlines(): + if re.search(r"#define CUDA_VERSION ", line) is not None: + cuda_version = line + break + + if cuda_version is None: + raise TypeError("CUDA_VERSION not found in cuda.h") + cuda_version = int(cuda_version.split()[2]) + return "%d%s%d" % ( + cuda_version // 1000, + delimeter, + (cuda_version % 1000) // 10, + ) + + +CUDA_HOME = os.environ.get("CUDA_HOME", False) +if not CUDA_HOME: + path_to_cuda_gdb = shutil.which("cuda-gdb") + if path_to_cuda_gdb is None: + raise OSError( + "Could not locate CUDA. " + "Please set the environment variable " + "CUDA_HOME to the path to the CUDA installation " + "and try again." + ) + CUDA_HOME = os.path.dirname(os.path.dirname(path_to_cuda_gdb)) + +if not os.path.isdir(CUDA_HOME): + raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") + +cuda_include_dir = os.path.join(CUDA_HOME, "include") +cupy_package_name = "cupy-cuda" + get_cuda_version_from_header( + cuda_include_dir +) +install_requires.append(cupy_package_name) + setup( name="dask-cudf", @@ -18,10 +86,11 @@ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], packages=find_packages(exclude=["tests", "tests.*"]), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, + extras_require=extras_require, )