From aa9459bda2aaf49288945db3a0b4bd2bbb78e546 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 18 Mar 2021 14:56:38 -0700 Subject: [PATCH 01/35] fix cudf build --- docker_build/Dockerfile | 78 ++++++++++++++++++++++++++++++++++++ python/cudf/requirements.txt | 44 ++++++++++++++++++++ python/cudf/setup.py | 1 + 3 files changed, 123 insertions(+) create mode 100644 docker_build/Dockerfile create mode 100644 python/cudf/requirements.txt diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile new file mode 100644 index 00000000000..7726d1d76dd --- /dev/null +++ b/docker_build/Dockerfile @@ -0,0 +1,78 @@ +# An integration test & dev container which builds and installs cuDF from main +ARG CUDA_VERSION=10.2 +ARG CUDA_SHORT_VERSION=${CUDA_VERSION} + +FROM nvidia/cuda:${CUDA_VERSION}-devel +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/lib +# Needed for cudf.concat(), avoids "OSError: library nvvm not found" +ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so +ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/ +# ENV DEBIAN_FRONTEND=noninteractive + +ENV CC=/usr/bin/gcc +ENV CXX=/usr/bin/g++ + +# Build env variables for arrow +ENV CMAKE_BUILD_TYPE=release +ENV PYARROW_WITH_PARQUET=1 +ENV PYARROW_WITH_CUDA=1 +ENV PYARROW_WITH_ORC=1 +ENV ARROW_HOME=/repos/dist +ENV LD_LIBRARY_PATH=/repos/dist/lib:$LD_LIBRARY_PATH + +# Build env variables for rmm +ENV INSTALL_PREFIX=/usr + + +RUN apt update -y --fix-missing && \ + apt upgrade -y && \ + apt install -y --no-install-recommends \ + git \ + libboost-all-dev \ + python3.7-dev \ + build-essential \ + cmake \ + software-properties-common \ + llvm-7 \ + autoconf \ + bison \ + flex \ + libboost-filesystem-dev \ + libboost-system-dev \ + libboost-regex-dev \ + libjemalloc-dev \ + wget \ + libssl-dev \ + protobuf-compiler \ + clang && \ + apt-get autoremove -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + add-apt-repository ppa:deadsnakes/ppa && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 && \ + apt-get install curl && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.7 get-pip.py + +# Install llvm 11.1 for apache +RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && tar -xvf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ + mv clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04 llvm-11.1 && cp -R llvm-11.1 /usr/lib/ && rm -rf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-get install llvm-10-dev llvm-9-dev -y && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install arrow from source +RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/dist/ && cd /repos/arrow && git checkout apache-arrow-1.0.1 && git submodule init && \ + git submodule && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ + cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ + pip install -r /repos/arrow/python/requirements-test.txt && cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ + make -j6 && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && pip install /repos/arrow/python/dist/*.whl + + +# Install rmm from source +RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd rmm/ && pip install clang==8.0.1 && pip install cmake==3.18 && pip install numba && ./build.sh + +ADD . /repos/cudf/ +# Install cudf from source +RUN cd /repos/cudf/ && git submodule update --init --recursive && ./build.sh && pip install -r python/cudf/requirements.txt --ignore-installed diff --git a/python/cudf/requirements.txt b/python/cudf/requirements.txt new file mode 100644 index 00000000000..f8f5c5f039e --- /dev/null +++ b/python/cudf/requirements.txt @@ -0,0 +1,44 @@ +clang==8.0.1 +cupy-cuda102 +cmake +wheel==0.34.2 +cmake-setuptools +numpy +pandas +fastavro +notebook +cython +fsspec +pytest +pytest-benchmark +pytest-xdist +sphinx +sphinx_rtd_theme +sphinxcontrib-websupport +nbsphinx +numpydoc +ipython +recommonmark +pandoc +partd +flake8==3.8.3 +black +isort==5.0.7 +mypy==0.782 +typing_extensions +pre_commit +dask +distributed +streamz +dlpack +rapidjson +flatbuffers +hypothesis +sphinx-markdown-tables +sphinx-copybutton +mimesis +packaging +protobuf +nvtx +cachetools +numba \ No newline at end of file diff --git a/python/cudf/setup.py b/python/cudf/setup.py index fdef940bc88..101ea135ebe 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -106,6 +106,7 @@ def run(self): "../../cpp/include", os.path.join(CUDF_ROOT, "include"), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), + os.path.join(CUDF_ROOT, "_deps/dlpack-src/include"), os.path.join( os.path.dirname(sysconfig.get_path("include")), "libcudf/libcudacxx", From a7dba6b0f0bb009ab1eca838faeca9c5b025d248 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 18 Mar 2021 18:38:42 -0700 Subject: [PATCH 02/35] add non interactive env --- docker_build/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 7726d1d76dd..d780ecb84fd 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -7,7 +7,7 @@ ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/lib # Needed for cudf.concat(), avoids "OSError: library nvvm not found" ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/ -# ENV DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive ENV CC=/usr/bin/gcc ENV CXX=/usr/bin/g++ @@ -17,6 +17,8 @@ ENV CMAKE_BUILD_TYPE=release ENV PYARROW_WITH_PARQUET=1 ENV PYARROW_WITH_CUDA=1 ENV PYARROW_WITH_ORC=1 +ENV PYARROW_WITH_DATASET=1 + ENV ARROW_HOME=/repos/dist ENV LD_LIBRARY_PATH=/repos/dist/lib:$LD_LIBRARY_PATH From 8f8874261c635c89bb1e3dd949f1404046f1b252 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 22 Mar 2021 13:22:17 -0700 Subject: [PATCH 03/35] fix nvrtc jitify issue --- cpp/cmake/Modules/StringifyJITHeaders.cmake | 4 ++++ cpp/src/binaryop/binaryop.cpp | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/Modules/StringifyJITHeaders.cmake b/cpp/cmake/Modules/StringifyJITHeaders.cmake index 0bfb37773dc..fb7ace05fa2 100644 --- a/cpp/cmake/Modules/StringifyJITHeaders.cmake +++ b/cpp/cmake/Modules/StringifyJITHeaders.cmake @@ -34,6 +34,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit + ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/release_assert.cuh.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit @@ -65,6 +66,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include ${CUDF_SOURCE_DIR}/include/cudf/wrappers/timestamps.hpp ${CUDF_SOURCE_DIR}/include/cudf/fixed_point/fixed_point.hpp ${CUDF_SOURCE_DIR}/include/cudf/wrappers/durations.hpp + ${CUDF_SOURCE_DIR}/include/cudf/detail/utilities/release_assert.cuh ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/chrono ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/climits ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/cstddef @@ -99,6 +101,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include COMMAND ${CUDF_BINARY_DIR}/stringify cudf/wrappers/timestamps.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit COMMAND ${CUDF_BINARY_DIR}/stringify cudf/fixed_point/fixed_point.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit COMMAND ${CUDF_BINARY_DIR}/stringify cudf/wrappers/durations.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit + COMMAND ${CUDF_BINARY_DIR}/stringify cudf/detail/utilities/release_assert.cuh > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/release_assert.cuh.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/chrono cuda_std_chrono > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/climits cuda_std_climits > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/cstddef cuda_std_cstddef > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit @@ -133,6 +136,7 @@ add_custom_target(stringify_run DEPENDS ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit + ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/release_assert.cuh.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 6b5afa69300..6097436da35 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -86,7 +87,8 @@ const std::vector header_names{"operation.h", cudf_utilities_bit_hpp, cudf_wrappers_timestamps_hpp, cudf_wrappers_durations_hpp, - cudf_fixed_point_fixed_point_hpp}; + cudf_fixed_point_fixed_point_hpp, + cudf_detail_utilities_release_assert_cuh}; std::istream* headers_code(std::string filename, std::iostream& stream) { From a3701cfb5706dd63f0e3761bf4f85cdbb3c020bf Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 23 Mar 2021 08:35:35 -0700 Subject: [PATCH 04/35] sync with update changes --- cpp/cmake/Modules/StringifyJITHeaders.cmake | 8 ++++---- cpp/src/binaryop/binaryop.cpp | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/cmake/Modules/StringifyJITHeaders.cmake b/cpp/cmake/Modules/StringifyJITHeaders.cmake index fb7ace05fa2..d67c546cd30 100644 --- a/cpp/cmake/Modules/StringifyJITHeaders.cmake +++ b/cpp/cmake/Modules/StringifyJITHeaders.cmake @@ -34,7 +34,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit - ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/release_assert.cuh.jit + ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/assert.cuh.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit @@ -66,7 +66,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include ${CUDF_SOURCE_DIR}/include/cudf/wrappers/timestamps.hpp ${CUDF_SOURCE_DIR}/include/cudf/fixed_point/fixed_point.hpp ${CUDF_SOURCE_DIR}/include/cudf/wrappers/durations.hpp - ${CUDF_SOURCE_DIR}/include/cudf/detail/utilities/release_assert.cuh + ${CUDF_SOURCE_DIR}/include/cudf/detail/utilities/assert.cuh ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/chrono ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/climits ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/cstddef @@ -101,7 +101,7 @@ add_custom_command(WORKING_DIRECTORY ${CUDF_SOURCE_DIR}/include COMMAND ${CUDF_BINARY_DIR}/stringify cudf/wrappers/timestamps.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit COMMAND ${CUDF_BINARY_DIR}/stringify cudf/fixed_point/fixed_point.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit COMMAND ${CUDF_BINARY_DIR}/stringify cudf/wrappers/durations.hpp > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit - COMMAND ${CUDF_BINARY_DIR}/stringify cudf/detail/utilities/release_assert.cuh > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/release_assert.cuh.jit + COMMAND ${CUDF_BINARY_DIR}/stringify cudf/detail/utilities/assert.cuh > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/assert.cuh.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/chrono cuda_std_chrono > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/climits cuda_std_climits > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit COMMAND ${CUDF_BINARY_DIR}/stringify ${LIBCUDACXX_INCLUDE_DIR}/cuda/std/cstddef cuda_std_cstddef > ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit @@ -136,7 +136,7 @@ add_custom_target(stringify_run DEPENDS ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/timestamps.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/fixed_point.hpp.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/durations.hpp.jit - ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/release_assert.cuh.jit + ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/assert.cuh.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/chrono.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/climits.jit ${CUDF_GENERATED_INCLUDE_DIR}/include/jit/libcudacxx/cuda/std/cstddef.jit diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index 6097436da35..9557670e967 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -25,11 +25,11 @@ #include #include +#include #include #include #include #include -#include #include #include @@ -88,7 +88,7 @@ const std::vector header_names{"operation.h", cudf_wrappers_timestamps_hpp, cudf_wrappers_durations_hpp, cudf_fixed_point_fixed_point_hpp, - cudf_detail_utilities_release_assert_cuh}; + cudf_detail_utilities_assert_cuh}; std::istream* headers_code(std::string filename, std::iostream& stream) { From 646061d5d97e9daab2f34217975853f7821e1c36 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 23 Mar 2021 18:41:34 -0700 Subject: [PATCH 05/35] add cuda lib dir --- python/cudf/setup.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 101ea135ebe..a326f26a860 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -36,6 +36,7 @@ raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") cuda_include_dir = os.path.join(CUDA_HOME, "include") +cuda_lib_dir = os.path.join(CUDA_HOME, "lib64") CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") @@ -118,9 +119,13 @@ def run(self): ], library_dirs=( pa.get_library_dirs() - + [get_python_lib(), os.path.join(os.sys.prefix, "lib")] + + [ + get_python_lib(), + os.path.join(os.sys.prefix, "lib"), + cuda_lib_dir, + ] ), - libraries=["cudf"] + pa.get_libraries() + ["arrow_cuda"], + libraries=["cudart", "cudf"] + pa.get_libraries() + ["arrow_cuda"], language="c++", extra_compile_args=["-std=c++14"], ) From 4bcecfa49228669ccbdfd49502fad1751c0a92b1 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 23 Mar 2021 18:47:30 -0700 Subject: [PATCH 06/35] remove 3.6 python usages --- CONTRIBUTING.md | 2 +- python/cudf/setup.py | 2 +- python/cudf_kafka/setup.py | 2 +- python/custreamz/setup.py | 2 +- python/dask_cudf/setup.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c332539ec7..98ecbcd3295 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -320,7 +320,7 @@ flag. Below is a list of the available arguments and their purpose: | `PYARROW_VERSION` | 1.0.1 | Not supported | set pyarrow version | | `CMAKE_VERSION` | newest | >=3.14 | set cmake version | | `CYTHON_VERSION` | 0.29 | Not supported | set Cython version | -| `PYTHON_VERSION` | 3.6 | 3.7 | set python version | +| `PYTHON_VERSION` | 3.7 | 3.8 | set python version | --- diff --git a/python/cudf/setup.py b/python/cudf/setup.py index a326f26a860..023caa0ef7c 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -144,8 +144,8 @@ def run(self): "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], # Include the separately-compiled shared library setup_requires=["cython", "protobuf"], diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 290dcc036af..536e529e169 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -81,8 +81,8 @@ "Topic :: Apache Kafka", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], # Include the separately-compiled shared library setup_requires=["cython"], diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py index 976412f6e18..c321e7ba17d 100644 --- a/python/custreamz/setup.py +++ b/python/custreamz/setup.py @@ -20,8 +20,8 @@ "Topic :: Apache Kafka", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], packages=find_packages(include=["custreamz", "custreamz.*"]), cmdclass=versioneer.get_cmdclass(), diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index d4809ff8f34..7ae816d3a52 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -18,8 +18,8 @@ "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], packages=find_packages(exclude=["tests", "tests.*"]), cmdclass=versioneer.get_cmdclass(), From 5ec449c16701ba0455ef0e9a54dc7e3586aa1055 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 23 Mar 2021 18:51:19 -0700 Subject: [PATCH 07/35] make changes to docker build --- docker_build/Dockerfile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index d780ecb84fd..04f5e377d2c 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -3,7 +3,8 @@ ARG CUDA_VERSION=10.2 ARG CUDA_SHORT_VERSION=${CUDA_VERSION} FROM nvidia/cuda:${CUDA_VERSION}-devel -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/lib +SHELL ["/bin/bash", "-c"] +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib # Needed for cudf.concat(), avoids "OSError: library nvvm not found" ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/ @@ -11,6 +12,9 @@ ENV DEBIAN_FRONTEND=noninteractive ENV CC=/usr/bin/gcc ENV CXX=/usr/bin/g++ +ENV CUDA_HOME=/usr/local/cuda-10.2 +ENV CUDA_PATH=/usr/local/cuda-10.2 +ENV PATH=/usr/local/cuda-10.2/lib64/:${PATH}:${CUDA_HOME}/bin # Build env variables for arrow ENV CMAKE_BUILD_TYPE=release @@ -20,7 +24,6 @@ ENV PYARROW_WITH_ORC=1 ENV PYARROW_WITH_DATASET=1 ENV ARROW_HOME=/repos/dist -ENV LD_LIBRARY_PATH=/repos/dist/lib:$LD_LIBRARY_PATH # Build env variables for rmm ENV INSTALL_PREFIX=/usr @@ -66,15 +69,19 @@ RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/c # Install arrow from source RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/dist/ && cd /repos/arrow && git checkout apache-arrow-1.0.1 && git submodule init && \ - git submodule && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ + git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ pip install -r /repos/arrow/python/requirements-test.txt && cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ make -j6 && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && pip install /repos/arrow/python/dist/*.whl # Install rmm from source -RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd rmm/ && pip install clang==8.0.1 && pip install cmake==3.18 && pip install numba && ./build.sh +RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && \ + cd rmm/ && pip install clang==8.0.1 && pip install cmake==3.18 && pip install numba && ./build.sh ADD . /repos/cudf/ + # Install cudf from source -RUN cd /repos/cudf/ && git submodule update --init --recursive && ./build.sh && pip install -r python/cudf/requirements.txt --ignore-installed +RUN cd /repos/cudf/ && git submodule update --init --recursive && pip install -r python/cudf/requirements.txt --ignore-installed + +RUN cd /repos/cudf/ && ./build.sh clean && ./build.sh From 22303df5fe697bf2696e27ab46ee77de0b29fda0 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 26 Mar 2021 21:55:16 -0700 Subject: [PATCH 08/35] changes to build --- conda/environments/cudf_dev_cuda11.1.yml | 68 +++++++++++++++++++ conda/environments/cudf_dev_cuda11.2.yml | 68 +++++++++++++++++++ docker_build/Dockerfile | 41 ++++++----- .../cuda-10.1/requirements-build.txt | 39 +++++++++++ .../cuda-10.2/requirements-build.txt} | 29 ++++---- .../cuda-11.0/requirements-build.txt | 39 +++++++++++ .../cuda-11.1/requirements-build.txt | 39 +++++++++++ .../cuda-11.2/requirements-build.txt | 39 +++++++++++ .../cudf/requirements/requirements-test.txt | 6 ++ 9 files changed, 333 insertions(+), 35 deletions(-) create mode 100644 conda/environments/cudf_dev_cuda11.1.yml create mode 100644 conda/environments/cudf_dev_cuda11.2.yml create mode 100644 python/cudf/requirements/cuda-10.1/requirements-build.txt rename python/cudf/{requirements.txt => requirements/cuda-10.2/requirements-build.txt} (60%) create mode 100644 python/cudf/requirements/cuda-11.0/requirements-build.txt create mode 100644 python/cudf/requirements/cuda-11.1/requirements-build.txt create mode 100644 python/cudf/requirements/cuda-11.2/requirements-build.txt create mode 100644 python/cudf/requirements/requirements-test.txt diff --git a/conda/environments/cudf_dev_cuda11.1.yml b/conda/environments/cudf_dev_cuda11.1.yml new file mode 100644 index 00000000000..887022a28ed --- /dev/null +++ b/conda/environments/cudf_dev_cuda11.1.yml @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +name: cudf_dev +channels: + - rapidsai + - nvidia + - rapidsai-nightly + - conda-forge + - defaults +dependencies: + - clang=8.0.1 + - clang-tools=8.0.1 + - cupy>7.1.0,<9.0.0a0 + - rmm=0.19.* + - cmake>=3.14 + - cmake_setuptools>=0.1.3 + - python>=3.6,<3.8 + - numba>=0.49,!=0.51.0 + - numpy + - pandas>=1.0,<1.3.0dev0 + - pyarrow=1.0.1 + - fastavro>=0.22.9 + - notebook>=0.5.0 + - cython>=0.29,<0.30 + - fsspec>=0.6.0 + - pytest + - pytest-benchmark + - pytest-xdist + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-websupport + - nbsphinx + - numpydoc + - ipython + - recommonmark + - pandoc=<2.0.0 + - cudatoolkit=11.1 + - pip + - partd + - flake8=3.8.3 + - black=19.10 + - isort=5.0.7 + - mypy=0.782 + - typing_extensions + - pre_commit + - dask>=2.22.0 + - distributed>=2.22.0 + - streamz + - dlpack + - arrow-cpp=1.0.1 + - arrow-cpp-proc * cuda + - boost-cpp>=1.72.0 + - double-conversion + - rapidjson + - flatbuffers + - hypothesis + - sphinx-markdown-tables + - sphinx-copybutton + - mimesis + - packaging + - protobuf + - nvtx>=0.2.1 + - cachetools + - pip: + - git+https://github.com/dask/dask.git@main + - git+https://github.com/dask/distributed.git@main + - git+https://github.com/python-streamz/streamz.git + - pyorc diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml new file mode 100644 index 00000000000..620a43ebdcc --- /dev/null +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -0,0 +1,68 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +name: cudf_dev +channels: + - rapidsai + - nvidia + - rapidsai-nightly + - conda-forge + - defaults +dependencies: + - clang=8.0.1 + - clang-tools=8.0.1 + - cupy>7.1.0,<9.0.0a0 + - rmm=0.19.* + - cmake>=3.14 + - cmake_setuptools>=0.1.3 + - python>=3.6,<3.8 + - numba>=0.49,!=0.51.0 + - numpy + - pandas>=1.0,<1.3.0dev0 + - pyarrow=1.0.1 + - fastavro>=0.22.9 + - notebook>=0.5.0 + - cython>=0.29,<0.30 + - fsspec>=0.6.0 + - pytest + - pytest-benchmark + - pytest-xdist + - sphinx + - sphinx_rtd_theme + - sphinxcontrib-websupport + - nbsphinx + - numpydoc + - ipython + - recommonmark + - pandoc=<2.0.0 + - cudatoolkit=11.2 + - pip + - partd + - flake8=3.8.3 + - black=19.10 + - isort=5.0.7 + - mypy=0.782 + - typing_extensions + - pre_commit + - dask>=2.22.0 + - distributed>=2.22.0 + - streamz + - dlpack + - arrow-cpp=1.0.1 + - arrow-cpp-proc * cuda + - boost-cpp>=1.72.0 + - double-conversion + - rapidjson + - flatbuffers + - hypothesis + - sphinx-markdown-tables + - sphinx-copybutton + - mimesis + - packaging + - protobuf + - nvtx>=0.2.1 + - cachetools + - pip: + - git+https://github.com/dask/dask.git@main + - git+https://github.com/dask/distributed.git@main + - git+https://github.com/python-streamz/streamz.git + - pyorc diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 04f5e377d2c..18f9577701e 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -1,8 +1,8 @@ # An integration test & dev container which builds and installs cuDF from main ARG CUDA_VERSION=10.2 -ARG CUDA_SHORT_VERSION=${CUDA_VERSION} - FROM nvidia/cuda:${CUDA_VERSION}-devel +ENV CUDA_SHORT_VERSION=10.2 + SHELL ["/bin/bash", "-c"] ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib # Needed for cudf.concat(), avoids "OSError: library nvvm not found" @@ -12,9 +12,9 @@ ENV DEBIAN_FRONTEND=noninteractive ENV CC=/usr/bin/gcc ENV CXX=/usr/bin/g++ -ENV CUDA_HOME=/usr/local/cuda-10.2 -ENV CUDA_PATH=/usr/local/cuda-10.2 -ENV PATH=/usr/local/cuda-10.2/lib64/:${PATH}:${CUDA_HOME}/bin +ENV CUDA_HOME=/usr/local/cuda +ENV CUDA_PATH=$CUDA_HOME +ENV PATH=${CUDA_HOME}/lib64/:${PATH}:${CUDA_HOME}/bin # Build env variables for arrow ENV CMAKE_BUILD_TYPE=release @@ -31,13 +31,15 @@ ENV INSTALL_PREFIX=/usr RUN apt update -y --fix-missing && \ apt upgrade -y && \ - apt install -y --no-install-recommends \ + apt install -y --no-install-recommends software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt update -y --fix-missing + +RUN apt install -y --no-install-recommends \ git \ libboost-all-dev \ - python3.7-dev \ + python3.8-dev \ build-essential \ - cmake \ - software-properties-common \ llvm-7 \ autoconf \ bison \ @@ -53,16 +55,19 @@ RUN apt update -y --fix-missing && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - add-apt-repository ppa:deadsnakes/ppa && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 && \ - apt-get install curl && \ - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ - python3.7 get-pip.py + update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1 && \ + wget https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py + +# Install cmake +RUN version=3.18 && build=5 && mkdir ~/temp && cd ~/temp && wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz && \ + tar -xzvf cmake-$version.$build.tar.gz && cd cmake-$version.$build/ && ./bootstrap && make -j$(nproc) && make install # Install llvm 11.1 for apache RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && tar -xvf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ mv clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04 llvm-11.1 && cp -R llvm-11.1 /usr/lib/ && rm -rf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-get update && \ apt-get install llvm-10-dev llvm-9-dev -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -72,16 +77,16 @@ RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/d git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ pip install -r /repos/arrow/python/requirements-test.txt && cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ - make -j6 && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && pip install /repos/arrow/python/dist/*.whl + make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && pip install /repos/arrow/python/dist/*.whl # Install rmm from source -RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && \ - cd rmm/ && pip install clang==8.0.1 && pip install cmake==3.18 && pip install numba && ./build.sh +RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd rmm/ && pip install clang==8.0.1 && pip install numba && ./build.sh ADD . /repos/cudf/ # Install cudf from source -RUN cd /repos/cudf/ && git submodule update --init --recursive && pip install -r python/cudf/requirements.txt --ignore-installed +RUN cd /repos/cudf/ && git submodule update --init --recursive && pip install -r python/cudf/requirements/cuda-$CUDA_SHORT_VERSION/requirements-build.txt --ignore-installed && \ + pip install -r python/cudf/requirements/requirements-test.txt --ignore-installed RUN cd /repos/cudf/ && ./build.sh clean && ./build.sh diff --git a/python/cudf/requirements/cuda-10.1/requirements-build.txt b/python/cudf/requirements/cuda-10.1/requirements-build.txt new file mode 100644 index 00000000000..1387e90499b --- /dev/null +++ b/python/cudf/requirements/cuda-10.1/requirements-build.txt @@ -0,0 +1,39 @@ +clang==8.0.1 +cupy-cuda101 +cmake +wheel==0.34.2 +cmake-setuptools>=0.1.3 +numpy +pandas>=1.0,<1.3.0dev0 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +sphinx +sphinx_rtd_theme +sphinxcontrib-websupport +nbsphinx +numpydoc +ipython +recommonmark +pandoc==2.0a4 +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +mypy==0.782 +typing_extensions +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz +dlpack +rapidjson +flatbuffers +sphinx-markdown-tables +sphinx-copybutton +packaging +protobuf +nvtx>=0.2.1 +cachetools +numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements.txt b/python/cudf/requirements/cuda-10.2/requirements-build.txt similarity index 60% rename from python/cudf/requirements.txt rename to python/cudf/requirements/cuda-10.2/requirements-build.txt index f8f5c5f039e..975cd39da23 100644 --- a/python/cudf/requirements.txt +++ b/python/cudf/requirements/cuda-10.2/requirements-build.txt @@ -2,16 +2,13 @@ clang==8.0.1 cupy-cuda102 cmake wheel==0.34.2 -cmake-setuptools +cmake-setuptools>=0.1.3 numpy -pandas -fastavro -notebook -cython -fsspec -pytest -pytest-benchmark -pytest-xdist +pandas>=1.0,<1.3.0dev0 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 sphinx sphinx_rtd_theme sphinxcontrib-websupport @@ -19,26 +16,24 @@ nbsphinx numpydoc ipython recommonmark -pandoc +pandoc==2.0a4 partd flake8==3.8.3 -black +black==19.10b0 isort==5.0.7 mypy==0.782 typing_extensions pre_commit -dask -distributed +dask>=2.22.0 +distributed>=2.22.0 streamz dlpack rapidjson flatbuffers -hypothesis sphinx-markdown-tables sphinx-copybutton -mimesis packaging protobuf -nvtx +nvtx>=0.2.1 cachetools -numba \ No newline at end of file +numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/requirements-build.txt b/python/cudf/requirements/cuda-11.0/requirements-build.txt new file mode 100644 index 00000000000..e899d53c818 --- /dev/null +++ b/python/cudf/requirements/cuda-11.0/requirements-build.txt @@ -0,0 +1,39 @@ +clang==8.0.1 +cupy-cuda110 +cmake +wheel==0.34.2 +cmake-setuptools>=0.1.3 +numpy +pandas>=1.0,<1.3.0dev0 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +sphinx +sphinx_rtd_theme +sphinxcontrib-websupport +nbsphinx +numpydoc +ipython +recommonmark +pandoc==2.0a4 +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +mypy==0.782 +typing_extensions +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz +dlpack +rapidjson +flatbuffers +sphinx-markdown-tables +sphinx-copybutton +packaging +protobuf +nvtx>=0.2.1 +cachetools +numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/requirements-build.txt b/python/cudf/requirements/cuda-11.1/requirements-build.txt new file mode 100644 index 00000000000..b3442fd5c66 --- /dev/null +++ b/python/cudf/requirements/cuda-11.1/requirements-build.txt @@ -0,0 +1,39 @@ +clang==8.0.1 +cupy-cuda111 +cmake +wheel==0.34.2 +cmake-setuptools>=0.1.3 +numpy +pandas>=1.0,<1.3.0dev0 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +sphinx +sphinx_rtd_theme +sphinxcontrib-websupport +nbsphinx +numpydoc +ipython +recommonmark +pandoc==2.0a4 +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +mypy==0.782 +typing_extensions +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz +dlpack +rapidjson +flatbuffers +sphinx-markdown-tables +sphinx-copybutton +packaging +protobuf +nvtx>=0.2.1 +cachetools +numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/requirements-build.txt b/python/cudf/requirements/cuda-11.2/requirements-build.txt new file mode 100644 index 00000000000..f32310956a2 --- /dev/null +++ b/python/cudf/requirements/cuda-11.2/requirements-build.txt @@ -0,0 +1,39 @@ +clang==8.0.1 +cupy-cuda112 +cmake +wheel==0.34.2 +cmake-setuptools>=0.1.3 +numpy +pandas>=1.0,<1.3.0dev0 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +sphinx +sphinx_rtd_theme +sphinxcontrib-websupport +nbsphinx +numpydoc +ipython +recommonmark +pandoc==2.0a4 +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +mypy==0.782 +typing_extensions +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz +dlpack +rapidjson +flatbuffers +sphinx-markdown-tables +sphinx-copybutton +packaging +protobuf +nvtx>=0.2.1 +cachetools +numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/requirements-test.txt b/python/cudf/requirements/requirements-test.txt new file mode 100644 index 00000000000..7dc2b0eb22a --- /dev/null +++ b/python/cudf/requirements/requirements-test.txt @@ -0,0 +1,6 @@ +pytest +pytest-benchmark +pytest-xdist +hypothesis +mimesis +pyorc \ No newline at end of file From a31a464b2996643814117e53c4428e8b75b6d116 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 26 Mar 2021 23:13:13 -0700 Subject: [PATCH 09/35] change cuda version and remove double arrow-cpp compilation --- docker_build/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 18f9577701e..e37b41d3ee5 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -1,7 +1,7 @@ # An integration test & dev container which builds and installs cuDF from main -ARG CUDA_VERSION=10.2 +ARG CUDA_VERSION=11.2.2 FROM nvidia/cuda:${CUDA_VERSION}-devel -ENV CUDA_SHORT_VERSION=10.2 +ENV CUDA_SHORT_VERSION=11.2 SHELL ["/bin/bash", "-c"] ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib @@ -77,7 +77,7 @@ RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/d git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ pip install -r /repos/arrow/python/requirements-test.txt && cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ - make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release --bundle-arrow-cpp bdist_wheel && pip install /repos/arrow/python/dist/*.whl + make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release bdist_wheel && pip install /repos/arrow/python/dist/*.whl # Install rmm from source From 705d7c47777541733d72114ba33300d89de4d921 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Sat, 27 Mar 2021 11:05:47 -0700 Subject: [PATCH 10/35] remove installing arrow test req --- docker_build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index e37b41d3ee5..326de69e59b 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -76,7 +76,7 @@ RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/c RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/dist/ && cd /repos/arrow && git checkout apache-arrow-1.0.1 && git submodule init && \ git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ - pip install -r /repos/arrow/python/requirements-test.txt && cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ + cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release bdist_wheel && pip install /repos/arrow/python/dist/*.whl From 98e8538da06e1bdd9e3164f0de9f942f888ac58b Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Mar 2021 08:58:32 -0700 Subject: [PATCH 11/35] introduce requirements-build & requirements-run.txt --- docker_build/Dockerfile | 5 ++- .../cuda-10.1/requirements-run.txt | 14 +++++++ .../cuda-10.2/requirements-build.txt | 39 ------------------- .../cuda-10.2/requirements-run.txt | 14 +++++++ .../cuda-11.0/requirements-build.txt | 39 ------------------- .../cuda-11.0/requirements-run.txt | 14 +++++++ .../cuda-11.1/requirements-build.txt | 39 ------------------- .../cuda-11.1/requirements-run.txt | 14 +++++++ .../cuda-11.2/requirements-build.txt | 39 ------------------- .../cuda-11.2/requirements-run.txt | 14 +++++++ .../{cuda-10.1 => }/requirements-build.txt | 13 ------- 11 files changed, 73 insertions(+), 171 deletions(-) create mode 100644 python/cudf/requirements/cuda-10.1/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-10.2/requirements-build.txt create mode 100644 python/cudf/requirements/cuda-10.2/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-11.0/requirements-build.txt create mode 100644 python/cudf/requirements/cuda-11.0/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-11.1/requirements-build.txt create mode 100644 python/cudf/requirements/cuda-11.1/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-11.2/requirements-build.txt create mode 100644 python/cudf/requirements/cuda-11.2/requirements-run.txt rename python/cudf/requirements/{cuda-10.1 => }/requirements-build.txt (68%) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 326de69e59b..63736a2438a 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -86,7 +86,8 @@ RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd rmm/ && pi ADD . /repos/cudf/ # Install cudf from source -RUN cd /repos/cudf/ && git submodule update --init --recursive && pip install -r python/cudf/requirements/cuda-$CUDA_SHORT_VERSION/requirements-build.txt --ignore-installed && \ - pip install -r python/cudf/requirements/requirements-test.txt --ignore-installed +RUN cd /repos/cudf/ && git submodule update --init --recursive && pip install -r python/cudf/requirements/requirements-build.txt --ignore-installed && \ + pip install -r python/cudf/requirements/cuda-$CUDA_SHORT_VERSION/requirements-run.txt --ignore-installed + RUN cd /repos/cudf/ && ./build.sh clean && ./build.sh diff --git a/python/cudf/requirements/cuda-10.1/requirements-run.txt b/python/cudf/requirements/cuda-10.1/requirements-run.txt new file mode 100644 index 00000000000..8a0303f6a32 --- /dev/null +++ b/python/cudf/requirements/cuda-10.1/requirements-run.txt @@ -0,0 +1,14 @@ +cupy-cuda101 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +ipython +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.2/requirements-build.txt b/python/cudf/requirements/cuda-10.2/requirements-build.txt deleted file mode 100644 index 975cd39da23..00000000000 --- a/python/cudf/requirements/cuda-10.2/requirements-build.txt +++ /dev/null @@ -1,39 +0,0 @@ -clang==8.0.1 -cupy-cuda102 -cmake -wheel==0.34.2 -cmake-setuptools>=0.1.3 -numpy -pandas>=1.0,<1.3.0dev0 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -sphinx -sphinx_rtd_theme -sphinxcontrib-websupport -nbsphinx -numpydoc -ipython -recommonmark -pandoc==2.0a4 -partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -mypy==0.782 -typing_extensions -pre_commit -dask>=2.22.0 -distributed>=2.22.0 -streamz -dlpack -rapidjson -flatbuffers -sphinx-markdown-tables -sphinx-copybutton -packaging -protobuf -nvtx>=0.2.1 -cachetools -numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.2/requirements-run.txt b/python/cudf/requirements/cuda-10.2/requirements-run.txt new file mode 100644 index 00000000000..221523fe0bd --- /dev/null +++ b/python/cudf/requirements/cuda-10.2/requirements-run.txt @@ -0,0 +1,14 @@ +cupy-cuda102 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +ipython +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/requirements-build.txt b/python/cudf/requirements/cuda-11.0/requirements-build.txt deleted file mode 100644 index e899d53c818..00000000000 --- a/python/cudf/requirements/cuda-11.0/requirements-build.txt +++ /dev/null @@ -1,39 +0,0 @@ -clang==8.0.1 -cupy-cuda110 -cmake -wheel==0.34.2 -cmake-setuptools>=0.1.3 -numpy -pandas>=1.0,<1.3.0dev0 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -sphinx -sphinx_rtd_theme -sphinxcontrib-websupport -nbsphinx -numpydoc -ipython -recommonmark -pandoc==2.0a4 -partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -mypy==0.782 -typing_extensions -pre_commit -dask>=2.22.0 -distributed>=2.22.0 -streamz -dlpack -rapidjson -flatbuffers -sphinx-markdown-tables -sphinx-copybutton -packaging -protobuf -nvtx>=0.2.1 -cachetools -numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/requirements-run.txt b/python/cudf/requirements/cuda-11.0/requirements-run.txt new file mode 100644 index 00000000000..0bddef63c6c --- /dev/null +++ b/python/cudf/requirements/cuda-11.0/requirements-run.txt @@ -0,0 +1,14 @@ +cupy-cuda110 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +ipython +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/requirements-build.txt b/python/cudf/requirements/cuda-11.1/requirements-build.txt deleted file mode 100644 index b3442fd5c66..00000000000 --- a/python/cudf/requirements/cuda-11.1/requirements-build.txt +++ /dev/null @@ -1,39 +0,0 @@ -clang==8.0.1 -cupy-cuda111 -cmake -wheel==0.34.2 -cmake-setuptools>=0.1.3 -numpy -pandas>=1.0,<1.3.0dev0 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -sphinx -sphinx_rtd_theme -sphinxcontrib-websupport -nbsphinx -numpydoc -ipython -recommonmark -pandoc==2.0a4 -partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -mypy==0.782 -typing_extensions -pre_commit -dask>=2.22.0 -distributed>=2.22.0 -streamz -dlpack -rapidjson -flatbuffers -sphinx-markdown-tables -sphinx-copybutton -packaging -protobuf -nvtx>=0.2.1 -cachetools -numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/requirements-run.txt b/python/cudf/requirements/cuda-11.1/requirements-run.txt new file mode 100644 index 00000000000..4693116b980 --- /dev/null +++ b/python/cudf/requirements/cuda-11.1/requirements-run.txt @@ -0,0 +1,14 @@ +cupy-cuda111 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +ipython +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/requirements-build.txt b/python/cudf/requirements/cuda-11.2/requirements-build.txt deleted file mode 100644 index f32310956a2..00000000000 --- a/python/cudf/requirements/cuda-11.2/requirements-build.txt +++ /dev/null @@ -1,39 +0,0 @@ -clang==8.0.1 -cupy-cuda112 -cmake -wheel==0.34.2 -cmake-setuptools>=0.1.3 -numpy -pandas>=1.0,<1.3.0dev0 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -sphinx -sphinx_rtd_theme -sphinxcontrib-websupport -nbsphinx -numpydoc -ipython -recommonmark -pandoc==2.0a4 -partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -mypy==0.782 -typing_extensions -pre_commit -dask>=2.22.0 -distributed>=2.22.0 -streamz -dlpack -rapidjson -flatbuffers -sphinx-markdown-tables -sphinx-copybutton -packaging -protobuf -nvtx>=0.2.1 -cachetools -numba>=0.49.0,!=0.51.0 \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/requirements-run.txt b/python/cudf/requirements/cuda-11.2/requirements-run.txt new file mode 100644 index 00000000000..aa07ca588d6 --- /dev/null +++ b/python/cudf/requirements/cuda-11.2/requirements-run.txt @@ -0,0 +1,14 @@ +cupy-cuda112 +fastavro>=0.22.9 +notebook>=0.5.0 +cython>=0.29,<0.30 +fsspec>=0.6.0 +ipython +partd +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 +pre_commit +dask>=2.22.0 +distributed>=2.22.0 +streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.1/requirements-build.txt b/python/cudf/requirements/requirements-build.txt similarity index 68% rename from python/cudf/requirements/cuda-10.1/requirements-build.txt rename to python/cudf/requirements/requirements-build.txt index 1387e90499b..e772cd9cc95 100644 --- a/python/cudf/requirements/cuda-10.1/requirements-build.txt +++ b/python/cudf/requirements/requirements-build.txt @@ -1,32 +1,19 @@ clang==8.0.1 -cupy-cuda101 cmake wheel==0.34.2 cmake-setuptools>=0.1.3 numpy pandas>=1.0,<1.3.0dev0 -fastavro>=0.22.9 -notebook>=0.5.0 cython>=0.29,<0.30 -fsspec>=0.6.0 sphinx sphinx_rtd_theme sphinxcontrib-websupport nbsphinx numpydoc -ipython recommonmark pandoc==2.0a4 -partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 mypy==0.782 typing_extensions -pre_commit -dask>=2.22.0 -distributed>=2.22.0 -streamz dlpack rapidjson flatbuffers From 3eaa8182934c4fe2590d6a5afe22baf6d5c23fb8 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Mar 2021 10:00:09 -0700 Subject: [PATCH 12/35] remove not needed env vars --- docker_build/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 63736a2438a..d4ad02647ee 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -10,8 +10,6 @@ ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/ ENV DEBIAN_FRONTEND=noninteractive -ENV CC=/usr/bin/gcc -ENV CXX=/usr/bin/g++ ENV CUDA_HOME=/usr/local/cuda ENV CUDA_PATH=$CUDA_HOME ENV PATH=${CUDA_HOME}/lib64/:${PATH}:${CUDA_HOME}/bin From 9013c6c29db6fae2713abeaef8c7f4176e16d394 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Mar 2021 10:04:44 -0700 Subject: [PATCH 13/35] remove dev tooling packages --- python/cudf/requirements/cuda-10.1/requirements-run.txt | 4 ---- python/cudf/requirements/cuda-10.2/requirements-run.txt | 4 ---- python/cudf/requirements/cuda-11.0/requirements-run.txt | 4 ---- python/cudf/requirements/cuda-11.1/requirements-run.txt | 4 ---- python/cudf/requirements/cuda-11.2/requirements-run.txt | 4 ---- python/cudf/requirements/requirements-build.txt | 1 - 6 files changed, 21 deletions(-) diff --git a/python/cudf/requirements/cuda-10.1/requirements-run.txt b/python/cudf/requirements/cuda-10.1/requirements-run.txt index 8a0303f6a32..56c6660f62e 100644 --- a/python/cudf/requirements/cuda-10.1/requirements-run.txt +++ b/python/cudf/requirements/cuda-10.1/requirements-run.txt @@ -5,10 +5,6 @@ cython>=0.29,<0.30 fsspec>=0.6.0 ipython partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -pre_commit dask>=2.22.0 distributed>=2.22.0 streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.2/requirements-run.txt b/python/cudf/requirements/cuda-10.2/requirements-run.txt index 221523fe0bd..701ee0643fb 100644 --- a/python/cudf/requirements/cuda-10.2/requirements-run.txt +++ b/python/cudf/requirements/cuda-10.2/requirements-run.txt @@ -5,10 +5,6 @@ cython>=0.29,<0.30 fsspec>=0.6.0 ipython partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -pre_commit dask>=2.22.0 distributed>=2.22.0 streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/requirements-run.txt b/python/cudf/requirements/cuda-11.0/requirements-run.txt index 0bddef63c6c..39c36c1d5e8 100644 --- a/python/cudf/requirements/cuda-11.0/requirements-run.txt +++ b/python/cudf/requirements/cuda-11.0/requirements-run.txt @@ -5,10 +5,6 @@ cython>=0.29,<0.30 fsspec>=0.6.0 ipython partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -pre_commit dask>=2.22.0 distributed>=2.22.0 streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/requirements-run.txt b/python/cudf/requirements/cuda-11.1/requirements-run.txt index 4693116b980..76c5e897efa 100644 --- a/python/cudf/requirements/cuda-11.1/requirements-run.txt +++ b/python/cudf/requirements/cuda-11.1/requirements-run.txt @@ -5,10 +5,6 @@ cython>=0.29,<0.30 fsspec>=0.6.0 ipython partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -pre_commit dask>=2.22.0 distributed>=2.22.0 streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/requirements-run.txt b/python/cudf/requirements/cuda-11.2/requirements-run.txt index aa07ca588d6..6b772d95b94 100644 --- a/python/cudf/requirements/cuda-11.2/requirements-run.txt +++ b/python/cudf/requirements/cuda-11.2/requirements-run.txt @@ -5,10 +5,6 @@ cython>=0.29,<0.30 fsspec>=0.6.0 ipython partd -flake8==3.8.3 -black==19.10b0 -isort==5.0.7 -pre_commit dask>=2.22.0 distributed>=2.22.0 streamz \ No newline at end of file diff --git a/python/cudf/requirements/requirements-build.txt b/python/cudf/requirements/requirements-build.txt index e772cd9cc95..d9f31034df6 100644 --- a/python/cudf/requirements/requirements-build.txt +++ b/python/cudf/requirements/requirements-build.txt @@ -1,4 +1,3 @@ -clang==8.0.1 cmake wheel==0.34.2 cmake-setuptools>=0.1.3 From 8300a8b36739910485b11928cb3f46b7232cc16b Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Mar 2021 11:28:50 -0700 Subject: [PATCH 14/35] disable building gandiva for arrow --- docker_build/Dockerfile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index d4ad02647ee..2e49237f3b5 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -38,7 +38,6 @@ RUN apt install -y --no-install-recommends \ libboost-all-dev \ python3.8-dev \ build-essential \ - llvm-7 \ autoconf \ bison \ flex \ @@ -61,20 +60,11 @@ RUN apt install -y --no-install-recommends \ RUN version=3.18 && build=5 && mkdir ~/temp && cd ~/temp && wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz && \ tar -xzvf cmake-$version.$build.tar.gz && cd cmake-$version.$build/ && ./bootstrap && make -j$(nproc) && make install -# Install llvm 11.1 for apache -RUN wget https://github.com/llvm/llvm-project/releases/download/llvmorg-11.1.0/clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && tar -xvf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - mv clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04 llvm-11.1 && cp -R llvm-11.1 /usr/lib/ && rm -rf clang+llvm-11.1.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - apt-get update && \ - apt-get install llvm-10-dev llvm-9-dev -y && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - # Install arrow from source RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/dist/ && cd /repos/arrow && git checkout apache-arrow-1.0.1 && git submodule init && \ git submodule update && export PARQUET_TEST_DATA="${PWD}/cpp/submodules/parquet-testing/data" && export ARROW_TEST_DATA="${PWD}/testing/data" && \ cd /repos/arrow/cpp && mkdir release && cd /repos/arrow/cpp/release && pip install -r /repos/arrow/python/requirements-build.txt && \ - cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=ON -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ + cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME -DCMAKE_INSTALL_LIBDIR=lib -DARROW_FLIGHT=ON -DARROW_GANDIVA=OFF -DARROW_ORC=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_PARQUET=ON -DARROW_PYTHON=ON -DARROW_PLASMA=ON -DARROW_BUILD_TESTS=ON -DARROW_CUDA=ON -DARROW_DATASET=ON .. && \ make -j$(nproc) && make install && cd /repos/arrow/python/ && python setup.py build_ext --build-type=release bdist_wheel && pip install /repos/arrow/python/dist/*.whl From 317a9f0ba7980c55f3cd2e8afeb6ff275d40bce2 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 29 Mar 2021 14:56:14 -0500 Subject: [PATCH 15/35] Update python/cudf/requirements/requirements-build.txt Co-authored-by: Keith Kraus --- python/cudf/requirements/requirements-build.txt | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/python/cudf/requirements/requirements-build.txt b/python/cudf/requirements/requirements-build.txt index d9f31034df6..a1d39c8687f 100644 --- a/python/cudf/requirements/requirements-build.txt +++ b/python/cudf/requirements/requirements-build.txt @@ -1,25 +1,10 @@ -cmake -wheel==0.34.2 -cmake-setuptools>=0.1.3 numpy pandas>=1.0,<1.3.0dev0 cython>=0.29,<0.30 -sphinx -sphinx_rtd_theme -sphinxcontrib-websupport -nbsphinx -numpydoc -recommonmark -pandoc==2.0a4 -mypy==0.782 typing_extensions -dlpack -rapidjson flatbuffers -sphinx-markdown-tables -sphinx-copybutton packaging protobuf nvtx>=0.2.1 cachetools -numba>=0.49.0,!=0.51.0 \ No newline at end of file +numba>=0.49.0,!=0.51.0 From f67c7d0971f2bdb03f6c389f5655be184d23c2b7 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Mar 2021 12:56:54 -0700 Subject: [PATCH 16/35] remove clang --- docker_build/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 2e49237f3b5..61b071f874d 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -47,8 +47,7 @@ RUN apt install -y --no-install-recommends \ libjemalloc-dev \ wget \ libssl-dev \ - protobuf-compiler \ - clang && \ + protobuf-compiler && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ From 3f2c291398ea9b21746a4e727c671b75024ca629 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 29 Mar 2021 14:09:46 -0700 Subject: [PATCH 17/35] remove numba specific workaround variables --- docker_build/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 61b071f874d..0c4d510b196 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -5,9 +5,7 @@ ENV CUDA_SHORT_VERSION=11.2 SHELL ["/bin/bash", "-c"] ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/lib:/repos/dist/lib -# Needed for cudf.concat(), avoids "OSError: library nvvm not found" -ENV NUMBAPRO_NVVM=/usr/local/cuda/nvvm/lib64/libnvvm.so -ENV NUMBAPRO_LIBDEVICE=/usr/local/cuda/nvvm/libdevice/ + ENV DEBIAN_FRONTEND=noninteractive ENV CUDA_HOME=/usr/local/cuda From 5439d5847714f26f4d451e9c54058a8af75827e0 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Mar 2021 23:32:04 -0700 Subject: [PATCH 18/35] remove requirements.txt files --- docker_build/Dockerfile | 11 ++--- python/cudf/pyproject.toml | 36 +++++++++++++++ .../cuda-10.1/requirements-run.txt | 10 ----- .../cuda-10.2/requirements-run.txt | 10 ----- .../cuda-11.0/requirements-run.txt | 10 ----- .../cuda-11.1/requirements-run.txt | 10 ----- .../cuda-11.2/requirements-run.txt | 8 +--- .../cudf/requirements/requirements-build.txt | 10 ----- .../cudf/requirements/requirements-test.txt | 6 --- python/cudf/setup.cfg | 12 +++++ python/cudf/setup.py | 45 +++++++++++++++++-- 11 files changed, 97 insertions(+), 71 deletions(-) create mode 100644 python/cudf/pyproject.toml delete mode 100644 python/cudf/requirements/cuda-10.1/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-10.2/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-11.0/requirements-run.txt delete mode 100644 python/cudf/requirements/cuda-11.1/requirements-run.txt delete mode 100644 python/cudf/requirements/requirements-build.txt delete mode 100644 python/cudf/requirements/requirements-test.txt diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index 0c4d510b196..a2038ee2ce0 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -66,13 +66,14 @@ RUN git clone https://github.com/apache/arrow.git /repos/arrow && mkdir /repos/d # Install rmm from source -RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd rmm/ && pip install clang==8.0.1 && pip install numba && ./build.sh +RUN cd /repos/ && git clone https://github.com/rapidsai/rmm.git && cd /repos/rmm/ && ./build.sh librmm && pip install /repos/rmm/python/. ADD . /repos/cudf/ +# Build env for CUDF build +ENV CUDF_ROOT=/repos/cudf/cpp/build/ + # Install cudf from source -RUN cd /repos/cudf/ && git submodule update --init --recursive && pip install -r python/cudf/requirements/requirements-build.txt --ignore-installed && \ - pip install -r python/cudf/requirements/cuda-$CUDA_SHORT_VERSION/requirements-run.txt --ignore-installed - +RUN cd /repos/cudf/ && git submodule update --init --recursive && ./build.sh libcudf && \ + cd /repos/cudf/python/cudf/ && pip install . -RUN cd /repos/cudf/ && ./build.sh clean && ./build.sh diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml new file mode 100644 index 00000000000..8e4322c70de --- /dev/null +++ b/python/cudf/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] + +requires = [ + "wheel", + "setuptools", + "numpy", + "pandas>=1.0,<1.3.0dev0", + "cython>=0.29,<0.30", + "typing_extensions", + "flatbuffers", + "packaging", + "protobuf", + "nvtx>=0.2.1", + "cachetools", + "numba>=0.49.0,!=0.51.0" +] + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/cudf/requirements/cuda-10.1/requirements-run.txt b/python/cudf/requirements/cuda-10.1/requirements-run.txt deleted file mode 100644 index 56c6660f62e..00000000000 --- a/python/cudf/requirements/cuda-10.1/requirements-run.txt +++ /dev/null @@ -1,10 +0,0 @@ -cupy-cuda101 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -ipython -partd -dask>=2.22.0 -distributed>=2.22.0 -streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.2/requirements-run.txt b/python/cudf/requirements/cuda-10.2/requirements-run.txt deleted file mode 100644 index 701ee0643fb..00000000000 --- a/python/cudf/requirements/cuda-10.2/requirements-run.txt +++ /dev/null @@ -1,10 +0,0 @@ -cupy-cuda102 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -ipython -partd -dask>=2.22.0 -distributed>=2.22.0 -streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/requirements-run.txt b/python/cudf/requirements/cuda-11.0/requirements-run.txt deleted file mode 100644 index 39c36c1d5e8..00000000000 --- a/python/cudf/requirements/cuda-11.0/requirements-run.txt +++ /dev/null @@ -1,10 +0,0 @@ -cupy-cuda110 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -ipython -partd -dask>=2.22.0 -distributed>=2.22.0 -streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/requirements-run.txt b/python/cudf/requirements/cuda-11.1/requirements-run.txt deleted file mode 100644 index 76c5e897efa..00000000000 --- a/python/cudf/requirements/cuda-11.1/requirements-run.txt +++ /dev/null @@ -1,10 +0,0 @@ -cupy-cuda111 -fastavro>=0.22.9 -notebook>=0.5.0 -cython>=0.29,<0.30 -fsspec>=0.6.0 -ipython -partd -dask>=2.22.0 -distributed>=2.22.0 -streamz \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/requirements-run.txt b/python/cudf/requirements/cuda-11.2/requirements-run.txt index 6b772d95b94..858ba231ab0 100644 --- a/python/cudf/requirements/cuda-11.2/requirements-run.txt +++ b/python/cudf/requirements/cuda-11.2/requirements-run.txt @@ -1,10 +1,4 @@ cupy-cuda112 fastavro>=0.22.9 -notebook>=0.5.0 cython>=0.29,<0.30 -fsspec>=0.6.0 -ipython -partd -dask>=2.22.0 -distributed>=2.22.0 -streamz \ No newline at end of file +fsspec>=0.6.0 \ No newline at end of file diff --git a/python/cudf/requirements/requirements-build.txt b/python/cudf/requirements/requirements-build.txt deleted file mode 100644 index a1d39c8687f..00000000000 --- a/python/cudf/requirements/requirements-build.txt +++ /dev/null @@ -1,10 +0,0 @@ -numpy -pandas>=1.0,<1.3.0dev0 -cython>=0.29,<0.30 -typing_extensions -flatbuffers -packaging -protobuf -nvtx>=0.2.1 -cachetools -numba>=0.49.0,!=0.51.0 diff --git a/python/cudf/requirements/requirements-test.txt b/python/cudf/requirements/requirements-test.txt deleted file mode 100644 index 7dc2b0eb22a..00000000000 --- a/python/cudf/requirements/requirements-test.txt +++ /dev/null @@ -1,6 +0,0 @@ -pytest -pytest-benchmark -pytest-xdist -hypothesis -mimesis -pyorc \ No newline at end of file diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index 3067d2daafd..bb0c84690d9 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -75,3 +75,15 @@ python_files = python_functions = bench_* test_* + + +[options] +packages = find: +extra_requires = + pytest + pytest-benchmark + pytest-xdist + hypothesis + mimesis + pyorc +python_requires = >=3.6,<3.8 \ No newline at end of file diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 023caa0ef7c..ad8cd6371d7 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -1,5 +1,6 @@ # Copyright (c) 2018-2020, NVIDIA CORPORATION. import os +import re import shutil import subprocess import sys @@ -16,10 +17,45 @@ import versioneer -install_requires = ["numba", "cython"] +install_requires = [ + "numba>=0.49.0,!=0.51.0", + "Cython>=0.29,<0.30", + "fastavro>=0.22.9", + "fsspec>=0.6.0", + "numpy", + "pandas>=1.0,<1.3.0dev0", + "typing_extensions", + "protobuf", + "nvtx>=0.2.1", + "cachetools", + "packaging", +] cython_files = ["cudf/**/*.pyx"] + +def get_cuda_version_from_header(cuda_include_dir, delimeter=""): + + cuda_version = None + + with open( + os.path.join(cuda_include_dir, "cuda.h"), "r", encoding="utf-8" + ) as f: + for line in f.readlines(): + if re.search(r"#define CUDA_VERSION ", line) is not None: + cuda_version = line + break + + if cuda_version is None: + raise TypeError("CUDA_VERSION not found in cuda.h") + cuda_version = int(cuda_version.split()[2]) + return "%d%s%d" % ( + cuda_version // 1000, + delimeter, + (cuda_version % 1000) // 10, + ) + + CUDA_HOME = os.environ.get("CUDA_HOME", False) if not CUDA_HOME: path_to_cuda_gdb = shutil.which("cuda-gdb") @@ -37,6 +73,9 @@ cuda_include_dir = os.path.join(CUDA_HOME, "include") cuda_lib_dir = os.path.join(CUDA_HOME, "lib64") +install_requires.append( + "cupy-cuda" + get_cuda_version_from_header(cuda_include_dir) +) CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") @@ -103,8 +142,8 @@ def run(self): "*", sources=cython_files, include_dirs=[ - "../../cpp/include/cudf", - "../../cpp/include", + os.path.join(CUDF_ROOT, "../include/cudf"), + os.path.join(CUDF_ROOT, "../include"), os.path.join(CUDF_ROOT, "include"), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), os.path.join(CUDF_ROOT, "_deps/dlpack-src/include"), From a82472265f25beb0465e9079a45f6fe8bb9b816e Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Mar 2021 23:35:14 -0700 Subject: [PATCH 19/35] cleanup --- docker_build/Dockerfile | 3 ++- python/cudf/pyproject.toml | 2 ++ python/cudf/requirements/cuda-11.2/requirements-run.txt | 4 ---- python/cudf/setup.cfg | 2 +- python/cudf/setup.py | 3 ++- 5 files changed, 7 insertions(+), 7 deletions(-) delete mode 100644 python/cudf/requirements/cuda-11.2/requirements-run.txt diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index a2038ee2ce0..fb230198073 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -1,4 +1,5 @@ -# An integration test & dev container which builds and installs cuDF from main +# Copyright (c) 2021, NVIDIA CORPORATION. + ARG CUDA_VERSION=11.2.2 FROM nvidia/cuda:${CUDA_VERSION}-devel ENV CUDA_SHORT_VERSION=11.2 diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 8e4322c70de..a022c9c86fe 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -1,3 +1,5 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + [build-system] requires = [ diff --git a/python/cudf/requirements/cuda-11.2/requirements-run.txt b/python/cudf/requirements/cuda-11.2/requirements-run.txt deleted file mode 100644 index 858ba231ab0..00000000000 --- a/python/cudf/requirements/cuda-11.2/requirements-run.txt +++ /dev/null @@ -1,4 +0,0 @@ -cupy-cuda112 -fastavro>=0.22.9 -cython>=0.29,<0.30 -fsspec>=0.6.0 \ No newline at end of file diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index bb0c84690d9..4fb74c9a9b4 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2020, NVIDIA CORPORATION. +# Copyright (c) 2018-2021, NVIDIA CORPORATION. # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the diff --git a/python/cudf/setup.py b/python/cudf/setup.py index ad8cd6371d7..c496b1aa92f 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -1,4 +1,5 @@ -# Copyright (c) 2018-2020, NVIDIA CORPORATION. +# Copyright (c) 2018-2021, NVIDIA CORPORATION. + import os import re import shutil From 9e793ef0172f914df16328d18569bca2558f8fe3 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 30 Mar 2021 23:55:55 -0700 Subject: [PATCH 20/35] add for dask-cudf --- python/dask_cudf/pyproject.toml | 28 ++++++++++++++ python/dask_cudf/setup.py | 67 ++++++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 python/dask_cudf/pyproject.toml diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml new file mode 100644 index 00000000000..dfe475a2e46 --- /dev/null +++ b/python/dask_cudf/pyproject.toml @@ -0,0 +1,28 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", +] + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 7ae816d3a52..54466f14515 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,9 +1,73 @@ # Copyright (c) 2019, NVIDIA CORPORATION. +import os +import re +import shutil + from setuptools import find_packages, setup import versioneer -install_requires = ["cudf", "dask", "distributed"] +install_requires = [ + "cudf", + "dask>=2021.3.1", + "distributed>=2.22.0", + "fsspec>=0.6.0", + "numpy", + "pandas>=1.0,<1.3.0dev0", +] +extra_requires = [ + "numpy", + "pandas>=1.0,<1.3.0dev0", + "pytest", + "numba>=0.49.0,!=0.51.0", + "dask>=2021.3.1", + "distributed>=2.22.0", +] + + +def get_cuda_version_from_header(cuda_include_dir, delimeter=""): + + cuda_version = None + + with open( + os.path.join(cuda_include_dir, "cuda.h"), "r", encoding="utf-8" + ) as f: + for line in f.readlines(): + if re.search(r"#define CUDA_VERSION ", line) is not None: + cuda_version = line + break + + if cuda_version is None: + raise TypeError("CUDA_VERSION not found in cuda.h") + cuda_version = int(cuda_version.split()[2]) + return "%d%s%d" % ( + cuda_version // 1000, + delimeter, + (cuda_version % 1000) // 10, + ) + + +CUDA_HOME = os.environ.get("CUDA_HOME", False) +if not CUDA_HOME: + path_to_cuda_gdb = shutil.which("cuda-gdb") + if path_to_cuda_gdb is None: + raise OSError( + "Could not locate CUDA. " + "Please set the environment variable " + "CUDA_HOME to the path to the CUDA installation " + "and try again." + ) + CUDA_HOME = os.path.dirname(os.path.dirname(path_to_cuda_gdb)) + +if not os.path.isdir(CUDA_HOME): + raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") + +cuda_include_dir = os.path.join(CUDA_HOME, "include") +cupy_package_name = "cupy-cuda" + get_cuda_version_from_header( + cuda_include_dir +) +install_requires.append(cupy_package_name) + setup( name="dask-cudf", @@ -24,4 +88,5 @@ packages=find_packages(exclude=["tests", "tests.*"]), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, + extra_requires=extra_requires, ) From 71d5089c811658cd49de731642d50f511e9b3b51 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 11:58:22 -0700 Subject: [PATCH 21/35] pin to correct supported python version --- conda/environments/cudf_dev_cuda10.1.yml | 2 +- conda/environments/cudf_dev_cuda10.2.yml | 2 +- conda/environments/cudf_dev_cuda11.0.yml | 2 +- conda/environments/cudf_dev_cuda11.1.yml | 2 +- conda/environments/cudf_dev_cuda11.2.yml | 2 +- python/cudf/setup.cfg | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/environments/cudf_dev_cuda10.1.yml b/conda/environments/cudf_dev_cuda10.1.yml index fa0b1126190..d3596b1399f 100644 --- a/conda/environments/cudf_dev_cuda10.1.yml +++ b/conda/environments/cudf_dev_cuda10.1.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49.0,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 diff --git a/conda/environments/cudf_dev_cuda10.2.yml b/conda/environments/cudf_dev_cuda10.2.yml index 52d82c4f4ef..ddb55853bca 100644 --- a/conda/environments/cudf_dev_cuda10.2.yml +++ b/conda/environments/cudf_dev_cuda10.2.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 2e64365bdf6..4e46ee3fddf 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 diff --git a/conda/environments/cudf_dev_cuda11.1.yml b/conda/environments/cudf_dev_cuda11.1.yml index 887022a28ed..77a6a097a3c 100644 --- a/conda/environments/cudf_dev_cuda11.1.yml +++ b/conda/environments/cudf_dev_cuda11.1.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 620a43ebdcc..920353b3e39 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -14,7 +14,7 @@ dependencies: - rmm=0.19.* - cmake>=3.14 - cmake_setuptools>=0.1.3 - - python>=3.6,<3.8 + - python>=3.7,<3.9 - numba>=0.49,!=0.51.0 - numpy - pandas>=1.0,<1.3.0dev0 diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index 4fb74c9a9b4..f104152a062 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -86,4 +86,4 @@ extra_requires = hypothesis mimesis pyorc -python_requires = >=3.6,<3.8 \ No newline at end of file +python_requires = >=3.7,<3.9 \ No newline at end of file From 408ee6a499a6c371b09cb37f19861e750a42869a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 12:08:26 -0700 Subject: [PATCH 22/35] add dev requirements --- .../cuda-10.1/dev_requirements.txt | 36 +++++++++++++++++++ .../cuda-10.2/dev_requirements.txt | 36 +++++++++++++++++++ .../cuda-11.0/dev_requirements.txt | 36 +++++++++++++++++++ .../cuda-11.1/dev_requirements.txt | 36 +++++++++++++++++++ .../cuda-11.2/dev_requirements.txt | 36 +++++++++++++++++++ 5 files changed, 180 insertions(+) create mode 100644 python/cudf/requirements/cuda-10.1/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-10.2/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-11.0/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-11.1/dev_requirements.txt create mode 100644 python/cudf/requirements/cuda-11.2/dev_requirements.txt diff --git a/python/cudf/requirements/cuda-10.1/dev_requirements.txt b/python/cudf/requirements/cuda-10.1/dev_requirements.txt new file mode 100644 index 00000000000..bc5ea0db051 --- /dev/null +++ b/python/cudf/requirements/cuda-10.1/dev_requirements.txt @@ -0,0 +1,36 @@ +cupy-cuda101 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-10.2/dev_requirements.txt b/python/cudf/requirements/cuda-10.2/dev_requirements.txt new file mode 100644 index 00000000000..f530bbb5996 --- /dev/null +++ b/python/cudf/requirements/cuda-10.2/dev_requirements.txt @@ -0,0 +1,36 @@ +cupy-cuda102 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.0/dev_requirements.txt b/python/cudf/requirements/cuda-11.0/dev_requirements.txt new file mode 100644 index 00000000000..2940f78c0d4 --- /dev/null +++ b/python/cudf/requirements/cuda-11.0/dev_requirements.txt @@ -0,0 +1,36 @@ +cupy-cuda110 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.1/dev_requirements.txt b/python/cudf/requirements/cuda-11.1/dev_requirements.txt new file mode 100644 index 00000000000..a6e6cada01e --- /dev/null +++ b/python/cudf/requirements/cuda-11.1/dev_requirements.txt @@ -0,0 +1,36 @@ +cupy-cuda111 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file diff --git a/python/cudf/requirements/cuda-11.2/dev_requirements.txt b/python/cudf/requirements/cuda-11.2/dev_requirements.txt new file mode 100644 index 00000000000..a509df68182 --- /dev/null +++ b/python/cudf/requirements/cuda-11.2/dev_requirements.txt @@ -0,0 +1,36 @@ +cupy-cuda112 +cachetools +cmake +cmake-setuptools>=0.1.3 +cython>=0.29,<0.30 +dlpack +fastavro>=0.22.9 +flatbuffers +fsspec>=0.6.0 +hypothesis +mimesis +mypy==0.782 +nbsphinx +numba>=0.49.0,!=0.51.0 +numpy +numpydoc +nvtx>=0.2.1 +packaging +pandas>=1.0,<1.3.0dev0 +pandoc==2.0a4 +protobuf +pyorc +pytest +pytest-benchmark +pytest-xdist +rapidjson +recommonmark +setuptools +sphinx +sphinx-copybutton +sphinx-markdown-tables +sphinx_rtd_theme +sphinxcontrib-websupport +typing_extensions +typing_extensions +wheel \ No newline at end of file From 6c5ff1e2bba62e9a66a1dc37d769c15ea4ccc1b7 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 12:09:27 -0700 Subject: [PATCH 23/35] copyright --- python/cudf/requirements/cuda-10.1/dev_requirements.txt | 2 ++ python/cudf/requirements/cuda-10.2/dev_requirements.txt | 2 ++ python/cudf/requirements/cuda-11.0/dev_requirements.txt | 2 ++ python/cudf/requirements/cuda-11.1/dev_requirements.txt | 2 ++ python/cudf/requirements/cuda-11.2/dev_requirements.txt | 2 ++ 5 files changed, 10 insertions(+) diff --git a/python/cudf/requirements/cuda-10.1/dev_requirements.txt b/python/cudf/requirements/cuda-10.1/dev_requirements.txt index bc5ea0db051..7122eb2cb92 100644 --- a/python/cudf/requirements/cuda-10.1/dev_requirements.txt +++ b/python/cudf/requirements/cuda-10.1/dev_requirements.txt @@ -1,3 +1,5 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + cupy-cuda101 cachetools cmake diff --git a/python/cudf/requirements/cuda-10.2/dev_requirements.txt b/python/cudf/requirements/cuda-10.2/dev_requirements.txt index f530bbb5996..f93220c2361 100644 --- a/python/cudf/requirements/cuda-10.2/dev_requirements.txt +++ b/python/cudf/requirements/cuda-10.2/dev_requirements.txt @@ -1,3 +1,5 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + cupy-cuda102 cachetools cmake diff --git a/python/cudf/requirements/cuda-11.0/dev_requirements.txt b/python/cudf/requirements/cuda-11.0/dev_requirements.txt index 2940f78c0d4..f0e0cf7889d 100644 --- a/python/cudf/requirements/cuda-11.0/dev_requirements.txt +++ b/python/cudf/requirements/cuda-11.0/dev_requirements.txt @@ -1,3 +1,5 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + cupy-cuda110 cachetools cmake diff --git a/python/cudf/requirements/cuda-11.1/dev_requirements.txt b/python/cudf/requirements/cuda-11.1/dev_requirements.txt index a6e6cada01e..1ae0f80c931 100644 --- a/python/cudf/requirements/cuda-11.1/dev_requirements.txt +++ b/python/cudf/requirements/cuda-11.1/dev_requirements.txt @@ -1,3 +1,5 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + cupy-cuda111 cachetools cmake diff --git a/python/cudf/requirements/cuda-11.2/dev_requirements.txt b/python/cudf/requirements/cuda-11.2/dev_requirements.txt index a509df68182..f70a021c3f8 100644 --- a/python/cudf/requirements/cuda-11.2/dev_requirements.txt +++ b/python/cudf/requirements/cuda-11.2/dev_requirements.txt @@ -1,3 +1,5 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + cupy-cuda112 cachetools cmake From 3ac2f242590f0e4b73ab5189f501ec6e8aee28e1 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 12:23:07 -0700 Subject: [PATCH 24/35] handle dask-cudf --- conda/environments/cudf_dev_cuda10.1.yml | 1 - conda/environments/cudf_dev_cuda10.2.yml | 1 - conda/environments/cudf_dev_cuda11.0.yml | 1 - conda/environments/cudf_dev_cuda11.1.yml | 1 - conda/environments/cudf_dev_cuda11.2.yml | 1 - .../requirements/cuda-10.1/dev_requirements.txt | 12 ++++++++++++ .../requirements/cuda-10.2/dev_requirements.txt | 12 ++++++++++++ .../requirements/cuda-11.0/dev_requirements.txt | 12 ++++++++++++ .../requirements/cuda-11.1/dev_requirements.txt | 12 ++++++++++++ .../requirements/cuda-11.2/dev_requirements.txt | 12 ++++++++++++ python/dask_cudf/setup.py | 3 ++- 11 files changed, 62 insertions(+), 6 deletions(-) create mode 100644 python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt create mode 100644 python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt create mode 100644 python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt create mode 100644 python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt create mode 100644 python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt diff --git a/conda/environments/cudf_dev_cuda10.1.yml b/conda/environments/cudf_dev_cuda10.1.yml index d3596b1399f..2ec8af368f2 100644 --- a/conda/environments/cudf_dev_cuda10.1.yml +++ b/conda/environments/cudf_dev_cuda10.1.yml @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=10.1 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda10.2.yml b/conda/environments/cudf_dev_cuda10.2.yml index ddb55853bca..64dcc458196 100644 --- a/conda/environments/cudf_dev_cuda10.2.yml +++ b/conda/environments/cudf_dev_cuda10.2.yml @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=10.2 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 4e46ee3fddf..f464738eaa8 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=11.0 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda11.1.yml b/conda/environments/cudf_dev_cuda11.1.yml index 77a6a097a3c..0a2bb113d7e 100644 --- a/conda/environments/cudf_dev_cuda11.1.yml +++ b/conda/environments/cudf_dev_cuda11.1.yml @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=11.1 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 920353b3e39..0c2c5aed568 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -36,7 +36,6 @@ dependencies: - pandoc=<2.0.0 - cudatoolkit=11.2 - pip - - partd - flake8=3.8.3 - black=19.10 - isort=5.0.7 diff --git a/python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt b/python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt new file mode 100644 index 00000000000..e1268ec50a6 --- /dev/null +++ b/python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +cupy-cuda101 +dask>=2021.3.1 +distributed>=2.22.0 +fsspec>=0.6.0 +numba>=0.49.0,!=0.51.0 +numpy +pandas>=1.0,<1.3.0dev0 +pytest +setuptools +wheel diff --git a/python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt b/python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt new file mode 100644 index 00000000000..ea20c935452 --- /dev/null +++ b/python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +cupy-cuda102 +dask>=2021.3.1 +distributed>=2.22.0 +fsspec>=0.6.0 +numba>=0.49.0,!=0.51.0 +numpy +pandas>=1.0,<1.3.0dev0 +pytest +setuptools +wheel diff --git a/python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt b/python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt new file mode 100644 index 00000000000..571d2a248f1 --- /dev/null +++ b/python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +cupy-cuda110 +dask>=2021.3.1 +distributed>=2.22.0 +fsspec>=0.6.0 +numba>=0.49.0,!=0.51.0 +numpy +pandas>=1.0,<1.3.0dev0 +pytest +setuptools +wheel diff --git a/python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt b/python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt new file mode 100644 index 00000000000..93fe16f029d --- /dev/null +++ b/python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +cupy-cuda111 +dask>=2021.3.1 +distributed>=2.22.0 +fsspec>=0.6.0 +numba>=0.49.0,!=0.51.0 +numpy +pandas>=1.0,<1.3.0dev0 +pytest +setuptools +wheel \ No newline at end of file diff --git a/python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt b/python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt new file mode 100644 index 00000000000..14f840b686c --- /dev/null +++ b/python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt @@ -0,0 +1,12 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +cupy-cuda112 +dask>=2021.3.1 +distributed>=2.22.0 +fsspec>=0.6.0 +numba>=0.49.0,!=0.51.0 +numpy +pandas>=1.0,<1.3.0dev0 +pytest +setuptools +wheel diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 54466f14515..cd33f05199b 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,4 +1,5 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2019-2021, NVIDIA CORPORATION. + import os import re import shutil From 17cab9078304bb0f0f0b2a3f44aeeb8a3089134d Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 12:40:23 -0700 Subject: [PATCH 25/35] cudf_kafka and custreamz --- python/cudf/setup.py | 1 + python/cudf_kafka/pyproject.toml | 30 ++++++++++++++++++++++++++++++ python/cudf_kafka/setup.py | 3 ++- python/custreamz/pyproject.toml | 28 ++++++++++++++++++++++++++++ python/custreamz/setup.py | 1 + python/dask_cudf/setup.py | 20 +++++++++++--------- 6 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 python/cudf_kafka/pyproject.toml create mode 100644 python/custreamz/pyproject.toml diff --git a/python/cudf/setup.py b/python/cudf/setup.py index c496b1aa92f..feac82278e4 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -203,4 +203,5 @@ def run(self): cmdclass=cmdclass, install_requires=install_requires, zip_safe=False, + extras_requires={"test": ["pytest", "pytest-xdist"]}, ) diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml new file mode 100644 index 00000000000..9855188ac6c --- /dev/null +++ b/python/cudf_kafka/pyproject.toml @@ -0,0 +1,30 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", + "Cython>=0.29,<0.30", +] + + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 536e529e169..3803a783817 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -85,7 +85,7 @@ "Programming Language :: Python :: 3.8", ], # Include the separately-compiled shared library - setup_requires=["cython"], + setup_requires=["Cython>=0.29,<0.30"], ext_modules=cythonize( extensions, nthreads=nthreads, @@ -99,5 +99,6 @@ ), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, + extras_requires={"test": ["pytest", "pytest-xdist"]}, zip_safe=False, ) diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml new file mode 100644 index 00000000000..dfe475a2e46 --- /dev/null +++ b/python/custreamz/pyproject.toml @@ -0,0 +1,28 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +[build-system] + +requires = [ + "wheel", + "setuptools", +] + +[tool.black] +line-length = 79 +target-version = ["py36"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist +)/ +''' diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py index c321e7ba17d..238b50d5fc4 100644 --- a/python/custreamz/setup.py +++ b/python/custreamz/setup.py @@ -27,4 +27,5 @@ cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, zip_safe=False, + extras_requires={"test": ["pytest", "pytest-xdist"]}, ) diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index cd33f05199b..24b781c0620 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -16,14 +16,16 @@ "numpy", "pandas>=1.0,<1.3.0dev0", ] -extra_requires = [ - "numpy", - "pandas>=1.0,<1.3.0dev0", - "pytest", - "numba>=0.49.0,!=0.51.0", - "dask>=2021.3.1", - "distributed>=2.22.0", -] +extras_requires = { + "test": [ + "numpy", + "pandas>=1.0,<1.3.0dev0", + "pytest", + "numba>=0.49.0,!=0.51.0", + "dask>=2021.3.1", + "distributed>=2.22.0", + ] +} def get_cuda_version_from_header(cuda_include_dir, delimeter=""): @@ -89,5 +91,5 @@ def get_cuda_version_from_header(cuda_include_dir, delimeter=""): packages=find_packages(exclude=["tests", "tests.*"]), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, - extra_requires=extra_requires, + extras_requires=extras_requires, ) From 8e04fb042da3336673bd04fa784c1dfd4a7e53b4 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 16:01:24 -0700 Subject: [PATCH 26/35] reviews --- docker_build/Dockerfile | 2 +- python/cudf/pyproject.toml | 7 ------- python/cudf/setup.cfg | 12 ------------ python/cudf/setup.py | 13 ++++++++++++- python/cudf_kafka/setup.py | 2 +- python/custreamz/setup.py | 2 +- .../cuda-10.1 => }/dev_requirements.txt | 1 - .../requirements/cuda-10.2/dev_requirements.txt | 12 ------------ .../requirements/cuda-11.0/dev_requirements.txt | 12 ------------ .../requirements/cuda-11.1/dev_requirements.txt | 12 ------------ .../requirements/cuda-11.2/dev_requirements.txt | 12 ------------ python/dask_cudf/setup.py | 5 +++-- 12 files changed, 18 insertions(+), 74 deletions(-) rename python/dask_cudf/{requirements/cuda-10.1 => }/dev_requirements.txt (92%) delete mode 100644 python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt delete mode 100644 python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt delete mode 100644 python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt delete mode 100644 python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt diff --git a/docker_build/Dockerfile b/docker_build/Dockerfile index fb230198073..9216f15b419 100644 --- a/docker_build/Dockerfile +++ b/docker_build/Dockerfile @@ -76,5 +76,5 @@ ENV CUDF_ROOT=/repos/cudf/cpp/build/ # Install cudf from source RUN cd /repos/cudf/ && git submodule update --init --recursive && ./build.sh libcudf && \ - cd /repos/cudf/python/cudf/ && pip install . + pip install /repos/cudf/python/cudf/. diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index a022c9c86fe..ad6f008a2f9 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -5,16 +5,9 @@ requires = [ "wheel", "setuptools", - "numpy", - "pandas>=1.0,<1.3.0dev0", "cython>=0.29,<0.30", - "typing_extensions", - "flatbuffers", "packaging", - "protobuf", - "nvtx>=0.2.1", "cachetools", - "numba>=0.49.0,!=0.51.0" ] [tool.black] diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index f104152a062..697a689801f 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -75,15 +75,3 @@ python_files = python_functions = bench_* test_* - - -[options] -packages = find: -extra_requires = - pytest - pytest-benchmark - pytest-xdist - hypothesis - mimesis - pyorc -python_requires = >=3.7,<3.9 \ No newline at end of file diff --git a/python/cudf/setup.py b/python/cudf/setup.py index feac82278e4..136923a2f48 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -32,6 +32,17 @@ "packaging", ] +extras_require = { + "test": [ + "pytest", + "pytest-benchmark", + "pytest-xdist", + "hypothesis" "mimesis", + "pyorc", + "msgpack", + ] +} + cython_files = ["cudf/**/*.pyx"] @@ -203,5 +214,5 @@ def run(self): cmdclass=cmdclass, install_requires=install_requires, zip_safe=False, - extras_requires={"test": ["pytest", "pytest-xdist"]}, + extras_require=extras_require, ) diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 3803a783817..85401bcc2e6 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -99,6 +99,6 @@ ), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, - extras_requires={"test": ["pytest", "pytest-xdist"]}, + extras_require={"test": ["pytest", "pytest-xdist"]}, zip_safe=False, ) diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py index 238b50d5fc4..07a6b92f65d 100644 --- a/python/custreamz/setup.py +++ b/python/custreamz/setup.py @@ -27,5 +27,5 @@ cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, zip_safe=False, - extras_requires={"test": ["pytest", "pytest-xdist"]}, + extras_require={"test": ["pytest", "pytest-xdist"]}, ) diff --git a/python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt similarity index 92% rename from python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt rename to python/dask_cudf/dev_requirements.txt index e1268ec50a6..1dd9c41be86 100644 --- a/python/dask_cudf/requirements/cuda-10.1/dev_requirements.txt +++ b/python/dask_cudf/dev_requirements.txt @@ -1,6 +1,5 @@ # Copyright (c) 2021, NVIDIA CORPORATION. -cupy-cuda101 dask>=2021.3.1 distributed>=2.22.0 fsspec>=0.6.0 diff --git a/python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt b/python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt deleted file mode 100644 index ea20c935452..00000000000 --- a/python/dask_cudf/requirements/cuda-10.2/dev_requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. - -cupy-cuda102 -dask>=2021.3.1 -distributed>=2.22.0 -fsspec>=0.6.0 -numba>=0.49.0,!=0.51.0 -numpy -pandas>=1.0,<1.3.0dev0 -pytest -setuptools -wheel diff --git a/python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt b/python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt deleted file mode 100644 index 571d2a248f1..00000000000 --- a/python/dask_cudf/requirements/cuda-11.0/dev_requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. - -cupy-cuda110 -dask>=2021.3.1 -distributed>=2.22.0 -fsspec>=0.6.0 -numba>=0.49.0,!=0.51.0 -numpy -pandas>=1.0,<1.3.0dev0 -pytest -setuptools -wheel diff --git a/python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt b/python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt deleted file mode 100644 index 93fe16f029d..00000000000 --- a/python/dask_cudf/requirements/cuda-11.1/dev_requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. - -cupy-cuda111 -dask>=2021.3.1 -distributed>=2.22.0 -fsspec>=0.6.0 -numba>=0.49.0,!=0.51.0 -numpy -pandas>=1.0,<1.3.0dev0 -pytest -setuptools -wheel \ No newline at end of file diff --git a/python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt b/python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt deleted file mode 100644 index 14f840b686c..00000000000 --- a/python/dask_cudf/requirements/cuda-11.2/dev_requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. - -cupy-cuda112 -dask>=2021.3.1 -distributed>=2.22.0 -fsspec>=0.6.0 -numba>=0.49.0,!=0.51.0 -numpy -pandas>=1.0,<1.3.0dev0 -pytest -setuptools -wheel diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 24b781c0620..1318701b845 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -16,7 +16,8 @@ "numpy", "pandas>=1.0,<1.3.0dev0", ] -extras_requires = { + +extras_require = { "test": [ "numpy", "pandas>=1.0,<1.3.0dev0", @@ -91,5 +92,5 @@ def get_cuda_version_from_header(cuda_include_dir, delimeter=""): packages=find_packages(exclude=["tests", "tests.*"]), cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, - extras_requires=extras_requires, + extras_require=extras_require, ) From eecf6d213212b9d001be9da7cfd4063aea01bc55 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 16:05:46 -0700 Subject: [PATCH 27/35] add streamz and cudf_kafka dev reqs --- python/cudf_kafka/dev_requirements.txt | 8 ++++++++ python/custreamz/dev_requirements.txt | 9 +++++++++ 2 files changed, 17 insertions(+) create mode 100644 python/cudf_kafka/dev_requirements.txt create mode 100644 python/custreamz/dev_requirements.txt diff --git a/python/cudf_kafka/dev_requirements.txt b/python/cudf_kafka/dev_requirements.txt new file mode 100644 index 00000000000..49217e124ca --- /dev/null +++ b/python/cudf_kafka/dev_requirements.txt @@ -0,0 +1,8 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +python-confluent-kafka +pytest +setuptools +wheel +cython>=0.29,<0.30 +python-confluent-kafka \ No newline at end of file diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt new file mode 100644 index 00000000000..63a9d40f713 --- /dev/null +++ b/python/custreamz/dev_requirements.txt @@ -0,0 +1,9 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. + +dask>=2021.3.1 +distributed>=2.22.0 +streamz +python-confluent-kafka +pytest +setuptools +wheel From 9efc93e5e9f71ad8b684142fc4bd481582eb8789 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 16:08:20 -0700 Subject: [PATCH 28/35] update req --- python/cudf_kafka/dev_requirements.txt | 3 +++ python/custreamz/dev_requirements.txt | 3 +++ python/dask_cudf/dev_requirements.txt | 3 +++ 3 files changed, 9 insertions(+) diff --git a/python/cudf_kafka/dev_requirements.txt b/python/cudf_kafka/dev_requirements.txt index 49217e124ca..e7f0d8edf99 100644 --- a/python/cudf_kafka/dev_requirements.txt +++ b/python/cudf_kafka/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 python-confluent-kafka pytest setuptools diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt index 63a9d40f713..7e300376f4e 100644 --- a/python/custreamz/dev_requirements.txt +++ b/python/custreamz/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 dask>=2021.3.1 distributed>=2.22.0 streamz diff --git a/python/dask_cudf/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt index 1dd9c41be86..39da0da3012 100644 --- a/python/dask_cudf/dev_requirements.txt +++ b/python/dask_cudf/dev_requirements.txt @@ -9,3 +9,6 @@ pandas>=1.0,<1.3.0dev0 pytest setuptools wheel +flake8==3.8.3 +black==19.10b0 +isort==5.0.7 \ No newline at end of file From 7832b4c59b201ead957a74f88e47ad27ebbafaa3 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 16:13:14 -0700 Subject: [PATCH 29/35] add pyarrow comment --- python/cudf/requirements/cuda-10.1/dev_requirements.txt | 3 +++ python/cudf/requirements/cuda-10.2/dev_requirements.txt | 3 +++ python/cudf/requirements/cuda-11.0/dev_requirements.txt | 3 +++ python/cudf/requirements/cuda-11.1/dev_requirements.txt | 3 +++ python/cudf/requirements/cuda-11.2/dev_requirements.txt | 3 +++ 5 files changed, 15 insertions(+) diff --git a/python/cudf/requirements/cuda-10.1/dev_requirements.txt b/python/cudf/requirements/cuda-10.1/dev_requirements.txt index 7122eb2cb92..967974d38b5 100644 --- a/python/cudf/requirements/cuda-10.1/dev_requirements.txt +++ b/python/cudf/requirements/cuda-10.1/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + cupy-cuda101 cachetools cmake diff --git a/python/cudf/requirements/cuda-10.2/dev_requirements.txt b/python/cudf/requirements/cuda-10.2/dev_requirements.txt index f93220c2361..34450456b5a 100644 --- a/python/cudf/requirements/cuda-10.2/dev_requirements.txt +++ b/python/cudf/requirements/cuda-10.2/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + cupy-cuda102 cachetools cmake diff --git a/python/cudf/requirements/cuda-11.0/dev_requirements.txt b/python/cudf/requirements/cuda-11.0/dev_requirements.txt index f0e0cf7889d..278b1a6bf61 100644 --- a/python/cudf/requirements/cuda-11.0/dev_requirements.txt +++ b/python/cudf/requirements/cuda-11.0/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + cupy-cuda110 cachetools cmake diff --git a/python/cudf/requirements/cuda-11.1/dev_requirements.txt b/python/cudf/requirements/cuda-11.1/dev_requirements.txt index 1ae0f80c931..fafdc7d7d4f 100644 --- a/python/cudf/requirements/cuda-11.1/dev_requirements.txt +++ b/python/cudf/requirements/cuda-11.1/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + cupy-cuda111 cachetools cmake diff --git a/python/cudf/requirements/cuda-11.2/dev_requirements.txt b/python/cudf/requirements/cuda-11.2/dev_requirements.txt index f70a021c3f8..db434b7c8ec 100644 --- a/python/cudf/requirements/cuda-11.2/dev_requirements.txt +++ b/python/cudf/requirements/cuda-11.2/dev_requirements.txt @@ -1,5 +1,8 @@ # Copyright (c) 2021, NVIDIA CORPORATION. +# pyarrow gpu package will have to be built from source : +# https://arrow.apache.org/docs/python/install.html#installing-from-source + cupy-cuda112 cachetools cmake From 998516f4e8503c1ecdffaad96b36125eb780809f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 16:23:21 -0700 Subject: [PATCH 30/35] fix kafka build too --- python/cudf_kafka/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 85401bcc2e6..cd923310c12 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -47,9 +47,9 @@ "*", sources=cython_files, include_dirs=[ - "../../cpp/include/cudf", - "../../cpp/include", - "../../cpp/libcudf_kafka/include/cudf_kafka", + os.path.join(CUDF_ROOT, "../include/cudf"), + os.path.join(CUDF_ROOT, "../include"), + os.path.join(CUDF_ROOT, "../libcudf_kafka/include/cudf_kafka"), os.path.join(CUDF_ROOT, "include"), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), os.path.join( From 7d9776751f94bd9c14acd7b553b2cb3a061f3565 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 16:58:52 -0700 Subject: [PATCH 31/35] remove redundant packages --- python/cudf/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index ad6f008a2f9..630efd5b9ec 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -6,8 +6,6 @@ requires = [ "wheel", "setuptools", "cython>=0.29,<0.30", - "packaging", - "cachetools", ] [tool.black] From ec396e20d12527eedbd6c4f97777962d21bd197b Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 18:58:50 -0700 Subject: [PATCH 32/35] obtain accurate path irrespective of the current working directory --- python/cudf/setup.py | 7 ++++++- python/cudf_kafka/setup.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 136923a2f48..583339f3818 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -89,7 +89,12 @@ def get_cuda_version_from_header(cuda_include_dir, delimeter=""): "cupy-cuda" + get_cuda_version_from_header(cuda_include_dir) ) -CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") +CUDF_ROOT = os.environ.get( + "CUDF_ROOT", + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + ), +) try: nthreads = int(os.environ.get("PARALLEL_LEVEL", "0") or "0") diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index cd923310c12..5623a185fb2 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -32,7 +32,12 @@ cuda_include_dir = os.path.join(CUDA_HOME, "include") -CUDF_ROOT = os.environ.get("CUDF_ROOT", "../../cpp/build/") +CUDF_ROOT = os.environ.get( + "CUDF_ROOT", + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + ), +) CUDF_KAFKA_ROOT = os.environ.get( "CUDF_KAFKA_ROOT", "../../libcudf_kafka/build" ) From 0b7a74f7999c6489edd20bbc79438d686c215d0e Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 31 Mar 2021 19:55:55 -0700 Subject: [PATCH 33/35] resolve full path --- python/cudf/setup.py | 12 +++++++----- python/cudf_kafka/setup.py | 14 +++++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 583339f3818..8e691b99d25 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -91,8 +91,10 @@ def get_cuda_version_from_header(cuda_include_dir, delimeter=""): CUDF_ROOT = os.environ.get( "CUDF_ROOT", - os.path.join( - os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + os.path.abspath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + ) ), ) @@ -159,9 +161,9 @@ def run(self): "*", sources=cython_files, include_dirs=[ - os.path.join(CUDF_ROOT, "../include/cudf"), - os.path.join(CUDF_ROOT, "../include"), - os.path.join(CUDF_ROOT, "include"), + os.path.abspath(os.path.join(CUDF_ROOT, "../include/cudf")), + os.path.abspath(os.path.join(CUDF_ROOT, "../include")), + os.path.abspath(os.path.join(CUDF_ROOT, "include")), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), os.path.join(CUDF_ROOT, "_deps/dlpack-src/include"), os.path.join( diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 5623a185fb2..f7523dda503 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -34,8 +34,10 @@ CUDF_ROOT = os.environ.get( "CUDF_ROOT", - os.path.join( - os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + os.path.abspath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "../../cpp/build/" + ) ), ) CUDF_KAFKA_ROOT = os.environ.get( @@ -52,9 +54,11 @@ "*", sources=cython_files, include_dirs=[ - os.path.join(CUDF_ROOT, "../include/cudf"), - os.path.join(CUDF_ROOT, "../include"), - os.path.join(CUDF_ROOT, "../libcudf_kafka/include/cudf_kafka"), + os.path.abspath(os.path.join(CUDF_ROOT, "../include/cudf")), + os.path.abspath(os.path.join(CUDF_ROOT, "../include")), + os.path.abspath( + os.path.join(CUDF_ROOT, "../libcudf_kafka/include/cudf_kafka") + ), os.path.join(CUDF_ROOT, "include"), os.path.join(CUDF_ROOT, "_deps/libcudacxx-src/include"), os.path.join( From 30ec5dd400e4e2d98c8f0ef6aefcbec96f870399 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 1 Apr 2021 09:02:01 -0500 Subject: [PATCH 34/35] Update conda/environments/cudf_dev_cuda11.1.yml Co-authored-by: AJ Schmidt --- conda/environments/cudf_dev_cuda11.1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/cudf_dev_cuda11.1.yml b/conda/environments/cudf_dev_cuda11.1.yml index 0a2bb113d7e..fae94974502 100644 --- a/conda/environments/cudf_dev_cuda11.1.yml +++ b/conda/environments/cudf_dev_cuda11.1.yml @@ -42,7 +42,7 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask>=2.22.0 + - dask>=2021.3.1 - distributed>=2.22.0 - streamz - dlpack From b13d04393922cc3b771f4b6848a42cd1c3c27085 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 1 Apr 2021 09:02:08 -0500 Subject: [PATCH 35/35] Update conda/environments/cudf_dev_cuda11.2.yml Co-authored-by: AJ Schmidt --- conda/environments/cudf_dev_cuda11.2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 0c2c5aed568..fa0a69ad1ea 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -42,7 +42,7 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask>=2.22.0 + - dask>=2021.3.1 - distributed>=2.22.0 - streamz - dlpack