diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000000..7f1c708c9a7 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,77 @@ +name: cuDF wheels + +on: + workflow_call: + inputs: + versioneer-override: + type: string + default: '' + build-tag: + type: string + default: '' + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + build-type: + type: string + default: nightly + +concurrency: + group: "cudf-${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +jobs: + cudf-wheels: + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux.yml@main + with: + repo: rapidsai/cudf + + build-type: ${{ inputs.build-type }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + + package-dir: python/cudf + package-name: cudf + + python-package-versioneer-override: ${{ inputs.versioneer-override }} + python-package-build-tag: ${{ inputs.build-tag }} + + skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" + + test-extras: test + + # Have to manually specify the cupy install location on arm. + # Have to also manually install tokenizers==0.10.2, which is the last tokenizers + # to have a binary aarch64 wheel available on PyPI + # Otherwise, the tokenizers sdist is used, which needs a Rust compiler + test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64" + + test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests" + secrets: inherit + dask_cudf-wheel: + needs: cudf-wheels + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure.yml@main + with: + repo: rapidsai/cudf + + build-type: ${{ inputs.build-type }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + + package-dir: python/dask_cudf + package-name: dask_cudf + + python-package-versioneer-override: ${{ inputs.versioneer-override }} + python-package-build-tag: ${{ inputs.build-tag }} + + test-extras: test + test-unittest: "pytest -v -n 8 ./python/dask_cudf/dask_cudf/tests" + secrets: inherit diff --git a/.gitignore b/.gitignore index 91a7ecc49f7..1867e65b7be 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,6 @@ docs/cudf/source/api_docs/generated/* docs/cudf/source/api_docs/api/* docs/cudf/source/user_guide/example_output/* docs/cudf/source/user_guide/cudf.*Dtype.*.rst + +# cibuildwheel +/wheelhouse diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7e8ee5b60bf..79951cdabbe 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -48,6 +48,8 @@ option(BUILD_TESTS "Configure CMake to build tests" ON) option(BUILD_BENCHMARKS "Configure CMake to build (google & nvbench) benchmarks" OFF) option(BUILD_SHARED_LIBS "Build cuDF shared libraries" ON) option(JITIFY_USE_CACHE "Use a file cache for JIT compiled kernels" ON) +option(CUDF_BUILD_TESTUTIL "Whether to build the test utilities contained in libcudf" ON) +mark_as_advanced(CUDF_BUILD_TESTUTIL) option(CUDF_USE_PROPRIETARY_NVCOMP "Download and use NVCOMP with proprietary extensions" ON) option(CUDF_USE_ARROW_STATIC "Build and statically link Arrow libraries" OFF) option(CUDF_ENABLE_ARROW_ORC "Build the Arrow ORC adapter" OFF) @@ -94,6 +96,12 @@ message(VERBOSE "CUDF: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}" rapids_cmake_build_type("Release") set(CUDF_BUILD_TESTS ${BUILD_TESTS}) set(CUDF_BUILD_BENCHMARKS ${BUILD_BENCHMARKS}) +if(BUILD_TESTS AND NOT CUDF_BUILD_TESTUTIL) + message( + FATAL_ERROR + "Tests cannot be built without building cudf test utils. Please set CUDF_BUILD_TESTUTIL=ON or BUILD_TESTS=OFF" + ) +endif() set(CUDF_CXX_FLAGS "") set(CUDF_CUDA_FLAGS "") @@ -133,12 +141,14 @@ include(cmake/Modules/ConfigureCUDA.cmake) # set other CUDA compilation flags # find zlib rapids_find_package(ZLIB REQUIRED) -# find Threads (needed by cudftestutil) -rapids_find_package( - Threads REQUIRED - BUILD_EXPORT_SET cudf-exports - INSTALL_EXPORT_SET cudf-exports -) +if(CUDF_BUILD_TESTUTIL) + # find Threads (needed by cudftestutil) + rapids_find_package( + Threads REQUIRED + BUILD_EXPORT_SET cudf-exports + INSTALL_EXPORT_SET cudf-exports + ) +endif() # add third party dependencies using CPM rapids_cpm_init() @@ -160,7 +170,9 @@ rapids_cpm_libcudacxx(BUILD_EXPORT_SET cudf-exports INSTALL_EXPORT_SET cudf-expo # find cuCollections Should come after including thrust and libcudacxx include(cmake/thirdparty/get_cucollections.cmake) # find or install GoogleTest -include(cmake/thirdparty/get_gtest.cmake) +if(CUDF_BUILD_TESTUTIL) + include(cmake/thirdparty/get_gtest.cmake) +endif() # preprocess jitify-able kernels include(cmake/Modules/JitifyPreprocessKernels.cmake) # find cuFile @@ -693,46 +705,48 @@ add_library(cudf::cudf ALIAS cudf) # ################################################################################################## # * build cudftestutil ---------------------------------------------------------------------------- -add_library( - cudftestutil STATIC - tests/io/metadata_utilities.cpp - tests/utilities/base_fixture.cpp - tests/utilities/column_utilities.cu - tests/utilities/table_utilities.cu - tests/utilities/tdigest_utilities.cu -) +if(CUDF_BUILD_TESTUTIL) + add_library( + cudftestutil STATIC + tests/io/metadata_utilities.cpp + tests/utilities/base_fixture.cpp + tests/utilities/column_utilities.cu + tests/utilities/table_utilities.cu + tests/utilities/tdigest_utilities.cu + ) -set_target_properties( - cudftestutil - PROPERTIES BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" - # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON - INTERFACE_POSITION_INDEPENDENT_CODE ON -) + set_target_properties( + cudftestutil + PROPERTIES BUILD_RPATH "\$ORIGIN" + INSTALL_RPATH "\$ORIGIN" + # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CUDA_STANDARD 17 + CUDA_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + INTERFACE_POSITION_INDEPENDENT_CODE ON + ) -target_compile_options( - cudftestutil PUBLIC "$:${CUDF_CXX_FLAGS}>>" - "$:${CUDF_CUDA_FLAGS}>>" -) + target_compile_options( + cudftestutil PUBLIC "$:${CUDF_CXX_FLAGS}>>" + "$:${CUDF_CUDA_FLAGS}>>" + ) -target_link_libraries( - cudftestutil - PUBLIC GTest::gmock GTest::gtest Threads::Threads cudf - PRIVATE $ -) + target_link_libraries( + cudftestutil + PUBLIC GTest::gmock GTest::gtest Threads::Threads cudf + PRIVATE $ + ) -target_include_directories( - cudftestutil PUBLIC "$" - "$" -) + target_include_directories( + cudftestutil PUBLIC "$" + "$" + ) -add_library(cudf::cudftestutil ALIAS cudftestutil) + add_library(cudf::cudftestutil ALIAS cudftestutil) +endif() # ################################################################################################## # * add tests ------------------------------------------------------------------------------------- @@ -787,24 +801,26 @@ install(DIRECTORY ${CUDF_SOURCE_DIR}/include/cudf ${CUDF_SOURCE_DIR}/include/cud ${CUDF_SOURCE_DIR}/include/nvtext DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) -install( - TARGETS cudftestutil - DESTINATION ${lib_dir} - EXPORT cudf-testing-exports -) +if(CUDF_BUILD_TESTUTIL) + install( + TARGETS cudftestutil + DESTINATION ${lib_dir} + EXPORT cudf-testing-exports + ) -install( - EXPORT cudf-testing-exports - FILE cudf-testing-targets.cmake - NAMESPACE cudf:: - DESTINATION "${lib_dir}/cmake/cudf" -) + install( + EXPORT cudf-testing-exports + FILE cudf-testing-targets.cmake + NAMESPACE cudf:: + DESTINATION "${lib_dir}/cmake/cudf" + ) -include("${rapids-cmake-dir}/export/write_dependencies.cmake") -rapids_export_write_dependencies( - INSTALL cudf-testing-exports - "${PROJECT_BINARY_DIR}/rapids-cmake/cudf/export/cudf-testing-dependencies.cmake" -) + include("${rapids-cmake-dir}/export/write_dependencies.cmake") + rapids_export_write_dependencies( + INSTALL cudf-testing-exports + "${PROJECT_BINARY_DIR}/rapids-cmake/cudf/export/cudf-testing-dependencies.cmake" + ) +endif() set(doc_string [=[ @@ -894,6 +910,7 @@ if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/cudf-testing-targets.cmake") endif() ]=] ) + string(APPEND build_code_string "${common_code_string}") rapids_export( @@ -905,15 +922,16 @@ rapids_export( FINAL_CODE_BLOCK build_code_string ) -export( - EXPORT cudf-testing-exports - FILE ${CUDF_BINARY_DIR}/cudf-testing-targets.cmake - NAMESPACE cudf:: -) -rapids_export_write_dependencies( - BUILD cudf-testing-exports "${CUDF_BINARY_DIR}/cudf-testing-dependencies.cmake" -) - +if(CUDF_BUILD_TESTUTIL) + export( + EXPORT cudf-testing-exports + FILE ${CUDF_BINARY_DIR}/cudf-testing-targets.cmake + NAMESPACE cudf:: + ) + rapids_export_write_dependencies( + BUILD cudf-testing-exports "${CUDF_BINARY_DIR}/cudf-testing-dependencies.cmake" + ) +endif() # ################################################################################################## # * make documentation ---------------------------------------------------------------------------- diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 8a3224237b6..9e5f845496e 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -31,9 +31,15 @@ project( option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" OFF ) +option(CUDF_BUILD_WHEELS "Whether this build is generating a Python wheel." OFF) option(USE_LIBARROW_FROM_PYARROW "Use the libarrow contained within pyarrow." OFF) mark_as_advanced(USE_LIBARROW_FROM_PYARROW) +# Always build wheels against the pyarrow libarrow. +if(CUDF_BUILD_WHEELS) + set(USE_LIBARROW_FROM_PYARROW ON) +endif() + # If the user requested it we attempt to find CUDF. if(FIND_CUDF_CPP) if(USE_LIBARROW_FROM_PYARROW) @@ -69,8 +75,32 @@ if(NOT cudf_FOUND) set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) - add_subdirectory(../../cpp cudf-cpp) + set(_exclude_from_all "") + if(CUDF_BUILD_WHEELS) + # We don't build C++ tests when building wheels, so we can also omit the test util and shrink + # the wheel by avoiding embedding GTest. + set(CUDF_BUILD_TESTUTIL OFF) + + # Statically link cudart if building wheels + set(CUDA_STATIC_RUNTIME ON) + + # Need to set this so all the nvcomp targets are global, not only nvcomp::nvcomp + # https://cmake.org/cmake/help/latest/variable/CMAKE_FIND_PACKAGE_TARGETS_GLOBAL.html#variable:CMAKE_FIND_PACKAGE_TARGETS_GLOBAL + set(CMAKE_FIND_PACKAGE_TARGETS_GLOBAL ON) + + # Don't install the cuDF C++ targets into wheels + set(_exclude_from_all EXCLUDE_FROM_ALL) + endif() + + add_subdirectory(../../cpp cudf-cpp ${_exclude_from_all}) + + if(CUDF_BUILD_WHEELS) + include(cmake/Modules/WheelHelpers.cmake) + get_target_property(_nvcomp_link_libs nvcomp::nvcomp INTERFACE_LINK_LIBRARIES) + # Ensure all the shared objects we need at runtime are in the wheel + add_target_libs_to_wheel(LIB_DIR cudf TARGETS arrow_shared nvcomp::nvcomp ${_nvcomp_link_libs}) + endif() # Since there are multiple subpackages of cudf._lib that require access to libcudf, we place the # library in the cudf directory as a single source of truth and modify the other rpaths # appropriately. diff --git a/python/cudf/LICENSE b/python/cudf/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/cudf/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/cudf/_custom_build/backend.py b/python/cudf/_custom_build/backend.py new file mode 100644 index 00000000000..37b7edf2432 --- /dev/null +++ b/python/cudf/_custom_build/backend.py @@ -0,0 +1,37 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +"""Custom build backend for cudf to get versioned requirements. + +Based on https://setuptools.pypa.io/en/latest/build_meta.html +""" +import os +from functools import wraps + +from setuptools import build_meta as _orig + +# Alias the required bits +build_wheel = _orig.build_wheel +build_sdist = _orig.build_sdist + + +def replace_requirements(func): + @wraps(func) + def wrapper(config_settings=None): + orig_list = getattr(_orig, func.__name__)(config_settings) + append_list = [ + f"rmm{os.getenv('RAPIDS_PY_WHEEL_CUDA_SUFFIX', default='')}" + ] + return orig_list + append_list + + return wrapper + + +get_requires_for_build_wheel = replace_requirements( + _orig.get_requires_for_build_wheel +) +get_requires_for_build_sdist = replace_requirements( + _orig.get_requires_for_build_sdist +) +get_requires_for_build_editable = replace_requirements( + _orig.get_requires_for_build_editable +) diff --git a/python/cudf/cmake/Modules/WheelHelpers.cmake b/python/cudf/cmake/Modules/WheelHelpers.cmake new file mode 100644 index 00000000000..28ea33240fa --- /dev/null +++ b/python/cudf/cmake/Modules/WheelHelpers.cmake @@ -0,0 +1,71 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +include_guard(GLOBAL) + +# Making libraries available inside wheels by installing the associated targets. +function(add_target_libs_to_wheel) + list(APPEND CMAKE_MESSAGE_CONTEXT "add_target_libs_to_wheel") + + set(options "") + set(one_value "LIB_DIR") + set(multi_value "TARGETS") + cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN}) + + message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__LIB_DIR}'") + + foreach(target IN LISTS __TARGETS) + + if(NOT TARGET ${target}) + message(VERBOSE "No target named ${target}") + continue() + endif() + + get_target_property(alias_target ${target} ALIASED_TARGET) + if(alias_target) + set(target ${alias_target}) + endif() + + get_target_property(is_imported ${target} IMPORTED) + if(NOT is_imported) + # If the target isn't imported, install it into the the wheel + install(TARGETS ${target} DESTINATION ${__LIB_DIR}) + message(VERBOSE "install(TARGETS ${target} DESTINATION ${__LIB_DIR})") + else() + # If the target is imported, make sure it's global + get_target_property(already_global ${target} IMPORTED_GLOBAL) + if(NOT already_global) + set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + endif() + + # Find the imported target's library so we can copy it into the wheel + set(lib_loc) + foreach(prop IN ITEMS IMPORTED_LOCATION IMPORTED_LOCATION_RELEASE IMPORTED_LOCATION_DEBUG) + get_target_property(lib_loc ${target} ${prop}) + if(lib_loc) + message(VERBOSE "Found ${prop} for ${target}: ${lib_loc}") + break() + endif() + message(VERBOSE "${target} has no value for property ${prop}") + endforeach() + + if(NOT lib_loc) + message(FATAL_ERROR "Found no libs to install for target ${target}") + endif() + + # Copy the imported library into the wheel + install(FILES ${lib_loc} DESTINATION ${__LIB_DIR}) + message(VERBOSE "install(FILES ${lib_loc} DESTINATION ${__LIB_DIR})") + endif() + endforeach() +endfunction() diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index df17b8f2032..c6d4becdbec 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -54,6 +54,7 @@ set(cython_sources utils.pyx ) set(linked_libraries cudf::cudf) + rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" @@ -63,10 +64,24 @@ rapids_cython_create_modules( # TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was # fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in # CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24. -find_package(Python REQUIRED COMPONENTS Development NumPy) +# find_package(Python REQUIRED COMPONENTS Development NumPy) + +# Note: The bug noted above prevents us from finding NumPy successfully using FindPython.cmake +# inside the manylinux images used to build wheels because manylinux images do not contain +# libpython.so and therefore Development cannot be found. Until we upgrade to CMake 3.24, we should +# use FindNumpy.cmake instead (provided by scikit-build). When we switch to 3.24 we can try +# switching back, but it may not work if that implicitly still requires Python libraries. In that +# case we'll need to follow up with the CMake team to remove that dependency. The stopgap solution +# is to unpack the static lib tarballs in the wheel building jobs so that there are at least static +# libs to be found, but that should be a last resort since it implies a dependency that isn't really +# necessary. The relevant command is tar -xf /opt/_internal/static-libs-for-embedding-only.tar.xz -C +# /opt/_internal" +find_package(NumPy REQUIRED) set(targets_using_numpy interop avro csv orc json parquet) foreach(target IN LISTS targets_using_numpy) - target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}") + target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}") + # Switch to the line below when we switch back to FindPython.cmake in CMake 3.24. + # target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}") endforeach() add_subdirectory(io) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 52490444dba..92b86649564 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -9,4 +9,10 @@ requires = [ "scikit-build>=0.13.1", "cmake>=3.23.1", "ninja", + "numpy", + "pyarrow==9.0.0", + "protoc-wheel", + "versioneer", ] +build-backend = "backend" +backend-path = ["_custom_build"] diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 9f22f87e240..2d5defc2849 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -1,16 +1,16 @@ # Copyright (c) 2018-2022, NVIDIA CORPORATION. import os -import re -import shutil import versioneer from setuptools import find_packages from skbuild import setup +cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") + install_requires = [ "cachetools", - "cuda-python>=11.5,<11.7.1", + "cuda-python>=11.7.1,<12.0", "fsspec>=0.6.0", "numba>=0.56.2", "numpy", @@ -19,6 +19,11 @@ "pandas>=1.0,<1.6.0dev0", "protobuf>=3.20.1,<3.21.0a0", "typing_extensions", + "pyarrow==9.0.0", + f"rmm{cuda_suffix}", + f"ptxcompiler{cuda_suffix}", + f"cubinlinker{cuda_suffix}", + "cupy-cuda11x", ] extras_require = { @@ -33,55 +38,24 @@ "pyorc", "msgpack", "transformers<=4.10.3", + "tzdata", ] } +if "RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE" in os.environ: + orig_get_versions = versioneer.get_versions -def get_cuda_version_from_header(cuda_include_dir, delimeter=""): - - cuda_version = None - - with open(os.path.join(cuda_include_dir, "cuda.h"), encoding="utf-8") as f: - for line in f.readlines(): - if re.search(r"#define CUDA_VERSION ", line) is not None: - cuda_version = line - break - - if cuda_version is None: - raise TypeError("CUDA_VERSION not found in cuda.h") - cuda_version = int(cuda_version.split()[2]) - return "%d%s%d" % ( - cuda_version // 1000, - delimeter, - (cuda_version % 1000) // 10, - ) - + version_override = os.environ["RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE"] -CUDA_HOME = os.environ.get("CUDA_HOME", False) -if not CUDA_HOME: - path_to_cuda_gdb = shutil.which("cuda-gdb") - if path_to_cuda_gdb is None: - raise OSError( - "Could not locate CUDA. " - "Please set the environment variable " - "CUDA_HOME to the path to the CUDA installation " - "and try again." - ) - CUDA_HOME = os.path.dirname(os.path.dirname(path_to_cuda_gdb)) - -if not os.path.isdir(CUDA_HOME): - raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") - -cuda_include_dir = os.path.join(CUDA_HOME, "include") -install_requires.append( - "cupy-cuda" - + get_cuda_version_from_header(cuda_include_dir) - + ">=9.5.0,<12.0.0a0" -) + def get_versions(): + data = orig_get_versions() + data["version"] = version_override + return data + versioneer.get_versions = get_versions setup( - name="cudf", + name=f"cudf{cuda_suffix}", version=versioneer.get_version(), description="cuDF - GPU Dataframe", url="https://github.com/rapidsai/cudf", @@ -96,11 +70,12 @@ def get_cuda_version_from_header(cuda_include_dir, delimeter=""): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", ], + cmdclass=versioneer.get_cmdclass(), + include_package_data=True, packages=find_packages(include=["cudf", "cudf.*"]), package_data={ key: ["*.pxd"] for key in find_packages(include=["cudf._lib*"]) }, - cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, extras_require=extras_require, zip_safe=False, diff --git a/python/dask_cudf/LICENSE b/python/dask_cudf/LICENSE new file mode 120000 index 00000000000..30cff7403da --- /dev/null +++ b/python/dask_cudf/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 4fa2af89b9d..d9d4da9c4ab 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,19 +1,20 @@ # Copyright (c) 2019-2022, NVIDIA CORPORATION. import os -import re -import shutil import versioneer from setuptools import find_packages, setup +cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") + install_requires = [ - "cudf", - "dask>=2022.9.2", - "distributed>=2022.9.2", + "dask==2022.9.2", + "distributed==2022.9.2", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.6.0dev0", + f"cudf{cuda_suffix}", + "cupy-cuda11x", ] extras_require = { @@ -21,58 +22,25 @@ "numpy", "pandas>=1.0,<1.6.0dev0", "pytest", + "pytest-xdist", "numba>=0.56.2", - "dask>=2021.09.1", - "distributed>=2021.09.1", ] } +if "RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE" in os.environ: + orig_get_versions = versioneer.get_versions -def get_cuda_version_from_header(cuda_include_dir, delimeter=""): - - cuda_version = None - - with open(os.path.join(cuda_include_dir, "cuda.h"), encoding="utf-8") as f: - for line in f.readlines(): - if re.search(r"#define CUDA_VERSION ", line) is not None: - cuda_version = line - break - - if cuda_version is None: - raise TypeError("CUDA_VERSION not found in cuda.h") - cuda_version = int(cuda_version.split()[2]) - return "%d%s%d" % ( - cuda_version // 1000, - delimeter, - (cuda_version % 1000) // 10, - ) + version_override = os.environ["RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE"] + def get_versions(): + data = orig_get_versions() + data["version"] = version_override + return data -CUDA_HOME = os.environ.get("CUDA_HOME", False) -if not CUDA_HOME: - path_to_cuda_gdb = shutil.which("cuda-gdb") - if path_to_cuda_gdb is None: - raise OSError( - "Could not locate CUDA. " - "Please set the environment variable " - "CUDA_HOME to the path to the CUDA installation " - "and try again." - ) - CUDA_HOME = os.path.dirname(os.path.dirname(path_to_cuda_gdb)) - -if not os.path.isdir(CUDA_HOME): - raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") - -cuda_include_dir = os.path.join(CUDA_HOME, "include") -install_requires.append( - "cupy-cuda" - + get_cuda_version_from_header(cuda_include_dir) - + ">=9.5.0,<12.0.0a0" -) - + versioneer.get_versions = get_versions setup( - name="dask-cudf", + name=f"dask-cudf{cuda_suffix}", version=versioneer.get_version(), description="Utilities for Dask and cuDF interactions", url="https://github.com/rapidsai/cudf",