diff --git a/build.sh b/build.sh index e5beb51dedf..3945f072cf9 100755 --- a/build.sh +++ b/build.sh @@ -332,8 +332,7 @@ fi if buildAll || hasArg cudf; then cd ${REPODIR}/python/cudf - SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} ${EXTRA_CMAKE_ARGS}" \ - SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \ + SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};-DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES};${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps . fi @@ -369,15 +368,12 @@ fi # build cudf_kafka Python package if hasArg cudf_kafka; then cd ${REPODIR}/python/cudf_kafka - SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} ${EXTRA_CMAKE_ARGS}" \ - SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \ + SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR};${EXTRA_CMAKE_ARGS}" python -m pip install --no-build-isolation --no-deps . fi # build custreamz Python package if hasArg custreamz; then cd ${REPODIR}/python/custreamz - SKBUILD_CONFIGURE_OPTIONS="-DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR}" \ - SKBUILD_BUILD_OPTIONS="-j${PARALLEL_LEVEL:-1}" \ - python -m pip install --no-build-isolation --no-deps . + python -m pip install --no-build-isolation --no-deps . fi diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index d36b609799b..be5705a9548 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -23,7 +23,7 @@ manylinux="manylinux_${manylinux_version}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep -python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas_tests] +python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,pandas-tests] git checkout $COMMIT diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index 7eab3221e5e..4f1e4bbf993 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -46,7 +46,7 @@ else RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep - python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf_pandas_tests] + python -m pip install $(ls ./local-cudf-dep/cudf*.whl)[test,cudf-pandas-tests] fi python -m pytest -p cudf.pandas ./python/cudf/cudf_pandas_tests/ diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8e0a7bc5495..c42a4718aca 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -86,7 +86,7 @@ dependencies: - rich - rmm==24.2.* - s3fs>=2022.3.0 -- scikit-build>=0.13.1 +- scikit-build-core>=0.7.0 - scipy - spdlog>=1.12.0,<1.13 - sphinx diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index e52e0adb163..6a134c9ace6 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -83,7 +83,7 @@ dependencies: - rich - rmm==24.2.* - s3fs>=2022.3.0 -- scikit-build>=0.13.1 +- scikit-build-core>=0.7.0 - scipy - spdlog>=1.12.0,<1.13 - sphinx diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 27edde1c98a..bc91ee61f6f 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -58,7 +58,7 @@ requirements: - protobuf ==4.24.* - python - cython >=3.0.3 - - scikit-build >=0.13.1 + - scikit-build-core >=0.7.0 - setuptools - dlpack >=0.5,<0.6.0a0 - pyarrow ==14.0.1.* diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index 343ec2519f1..872324d3f73 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -57,7 +57,7 @@ requirements: - cuda-version ={{ cuda_version }} - cudf ={{ version }} - libcudf_kafka ={{ version }} - - scikit-build >=0.13.1 + - scikit-build-core >=0.7.0 - setuptools {% if cuda_major == "12" %} - cuda-cudart-dev diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index d971a32db70..1bead93c9cc 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -77,6 +77,7 @@ list(POP_BACK CMAKE_PREFIX_PATH) # benchmarks will not work without updating GBench (and possibly NVBench) builds. We are currently # ignoring these limitations since we don't anticipate using this feature except for building # wheels. + enable_language(C) execute_process( COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6 OUTPUT_VARIABLE GLIBC_EXECUTABLE diff --git a/dependencies.yaml b/dependencies.yaml index 4a1c2ad1cd3..ebf454a9522 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -71,10 +71,8 @@ files: extras: table: build-system includes: - - build_all - build_python_common - build_python_cudf - - build_wheels py_run_cudf: output: pyproject pyproject_dir: python/cudf @@ -99,7 +97,7 @@ files: pyproject_dir: python/cudf extras: table: project.optional-dependencies - key: pandas_tests + key: pandas-tests includes: - test_python_pandas_cudf py_test_cudf_pandas: @@ -107,7 +105,7 @@ files: pyproject_dir: python/cudf extras: table: project.optional-dependencies - key: cudf_pandas_tests + key: cudf-pandas-tests includes: - test_python_cudf_pandas py_build_dask_cudf: @@ -143,7 +141,6 @@ files: table: build-system includes: - build_python_common - - build_wheels py_run_cudf_kafka: output: pyproject pyproject_dir: python/cudf_kafka @@ -193,12 +190,10 @@ channels: dependencies: build_all: common: - - output_types: [conda, requirements, pyproject] - packages: - - &cmake_ver cmake>=3.26.4 - - ninja - output_types: conda packages: + - &cmake_ver cmake>=3.26.4 + - &ninja ninja - c-compiler - cxx-compiler - dlpack>=0.5,<0.6.0a0 @@ -257,15 +252,20 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: + - *cmake_ver - cython>=3.0.3 + - *ninja # TODO: Pin to numpy<1.25 until cudf requires pandas 2 - &numpy numpy>=1.21,<1.25 - - scikit-build>=0.13.1 - - output_types: [conda, requirements, pyproject] - packages: # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - pyarrow==14.0.1.* + - output_types: conda + packages: + - scikit-build-core>=0.7.0 + - output_types: [requirements, pyproject] + packages: + - scikit-build-core[pyproject]>=0.7.0 build_python_cudf: common: - output_types: conda diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 8a9ea00d640..b3e1f13ab85 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -23,11 +23,7 @@ rapids_cuda_init_architectures(cudf-python) project( cudf-python VERSION ${cudf_version} - LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C - # language to be enabled here. The test project that is built in scikit-build to verify - # various linking options for the python library is hardcoded to build with C, so until - # that is fixed we need to keep C. - C CXX CUDA + LANGUAGES CXX CUDA ) option(FIND_CUDF_CPP "Search for existing CUDF C++ installations before defaulting to local files" @@ -68,7 +64,7 @@ else() set(cudf_FOUND OFF) endif() -include(rapids-cython) +include(rapids-cython-core) if(NOT cudf_FOUND) set(BUILD_TESTS OFF) diff --git a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake b/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake index 8dc47c470ba..d432f9fe1f5 100644 --- a/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake +++ b/python/cudf/cmake/Modules/LinkPyarrowHeaders.cmake @@ -13,22 +13,7 @@ # ============================================================================= include_guard(GLOBAL) -# TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was -# fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in -# CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24. -# find_package(Python REQUIRED COMPONENTS Development NumPy) - -# Note: The bug noted above prevents us from finding NumPy successfully using FindPython.cmake -# inside the manylinux images used to build wheels because manylinux images do not contain -# libpython.so and therefore Development cannot be found. Until we upgrade to CMake 3.24, we should -# use FindNumpy.cmake instead (provided by scikit-build). When we switch to 3.24 we can try -# switching back, but it may not work if that implicitly still requires Python libraries. In that -# case we'll need to follow up with the CMake team to remove that dependency. The stopgap solution -# is to unpack the static lib tarballs in the wheel building jobs so that there are at least static -# libs to be found, but that should be a last resort since it implies a dependency that isn't really -# necessary. The relevant command is tar -xf /opt/_internal/static-libs-for-embedding-only.tar.xz -C -# /opt/_internal" -find_package(NumPy REQUIRED) +find_package(Python REQUIRED COMPONENTS Development NumPy) execute_process( COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())" @@ -49,7 +34,7 @@ endif() function(link_to_pyarrow_headers targets) foreach(target IN LISTS targets) # PyArrow headers require numpy headers. - target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}") + target_include_directories(${target} PRIVATE "${Python_NumPy_INCLUDE_DIRS}") target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}") endforeach() endfunction() diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 6d4c2a33316..b67c26f779f 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -63,7 +63,7 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf ) -target_link_libraries(strings_udf cudf_strings_udf) +target_link_libraries(strings_udf PUBLIC cudf_strings_udf) link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}") diff --git a/python/cudf/cudf/_lib/io/CMakeLists.txt b/python/cudf/cudf/_lib/io/CMakeLists.txt index 1f93bfa8b55..2408fa1c12f 100644 --- a/python/cudf/cudf/_lib/io/CMakeLists.txt +++ b/python/cudf/cudf/_lib/io/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,8 +20,4 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX io_ ASSOCIATED_TARGETS cudf ) -set(targets_using_numpy_and_arrow_headers io_datasource io_utils) -foreach(target IN LISTS targets_using_numpy_and_arrow_headers) - target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}") - target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}") -endforeach() +link_to_pyarrow_headers("${RAPIDS_CYTHON_CREATED_TARGETS}") diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index d32284c0c5d..879604ce0fc 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -1,7 +1,7 @@ # Copyright (c) 2021-2023, NVIDIA CORPORATION. [build-system] -build-backend = "setuptools.build_meta" +build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.3", @@ -10,9 +10,7 @@ requires = [ "protoc-wheel", "pyarrow==14.0.1.*", "rmm==24.2.*", - "scikit-build>=0.13.1", - "setuptools", - "wheel", + "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] @@ -71,7 +69,7 @@ test = [ "transformers==4.24.0", "tzdata", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -pandas_tests = [ +pandas-tests = [ "beautifulsoup4", "blosc", "boto3", @@ -114,7 +112,7 @@ pandas_tests = [ "xlwt", "zstandard", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -cudf_pandas_tests = [ +cudf-pandas-tests = [ "ipython", "openpyxl", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -123,12 +121,6 @@ cudf_pandas_tests = [ Homepage = "https://github.com/rapidsai/cudf" Documentation = "https://docs.rapids.ai/api/cudf/stable/" -[tool.setuptools] -license-files = ["LICENSE"] - -[tool.setuptools.dynamic] -version = {file = "cudf/VERSION"} - [tool.isort] line_length = 79 multi_line_output = 3 @@ -171,3 +163,16 @@ skip = [ "dist", "__init__.py", ] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["cudf"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "cudf/VERSION" +regex = "(?P.*)" diff --git a/python/cudf/setup.py b/python/cudf/setup.py deleted file mode 100644 index 984cd63a7c9..00000000000 --- a/python/cudf/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. - -from setuptools import find_packages -from skbuild import setup - -packages = find_packages(include=["cudf*", "udf_cpp*"]) -setup( - packages=packages, - package_data={ - key: ["VERSION", "*.pxd", "*.hpp", "*.cuh"] for key in packages - }, - zip_safe=False, -) diff --git a/python/cudf_kafka/CMakeLists.txt b/python/cudf_kafka/CMakeLists.txt index 1e21c873585..116bc7a3beb 100644 --- a/python/cudf_kafka/CMakeLists.txt +++ b/python/cudf_kafka/CMakeLists.txt @@ -21,11 +21,7 @@ include(../../fetch_rapids.cmake) project( cudf-kafka-python VERSION ${cudf_kafka_version} - LANGUAGES # TODO: Building Python extension modules via the python_extension_module requires the C - # language to be enabled here. The test project that is built in scikit-build to verify - # various linking options for the python library is hardcoded to build with C, so until - # that is fixed we need to keep C. - C CXX + LANGUAGES CXX ) find_package(cudf_kafka ${cudf_kafka_version} REQUIRED) @@ -37,7 +33,7 @@ if(NOT cudf_kafka_FOUND) ) endif() -include(rapids-cython) +include(rapids-cython-core) rapids_cython_init() add_subdirectory(cudf_kafka/_lib) diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 05b1db3b9ac..062a0224c1f 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -1,14 +1,14 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. [build-system] - +build-backend = "scikit_build_core.build" requires = [ + "cmake>=3.26.4", "cython>=3.0.3", + "ninja", "numpy>=1.21,<1.25", "pyarrow==14.0.1.*", - "scikit-build>=0.13.1", - "setuptools", - "wheel", + "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] @@ -36,12 +36,6 @@ test = [ Homepage = "https://github.com/rapidsai/cudf" Documentation = "https://docs.rapids.ai/api/cudf/stable/" -[tool.setuptools] -license-files = ["LICENSE"] - -[tool.setuptools.dynamic] -version = {file = "cudf_kafka/VERSION"} - [tool.isort] line_length = 79 multi_line_output = 3 @@ -87,3 +81,16 @@ skip = [ "dist", "__init__.py", ] + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["cudf_kafka"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "cudf_kafka/VERSION" +regex = "(?P.*)" diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py deleted file mode 100644 index 6a99e9ed968..00000000000 --- a/python/cudf_kafka/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. -from setuptools import find_packages -from skbuild import setup - -packages = find_packages(include=["cudf_kafka*"]) - -setup( - packages=packages, - package_data={ - key: ["VERSION", "*.pxd", "*.hpp", "*.cuh"] for key in packages - }, - zip_safe=False, -) diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py deleted file mode 100644 index 04943bf88e2..00000000000 --- a/python/custreamz/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -from setuptools import setup - -setup( - package_data={"custreamz": ["VERSION"]}, -) diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 890be46b974..55481dc473f 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -9,7 +9,7 @@ requires = [ [project] name = "dask_cudf" -dynamic = ["version", "entry-points"] +dynamic = ["version"] description = "Utilities for Dask and cuDF interactions" readme = { file = "README.md", content-type = "text/markdown" } authors = [ @@ -35,6 +35,10 @@ classifiers = [ "Programming Language :: Python :: 3.10", ] +[project.entry-points."dask.dataframe.backends"] +cudf = "dask_cudf.backends:CudfBackendEntrypoint" + + [project.optional-dependencies] test = [ "dask-cuda==24.2.*", diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py deleted file mode 100644 index c6ce219d32f..00000000000 --- a/python/dask_cudf/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright (c) 2019-2023, NVIDIA CORPORATION. - -from setuptools import find_packages, setup - -packages = find_packages(exclude=["tests", "tests.*"]) - -setup( - include_package_data=True, - packages=packages, - package_data={key: ["VERSION"] for key in packages}, - entry_points={ - "dask.dataframe.backends": [ - "cudf = dask_cudf.backends:CudfBackendEntrypoint", - ] - }, - zip_safe=False, -)