merge branch-0.37

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -14,7 +14,7 @@ dependencies:
 - cmake>=3.26.4
 - cuda-version=11.8
 - cudatoolkit
-- cudf==24.2.*
+- cudf==24.4.*
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
@@ -39,7 +39,7 @@ dependencies:
 - pytest-asyncio
 - pytest-rerunfailures
 - python>=3.9,<3.11
-- rmm==24.2.*
+- rmm==24.4.*
 - scikit-build-core>=0.7.0
 - spdlog>=1.12.0,<1.13
 - ucx

conda/environments/all_cuda-122_arch-x86_64.yaml

@@ -14,7 +14,7 @@ dependencies:
 - cmake>=3.26.4
 - cuda-cudart-dev
 - cuda-version=12.2
-- cudf==24.2.*
+- cudf==24.4.*
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
@@ -39,7 +39,7 @@ dependencies:
 - pytest-asyncio
 - pytest-rerunfailures
 - python>=3.9,<3.11
-- rmm==24.2.*
+- rmm==24.4.*
 - scikit-build-core>=0.7.0
 - spdlog>=1.12.0,<1.13
 - ucx

cpp/CMakeLists.txt

@@ -71,11 +71,25 @@ rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH)
 
 # ##################################################################################################
 # * compiler options ------------------------------------------------------------------------------
-rapids_find_package(
-  ucx REQUIRED
-  BUILD_EXPORT_SET ucxx-exports
-  INSTALL_EXPORT_SET ucxx-exports
-)
+# Due to https://github.com/openucx/ucx/issues/9614, we cannot export the ucx
+# dependency because users would then have no control over whether ucx is found
+# multiple times, causing potential configure errors. Therefore, we use a raw
+# find_package call instead of rapids_find_package and skip exporting the ucx
+# dependency. Consumers of ucxx must find ucx themselves. Once the above issue
+# is resolved (see https://github.com/openucx/ucx/pull/9622) we can remove the
+# above find_package in favor of the commented out lines below. For the same
+# reason, we must also gate this find_package call behind a check for the
+# target already existing so that consumers can use tools like CPM.cmake to
+# either find or build ucxx from source if it cannot be found (i.e. both cases
+# must allow prior finding of ucx).
+if(NOT TARGET ucx::ucp)
+  find_package(ucx REQUIRED)
+endif()
+#rapids_find_package(
+#  ucx REQUIRED
+#  BUILD_EXPORT_SET ucxx-exports
+#  INSTALL_EXPORT_SET ucxx-exports
+#)
 
 # ##################################################################################################
 # * dependencies ----------------------------------------------------------------------------------

dependencies.yaml

@@ -225,7 +225,7 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - cupy>=12.0.0
+          - &cupy_conda cupy>=12.0.0
     specific:
       - output_types: [requirements, pyproject]
         matrices:
@@ -235,7 +235,7 @@ dependencies:
           - matrix: {cuda: "11.*"}
             packages:
               - cupy-cuda11x>=12.0.0
-          - {matrix: null, packages: [cupy-cuda11x>=12.0.0]}
+          - {matrix: null, packages: [*cupy_conda]}
   depends_on_cudf:
     common:
       - output_types: conda

python/CMakeLists.txt

@@ -21,6 +21,7 @@ option(FIND_UCXX_CPP "Search for existing UCXX C++ installations before defaulti
 
 # If the user requested it we attempt to find UCXX.
 if(FIND_UCXX_CPP)
+  find_package(ucx REQUIRED)
   find_package(ucxx ${ucxx_version} REQUIRED COMPONENTS python)
 else()
   set(ucxx_FOUND OFF)

python/pyproject.toml

@@ -7,7 +7,7 @@ requires = [
     "cmake>=3.26.4",
     "cython>=3.0.0",
     "ninja",
-    "rmm==24.2.*",
+    "rmm==24.4.*",
     "scikit-build-core[pyproject]>=0.7.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 
@@ -40,8 +40,8 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     "cloudpickle",
-    "cudf==24.2.*",
-    "cupy-cuda11x>=12.0.0",
+    "cudf==24.4.*",
+    "cupy>=12.0.0",
     "dask",
     "distributed",
     "numba>=0.57.1",