Use new rapids-dask-dependency metapackage for managing dask vers…

…ions (#5649)

Currently dask versions are pinned as part of every release cycle and then unpinned for the next development cycle across all of RAPIDS. This introduces a great deal of churn. To centralize the dependency, we have created a metapackage to manage the required dask version and this PR introduces that metapackage as a dependency of cuml.



xref: rapidsai/cudf#14364

Authors:
  - GALI PREM SAGAR (https://github.com/galipremsagar)
  - Simon Adorf (https://github.com/csadorf)

Approvers:
  - Simon Adorf (https://github.com/csadorf)
  - Jake Awe (https://github.com/AyodeAwe)

URL: #5649

ci/build_wheel.sh

@@ -38,6 +38,10 @@ for dep in cudf pylibraft raft-dask rmm; do
     sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
 done
 
+for dep in dask-cuda rapids-dask-dependency; do
+    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
+done
+
 if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
     sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" ${pyproject_file}
     sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}

ci/release/update-version.sh

@@ -73,6 +73,7 @@ DEPENDENCIES=(
   librmm
   pylibraft
   raft-dask
+  rapids-dask-dependency
   rmm
 )
 for FILE in dependencies.yaml conda/environments/*.yaml; do
@@ -81,17 +82,13 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do
   done
 done
 
-sed_runner "s|/branch-.*?/|/branch-${NEXT_SHORT_TAG}/|g" README.md
-sed_runner "s|/branch-.*?/|/branch-${NEXT_SHORT_TAG}/|g" python/README.md
+sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" README.md
+sed_runner "s|/branch-[^/]*/|/branch-${NEXT_SHORT_TAG}/|g" python/README.md
+sed_runner "/- rapids-dask-dependency==/ s/==.*/==${NEXT_SHORT_TAG}\.*/g" python/README.md
 
 # Wheel builds clone cumlprims_mg, update its branch
 sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml
 
-# Wheel builds install dask-cuda from source, update its branch
-for FILE in .github/workflows/*.yaml; do
-  sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE};
-done
-
 # CI files
 for FILE in .github/workflows/*.yaml; do
   sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}"

ci/test_wheel.sh

@@ -12,9 +12,6 @@ if [[ "$(arch)" == "aarch64" ]]; then
     python -m pip install cmake
 fi
 
-# Always install latest dask for testing
-python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/[email protected]
-
 # echo to expand wildcard before adding `[extra]` requires for pip
 python -m pip install $(echo ./dist/cuml*.whl)[test]
 

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -16,12 +16,9 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
 - dask-ml
-- dask>=2023.9.2
-- distributed>=2023.9.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - gmock>=1.13.0
@@ -63,6 +60,7 @@ dependencies:
 - pytest-xdist
 - python>=3.9,<3.11
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - rmm==23.12.*
 - scikit-build>=0.13.1

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -18,12 +18,9 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
 - dask-ml
-- dask>=2023.9.2
-- distributed>=2023.9.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - gmock>=1.13.0
@@ -59,6 +56,7 @@ dependencies:
 - pytest-xdist
 - python>=3.9,<3.11
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - rmm==23.12.*
 - scikit-build>=0.13.1

conda/recipes/cuml/meta.yaml

@@ -76,15 +76,13 @@ requirements:
     - cudf ={{ minor_version }}
     - cupy >=12.0.0
     - dask-cudf ={{ minor_version }}
-    - dask >=2023.9.2
-    - dask-core>=2023.9.2
-    - distributed >=2023.9.2
     - joblib >=0.11
     - libcuml ={{ version }}
     - libcumlprims ={{ minor_version }}
     - pylibraft ={{ minor_version }}
     - python x.x
     - raft-dask ={{ minor_version }}
+    - rapids-dask-dependency ={{ minor_version }}
     - treelite {{ treelite_version }}
 
 tests:

dependencies.yaml

@@ -175,24 +175,20 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cudf==23.12.*
-          - dask>=2023.9.2
           - dask-cuda==23.12.*
           - dask-cudf==23.12.*
-          - distributed>=2023.9.2
           - joblib>=0.11
           - numba>=0.57
             # TODO: Is scipy really a hard dependency, or should
             # we make it optional (i.e. an extra for pip
             # installation/run_constrained for conda)?
           - scipy>=1.8.0
           - raft-dask==23.12.*
+          - rapids-dask-dependency==23.12.*
           - *treelite
       - output_types: [conda, requirements]
         packages:
           - cupy>=12.0.0
-      - output_types: conda
-        packages:
-          - dask-core>=2023.9.2
       - output_types: pyproject
         packages:
           - *treelite_runtime

python/README.md

@@ -38,7 +38,7 @@ example `setup.py --singlegpu`) are:
 
 RAFT's Python and Cython is located in the [RAFT repository](https://github.com/rapidsai/raft/python). It was designed to be included in projects as opposed to be distributed by itself, so at build time, **setup.py creates a symlink from cuML, located in `/python/cuml/raft/` to the Python folder of RAFT**.
 
-For developers that need to modify RAFT code, please refer to the [RAFT Developer Guide](https://github.com/rapidsai/raft/blob/branch-23.04/BUILD.md#developer-guide) for recommendations.
+For developers that need to modify RAFT code, please refer to the [RAFT Developer Guide](https://github.com/rapidsai/raft/blob/branch-23.12/docs/source/build.md) for recommendations.
 
 To configure RAFT at build time:
 
@@ -50,7 +50,7 @@ The RAFT Python code gets included in the cuML build and distributable artifacts
 
 ### Build Requirements
 
-cuML's convenience [development yaml files](https://github.com/rapidsai/cuml/tree/branch-23.04/environments) includes all dependencies required to build cuML.
+cuML's convenience [development yaml files](https://github.com/rapidsai/cuml/tree/branch-23.12/environments) includes all dependencies required to build cuML.
 
 To build cuML's Python package, the following dependencies are required:
 
@@ -70,8 +70,7 @@ Packages required for multigpu algorithms*:
 - ucx-py version matching the cuML version
 - dask-cudf version matching the cuML version
 - nccl>=2.5
-- dask>=2023.9.2
-- distributed>=2023.9.2
+- rapids-dask-dependency==23.12.*
 
 * this can be avoided with `--singlegpu` argument flag.
 

python/pyproject.toml

@@ -61,11 +61,10 @@ dependencies = [
     "cupy-cuda11x>=12.0.0",
     "dask-cuda==23.12.*",
     "dask-cudf==23.12.*",
-    "dask>=2023.9.2",
-    "distributed>=2023.9.2",
     "joblib>=0.11",
     "numba>=0.57",
     "raft-dask==23.12.*",
+    "rapids-dask-dependency==23.12.*",
     "scipy>=1.8.0",
     "treelite==3.9.1",
     "treelite_runtime==3.9.1",