Use new rapids-dask-dependency metapackage for managing dask vers…

…ions (#3991) Currently dask versions are pinned as part of every release cycle and then unpinned for the next development cycle across all of RAPIDS. This introduces a great deal of churn. To centralize the dependency, we have created a metapackage to manage the required dask version and this PR introduces that metapackage as a dependency of cugraph. xref: rapidsai/cudf#14364 Authors: - Chuck Hastings (https://github.com/ChuckHastings) - Rick Ratzel (https://github.com/rlratzel) - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) - Naim ([email protected]) Approvers: - Jake Awe (https://github.com/AyodeAwe)
rapidsai · Nov 14, 2023 · b90e1a5 · b90e1a5
1 parent 96a7a6e
commit b90e1a5
Show file tree

Hide file tree

Showing 11 changed files with 14 additions and 27 deletions.
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -40,8 +40,11 @@ for dep in rmm cudf raft-dask pylibcugraph pylibraft ucx-py; do
     sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
 done
 
-# dask-cuda doesn't get a suffix, but it does get an alpha spec.
-sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file}
+# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec.
+for dep in dask-cuda rapids-dask-dependency; do
+    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
+done
+
 
 if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
     sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -88,6 +88,7 @@ DEPENDENCIES=(
   raft-dask
   rmm
   ucx-py
+  rapids-dask-dependency
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do

diff --git a/ci/test_wheel_cugraph.sh b/ci/test_wheel_cugraph.sh
@@ -8,7 +8,4 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep
 python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl
 
-# Always install latest dask for testing
-python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main
-
 ./ci/test_wheel.sh cugraph python/cugraph
diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -20,11 +20,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
-- dask>=2023.7.1
-- distributed>=2023.7.1
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
@@ -62,6 +59,7 @@ dependencies:
 - pytest-xdist
 - python-louvain
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - requests
 - rmm==23.12.*

diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml
@@ -20,11 +20,8 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-core>=2023.9.2
 - dask-cuda==23.12.*
 - dask-cudf==23.12.*
-- dask>=2023.7.1
-- distributed>=2023.7.1
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
@@ -61,6 +58,7 @@ dependencies:
 - pytest-xdist
 - python-louvain
 - raft-dask==23.12.*
+- rapids-dask-dependency==23.12.*
 - recommonmark
 - requests
 - rmm==23.12.*

diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml
@@ -26,7 +26,7 @@ requirements:
     - python
     - scikit-build >=0.13.1
   run:
-    - distributed >=2023.9.2
+    - rapids-dask-dependency ={{ version }}
     - numba >=0.57
     - numpy >=1.21
     - python

diff --git a/conda/recipes/cugraph-service/meta.yaml b/conda/recipes/cugraph-service/meta.yaml
@@ -59,10 +59,10 @@ outputs:
         - cupy >=12.0.0
         - dask-cuda ={{ minor_version }}
         - dask-cudf ={{ minor_version }}
-        - distributed >=2023.9.2
         - numba >=0.57
         - numpy >=1.21
         - python
+        - rapids-dask-dependency ={{ minor_version }}
         - thriftpy2 >=0.4.15
         - ucx-py {{ ucx_py_version }}
 

diff --git a/conda/recipes/cugraph/meta.yaml b/conda/recipes/cugraph/meta.yaml
@@ -76,15 +76,13 @@ requirements:
     - cupy >=12.0.0
     - dask-cuda ={{ minor_version }}
     - dask-cudf ={{ minor_version }}
-    - dask >=2023.9.2
-    - dask-core >=2023.9.2
-    - distributed >=2023.9.2
     - fsspec>=0.6.0
     - libcugraph ={{ version }}
     - pylibcugraph ={{ version }}
     - pylibraft ={{ minor_version }}
     - python
     - raft-dask ={{ minor_version }}
+    - rapids-dask-dependency ={{ minor_version }}
     - requests
     - ucx-proc=*=gpu
     - ucx-py {{ ucx_py_version }}

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -377,15 +377,13 @@ dependencies:
     common:
       - output_types: [conda, pyproject]
         packages:
-          - &dask dask>=2023.7.1
-          - &distributed distributed>=2023.7.1
+          - &dask rapids-dask-dependency==23.12.*
           - &dask_cuda dask-cuda==23.12.*
           - &numba numba>=0.57
           - &ucx_py ucx-py==0.35.*
       - output_types: conda
         packages:
           - aiohttp
-          - &dask-core_conda dask-core>=2023.9.2
           - fsspec>=0.6.0
           - libcudf==23.12.*
           - requests
@@ -431,14 +429,10 @@ dependencies:
         packages:
           - *dask
           - *dask_cuda
-          - *distributed
           - *numba
           - *numpy
           - *thrift
           - *ucx_py
-      - output_types: conda
-        packages:
-          - *dask-core_conda
       - output_types: pyproject
         packages:
           - *cugraph

diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml
@@ -25,10 +25,9 @@ dependencies = [
     "cupy-cuda11x>=12.0.0",
     "dask-cuda==23.12.*",
     "dask-cudf==23.12.*",
-    "dask>=2023.7.1",
-    "distributed>=2023.7.1",
     "numba>=0.57",
     "numpy>=1.21",
+    "rapids-dask-dependency==23.12.*",
     "rmm==23.12.*",
     "thriftpy2",
     "ucx-py==0.35.*",

diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml
@@ -33,12 +33,11 @@ dependencies = [
     "cupy-cuda11x>=12.0.0",
     "dask-cuda==23.12.*",
     "dask-cudf==23.12.*",
-    "dask>=2023.7.1",
-    "distributed>=2023.7.1",
     "fsspec[http]>=0.6.0",
     "numba>=0.57",
     "pylibcugraph==23.12.*",
     "raft-dask==23.12.*",
+    "rapids-dask-dependency==23.12.*",
     "rmm==23.12.*",
     "ucx-py==0.35.*",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.