Unpin dask and distributed for development (#5209)

This PR unpins `dask` and `distributed` for `23.04` development.



xref : rapidsai/cudf#12710

Authors:
  - GALI PREM SAGAR (https://github.com/galipremsagar)

Approvers:
  - Ray Douglass (https://github.com/raydouglass)
  - Dante Gama Dessavre (https://github.com/dantegd)

URL: #5209

.github/workflows/pr.yaml

@@ -99,10 +99,10 @@ jobs:
       build_type: pull-request
       package-name: cuml
       # Always want to test against latest dask/distributed.
-      test-before-amd64: "pip install git+https://github.com/dask/dask.git@2023.1.1 git+https://github.com/dask/distributed.git@2023.1.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.02"
+      test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.04"
       # On arm also need to install cupy from the specific webpage and CMake
       # because treelite needs to be compiled (no wheels available for arm).
-      test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install cmake && pip install git+https://github.com/dask/dask.git@2023.1.1 git+https://github.com/dask/distributed.git@2023.1.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.02"
+      test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install cmake && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.04"
       # skipped test context: https://github.com/rapidsai/cuml/issues/5025
       # parallelization is based on current test memory usage
       test-unittest: "pytest -v ./python/cuml/tests -k 'not test_silhouette_score_batched and not test_sparse_pca_inputs' -n 8 --ignore=dask && pytest -v ./python/cuml/tests -k 'test_sparse_pca_inputs' --ignore=dask && pytest -v ./python/cuml/tests/dask"

.github/workflows/test.yaml

@@ -49,10 +49,10 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       package-name: cuml
-      test-before-amd64: "pip install git+https://github.com/dask/dask.git@2023.1.1 git+https://github.com/dask/distributed.git@2023.1.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.02"
+      test-before-amd64: "pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.04"
       # On arm also need to install cupy from the specific webpage and CMake
       # because treelite needs to be compiled (no wheels available for arm).
-      test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install cmake && pip install git+https://github.com/dask/dask.git@2023.1.1 git+https://github.com/dask/distributed.git@2023.1.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.02"
+      test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64 && pip install cmake && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.04"
       # skipped test context: https://github.com/rapidsai/cuml/issues/5025
       # parallelization is based on current test memory usage
       test-unittest: "pytest -v ./python/cuml/tests -k 'not test_silhouette_score_batched and not test_sparse_pca_inputs' -n 8 --ignore=dask && pytest -v ./python/cuml/tests -k 'test_sparse_pca_inputs' --ignore=dask && pytest -v ./python/cuml/tests/dask"

ci/gpu/build.sh

@@ -43,7 +43,7 @@ unset GIT_DESCRIBE_TAG
 export NUMBA_THREADING_LAYER=workqueue
 
 # Whether to install dask nightly or stable packages
-export INSTALL_DASK_MAIN=0
+export INSTALL_DASK_MAIN=1
 
 # Dask version to install when `INSTALL_DASK_MAIN=0`
 export DASK_STABLE_VERSION="2023.1.1"

ci/release/update-version.sh

@@ -66,7 +66,9 @@ sed_runner "s|/branch-.*?/|/branch-${NEXT_SHORT_TAG}/|g" python/README.md
 sed_runner "s/extra-repo-sha: branch-.*/extra-repo-sha: branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml
 
 # Wheel builds install dask-cuda from source, update its branch
-sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" .github/workflows/*.yaml
+for FILE in .github/workflows/*.yaml; do
+  sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE};
+done
 
 # Need to distutils-normalize the original version
 NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -18,8 +18,8 @@ dependencies:
 - dask-cuda=23.04.*
 - dask-cudf=23.04.*
 - dask-ml
-- dask==2023.1.1
-- distributed==2023.1.1
+- dask>=2023.1.1
+- distributed>=2023.1.1
 - doxygen=1.8.20
 - faiss-proc=*=cuda
 - gcc_linux-64=9.*

conda/recipes/cuml/meta.yaml

@@ -68,8 +68,8 @@ requirements:
     - cudf ={{ minor_version }}
     - cupy >=7.8.0,<12.0.0a0
     - dask-cudf ={{ minor_version }}
-    - dask ==2023.1.1
-    - distributed ==2023.1.1
+    - dask >=2023.1.1
+    - distributed >=2023.1.1
     - joblib >=0.11
     - libcuml ={{ version }}
     - libcumlprims ={{ minor_version }}

cpp/Doxyfile.in

@@ -38,7 +38,7 @@ PROJECT_NAME           = "cuML C++ API"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "23.02"
+PROJECT_NUMBER         = "23.04"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

dependencies.yaml

@@ -109,8 +109,8 @@ dependencies:
     common:
       - output_types: [conda, requirements]
         packages:
-          - dask==2023.1.1
-          - distributed==2023.1.1
+          - dask>=2023.1.1
+          - distributed>=2023.1.1
           - joblib>=0.11
       - output_types: conda
         packages: