[REVIEW] Unpin dask & distributed

ci/benchmark/build.sh

@@ -36,8 +36,8 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/"
 # like `/tmp` is.
 export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 
-# Dask & Distributed git tag
-export DASK_DISTRIBUTED_GIT_TAG='2022.01.0'
+# Dask & Distributed option to install main
+export INSTALL_DASK_MAIN=1
 
 function remove_libcudf_kernel_cache_dir {
     EXITCODE=$?
@@ -78,10 +78,15 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
 # conda install "your-pkg=1.0.0"
 
 # Install the master version of dask, distributed, and streamz
-logger "pip install git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps"
-pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
-logger "pip install git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps"
-pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
+# Install the main version of dask and distributed
+if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
+    gpuci_logger "conda install -c dask/label/dev dask"
+    conda install -c dask/label/dev dask
+else
+    gpuci_logger "conda install -c conda-forge dask"
+    conda install -c conda-forge dask
+fi
+
 logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps"
 pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps
 

ci/gpu/build.sh

@@ -31,7 +31,7 @@ export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 
 # Dask & Distributed git tag
-export DASK_DISTRIBUTED_GIT_TAG='2022.01.0'
+export INSTALL_DASK_MAIN=1
 
 # ucx-py version
 export UCX_PY_VERSION='0.25.*'
@@ -107,8 +107,13 @@ function install_dask {
     # Install the main version of dask, distributed, and streamz
     gpuci_logger "Install the main version of dask, distributed, and streamz"
     set -x
-    pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
-    pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
+    if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
+        gpuci_logger "conda install -c dask/label/dev dask"
+        conda install -c dask/label/dev dask
+    else
+        gpuci_logger "conda install -c conda-forge dask"
+        conda install -c conda-forge dask
+    fi
     # Need to uninstall streamz that is already in the env.
     pip uninstall -y streamz
     pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps

conda/environments/cudf_dev_cuda11.5.yml

@@ -5,6 +5,7 @@ channels:
   - rapidsai
   - nvidia
   - rapidsai-nightly
+  - dask/label/dev
   - conda-forge
 dependencies:
   - clang=11.1.0
@@ -42,8 +43,8 @@ dependencies:
   - pydocstyle=6.1.1
   - typing_extensions
   - pre-commit
-  - dask>=2021.11.1,<=2022.01.0
-  - distributed>=2021.11.1,<=2022.01.0
+  - dask>=2021.11.1
+  - distributed>=2021.11.1
   - streamz
   - arrow-cpp=5.0.0
   - dlpack>=0.5,<0.6.0a0
@@ -63,8 +64,6 @@ dependencies:
   - librdkafka=1.7.0
   - python-confluent-kafka=1.7.0
   - pip:
-      - git+https://github.com/dask/dask.git@main
-      - git+https://github.com/dask/distributed.git@main
       - git+https://github.com/python-streamz/streamz.git@master
       - pyorc
   - ptxcompiler  # [linux64]

conda/recipes/custreamz/meta.yaml

@@ -32,8 +32,8 @@ requirements:
     - python
     - streamz
     - cudf {{ version }}
-    - dask>=2021.11.1,<=2022.01.0
-    - distributed>=2021.11.1,<=2022.01.0
+    - dask>=2021.11.1
+    - distributed>=2021.11.1
     - python-confluent-kafka >=1.7.0,<1.8.0a0
     - cudf_kafka {{ version }}
 

conda/recipes/dask-cudf/meta.yaml

@@ -27,14 +27,14 @@ requirements:
   host:
     - python
     - cudf {{ version }}
-    - dask>=2021.11.1,<=2022.01.0
-    - distributed>=2021.11.1,<=2022.01.0
+    - dask>=2021.11.1
+    - distributed>=2021.11.1
     - cudatoolkit {{ cuda_version }}
   run:
     - python
     - cudf {{ version }}
-    - dask>=2021.11.1,<=2022.01.0
-    - distributed>=2021.11.1,<=2022.01.0
+    - dask>=2021.11.1
+    - distributed>=2021.11.1
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
 
 test:                                   # [linux64]

python/custreamz/dev_requirements.txt

@@ -3,8 +3,8 @@
 flake8==3.8.3
 black==19.10b0
 isort==5.6.4
-dask>=2021.11.1,<=2022.01.0
-distributed>=2021.11.1,<=2022.01.0
+dask>=2021.11.1
+distributed>=2021.11.1
 streamz
 python-confluent-kafka
 pytest

python/dask_cudf/dev_requirements.txt

@@ -1,7 +1,7 @@
 # Copyright (c) 2021, NVIDIA CORPORATION.
 
-dask>=2021.11.1,<=2022.01.0
-distributed>=2021.11.1,<=2022.01.0
+dask>=2021.11.1
+distributed>=2021.11.1
 fsspec>=0.6.0
 numba>=0.53.1
 numpy

python/dask_cudf/setup.py

@@ -10,8 +10,8 @@
 
 install_requires = [
     "cudf",
-    "dask>=2021.11.1,<=2022.01.0",
-    "distributed>=2021.11.1,<=2022.01.0",
+    "dask>=2021.11.1",
+    "distributed>=2021.11.1",
     "fsspec>=0.6.0",
     "numpy",
     "pandas>=1.0,<1.4.0dev0",