[REVIEW] Unpin dask & distributed

ci/benchmark/build.sh

@@ -36,8 +36,8 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/"
 # like `/tmp` is.
 export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 
-# Dask & Distributed git tag
-export DASK_DISTRIBUTED_GIT_TAG='2022.01.0'
+# Dask & Distributed option to install main(nightly) or `conda-forge` packages.
+export INSTALL_DASK_MAIN=1
 
 function remove_libcudf_kernel_cache_dir {
     EXITCODE=$?
@@ -77,11 +77,16 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \
 # conda remove -f rapids-build-env rapids-notebook-env
 # conda install "your-pkg=1.0.0"
 
-# Install the master version of dask, distributed, and streamz
-logger "pip install git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps"
-pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
-logger "pip install git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps"
-pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
+# Install the conda-forge or nightly version of dask and distributed
+if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
+    gpuci_logger "gpuci_mamba_retry update dask"
+    gpuci_mamba_retry update dask
+else
+    gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 --force-reinstall"
+    gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 --force-reinstall
+fi
+
+# Install the master version of streamz
 logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps"
 pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps
 

ci/cpu/build.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018-2021, NVIDIA CORPORATION.
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 ##############################################
 # cuDF CPU conda build script for CI         #
 ##############################################
@@ -46,9 +46,10 @@ gpuci_logger "Activate conda env"
 . /opt/conda/etc/profile.d/conda.sh
 conda activate rapids
 
-# Remove rapidsai-nightly channel if we are building main branch
+# Remove `rapidsai-nightly` & `dask/label/dev` channel if we are building main branch
 if [ "$SOURCE_BRANCH" = "main" ]; then
   conda config --system --remove channels rapidsai-nightly
+  conda config --system --remove channels dask/label/dev
 fi
 
 gpuci_logger "Check compiler versions"

ci/gpu/build.sh

@@ -30,8 +30,8 @@ export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/"
 export GIT_DESCRIBE_TAG=`git describe --tags`
 export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 
-# Dask & Distributed git tag
-export DASK_DISTRIBUTED_GIT_TAG='2022.01.0'
+# Dask & Distributed option to install main(nightly) or `conda-forge` packages.
+export INSTALL_DASK_MAIN=1
 
 # ucx-py version
 export UCX_PY_VERSION='0.25.*'
@@ -108,11 +108,19 @@ conda config --show-sources
 conda list --show-channel-urls
 
 function install_dask {
-    # Install the main version of dask, distributed, and streamz
-    gpuci_logger "Install the main version of dask, distributed, and streamz"
+    # Install the conda-forge or nightly version of dask and distributed
+    gpuci_logger "Install the conda-forge or nightly version of dask and distributed"
     set -x
-    pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
-    pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps
+    if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then
+        gpuci_logger "gpuci_mamba_retry update dask"
+        gpuci_mamba_retry update dask
+        conda list
+    else
+        gpuci_logger "gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 --force-reinstall"
+        gpuci_mamba_retry install conda-forge::dask>=2022.02.1 conda-forge::distributed>=2022.02.1 --force-reinstall
+    fi
+    # Install the main version of streamz
+    gpuci_logger "Install the main version of streamz"
     # Need to uninstall streamz that is already in the env.
     pip uninstall -y streamz
     pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps

conda/environments/cudf_dev_cuda11.5.yml

@@ -5,6 +5,7 @@ channels:
   - rapidsai
   - nvidia
   - rapidsai-nightly
+  - dask/label/dev
   - conda-forge
 dependencies:
   - clang=11.1.0
@@ -42,8 +43,8 @@ dependencies:
   - pydocstyle=6.1.1
   - typing_extensions
   - pre-commit
-  - dask>=2021.11.1,<=2022.01.0
-  - distributed>=2021.11.1,<=2022.01.0
+  - dask>=2022.02.1
+  - distributed>=2022.02.1
   - streamz
   - arrow-cpp=6.0.1
   - dlpack>=0.5,<0.6.0a0
@@ -63,8 +64,6 @@ dependencies:
   - librdkafka=1.7.0
   - python-confluent-kafka=1.7.0
   - pip:
-      - git+https://github.com/dask/dask.git@main
-      - git+https://github.com/dask/distributed.git@main
       - git+https://github.com/python-streamz/streamz.git@master
       - pyorc
   - ptxcompiler  # [linux64]

conda/recipes/custreamz/meta.yaml

@@ -32,8 +32,8 @@ requirements:
     - python
     - streamz
     - cudf {{ version }}
-    - dask>=2021.11.1,<=2022.01.0
-    - distributed>=2021.11.1,<=2022.01.0
+    - dask>=2022.02.1
+    - distributed>=2022.02.1
     - python-confluent-kafka >=1.7.0,<1.8.0a0
     - cudf_kafka {{ version }}
 

conda/recipes/dask-cudf/meta.yaml

@@ -1,4 +1,4 @@
-# Copyright (c) 2018-2019, NVIDIA CORPORATION.
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
 
 {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
 {% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
@@ -27,14 +27,14 @@ requirements:
   host:
     - python
     - cudf {{ version }}
-    - dask>=2021.11.1,<=2022.01.0
-    - distributed>=2021.11.1,<=2022.01.0
+    - dask>=2022.02.1
+    - distributed>=2022.02.1
     - cudatoolkit {{ cuda_version }}
   run:
     - python
     - cudf {{ version }}
-    - dask>=2021.11.1,<=2022.01.0
-    - distributed>=2021.11.1,<=2022.01.0
+    - dask>=2022.02.1
+    - distributed>=2022.02.1
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
 
 test:                                   # [linux64]

python/custreamz/dev_requirements.txt

@@ -3,8 +3,8 @@
 flake8==3.8.3
 black==19.10b0
 isort==5.6.4
-dask>=2021.11.1,<=2022.01.0
-distributed>=2021.11.1,<=2022.01.0
+dask>=2022.02.1
+distributed>=2022.02.1
 streamz
 python-confluent-kafka
 pytest

python/dask_cudf/dev_requirements.txt

@@ -1,7 +1,7 @@
 # Copyright (c) 2021, NVIDIA CORPORATION.
 
-dask>=2021.11.1,<=2022.01.0
-distributed>=2021.11.1,<=2022.01.0
+dask>=2022.02.1
+distributed>=2022.02.1
 fsspec>=0.6.0
 numba>=0.53.1
 numpy

python/dask_cudf/setup.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.
+# Copyright (c) 2019-2022, NVIDIA CORPORATION.
 
 import os
 import re
@@ -10,8 +10,8 @@
 
 install_requires = [
     "cudf",
-    "dask>=2021.11.1,<=2022.01.0",
-    "distributed>=2021.11.1,<=2022.01.0",
+    "dask>=2022.02.1",
+    "distributed>=2022.02.1",
     "fsspec>=0.6.0",
     "numpy",
     "pandas>=1.0,<1.4.0dev0",