Unpin dask and distributed for development (#12302)

This PR unpins `dask` and `distributed` to `2022.12.0+` for `23.02` development. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Ray Douglass (https://github.com/raydouglass) - Charles Blackmon-Luca (https://github.com/charlesbluca) URL: #12302
rapidsai · Dec 12, 2022 · b448ac2 · b448ac2
1 parent e9d52a4
commit b448ac2
Show file tree

Hide file tree

Showing 15 changed files with 19 additions and 51 deletions.
diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh
@@ -37,10 +37,10 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/"
 export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache"
 
 # Dask & Distributed option to install main(nightly) or `conda-forge` packages.
-export INSTALL_DASK_MAIN=0
+export INSTALL_DASK_MAIN=1
 
 # Dask version to install when `INSTALL_DASK_MAIN=0`
-export DASK_STABLE_VERSION="2022.11.1"
+export DASK_STABLE_VERSION="2022.12.0"
 
 function remove_libcudf_kernel_cache_dir {
     EXITCODE=$?

diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh
@@ -35,7 +35,7 @@ export CONDA_BLD_DIR="$WORKSPACE/.conda-bld"
 
 # Whether to keep `dask/label/dev` channel in the env. If INSTALL_DASK_MAIN=0,
 # `dask/label/dev` channel is removed.
-export INSTALL_DASK_MAIN=0
+export INSTALL_DASK_MAIN=1
 
 # Switch to project root; also root of repo checkout
 cd "$WORKSPACE"

diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh
@@ -39,10 +39,10 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'`
 unset GIT_DESCRIBE_TAG
 
 # Dask & Distributed option to install main(nightly) or `conda-forge` packages.
-export INSTALL_DASK_MAIN=0
+export INSTALL_DASK_MAIN=1
 
 # Dask version to install when `INSTALL_DASK_MAIN=0`
-export DASK_STABLE_VERSION="2022.11.1"
+export DASK_STABLE_VERSION="2022.12.0"
 
 # ucx-py version
 export UCX_PY_VERSION='0.30.*'

diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml
@@ -22,8 +22,8 @@ dependencies:
 - cxx-compiler
 - cython>=0.29,<0.30
 - dask-cuda=23.02.*
-- dask>=2022.11.1
-- distributed>=2022.11.1
+- dask>=2022.12.0
+- distributed>=2022.12.0
 - dlpack>=0.5,<0.6.0a0
 - doxygen=1.8.20
 - fastavro>=0.22.9

diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml
@@ -40,8 +40,8 @@ requirements:
     - python
     - streamz
     - cudf ={{ version }}
-    - dask==2022.11.1
-    - distributed==2022.11.1
+    - dask>=2022.12.0
+    - distributed>=2022.12.0
     - python-confluent-kafka >=1.7.0,<1.8.0a0
     - cudf_kafka ={{ version }}
 

diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml
@@ -35,14 +35,14 @@ requirements:
   host:
     - python
     - cudf ={{ version }}
-    - dask==2022.11.1
-    - distributed==2022.11.1
+    - dask>=2022.12.0
+    - distributed>=2022.12.0
     - cudatoolkit ={{ cuda_version }}
   run:
     - python
     - cudf ={{ version }}
-    - dask==2022.11.1
-    - distributed==2022.11.1
+    - dask>=2022.12.0
+    - distributed>=2022.12.0
     - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }}
 
 # TODO: Remove the linux64 tags on tests after disabling gpuCI / Jenkins

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -207,8 +207,8 @@ dependencies:
       - output_types: [conda, requirements]
         packages:
           - cachetools
-          - dask>=2022.11.1
-          - distributed>=2022.11.1
+          - dask>=2022.12.0
+          - distributed>=2022.12.0
           - fsspec>=0.6.0
           - numba>=0.56.2
           - numpy

diff --git a/python/dask_cudf/dask_cudf/core.py b/python/dask_cudf/dask_cudf/core.py
@@ -5,11 +5,8 @@
 
 import numpy as np
 import pandas as pd
-from packaging.version import parse as parse_version
 from tlz import partition_all
 
-import dask
-import dask.dataframe.optimize
 from dask import dataframe as dd
 from dask.base import normalize_token, tokenize
 from dask.dataframe.core import (
@@ -30,12 +27,6 @@
 from dask_cudf.accessors import ListMethods, StructMethods
 from dask_cudf.sorting import _get_shuffle_type
 
-DASK_BACKEND_SUPPORT = parse_version(dask.__version__) >= parse_version(
-    "2022.10.0"
-)
-# TODO: Remove DASK_BACKEND_SUPPORT throughout codebase
-# when dask_cudf is pinned to dask>=2022.10.0
-
 
 class _Frame(dd.core._Frame, OperatorMethodMixin):
     """Superclass for DataFrame and Series

diff --git a/python/dask_cudf/dask_cudf/groupby.py b/python/dask_cudf/dask_cudf/groupby.py
@@ -219,9 +219,7 @@ def aggregate(self, arg, split_every=None, split_out=1, shuffle=None):
             arg,
             split_every=split_every,
             split_out=split_out,
-            # TODO: Change following line to `shuffle=shuffle,`
-            # when dask_cudf is pinned to dask>2022.8.0
-            **({} if shuffle is None else {"shuffle": shuffle}),
+            shuffle=shuffle,
         )
 
 
@@ -351,9 +349,7 @@ def aggregate(self, arg, split_every=None, split_out=1, shuffle=None):
             arg,
             split_every=split_every,
             split_out=split_out,
-            # TODO: Change following line to `shuffle=shuffle,`
-            # when dask_cudf is pinned to dask>2022.8.0
-            **({} if shuffle is None else {"shuffle": shuffle}),
+            shuffle=shuffle,
         )
 
 

diff --git a/python/dask_cudf/dask_cudf/io/tests/test_csv.py b/python/dask_cudf/dask_cudf/io/tests/test_csv.py
@@ -16,10 +16,6 @@
 import dask_cudf
 
 
-@pytest.mark.skipif(
-    not dask_cudf.core.DASK_BACKEND_SUPPORT,
-    reason="No backend-dispatch support",
-)
 def test_csv_roundtrip_backend_dispatch(tmp_path):
     # Test ddf.read_csv cudf-backend dispatch
     df = cudf.DataFrame({"x": [1, 2, 3, 4], "id": ["a", "b", "c", "d"]})

diff --git a/python/dask_cudf/dask_cudf/io/tests/test_json.py b/python/dask_cudf/dask_cudf/io/tests/test_json.py
@@ -12,10 +12,6 @@
 import dask_cudf
 
 
-@pytest.mark.skipif(
-    not dask_cudf.core.DASK_BACKEND_SUPPORT,
-    reason="No backend-dispatch support",
-)
 def test_read_json_backend_dispatch(tmp_path):
     # Test ddf.read_json cudf-backend dispatch
     df1 = dask.datasets.timeseries(

diff --git a/python/dask_cudf/dask_cudf/io/tests/test_orc.py b/python/dask_cudf/dask_cudf/io/tests/test_orc.py
@@ -17,10 +17,6 @@
 sample_orc = os.path.join(cur_dir, "data/orc/sample.orc")
 
 
-@pytest.mark.skipif(
-    not dask_cudf.core.DASK_BACKEND_SUPPORT,
-    reason="No backend-dispatch support",
-)
 def test_read_orc_backend_dispatch():
     # Test ddf.read_orc cudf-backend dispatch
     df1 = cudf.read_orc(sample_orc)

diff --git a/python/dask_cudf/dask_cudf/io/tests/test_parquet.py b/python/dask_cudf/dask_cudf/io/tests/test_parquet.py
@@ -46,10 +46,6 @@ def _divisions(setting):
     return {"gather_statistics": setting}
 
 
-@pytest.mark.skipif(
-    not dask_cudf.core.DASK_BACKEND_SUPPORT,
-    reason="No backend-dispatch support",
-)
 def test_roundtrip_backend_dispatch(tmpdir):
     # Test ddf.read_parquet cudf-backend dispatch
     tmpdir = str(tmpdir)

diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py
@@ -17,9 +17,6 @@
 import dask_cudf as dgd
 
 
-@pytest.mark.skipif(
-    not dgd.core.DASK_BACKEND_SUPPORT, reason="No backend-dispatch support"
-)
 def test_from_dict_backend_dispatch():
     # Test ddf.from_dict cudf-backend dispatch
     np.random.seed(0)

diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py
@@ -8,8 +8,8 @@
 cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="")
 
 install_requires = [
-    "dask==2022.11.1",
-    "distributed==2022.11.1",
+    "dask>=2022.12.0",
+    "distributed>=2022.12.0",
     "fsspec>=0.6.0",
     "numpy",
     "pandas>=1.0,<1.6.0dev0",