From 279d8555194e774a2aeeec0ed6efb2a8fd3927f3 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 23 Sep 2021 09:13:37 -0700 Subject: [PATCH 1/4] pin max dask & distributed versions --- ci/benchmark/build.sh | 8 ++++---- ci/gpu/build.sh | 4 ++-- conda/environments/cudf_dev_cuda11.0.yml | 8 ++++---- conda/environments/cudf_dev_cuda11.2.yml | 8 ++++---- conda/recipes/custreamz/meta.yaml | 4 ++-- conda/recipes/dask-cudf/meta.yaml | 8 ++++---- python/custreamz/dev_requirements.txt | 4 ++-- python/dask_cudf/dev_requirements.txt | 4 ++-- python/dask_cudf/setup.py | 8 ++++---- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh index e73153ce0c3..7b7799fe93b 100755 --- a/ci/benchmark/build.sh +++ b/ci/benchmark/build.sh @@ -75,10 +75,10 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ # conda install "your-pkg=1.0.0" # Install the master version of dask, distributed, and streamz -logger "pip install git+https://github.com/dask/distributed.git@main --upgrade --no-deps" -pip install "git+https://github.com/dask/distributed.git@main" --upgrade --no-deps -logger "pip install git+https://github.com/dask/dask.git@main --upgrade --no-deps" -pip install "git+https://github.com/dask/dask.git@main" --upgrade --no-deps +logger "pip install git+https://github.com/dask/distributed.git@2021.09.1 --upgrade --no-deps" +pip install "git+https://github.com/dask/distributed.git@2021.09.1" --upgrade --no-deps +logger "pip install git+https://github.com/dask/dask.git@2021.09.1 --upgrade --no-deps" +pip install "git+https://github.com/dask/dask.git@2021.09.1" --upgrade --no-deps logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps" pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 918e4760c71..a1f74b87aeb 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -101,8 +101,8 @@ function install_dask { # Install the main version of dask, distributed, and streamz gpuci_logger "Install the main version of dask, distributed, and streamz" set -x - pip install "git+https://github.com/dask/distributed.git@2021.07.1" --upgrade --no-deps - pip install "git+https://github.com/dask/dask.git@2021.07.1" --upgrade --no-deps + pip install "git+https://github.com/dask/distributed.git@2021.09.1" --upgrade --no-deps + pip install "git+https://github.com/dask/dask.git@2021.09.1" --upgrade --no-deps # Need to uninstall streamz that is already in the env. pip uninstall -y streamz pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index f975aded863..f19a121f061 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -39,8 +39,8 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask>=2021.6.0 - - distributed>=2021.6.0 + - dask>=2021.6.0,<=2021.09.1 + - distributed>=2021.6.0,<=2021.09.1 - streamz - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 @@ -58,7 +58,7 @@ dependencies: - transformers - pydata-sphinx-theme - pip: - - git+https://github.com/dask/dask.git@2021.07.1 - - git+https://github.com/dask/distributed.git@2021.07.1 + - git+https://github.com/dask/dask.git@2021.09.1 + - git+https://github.com/dask/distributed.git@2021.09.1 - git+https://github.com/python-streamz/streamz.git@master - pyorc diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index f1487ee3e4b..38572ace443 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -39,8 +39,8 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask>=2021.6.0 - - distributed>=2021.6.0 + - dask>=2021.6.0,<=2021.09.1 + - distributed>=2021.6.0,<=2021.09.1 - streamz - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 @@ -58,7 +58,7 @@ dependencies: - transformers - pydata-sphinx-theme - pip: - - git+https://github.com/dask/dask.git@2021.07.1 - - git+https://github.com/dask/distributed.git@2021.07.1 + - git+https://github.com/dask/dask.git@2021.09.1 + - git+https://github.com/dask/distributed.git@2021.09.1 - git+https://github.com/python-streamz/streamz.git@master - pyorc diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index d0965e97567..7c3c1877c76 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -31,8 +31,8 @@ requirements: - python - streamz - cudf {{ version }} - - dask>=2021.6.0 - - distributed>=2021.6.0 + - dask>=2021.6.0,<=2021.09.1 + - distributed>=2021.6.0,<=2021.09.1 - python-confluent-kafka - cudf_kafka {{ version }} diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 1b2c4efd610..c7f154395bf 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -26,13 +26,13 @@ requirements: host: - python - cudf {{ version }} - - dask>=2021.6.0 - - distributed>=2021.6.0 + - dask>=2021.6.0,<=2021.09.1 + - distributed>=2021.6.0,<=2021.09.1 run: - python - cudf {{ version }} - - dask>=2021.6.0 - - distributed>=2021.6.0 + - dask>=2021.6.0,<=2021.09.1 + - distributed>=2021.6.0,<=2021.09.1 test: # [linux64] requires: # [linux64] diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt index 61e4817b1c2..bd94b4d6d52 100644 --- a/python/custreamz/dev_requirements.txt +++ b/python/custreamz/dev_requirements.txt @@ -3,8 +3,8 @@ flake8==3.8.3 black==19.10b0 isort==5.6.4 -dask>=2021.6.0 -distributed>=2021.6.0 +dask>=2021.6.0,<=2021.09.1 +distributed>=2021.6.0,<=2021.09.1 streamz python-confluent-kafka pytest diff --git a/python/dask_cudf/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt index 0b601180711..282f66d0d04 100644 --- a/python/dask_cudf/dev_requirements.txt +++ b/python/dask_cudf/dev_requirements.txt @@ -1,7 +1,7 @@ # Copyright (c) 2021, NVIDIA CORPORATION. -dask>=2021.6.0 -distributed>=2021.6.0 +dask>=2021.6.0,<=2021.09.1 +distributed>=2021.6.0,<=2021.09.1 fsspec>=0.6.0 numba>=0.53.1 numpy diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index c4cb57ff89a..84178caeb72 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -10,8 +10,8 @@ install_requires = [ "cudf", - "dask>=2021.6.0", - "distributed>=2021.6.0", + "dask>=2021.6.0,<=2021.09.1", + "distributed>=2021.6.0,<=2021.09.1", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.4.0dev0", @@ -23,8 +23,8 @@ "pandas>=1.0,<1.4.0dev0", "pytest", "numba>=0.53.1", - "dask>=2021.6.0", - "distributed>=2021.6.0", + "dask>=2021.6.0,<=2021.09.1", + "distributed>=2021.6.0,<=2021.09.1", ] } From 96d81ae0f26c85688bba524a8b3b660e1ef83e21 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 23 Sep 2021 12:50:27 -0500 Subject: [PATCH 2/4] Apply suggestions from code review --- conda/environments/cudf_dev_cuda11.0.yml | 4 ++-- conda/environments/cudf_dev_cuda11.2.yml | 4 ++-- conda/recipes/custreamz/meta.yaml | 4 ++-- conda/recipes/dask-cudf/meta.yaml | 8 ++++---- python/custreamz/dev_requirements.txt | 4 ++-- python/dask_cudf/dev_requirements.txt | 4 ++-- python/dask_cudf/setup.py | 8 ++++---- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index f19a121f061..1ba6de034a5 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -39,8 +39,8 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask>=2021.6.0,<=2021.09.1 - - distributed>=2021.6.0,<=2021.09.1 + - dask=2021.9.1 + - distributed=2021.9.1 - streamz - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 38572ace443..b316ace6364 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -39,8 +39,8 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask>=2021.6.0,<=2021.09.1 - - distributed>=2021.6.0,<=2021.09.1 + - dask=2021.9.1 + - distributed=2021.9.1 - streamz - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index 7c3c1877c76..6e6d33cf03f 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -31,8 +31,8 @@ requirements: - python - streamz - cudf {{ version }} - - dask>=2021.6.0,<=2021.09.1 - - distributed>=2021.6.0,<=2021.09.1 + - dask=2021.9.1 + - distributed=2021.9.1 - python-confluent-kafka - cudf_kafka {{ version }} diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index c7f154395bf..b8c88480506 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -26,13 +26,13 @@ requirements: host: - python - cudf {{ version }} - - dask>=2021.6.0,<=2021.09.1 - - distributed>=2021.6.0,<=2021.09.1 + - dask=2021.9.1 + - distributed=2021.9.1 run: - python - cudf {{ version }} - - dask>=2021.6.0,<=2021.09.1 - - distributed>=2021.6.0,<=2021.09.1 + - dask=2021.9.1 + - distributed=2021.9.1 test: # [linux64] requires: # [linux64] diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt index bd94b4d6d52..1f0c6a4d9cf 100644 --- a/python/custreamz/dev_requirements.txt +++ b/python/custreamz/dev_requirements.txt @@ -3,8 +3,8 @@ flake8==3.8.3 black==19.10b0 isort==5.6.4 -dask>=2021.6.0,<=2021.09.1 -distributed>=2021.6.0,<=2021.09.1 +dask==2021.9.1 +distributed==2021.9.1 streamz python-confluent-kafka pytest diff --git a/python/dask_cudf/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt index 282f66d0d04..afcab224a2e 100644 --- a/python/dask_cudf/dev_requirements.txt +++ b/python/dask_cudf/dev_requirements.txt @@ -1,7 +1,7 @@ # Copyright (c) 2021, NVIDIA CORPORATION. -dask>=2021.6.0,<=2021.09.1 -distributed>=2021.6.0,<=2021.09.1 +dask==2021.9.1 +distributed==2021.9.1 fsspec>=0.6.0 numba>=0.53.1 numpy diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 84178caeb72..7b88034173f 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -10,8 +10,8 @@ install_requires = [ "cudf", - "dask>=2021.6.0,<=2021.09.1", - "distributed>=2021.6.0,<=2021.09.1", + "dask==2021.9.1", + "distributed==2021.9.1", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.4.0dev0", @@ -23,8 +23,8 @@ "pandas>=1.0,<1.4.0dev0", "pytest", "numba>=0.53.1", - "dask>=2021.6.0,<=2021.09.1", - "distributed>=2021.6.0,<=2021.09.1", + "dask==2021.9.1", + "distributed==2021.9.1", ] } From a4434ab187905464cc31b51c6c7ef65e355fc427 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 23 Sep 2021 12:57:51 -0500 Subject: [PATCH 3/4] Apply suggestions from code review --- conda/environments/cudf_dev_cuda11.0.yml | 4 ++-- conda/environments/cudf_dev_cuda11.2.yml | 4 ++-- conda/recipes/custreamz/meta.yaml | 4 ++-- conda/recipes/dask-cudf/meta.yaml | 8 ++++---- python/custreamz/dev_requirements.txt | 4 ++-- python/dask_cudf/dev_requirements.txt | 4 ++-- python/dask_cudf/setup.py | 8 ++++---- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 1ba6de034a5..1476c294682 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -39,8 +39,8 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask=2021.9.1 - - distributed=2021.9.1 + - dask=2021.09.1 + - distributed=2021.09.1 - streamz - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index b316ace6364..37f1899fcf5 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -39,8 +39,8 @@ dependencies: - mypy=0.782 - typing_extensions - pre_commit - - dask=2021.9.1 - - distributed=2021.9.1 + - dask=2021.09.1 + - distributed=2021.09.1 - streamz - arrow-cpp=5.0.0 - dlpack>=0.5,<0.6.0a0 diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index 6e6d33cf03f..db8aa8e6c85 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -31,8 +31,8 @@ requirements: - python - streamz - cudf {{ version }} - - dask=2021.9.1 - - distributed=2021.9.1 + - dask=2021.09.1 + - distributed=2021.09.1 - python-confluent-kafka - cudf_kafka {{ version }} diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index b8c88480506..45d96a2de85 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -26,13 +26,13 @@ requirements: host: - python - cudf {{ version }} - - dask=2021.9.1 - - distributed=2021.9.1 + - dask=2021.09.1 + - distributed=2021.09.1 run: - python - cudf {{ version }} - - dask=2021.9.1 - - distributed=2021.9.1 + - dask=2021.09.1 + - distributed=2021.09.1 test: # [linux64] requires: # [linux64] diff --git a/python/custreamz/dev_requirements.txt b/python/custreamz/dev_requirements.txt index 1f0c6a4d9cf..2f2a45dbe05 100644 --- a/python/custreamz/dev_requirements.txt +++ b/python/custreamz/dev_requirements.txt @@ -3,8 +3,8 @@ flake8==3.8.3 black==19.10b0 isort==5.6.4 -dask==2021.9.1 -distributed==2021.9.1 +dask==2021.09.1 +distributed==2021.09.1 streamz python-confluent-kafka pytest diff --git a/python/dask_cudf/dev_requirements.txt b/python/dask_cudf/dev_requirements.txt index afcab224a2e..7d41184feae 100644 --- a/python/dask_cudf/dev_requirements.txt +++ b/python/dask_cudf/dev_requirements.txt @@ -1,7 +1,7 @@ # Copyright (c) 2021, NVIDIA CORPORATION. -dask==2021.9.1 -distributed==2021.9.1 +dask==2021.09.1 +distributed==2021.09.1 fsspec>=0.6.0 numba>=0.53.1 numpy diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 7b88034173f..515469f8b6c 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -10,8 +10,8 @@ install_requires = [ "cudf", - "dask==2021.9.1", - "distributed==2021.9.1", + "dask==2021.09.1", + "distributed==2021.09.1", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.4.0dev0", @@ -23,8 +23,8 @@ "pandas>=1.0,<1.4.0dev0", "pytest", "numba>=0.53.1", - "dask==2021.9.1", - "distributed==2021.9.1", + "dask==2021.09.1", + "distributed==2021.09.1", ] } From 2325d1989823ddbbc7fdc56efa449f27d5a66122 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 23 Sep 2021 11:29:19 -0700 Subject: [PATCH 4/4] make an env var --- ci/benchmark/build.sh | 11 +++++++---- ci/gpu/build.sh | 7 +++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh index 7b7799fe93b..c2544ff7ffe 100755 --- a/ci/benchmark/build.sh +++ b/ci/benchmark/build.sh @@ -36,6 +36,9 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" # like `/tmp` is. export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" +# Dask & Distributed git tag +export DASK_DISTRIBUTED_GIT_TAG='2021.09.1' + function remove_libcudf_kernel_cache_dir { EXITCODE=$? logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" @@ -75,10 +78,10 @@ conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ # conda install "your-pkg=1.0.0" # Install the master version of dask, distributed, and streamz -logger "pip install git+https://github.com/dask/distributed.git@2021.09.1 --upgrade --no-deps" -pip install "git+https://github.com/dask/distributed.git@2021.09.1" --upgrade --no-deps -logger "pip install git+https://github.com/dask/dask.git@2021.09.1 --upgrade --no-deps" -pip install "git+https://github.com/dask/dask.git@2021.09.1" --upgrade --no-deps +logger "pip install git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps" +pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps +logger "pip install git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG --upgrade --no-deps" +pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps" pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index a1f74b87aeb..7c5b9d836dd 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -30,6 +30,9 @@ export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` +# Dask & Distributed git tag +export DASK_DISTRIBUTED_GIT_TAG='2021.09.1' + ################################################################################ # TRAP - Setup trap for removing jitify cache ################################################################################ @@ -101,8 +104,8 @@ function install_dask { # Install the main version of dask, distributed, and streamz gpuci_logger "Install the main version of dask, distributed, and streamz" set -x - pip install "git+https://github.com/dask/distributed.git@2021.09.1" --upgrade --no-deps - pip install "git+https://github.com/dask/dask.git@2021.09.1" --upgrade --no-deps + pip install "git+https://github.com/dask/distributed.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps + pip install "git+https://github.com/dask/dask.git@$DASK_DISTRIBUTED_GIT_TAG" --upgrade --no-deps # Need to uninstall streamz that is already in the env. pip uninstall -y streamz pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps