From a19431f5802ecc022513a132dd8d309a71f4c7c3 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 Nov 2023 13:59:03 -0800 Subject: [PATCH 1/9] Update dependency lists --- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 +--- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 +--- conda/recipes/dask-cudf/meta.yaml | 8 +------- dependencies.yaml | 7 ++----- python/dask_cudf/pyproject.toml | 3 +-- 5 files changed, 6 insertions(+), 20 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 3d3d4f15d05..3366ccbb331 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -25,10 +25,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - dlpack>=0.5,<0.6.0a0 - doxygen=1.9.1 - fastavro>=0.22.9 @@ -80,6 +77,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 +- rapids_dask_dependency==23.12.00.* - rmm==23.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 4f39424bbc6..1cd1be391e0 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -27,10 +27,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - dlpack>=0.5,<0.6.0a0 - doxygen=1.9.1 - fastavro>=0.22.9 @@ -78,6 +75,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 +- rapids_dask_dependency==23.12.00.* - rmm==23.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 9dc9f76d9f5..aed2d250d5b 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -37,17 +37,11 @@ build: requirements: host: - python - - cudf ={{ version }} - - dask >=2023.9.2 - - dask-core >=2023.9.2 - - distributed >=2023.9.2 - cuda-version ={{ cuda_version }} run: - python - cudf ={{ version }} - - dask >=2023.9.2 - - dask-core >=2023.9.2 - - distributed >=2023.9.2 + - rapids_dask_dependency==23.12.00.* - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} test: diff --git a/dependencies.yaml b/dependencies.yaml index bb61e244b97..a4455110dec 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -478,16 +478,13 @@ dependencies: - ptxcompiler-cu11 run_dask_cudf: common: - - output_types: [conda, requirements, pyproject] - packages: - - dask>=2023.9.2 - - distributed>=2023.9.2 - output_types: conda packages: + - rapids_dask_dependency==23.12.00.* - cupy>=12.0.0 - - dask-core>=2023.9.2 # dask-core in conda is the actual package & dask is the meta package - output_types: pyproject packages: + - rapids_dask_dependency==23.12.* - &cudf cudf==23.12.* - *cupy_pip run_cudf_kafka: diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 32c7bb9fd15..aeccf83b73a 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -20,11 +20,10 @@ requires-python = ">=3.9" dependencies = [ "cudf==23.12.*", "cupy-cuda11x>=12.0.0", - "dask>=2023.9.2", - "distributed>=2023.9.2", "fsspec>=0.6.0", "numpy>=1.21,<1.25", "pandas>=1.3,<1.6.0dev0", + "rapids_dask_dependency==23.12.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From f2794a9a39611dfdbc5c2234ff3890826d3a41c2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 Nov 2023 14:02:11 -0800 Subject: [PATCH 2/9] Update wheel building to stop needing manual installations --- ci/build_wheel.sh | 1 + ci/test_wheel_dask_cudf.sh | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index a770c260bef..4fb0ce85d97 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -36,6 +36,7 @@ fi if [[ ${package_name} == "dask_cudf" ]]; then sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} # ptxcompiler and cubinlinker aren't version constrained diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 118bea753d0..e9162b816aa 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -23,9 +23,6 @@ manylinux="manylinux_${manylinux_version}" RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-cudf-dep python -m pip install --no-deps ./local-cudf-dep/cudf*.whl -# Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.12 - # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cudf*.whl)[test] From 1c4dbd565adde35c0689ccfddc48954653130104 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 6 Nov 2023 15:51:30 -0800 Subject: [PATCH 3/9] Update wheel dependency with alpha spec --- ci/build_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 4fb0ce85d97..f87fc54da68 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -37,6 +37,7 @@ fi if [[ ${package_name} == "dask_cudf" ]]; then sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/rapids_dask_dependency==(.*)\"/rapids_dask_dependency==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} # ptxcompiler and cubinlinker aren't version constrained From a95a5bc7be91a78e64688ac46cce7cffbdff0b4c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Nov 2023 10:10:45 -0800 Subject: [PATCH 4/9] Rename the package --- ci/build_wheel.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/dask-cudf/meta.yaml | 2 +- dependencies.yaml | 4 ++-- python/dask_cudf/pyproject.toml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index f87fc54da68..61dc64ee2ef 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -37,7 +37,7 @@ fi if [[ ${package_name} == "dask_cudf" ]]; then sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/rapids_dask_dependency==(.*)\"/rapids_dask_dependency==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} # ptxcompiler and cubinlinker aren't version constrained diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 3366ccbb331..d50fcef405c 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -77,7 +77,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 -- rapids_dask_dependency==23.12.00.* +- rapids-dask-dependency==23.12.00.* - rmm==23.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 1cd1be391e0..246d58fb232 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -75,7 +75,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 -- rapids_dask_dependency==23.12.00.* +- rapids-dask-dependency==23.12.00.* - rmm==23.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index aed2d250d5b..fee87dfb994 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -41,7 +41,7 @@ requirements: run: - python - cudf ={{ version }} - - rapids_dask_dependency==23.12.00.* + - rapids-dask-dependency==23.12.00.* - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} test: diff --git a/dependencies.yaml b/dependencies.yaml index a4455110dec..7969431fcba 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -480,11 +480,11 @@ dependencies: common: - output_types: conda packages: - - rapids_dask_dependency==23.12.00.* + - rapids-dask-dependency==23.12.00.* - cupy>=12.0.0 - output_types: pyproject packages: - - rapids_dask_dependency==23.12.* + - rapids-dask-dependency==23.12.* - &cudf cudf==23.12.* - *cupy_pip run_cudf_kafka: diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index aeccf83b73a..0306da3de46 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "fsspec>=0.6.0", "numpy>=1.21,<1.25", "pandas>=1.3,<1.6.0dev0", - "rapids_dask_dependency==23.12.*", + "rapids-dask-dependency==23.12.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", From 443f01836b549db9ca3137016b5c519a2cd07cf2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Nov 2023 11:10:40 -0800 Subject: [PATCH 5/9] Update update-version.sh --- ci/release/update-version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 7574b4174e9..843abd3c3c1 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -80,6 +80,7 @@ DEPENDENCIES=( kvikio libkvikio librmm + rapids-dask-dependency rmm ) for DEP in "${DEPENDENCIES[@]}"; do From 6013c37065e15958740cb66a47b447410f9f3579 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Nov 2023 11:53:31 -0800 Subject: [PATCH 6/9] Update conda/recipes/dask-cudf/meta.yaml Co-authored-by: GALI PREM SAGAR --- conda/recipes/dask-cudf/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index fee87dfb994..42d67a6cf9a 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -41,7 +41,7 @@ requirements: run: - python - cudf ={{ version }} - - rapids-dask-dependency==23.12.00.* + - rapids-dask-dependency ={{ version }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} test: From 5a308e44e9c963cfe6159dd88db6f36d866a2856 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Nov 2023 16:15:24 -0800 Subject: [PATCH 7/9] Make pip/conda dependencies consistent and fix recipe --- conda/recipes/dask-cudf/meta.yaml | 2 +- dependencies.yaml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 42d67a6cf9a..16638926492 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -41,7 +41,7 @@ requirements: run: - python - cudf ={{ version }} - - rapids-dask-dependency ={{ version }} + - rapids-dask-dependency ={{ minor_version }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} test: diff --git a/dependencies.yaml b/dependencies.yaml index 7969431fcba..52931ba10f1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -478,13 +478,14 @@ dependencies: - ptxcompiler-cu11 run_dask_cudf: common: + - output_types: [conda, requirements, pyproject] + packages: + - rapids-dask-dependency==23.12.* - output_types: conda packages: - - rapids-dask-dependency==23.12.00.* - cupy>=12.0.0 - output_types: pyproject packages: - - rapids-dask-dependency==23.12.* - &cudf cudf==23.12.* - *cupy_pip run_cudf_kafka: From 55a781225c00be00e791d661c773a6d9025c7a05 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 Nov 2023 16:22:28 -0800 Subject: [PATCH 8/9] dfg --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d50fcef405c..41a868f86ff 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -77,7 +77,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 -- rapids-dask-dependency==23.12.00.* +- rapids-dask-dependency==23.12.* - rmm==23.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 246d58fb232..cabb369e5dc 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -75,7 +75,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.9,<3.11 - pytorch<1.12.0 -- rapids-dask-dependency==23.12.00.* +- rapids-dask-dependency==23.12.* - rmm==23.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 From bf6d71bb3d1a1e173cdc5bc131bce95412a41969 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 13 Nov 2023 10:26:52 -0600 Subject: [PATCH 9/9] Apply suggestions from code review --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - conda/environments/all_cuda-120_arch-x86_64.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 0b69ca2435e..9b85888a7b3 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -25,7 +25,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.3 -- dask-core>=2023.9.2 - dask-cuda==23.12.* - dlpack>=0.5,<0.6.0a0 - doxygen=1.9.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 715ea65cc96..da2b4e109b3 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -27,7 +27,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.3 -- dask-core>=2023.9.2 - dask-cuda==23.12.* - dlpack>=0.5,<0.6.0a0 - doxygen=1.9.1