From d500828689325c684cc4d66a9014c0fe144c0f7c Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 7 Nov 2023 11:05:45 -0800 Subject: [PATCH 1/3] Use rapids meta package --- ci/build_wheel.sh | 1 + ci/test_wheel_raft_dask.sh | 3 --- conda/environments/all_cuda-118_arch-aarch64.yaml | 4 +--- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 +--- conda/environments/all_cuda-120_arch-aarch64.yaml | 4 +--- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 +--- conda/recipes/raft-dask/meta.yaml | 4 +--- dependencies.yaml | 5 ++--- python/raft-dask/pyproject.toml | 3 +-- 9 files changed, 9 insertions(+), 23 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 3dc7740a1e..d83aecf7ba 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -39,6 +39,7 @@ fi if [[ ${package_name} == "raft-dask" ]]; then sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} fi diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index a20e950313..b70563b7a1 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -11,9 +11,6 @@ RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl -# Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.12 - # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/raft_dask*.whl)[test] diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 762d436028..b39bf868ac 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -20,10 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - gmock>=1.13.0 @@ -49,6 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.00.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 48cd34c6ca..d59f9632a4 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -20,10 +20,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 @@ -49,6 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.00.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-aarch64.yaml b/conda/environments/all_cuda-120_arch-aarch64.yaml index 9108fb6215..fcee527c8b 100644 --- a/conda/environments/all_cuda-120_arch-aarch64.yaml +++ b/conda/environments/all_cuda-120_arch-aarch64.yaml @@ -21,10 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-aarch64=11.* - gmock>=1.13.0 @@ -45,6 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.00.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 8f1fbf6744..5fad367f38 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -21,10 +21,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask-core>=2023.9.2 - dask-cuda==23.12.* -- dask>=2023.9.2 -- distributed>=2023.9.2 - doxygen>=1.8.20 - gcc_linux-64=11.* - gmock>=1.13.0 @@ -45,6 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov +- rapids-dask-dependency==23.12.00.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml index db4fc860df..eae5a6affe 100644 --- a/conda/recipes/raft-dask/meta.yaml +++ b/conda/recipes/raft-dask/meta.yaml @@ -60,10 +60,8 @@ requirements: - cudatoolkit {% endif %} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - - dask >=2023.9.2 - - dask-core >=2023.9.2 - dask-cuda ={{ minor_version }} - - distributed >=2023.9.2 + - rapids-dask-dependency ={{ minor_version }} - joblib >=0.11 - nccl >=2.9.9 - pylibraft {{ version }} diff --git a/dependencies.yaml b/dependencies.yaml index aba81d7ed9..699f451595 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -436,21 +436,20 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - dask>=2023.9.2 - dask-cuda==23.12.* - - distributed>=2023.9.2 - joblib>=0.11 - numba>=0.57 - *numpy - ucx-py==0.35.* - output_types: conda packages: - - dask-core>=2023.9.2 + - rapids-dask-dependency==23.12.00.* - ucx>=1.13.0 - ucx-proc=*=gpu - &ucx_py_conda ucx-py==0.35.* - output_types: pyproject packages: + - rapids-dask-dependency==23.12.* - &pylibraft_conda pylibraft==23.12.* - output_types: requirements packages: diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 7abb6231a1..be030f839d 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -35,12 +35,11 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "dask-cuda==23.12.*", - "dask>=2023.9.2", - "distributed>=2023.9.2", "joblib>=0.11", "numba>=0.57", "numpy>=1.21", "pylibraft==23.12.*", + "rapids-dask-dependency==23.12.*", "ucx-py==0.35.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From ff3aa2a5c8f3532193a3308623abc28b79ea2c3f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 8 Nov 2023 08:42:11 -0800 Subject: [PATCH 2/3] fix package names and update update scripts --- ci/release/update-version.sh | 6 +----- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- dependencies.yaml | 3 +-- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 471987eb40..0be1f1ad82 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -57,6 +57,7 @@ DEPENDENCIES=( dask-cuda pylibraft rmm + rapids-dask-dependency # ucx-py is handled separately below ) for FILE in dependencies.yaml conda/environments/*.yaml; do @@ -75,11 +76,6 @@ done sed_runner "/^ucx_py_version:$/ {n;s/.*/ - \"${NEXT_UCX_PY_VERSION}\"/}" conda/recipes/raft-dask/conda_build_config.yaml -# Wheel builds install dask-cuda from source, update its branch -for FILE in .github/workflows/*.yaml; do - sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE}; -done - for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" done diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index b39bf868ac..c28f1961e6 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -46,7 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov -- rapids-dask-dependency==23.12.00.* +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d59f9632a4..9b7c110bc3 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -46,7 +46,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov -- rapids-dask-dependency==23.12.00.* +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-aarch64.yaml b/conda/environments/all_cuda-120_arch-aarch64.yaml index fcee527c8b..8d614d3c2c 100644 --- a/conda/environments/all_cuda-120_arch-aarch64.yaml +++ b/conda/environments/all_cuda-120_arch-aarch64.yaml @@ -42,7 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov -- rapids-dask-dependency==23.12.00.* +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 5fad367f38..f9d65cee39 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -42,7 +42,7 @@ dependencies: - pydata-sphinx-theme - pytest - pytest-cov -- rapids-dask-dependency==23.12.00.* +- rapids-dask-dependency==23.12.* - recommonmark - rmm==23.12.* - scikit-build>=0.13.1 diff --git a/dependencies.yaml b/dependencies.yaml index 699f451595..419cd0c22e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -440,16 +440,15 @@ dependencies: - joblib>=0.11 - numba>=0.57 - *numpy + - rapids-dask-dependency==23.12.* - ucx-py==0.35.* - output_types: conda packages: - - rapids-dask-dependency==23.12.00.* - ucx>=1.13.0 - ucx-proc=*=gpu - &ucx_py_conda ucx-py==0.35.* - output_types: pyproject packages: - - rapids-dask-dependency==23.12.* - &pylibraft_conda pylibraft==23.12.* - output_types: requirements packages: From 7d42c1a52a190ed23d132873a3e09c69930c5516 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 8 Nov 2023 15:53:39 -0800 Subject: [PATCH 3/3] alpha spec for dask-cuda --- ci/build_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d83aecf7ba..5d06e46303 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -40,6 +40,7 @@ if [[ ${package_name} == "raft-dask" ]]; then sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} fi