From ae4c7e3ce4fe100eb919ca00fa34461e44078ba9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 24 Jul 2024 18:30:53 -0500 Subject: [PATCH] split up CUDA-suffixed dependencies in dependencies.yaml (#16183) Contributes to https://github.com/rapidsai/build-planning/issues/31 Follow-up to #15245 RAPIDS DLFW builds prefer to build all RAPIDS packages together without CUDA suffixes, leading to the following set of requirements for `cudf` wheels built there: * project name must be `cudf` (not `cudf-cu12`) * all dependencies must be unsuffixed (e.g. `rmm` not `rmm-cu12`) * the correct set of dependencies based on CUDA version must be expressed in the wheel metadata (e.g. `cubinlinker` and `ptxcompiler` on CUDA 11, `pynvjitlink` on CUDA 12) To meet all 3 of those, this proposes decomposing CUDA-suffixed dependencies in `dependencies.yaml` into two lists... `cuda_suffixed="true"` and `cuda_suffixed="false"`. That'd allow DLFW builds to do the following to meet its requirements: ```shell pip wheel \ -C rapidsai.disable-cuda=true \ -C rapidsai.matrix-entry="cuda=12.5;cuda_suffixed=false" \ . ``` Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/16183 --- ci/release/update-version.sh | 5 ++- dependencies.yaml | 71 ++++++++++++++++++++++--------- python/cudf/pyproject.toml | 1 + python/cudf_kafka/pyproject.toml | 1 + python/cudf_polars/pyproject.toml | 1 + python/custreamz/pyproject.toml | 1 + python/dask_cudf/pyproject.toml | 1 + 7 files changed, 59 insertions(+), 22 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index f629de64905..ad96aff3930 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -68,15 +68,18 @@ done # README.md update sed_runner "s/version == ${CURRENT_SHORT_TAG}/version == ${NEXT_SHORT_TAG}/g" README.md sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md +sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" python/cudf_polars/docs/overview.md +sed_runner "s/branch-${CURRENT_SHORT_TAG}/branch-${NEXT_SHORT_TAG}/g" python/cudf_polars/docs/overview.md # Libcudf examples update sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/versions.cmake # CI files -for FILE in .github/workflows/*.yaml; do +for FILE in .github/workflows/*.yaml .github/workflows/*.yml; do sed_runner "/shared-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" sed_runner "s/dask-cuda.git@branch-[^\"\s]\+/dask-cuda.git@branch-${NEXT_SHORT_TAG}/g" ${FILE}; done +sed_runner "s/branch-[0-9]+\.[0-9]+/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cudf_polars.sh # Java files NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}-SNAPSHOT" diff --git a/dependencies.yaml b/dependencies.yaml index a19574b7658..48433d8e5c1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -329,7 +329,7 @@ dependencies: common: - output_types: conda packages: - - &rmm_conda rmm==24.8.*,>=0.0.0a0 + - &rmm_unsuffixed rmm==24.8.*,>=0.0.0a0 - pip - pip: - git+https://github.com/python-streamz/streamz.git@master @@ -343,13 +343,17 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} - packages: &build_python_packages_cu12 + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: - rmm-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} - packages: &build_python_packages_cu11 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: - rmm-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: [*rmm_conda] } + - {matrix: null, packages: [*rmm_unsuffixed]} libarrow_build: common: - output_types: conda @@ -567,7 +571,7 @@ dependencies: - typing_extensions>=4.0.0 - output_types: conda packages: - - *rmm_conda + - *rmm_unsuffixed - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -588,23 +592,40 @@ dependencies: matrices: - matrix: {cuda: "12.*"} packages: - - pynvjitlink>=0.0.0a0 + - &pynvjitlink_unsuffixed pynvjitlink>=0.0.0a0 - matrix: {cuda: "11.*"} packages: - - cubinlinker - - ptxcompiler + - &cubinlinker_unsuffixed cubinlinker + - &ptxcompiler_unsuffixed ptxcompiler - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - rmm-cu12==24.8.*,>=0.0.0a0 - pynvjitlink-cu12>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "false" + packages: + - *rmm_unsuffixed + - *pynvjitlink_unsuffixed + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - rmm-cu11==24.8.*,>=0.0.0a0 - cubinlinker-cu11 - ptxcompiler-cu11 - - {matrix: null, packages: [cubinlinker, ptxcompiler, *rmm_conda]} + - matrix: + cuda: "11.*" + cuda_suffixed: "false" + packages: &run_cudf_cu11_unsuffixed + - *cubinlinker_unsuffixed + - *ptxcompiler_unsuffixed + - *rmm_unsuffixed + - {matrix: null, packages: *run_cudf_cu11_unsuffixed} run_cudf_polars: common: - output_types: [conda, requirements, pyproject] @@ -706,7 +727,7 @@ dependencies: common: - output_types: conda packages: - - &cudf_conda cudf==24.8.*,>=0.0.0a0 + - &cudf_unsuffixed cudf==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -716,18 +737,22 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - cudf-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - cudf-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: [*cudf_conda]} + - {matrix: null, packages: [*cudf_unsuffixed]} depends_on_cudf_kafka: common: - output_types: conda packages: - - &cudf_kafka_conda cudf_kafka==24.8.*,>=0.0.0a0 + - &cudf_kafka_unsuffixed cudf_kafka==24.8.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -737,13 +762,17 @@ dependencies: specific: - output_types: [requirements, pyproject] matrices: - - matrix: {cuda: "12.*"} + - matrix: + cuda: "12.*" + cuda_suffixed: "true" packages: - cudf_kafka-cu12==24.8.*,>=0.0.0a0 - - matrix: {cuda: "11.*"} + - matrix: + cuda: "11.*" + cuda_suffixed: "true" packages: - cudf_kafka-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: [*cudf_kafka_conda]} + - {matrix: null, packages: [*cudf_kafka_unsuffixed]} depends_on_cupy: common: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index dcb33b1fc1a..30b0f6249f9 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -120,6 +120,7 @@ skip = [ [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index badfdf06d15..eba4e808a89 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -100,6 +100,7 @@ regex = "(?P.*)" [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.26.4,!=3.30.0", "cython>=3.0.3", diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 0b559f7a8e9..def1d086cc1 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -182,3 +182,4 @@ docstring-code-format = true [tool.rapids-build-backend] build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 7b99e041b54..70d16c4b07f 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -49,6 +49,7 @@ Homepage = "https://github.com/rapidsai/cudf" [tool.rapids-build-backend] build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" [tool.setuptools] license-files = ["LICENSE"] diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 9b2e3a5a7b1..16e07428d6b 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -58,6 +58,7 @@ Homepage = "https://github.com/rapidsai/cudf" [tool.rapids-build-backend] build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" [tool.setuptools] license-files = ["LICENSE"]