From 3cac1743941fa390a1c23d8c5f24dea53cfe8bf2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Aug 2023 08:57:28 -0700 Subject: [PATCH 1/7] Centralize logic --- ci/build_wheel.sh | 51 +++++++++++++++++++++++++ ci/build_wheel_cudf.sh | 23 +++-------- ci/build_wheel_dask_cudf.sh | 20 ++-------- ci/release/apply_wheel_modifications.sh | 32 ---------------- 4 files changed, 60 insertions(+), 66 deletions(-) create mode 100755 ci/build_wheel.sh delete mode 100755 ci/release/apply_wheel_modifications.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100755 index 00000000000..29c8312d353 --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name=$1 +package_dir=$2 + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" + +sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +for dep in rmm cudf ptxcompiler cubinlinker; do + sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +done + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} + sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" ${pyproject_file} + sed -i "/ptxcompiler/d" ${pyproject_file} + sed -i "/cubinlinker/d" ${pyproject_file} +fi + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index c20a5162788..88615b7f09e 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -3,24 +3,13 @@ set -euo pipefail -source rapids-configure-sccache -source rapids-date-string +package_dir="python/cudf" -# Use gha-tools rapids-pip-wheel-version to generate wheel version then -# update the necessary files -version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" +export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +./ci/build_wheel.sh cudf ${package_dir} -ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" -echo "The package name and/or version was modified in the package source. The git diff is:" -git diff +mkdir -p ${package_dir}/final_dist +python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* -cd python/cudf - -SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* - -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/build_wheel_dask_cudf.sh b/ci/build_wheel_dask_cudf.sh index 9af90d3a863..e1a303111fd 100755 --- a/ci/build_wheel_dask_cudf.sh +++ b/ci/build_wheel_dask_cudf.sh @@ -3,22 +3,8 @@ set -euo pipefail -source rapids-configure-sccache -source rapids-date-string +export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" -# Use gha-tools rapids-pip-wheel-version to generate wheel version then -# update the necessary files -version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" +./ci/build_wheel.sh dask_cudf python/dask_cudf -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" -echo "The package name and/or version was modified in the package source. The git diff is:" -git diff - -cd python/dask_cudf - -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 dist +RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh deleted file mode 100755 index 9d337aaa057..00000000000 --- a/ci/release/apply_wheel_modifications.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Usage: bash apply_wheel_modifications.sh - -VERSION=${1} -CUDA_SUFFIX=${2} - -# pyproject.toml versions -sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cudf/pyproject.toml -sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/dask_cudf/pyproject.toml -sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/cudf_kafka/pyproject.toml -sed -i "s/^version = .*/version = \"${VERSION}\"/g" python/custreamz/pyproject.toml - -# cudf pyproject.toml cuda suffixes -sed -i "s/^name = \"cudf\"/name = \"cudf${CUDA_SUFFIX}\"/g" python/cudf/pyproject.toml -sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/pyproject.toml -sed -i "s/ptxcompiler/ptxcompiler${CUDA_SUFFIX}/g" python/cudf/pyproject.toml -sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/pyproject.toml - -# dask_cudf pyproject.toml cuda suffixes -sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/pyproject.toml -# Need to provide the == to avoid modifying the URL -sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml - -if [[ $CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" python/cudf/pyproject.toml - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml - sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" python/{cudf,dask_cudf}/pyproject.toml - sed -i "/ptxcompiler/d" python/cudf/pyproject.toml - sed -i "/cubinlinker/d" python/cudf/pyproject.toml -fi From 4c350212bc77db3cbe83b99ebbe87276dc2c9a34 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Aug 2023 10:29:29 -0700 Subject: [PATCH 2/7] Make sure suffix exists in outer script --- ci/build_wheel_cudf.sh | 1 + ci/build_wheel_dask_cudf.sh | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cudf.sh b/ci/build_wheel_cudf.sh index 88615b7f09e..7d3919b2d72 100755 --- a/ci/build_wheel_cudf.sh +++ b/ci/build_wheel_cudf.sh @@ -12,4 +12,5 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" mkdir -p ${package_dir}/final_dist python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/* +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist diff --git a/ci/build_wheel_dask_cudf.sh b/ci/build_wheel_dask_cudf.sh index e1a303111fd..ec3f6b80083 100755 --- a/ci/build_wheel_dask_cudf.sh +++ b/ci/build_wheel_dask_cudf.sh @@ -3,8 +3,7 @@ set -euo pipefail -export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" - ./ci/build_wheel.sh dask_cudf python/dask_cudf +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist From 902fa7def2a949bb0475be4b2530b1d527687848 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Aug 2023 11:07:29 -0700 Subject: [PATCH 3/7] Move the comparator into the capture so that ptxcompiler and cubinlinker work too --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 29c8312d353..6b4330e0a97 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -35,7 +35,7 @@ if ! rapids-is-release-build; then fi for dep in rmm cudf ptxcompiler cubinlinker; do - sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/${dep}(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} done if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then From 3016a00580bac02661d9fc14c076e59dc91a8da0 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Aug 2023 11:43:19 -0700 Subject: [PATCH 4/7] Fix the sed logic --- ci/build_wheel.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 6b4330e0a97..c656dff7bd7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -34,9 +34,14 @@ if ! rapids-is-release-build; then alpha_spec=',>=0.0.0a0' fi -for dep in rmm cudf ptxcompiler cubinlinker; do - sed -r -i "s/${dep}(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -done +if [[ ${package_name} == "dask_cudf" ]]; then + sed -r -i "s/cudf(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} +else + sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + # ptxcompiler and cubinlinker aren't version constrained + sed -r -i "s/ptxcompiler\"/ptxcompiler${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + sed -r -i "s/cubinlinker\"/cubinlinker${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +fi if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} From a0fa3e6b42e224f5783fbfb0fada376020171deb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Aug 2023 13:30:32 -0700 Subject: [PATCH 5/7] Fix regex in dask-cudf --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c656dff7bd7..442aa159177 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -35,7 +35,7 @@ if ! rapids-is-release-build; then fi if [[ ${package_name} == "dask_cudf" ]]; then - sed -r -i "s/cudf(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} else sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} # ptxcompiler and cubinlinker aren't version constrained From 84a86dad1b27077f11d8421c718ad595e4587d34 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Aug 2023 15:15:10 -0700 Subject: [PATCH 6/7] More typo fixes --- ci/build_wheel_dask_cudf.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_dask_cudf.sh b/ci/build_wheel_dask_cudf.sh index ec3f6b80083..47e35c46004 100755 --- a/ci/build_wheel_dask_cudf.sh +++ b/ci/build_wheel_dask_cudf.sh @@ -3,7 +3,9 @@ set -euo pipefail -./ci/build_wheel.sh dask_cudf python/dask_cudf +package_dir="python/dask_cudf" + +./ci/build_wheel.sh dask_cudf ${package_dir} RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist +RAPIDS_PY_WHEEL_NAME="dask_cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist From 0455387c874c0479e9bd777e0ac294f53682f55a Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 31 Aug 2023 10:33:08 -0700 Subject: [PATCH 7/7] Remove now superfluous numba patch --- ci/build_wheel.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 442aa159177..06d0c3c7a56 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -46,7 +46,6 @@ fi if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} - sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" ${pyproject_file} sed -i "/ptxcompiler/d" ${pyproject_file} sed -i "/cubinlinker/d" ${pyproject_file} fi