diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e99aa12..89546be1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -49,7 +49,7 @@ repos: - --fix - --rapids-version=24.12 - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 + rev: v1.16.0 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 03d4449b..98410678 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -17,7 +17,11 @@ rapids-print-env rapids-logger "Begin C++ and Python builds" +sccache --zero-stats + rapids-conda-retry mambabuild \ conda/recipes/ucxx +sccache --show-adv-stats + rapids-upload-conda-to-s3 cpp diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 2d690d02..b1ede832 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -9,33 +9,18 @@ package_dir=$2 source rapids-configure-sccache source rapids-date-string -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - rapids-generate-version > ./VERSION -if [[ ${package_name} == "distributed-ucxx" ]]; then - python -m pip wheel "${package_dir}/" -w "${package_dir}/dist" -vvv --no-deps --disable-pip-version-check - - RAPIDS_PY_WHEEL_NAME="distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/dist -elif [[ ${package_name} == "libucxx" ]]; then - SKBUILD_CMAKE_ARGS="-DUCXX_ENABLE_RMM=ON" \ - python -m pip wheel "${package_dir}"/ -w "${package_dir}"/dist -vvv --no-deps --disable-pip-version-check - - python -m auditwheel repair -w ${package_dir}/final_dist --exclude "libucp.so.0" ${package_dir}/dist/* - - RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp ${package_dir}/final_dist -elif [[ ${package_name} == "ucxx" ]]; then - CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libucxx_dist) - echo "libucxx-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/libucxx_*.whl)" > "${package_dir}/constraints.txt" +cd "${package_dir}" - PIP_CONSTRAINT="${package_dir}/constraints.txt" \ - SKBUILD_CMAKE_ARGS="-DFIND_UCXX_CPP=ON;-DCMAKE_INSTALL_LIBDIR=ucxx/lib64;-DCMAKE_INSTALL_INCLUDEDIR=ucxx/include" \ - python -m pip wheel "${package_dir}"/ -w "${package_dir}"/dist -vvv --no-deps --disable-pip-version-check +sccache --zero-stats - python -m auditwheel repair -w ${package_dir}/final_dist --exclude "libucp.so.0" --exclude "libucxx.so" ${package_dir}/dist/* +rapids-logger "Building '${package_name}' wheel" +python -m pip wheel \ + -w dist \ + -v \ + --no-deps \ + --disable-pip-version-check \ + . - RAPIDS_PY_WHEEL_NAME="ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/final_dist -else - echo "Unknown package '${package_name}'" - exit 1 -fi +sccache --show-adv-stats diff --git a/ci/build_wheel_distributed_ucxx.sh b/ci/build_wheel_distributed_ucxx.sh index 77c2d988..f6ee95a7 100755 --- a/ci/build_wheel_distributed_ucxx.sh +++ b/ci/build_wheel_distributed_ucxx.sh @@ -5,4 +5,8 @@ set -euo pipefail package_dir="python/distributed-ucxx" -./ci/build_wheel.sh distributed-ucxx ${package_dir} +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +./ci/build_wheel.sh distributed-ucxx "${package_dir}" + +RAPIDS_PY_WHEEL_NAME="distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${package_dir}/dist" diff --git a/ci/build_wheel_libucxx.sh b/ci/build_wheel_libucxx.sh index c5798f2f..e9262077 100755 --- a/ci/build_wheel_libucxx.sh +++ b/ci/build_wheel_libucxx.sh @@ -3,6 +3,38 @@ set -euo pipefail +package_name="libucxx" package_dir="python/libucxx" -./ci/build_wheel.sh libucxx ${package_dir} +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + +export SKBUILD_CMAKE_ARGS="-DUCXX_ENABLE_RMM=ON" + +./ci/build_wheel.sh "${package_name}" "${package_dir}" + +mkdir -p "${package_dir}/final_dist" +python -m auditwheel repair \ + --exclude "libucp.so.0" \ + -w "${package_dir}/final_dist" \ + ${package_dir}/dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp "${package_dir}/final_dist" diff --git a/ci/build_wheel_ucxx.sh b/ci/build_wheel_ucxx.sh index 04897ffe..78602085 100755 --- a/ci/build_wheel_ucxx.sh +++ b/ci/build_wheel_ucxx.sh @@ -3,6 +3,29 @@ set -euo pipefail +package_name="ucxx" package_dir="python/ucxx" -./ci/build_wheel.sh ucxx ${package_dir} +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# Downloads libucxx wheel from this current build, +# then ensures 'ucxx' wheel builds always use the 'libucxx' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libucxx_dist +echo "libucxx-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libucxx_dist/libucxx_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + +export SKBUILD_CMAKE_ARGS="-DFIND_UCXX_CPP=ON;-DCMAKE_INSTALL_LIBDIR=ucxx/lib64;-DCMAKE_INSTALL_INCLUDEDIR=ucxx/include" + +./ci/build_wheel.sh "${package_name}" "${package_dir}" + +mkdir -p "${package_dir}/final_dist" +python -m auditwheel repair \ + --exclude "libucp.so.0" \ + --exclude "libucxx.so" \ + -w "${package_dir}/final_dist" \ + ${package_dir}/dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${package_dir}/final_dist"