Build KvikIO as a shared library

README.md

@@ -6,7 +6,6 @@ KvikIO (pronounced "kuh-VICK-eye-oh", see [here](https://ordnet.dk/ddo_en/dict?q
 bindings to [cuFile](https://docs.nvidia.com/gpudirect-storage/api-reference-guide/index.html),
 which enables [GPUDirect Storage (GDS)](https://developer.nvidia.com/blog/gpudirect-storage/).
 KvikIO also works efficiently when GDS isn't available and can read/write both host and device data seamlessly.
-The C++ library is header-only making it easy to include in [existing projects](https://github.com/rapidsai/kvikio/blob/HEAD/cpp/examples/downstream/).
 
 
 ### Features

ci/build_wheel.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_name=$1
+package_dir=$2
+
+source rapids-configure-sccache
+source rapids-date-string
+
+rapids-generate-version > ./VERSION
+
+cd "${package_dir}"
+
+sccache --zero-stats
+
+rapids-logger "Building '${package_name}' wheel"
+python -m pip wheel \
+    -w dist \
+    -v \
+    --no-deps \
+    --disable-pip-version-check \
+    .
+
+sccache --show-adv-stats

ci/build_wheel_cpp.sh

@@ -6,19 +6,34 @@ set -euo pipefail
 package_name="libkvikio"
 package_dir="python/libkvikio"
 
-source rapids-configure-sccache
-source rapids-date-string
+rapids-logger "Generating build requirements"
 
-rapids-generate-version > ./VERSION
+rapids-dependency-file-generator \
+  --output requirements \
+  --file-key "py_build_${package_name}" \
+  --file-key "py_rapids_build_${package_name}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \
+| tee /tmp/requirements-build.txt
 
-cd "${package_dir}"
+rapids-logger "Installing build requirements"
+python -m pip install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt
 
-sccache --zero-stats
+# build with '--no-build-isolation', for better sccache hit rate
+# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735)
+export PIP_NO_BUILD_ISOLATION=0
 
-python -m pip install wheel
-python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check
-
-sccache --show-adv-stats
+export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON"
+./ci/build_wheel.sh "${package_name}" "${package_dir}"
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist
+
+mkdir -p ${package_dir}/final_dist
+python -m auditwheel repair \
+    --exclude libnvcomp.so.4 \
+    -w ${package_dir}/final_dist \
+    ${package_dir}/dist/*
+
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp "${package_dir}/final_dist"

ci/build_wheel_python.sh

@@ -6,35 +6,23 @@ set -euo pipefail
 package_name="kvikio"
 package_dir="python/kvikio"
 
-source rapids-configure-sccache
-source rapids-date-string
-
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-rapids-generate-version > ./VERSION
-
-CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist)
-
-cd "${package_dir}"
-
-# ensure 'kvikio' wheel builds always use the 'libkvikio' just built in the same CI run
+# Ensure 'kvikio' wheel builds always use the 'libkvikio' just built in the same CI run
 #
-# using env variable PIP_CONSTRAINT is necessary to ensure the constraints
+# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints
 # are used when creating the isolated build environment
-echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/libkvikio_*.whl)" > ./constraints.txt
-
-sccache --zero-stats
-
-PIP_CONSTRAINT="${PWD}/constraints.txt" \
-SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" \
-    python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check
+RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist
+echo "libkvikio-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libkvikio_dist/libkvikio_*.whl)" > /tmp/constraints.txt
+export PIP_CONSTRAINT="/tmp/constraints.txt"
 
-sccache --show-adv-stats
+export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON"
+./ci/build_wheel.sh "${package_name}" "${package_dir}"
 
-mkdir -p final_dist
 python -m auditwheel repair \
+    --exclude libkvikio.so \
     --exclude libnvcomp.so.4 \
-    -w final_dist \
-    dist/*
+    -w ${package_dir}/final_dist \
+    ${package_dir}/dist/*
 
-RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist
+RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python ${package_dir}/final_dist

ci/test_wheel.sh

@@ -4,10 +4,14 @@
 set -eou pipefail
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
-WHEELHOUSE="${PWD}/dist/"
-RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}"
 
-python -m pip install "$(echo ${WHEELHOUSE}/kvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
+# Download and install the libkvikio and kvikio wheels built in the previous step
+RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./dist
+RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
+
+python -m pip install -v \
+  "$(echo ./dist/libkvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \
+  "$(echo ./dist/kvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
 
 # If running CUDA 11.8 on arm64, we skip tests marked "cufile" since
 # cuFile didn't support arm until 12.4