Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade wheels to use arrow 13 #14339

Merged
merged 21 commits into from
Oct 31, 2023
Merged
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
@@ -101,6 +101,7 @@ jobs:
uses: rapidsai/shared-workflows/.github/workflows/[email protected]
with:
build_type: pull-request
build-2_28-wheels: "true"
script: "ci/build_wheel_cudf.sh"
wheel-tests-cudf:
needs: wheel-build-cudf
6 changes: 5 additions & 1 deletion ci/build_wheel_cudf.sh
Original file line number Diff line number Diff line change
@@ -9,8 +9,12 @@ export SKBUILD_CONFIGURE_OPTIONS="-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF"

./ci/build_wheel.sh cudf ${package_dir}

manylinux="manylinux_2_17"
mkdir -p ${package_dir}/final_dist
if command -v dnf >/dev/null 2>&1 ; then
vyasr marked this conversation as resolved.
Show resolved Hide resolved
manylinux="manylinux_2_28"
fi
python -m auditwheel repair -w ${package_dir}/final_dist ${package_dir}/dist/*

RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist
RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/final_dist
14 changes: 13 additions & 1 deletion ci/test_wheel_cudf.sh
Original file line number Diff line number Diff line change
@@ -3,8 +3,20 @@

set -eou pipefail

# Set the manylinux version to pull the newer ABI wheels when testing on Ubuntu
# 20.04 or later. Note that this logic is specific to the current set of
# citestwheel images and would need to change if the images change.
manylinux="manylinux_2_17"
if command -v lsb_release >/dev/null 2>&1 ; then
vyasr marked this conversation as resolved.
Show resolved Hide resolved
release_info=$(lsb_release -r)
major_version=$(echo "$release_info" | awk '{print $2}' | cut -d. -f1)
if [[ ${major_version} -ge 20 ]]; then
manylinux="manylinux_2_28"
fi
fi

RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
RAPIDS_PY_WHEEL_NAME="cudf_${manylinux}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist

# echo to expand wildcard before adding `[extra]` requires for pip
python -m pip install $(echo ./dist/cudf*.whl)[test]
44 changes: 25 additions & 19 deletions cpp/cmake/thirdparty/get_arrow.cmake
Original file line number Diff line number Diff line change
@@ -62,10 +62,26 @@ function(find_libarrow_in_python_wheel PYARROW_VERSION)
# to builds of GTest. Similarly, benchmarks will not work without updating GBench (and possibly
vyasr marked this conversation as resolved.
Show resolved Hide resolved
# NVBench) builds. We are currently ignoring these limitations since we don't anticipate using
# this feature except for building wheels.
target_compile_options(
Arrow::Arrow INTERFACE "$<$<COMPILE_LANGUAGE:CXX>:-D_GLIBCXX_USE_CXX11_ABI=0>"
"$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-D_GLIBCXX_USE_CXX11_ABI=0>"
EXECUTE_PROCESS(
COMMAND ${CMAKE_C_COMPILER} -print-file-name=libc.so.6
OUTPUT_VARIABLE GLIBC_EXECUTABLE
OUTPUT_STRIP_TRAILING_WHITESPACE
)
EXECUTE_PROCESS(
COMMAND ${GLIBC_EXECUTABLE}
OUTPUT_VARIABLE GLIBC_OUTPUT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
STRING(REGEX MATCH "stable release version ([0-9]+\\.[0-9]+)" GLIBC_VERSION ${GLIBC_OUTPUT})
STRING(REPLACE "stable release version " "" GLIBC_VERSION ${GLIBC_VERSION})
STRING(REPLACE "." ";" GLIBC_VERSION_LIST ${GLIBC_VERSION})
LIST(GET GLIBC_VERSION_LIST 1 GLIBC_VERSION_MINOR)
if(GLIBC_VERSION_MINOR LESS 28)
target_compile_options(
Arrow::Arrow INTERFACE "$<$<COMPILE_LANGUAGE:CXX>:-D_GLIBCXX_USE_CXX11_ABI=0>"
"$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-D_GLIBCXX_USE_CXX11_ABI=0>"
)
endif()

rapids_export_package(BUILD Arrow cudf-exports)
rapids_export_package(INSTALL Arrow cudf-exports)
@@ -408,22 +424,12 @@ function(find_and_configure_arrow VERSION BUILD_STATIC ENABLE_S3 ENABLE_ORC ENAB
endfunction()

if(NOT DEFINED CUDF_VERSION_Arrow)
# Temporarily use Arrow 12.0.1 in wheels and Arrow 13.0.0 otherwise
if(USE_LIBARROW_FROM_PYARROW)
set(CUDF_VERSION_Arrow
# This version must be kept in sync with the libarrow version pinned for builds in
# dependencies.yaml.
12.0.1
CACHE STRING "The version of Arrow to find (or build)"
)
else()
set(CUDF_VERSION_Arrow
# This version must be kept in sync with the libarrow version pinned for builds in
# dependencies.yaml.
13.0.0
CACHE STRING "The version of Arrow to find (or build)"
)
endif()
set(CUDF_VERSION_Arrow
# This version must be kept in sync with the libarrow version pinned for builds in
# dependencies.yaml.
13.0.0
CACHE STRING "The version of Arrow to find (or build)"
)
endif()

find_and_configure_arrow(
4 changes: 2 additions & 2 deletions dependencies.yaml
Original file line number Diff line number Diff line change
@@ -251,7 +251,7 @@ dependencies:
packages:
# Hard pin the patch version used during the build. This must be kept
# in sync with the version pinned in get_arrow.cmake.
- pyarrow==12.0.1.*
- pyarrow==13.0.0.*
vyasr marked this conversation as resolved.
Show resolved Hide resolved
build_python:
common:
- output_types: [conda, requirements, pyproject]
@@ -279,7 +279,7 @@ dependencies:
- output_types: [requirements, pyproject]
packages:
# Allow runtime version to float up to minor version
- pyarrow==12.*
- pyarrow==13.*
vyasr marked this conversation as resolved.
Show resolved Hide resolved
cudatoolkit:
specific:
- output_types: conda
4 changes: 2 additions & 2 deletions python/cudf/pyproject.toml
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ requires = [
"ninja",
"numpy>=1.21,<1.25",
"protoc-wheel",
"pyarrow==12.0.1.*",
"pyarrow==13.0.0.*",
"rmm==23.12.*",
"scikit-build>=0.13.1",
"setuptools",
@@ -38,7 +38,7 @@ dependencies = [
"pandas>=1.3,<1.6.0dev0",
"protobuf>=4.21,<5",
"ptxcompiler",
"pyarrow==12.*",
"pyarrow==13.*",
"rmm==23.12.*",
"typing_extensions>=4.0.0",
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
2 changes: 1 addition & 1 deletion python/cudf_kafka/pyproject.toml
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@
requires = [
"cython>=3.0.0",
"numpy>=1.21,<1.25",
"pyarrow==12.0.1.*",
"pyarrow==13.0.0.*",
"setuptools",
"wheel",
] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.