[REVIEW] Upgrade to arrow-10.0.1

ci/cpu/build.sh

@@ -78,7 +78,8 @@ conda config --set ssl_verify False
 
 # TODO: Move boa install to gpuci/rapidsai
 gpuci_mamba_retry install boa
-
+gpuci_conda_retry remove --force pyarrow arrow-cpp
+gpuci_mamba_retry install -y "pyarrow=10.0.1" "arrow-cpp=10.0.1"
 ################################################################################
 # BUILD - Conda package builds
 ################################################################################

ci/gpu/build.sh

@@ -120,6 +120,9 @@ function install_dask {
 
 install_dask
 
+gpuci_conda_retry remove --force pyarrow arrow-cpp
+gpuci_mamba_retry install -y "pyarrow=10.0.1" "arrow-cpp=10.0.1"
+
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
 
     gpuci_logger "Install dependencies"

conda/environments/all_cuda-115_arch-x86_64.yaml

@@ -9,7 +9,8 @@ channels:
 - nvidia
 dependencies:
 - aiobotocore>=2.2.0
-- arrow-cpp=9
+# - libarrow=10.0.1
+- arrow-cpp=10.0.1
 - boto3>=1.21.21
 - botocore>=1.24.21
 - c-compiler
@@ -48,9 +49,9 @@ dependencies:
 - pandoc<=2.0.0
 - pip
 - pre-commit
-- protobuf>=3.20.1,<3.21.0a0
+- protobuf=4.21
 - ptxcompiler
-- pyarrow=9.0.0
+- pyarrow=10.0.1
 - pydata-sphinx-theme
 - pyorc
 - pytest

conda/recipes/cudf/meta.yaml

@@ -52,7 +52,7 @@ requirements:
     - setuptools
     - numba >=0.56.2
     - dlpack>=0.5,<0.6.0a0
-    - pyarrow =9
+    - pyarrow =10
     - libcudf ={{ version }}
     - rmm ={{ minor_version }}
     - cudatoolkit ={{ cuda_version }}

conda/recipes/libcudf/conda_build_config.yaml

@@ -17,7 +17,7 @@ gtest_version:
   - "=1.10.0"
 
 arrow_cpp_version:
-  - "=9"
+  - "=10.0.1"
 
 dlpack_version:
   - ">=0.5,<0.6.0a0"

cpp/cmake/thirdparty/get_arrow.cmake

@@ -365,7 +365,7 @@ endfunction()
 
 if(NOT DEFINED CUDF_VERSION_Arrow)
   set(CUDF_VERSION_Arrow
-      9.0.0
+      10.0.1
       CACHE STRING "The version of Arrow to find (or build)"
   )
 endif()

dependencies.yaml

@@ -57,12 +57,12 @@ dependencies:
           - cython>=0.29,<0.30
           - dlpack>=0.5,<0.6.0a0
           - ninja
-          - pyarrow=9.0.0
+          - pyarrow=10.0.1
           - rmm=23.02.*
           - scikit-build>=0.13.1
       - output_types: conda
         packages:
-          - arrow-cpp=9
+          - arrow-cpp=10
           - c-compiler
           - cxx-compiler
           - librdkafka=1.7.0

python/cudf/cudf/_lib/CMakeLists.txt

@@ -89,6 +89,19 @@ foreach(target IN LISTS targets_using_dlpack)
   target_include_directories(${target} PRIVATE "${DLPACK_INCLUDE_DIR}")
 endforeach()
 
+find_package(Python 3.8 REQUIRED COMPONENTS Interpreter)
+
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())"
+  OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+set(targets_using_arrow_headers interop avro csv orc json parquet)
+foreach(target IN LISTS targets_using_arrow_headers)
+  target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}")
+endforeach()
+
 add_subdirectory(io)
 add_subdirectory(nvtext)
 add_subdirectory(strings)

python/cudf/cudf/_lib/io/CMakeLists.txt

@@ -20,7 +20,16 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX io_ ASSOCIATED_TARGETS cudf
 )
 
-set(targets_using_numpy io_datasource io_utils)
-foreach(target IN LISTS targets_using_numpy)
+find_package(Python 3.8 REQUIRED COMPONENTS Interpreter)
+
+execute_process(
+  COMMAND "${Python_EXECUTABLE}" -c "import pyarrow; print(pyarrow.get_include())"
+  OUTPUT_VARIABLE PYARROW_INCLUDE_DIR
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+
+set(targets_using_numpy_and_arrow_headers io_datasource io_utils)
+foreach(target IN LISTS targets_using_numpy_and_arrow_headers)
   target_include_directories(${target} PRIVATE "${NumPy_INCLUDE_DIRS}")
+  target_include_directories(${target} PRIVATE "${PYARROW_INCLUDE_DIR}")
 endforeach()