[REVIEW] Upgrade to arrow-11

ci/build_cpp.sh

@@ -1,12 +1,15 @@
 #!/bin/bash
-# Copyright (c) 2022, NVIDIA CORPORATION.
+# Copyright (c) 2022-2023, NVIDIA CORPORATION.
 
 set -euo pipefail
 
 source rapids-env-update
 
 export CMAKE_GENERATOR=Ninja
 
+gpuci_conda_retry remove --force pyarrow arrow-cpp openssl librdkafka
+rapids-mamba-retry install -y "pyarrow=11" "libarrow=11" "librdkafka>=1.9.0,<1.10.0a0""
+
 rapids-print-env
 
 rapids-logger "Begin cpp build"

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -30,8 +30,8 @@ dependencies:
 - gcc_linux-64=9.*
 - hypothesis
 - ipython
-- libarrow=10
-- librdkafka=1.7.0
+- libarrow=11
+- librdkafka>=1.9.0,<1.10.0a0
 - mimesis>=4.1.0
 - moto>=4.0.8
 - myst-nb
@@ -45,20 +45,20 @@ dependencies:
 - nvtx>=0.2.1
 - packaging
 - pandas>=1.0,<1.6.0dev0
-- pandoc<=2.0.0
+- pandoc
 - pip
 - pre-commit
 - protobuf=4.21
 - ptxcompiler
-- pyarrow=10
+- pyarrow=11
 - pydata-sphinx-theme
 - pyorc
 - pytest
 - pytest-benchmark
 - pytest-cases
 - pytest-cov
 - pytest-xdist
-- python-confluent-kafka=1.7.0
+- python-confluent-kafka>=1.9.0,<1.10.0a0
 - python-snappy>=0.6.0
 - python>=3.8,<3.11
 - pytorch<1.12.0

conda/recipes/cudf/meta.yaml

@@ -52,7 +52,7 @@ requirements:
     - setuptools
     - numba >=0.56.2
     - dlpack >=0.5,<0.6.0a0
-    - pyarrow =10
+    - pyarrow =11
     - libcudf ={{ version }}
     - rmm ={{ minor_version }}
     - cudatoolkit ={{ cuda_version }}

conda/recipes/cudf_kafka/meta.yaml

@@ -46,7 +46,7 @@ requirements:
   run:
     - python
     - libcudf_kafka ={{ version }}
-    - python-confluent-kafka >=1.7.0,<1.8.0a0
+    - python-confluent-kafka >=1.9.0,<1.10.0a0
     - cudf ={{ version }}
 
 test:

conda/recipes/libcudf/conda_build_config.yaml

@@ -17,10 +17,10 @@ gtest_version:
   - "=1.10.0"
 
 libarrow_version:
-  - "=10"
+  - "=11"
 
 dlpack_version:
   - ">=0.5,<0.6.0a0"
 
 librdkafka_version:
-  - ">=1.7.0,<1.8.0a0"
+  - ">=1.9.0,<1.10.0a0"

cpp/cmake/thirdparty/get_arrow.cmake

@@ -379,7 +379,7 @@ endfunction()
 
 if(NOT DEFINED CUDF_VERSION_Arrow)
   set(CUDF_VERSION_Arrow
-      10.0.1
+      11.0.0
       CACHE STRING "The version of Arrow to find (or build)"
   )
 endif()

dependencies.yaml

@@ -58,15 +58,15 @@ dependencies:
           - cython>=0.29,<0.30
           - dlpack>=0.5,<0.6.0a0
           - ninja
-          - pyarrow=10
+          - pyarrow=11
           - rmm=23.04.*
           - scikit-build>=0.13.1
       - output_types: conda
         packages:
-          - libarrow=10
+          - libarrow=11
           - c-compiler
           - cxx-compiler
-          - librdkafka=1.7.0
+          - librdkafka>=1.9.0,<1.10.0a0
           - protobuf=4.21
     specific:
       - output_types: conda
@@ -128,7 +128,7 @@ dependencies:
           - myst-nb
           - nbsphinx
           - numpydoc
-          - pandoc<=2.0.0 # We should check and fix all "<=" pinnings
+          - pandoc
           - pydata-sphinx-theme
           - sphinx
           - sphinx-autobuild
@@ -234,7 +234,7 @@ dependencies:
           - nvtx>=0.2.1
           - packaging
           - pandas>=1.0,<1.6.0dev0
-          - python-confluent-kafka=1.7.0
+          - python-confluent-kafka>=1.9.0,<1.10.0a0
           - streamz
           - typing_extensions
       - output_types: conda

python/cudf/pyproject.toml

@@ -11,7 +11,7 @@ requires = [
     "ninja",
     "numpy",
     # Hard pin the patch version used during the build.
-    "pyarrow==10.0.1",
+    "pyarrow==11.0.0",
     "protoc-wheel",
     "versioneer",
 ]

python/cudf/setup.py

@@ -20,7 +20,7 @@
     "protobuf==4.21",
     "typing_extensions",
     # Allow floating minor versions for Arrow.
-    "pyarrow==10",
+    "pyarrow==11",
     f"rmm{cuda_suffix}==23.4.*",
     f"ptxcompiler{cuda_suffix}",
     f"cubinlinker{cuda_suffix}",