[REVIEW] Upgrade to arrow-11

ci/build_cpp.sh

@@ -7,6 +7,9 @@ source rapids-env-update
 
 export CMAKE_GENERATOR=Ninja
 
+gpuci_conda_retry remove --force pyarrow arrow-cpp openssl librdkafka
+rapids-mamba-retry install -y "pyarrow=11" "libarrow=11" "librdkafka>=1.9.0,<1.10.0a0"
+
 rapids-print-env
 
 rapids-logger "Begin cpp build"

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -34,8 +34,8 @@ dependencies:
 - gtest==1.10.0.*
 - hypothesis
 - ipython
-- libarrow==10.0.1.*
-- librdkafka=1.7.0
+- libarrow==11.0.0.*
+- librdkafka>=1.9.0,<1.10.0a0
 - librmm==23.6.*
 - mimesis>=4.1.0
 - moto>=4.0.8
@@ -56,15 +56,15 @@ dependencies:
 - pre-commit
 - protobuf>=4.21.6,<4.22
 - ptxcompiler
-- pyarrow==10.0.1.*
+- pyarrow==11.0.0.*
 - pydata-sphinx-theme
 - pyorc
 - pytest
 - pytest-benchmark
 - pytest-cases
 - pytest-cov
 - pytest-xdist
-- python-confluent-kafka==1.7.0
+- python-confluent-kafka>=1.9.0,<1.10.0a0
 - python-snappy>=0.6.0
 - python>=3.8,<3.11
 - pytorch<1.12.0

conda/recipes/cudf/meta.yaml

@@ -54,7 +54,7 @@ requirements:
     - setuptools
     - numba >=0.56.4,<0.57
     - dlpack >=0.5,<0.6.0a0
-    - pyarrow =10
+    - pyarrow =11
     - libcudf ={{ version }}
     - rmm ={{ minor_version }}
     - cudatoolkit ={{ cuda_version }}

conda/recipes/cudf_kafka/meta.yaml

@@ -48,7 +48,7 @@ requirements:
   run:
     - python
     - libcudf_kafka ={{ version }}
-    - python-confluent-kafka >=1.7.0,<1.8.0a0
+    - python-confluent-kafka >=1.9.0,<1.10.0a0
     - cudf ={{ version }}
 
 test:

conda/recipes/libcudf/conda_build_config.yaml

@@ -17,13 +17,13 @@ gtest_version:
   - "=1.10.0"
 
 libarrow_version:
-  - "=10"
+  - "=11"
 
 dlpack_version:
   - ">=0.5,<0.6.0a0"
 
 librdkafka_version:
-  - ">=1.7.0,<1.8.0a0"
+  - ">=1.9.0,<1.10.0a0"
 
 fmt_version:
   - ">=9.1.0,<10"

cpp/cmake/thirdparty/get_arrow.cmake

@@ -381,7 +381,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow)
   set(CUDF_VERSION_Arrow
       # This version must be kept in sync with the libarrow version pinned for builds in
       # dependencies.yaml.
-      10.0.1
+      11.0.0
       CACHE STRING "The version of Arrow to find (or build)"
   )
 endif()

dependencies.yaml

@@ -216,8 +216,8 @@ dependencies:
           - &gmock gmock==1.10.0.*
           # Hard pin the patch version used during the build. This must be kept
           # in sync with the version pinned in get_arrow.cmake.
-          - libarrow==10.0.1.*
-          - librdkafka=1.7.0
+          - libarrow==11.0.0.*
+          - librdkafka>=1.9.0,<1.10.0a0
           - spdlog>=1.11.0,<1.12
   build_wheels:
     common:
@@ -232,7 +232,7 @@ dependencies:
           - cython>=0.29,<0.30
           # Hard pin the patch version used during the build. This must be kept
           # in sync with the version pinned in get_arrow.cmake.
-          - pyarrow==10.0.1.*
+          - pyarrow==11.0.0.*
           - numpy>=1.21
   build_python:
     common:
@@ -251,13 +251,13 @@ dependencies:
       - output_types: [conda, requirements]
         packages:
           # Allow runtime version to float up to minor version
-          - libarrow==10.*
+          - libarrow==11.*
   pyarrow_run:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           # Allow runtime version to float up to minor version
-          - pyarrow==10.*
+          - pyarrow==11.*
   cudatoolkit:
     specific:
       - output_types: conda
@@ -392,11 +392,11 @@ dependencies:
     common:
       - output_types: conda
         packages:
-          - python-confluent-kafka==1.7.0
+          - python-confluent-kafka>=1.9.0,<1.10.0a0
       - output_types: [requirements, pyproject]
         packages:
           - *cudf
-          - confluent-kafka==1.7.0
+          - confluent-kafka>=1.9.0,<1.10.0a0
   run_custreamz:
     common:
       - output_types: [conda, requirements, pyproject]

python/cudf/pyproject.toml

@@ -8,7 +8,7 @@ requires = [
     "ninja",
     "numpy>=1.21",
     "protoc-wheel",
-    "pyarrow==10.0.1.*",
+    "pyarrow==11.0.0.*",
     "rmm==23.6.*",
     "scikit-build>=0.13.1",
     "setuptools",
@@ -38,7 +38,7 @@ dependencies = [
     "pandas>=1.3,<1.6.0dev0",
     "protobuf>=4.21.6,<4.22",
     "ptxcompiler",
-    "pyarrow==10.*",
+    "pyarrow==11.*",
     "rmm==23.6.*",
     "typing_extensions",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.

python/cudf_kafka/pyproject.toml

@@ -5,7 +5,7 @@
 requires = [
     "cython>=0.29,<0.30",
     "numpy>=1.21",
-    "pyarrow==10.0.1.*",
+    "pyarrow==11.0.0.*",
     "setuptools",
     "wheel",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
@@ -21,7 +21,7 @@ authors = [
 license = { text = "Apache 2.0" }
 requires-python = ">=3.8"
 dependencies = [
-    "confluent-kafka==1.7.0",
+    "confluent-kafka>=1.9.0,<1.10.0a0",
     "cudf==23.6.*",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.