From 77f3d576f9b70a789ceddaae81ef8a5f97664f35 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 21 Jul 2023 08:18:36 -0700 Subject: [PATCH 1/7] Upgrade to arrow 12 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 ++-- conda/environments/all_cuda-120_arch-x86_64.yaml | 4 ++-- conda/recipes/libcudf/conda_build_config.yaml | 2 +- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- dependencies.yaml | 8 ++++---- python/cudf/pyproject.toml | 2 +- python/cudf_kafka/pyproject.toml | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 71e5d129d80..d284cfe33db 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -38,7 +38,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==11.0.0.* +- libarrow==12.0.1.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 @@ -66,7 +66,7 @@ dependencies: - pre-commit - protobuf>=4.21.6,<4.22 - ptxcompiler -- pyarrow==11.0.0.* +- pyarrow==12.0.1.* - pydata-sphinx-theme - pyorc - pytest diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index ba42f9a8165..603f6f1dbfb 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -39,7 +39,7 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow==11.0.0.* +- libarrow==12.0.1.* - libcufile-dev - libcurand-dev - libkvikio==23.8.* @@ -63,7 +63,7 @@ dependencies: - pip - pre-commit - protobuf>=4.21.6,<4.22 -- pyarrow==11.0.0.* +- pyarrow==12.0.1.* - pydata-sphinx-theme - pyorc - pytest diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 4d03dab2f35..0397045786b 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -23,7 +23,7 @@ gtest_version: - ">=1.13.0" libarrow_version: - - "=11" + - "=12" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index c877c9c6466..036ef880f99 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -381,7 +381,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 11.0.0 + 12.0.1 CACHE STRING "The version of Arrow to find (or build)" ) endif() diff --git a/dependencies.yaml b/dependencies.yaml index 183451670bc..5b77d7c1b71 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -223,7 +223,7 @@ dependencies: - &gmock gmock>=1.13.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - libarrow==11.0.0.* + - libarrow==12.0.1.* - librdkafka>=1.9.0,<1.10.0a0 - spdlog>=1.11.0,<1.12 specific: @@ -253,7 +253,7 @@ dependencies: - cython>=0.29,<0.30 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==11.0.0.* + - pyarrow==12.0.1.* - numpy>=1.21 build_python: common: @@ -272,13 +272,13 @@ dependencies: - output_types: [conda, requirements] packages: # Allow runtime version to float up to minor version - - libarrow==11.* + - libarrow==12.* pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to minor version - - pyarrow==11.* + - pyarrow==12.* cudatoolkit: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 109a2c83b46..33c1bb83075 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21", "protoc-wheel", - "pyarrow==11.0.0.*", + "pyarrow==12.0.1.*", "rmm==23.8.*", "scikit-build>=0.13.1", "setuptools", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 3fb04402ccb..057a5f3aa21 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "cython>=0.29,<0.30", "numpy>=1.21", - "pyarrow==11.0.0.*", + "pyarrow==12.0.1.*", "setuptools", "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 4057f12f6c6321d43ba799abfda842ead0c3b436 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 27 Jul 2023 10:01:28 -0500 Subject: [PATCH 2/7] Update pyproject.toml --- python/cudf/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9ca922c83f7..9ffefad32d0 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "pandas>=1.3,<1.6.0dev0", "protobuf>=4.21.6,<4.22", "ptxcompiler", - "pyarrow==11.*", + "pyarrow==12.*", "rmm==23.10.*", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From cca834a3b34d405adc8b20017a43e89ce733b333 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 27 Jul 2023 15:12:24 -0700 Subject: [PATCH 3/7] fix --- conda/recipes/cudf/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 8f1f55b1df8..0d302f711ea 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -61,7 +61,7 @@ requirements: - scikit-build >=0.13.1 - setuptools - dlpack >=0.5,<0.6.0a0 - - pyarrow =11 + - pyarrow =12 - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} From bc03aed55ceab2978d312c9dc460e3332c46b076 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 28 Jul 2023 11:54:28 -0700 Subject: [PATCH 4/7] test --- ci/test_python_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 0e922c105dd..ce3c4aedc30 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -29,7 +29,7 @@ RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env - +conda config --system --remove channels rapidsai rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ From e1ff9b7d0deace9a830e4991b939784601da618f Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Fri, 28 Jul 2023 12:06:32 -0700 Subject: [PATCH 5/7] protobuf --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 4 ++-- dependencies.yaml | 2 +- python/cudf/pyproject.toml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 96d21c3491b..0e0554c1080 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -64,7 +64,7 @@ dependencies: - pandoc - pip - pre-commit -- protobuf>=4.21.6,<4.22 +- protobuf>=4.21.6,<4.23.4.0a0 - ptxcompiler - pyarrow==12.0.1.* - pydata-sphinx-theme diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 565cb3f8fee..ad3231a6c70 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -62,7 +62,7 @@ dependencies: - pandoc - pip - pre-commit -- protobuf>=4.21.6,<4.22 +- protobuf>=4.21.6,<4.23.4.0a0 - pyarrow==12.0.1.* - pydata-sphinx-theme - pyorc diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 0d302f711ea..04e71d5186a 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -55,7 +55,7 @@ requirements: - cuda-version ={{ cuda_version }} - sysroot_{{ target_platform }} {{ sysroot_version }} host: - - protobuf >=4.21.6,<4.22 + - protobuf >=4.21.6,<4.23.4.0a0 - python - cython >=0.29,<0.30 - scikit-build >=0.13.1 @@ -73,7 +73,7 @@ requirements: {% endif %} - cuda-version ={{ cuda_version }} run: - - protobuf >=4.21.6,<4.22 + - protobuf >=4.21.6,<4.23.4.0a0 - python - typing_extensions >=4.0.0 - pandas >=1.3,<1.6.0dev0 diff --git a/dependencies.yaml b/dependencies.yaml index 37797a454a9..846a8f52fba 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -263,7 +263,7 @@ dependencies: - rmm==23.10.* - output_types: conda packages: - - &protobuf protobuf>=4.21.6,<4.22 + - &protobuf protobuf>=4.21.6,<4.23.4.0a0 - output_types: pyproject packages: - protoc-wheel diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9ffefad32d0..e6c5f949b2b 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "nvtx>=0.2.1", "packaging", "pandas>=1.3,<1.6.0dev0", - "protobuf>=4.21.6,<4.22", + "protobuf>=4.21.6,<4.23.4.0a0", "ptxcompiler", "pyarrow==12.*", "rmm==23.10.*", From 3d9d49149aeadbe9a4b3a74ad3199ce748dfba47 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 28 Jul 2023 15:26:24 -0500 Subject: [PATCH 6/7] Update test_python_common.sh --- ci/test_python_common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index ce3c4aedc30..0e922c105dd 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -29,7 +29,7 @@ RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env -conda config --system --remove channels rapidsai + rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ From caa624f6a84095397e1b362f00bf6b63eaba070f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 28 Jul 2023 15:21:54 -0700 Subject: [PATCH 7/7] Finalize the S3 fs after using --- cpp/tests/io/arrow_io_source_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/tests/io/arrow_io_source_test.cpp b/cpp/tests/io/arrow_io_source_test.cpp index 2961deec384..d7f1879040b 100644 --- a/cpp/tests/io/arrow_io_source_test.cpp +++ b/cpp/tests/io/arrow_io_source_test.cpp @@ -87,6 +87,7 @@ TEST_F(ArrowIOTest, S3FileSystem) ASSERT_EQ(1, tbl.tbl->num_columns()); // Only single column specified in reader_options ASSERT_EQ(244, tbl.tbl->num_rows()); // known number of rows from the S3 file } + CUDF_EXPECTS(arrow::fs::EnsureS3Finalized().ok(), "Failed to finalize s3 filesystem"); } CUDF_TEST_PROGRAM_MAIN()