From ec1b08192091b6b3a83fa4c1ef62020c14798f81 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 21 Feb 2024 18:18:20 +0000 Subject: [PATCH 1/4] Upgrade to arrow 15 --- .../all_cuda-118_arch-x86_64.yaml | 10 ++++----- .../all_cuda-122_arch-x86_64.yaml | 10 ++++----- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- cpp/cmake/thirdparty/get_arrow.cmake | 4 ++-- dependencies.yaml | 22 +++++++++---------- python/cudf/pyproject.toml | 4 ++-- python/cudf_kafka/pyproject.toml | 2 +- 8 files changed, 27 insertions(+), 29 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index fa4ef8ddf68..765542a853e 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,15 +40,15 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==14.0.1.* -- libarrow-dataset==14.0.1.* -- libarrow==14.0.1.* +- libarrow-acero==15.0.0.* +- libarrow-dataset==15.0.0.* +- libarrow==15.0.0.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 - libcurand=10.3.0.86 - libkvikio==24.4.* -- libparquet==14.0.1.* +- libparquet==15.0.0.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.4.* - make @@ -71,7 +71,7 @@ dependencies: - pre-commit - protobuf>=4.21,<5 - ptxcompiler -- pyarrow==14.0.1.* +- pyarrow==15.0.0.* - pydata-sphinx-theme!=0.14.2 - pytest-benchmark - pytest-cases>=3.8.2 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index c0950c7da98..86f9c5199d3 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -41,13 +41,13 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==14.0.1.* -- libarrow-dataset==14.0.1.* -- libarrow==14.0.1.* +- libarrow-acero==15.0.0.* +- libarrow-dataset==15.0.0.* +- libarrow==15.0.0.* - libcufile-dev - libcurand-dev - libkvikio==24.4.* -- libparquet==14.0.1.* +- libparquet==15.0.0.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.4.* - make @@ -68,7 +68,7 @@ dependencies: - pip - pre-commit - protobuf>=4.21,<5 -- pyarrow==14.0.1.* +- pyarrow==15.0.0.* - pydata-sphinx-theme!=0.14.2 - pynvjitlink - pytest-benchmark diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 018380bbbd2..176fb8b005f 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -65,7 +65,7 @@ requirements: - scikit-build-core >=0.7.0 - setuptools - dlpack >=0.5,<0.6.0a0 - - pyarrow ==14.0.1.* + - pyarrow ==15.0.0.* - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 9ed8c94f2bb..f2b1264b668 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -23,7 +23,7 @@ gtest_version: - ">=1.13.0" libarrow_version: - - "==14.0.1" + - "==15.0.0" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 1bead93c9cc..e62cc7f5776 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2020-2023, NVIDIA CORPORATION. +# Copyright (c) 2020-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -441,7 +441,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 14.0.1 + 15.0.0 CACHE STRING "The version of Arrow to find (or build)" ) endif() diff --git a/dependencies.yaml b/dependencies.yaml index c4c2cd3c764..1434f8c87ea 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -261,7 +261,7 @@ dependencies: - &numpy numpy>=1.21 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==14.0.1.* + - pyarrow==15.0.0.* - output_types: conda packages: - scikit-build-core>=0.7.0 @@ -306,27 +306,25 @@ dependencies: packages: # Hard pin the Arrow patch version used during the build. This must # be kept in sync with the version pinned in get_arrow.cmake. - - libarrow-acero==14.0.1.* - - libarrow-dataset==14.0.1.* - - libarrow==14.0.1.* - - libparquet==14.0.1.* + - libarrow-acero==15.0.0.* + - libarrow-dataset==15.0.0.* + - libarrow==15.0.0.* + - libparquet==15.0.0.* libarrow_run: common: - output_types: conda packages: # Allow runtime version to float up to minor version - # Disallow libarrow 14.0.0 due to a CVE - - libarrow-acero>=14.0.1,<15.0.0a0 - - libarrow-dataset>=14.0.1,<15.0.0a0 - - libarrow>=14.0.1,<15.0.0a0 - - libparquet>=14.0.1,<15.0.0a0 + - libarrow-acero>=15.0.0,<16.0.0a0 + - libarrow-dataset>=15.0.0,<16.0.0a0 + - libarrow>=15.0.0,<16.0.0a0 + - libparquet>=15.0.0,<16.0.0a0 pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to minor version - # Disallow pyarrow 14.0.0 due to a CVE - - pyarrow>=14.0.1,<15.0.0a0 + - pyarrow>=15.0.0,<16.0.0a0 cuda_version: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 7f2d8e438d2..152f0b26827 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21", "protoc-wheel", - "pyarrow==14.0.1.*", + "pyarrow==15.0.0.*", "rmm==24.4.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -36,7 +36,7 @@ dependencies = [ "pandas>=2.0,<2.1.5dev0", "protobuf>=4.21,<5", "ptxcompiler", - "pyarrow>=14.0.1,<15.0.0a0", + "pyarrow>=15.0.0,<16.0.0a0", "rich", "rmm==24.4.*", "typing_extensions>=4.0.0", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index af59efa9777..ceff272b83e 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "cython>=3.0.3", "ninja", "numpy>=1.21", - "pyarrow==14.0.1.*", + "pyarrow==15.0.0.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From defb7aab196e5738f19220f5cd80c502c395f6da Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 21 Feb 2024 19:06:38 +0000 Subject: [PATCH 2/4] upgrade to 14.0.2 --- .../all_cuda-118_arch-x86_64.yaml | 10 +++++----- .../all_cuda-122_arch-x86_64.yaml | 10 +++++----- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- dependencies.yaml | 20 +++++++++---------- python/cudf/pyproject.toml | 4 ++-- python/cudf_kafka/pyproject.toml | 2 +- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 765542a853e..625e6c6e9db 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -40,15 +40,15 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==15.0.0.* -- libarrow-dataset==15.0.0.* -- libarrow==15.0.0.* +- libarrow-acero==14.0.2.* +- libarrow-dataset==14.0.2.* +- libarrow==14.0.2.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 - libcurand=10.3.0.86 - libkvikio==24.4.* -- libparquet==15.0.0.* +- libparquet==14.0.2.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.4.* - make @@ -71,7 +71,7 @@ dependencies: - pre-commit - protobuf>=4.21,<5 - ptxcompiler -- pyarrow==15.0.0.* +- pyarrow==14.0.2.* - pydata-sphinx-theme!=0.14.2 - pytest-benchmark - pytest-cases>=3.8.2 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 86f9c5199d3..871f00a0e8e 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -41,13 +41,13 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==15.0.0.* -- libarrow-dataset==15.0.0.* -- libarrow==15.0.0.* +- libarrow-acero==14.0.2.* +- libarrow-dataset==14.0.2.* +- libarrow==14.0.2.* - libcufile-dev - libcurand-dev - libkvikio==24.4.* -- libparquet==15.0.0.* +- libparquet==14.0.2.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.4.* - make @@ -68,7 +68,7 @@ dependencies: - pip - pre-commit - protobuf>=4.21,<5 -- pyarrow==15.0.0.* +- pyarrow==14.0.2.* - pydata-sphinx-theme!=0.14.2 - pynvjitlink - pytest-benchmark diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 176fb8b005f..d32e6932598 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -65,7 +65,7 @@ requirements: - scikit-build-core >=0.7.0 - setuptools - dlpack >=0.5,<0.6.0a0 - - pyarrow ==15.0.0.* + - pyarrow ==14.0.2.* - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index f2b1264b668..603cbd8fc2a 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -23,7 +23,7 @@ gtest_version: - ">=1.13.0" libarrow_version: - - "==15.0.0" + - "==14.0.2" dlpack_version: - ">=0.5,<0.6.0a0" diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index e62cc7f5776..114a1f98a68 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -441,7 +441,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 15.0.0 + 14.0.2 CACHE STRING "The version of Arrow to find (or build)" ) endif() diff --git a/dependencies.yaml b/dependencies.yaml index 1434f8c87ea..9c143ca8bea 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -261,7 +261,7 @@ dependencies: - &numpy numpy>=1.21 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==15.0.0.* + - pyarrow==14.0.2.* - output_types: conda packages: - scikit-build-core>=0.7.0 @@ -306,25 +306,25 @@ dependencies: packages: # Hard pin the Arrow patch version used during the build. This must # be kept in sync with the version pinned in get_arrow.cmake. - - libarrow-acero==15.0.0.* - - libarrow-dataset==15.0.0.* - - libarrow==15.0.0.* - - libparquet==15.0.0.* + - libarrow-acero==14.0.2.* + - libarrow-dataset==14.0.2.* + - libarrow==14.0.2.* + - libparquet==14.0.2.* libarrow_run: common: - output_types: conda packages: # Allow runtime version to float up to minor version - - libarrow-acero>=15.0.0,<16.0.0a0 - - libarrow-dataset>=15.0.0,<16.0.0a0 - - libarrow>=15.0.0,<16.0.0a0 - - libparquet>=15.0.0,<16.0.0a0 + - libarrow-acero>=14.0.2,<15.0.0a0 + - libarrow-dataset>=14.0.2,<15.0.0a0 + - libarrow>=14.0.2,<15.0.0a0 + - libparquet>=14.0.2,<15.0.0a0 pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to minor version - - pyarrow>=15.0.0,<16.0.0a0 + - pyarrow>=14.0.2,<15.0.0a0 cuda_version: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 152f0b26827..29ee1037d81 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "ninja", "numpy>=1.21", "protoc-wheel", - "pyarrow==15.0.0.*", + "pyarrow==14.0.2.*", "rmm==24.4.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -36,7 +36,7 @@ dependencies = [ "pandas>=2.0,<2.1.5dev0", "protobuf>=4.21,<5", "ptxcompiler", - "pyarrow>=15.0.0,<16.0.0a0", + "pyarrow>=14.0.2,<15.0.0a0", "rich", "rmm==24.4.*", "typing_extensions>=4.0.0", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index ceff272b83e..216d83940ce 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "cython>=3.0.3", "ninja", "numpy>=1.21", - "pyarrow==15.0.0.*", + "pyarrow==14.0.2.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From ed05ac771a993270af664c39715f94aca7a0a0bd Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 21 Feb 2024 13:37:44 -0600 Subject: [PATCH 3/4] Apply suggestions from code review --- dependencies.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 9c143ca8bea..c5797fbe40a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -315,16 +315,18 @@ dependencies: - output_types: conda packages: # Allow runtime version to float up to minor version - - libarrow-acero>=14.0.2,<15.0.0a0 - - libarrow-dataset>=14.0.2,<15.0.0a0 - - libarrow>=14.0.2,<15.0.0a0 - - libparquet>=14.0.2,<15.0.0a0 + # Disallow libarrow 14.0.0 due to a CVE + - libarrow-acero>=14.0.1,<15.0.0a0 + - libarrow-dataset>=14.0.1,<15.0.0a0 + - libarrow>=14.0.1,<15.0.0a0 + - libparquet>=14.0.1,<15.0.0a0 pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to minor version - - pyarrow>=14.0.2,<15.0.0a0 + # Disallow pyarrow 14.0.0 due to a CVE + - pyarrow>=14.0.1,<15.0.0a0 cuda_version: specific: - output_types: conda From c395fe4889a07d5d5fa004a9bfec7da932618aac Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 21 Feb 2024 13:39:55 -0600 Subject: [PATCH 4/4] Apply suggestions from code review --- python/cudf/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 29ee1037d81..82ac84a4022 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "pandas>=2.0,<2.1.5dev0", "protobuf>=4.21,<5", "ptxcompiler", - "pyarrow>=14.0.2,<15.0.0a0", + "pyarrow>=14.0.1,<15.0.0a0", "rich", "rmm==24.4.*", "typing_extensions>=4.0.0",