From 2eb13701f5c983529ea49443d750e2bfafb80296 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 20 May 2024 16:35:59 +0000 Subject: [PATCH 1/8] Upgrade to 16.1 --- .../all_cuda-118_arch-x86_64.yaml | 10 +++++----- .../all_cuda-122_arch-x86_64.yaml | 10 +++++----- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- dependencies.yaml | 20 +++++++++---------- python/cudf/pyproject.toml | 4 ++-- python/cudf_kafka/pyproject.toml | 2 +- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 48699b81eed..804b09bab59 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -36,15 +36,15 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==16.0.0.* -- libarrow-dataset==16.0.0.* -- libarrow==16.0.0.* +- libarrow-acero==16.1.0.* +- libarrow-dataset==16.1.0.* +- libarrow==16.1.0.* - libcufile-dev=1.4.0.31 - libcufile=1.4.0.31 - libcurand-dev=10.3.0.86 - libcurand=10.3.0.86 - libkvikio==24.6.* -- libparquet==16.0.0.* +- libparquet==16.1.0.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.6.* - make @@ -66,7 +66,7 @@ dependencies: - pip - pre-commit - ptxcompiler -- pyarrow==16.0.0.* +- pyarrow==16.1.0.* - pydata-sphinx-theme!=0.14.2 - pytest-benchmark - pytest-cases>=3.8.2 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index d06a727f331..89eac98f652 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -37,13 +37,13 @@ dependencies: - hypothesis - identify>=2.5.20 - ipython -- libarrow-acero==16.0.0.* -- libarrow-dataset==16.0.0.* -- libarrow==16.0.0.* +- libarrow-acero==16.1.0.* +- libarrow-dataset==16.1.0.* +- libarrow==16.1.0.* - libcufile-dev - libcurand-dev - libkvikio==24.6.* -- libparquet==16.0.0.* +- libparquet==16.1.0.* - librdkafka>=1.9.0,<1.10.0a0 - librmm==24.6.* - make @@ -63,7 +63,7 @@ dependencies: - pandoc - pip - pre-commit -- pyarrow==16.0.0.* +- pyarrow==16.1.0.* - pydata-sphinx-theme!=0.14.2 - pynvjitlink - pytest-benchmark diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 12e29c77a98..8b74f90271d 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -64,7 +64,7 @@ requirements: - scikit-build-core >=0.7.0 - dlpack >=0.8,<1.0 - numpy 1.23 - - pyarrow ==16.0.0.* + - pyarrow ==16.1.0.* - libcudf ={{ version }} - rmm ={{ minor_version }} {% if cuda_major == "11" %} diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 61ffcf3c3de..c01178bf732 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -20,7 +20,7 @@ cmake_version: - ">=3.26.4" libarrow_version: - - "==16.0.0" + - "==16.1.0" dlpack_version: - ">=0.8,<1.0" diff --git a/dependencies.yaml b/dependencies.yaml index 4f8f3c16ea1..1796aeea59b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -289,7 +289,7 @@ dependencies: - cython>=3.0.3 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - - pyarrow==16.0.0.* + - pyarrow==16.1.0.* - output_types: conda packages: - scikit-build-core>=0.7.0 @@ -332,25 +332,25 @@ dependencies: packages: # Hard pin the Arrow patch version used during the build. This must # be kept in sync with the version pinned in get_arrow.cmake. - - libarrow-acero==16.0.0.* - - libarrow-dataset==16.0.0.* - - libarrow==16.0.0.* - - libparquet==16.0.0.* + - libarrow-acero==16.1.0.* + - libarrow-dataset==16.1.0.* + - libarrow==16.1.0.* + - libparquet==16.1.0.* libarrow_run: common: - output_types: conda packages: # Allow runtime version to float up to minor version - - libarrow-acero>=16.0.0,<17.0.0a0 - - libarrow-dataset>=16.0.0,<17.0.0a0 - - libarrow>=16.0.0,<17.0.0a0 - - libparquet>=16.0.0,<17.0.0a0 + - libarrow-acero>=16.1.0,<17.0.0a0 + - libarrow-dataset>=16.1.0,<17.0.0a0 + - libarrow>=16.1.0,<17.0.0a0 + - libparquet>=16.1.0,<17.0.0a0 pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to patch version - - pyarrow>=16.0.0,<16.1.0a0 + - pyarrow>=16.1.0,<16.1.0a0 cuda_version: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 826362f0632..dda497f8aff 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "cython>=3.0.3", "ninja", "numpy==1.23.*", - "pyarrow==16.0.0.*", + "pyarrow==16.1.0.*", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -34,7 +34,7 @@ dependencies = [ "packaging", "pandas>=2.0,<2.2.3dev0", "ptxcompiler", - "pyarrow>=16.0.0,<16.1.0a0", + "pyarrow>=16.1.0,<16.1.0a0", "rich", "rmm==24.6.*", "typing_extensions>=4.0.0", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 787dd8a97d7..80e30e000c0 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -7,7 +7,7 @@ requires = [ "cython>=3.0.3", "ninja", "numpy==1.23.*", - "pyarrow==16.0.0.*", + "pyarrow==16.1.0.*", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 1a4d05a01a2f87053e5c851bee7ef2ed0f788000 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 21 May 2024 00:06:37 +0000 Subject: [PATCH 2/8] Drop workarounds for 16.0 --- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/libcudf/meta.yaml | 9 --------- cpp/cmake/thirdparty/get_arrow.cmake | 2 +- 3 files changed, 2 insertions(+), 11 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 8b74f90271d..e7245e67659 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -82,7 +82,7 @@ requirements: - cupy >=12.0.0 - numba >=0.57 - {{ pin_compatible('numpy', max_pin='x') }} - - {{ pin_compatible('pyarrow', max_pin='x') }} + - {{ pin_compatible('pyarrow', max_pin='x.x') }} - libcudf ={{ version }} - {{ pin_compatible('rmm', max_pin='x.x') }} - fsspec >=0.6.0 diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index ad2e840c71d..76115362b6c 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -86,9 +86,6 @@ outputs: {% else %} - {{ compiler('cuda') }} {% endif %} - # TODO: start taking libarrow's run exports again wwhen they're correct for 16.0 - # ref: https://github.com/conda-forge/arrow-cpp-feedstock/issues/1418 - - libarrow requirements: build: - cmake {{ cmake_version }} @@ -108,12 +105,6 @@ outputs: - librmm ={{ minor_version }} - libkvikio ={{ minor_version }} - dlpack {{ dlpack_version }} - # TODO: start taking libarrow's run exports again wwhen they're correct for 16.0 - # ref: https://github.com/conda-forge/arrow-cpp-feedstock/issues/1418 - - libarrow>=16.0.0,<16.1.0a0 - - libarrow-acero>=16.0.0,<16.1.0a0 - - libarrow-dataset>=16.0.0,<16.1.0a0 - - libparquet>=16.0.0,<16.1.0a0 test: commands: - test -f $PREFIX/lib/libcudf.so diff --git a/cpp/cmake/thirdparty/get_arrow.cmake b/cpp/cmake/thirdparty/get_arrow.cmake index 73e66cce608..0afdc526981 100644 --- a/cpp/cmake/thirdparty/get_arrow.cmake +++ b/cpp/cmake/thirdparty/get_arrow.cmake @@ -430,7 +430,7 @@ if(NOT DEFINED CUDF_VERSION_Arrow) set(CUDF_VERSION_Arrow # This version must be kept in sync with the libarrow version pinned for builds in # dependencies.yaml. - 16.0.0 + 16.1.0 CACHE STRING "The version of Arrow to find (or build)" ) endif() From ddfac77f8608d6ece81fd67422a22d7d98279bc5 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 May 2024 19:19:26 -0500 Subject: [PATCH 3/8] Apply suggestions from code review Co-authored-by: Bradley Dice --- dependencies.yaml | 10 +++++----- python/cudf/pyproject.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 1796aeea59b..fb688e1d36e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -341,16 +341,16 @@ dependencies: - output_types: conda packages: # Allow runtime version to float up to minor version - - libarrow-acero>=16.1.0,<17.0.0a0 - - libarrow-dataset>=16.1.0,<17.0.0a0 - - libarrow>=16.1.0,<17.0.0a0 - - libparquet>=16.1.0,<17.0.0a0 + - libarrow-acero>=16.1.0,<16.2.0a0 + - libarrow-dataset>=16.1.0,<16.2.0a0 + - libarrow>=16.1.0,<16.2.0a0 + - libparquet>=16.1.0,<16.2.0a0 pyarrow_run: common: - output_types: [conda, requirements, pyproject] packages: # Allow runtime version to float up to patch version - - pyarrow>=16.1.0,<16.1.0a0 + - pyarrow>=16.1.0,<16.2.0a0 cuda_version: specific: - output_types: conda diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index dda497f8aff..38aa6eeb24e 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "packaging", "pandas>=2.0,<2.2.3dev0", "ptxcompiler", - "pyarrow>=16.1.0,<16.1.0a0", + "pyarrow>=16.1.0,<16.2.0a0", "rich", "rmm==24.6.*", "typing_extensions>=4.0.0", From 231c076f5a8253d449adff771c068aea66bdb3c8 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 May 2024 19:19:56 -0500 Subject: [PATCH 4/8] Update dependencies.yaml --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index fb688e1d36e..0844d86fb66 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -340,7 +340,7 @@ dependencies: common: - output_types: conda packages: - # Allow runtime version to float up to minor version + # Allow runtime version to float up to patch version - libarrow-acero>=16.1.0,<16.2.0a0 - libarrow-dataset>=16.1.0,<16.2.0a0 - libarrow>=16.1.0,<16.2.0a0 From 7dcc4e9454e15ffe159dc4ca06090909bd64f1dc Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 21 May 2024 16:43:02 +0000 Subject: [PATCH 5/8] xfail a pytest specific to rockylinux8 --- python/cudf/cudf/tests/test_orc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index a9bca7d8b98..2d179a9107a 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -3,6 +3,7 @@ import datetime import decimal import os +import platform import random from io import BytesIO from string import ascii_lowercase @@ -1833,6 +1834,10 @@ def test_orc_writer_negative_timestamp(negative_timestamp_df): ) +@pytest.mark.xfail( + condition=platform.system() == "Linux" and "rocky" in platform.release(), + reason="Bug specific to rockylinux8: https://github.com/rapidsai/cudf/issues/15802", +) def test_orc_reader_apache_negative_timestamp(datadir): path = datadir / "TestOrcFile.apache_timestamp.orc" From 9fe6dd547bc822afa23e045974f65e8cba05001c Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 21 May 2024 17:52:54 +0000 Subject: [PATCH 6/8] test --- python/cudf/cudf/tests/test_orc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 2d179a9107a..676e3de3b6c 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -1840,7 +1840,7 @@ def test_orc_writer_negative_timestamp(negative_timestamp_df): ) def test_orc_reader_apache_negative_timestamp(datadir): path = datadir / "TestOrcFile.apache_timestamp.orc" - + print(platform.system(), platform.release()) pdf = pd.read_orc(path) gdf = cudf.read_orc(path) From e45ce4e1d55d20f888195a4a6a335bb755963f1a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 21 May 2024 19:03:31 +0000 Subject: [PATCH 7/8] test --- python/cudf/cudf/tests/test_orc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 676e3de3b6c..cf9f5454459 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -1840,7 +1840,9 @@ def test_orc_writer_negative_timestamp(negative_timestamp_df): ) def test_orc_reader_apache_negative_timestamp(datadir): path = datadir / "TestOrcFile.apache_timestamp.orc" - print(platform.system(), platform.release()) + print(platform.system()) + print(platform.release()) + print(platform.uname()) pdf = pd.read_orc(path) gdf = cudf.read_orc(path) From 3e85eaf413d9c74ef025e61e142ed5a46c10f265 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 21 May 2024 20:07:56 +0000 Subject: [PATCH 8/8] skip pytest --- python/cudf/cudf/tests/test_orc.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index cf9f5454459..83b7353ad89 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -3,7 +3,6 @@ import datetime import decimal import os -import platform import random from io import BytesIO from string import ascii_lowercase @@ -1834,15 +1833,12 @@ def test_orc_writer_negative_timestamp(negative_timestamp_df): ) -@pytest.mark.xfail( - condition=platform.system() == "Linux" and "rocky" in platform.release(), +@pytest.mark.skip( reason="Bug specific to rockylinux8: https://github.com/rapidsai/cudf/issues/15802", ) def test_orc_reader_apache_negative_timestamp(datadir): path = datadir / "TestOrcFile.apache_timestamp.orc" - print(platform.system()) - print(platform.release()) - print(platform.uname()) + pdf = pd.read_orc(path) gdf = cudf.read_orc(path)