From a8ef4d4e49d584d1553283f6cf9aef17baaca62f Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 4 Aug 2023 16:19:33 -0700 Subject: [PATCH] Update to Cython 3.0.0 (#13777) This PR contains the minimal set of changes to compile using Cython 3 without warnings. Future PRs can be made to take advantage of new or improved features. The specific changes are: - Ensuring `nogil` always comes after `except`. `except * nogil` is a compile-time error in Cython 3 - Removing any extern cdef functions that uses C++ rvalues. These were never supported by Cython, but prior to 3.0 they were silently ignored whereas now Cython throws warnings during compilation - Relative imports are no longer off by one level in pxd files and must be adjusted accordingly (see https://github.com/cython/cython/issues/3442) There are a large number of outstanding warnings due to https://github.com/NVIDIA/cuda-python/issues/44. cuda-python for CUDA 12 has the necessary fix, but we will need a cuda-python 11.8.* bugfix with a backport to make those warnings go away. There are also warnings coming from pyarrow due to https://github.com/apache/arrow/issues/34564. pyarrow 12 contains the necessary fixes, so this issues should be resolved once #13728 is merged. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Ray Douglass (https://github.com/raydouglass) - Bradley Dice (https://github.com/bdice) - Benjamin Zaitlen (https://github.com/quasiben) - GALI PREM SAGAR (https://github.com/galipremsagar) - https://github.com/jakirkham URL: https://github.com/rapidsai/cudf/pull/13777 --- ci/test_wheel_dask_cudf.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-120_arch-x86_64.yaml | 2 +- conda/recipes/cudf/meta.yaml | 2 +- conda/recipes/cudf_kafka/meta.yaml | 2 +- dependencies.yaml | 2 +- python/cudf/cudf/_lib/cpp/column/column.pxd | 8 +------- python/cudf/cudf/_lib/cpp/column/column_view.pxd | 3 +-- python/cudf/cudf/_lib/cpp/table/table.pxd | 3 +-- python/cudf/cudf/_lib/csv.pyx | 2 +- python/cudf/cudf/_lib/io/datasource.pxd | 4 ++-- python/cudf/cudf/_lib/io/datasource.pyx | 4 ++-- python/cudf/cudf/_lib/json.pyx | 4 ++-- python/cudf/cudf/_lib/pylibcudf/__init__.pxd | 14 +++++--------- python/cudf/pyproject.toml | 2 +- python/cudf_kafka/pyproject.toml | 2 +- 16 files changed, 23 insertions(+), 35 deletions(-) diff --git a/ci/test_wheel_dask_cudf.sh b/ci/test_wheel_dask_cudf.sh index 9e190ea9321..38fce04459e 100755 --- a/ci/test_wheel_dask_cudf.sh +++ b/ci/test_wheel_dask_cudf.sh @@ -11,7 +11,7 @@ RAPIDS_PY_WHEEL_NAME="cudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from python -m pip install --no-deps ./local-cudf-dep/cudf*.whl # Always install latest dask for testing -python -m pip install git+https://github.com/dask/dask.git@2023.7.1 git+https://github.com/dask/distributed.git@2023.7.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.08 +python -m pip install git+https://github.com/dask/dask.git@2023.7.1 git+https://github.com/dask/distributed.git@2023.7.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.10 # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cudf*.whl)[test] diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8f04f7c06b8..06a3635bb05 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -22,7 +22,7 @@ dependencies: - cudatoolkit - cupy>=12.0.0 - cxx-compiler -- cython>=0.29,<0.30 +- cython>=3.0.0 - dask-core==2023.7.1 - dask-cuda==23.10.* - dask==2023.7.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 19ca856df20..9273dd14867 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -23,7 +23,7 @@ dependencies: - cuda-version=12.0 - cupy>=12.0.0 - cxx-compiler -- cython>=0.29,<0.30 +- cython>=3.0.0 - dask-core==2023.7.1 - dask-cuda==23.10.* - dask==2023.7.1 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 612d2b177f6..a909b72c878 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -57,7 +57,7 @@ requirements: host: - protobuf ==4.21.* - python - - cython >=0.29,<0.30 + - cython >=3.0.0 - scikit-build >=0.13.1 - setuptools - dlpack >=0.5,<0.6.0a0 diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml index fff5bf1e840..ec0cc402511 100644 --- a/conda/recipes/cudf_kafka/meta.yaml +++ b/conda/recipes/cudf_kafka/meta.yaml @@ -43,7 +43,7 @@ requirements: - sysroot_{{ target_platform }} {{ sysroot_version }} host: - python - - cython >=0.29,<0.30 + - cython >=3.0.0 - cuda-version ={{ cuda_version }} - cudf ={{ version }} - libcudf_kafka ={{ version }} diff --git a/dependencies.yaml b/dependencies.yaml index b0022e8564c..3e5e703c239 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -250,7 +250,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - cython>=0.29,<0.30 + - cython>=3.0.0 # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - pyarrow==12.0.1.* diff --git a/python/cudf/cudf/_lib/cpp/column/column.pxd b/python/cudf/cudf/_lib/cpp/column/column.pxd index 205a1548c54..136f1d795a9 100644 --- a/python/cudf/cudf/_lib/cpp/column/column.pxd +++ b/python/cudf/cudf/_lib/cpp/column/column.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.memory cimport unique_ptr @@ -20,12 +20,6 @@ cdef extern from "cudf/column/column.hpp" namespace "cudf" nogil: column() except + column(const column& other) except + - column( - data_type dtype, - size_type size, - device_buffer&& data - ) except + - column(column_view view) except + size_type size() except + diff --git a/python/cudf/cudf/_lib/cpp/column/column_view.pxd b/python/cudf/cudf/_lib/cpp/column/column_view.pxd index 39c1c958531..edd013d9340 100644 --- a/python/cudf/cudf/_lib/cpp/column/column_view.pxd +++ b/python/cudf/cudf/_lib/cpp/column/column_view.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp cimport bool from libcpp.vector cimport vector @@ -12,7 +12,6 @@ cdef extern from "cudf/column/column_view.hpp" namespace "cudf" nogil: column_view(const column_view& other) except + column_view& operator=(const column_view&) except + - column_view& operator=(column_view&&) except + column_view( data_type type, diff --git a/python/cudf/cudf/_lib/cpp/table/table.pxd b/python/cudf/cudf/_lib/cpp/table/table.pxd index d7f3de76c63..ac93e3def19 100644 --- a/python/cudf/cudf/_lib/cpp/table/table.pxd +++ b/python/cudf/cudf/_lib/cpp/table/table.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector @@ -11,7 +11,6 @@ from cudf._lib.cpp.types cimport size_type cdef extern from "cudf/table/table.hpp" namespace "cudf" nogil: cdef cppclass table: table(const table&) except + - table(vector[unique_ptr[column]]&& columns) except + table(table_view) except + size_type num_columns() except + size_type num_rows() except + diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index df6ed89ac7e..3f275e2635f 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -543,7 +543,7 @@ def write_csv( ) -cdef data_type _get_cudf_data_type_from_dtype(object dtype) except +: +cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *: # TODO: Remove this work-around Dictionary types # in libcudf are fully mapped to categorical columns: # https://github.com/rapidsai/cudf/issues/3960 diff --git a/python/cudf/cudf/_lib/io/datasource.pxd b/python/cudf/cudf/_lib/io/datasource.pxd index a7a3731a0e6..e8fe79d2685 100644 --- a/python/cudf/cudf/_lib/io/datasource.pxd +++ b/python/cudf/cudf/_lib/io/datasource.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp.memory cimport shared_ptr @@ -6,7 +6,7 @@ from cudf._lib.cpp.io.types cimport arrow_io_source, datasource cdef class Datasource: - cdef datasource* get_datasource(self) nogil except * + cdef datasource* get_datasource(self) except * nogil cdef class NativeFileDatasource(Datasource): cdef shared_ptr[arrow_io_source] c_datasource diff --git a/python/cudf/cudf/_lib/io/datasource.pyx b/python/cudf/cudf/_lib/io/datasource.pyx index 7402779a6ac..b39a1aee9b8 100644 --- a/python/cudf/cudf/_lib/io/datasource.pyx +++ b/python/cudf/cudf/_lib/io/datasource.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libcpp.memory cimport shared_ptr from pyarrow.includes.libarrow cimport CRandomAccessFile @@ -8,7 +8,7 @@ from cudf._lib.cpp.io.types cimport arrow_io_source, datasource cdef class Datasource: - cdef datasource* get_datasource(self) nogil except *: + cdef datasource* get_datasource(self) except * nogil: with gil: raise NotImplementedError("get_datasource() should not " + "be directly invoked here") diff --git a/python/cudf/cudf/_lib/json.pyx b/python/cudf/cudf/_lib/json.pyx index af4232a8734..611baed7fd7 100644 --- a/python/cudf/cudf/_lib/json.pyx +++ b/python/cudf/cudf/_lib/json.pyx @@ -211,7 +211,7 @@ def write_json( ) -cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except +: +cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except *: cdef schema_element s_element cdef data_type lib_type if cudf.api.types.is_categorical_dtype(dtype): @@ -236,7 +236,7 @@ cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except +: return s_element -cdef data_type _get_cudf_data_type_from_dtype(object dtype) except +: +cdef data_type _get_cudf_data_type_from_dtype(object dtype) except *: if cudf.api.types.is_categorical_dtype(dtype): raise NotImplementedError( "CategoricalDtype as dtype is not yet " diff --git a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd index 2d2bfe8ef8e..b4f8bfad4fb 100644 --- a/python/cudf/cudf/_lib/pylibcudf/__init__.pxd +++ b/python/cudf/cudf/_lib/pylibcudf/__init__.pxd @@ -1,15 +1,11 @@ # Copyright (c) 2023, NVIDIA CORPORATION. # TODO: Verify consistent usage of relative/absolute imports in pylibcudf. -# Relative Cython imports always look one level too high. This is a known bug -# https://github.com/cython/cython/issues/3442 -# that is fixed in Cython 3 -# https://github.com/cython/cython/pull/4552 -from .pylibcudf cimport copying -from .pylibcudf.column cimport Column -from .pylibcudf.gpumemoryview cimport gpumemoryview -from .pylibcudf.table cimport Table -from .pylibcudf.types cimport DataType, TypeId +from . cimport copying +from .column cimport Column +from .gpumemoryview cimport gpumemoryview +from .table cimport Table +from .types cimport DataType, TypeId __all__ = [ "Column", diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 5cd63893844..574769f68d1 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" requires = [ "cmake>=3.26.4", - "cython>=0.29,<0.30", + "cython>=3.0.0", "ninja", "numpy>=1.21", "protoc-wheel", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 081a2f69800..a6ef867451b 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -3,7 +3,7 @@ [build-system] requires = [ - "cython>=0.29,<0.30", + "cython>=3.0.0", "numpy>=1.21", "pyarrow==12.0.1.*", "setuptools",