From 15e870f08fa1a11406fa1b47ebc0a327029940b6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 May 2023 15:43:19 -0700 Subject: [PATCH 01/17] Update cupy dependency --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- dependencies.yaml | 6 +++--- python/cudf/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index a67b9fbce4e..2c0b5448e80 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -18,7 +18,7 @@ dependencies: - cuda-python>=11.7.1,<12.0 - cuda-sanitizer-api=11.8.86 - cudatoolkit=11.8 -- cupy>=9.5.0,<12.0.0a0 +- cupy>=12.0.0 - cxx-compiler - cython>=0.29,<0.30 - dask-core==2023.3.2 diff --git a/dependencies.yaml b/dependencies.yaml index cf23bf93904..2c8b9430415 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -358,7 +358,7 @@ dependencies: - output_types: conda packages: - cubinlinker - - cupy>=9.5.0,<12.0.0a0 + - cupy>=12.0.0 - pip - pip: - git+https://github.com/python-streamz/streamz.git@master @@ -374,7 +374,7 @@ dependencies: - output_types: pyproject packages: - cubinlinker - - &cupy_pip cupy-cuda11x>=9.5.0,<12.0.0a0 + - &cupy_pip cupy-cuda11x>=12.0.0 - ptxcompiler specific: - output_types: requirements @@ -382,7 +382,7 @@ dependencies: - matrix: arch: x86_64 packages: - - cupy-cuda115>=9.5.0,<12.0.0a0 # TODO: This might change to cupy-cuda11x? + - cupy-cuda115>=12.0.0 - matrix: arch: aarch64 packages: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index cccf9e74f2f..2a686fe3016 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "cachetools", "cubinlinker", "cuda-python>=11.7.1,<12.0", - "cupy-cuda11x>=9.5.0,<12.0.0a0", + "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numba>=0.56.4,<0.57", "numpy>=1.21", diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 34a10cc2c71..c6c43661e9e 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -19,7 +19,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cudf==23.6.*", - "cupy-cuda11x>=9.5.0,<12.0.0a0", + "cupy-cuda11x>=12.0.0", "dask==2023.3.2", "distributed==2023.3.2.1", "fsspec>=0.6.0", From 41d158cd9b13ca9c94ff991f8ac4c051801b668b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 May 2023 17:44:01 -0700 Subject: [PATCH 02/17] Add heaviside to list of xfails for ufunc test --- python/cudf/cudf/tests/test_array_ufunc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index ac77c6b89f3..86a7c34be3b 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -358,6 +358,7 @@ def test_ufunc_dataframe(request, ufunc, has_nulls, indexed): "fmax", "fmin", "fmod", + "heaviside", "gcd", "hypot", "lcm", From 582ddc3d76bdf32aa89929f15c4f8d25fdcf7970 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 3 May 2023 18:00:02 -0700 Subject: [PATCH 03/17] Update copyright --- python/cudf/cudf/tests/test_array_ufunc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index 86a7c34be3b..81950bb8bde 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import operator import warnings From 012af07d726b9f5b78e2450afebc54caa46c6083 Mon Sep 17 00:00:00 2001 From: divyegala Date: Wed, 3 May 2023 20:23:03 -0700 Subject: [PATCH 04/17] fix bad merge --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 8 ++++---- .github/workflows/test.yaml | 4 ++-- ci/release/apply_wheel_modifications.sh | 5 +++++ dependencies.yaml | 21 +++++++++++++++++++-- python/cudf/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 7 files changed, 36 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ead85373eaf..2d592d3f247 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,7 +66,7 @@ jobs: run_script: "ci/build_docs.sh" wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -78,7 +78,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -99,7 +99,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@cuda-120-pip with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 98f717bf40c..1c3234edd23 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -98,7 +98,7 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@cuda-120-pip with: build_type: pull-request package-name: cudf @@ -107,7 +107,7 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120-pip with: build_type: pull-request package-name: cudf @@ -118,7 +118,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-tests-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@cuda-120-pip with: build_type: pull-request package-name: dask_cudf @@ -127,7 +127,7 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@cuda-120-pip with: build_type: pull-request package-name: dask_cudf diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 08e7925ebbf..e2e71daaa87 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -79,7 +79,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@cuda-120-pip with: build_type: nightly branch: ${{ inputs.branch }} @@ -90,7 +90,7 @@ jobs: test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests" wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@manylinux_v2 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@cuda-120-pip with: build_type: nightly branch: ${{ inputs.branch }} diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index 0c55c4b9141..b4ccba7537a 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -22,3 +22,8 @@ sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/pyproject.toml sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/pyproject.toml # Need to provide the == to avoid modifying the URL sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml + +if [[ $CUDA_SUFFIX == "-cu12" ]]; then + sed -i "/cuda-python/ s/>=11.7.1,<12.0/>=12.0,<13.0/g" python/{cudf,dask_cudf}/pyproject.toml + sed -i "/cupy-cuda11x/ s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml +fi diff --git a/dependencies.yaml b/dependencies.yaml index cf23bf93904..69d3d8a0018 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -374,9 +374,17 @@ dependencies: - output_types: pyproject packages: - cubinlinker - - &cupy_pip cupy-cuda11x>=9.5.0,<12.0.0a0 - ptxcompiler specific: + - output_types: pyproject + matrices: + - matrix: + cuda: "12.0" + packages: + - &cupy_cu12_pip cupy-cuda12x>=12.0.0 + - matrix: # All CUDA 11 versions + packages: + - &cupy_cu11_pip cupy-cuda11x>=12.0.0 - output_types: requirements matrices: - matrix: @@ -399,7 +407,16 @@ dependencies: - output_types: pyproject packages: - &cudf cudf==23.6.* - - *cupy_pip + specific: + - output_types: pyproject + matrices: + - matrix: + cuda: "12.0" + packages: + - *cupy_cu12_pip + - matrix: # All CUDA 11 versions + packages: + - *cupy_cu11_pip run_cudf_kafka: common: - output_types: [requirements, pyproject] diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index cccf9e74f2f..2a686fe3016 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "cachetools", "cubinlinker", "cuda-python>=11.7.1,<12.0", - "cupy-cuda11x>=9.5.0,<12.0.0a0", + "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numba>=0.56.4,<0.57", "numpy>=1.21", diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 34a10cc2c71..c6c43661e9e 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -19,7 +19,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cudf==23.6.*", - "cupy-cuda11x>=9.5.0,<12.0.0a0", + "cupy-cuda11x>=12.0.0", "dask==2023.3.2", "distributed==2023.3.2.1", "fsspec>=0.6.0", From d55047dd2048861643ffd0a309189e9255dd315c Mon Sep 17 00:00:00 2001 From: divyegala Date: Thu, 4 May 2023 06:53:08 -0700 Subject: [PATCH 05/17] cuda-python dependency only for cudf --- ci/release/apply_wheel_modifications.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index b4ccba7537a..95698f72814 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -24,6 +24,6 @@ sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/das sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml if [[ $CUDA_SUFFIX == "-cu12" ]]; then - sed -i "/cuda-python/ s/>=11.7.1,<12.0/>=12.0,<13.0/g" python/{cudf,dask_cudf}/pyproject.toml + sed -i "/cuda-python/ s/>=11.7.1,<12.0/>=12.0,<13.0/g" python/cudf/pyproject.toml sed -i "/cupy-cuda11x/ s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml fi From 1b7a96d4ae1183c0b2e52e115025c76371ef5cd5 Mon Sep 17 00:00:00 2001 From: divyegala Date: Thu, 4 May 2023 08:22:19 -0700 Subject: [PATCH 06/17] better sed commands --- ci/release/apply_wheel_modifications.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index 95698f72814..d138be3b223 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -24,6 +24,6 @@ sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/das sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml if [[ $CUDA_SUFFIX == "-cu12" ]]; then - sed -i "/cuda-python/ s/>=11.7.1,<12.0/>=12.0,<13.0/g" python/cudf/pyproject.toml - sed -i "/cupy-cuda11x/ s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml + sed -i "s/cuda-python.*/cuda-python>=12.0,<13.0\",/g" python/cudf/pyproject.toml + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml fi From 12b498391765cc1665d6c0a3e9ad1777e87ded41 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Thu, 4 May 2023 13:38:23 -0400 Subject: [PATCH 07/17] sed cuda-python replacing only version --- ci/release/apply_wheel_modifications.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index d138be3b223..2af1989ba06 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -24,6 +24,6 @@ sed -i "s/^name = \"dask_cudf\"/name = \"dask_cudf${CUDA_SUFFIX}\"/g" python/das sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml if [[ $CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python.*/cuda-python>=12.0,<13.0\",/g" python/cudf/pyproject.toml + sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" python/cudf/pyproject.toml sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml fi From 5f321e1596808d0813d44cc8c849dae12abf3c44 Mon Sep 17 00:00:00 2001 From: divyegala Date: Thu, 4 May 2023 10:40:02 -0700 Subject: [PATCH 08/17] remove cupy from workflows --- .github/workflows/pr.yaml | 2 -- .github/workflows/test.yaml | 1 - 2 files changed, 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 00afd8e8374..a4c10a7d12e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -111,8 +111,6 @@ jobs: with: build_type: pull-request package-name: cudf - # Install cupy-cuda11x for arm from a special index url - test-before-arm64: "python -m pip install 'cupy-cuda11x>=12.0.0' -f https://pip.cupy.dev/aarch64" test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests" test-smoketest: "python ./ci/wheel_smoke_test_cudf.py" wheel-build-dask-cudf: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c333579406f..6ac65fce9f3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -86,7 +86,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: cudf - test-before-arm64: "python -m pip install 'cupy-cuda11x>=12.0.0' -f https://pip.cupy.dev/aarch64" test-unittest: "python -m pytest -n 8 ./python/cudf/cudf/tests" wheel-tests-dask-cudf: secrets: inherit From d1850caf6f9f16a8aba0eac517150cb45d4523d7 Mon Sep 17 00:00:00 2001 From: divyegala Date: Thu, 4 May 2023 13:00:34 -0700 Subject: [PATCH 09/17] pick numba>=0.57 for cuda 12 --- ci/release/apply_wheel_modifications.sh | 1 + dependencies.yaml | 22 +++------------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index 2af1989ba06..3c3bb36a158 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -26,4 +26,5 @@ sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/pyproject.toml if [[ $CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" python/cudf/pyproject.toml sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml + sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" python/{cudf,dask_cudf}/pyproject.toml fi diff --git a/dependencies.yaml b/dependencies.yaml index 8ad59383bc7..c46865f8cef 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -374,17 +374,9 @@ dependencies: - output_types: pyproject packages: - cubinlinker + - &cupy_pip cupy-cuda11x>=12.0.0 - ptxcompiler specific: - - output_types: pyproject - matrices: - - matrix: - cuda: "12.0" - packages: - - &cupy_cu12_pip cupy-cuda12x>=12.0.0 - - matrix: # All CUDA 11 versions - packages: - - &cupy_cu11_pip cupy-cuda11x>=12.0.0 - output_types: requirements matrices: - matrix: @@ -403,20 +395,12 @@ dependencies: - distributed==2023.3.2.1 - output_types: conda packages: + - cupy>=12.0.0 - dask-core==2023.3.2 # dask-core in conda is the actual package & dask is the meta package - output_types: pyproject packages: - &cudf cudf==23.6.* - specific: - - output_types: pyproject - matrices: - - matrix: - cuda: "12.0" - packages: - - *cupy_cu12_pip - - matrix: # All CUDA 11 versions - packages: - - *cupy_cu11_pip + - *cupy_pip run_cudf_kafka: common: - output_types: [requirements, pyproject] From ebcf248a33b534d959d0389da2194d0bde1e8f6a Mon Sep 17 00:00:00 2001 From: divyegala Date: Thu, 4 May 2023 16:13:01 -0700 Subject: [PATCH 10/17] no dependency on ptxcompiler,cubinliner for cuda12 --- ci/release/apply_wheel_modifications.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh index 3c3bb36a158..3de1814dfaf 100755 --- a/ci/release/apply_wheel_modifications.sh +++ b/ci/release/apply_wheel_modifications.sh @@ -27,4 +27,6 @@ if [[ $CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9]*/cuda-python>=12.0,<13.0/g" python/cudf/pyproject.toml sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/{cudf,dask_cudf}/pyproject.toml sed -i "s/numba[<=>\.,0-9]*/numba>=0.57/g" python/{cudf,dask_cudf}/pyproject.toml + sed -i "/ptxcompiler/d" python/cudf/pyproject.toml + sed -i "/cubinlinker/d" python/cudf/pyproject.toml fi From 0a83166e24ad89a7a3d0d0767878cc3681dcc5ed Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 5 May 2023 08:30:18 -0700 Subject: [PATCH 11/17] Also make cubinlinker import local --- python/cudf/cudf/core/udf/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/udf/utils.py b/python/cudf/cudf/core/udf/utils.py index 0b7544752e2..d890b94127f 100644 --- a/python/cudf/cudf/core/udf/utils.py +++ b/python/cudf/cudf/core/udf/utils.py @@ -8,7 +8,6 @@ import cupy as cp import llvmlite.binding as ll import numpy as np -from cubinlinker.patch import _numba_version_ok, get_logger, new_patched_linker from cuda import cudart from numba import cuda, typeof from numba.core.datamodel import default_manager, models @@ -51,9 +50,6 @@ _cudf_str_dtype = dtype(str) -logger = get_logger() - - JIT_SUPPORTED_TYPES = ( NUMERIC_TYPES | BOOL_TYPES @@ -460,9 +456,17 @@ def _setup_numba_linker(path): def maybe_patch_numba_linker( driver_version, runtime_version, ptx_toolkit_version ): + from cubinlinker.patch import ( + _numba_version_ok, + get_logger, + new_patched_linker, + ) + # Numba thinks cubinlinker is only needed if the driver is older than # the ctk, but when PTX files are present, it might also need to patch # because those PTX files may newer than the driver as well + logger = get_logger() + if (driver_version < ptx_toolkit_version) or ( driver_version < runtime_version ): From f25386fc0b2d7b711267155f55d7d0ab8fab4dc4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 5 May 2023 10:31:47 -0700 Subject: [PATCH 12/17] Temporarily bound numpy to avoid deprecation warning --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/recipes/cudf/meta.yaml | 2 +- dependencies.yaml | 2 +- python/cudf/pyproject.toml | 2 +- python/cudf_kafka/pyproject.toml | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 2c0b5448e80..9ae9c9fc2be 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -49,6 +49,7 @@ dependencies: - notebook - numba>=0.56.4,<0.57 - numpy>=1.21 +- numpy>=1.21,<1.24 - numpydoc - nvcc_linux-64=11.8 - nvtx>=0.2.1 diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 327f7f90a82..af23750b516 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -66,7 +66,7 @@ requirements: - pandas >=1.3,<1.6.0dev0 - cupy >=12.0.0 - numba >=0.56.4,<0.57 - - numpy >=1.21 + - numpy>=1.21,<1.24 # Temporarily upper bound numpy to avoid overflow deprecations - {{ pin_compatible('pyarrow', max_pin='x.x.x') }} - libcudf {{ version }} - fastavro >=0.22.0 diff --git a/dependencies.yaml b/dependencies.yaml index c46865f8cef..72710559adf 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -233,7 +233,7 @@ dependencies: # Hard pin the patch version used during the build. This must be kept # in sync with the version pinned in get_arrow.cmake. - pyarrow==11.0.0.* - - numpy>=1.21 + - numpy>=1.21,<1.24 # Temporarily upper bound numpy to avoid overflow deprecations build_python: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 2a686fe3016..8f012e1ad5f 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "cmake>=3.23.1,!=3.25.0", "cython>=0.29,<0.30", "ninja", - "numpy>=1.21", + "numpy>=1.21,<1.24", "protoc-wheel", "pyarrow==11.0.0.*", "rmm==23.6.*", diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index d8b97f52864..df0825c846a 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "cython>=0.29,<0.30", - "numpy>=1.21", + "numpy>=1.21,<1.24", "pyarrow==11.0.0.*", "setuptools", "wheel", From a0e6f0595f01fe314c2bae1d14cdfacd51a8b5f2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 5 May 2023 11:03:55 -0700 Subject: [PATCH 13/17] Update one more numpy requirement --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - dependencies.yaml | 2 +- python/cudf/pyproject.toml | 2 +- python/dask_cudf/pyproject.toml | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 9ae9c9fc2be..5983aa4b267 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -48,7 +48,6 @@ dependencies: - ninja - notebook - numba>=0.56.4,<0.57 -- numpy>=1.21 - numpy>=1.21,<1.24 - numpydoc - nvcc_linux-64=11.8 diff --git a/dependencies.yaml b/dependencies.yaml index 72710559adf..b23de2691ef 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -341,7 +341,7 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - fsspec>=0.6.0 - - numpy>=1.21 + - numpy>=1.21,<1.24 # Temporarily upper bound numpy to avoid overflow deprecations - pandas>=1.3,<1.6.0dev0 run_cudf: common: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 8f012e1ad5f..d13324a7404 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -32,7 +32,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "fsspec>=0.6.0", "numba>=0.56.4,<0.57", - "numpy>=1.21", + "numpy>=1.21,<1.24", "nvtx>=0.2.1", "packaging", "pandas>=1.3,<1.6.0dev0", diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index c6c43661e9e..ff2a3f2d095 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -23,7 +23,7 @@ dependencies = [ "dask==2023.3.2", "distributed==2023.3.2.1", "fsspec>=0.6.0", - "numpy>=1.21", + "numpy>=1.21,<1.24", "pandas>=1.3,<1.6.0dev0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From b4a1b1471d2a25104eb99a7edeaef3ea10b99486 Mon Sep 17 00:00:00 2001 From: divyegala Date: Fri, 5 May 2023 11:39:36 -0700 Subject: [PATCH 14/17] unevaluated strings --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a4c10a7d12e..cc7309802a5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -121,7 +121,7 @@ jobs: build_type: pull-request package-name: dask_cudf package-dir: python/dask_cudf - before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf && python -m pip install --no-deps ./local-cudf/cudf*.whl" + before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-cudf && python -m pip install --no-deps ./local-cudf/cudf*.whl" wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit @@ -130,5 +130,5 @@ jobs: build_type: pull-request package-name: dask_cudf # Install the cudf we just built, and also test against latest dask/distributed/dask-cuda. - test-before: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" + test-before: "RAPIDS_PY_WHEEL_NAME=cudf_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-cudf-dep && python -m pip install --no-deps ./local-cudf-dep/cudf*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06" test-unittest: "python -m pytest -n 8 ./python/dask_cudf/dask_cudf/tests" From 52358730d10cefd35f1fc24b6c130ab32995484d Mon Sep 17 00:00:00 2001 From: divyegala Date: Fri, 5 May 2023 11:45:51 -0700 Subject: [PATCH 15/17] bump From 7091cd95892531ef5793c3ebc5432a3b0183098f Mon Sep 17 00:00:00 2001 From: divyegala Date: Fri, 5 May 2023 11:46:45 -0700 Subject: [PATCH 16/17] bump From 34e6aedaf6f766f6d1050fc64e32e3bbd9fd8cb5 Mon Sep 17 00:00:00 2001 From: divyegala Date: Fri, 5 May 2023 12:36:13 -0700 Subject: [PATCH 17/17] bump