From e3ce7cb7ce0b5592726e8352d9054c33996b96ab Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Mon, 15 Nov 2021 12:57:11 -0500 Subject: [PATCH 01/19] Use ptxcompiler to patch Numba at runtime to support CUDA-enhanced compatibility --- conda/environments/cudf_dev_cuda11.0.yml | 1 + conda/environments/cudf_dev_cuda11.2.yml | 1 + conda/environments/cudf_dev_cuda11.5.yml | 1 + conda/recipes/cudf/meta.yaml | 2 +- python/cudf/cudf/__init__.py | 7 +++++++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 803e4f0ba26..fc10ea57891 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -66,3 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc + - ptxcompiler \ No newline at end of file diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 2281d361ebd..33154588b22 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -66,3 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc + - ptxcompiler \ No newline at end of file diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index 63800fe786b..ca1788d5995 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -66,3 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc + - ptxcompiler \ No newline at end of file diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 615135a6f6b..3114c1953ef 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -51,7 +51,7 @@ requirements: - nvtx >=0.2.1 - packaging - cachetools - + - ptxcompiler # CUDA enhanced compatibility. See https://github.com/rapidsai/ptxcompiler test: # [linux64] requires: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index bc35551b5bd..31ea8808bc8 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -101,10 +101,16 @@ from cudf.core.tools.datetimes import date_range from cudf.utils.dtypes import _NA_REP from cudf.utils.utils import set_allocator +from ptxcompiler.patch import patch_numba_codegen_if_needed cuda.set_memory_manager(rmm.RMMNumbaManager) cupy.cuda.set_allocator(rmm.rmm_cupy_allocator) +# Patch Numba to support CUDA enhanced compatibility. +# See https://github.com/rapidsai/ptxcompiler for +# details. +patch_numba_codegen_if_needed() + try: # Numba 0.54: Disable low occupancy warnings numba_config.CUDA_LOW_OCCUPANCY_WARNINGS = 0 @@ -113,5 +119,6 @@ pass del numba_config + __version__ = get_versions()["version"] del get_versions From 7f5c4b92be87bed7e83e460bed60750e39b36ca9 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 08:41:40 -0500 Subject: [PATCH 02/19] del function that we don't want to expose --- python/cudf/cudf/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index 31ea8808bc8..53540a8d5c1 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -110,6 +110,7 @@ # See https://github.com/rapidsai/ptxcompiler for # details. patch_numba_codegen_if_needed() +del patch_numba_codegen_if_needed try: # Numba 0.54: Disable low occupancy warnings From c06cda21bfeb5e832493e5a0f0fde5cc537fe64a Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 08:45:33 -0500 Subject: [PATCH 03/19] Newlines --- conda/environments/cudf_dev_cuda11.0.yml | 3 ++- conda/environments/cudf_dev_cuda11.2.yml | 2 +- conda/environments/cudf_dev_cuda11.5.yml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index fc10ea57891..f4db5fc6fd3 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -66,4 +66,5 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - - ptxcompiler \ No newline at end of file + - ptxcompiler + diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 33154588b22..23dcb27362a 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -66,4 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - - ptxcompiler \ No newline at end of file + - ptxcompiler diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index ca1788d5995..da1260de3cd 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -66,4 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - - ptxcompiler \ No newline at end of file + - ptxcompiler From 3438bdd5cb67fed86dc0616707d2651413b0eabe Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 15:33:25 -0500 Subject: [PATCH 04/19] Newlines --- conda/environments/cudf_dev_cuda11.0.yml | 1 - conda/recipes/cudf/meta.yaml | 1 + python/cudf/cudf/__init__.py | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index f4db5fc6fd3..4ccb790d574 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -67,4 +67,3 @@ dependencies: - git+https://github.com/python-streamz/streamz.git@master - pyorc - ptxcompiler - diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 3114c1953ef..28479de967a 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -52,6 +52,7 @@ requirements: - packaging - cachetools - ptxcompiler # CUDA enhanced compatibility. See https://github.com/rapidsai/ptxcompiler + test: # [linux64] requires: # [linux64] - cudatoolkit {{ cuda_version }}.* # [linux64] diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index 53540a8d5c1..50be8cc25c8 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -120,6 +120,5 @@ pass del numba_config - __version__ = get_versions()["version"] del get_versions From 61ff5f2ab26583ec7e3bb20453b54558239030d8 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 15:59:54 -0500 Subject: [PATCH 05/19] Don't use debug=True in numba kernels --- .../cudf/tests/test_extension_compilation.py | 47 +++++++++++-------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/python/cudf/cudf/tests/test_extension_compilation.py b/python/cudf/cudf/tests/test_extension_compilation.py index 39fa7b11ce2..775ef21d1a8 100644 --- a/python/cudf/cudf/tests/test_extension_compilation.py +++ b/python/cudf/cudf/tests/test_extension_compilation.py @@ -1,5 +1,6 @@ import operator +import cupy as cp import pytest from numba import cuda, types from numba.cuda import compile_ptx @@ -71,8 +72,8 @@ def test_execute_masked_binary(op, ty): def func(x, y): return op(x, y) - @cuda.jit(debug=True) - def test_kernel(x, y): + @cuda.jit + def test_kernel(x, y, err): # Reference result with unmasked value u = func(x, y) @@ -87,14 +88,15 @@ def test_kernel(x, y): # Check masks are as expected, and unmasked result matches masked # result if r0.valid: - raise RuntimeError("Expected r0 to be invalid") + err[0] = 1 if not r1.valid: - raise RuntimeError("Expected r1 to be valid") + err[0] = 2 if u != r1.value: - print("Values: ", u, r1.value) - raise RuntimeError("u != r1.value") + err[0] = 3 - test_kernel[1, 1](1, 2) + err = cp.asarray([0], dtype="int8") + test_kernel[1, 1](1, 2, err) + assert err[0] == 0 @pytest.mark.parametrize("op", ops) @@ -187,18 +189,20 @@ def test_is_na(fn): device_fn = cuda.jit(device=True)(fn) - @cuda.jit(debug=True) - def test_kernel(): + @cuda.jit + def test_kernel(err): valid_is_na = device_fn(valid) invalid_is_na = device_fn(invalid) if valid_is_na: - raise RuntimeError("Valid masked value is NA and should not be") + err[0] = 1 if not invalid_is_na: - raise RuntimeError("Invalid masked value is not NA and should be") + err[0] = 2 + err = cp.asarray([0], dtype="int8") test_kernel[1, 1]() + assert err[0] == 0 def func_lt_na(x): @@ -271,8 +275,8 @@ def test_na_masked_comparisons(fn, ty): device_fn = cuda.jit(device=True)(fn) - @cuda.jit(debug=True) - def test_kernel(): + @cuda.jit + def test_kernel(err): unmasked = ty(1) valid_masked = Masked(unmasked, True) invalid_masked = Masked(unmasked, False) @@ -281,12 +285,14 @@ def test_kernel(): invalid_cmp_na = device_fn(invalid_masked) if valid_cmp_na: - raise RuntimeError("Valid masked value compared True with NA") + err[0] = 1 if invalid_cmp_na: - raise RuntimeError("Invalid masked value compared True with NA") + err[0] = 2 - test_kernel[1, 1]() + err = cp.asarray([0], dtype="int8") + test_kernel[1, 1](err) + assert err[0] == 0 # xfail because scalars do not yet cast for a comparison to NA @@ -297,13 +303,16 @@ def test_na_scalar_comparisons(fn, ty): device_fn = cuda.jit(device=True)(fn) - @cuda.jit(debug=True) - def test_kernel(): + @cuda.jit + def test_kernel(err): unmasked = ty(1) unmasked_cmp_na = device_fn(unmasked) if unmasked_cmp_na: + err[0] = 1 raise RuntimeError("Unmasked value compared True with NA") - test_kernel[1, 1]() + err = cp.asarray([0], dtype="int8") + test_kernel[1, 1](err) + assert err[0] == 0 From 17287cb739cb76037cea032260ebd33a90b03735 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 16:09:48 -0500 Subject: [PATCH 06/19] Describe change --- python/cudf/cudf/tests/test_extension_compilation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/cudf/cudf/tests/test_extension_compilation.py b/python/cudf/cudf/tests/test_extension_compilation.py index 775ef21d1a8..e417eec1f57 100644 --- a/python/cudf/cudf/tests/test_extension_compilation.py +++ b/python/cudf/cudf/tests/test_extension_compilation.py @@ -88,6 +88,11 @@ def test_kernel(x, y, err): # Check masks are as expected, and unmasked result matches masked # result if r0.valid: + # TODO: ideally, we would raise an exception here rather + # than return an "error code", and that is what the + # previous version of this (and below) tests did. But, + # Numba kernels cannot currently use `debug=True` with + # CUDA enhanced compatibility. err[0] = 1 if not r1.valid: err[0] = 2 From 97c694652c473a3f7d763b2ad614a3d07fa79d7c Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 16:10:34 -0500 Subject: [PATCH 07/19] Improve description --- python/cudf/cudf/tests/test_extension_compilation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_extension_compilation.py b/python/cudf/cudf/tests/test_extension_compilation.py index e417eec1f57..e5cb539bf10 100644 --- a/python/cudf/cudf/tests/test_extension_compilation.py +++ b/python/cudf/cudf/tests/test_extension_compilation.py @@ -92,7 +92,9 @@ def test_kernel(x, y, err): # than return an "error code", and that is what the # previous version of this (and below) tests did. But, # Numba kernels cannot currently use `debug=True` with - # CUDA enhanced compatibility. + # CUDA enhanced compatibility. Once a solution to that is + # reached, we should switch back to raising exceptions + # here. err[0] = 1 if not r1.valid: err[0] = 2 From 95963f580cc9152d7a84ebffae286e6d02f8d975 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 16:38:12 -0500 Subject: [PATCH 08/19] Change numba version --- conda/environments/cudf_dev_cuda11.0.yml | 2 +- conda/environments/cudf_dev_cuda11.2.yml | 2 +- conda/environments/cudf_dev_cuda11.5.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 4ccb790d574..1a914368257 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.20.1 - cmake_setuptools>=0.1.3 - python>=3.7,<3.9 - - numba>=0.53.1 + - numba>=0.54 - numpy - pandas>=1.0,<1.4.0dev0 - pyarrow=5.0.0=*cuda diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 23dcb27362a..940111e08db 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.20.1 - cmake_setuptools>=0.1.3 - python>=3.7,<3.9 - - numba>=0.53.1 + - numba>=0.54 - numpy - pandas>=1.0,<1.4.0dev0 - pyarrow=5.0.0=*cuda diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index da1260de3cd..123590ac8cc 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -14,7 +14,7 @@ dependencies: - cmake>=3.20.1 - cmake_setuptools>=0.1.3 - python>=3.7,<3.9 - - numba>=0.53.1 + - numba>=0.54 - numpy - pandas>=1.0,<1.4.0dev0 - pyarrow=5.0.0=*cuda From 9c66ae368e38a8a5dee142a2caceb03ab003e32e Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Tue, 16 Nov 2021 17:40:33 -0500 Subject: [PATCH 09/19] Remove raise --- python/cudf/cudf/tests/test_extension_compilation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_extension_compilation.py b/python/cudf/cudf/tests/test_extension_compilation.py index e5cb539bf10..de75d19c4a3 100644 --- a/python/cudf/cudf/tests/test_extension_compilation.py +++ b/python/cudf/cudf/tests/test_extension_compilation.py @@ -318,7 +318,6 @@ def test_kernel(err): if unmasked_cmp_na: err[0] = 1 - raise RuntimeError("Unmasked value compared True with NA") err = cp.asarray([0], dtype="int8") test_kernel[1, 1](err) From 9aebb54712cebc1b18f5b0e512741165de460e5d Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 16 Nov 2021 17:47:39 -0600 Subject: [PATCH 10/19] fix pytests for latest numpy --- python/cudf/cudf/tests/test_concat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/tests/test_concat.py b/python/cudf/cudf/tests/test_concat.py index bb96f3c4290..46707a283af 100644 --- a/python/cudf/cudf/tests/test_concat.py +++ b/python/cudf/cudf/tests/test_concat.py @@ -576,7 +576,7 @@ def test_concat_empty_dataframes(df, other, ignore_index): if expected.shape != df.shape: for key, col in actual[actual.columns].iteritems(): if is_categorical_dtype(col.dtype): - if expected[key].dtype != "category": + if not is_categorical_dtype(expected[key].dtype): # TODO: Pandas bug: # https://github.com/pandas-dev/pandas/issues/42840 expected[key] = expected[key].fillna("-1").astype("str") @@ -1186,7 +1186,7 @@ def test_concat_join_empty_dataframes( if axis == 0: for key, col in actual[actual.columns].iteritems(): if is_categorical_dtype(col.dtype): - if expected[key].dtype != "category": + if not is_categorical_dtype(expected[key].dtype): # TODO: Pandas bug: # https://github.com/pandas-dev/pandas/issues/42840 expected[key] = ( From 9eb1579a92f177e7d63cdc90cccf4ed75d337898 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 17 Nov 2021 07:22:38 -0800 Subject: [PATCH 11/19] upgrade clang to 11.1.0 --- conda/environments/cudf_dev_cuda11.0.yml | 4 ++-- conda/environments/cudf_dev_cuda11.2.yml | 4 ++-- conda/environments/cudf_dev_cuda11.5.yml | 4 ++-- cpp/scripts/run-clang-format.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 803e4f0ba26..2669635622b 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -7,8 +7,8 @@ channels: - rapidsai-nightly - conda-forge dependencies: - - clang=11.0.0 - - clang-tools=11.0.0 + - clang=11.1.0 + - clang-tools=11.1.0 - cupy>7.1.0,<10.0.0a0 - rmm=21.12.* - cmake>=3.20.1 diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 2281d361ebd..d14bd70a3c9 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -7,8 +7,8 @@ channels: - rapidsai-nightly - conda-forge dependencies: - - clang=11.0.0 - - clang-tools=11.0.0 + - clang=11.1.0 + - clang-tools=11.1.0 - cupy>7.1.0,<10.0.0a0 - rmm=21.12.* - cmake>=3.20.1 diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index 63800fe786b..728c91c7887 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -7,8 +7,8 @@ channels: - rapidsai-nightly - conda-forge dependencies: - - clang=11.0.0 - - clang-tools=11.0.0 + - clang=11.1.0 + - clang-tools=11.1.0 - cupy>7.1.0,<10.0.0a0 - rmm=21.12.* - cmake>=3.20.1 diff --git a/cpp/scripts/run-clang-format.py b/cpp/scripts/run-clang-format.py index 178bf2f0c78..a7c83da22c5 100755 --- a/cpp/scripts/run-clang-format.py +++ b/cpp/scripts/run-clang-format.py @@ -22,7 +22,7 @@ import sys import tempfile -EXPECTED_VERSION = "11.0.0" +EXPECTED_VERSION = "11.1.0" VERSION_REGEX = re.compile(r"clang-format version ([0-9.]+)") # NOTE: populate this list with more top-level dirs as we add more of them to # the cudf repo From b192ab92e14d15d8935296e976d7bcfa0b2504fe Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 17 Nov 2021 14:16:00 -0600 Subject: [PATCH 12/19] Update python/cudf/cudf/tests/test_extension_compilation.py Co-authored-by: Graham Markall <535640+gmarkall@users.noreply.github.com> --- python/cudf/cudf/tests/test_extension_compilation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf/cudf/tests/test_extension_compilation.py b/python/cudf/cudf/tests/test_extension_compilation.py index de75d19c4a3..47c9448cf63 100644 --- a/python/cudf/cudf/tests/test_extension_compilation.py +++ b/python/cudf/cudf/tests/test_extension_compilation.py @@ -208,7 +208,7 @@ def test_kernel(err): err[0] = 2 err = cp.asarray([0], dtype="int8") - test_kernel[1, 1]() + test_kernel[1, 1](err) assert err[0] == 0 From e63b0a9738d565db55ea22c0614c52b754e772ae Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Wed, 17 Nov 2021 19:59:18 -0500 Subject: [PATCH 13/19] Update python/cudf/cudf/__init__.py Co-authored-by: jakirkham --- python/cudf/cudf/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index 50be8cc25c8..aa50e5422cf 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -101,7 +101,16 @@ from cudf.core.tools.datetimes import date_range from cudf.utils.dtypes import _NA_REP from cudf.utils.utils import set_allocator -from ptxcompiler.patch import patch_numba_codegen_if_needed +try: + from ptxcompiler.patch import patch_numba_codegen_if_needed +except ImportError: + pass +else: + # Patch Numba to support CUDA enhanced compatibility. + # See https://github.com/rapidsai/ptxcompiler for + # details. + patch_numba_codegen_if_needed() + del patch_numba_codegen_if_needed cuda.set_memory_manager(rmm.RMMNumbaManager) cupy.cuda.set_allocator(rmm.rmm_cupy_allocator) From 74a6c79c7e8e80232d6d68e2b04680432c855e6a Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Thu, 18 Nov 2021 07:21:39 -0500 Subject: [PATCH 14/19] Remove duplicated code --- python/cudf/cudf/__init__.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index aa50e5422cf..b24e71e7785 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -101,6 +101,7 @@ from cudf.core.tools.datetimes import date_range from cudf.utils.dtypes import _NA_REP from cudf.utils.utils import set_allocator + try: from ptxcompiler.patch import patch_numba_codegen_if_needed except ImportError: @@ -115,12 +116,6 @@ cuda.set_memory_manager(rmm.RMMNumbaManager) cupy.cuda.set_allocator(rmm.rmm_cupy_allocator) -# Patch Numba to support CUDA enhanced compatibility. -# See https://github.com/rapidsai/ptxcompiler for -# details. -patch_numba_codegen_if_needed() -del patch_numba_codegen_if_needed - try: # Numba 0.54: Disable low occupancy warnings numba_config.CUDA_LOW_OCCUPANCY_WARNINGS = 0 From 3675c0c344eec0f9ad424020bc83472e361ad0bc Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Thu, 18 Nov 2021 09:17:14 -0500 Subject: [PATCH 15/19] Change numba pin --- conda/recipes/cudf/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 28479de967a..ffce5db6a49 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -29,7 +29,7 @@ requirements: - python - cython >=0.29,<0.30 - setuptools - - numba >=0.53.1 + - numba >=0.54 - dlpack>=0.5,<0.6.0a0 - pyarrow 5.0.0 *cuda - libcudf {{ version }} @@ -41,7 +41,7 @@ requirements: - typing_extensions - pandas >=1.0,<1.4.0dev0 - cupy >7.1.0,<10.0.0a0 - - numba >=0.53.1 + - numba >=0.54 - numpy - {{ pin_compatible('pyarrow', max_pin='x.x.x') }} *cuda - fastavro >=0.22.0 From 8455c6042eaee6040c59f386f56b07bf08db6fb5 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Mon, 22 Nov 2021 10:33:57 -0500 Subject: [PATCH 16/19] Update conda/environments/cudf_dev_cuda11.5.yml --- conda/environments/cudf_dev_cuda11.5.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index 7f10a721a29..b93beb48a3c 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -66,4 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - - ptxcompiler + - ptxcompiler # [linux64] From f8019debca14879fd128a49c02a11d57e280be84 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Mon, 22 Nov 2021 10:34:06 -0500 Subject: [PATCH 17/19] Update conda/environments/cudf_dev_cuda11.2.yml --- conda/environments/cudf_dev_cuda11.2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/cudf_dev_cuda11.2.yml b/conda/environments/cudf_dev_cuda11.2.yml index 32568221b35..ec35df05e9b 100644 --- a/conda/environments/cudf_dev_cuda11.2.yml +++ b/conda/environments/cudf_dev_cuda11.2.yml @@ -66,4 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - - ptxcompiler + - ptxcompiler # [linux64] From 65da07dec09c2fa92eb1bb0ed624498eaebd0bb7 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath <3190405+shwina@users.noreply.github.com> Date: Mon, 22 Nov 2021 10:34:15 -0500 Subject: [PATCH 18/19] Update conda/recipes/cudf/meta.yaml --- conda/recipes/cudf/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index fc2afb18790..46eefbc825f 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -51,7 +51,7 @@ requirements: - nvtx >=0.2.1 - packaging - cachetools - - ptxcompiler # CUDA enhanced compatibility. See https://github.com/rapidsai/ptxcompiler + - ptxcompiler # [linux64] # CUDA enhanced compatibility. See https://github.com/rapidsai/ptxcompiler test: # [linux64] requires: # [linux64] From 69dca77bb0978201de7bf9c31fc7aaf564ff0390 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Mon, 22 Nov 2021 10:35:02 -0500 Subject: [PATCH 19/19] One more linux64 --- conda/environments/cudf_dev_cuda11.0.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/cudf_dev_cuda11.0.yml b/conda/environments/cudf_dev_cuda11.0.yml index 5bea2a9abf2..aa6e7aebdaf 100644 --- a/conda/environments/cudf_dev_cuda11.0.yml +++ b/conda/environments/cudf_dev_cuda11.0.yml @@ -66,4 +66,4 @@ dependencies: - git+https://github.com/dask/distributed.git@main - git+https://github.com/python-streamz/streamz.git@master - pyorc - - ptxcompiler + - ptxcompiler # [linux64]