diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index b1bfac2a1dd..c851f65d4f6 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -28,8 +28,11 @@ if [[ $(arch) == "aarch64" ]]; then DESELECTED_TESTS+=("tests/unit/operations/test_join.py::test_join_4_columns_with_validity") else # Ensure that we don't run dbgen when it uses newer symbols than supported by the glibc version in the CI image. + # Allow errors since any of these commands could produce empty results that would cause the script to fail. + set +e glibc_minor_version=$(ldd --version | head -1 | grep -o "[0-9]\.[0-9]\+" | tail -1 | cut -d '.' -f2) latest_glibc_symbol_found=$(nm py-polars/tests/benchmark/data/pdsh/dbgen/dbgen | grep GLIBC | grep -o "[0-9]\.[0-9]\+" | sort --version-sort | tail -1 | cut -d "." -f 2) + set -e if [[ ${glibc_minor_version} -lt ${latest_glibc_symbol_found} ]]; then DESELECTED_TESTS+=("tests/benchmark/test_pdsh.py::test_pdsh") fi diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index ecc490b378b..97c72ec8042 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -71,6 +71,7 @@ dependencies: - ptxcompiler - pyarrow>=14.0.0,<19.0.0a0 - pydata-sphinx-theme!=0.14.2 +- pynvml>=11.4.1,<12.0.0a0 - pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 4559829ac3a..84b58b6d7a4 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -69,6 +69,7 @@ dependencies: - pyarrow>=14.0.0,<19.0.0a0 - pydata-sphinx-theme!=0.14.2 - pynvjitlink>=0.0.0a0 +- pynvml>=11.4.1,<12.0.0a0 - pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index 1e6c0a35a09..74ecded8ead 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -43,6 +43,7 @@ requirements: run: - python - cudf ={{ version }} + - pynvml >=11.4.1,<12.0.0a0 - rapids-dask-dependency ={{ minor_version }} - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} diff --git a/dependencies.yaml b/dependencies.yaml index 631ce12f0b0..3976696a41c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -758,6 +758,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: + - pynvml>=11.4.1,<12.0.0a0 - rapids-dask-dependency==25.2.*,>=0.0.0a0 run_custreamz: common: diff --git a/python/dask_cudf/dask_cudf/io/parquet.py b/python/dask_cudf/dask_cudf/io/parquet.py index bf8fae552c2..bbedd046760 100644 --- a/python/dask_cudf/dask_cudf/io/parquet.py +++ b/python/dask_cudf/dask_cudf/io/parquet.py @@ -55,7 +55,7 @@ def _get_device_size(): handle = pynvml.nvmlDeviceGetHandleByIndex(int(index)) return pynvml.nvmlDeviceGetMemoryInfo(handle).total - except (ImportError, ValueError): + except ValueError: # Fall back to a conservative 8GiB default return 8 * 1024**3 diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 9364cc7647f..33ba8fe083f 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -24,6 +24,7 @@ dependencies = [ "fsspec>=0.6.0", "numpy>=1.23,<3.0a0", "pandas>=2.0,<2.2.4dev0", + "pynvml>=11.4.1,<12.0.0a0", "rapids-dask-dependency==25.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [