Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support constructing cudf.Scalar objects from host side lists #8459

Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions python/cudf/cudf/_lib/cpp/scalar/scalar.pxd
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,5 @@ cdef extern from "cudf/scalar/scalar.hpp" namespace "cudf" nogil:
# TODO: Figure out how to add an int32 overload of value()

cdef cppclass list_scalar(scalar):
list_scalar(column_view col) except +
column_view view() except +
17 changes: 16 additions & 1 deletion python/cudf/cudf/_lib/scalar.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import decimal
import numpy as np
import pandas as pd

import pyarrow as pa
from libc.stdint cimport (
int8_t,
int16_t,
Expand Down Expand Up @@ -82,6 +82,9 @@ cdef class DeviceScalar:
if isinstance(dtype, cudf.Decimal64Dtype):
_set_decimal64_from_scalar(
self.c_value, value, dtype, valid)
elif isinstance(dtype, cudf.ListDtype):
_set_list_from_pylist(
self.c_value, value, dtype, valid)
elif pd.api.types.is_string_dtype(dtype):
_set_string_from_np_string(self.c_value, value, valid)
elif pd.api.types.is_numeric_dtype(dtype):
Expand Down Expand Up @@ -295,6 +298,18 @@ cdef _set_decimal64_from_scalar(unique_ptr[scalar]& s,
)
)

cdef _set_list_from_pylist(unique_ptr[scalar]& s,
object value,
object dtype,
bool valid=True):
cdef Column col = cudf.core.column.as_column(
pa.array(value)
)
cdef column_view col_view = col.view()
s.reset(
new list_scalar(col_view)
)

cdef _get_py_list_from_list(unique_ptr[scalar]& s):

if not s.get()[0].is_valid():
Expand Down
11 changes: 10 additions & 1 deletion python/cudf/cudf/core/scalar.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from cudf._lib.scalar import DeviceScalar, _is_null_host_scalar
from cudf.core.column.column import ColumnBase
from cudf.core.dtypes import Decimal64Dtype
from cudf.core.dtypes import Decimal64Dtype, ListDtype
from cudf.core.index import BaseIndex
from cudf.core.series import Series
from cudf.utils.dtypes import (
Expand Down Expand Up @@ -118,6 +118,15 @@ def _device_value_to_host(self):
def _preprocess_host_value(self, value, dtype):
valid = not _is_null_host_scalar(value)

if isinstance(value, list):
if dtype is not None:
raise TypeError("Lists may not be cast to a different dtype")
else:
dtype = ListDtype.from_arrow(
pa.infer_type([value], from_pandas=True)
)
return value, dtype

if isinstance(dtype, Decimal64Dtype):
value = pa.scalar(
value, type=pa.decimal128(dtype.precision, dtype.scale)
Expand Down
67 changes: 55 additions & 12 deletions python/cudf/cudf/tests/test_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import cudf
from cudf import NA
from cudf._lib.copying import get_element
from cudf.tests.utils import assert_eq


Expand Down Expand Up @@ -336,25 +337,67 @@ def test_concatenate_list_with_nonlist():


@pytest.mark.parametrize(
"indata,expect",
"data",
[
([1], [1]),
([1, 2, 3], [1, 2, 3]),
([[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]),
([None], [NA]),
([1, None, 3], [1, NA, 3]),
([[1, None, 3], [None, 5, 6]], [[1, NA, 3], [NA, 5, 6]]),
marlenezw marked this conversation as resolved.
Show resolved Hide resolved
[1],
[1, 2, 3],
[[1, 2, 3], [4, 5, 6]],
[NA],
[1, NA, 3],
[[1, NA, 3], [NA, 5, 6]],
],
)
def test_list_getitem(indata, expect):
list_sr = cudf.Series([indata])
# __getitem__ shall fill None with cudf.NA
assert list_sr[0] == expect
def test_list_getitem(data):
list_sr = cudf.Series([data])
assert list_sr[0] == data


@pytest.mark.parametrize(
"input_obj", [[[1, cudf.NA, 3]], [[1, cudf.NA, 3], [4, 5, cudf.NA]]]
"data",
[
[1, 2, 3],
[[1, 2, 3], [4, 5, 6]],
["a", "b", "c"],
[["a", "b", "c"], ["d", "e", "f"]],
[1.1, 2.2, 3.3],
[[1.1, 2.2, 3.3], [4.4, 5.5, 6.6]],
[1, NA, 3],
[[1, NA, 3], [4, 5, NA]],
["a", NA, "c"],
[["a", NA, "c"], ["d", "e", NA]],
[1.1, NA, 3.3],
[[1.1, NA, 3.3], [4.4, 5.5, NA]],
],
)
def test_list_scalar_host_construction(data):
slr = cudf.Scalar(data)
assert slr.value == data


@pytest.mark.parametrize(
"data",
[
[1, 2, 3],
[[1, 2, 3], [4, 5, 6]],
["a", "b", "c"],
[["a", "b", "c"], ["d", "e", "f"]],
[1.1, 2.2, 3.3],
[[1.1, 2.2, 3.3], [4.4, 5.5, 6.6]],
[1, NA, 3],
[[1, NA, 3], [4, 5, NA]],
["a", NA, "c"],
[["a", NA, "c"], ["d", "e", NA]],
[1.1, NA, 3.3],
[[1.1, NA, 3.3], [4.4, 5.5, NA]],
],
)
def test_list_scalar_device_construction(data):
col = cudf.Series([data])._column
slr = get_element(col, 0)
assert slr.value == data


@pytest.mark.parametrize("input_obj", [[[1, NA, 3]], [[1, NA, 3], [4, 5, NA]]])
def test_construction_series_with_nulls(input_obj):
expect = pa.array(input_obj, from_pandas=True)
got = cudf.Series(input_obj).to_arrow()
Expand Down