Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix has_nonempty_nulls ignoring column offset #13647

Merged
merged 2 commits into from
Jun 29, 2023
Merged

Fix has_nonempty_nulls ignoring column offset #13647

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
02e67c3
Fix `has_nonempty_nulls`
ttnghia Jun 29, 2023
d11a90e
Add test
ttnghia Jun 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions cpp/src/copying/purge_nonempty_nulls.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,10 @@ bool has_nonempty_null_rows(cudf::column_view const& input, rmm::cuda_stream_vie

// Cross-reference nullmask and offsets.
auto const type = input.type().id();
auto const offsets = (type == type_id::STRING) ? (strings_column_view{input}).offsets()
: (lists_column_view{input}).offsets();
auto const offsets = (type == type_id::STRING) ? (strings_column_view{input}).offsets_begin()
: (lists_column_view{input}).offsets_begin();
Comment on lines +45 to +46
Copy link
Contributor Author

@ttnghia ttnghia Jun 29, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI: offsets_begin = offsets().begin<offset_type>() + offset().

auto const d_input = cudf::column_device_view::create(input, stream);
auto const is_dirty_row = [d_input = *d_input, offsets = offsets.begin<size_type>()] __device__(
size_type const& row_idx) {
auto const is_dirty_row = [d_input = *d_input, offsets] __device__(size_type const& row_idx) {
return d_input.is_null_nocheck(row_idx) && (offsets[row_idx] != offsets[row_idx + 1]);
};

Expand Down
39 changes: 35 additions & 4 deletions cpp/tests/copying/purge_nonempty_nulls_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,41 @@ using gather_map_t = cudf::test::fixed_width_column_wrapper<cudf::size_type>;
template <typename T>
using LCW = cudf::test::lists_column_wrapper<T, int32_t>;

struct HasNonEmptyNullsTest : public cudf::test::BaseFixture {};

TEST_F(HasNonEmptyNullsTest, TrivialTest)
{
auto const input = LCW<T>{{{{1, 2, 3, 4}, null_at(2)},
{5},
{6, 7}, // <--- Will be set to NULL. Unsanitized row.
{8, 9, 10}},
no_nulls()}
.release();
EXPECT_FALSE(cudf::may_have_nonempty_nulls(*input));
EXPECT_FALSE(cudf::has_nonempty_nulls(*input));

// Set nullmask, post construction.
cudf::detail::set_null_mask(
input->mutable_view().null_mask(), 2, 3, false, cudf::get_default_stream());
input->set_null_count(1);
EXPECT_TRUE(cudf::may_have_nonempty_nulls(*input));
EXPECT_TRUE(cudf::has_nonempty_nulls(*input));
}

TEST_F(HasNonEmptyNullsTest, SlicedInputTest)
{
auto const input = cudf::test::strings_column_wrapper{
{"" /*NULL*/, "111", "222", "333", "444", "" /*NULL*/, "", "777", "888", "" /*NULL*/, "101010"},
cudf::test::iterators::nulls_at({0, 5, 9})};

// Split into 2 columns from rows [0, 2) and [2, 10).
auto const result = cudf::split(input, {2});
for (auto const& col : result) {
EXPECT_TRUE(cudf::may_have_nonempty_nulls(col));
EXPECT_FALSE(cudf::has_nonempty_nulls(col));
}
}

struct PurgeNonEmptyNullsTest : public cudf::test::BaseFixture {
/// Helper to run gather() on a single column, and extract the single column from the result.
std::unique_ptr<cudf::column> gather(cudf::column_view const& input,
Expand Down Expand Up @@ -69,15 +104,11 @@ TEST_F(PurgeNonEmptyNullsTest, SingleLevelList)
{8, 9, 10}},
no_nulls()}
.release();
EXPECT_FALSE(cudf::may_have_nonempty_nulls(*input));
EXPECT_FALSE(cudf::has_nonempty_nulls(*input));

// Set nullmask, post construction.
cudf::detail::set_null_mask(
input->mutable_view().null_mask(), 2, 3, false, cudf::get_default_stream());
input->set_null_count(1);
EXPECT_TRUE(cudf::may_have_nonempty_nulls(*input));
EXPECT_TRUE(cudf::has_nonempty_nulls(*input));

test_purge(*input);

Expand Down