Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow only scale=0 fixed-point values in fixed_width_column_wrapper #16120

Merged
merged 1 commit into from
Jul 11, 2024
Merged

Allow only scale=0 fixed-point values in fixed_width_column_wrapper #16120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cpp/include/cudf_test/column_wrapper.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,9 @@ rmm::device_buffer make_elements(InputIterator begin, InputIterator end)
using namespace numeric;
using RepType = typename ElementTo::rep;

CUDF_EXPECTS(std::all_of(begin, end, [](ElementFrom v) { return v.scale() == 0; }),
"Only zero-scale fixed-point values are supported");

auto to_rep = [](ElementTo fp) { return fp.value(); };
auto transformer_begin = thrust::make_transform_iterator(begin, to_rep);
auto const size = cudf::distance(begin, end);
Expand Down
55 changes: 14 additions & 41 deletions cpp/tests/io/orc_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ using int32_col = column_wrapper<int32_t>;
using int64_col = column_wrapper<int64_t>;
using float32_col = column_wrapper<float>;
using float64_col = column_wrapper<double>;
using dec32_col = column_wrapper<numeric::decimal32>;
using dec64_col = column_wrapper<numeric::decimal64>;
using dec128_col = column_wrapper<numeric::decimal128>;
using dec32_col = cudf::test::fixed_point_column_wrapper<numeric::decimal32::rep>;
using dec64_col = cudf::test::fixed_point_column_wrapper<numeric::decimal64::rep>;
using dec128_col = cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep>;
using struct_col = cudf::test::structs_column_wrapper;
template <typename T>
using list_col = cudf::test::lists_column_wrapper<T>;
Expand Down Expand Up @@ -355,21 +355,15 @@ TEST_F(OrcWriterTest, MultiColumn)
auto col4_data = random_values<float>(num_rows);
auto col5_data = random_values<double>(num_rows);
auto col6_vals = random_values<int64_t>(num_rows);
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{col6_vals[i], numeric::scale_type{12}};
});
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{col6_vals[i], numeric::scale_type{-12}};
});

bool_col col0(col0_data.begin(), col0_data.end());
int8_col col1(col1_data.begin(), col1_data.end());
int16_col col2(col2_data.begin(), col2_data.end());
int32_col col3(col3_data.begin(), col3_data.end());
float32_col col4(col4_data.begin(), col4_data.end());
float64_col col5(col5_data.begin(), col5_data.end());
dec128_col col6(col6_data, col6_data + num_rows);
dec128_col col7(col7_data, col7_data + num_rows);
dec128_col col6{col6_vals.begin(), col6_vals.end(), numeric::scale_type{12}};
dec128_col col7{col6_vals.begin(), col6_vals.end(), numeric::scale_type{-12}};

list_col<int64_t> col8{
{9, 8}, {7, 6, 5}, {}, {4}, {3, 2, 1, 0}, {20, 21, 22, 23, 24}, {}, {66, 666}, {}, {-1, -2}};
Expand Down Expand Up @@ -416,9 +410,6 @@ TEST_F(OrcWriterTest, MultiColumnWithNulls)
auto col4_data = random_values<float>(num_rows);
auto col5_data = random_values<double>(num_rows);
auto col6_vals = random_values<int32_t>(num_rows);
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal64{col6_vals[i], numeric::scale_type{2}};
});
auto col0_mask =
cudf::detail::make_counting_transform_iterator(0, [](auto i) { return (i % 2); });
auto col1_mask =
Expand All @@ -438,7 +429,7 @@ TEST_F(OrcWriterTest, MultiColumnWithNulls)
int32_col col3{col3_data.begin(), col3_data.end(), col3_mask};
float32_col col4{col4_data.begin(), col4_data.end(), col4_mask};
float64_col col5{col5_data.begin(), col5_data.end(), col5_mask};
dec64_col col6{col6_data, col6_data + num_rows, col6_mask};
dec64_col col6{col6_vals.begin(), col6_vals.end(), col6_mask, numeric::scale_type{2}};
list_col<int32_t> col7{
{{9, 8}, {7, 6, 5}, {}, {4}, {3, 2, 1, 0}, {20, 21, 22, 23, 24}, {}, {66, 666}, {}, {-1, -2}},
col0_mask};
Expand Down Expand Up @@ -541,14 +532,11 @@ TEST_F(OrcWriterTest, SlicedTable)
auto seq_col0 = random_values<int32_t>(num_rows);
auto seq_col2 = random_values<float>(num_rows);
auto vals_col3 = random_values<int32_t>(num_rows);
auto seq_col3 = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal64{vals_col3[i], numeric::scale_type{2}};
});

int32_col col0(seq_col0.begin(), seq_col0.end());
str_col col1(strings.begin(), strings.end());
float32_col col2(seq_col2.begin(), seq_col2.end());
dec64_col col3(seq_col3, seq_col3 + num_rows);
dec64_col col3{vals_col3.begin(), vals_col3.end(), numeric::scale_type{2}};

list_col<int64_t> col4{
{9, 8}, {7, 6, 5}, {}, {4}, {3, 2, 1, 0}, {20, 21, 22, 23, 24}, {}, {66, 666}};
Expand Down Expand Up @@ -1213,11 +1201,8 @@ TEST_P(OrcWriterTestDecimal, Decimal64)

// Using int16_t because scale causes values to overflow if they already require 32 bits
auto const vals = random_values<int32_t>(num_rows);
auto data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal64{vals[i], numeric::scale_type{scale}};
});
auto mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 7 == 0; });
dec64_col col{data, data + num_rows, mask};
dec64_col col{vals.begin(), vals.end(), mask, numeric::scale_type{scale}};
cudf::table_view tbl({static_cast<cudf::column_view>(col)});

auto filepath = temp_env->get_temp_filepath("Decimal64.orc");
Expand All @@ -1244,11 +1229,8 @@ TEST_F(OrcWriterTest, Decimal32)

// Using int16_t because scale causes values to overflow if they already require 32 bits
auto const vals = random_values<int16_t>(num_rows);
auto data = cudf::detail::make_counting_transform_iterator(0, [&vals](auto i) {
return numeric::decimal32{vals[i], numeric::scale_type{2}};
});
auto mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 13; });
dec32_col col{data, data + num_rows, mask};
dec32_col col{vals.begin(), vals.end(), mask, numeric::scale_type{2}};
cudf::table_view expected({col});

auto filepath = temp_env->get_temp_filepath("Decimal32.orc");
Expand Down Expand Up @@ -1527,12 +1509,9 @@ TEST_F(OrcReaderTest, DecimalOptions)
{
constexpr auto num_rows = 10;
auto col_vals = random_values<int64_t>(num_rows);
auto col_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{col_vals[i], numeric::scale_type{2}};
});
auto mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 3 == 0; });

dec128_col col{col_data, col_data + num_rows, mask};
dec128_col col{col_vals.begin(), col_vals.end(), mask, numeric::scale_type{2}};
table_view expected({col});

cudf::io::table_input_metadata expected_metadata(expected);
Expand All @@ -1555,15 +1534,9 @@ TEST_F(OrcWriterTest, DecimalOptionsNested)
{
auto const num_rows = 100;

auto dec_vals = random_values<int32_t>(num_rows);
auto dec1_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal64{dec_vals[i], numeric::scale_type{2}};
});
auto dec2_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{dec_vals[i], numeric::scale_type{2}};
});
dec64_col dec1_col(dec1_data, dec1_data + num_rows);
dec128_col dec2_col(dec2_data, dec2_data + num_rows);
auto dec_vals = random_values<int32_t>(num_rows);
dec64_col dec1_col{dec_vals.begin(), dec_vals.end(), numeric::scale_type{2}};
dec128_col dec2_col{dec_vals.begin(), dec_vals.end(), numeric::scale_type{2}};
auto child_struct_col = cudf::test::structs_column_wrapper{dec1_col, dec2_col};

auto int_vals = random_values<int32_t>(num_rows);
Expand Down Expand Up @@ -1974,7 +1947,7 @@ TEST_F(OrcStatisticsTest, Empty)
int32_col col0{};
float64_col col1{};
str_col col2{};
dec64_col col3{};
dec64_col col3{{}, numeric::scale_type{0}};
column_wrapper<cudf::timestamp_ns, cudf::timestamp_ns::rep> col4;
bool_col col5{};
table_view expected({col0, col1, col2, col3, col4, col5});
Expand Down
34 changes: 12 additions & 22 deletions cpp/tests/io/parquet_v2_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,25 +47,20 @@ TEST_P(ParquetV2Test, MultiColumn)
auto col6_vals = random_values<int16_t>(num_rows);
auto col7_vals = random_values<int32_t>(num_rows);
auto col8_vals = random_values<int64_t>(num_rows);
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&col6_vals](auto i) {
return numeric::decimal32{col6_vals[i], numeric::scale_type{5}};
});
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&col7_vals](auto i) {
return numeric::decimal64{col7_vals[i], numeric::scale_type{-5}};
});
auto col8_data = cudf::detail::make_counting_transform_iterator(0, [&col8_vals](auto i) {
return numeric::decimal128{col8_vals[i], numeric::scale_type{-6}};
});

// column_wrapper<bool> col0{col0_data.begin(), col0_data.end(), no_nulls()};
column_wrapper<int8_t> col1{col1_data.begin(), col1_data.end(), no_nulls()};
column_wrapper<int16_t> col2{col2_data.begin(), col2_data.end(), no_nulls()};
column_wrapper<int32_t> col3{col3_data.begin(), col3_data.end(), no_nulls()};
column_wrapper<float> col4{col4_data.begin(), col4_data.end(), no_nulls()};
column_wrapper<double> col5{col5_data.begin(), col5_data.end(), no_nulls()};
column_wrapper<numeric::decimal32> col6{col6_data, col6_data + num_rows, no_nulls()};
column_wrapper<numeric::decimal64> col7{col7_data, col7_data + num_rows, no_nulls()};
column_wrapper<numeric::decimal128> col8{col8_data, col8_data + num_rows, no_nulls()};

cudf::test::fixed_point_column_wrapper<numeric::decimal32::rep> col6(
col6_vals.begin(), col6_vals.end(), no_nulls(), numeric::scale_type{5});
cudf::test::fixed_point_column_wrapper<numeric::decimal64::rep> col7(
col7_vals.begin(), col7_vals.end(), no_nulls(), numeric::scale_type{-5});
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col8(
col8_vals.begin(), col8_vals.end(), no_nulls(), numeric::scale_type{-6});

auto expected = table_view{{col1, col2, col3, col4, col5, col6, col7, col8}};

Expand Down Expand Up @@ -109,14 +104,6 @@ TEST_P(ParquetV2Test, MultiColumnWithNulls)
auto col5_data = random_values<double>(num_rows);
auto col6_vals = random_values<int32_t>(num_rows);
auto col7_vals = random_values<int64_t>(num_rows);
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&col6_vals](auto i) {
return numeric::decimal32{col6_vals[i], numeric::scale_type{-2}};
});
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&col7_vals](auto i) {
return numeric::decimal64{col7_vals[i], numeric::scale_type{-8}};
});
// auto col0_mask = cudf::detail::make_counting_transform_iterator(
// 0, [](auto i) { return (i % 2); });
auto col1_mask =
cudf::detail::make_counting_transform_iterator(0, [](auto i) { return (i < 10); });
auto col2_mask = no_nulls();
Expand All @@ -138,8 +125,11 @@ TEST_P(ParquetV2Test, MultiColumnWithNulls)
column_wrapper<int32_t> col3{col3_data.begin(), col3_data.end(), col3_mask};
column_wrapper<float> col4{col4_data.begin(), col4_data.end(), col4_mask};
column_wrapper<double> col5{col5_data.begin(), col5_data.end(), col5_mask};
column_wrapper<numeric::decimal32> col6{col6_data, col6_data + num_rows, col6_mask};
column_wrapper<numeric::decimal64> col7{col7_data, col7_data + num_rows, col7_mask};

cudf::test::fixed_point_column_wrapper<numeric::decimal32::rep> col6(
col6_vals.begin(), col6_vals.end(), col6_mask, numeric::scale_type{-2});
cudf::test::fixed_point_column_wrapper<numeric::decimal64::rep> col7(
col7_vals.begin(), col7_vals.end(), col7_mask, numeric::scale_type{-8});

auto expected = table_view{{/*col0, */ col1, col2, col3, col4, col5, col6, col7}};

Expand Down
17 changes: 7 additions & 10 deletions cpp/tests/reshape/interleave_columns_tests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,19 +363,16 @@ TYPED_TEST(FixedPointTestAllReps, FixedPointInterleave)
{
using namespace numeric;
using decimalXX = TypeParam;
using RepType = typename decimalXX::rep;

for (int i = 0; i > -4; --i) {
auto const ONE = decimalXX{1, scale_type{i}};
auto const TWO = decimalXX{2, scale_type{i}};
auto const FOUR = decimalXX{4, scale_type{i}};
auto const FIVE = decimalXX{5, scale_type{i}};
auto const a = cudf::test::fixed_point_column_wrapper<RepType>({1, 4}, scale_type{i});
auto const b = cudf::test::fixed_point_column_wrapper<RepType>({2, 5}, scale_type{i});

auto const a = cudf::test::fixed_width_column_wrapper<decimalXX>({ONE, FOUR});
auto const b = cudf::test::fixed_width_column_wrapper<decimalXX>({TWO, FIVE});

auto const input = cudf::table_view{std::vector<cudf::column_view>{a, b}};
auto const expected = cudf::test::fixed_width_column_wrapper<decimalXX>({ONE, TWO, FOUR, FIVE});
auto const actual = cudf::interleave_columns(input);
auto const input = cudf::table_view{std::vector<cudf::column_view>{a, b}};
auto const expected =
cudf::test::fixed_point_column_wrapper<RepType>({1, 2, 4, 5}, scale_type{i});
auto const actual = cudf::interleave_columns(input);

CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, actual->view());
}
Expand Down
24 changes: 8 additions & 16 deletions cpp/tests/streams/io/csv_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,21 +39,17 @@ TEST_F(CSVTest, CSVWriter)

std::vector<size_t> zeros(num_rows, 0);
std::vector<size_t> ones(num_rows, 1);
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones[i], numeric::scale_type{12}};
});
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones[i], numeric::scale_type{-12}};
});

cudf::test::fixed_width_column_wrapper<bool> col0(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<int8_t> col1(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<int16_t> col2(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<int32_t> col3(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<float> col4(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<double> col5(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<numeric::decimal128> col6(col6_data, col6_data + num_rows);
cudf::test::fixed_width_column_wrapper<numeric::decimal128> col7(col7_data, col7_data + num_rows);
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col6(
ones.begin(), ones.end(), numeric::scale_type{12});
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col7(
ones.begin(), ones.end(), numeric::scale_type{-12});

std::vector<std::string> col8_data(num_rows, "rapids");
cudf::test::strings_column_wrapper col8(col8_data.begin(), col8_data.end());
Expand All @@ -72,21 +68,17 @@ TEST_F(CSVTest, CSVReader)

std::vector<size_t> zeros(num_rows, 0);
std::vector<size_t> ones(num_rows, 1);
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones[i], numeric::scale_type{12}};
});
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones[i], numeric::scale_type{-12}};
});

cudf::test::fixed_width_column_wrapper<bool> col0(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<int8_t> col1(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<int16_t> col2(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<int32_t> col3(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<float> col4(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<double> col5(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<numeric::decimal128> col6(col6_data, col6_data + num_rows);
cudf::test::fixed_width_column_wrapper<numeric::decimal128> col7(col7_data, col7_data + num_rows);
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col6(
ones.begin(), ones.end(), numeric::scale_type{12});
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col7(
ones.begin(), ones.end(), numeric::scale_type{-12});

std::vector<std::string> col8_data(num_rows, "rapids");
cudf::test::strings_column_wrapper col8(col8_data.begin(), col8_data.end());
Expand Down
20 changes: 4 additions & 16 deletions cpp/tests/streams/io/orc_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,22 +59,10 @@ cudf::table construct_table()
cudf::test::fixed_width_column_wrapper<int32_t> col3(zeros_iterator, zeros_iterator + num_rows);
cudf::test::fixed_width_column_wrapper<float> col4(zeros_iterator, zeros_iterator + num_rows);
cudf::test::fixed_width_column_wrapper<double> col5(zeros_iterator, zeros_iterator + num_rows);

cudf::test::fixed_width_column_wrapper<numeric::decimal128> col6 = [&ones_iterator] {
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones_iterator[i], numeric::scale_type{12}};
});
return cudf::test::fixed_width_column_wrapper<numeric::decimal128>(col6_data,
col6_data + num_rows);
}();

cudf::test::fixed_width_column_wrapper<numeric::decimal128> col7 = [&ones_iterator] {
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones_iterator[i], numeric::scale_type{-12}};
});
return cudf::test::fixed_width_column_wrapper<numeric::decimal128>(col7_data,
col7_data + num_rows);
}();
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col6(
ones_iterator, ones_iterator + num_rows, numeric::scale_type{12});
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col7(
ones_iterator, ones_iterator + num_rows, numeric::scale_type{-12});

cudf::test::lists_column_wrapper<int64_t> col8 = [] {
auto col8_mask =
Expand Down
18 changes: 4 additions & 14 deletions cpp/tests/streams/io/parquet_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,20 +55,10 @@ cudf::table construct_table()
cudf::test::fixed_width_column_wrapper<int32_t> col3(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<float> col4(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<double> col5(zeros.begin(), zeros.end());
cudf::test::fixed_width_column_wrapper<numeric::decimal128> col6 = [&ones] {
auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones[i], numeric::scale_type{12}};
});
return cudf::test::fixed_width_column_wrapper<numeric::decimal128>(col6_data,
col6_data + num_rows);
}();
cudf::test::fixed_width_column_wrapper<numeric::decimal128> col7 = [&ones] {
auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) {
return numeric::decimal128{ones[i], numeric::scale_type{-12}};
});
return cudf::test::fixed_width_column_wrapper<numeric::decimal128>(col7_data,
col7_data + num_rows);
}();
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col6(
ones.begin(), ones.end(), numeric::scale_type{12});
cudf::test::fixed_point_column_wrapper<numeric::decimal128::rep> col7(
ones.begin(), ones.end(), numeric::scale_type{-12});

cudf::test::lists_column_wrapper<int64_t> col8{
{1, 1}, {1, 1, 1}, {}, {1}, {1, 1, 1, 1}, {1, 1, 1, 1, 1}, {}, {1, -1}, {}, {-1, -1}};
Expand Down
Loading