From 8bfae4ee6bb0edd383a494e9351f2c0d04309b52 Mon Sep 17 00:00:00 2001 From: David Wendt Date: Thu, 27 Jun 2024 18:47:05 -0400 Subject: [PATCH] Allow only scale=0 fixed-point values in fixed_width_column_wrapper --- cpp/include/cudf_test/column_wrapper.hpp | 3 + cpp/tests/io/orc_test.cpp | 55 +++++-------------- cpp/tests/io/parquet_v2_test.cpp | 34 ++++-------- .../reshape/interleave_columns_tests.cpp | 17 +++--- cpp/tests/streams/io/csv_test.cpp | 24 +++----- cpp/tests/streams/io/orc_test.cpp | 20 ++----- cpp/tests/streams/io/parquet_test.cpp | 18 ++---- 7 files changed, 52 insertions(+), 119 deletions(-) diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp index 7363f965af8..2abd6f0abac 100644 --- a/cpp/include/cudf_test/column_wrapper.hpp +++ b/cpp/include/cudf_test/column_wrapper.hpp @@ -226,6 +226,9 @@ rmm::device_buffer make_elements(InputIterator begin, InputIterator end) using namespace numeric; using RepType = typename ElementTo::rep; + CUDF_EXPECTS(std::all_of(begin, end, [](ElementFrom v) { return v.scale() == 0; }), + "Only zero-scale fixed-point values are supported"); + auto to_rep = [](ElementTo fp) { return fp.value(); }; auto transformer_begin = thrust::make_transform_iterator(begin, to_rep); auto const size = cudf::distance(begin, end); diff --git a/cpp/tests/io/orc_test.cpp b/cpp/tests/io/orc_test.cpp index b5e080f3cc5..39ba62952b4 100644 --- a/cpp/tests/io/orc_test.cpp +++ b/cpp/tests/io/orc_test.cpp @@ -54,9 +54,9 @@ using int32_col = column_wrapper; using int64_col = column_wrapper; using float32_col = column_wrapper; using float64_col = column_wrapper; -using dec32_col = column_wrapper; -using dec64_col = column_wrapper; -using dec128_col = column_wrapper; +using dec32_col = cudf::test::fixed_point_column_wrapper; +using dec64_col = cudf::test::fixed_point_column_wrapper; +using dec128_col = cudf::test::fixed_point_column_wrapper; using struct_col = cudf::test::structs_column_wrapper; template using list_col = cudf::test::lists_column_wrapper; @@ -355,12 +355,6 @@ TEST_F(OrcWriterTest, MultiColumn) auto col4_data = random_values(num_rows); auto col5_data = random_values(num_rows); auto col6_vals = random_values(num_rows); - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{col6_vals[i], numeric::scale_type{12}}; - }); - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{col6_vals[i], numeric::scale_type{-12}}; - }); bool_col col0(col0_data.begin(), col0_data.end()); int8_col col1(col1_data.begin(), col1_data.end()); @@ -368,8 +362,8 @@ TEST_F(OrcWriterTest, MultiColumn) int32_col col3(col3_data.begin(), col3_data.end()); float32_col col4(col4_data.begin(), col4_data.end()); float64_col col5(col5_data.begin(), col5_data.end()); - dec128_col col6(col6_data, col6_data + num_rows); - dec128_col col7(col7_data, col7_data + num_rows); + dec128_col col6{col6_vals.begin(), col6_vals.end(), numeric::scale_type{12}}; + dec128_col col7{col6_vals.begin(), col6_vals.end(), numeric::scale_type{-12}}; list_col col8{ {9, 8}, {7, 6, 5}, {}, {4}, {3, 2, 1, 0}, {20, 21, 22, 23, 24}, {}, {66, 666}, {}, {-1, -2}}; @@ -416,9 +410,6 @@ TEST_F(OrcWriterTest, MultiColumnWithNulls) auto col4_data = random_values(num_rows); auto col5_data = random_values(num_rows); auto col6_vals = random_values(num_rows); - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal64{col6_vals[i], numeric::scale_type{2}}; - }); auto col0_mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return (i % 2); }); auto col1_mask = @@ -438,7 +429,7 @@ TEST_F(OrcWriterTest, MultiColumnWithNulls) int32_col col3{col3_data.begin(), col3_data.end(), col3_mask}; float32_col col4{col4_data.begin(), col4_data.end(), col4_mask}; float64_col col5{col5_data.begin(), col5_data.end(), col5_mask}; - dec64_col col6{col6_data, col6_data + num_rows, col6_mask}; + dec64_col col6{col6_vals.begin(), col6_vals.end(), col6_mask, numeric::scale_type{2}}; list_col col7{ {{9, 8}, {7, 6, 5}, {}, {4}, {3, 2, 1, 0}, {20, 21, 22, 23, 24}, {}, {66, 666}, {}, {-1, -2}}, col0_mask}; @@ -541,14 +532,11 @@ TEST_F(OrcWriterTest, SlicedTable) auto seq_col0 = random_values(num_rows); auto seq_col2 = random_values(num_rows); auto vals_col3 = random_values(num_rows); - auto seq_col3 = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal64{vals_col3[i], numeric::scale_type{2}}; - }); int32_col col0(seq_col0.begin(), seq_col0.end()); str_col col1(strings.begin(), strings.end()); float32_col col2(seq_col2.begin(), seq_col2.end()); - dec64_col col3(seq_col3, seq_col3 + num_rows); + dec64_col col3{vals_col3.begin(), vals_col3.end(), numeric::scale_type{2}}; list_col col4{ {9, 8}, {7, 6, 5}, {}, {4}, {3, 2, 1, 0}, {20, 21, 22, 23, 24}, {}, {66, 666}}; @@ -1213,11 +1201,8 @@ TEST_P(OrcWriterTestDecimal, Decimal64) // Using int16_t because scale causes values to overflow if they already require 32 bits auto const vals = random_values(num_rows); - auto data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal64{vals[i], numeric::scale_type{scale}}; - }); auto mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 7 == 0; }); - dec64_col col{data, data + num_rows, mask}; + dec64_col col{vals.begin(), vals.end(), mask, numeric::scale_type{scale}}; cudf::table_view tbl({static_cast(col)}); auto filepath = temp_env->get_temp_filepath("Decimal64.orc"); @@ -1244,11 +1229,8 @@ TEST_F(OrcWriterTest, Decimal32) // Using int16_t because scale causes values to overflow if they already require 32 bits auto const vals = random_values(num_rows); - auto data = cudf::detail::make_counting_transform_iterator(0, [&vals](auto i) { - return numeric::decimal32{vals[i], numeric::scale_type{2}}; - }); auto mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 13; }); - dec32_col col{data, data + num_rows, mask}; + dec32_col col{vals.begin(), vals.end(), mask, numeric::scale_type{2}}; cudf::table_view expected({col}); auto filepath = temp_env->get_temp_filepath("Decimal32.orc"); @@ -1527,12 +1509,9 @@ TEST_F(OrcReaderTest, DecimalOptions) { constexpr auto num_rows = 10; auto col_vals = random_values(num_rows); - auto col_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{col_vals[i], numeric::scale_type{2}}; - }); auto mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 3 == 0; }); - dec128_col col{col_data, col_data + num_rows, mask}; + dec128_col col{col_vals.begin(), col_vals.end(), mask, numeric::scale_type{2}}; table_view expected({col}); cudf::io::table_input_metadata expected_metadata(expected); @@ -1555,15 +1534,9 @@ TEST_F(OrcWriterTest, DecimalOptionsNested) { auto const num_rows = 100; - auto dec_vals = random_values(num_rows); - auto dec1_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal64{dec_vals[i], numeric::scale_type{2}}; - }); - auto dec2_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{dec_vals[i], numeric::scale_type{2}}; - }); - dec64_col dec1_col(dec1_data, dec1_data + num_rows); - dec128_col dec2_col(dec2_data, dec2_data + num_rows); + auto dec_vals = random_values(num_rows); + dec64_col dec1_col{dec_vals.begin(), dec_vals.end(), numeric::scale_type{2}}; + dec128_col dec2_col{dec_vals.begin(), dec_vals.end(), numeric::scale_type{2}}; auto child_struct_col = cudf::test::structs_column_wrapper{dec1_col, dec2_col}; auto int_vals = random_values(num_rows); @@ -1974,7 +1947,7 @@ TEST_F(OrcStatisticsTest, Empty) int32_col col0{}; float64_col col1{}; str_col col2{}; - dec64_col col3{}; + dec64_col col3{{}, numeric::scale_type{0}}; column_wrapper col4; bool_col col5{}; table_view expected({col0, col1, col2, col3, col4, col5}); diff --git a/cpp/tests/io/parquet_v2_test.cpp b/cpp/tests/io/parquet_v2_test.cpp index f106fd5a487..9e66fc9409f 100644 --- a/cpp/tests/io/parquet_v2_test.cpp +++ b/cpp/tests/io/parquet_v2_test.cpp @@ -47,15 +47,6 @@ TEST_P(ParquetV2Test, MultiColumn) auto col6_vals = random_values(num_rows); auto col7_vals = random_values(num_rows); auto col8_vals = random_values(num_rows); - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&col6_vals](auto i) { - return numeric::decimal32{col6_vals[i], numeric::scale_type{5}}; - }); - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&col7_vals](auto i) { - return numeric::decimal64{col7_vals[i], numeric::scale_type{-5}}; - }); - auto col8_data = cudf::detail::make_counting_transform_iterator(0, [&col8_vals](auto i) { - return numeric::decimal128{col8_vals[i], numeric::scale_type{-6}}; - }); // column_wrapper col0{col0_data.begin(), col0_data.end(), no_nulls()}; column_wrapper col1{col1_data.begin(), col1_data.end(), no_nulls()}; @@ -63,9 +54,13 @@ TEST_P(ParquetV2Test, MultiColumn) column_wrapper col3{col3_data.begin(), col3_data.end(), no_nulls()}; column_wrapper col4{col4_data.begin(), col4_data.end(), no_nulls()}; column_wrapper col5{col5_data.begin(), col5_data.end(), no_nulls()}; - column_wrapper col6{col6_data, col6_data + num_rows, no_nulls()}; - column_wrapper col7{col7_data, col7_data + num_rows, no_nulls()}; - column_wrapper col8{col8_data, col8_data + num_rows, no_nulls()}; + + cudf::test::fixed_point_column_wrapper col6( + col6_vals.begin(), col6_vals.end(), no_nulls(), numeric::scale_type{5}); + cudf::test::fixed_point_column_wrapper col7( + col7_vals.begin(), col7_vals.end(), no_nulls(), numeric::scale_type{-5}); + cudf::test::fixed_point_column_wrapper col8( + col8_vals.begin(), col8_vals.end(), no_nulls(), numeric::scale_type{-6}); auto expected = table_view{{col1, col2, col3, col4, col5, col6, col7, col8}}; @@ -109,14 +104,6 @@ TEST_P(ParquetV2Test, MultiColumnWithNulls) auto col5_data = random_values(num_rows); auto col6_vals = random_values(num_rows); auto col7_vals = random_values(num_rows); - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&col6_vals](auto i) { - return numeric::decimal32{col6_vals[i], numeric::scale_type{-2}}; - }); - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&col7_vals](auto i) { - return numeric::decimal64{col7_vals[i], numeric::scale_type{-8}}; - }); - // auto col0_mask = cudf::detail::make_counting_transform_iterator( - // 0, [](auto i) { return (i % 2); }); auto col1_mask = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return (i < 10); }); auto col2_mask = no_nulls(); @@ -138,8 +125,11 @@ TEST_P(ParquetV2Test, MultiColumnWithNulls) column_wrapper col3{col3_data.begin(), col3_data.end(), col3_mask}; column_wrapper col4{col4_data.begin(), col4_data.end(), col4_mask}; column_wrapper col5{col5_data.begin(), col5_data.end(), col5_mask}; - column_wrapper col6{col6_data, col6_data + num_rows, col6_mask}; - column_wrapper col7{col7_data, col7_data + num_rows, col7_mask}; + + cudf::test::fixed_point_column_wrapper col6( + col6_vals.begin(), col6_vals.end(), col6_mask, numeric::scale_type{-2}); + cudf::test::fixed_point_column_wrapper col7( + col7_vals.begin(), col7_vals.end(), col7_mask, numeric::scale_type{-8}); auto expected = table_view{{/*col0, */ col1, col2, col3, col4, col5, col6, col7}}; diff --git a/cpp/tests/reshape/interleave_columns_tests.cpp b/cpp/tests/reshape/interleave_columns_tests.cpp index bc7488bbf9e..de155c35a5e 100644 --- a/cpp/tests/reshape/interleave_columns_tests.cpp +++ b/cpp/tests/reshape/interleave_columns_tests.cpp @@ -363,19 +363,16 @@ TYPED_TEST(FixedPointTestAllReps, FixedPointInterleave) { using namespace numeric; using decimalXX = TypeParam; + using RepType = typename decimalXX::rep; for (int i = 0; i > -4; --i) { - auto const ONE = decimalXX{1, scale_type{i}}; - auto const TWO = decimalXX{2, scale_type{i}}; - auto const FOUR = decimalXX{4, scale_type{i}}; - auto const FIVE = decimalXX{5, scale_type{i}}; + auto const a = cudf::test::fixed_point_column_wrapper({1, 4}, scale_type{i}); + auto const b = cudf::test::fixed_point_column_wrapper({2, 5}, scale_type{i}); - auto const a = cudf::test::fixed_width_column_wrapper({ONE, FOUR}); - auto const b = cudf::test::fixed_width_column_wrapper({TWO, FIVE}); - - auto const input = cudf::table_view{std::vector{a, b}}; - auto const expected = cudf::test::fixed_width_column_wrapper({ONE, TWO, FOUR, FIVE}); - auto const actual = cudf::interleave_columns(input); + auto const input = cudf::table_view{std::vector{a, b}}; + auto const expected = + cudf::test::fixed_point_column_wrapper({1, 2, 4, 5}, scale_type{i}); + auto const actual = cudf::interleave_columns(input); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, actual->view()); } diff --git a/cpp/tests/streams/io/csv_test.cpp b/cpp/tests/streams/io/csv_test.cpp index 6e27db02d56..42894a0ebcb 100644 --- a/cpp/tests/streams/io/csv_test.cpp +++ b/cpp/tests/streams/io/csv_test.cpp @@ -39,12 +39,6 @@ TEST_F(CSVTest, CSVWriter) std::vector zeros(num_rows, 0); std::vector ones(num_rows, 1); - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones[i], numeric::scale_type{12}}; - }); - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones[i], numeric::scale_type{-12}}; - }); cudf::test::fixed_width_column_wrapper col0(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col1(zeros.begin(), zeros.end()); @@ -52,8 +46,10 @@ TEST_F(CSVTest, CSVWriter) cudf::test::fixed_width_column_wrapper col3(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col4(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col5(zeros.begin(), zeros.end()); - cudf::test::fixed_width_column_wrapper col6(col6_data, col6_data + num_rows); - cudf::test::fixed_width_column_wrapper col7(col7_data, col7_data + num_rows); + cudf::test::fixed_point_column_wrapper col6( + ones.begin(), ones.end(), numeric::scale_type{12}); + cudf::test::fixed_point_column_wrapper col7( + ones.begin(), ones.end(), numeric::scale_type{-12}); std::vector col8_data(num_rows, "rapids"); cudf::test::strings_column_wrapper col8(col8_data.begin(), col8_data.end()); @@ -72,12 +68,6 @@ TEST_F(CSVTest, CSVReader) std::vector zeros(num_rows, 0); std::vector ones(num_rows, 1); - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones[i], numeric::scale_type{12}}; - }); - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones[i], numeric::scale_type{-12}}; - }); cudf::test::fixed_width_column_wrapper col0(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col1(zeros.begin(), zeros.end()); @@ -85,8 +75,10 @@ TEST_F(CSVTest, CSVReader) cudf::test::fixed_width_column_wrapper col3(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col4(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col5(zeros.begin(), zeros.end()); - cudf::test::fixed_width_column_wrapper col6(col6_data, col6_data + num_rows); - cudf::test::fixed_width_column_wrapper col7(col7_data, col7_data + num_rows); + cudf::test::fixed_point_column_wrapper col6( + ones.begin(), ones.end(), numeric::scale_type{12}); + cudf::test::fixed_point_column_wrapper col7( + ones.begin(), ones.end(), numeric::scale_type{-12}); std::vector col8_data(num_rows, "rapids"); cudf::test::strings_column_wrapper col8(col8_data.begin(), col8_data.end()); diff --git a/cpp/tests/streams/io/orc_test.cpp b/cpp/tests/streams/io/orc_test.cpp index 401c7049381..cc43bf15b5d 100644 --- a/cpp/tests/streams/io/orc_test.cpp +++ b/cpp/tests/streams/io/orc_test.cpp @@ -59,22 +59,10 @@ cudf::table construct_table() cudf::test::fixed_width_column_wrapper col3(zeros_iterator, zeros_iterator + num_rows); cudf::test::fixed_width_column_wrapper col4(zeros_iterator, zeros_iterator + num_rows); cudf::test::fixed_width_column_wrapper col5(zeros_iterator, zeros_iterator + num_rows); - - cudf::test::fixed_width_column_wrapper col6 = [&ones_iterator] { - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones_iterator[i], numeric::scale_type{12}}; - }); - return cudf::test::fixed_width_column_wrapper(col6_data, - col6_data + num_rows); - }(); - - cudf::test::fixed_width_column_wrapper col7 = [&ones_iterator] { - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones_iterator[i], numeric::scale_type{-12}}; - }); - return cudf::test::fixed_width_column_wrapper(col7_data, - col7_data + num_rows); - }(); + cudf::test::fixed_point_column_wrapper col6( + ones_iterator, ones_iterator + num_rows, numeric::scale_type{12}); + cudf::test::fixed_point_column_wrapper col7( + ones_iterator, ones_iterator + num_rows, numeric::scale_type{-12}); cudf::test::lists_column_wrapper col8 = [] { auto col8_mask = diff --git a/cpp/tests/streams/io/parquet_test.cpp b/cpp/tests/streams/io/parquet_test.cpp index b277d184e3a..9d2dec2d697 100644 --- a/cpp/tests/streams/io/parquet_test.cpp +++ b/cpp/tests/streams/io/parquet_test.cpp @@ -55,20 +55,10 @@ cudf::table construct_table() cudf::test::fixed_width_column_wrapper col3(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col4(zeros.begin(), zeros.end()); cudf::test::fixed_width_column_wrapper col5(zeros.begin(), zeros.end()); - cudf::test::fixed_width_column_wrapper col6 = [&ones] { - auto col6_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones[i], numeric::scale_type{12}}; - }); - return cudf::test::fixed_width_column_wrapper(col6_data, - col6_data + num_rows); - }(); - cudf::test::fixed_width_column_wrapper col7 = [&ones] { - auto col7_data = cudf::detail::make_counting_transform_iterator(0, [&](auto i) { - return numeric::decimal128{ones[i], numeric::scale_type{-12}}; - }); - return cudf::test::fixed_width_column_wrapper(col7_data, - col7_data + num_rows); - }(); + cudf::test::fixed_point_column_wrapper col6( + ones.begin(), ones.end(), numeric::scale_type{12}); + cudf::test::fixed_point_column_wrapper col7( + ones.begin(), ones.end(), numeric::scale_type{-12}); cudf::test::lists_column_wrapper col8{ {1, 1}, {1, 1, 1}, {}, {1}, {1, 1, 1, 1}, {1, 1, 1, 1, 1}, {}, {1, -1}, {}, {-1, -1}};