Approximate overflow detection in ORC statistics

cpp/src/io/statistics/typed_statistics_chunk.cuh

@@ -237,14 +237,23 @@ template <typename T, bool include_aggregate>
 __inline__ __device__ statistics_chunk
 get_untyped_chunk(const typed_statistics_chunk<T, include_aggregate>& chunk)
 {
+  using E = typename detail::extrema_type<T>::type;
   statistics_chunk stat;
   stat.non_nulls  = chunk.non_nulls;
   stat.null_count = chunk.num_rows - chunk.non_nulls;
   stat.has_minmax = chunk.has_minmax;
-  stat.has_sum =
-    chunk.has_minmax;  // If a valid input was encountered we assume that the sum is valid
+  stat.has_sum    = [&]() {
+    if (!chunk.has_minmax) return false;
+    // invalidate the sum if overlow or underflow is possible
+    if constexpr (std::is_floating_point_v<E> or std::is_integral_v<E>) {
+      return std::numeric_limits<E>::max() / chunk.non_nulls >=
+               static_cast<E>(chunk.maximum_value) and
+             std::numeric_limits<E>::lowest() / chunk.non_nulls <=
+               static_cast<E>(chunk.minimum_value);
+    }
+    return true;
+  }();
   if (chunk.has_minmax) {
-    using E = typename detail::extrema_type<T>::type;
     if constexpr (std::is_floating_point_v<E>) {
       union_member::get<E>(stat.min_value) =
         (chunk.minimum_value != 0.0) ? chunk.minimum_value : CUDART_NEG_ZERO;

cpp/tests/io/orc_test.cpp

@@ -1197,4 +1197,46 @@ TEST_F(OrcWriterTest, Decimal32)
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(col64, result.tbl->view().column(0));
 }
 
+TEST_F(OrcStatisticsTest, Overflow)
+{
+  int num_rows       = 10;
+  auto too_large_seq = cudf::detail::make_counting_transform_iterator(
+    0, [](auto i) { return i * (std::numeric_limits<int64_t>::max() / 20); });
+  auto too_small_seq = cudf::detail::make_counting_transform_iterator(
+    0, [](auto i) { return i * (std::numeric_limits<int64_t>::min() / 20); });
+  auto not_too_large_seq = cudf::detail::make_counting_transform_iterator(
+    0, [](auto i) { return i * (std::numeric_limits<int64_t>::max() / 200); });
+  auto not_too_small_seq = cudf::detail::make_counting_transform_iterator(
+    0, [](auto i) { return i * (std::numeric_limits<int64_t>::min() / 200); });
+  auto validity = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 2; });
+
+  column_wrapper<int64_t, typename decltype(too_large_seq)::value_type> col1(
+    too_large_seq, too_large_seq + num_rows, validity);
+  column_wrapper<int64_t, typename decltype(too_small_seq)::value_type> col2(
+    too_small_seq, too_small_seq + num_rows, validity);
+  column_wrapper<int64_t, typename decltype(not_too_large_seq)::value_type> col3(
+    not_too_large_seq, not_too_large_seq + num_rows, validity);
+  column_wrapper<int64_t, typename decltype(not_too_small_seq)::value_type> col4(
+    not_too_small_seq, not_too_small_seq + num_rows, validity);
+  table_view tbl({col1, col2, col3, col4});
+
+  auto filepath = temp_env->get_temp_filepath("OrcStatsMerge.orc");
+
+  cudf_io::orc_writer_options out_opts =
+    cudf_io::orc_writer_options::builder(cudf_io::sink_info{filepath}, tbl);
+  cudf_io::write_orc(out_opts);
+
+  auto const stats = cudf_io::read_parsed_orc_statistics(cudf_io::source_info{filepath});
+
+  auto check_sum_exist = [&](int idx, bool expected) {
+    auto const& s  = stats.file_stats[idx];
+    auto const& ts = std::get<cudf_io::integer_statistics>(s.type_specific_stats);
+    EXPECT_EQ(ts.sum.has_value(), expected);
+  };
+  check_sum_exist(1, false);
+  check_sum_exist(2, false);
+  check_sum_exist(3, true);
+  check_sum_exist(4, true);
+}
+
 CUDF_TEST_PROGRAM_MAIN()