From 241825a7b4db1713f44d8c298e08364b1eea9a32 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Mon, 11 Mar 2024 22:54:33 +0000 Subject: [PATCH] Implement grouped product scan (#15254) Although cumulative products are implemented for whole-frame scans, they were not for grouped aggregations. Plumb through the necessary machinery to enable this. Only enabled for floating and integral types: the units make no sense for durations. As for the whole-frame product aggregation, it is very easy to overflow the output type. For floating types this will result in `+/- inf` as the result. For signed integral types, behaviour is undefined on overflow. - Closes #15253 Authors: - Lawrence Mitchell (https://github.com/wence-) - Bradley Dice (https://github.com/bdice) Approvers: - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/15254 --- cpp/CMakeLists.txt | 1 + .../cudf/detail/aggregation/aggregation.hpp | 1 + cpp/src/aggregation/aggregation.cpp | 4 +- cpp/src/groupby/sort/group_product_scan.cu | 41 +++++ cpp/src/groupby/sort/group_scan.hpp | 19 ++- cpp/src/groupby/sort/group_scan_util.cuh | 2 + cpp/src/groupby/sort/scan.cpp | 12 ++ cpp/tests/CMakeLists.txt | 1 + cpp/tests/groupby/product_scan_tests.cpp | 142 ++++++++++++++++++ python/cudf/cudf/_lib/aggregation.pyx | 1 + python/cudf/cudf/_lib/groupby.pyx | 2 +- python/cudf/cudf/tests/test_groupby.py | 4 +- 12 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 cpp/src/groupby/sort/group_product_scan.cu create mode 100644 cpp/tests/groupby/product_scan_tests.cpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 5ccc2e76101..4f64c094ead 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -339,6 +339,7 @@ add_library( src/groupby/sort/group_count_scan.cu src/groupby/sort/group_max_scan.cu src/groupby/sort/group_min_scan.cu + src/groupby/sort/group_product_scan.cu src/groupby/sort/group_rank_scan.cu src/groupby/sort/group_replace_nulls.cu src/groupby/sort/group_sum_scan.cu diff --git a/cpp/include/cudf/detail/aggregation/aggregation.hpp b/cpp/include/cudf/detail/aggregation/aggregation.hpp index a8f164646a5..87c0f8ec7f1 100644 --- a/cpp/include/cudf/detail/aggregation/aggregation.hpp +++ b/cpp/include/cudf/detail/aggregation/aggregation.hpp @@ -170,6 +170,7 @@ class sum_aggregation final : public rolling_aggregation, * @brief Derived class for specifying a product aggregation */ class product_aggregation final : public groupby_aggregation, + public groupby_scan_aggregation, public reduce_aggregation, public scan_aggregation, public segmented_reduce_aggregation { diff --git a/cpp/src/aggregation/aggregation.cpp b/cpp/src/aggregation/aggregation.cpp index b3f2a774a60..adee9147740 100644 --- a/cpp/src/aggregation/aggregation.cpp +++ b/cpp/src/aggregation/aggregation.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2023, NVIDIA CORPORATION. + * Copyright (c) 2019-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -429,6 +429,8 @@ std::unique_ptr make_product_aggregation() } template std::unique_ptr make_product_aggregation(); template std::unique_ptr make_product_aggregation(); +template std::unique_ptr +make_product_aggregation(); template std::unique_ptr make_product_aggregation(); template std::unique_ptr make_product_aggregation(); template std::unique_ptr diff --git a/cpp/src/groupby/sort/group_product_scan.cu b/cpp/src/groupby/sort/group_product_scan.cu new file mode 100644 index 00000000000..e1a615730dd --- /dev/null +++ b/cpp/src/groupby/sort/group_product_scan.cu @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "groupby/sort/group_scan_util.cuh" + +#include + +namespace cudf { +namespace groupby { +namespace detail { +std::unique_ptr product_scan(column_view const& values, + size_type num_groups, + cudf::device_span group_labels, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + return type_dispatcher(values.type(), + group_scan_dispatcher{}, + values, + num_groups, + group_labels, + stream, + mr); +} + +} // namespace detail +} // namespace groupby +} // namespace cudf diff --git a/cpp/src/groupby/sort/group_scan.hpp b/cpp/src/groupby/sort/group_scan.hpp index dc0eb691748..fd53046f7e2 100644 --- a/cpp/src/groupby/sort/group_scan.hpp +++ b/cpp/src/groupby/sort/group_scan.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,23 @@ std::unique_ptr sum_scan(column_view const& values, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr); +/** + * @brief Internal API to calculate groupwise cumulative product + * + * Behaviour is undefined for signed integral types if any groupwise product overflows the type. + * + * @param values Grouped values to get product of + * @param num_groups Number of groups + * @param group_labels ID of group that the corresponding value belongs to + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + */ +std::unique_ptr product_scan(column_view const& values, + size_type num_groups, + device_span group_labels, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + /** * @brief Internal API to calculate groupwise cumulative minimum value * diff --git a/cpp/src/groupby/sort/group_scan_util.cuh b/cpp/src/groupby/sort/group_scan_util.cuh index 1cfbf400062..2ebc8ba7d5d 100644 --- a/cpp/src/groupby/sort/group_scan_util.cuh +++ b/cpp/src/groupby/sort/group_scan_util.cuh @@ -74,6 +74,8 @@ static constexpr bool is_group_scan_supported() { if (K == aggregation::SUM) return cudf::is_numeric() || cudf::is_duration() || cudf::is_fixed_point(); + else if (K == aggregation::PRODUCT) + return cudf::is_numeric(); else if (K == aggregation::MIN or K == aggregation::MAX) return not cudf::is_dictionary() and (is_relationally_comparable() or std::is_same_v); diff --git a/cpp/src/groupby/sort/scan.cpp b/cpp/src/groupby/sort/scan.cpp index ae183474810..45c232aa3aa 100644 --- a/cpp/src/groupby/sort/scan.cpp +++ b/cpp/src/groupby/sort/scan.cpp @@ -85,6 +85,18 @@ void scan_result_functor::operator()(aggregation const& agg) get_grouped_values(), helper.num_groups(stream), helper.group_labels(stream), stream, mr)); } +template <> +void scan_result_functor::operator()(aggregation const& agg) +{ + if (cache.has_result(values, agg)) return; + + cache.add_result( + values, + agg, + detail::product_scan( + get_grouped_values(), helper.num_groups(stream), helper.group_labels(stream), stream, mr)); +} + template <> void scan_result_functor::operator()(aggregation const& agg) { diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 0eaa87f0ece..9dbf278c71d 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -132,6 +132,7 @@ ConfigureTest( groupby/min_scan_tests.cpp groupby/nth_element_tests.cpp groupby/nunique_tests.cpp + groupby/product_scan_tests.cpp groupby/product_tests.cpp groupby/quantile_tests.cpp groupby/rank_scan_tests.cpp diff --git a/cpp/tests/groupby/product_scan_tests.cpp b/cpp/tests/groupby/product_scan_tests.cpp new file mode 100644 index 00000000000..6010abd8a20 --- /dev/null +++ b/cpp/tests/groupby/product_scan_tests.cpp @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2024, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +#include + +using key_wrapper = cudf::test::fixed_width_column_wrapper; + +template +struct groupby_product_scan_test : public cudf::test::BaseFixture { + using V = T; + using R = cudf::detail::target_type_t; + using value_wrapper = cudf::test::fixed_width_column_wrapper; + using result_wrapper = cudf::test::fixed_width_column_wrapper; +}; + +using supported_types = + cudf::test::Concat>; + +TYPED_TEST_SUITE(groupby_product_scan_test, supported_types); + +TYPED_TEST(groupby_product_scan_test, basic) +{ + using value_wrapper = typename TestFixture::value_wrapper; + using result_wrapper = typename TestFixture::result_wrapper; + + // clang-format off + key_wrapper keys {1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + value_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + + key_wrapper expect_keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + result_wrapper expect_vals{0, 0, 0, 1, 4, 20, 180, 2, 14, 112}; + // clang-format on + auto agg = cudf::make_product_aggregation(); + test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); +} + +TYPED_TEST(groupby_product_scan_test, pre_sorted) +{ + using value_wrapper = typename TestFixture::value_wrapper; + using result_wrapper = typename TestFixture::result_wrapper; + + // clang-format off + key_wrapper keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; + value_wrapper vals{0, 3, 6, 1, 4, 5, 9, 2, 7, 8}; + + key_wrapper expect_keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; + result_wrapper expect_vals{0, 0, 0, 1, 4, 20, 180, 2, 14, 112}; + // clang-format on + + auto agg = cudf::make_product_aggregation(); + test_single_scan(keys, + vals, + expect_keys, + expect_vals, + std::move(agg), + cudf::null_policy::EXCLUDE, + cudf::sorted::YES); +} + +TYPED_TEST(groupby_product_scan_test, empty_cols) +{ + using value_wrapper = typename TestFixture::value_wrapper; + using result_wrapper = typename TestFixture::result_wrapper; + + key_wrapper keys{}; + value_wrapper vals{}; + + key_wrapper expect_keys{}; + result_wrapper expect_vals{}; + + auto agg = cudf::make_product_aggregation(); + test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); +} + +TYPED_TEST(groupby_product_scan_test, zero_valid_keys) +{ + using value_wrapper = typename TestFixture::value_wrapper; + using result_wrapper = typename TestFixture::result_wrapper; + + key_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); + value_wrapper vals{3, 4, 5}; + key_wrapper expect_keys{}; + result_wrapper expect_vals{}; + + auto agg = cudf::make_product_aggregation(); + test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); +} + +TYPED_TEST(groupby_product_scan_test, zero_valid_values) +{ + using value_wrapper = typename TestFixture::value_wrapper; + using result_wrapper = typename TestFixture::result_wrapper; + + key_wrapper keys{1, 1, 1}; + value_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); + key_wrapper expect_keys{1, 1, 1}; + result_wrapper expect_vals({3, 4, 5}, cudf::test::iterators::all_nulls()); + + auto agg = cudf::make_product_aggregation(); + test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); +} + +TYPED_TEST(groupby_product_scan_test, null_keys_and_values) +{ + using value_wrapper = typename TestFixture::value_wrapper; + using result_wrapper = typename TestFixture::result_wrapper; + + // clang-format off + key_wrapper keys( {1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + value_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + + // { 1, 1, 1, 2, 2, 2, 2, 3, *, 3, 4}; + key_wrapper expect_keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, cudf::test::iterators::no_nulls()); + // { -, 3, 6, 1, 4, -, 9, 2, _, 8, -} + result_wrapper expect_vals({-1, 3, 18, 1, 4, -1, 36, 2, 16, -1}, + { 0, 1, 1, 1, 1, 0, 1, 1, 1, 0}); + // clang-format on + + auto agg = cudf::make_product_aggregation(); + test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); +} diff --git a/python/cudf/cudf/_lib/aggregation.pyx b/python/cudf/cudf/_lib/aggregation.pyx index de3cbb07c37..11f801ba772 100644 --- a/python/cudf/cudf/_lib/aggregation.pyx +++ b/python/cudf/cudf/_lib/aggregation.pyx @@ -150,6 +150,7 @@ class Aggregation: cumsum = sum cummin = min cummax = max + cumprod = product @classmethod def rank(cls, method, ascending, na_option, pct): diff --git a/python/cudf/cudf/_lib/groupby.pyx b/python/cudf/cudf/_lib/groupby.pyx index 05300a41009..d5e97439180 100644 --- a/python/cudf/cudf/_lib/groupby.pyx +++ b/python/cudf/cudf/_lib/groupby.pyx @@ -216,7 +216,7 @@ cdef class GroupBy: return columns_from_pylibcudf_table(replaced) -_GROUPBY_SCANS = {"cumcount", "cumsum", "cummin", "cummax", "rank"} +_GROUPBY_SCANS = {"cumcount", "cumsum", "cummin", "cummax", "cumprod", "rank"} def _is_all_scan_aggregate(all_aggs): diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index f856bbedca2..bc2aaab1286 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -2319,7 +2319,9 @@ def test_groupby_unique(by, data, dtype): @pytest.mark.parametrize("nelem", [2, 3, 100, 1000]) -@pytest.mark.parametrize("func", ["cummin", "cummax", "cumcount", "cumsum"]) +@pytest.mark.parametrize( + "func", ["cummin", "cummax", "cumcount", "cumsum", "cumprod"] +) def test_groupby_2keys_scan(nelem, func): pdf = make_frame(pd.DataFrame, nelem=nelem) expect_df = pdf.groupby(["x", "y"], sort=True).agg(func)