Support null_policy::EXCLUDE for COLLECT rolling aggregation

cpp/include/cudf_test/iterator_utilities.hpp

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cudf/detail/iterator.cuh>
+#include <cudf/types.hpp>
+
+#include <thrust/iterator/transform_iterator.h>
+
+#include <set>
+
+namespace cudf {
+namespace test {
+
+/**
+ * @brief Bool iterator for marking (possibly multiple) null elements in a column_wrapper.
+ *
+ * The returned iterator yields `false` (to mark `null`) at all the specified indices,
+ * and yields `true` (to mark valid rows) for all other indices. E.g.
+ *
+ * @code
+ * auto iter = iterator_with_null_at(std::vector<size_type>{8,9});
+ * iter[6] == true;  // i.e. Valid row at index 6.
+ * iter[7] == true;  // i.e. Valid row at index 7.
+ * iter[8] == false; // i.e. Invalid row at index 8.
+ * iter[9] == false; // i.e. Invalid row at index 9.
+ * @endcode
+ *
+ * @param indices The collection of indices for which the validity iterator
+ *                must return `false` (i.e. null)
+ * @return auto Validity iterator
+ */
+static auto iterator_with_null_at(std::vector<cudf::size_type> const& indices)
+{
+  return cudf::detail::make_counting_transform_iterator(0, [indices](auto i) {
+    return std::find(indices.begin(), indices.end(), i) == indices.cend();
+  });
+}
+
+/**
+ * @brief Bool iterator for marking a single null element in a column_wrapper
+ *
+ * The returned iterator yields `false` (to mark `null`) at the specified index,
+ * and yields `true` (to mark valid rows) for all other indices. E.g.
+ *
+ * @code
+ * auto iter = iterator_with_null_at(8);
+ * iter[7] == true;  // i.e. Valid row at index 7.
+ * iter[8] == false; // i.e. Invalid row at index 8.
+ * @endcode
+ *
+ * @param index The index for which the validity iterator must return `false` (i.e. null)
+ * @return auto Validity iterator
+ */
+static auto iterator_with_null_at(cudf::size_type const& index)
+{
+  return iterator_with_null_at(std::vector<cudf::size_type>{index});
+}
+
+}  // namespace test
+}  // namespace cudf

cpp/tests/collect_list/collect_list_test.cpp

@@ -18,6 +18,7 @@
 #include <cudf_test/column_utilities.hpp>
 #include <cudf_test/column_wrapper.hpp>
 #include <cudf_test/cudf_gtest.hpp>
+#include <cudf_test/iterator_utilities.hpp>
 #include <cudf_test/type_lists.hpp>
 
 #include <cudf/aggregation.hpp>
@@ -269,7 +270,7 @@ TYPED_TEST(TypedCollectListTest, RollingWindowWithNullInputsHonoursMinPeriods)
       expected_result_child_values.begin(), expected_result_child_values.end());
     auto expected_offsets  = fixed_width_column_wrapper<size_type>{0, 0, 2, 4, 6, 8, 8}.release();
     auto expected_num_rows = expected_offsets->size() - 1;
-    auto null_mask_iter    = make_counting_transform_iterator(
+    auto null_mask_iter    = cudf::detail::make_counting_transform_iterator(
       size_type{0}, [expected_num_rows](auto i) { return i != 0 && i != (expected_num_rows - 1); });
 
     auto expected_result = make_lists_column(
@@ -330,7 +331,7 @@ TYPED_TEST(TypedCollectListTest, RollingWindowWithNullInputsHonoursMinPeriods)
 
     auto expected_offsets  = fixed_width_column_wrapper<size_type>{0, 0, 3, 5, 8, 8, 8}.release();
     auto expected_num_rows = expected_offsets->size() - 1;
-    auto null_mask_iter    = make_counting_transform_iterator(
+    auto null_mask_iter    = cudf::detail::make_counting_transform_iterator(
       size_type{0}, [expected_num_rows](auto i) { return i > 0 && i < 4; });
 
     auto expected_result = make_lists_column(
@@ -677,21 +678,20 @@ TYPED_TEST(TypedCollectListTest, GroupedTimeRangeRollingWindowWithNulls)
                                       min_periods,
                                       make_collect_aggregation());
 
-  auto null_at = [](auto null_idx) {
-    return make_counting_transform_iterator(0, [null_idx](auto i) { return i != null_idx; });
-  };
+  auto null_at_0 = iterator_with_null_at(0);
+  auto null_at_1 = iterator_with_null_at(1);
 
   // In the results, `11` and `21` should be nulls.
   auto const expected_result = lists_column_wrapper<T, int32_t>{
-    {{10, 11, 12, 13}, null_at(1)},
-    {{10, 11, 12, 13}, null_at(1)},
-    {{10, 11, 12, 13, 14}, null_at(1)},
-    {{10, 11, 12, 13, 14}, null_at(1)},
-    {{10, 11, 12, 13, 14}, null_at(1)},
-    {{20}, null_at(1)},
-    {{21, 22}, null_at(0)},
-    {{21, 22, 23}, null_at(0)},
-    {{21, 22, 23}, null_at(0)}}.release();
+    {{10, 11, 12, 13}, null_at_1},
+    {{10, 11, 12, 13}, null_at_1},
+    {{10, 11, 12, 13, 14}, null_at_1},
+    {{10, 11, 12, 13, 14}, null_at_1},
+    {{10, 11, 12, 13, 14}, null_at_1},
+    {{20}, null_at_1},
+    {{21, 22}, null_at_0},
+    {{21, 22, 23}, null_at_0},
+    {{21, 22, 23}, null_at_0}}.release();
 
   CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_result->view(), result->view());
 
@@ -793,22 +793,21 @@ TEST_F(CollectListTest, GroupedTimeRangeRollingWindowOnStringsWithNulls)
                                       min_periods,
                                       make_collect_aggregation());
 
-  auto null_at = [](auto null_idx) {
-    return make_counting_transform_iterator(0, [null_idx](auto i) { return i != null_idx; });
-  };
+  auto null_at_0 = iterator_with_null_at(0);
+  auto null_at_1 = iterator_with_null_at(1);
 
   // In the results, `11` and `21` should be nulls.
   auto const expected_result = lists_column_wrapper<cudf::string_view>{
-    {{"10", "11", "12", "13"}, null_at(1)},
-    {{"10", "11", "12", "13"}, null_at(1)},
-    {{"10", "11", "12", "13", "14"}, null_at(1)},
-    {{"10", "11", "12", "13", "14"}, null_at(1)},
-    {{"10", "11", "12", "13", "14"}, null_at(1)},
+    {{"10", "11", "12", "13"}, null_at_1},
+    {{"10", "11", "12", "13"}, null_at_1},
+    {{"10", "11", "12", "13", "14"}, null_at_1},
+    {{"10", "11", "12", "13", "14"}, null_at_1},
+    {{"10", "11", "12", "13", "14"}, null_at_1},
     {"20"},
-    {{"21", "22"}, null_at(0)},
-    {{"21", "22", "23"}, null_at(0)},
+    {{"21", "22"}, null_at_0},
+    {{"21", "22", "23"}, null_at_0},
     {{"21", "22", "23"},
-     null_at(0)}}.release();
+     null_at_0}}.release();
 
   CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_result->view(), result->view());
 
@@ -985,22 +984,20 @@ TYPED_TEST(TypedCollectListTest, GroupedTimeRangeRollingWindowWithNullsAndMinPer
                                       min_periods,
                                       make_collect_aggregation());
 
-  auto null_at = [](auto null_idx) {
-    return make_counting_transform_iterator(0, [null_idx](auto i) { return i != null_idx; });
-  };
+  auto null_at_1 = iterator_with_null_at(1);
 
   // In the results, `11` and `21` should be nulls.
   auto const expected_result = lists_column_wrapper<T, int32_t>{
-    {{{10, 11, 12, 13}, null_at(1)},
-     {{10, 11, 12, 13}, null_at(1)},
-     {{10, 11, 12, 13, 14}, null_at(1)},
-     {{10, 11, 12, 13, 14}, null_at(1)},
-     {{10, 11, 12, 13, 14}, null_at(1)},
+    {{{10, 11, 12, 13}, null_at_1},
+     {{10, 11, 12, 13}, null_at_1},
+     {{10, 11, 12, 13, 14}, null_at_1},
+     {{10, 11, 12, 13, 14}, null_at_1},
+     {{10, 11, 12, 13, 14}, null_at_1},
      {},
      {},
      {},
      {}},
-    make_counting_transform_iterator(0, [](auto i) {
+    cudf::detail::make_counting_transform_iterator(0, [](auto i) {
       return i < 5;
     })}.release();
 
@@ -1027,7 +1024,7 @@ TYPED_TEST(TypedCollectListTest, GroupedTimeRangeRollingWindowWithNullsAndMinPer
      {},
      {},
      {}},
-    make_counting_transform_iterator(
+    cudf::detail::make_counting_transform_iterator(
       0, [](auto i) { return i < 5; })}.release();
 
   CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_result_with_nulls_excluded->view(),
@@ -1113,22 +1110,20 @@ TEST_F(CollectListTest, GroupedTimeRangeRollingWindowOnStringsWithNullsAndMinPer
                                       min_periods,
                                       make_collect_aggregation());
 
-  auto null_at = [](auto null_idx) {
-    return make_counting_transform_iterator(0, [null_idx](auto i) { return i != null_idx; });
-  };
+  auto null_at_1 = iterator_with_null_at(1);
 
   // In the results, `11` and `21` should be nulls.
   auto const expected_result = lists_column_wrapper<cudf::string_view>{
-    {{{"10", "11", "12", "13"}, null_at(1)},
-     {{"10", "11", "12", "13"}, null_at(1)},
-     {{"10", "11", "12", "13", "14"}, null_at(1)},
-     {{"10", "11", "12", "13", "14"}, null_at(1)},
-     {{"10", "11", "12", "13", "14"}, null_at(1)},
+    {{{"10", "11", "12", "13"}, null_at_1},
+     {{"10", "11", "12", "13"}, null_at_1},
+     {{"10", "11", "12", "13", "14"}, null_at_1},
+     {{"10", "11", "12", "13", "14"}, null_at_1},
+     {{"10", "11", "12", "13", "14"}, null_at_1},
      {},
      {},
      {},
      {}},
-    make_counting_transform_iterator(0, [](auto i) {
+    cudf::detail::make_counting_transform_iterator(0, [](auto i) {
       return i < 5;
     })}.release();
 
@@ -1155,7 +1150,7 @@ TEST_F(CollectListTest, GroupedTimeRangeRollingWindowOnStringsWithNullsAndMinPer
      {},
      {},
      {}},
-    make_counting_transform_iterator(
+    cudf::detail::make_counting_transform_iterator(
       0, [](auto i) { return i < 5; })}.release();
 
   CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_result_with_nulls_excluded->view(),