From 37fe468faa48c4000387a3830b0170cb1b7c5f07 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Tue, 7 Feb 2023 18:11:53 -0600 Subject: [PATCH 01/69] Enable doctests for GroupBy methods (#12658) This PR enables doctests for some GroupBy methods that are not currently tested due to not meeting the inclusion criteria in our doctest class. This includes enabling tests for `GroupBy.apply` with `engine='jit'`. came up during https://github.com/rapidsai/cudf/pull/11452 Authors: - https://github.com/brandon-b-miller Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12658 --- python/cudf/cudf/core/groupby/__init__.py | 7 ++- python/cudf/cudf/core/groupby/groupby.py | 60 ++++++++++++----------- python/cudf/cudf/tests/test_doctests.py | 8 ++- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/python/cudf/cudf/core/groupby/__init__.py b/python/cudf/cudf/core/groupby/__init__.py index bb21dd1729d..4375ed3e3da 100644 --- a/python/cudf/cudf/core/groupby/__init__.py +++ b/python/cudf/cudf/core/groupby/__init__.py @@ -1,3 +1,8 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from cudf.core.groupby.groupby import GroupBy, Grouper + +__all__ = [ + "GroupBy", + "Grouper", +] diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 91e00eb43f3..04a71b77413 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -111,7 +111,7 @@ def _quantile_75(x): ... 'Max Speed': [380., 370., 24., 26.], ... }}) >>> df - Animal Max Speed + Animal Max Speed 0 Falcon 380.0 1 Falcon 370.0 2 Parrot 24.0 @@ -420,8 +420,11 @@ def agg(self, func): Examples -------- >>> import cudf - >>> a = cudf.DataFrame( - {'a': [1, 1, 2], 'b': [1, 2, 3], 'c': [2, 2, 1]}) + >>> a = cudf.DataFrame({ + ... 'a': [1, 1, 2], + ... 'b': [1, 2, 3], + ... 'c': [2, 2, 1] + ... }) >>> a.groupby('a').agg('sum') b c a @@ -430,6 +433,12 @@ def agg(self, func): Specifying a list of aggregations to perform on each column. + >>> import cudf + >>> a = cudf.DataFrame({ + ... 'a': [1, 1, 2], + ... 'b': [1, 2, 3], + ... 'c': [2, 2, 1] + ... }) >>> a.groupby('a').agg(['sum', 'min']) b c sum min sum min @@ -439,6 +448,12 @@ def agg(self, func): Using a dict to specify aggregations to perform per column. + >>> import cudf + >>> a = cudf.DataFrame({ + ... 'a': [1, 1, 2], + ... 'b': [1, 2, 3], + ... 'c': [2, 2, 1] + ... }) >>> a.groupby('a').agg({'a': 'max', 'b': ['min', 'mean']}) a b max min mean @@ -448,6 +463,12 @@ def agg(self, func): Using lambdas/callables to specify aggregations taking parameters. + >>> import cudf + >>> a = cudf.DataFrame({ + ... 'a': [1, 1, 2], + ... 'b': [1, 2, 3], + ... 'c': [2, 2, 1] + ... }) >>> f1 = lambda x: x.quantile(0.5); f1.__name__ = "q0.5" >>> f2 = lambda x: x.quantile(0.75); f2.__name__ = "q0.75" >>> a.groupby('a').agg([f1, f2]) @@ -905,6 +926,7 @@ def mult(df): .. code-block:: + >>> import pandas as pd >>> df = pd.DataFrame({ ... 'a': [1, 1, 2, 2], ... 'b': [1, 2, 1, 2], @@ -1218,10 +1240,12 @@ def describe(self, include=None, exclude=None): Examples -------- >>> import cudf - >>> gdf = cudf.DataFrame({"Speed": [380.0, 370.0, 24.0, 26.0], - "Score": [50, 30, 90, 80]}) + >>> gdf = cudf.DataFrame({ + ... "Speed": [380.0, 370.0, 24.0, 26.0], + ... "Score": [50, 30, 90, 80], + ... }) >>> gdf - Speed Score + Speed Score 0 380.0 50 1 370.0 30 2 24.0 90 @@ -1290,7 +1314,7 @@ def corr(self, method="pearson", min_periods=1): ... "val2": [4, 5, 6, 1, 2, 9, 8, 5, 1], ... "val3": [4, 5, 6, 1, 2, 9, 8, 5, 1]}) >>> gdf - id val1 val2 val3 + id val1 val2 val3 0 a 5 4 4 1 a 4 5 5 2 a 6 6 6 @@ -1652,28 +1676,6 @@ def fillna( Returns ------- DataFrame or Series - - .. pandas-compat:: - **groupby.fillna** - - This function may return result in different format to the method - Pandas supports. For example: - - .. code-block:: - - >>> df = pd.DataFrame({'k': [1, 1, 2], 'v': [2, None, 4]}) - >>> gdf = cudf.from_pandas(df) - >>> df.groupby('k').fillna({'v': 4}) # pandas - v - k - 1 0 2.0 - 1 4.0 - 2 2 4.0 - >>> gdf.groupby('k').fillna({'v': 4}) # cudf - v - 0 2.0 - 1 4.0 - 2 4.0 """ if inplace: raise NotImplementedError("Does not support inplace yet.") diff --git a/python/cudf/cudf/tests/test_doctests.py b/python/cudf/cudf/tests/test_doctests.py index dbb5c548166..0da5c6b04d6 100644 --- a/python/cudf/cudf/tests/test_doctests.py +++ b/python/cudf/cudf/tests/test_doctests.py @@ -1,8 +1,9 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import contextlib import doctest import inspect import io +import itertools import os import numpy as np @@ -12,6 +13,9 @@ pytestmark = pytest.mark.filterwarnings("ignore::FutureWarning") +# modules that will be searched for doctests +tests = [cudf, cudf.core.groupby] + def _name_in_all(parent, name): return name in getattr(parent, "__all__", []) @@ -78,7 +82,7 @@ def chdir_to_tmp_path(cls, tmp_path): @pytest.mark.parametrize( "docstring", - _find_doctests_in_obj(cudf), + itertools.chain(*[_find_doctests_in_obj(mod) for mod in tests]), ids=lambda docstring: docstring.name, ) def test_docstring(self, docstring): From b87b64f1b1bc00b36e83f05187a2eeaaf1229029 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Tue, 7 Feb 2023 20:11:07 -0500 Subject: [PATCH 02/69] Convert `rank` to use to experimental row comparators (#12481) Converts the `rank` function to use experimental row comparators, which support list and struct types. Part of #11844. [Throughput benchmarks](https://github.com/rapidsai/cudf/pull/12481#issuecomment-1416384229) are available below. It seems like when `size_bytes` is constrained, the generator generates fewer rows in `list` types for increasing depths. That's why, `depth=4` has a higher throughput than `depth=1` because the number of leaf elements generated are the same, but with much fewer rows. Authors: - Divye Gala (https://github.com/divyegala) - Jordan Jacobelli (https://github.com/Ethyling) Approvers: - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12481 --- cpp/benchmarks/CMakeLists.txt | 5 +- cpp/benchmarks/sort/nested_types_common.hpp | 85 +++ cpp/benchmarks/sort/rank.cpp | 4 +- cpp/benchmarks/sort/rank_lists.cpp | 49 ++ cpp/benchmarks/sort/rank_structs.cpp | 47 ++ cpp/benchmarks/sort/rank_types_common.hpp | 52 ++ cpp/benchmarks/sort/sort_lists.cpp | 16 +- cpp/benchmarks/sort/sort_structs.cpp | 52 +- cpp/src/sort/rank.cu | 39 +- cpp/tests/sort/rank_test.cpp | 556 ++++++++++++++++++-- 10 files changed, 776 insertions(+), 129 deletions(-) create mode 100644 cpp/benchmarks/sort/nested_types_common.hpp create mode 100644 cpp/benchmarks/sort/rank_lists.cpp create mode 100644 cpp/benchmarks/sort/rank_structs.cpp create mode 100644 cpp/benchmarks/sort/rank_types_common.hpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 6f67cb32b0a..c5ae3345da5 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -169,7 +169,10 @@ ConfigureNVBench(SEARCH_NVBENCH search/contains.cpp) # ################################################################################################## # * sort benchmark -------------------------------------------------------------------------------- ConfigureBench(SORT_BENCH sort/rank.cpp sort/sort.cpp sort/sort_strings.cpp) -ConfigureNVBench(SORT_NVBENCH sort/segmented_sort.cpp sort/sort_lists.cpp sort/sort_structs.cpp) +ConfigureNVBench( + SORT_NVBENCH sort/rank_lists.cpp sort/rank_structs.cpp sort/segmented_sort.cpp + sort/sort_lists.cpp sort/sort_structs.cpp +) # ################################################################################################## # * quantiles benchmark diff --git a/cpp/benchmarks/sort/nested_types_common.hpp b/cpp/benchmarks/sort/nested_types_common.hpp new file mode 100644 index 00000000000..c4851823534 --- /dev/null +++ b/cpp/benchmarks/sort/nested_types_common.hpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include + +#include + +#include + +inline std::unique_ptr create_lists_data(nvbench::state& state) +{ + const size_t size_bytes(state.get_int64("size_bytes")); + const cudf::size_type depth{static_cast(state.get_int64("depth"))}; + auto const null_frequency{state.get_float64("null_frequency")}; + + data_profile table_profile; + table_profile.set_distribution_params(cudf::type_id::LIST, distribution_id::UNIFORM, 0, 5); + table_profile.set_list_depth(depth); + table_profile.set_null_probability(null_frequency); + return create_random_table({cudf::type_id::LIST}, table_size_bytes{size_bytes}, table_profile); +} + +inline std::unique_ptr create_structs_data(nvbench::state& state, + cudf::size_type const n_cols = 1) +{ + using Type = int; + using column_wrapper = cudf::test::fixed_width_column_wrapper; + std::default_random_engine generator; + std::uniform_int_distribution distribution(0, 100); + + const cudf::size_type n_rows{static_cast(state.get_int64("NumRows"))}; + const cudf::size_type depth{static_cast(state.get_int64("Depth"))}; + const bool nulls{static_cast(state.get_int64("Nulls"))}; + + // Create columns with values in the range [0,100) + std::vector columns; + columns.reserve(n_cols); + std::generate_n(std::back_inserter(columns), n_cols, [&]() { + auto const elements = cudf::detail::make_counting_transform_iterator( + 0, [&](auto row) { return distribution(generator); }); + if (!nulls) return column_wrapper(elements, elements + n_rows); + auto valids = + cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 10 != 0; }); + return column_wrapper(elements, elements + n_rows, valids); + }); + + std::vector> cols; + std::transform(columns.begin(), columns.end(), std::back_inserter(cols), [](column_wrapper& col) { + return col.release(); + }); + + std::vector> child_cols = std::move(cols); + // Nest the child columns in a struct, then nest that struct column inside another + // struct column up to the desired depth + for (int i = 0; i < depth; i++) { + std::vector struct_validity; + std::uniform_int_distribution bool_distribution(0, 100 * (i + 1)); + std::generate_n( + std::back_inserter(struct_validity), n_rows, [&]() { return bool_distribution(generator); }); + cudf::test::structs_column_wrapper struct_col(std::move(child_cols), struct_validity); + child_cols = std::vector>{}; + child_cols.push_back(struct_col.release()); + } + + // Create table view + return std::make_unique(std::move(child_cols)); +} diff --git a/cpp/benchmarks/sort/rank.cpp b/cpp/benchmarks/sort/rank.cpp index 2c26f4fa15d..6d0a8e5aedd 100644 --- a/cpp/benchmarks/sort/rank.cpp +++ b/cpp/benchmarks/sort/rank.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ static void BM_rank(benchmark::State& state, bool nulls) // Create columns with values in the range [0,100) data_profile profile = data_profile_builder().cardinality(0).distribution( cudf::type_to_id(), distribution_id::UNIFORM, 0, 100); - profile.set_null_probability(nulls ? std::optional{0.01} : std::nullopt); + profile.set_null_probability(nulls ? std::optional{0.2} : std::nullopt); auto keys = create_random_column(cudf::type_to_id(), row_count{n_rows}, profile); for (auto _ : state) { diff --git a/cpp/benchmarks/sort/rank_lists.cpp b/cpp/benchmarks/sort/rank_lists.cpp new file mode 100644 index 00000000000..f467b639810 --- /dev/null +++ b/cpp/benchmarks/sort/rank_lists.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nested_types_common.hpp" +#include "rank_types_common.hpp" + +#include + +#include + +#include + +template +void nvbench_rank_lists(nvbench::state& state, nvbench::type_list>) +{ + cudf::rmm_pool_raii pool_raii; + + auto const table = create_lists_data(state); + + auto const null_frequency{state.get_float64("null_frequency")}; + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + cudf::rank(table->view().column(0), + method, + cudf::order::ASCENDING, + null_frequency ? cudf::null_policy::INCLUDE : cudf::null_policy::EXCLUDE, + cudf::null_order::AFTER, + rmm::mr::get_current_device_resource()); + }); +} + +NVBENCH_BENCH_TYPES(nvbench_rank_lists, NVBENCH_TYPE_AXES(methods)) + .set_name("rank_lists") + .add_int64_power_of_two_axis("size_bytes", {10, 18, 24, 28}) + .add_int64_axis("depth", {1, 4}) + .add_float64_axis("null_frequency", {0, 0.2}); diff --git a/cpp/benchmarks/sort/rank_structs.cpp b/cpp/benchmarks/sort/rank_structs.cpp new file mode 100644 index 00000000000..c1e2c5bd7dc --- /dev/null +++ b/cpp/benchmarks/sort/rank_structs.cpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "nested_types_common.hpp" +#include "rank_types_common.hpp" + +#include + +#include + +template +void nvbench_rank_structs(nvbench::state& state, nvbench::type_list>) +{ + cudf::rmm_pool_raii pool_raii; + + auto const table = create_structs_data(state); + + const bool nulls{static_cast(state.get_int64("Nulls"))}; + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + cudf::rank(table->view().column(0), + method, + cudf::order::ASCENDING, + nulls ? cudf::null_policy::INCLUDE : cudf::null_policy::EXCLUDE, + cudf::null_order::AFTER, + rmm::mr::get_current_device_resource()); + }); +} + +NVBENCH_BENCH_TYPES(nvbench_rank_structs, NVBENCH_TYPE_AXES(methods)) + .set_name("rank_structs") + .add_int64_power_of_two_axis("NumRows", {10, 18, 26}) + .add_int64_axis("Depth", {0, 1, 8}) + .add_int64_axis("Nulls", {0, 1}); diff --git a/cpp/benchmarks/sort/rank_types_common.hpp b/cpp/benchmarks/sort/rank_types_common.hpp new file mode 100644 index 00000000000..adb58606c42 --- /dev/null +++ b/cpp/benchmarks/sort/rank_types_common.hpp @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include + +enum class rank_method : int32_t {}; + +NVBENCH_DECLARE_ENUM_TYPE_STRINGS( + cudf::rank_method, + [](cudf::rank_method value) { + switch (value) { + case cudf::rank_method::FIRST: return "FIRST"; + case cudf::rank_method::AVERAGE: return "AVERAGE"; + case cudf::rank_method::MIN: return "MIN"; + case cudf::rank_method::MAX: return "MAX"; + case cudf::rank_method::DENSE: return "DENSE"; + default: return "unknown"; + } + }, + [](cudf::rank_method value) { + switch (value) { + case cudf::rank_method::FIRST: return "cudf::rank_method::FIRST"; + case cudf::rank_method::AVERAGE: return "cudf::rank_method::AVERAGE"; + case cudf::rank_method::MIN: return "cudf::rank_method::MIN"; + case cudf::rank_method::MAX: return "cudf::rank_method::MAX"; + case cudf::rank_method::DENSE: return "cudf::rank_method::DENSE"; + default: return "unknown"; + } + }) + +using methods = nvbench::enum_type_list; diff --git a/cpp/benchmarks/sort/sort_lists.cpp b/cpp/benchmarks/sort/sort_lists.cpp index dac865de479..14cc60cbfe7 100644 --- a/cpp/benchmarks/sort/sort_lists.cpp +++ b/cpp/benchmarks/sort/sort_lists.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ * limitations under the License. */ -#include -#include +#include "nested_types_common.hpp" #include @@ -25,16 +24,7 @@ void nvbench_sort_lists(nvbench::state& state) { cudf::rmm_pool_raii pool_raii; - const size_t size_bytes(state.get_int64("size_bytes")); - const cudf::size_type depth{static_cast(state.get_int64("depth"))}; - auto const null_frequency{state.get_float64("null_frequency")}; - - data_profile table_profile; - table_profile.set_distribution_params(cudf::type_id::LIST, distribution_id::UNIFORM, 0, 5); - table_profile.set_list_depth(depth); - table_profile.set_null_probability(null_frequency); - auto const table = - create_random_table({cudf::type_id::LIST}, table_size_bytes{size_bytes}, table_profile); + auto const table = create_lists_data(state); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { rmm::cuda_stream_view stream_view{launch.get_stream()}; diff --git a/cpp/benchmarks/sort/sort_structs.cpp b/cpp/benchmarks/sort/sort_structs.cpp index 9b6c32940f5..22a6780c237 100644 --- a/cpp/benchmarks/sort/sort_structs.cpp +++ b/cpp/benchmarks/sort/sort_structs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,65 +14,21 @@ * limitations under the License. */ -#include - -#include +#include "nested_types_common.hpp" #include #include -#include - void nvbench_sort_struct(nvbench::state& state) { cudf::rmm_pool_raii pool_raii; - using Type = int; - using column_wrapper = cudf::test::fixed_width_column_wrapper; - std::default_random_engine generator; - std::uniform_int_distribution distribution(0, 100); - - const cudf::size_type n_rows{static_cast(state.get_int64("NumRows"))}; - const cudf::size_type n_cols{1}; - const cudf::size_type depth{static_cast(state.get_int64("Depth"))}; - const bool nulls{static_cast(state.get_int64("Nulls"))}; - - // Create columns with values in the range [0,100) - std::vector columns; - columns.reserve(n_cols); - std::generate_n(std::back_inserter(columns), n_cols, [&]() { - auto const elements = cudf::detail::make_counting_transform_iterator( - 0, [&](auto row) { return distribution(generator); }); - if (!nulls) return column_wrapper(elements, elements + n_rows); - auto valids = - cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i % 10 != 0; }); - return column_wrapper(elements, elements + n_rows, valids); - }); - - std::vector> cols; - std::transform(columns.begin(), columns.end(), std::back_inserter(cols), [](column_wrapper& col) { - return col.release(); - }); - - std::vector> child_cols = std::move(cols); - // Lets add some layers - for (int i = 0; i < depth; i++) { - std::vector struct_validity; - std::uniform_int_distribution bool_distribution(0, 100 * (i + 1)); - std::generate_n( - std::back_inserter(struct_validity), n_rows, [&]() { return bool_distribution(generator); }); - cudf::test::structs_column_wrapper struct_col(std::move(child_cols), struct_validity); - child_cols = std::vector>{}; - child_cols.push_back(struct_col.release()); - } - - // Create table view - auto const input = cudf::table(std::move(child_cols)); + auto const input = create_structs_data(state); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { rmm::cuda_stream_view stream_view{launch.get_stream()}; - cudf::detail::sorted_order(input, {}, {}, stream_view, rmm::mr::get_current_device_resource()); + cudf::detail::sorted_order(*input, {}, {}, stream_view, rmm::mr::get_current_device_resource()); }); } diff --git a/cpp/src/sort/rank.cu b/cpp/src/sort/rank.cu index 99e99704c10..461e978643f 100644 --- a/cpp/src/sort/rank.cu +++ b/cpp/src/sort/rank.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ #include #include #include -#include +#include #include #include #include @@ -47,37 +47,26 @@ namespace cudf { namespace detail { namespace { -// Functor to identify unique elements in a sorted order table/column -template -struct unique_comparator { - unique_comparator(table_device_view device_table, Iterator const sorted_order, bool has_nulls) - : comparator(nullate::DYNAMIC{has_nulls}, device_table, device_table, null_equality::EQUAL), - permute(sorted_order) - { - } - __device__ ReturnType operator()(size_type index) const noexcept - { - return index == 0 || not comparator(permute[index], permute[index - 1]); - }; - - private: - row_equality_comparator comparator; - Iterator const permute; -}; // Assign rank from 1 to n unique values. Equal values get same rank value. rmm::device_uvector sorted_dense_rank(column_view input_col, column_view sorted_order_view, rmm::cuda_stream_view stream) { - auto device_table = table_device_view::create(table_view{{input_col}}, stream); + auto const t_input = table_view{{input_col}}; + auto const comparator = cudf::experimental::row::equality::self_comparator{t_input, stream}; + auto const device_comparator = comparator.equal_to(nullate::DYNAMIC{has_nested_nulls(t_input)}); + + auto const sorted_index_order = thrust::make_permutation_iterator( + sorted_order_view.begin(), thrust::make_counting_iterator(0)); + auto conv = [permute = sorted_index_order, device_comparator] __device__(size_type index) { + return static_cast(index == 0 || + not device_comparator(permute[index], permute[index - 1])); + }; + auto const unique_it = cudf::detail::make_counting_transform_iterator(0, conv); + auto const input_size = input_col.size(); rmm::device_uvector dense_rank_sorted(input_size, stream); - auto sorted_index_order = thrust::make_permutation_iterator( - sorted_order_view.begin(), thrust::make_counting_iterator(0)); - auto conv = unique_comparator( - *device_table, sorted_index_order, input_col.has_nulls()); - auto unique_it = cudf::detail::make_counting_transform_iterator(0, conv); thrust::inclusive_scan( rmm::exec_policy(stream), unique_it, unique_it + input_size, dense_rank_sorted.data()); diff --git a/cpp/tests/sort/rank_test.cpp b/cpp/tests/sort/rank_test.cpp index 8461b0a1984..2722c1dfdad 100644 --- a/cpp/tests/sort/rank_test.cpp +++ b/cpp/tests/sort/rank_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,13 @@ #include #include +template +using lists_col = cudf::test::lists_column_wrapper; +using structs_col = cudf::test::structs_column_wrapper; + +using cudf::test::iterators::null_at; +using cudf::test::iterators::nulls_at; + namespace { void run_rank_test(cudf::table_view input, cudf::table_view expected, @@ -50,10 +58,9 @@ void run_rank_test(cudf::table_view input, } using input_arg_t = std::tuple; -input_arg_t asce_keep{cudf::order::ASCENDING, cudf::null_policy::EXCLUDE, cudf::null_order::AFTER}; -input_arg_t asce_top{cudf::order::ASCENDING, cudf::null_policy::INCLUDE, cudf::null_order::BEFORE}; -input_arg_t asce_bottom{ - cudf::order::ASCENDING, cudf::null_policy::INCLUDE, cudf::null_order::AFTER}; +input_arg_t asc_keep{cudf::order::ASCENDING, cudf::null_policy::EXCLUDE, cudf::null_order::AFTER}; +input_arg_t asc_top{cudf::order::ASCENDING, cudf::null_policy::INCLUDE, cudf::null_order::BEFORE}; +input_arg_t asc_bottom{cudf::order::ASCENDING, cudf::null_policy::INCLUDE, cudf::null_order::AFTER}; input_arg_t desc_keep{ cudf::order::DESCENDING, cudf::null_policy::EXCLUDE, cudf::null_order::BEFORE}; @@ -105,7 +112,7 @@ TYPED_TEST_SUITE(Rank, cudf::test::NumericTypes); // fixed_width_column_wrapper col1{{ 5, 4, 3, 5, 8, 5}}; // 3, 2, 1, 4, 6, 5 -TYPED_TEST(Rank, first_asce_keep) +TYPED_TEST(Rank, first_asc_keep) { // ASCENDING cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 4, 6, 5}}; @@ -113,25 +120,25 @@ TYPED_TEST(Rank, first_asce_keep) {1, 1, 0, 1, 1, 1}}; // KEEP cudf::test::fixed_width_column_wrapper col3_rank{{2, 5, 1, 3, 6, 4}, {1, 1, 1, 1, 1, 1}}; - this->run_all_tests(cudf::rank_method::FIRST, asce_keep, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::FIRST, asc_keep, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, first_asce_top) +TYPED_TEST(Rank, first_asc_top) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 4, 6, 5}}; cudf::test::fixed_width_column_wrapper col2_rank{ {3, 2, 1, 4, 6, 5}}; // BEFORE = TOP cudf::test::fixed_width_column_wrapper col3_rank{{2, 5, 1, 3, 6, 4}}; - this->run_all_tests(cudf::rank_method::FIRST, asce_top, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::FIRST, asc_top, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, first_asce_bottom) +TYPED_TEST(Rank, first_asc_bottom) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 4, 6, 5}}; cudf::test::fixed_width_column_wrapper col2_rank{ {2, 1, 6, 3, 5, 4}}; // AFTER = BOTTOM cudf::test::fixed_width_column_wrapper col3_rank{{2, 5, 1, 3, 6, 4}}; - this->run_all_tests(cudf::rank_method::FIRST, asce_bottom, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::FIRST, asc_bottom, col1_rank, col2_rank, col3_rank); } TYPED_TEST(Rank, first_desc_keep) @@ -163,30 +170,30 @@ TYPED_TEST(Rank, first_desc_bottom) this->run_all_tests(cudf::rank_method::FIRST, desc_bottom, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, dense_asce_keep) +TYPED_TEST(Rank, dense_asc_keep) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 3, 4, 3}}; cudf::test::fixed_width_column_wrapper col2_rank{{2, 1, -1, 2, 3, 2}, {1, 1, 0, 1, 1, 1}}; cudf::test::fixed_width_column_wrapper col3_rank{{2, 3, 1, 2, 4, 2}, {1, 1, 1, 1, 1, 1}}; - this->run_all_tests(cudf::rank_method::DENSE, asce_keep, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::DENSE, asc_keep, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, dense_asce_top) +TYPED_TEST(Rank, dense_asc_top) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 3, 4, 3}}; cudf::test::fixed_width_column_wrapper col2_rank{{3, 2, 1, 3, 4, 3}}; cudf::test::fixed_width_column_wrapper col3_rank{{2, 3, 1, 2, 4, 2}}; - this->run_all_tests(cudf::rank_method::DENSE, asce_top, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::DENSE, asc_top, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, dense_asce_bottom) +TYPED_TEST(Rank, dense_asc_bottom) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 3, 4, 3}}; cudf::test::fixed_width_column_wrapper col2_rank{{2, 1, 4, 2, 3, 2}}; cudf::test::fixed_width_column_wrapper col3_rank{{2, 3, 1, 2, 4, 2}}; - this->run_all_tests(cudf::rank_method::DENSE, asce_bottom, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::DENSE, asc_bottom, col1_rank, col2_rank, col3_rank); } TYPED_TEST(Rank, dense_desc_keep) @@ -215,30 +222,30 @@ TYPED_TEST(Rank, dense_desc_bottom) this->run_all_tests(cudf::rank_method::DENSE, desc_bottom, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, min_asce_keep) +TYPED_TEST(Rank, min_asc_keep) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 3, 6, 3}}; cudf::test::fixed_width_column_wrapper col2_rank{{2, 1, -1, 2, 5, 2}, {1, 1, 0, 1, 1, 1}}; cudf::test::fixed_width_column_wrapper col3_rank{{2, 5, 1, 2, 6, 2}, {1, 1, 1, 1, 1, 1}}; - this->run_all_tests(cudf::rank_method::MIN, asce_keep, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::MIN, asc_keep, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, min_asce_top) +TYPED_TEST(Rank, min_asc_top) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 3, 6, 3}}; cudf::test::fixed_width_column_wrapper col2_rank{{3, 2, 1, 3, 6, 3}}; cudf::test::fixed_width_column_wrapper col3_rank{{2, 5, 1, 2, 6, 2}}; - this->run_all_tests(cudf::rank_method::MIN, asce_top, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::MIN, asc_top, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, min_asce_bottom) +TYPED_TEST(Rank, min_asc_bottom) { cudf::test::fixed_width_column_wrapper col1_rank{{3, 2, 1, 3, 6, 3}}; cudf::test::fixed_width_column_wrapper col2_rank{{2, 1, 6, 2, 5, 2}}; cudf::test::fixed_width_column_wrapper col3_rank{{2, 5, 1, 2, 6, 2}}; - this->run_all_tests(cudf::rank_method::MIN, asce_bottom, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::MIN, asc_bottom, col1_rank, col2_rank, col3_rank); } TYPED_TEST(Rank, min_desc_keep) @@ -267,30 +274,30 @@ TYPED_TEST(Rank, min_desc_bottom) this->run_all_tests(cudf::rank_method::MIN, desc_bottom, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, max_asce_keep) +TYPED_TEST(Rank, max_asc_keep) { cudf::test::fixed_width_column_wrapper col1_rank{{5, 2, 1, 5, 6, 5}}; cudf::test::fixed_width_column_wrapper col2_rank{{4, 1, -1, 4, 5, 4}, {1, 1, 0, 1, 1, 1}}; cudf::test::fixed_width_column_wrapper col3_rank{{4, 5, 1, 4, 6, 4}, {1, 1, 1, 1, 1, 1}}; - this->run_all_tests(cudf::rank_method::MAX, asce_keep, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::MAX, asc_keep, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, max_asce_top) +TYPED_TEST(Rank, max_asc_top) { cudf::test::fixed_width_column_wrapper col1_rank{{5, 2, 1, 5, 6, 5}}; cudf::test::fixed_width_column_wrapper col2_rank{{5, 2, 1, 5, 6, 5}}; cudf::test::fixed_width_column_wrapper col3_rank{{4, 5, 1, 4, 6, 4}}; - this->run_all_tests(cudf::rank_method::MAX, asce_top, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::MAX, asc_top, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, max_asce_bottom) +TYPED_TEST(Rank, max_asc_bottom) { cudf::test::fixed_width_column_wrapper col1_rank{{5, 2, 1, 5, 6, 5}}; cudf::test::fixed_width_column_wrapper col2_rank{{4, 1, 6, 4, 5, 4}}; cudf::test::fixed_width_column_wrapper col3_rank{{4, 5, 1, 4, 6, 4}}; - this->run_all_tests(cudf::rank_method::MAX, asce_bottom, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::MAX, asc_bottom, col1_rank, col2_rank, col3_rank); } TYPED_TEST(Rank, max_desc_keep) @@ -319,28 +326,28 @@ TYPED_TEST(Rank, max_desc_bottom) this->run_all_tests(cudf::rank_method::MAX, desc_bottom, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, average_asce_keep) +TYPED_TEST(Rank, average_asc_keep) { cudf::test::fixed_width_column_wrapper col1_rank{{4, 2, 1, 4, 6, 4}}; cudf::test::fixed_width_column_wrapper col2_rank{{3, 1, -1, 3, 5, 3}, {1, 1, 0, 1, 1, 1}}; cudf::test::fixed_width_column_wrapper col3_rank{{3, 5, 1, 3, 6, 3}, {1, 1, 1, 1, 1, 1}}; - this->run_all_tests(cudf::rank_method::AVERAGE, asce_keep, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::AVERAGE, asc_keep, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, average_asce_top) +TYPED_TEST(Rank, average_asc_top) { cudf::test::fixed_width_column_wrapper col1_rank{{4, 2, 1, 4, 6, 4}}; cudf::test::fixed_width_column_wrapper col2_rank{{4, 2, 1, 4, 6, 4}}; cudf::test::fixed_width_column_wrapper col3_rank{{3, 5, 1, 3, 6, 3}}; - this->run_all_tests(cudf::rank_method::AVERAGE, asce_top, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::AVERAGE, asc_top, col1_rank, col2_rank, col3_rank); } -TYPED_TEST(Rank, average_asce_bottom) +TYPED_TEST(Rank, average_asc_bottom) { cudf::test::fixed_width_column_wrapper col1_rank{{4, 2, 1, 4, 6, 4}}; cudf::test::fixed_width_column_wrapper col2_rank{{3, 1, 6, 3, 5, 3}}; cudf::test::fixed_width_column_wrapper col3_rank{{3, 5, 1, 3, 6, 3}}; - this->run_all_tests(cudf::rank_method::AVERAGE, asce_bottom, col1_rank, col2_rank, col3_rank); + this->run_all_tests(cudf::rank_method::AVERAGE, asc_bottom, col1_rank, col2_rank, col3_rank); } TYPED_TEST(Rank, average_desc_keep) @@ -368,30 +375,30 @@ TYPED_TEST(Rank, average_desc_bottom) } // percentage==true (dense, not-dense) -TYPED_TEST(Rank, dense_asce_keep_pct) +TYPED_TEST(Rank, dense_asc_keep_pct) { cudf::test::fixed_width_column_wrapper col1_rank{{0.75, 0.5, 0.25, 0.75, 1., 0.75}}; cudf::test::fixed_width_column_wrapper col2_rank{ {2.0 / 3.0, 1.0 / 3.0, -1., 2.0 / 3.0, 1., 2.0 / 3.0}, {1, 1, 0, 1, 1, 1}}; cudf::test::fixed_width_column_wrapper col3_rank{{0.5, 0.75, 0.25, 0.5, 1., 0.5}, {1, 1, 1, 1, 1, 1}}; - this->run_all_tests(cudf::rank_method::DENSE, asce_keep, col1_rank, col2_rank, col3_rank, true); + this->run_all_tests(cudf::rank_method::DENSE, asc_keep, col1_rank, col2_rank, col3_rank, true); } -TYPED_TEST(Rank, dense_asce_top_pct) +TYPED_TEST(Rank, dense_asc_top_pct) { cudf::test::fixed_width_column_wrapper col1_rank{{0.75, 0.5, 0.25, 0.75, 1., 0.75}}; cudf::test::fixed_width_column_wrapper col2_rank{{0.75, 0.5, 0.25, 0.75, 1., 0.75}}; cudf::test::fixed_width_column_wrapper col3_rank{{0.5, 0.75, 0.25, 0.5, 1., 0.5}}; - this->run_all_tests(cudf::rank_method::DENSE, asce_top, col1_rank, col2_rank, col3_rank, true); + this->run_all_tests(cudf::rank_method::DENSE, asc_top, col1_rank, col2_rank, col3_rank, true); } -TYPED_TEST(Rank, dense_asce_bottom_pct) +TYPED_TEST(Rank, dense_asc_bottom_pct) { cudf::test::fixed_width_column_wrapper col1_rank{{0.75, 0.5, 0.25, 0.75, 1., 0.75}}; cudf::test::fixed_width_column_wrapper col2_rank{{0.5, 0.25, 1., 0.5, 0.75, 0.5}}; cudf::test::fixed_width_column_wrapper col3_rank{{0.5, 0.75, 0.25, 0.5, 1., 0.5}}; - this->run_all_tests(cudf::rank_method::DENSE, asce_bottom, col1_rank, col2_rank, col3_rank, true); + this->run_all_tests(cudf::rank_method::DENSE, asc_bottom, col1_rank, col2_rank, col3_rank, true); } TYPED_TEST(Rank, min_desc_keep_pct) @@ -444,3 +451,472 @@ TEST_F(RankLarge, average_large) cudf::test::fixed_width_column_wrapper expected(iter + 1, iter + 10559); CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); } + +template +struct RankListAndStruct : public cudf::test::BaseFixture { + void run_all_tests(cudf::rank_method method, + input_arg_t input_arg, + cudf::column_view const list_rank, + cudf::column_view const struct_rank, + bool percentage = false) + { + if constexpr (std::is_same_v) { return; } + /* + [ + [], + [1], + [2, 2], + [2, 3], + [2, 2], + [1], + [], + NULL + [2], + NULL, + [1] + ] + */ + auto list_col = + lists_col{{{}, {1}, {2, 2}, {2, 3}, {2, 2}, {1}, {}, {} /*NULL*/, {2}, {} /*NULL*/, {1}}, + nulls_at({7, 9})}; + + // clang-format off + /* + +------------+ + | s| + +------------+ + 0 | {0, null}| + 1 | {1, null}| + 2 | null| + 3 |{null, null}| + 4 | null| + 5 |{null, null}| + 6 | {null, 1}| + 7 | {null, 0}| + +------------+ + */ + std::vector struct_valids{1, 1, 0, 1, 0, 1, 1, 1}; + auto col1 = cudf::test::fixed_width_column_wrapper{{ 0, 1, 9, -1, 9, -1, -1, -1}, {1, 1, 1, 0, 1, 0, 0, 0}}; + auto col2 = cudf::test::fixed_width_column_wrapper{{-1, -1, 9, -1, 9, -1, 1, 0}, {0, 0, 1, 0, 1, 0, 1, 1}}; + auto struct_col = cudf::test::structs_column_wrapper{{col1, col2}, struct_valids}.release(); + // clang-format on + + for (auto const& test_case : { + // Non-null column + test_case_t{cudf::table_view{{list_col}}, cudf::table_view{{list_rank}}}, + // Null column + test_case_t{cudf::table_view{{struct_col->view()}}, cudf::table_view{{struct_rank}}}, + }) { + auto [input, output] = test_case; + + run_rank_test(input, + output, + method, + std::get<0>(input_arg), + std::get<1>(input_arg), + std::get<2>(input_arg), + percentage); + } + } +}; + +TYPED_TEST_SUITE(RankListAndStruct, cudf::test::NumericTypes); + +TYPED_TEST(RankListAndStruct, first_asc_keep) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper list_rank{ + {1, 3, 7, 9, 8, 4, 2, -1, 6, -1, 5}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{1, 2, -1, 5, -1, 6, 4, 3}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::FIRST, asc_keep, list_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, first_asc_top) +{ + // ASCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 3, 5, 9, 11, 10, 6, 4, 1, 8, 2, 7}; + cudf::test::fixed_width_column_wrapper struct_rank{7, 8, 1, 3, 2, 4, 6, 5}; + this->run_all_tests(cudf::rank_method::FIRST, asc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, first_asc_bottom) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 1, 3, 7, 9, 8, 4, 2, 10, 6, 11, 5}; + cudf::test::fixed_width_column_wrapper struct_rank{1, 2, 7, 5, 8, 6, 4, 3}; + this->run_all_tests(cudf::rank_method::FIRST, asc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, first_desc_keep) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + {8, 5, 2, 1, 3, 6, 9, -1, 4, -1, 7}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{2, 1, -1, 5, -1, 6, 3, 4}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::FIRST, desc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, first_desc_top) +{ + // DESCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 10, 7, 4, 3, 5, 8, 11, 1, 6, 2, 9}; + cudf::test::fixed_width_column_wrapper struct_rank{8, 7, 1, 3, 2, 4, 5, 6}; + this->run_all_tests(cudf::rank_method::FIRST, desc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, first_desc_bottom) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 8, 5, 2, 1, 3, 6, 9, 10, 4, 11, 7}; + cudf::test::fixed_width_column_wrapper struct_rank{2, 1, 7, 5, 8, 6, 3, 4}; + this->run_all_tests(cudf::rank_method::FIRST, desc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_asc_keep) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + {1, 2, 4, 5, 4, 2, 1, -1, 3, -1, 2}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{1, 2, -1, 5, -1, 5, 4, 3}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::DENSE, asc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_asc_top) +{ + // ASCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{2, 3, 5, 6, 5, 3, 2, 1, 4, 1, 3}; + cudf::test::fixed_width_column_wrapper struct_rank{5, 6, 1, 2, 1, 2, 4, 3}; + this->run_all_tests(cudf::rank_method::DENSE, asc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_asc_bottom) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{1, 2, 4, 5, 4, 2, 1, 6, 3, 6, 2}; + cudf::test::fixed_width_column_wrapper struct_rank{1, 2, 6, 5, 6, 5, 4, 3}; + this->run_all_tests(cudf::rank_method::DENSE, asc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_desc_keep) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + {5, 4, 2, 1, 2, 4, 5, -1, 3, -1, 4}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{2, 1, -1, 5, -1, 5, 3, 4}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::DENSE, desc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_desc_top) +{ + // DESCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{6, 5, 3, 2, 3, 5, 6, 1, 4, 1, 5}; + cudf::test::fixed_width_column_wrapper struct_rank{6, 5, 1, 2, 1, 2, 3, 4}; + this->run_all_tests(cudf::rank_method::DENSE, desc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_desc_bottom) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{5, 4, 2, 1, 2, 4, 5, 6, 3, 6, 4}; + cudf::test::fixed_width_column_wrapper struct_rank{2, 1, 6, 5, 6, 5, 3, 4}; + this->run_all_tests(cudf::rank_method::DENSE, desc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, min_asc_keep) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + {1, 3, 7, 9, 7, 3, 1, -1, 6, -1, 3}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{1, 2, -1, 5, -1, 5, 4, 3}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::MIN, asc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, min_asc_top) +{ + // ASCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 3, 5, 9, 11, 9, 5, 3, 1, 8, 1, 5}; + cudf::test::fixed_width_column_wrapper struct_rank{7, 8, 1, 3, 1, 3, 6, 5}; + this->run_all_tests(cudf::rank_method::MIN, asc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, min_asc_bottom) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 1, 3, 7, 9, 7, 3, 1, 10, 6, 10, 3}; + cudf::test::fixed_width_column_wrapper struct_rank{1, 2, 7, 5, 7, 5, 4, 3}; + this->run_all_tests(cudf::rank_method::MIN, asc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, min_desc_keep) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + {8, 5, 2, 1, 2, 5, 8, -1, 4, -1, 5}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{2, 1, -1, 5, -1, 5, 3, 4}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::MIN, desc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, min_desc_top) +{ + // DESCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 10, 7, 4, 3, 4, 7, 10, 1, 6, 1, 7}; + cudf::test::fixed_width_column_wrapper struct_rank{8, 7, 1, 3, 1, 3, 5, 6}; + this->run_all_tests(cudf::rank_method::MIN, desc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, min_desc_bottom) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 8, 5, 2, 1, 2, 5, 8, 10, 4, 10, 5}; + cudf::test::fixed_width_column_wrapper struct_rank{2, 1, 7, 5, 7, 5, 3, 4}; + this->run_all_tests(cudf::rank_method::MIN, desc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, max_asc_keep) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + {2, 5, 8, 9, 8, 5, 2, -1, 6, -1, 5}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{1, 2, -1, 6, -1, 6, 4, 3}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::MAX, asc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, max_asc_top) +{ + // ASCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 4, 7, 10, 11, 10, 7, 4, 2, 8, 2, 7}; + cudf::test::fixed_width_column_wrapper struct_rank{7, 8, 2, 4, 2, 4, 6, 5}; + this->run_all_tests(cudf::rank_method::MAX, asc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, max_asc_bottom) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 2, 5, 8, 9, 8, 5, 2, 11, 6, 11, 5}; + cudf::test::fixed_width_column_wrapper struct_rank{1, 2, 8, 6, 8, 6, 4, 3}; + this->run_all_tests(cudf::rank_method::MAX, asc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, max_desc_keep) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + {9, 7, 3, 1, 3, 7, 9, -1, 4, -1, 7}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{{2, 1, -1, 6, -1, 6, 3, 4}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::MAX, desc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, max_desc_top) +{ + // DESCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 11, 9, 5, 3, 5, 9, 11, 2, 6, 2, 9}; + cudf::test::fixed_width_column_wrapper struct_rank{8, 7, 2, 4, 2, 4, 5, 6}; + this->run_all_tests(cudf::rank_method::MAX, desc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, max_desc_bottom) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 9, 7, 3, 1, 3, 7, 9, 11, 4, 11, 7}; + cudf::test::fixed_width_column_wrapper struct_rank{2, 1, 8, 6, 8, 6, 3, 4}; + this->run_all_tests(cudf::rank_method::MAX, desc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, average_asc_keep) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + {1.5, 4.0, 7.5, 9.0, 7.5, 4.0, 1.5, -1.0, 6.0, -1.0, 4.0}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{ + {1.0, 2.0, -1.0, 5.5, -1.0, 5.5, 4.0, 3.0}, nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::AVERAGE, asc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, average_asc_top) +{ + // ASCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 3.5, 6.0, 9.5, 11.0, 9.5, 6.0, 3.5, 1.5, 8.0, 1.5, 6.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 7.0, 8.0, 1.5, 3.5, 1.5, 3.5, 6.0, 5.0}; + this->run_all_tests(cudf::rank_method::AVERAGE, asc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, average_asc_bottom) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 1.5, 4.0, 7.5, 9.0, 7.5, 4.0, 1.5, 10.5, 6.0, 10.5, 4.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 1.0, 2.0, 7.5, 5.5, 7.5, 5.5, 4.0, 3.0}; + this->run_all_tests(cudf::rank_method::AVERAGE, asc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, average_desc_keep) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + {8.5, 6.0, 2.5, 1.0, 2.5, 6.0, 8.5, -1.0, 4.0, -1.0, 6.0}, nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{ + {2.0, 1.0, -1.0, 5.5, -1.0, 5.5, 3.0, 4.0}, nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::AVERAGE, desc_keep, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, average_desc_top) +{ + // DESCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{ + 10.5, 8.0, 4.5, 3.0, 4.5, 8.0, 10.5, 1.5, 6.0, 1.5, 8.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 8.0, 7.0, 1.5, 3.5, 1.5, 3.5, 5.0, 6.0}; + this->run_all_tests(cudf::rank_method::AVERAGE, desc_top, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, average_desc_bottom) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{ + 8.5, 6.0, 2.5, 1.0, 2.5, 6.0, 8.5, 10.5, 4.0, 10.5, 6.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 2.0, 1.0, 7.5, 5.5, 7.5, 5.5, 3.0, 4.0}; + this->run_all_tests(cudf::rank_method::AVERAGE, desc_bottom, col_rank, struct_rank); +} + +TYPED_TEST(RankListAndStruct, dense_asc_keep_pct) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{{1.0 / 5.0, + 2.0 / 5.0, + 4.0 / 5.0, + 1.0, + 4.0 / 5.0, + 2.0 / 5.0, + 1.0 / 5.0, + -1.0, + 3.0 / 5.0, + -1.0, + 2.0 / 5.0}, + nulls_at({7, 9})}; + + cudf::test::fixed_width_column_wrapper struct_rank{ + {1.0 / 5.0, 2.0 / 5.0, -1.0, 1.0, -1.0, 1.0, 4.0 / 5.0, 3.0 / 5.0}, nulls_at({2, 4})}; + + this->run_all_tests(cudf::rank_method::DENSE, asc_keep, col_rank, struct_rank, true); +} + +TYPED_TEST(RankListAndStruct, dense_asc_top_pct) +{ + // ASCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{1.0 / 3.0, + 1.0 / 2.0, + 5.0 / 6.0, + 1.0, + 5.0 / 6.0, + 1.0 / 2.0, + 1.0 / 3.0, + 1.0 / 6.0, + 2.0 / 3.0, + 1.0 / 6.0, + 1.0 / 2.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 5.0 / 6.0, 1.0, 1.0 / 6.0, 2.0 / 6.0, 1.0 / 6.0, 2.0 / 6.0, 4.0 / 6.0, 3.0 / 6.0}; + this->run_all_tests(cudf::rank_method::DENSE, asc_top, col_rank, struct_rank, true); +} + +TYPED_TEST(RankListAndStruct, dense_asc_bottom_pct) +{ + // ASCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{1.0 / 6.0, + 1.0 / 3.0, + 2.0 / 3.0, + 5.0 / 6.0, + 2.0 / 3.0, + 1.0 / 3.0, + 1.0 / 6.0, + 1.0, + 1.0 / 2.0, + 1.0, + 1.0 / 3.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 1.0 / 6.0, 2.0 / 6.0, 1.0, 5.0 / 6.0, 1.0, 5.0 / 6.0, 4.0 / 6.0, 3.0 / 6.0}; + this->run_all_tests(cudf::rank_method::DENSE, asc_bottom, col_rank, struct_rank, true); +} + +TYPED_TEST(RankListAndStruct, min_desc_keep_pct) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{{8.0 / 9.0, + 5.0 / 9.0, + 2.0 / 9.0, + 1.0 / 9.0, + 2.0 / 9.0, + 5.0 / 9.0, + 8.0 / 9.0, + -1.0, + 4.0 / 9.0, + -1.0, + 5.0 / 9.0}, + nulls_at({7, 9})}; + cudf::test::fixed_width_column_wrapper struct_rank{ + {2.0 / 6.0, 1.0 / 6.0, -1.0, 5.0 / 6.0, -1.0, 5.0 / 6.0, 3.0 / 6.0, 4.0 / 6.0}, + nulls_at({2, 4})}; + this->run_all_tests(cudf::rank_method::MIN, desc_keep, col_rank, struct_rank, true); +} + +TYPED_TEST(RankListAndStruct, min_desc_top_pct) +{ + // DESCENDING and null_order::AFTER + cudf::test::fixed_width_column_wrapper col_rank{10.0 / 11.0, + 7.0 / 11.0, + 4.0 / 11.0, + 3.0 / 11.0, + 4.0 / 11.0, + 7.0 / 11.0, + 10.0 / 11.0, + 1.0 / 11.0, + 6.0 / 11.0, + 1.0 / 11.0, + 7.0 / 11.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 1.0, 7.0 / 8.0, 1.0 / 8.0, 3.0 / 8.0, 1.0 / 8.0, 3.0 / 8.0, 5.0 / 8.0, 6.0 / 8.0}; + this->run_all_tests(cudf::rank_method::MIN, desc_top, col_rank, struct_rank, true); +} + +TYPED_TEST(RankListAndStruct, min_desc_bottom_pct) +{ + // DESCENDING and null_order::BEFORE + cudf::test::fixed_width_column_wrapper col_rank{8.0 / 11.0, + 5.0 / 11.0, + 2.0 / 11.0, + 1.0 / 11.0, + 2.0 / 11.0, + 5.0 / 11.0, + 8.0 / 11.0, + 10.0 / 11.0, + 4.0 / 11.0, + 10.0 / 11.0, + 5.0 / 11.0}; + cudf::test::fixed_width_column_wrapper struct_rank{ + 2.0 / 8.0, 1.0 / 8.0, 7.0 / 8.0, 5.0 / 8.0, 7.0 / 8.0, 5.0 / 8.0, 3.0 / 8.0, 4.0 / 8.0}; + this->run_all_tests(cudf::rank_method::MIN, desc_bottom, col_rank, struct_rank, true); +} From fea6288f5f114790fd7d075a6b6c26b2f78a8316 Mon Sep 17 00:00:00 2001 From: Cindy Jiang <47068112+cindyyuanjiang@users.noreply.github.com> Date: Tue, 7 Feb 2023 17:15:28 -0800 Subject: [PATCH 03/69] Add `regex_program` strings splitting java APIs and tests (#12713) This PR adds [split_re, rsplit_re, split_record_re, rsplit_record_re](https://docs.rapids.ai/api/libcudf/nightly/split__re_8hpp.html) related `regex_program` java APIs and unit tests. Part of work for https://github.com/NVIDIA/spark-rapids/issues/7295. Authors: - Cindy Jiang (https://github.com/cindyyuanjiang) Approvers: - Jason Lowe (https://github.com/jlowe) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12713 --- .../main/java/ai/rapids/cudf/ColumnView.java | 152 +++++++++++++++--- java/src/main/native/src/ColumnViewJni.cpp | 98 +++++++---- .../java/ai/rapids/cudf/ColumnVectorTest.java | 60 ++++--- 3 files changed, 234 insertions(+), 76 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 4daa3c17cfc..2d0bf28225f 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -2531,12 +2531,34 @@ public final ColumnVector stringLocate(Scalar substring, int start, int end) { * regular expression pattern or just by a string literal delimiter. * @return list of strings columns as a table. */ + @Deprecated public final Table stringSplit(String pattern, int limit, boolean splitByRegex) { + if (splitByRegex) { + return stringSplit(new RegexProgram(pattern, CaptureGroups.NON_CAPTURE), limit); + } else { + return stringSplit(pattern, limit); + } + } + + /** + * Returns a list of columns by splitting each string using the specified regex program pattern. + * The number of rows in the output columns will be the same as the input column. Null entries + * are added for the rows where split results have been exhausted. Null input entries result in + * all nulls in the corresponding rows of the output columns. + * + * @param regexProg the regex program with UTF-8 encoded string identifying the split pattern + * for each input string. + * @param limit the maximum size of the list resulting from splitting each input string, + * or -1 for all possible splits. Note that limit = 0 (all possible splits without + * trailing empty strings) and limit = 1 (no split at all) are not supported. + * @return list of strings columns as a table. + */ + public final Table stringSplit(RegexProgram regexProg, int limit) { assert type.equals(DType.STRING) : "column type must be a String"; - assert pattern != null : "pattern is null"; - assert pattern.length() > 0 : "empty pattern is not supported"; + assert regexProg != null : "regex program is null"; assert limit != 0 && limit != 1 : "split limit == 0 and limit == 1 are not supported"; - return new Table(stringSplit(this.getNativeView(), pattern, limit, splitByRegex)); + return new Table(stringSplitRe(this.getNativeView(), regexProg.pattern(), regexProg.combinedFlags(), + regexProg.capture().nativeId, limit)); } /** @@ -2550,6 +2572,7 @@ public final Table stringSplit(String pattern, int limit, boolean splitByRegex) * regular expression pattern or just by a string literal delimiter. * @return list of strings columns as a table. */ + @Deprecated public final Table stringSplit(String pattern, boolean splitByRegex) { return stringSplit(pattern, -1, splitByRegex); } @@ -2567,7 +2590,10 @@ public final Table stringSplit(String pattern, boolean splitByRegex) { * @return list of strings columns as a table. */ public final Table stringSplit(String delimiter, int limit) { - return stringSplit(delimiter, limit, false); + assert type.equals(DType.STRING) : "column type must be a String"; + assert delimiter != null : "delimiter is null"; + assert limit != 0 && limit != 1 : "split limit == 0 and limit == 1 are not supported"; + return new Table(stringSplit(this.getNativeView(), delimiter, limit)); } /** @@ -2580,7 +2606,21 @@ public final Table stringSplit(String delimiter, int limit) { * @return list of strings columns as a table. */ public final Table stringSplit(String delimiter) { - return stringSplit(delimiter, -1, false); + return stringSplit(delimiter, -1); + } + + /** + * Returns a list of columns by splitting each string using the specified regex program pattern. + * The number of rows in the output columns will be the same as the input column. Null entries + * are added for the rows where split results have been exhausted. Null input entries result in + * all nulls in the corresponding rows of the output columns. + * + * @param regexProg the regex program with UTF-8 encoded string identifying the split pattern + * for each input string. + * @return list of strings columns as a table. + */ + public final Table stringSplit(RegexProgram regexProg) { + return stringSplit(regexProg, -1); } /** @@ -2595,13 +2635,33 @@ public final Table stringSplit(String delimiter) { * regular expression pattern or just by a string literal delimiter. * @return a LIST column of string elements. */ + @Deprecated public final ColumnVector stringSplitRecord(String pattern, int limit, boolean splitByRegex) { + if (splitByRegex) { + return stringSplitRecord(new RegexProgram(pattern, CaptureGroups.NON_CAPTURE), limit); + } else { + return stringSplitRecord(pattern, limit); + } + } + + /** + * Returns a column that are lists of strings in which each list is made by splitting the + * corresponding input string using the specified regex program pattern. + * + * @param regexProg the regex program with UTF-8 encoded string identifying the split pattern + * for each input string. + * @param limit the maximum size of the list resulting from splitting each input string, + * or -1 for all possible splits. Note that limit = 0 (all possible splits without + * trailing empty strings) and limit = 1 (no split at all) are not supported. + * @return a LIST column of string elements. + */ + public final ColumnVector stringSplitRecord(RegexProgram regexProg, int limit) { assert type.equals(DType.STRING) : "column type must be String"; - assert pattern != null : "pattern is null"; - assert pattern.length() > 0 : "empty pattern is not supported"; + assert regexProg != null : "regex program is null"; assert limit != 0 && limit != 1 : "split limit == 0 and limit == 1 are not supported"; return new ColumnVector( - stringSplitRecord(this.getNativeView(), pattern, limit, splitByRegex)); + stringSplitRecordRe(this.getNativeView(), regexProg.pattern(), regexProg.combinedFlags(), + regexProg.capture().nativeId, limit)); } /** @@ -2613,6 +2673,7 @@ public final ColumnVector stringSplitRecord(String pattern, int limit, boolean s * regular expression pattern or just by a string literal delimiter. * @return a LIST column of string elements. */ + @Deprecated public final ColumnVector stringSplitRecord(String pattern, boolean splitByRegex) { return stringSplitRecord(pattern, -1, splitByRegex); } @@ -2628,7 +2689,10 @@ public final ColumnVector stringSplitRecord(String pattern, boolean splitByRegex * @return a LIST column of string elements. */ public final ColumnVector stringSplitRecord(String delimiter, int limit) { - return stringSplitRecord(delimiter, limit, false); + assert type.equals(DType.STRING) : "column type must be String"; + assert delimiter != null : "delimiter is null"; + assert limit != 0 && limit != 1 : "split limit == 0 and limit == 1 are not supported"; + return new ColumnVector(stringSplitRecord(this.getNativeView(), delimiter, limit)); } /** @@ -2639,7 +2703,19 @@ public final ColumnVector stringSplitRecord(String delimiter, int limit) { * @return a LIST column of string elements. */ public final ColumnVector stringSplitRecord(String delimiter) { - return stringSplitRecord(delimiter, -1, false); + return stringSplitRecord(delimiter, -1); + } + + /** + * Returns a column that are lists of strings in which each list is made by splitting the + * corresponding input string using the specified regex program pattern. + * + * @param regexProg the regex program with UTF-8 encoded string identifying the split pattern + * for each input string. + * @return a LIST column of string elements. + */ + public final ColumnVector stringSplitRecord(RegexProgram regexProg) { + return stringSplitRecord(regexProg, -1); } /** @@ -3958,36 +4034,64 @@ private static native long repeatStringsWithColumnRepeatTimes(long stringsHandle private static native long substringLocate(long columnView, long substringScalar, int start, int end); /** - * Returns a list of columns by splitting each string using the specified pattern. The number of - * rows in the output columns will be the same as the input column. Null entries are added for a - * row where split results have been exhausted. Null input entries result in all nulls in the - * corresponding rows of the output columns. + * Returns a list of columns by splitting each string using the specified string literal + * delimiter. The number of rows in the output columns will be the same as the input column. + * Null entries are added for the rows where split results have been exhausted. Null input entries + * result in all nulls in the corresponding rows of the output columns. * * @param nativeHandle native handle of the input strings column that being operated on. - * @param pattern UTF-8 encoded string identifying the split pattern for each input string. + * @param delimiter UTF-8 encoded string identifying the split delimiter for each input string. + * @param limit the maximum size of the list resulting from splitting each input string, + * or -1 for all possible splits. Note that limit = 0 (all possible splits without + * trailing empty strings) and limit = 1 (no split at all) are not supported. + */ + private static native long[] stringSplit(long nativeHandle, String delimiter, int limit); + + /** + * Returns a list of columns by splitting each string using the specified regular expression + * pattern. The number of rows in the output columns will be the same as the input column. + * Null entries are added for the rows where split results have been exhausted. Null input entries + * result in all nulls in the corresponding rows of the output columns. + * + * @param nativeHandle native handle of the input strings column that being operated on. + * @param pattern UTF-8 encoded string identifying the split regular expression pattern for + * each input string. + * @param flags regex flags setting. + * @param capture capture groups setting. * @param limit the maximum size of the list resulting from splitting each input string, * or -1 for all possible splits. Note that limit = 0 (all possible splits without * trailing empty strings) and limit = 1 (no split at all) are not supported. - * @param splitByRegex a boolean flag indicating whether the input strings will be split by a - * regular expression pattern or just by a string literal delimiter. */ - private static native long[] stringSplit(long nativeHandle, String pattern, int limit, - boolean splitByRegex); + private static native long[] stringSplitRe(long nativeHandle, String pattern, int flags, + int capture, int limit); /** * Returns a column that are lists of strings in which each list is made by splitting the * corresponding input string using the specified string literal delimiter. * * @param nativeHandle native handle of the input strings column that being operated on. - * @param pattern UTF-8 encoded string identifying the split pattern for each input string. + * @param delimiter UTF-8 encoded string identifying the split delimiter for each input string. + * @param limit the maximum size of the list resulting from splitting each input string, + * or -1 for all possible splits. Note that limit = 0 (all possible splits without + * trailing empty strings) and limit = 1 (no split at all) are not supported. + */ + private static native long stringSplitRecord(long nativeHandle, String delimiter, int limit); + + /** + * Returns a column that are lists of strings in which each list is made by splitting the + * corresponding input string using the specified regular expression pattern. + * + * @param nativeHandle native handle of the input strings column that being operated on. + * @param pattern UTF-8 encoded string identifying the split regular expression pattern for + * each input string. + * @param flags regex flags setting. + * @param capture capture groups setting. * @param limit the maximum size of the list resulting from splitting each input string, * or -1 for all possible splits. Note that limit = 0 (all possible splits without * trailing empty strings) and limit = 1 (no split at all) are not supported. - * @param splitByRegex a boolean flag indicating whether the input strings will be split by a - * regular expression pattern or just by a string literal delimiter. */ - private static native long stringSplitRecord(long nativeHandle, String pattern, int limit, - boolean splitByRegex); + private static native long stringSplitRecordRe(long nativeHandle, String pattern, int flags, + int capture, int limit); /** * Native method to calculate substring from a given string column. 0 indexing. diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index bfa3fa0a522..958efd364ed 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -681,9 +681,8 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_reverseStringsOrLists(JNI JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_ColumnView_stringSplit(JNIEnv *env, jclass, jlong input_handle, - jstring pattern_obj, - jint limit, - jboolean split_by_regex) { + jstring delimiter_obj, + jint limit) { JNI_NULL_CHECK(env, input_handle, "input_handle is null", 0); if (limit == 0 || limit == 1) { @@ -697,21 +696,42 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_ColumnView_stringSplit(JNIEnv * try { cudf::jni::auto_set_device(env); - auto const input = reinterpret_cast(input_handle); - auto const strs_input = cudf::strings_column_view{*input}; + auto const input = reinterpret_cast(input_handle); + auto const strings_column = cudf::strings_column_view{*input}; + auto const delimiter_jstr = cudf::jni::native_jstring(env, delimiter_obj); + auto const delimiter = std::string(delimiter_jstr.get(), delimiter_jstr.size_bytes()); + auto const max_split = limit > 1 ? limit - 1 : limit; + auto result = cudf::strings::split(strings_column, cudf::string_scalar{delimiter}, max_split); + return cudf::jni::convert_table_for_return(env, std::move(result)); + } + CATCH_STD(env, 0); +} - auto const pattern_jstr = cudf::jni::native_jstring(env, pattern_obj); - if (pattern_jstr.is_empty()) { - // Java's split API produces different behaviors than cudf when splitting with empty - // pattern. - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "Empty pattern is not supported", 0); - } +JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_ColumnView_stringSplitRe( + JNIEnv *env, jclass, jlong input_handle, jstring pattern_obj, jint regex_flags, + jint capture_groups, jint limit) { + JNI_NULL_CHECK(env, input_handle, "input_handle is null", 0); + if (limit == 0 || limit == 1) { + // Cannot achieve the results of splitting with limit == 0 or limit == 1. + // This is because cudf operates on a different parameter (`max_split`) which is converted from + // limit. When limit == 0 or limit == 1, max_split will be non-positive and will result in an + // unlimited split. + JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + "limit == 0 and limit == 1 are not supported", 0); + } + + try { + cudf::jni::auto_set_device(env); + auto const input = reinterpret_cast(input_handle); + auto const strings_column = cudf::strings_column_view{*input}; + auto const pattern_jstr = cudf::jni::native_jstring(env, pattern_obj); auto const pattern = std::string(pattern_jstr.get(), pattern_jstr.size_bytes()); auto const max_split = limit > 1 ? limit - 1 : limit; - auto result = split_by_regex ? - cudf::strings::split_re(strs_input, pattern, max_split) : - cudf::strings::split(strs_input, cudf::string_scalar{pattern}, max_split); + auto const flags = static_cast(regex_flags); + auto const groups = static_cast(capture_groups); + auto const regex_prog = cudf::strings::regex_program::create(pattern, flags, groups); + auto result = cudf::strings::split_re(strings_column, *regex_prog, max_split); return cudf::jni::convert_table_for_return(env, std::move(result)); } CATCH_STD(env, 0); @@ -719,9 +739,8 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_ColumnView_stringSplit(JNIEnv * JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_stringSplitRecord(JNIEnv *env, jclass, jlong input_handle, - jstring pattern_obj, - jint limit, - jboolean split_by_regex) { + jstring delimiter_obj, + jint limit) { JNI_NULL_CHECK(env, input_handle, "input_handle is null", 0); if (limit == 0 || limit == 1) { @@ -735,22 +754,43 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_stringSplitRecord(JNIEnv try { cudf::jni::auto_set_device(env); - auto const input = reinterpret_cast(input_handle); - auto const strs_input = cudf::strings_column_view{*input}; + auto const input = reinterpret_cast(input_handle); + auto const strings_column = cudf::strings_column_view{*input}; + auto const delimiter_jstr = cudf::jni::native_jstring(env, delimiter_obj); + auto const delimiter = std::string(delimiter_jstr.get(), delimiter_jstr.size_bytes()); + auto const max_split = limit > 1 ? limit - 1 : limit; + auto result = + cudf::strings::split_record(strings_column, cudf::string_scalar{delimiter}, max_split); + return release_as_jlong(result); + } + CATCH_STD(env, 0); +} - auto const pattern_jstr = cudf::jni::native_jstring(env, pattern_obj); - if (pattern_jstr.is_empty()) { - // Java's split API produces different behaviors than cudf when splitting with empty - // pattern. - JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", "Empty pattern is not supported", 0); - } +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_stringSplitRecordRe( + JNIEnv *env, jclass, jlong input_handle, jstring pattern_obj, jint regex_flags, + jint capture_groups, jint limit) { + JNI_NULL_CHECK(env, input_handle, "input_handle is null", 0); + if (limit == 0 || limit == 1) { + // Cannot achieve the results of splitting with limit == 0 or limit == 1. + // This is because cudf operates on a different parameter (`max_split`) which is converted from + // limit. When limit == 0 or limit == 1, max_split will be non-positive and will result in an + // unlimited split. + JNI_THROW_NEW(env, "java/lang/IllegalArgumentException", + "limit == 0 and limit == 1 are not supported", 0); + } + + try { + cudf::jni::auto_set_device(env); + auto const input = reinterpret_cast(input_handle); + auto const strings_column = cudf::strings_column_view{*input}; + auto const pattern_jstr = cudf::jni::native_jstring(env, pattern_obj); auto const pattern = std::string(pattern_jstr.get(), pattern_jstr.size_bytes()); auto const max_split = limit > 1 ? limit - 1 : limit; - auto result = - split_by_regex ? - cudf::strings::split_record_re(strs_input, pattern, max_split) : - cudf::strings::split_record(strs_input, cudf::string_scalar{pattern}, max_split); + auto const flags = static_cast(regex_flags); + auto const groups = static_cast(capture_groups); + auto const regex_prog = cudf::strings::regex_program::create(pattern, flags, groups); + auto result = cudf::strings::split_record_re(strings_column, *regex_prog, max_split); return release_as_jlong(result); } CATCH_STD(env, 0); diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 46264b7d668..ab4baf74277 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -4990,28 +4990,29 @@ void testReverseList() { void testStringSplit() { String pattern = " "; try (ColumnVector v = ColumnVector.fromStrings("Héllo there all", "thésé", null, "", - "ARé some things", "test strings here"); + "ARé some things", "test strings here"); Table expectedSplitLimit2 = new Table.TestBuilder() - .column("Héllo", "thésé", null, "", "ARé", "test") - .column("there all", null, null, null, "some things", "strings here") - .build(); + .column("Héllo", "thésé", null, "", "ARé", "test") + .column("there all", null, null, null, "some things", "strings here") + .build(); Table expectedSplitAll = new Table.TestBuilder() - .column("Héllo", "thésé", null, "", "ARé", "test") - .column("there", null, null, null, "some", "strings") - .column("all", null, null, null, "things", "here") - .build(); + .column("Héllo", "thésé", null, "", "ARé", "test") + .column("there", null, null, null, "some", "strings") + .column("all", null, null, null, "things", "here") + .build(); Table resultSplitLimit2 = v.stringSplit(pattern, 2); Table resultSplitAll = v.stringSplit(pattern)) { - assertTablesAreEqual(expectedSplitLimit2, resultSplitLimit2); - assertTablesAreEqual(expectedSplitAll, resultSplitAll); + assertTablesAreEqual(expectedSplitLimit2, resultSplitLimit2); + assertTablesAreEqual(expectedSplitAll, resultSplitAll); } } @Test void testStringSplitByRegularExpression() { String pattern = "[_ ]"; + RegexProgram regexProg = new RegexProgram(pattern, CaptureGroups.NON_CAPTURE); try (ColumnVector v = ColumnVector.fromStrings("Héllo_there all", "thésé", null, "", - "ARé some_things", "test_strings_here"); + "ARé some_things", "test_strings_here"); Table expectedSplitLimit2 = new Table.TestBuilder() .column("Héllo", "thésé", null, "", "ARé", "test") .column("there all", null, null, null, "some_things", "strings_here") @@ -5020,11 +5021,17 @@ void testStringSplitByRegularExpression() { .column("Héllo", "thésé", null, "", "ARé", "test") .column("there", null, null, null, "some", "strings") .column("all", null, null, null, "things", "here") - .build(); - Table resultSplitLimit2 = v.stringSplit(pattern, 2, true); - Table resultSplitAll = v.stringSplit(pattern, true)) { - assertTablesAreEqual(expectedSplitLimit2, resultSplitLimit2); - assertTablesAreEqual(expectedSplitAll, resultSplitAll); + .build()) { + try (Table resultSplitLimit2 = v.stringSplit(pattern, 2, true); + Table resultSplitAll = v.stringSplit(pattern, true)) { + assertTablesAreEqual(expectedSplitLimit2, resultSplitLimit2); + assertTablesAreEqual(expectedSplitAll, resultSplitAll); + } + try (Table resultSplitLimit2 = v.stringSplit(regexProg, 2); + Table resultSplitAll = v.stringSplit(regexProg)) { + assertTablesAreEqual(expectedSplitLimit2, resultSplitLimit2); + assertTablesAreEqual(expectedSplitAll, resultSplitAll); + } } } @@ -5032,7 +5039,7 @@ void testStringSplitByRegularExpression() { void testStringSplitRecord() { String pattern = " "; try (ColumnVector v = ColumnVector.fromStrings("Héllo there all", "thésé", null, "", - "ARé some things", "test strings here"); + "ARé some things", "test strings here"); ColumnVector expectedSplitLimit2 = ColumnVector.fromLists( new HostColumnVector.ListType(true, new HostColumnVector.BasicType(true, DType.STRING)), @@ -5061,8 +5068,9 @@ void testStringSplitRecord() { @Test void testStringSplitRecordByRegularExpression() { String pattern = "[_ ]"; + RegexProgram regexProg = new RegexProgram(pattern, CaptureGroups.NON_CAPTURE); try (ColumnVector v = ColumnVector.fromStrings("Héllo_there all", "thésé", null, "", - "ARé some_things", "test_strings_here"); + "ARé some_things", "test_strings_here"); ColumnVector expectedSplitLimit2 = ColumnVector.fromLists( new HostColumnVector.ListType(true, new HostColumnVector.BasicType(true, DType.STRING)), @@ -5080,11 +5088,17 @@ void testStringSplitRecordByRegularExpression() { null, Arrays.asList(""), Arrays.asList("ARé", "some", "things"), - Arrays.asList("test", "strings", "here")); - ColumnVector resultSplitLimit2 = v.stringSplitRecord(pattern, 2, true); - ColumnVector resultSplitAll = v.stringSplitRecord(pattern, true)) { - assertColumnsAreEqual(expectedSplitLimit2, resultSplitLimit2); - assertColumnsAreEqual(expectedSplitAll, resultSplitAll); + Arrays.asList("test", "strings", "here"))) { + try (ColumnVector resultSplitLimit2 = v.stringSplitRecord(pattern, 2, true); + ColumnVector resultSplitAll = v.stringSplitRecord(pattern, true)) { + assertColumnsAreEqual(expectedSplitLimit2, resultSplitLimit2); + assertColumnsAreEqual(expectedSplitAll, resultSplitAll); + } + try (ColumnVector resultSplitLimit2 = v.stringSplitRecord(regexProg, 2); + ColumnVector resultSplitAll = v.stringSplitRecord(regexProg)) { + assertColumnsAreEqual(expectedSplitLimit2, resultSplitLimit2); + assertColumnsAreEqual(expectedSplitAll, resultSplitAll); + } } } From b8ae0e4b41c541c5b2b27417af30fa1b9afcbdce Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Tue, 7 Feb 2023 23:33:56 -0600 Subject: [PATCH 04/69] Support conversion to/from cudf in dask.dataframe.core.to_backend (#12380) This PR corresponds to the `cudf` component of https://github.com/dask/dask/pull/9758 Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12380 --- python/dask_cudf/dask_cudf/backends.py | 194 +++++++++++------- python/dask_cudf/dask_cudf/tests/test_core.py | 55 ++++- 2 files changed, 170 insertions(+), 79 deletions(-) diff --git a/python/dask_cudf/dask_cudf/backends.py b/python/dask_cudf/dask_cudf/backends.py index b6be5ade6ba..821ec103204 100644 --- a/python/dask_cudf/dask_cudf/backends.py +++ b/python/dask_cudf/dask_cudf/backends.py @@ -11,6 +11,10 @@ import dask.dataframe as dd from dask import config +from dask.dataframe.backends import ( + DataFrameBackendEntrypoint, + PandasBackendEntrypoint, +) from dask.dataframe.core import get_parallel_type, meta_nonempty from dask.dataframe.dispatch import ( categorical_dtype_dispatch, @@ -30,7 +34,7 @@ make_meta_obj, ) from dask.sizeof import sizeof as sizeof_dispatch -from dask.utils import is_arraylike +from dask.utils import Dispatch, is_arraylike import cudf from cudf.api.types import is_string_dtype @@ -446,91 +450,127 @@ def _default_backend(func, *args, **kwargs): return func(*args, **kwargs) -try: +def _unsupported_kwargs(old, new, kwargs): + # Utility to raise a meaningful error when + # unsupported kwargs are encountered within + # ``to_backend_dispatch`` + if kwargs: + raise ValueError( + f"Unsupported key-word arguments used in `to_backend` " + f"for {old}-to-{new} conversion: {kwargs}" + ) - # Define "cudf" backend engine to be registered with Dask - from dask.dataframe.backends import DataFrameBackendEntrypoint - - class CudfBackendEntrypoint(DataFrameBackendEntrypoint): - """Backend-entrypoint class for Dask-DataFrame - - This class is registered under the name "cudf" for the - ``dask.dataframe.backends`` entrypoint in ``setup.cfg``. - Dask-DataFrame will use the methods defined in this class - in place of ``dask.dataframe.`` when the - "dataframe.backend" configuration is set to "cudf": - - Examples - -------- - >>> import dask - >>> import dask.dataframe as dd - >>> with dask.config.set({"dataframe.backend": "cudf"}): - ... ddf = dd.from_dict({"a": range(10)}) - >>> type(ddf) - - """ - - @staticmethod - def from_dict( - data, - npartitions, - orient="columns", - dtype=None, - columns=None, - constructor=cudf.DataFrame, - ): - - return _default_backend( - dd.from_dict, - data, - npartitions=npartitions, - orient=orient, - dtype=dtype, - columns=columns, - constructor=constructor, - ) - @staticmethod - def read_parquet(*args, engine=None, **kwargs): - from dask_cudf.io.parquet import CudfEngine +# Register cudf->pandas +to_pandas_dispatch = PandasBackendEntrypoint.to_backend_dispatch() - return _default_backend( - dd.read_parquet, - *args, - engine=CudfEngine, - **kwargs, - ) - @staticmethod - def read_json(*args, **kwargs): - from dask_cudf.io.json import read_json +@to_pandas_dispatch.register((cudf.DataFrame, cudf.Series, cudf.Index)) +def to_pandas_dispatch_from_cudf(data, nullable=False, **kwargs): + _unsupported_kwargs("cudf", "pandas", kwargs) + return data.to_pandas(nullable=nullable) - return read_json(*args, **kwargs) - @staticmethod - def read_orc(*args, **kwargs): - from dask_cudf.io import read_orc +# Register pandas->cudf +to_cudf_dispatch = Dispatch("to_cudf_dispatch") - return read_orc(*args, **kwargs) - @staticmethod - def read_csv(*args, **kwargs): - from dask_cudf.io import read_csv +@to_cudf_dispatch.register((pd.DataFrame, pd.Series, pd.Index)) +def to_cudf_dispatch_from_pandas(data, nan_as_null=None, **kwargs): + _unsupported_kwargs("pandas", "cudf", kwargs) + return cudf.from_pandas(data, nan_as_null=nan_as_null) - return read_csv(*args, **kwargs) - @staticmethod - def read_hdf(*args, **kwargs): - from dask_cudf import from_dask_dataframe +# Define "cudf" backend engine to be registered with Dask +class CudfBackendEntrypoint(DataFrameBackendEntrypoint): + """Backend-entrypoint class for Dask-DataFrame - # HDF5 reader not yet implemented in cudf - warnings.warn( - "read_hdf is not yet implemented in cudf/dask_cudf. " - "Moving to cudf from pandas. Expect poor performance!" - ) - return from_dask_dataframe( - _default_backend(dd.read_hdf, *args, **kwargs) - ) + This class is registered under the name "cudf" for the + ``dask.dataframe.backends`` entrypoint in ``setup.cfg``. + Dask-DataFrame will use the methods defined in this class + in place of ``dask.dataframe.`` when the + "dataframe.backend" configuration is set to "cudf": -except ImportError: - pass + Examples + -------- + >>> import dask + >>> import dask.dataframe as dd + >>> with dask.config.set({"dataframe.backend": "cudf"}): + ... ddf = dd.from_dict({"a": range(10)}) + >>> type(ddf) + + """ + + @classmethod + def to_backend_dispatch(cls): + return to_cudf_dispatch + + @classmethod + def to_backend(cls, data: dd.core._Frame, **kwargs): + if isinstance(data._meta, (cudf.DataFrame, cudf.Series, cudf.Index)): + # Already a cudf-backed collection + _unsupported_kwargs("cudf", "cudf", kwargs) + return data + return data.map_partitions(cls.to_backend_dispatch(), **kwargs) + + @staticmethod + def from_dict( + data, + npartitions, + orient="columns", + dtype=None, + columns=None, + constructor=cudf.DataFrame, + ): + + return _default_backend( + dd.from_dict, + data, + npartitions=npartitions, + orient=orient, + dtype=dtype, + columns=columns, + constructor=constructor, + ) + + @staticmethod + def read_parquet(*args, engine=None, **kwargs): + from dask_cudf.io.parquet import CudfEngine + + return _default_backend( + dd.read_parquet, + *args, + engine=CudfEngine, + **kwargs, + ) + + @staticmethod + def read_json(*args, **kwargs): + from dask_cudf.io.json import read_json + + return read_json(*args, **kwargs) + + @staticmethod + def read_orc(*args, **kwargs): + from dask_cudf.io import read_orc + + return read_orc(*args, **kwargs) + + @staticmethod + def read_csv(*args, **kwargs): + from dask_cudf.io import read_csv + + return read_csv(*args, **kwargs) + + @staticmethod + def read_hdf(*args, **kwargs): + from dask_cudf import from_dask_dataframe + + # HDF5 reader not yet implemented in cudf + warnings.warn( + "read_hdf is not yet implemented in cudf/dask_cudf. " + "Moving to cudf from pandas. Expect poor performance!" + ) + return from_dask_dataframe( + _default_backend(dd.read_hdf, *args, **kwargs) + ) diff --git a/python/dask_cudf/dask_cudf/tests/test_core.py b/python/dask_cudf/dask_cudf/tests/test_core.py index ee8229bc7e8..7f8876c8564 100644 --- a/python/dask_cudf/dask_cudf/tests/test_core.py +++ b/python/dask_cudf/dask_cudf/tests/test_core.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd import pytest +from packaging import version import dask from dask import dataframe as dd @@ -31,6 +32,58 @@ def test_from_dict_backend_dispatch(): dd.assert_eq(expect, ddf) +def test_to_backend(): + np.random.seed(0) + data = { + "x": np.random.randint(0, 5, size=10000), + "y": np.random.normal(size=10000), + } + with dask.config.set({"dataframe.backend": "pandas"}): + ddf = dd.from_dict(data, npartitions=2) + assert isinstance(ddf._meta, pd.DataFrame) + + gdf = ddf.to_backend("cudf") + assert isinstance(gdf, dgd.DataFrame) + dd.assert_eq(cudf.DataFrame(data), ddf) + + assert isinstance(gdf.to_backend()._meta, pd.DataFrame) + + +def test_to_backend_kwargs(): + data = {"x": [0, 2, np.nan, 3, 4, 5]} + with dask.config.set({"dataframe.backend": "pandas"}): + dser = dd.from_dict(data, npartitions=2)["x"] + assert isinstance(dser._meta, pd.Series) + + # Using `nan_as_null=False` will result in a cudf-backed + # Series with a NaN element (ranther than ) + gser_nan = dser.to_backend("cudf", nan_as_null=False) + assert isinstance(gser_nan, dgd.Series) + assert np.isnan(gser_nan.compute()).sum() == 1 + + # Using `nan_as_null=True` will result in a cudf-backed + # Series with a element (ranther than NaN) + gser_null = dser.to_backend("cudf", nan_as_null=True) + assert isinstance(gser_null, dgd.Series) + assert np.isnan(gser_null.compute()).sum() == 0 + + # Check `nullable` argument for `cudf.Series.to_pandas` + dser_null = gser_null.to_backend("pandas", nullable=False) + assert dser_null.compute().dtype == "float" + dser_null = gser_null.to_backend("pandas", nullable=True) + assert isinstance(dser_null.compute().dtype, pd.Float64Dtype) + + # Check unsupported arguments + with pytest.raises(ValueError, match="pandas-to-cudf"): + dser.to_backend("cudf", bad_arg=True) + + with pytest.raises(ValueError, match="cudf-to-cudf"): + gser_null.to_backend("cudf", bad_arg=True) + + with pytest.raises(ValueError, match="cudf-to-pandas"): + gser_null.to_backend("pandas", bad_arg=True) + + def test_from_cudf(): np.random.seed(0) @@ -547,8 +600,6 @@ def test_unary_ops(func, gdf, gddf): # Fixed in https://github.com/dask/dask/pull/4657 if isinstance(p, cudf.Index): - from packaging import version - if version.parse(dask.__version__) < version.parse("1.1.6"): pytest.skip( "dask.dataframe assert_eq index check hardcoded to " From 8ad4166c7026482a53a60f47b56dd5e1dec1a463 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:39:42 -0500 Subject: [PATCH 05/69] Remove cudf::strings::repeat_strings_output_sizes and optional parameter from cudf::strings::repeat_strings (#12609) Removes `cudf::strings::repeat_strings_output_sizes` and the optional sizes parameter from `cudf::strings::repeat_strings`. This function (and corresponding optional parameter) is no longer needed now that the internal utilities will throw an error if the column output size exceeds the maximum. Closes #12542 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Karthikeyan (https://github.com/karthikeyann) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12609 --- cpp/benchmarks/string/repeat_strings.cpp | 56 +----- cpp/include/cudf/strings/repeat_strings.hpp | 94 +++------- cpp/src/strings/repeat_strings.cu | 194 +++----------------- cpp/tests/strings/repeat_strings_tests.cpp | 121 +----------- 4 files changed, 56 insertions(+), 409 deletions(-) diff --git a/cpp/benchmarks/string/repeat_strings.cpp b/cpp/benchmarks/string/repeat_strings.cpp index 1844e93bc53..fe015b27f13 100644 --- a/cpp/benchmarks/string/repeat_strings.cpp +++ b/cpp/benchmarks/string/repeat_strings.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,42 +79,6 @@ static void BM_repeat_strings_column_times(benchmark::State& state) (strings_col.chars_size() + repeat_times_col.size() * sizeof(int32_t))); } -static void BM_compute_output_strings_sizes(benchmark::State& state) -{ - auto const n_rows = static_cast(state.range(0)); - auto const max_str_length = static_cast(state.range(1)); - auto const table = create_data_table(2, n_rows, max_str_length); - auto const strings_col = cudf::strings_column_view(table->view().column(0)); - auto const repeat_times_col = table->view().column(1); - - for ([[maybe_unused]] auto _ : state) { - [[maybe_unused]] cuda_event_timer raii(state, true, cudf::get_default_stream()); - cudf::strings::repeat_strings_output_sizes(strings_col, repeat_times_col); - } - - state.SetBytesProcessed(state.iterations() * - (strings_col.chars_size() + repeat_times_col.size() * sizeof(int32_t))); -} - -static void BM_repeat_strings_column_times_precomputed_sizes(benchmark::State& state) -{ - auto const n_rows = static_cast(state.range(0)); - auto const max_str_length = static_cast(state.range(1)); - auto const table = create_data_table(2, n_rows, max_str_length); - auto const strings_col = cudf::strings_column_view(table->view().column(0)); - auto const repeat_times_col = table->view().column(1); - [[maybe_unused]] auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(strings_col, repeat_times_col); - - for ([[maybe_unused]] auto _ : state) { - [[maybe_unused]] cuda_event_timer raii(state, true, cudf::get_default_stream()); - cudf::strings::repeat_strings(strings_col, repeat_times_col, *sizes); - } - - state.SetBytesProcessed(state.iterations() * - (strings_col.chars_size() + repeat_times_col.size() * sizeof(int32_t))); -} - static void generate_bench_args(benchmark::internal::Benchmark* b) { int const min_rows = 1 << 8; @@ -145,23 +109,5 @@ class RepeatStrings : public cudf::benchmark { ->UseManualTime() \ ->Unit(benchmark::kMillisecond); -#define COMPUTE_OUTPUT_STRINGS_SIZES_BENCHMARK_DEFINE(name) \ - BENCHMARK_DEFINE_F(RepeatStrings, name) \ - (::benchmark::State & st) { BM_compute_output_strings_sizes(st); } \ - BENCHMARK_REGISTER_F(RepeatStrings, name) \ - ->Apply(generate_bench_args) \ - ->UseManualTime() \ - ->Unit(benchmark::kMillisecond); - -#define REPEAT_STRINGS_COLUMN_TIMES_PRECOMPUTED_SIZES_BENCHMARK_DEFINE(name) \ - BENCHMARK_DEFINE_F(RepeatStrings, name) \ - (::benchmark::State & st) { BM_repeat_strings_column_times_precomputed_sizes(st); } \ - BENCHMARK_REGISTER_F(RepeatStrings, name) \ - ->Apply(generate_bench_args) \ - ->UseManualTime() \ - ->Unit(benchmark::kMillisecond); - REPEAT_STRINGS_SCALAR_TIMES_BENCHMARK_DEFINE(scalar_times) REPEAT_STRINGS_COLUMN_TIMES_BENCHMARK_DEFINE(column_times) -COMPUTE_OUTPUT_STRINGS_SIZES_BENCHMARK_DEFINE(compute_output_strings_sizes) -REPEAT_STRINGS_COLUMN_TIMES_PRECOMPUTED_SIZES_BENCHMARK_DEFINE(precomputed_sizes) diff --git a/cpp/include/cudf/strings/repeat_strings.hpp b/cpp/include/cudf/strings/repeat_strings.hpp index 0e6ee2126d3..26fe5f95983 100644 --- a/cpp/include/cudf/strings/repeat_strings.hpp +++ b/cpp/include/cudf/strings/repeat_strings.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,15 +32,15 @@ namespace strings { */ /** - * @brief Repeat the given string scalar by a given number of times. + * @brief Repeat the given string scalar a given number of times * * An output string scalar is generated by repeating the input string by a number of times given by - * the @p `repeat_times` parameter. + * the `repeat_times` parameter. * * In special cases: - * - If @p `repeat_times` is not a positive value, an empty (valid) string scalar will be returned. + * - If `repeat_times` is not a positive value, an empty (valid) string scalar will be returned. * - An invalid input scalar will always result in an invalid output scalar regardless of the - * value of @p `repeat_times` parameter. + * value of `repeat_times` parameter. * * @code{.pseudo} * Example: @@ -50,13 +50,13 @@ namespace strings { * @endcode * * @throw cudf::logic_error if the size of the output string scalar exceeds the maximum value that - * can be stored by the index type - * (i.e., @code input.size() * repeat_times > numeric_limits::max() @endcode). + * can be stored by the index type: + * `input.size() * repeat_times > max of size_type` * - * @param input The scalar containing the string to repeat. - * @param repeat_times The number of times the input string is repeated. - * @param mr Device memory resource used to allocate the returned string scalar. - * @return New string scalar in which the input string is repeated. + * @param input The scalar containing the string to repeat + * @param repeat_times The number of times the input string is repeated + * @param mr Device memory resource used to allocate the returned string scalar + * @return New string scalar in which the input string is repeated */ std::unique_ptr repeat_string( string_scalar const& input, @@ -64,19 +64,16 @@ std::unique_ptr repeat_string( rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); /** - * @brief Repeat each string in the given strings column by a given number of times. + * @brief Repeat each string in the given strings column a given number of times * - * An output strings column is generated by repeating each string from the input strings column by a - * number of times given by the @p `repeat_times` parameter. + * An output strings column is generated by repeating each string from the input strings column by + * the number of times given by the `repeat_times` parameter. * * In special cases: - * - If @p `repeat_times` is not a positive number, a non-null input string will always result in + * - If `repeat_times` is not a positive number, a non-null input string will always result in * an empty output string. * - A null input string will always result in a null output string regardless of the value of the - * @p `repeat_times` parameter. - * - * The caller is responsible for checking the output column size will not exceed the maximum size of - * a strings column (number of total characters is less than the max size_type value). + * `repeat_times` parameter. * * @code{.pseudo} * Example: @@ -85,10 +82,10 @@ std::unique_ptr repeat_string( * out is ['aaaaaa', null, '', 'bbcbbcbbc'] * @endcode * - * @param input The column containing strings to repeat. - * @param repeat_times The number of times each input string is repeated. - * @param mr Device memory resource used to allocate the returned strings column. - * @return New column containing the repeated strings. + * @param input The column containing strings to repeat + * @param repeat_times The number of times each input string is repeated + * @param mr Device memory resource used to allocate the returned strings column + * @return New column containing the repeated strings */ std::unique_ptr repeat_strings( strings_column_view const& input, @@ -97,11 +94,10 @@ std::unique_ptr repeat_strings( /** * @brief Repeat each string in the given strings column by the numbers of times given in another - * numeric column. + * numeric column * * An output strings column is generated by repeating each of the input string by a number of times - * given by the corresponding row in a @p `repeat_times` numeric column. The computational time can - * be reduced if sizes of the output strings are known and provided. + * given by the corresponding row in a `repeat_times` numeric column. * * In special cases: * - Any null row (from either the input strings column or the `repeat_times` column) will always @@ -109,9 +105,6 @@ std::unique_ptr repeat_strings( * - If any value in the `repeat_times` column is not a positive number and its corresponding input * string is not null, the output string will be an empty string. * - * The caller is responsible for checking the output column size will not exceed the maximum size of - * a strings column (number of total characters is less than the max size_type value). - * * @code{.pseudo} * Example: * strs = ['aa', null, '', 'bbc-'] @@ -120,51 +113,16 @@ std::unique_ptr repeat_strings( * out is ['aa', null, '', 'bbc-bbc-bbc-bbc-'] * @endcode * - * @throw cudf::logic_error if the input `repeat_times` column has data type other than integer. + * @throw cudf::logic_error if the input `repeat_times` is not an integer type * @throw cudf::logic_error if the input columns have different sizes. * - * @param input The column containing strings to repeat. + * @param input The column containing strings to repeat * @param repeat_times The column containing numbers of times that the corresponding input strings - * are repeated. - * @param output_strings_sizes The optional column containing pre-computed sizes of the output - * strings. - * @param mr Device memory resource used to allocate the returned strings column. + * are repeated + * @param mr Device memory resource used to allocate the returned strings column * @return New column containing the repeated strings. */ std::unique_ptr repeat_strings( - strings_column_view const& input, - column_view const& repeat_times, - std::optional output_strings_sizes = std::nullopt, - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); - -/** - * @brief Compute sizes of the output strings if each string in the input strings column - * is repeated by the numbers of times given in another numeric column. - * - * The output column storing string output sizes is not nullable. These string sizes are - * also summed up and returned (in an `int64_t` value), which can be used to detect if the input - * strings column can be safely repeated without data corruption due to overflow in string indexing. - * - * @code{.pseudo} - * Example: - * strs = ['aa', null, '', 'bbc-'] - * repeat_times = [ 1, 2, 3, 4 ] - * [output_sizes, total_size] = repeat_strings_output_sizes(strs, repeat_times) - * out is [2, 0, 0, 16], and total_size = 18 - * @endcode - * - * @throw cudf::logic_error if the input `repeat_times` column has data type other than integer. - * @throw cudf::logic_error if the input columns have different sizes. - * - * @param input The column containing strings to repeat. - * @param repeat_times The column containing numbers of times that the corresponding input strings - * are repeated. - * @param mr Device memory resource used to allocate the returned strings column. - * @return A pair with the first item is an int32_t column containing sizes of the output strings, - * and the second item is an int64_t number containing the total sizes (in bytes) of the - * output strings column. - */ -std::pair, int64_t> repeat_strings_output_sizes( strings_column_view const& input, column_view const& repeat_times, rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()); diff --git a/cpp/src/strings/repeat_strings.cu b/cpp/src/strings/repeat_strings.cu index cc283fbcee2..3784b535a5b 100644 --- a/cpp/src/strings/repeat_strings.cu +++ b/cpp/src/strings/repeat_strings.cu @@ -176,7 +176,7 @@ namespace { * separate number of times. */ template -struct compute_size_and_repeat_separately_fn { +struct compute_sizes_and_repeat_fn { column_device_view const strings_dv; column_device_view const repeat_times_dv; Iterator const repeat_times_iter; @@ -189,146 +189,63 @@ struct compute_size_and_repeat_separately_fn { // If d_chars != nullptr: only repeat strings. char* d_chars{nullptr}; - __device__ int64_t operator()(size_type const idx) const noexcept + __device__ void operator()(size_type const idx) const noexcept { auto const string_is_valid = !strings_has_nulls || strings_dv.is_valid_nocheck(idx); auto const rtimes_is_valid = !rtimes_has_nulls || repeat_times_dv.is_valid_nocheck(idx); // Any null input (either string or repeat_times value) will result in a null output. auto const is_valid = string_is_valid && rtimes_is_valid; + if (!is_valid) { + if (!d_chars) { d_offsets[idx] = 0; } + return; + } - // When the input string is null, `repeat_times` and `string_size` are also set to 0. - // This makes sure that if `repeat_times > 0` then we will always have a valid input string, - // and if `repeat_times <= 0` we will never copy anything to the output. - auto const repeat_times = is_valid ? repeat_times_iter[idx] : size_type{0}; - auto const string_size = - is_valid ? strings_dv.element(idx).size_bytes() : size_type{0}; - - // The output_size is returned, and it needs to be an int64_t number to prevent overflow. - auto const output_size = - repeat_times > 0 ? static_cast(repeat_times) * static_cast(string_size) - : int64_t{0}; + auto repeat_times = repeat_times_iter[idx]; + auto const d_str = strings_dv.element(idx); if (!d_chars) { - // If overflow happen, the stored value of output string size will be incorrect due to - // downcasting. In such cases, the entire output string size array should be discarded. - d_offsets[idx] = static_cast(output_size); - } else if (repeat_times > 0 && string_size > 0) { - auto const d_str = strings_dv.element(idx); - auto const input_ptr = d_str.data(); - auto output_ptr = d_chars + d_offsets[idx]; - for (size_type repeat_idx = 0; repeat_idx < repeat_times; ++repeat_idx) { - output_ptr = copy_and_increment(output_ptr, input_ptr, string_size); + // repeat_times could be negative + d_offsets[idx] = (repeat_times > 0) ? (repeat_times * d_str.size_bytes()) : 0; + } else { + auto output_ptr = d_chars + d_offsets[idx]; + while (repeat_times-- > 0) { + output_ptr = copy_and_increment(output_ptr, d_str.data(), d_str.size_bytes()); } } - - // The output_size value may be used to sum up to detect overflow at the caller site. - // The caller can detect overflow easily by checking `SUM(output_size) > INT_MAX`. - return output_size; } }; -/** - * @brief Creates child offsets and chars columns by applying the template function that - * can be used for computing the output size of each string as well as create the output. - * - * This function is similar to `strings::detail::make_strings_children`, except that it accepts an - * optional input `std::optional` that can contain the precomputed sizes of the output - * strings. - * - * @deprecated This will be removed with issue 12542 - */ -template -auto make_strings_children(Func fn, - size_type exec_size, - size_type strings_count, - std::optional output_strings_sizes, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - auto offsets_column = make_numeric_column( - data_type{type_id::INT32}, strings_count + 1, mask_state::UNALLOCATED, stream, mr); - - auto offsets_view = offsets_column->mutable_view(); - auto d_offsets = offsets_view.template data(); - fn.d_offsets = d_offsets; - - // This may be called twice -- once for offsets and once for chars. - auto for_each_fn = [exec_size, stream](Func& fn) { - thrust::for_each_n( - rmm::exec_policy(stream), thrust::make_counting_iterator(0), exec_size, fn); - }; - - if (!output_strings_sizes.has_value()) { - // Compute the output sizes only if they are not given. - for_each_fn(fn); - - // Compute the offsets values. - auto const bytes = - cudf::detail::sizes_to_offsets(d_offsets, d_offsets + strings_count + 1, d_offsets, stream); - CUDF_EXPECTS(bytes <= static_cast(std::numeric_limits::max()), - "Size of output exceeds column size limit"); - } else { - // Compute the offsets values from the provided output string sizes. - auto const string_sizes = output_strings_sizes.value(); - CUDF_CUDA_TRY(cudaMemsetAsync(d_offsets, 0, sizeof(offset_type), stream.value())); - thrust::inclusive_scan(rmm::exec_policy(stream), - string_sizes.template begin(), - string_sizes.template end(), - d_offsets + 1); - } - - // Now build the chars column - auto const bytes = cudf::detail::get_value(offsets_view, strings_count, stream); - auto chars_column = create_chars_child_column(bytes, stream, mr); - - // Execute the function fn again to fill the chars column. - // Note that if the output chars column has zero size, the function fn should not be called to - // avoid accidentally overwriting the offsets. - if (bytes > 0) { - fn.d_chars = chars_column->mutable_view().template data(); - for_each_fn(fn); - } - - return std::pair(std::move(offsets_column), std::move(chars_column)); -} - } // namespace std::unique_ptr repeat_strings(strings_column_view const& input, column_view const& repeat_times, - std::optional output_strings_sizes, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { CUDF_EXPECTS(input.size() == repeat_times.size(), "The input columns must have the same size."); CUDF_EXPECTS(cudf::is_index_type(repeat_times.type()), "repeat_strings expects an integer type for the `repeat_times` input column."); - if (output_strings_sizes.has_value()) { - auto const output_sizes = output_strings_sizes.value(); - CUDF_EXPECTS(input.size() == output_sizes.size() && - (!output_sizes.nullable() || !output_sizes.has_nulls()), - "The given column of output string sizes is invalid."); - } auto const strings_count = input.size(); if (strings_count == 0) { return make_empty_column(type_id::STRING); } auto const strings_dv_ptr = column_device_view::create(input.parent(), stream); auto const repeat_times_dv_ptr = column_device_view::create(repeat_times, stream); - auto const strings_has_nulls = input.has_nulls(); - auto const rtimes_has_nulls = repeat_times.has_nulls(); auto const repeat_times_iter = cudf::detail::indexalator_factory::make_input_iterator(repeat_times); - auto const fn = compute_size_and_repeat_separately_fn{ - *strings_dv_ptr, *repeat_times_dv_ptr, repeat_times_iter, strings_has_nulls, rtimes_has_nulls}; - - auto [offsets_column, chars_column] = - make_strings_children(fn, strings_count, strings_count, output_strings_sizes, stream, mr); - - // We generate new bitmask by AND of the input columns' bitmasks. - // Note that if the input columns are nullable, the output column will also be nullable (which may - // not have nulls). + auto const fn = + compute_sizes_and_repeat_fn{*strings_dv_ptr, + *repeat_times_dv_ptr, + repeat_times_iter, + input.has_nulls(), + repeat_times.has_nulls()}; + + auto [offsets_column, chars_column] = make_strings_children(fn, strings_count, stream, mr); + + // We generate new bitmask by AND of the two input columns' bitmasks. + // Note that if either of the input columns are nullable, the output column will also be nullable + // but may not have nulls. auto [null_mask, null_count] = cudf::detail::bitmask_and(table_view{{input.parent(), repeat_times}}, stream, mr); @@ -338,52 +255,6 @@ std::unique_ptr repeat_strings(strings_column_view const& input, null_count, std::move(null_mask)); } - -std::pair, int64_t> repeat_strings_output_sizes( - strings_column_view const& input, - column_view const& repeat_times, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_EXPECTS(input.size() == repeat_times.size(), "The input columns must have the same size."); - CUDF_EXPECTS( - cudf::is_index_type(repeat_times.type()), - "repeat_strings_output_sizes expects an integer type for the `repeat_times` input column."); - - auto const strings_count = input.size(); - if (strings_count == 0) { - return std::pair(make_empty_column(type_to_id()), int64_t{0}); - } - - auto output_sizes = make_numeric_column( - data_type{type_to_id()}, strings_count, mask_state::UNALLOCATED, stream, mr); - - auto const strings_dv_ptr = column_device_view::create(input.parent(), stream); - auto const repeat_times_dv_ptr = column_device_view::create(repeat_times, stream); - auto const strings_has_nulls = input.has_nulls(); - auto const rtimes_has_nulls = repeat_times.has_nulls(); - auto const repeat_times_iter = - cudf::detail::indexalator_factory::make_input_iterator(repeat_times); - - auto const fn = compute_size_and_repeat_separately_fn{ - *strings_dv_ptr, - *repeat_times_dv_ptr, - repeat_times_iter, - strings_has_nulls, - rtimes_has_nulls, - output_sizes->mutable_view().template begin()}; - - auto const total_bytes = - thrust::transform_reduce(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(strings_count), - fn, - int64_t{0}, - thrust::plus{}); - - return std::pair(std::move(output_sizes), total_bytes); -} - } // namespace detail std::unique_ptr repeat_string(string_scalar const& input, @@ -404,21 +275,10 @@ std::unique_ptr repeat_strings(strings_column_view const& input, std::unique_ptr repeat_strings(strings_column_view const& input, column_view const& repeat_times, - std::optional output_strings_sizes, rmm::mr::device_memory_resource* mr) { CUDF_FUNC_RANGE(); - return detail::repeat_strings( - input, repeat_times, output_strings_sizes, cudf::get_default_stream(), mr); -} - -std::pair, int64_t> repeat_strings_output_sizes( - strings_column_view const& input, - column_view const& repeat_times, - rmm::mr::device_memory_resource* mr) -{ - CUDF_FUNC_RANGE(); - return detail::repeat_strings_output_sizes(input, repeat_times, cudf::get_default_stream(), mr); + return detail::repeat_strings(input, repeat_times, cudf::get_default_stream(), mr); } } // namespace strings diff --git a/cpp/tests/strings/repeat_strings_tests.cpp b/cpp/tests/strings/repeat_strings_tests.cpp index 69d0494c253..e75409d9f39 100644 --- a/cpp/tests/strings/repeat_strings_tests.cpp +++ b/cpp/tests/strings/repeat_strings_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -207,20 +207,6 @@ TEST_F(RepeatStringsTest, StringsColumnWithColumnRepeatTimesInvalidInput) EXPECT_THROW(cudf::strings::repeat_strings(strs_cv, repeat_times), cudf::logic_error); } - // Sizes mismatched between strings column and output_strings_sizes column. - { - auto const repeat_times = int32s_col{1, 2}; - auto const sizes = int32s_col{1, 2, 3, 4, 5}; - EXPECT_THROW(cudf::strings::repeat_strings(strs_cv, repeat_times, sizes), cudf::logic_error); - } - - // output_strings_sizes column has nulls. - { - auto const repeat_times = int32s_col{1, 2}; - auto const sizes = int32s_col{{null, 2}, null_at(0)}; - EXPECT_THROW(cudf::strings::repeat_strings(strs_cv, repeat_times, sizes), cudf::logic_error); - } - // Invalid data type for repeat_times column. { auto const repeat_times = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6}; @@ -243,11 +229,7 @@ TEST_F(RepeatStringsTest, StringsColumnWithColumnRepeatTimesOverflowOutput) auto const repeat_times = int32s_col{half_max, half_max, half_max, half_max, half_max, half_max, half_max}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(strs_cv, repeat_times); - (void)sizes; - auto const expected_bytes = static_cast(half_max) * int64_t{1 + 2 + 3 + 4 + 5 + 6 + 7}; - EXPECT_EQ(expected_bytes, total_bytes); + EXPECT_THROW(cudf::strings::repeat_strings(strs_cv, repeat_times), cudf::logic_error); } TYPED_TEST(RepeatStringsTypedTest, StringsColumnNoNullWithScalarRepeatTimes) @@ -301,15 +283,6 @@ TYPED_TEST(RepeatStringsTypedTest, StringsColumnNoNullWithColumnRepeatTimes) auto results = cudf::strings::repeat_strings(strs_cv, repeat_times); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{6, 12, 27, 0, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(strs_cv, repeat_times); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(45, total_bytes); - - results = cudf::strings::repeat_strings(strs_cv, repeat_times, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // repeat_times column has nulls. @@ -320,15 +293,6 @@ TYPED_TEST(RepeatStringsTypedTest, StringsColumnNoNullWithColumnRepeatTimes) auto results = cudf::strings::repeat_strings(strs_cv, repeat_times); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{6, 0, 27, 12, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(strs_cv, repeat_times); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(45, total_bytes); - - results = cudf::strings::repeat_strings(strs_cv, repeat_times, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } } @@ -377,15 +341,6 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnNoNullWithColumnRepeatTime auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{6, 12, 27}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(45, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // Sliced the middle of the column. @@ -397,15 +352,6 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnNoNullWithColumnRepeatTime auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{12, 27}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(39, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // Sliced the second half of the column. @@ -417,15 +363,6 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnNoNullWithColumnRepeatTime auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{27, 12, 12}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(51, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } } @@ -520,15 +457,6 @@ TYPED_TEST(RepeatStringsTypedTest, StringsColumnWithNullsWithColumnRepeatTimes) auto results = cudf::strings::repeat_strings(strs_cv, repeat_times); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{6, 0, 18, 0, 0, 0, 12, 12, 0, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(strs_cv, repeat_times); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(48, total_bytes); - - results = cudf::strings::repeat_strings(strs_cv, repeat_times, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // repeat_times column has nulls. @@ -549,15 +477,6 @@ TYPED_TEST(RepeatStringsTypedTest, StringsColumnWithNullsWithColumnRepeatTimes) auto results = cudf::strings::repeat_strings(strs_cv, repeat_times); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{6, 0, 0, 0, 0, 0, 12, 0, 0, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(strs_cv, repeat_times); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(18, total_bytes); - - results = cudf::strings::repeat_strings(strs_cv, repeat_times, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } } @@ -631,15 +550,6 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnWithNullsWithColumnRepeatT auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{6, 0, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(6, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // Sliced the middle of the column. @@ -652,15 +562,6 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnWithNullsWithColumnRepeatT auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{0, 0, 0, 0, 12}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(12, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // Sliced the second half of the column, output has nulls. @@ -672,15 +573,6 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnWithNullsWithColumnRepeatT auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{12, 0, 0, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(12, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_strs, *results, verbosity); } // Sliced the second half of the column, output does not have null. @@ -693,14 +585,5 @@ TYPED_TEST(RepeatStringsTypedTest, SlicedStringsColumnWithNullsWithColumnRepeatT auto results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_strs, *results, verbosity); - - auto const expected_sizes = int32s_col{0, 0}; - auto const [sizes, total_bytes] = - cudf::strings::repeat_strings_output_sizes(sliced_strs_cv, sliced_rtimes); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected_sizes, *sizes, verbosity); - EXPECT_EQ(0, total_bytes); - - results = cudf::strings::repeat_strings(sliced_strs_cv, sliced_rtimes, *sizes); - CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected_strs, *results, verbosity); } } From 476d5bbf9cfdbcef024bdccc29f30cd1c6fdbc94 Mon Sep 17 00:00:00 2001 From: nvdbaranec <56695930+nvdbaranec@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:02:08 -0600 Subject: [PATCH 06/69] Handle parquet list data corner case (#12698) Fixes an issue with a particular arrangement of page data related to lists. Specifically, it is possible for page `N` to contain "0" rows because the values for the row it is a part of start on page `N-1` and end on page `N+1`. This was defeating logic in the decode kernel that would erroneously cause these values to be skipped. Similar to https://github.com/rapidsai/cudf/pull/12488 this is only reproducible with data out in the wild. In this case, we have a file that we could in theory check in to create a test with, but it is 16 MB so it's fairly large. Looking for feedback on whether this is too big. Authors: - https://github.com/nvdbaranec - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Bradley Dice (https://github.com/bdice) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/12698 --- cpp/src/io/parquet/page_data.cu | 50 +++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/cpp/src/io/parquet/page_data.cu b/cpp/src/io/parquet/page_data.cu index 23d130e1585..ee115e7432a 100644 --- a/cpp/src/io/parquet/page_data.cu +++ b/cpp/src/io/parquet/page_data.cu @@ -104,20 +104,41 @@ struct page_state_s { * specified row bounds * * @param s The page to be checked - * @param min_row The starting row index + * @param start_row The starting row index * @param num_rows The number of rows * * @return True if the page spans the beginning or the end of the row bounds */ -inline __device__ bool is_bounds_page(page_state_s* const s, size_t min_row, size_t num_rows) +inline __device__ bool is_bounds_page(page_state_s* const s, size_t start_row, size_t num_rows) { size_t const page_begin = s->col.start_row + s->page.chunk_row; size_t const page_end = page_begin + s->page.num_rows; - size_t const begin = min_row; - size_t const end = min_row + num_rows; + size_t const begin = start_row; + size_t const end = start_row + num_rows; + return ((page_begin <= begin && page_end >= begin) || (page_begin <= end && page_end >= end)); } +/** + * @brief Returns whether or not a page is completely contained within the specified + * row bounds + * + * @param s The page to be checked + * @param start_row The starting row index + * @param num_rows The number of rows + * + * @return True if the page is completely contained within the row bounds + */ +inline __device__ bool is_page_contained(page_state_s* const s, size_t start_row, size_t num_rows) +{ + size_t const page_begin = s->col.start_row + s->page.chunk_row; + size_t const page_end = page_begin + s->page.num_rows; + size_t const begin = start_row; + size_t const end = start_row + num_rows; + + return page_begin >= begin && page_end <= end; +} + /** * @brief Read a 32-bit varint integer * @@ -1728,10 +1749,11 @@ __global__ void __launch_bounds__(block_size) auto const thread_depth = depth + t; if (thread_depth < s->page.num_output_nesting_levels) { // if we are not a bounding page (as checked above) then we are either - // returning 0 rows from the page (completely outside the bounds) or all - // rows in the page (completely within the bounds) + // returning all rows/values from this page, or 0 of them pp->nesting[thread_depth].batch_size = - s->num_rows == 0 ? 0 : pp->nesting[thread_depth].size; + (s->num_rows == 0 && !is_page_contained(s, min_row, num_rows)) + ? 0 + : pp->nesting[thread_depth].size; } depth += blockDim.x; } @@ -1838,7 +1860,19 @@ __global__ void __launch_bounds__(block_size) gpuDecodePageData( bool const has_repetition = s->col.max_level[level_type::REPETITION] > 0; // if we have no work to do (eg, in a skip_rows/num_rows case) in this page. - if (s->num_rows == 0 && !(has_repetition && is_bounds_page(s, min_row, num_rows))) { return; } + // + // corner case: in the case of lists, we can have pages that contain "0" rows if the current row + // starts before this page and ends after this page: + // P0 P1 P2 + // |---------|---------|----------| + // ^------------------^ + // row start row end + // P1 will contain 0 rows + // + if (s->num_rows == 0 && !(has_repetition && (is_bounds_page(s, min_row, num_rows) || + is_page_contained(s, min_row, num_rows)))) { + return; + } if (s->dict_base) { out_thread0 = (s->dict_bits > 0) ? 64 : 32; From 89ec635dceacde2b6715af253029ef317905df4e Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Wed, 8 Feb 2023 12:17:50 -0500 Subject: [PATCH 07/69] Update shared workflow branches (#12733) This PR updates the branch reference used for our shared workflows. Authors: - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/12733 --- .github/workflows/build.yaml | 14 +++++++------- .github/workflows/pr.yaml | 26 +++++++++++++------------- .github/workflows/test.yaml | 14 +++++++------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3366554db30..26d07515f70 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -55,7 +55,7 @@ jobs: skip_upload_pkgs: libcudf-example wheel-build-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -67,7 +67,7 @@ jobs: wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -77,7 +77,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-publish-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -88,7 +88,7 @@ jobs: wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-publish.yml@branch-23.04 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cf20b0006a2..f33fc15c52f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,32 +25,32 @@ jobs: - wheel-build-dask-cudf - wheel-tests-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@branch-23.04 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.04 conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@branch-23.04 with: build_type: pull-request conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.04 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@branch-23.04 with: build_type: pull-request conda-python-cudf-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.04 with: build_type: pull-request test_script: "ci/test_python_cudf.sh" @@ -58,14 +58,14 @@ jobs: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.04 with: build_type: pull-request test_script: "ci/test_python_other.sh" conda-java-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: pull-request node_type: "gpu-latest-1" @@ -75,7 +75,7 @@ jobs: conda-notebook-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: pull-request node_type: "gpu-latest-1" @@ -85,7 +85,7 @@ jobs: wheel-build-cudf: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.04 with: build_type: pull-request package-name: cudf @@ -94,7 +94,7 @@ jobs: wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.04 with: build_type: pull-request package-name: cudf @@ -106,7 +106,7 @@ jobs: wheel-build-dask-cudf: needs: wheel-tests-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-build.yml@branch-23.04 with: build_type: pull-request package-name: dask_cudf @@ -115,7 +115,7 @@ jobs: wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.04 with: build_type: pull-request package-name: dask_cudf diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1b117bb2f4f..ff19d51f8ef 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-cudf-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -34,7 +34,7 @@ jobs: conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -43,7 +43,7 @@ jobs: test_script: "ci/test_python_other.sh" conda-java-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -55,7 +55,7 @@ jobs: run_script: "ci/test_java.sh" conda-notebook-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -67,7 +67,7 @@ jobs: run_script: "ci/test_notebooks.sh" wheel-tests-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} @@ -78,7 +78,7 @@ jobs: test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests" wheel-tests-dask-cudf: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.02 + uses: rapidsai/shared-action-workflows/.github/workflows/wheels-pure-test.yml@branch-23.04 with: build_type: nightly branch: ${{ inputs.branch }} From d3f9dafa49c973c5e5d8b8a9336bbc92555ea0c3 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 8 Feb 2023 11:41:10 -0600 Subject: [PATCH 08/69] Fix faulty conditional logic in JIT `GroupBy.apply` (#12706) Closes https://github.com/rapidsai/cudf/issues/12686 Authors: - https://github.com/brandon-b-miller Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12706 --- python/cudf/cudf/tests/test_groupby.py | 17 +++++++++++++++++ python/cudf/udf_cpp/groupby/function.cu | 6 +++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index c5b330fd89c..1fea3c7a37e 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -519,6 +519,23 @@ def test_groupby_apply_jit_args(func, args, groupby_jit_data): run_groupby_apply_jit_test(groupby_jit_data, func, ["key1", "key2"], *args) +def test_groupby_apply_jit_block_divergence(): + # https://github.com/rapidsai/cudf/issues/12686 + df = cudf.DataFrame( + { + "a": [0, 0, 0, 1, 1, 1], + "b": [1, 1, 1, 2, 3, 4], + } + ) + + def diverging_block(grp_df): + if grp_df["a"].mean() > 0: + return grp_df["b"].mean() + return 0 + + run_groupby_apply_jit_test(df, diverging_block, ["a"]) + + @pytest.mark.parametrize("nelem", [2, 3, 100, 500, 1000]) @pytest.mark.parametrize( "func", diff --git a/python/cudf/udf_cpp/groupby/function.cu b/python/cudf/udf_cpp/groupby/function.cu index f94f99c4b49..782371b8a44 100644 --- a/python/cudf/udf_cpp/groupby/function.cu +++ b/python/cudf/udf_cpp/groupby/function.cu @@ -284,7 +284,7 @@ extern "C" { __device__ int name##_##cname(return_type* numba_return_value, type* const data, int64_t size) \ { \ return_type const res = name(data, size); \ - if (threadIdx.x == 0) { *numba_return_value = res; } \ + *numba_return_value = res; \ __syncthreads(); \ return 0; \ } @@ -309,8 +309,8 @@ extern "C" { __device__ int name##_##cname( \ int64_t* numba_return_value, type* const data, int64_t* index, int64_t size) \ { \ - auto const res = name(data, index, size); \ - if (threadIdx.x == 0) { *numba_return_value = res; } \ + auto const res = name(data, index, size); \ + *numba_return_value = res; \ __syncthreads(); \ return 0; \ } From 0161ba896a1d70ba3e049bdbb3d649cedba2aeb0 Mon Sep 17 00:00:00 2001 From: Cindy Jiang <47068112+cindyyuanjiang@users.noreply.github.com> Date: Wed, 8 Feb 2023 10:48:38 -0800 Subject: [PATCH 09/69] Add `regex_program` strings replacing java APIs and tests (#12701) This PR adds [replace_re, replace_with_backrefs](https://docs.rapids.ai/api/libcudf/nightly/replace__re_8hpp.html) related `regex_program` java APIs and unit tests. Part of work for https://github.com/NVIDIA/spark-rapids/issues/7295. Authors: - Cindy Jiang (https://github.com/cindyyuanjiang) Approvers: - Jason Lowe (https://github.com/jlowe) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12701 --- .../main/java/ai/rapids/cudf/ColumnView.java | 71 +++++++++++--- java/src/main/native/src/ColumnViewJni.cpp | 43 ++++---- .../java/ai/rapids/cudf/ColumnVectorTest.java | 98 ++++++++++++------- 3 files changed, 149 insertions(+), 63 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 2d0bf28225f..0cb9ed37d9f 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -2922,8 +2922,21 @@ public final ColumnVector stringReplace(Scalar target, Scalar replace) { * @param repl The string scalar to replace for each pattern match. * @return A new column vector containing the string results. */ + @Deprecated public final ColumnVector replaceRegex(String pattern, Scalar repl) { - return replaceRegex(pattern, repl, -1); + return replaceRegex(new RegexProgram(pattern, CaptureGroups.NON_CAPTURE), repl); + } + + /** + * For each string, replaces any character sequence matching the given regex program pattern + * using the replacement string scalar. + * + * @param regexProg The regex program with pattern to search within each string. + * @param repl The string scalar to replace for each pattern match. + * @return A new column vector containing the string results. + */ + public final ColumnVector replaceRegex(RegexProgram regexProg, Scalar repl) { + return replaceRegex(regexProg, repl, -1); } /** @@ -2935,12 +2948,27 @@ public final ColumnVector replaceRegex(String pattern, Scalar repl) { * @param maxRepl The maximum number of times a replacement should occur within each string. * @return A new column vector containing the string results. */ + @Deprecated public final ColumnVector replaceRegex(String pattern, Scalar repl, int maxRepl) { + return replaceRegex(new RegexProgram(pattern, CaptureGroups.NON_CAPTURE), repl, maxRepl); + } + + /** + * For each string, replaces any character sequence matching the given regex program pattern + * using the replacement string scalar. + * + * @param regexProg The regex program with pattern to search within each string. + * @param repl The string scalar to replace for each pattern match. + * @param maxRepl The maximum number of times a replacement should occur within each string. + * @return A new column vector containing the string results. + */ + public final ColumnVector replaceRegex(RegexProgram regexProg, Scalar repl, int maxRepl) { if (!repl.getType().equals(DType.STRING)) { throw new IllegalArgumentException("Replacement must be a string scalar"); } - return new ColumnVector(replaceRegex(getNativeView(), pattern, repl.getScalarHandle(), - maxRepl)); + assert regexProg != null : "regex program may not be null"; + return new ColumnVector(replaceRegex(getNativeView(), regexProg.pattern(), regexProg.combinedFlags(), + regexProg.capture().nativeId, repl.getScalarHandle(), maxRepl)); } /** @@ -2966,9 +2994,26 @@ public final ColumnVector replaceMultiRegex(String[] patterns, ColumnView repls) * @param replace The replacement template for creating the output string. * @return A new java column vector containing the string results. */ + @Deprecated public final ColumnVector stringReplaceWithBackrefs(String pattern, String replace) { - return new ColumnVector(stringReplaceWithBackrefs(getNativeView(), pattern, - replace)); + return stringReplaceWithBackrefs(new RegexProgram(pattern), replace); + } + + /** + * For each string, replaces any character sequence matching the given regex program + * pattern using the replace template for back-references. + * + * Any null string entries return corresponding null output column entries. + * + * @param regexProg The regex program with pattern to search within each string. + * @param replace The replacement template for creating the output string. + * @return A new java column vector containing the string results. + */ + public final ColumnVector stringReplaceWithBackrefs(RegexProgram regexProg, String replace) { + assert regexProg != null : "regex program may not be null"; + return new ColumnVector( + stringReplaceWithBackrefs(getNativeView(), regexProg.pattern(), regexProg.combinedFlags(), + regexProg.capture().nativeId, replace)); } /** @@ -4129,12 +4174,14 @@ private static native long substringColumn(long columnView, long startColumn, lo * Native method for replacing each regular expression pattern match with the specified * replacement string. * @param columnView native handle of the cudf::column_view being operated on. - * @param pattern The regular expression pattern to search within each string. + * @param pattern regular expression pattern to search within each string. + * @param flags regex flags setting. + * @param capture capture groups setting. * @param repl native handle of the cudf::scalar containing the replacement string. * @param maxRepl maximum number of times to replace the pattern within a string * @return native handle of the resulting cudf column containing the string results. */ - private static native long replaceRegex(long columnView, String pattern, + private static native long replaceRegex(long columnView, String pattern, int flags, int capture, long repl, long maxRepl) throws CudfException; /** @@ -4148,15 +4195,17 @@ private static native long replaceMultiRegex(long columnView, String[] patterns, long repls) throws CudfException; /** - * Native method for replacing any character sequence matching the given pattern - * using the replace template for back-references. + * Native method for replacing any character sequence matching the given regex program + * pattern using the replace template for back-references. * @param columnView native handle of the cudf::column_view being operated on. * @param pattern The regular expression patterns to search within each string. + * @param flags Regex flags setting. + * @param capture Capture groups setting. * @param replace The replacement template for creating the output string. * @return native handle of the resulting cudf column containing the string results. */ - private static native long stringReplaceWithBackrefs(long columnView, String pattern, - String replace) throws CudfException; + private static native long stringReplaceWithBackrefs(long columnView, String pattern, int flags, + int capture, String replace) throws CudfException; /** * Native method for checking if strings in a column starts with a specified comparison string. diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index 958efd364ed..c42cc430560 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -1606,21 +1606,24 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_mapContains(JNIEnv *env, CATCH_STD(env, 0); } -JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_replaceRegex(JNIEnv *env, jclass, - jlong j_column_view, - jstring j_pattern, jlong j_repl, - jlong j_maxrepl) { +JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_replaceRegex( + JNIEnv *env, jclass, jlong j_column_view, jstring j_pattern, jint regex_flags, + jint capture_groups, jlong j_repl, jlong j_maxrepl) { JNI_NULL_CHECK(env, j_column_view, "column is null", 0); JNI_NULL_CHECK(env, j_pattern, "pattern string is null", 0); JNI_NULL_CHECK(env, j_repl, "replace scalar is null", 0); try { cudf::jni::auto_set_device(env); - auto cv = reinterpret_cast(j_column_view); - cudf::strings_column_view scv(*cv); - cudf::jni::native_jstring pattern(env, j_pattern); - auto repl = reinterpret_cast(j_repl); - return release_as_jlong(cudf::strings::replace_re(scv, pattern.get(), *repl, j_maxrepl)); + auto const cv = reinterpret_cast(j_column_view); + auto const strings_column = cudf::strings_column_view{*cv}; + auto const pattern = cudf::jni::native_jstring(env, j_pattern); + auto const flags = static_cast(regex_flags); + auto const groups = static_cast(capture_groups); + auto const regex_prog = cudf::strings::regex_program::create(pattern.get(), flags, groups); + auto const repl = reinterpret_cast(j_repl); + return release_as_jlong( + cudf::strings::replace_re(strings_column, *regex_prog, *repl, j_maxrepl)); } CATCH_STD(env, 0); } @@ -1646,19 +1649,23 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_replaceMultiRegex(JNIEnv } JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_stringReplaceWithBackrefs( - JNIEnv *env, jclass, jlong column_view, jstring patternObj, jstring replaceObj) { + JNIEnv *env, jclass, jlong j_column_view, jstring pattern_obj, jint regex_flags, + jint capture_groups, jstring replace_obj) { - JNI_NULL_CHECK(env, column_view, "column is null", 0); - JNI_NULL_CHECK(env, patternObj, "pattern string is null", 0); - JNI_NULL_CHECK(env, replaceObj, "replace string is null", 0); + JNI_NULL_CHECK(env, j_column_view, "column is null", 0); + JNI_NULL_CHECK(env, pattern_obj, "pattern string is null", 0); + JNI_NULL_CHECK(env, replace_obj, "replace string is null", 0); try { cudf::jni::auto_set_device(env); - cudf::column_view *cv = reinterpret_cast(column_view); - cudf::strings_column_view scv(*cv); - cudf::jni::native_jstring ss_pattern(env, patternObj); - cudf::jni::native_jstring ss_replace(env, replaceObj); + auto const cv = reinterpret_cast(j_column_view); + auto const strings_column = cudf::strings_column_view{*cv}; + auto const pattern = cudf::jni::native_jstring(env, pattern_obj); + auto const flags = static_cast(regex_flags); + auto const groups = static_cast(capture_groups); + auto const regex_prog = cudf::strings::regex_program::create(pattern.get(), flags, groups); + cudf::jni::native_jstring ss_replace(env, replace_obj); return release_as_jlong( - cudf::strings::replace_with_backrefs(scv, ss_pattern.get(), ss_replace.get())); + cudf::strings::replace_with_backrefs(strings_column, *regex_prog, ss_replace.get())); } CATCH_STD(env, 0); } diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index ab4baf74277..db64dcb08c7 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -5147,29 +5147,42 @@ void teststringReplaceThrowsException() { @Test void testReplaceRegex() { - try (ColumnVector v = - ColumnVector.fromStrings("title and Title with title", "nothing", null, "Title"); - Scalar repl = Scalar.fromString("Repl"); - ColumnVector actual = v.replaceRegex("[tT]itle", repl); - ColumnVector expected = - ColumnVector.fromStrings("Repl and Repl with Repl", "nothing", null, "Repl")) { - assertColumnsAreEqual(expected, actual); - } + try (ColumnVector v = ColumnVector.fromStrings("title and Title with title", "nothing", null, "Title"); + Scalar repl = Scalar.fromString("Repl")) { + String pattern = "[tT]itle"; + RegexProgram regexProg = new RegexProgram(pattern, CaptureGroups.NON_CAPTURE); - try (ColumnVector v = - ColumnVector.fromStrings("title and Title with title", "nothing", null, "Title"); - Scalar repl = Scalar.fromString("Repl"); - ColumnVector actual = v.replaceRegex("[tT]itle", repl, 0)) { - assertColumnsAreEqual(v, actual); - } + try (ColumnVector actual = v.replaceRegex(pattern, repl); + ColumnVector expected = + ColumnVector.fromStrings("Repl and Repl with Repl", "nothing", null, "Repl")) { + assertColumnsAreEqual(expected, actual); + } - try (ColumnVector v = - ColumnVector.fromStrings("title and Title with title", "nothing", null, "Title"); - Scalar repl = Scalar.fromString("Repl"); - ColumnVector actual = v.replaceRegex("[tT]itle", repl, 1); - ColumnVector expected = - ColumnVector.fromStrings("Repl and Title with title", "nothing", null, "Repl")) { - assertColumnsAreEqual(expected, actual); + try (ColumnVector actual = v.replaceRegex(pattern, repl, 0)) { + assertColumnsAreEqual(v, actual); + } + + try (ColumnVector actual = v.replaceRegex(pattern, repl, 1); + ColumnVector expected = + ColumnVector.fromStrings("Repl and Title with title", "nothing", null, "Repl")) { + assertColumnsAreEqual(expected, actual); + } + + try (ColumnVector actual = v.replaceRegex(regexProg, repl); + ColumnVector expected = + ColumnVector.fromStrings("Repl and Repl with Repl", "nothing", null, "Repl")) { + assertColumnsAreEqual(expected, actual); + } + + try (ColumnVector actual = v.replaceRegex(regexProg, repl, 0)) { + assertColumnsAreEqual(v, actual); + } + + try (ColumnVector actual = v.replaceRegex(regexProg, repl, 1); + ColumnVector expected = + ColumnVector.fromStrings("Repl and Title with title", "nothing", null, "Repl")) { + assertColumnsAreEqual(expected, actual); + } } } @@ -5188,45 +5201,55 @@ void testReplaceMultiRegex() { @Test void testStringReplaceWithBackrefs() { - try (ColumnVector v = ColumnVector.fromStrings("

title

", "

another title

", - null); + try (ColumnVector v = ColumnVector.fromStrings("

title

", "

another title

", null); ColumnVector expected = ColumnVector.fromStrings("

title

", "

another title

", null); - ColumnVector actual = v.stringReplaceWithBackrefs("

(.*)

", "

\\1

")) { + ColumnVector actual = v.stringReplaceWithBackrefs("

(.*)

", "

\\1

"); + ColumnVector actualRe = + v.stringReplaceWithBackrefs(new RegexProgram("

(.*)

"), "

\\1

")) { assertColumnsAreEqual(expected, actual); + assertColumnsAreEqual(expected, actualRe); } try (ColumnVector v = ColumnVector.fromStrings("2020-1-01", "2020-2-02", null); ColumnVector expected = ColumnVector.fromStrings("2020-01-01", "2020-02-02", null); - ColumnVector actual = v.stringReplaceWithBackrefs("-([0-9])-", "-0\\1-")) { + ColumnVector actual = v.stringReplaceWithBackrefs("-([0-9])-", "-0\\1-"); + ColumnVector actualRe = + v.stringReplaceWithBackrefs(new RegexProgram("-([0-9])-"), "-0\\1-")) { assertColumnsAreEqual(expected, actual); + assertColumnsAreEqual(expected, actualRe); } - try (ColumnVector v = ColumnVector.fromStrings("2020-01-1", "2020-02-2", - "2020-03-3invalid", null); + try (ColumnVector v = ColumnVector.fromStrings("2020-01-1", "2020-02-2", "2020-03-3invalid", null); ColumnVector expected = ColumnVector.fromStrings("2020-01-01", "2020-02-02", "2020-03-3invalid", null); - ColumnVector actual = v.stringReplaceWithBackrefs( - "-([0-9])$", "-0\\1")) { + ColumnVector actual = v.stringReplaceWithBackrefs("-([0-9])$", "-0\\1"); + ColumnVector actualRe = + v.stringReplaceWithBackrefs(new RegexProgram("-([0-9])$"), "-0\\1")) { assertColumnsAreEqual(expected, actual); + assertColumnsAreEqual(expected, actualRe); } try (ColumnVector v = ColumnVector.fromStrings("2020-01-1 random_text", "2020-02-2T12:34:56", - "2020-03-3invalid", null); + "2020-03-3invalid", null); ColumnVector expected = ColumnVector.fromStrings("2020-01-01 random_text", "2020-02-02T12:34:56", "2020-03-3invalid", null); - ColumnVector actual = v.stringReplaceWithBackrefs( - "-([0-9])([ T])", "-0\\1\\2")) { + ColumnVector actual = v.stringReplaceWithBackrefs("-([0-9])([ T])", "-0\\1\\2"); + ColumnVector actualRe = + v.stringReplaceWithBackrefs(new RegexProgram("-([0-9])([ T])"), "-0\\1\\2")) { assertColumnsAreEqual(expected, actual); + assertColumnsAreEqual(expected, actualRe); } // test zero as group index try (ColumnVector v = ColumnVector.fromStrings("aa-11 b2b-345", "aa-11a 1c-2b2 b2-c3", "11-aa", null); ColumnVector expected = ColumnVector.fromStrings("aa-11:aa:11; b2b-345:b:345;", "aa-11:aa:11;a 1c-2:c:2;b2 b2-c3", "11-aa", null); - ColumnVector actual = v.stringReplaceWithBackrefs( - "([a-z]+)-([0-9]+)", "${0}:${1}:${2};")) { + ColumnVector actual = v.stringReplaceWithBackrefs("([a-z]+)-([0-9]+)", "${0}:${1}:${2};"); + ColumnVector actualRe = + v.stringReplaceWithBackrefs(new RegexProgram("([a-z]+)-([0-9]+)"), "${0}:${1}:${2};")) { assertColumnsAreEqual(expected, actual); + assertColumnsAreEqual(expected, actualRe); } // group index exceeds group count @@ -5236,6 +5259,13 @@ void testStringReplaceWithBackrefs() { } }); + // group index exceeds group count + assertThrows(CudfException.class, () -> { + try (ColumnVector v = ColumnVector.fromStrings("ABC123defgh"); + ColumnVector r = + v.stringReplaceWithBackrefs(new RegexProgram("([A-Z]+)([0-9]+)([a-z]+)"), "\\4")) { + } + }); } @Test From c20c8b42215e38bee207b49dad6e28ea04ccbd8c Mon Sep 17 00:00:00 2001 From: Sevag H Date: Wed, 8 Feb 2023 16:50:48 -0500 Subject: [PATCH 10/69] Bump pinned rapids wheel deps to 23.4 (#12735) We introduced a change to pin RAPIDS wheel dependencies to the same release version. However, branch 23.04 was created before that last PR was merged, so as of now cudf's 23.4 wheels are installing 23.2 RAPIDS dependencies. This PR updates those pins to the current release. Authors: - Sevag H (https://github.com/sevagh) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12735 --- python/cudf/setup.py | 2 +- python/dask_cudf/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 48199d83478..88bc2cfae28 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -21,7 +21,7 @@ "typing_extensions", # Allow floating minor versions for Arrow. "pyarrow==10", - f"rmm{cuda_suffix}==23.2.*", + f"rmm{cuda_suffix}==23.4.*", f"ptxcompiler{cuda_suffix}", f"cubinlinker{cuda_suffix}", "cupy-cuda11x", diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 4b420b1b97c..be4c704019d 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -13,7 +13,7 @@ "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.6.0dev0", - f"cudf{cuda_suffix}==23.2.*", + f"cudf{cuda_suffix}==23.4.*", "cupy-cuda11x", ] From 3e4ff2afb8a5ca7f1ce051f2d86945688dfb21f9 Mon Sep 17 00:00:00 2001 From: Ajay Thorve Date: Wed, 8 Feb 2023 16:25:28 -0800 Subject: [PATCH 11/69] Reduce error handling verbosity in CI tests scripts (#12738) This PR adds a less verbose [trap method](https://github.com/rapidsai/cugraph/blob/f2b081075704aabc789603e14ce552eac3fbe692/ci/test.sh#L19), for error handling to help ensure that we capture all potential error codes in our test scripts, and works as follows: - setting an environment variable, EXITCODE, with a default value of 0 - setting a trap statement triggered by ERR signals which will set EXITCODE=1 when any commands return a non-zero exit code cc @ajschmidt8 Authors: - Ajay Thorve (https://github.com/AjayThorve) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12738 --- ci/test_cpp.sh | 12 ++++-------- ci/test_java.sh | 13 ++++--------- ci/test_notebooks.sh | 9 ++++----- ci/test_python_common.sh | 3 +-- ci/test_python_cudf.sh | 25 +++++-------------------- ci/test_python_other.sh | 31 +++++-------------------------- 6 files changed, 23 insertions(+), 70 deletions(-) diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 5b1e8aa398c..b3d7919b279 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -21,7 +21,6 @@ set -u CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" -SUITEERROR=0 rapids-print-env @@ -32,6 +31,8 @@ rapids-mamba-retry install \ rapids-logger "Check GPU usage" nvidia-smi +EXITCODE=0 +trap "EXITCODE=1" ERR set +e # TODO: Disabling stream identification for now. @@ -61,12 +62,6 @@ for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do #else # GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} #fi - - exitcode=$? - if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: GTest ${gt}" - fi done if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then @@ -85,4 +80,5 @@ if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then # TODO: test-results/*.cs.log are processed in gpuci fi -exit ${SUITEERROR} +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/test_java.sh b/ci/test_java.sh index 27a1f2aa46f..f905aaa1178 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -29,22 +29,17 @@ rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ libcudf -SUITEERROR=0 - rapids-logger "Check GPU usage" nvidia-smi +EXITCODE=0 +trap "EXITCODE=1" ERR set +e rapids-logger "Run Java tests" pushd java mvn test -B -DCUDF_JNI_ARROW_STATIC=OFF -DCUDF_JNI_ENABLE_PROFILING=OFF -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in cudf Java" -fi popd -exit ${SUITEERROR} +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index f1e17162195..7f5f35219b0 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -36,9 +36,8 @@ pushd notebooks # (space-separated list of filenames without paths) SKIPNBS="" -# Set SUITEERROR to failure if any run fails -SUITEERROR=0 - +EXITCODE=0 +trap "EXITCODE=1" ERR set +e for nb in $(find . -name "*.ipynb"); do nbBasename=$(basename ${nb}) @@ -55,8 +54,8 @@ for nb in $(find . -name "*.ipynb"); do else nvidia-smi ${NBTEST} ${nbBasename} - SUITEERROR=$((SUITEERROR | $?)) fi done -exit ${SUITEERROR} +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 107540c0192..0e922c105dd 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Common setup steps shared by Python test jobs @@ -27,7 +27,6 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" -SUITEERROR=0 rapids-print-env diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index bea162a9318..337ef38cf97 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Common setup steps shared by Python test jobs source "$(dirname "$0")/test_python_common.sh" @@ -7,6 +7,8 @@ source "$(dirname "$0")/test_python_common.sh" rapids-logger "Check GPU usage" nvidia-smi +EXITCODE=0 +trap "EXITCODE=1" ERR set +e rapids-logger "pytest cudf" @@ -24,12 +26,6 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-coverage.xml" \ --cov-report=term \ tests -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in cudf" -fi popd # Run benchmarks with both cudf and pandas to ensure compatibility is maintained. @@ -48,12 +44,6 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-coverage.xml" \ --cov-report=term \ benchmarks -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in cudf" -fi rapids-logger "pytest for cudf benchmarks using pandas" CUDF_BENCHMARKS_USE_PANDAS=ON \ @@ -67,12 +57,7 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-benchmark-pandas-coverage.xml" \ --cov-report=term \ benchmarks -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in cudf" -fi popd -exit ${SUITEERROR} +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index d7a5e288193..b79cd44cdbe 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # Common setup steps shared by Python test jobs source "$(dirname "$0")/test_python_common.sh" @@ -12,6 +12,8 @@ rapids-mamba-retry install \ rapids-logger "Check GPU usage" nvidia-smi +EXITCODE=0 +trap "EXITCODE=1" ERR set +e rapids-logger "pytest dask_cudf" @@ -26,12 +28,6 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/dask-cudf-coverage.xml" \ --cov-report=term \ dask_cudf -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in dask-cudf" -fi popd rapids-logger "pytest custreamz" @@ -46,12 +42,6 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/custreamz-coverage.xml" \ --cov-report=term \ custreamz -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in custreamz" -fi popd set -e @@ -73,12 +63,6 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/strings-udf-coverage.xml" \ --cov-report=term \ tests -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in strings_udf" -fi popd rapids-logger "pytest cudf with strings_udf" @@ -94,12 +78,7 @@ pytest \ --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-strings-udf-coverage.xml" \ --cov-report=term \ tests/test_udf_masked_ops.py -exitcode=$? - -if (( ${exitcode} != 0 )); then - SUITEERROR=${exitcode} - echo "FAILED: 1 or more tests in cudf with strings_udf" -fi popd -exit ${SUITEERROR} +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} From 74efb790cfbf5f9563ff88f9c835dc03570605f9 Mon Sep 17 00:00:00 2001 From: Liangcai Li Date: Thu, 9 Feb 2023 10:20:45 +0800 Subject: [PATCH 12/69] Allow setting the seed argument for hash partition (#12715) This PR is exposing the `seed` parameter for the JNI hash partition APIs to support customizing the hash algorithm seed. The existing tests should cover this change. Authors: - Liangcai Li (https://github.com/firestarman) Approvers: - Nghia Truong (https://github.com/ttnghia) - Jason Lowe (https://github.com/jlowe) URL: https://github.com/rapidsai/cudf/pull/12715 --- java/src/main/java/ai/rapids/cudf/Table.java | 16 ++++++++++++++++ java/src/main/native/src/TableJni.cpp | 7 ++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/java/src/main/java/ai/rapids/cudf/Table.java b/java/src/main/java/ai/rapids/cudf/Table.java index 3eed7e45eed..3ccab70ccda 100644 --- a/java/src/main/java/ai/rapids/cudf/Table.java +++ b/java/src/main/java/ai/rapids/cudf/Table.java @@ -194,6 +194,7 @@ private static native long[] hashPartition(long inputTable, int[] columnsToHash, int hashTypeId, int numberOfPartitions, + int seed, int[] outputOffsets) throws CudfException; private static native long[] roundRobinPartition(long inputTable, @@ -4253,12 +4254,27 @@ public PartitionedTable hashPartition(int numberOfPartitions) { * {@link Table} class */ public PartitionedTable hashPartition(HashType type, int numberOfPartitions) { + final int DEFAULT_HASH_SEED = 0; + return hashPartition(type, numberOfPartitions, DEFAULT_HASH_SEED); + } + + /** + * Hash partition a table into the specified number of partitions. + * @param type the type of hash to use. Depending on the type of hash different restrictions + * on the hash column(s) may exist. Not all hash functions are guaranteed to work + * besides IDENTITY and MURMUR3. + * @param numberOfPartitions number of partitions to use + * @param seed the seed value for hashing + * @return Table that exposes a limited functionality of the {@link Table} class + */ + public PartitionedTable hashPartition(HashType type, int numberOfPartitions, int seed) { int[] partitionOffsets = new int[numberOfPartitions]; return new PartitionedTable(new Table(Table.hashPartition( operation.table.nativeHandle, operation.indices, type.nativeId, partitionOffsets.length, + seed, partitionOffsets)), partitionOffsets); } } diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index 3d730ff61a1..0b3ccb59a39 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -2655,7 +2655,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_partition(JNIEnv *env, jc JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_hashPartition( JNIEnv *env, jclass, jlong input_table, jintArray columns_to_hash, jint hash_function, - jint number_of_partitions, jintArray output_offsets) { + jint number_of_partitions, jint seed, jintArray output_offsets) { JNI_NULL_CHECK(env, input_table, "input table is null", NULL); JNI_NULL_CHECK(env, columns_to_hash, "columns_to_hash is null", NULL); @@ -2665,6 +2665,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_hashPartition( try { cudf::jni::auto_set_device(env); auto const hash_func = static_cast(hash_function); + auto const hash_seed = static_cast(seed); auto const n_input_table = reinterpret_cast(input_table); cudf::jni::native_jintArray n_columns_to_hash(env, columns_to_hash); JNI_ARG_CHECK(env, n_columns_to_hash.size() > 0, "columns_to_hash is zero", NULL); @@ -2672,8 +2673,8 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_hashPartition( std::vector columns_to_hash_vec(n_columns_to_hash.begin(), n_columns_to_hash.end()); - auto [partitioned_table, partition_offsets] = - cudf::hash_partition(*n_input_table, columns_to_hash_vec, number_of_partitions, hash_func); + auto [partitioned_table, partition_offsets] = cudf::hash_partition( + *n_input_table, columns_to_hash_vec, number_of_partitions, hash_func, hash_seed); cudf::jni::native_jintArray n_output_offsets(env, output_offsets); std::copy(partition_offsets.begin(), partition_offsets.end(), n_output_offsets.begin()); From b0335f0c928b748b7c107a159cf4bb17c0839bd1 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Thu, 9 Feb 2023 13:56:06 +0100 Subject: [PATCH 13/69] `partition_by_hash()`: use `_split()` (#12704) Reduce overhead of `Frame.partition_by_hash()` by calling `Frame._split()`. #### Benchmark Small benchmark of this PR shows ~1.5x speedup: https://gist.github.com/madsbk/308df2dd58309510610fd27e0529f862 ## Authors: - Mads R. B. Kristensen (https://github.com/madsbk) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12704 --- python/cudf/cudf/core/dataframe.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1ebf59ba6e4..535fe2352aa 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -4614,10 +4614,13 @@ def partition_by_hash(self, columns, nparts, keep_index=True): self._column_names, self._index_names if keep_index else None, ) - # Slice into partition - ret = [outdf[s:e] for s, e in zip(offsets, offsets[1:] + [None])] - if not keep_index: - ret = [df.reset_index(drop=True) for df in ret] + # Slice into partitions. Notice, `hash_partition` returns the start + # offset of each partition thus we skip the first offset + ret = outdf._split(offsets[1:], keep_index=keep_index) + + # Calling `_split()` on an empty dataframe returns an empty list + # so we add empty partitions here + ret += [self._empty_like(keep_index) for _ in range(nparts - len(ret))] return ret def info( From 8d17379bd821edc67c3a3f93801d3e79b8e2d97f Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 9 Feb 2023 08:33:46 -0500 Subject: [PATCH 14/69] Fix memcheck read error in compound segmented reduce (#12722) Fixes an out-of-bounds memory read in the compound segmented reduction logic. The number of segments was computed incorrectly causing an extra read passed the end of the valid-counts vector. This was found by running compute-sanitizer test on the reductions gtests as follows: ``` compute-sanitizer --tool memcheck --demangle full gtests/REDUCTIONS_TEST --rmm_mode=cuda ``` The number of segments is 1 less than the number of offsets. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/12722 --- cpp/include/cudf/detail/segmented_reduction.cuh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/include/cudf/detail/segmented_reduction.cuh b/cpp/include/cudf/detail/segmented_reduction.cuh index 9a49c1abe38..1c39d5eab1e 100644 --- a/cpp/include/cudf/detail/segmented_reduction.cuh +++ b/cpp/include/cudf/detail/segmented_reduction.cuh @@ -145,10 +145,10 @@ void segmented_reduce(InputIterator d_in, size_type* d_valid_counts, rmm::cuda_stream_view stream) { - using OutputType = typename thrust::iterator_value::type; - using IntermediateType = typename thrust::iterator_value::type; - auto num_segments = static_cast(std::distance(d_offset_begin, d_offset_end)); - auto const binary_op = op.get_binary_op(); + using OutputType = typename thrust::iterator_value::type; + using IntermediateType = typename thrust::iterator_value::type; + auto num_segments = static_cast(std::distance(d_offset_begin, d_offset_end)) - 1; + auto const binary_op = op.get_binary_op(); auto const initial_value = op.template get_identity(); rmm::device_uvector intermediate_result{static_cast(num_segments), From 3b2682f0a8d3cfe3b106d2922b4a425dda0162e1 Mon Sep 17 00:00:00 2001 From: Raymond Douglass Date: Thu, 9 Feb 2023 10:08:45 -0500 Subject: [PATCH 15/69] update changelog --- CHANGELOG.md | 242 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 240 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d653d503a1e..4acad48eabf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,244 @@ -# cuDF 23.02.00 (Date TBD) +# cuDF 23.02.00 (9 Feb 2023) -Please see https://github.com/rapidsai/cudf/releases/tag/v23.02.00a for the latest changes to this development branch. +## 🚨 Breaking Changes + +- Pin `dask` and `distributed` for release ([#12695](https://github.com/rapidsai/cudf/pull/12695)) [@galipremsagar](https://github.com/galipremsagar) +- Change ways to access `ptr` in `Buffer` ([#12587](https://github.com/rapidsai/cudf/pull/12587)) [@galipremsagar](https://github.com/galipremsagar) +- Remove column names ([#12578](https://github.com/rapidsai/cudf/pull/12578)) [@vuule](https://github.com/vuule) +- Default `cudf::io::read_json` to nested JSON parser ([#12544](https://github.com/rapidsai/cudf/pull/12544)) [@vuule](https://github.com/vuule) +- Switch `engine=cudf` to the new `JSON` reader ([#12509](https://github.com/rapidsai/cudf/pull/12509)) [@galipremsagar](https://github.com/galipremsagar) +- Add trailing comma support for nested JSON reader ([#12448](https://github.com/rapidsai/cudf/pull/12448)) [@karthikeyann](https://github.com/karthikeyann) +- Upgrade to `arrow-10.0.1` ([#12327](https://github.com/rapidsai/cudf/pull/12327)) [@galipremsagar](https://github.com/galipremsagar) +- Fail loudly to avoid data corruption with unsupported input in `read_orc` ([#12325](https://github.com/rapidsai/cudf/pull/12325)) [@vuule](https://github.com/vuule) +- CSV, JSON reader to infer integer column with nulls as int64 instead of float64 ([#12309](https://github.com/rapidsai/cudf/pull/12309)) [@karthikeyann](https://github.com/karthikeyann) +- Remove deprecated code for 23.02 ([#12281](https://github.com/rapidsai/cudf/pull/12281)) [@vyasr](https://github.com/vyasr) +- Null element for parsing error in numeric types in JSON, CSV reader ([#12272](https://github.com/rapidsai/cudf/pull/12272)) [@karthikeyann](https://github.com/karthikeyann) +- Purge non-empty nulls for `superimpose_nulls` and `push_down_nulls` ([#12239](https://github.com/rapidsai/cudf/pull/12239)) [@ttnghia](https://github.com/ttnghia) +- Rename `cudf::structs::detail::superimpose_parent_nulls` APIs ([#12230](https://github.com/rapidsai/cudf/pull/12230)) [@ttnghia](https://github.com/ttnghia) +- Remove JIT type names, refactor id_to_type. ([#12158](https://github.com/rapidsai/cudf/pull/12158)) [@bdice](https://github.com/bdice) +- Floor division uses integer division for integral arguments ([#12131](https://github.com/rapidsai/cudf/pull/12131)) [@wence-](https://github.com/wence-) + +## 🐛 Bug Fixes + +- Fix a mask data corruption in UDF ([#12647](https://github.com/rapidsai/cudf/pull/12647)) [@galipremsagar](https://github.com/galipremsagar) +- pre-commit: Update isort version to 5.12.0 ([#12645](https://github.com/rapidsai/cudf/pull/12645)) [@wence-](https://github.com/wence-) +- tests: Skip cuInit tests if cuda-gdb is not found or not working ([#12644](https://github.com/rapidsai/cudf/pull/12644)) [@wence-](https://github.com/wence-) +- Revert regex program java APIs and tests ([#12639](https://github.com/rapidsai/cudf/pull/12639)) [@cindyyuanjiang](https://github.com/cindyyuanjiang) +- Fix leaks in ColumnVectorTest ([#12625](https://github.com/rapidsai/cudf/pull/12625)) [@jlowe](https://github.com/jlowe) +- Handle when spillable buffers own each other ([#12607](https://github.com/rapidsai/cudf/pull/12607)) [@madsbk](https://github.com/madsbk) +- Fix incorrect null counts for sliced columns in JCudfSerialization ([#12589](https://github.com/rapidsai/cudf/pull/12589)) [@jlowe](https://github.com/jlowe) +- lists: Transfer dtypes correctly through list.get ([#12586](https://github.com/rapidsai/cudf/pull/12586)) [@wence-](https://github.com/wence-) +- timedelta: Don't go via float intermediates for floordiv ([#12585](https://github.com/rapidsai/cudf/pull/12585)) [@wence-](https://github.com/wence-) +- Fixing BUG, `get_next_chunk()` should use the blocking function `device_read()` ([#12584](https://github.com/rapidsai/cudf/pull/12584)) [@madsbk](https://github.com/madsbk) +- Make JNI QuoteStyle accessible outside ai.rapids.cudf ([#12572](https://github.com/rapidsai/cudf/pull/12572)) [@mythrocks](https://github.com/mythrocks) +- `partition_by_hash()`: support index ([#12554](https://github.com/rapidsai/cudf/pull/12554)) [@madsbk](https://github.com/madsbk) +- Mixed Join benchmark bug due to wrong conditional column ([#12553](https://github.com/rapidsai/cudf/pull/12553)) [@divyegala](https://github.com/divyegala) +- Update List Lexicographical Comparator ([#12538](https://github.com/rapidsai/cudf/pull/12538)) [@divyegala](https://github.com/divyegala) +- Dynamically read PTX version ([#12534](https://github.com/rapidsai/cudf/pull/12534)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- build.sh switch to use `RAPIDS` magic value ([#12525](https://github.com/rapidsai/cudf/pull/12525)) [@robertmaynard](https://github.com/robertmaynard) +- Loosen runtime arrow pinning ([#12522](https://github.com/rapidsai/cudf/pull/12522)) [@vyasr](https://github.com/vyasr) +- Enable metadata transfer for complex types in transpose ([#12491](https://github.com/rapidsai/cudf/pull/12491)) [@galipremsagar](https://github.com/galipremsagar) +- Fix issues with parquet chunked reader ([#12488](https://github.com/rapidsai/cudf/pull/12488)) [@nvdbaranec](https://github.com/nvdbaranec) +- Fix missing metadata transfer in concat for `ListColumn` ([#12487](https://github.com/rapidsai/cudf/pull/12487)) [@galipremsagar](https://github.com/galipremsagar) +- Rename libcudf substring source files to slice ([#12484](https://github.com/rapidsai/cudf/pull/12484)) [@davidwendt](https://github.com/davidwendt) +- Fix compile issue with arrow 10 ([#12465](https://github.com/rapidsai/cudf/pull/12465)) [@ttnghia](https://github.com/ttnghia) +- Fix List offsets bug in mixed type list column in nested JSON reader ([#12447](https://github.com/rapidsai/cudf/pull/12447)) [@karthikeyann](https://github.com/karthikeyann) +- Fix xfail incompatibilities ([#12423](https://github.com/rapidsai/cudf/pull/12423)) [@vyasr](https://github.com/vyasr) +- Fix bug in Parquet column index encoding ([#12404](https://github.com/rapidsai/cudf/pull/12404)) [@etseidl](https://github.com/etseidl) +- When building Arrow shared look for a shared OpenSSL ([#12396](https://github.com/rapidsai/cudf/pull/12396)) [@robertmaynard](https://github.com/robertmaynard) +- Fix get_json_object to return empty column on empty input ([#12384](https://github.com/rapidsai/cudf/pull/12384)) [@davidwendt](https://github.com/davidwendt) +- Pin arrow 9 in testing dependencies to prevent conda solve issues ([#12377](https://github.com/rapidsai/cudf/pull/12377)) [@vyasr](https://github.com/vyasr) +- Fix reductions any/all return value for empty input ([#12374](https://github.com/rapidsai/cudf/pull/12374)) [@davidwendt](https://github.com/davidwendt) +- Fix debug compile errors in parquet.hpp ([#12372](https://github.com/rapidsai/cudf/pull/12372)) [@davidwendt](https://github.com/davidwendt) +- Purge non-empty nulls in `cudf::make_lists_column` ([#12370](https://github.com/rapidsai/cudf/pull/12370)) [@ttnghia](https://github.com/ttnghia) +- Use correct memory resource in io::make_column ([#12364](https://github.com/rapidsai/cudf/pull/12364)) [@vyasr](https://github.com/vyasr) +- Add code to detect possible malformed page data in parquet files. ([#12360](https://github.com/rapidsai/cudf/pull/12360)) [@nvdbaranec](https://github.com/nvdbaranec) +- Fail loudly to avoid data corruption with unsupported input in `read_orc` ([#12325](https://github.com/rapidsai/cudf/pull/12325)) [@vuule](https://github.com/vuule) +- Fix NumericPairIteratorTest for float values ([#12306](https://github.com/rapidsai/cudf/pull/12306)) [@davidwendt](https://github.com/davidwendt) +- Fixes memory allocation in nested JSON tokenizer ([#12300](https://github.com/rapidsai/cudf/pull/12300)) [@elstehle](https://github.com/elstehle) +- Reconstruct dtypes correctly for list aggs of struct columns ([#12290](https://github.com/rapidsai/cudf/pull/12290)) [@wence-](https://github.com/wence-) +- Fix regex \A and \Z to strictly match string begin/end ([#12282](https://github.com/rapidsai/cudf/pull/12282)) [@davidwendt](https://github.com/davidwendt) +- Fix compile issue in `json_chunked_reader.cpp` ([#12280](https://github.com/rapidsai/cudf/pull/12280)) [@ttnghia](https://github.com/ttnghia) +- Change reductions any/all to return valid values for empty input ([#12279](https://github.com/rapidsai/cudf/pull/12279)) [@davidwendt](https://github.com/davidwendt) +- Only exclude join keys that are indices from key columns ([#12271](https://github.com/rapidsai/cudf/pull/12271)) [@wence-](https://github.com/wence-) +- Fix spill to device limit ([#12252](https://github.com/rapidsai/cudf/pull/12252)) [@madsbk](https://github.com/madsbk) +- Correct behaviour of sort in `concat` for singleton concatenations ([#12247](https://github.com/rapidsai/cudf/pull/12247)) [@wence-](https://github.com/wence-) +- Purge non-empty nulls for `superimpose_nulls` and `push_down_nulls` ([#12239](https://github.com/rapidsai/cudf/pull/12239)) [@ttnghia](https://github.com/ttnghia) +- Patch CUB DeviceSegmentedSort and remove workaround ([#12234](https://github.com/rapidsai/cudf/pull/12234)) [@davidwendt](https://github.com/davidwendt) +- Fix memory leak in udf_string::assign(&&) function ([#12206](https://github.com/rapidsai/cudf/pull/12206)) [@davidwendt](https://github.com/davidwendt) +- Workaround thrust-copy-if limit in json get_tree_representation ([#12190](https://github.com/rapidsai/cudf/pull/12190)) [@davidwendt](https://github.com/davidwendt) +- Fix page size calculation in Parquet writer ([#12182](https://github.com/rapidsai/cudf/pull/12182)) [@etseidl](https://github.com/etseidl) +- Add cudf::detail::sizes_to_offsets_iterator to allow checking overflow in offsets ([#12180](https://github.com/rapidsai/cudf/pull/12180)) [@davidwendt](https://github.com/davidwendt) +- Workaround thrust-copy-if limit in wordpiece-tokenizer ([#12168](https://github.com/rapidsai/cudf/pull/12168)) [@davidwendt](https://github.com/davidwendt) +- Floor division uses integer division for integral arguments ([#12131](https://github.com/rapidsai/cudf/pull/12131)) [@wence-](https://github.com/wence-) + +## 📖 Documentation + +- Fix link to NVTX ([#12598](https://github.com/rapidsai/cudf/pull/12598)) [@sameerz](https://github.com/sameerz) +- Include missing groupby functions in documentation ([#12580](https://github.com/rapidsai/cudf/pull/12580)) [@quasiben](https://github.com/quasiben) +- Fix documentation author ([#12527](https://github.com/rapidsai/cudf/pull/12527)) [@bdice](https://github.com/bdice) +- Update libcudf reduction docs for casting output types ([#12526](https://github.com/rapidsai/cudf/pull/12526)) [@davidwendt](https://github.com/davidwendt) +- Add JSON reader page in user guide ([#12499](https://github.com/rapidsai/cudf/pull/12499)) [@GregoryKimball](https://github.com/GregoryKimball) +- Link unsupported iteration API docstrings ([#12482](https://github.com/rapidsai/cudf/pull/12482)) [@galipremsagar](https://github.com/galipremsagar) +- `strings_udf` doc update ([#12469](https://github.com/rapidsai/cudf/pull/12469)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Update cudf_assert docs with correct NDEBUG behavior ([#12464](https://github.com/rapidsai/cudf/pull/12464)) [@robertmaynard](https://github.com/robertmaynard) +- Update pre-commit hooks guide ([#12395](https://github.com/rapidsai/cudf/pull/12395)) [@bdice](https://github.com/bdice) +- Update test docs to not use detail comparison utilities ([#12332](https://github.com/rapidsai/cudf/pull/12332)) [@PointKernel](https://github.com/PointKernel) +- Fix doxygen description for regex_program::compute_working_memory_size ([#12329](https://github.com/rapidsai/cudf/pull/12329)) [@davidwendt](https://github.com/davidwendt) +- Add eval to docs. ([#12322](https://github.com/rapidsai/cudf/pull/12322)) [@vyasr](https://github.com/vyasr) +- Turn on xfail_strict=true ([#12244](https://github.com/rapidsai/cudf/pull/12244)) [@wence-](https://github.com/wence-) +- Update 10 minutes to cuDF ([#12114](https://github.com/rapidsai/cudf/pull/12114)) [@wence-](https://github.com/wence-) + +## 🚀 New Features + +- Use kvikIO as the default IO backend ([#12574](https://github.com/rapidsai/cudf/pull/12574)) [@vuule](https://github.com/vuule) +- Use `has_nonempty_nulls` instead of `may_contain_non_empty_nulls` in `superimpose_nulls` and `push_down_nulls` ([#12560](https://github.com/rapidsai/cudf/pull/12560)) [@ttnghia](https://github.com/ttnghia) +- Add strings methods removeprefix and removesuffix ([#12557](https://github.com/rapidsai/cudf/pull/12557)) [@davidwendt](https://github.com/davidwendt) +- Add `regex_program` java APIs and unit tests ([#12548](https://github.com/rapidsai/cudf/pull/12548)) [@cindyyuanjiang](https://github.com/cindyyuanjiang) +- Default `cudf::io::read_json` to nested JSON parser ([#12544](https://github.com/rapidsai/cudf/pull/12544)) [@vuule](https://github.com/vuule) +- Make string quoting optional on CSV write ([#12539](https://github.com/rapidsai/cudf/pull/12539)) [@mythrocks](https://github.com/mythrocks) +- Use new nvCOMP API to optimize the compression temp memory size ([#12533](https://github.com/rapidsai/cudf/pull/12533)) [@vuule](https://github.com/vuule) +- Support "values" orient (array of arrays) in Nested JSON reader ([#12498](https://github.com/rapidsai/cudf/pull/12498)) [@karthikeyann](https://github.com/karthikeyann) +- `one_hot_encode` to use experimental row comparators ([#12478](https://github.com/rapidsai/cudf/pull/12478)) [@divyegala](https://github.com/divyegala) +- Support %W and %w format specifiers in cudf::strings::to_timestamps ([#12475](https://github.com/rapidsai/cudf/pull/12475)) [@davidwendt](https://github.com/davidwendt) +- Add JSON Writer ([#12474](https://github.com/rapidsai/cudf/pull/12474)) [@karthikeyann](https://github.com/karthikeyann) +- Refactor `thrust_copy_if` into `cudf::detail::copy_if_safe` ([#12455](https://github.com/rapidsai/cudf/pull/12455)) [@ttnghia](https://github.com/ttnghia) +- Add trailing comma support for nested JSON reader ([#12448](https://github.com/rapidsai/cudf/pull/12448)) [@karthikeyann](https://github.com/karthikeyann) +- Extract `tokenize_json.hpp` detail header from `src/io/json/nested_json.hpp` ([#12432](https://github.com/rapidsai/cudf/pull/12432)) [@ttnghia](https://github.com/ttnghia) +- JNI bindings to write CSV ([#12425](https://github.com/rapidsai/cudf/pull/12425)) [@mythrocks](https://github.com/mythrocks) +- Nested JSON depth benchmark ([#12371](https://github.com/rapidsai/cudf/pull/12371)) [@karthikeyann](https://github.com/karthikeyann) +- Implement `lists::reverse` ([#12336](https://github.com/rapidsai/cudf/pull/12336)) [@ttnghia](https://github.com/ttnghia) +- Use `device_read` in experimental `read_json` ([#12314](https://github.com/rapidsai/cudf/pull/12314)) [@vuule](https://github.com/vuule) +- Implement JNI for `strings::reverse` ([#12283](https://github.com/rapidsai/cudf/pull/12283)) [@ttnghia](https://github.com/ttnghia) +- Null element for parsing error in numeric types in JSON, CSV reader ([#12272](https://github.com/rapidsai/cudf/pull/12272)) [@karthikeyann](https://github.com/karthikeyann) +- Add cudf::strings:like function with multiple patterns ([#12269](https://github.com/rapidsai/cudf/pull/12269)) [@davidwendt](https://github.com/davidwendt) +- Add environment variable to control host memory allocation in `hostdevice_vector` ([#12251](https://github.com/rapidsai/cudf/pull/12251)) [@vuule](https://github.com/vuule) +- Add cudf::strings::reverse function ([#12227](https://github.com/rapidsai/cudf/pull/12227)) [@davidwendt](https://github.com/davidwendt) +- Selectively use dictionary encoding in Parquet writer ([#12211](https://github.com/rapidsai/cudf/pull/12211)) [@etseidl](https://github.com/etseidl) +- Support `replace` in `strings_udf` ([#12207](https://github.com/rapidsai/cudf/pull/12207)) [@brandon-b-miller](https://github.com/brandon-b-miller) +- Add support to read binary encoded decimals in parquet ([#12205](https://github.com/rapidsai/cudf/pull/12205)) [@PointKernel](https://github.com/PointKernel) +- Support regex EOL where the string ends with a new-line character ([#12181](https://github.com/rapidsai/cudf/pull/12181)) [@davidwendt](https://github.com/davidwendt) +- Updating `stream_compaction/unique` to use new row comparators ([#12159](https://github.com/rapidsai/cudf/pull/12159)) [@divyegala](https://github.com/divyegala) +- Add device buffer datasource ([#12024](https://github.com/rapidsai/cudf/pull/12024)) [@PointKernel](https://github.com/PointKernel) +- Implement groupby apply with JIT ([#11452](https://github.com/rapidsai/cudf/pull/11452)) [@bwyogatama](https://github.com/bwyogatama) + +## 🛠️ Improvements + +- Update shared workflow branches ([#12696](https://github.com/rapidsai/cudf/pull/12696)) [@ajschmidt8](https://github.com/ajschmidt8) +- Pin `dask` and `distributed` for release ([#12695](https://github.com/rapidsai/cudf/pull/12695)) [@galipremsagar](https://github.com/galipremsagar) +- Don't upload `libcudf-example` to Anaconda.org ([#12671](https://github.com/rapidsai/cudf/pull/12671)) [@ajschmidt8](https://github.com/ajschmidt8) +- Pin wheel dependencies to same RAPIDS release ([#12659](https://github.com/rapidsai/cudf/pull/12659)) [@sevagh](https://github.com/sevagh) +- Use CTK 118/cp310 branch of wheel workflows ([#12602](https://github.com/rapidsai/cudf/pull/12602)) [@sevagh](https://github.com/sevagh) +- Change ways to access `ptr` in `Buffer` ([#12587](https://github.com/rapidsai/cudf/pull/12587)) [@galipremsagar](https://github.com/galipremsagar) +- Version a parquet writer xfail ([#12579](https://github.com/rapidsai/cudf/pull/12579)) [@galipremsagar](https://github.com/galipremsagar) +- Remove column names ([#12578](https://github.com/rapidsai/cudf/pull/12578)) [@vuule](https://github.com/vuule) +- Parquet reader optimization to address V100 regression. ([#12577](https://github.com/rapidsai/cudf/pull/12577)) [@nvdbaranec](https://github.com/nvdbaranec) +- Add support for `category` dtypes in CSV reader ([#12571](https://github.com/rapidsai/cudf/pull/12571)) [@galipremsagar](https://github.com/galipremsagar) +- Remove `spill_lock` parameter from `SpillableBuffer.get_ptr()` ([#12564](https://github.com/rapidsai/cudf/pull/12564)) [@madsbk](https://github.com/madsbk) +- Optimize `cudf::make_lists_column` ([#12547](https://github.com/rapidsai/cudf/pull/12547)) [@ttnghia](https://github.com/ttnghia) +- Remove `cudf::strings::repeat_strings_output_sizes` from Java and JNI ([#12546](https://github.com/rapidsai/cudf/pull/12546)) [@ttnghia](https://github.com/ttnghia) +- Test that cuInit is not called when RAPIDS_NO_INITIALIZE is set ([#12545](https://github.com/rapidsai/cudf/pull/12545)) [@wence-](https://github.com/wence-) +- Rework repeat_strings to use sizes-to-offsets utility ([#12543](https://github.com/rapidsai/cudf/pull/12543)) [@davidwendt](https://github.com/davidwendt) +- Replace exclusive_scan with sizes_to_offsets in cudf::lists::sequences ([#12541](https://github.com/rapidsai/cudf/pull/12541)) [@davidwendt](https://github.com/davidwendt) +- Rework nvtext::ngrams_tokenize to use sizes-to-offsets utility ([#12540](https://github.com/rapidsai/cudf/pull/12540)) [@davidwendt](https://github.com/davidwendt) +- Fix binary-ops gtests coded in namespace cudf::test ([#12536](https://github.com/rapidsai/cudf/pull/12536)) [@davidwendt](https://github.com/davidwendt) +- More `[@acquire_spill_lock()` and `as_buffer(..., exposed=False)` ([#12535](https://github.com/rapidsai/cudf/pull/12535)) @madsbk](https://github.com/acquire_spill_lock()` and `as_buffer(..., exposed=False)` ([#12535](https://github.com/rapidsai/cudf/pull/12535)) @madsbk) +- Guard CUDA runtime APIs with error checking ([#12531](https://github.com/rapidsai/cudf/pull/12531)) [@PointKernel](https://github.com/PointKernel) +- Update TODOs from issue 10432. ([#12528](https://github.com/rapidsai/cudf/pull/12528)) [@bdice](https://github.com/bdice) +- Update rapids-cmake definitions version in GitHub Actions style checks. ([#12511](https://github.com/rapidsai/cudf/pull/12511)) [@bdice](https://github.com/bdice) +- Switch `engine=cudf` to the new `JSON` reader ([#12509](https://github.com/rapidsai/cudf/pull/12509)) [@galipremsagar](https://github.com/galipremsagar) +- Fix SUM/MEAN aggregation type support. ([#12503](https://github.com/rapidsai/cudf/pull/12503)) [@bdice](https://github.com/bdice) +- Stop using pandas._testing ([#12492](https://github.com/rapidsai/cudf/pull/12492)) [@vyasr](https://github.com/vyasr) +- Fix ROLLING_TEST gtests coded in namespace cudf::test ([#12490](https://github.com/rapidsai/cudf/pull/12490)) [@davidwendt](https://github.com/davidwendt) +- Fix erroneously skipped ORC ZSTD test ([#12486](https://github.com/rapidsai/cudf/pull/12486)) [@vuule](https://github.com/vuule) +- Rework nvtext::generate_character_ngrams to use make_strings_children ([#12480](https://github.com/rapidsai/cudf/pull/12480)) [@davidwendt](https://github.com/davidwendt) +- Raise warnings as errors in the test suite ([#12468](https://github.com/rapidsai/cudf/pull/12468)) [@vyasr](https://github.com/vyasr) +- Remove `int32` hard-coding in python ([#12467](https://github.com/rapidsai/cudf/pull/12467)) [@galipremsagar](https://github.com/galipremsagar) +- Use cudaMemcpyDefault. ([#12466](https://github.com/rapidsai/cudf/pull/12466)) [@bdice](https://github.com/bdice) +- Update workflows for nightly tests ([#12462](https://github.com/rapidsai/cudf/pull/12462)) [@ajschmidt8](https://github.com/ajschmidt8) +- Build CUDA `11.8` and Python `3.10` Packages ([#12457](https://github.com/rapidsai/cudf/pull/12457)) [@ajschmidt8](https://github.com/ajschmidt8) +- JNI build image default as cuda11.8 ([#12441](https://github.com/rapidsai/cudf/pull/12441)) [@pxLi](https://github.com/pxLi) +- Re-enable `Recently Updated` Check ([#12435](https://github.com/rapidsai/cudf/pull/12435)) [@ajschmidt8](https://github.com/ajschmidt8) +- Rework remaining cudf::strings::from_xyz functions to use make_strings_children ([#12434](https://github.com/rapidsai/cudf/pull/12434)) [@vuule](https://github.com/vuule) +- Build wheels alongside conda CI ([#12427](https://github.com/rapidsai/cudf/pull/12427)) [@sevagh](https://github.com/sevagh) +- Remove arguments for checking exception messages in Python ([#12424](https://github.com/rapidsai/cudf/pull/12424)) [@vyasr](https://github.com/vyasr) +- Clean up cuco usage ([#12421](https://github.com/rapidsai/cudf/pull/12421)) [@PointKernel](https://github.com/PointKernel) +- Fix warnings in remaining modules ([#12406](https://github.com/rapidsai/cudf/pull/12406)) [@vyasr](https://github.com/vyasr) +- Update `ops-bot.yaml` ([#12402](https://github.com/rapidsai/cudf/pull/12402)) [@ajschmidt8](https://github.com/ajschmidt8) +- Rework cudf::strings::integers_to_ipv4 to use make_strings_children utility ([#12401](https://github.com/rapidsai/cudf/pull/12401)) [@davidwendt](https://github.com/davidwendt) +- Use `numpy.empty()` instead of `bytearray` to allocate host memory for spilling ([#12399](https://github.com/rapidsai/cudf/pull/12399)) [@madsbk](https://github.com/madsbk) +- Deprecate chunksize from dask_cudf.read_csv ([#12394](https://github.com/rapidsai/cudf/pull/12394)) [@rjzamora](https://github.com/rjzamora) +- Expose the RMM pool size in JNI ([#12390](https://github.com/rapidsai/cudf/pull/12390)) [@revans2](https://github.com/revans2) +- Fix COPYING_TEST: gtests coded in namespace cudf::test ([#12387](https://github.com/rapidsai/cudf/pull/12387)) [@davidwendt](https://github.com/davidwendt) +- Rework cudf::strings::url_encode to use make_strings_children utility ([#12385](https://github.com/rapidsai/cudf/pull/12385)) [@davidwendt](https://github.com/davidwendt) +- Use make_strings_children in parse_data nested json reader ([#12382](https://github.com/rapidsai/cudf/pull/12382)) [@karthikeyann](https://github.com/karthikeyann) +- Fix warnings in test_datetime.py ([#12381](https://github.com/rapidsai/cudf/pull/12381)) [@vyasr](https://github.com/vyasr) +- Mixed Join Benchmarks ([#12375](https://github.com/rapidsai/cudf/pull/12375)) [@divyegala](https://github.com/divyegala) +- Fix warnings in dataframe.py ([#12369](https://github.com/rapidsai/cudf/pull/12369)) [@vyasr](https://github.com/vyasr) +- Update conda recipes. ([#12368](https://github.com/rapidsai/cudf/pull/12368)) [@bdice](https://github.com/bdice) +- Use gpu-latest-1 runner tag ([#12366](https://github.com/rapidsai/cudf/pull/12366)) [@bdice](https://github.com/bdice) +- Rework cudf::strings::from_booleans to use make_strings_children ([#12365](https://github.com/rapidsai/cudf/pull/12365)) [@vuule](https://github.com/vuule) +- Fix warnings in test modules up to test_dataframe.py ([#12355](https://github.com/rapidsai/cudf/pull/12355)) [@vyasr](https://github.com/vyasr) +- JSON column performance optimization - struct column nulls ([#12354](https://github.com/rapidsai/cudf/pull/12354)) [@karthikeyann](https://github.com/karthikeyann) +- Accelerate stable-segmented-sort with CUB segmented sort ([#12347](https://github.com/rapidsai/cudf/pull/12347)) [@davidwendt](https://github.com/davidwendt) +- Add size check to make_offsets_child_column utility ([#12345](https://github.com/rapidsai/cudf/pull/12345)) [@davidwendt](https://github.com/davidwendt) +- Enable max compression ratio small block optimization for ZSTD ([#12338](https://github.com/rapidsai/cudf/pull/12338)) [@vuule](https://github.com/vuule) +- Fix warnings in test_monotonic.py ([#12334](https://github.com/rapidsai/cudf/pull/12334)) [@vyasr](https://github.com/vyasr) +- Improve JSON column creation performance (list offsets) ([#12330](https://github.com/rapidsai/cudf/pull/12330)) [@karthikeyann](https://github.com/karthikeyann) +- Upgrade to `arrow-10.0.1` ([#12327](https://github.com/rapidsai/cudf/pull/12327)) [@galipremsagar](https://github.com/galipremsagar) +- Fix warnings in test_orc.py ([#12326](https://github.com/rapidsai/cudf/pull/12326)) [@vyasr](https://github.com/vyasr) +- Fix warnings in test_groupby.py ([#12324](https://github.com/rapidsai/cudf/pull/12324)) [@vyasr](https://github.com/vyasr) +- Fix `test_notebooks.sh` ([#12323](https://github.com/rapidsai/cudf/pull/12323)) [@ajschmidt8](https://github.com/ajschmidt8) +- Fix transform gtests coded in namespace cudf::test ([#12321](https://github.com/rapidsai/cudf/pull/12321)) [@davidwendt](https://github.com/davidwendt) +- Fix `check_style.sh` script ([#12320](https://github.com/rapidsai/cudf/pull/12320)) [@ajschmidt8](https://github.com/ajschmidt8) +- Rework cudf::strings::from_timestamps to use make_strings_children ([#12317](https://github.com/rapidsai/cudf/pull/12317)) [@davidwendt](https://github.com/davidwendt) +- Fix warnings in test_index.py ([#12313](https://github.com/rapidsai/cudf/pull/12313)) [@vyasr](https://github.com/vyasr) +- Fix warnings in test_multiindex.py ([#12310](https://github.com/rapidsai/cudf/pull/12310)) [@vyasr](https://github.com/vyasr) +- CSV, JSON reader to infer integer column with nulls as int64 instead of float64 ([#12309](https://github.com/rapidsai/cudf/pull/12309)) [@karthikeyann](https://github.com/karthikeyann) +- Fix warnings in test_indexing.py ([#12305](https://github.com/rapidsai/cudf/pull/12305)) [@vyasr](https://github.com/vyasr) +- Fix warnings in test_joining.py ([#12304](https://github.com/rapidsai/cudf/pull/12304)) [@vyasr](https://github.com/vyasr) +- Unpin `dask` and `distributed` for development ([#12302](https://github.com/rapidsai/cudf/pull/12302)) [@galipremsagar](https://github.com/galipremsagar) +- Re-enable `sccache` for Jenkins builds ([#12297](https://github.com/rapidsai/cudf/pull/12297)) [@ajschmidt8](https://github.com/ajschmidt8) +- Define needs for pr-builder workflow. ([#12296](https://github.com/rapidsai/cudf/pull/12296)) [@bdice](https://github.com/bdice) +- Forward merge 22.12 into 23.02 ([#12294](https://github.com/rapidsai/cudf/pull/12294)) [@vyasr](https://github.com/vyasr) +- Fix warnings in test_stats.py ([#12293](https://github.com/rapidsai/cudf/pull/12293)) [@vyasr](https://github.com/vyasr) +- Fix table gtests coded in namespace cudf::test ([#12292](https://github.com/rapidsai/cudf/pull/12292)) [@davidwendt](https://github.com/davidwendt) +- Change cython for regex calls to use cudf::strings::regex_program ([#12289](https://github.com/rapidsai/cudf/pull/12289)) [@davidwendt](https://github.com/davidwendt) +- Improved error reporting when reading multiple JSON files ([#12285](https://github.com/rapidsai/cudf/pull/12285)) [@vuule](https://github.com/vuule) +- Deprecate Frame.sum_of_squares ([#12284](https://github.com/rapidsai/cudf/pull/12284)) [@vyasr](https://github.com/vyasr) +- Remove deprecated code for 23.02 ([#12281](https://github.com/rapidsai/cudf/pull/12281)) [@vyasr](https://github.com/vyasr) +- Clean up handling of max_page_size_bytes in Parquet writer ([#12277](https://github.com/rapidsai/cudf/pull/12277)) [@etseidl](https://github.com/etseidl) +- Fix replace gtests coded in namespace cudf::test ([#12270](https://github.com/rapidsai/cudf/pull/12270)) [@davidwendt](https://github.com/davidwendt) +- Add pandas nullable type support in `Index.to_pandas` ([#12268](https://github.com/rapidsai/cudf/pull/12268)) [@galipremsagar](https://github.com/galipremsagar) +- Rework nvtext::detokenize to use indexalator for row indices ([#12267](https://github.com/rapidsai/cudf/pull/12267)) [@davidwendt](https://github.com/davidwendt) +- Fix reduction gtests coded in namespace cudf::test ([#12257](https://github.com/rapidsai/cudf/pull/12257)) [@davidwendt](https://github.com/davidwendt) +- Remove default parameters from cudf::detail::sort function declarations ([#12254](https://github.com/rapidsai/cudf/pull/12254)) [@davidwendt](https://github.com/davidwendt) +- Add `duplicated` support for `Series`, `DataFrame` and `Index` ([#12246](https://github.com/rapidsai/cudf/pull/12246)) [@galipremsagar](https://github.com/galipremsagar) +- Replace column/table test utilities with macros ([#12242](https://github.com/rapidsai/cudf/pull/12242)) [@PointKernel](https://github.com/PointKernel) +- Rework cudf::strings::pad and zfill to use make_strings_children ([#12238](https://github.com/rapidsai/cudf/pull/12238)) [@davidwendt](https://github.com/davidwendt) +- Fix sort gtests coded in namespace cudf::test ([#12237](https://github.com/rapidsai/cudf/pull/12237)) [@davidwendt](https://github.com/davidwendt) +- Wrapping concat and file writes in `[@acquire_spill_lock()` ([#12232](https://github.com/rapidsai/cudf/pull/12232)) @madsbk](https://github.com/acquire_spill_lock()` ([#12232](https://github.com/rapidsai/cudf/pull/12232)) @madsbk) +- Rename `cudf::structs::detail::superimpose_parent_nulls` APIs ([#12230](https://github.com/rapidsai/cudf/pull/12230)) [@ttnghia](https://github.com/ttnghia) +- Cover parsing to decimal types in `read_json` tests ([#12229](https://github.com/rapidsai/cudf/pull/12229)) [@vuule](https://github.com/vuule) +- Spill Statistics ([#12223](https://github.com/rapidsai/cudf/pull/12223)) [@madsbk](https://github.com/madsbk) +- Use CUDF_JNI_ENABLE_PROFILING to conditionally enable profiling support. ([#12221](https://github.com/rapidsai/cudf/pull/12221)) [@bdice](https://github.com/bdice) +- Clean up of `test_spilling.py` ([#12220](https://github.com/rapidsai/cudf/pull/12220)) [@madsbk](https://github.com/madsbk) +- Simplify repetitive boolean logic ([#12218](https://github.com/rapidsai/cudf/pull/12218)) [@vuule](https://github.com/vuule) +- Add `Series.hasnans` and `Index.hasnans` ([#12214](https://github.com/rapidsai/cudf/pull/12214)) [@galipremsagar](https://github.com/galipremsagar) +- Add cudf::strings:udf::replace function ([#12210](https://github.com/rapidsai/cudf/pull/12210)) [@davidwendt](https://github.com/davidwendt) +- Adds in new java APIs for appending byte arrays to host columnar data ([#12208](https://github.com/rapidsai/cudf/pull/12208)) [@revans2](https://github.com/revans2) +- Remove Python dependencies from Java CI. ([#12193](https://github.com/rapidsai/cudf/pull/12193)) [@bdice](https://github.com/bdice) +- Fix null order in sort-based groupby and improve groupby tests ([#12191](https://github.com/rapidsai/cudf/pull/12191)) [@divyegala](https://github.com/divyegala) +- Move strings children functions from cudf/strings/detail/utilities.cuh to new header ([#12185](https://github.com/rapidsai/cudf/pull/12185)) [@davidwendt](https://github.com/davidwendt) +- Clean up existing JNI scalar to column code ([#12173](https://github.com/rapidsai/cudf/pull/12173)) [@revans2](https://github.com/revans2) +- Remove JIT type names, refactor id_to_type. ([#12158](https://github.com/rapidsai/cudf/pull/12158)) [@bdice](https://github.com/bdice) +- Update JNI version to 23.02.0-SNAPSHOT ([#12129](https://github.com/rapidsai/cudf/pull/12129)) [@pxLi](https://github.com/pxLi) +- Minor refactor of cpp/src/io/parquet/page_data.cu ([#12126](https://github.com/rapidsai/cudf/pull/12126)) [@etseidl](https://github.com/etseidl) +- Add codespell as a linter ([#12097](https://github.com/rapidsai/cudf/pull/12097)) [@benfred](https://github.com/benfred) +- Enable specifying exceptions in error macros ([#12078](https://github.com/rapidsai/cudf/pull/12078)) [@vyasr](https://github.com/vyasr) +- Move `_label_encoding` from Series to Column ([#12040](https://github.com/rapidsai/cudf/pull/12040)) [@shwina](https://github.com/shwina) +- Add GitHub Actions Workflows ([#12002](https://github.com/rapidsai/cudf/pull/12002)) [@ajschmidt8](https://github.com/ajschmidt8) +- Consolidate dask-cudf `groupby_agg` calls in one place ([#10835](https://github.com/rapidsai/cudf/pull/10835)) [@charlesbluca](https://github.com/charlesbluca) # cuDF 22.12.00 (8 Dec 2022) From ac60656bc929c08f522d23328a515caa52cee989 Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Thu, 9 Feb 2023 11:13:21 -0500 Subject: [PATCH 16/69] Fix update-version.sh (#12745) Fix the `update-version.sh` script by removing a file that no longer exists Authors: - Ray Douglass (https://github.com/raydouglass) Approvers: - Sevag H (https://github.com/sevagh) --- ci/release/update-version.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 555a67d9cd6..96099b0512d 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -43,9 +43,6 @@ sed_runner 's/'"cudf_version .*)"'/'"cudf_version ${NEXT_FULL_TAG})"'/g' python/ # Strings UDF update sed_runner 's/'"strings_udf_version .*)"'/'"strings_udf_version ${NEXT_FULL_TAG})"'/g' python/strings_udf/CMakeLists.txt -# Groupby UDF update -sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' python/cudf/udf_cpp/CMakeLists.txt - # cpp libcudf_kafka update sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/libcudf_kafka/CMakeLists.txt From 0cab19aca4c65453b9d1715b34999dbfc5d0a162 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Thu, 9 Feb 2023 16:52:12 -0500 Subject: [PATCH 17/69] Reduce the number of test cases in multibyte_split benchmark (#12737) Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12737 --- cpp/benchmarks/io/text/multibyte_split.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/cpp/benchmarks/io/text/multibyte_split.cpp b/cpp/benchmarks/io/text/multibyte_split.cpp index c3b7c585055..261243d29fb 100644 --- a/cpp/benchmarks/io/text/multibyte_split.cpp +++ b/cpp/benchmarks/io/text/multibyte_split.cpp @@ -209,10 +209,21 @@ using source_type_list = nvbench::enum_type_list; -NVBENCH_BENCH_TYPES(bench_multibyte_split, NVBENCH_TYPE_AXES(source_type_list)) - .set_name("multibyte_split") +NVBENCH_BENCH_TYPES(bench_multibyte_split, + NVBENCH_TYPE_AXES(nvbench::enum_type_list)) + .set_name("multibyte_split_delimiters") + .set_min_samples(4) .add_int64_axis("strip_delimiters", {0, 1}) .add_int64_axis("delim_size", {1, 4, 7}) .add_int64_axis("delim_percent", {1, 25}) + .add_int64_power_of_two_axis("size_approx", {15}) + .add_int64_axis("byte_range_percent", {50}); + +NVBENCH_BENCH_TYPES(bench_multibyte_split, NVBENCH_TYPE_AXES(source_type_list)) + .set_name("multibyte_split_source") + .set_min_samples(4) + .add_int64_axis("strip_delimiters", {1}) + .add_int64_axis("delim_size", {1}) + .add_int64_axis("delim_percent", {1}) .add_int64_power_of_two_axis("size_approx", {15, 30}) - .add_int64_axis("byte_range_percent", {1, 5, 25, 50, 100}); + .add_int64_axis("byte_range_percent", {10, 100}); From c931d5abd8b32fa9106cf6dd004ae0fdfde466b9 Mon Sep 17 00:00:00 2001 From: Yunsong Wang Date: Fri, 10 Feb 2023 11:48:08 -0500 Subject: [PATCH 18/69] Update default data source in cuio reader benchmarks (#12740) Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12740 --- cpp/benchmarks/io/csv/csv_reader_input.cpp | 21 +++++++++++-------- cpp/benchmarks/io/orc/orc_reader_input.cpp | 19 ++++++++++------- .../io/parquet/parquet_reader_input.cpp | 21 +++++++++++-------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/cpp/benchmarks/io/csv/csv_reader_input.cpp b/cpp/benchmarks/io/csv/csv_reader_input.cpp index 27fea856332..a68f689e4db 100644 --- a/cpp/benchmarks/io/csv/csv_reader_input.cpp +++ b/cpp/benchmarks/io/csv/csv_reader_input.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -64,19 +64,20 @@ void csv_read_common(DataType const& data_types, state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); } -template -void BM_csv_read_input(nvbench::state& state, nvbench::type_list>) +template +void BM_csv_read_input(nvbench::state& state, + nvbench::type_list, nvbench::enum_type>) { cudf::rmm_pool_raii rmm_pool; auto const d_type = get_type_or_group(static_cast(DataType)); - auto const source_type = io_type::FILEPATH; + auto const source_type = IOType; csv_read_common(d_type, source_type, state); } -template -void BM_csv_read_io(nvbench::state& state, nvbench::type_list>) +template +void BM_csv_read_io(nvbench::state& state, nvbench::type_list>) { cudf::rmm_pool_raii rmm_pool; @@ -86,7 +87,7 @@ void BM_csv_read_io(nvbench::state& state, nvbench::type_list(data_type::TIMESTAMP), static_cast(data_type::DURATION), static_cast(data_type::STRING)}); - auto const source_type = IO; + auto const source_type = IOType; csv_read_common(d_type, source_type, state); } @@ -101,9 +102,11 @@ using d_type_list = nvbench::enum_type_list; -NVBENCH_BENCH_TYPES(BM_csv_read_input, NVBENCH_TYPE_AXES(d_type_list)) +NVBENCH_BENCH_TYPES(BM_csv_read_input, + NVBENCH_TYPE_AXES(d_type_list, + nvbench::enum_type_list)) .set_name("csv_read_data_type") - .set_type_axes_names({"data_type"}) + .set_type_axes_names({"data_type", "io"}) .set_min_samples(4); NVBENCH_BENCH_TYPES(BM_csv_read_io, NVBENCH_TYPE_AXES(io_list)) diff --git a/cpp/benchmarks/io/orc/orc_reader_input.cpp b/cpp/benchmarks/io/orc/orc_reader_input.cpp index 3f8c096140e..a57a12debc6 100644 --- a/cpp/benchmarks/io/orc/orc_reader_input.cpp +++ b/cpp/benchmarks/io/orc/orc_reader_input.cpp @@ -57,8 +57,9 @@ void orc_read_common(cudf::io::orc_writer_options const& opts, state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); } -template -void BM_orc_read_data(nvbench::state& state, nvbench::type_list>) +template +void BM_orc_read_data(nvbench::state& state, + nvbench::type_list, nvbench::enum_type>) { cudf::rmm_pool_raii rmm_pool; @@ -72,17 +73,17 @@ void BM_orc_read_data(nvbench::state& state, nvbench::type_listview(); - cuio_source_sink_pair source_sink(io_type::HOST_BUFFER); + cuio_source_sink_pair source_sink(IOType); cudf::io::orc_writer_options opts = cudf::io::orc_writer_options::builder(source_sink.make_sink_info(), view); orc_read_common(opts, source_sink, state); } -template +template void BM_orc_read_io_compression( nvbench::state& state, - nvbench::type_list, nvbench::enum_type>) + nvbench::type_list, nvbench::enum_type>) { cudf::rmm_pool_raii rmm_pool; @@ -103,7 +104,7 @@ void BM_orc_read_io_compression( data_profile_builder().cardinality(cardinality).avg_run_length(run_length)); auto const view = tbl->view(); - cuio_source_sink_pair source_sink(IO); + cuio_source_sink_pair source_sink(IOType); cudf::io::orc_writer_options opts = cudf::io::orc_writer_options::builder(source_sink.make_sink_info(), view) .compression(Compression); @@ -126,9 +127,11 @@ using io_list = nvbench::enum_type_list; -NVBENCH_BENCH_TYPES(BM_orc_read_data, NVBENCH_TYPE_AXES(d_type_list)) +NVBENCH_BENCH_TYPES(BM_orc_read_data, + NVBENCH_TYPE_AXES(d_type_list, + nvbench::enum_type_list)) .set_name("orc_read_decode") - .set_type_axes_names({"data_type"}) + .set_type_axes_names({"data_type", "io"}) .set_min_samples(4) .add_int64_axis("cardinality", {0, 1000}) .add_int64_axis("run_length", {1, 32}); diff --git a/cpp/benchmarks/io/parquet/parquet_reader_input.cpp b/cpp/benchmarks/io/parquet/parquet_reader_input.cpp index 36a62903f31..fba69cb2b0f 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_input.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_input.cpp @@ -57,8 +57,10 @@ void parquet_read_common(cudf::io::parquet_writer_options const& write_opts, state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); } -template -void BM_parquet_read_data(nvbench::state& state, nvbench::type_list>) +template +void BM_parquet_read_data( + nvbench::state& state, + nvbench::type_list, nvbench::enum_type>) { cudf::rmm_pool_raii rmm_pool; @@ -66,7 +68,6 @@ void BM_parquet_read_data(nvbench::state& state, nvbench::type_listview(); - cuio_source_sink_pair source_sink(source_type); + cuio_source_sink_pair source_sink(IOType); cudf::io::parquet_writer_options write_opts = cudf::io::parquet_writer_options::builder(source_sink.make_sink_info(), view) .compression(compression); @@ -82,10 +83,10 @@ void BM_parquet_read_data(nvbench::state& state, nvbench::type_list +template void BM_parquet_read_io_compression( nvbench::state& state, - nvbench::type_list, nvbench::enum_type>) + nvbench::type_list, nvbench::enum_type>) { cudf::rmm_pool_raii rmm_pool; @@ -101,7 +102,7 @@ void BM_parquet_read_io_compression( cudf::size_type const cardinality = state.get_int64("cardinality"); cudf::size_type const run_length = state.get_int64("run_length"); auto const compression = Compression; - auto const source_type = IO; + auto const source_type = IOType; auto const tbl = create_random_table(cycle_dtypes(d_type, num_cols), @@ -133,9 +134,11 @@ using io_list = nvbench::enum_type_list; -NVBENCH_BENCH_TYPES(BM_parquet_read_data, NVBENCH_TYPE_AXES(d_type_list)) +NVBENCH_BENCH_TYPES(BM_parquet_read_data, + NVBENCH_TYPE_AXES(d_type_list, + nvbench::enum_type_list)) .set_name("parquet_read_decode") - .set_type_axes_names({"data_type"}) + .set_type_axes_names({"data_type", "io"}) .set_min_samples(4) .add_int64_axis("cardinality", {0, 1000}) .add_int64_axis("run_length", {1, 32}); From 048f9368d95189cf398b0389702c84891ade3cb1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 10 Feb 2023 11:29:49 -0600 Subject: [PATCH 19/69] Unpin `dask` and `distributed` for development (#12710) This PR unpins `dask` and `distributed` for `23.04` development. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Peter Andreas Entschev (https://github.com/pentschev) - Ray Douglass (https://github.com/raydouglass) URL: https://github.com/rapidsai/cudf/pull/12710 --- ci/benchmark/build.sh | 2 +- ci/cpu/build.sh | 2 +- ci/gpu/build.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 4 ++-- conda/recipes/custreamz/meta.yaml | 4 ++-- conda/recipes/dask-cudf/meta.yaml | 8 ++++---- conda/recipes/dask-cudf/run_test.sh | 2 +- dependencies.yaml | 4 ++-- python/dask_cudf/setup.py | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh index c27fe23d078..e221424d1cd 100755 --- a/ci/benchmark/build.sh +++ b/ci/benchmark/build.sh @@ -37,7 +37,7 @@ export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" # Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=0 +export INSTALL_DASK_MAIN=1 # Dask version to install when `INSTALL_DASK_MAIN=0` export DASK_STABLE_VERSION="2023.1.1" diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh index 5b4a201e5e9..3aa00ff7de9 100755 --- a/ci/cpu/build.sh +++ b/ci/cpu/build.sh @@ -35,7 +35,7 @@ export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" # Whether to keep `dask/label/dev` channel in the env. If INSTALL_DASK_MAIN=0, # `dask/label/dev` channel is removed. -export INSTALL_DASK_MAIN=0 +export INSTALL_DASK_MAIN=1 # Switch to project root; also root of repo checkout cd "$WORKSPACE" diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 0e790ba05ec..ff40fdf6c9f 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -39,7 +39,7 @@ export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` unset GIT_DESCRIBE_TAG # Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=0 +export INSTALL_DASK_MAIN=1 # Dask version to install when `INSTALL_DASK_MAIN=0` export DASK_STABLE_VERSION="2023.1.1" diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 4f62e48a6f1..675df3891c3 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -21,8 +21,8 @@ dependencies: - cxx-compiler - cython>=0.29,<0.30 - dask-cuda=23.04.* -- dask==2023.1.1 -- distributed==2023.1.1 +- dask>=2023.1.1 +- distributed>=2023.1.1 - dlpack>=0.5,<0.6.0a0 - doxygen=1.8.20 - fastavro>=0.22.9 diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml index 24f53289754..af5705341e6 100644 --- a/conda/recipes/custreamz/meta.yaml +++ b/conda/recipes/custreamz/meta.yaml @@ -39,8 +39,8 @@ requirements: - python - streamz - cudf ={{ version }} - - dask ==2023.1.1 - - distributed ==2023.1.1 + - dask >=2023.1.1 + - distributed >=2023.1.1 - python-confluent-kafka >=1.7.0,<1.8.0a0 - cudf_kafka ={{ version }} diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml index bc1c4783361..3ee3d4d3952 100644 --- a/conda/recipes/dask-cudf/meta.yaml +++ b/conda/recipes/dask-cudf/meta.yaml @@ -35,14 +35,14 @@ requirements: host: - python - cudf ={{ version }} - - dask ==2023.1.1 - - distributed ==2023.1.1 + - dask >=2023.1.1 + - distributed >=2023.1.1 - cudatoolkit ={{ cuda_version }} run: - python - cudf ={{ version }} - - dask ==2023.1.1 - - distributed ==2023.1.1 + - dask >=2023.1.1 + - distributed >=2023.1.1 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} test: diff --git a/conda/recipes/dask-cudf/run_test.sh b/conda/recipes/dask-cudf/run_test.sh index 3b1fc46c4f4..78be90757a2 100644 --- a/conda/recipes/dask-cudf/run_test.sh +++ b/conda/recipes/dask-cudf/run_test.sh @@ -18,7 +18,7 @@ if [ "${ARCH}" = "aarch64" ]; then fi # Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=0 +export INSTALL_DASK_MAIN=1 # Dask version to install when `INSTALL_DASK_MAIN=0` export DASK_STABLE_VERSION="2023.1.1" diff --git a/dependencies.yaml b/dependencies.yaml index 0a3a2ce7828..ae8eac4ea30 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -226,8 +226,8 @@ dependencies: - output_types: [conda, requirements] packages: - cachetools - - dask==2023.1.1 - - distributed==2023.1.1 + - dask>=2023.1.1 + - distributed>=2023.1.1 - fsspec>=0.6.0 - numba>=0.56.2 - numpy diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index be4c704019d..04145d23978 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -8,8 +8,8 @@ cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") install_requires = [ - "dask==2023.1.1", - "distributed==2023.1.1", + "dask>=2023.1.1", + "distributed>=2023.1.1", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.6.0dev0", From c4a1389bca6f2fd521bd5e768eda7407aa3e66b5 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Fri, 10 Feb 2023 11:36:23 -0600 Subject: [PATCH 20/69] Fix `Series` comparison vs scalars (#12519) Fixes an issue where this happens: ```python import cudf cudf.Series(['a','b','c']) == 1 ``` ``` File "/raid/brmiller/anaconda/envs/cudf_dev/lib/python3.9/site-packages/cudf/core/mixins/mixin_factory.py", line 11, in wrapper return method(self, *args1, *args2, **kwargs1, **kwargs2) File "/raid/brmiller/anaconda/envs/cudf_dev/lib/python3.9/site-packages/cudf/core/indexed_frame.py", line 3278, in _binaryop ColumnAccessor(type(self)._colwise_binop(operands, op)), File "/raid/brmiller/anaconda/envs/cudf_dev/lib/python3.9/site-packages/cudf/core/column_accessor.py", line 124, in __init__ column_length = len(data[next(iter(data))]) TypeError: object of type 'bool' has no len() ``` It turns out this happens because `StringColumn`'s `normalize_binop_value` method returns `NotImplemented` for scalars that are not of dtype `object`. This eventually causes python to dispatch to the python scalar class' `__eq__` which returns the scalar `False` when encountering a cuDF object. cuDF expects a column object at this point but has a scalar. This in turn causes cuDF to try and construct a `ColumnAccessor` around a dict that looks like `{'name', False}` ultimately throwing the error. This PR proposes to earlystop this behavior according to the rules for comparing python string scalars with other objects: - Always return `False` for `__eq__` even if the character in the string is equivalent to whatever is being compared - Always return `True` for `__ne__` ditto above. - Copy the input mask This should align us with pandas behavior for this case: ```python >>> pd.Series(['a','b', 'c'], dtype='string') == 1 0 False 1 False 2 False dtype: boolean >>> pd.Series(['a','b', 'c'], dtype='string') != 1 0 True 1 True 2 True dtype: boolean ``` EDIT: Updating this PR to handle a similar issue resulting in the same error when comparing datetime series to strings that contain valid datetimes, such as `20110101`. Authors: - https://github.com/brandon-b-miller Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12519 --- python/cudf/cudf/core/column/datetime.py | 7 +- python/cudf/cudf/core/column/string.py | 13 +++- python/cudf/cudf/tests/test_binops.py | 81 ++++++++++++++++++------ python/cudf/cudf/tests/test_datetime.py | 26 ++++++++ 4 files changed, 104 insertions(+), 23 deletions(-) diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index 56436ac141d..0c546168fe3 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. from __future__ import annotations @@ -261,6 +261,11 @@ def normalize_binop_value(self, other: DatetimeLikeScalar) -> ScalarLike: return cudf.Scalar(None, dtype=other.dtype) return cudf.Scalar(other) + elif isinstance(other, str): + try: + return cudf.Scalar(other, dtype=self.dtype) + except ValueError: + pass return NotImplemented diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 9c30585a541..ce8bc3da08b 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5665,7 +5665,7 @@ def normalize_binop_value( and other.dtype == "object" ): return other - if isinstance(other, str): + if is_scalar(other): return cudf.Scalar(other) return NotImplemented @@ -5701,6 +5701,17 @@ def _binaryop( return NotImplemented if isinstance(other, (StringColumn, str, cudf.Scalar)): + if isinstance(other, cudf.Scalar) and other.dtype != "O": + if op in { + "__eq__", + "__ne__", + }: + return column.full( + len(self), op == "__ne__", dtype="bool" + ).set_mask(self.mask) + else: + return NotImplemented + if op == "__add__": if isinstance(other, cudf.Scalar): other = cast( diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index e5ade1326c9..7d01f89eada 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. import decimal import operator @@ -320,29 +320,68 @@ def test_series_compare_nulls(cmpop, dtypes): utils.assert_eq(expect, got) -@pytest.mark.parametrize( - "obj", [pd.Series(["a", "b", None, "d", "e", None], dtype="string"), "a"] -) -@pytest.mark.parametrize("cmpop", _cmpops) -@pytest.mark.parametrize( - "cmp_obj", - [pd.Series(["b", "a", None, "d", "f", None], dtype="string"), "a"], -) -def test_string_series_compare(obj, cmpop, cmp_obj): +@pytest.fixture +def str_series_cmp_data(): + return pd.Series(["a", "b", None, "d", "e", None], dtype="string") - g_obj = obj - if isinstance(g_obj, pd.Series): - g_obj = Series.from_pandas(g_obj) - g_cmp_obj = cmp_obj - if isinstance(g_cmp_obj, pd.Series): - g_cmp_obj = Series.from_pandas(g_cmp_obj) - got = cmpop(g_obj, g_cmp_obj) - expected = cmpop(obj, cmp_obj) - if isinstance(expected, pd.Series): - expected = cudf.from_pandas(expected) +@pytest.fixture(ids=[op.__name__ for op in _cmpops], params=_cmpops) +def str_series_compare_str_cmpop(request): + return request.param - utils.assert_eq(expected, got) + +@pytest.fixture(ids=["eq", "ne"], params=[operator.eq, operator.ne]) +def str_series_compare_num_cmpop(request): + return request.param + + +@pytest.fixture(ids=["int", "float", "bool"], params=[1, 1.5, True]) +def cmp_scalar(request): + return request.param + + +def test_str_series_compare_str( + str_series_cmp_data, str_series_compare_str_cmpop +): + expect = str_series_compare_str_cmpop(str_series_cmp_data, "a") + got = str_series_compare_str_cmpop( + Series.from_pandas(str_series_cmp_data), "a" + ) + + utils.assert_eq(expect, got.to_pandas(nullable=True)) + + +def test_str_series_compare_str_reflected( + str_series_cmp_data, str_series_compare_str_cmpop +): + expect = str_series_compare_str_cmpop("a", str_series_cmp_data) + got = str_series_compare_str_cmpop( + "a", Series.from_pandas(str_series_cmp_data) + ) + + utils.assert_eq(expect, got.to_pandas(nullable=True)) + + +def test_str_series_compare_num( + str_series_cmp_data, str_series_compare_num_cmpop, cmp_scalar +): + expect = str_series_compare_num_cmpop(str_series_cmp_data, cmp_scalar) + got = str_series_compare_num_cmpop( + Series.from_pandas(str_series_cmp_data), cmp_scalar + ) + + utils.assert_eq(expect, got.to_pandas(nullable=True)) + + +def test_str_series_compare_num_reflected( + str_series_cmp_data, str_series_compare_num_cmpop, cmp_scalar +): + expect = str_series_compare_num_cmpop(cmp_scalar, str_series_cmp_data) + got = str_series_compare_num_cmpop( + cmp_scalar, Series.from_pandas(str_series_cmp_data) + ) + + utils.assert_eq(expect, got.to_pandas(nullable=True)) @pytest.mark.parametrize("obj_class", ["Series", "Index"]) diff --git a/python/cudf/cudf/tests/test_datetime.py b/python/cudf/cudf/tests/test_datetime.py index 5616cea42ba..1211938ff10 100644 --- a/python/cudf/cudf/tests/test_datetime.py +++ b/python/cudf/cudf/tests/test_datetime.py @@ -22,6 +22,15 @@ expect_warning_if, ) +_cmpops = [ + operator.lt, + operator.gt, + operator.le, + operator.ge, + operator.eq, + operator.ne, +] + def data1(): return pd.date_range("20010101", "20020215", freq="400h", name="times") @@ -986,6 +995,23 @@ def test_datetime_series_ops_with_scalars(data, other_scalars, dtype, op): ) +@pytest.mark.parametrize("data", ["20110101", "20120101", "20130101"]) +@pytest.mark.parametrize("other_scalars", ["20110101", "20120101", "20130101"]) +@pytest.mark.parametrize("op", _cmpops) +@pytest.mark.parametrize( + "dtype", + ["datetime64[ns]", "datetime64[us]", "datetime64[ms]", "datetime64[s]"], +) +def test_datetime_series_cmpops_with_scalars(data, other_scalars, dtype, op): + gsr = cudf.Series(data=data, dtype=dtype) + psr = gsr.to_pandas() + + expect = op(psr, other_scalars) + got = op(gsr, other_scalars) + + assert_eq(expect, got) + + @pytest.mark.parametrize( "data", [ From 2d7e79a7ae4f40fdb278798d02b79480aaef1adf Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 10 Feb 2023 15:37:30 -0500 Subject: [PATCH 21/69] Add nvbench environment class for initializing RMM in benchmarks (#12728) Adds an environment class to initialize the RMM pool memory resource manager before running benchmarks through nvbench. This removes the need to initialize RMM on every benchmark call which improves benchmark run performance and fixes some GPU metrics gathering when run under nsys. This requires a patch to the nvbench source to enable this feature and is part of the following pull request: https://github.com/NVIDIA/nvbench/pull/123 The patch can be removed once the PR is merged and source available to the libcudf build. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Divye Gala (https://github.com/divyegala) - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) - Robert Maynard (https://github.com/robertmaynard) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/12728 --- cpp/CMakeLists.txt | 9 ++--- cpp/benchmarks/fixture/rmm_pool_raii.hpp | 13 +++++++- cpp/benchmarks/groupby/group_max.cpp | 3 +- cpp/benchmarks/groupby/group_nunique.cpp | 3 +- cpp/benchmarks/groupby/group_rank.cpp | 3 +- cpp/benchmarks/groupby/group_struct_keys.cpp | 4 +-- cpp/benchmarks/io/csv/csv_reader_input.cpp | 4 --- cpp/benchmarks/io/csv/csv_reader_options.cpp | 4 +-- cpp/benchmarks/io/fst.cu | 14 +------- cpp/benchmarks/io/json/nested_json.cpp | 6 ---- cpp/benchmarks/io/orc/orc_reader_input.cpp | 4 --- cpp/benchmarks/io/orc/orc_reader_options.cpp | 4 +-- cpp/benchmarks/io/orc/orc_writer.cpp | 8 +---- cpp/benchmarks/io/orc/orc_writer_chunks.cpp | 6 +--- .../io/parquet/parquet_reader_input.cpp | 4 --- .../io/parquet/parquet_reader_options.cpp | 2 -- cpp/benchmarks/io/parquet/parquet_writer.cpp | 6 +--- .../io/parquet/parquet_writer_chunks.cpp | 6 +--- cpp/benchmarks/io/text/multibyte_split.cpp | 2 -- cpp/benchmarks/join/join.cu | 9 ----- cpp/benchmarks/join/mixed_join.cu | 15 --------- cpp/benchmarks/reduction/distinct_count.cpp | 4 +-- cpp/benchmarks/reduction/rank.cpp | 4 +-- cpp/benchmarks/reduction/scan_structs.cpp | 4 +-- cpp/benchmarks/reduction/segment_reduce.cu | 5 +-- cpp/benchmarks/search/contains.cpp | 3 +- cpp/benchmarks/sort/rank_lists.cpp | 2 -- cpp/benchmarks/sort/rank_structs.cpp | 2 -- cpp/benchmarks/sort/segmented_sort.cpp | 4 +-- cpp/benchmarks/sort/sort_lists.cpp | 2 -- cpp/benchmarks/sort/sort_structs.cpp | 2 -- cpp/benchmarks/stream_compaction/distinct.cpp | 6 +--- cpp/benchmarks/stream_compaction/unique.cpp | 4 --- cpp/benchmarks/string/like.cpp | 3 +- cpp/benchmarks/string/reverse.cpp | 3 +- cpp/cmake/thirdparty/get_nvbench.cmake | 33 +++++++++++++++++++ .../patches/nvbench_global_setup.diff | 27 +++++++++++++++ .../thirdparty/patches/nvbench_override.json | 14 ++++++++ 38 files changed, 109 insertions(+), 142 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_nvbench.cmake create mode 100644 cpp/cmake/thirdparty/patches/nvbench_global_setup.diff create mode 100644 cpp/cmake/thirdparty/patches/nvbench_override.json diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 19c118016bf..a635e655c39 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -784,12 +784,9 @@ if(CUDF_BUILD_BENCHMARKS) include(${rapids-cmake-dir}/cpm/gbench.cmake) rapids_cpm_gbench() - # Find or install NVBench Temporarily force downloading of fmt because current versions of nvbench - # do not support the latest version of fmt, which is automatically pulled into our conda - # environments by mamba. - set(CPM_DOWNLOAD_fmt TRUE) - include(${rapids-cmake-dir}/cpm/nvbench.cmake) - rapids_cpm_nvbench() + # Find or install nvbench + include(cmake/thirdparty/get_nvbench.cmake) + add_subdirectory(benchmarks) endif() diff --git a/cpp/benchmarks/fixture/rmm_pool_raii.hpp b/cpp/benchmarks/fixture/rmm_pool_raii.hpp index 60586ef878b..465c53a91ea 100644 --- a/cpp/benchmarks/fixture/rmm_pool_raii.hpp +++ b/cpp/benchmarks/fixture/rmm_pool_raii.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,4 +69,15 @@ class rmm_pool_raii { std::shared_ptr mr; }; +/** + * Base fixture for cudf benchmarks using nvbench. + * + * Initializes the default memory resource to use the RMM pool device resource. + */ +struct nvbench_base_fixture { + rmm_pool_raii _mr; +}; + } // namespace cudf + +#define NVBENCH_ENVIRONMENT cudf::nvbench_base_fixture diff --git a/cpp/benchmarks/groupby/group_max.cpp b/cpp/benchmarks/groupby/group_max.cpp index 4956cce0daf..077558f8709 100644 --- a/cpp/benchmarks/groupby/group_max.cpp +++ b/cpp/benchmarks/groupby/group_max.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,6 @@ template void bench_groupby_max(nvbench::state& state, nvbench::type_list) { - cudf::rmm_pool_raii pool_raii; const auto size = static_cast(state.get_int64("num_rows")); auto const keys = [&] { diff --git a/cpp/benchmarks/groupby/group_nunique.cpp b/cpp/benchmarks/groupby/group_nunique.cpp index 05698c04058..f74ed95200e 100644 --- a/cpp/benchmarks/groupby/group_nunique.cpp +++ b/cpp/benchmarks/groupby/group_nunique.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,6 @@ auto make_aggregation_request_vector(cudf::column_view const& values, Args&&... template void bench_groupby_nunique(nvbench::state& state, nvbench::type_list) { - cudf::rmm_pool_raii pool_raii; const auto size = static_cast(state.get_int64("num_rows")); auto const keys = [&] { diff --git a/cpp/benchmarks/groupby/group_rank.cpp b/cpp/benchmarks/groupby/group_rank.cpp index f573b63a75d..2a70b95890b 100644 --- a/cpp/benchmarks/groupby/group_rank.cpp +++ b/cpp/benchmarks/groupby/group_rank.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,6 @@ static void nvbench_groupby_rank(nvbench::state& state, { using namespace cudf; constexpr auto dtype = type_to_id(); - cudf::rmm_pool_raii pool_raii; bool const is_sorted = state.get_int64("is_sorted"); cudf::size_type const column_size = state.get_int64("data_size"); diff --git a/cpp/benchmarks/groupby/group_struct_keys.cpp b/cpp/benchmarks/groupby/group_struct_keys.cpp index cc6f0faaf41..53ef12ffeaa 100644 --- a/cpp/benchmarks/groupby/group_struct_keys.cpp +++ b/cpp/benchmarks/groupby/group_struct_keys.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,8 +29,6 @@ void bench_groupby_struct_keys(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; - using Type = int; using column_wrapper = cudf::test::fixed_width_column_wrapper; std::default_random_engine generator; diff --git a/cpp/benchmarks/io/csv/csv_reader_input.cpp b/cpp/benchmarks/io/csv/csv_reader_input.cpp index a68f689e4db..026045acee7 100644 --- a/cpp/benchmarks/io/csv/csv_reader_input.cpp +++ b/cpp/benchmarks/io/csv/csv_reader_input.cpp @@ -68,8 +68,6 @@ template void BM_csv_read_input(nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group(static_cast(DataType)); auto const source_type = IOType; @@ -79,8 +77,6 @@ void BM_csv_read_input(nvbench::state& state, template void BM_csv_read_io(nvbench::state& state, nvbench::type_list>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL), static_cast(data_type::FLOAT), static_cast(data_type::DECIMAL), diff --git a/cpp/benchmarks/io/csv/csv_reader_options.cpp b/cpp/benchmarks/io/csv/csv_reader_options.cpp index 04522c16d5c..2d0e0e5754e 100644 --- a/cpp/benchmarks/io/csv/csv_reader_options.cpp +++ b/cpp/benchmarks/io/csv/csv_reader_options.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,8 +32,6 @@ void BM_csv_read_varying_options( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const data_types = dtypes_for_column_selection(get_type_or_group({static_cast(data_type::INTEGRAL), static_cast(data_type::FLOAT), diff --git a/cpp/benchmarks/io/fst.cu b/cpp/benchmarks/io/fst.cu index 6d318db12de..7acf69e9d8e 100644 --- a/cpp/benchmarks/io/fst.cu +++ b/cpp/benchmarks/io/fst.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -78,9 +78,6 @@ constexpr std::size_t single_item = 1; void BM_FST_JSON(nvbench::state& state) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - CUDF_EXPECTS(state.get_int64("string_size") <= std::numeric_limits::max(), "Benchmarks only support up to size_type's maximum number of items"); auto const string_size{size_type(state.get_int64("string_size"))}; @@ -116,9 +113,6 @@ void BM_FST_JSON(nvbench::state& state) void BM_FST_JSON_no_outidx(nvbench::state& state) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - CUDF_EXPECTS(state.get_int64("string_size") <= std::numeric_limits::max(), "Benchmarks only support up to size_type's maximum number of items"); auto const string_size{size_type(state.get_int64("string_size"))}; @@ -154,9 +148,6 @@ void BM_FST_JSON_no_outidx(nvbench::state& state) void BM_FST_JSON_no_out(nvbench::state& state) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - CUDF_EXPECTS(state.get_int64("string_size") <= std::numeric_limits::max(), "Benchmarks only support up to size_type's maximum number of items"); auto const string_size{size_type(state.get_int64("string_size"))}; @@ -190,9 +181,6 @@ void BM_FST_JSON_no_out(nvbench::state& state) void BM_FST_JSON_no_str(nvbench::state& state) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - CUDF_EXPECTS(state.get_int64("string_size") <= std::numeric_limits::max(), "Benchmarks only support up to size_type's maximum number of items"); auto const string_size{size_type(state.get_int64("string_size"))}; diff --git a/cpp/benchmarks/io/json/nested_json.cpp b/cpp/benchmarks/io/json/nested_json.cpp index 2abae88dca3..416cf403671 100644 --- a/cpp/benchmarks/io/json/nested_json.cpp +++ b/cpp/benchmarks/io/json/nested_json.cpp @@ -157,9 +157,6 @@ auto make_test_json_data(cudf::size_type string_size, rmm::cuda_stream_view stre void BM_NESTED_JSON(nvbench::state& state) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - auto const string_size{cudf::size_type(state.get_int64("string_size"))}; auto const default_options = cudf::io::json_reader_options{}; @@ -189,9 +186,6 @@ NVBENCH_BENCH(BM_NESTED_JSON) void BM_NESTED_JSON_DEPTH(nvbench::state& state) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - auto const string_size{cudf::size_type(state.get_int64("string_size"))}; auto const depth{cudf::size_type(state.get_int64("depth"))}; diff --git a/cpp/benchmarks/io/orc/orc_reader_input.cpp b/cpp/benchmarks/io/orc/orc_reader_input.cpp index a57a12debc6..4705c083c02 100644 --- a/cpp/benchmarks/io/orc/orc_reader_input.cpp +++ b/cpp/benchmarks/io/orc/orc_reader_input.cpp @@ -61,8 +61,6 @@ template void BM_orc_read_data(nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group(static_cast(DataType)); cudf::size_type const cardinality = state.get_int64("cardinality"); cudf::size_type const run_length = state.get_int64("run_length"); @@ -85,8 +83,6 @@ void BM_orc_read_io_compression( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL_SIGNED), static_cast(data_type::FLOAT), static_cast(data_type::DECIMAL), diff --git a/cpp/benchmarks/io/orc/orc_reader_options.cpp b/cpp/benchmarks/io/orc/orc_reader_options.cpp index 1b7d33ccd19..1e841f744ae 100644 --- a/cpp/benchmarks/io/orc/orc_reader_options.cpp +++ b/cpp/benchmarks/io/orc/orc_reader_options.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,8 +56,6 @@ void BM_orc_read_varying_options(nvbench::state& state, nvbench::enum_type, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const num_chunks = RowSelection == row_selection::ALL ? 1 : chunked_read_num_chunks; auto const use_index = UsesIndex == uses_index::YES; diff --git a/cpp/benchmarks/io/orc/orc_writer.cpp b/cpp/benchmarks/io/orc/orc_writer.cpp index 545f8d10122..67bf4cb750b 100644 --- a/cpp/benchmarks/io/orc/orc_writer.cpp +++ b/cpp/benchmarks/io/orc/orc_writer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,8 +46,6 @@ constexpr cudf::size_type num_cols = 64; template void BM_orc_write_encode(nvbench::state& state, nvbench::type_list>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group(static_cast(DataType)); cudf::size_type const cardinality = state.get_int64("cardinality"); cudf::size_type const run_length = state.get_int64("run_length"); @@ -90,8 +88,6 @@ void BM_orc_write_io_compression( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL_SIGNED), static_cast(data_type::FLOAT), static_cast(data_type::DECIMAL), @@ -141,8 +137,6 @@ void BM_orc_write_statistics( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL_SIGNED), static_cast(data_type::FLOAT), static_cast(data_type::DECIMAL), diff --git a/cpp/benchmarks/io/orc/orc_writer_chunks.cpp b/cpp/benchmarks/io/orc/orc_writer_chunks.cpp index 592eae96362..eda70bc05e6 100644 --- a/cpp/benchmarks/io/orc/orc_writer_chunks.cpp +++ b/cpp/benchmarks/io/orc/orc_writer_chunks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,8 +35,6 @@ constexpr int64_t data_size = 512 << 20; void nvbench_orc_write(nvbench::state& state) { - cudf::rmm_pool_raii rmm_pool; - cudf::size_type num_cols = state.get_int64("num_columns"); auto tbl = create_random_table( @@ -79,8 +77,6 @@ void nvbench_orc_write(nvbench::state& state) void nvbench_orc_chunked_write(nvbench::state& state) { - cudf::rmm_pool_raii rmm_pool; - cudf::size_type num_cols = state.get_int64("num_columns"); cudf::size_type num_tables = state.get_int64("num_chunks"); diff --git a/cpp/benchmarks/io/parquet/parquet_reader_input.cpp b/cpp/benchmarks/io/parquet/parquet_reader_input.cpp index fba69cb2b0f..e04dfbbc799 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_input.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_input.cpp @@ -62,8 +62,6 @@ void BM_parquet_read_data( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group(static_cast(DataType)); cudf::size_type const cardinality = state.get_int64("cardinality"); cudf::size_type const run_length = state.get_int64("run_length"); @@ -88,8 +86,6 @@ void BM_parquet_read_io_compression( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL), static_cast(data_type::FLOAT), static_cast(data_type::DECIMAL), diff --git a/cpp/benchmarks/io/parquet/parquet_reader_options.cpp b/cpp/benchmarks/io/parquet/parquet_reader_options.cpp index 6e187afd6ab..3fd46fa08f2 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader_options.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader_options.cpp @@ -57,8 +57,6 @@ void BM_parquet_read_options(nvbench::state& state, nvbench::enum_type, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto constexpr str_to_categories = ConvertsStrings == converts_strings::YES; auto constexpr uses_pd_metadata = UsesPandasMetadata == uses_pandas_metadata::YES; diff --git a/cpp/benchmarks/io/parquet/parquet_writer.cpp b/cpp/benchmarks/io/parquet/parquet_writer.cpp index a0b076abfda..d3d22e06086 100644 --- a/cpp/benchmarks/io/parquet/parquet_writer.cpp +++ b/cpp/benchmarks/io/parquet/parquet_writer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,8 +46,6 @@ constexpr cudf::size_type num_cols = 64; template void BM_parq_write_encode(nvbench::state& state, nvbench::type_list>) { - cudf::rmm_pool_raii rmm_pool; - auto const data_types = get_type_or_group(static_cast(DataType)); cudf::size_type const cardinality = state.get_int64("cardinality"); cudf::size_type const run_length = state.get_int64("run_length"); @@ -90,8 +88,6 @@ void BM_parq_write_io_compression( nvbench::state& state, nvbench::type_list, nvbench::enum_type>) { - cudf::rmm_pool_raii rmm_pool; - auto const data_types = get_type_or_group({static_cast(data_type::INTEGRAL), static_cast(data_type::FLOAT), static_cast(data_type::DECIMAL), diff --git a/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp b/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp index 11b29cc2297..ed70f53cad8 100644 --- a/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp +++ b/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,8 +33,6 @@ constexpr int64_t data_size = 512 << 20; void PQ_write(nvbench::state& state) { - cudf::rmm_pool_raii rmm_pool; - cudf::size_type const num_cols = state.get_int64("num_cols"); auto const tbl = create_random_table(cycle_dtypes({cudf::type_id::INT32}, num_cols), @@ -67,8 +65,6 @@ void PQ_write(nvbench::state& state) void PQ_write_chunked(nvbench::state& state) { - cudf::rmm_pool_raii rmm_pool; - cudf::size_type const num_cols = state.get_int64("num_cols"); cudf::size_type const num_tables = state.get_int64("num_chunks"); diff --git a/cpp/benchmarks/io/text/multibyte_split.cpp b/cpp/benchmarks/io/text/multibyte_split.cpp index 261243d29fb..41b5ddb567e 100644 --- a/cpp/benchmarks/io/text/multibyte_split.cpp +++ b/cpp/benchmarks/io/text/multibyte_split.cpp @@ -116,8 +116,6 @@ template static void bench_multibyte_split(nvbench::state& state, nvbench::type_list>) { - cudf::rmm_pool_raii pool_raii; - auto const delim_size = state.get_int64("delim_size"); auto const delim_percent = state.get_int64("delim_percent"); auto const file_size_approx = state.get_int64("size_approx"); diff --git a/cpp/benchmarks/join/join.cu b/cpp/benchmarks/join/join.cu index 053eb6c2852..647e37aa97d 100644 --- a/cpp/benchmarks/join/join.cu +++ b/cpp/benchmarks/join/join.cu @@ -23,9 +23,6 @@ void nvbench_inner_join(nvbench::state& state, { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_input, cudf::table_view const& right_input, cudf::null_equality compare_nulls, @@ -43,9 +40,6 @@ void nvbench_left_join(nvbench::state& state, { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_input, cudf::table_view const& right_input, cudf::null_equality compare_nulls, @@ -63,9 +57,6 @@ void nvbench_full_join(nvbench::state& state, { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_input, cudf::table_view const& right_input, cudf::null_equality compare_nulls, diff --git a/cpp/benchmarks/join/mixed_join.cu b/cpp/benchmarks/join/mixed_join.cu index b7da5e2c0b3..1420625bbcd 100644 --- a/cpp/benchmarks/join/mixed_join.cu +++ b/cpp/benchmarks/join/mixed_join.cu @@ -23,9 +23,6 @@ void nvbench_mixed_inner_join( { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_equality_input, cudf::table_view const& right_equality_input, cudf::table_view const& left_conditional_input, @@ -50,9 +47,6 @@ void nvbench_mixed_left_join( { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_equality_input, cudf::table_view const& right_equality_input, cudf::table_view const& left_conditional_input, @@ -77,9 +71,6 @@ void nvbench_mixed_full_join( { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_equality_input, cudf::table_view const& right_equality_input, cudf::table_view const& left_conditional_input, @@ -104,9 +95,6 @@ void nvbench_mixed_left_semi_join( { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_equality_input, cudf::table_view const& right_equality_input, cudf::table_view const& left_conditional_input, @@ -131,9 +119,6 @@ void nvbench_mixed_left_anti_join( { skip_helper(state); - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii pool_raii; - auto join = [](cudf::table_view const& left_equality_input, cudf::table_view const& right_equality_input, cudf::table_view const& left_conditional_input, diff --git a/cpp/benchmarks/reduction/distinct_count.cpp b/cpp/benchmarks/reduction/distinct_count.cpp index 489d7935809..d2218c270a8 100644 --- a/cpp/benchmarks/reduction/distinct_count.cpp +++ b/cpp/benchmarks/reduction/distinct_count.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,8 +24,6 @@ template static void bench_reduction_distinct_count(nvbench::state& state, nvbench::type_list) { - cudf::rmm_pool_raii pool_raii; - auto const dtype = cudf::type_to_id(); auto const size = static_cast(state.get_int64("num_rows")); auto const null_probability = state.get_float64("null_probability"); diff --git a/cpp/benchmarks/reduction/rank.cpp b/cpp/benchmarks/reduction/rank.cpp index 5022e029d97..41295f787fc 100644 --- a/cpp/benchmarks/reduction/rank.cpp +++ b/cpp/benchmarks/reduction/rank.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,8 +26,6 @@ template static void nvbench_reduction_scan(nvbench::state& state, nvbench::type_list) { - cudf::rmm_pool_raii pool_raii; - auto const dtype = cudf::type_to_id(); double const null_probability = state.get_float64("null_probability"); diff --git a/cpp/benchmarks/reduction/scan_structs.cpp b/cpp/benchmarks/reduction/scan_structs.cpp index 92016041c9a..d5b19faf773 100644 --- a/cpp/benchmarks/reduction/scan_structs.cpp +++ b/cpp/benchmarks/reduction/scan_structs.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,6 @@ static constexpr cudf::size_type max_str_length = 32; static void nvbench_structs_scan(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; - auto const null_probability = [&] { auto const null_prob_val = state.get_float64("null_probability"); return null_prob_val > 0 ? std::optional{null_prob_val} : std::nullopt; diff --git a/cpp/benchmarks/reduction/segment_reduce.cu b/cpp/benchmarks/reduction/segment_reduce.cu index e063adb25f9..127b3598dae 100644 --- a/cpp/benchmarks/reduction/segment_reduce.cu +++ b/cpp/benchmarks/reduction/segment_reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -86,9 +86,6 @@ template void BM_Simple_Segmented_Reduction(nvbench::state& state, nvbench::type_list>) { - // TODO: to be replaced by nvbench fixture once it's ready - cudf::rmm_pool_raii rmm_pool; - auto const column_size{cudf::size_type(state.get_int64("column_size"))}; auto const num_segments{cudf::size_type(state.get_int64("num_segments"))}; diff --git a/cpp/benchmarks/search/contains.cpp b/cpp/benchmarks/search/contains.cpp index 8daa975d4ed..01a0a37b21a 100644 --- a/cpp/benchmarks/search/contains.cpp +++ b/cpp/benchmarks/search/contains.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,6 @@ std::unique_ptr create_column_data(cudf::size_type n_rows, bool ha static void nvbench_contains_scalar(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; using Type = int; auto const has_nulls = static_cast(state.get_int64("has_nulls")); diff --git a/cpp/benchmarks/sort/rank_lists.cpp b/cpp/benchmarks/sort/rank_lists.cpp index f467b639810..49dc409ebfc 100644 --- a/cpp/benchmarks/sort/rank_lists.cpp +++ b/cpp/benchmarks/sort/rank_lists.cpp @@ -26,8 +26,6 @@ template void nvbench_rank_lists(nvbench::state& state, nvbench::type_list>) { - cudf::rmm_pool_raii pool_raii; - auto const table = create_lists_data(state); auto const null_frequency{state.get_float64("null_frequency")}; diff --git a/cpp/benchmarks/sort/rank_structs.cpp b/cpp/benchmarks/sort/rank_structs.cpp index c1e2c5bd7dc..c0227e85191 100644 --- a/cpp/benchmarks/sort/rank_structs.cpp +++ b/cpp/benchmarks/sort/rank_structs.cpp @@ -24,8 +24,6 @@ template void nvbench_rank_structs(nvbench::state& state, nvbench::type_list>) { - cudf::rmm_pool_raii pool_raii; - auto const table = create_structs_data(state); const bool nulls{static_cast(state.get_int64("Nulls"))}; diff --git a/cpp/benchmarks/sort/segmented_sort.cpp b/cpp/benchmarks/sort/segmented_sort.cpp index e3459291caf..22d2b1c4029 100644 --- a/cpp/benchmarks/sort/segmented_sort.cpp +++ b/cpp/benchmarks/sort/segmented_sort.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,8 +26,6 @@ void nvbench_segmented_sort(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; - auto const stable = static_cast(state.get_int64("stable")); auto const dtype = cudf::type_to_id(); auto const size_bytes = static_cast(state.get_int64("size_bytes")); diff --git a/cpp/benchmarks/sort/sort_lists.cpp b/cpp/benchmarks/sort/sort_lists.cpp index 14cc60cbfe7..b55b60f5ec9 100644 --- a/cpp/benchmarks/sort/sort_lists.cpp +++ b/cpp/benchmarks/sort/sort_lists.cpp @@ -22,8 +22,6 @@ void nvbench_sort_lists(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; - auto const table = create_lists_data(state); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { diff --git a/cpp/benchmarks/sort/sort_structs.cpp b/cpp/benchmarks/sort/sort_structs.cpp index 22a6780c237..1d54fa42f6f 100644 --- a/cpp/benchmarks/sort/sort_structs.cpp +++ b/cpp/benchmarks/sort/sort_structs.cpp @@ -22,8 +22,6 @@ void nvbench_sort_struct(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; - auto const input = create_structs_data(state); state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { diff --git a/cpp/benchmarks/stream_compaction/distinct.cpp b/cpp/benchmarks/stream_compaction/distinct.cpp index 512554ff1bc..81eafa3044f 100644 --- a/cpp/benchmarks/stream_compaction/distinct.cpp +++ b/cpp/benchmarks/stream_compaction/distinct.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,8 +29,6 @@ NVBENCH_DECLARE_TYPE_STRINGS(cudf::timestamp_ms, "cudf::timestamp_ms", "cudf::ti template void nvbench_distinct(nvbench::state& state, nvbench::type_list) { - cudf::rmm_pool_raii pool_raii; - cudf::size_type const num_rows = state.get_int64("NumRows"); data_profile profile = data_profile_builder().cardinality(0).null_probability(0.01).distribution( @@ -61,8 +59,6 @@ NVBENCH_BENCH_TYPES(nvbench_distinct, NVBENCH_TYPE_AXES(data_type)) template void nvbench_distinct_list(nvbench::state& state, nvbench::type_list) { - cudf::rmm_pool_raii pool_raii; - auto const size = state.get_int64("ColumnSize"); auto const dtype = cudf::type_to_id(); double const null_probability = state.get_float64("null_probability"); diff --git a/cpp/benchmarks/stream_compaction/unique.cpp b/cpp/benchmarks/stream_compaction/unique.cpp index 9a0f4c3b743..dafb9d506c7 100644 --- a/cpp/benchmarks/stream_compaction/unique.cpp +++ b/cpp/benchmarks/stream_compaction/unique.cpp @@ -54,8 +54,6 @@ void nvbench_unique(nvbench::state& state, nvbench::type_list(); double const null_probability = state.get_float64("null_probability"); diff --git a/cpp/benchmarks/string/like.cpp b/cpp/benchmarks/string/like.cpp index de7382f5a75..d86c31480dd 100644 --- a/cpp/benchmarks/string/like.cpp +++ b/cpp/benchmarks/string/like.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,7 +71,6 @@ std::unique_ptr build_input_column(cudf::size_type n_rows, int32_t static void bench_like(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; auto const n_rows = static_cast(state.get_int64("num_rows")); auto const hit_rate = static_cast(state.get_int64("hit_rate")); diff --git a/cpp/benchmarks/string/reverse.cpp b/cpp/benchmarks/string/reverse.cpp index 7b08897079b..4c3846c79bb 100644 --- a/cpp/benchmarks/string/reverse.cpp +++ b/cpp/benchmarks/string/reverse.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ static void bench_reverse(nvbench::state& state) { - cudf::rmm_pool_raii pool_raii; auto const num_rows = static_cast(state.get_int64("num_rows")); auto const row_width = static_cast(state.get_int64("row_width")); diff --git a/cpp/cmake/thirdparty/get_nvbench.cmake b/cpp/cmake/thirdparty/get_nvbench.cmake new file mode 100644 index 00000000000..3a39e6c7ad1 --- /dev/null +++ b/cpp/cmake/thirdparty/get_nvbench.cmake @@ -0,0 +1,33 @@ +# ============================================================================= +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# This function finds nvbench and applies any needed patches. +function(find_and_configure_nvbench) + + include(${rapids-cmake-dir}/cpm/nvbench.cmake) + include(${rapids-cmake-dir}/cpm/package_override.cmake) + + # Find or install NVBench Temporarily force downloading of fmt because current versions of nvbench + # do not support the latest version of fmt, which is automatically pulled into our conda + # environments by mamba. + set(CPM_DOWNLOAD_fmt TRUE) + + set(cudf_patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/patches") + rapids_cpm_package_override("${cudf_patch_dir}/nvbench_override.json") + + rapids_cpm_nvbench() + +endfunction() + +find_and_configure_nvbench() diff --git a/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff b/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff new file mode 100644 index 00000000000..4e18385f664 --- /dev/null +++ b/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff @@ -0,0 +1,27 @@ +diff --git a/nvbench/main.cuh b/nvbench/main.cuh +index 4c1588c..3ba2b99 100644 +--- a/nvbench/main.cuh ++++ b/nvbench/main.cuh +@@ -54,6 +54,14 @@ + // clang-format on + #endif + ++#ifndef NVBENCH_ENVIRONMENT ++namespace nvbench { ++struct no_environment ++{}; ++} ++#define NVBENCH_ENVIRONMENT nvbench::no_environment ++#endif ++ + #define NVBENCH_MAIN_PARSE(argc, argv) \ + nvbench::option_parser parser; \ + parser.parse(argc, argv) +@@ -77,6 +85,7 @@ + printer.set_total_state_count(total_states); \ + \ + printer.set_completed_state_count(0); \ ++ NVBENCH_ENVIRONMENT{}; \ + for (auto &bench_ptr : benchmarks) \ + { \ + bench_ptr->set_printer(printer); \ diff --git a/cpp/cmake/thirdparty/patches/nvbench_override.json b/cpp/cmake/thirdparty/patches/nvbench_override.json new file mode 100644 index 00000000000..ad9b19c29c1 --- /dev/null +++ b/cpp/cmake/thirdparty/patches/nvbench_override.json @@ -0,0 +1,14 @@ + +{ + "packages" : { + "nvbench" : { + "patches" : [ + { + "file" : "${current_json_dir}/nvbench_global_setup.diff", + "issue" : "Fix add support for global setup to initialize RMM in nvbench [https://github.com/NVIDIA/nvbench/pull/123]", + "fixed_in" : "" + } + ] + } + } +} From 1c0224f15513709d5a6aab59c3503c0a6af59835 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 Feb 2023 14:13:20 -0800 Subject: [PATCH 22/69] Reenable stream identification library in CI (#12714) This PR reenables the preload library introduced for verifying stream usage in libcudf in #11875. This library was disabled during the GitHub Actions migration. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Yunsong Wang (https://github.com/PointKernel) - AJ Schmidt (https://github.com/ajschmidt8) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12714 --- ci/test_cpp.sh | 38 ++++++------ cpp/CMakeLists.txt | 42 ++++++++++++- cpp/tests/CMakeLists.txt | 13 ++++ .../test_default_stream_identification.cu | 4 +- .../identify_stream_usage.cpp | 2 +- .../identify_stream_usage/CMakeLists.txt | 60 ------------------- python/cudf/CMakeLists.txt | 1 + 7 files changed, 78 insertions(+), 82 deletions(-) rename cpp/tests/{utilities => }/identify_stream_usage/test_default_stream_identification.cu (93%) rename cpp/tests/utilities/{identify_stream_usage => }/identify_stream_usage.cpp (99%) delete mode 100644 cpp/tests/utilities/identify_stream_usage/CMakeLists.txt diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index b3d7919b279..d6681881419 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -35,13 +35,10 @@ EXITCODE=0 trap "EXITCODE=1" ERR set +e -# TODO: Disabling stream identification for now. -# Set up library for finding incorrect default stream usage. -#pushd "cpp/tests/utilities/identify_stream_usage/" -#mkdir build && cd build && cmake .. -GNinja && ninja && ninja test -#STREAM_IDENTIFY_LIB="$(realpath build/libidentify_stream_usage.so)" -#echo "STREAM_IDENTIFY_LIB=${STREAM_IDENTIFY_LIB}" -#popd +# Get library for finding incorrect default stream usage. +STREAM_IDENTIFY_LIB="${CONDA_PREFIX}/lib/libcudf_identify_stream_usage.so" + +echo "STREAM_IDENTIFY_LIB=${STREAM_IDENTIFY_LIB}" # Run libcudf and libcudf_kafka gtests from libcudf-tests package rapids-logger "Run gtests" @@ -51,17 +48,22 @@ rapids-logger "Run gtests" for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do test_name=$(basename ${gt}) echo "Running gtest $test_name" - ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} - # TODO: Disabling stream identification for now. - #if [[ ${test_name} == "SPAN_TEST" ]]; then - # # This one test is specifically designed to test using a thrust device - # # vector, so we expect and allow it to include default stream usage. - # gtest_filter="SpanTest.CanConstructFromDeviceContainers" - # GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="-${gtest_filter}" && \ - # ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="${gtest_filter}" - #else - # GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} - #fi + + # TODO: This strategy for using the stream lib will need to change when we + # switch to invoking ctest. For one, we will want to set the test + # properties to use the lib (which means that the decision will be made at + # CMake-configure time instead of runtime). We may also need to leverage + # something like gtest_discover_tests to be able to filter on the + # underlying test names. + if [[ ${test_name} == "SPAN_TEST" ]]; then + # This one test is specifically designed to test using a thrust device + # vector, so we expect and allow it to include default stream usage. + gtest_filter="SpanTest.CanConstructFromDeviceContainers" + GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="-${gtest_filter}" && \ + ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} --gtest_filter="${gtest_filter}" + else + GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR} + fi done if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index a635e655c39..d402a47628c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,6 +71,18 @@ option(CUDA_ENABLE_LINEINFO option(CUDA_WARNINGS_AS_ERRORS "Enable -Werror=all-warnings for all CUDA compilation" ON) # cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF) + +set(DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL ON) +if(${CUDA_STATIC_RUNTIME}) + set(DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL OFF) +endif() +option( + CUDF_BUILD_STREAMS_TEST_UTIL + "Whether to build the utilities for stream testing contained in libcudf" + ${DEFAULT_CUDF_BUILD_STREAMS_TEST_UTIL} +) +mark_as_advanced(CUDF_BUILD_STREAMS_TEST_UTIL) + option(USE_LIBARROW_FROM_PYARROW "Use the libarrow contained within pyarrow." OFF) mark_as_advanced(USE_LIBARROW_FROM_PYARROW) @@ -754,10 +766,34 @@ if(CUDF_BUILD_TESTUTIL) cudftestutil PUBLIC "$" "$" ) - add_library(cudf::cudftestutil ALIAS cudftestutil) endif() + +# * build cudf_identify_stream_usage -------------------------------------------------------------- + +if(CUDF_BUILD_STREAMS_TEST_UTIL) + if(CUDA_STATIC_RUNTIME) + message( + FATAL_ERROR + "Stream identification cannot be used with a static CUDA runtime. Please set CUDA_STATIC_RUNTIME=OFF or CUDF_BUILD_STREAMS_TEST_UTIL=OFF." + ) + endif() + + # Libraries for stream-related testing. + add_library(cudf_identify_stream_usage SHARED tests/utilities/identify_stream_usage.cpp) + + set_target_properties( + cudf_identify_stream_usage + PROPERTIES # set target compile options + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + POSITION_INDEPENDENT_CODE ON + ) + target_link_libraries(cudf_identify_stream_usage PUBLIC CUDA::cudart rmm::rmm) + add_library(cudf::cudf_identify_stream_usage ALIAS cudf_identify_stream_usage) +endif() + # ################################################################################################## # * add tests ------------------------------------------------------------------------------------- @@ -830,6 +866,10 @@ if(CUDF_BUILD_TESTUTIL) ) endif() +if(CUDF_BUILD_STREAMS_TEST_UTIL) + install(TARGETS cudf_identify_stream_usage DESTINATION ${lib_dir}) +endif() + set(doc_string [=[ Provide targets for the cudf library. diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 053acafdd3d..83a1c14438b 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -508,6 +508,19 @@ ConfigureTest( # * bin tests ---------------------------------------------------------------------------------- ConfigureTest(LABEL_BINS_TEST labeling/label_bins_tests.cpp) +# ################################################################################################## +# * stream identification tests ------------------------------------------------------------------- +ConfigureTest( + STREAM_IDENTIFICATION_TEST identify_stream_usage/test_default_stream_identification.cu +) +# Note that this only works when the test is invoked via ctest. At the moment CI is running all +# tests by manually invoking the executable, so we'll have to manually pass this environment +# variable in that setup. +set_tests_properties( + STREAM_IDENTIFICATION_TEST PROPERTIES ENVIRONMENT + LD_PRELOAD=$ +) + # ################################################################################################## # enable testing ################################################################################ # ################################################################################################## diff --git a/cpp/tests/utilities/identify_stream_usage/test_default_stream_identification.cu b/cpp/tests/identify_stream_usage/test_default_stream_identification.cu similarity index 93% rename from cpp/tests/utilities/identify_stream_usage/test_default_stream_identification.cu rename to cpp/tests/identify_stream_usage/test_default_stream_identification.cu index 022244b148b..28bb47af40d 100644 --- a/cpp/tests/utilities/identify_stream_usage/test_default_stream_identification.cu +++ b/cpp/tests/identify_stream_usage/test_default_stream_identification.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ void test_cudaLaunchKernel() try { kernel<<<1, 1>>>(); - } catch (std::runtime_error) { + } catch (std::runtime_error&) { return; } throw std::runtime_error("No exception raised for kernel on default stream!"); diff --git a/cpp/tests/utilities/identify_stream_usage/identify_stream_usage.cpp b/cpp/tests/utilities/identify_stream_usage.cpp similarity index 99% rename from cpp/tests/utilities/identify_stream_usage/identify_stream_usage.cpp rename to cpp/tests/utilities/identify_stream_usage.cpp index 4a1a8f04791..87301a7d49d 100644 --- a/cpp/tests/utilities/identify_stream_usage/identify_stream_usage.cpp +++ b/cpp/tests/utilities/identify_stream_usage.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/cpp/tests/utilities/identify_stream_usage/CMakeLists.txt b/cpp/tests/utilities/identify_stream_usage/CMakeLists.txt deleted file mode 100644 index 89f40303550..00000000000 --- a/cpp/tests/utilities/identify_stream_usage/CMakeLists.txt +++ /dev/null @@ -1,60 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR) - -if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake) - file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-22.12/RAPIDS.cmake - ${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake - ) -endif() -include(${CMAKE_CURRENT_BINARY_DIR}/CUDF_RAPIDS.cmake) - -project( - IDENTIFY_STREAM_USAGE - VERSION 0.0.1 - LANGUAGES CXX CUDA -) - -include(rapids-cpm) -include(${rapids-cmake-dir}/cpm/rmm.cmake) -rapids_cpm_init() -rapids_cpm_rmm() - -set(CMAKE_CUDA_RUNTIME_LIBRARY SHARED) -add_library(identify_stream_usage SHARED identify_stream_usage.cpp) - -find_package(CUDAToolkit REQUIRED) - -set_target_properties(identify_stream_usage PROPERTIES CUDA_RUNTIME_LIBRARY SHARED) -target_link_libraries(identify_stream_usage PUBLIC CUDA::cudart rmm::rmm) - -set_target_properties( - identify_stream_usage - PROPERTIES # set target compile options - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED ON - POSITION_INDEPENDENT_CODE ON -) - -# Add the test file. -include(CTest) - -add_executable(Tests test_default_stream_identification.cu) -add_test(NAME default_stream_identification COMMAND Tests) - -set_tests_properties( - default_stream_identification PROPERTIES ENVIRONMENT - LD_PRELOAD=$ -) diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 5223bc0a5c7..7457b770b13 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -89,6 +89,7 @@ if(NOT cudf_FOUND) # We don't build C++ tests when building wheels, so we can also omit the test util and shrink # the wheel by avoiding embedding GTest. set(CUDF_BUILD_TESTUTIL OFF) + set(CUDF_BUILD_STREAMS_TEST_UTIL OFF) # Statically link cudart if building wheels set(CUDA_STATIC_RUNTIME ON) From 8630e7cbc3d7ecb23fec5d5b837b15b9a999f323 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 10 Feb 2023 19:08:34 -0600 Subject: [PATCH 23/69] Remove gpuCI scripts. (#12712) Since migrating to GitHub Actions, the gpuCI scripts are no longer needed. This PR removes those outdated gpuCI scripts. Authors: - Bradley Dice (https://github.com/bdice) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Ray Douglass (https://github.com/raydouglass) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12712 --- README.md | 2 - ci/benchmark/build.sh | 196 ----------------- ci/checks/changelog.sh | 39 ---- ci/checks/style.sh | 23 -- ci/cpu/build.sh | 150 ------------- ci/cpu/prebuild.sh | 15 -- ci/cpu/upload.sh | 65 ------ ci/gpu/build.sh | 324 ---------------------------- ci/gpu/java.sh | 115 ---------- ci/gpu/test-notebooks.sh | 48 ----- ci/local/README.md | 57 ----- ci/local/build.sh | 146 ------------- ci/release/update-version.sh | 5 - ci/test_cpp.sh | 2 +- ci/test_python_cudf.sh | 1 - conda/recipes/dask-cudf/run_test.sh | 8 +- 16 files changed, 5 insertions(+), 1191 deletions(-) delete mode 100755 ci/benchmark/build.sh delete mode 100755 ci/checks/changelog.sh delete mode 100755 ci/checks/style.sh delete mode 100755 ci/cpu/build.sh delete mode 100755 ci/cpu/prebuild.sh delete mode 100755 ci/cpu/upload.sh delete mode 100755 ci/gpu/build.sh delete mode 100755 ci/gpu/java.sh delete mode 100755 ci/gpu/test-notebooks.sh delete mode 100644 ci/local/README.md delete mode 100755 ci/local/build.sh diff --git a/README.md b/README.md index 68c2d4f6276..36c1ff1d1fa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ #
 cuDF - GPU DataFrames
-[![Build Status](https://gpuci.gpuopenanalytics.com/job/rapidsai/job/gpuci/job/cudf/job/branches/job/cudf-branch-pipeline/badge/icon)](https://gpuci.gpuopenanalytics.com/job/rapidsai/job/gpuci/job/cudf/job/branches/job/cudf-branch-pipeline/) - **NOTE:** For the latest stable [README.md](https://github.com/rapidsai/cudf/blob/main/README.md) ensure you are on the `main` branch. ## Resources diff --git a/ci/benchmark/build.sh b/ci/benchmark/build.sh deleted file mode 100755 index e221424d1cd..00000000000 --- a/ci/benchmark/build.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020-2023, NVIDIA CORPORATION. -######################################### -# cuDF GPU build and test script for CI # -######################################### -set -e -NUMARGS=$# -ARGS=$* - -# Logger function for build status output -function logger() { - echo -e "\n>>>> $@\n" -} - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=4 -export CUDA_REL=${CUDA_VERSION%.*} -export HOME="$WORKSPACE" - -# Parse git describe -cd "$WORKSPACE" -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -# Set Benchmark Vars -export GBENCH_BENCHMARKS_DIR="$WORKSPACE/cpp/build/gbenchmarks/" - -# Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache because -# it's local to the container's virtual file system, and not shared with other CI jobs -# like `/tmp` is. -export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" - -# Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=1 - -# Dask version to install when `INSTALL_DASK_MAIN=0` -export DASK_STABLE_VERSION="2023.1.1" - -function remove_libcudf_kernel_cache_dir { - EXITCODE=$? - logger "removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" - exit $EXITCODE -} - -trap remove_libcudf_kernel_cache_dir EXIT - -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" || logger "could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" - -################################################################################ -# SETUP - Check environment -################################################################################ - -logger "Check environment..." -env - -logger "Check GPU usage..." -nvidia-smi - -logger "Activate conda env..." -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Enter dependencies to be shown in ASV tooltips. -CUDF_DEPS=(librmm) -LIBCUDF_DEPS=(librmm) - -conda install "rmm=$MINOR_VERSION.*" "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rapids-notebook-env=$MINOR_VERSION.*" \ - rapids-pytest-benchmark - -# https://docs.rapids.ai/maintainers/depmgmt/ -# conda remove -f rapids-build-env rapids-notebook-env -# conda install "your-pkg=1.0.0" - -# Install the conda-forge or nightly version of dask and distributed -if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then - gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" - gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" -else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" - gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall -fi - -# Install the master version of streamz -logger "pip install git+https://github.com/python-streamz/streamz.git@master --upgrade --no-deps" -pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps - -logger "Check versions..." -python --version - -conda info -conda config --show-sources -conda list --show-channel-urls - -################################################################################ -# BUILD - Build libcudf, cuDF and dask_cudf from source -################################################################################ - -logger "Build libcudf..." -"$WORKSPACE/build.sh" clean libcudf cudf dask_cudf benchmarks tests --ptds - -################################################################################ -# BENCHMARK - Run and parse libcudf and cuDF benchmarks -################################################################################ - -logger "Running benchmarks..." - -#Download GBench results Parser -curl -L https://raw.githubusercontent.com/rapidsai/benchmark/main/parser/GBenchToASV.py --output GBenchToASV.py - -### -# Generate Metadata for dependencies -### - -# Concatenate dependency arrays, convert to JSON array, -# and remove duplicates. -X=("${CUDF_DEPS[@]}" "${LIBCUDF_DEPS[@]}") -DEPS=$(printf '%s\n' "${X[@]}" | jq -R . | jq -s 'unique') - -# Build object with k/v pairs of "dependency:version" -DEP_VER_DICT=$(jq -n '{}') -for DEP in $(echo "${DEPS}" | jq -r '.[]'); do - VER=$(conda list | grep "^${DEP}" | awk '{print $2"-"$3}') - DEP_VER_DICT=$(echo "${DEP_VER_DICT}" | jq -c --arg DEP "${DEP}" --arg VER "${VER}" '. + { ($DEP): $VER }') -done - -# Pass in an array of dependencies to get a dict of "dependency:version" -function getReqs() { - local DEPS_ARR=("$@") - local REQS="{}" - for DEP in "${DEPS_ARR[@]}"; do - VER=$(echo "${DEP_VER_DICT}" | jq -r --arg DEP "${DEP}" '.[$DEP]') - REQS=$(echo "${REQS}" | jq -c --arg DEP "${DEP}" --arg VER "${VER}" '. + { ($DEP): $VER }') - done - - echo "${REQS}" -} - -### -# Run LIBCUDF Benchmarks -### - -REQS=$(getReqs "${LIBCUDF_DEPS[@]}") - -mkdir -p "$WORKSPACE/tmp/benchmark" -touch "$WORKSPACE/tmp/benchmark/benchmarks.txt" -ls ${GBENCH_BENCHMARKS_DIR} > "$WORKSPACE/tmp/benchmark/benchmarks.txt" - -#Disable error aborting while tests run, failed tests will not generate data -logger "Running libcudf GBenchmarks..." -cd ${GBENCH_BENCHMARKS_DIR} -set +e -while read BENCH; -do - nvidia-smi - ./${BENCH} --benchmark_out=${BENCH}.json --benchmark_out_format=json - EXITCODE=$? - if [[ ${EXITCODE} != 0 ]]; then - rm ./${BENCH}.json - JOBEXITCODE=1 - fi -done < "$WORKSPACE/tmp/benchmark/benchmarks.txt" -set -e - -rm "$WORKSPACE/tmp/benchmark/benchmarks.txt" -cd "$WORKSPACE" -mv ${GBENCH_BENCHMARKS_DIR}/*.json "$WORKSPACE/tmp/benchmark/" -python GBenchToASV.py -d "$WORKSPACE/tmp/benchmark/" -t ${S3_ASV_DIR} -n libcudf -b branch-${MINOR_VERSION} -r "${REQS}" - -### -# Run Python Benchmarks -### - -#REQS=$(getReqs "${CUDF_DEPS[@]}") - -#BENCHMARK_META=$(jq -n \ -# --arg NODE "${NODE_NAME}" \ -# --arg BRANCH "branch-${MINOR_VERSION}" \ -# --argjson REQS "${REQS}" ' -# { -# "machineName": $NODE, -# "commitBranch": $BRANCH, -# "requirements": $REQS -# } -#') - -#echo "Benchmark meta:" -#echo "${BENCHMARK_META}" | jq "." diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh deleted file mode 100755 index 0dfcf27298e..00000000000 --- a/ci/checks/changelog.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018, NVIDIA CORPORATION. -######################### -# cuDF CHANGELOG Tester # -######################### - -# Checkout main for comparison -git checkout --force --quiet main - -# Switch back to tip of PR branch -git checkout --force --quiet current-pr-branch - -# Ignore errors during searching -set +e - -# Get list of modified files between main and PR branch -CHANGELOG=`git diff --name-only main...current-pr-branch | grep CHANGELOG.md` -# Check if CHANGELOG has PR ID -PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` -RETVAL=0 - -# Return status of check result -if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then - echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" -else - echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" - RETVAL=1 -fi - -cat << EOF - It should be placed under the section for the appropriate release. - It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. - It should be formatted as '- PR # ' - Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' - - -EOF - -exit $RETVAL diff --git a/ci/checks/style.sh b/ci/checks/style.sh deleted file mode 100755 index d32d88f5574..00000000000 --- a/ci/checks/style.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -##################### -# cuDF Style Tester # -##################### - -# Ignore errors and set path -set +e -PATH=/conda/bin:$PATH -LC_ALL=C.UTF-8 -LANG=C.UTF-8 - -# Activate common conda env -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -FORMAT_FILE_URL=https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-23.04/cmake-format-rapids-cmake.json -export RAPIDS_CMAKE_FORMAT_FILE=/tmp/rapids_cmake_ci/cmake-formats-rapids-cmake.json -mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) -wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} - -# Run pre-commit checks -pre-commit run --hook-stage manual --all-files --show-diff-on-failure diff --git a/ci/cpu/build.sh b/ci/cpu/build.sh deleted file mode 100755 index 3aa00ff7de9..00000000000 --- a/ci/cpu/build.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2023, NVIDIA CORPORATION. -############################################## -# cuDF CPU conda build script for CI # -############################################## -set -e - -# Set path and build parallel level -# FIXME: PATH variable shouldn't be necessary. -# This should be removed once we either stop using the `remote-docker-plugin` -# or the following issue is addressed: https://github.com/gpuopenanalytics/remote-docker-plugin/issues/47 -export PATH=/usr/local/gcc9/bin:/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Determine CUDA release version -export CUDA_REL=${CUDA_VERSION%.*} - -# Setup 'gpuci_conda_retry' for build retries (results in 2 total attempts) -export GPUCI_CONDA_RETRY_MAX=1 -export GPUCI_CONDA_RETRY_SLEEP=30 - -# Workaround to keep Jenkins builds working -# until we migrate fully to GitHub Actions -export RAPIDS_CUDA_VERSION="${CUDA}" -export SCCACHE_BUCKET=rapids-sccache -export SCCACHE_REGION=us-west-2 -export SCCACHE_IDLE_TIMEOUT=32768 - -# Use Ninja to build, setup Conda Build Dir -export CMAKE_GENERATOR="Ninja" -export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" - -# Whether to keep `dask/label/dev` channel in the env. If INSTALL_DASK_MAIN=0, -# `dask/label/dev` channel is removed. -export INSTALL_DASK_MAIN=1 - -# Switch to project root; also root of repo checkout -cd "$WORKSPACE" - -# If nightly build, append current YYMMDD to version -if [[ "$BUILD_MODE" = "branch" && "$SOURCE_BRANCH" = branch-* ]] ; then - export VERSION_SUFFIX=`date +%y%m%d` -fi - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment variables" -env - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Remove `rapidsai-nightly` & `dask/label/dev` channel if we are building main branch -if [ "$SOURCE_BRANCH" = "main" ]; then - conda config --system --remove channels rapidsai-nightly - conda config --system --remove channels dask/label/dev -elif [[ "${INSTALL_DASK_MAIN}" == 0 ]]; then - # Remove `dask/label/dev` channel if INSTALL_DASK_MAIN=0 - conda config --system --remove channels dask/label/dev -fi - -gpuci_logger "Check compiler versions" -python --version - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -# FIX Added to deal with Anancoda SSL verification issues during conda builds -conda config --set ssl_verify False - -# TODO: Move boa install to gpuci/rapidsai -gpuci_mamba_retry install boa -################################################################################ -# BUILD - Conda package builds -################################################################################ - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - CONDA_BUILD_ARGS="" - CONDA_CHANNEL="" -else - CONDA_BUILD_ARGS="--dirty --no-remove-work-dir" - CONDA_CHANNEL="-c $WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" -fi - -if [ "$BUILD_LIBCUDF" == '1' ]; then - gpuci_logger "Build conda pkg for libcudf" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/libcudf $CONDA_BUILD_ARGS - - # BUILD_LIBCUDF == 1 means this job is being run on the cpu_build jobs - # that is where we must also build the strings_udf package - mkdir -p ${CONDA_BLD_DIR}/strings_udf/work - STRINGS_UDF_BUILD_DIR=${CONDA_BLD_DIR}/strings_udf/work - gpuci_logger "Build conda pkg for cudf (python 3.8), for strings_udf" - gpuci_conda_retry mambabuild --no-build-id --croot ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/cudf ${CONDA_BUILD_ARGS} --python=3.8 - gpuci_logger "Build conda pkg for cudf (python 3.9), for strings_udf" - gpuci_conda_retry mambabuild --no-build-id --croot ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/cudf ${CONDA_BUILD_ARGS} --python=3.9 - - gpuci_logger "Build conda pkg for strings_udf (python 3.8)" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} -c ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.8 - gpuci_logger "Build conda pkg for strings_udf (python 3.9)" - gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} -c ${STRINGS_UDF_BUILD_DIR} -c ${CONDA_BLD_DIR} conda/recipes/strings_udf $CONDA_BUILD_ARGS --python=3.9 - - mkdir -p ${CONDA_BLD_DIR}/libcudf/work - cp -r ${CONDA_BLD_DIR}/work/* ${CONDA_BLD_DIR}/libcudf/work - gpuci_logger "sccache stats" - sccache --show-stats - - # Copy libcudf build metrics results - LIBCUDF_BUILD_DIR=$CONDA_BLD_DIR/libcudf/work/cpp/build - echo "Checking for build metrics log $LIBCUDF_BUILD_DIR/ninja_log.html" - if [[ -f "$LIBCUDF_BUILD_DIR/ninja_log.html" ]]; then - gpuci_logger "Copying build metrics results" - mkdir -p "$WORKSPACE/build-metrics" - cp "$LIBCUDF_BUILD_DIR/ninja_log.html" "$WORKSPACE/build-metrics/BuildMetrics.html" - cp "$LIBCUDF_BUILD_DIR/ninja.log" "$WORKSPACE/build-metrics/ninja.log" - fi -fi - -if [ "$BUILD_CUDF" == '1' ]; then - gpuci_logger "Build conda pkg for cudf" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for dask-cudf" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for cudf_kafka" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for custreamz" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - - gpuci_logger "Build conda pkg for strings_udf" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/strings_udf --python=$PYTHON $CONDA_BUILD_ARGS $CONDA_CHANNEL - -fi -################################################################################ -# UPLOAD - Conda packages -################################################################################ - -# Uploads disabled due to new GH Actions implementation -# gpuci_logger "Upload conda pkgs" -# source ci/cpu/upload.sh diff --git a/ci/cpu/prebuild.sh b/ci/cpu/prebuild.sh deleted file mode 100755 index 32589042f7f..00000000000 --- a/ci/cpu/prebuild.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -set -e - -#Always upload cudf packages -export UPLOAD_CUDF=1 -export UPLOAD_LIBCUDF=1 -export UPLOAD_CUDF_KAFKA=1 - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - #If project flash is not activate, always build both - export BUILD_LIBCUDF=1 - export BUILD_CUDF=1 -fi diff --git a/ci/cpu/upload.sh b/ci/cpu/upload.sh deleted file mode 100755 index 82c58673605..00000000000 --- a/ci/cpu/upload.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -set -e - -# Setup 'gpuci_retry' for upload retries (results in 4 total attempts) -export GPUCI_RETRY_MAX=3 -export GPUCI_RETRY_SLEEP=30 - -# Set default label options if they are not defined elsewhere -export LABEL_OPTION=${LABEL_OPTION:-"--label main"} - -# Skip uploads unless BUILD_MODE == "branch" -if [ "${BUILD_MODE}" != "branch" ]; then - echo "Skipping upload" - return 0 -fi - -# Skip uploads if there is no upload key -if [ -z "$MY_UPLOAD_KEY" ]; then - echo "No upload key" - return 0 -fi - -################################################################################ -# UPLOAD - Conda packages -################################################################################ - -gpuci_logger "Starting conda uploads" -if [[ "$BUILD_LIBCUDF" == "1" && "$UPLOAD_LIBCUDF" == "1" ]]; then - export LIBCUDF_FILES=$(conda build --no-build-id --croot "${CONDA_BLD_DIR}" conda/recipes/libcudf --output) - LIBCUDF_FILES=$(echo "$LIBCUDF_FILES" | sed 's/.*libcudf-example.*//') # skip libcudf-example pkg upload - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress $LIBCUDF_FILES -fi - -if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF" == "1" ]]; then - export CUDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/cudf --python=$PYTHON --output) - test -e ${CUDF_FILE} - echo "Upload cudf: ${CUDF_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_FILE} --no-progress - - export STRINGS_UDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/strings_udf --python=$PYTHON --output -c "${CONDA_BLD_DIR}") - test -e ${STRINGS_UDF_FILE} - echo "Upload strings_udf: ${STRINGS_UDF_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${STRINGS_UDF_FILE} --no-progress - - export DASK_CUDF_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/dask-cudf --python=$PYTHON --output) - test -e ${DASK_CUDF_FILE} - echo "Upload dask-cudf: ${DASK_CUDF_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${DASK_CUDF_FILE} --no-progress - - export CUSTREAMZ_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/custreamz --python=$PYTHON --output) - test -e ${CUSTREAMZ_FILE} - echo "Upload custreamz: ${CUSTREAMZ_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUSTREAMZ_FILE} --no-progress -fi - -if [[ "$BUILD_CUDF" == "1" && "$UPLOAD_CUDF_KAFKA" == "1" ]]; then - export CUDF_KAFKA_FILE=$(conda build --croot "${CONDA_BLD_DIR}" conda/recipes/cudf_kafka --python=$PYTHON --output) - test -e ${CUDF_KAFKA_FILE} - echo "Upload cudf_kafka: ${CUDF_KAFKA_FILE}" - gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUDF_KAFKA_FILE} --no-progress -fi diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh deleted file mode 100755 index ff40fdf6c9f..00000000000 --- a/ci/gpu/build.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2023, NVIDIA CORPORATION. -############################################## -# cuDF GPU build and test script for CI # -############################################## -set -e -NUMARGS=$# -ARGS=$* - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Switch to project root; also root of repo checkout -cd "$WORKSPACE" - -# Determine CUDA release version -export CUDA_REL=${CUDA_VERSION%.*} -export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" - -# Workaround to keep Jenkins builds working -# until we migrate fully to GitHub Actions -export RAPIDS_CUDA_VERSION="${CUDA}" -export SCCACHE_BUCKET=rapids-sccache -export SCCACHE_REGION=us-west-2 -export SCCACHE_IDLE_TIMEOUT=32768 - -# Parse git describe -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` -unset GIT_DESCRIBE_TAG - -# Dask & Distributed option to install main(nightly) or `conda-forge` packages. -export INSTALL_DASK_MAIN=1 - -# Dask version to install when `INSTALL_DASK_MAIN=0` -export DASK_STABLE_VERSION="2023.1.1" - -# ucx-py version -export UCX_PY_VERSION='0.31.*' - -################################################################################ -# TRAP - Setup trap for removing jitify cache -################################################################################ - -# Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache -# because it's local to the container's virtual file system, and not shared with -# other CI jobs like `/tmp` is -export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" - -function remove_libcudf_kernel_cache_dir { - EXITCODE=$? - gpuci_logger "TRAP: Removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" - exit $EXITCODE -} - -# Set trap to run on exit -gpuci_logger "TRAP: Set trap to remove jitify cache on exit" -trap remove_libcudf_kernel_cache_dir EXIT - -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment variables" -env - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -# Remove `dask/label/dev` channel if INSTALL_DASK_MAIN=0 -if [ "$SOURCE_BRANCH" != "main" ] && [[ "${INSTALL_DASK_MAIN}" == 0 ]]; then - conda config --system --remove channels dask/label/dev - gpuci_mamba_retry install conda-forge::dask==$DASK_STABLE_VERSION conda-forge::distributed==$DASK_STABLE_VERSION conda-forge::dask-core==$DASK_STABLE_VERSION --force-reinstall -fi - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls -gpuci_logger "Check compiler versions" -python --version - -function install_dask { - # Install the conda-forge or nightly version of dask and distributed - gpuci_logger "Install the conda-forge or nightly version of dask and distributed" - set -x - if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then - gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" - gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" - conda list - else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" - gpuci_mamba_retry install conda-forge::dask==$DASK_STABLE_VERSION conda-forge::distributed==$DASK_STABLE_VERSION conda-forge::dask-core==$DASK_STABLE_VERSION --force-reinstall - fi - # Install the main version of streamz - gpuci_logger "Install the main version of streamz" - # Need to uninstall streamz that is already in the env. - pip uninstall -y streamz - pip install "git+https://github.com/python-streamz/streamz.git@master" --upgrade --no-deps - set +x -} - -install_dask - -if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then - - gpuci_logger "Install dependencies" - gpuci_mamba_retry install -y \ - "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rapids-notebook-env=$MINOR_VERSION.*" \ - "dask-cuda=${MINOR_VERSION}" \ - "rmm=$MINOR_VERSION.*" \ - "ucx-py=${UCX_PY_VERSION}" - - # https://docs.rapids.ai/maintainers/depmgmt/ - # gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env - # gpuci_mamba_retry install -y "your-pkg=1.0.0" - - ################################################################################ - # BUILD - Build libcudf, cuDF, libcudf_kafka, dask_cudf, and strings_udf from source - ################################################################################ - - gpuci_logger "Build from source" - "$WORKSPACE/build.sh" clean libcudf cudf dask_cudf libcudf_kafka cudf_kafka strings_udf benchmarks tests --ptds - - ################################################################################ - # TEST - Run GoogleTest - ################################################################################ - - set +e -Eo pipefail - EXITCODE=0 - trap "EXITCODE=1" ERR - - - if hasArg --skip-tests; then - gpuci_logger "Skipping Tests" - exit 0 - else - gpuci_logger "Check GPU usage" - nvidia-smi - - gpuci_logger "GoogleTests" - set -x - cd "$WORKSPACE/cpp/build" - - for gt in "$WORKSPACE/cpp/build/gtests/"* ; do - test_name=$(basename ${gt}) - echo "Running GoogleTest $test_name" - ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" - done - fi -else - #Project Flash - - if hasArg --skip-tests; then - gpuci_logger "Skipping Tests" - exit 0 - fi - - gpuci_logger "Check GPU usage" - nvidia-smi - - gpuci_logger "Installing libcudf, libcudf_kafka and libcudf-tests" - gpuci_mamba_retry install -y -c ${CONDA_ARTIFACT_PATH} libcudf libcudf_kafka libcudf-tests - - # TODO: Move boa install to gpuci/rapidsai - gpuci_mamba_retry install boa - gpuci_logger "Building cudf, dask-cudf, cudf_kafka and custreamz" - export CONDA_BLD_DIR="$WORKSPACE/.conda-bld" - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/dask-cudf --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/cudf_kafka --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - gpuci_conda_retry mambabuild --croot ${CONDA_BLD_DIR} conda/recipes/custreamz --python=$PYTHON -c ${CONDA_ARTIFACT_PATH} - - # the CUDA component of strings_udf must be built on cuda 11.5 just like libcudf - # but because there is no separate python package, we must also build the python on the 11.5 jobs - # this means that at this point (on the GPU test jobs) the whole package is already built and has been - # copied by CI from the upstream 11.5 jobs into $CONDA_ARTIFACT_PATH - gpuci_logger "Installing cudf, dask-cudf, cudf_kafka, and custreamz" - gpuci_mamba_retry install cudf dask-cudf cudf_kafka custreamz -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" - - gpuci_logger "Check current conda environment" - conda list --show-channel-urls - - gpuci_logger "GoogleTests" - - # Set up library for finding incorrect default stream usage. - cd "$WORKSPACE/cpp/tests/utilities/identify_stream_usage/" - mkdir build && cd build && cmake .. -GNinja && ninja && ninja test - STREAM_IDENTIFY_LIB="$WORKSPACE/cpp/tests/utilities/identify_stream_usage/build/libidentify_stream_usage.so" - - # Run libcudf and libcudf_kafka gtests from libcudf-tests package - for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do - test_name=$(basename ${gt}) - - echo "Running GoogleTest $test_name" - if [[ ${test_name} == "SPAN_TEST" ]]; then - # This one test is specifically designed to test using a thrust device - # vector, so we expect and allow it to include default stream usage. - gtest_filter="SpanTest.CanConstructFromDeviceContainers" - GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" --gtest_filter="-${gtest_filter}" - ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" --gtest_filter="${gtest_filter}" - else - GTEST_CUDF_STREAM_MODE="custom" LD_PRELOAD=${STREAM_IDENTIFY_LIB} ${gt} --gtest_output=xml:"$WORKSPACE/test-results/" - fi - done - - export LIB_BUILD_DIR="$WORKSPACE/ci/artifacts/cudf/cpu/libcudf_work/cpp/build" - # Copy libcudf build time results - echo "Checking for build time log $LIB_BUILD_DIR/ninja_log.xml" - if [[ -f "$LIB_BUILD_DIR/ninja_log.xml" ]]; then - gpuci_logger "Copying build time results" - cp "$LIB_BUILD_DIR/ninja_log.xml" "$WORKSPACE/test-results/buildtimes-junit.xml" - fi - - ################################################################################ - # MEMCHECK - Run compute-sanitizer on GoogleTest (only in nightly builds) - ################################################################################ - if [[ "$BUILD_MODE" == "branch" && "$BUILD_TYPE" == "gpu" ]]; then - if [[ "$COMPUTE_SANITIZER_ENABLE" == "true" ]]; then - gpuci_logger "Memcheck on GoogleTests with rmm_mode=cuda" - export GTEST_CUDF_RMM_MODE=cuda - COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" - mkdir -p "$WORKSPACE/test-results/" - for gt in "$CONDA_PREFIX/bin/gtests/libcudf"*/* ; do - test_name=$(basename ${gt}) - if [[ "$test_name" == "ERROR_TEST" ]]; then - continue - fi - echo "Running GoogleTest $test_name" - ${COMPUTE_SANITIZER_CMD} ${gt} | tee "$WORKSPACE/test-results/${test_name}.cs.log" - done - unset GTEST_CUDF_RMM_MODE - # test-results/*.cs.log are processed in gpuci - fi - fi -fi - -# Both regular and Project Flash proceed here - -# set environment variable for numpy 1.16 -# will be enabled for later versions by default -np_ver=$(python -c "import numpy; print('.'.join(numpy.__version__.split('.')[:-1]))") -if [ "$np_ver" == "1.16" ];then - export NUMPY_EXPERIMENTAL_ARRAY_FUNCTION=1 -fi - -################################################################################ -# TEST - Run py.test, notebooks -################################################################################ - -cd "$WORKSPACE/python/cudf/cudf" -# It is essential to cd into $WORKSPACE/python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. -gpuci_logger "Check conda packages" -conda list -gpuci_logger "Python py.test for cuDF" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope tests - -gpuci_logger "Python py.tests for cuDF with spilling (CUDF_SPILL_DEVICE_LIMIT=1)" -# Due to time concerns, we only run tests marked "spilling" -CUDF_SPILL=on CUDF_SPILL_DEVICE_LIMIT=1 py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov-append --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope -m spilling tests - -cd "$WORKSPACE/python/dask_cudf" -gpuci_logger "Python py.test for dask-cudf" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/dask-cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-dask-cudf.xml" -v --cov-config=.coveragerc --cov=dask_cudf --cov-report=xml:"$WORKSPACE/python/dask_cudf/dask-cudf-coverage.xml" --cov-report term dask_cudf - -cd "$WORKSPACE/python/custreamz" -gpuci_logger "Python py.test for cuStreamz" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml="$WORKSPACE/junit-custreamz.xml" -v --cov-config=.coveragerc --cov=custreamz --cov-report=xml:"$WORKSPACE/python/custreamz/custreamz-coverage.xml" --cov-report term custreamz - - -# only install strings_udf after cuDF is finished testing without its presence -gpuci_logger "Installing strings_udf" -gpuci_mamba_retry install strings_udf -c "${CONDA_BLD_DIR}" -c "${CONDA_ARTIFACT_PATH}" - -cd "$WORKSPACE/python/strings_udf/strings_udf" -gpuci_logger "Python py.test for strings_udf" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/strings-udf-cuda-tmp" --junitxml="$WORKSPACE/junit-strings-udf.xml" -v --cov-config=.coveragerc --cov=strings_udf --cov-report=xml:"$WORKSPACE/python/strings_udf/strings-udf-coverage.xml" --cov-report term tests - -# retest cuDF UDFs -cd "$WORKSPACE/python/cudf/cudf" -gpuci_logger "Python py.test retest cuDF UDFs" -py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-strings-udf-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf-strings-udf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-strings-udf-coverage.xml" --cov-report term --dist=loadscope tests/test_udf_masked_ops.py - - -# Run benchmarks with both cudf and pandas to ensure compatibility is maintained. -# Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. -# Therefore, these runs only verify that benchmarks are valid. -# They do not generate meaningful performance measurements. -cd "$WORKSPACE/python/cudf" -gpuci_logger "Python pytest for cuDF benchmarks" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" -v --dist=loadscope benchmarks - -gpuci_logger "Python pytest for cuDF benchmarks using pandas" -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" -v --dist=loadscope benchmarks - -gpuci_logger "Test notebooks" -"$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log -python "$WORKSPACE/ci/utils/nbtestlog2junitxml.py" nbtest.log - -if [ -n "${CODECOV_TOKEN}" ]; then - codecov -t $CODECOV_TOKEN -fi - -return ${EXITCODE} diff --git a/ci/gpu/java.sh b/ci/gpu/java.sh deleted file mode 100755 index 2db9cd57eb8..00000000000 --- a/ci/gpu/java.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. -############################################## -# cuDF GPU build and test script for CI # -############################################## -set -e -NUMARGS=$# -ARGS=$* - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Switch to project root; also root of repo checkout -cd "$WORKSPACE" - -# Determine CUDA release version -export CUDA_REL=${CUDA_VERSION%.*} -export CONDA_ARTIFACT_PATH="$WORKSPACE/ci/artifacts/cudf/cpu/.conda-bld/" - -# Parse git describe -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -################################################################################ -# TRAP - Setup trap for removing jitify cache -################################################################################ - -# Set `LIBCUDF_KERNEL_CACHE_PATH` environment variable to $HOME/.jitify-cache -# because it's local to the container's virtual file system, and not shared with -# other CI jobs like `/tmp` is -export LIBCUDF_KERNEL_CACHE_PATH="$HOME/.jitify-cache" - -function remove_libcudf_kernel_cache_dir { - EXITCODE=$? - gpuci_logger "TRAP: Removing kernel cache dir: $LIBCUDF_KERNEL_CACHE_PATH" - rm -rf "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not rm -rf $LIBCUDF_KERNEL_CACHE_PATH" - exit $EXITCODE -} - -# Set trap to run on exit -gpuci_logger "TRAP: Set trap to remove jitify cache on exit" -trap remove_libcudf_kernel_cache_dir EXIT - -mkdir -p "$LIBCUDF_KERNEL_CACHE_PATH" \ - || gpuci_logger "[ERROR] TRAP: Could not mkdir -p $LIBCUDF_KERNEL_CACHE_PATH" - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment variables" -env - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -gpuci_logger "Install dependencies" -gpuci_mamba_retry install -y \ - "cudatoolkit=$CUDA_REL" \ - "rapids-build-env=$MINOR_VERSION.*" \ - "rmm=$MINOR_VERSION.*" \ - "openjdk=8.*" \ - "maven" -# "mamba install openjdk" adds an activation script to set JAVA_HOME but this is -# not triggered on installation. Re-activating the conda environment will set -# this environment variable so that CMake can find JNI. -conda activate rapids - -# https://docs.rapids.ai/maintainers/depmgmt/ -# gpuci_conda_retry remove --force rapids-build-env rapids-notebook-env -# gpuci_mamba_retry install -y "your-pkg=1.0.0" - -gpuci_logger "Check conda environment" -conda info -conda config --show-sources -conda list --show-channel-urls - -################################################################################ -# INSTALL - Install libcudf artifacts -################################################################################ - -gpuci_logger "Installing libcudf" -gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcudf - -################################################################################ -# TEST - Run java tests -################################################################################ - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "Running Java Tests" -cd ${WORKSPACE}/java -mvn test -B -DCUDF_JNI_ARROW_STATIC=OFF - -return ${EXITCODE} diff --git a/ci/gpu/test-notebooks.sh b/ci/gpu/test-notebooks.sh deleted file mode 100755 index 36d093d0d28..00000000000 --- a/ci/gpu/test-notebooks.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -NOTEBOOKS_DIR="$WORKSPACE/notebooks" -NBTEST="$WORKSPACE/ci/utils/nbtest.sh" -LIBCUDF_KERNEL_CACHE_PATH="$WORKSPACE/.jitcache" - -cd ${NOTEBOOKS_DIR} -TOPLEVEL_NB_FOLDERS=$(find . -name *.ipynb |cut -d'/' -f2|sort -u) - -# Add notebooks that should be skipped here -# (space-separated list of filenames without paths) - -SKIPNBS="" - -## Check env -env - -EXITCODE=0 - -# Always run nbtest in all TOPLEVEL_NB_FOLDERS, set EXITCODE to failure -# if any run fails - -cd ${NOTEBOOKS_DIR} -for nb in $(find . -name "*.ipynb"); do - nbBasename=$(basename ${nb}) - # Skip all NBs that use dask (in the code or even in their name) - if ((echo ${nb}|grep -qi dask) || \ - (grep -q dask ${nb})); then - echo "--------------------------------------------------------------------------------" - echo "SKIPPING: ${nb} (suspected Dask usage, not currently automatable)" - echo "--------------------------------------------------------------------------------" - elif (echo " ${SKIPNBS} " | grep -q " ${nbBasename} "); then - echo "--------------------------------------------------------------------------------" - echo "SKIPPING: ${nb} (listed in skip list)" - echo "--------------------------------------------------------------------------------" - else - nvidia-smi - ${NBTEST} ${nbBasename} - EXITCODE=$((EXITCODE | $?)) - rm -rf ${LIBCUDF_KERNEL_CACHE_PATH}/* - fi -done - - -nvidia-smi - -exit ${EXITCODE} diff --git a/ci/local/README.md b/ci/local/README.md deleted file mode 100644 index 7754bcaf647..00000000000 --- a/ci/local/README.md +++ /dev/null @@ -1,57 +0,0 @@ -## Purpose - -This script is designed for developer and contributor use. This tool mimics the actions of gpuCI on your local machine. This allows you to test and even debug your code inside a gpuCI base container before pushing your code as a GitHub commit. -The script can be helpful in locally triaging and debugging RAPIDS continuous integration failures. - -## Requirements - -``` -nvidia-docker -``` - -## Usage - -``` -bash build.sh [-h] [-H] [-s] [-r ] [-i ] -Build and test your local repository using a base gpuCI Docker image - -where: - -H Show this help text - -r Path to repository (defaults to working directory) - -i Use Docker image (default is gpuci/rapidsai:${NIGHTLY_VERSION}-cuda11.5-devel-ubuntu20.04-py3.8) - -s Skip building and testing and start an interactive shell in a container of the Docker image -``` - -Example Usage: -`bash build.sh -r ~/rapids/cudf -i gpuci/rapidsai:22.02-cuda11.5-devel-ubuntu20.04-py3.8` - -For a full list of available gpuCI docker images, visit our [DockerHub](https://hub.docker.com/r/gpuci/rapidsai/tags) page. - -Style Check: -```bash -$ bash ci/local/build.sh -r ~/rapids/cudf -s -$ source activate rapids # Activate gpuCI conda environment -$ cd rapids -$ flake8 python -``` - -## Information - -There are some caveats to be aware of when using this script, especially if you plan on developing from within the container itself. - - -### Docker Image Build Repository - -The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/cudf/build_rapidsai_cuda11.5-ubuntu20.04-py3.8/`. Feel free to remove this directory after the script is finished. - -*Note*: The script *will not* override your local build repository. Your local environment stays in tact. - - -### Where The User is Dumped - -The script will build your repository and run all tests. If any tests fail, it dumps the user into the docker container itself to allow you to debug from within the container. If all the tests pass as expected the container exits and is automatically removed. Remember to exit the container if tests fail and you do not wish to debug within the container itself. - - -### Container File Structure - -Your repository will be located in the `/rapids/` folder of the container. This folder is volume mounted from the local machine. Any changes to the code in this repository are replicated onto the local machine. The `cpp/build` and `python/build` directories within your repository is on a separate mount to avoid conflicting with your local build artifacts. diff --git a/ci/local/build.sh b/ci/local/build.sh deleted file mode 100755 index f6479cd76cc..00000000000 --- a/ci/local/build.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash -# Copyright (c) 2019-2022, NVIDIA CORPORATION. - -GIT_DESCRIBE_TAG=`git describe --tags` -MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -DOCKER_IMAGE="gpuci/rapidsai:${MINOR_VERSION}-cuda11.5-devel-ubuntu20.04-py3.8" -REPO_PATH=${PWD} -RAPIDS_DIR_IN_CONTAINER="/rapids" -CPP_BUILD_DIR="cpp/build" -PYTHON_BUILD_DIR="python/build" -CONTAINER_SHELL_ONLY=0 - -SHORTHELP="$(basename "$0") [-h] [-H] [-s] [-r ] [-i ]" -LONGHELP="${SHORTHELP} -Build and test your local repository using a base gpuCI Docker image - -where: - -H Show this help text - -r Path to repository (defaults to working directory) - -i Use Docker image (default is ${DOCKER_IMAGE}) - -s Skip building and testing and start an interactive shell in a container of the Docker image -" - -# Limit GPUs available to container based on CUDA_VISIBLE_DEVICES -if [[ -z "${CUDA_VISIBLE_DEVICES}" ]]; then - NVIDIA_VISIBLE_DEVICES="all" -else - NVIDIA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES} -fi - -while getopts ":hHr:i:s" option; do - case ${option} in - r) - REPO_PATH=${OPTARG} - ;; - i) - DOCKER_IMAGE=${OPTARG} - ;; - s) - CONTAINER_SHELL_ONLY=1 - ;; - h) - echo "${SHORTHELP}" - exit 0 - ;; - H) - echo "${LONGHELP}" - exit 0 - ;; - *) - echo "ERROR: Invalid flag" - echo "${SHORTHELP}" - exit 1 - ;; - esac -done - -REPO_PATH_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")" -CPP_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${CPP_BUILD_DIR}" -PYTHON_BUILD_DIR_IN_CONTAINER="${RAPIDS_DIR_IN_CONTAINER}/$(basename "${REPO_PATH}")/${PYTHON_BUILD_DIR}" - - -# BASE_CONTAINER_BUILD_DIR is named after the image name, allowing for -# multiple image builds to coexist on the local filesystem. This will -# be mapped to the typical BUILD_DIR inside of the container. Builds -# running in the container generate build artifacts just as they would -# in a bare-metal environment, and the host filesystem is able to -# maintain the host build in BUILD_DIR as well. -# shellcheck disable=SC2001,SC2005,SC2046 -BASE_CONTAINER_BUILD_DIR=${REPO_PATH}/build_$(echo $(basename "${DOCKER_IMAGE}")|sed -e 's/:/_/g') -CPP_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/cpp -PYTHON_CONTAINER_BUILD_DIR=${BASE_CONTAINER_BUILD_DIR}/python - - -BUILD_SCRIPT="#!/bin/bash -set -e -WORKSPACE=${REPO_PATH_IN_CONTAINER} -PREBUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/prebuild.sh -BUILD_SCRIPT=${REPO_PATH_IN_CONTAINER}/ci/gpu/build.sh -if [ -f \${PREBUILD_SCRIPT} ]; then - source \${PREBUILD_SCRIPT} -fi -yes | source \${BUILD_SCRIPT} -" - -if (( CONTAINER_SHELL_ONLY == 0 )); then - COMMAND="${CPP_BUILD_DIR_IN_CONTAINER}/build.sh || bash" -else - COMMAND="bash" -fi - -# Create the build dir for the container to mount, generate the build script inside of it -mkdir -p "${BASE_CONTAINER_BUILD_DIR}" -mkdir -p "${CPP_CONTAINER_BUILD_DIR}" -mkdir -p "${PYTHON_CONTAINER_BUILD_DIR}" -# Create build directories. This is to ensure correct owner for directories. If -# directories don't exist there is side effect from docker volume mounting creating build -# directories owned by root(volume mount point(s)) -mkdir -p "${REPO_PATH}/${CPP_BUILD_DIR}" -mkdir -p "${REPO_PATH}/${PYTHON_BUILD_DIR}" - -echo "${BUILD_SCRIPT}" > "${CPP_CONTAINER_BUILD_DIR}/build.sh" -chmod ugo+x "${CPP_CONTAINER_BUILD_DIR}/build.sh" - -# Mount passwd and group files to docker. This allows docker to resolve username and group -# avoiding these nags: -# * groups: cannot find name for group ID ID -# * I have no name!@id:/$ -# For ldap user user information is not present in system /etc/passwd and /etc/group files. -# Hence we generate dummy files for ldap users which docker uses to resolve username and group - -PASSWD_FILE="/etc/passwd" -GROUP_FILE="/etc/group" - -USER_FOUND=$(grep -wc "$(whoami)" < "$PASSWD_FILE") -if [ "$USER_FOUND" == 0 ]; then - echo "Local User not found, LDAP WAR for docker mounts activated. Creating dummy passwd and group" - echo "files to allow docker resolve username and group" - cp "$PASSWD_FILE" /tmp/passwd - PASSWD_FILE="/tmp/passwd" - cp "$GROUP_FILE" /tmp/group - GROUP_FILE="/tmp/group" - echo "$(whoami):x:$(id -u):$(id -g):$(whoami),,,:$HOME:$SHELL" >> "$PASSWD_FILE" - echo "$(whoami):x:$(id -g):" >> "$GROUP_FILE" -fi - -# Run the generated build script in a container -docker pull "${DOCKER_IMAGE}" - -DOCKER_MAJOR=$(docker -v|sed 's/[^[0-9]*\([0-9]*\).*/\1/') -GPU_OPTS="--gpus device=${NVIDIA_VISIBLE_DEVICES}" -if [ "$DOCKER_MAJOR" -lt 19 ] -then - GPU_OPTS="--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES='${NVIDIA_VISIBLE_DEVICES}'" -fi - -docker run --rm -it ${GPU_OPTS} \ - -u "$(id -u)":"$(id -g)" \ - -v "${REPO_PATH}":"${REPO_PATH_IN_CONTAINER}" \ - -v "${CPP_CONTAINER_BUILD_DIR}":"${CPP_BUILD_DIR_IN_CONTAINER}" \ - -v "${PYTHON_CONTAINER_BUILD_DIR}":"${PYTHON_BUILD_DIR_IN_CONTAINER}" \ - -v "$PASSWD_FILE":/etc/passwd:ro \ - -v "$GROUP_FILE":/etc/group:ro \ - --cap-add=SYS_PTRACE \ - "${DOCKER_IMAGE}" bash -c "${COMMAND}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 96099b0512d..d2be7d5f222 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -53,7 +53,6 @@ sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake # cmake-format rapids-cmake definitions -sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHORT_TAG}\/cmake-format-rapids-cmake.json"'/g' ci/checks/style.sh sed_runner 's/'"branch-.*\/cmake-format-rapids-cmake.json"'/'"branch-${NEXT_SHORT_TAG}\/cmake-format-rapids-cmake.json"'/g' ci/check_style.sh # doxyfile update @@ -81,10 +80,6 @@ sed_runner "s/cudf=${CURRENT_SHORT_TAG}/cudf=${NEXT_SHORT_TAG}/g" README.md sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/basic/CMakeLists.txt sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_TAG}/" cpp/examples/strings/CMakeLists.txt -# ucx-py version update -sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/build.sh -sed_runner "s/export UCX_PY_VERSION=.*/export UCX_PY_VERSION='${NEXT_UCX_PY_VERSION}'/g" ci/gpu/java.sh - # Need to distutils-normalize the original version NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index d6681881419..0be72486319 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -79,7 +79,7 @@ if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then ${COMPUTE_SANITIZER_CMD} ${gt} | tee "${RAPIDS_TESTS_DIR}${test_name}.cs.log" done unset GTEST_CUDF_RMM_MODE - # TODO: test-results/*.cs.log are processed in gpuci + # TODO: test-results/*.cs.log are processed in CI fi rapids-logger "Test script exiting with value: $EXITCODE" diff --git a/ci/test_python_cudf.sh b/ci/test_python_cudf.sh index 337ef38cf97..bb33d8473ce 100755 --- a/ci/test_python_cudf.sh +++ b/ci/test_python_cudf.sh @@ -13,7 +13,6 @@ set +e rapids-logger "pytest cudf" pushd python/cudf/cudf -# (TODO: Copied the comment below from gpuCI, need to verify on GitHub Actions) # It is essential to cd into python/cudf/cudf as `pytest-xdist` + `coverage` seem to work only at this directory level. pytest \ --cache-clear \ diff --git a/conda/recipes/dask-cudf/run_test.sh b/conda/recipes/dask-cudf/run_test.sh index 78be90757a2..0c2f628dcf2 100644 --- a/conda/recipes/dask-cudf/run_test.sh +++ b/conda/recipes/dask-cudf/run_test.sh @@ -25,11 +25,11 @@ export DASK_STABLE_VERSION="2023.1.1" # Install the conda-forge or nightly version of dask and distributed if [[ "${INSTALL_DASK_MAIN}" == 1 ]]; then - gpuci_logger "gpuci_mamba_retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" - gpuci_mamba_retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" + rapids-logger "rapids-mamba-retry install -c dask/label/dev 'dask/label/dev::dask' 'dask/label/dev::distributed'" + rapids-mamba-retry install -c dask/label/dev "dask/label/dev::dask" "dask/label/dev::distributed" else - gpuci_logger "gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" - gpuci_mamba_retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall + rapids-logger "rapids-mamba-retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall" + rapids-mamba-retry install conda-forge::dask=={$DASK_STABLE_VERSION} conda-forge::distributed=={$DASK_STABLE_VERSION} conda-forge::dask-core=={$DASK_STABLE_VERSION} --force-reinstall fi logger "python -c 'import dask_cudf'" From bad94b966771890ff2fa73e3663b46c5d74840f2 Mon Sep 17 00:00:00 2001 From: Chong Gao Date: Mon, 13 Feb 2023 10:02:50 +0800 Subject: [PATCH 24/69] JNI switches to nested JSON reader (#12732) JNI switches to nested JSON reader closes https://github.com/NVIDIA/spark-rapids/issues/7518 Note: The new reader read `05/03/2001` as String, so I removed the timestamps in the test case `testReadJSONBufferInferred` Authors: - Chong Gao (https://github.com/res-life) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12732 --- java/src/main/native/src/TableJni.cpp | 6 ++-- .../test/java/ai/rapids/cudf/TableTest.java | 30 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/java/src/main/native/src/TableJni.cpp b/java/src/main/native/src/TableJni.cpp index 0b3ccb59a39..ddcc8644a9c 100644 --- a/java/src/main/native/src/TableJni.cpp +++ b/java/src/main/native/src/TableJni.cpp @@ -1334,8 +1334,7 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_Table_readAndInferJSON( cudf::io::json_reader_options_builder opts = cudf::io::json_reader_options::builder(source) .dayfirst(static_cast(day_first)) - .lines(static_cast(lines)) - .legacy(true); + .lines(static_cast(lines)); auto result = std::make_unique(cudf::io::read_json(opts.build())); @@ -1441,8 +1440,7 @@ JNIEXPORT jlongArray JNICALL Java_ai_rapids_cudf_Table_readJSON( cudf::io::json_reader_options_builder opts = cudf::io::json_reader_options::builder(source) .dayfirst(static_cast(day_first)) - .lines(static_cast(lines)) - .legacy(true); + .lines(static_cast(lines)); if (!n_col_names.is_null() && data_types.size() > 0) { if (n_col_names.size() != n_types.size()) { diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 1656d871c2d..4f00bc7493d 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -335,22 +335,38 @@ void testReadJSONBufferInferred() { JSONOptions opts = JSONOptions.builder() .withDayFirst(true) .build(); - byte[] data = ("[false,A,1,2,05/03/2001]\n" + - "[true,B,2,3,31/10/2010]'\n" + - "[false,C,3,4,20/10/1994]\n" + - "[true,D,4,5,18/10/1990]").getBytes(StandardCharsets.UTF_8); + byte[] data = ("[false,A,1,2]\n" + + "[true,B,2,3]\n" + + "[false,C,3,4]\n" + + "[true,D,4,5]").getBytes(StandardCharsets.UTF_8); try (Table expected = new Table.TestBuilder() .column(false, true, false, true) .column("A", "B", "C", "D") .column(1L, 2L, 3L, 4L) .column(2L, 3L, 4L, 5L) - .timestampMillisecondsColumn(983750400000L, 1288483200000L, 782611200000L, 656208000000L) .build(); Table table = Table.readJSON(Schema.INFERRED, opts, data)) { assertTablesAreEqual(expected, table); } } + @Test + void testReadJSONSubColumns() { + // JSON file has 2 columns, here only read 1 column + Schema schema = Schema.builder() + .column(DType.INT32, "age") + .build(); + JSONOptions opts = JSONOptions.builder() + .withLines(true) + .build(); + try (Table expected = new Table.TestBuilder() + .column(null, 30, 19) + .build(); + Table table = Table.readJSON(schema, opts, TEST_SIMPLE_JSON_FILE)) { + assertTablesAreEqual(expected, table); + } + } + @Test void testReadJSONBuffer() { // JSON reader will set the column according to the iterator if can't infer the name @@ -363,7 +379,7 @@ void testReadJSONBuffer() { JSONOptions opts = JSONOptions.builder() .build(); byte[] data = ("[A,1,2]\n" + - "[B,2,3]'\n" + + "[B,2,3]\n" + "[C,3,4]\n" + "[D,4,5]").getBytes(StandardCharsets.UTF_8); try (Table expected = new Table.TestBuilder() @@ -389,7 +405,7 @@ void testReadJSONBufferWithOffset() { .build(); int bytesToIgnore = 8; byte[] data = ("[A,1,2]\n" + - "[B,2,3]'\n" + + "[B,2,3]\n" + "[C,3,4]\n" + "[D,4,5]").getBytes(StandardCharsets.UTF_8); try (Table expected = new Table.TestBuilder() From 53183cd86a74a04c2357f47120b39a4b11c7dad5 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Tue, 14 Feb 2023 01:01:08 +0530 Subject: [PATCH 25/69] Fix missing trailing comma in json writer (#12688) Fix missing trailing comma in json writer for non-Lines json format. Updates default rows_per_chunk because `8` is too small and writer is slow. closes #12687 Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - Bradley Dice (https://github.com/bdice) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12688 --- cpp/src/io/json/write_json.cu | 23 +++++++++++------------ python/cudf/cudf/_lib/json.pyx | 2 +- python/cudf/cudf/tests/test_json.py | 15 ++++++++++++--- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/cpp/src/io/json/write_json.cu b/cpp/src/io/json/write_json.cu index 49d035e6cb9..9849629015d 100644 --- a/cpp/src/io/json/write_json.cu +++ b/cpp/src/io/json/write_json.cu @@ -581,14 +581,13 @@ std::unique_ptr make_column_names_column(host_span 0, "Unexpected empty strings column."); - string_scalar d_line_terminator{line_terminator}; auto p_str_col_w_nl = cudf::strings::detail::join_strings(str_column_view, d_line_terminator, string_scalar("", false), @@ -609,15 +608,6 @@ void write_chunked(data_sink* out_sink, out_sink->host_write(h_bytes.data(), total_num_bytes); } - - // Needs newline at the end, to separate from next chunk - if (options.is_enabled_lines()) { - if (out_sink->is_device_write_preferred(d_line_terminator.size())) { - out_sink->device_write(d_line_terminator.data(), d_line_terminator.size(), stream); - } else { - out_sink->host_write(line_terminator.data(), line_terminator.size()); - } - } } void write_json(data_sink* out_sink, @@ -697,7 +687,16 @@ void write_json(data_sink* out_sink, // struct converter for the table auto str_concat_col = converter(sub_view.begin(), sub_view.end(), user_column_names); - write_chunked(out_sink, str_concat_col->view(), line_terminator, options, stream, mr); + write_chunked(out_sink, str_concat_col->view(), d_line_terminator, options, stream, mr); + + // Needs line_terminator at the end, to separate from next chunk + if (&sub_view != &vector_views.back() or options.is_enabled_lines()) { + if (out_sink->is_device_write_preferred(d_line_terminator.size())) { + out_sink->device_write(d_line_terminator.data(), d_line_terminator.size(), stream); + } else { + out_sink->host_write(line_terminator.data(), line_terminator.size()); + } + } } } else { if (options.is_enabled_lines()) { diff --git a/python/cudf/cudf/_lib/json.pyx b/python/cudf/cudf/_lib/json.pyx index a40ba7862b2..2339b874ea0 100644 --- a/python/cudf/cudf/_lib/json.pyx +++ b/python/cudf/cudf/_lib/json.pyx @@ -156,7 +156,7 @@ def write_json( bool include_nulls=True, bool lines=False, bool index=False, - int rows_per_chunk=8, + int rows_per_chunk=1024*256, # 256K rows ): """ Cython function to call into libcudf API, see `write_json`. diff --git a/python/cudf/cudf/tests/test_json.py b/python/cudf/cudf/tests/test_json.py index 81acb43ee7d..b778db4465f 100644 --- a/python/cudf/cudf/tests/test_json.py +++ b/python/cudf/cudf/tests/test_json.py @@ -187,14 +187,23 @@ def test_json_writer(tmpdir, pdf, gdf): assert_eq(pdf_string, gdf_string) -def test_cudf_json_writer(pdf): +@pytest.mark.parametrize( + "lines", [True, False], ids=["lines=True", "lines=False"] +) +def test_cudf_json_writer(pdf, lines): # removing datetime column because pandas doesn't support it for col_name in pdf.columns: if "datetime" in col_name: pdf.drop(col_name, axis=1, inplace=True) gdf = cudf.DataFrame.from_pandas(pdf) - pdf_string = pdf.to_json(orient="records", lines=True) - gdf_string = gdf.to_json(orient="records", lines=True, engine="cudf") + pdf_string = pdf.to_json(orient="records", lines=lines) + gdf_string = gdf.to_json(orient="records", lines=lines, engine="cudf") + + assert_eq(pdf_string, gdf_string) + + gdf_string = gdf.to_json( + orient="records", lines=lines, engine="cudf", rows_per_chunk=8 + ) assert_eq(pdf_string, gdf_string) From 12410d90f54d8f838e5328928f092a692620d628 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 15 Feb 2023 09:54:48 -0800 Subject: [PATCH 26/69] Stop force pulling fmt in nvbench. (#12768) This undoes https://github.com/rapidsai/cudf/pull/12064 and resolves #12065. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Robert Maynard (https://github.com/robertmaynard) Approvers: - Bradley Dice (https://github.com/bdice) --- cpp/cmake/thirdparty/get_nvbench.cmake | 5 ----- .../patches/nvbench_global_setup.diff | 20 +++++++++---------- .../thirdparty/patches/nvbench_override.json | 5 +++++ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cpp/cmake/thirdparty/get_nvbench.cmake b/cpp/cmake/thirdparty/get_nvbench.cmake index 3a39e6c7ad1..f0642145fa0 100644 --- a/cpp/cmake/thirdparty/get_nvbench.cmake +++ b/cpp/cmake/thirdparty/get_nvbench.cmake @@ -18,11 +18,6 @@ function(find_and_configure_nvbench) include(${rapids-cmake-dir}/cpm/nvbench.cmake) include(${rapids-cmake-dir}/cpm/package_override.cmake) - # Find or install NVBench Temporarily force downloading of fmt because current versions of nvbench - # do not support the latest version of fmt, which is automatically pulled into our conda - # environments by mamba. - set(CPM_DOWNLOAD_fmt TRUE) - set(cudf_patch_dir "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/patches") rapids_cpm_package_override("${cudf_patch_dir}/nvbench_override.json") diff --git a/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff b/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff index 4e18385f664..0487b0a1ac3 100644 --- a/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff +++ b/cpp/cmake/thirdparty/patches/nvbench_global_setup.diff @@ -1,5 +1,5 @@ diff --git a/nvbench/main.cuh b/nvbench/main.cuh -index 4c1588c..3ba2b99 100644 +index 0ba82d7..7ab02c1 100644 --- a/nvbench/main.cuh +++ b/nvbench/main.cuh @@ -54,6 +54,14 @@ @@ -14,14 +14,14 @@ index 4c1588c..3ba2b99 100644 +#define NVBENCH_ENVIRONMENT nvbench::no_environment +#endif + - #define NVBENCH_MAIN_PARSE(argc, argv) \ - nvbench::option_parser parser; \ + #define NVBENCH_MAIN_PARSE(argc, argv) \ + nvbench::option_parser parser; \ parser.parse(argc, argv) @@ -77,6 +85,7 @@ - printer.set_total_state_count(total_states); \ - \ - printer.set_completed_state_count(0); \ -+ NVBENCH_ENVIRONMENT{}; \ - for (auto &bench_ptr : benchmarks) \ - { \ - bench_ptr->set_printer(printer); \ + printer.set_total_state_count(total_states); \ + \ + printer.set_completed_state_count(0); \ ++ NVBENCH_ENVIRONMENT(); \ + for (auto &bench_ptr : benchmarks) \ + { \ + bench_ptr->set_printer(printer); \ diff --git a/cpp/cmake/thirdparty/patches/nvbench_override.json b/cpp/cmake/thirdparty/patches/nvbench_override.json index ad9b19c29c1..7be868081b6 100644 --- a/cpp/cmake/thirdparty/patches/nvbench_override.json +++ b/cpp/cmake/thirdparty/patches/nvbench_override.json @@ -7,6 +7,11 @@ "file" : "${current_json_dir}/nvbench_global_setup.diff", "issue" : "Fix add support for global setup to initialize RMM in nvbench [https://github.com/NVIDIA/nvbench/pull/123]", "fixed_in" : "" + }, + { + "file" : "nvbench/use_existing_fmt.diff", + "issue" : "Fix add support for using an existing fmt [https://github.com/NVIDIA/nvbench/pull/125]", + "fixed_in" : "" } ] } From d787ff2cd3b8df6a779674bd59ff9ef37e41d6bf Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 15 Feb 2023 20:40:19 -0600 Subject: [PATCH 27/69] Add `GroupBy.dtypes` (#12783) This PR adds `dtypes` property to `GroupBy`, this will also fix some upstream dask breaking changes introduced in: https://github.com/dask/dask/pull/9889 Issue was discovered in: https://github.com/rapidsai/cudf/pull/12768#issuecomment-1430650703 Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/12783 --- python/cudf/cudf/core/groupby/groupby.py | 31 ++++++++++++++++++++++++ python/cudf/cudf/tests/test_groupby.py | 12 +++++++++ python/cudf_kafka/setup.py | 12 ++++++--- 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 04a71b77413..8ff3e17d6ff 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -275,6 +275,37 @@ def __iter__(self): for i, name in enumerate(group_names): yield name, grouped_values[offsets[i] : offsets[i + 1]] + @property + def dtypes(self): + """ + Return the dtypes in this group. + + Returns + ------- + pandas.DataFrame + The data type of each column of the group. + + Examples + -------- + >>> import cudf + >>> df = cudf.DataFrame({'a': [1, 2, 3, 3], 'b': ['x', 'y', 'z', 'a'], + ... 'c':[10, 11, 12, 12]}) + >>> df.groupby("a").dtypes + b c + a + 1 object int64 + 2 object int64 + 3 object int64 + """ + index = self.grouping.keys.unique().to_pandas() + return pd.DataFrame( + { + name: [self.obj._dtypes[name]] * len(index) + for name in self.grouping.values._column_names + }, + index=index, + ) + @cached_property def groups(self): """ diff --git a/python/cudf/cudf/tests/test_groupby.py b/python/cudf/cudf/tests/test_groupby.py index 1fea3c7a37e..97700779a89 100644 --- a/python/cudf/cudf/tests/test_groupby.py +++ b/python/cudf/cudf/tests/test_groupby.py @@ -2960,3 +2960,15 @@ def test_groupby_ngroup(by, ascending, df_ngroup): expected = df_ngroup.to_pandas().groupby(by).ngroup(ascending=ascending) actual = df_ngroup.groupby(by).ngroup(ascending=ascending) assert_eq(expected, actual, check_dtype=False) + + +@pytest.mark.parametrize( + "groups", ["a", "b", "c", ["a", "c"], ["a", "b", "c"]] +) +def test_groupby_dtypes(groups): + df = cudf.DataFrame( + {"a": [1, 2, 3, 3], "b": ["x", "y", "z", "a"], "c": [10, 11, 12, 12]} + ) + pdf = df.to_pandas() + + assert_eq(pdf.groupby(groups).dtypes, df.groupby(groups).dtypes) diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 6d8f954f337..499d1d4a84b 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -43,7 +43,7 @@ ), ) CUDF_KAFKA_ROOT = os.environ.get( - "CUDF_KAFKA_ROOT", "../../libcudf_kafka/build" + "CUDF_KAFKA_ROOT", "../../cpp/libcudf_kafka/build" ) try: @@ -72,8 +72,14 @@ pa.get_include(), cuda_include_dir, ], - library_dirs=([get_python_lib(), os.path.join(os.sys.prefix, "lib")]), - libraries=["cudf", "cudf_kafka"], + library_dirs=( + [ + get_python_lib(), + os.path.join(os.sys.prefix, "lib"), + CUDF_KAFKA_ROOT, + ] + ), + libraries=["cudf", "cudf_kafka", "fmt"], language="c++", extra_compile_args=["-std=c++17"], ) From e4ffcbb3b67fafd54ee4759da87aab9f98eac5cf Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Thu, 16 Feb 2023 01:46:20 -0500 Subject: [PATCH 28/69] Fast path for `experimental::row::equality` (#12676) This PR adds a fast path for primitive types similar to `experimental::row::lexicographic`. The compilation impact for building on bare-metal from source with command `./build.sh libcudf tests benchmarks` for baseline `16m43.607s` vs this branch `17m13.987s`. This PR is a part of https://github.com/rapidsai/cudf/issues/12593. Algorithms and benchmarks (those that were available are linked) affected by this change: `experimental::row::equality::self_comparator` - [x] [`group_nunique`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1414477008) - [x] [`group_rank_scan`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1414479087) - [x] [`rank_scan`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1414481112) - [x] `contains_table` - [x] [`distinct`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1414482768) - [x] [`unique`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1414483897) - [x] [`rank`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1421909885) `experimental::row::equality::two_table_comparator` - [x] `struct_binary_ops` - [x] `lists/contains` - [x] [`contains_scalar`](https://github.com/rapidsai/cudf/pull/12676#issuecomment-1414485080) (This algorithm does not need a primitive type optimization because the enclosing struct already type-dispatches based on nested vs non-nested types) - [x] `contains_table` - [x] `one_hot_encode` Authors: - Divye Gala (https://github.com/divyegala) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Nghia Truong (https://github.com/ttnghia) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/12676 --- .../cudf/table/experimental/row_operators.cuh | 79 ++++++++++- .../binaryop/compiled/struct_binary_ops.cuh | 77 +++++++--- cpp/src/groupby/hash/groupby.cu | 131 ++++++++++-------- cpp/src/groupby/sort/group_nunique.cu | 49 ++++--- cpp/src/groupby/sort/group_rank_scan.cu | 78 +++++++---- cpp/src/groupby/sort/sort_helper.cu | 29 ++-- cpp/src/lists/contains.cu | 4 +- cpp/src/reductions/scan/rank_scan.cu | 49 +++++-- cpp/src/search/contains_scalar.cu | 4 +- cpp/src/search/contains_table.cu | 75 +++++++--- cpp/src/sort/rank.cu | 50 +++++-- cpp/src/stream_compaction/distinct.cu | 13 +- cpp/src/stream_compaction/distinct_reduce.cu | 27 ++-- cpp/src/stream_compaction/distinct_reduce.cuh | 4 +- cpp/src/stream_compaction/unique.cu | 52 ++++--- cpp/src/transform/one_hot_encode.cu | 50 +++++-- .../table/experimental_row_operator_tests.cu | 62 ++++++--- 17 files changed, 589 insertions(+), 244 deletions(-) diff --git a/cpp/include/cudf/table/experimental/row_operators.cuh b/cpp/include/cudf/table/experimental/row_operators.cuh index 0dc0f4e5315..f9ffbfcdf7b 100644 --- a/cpp/include/cudf/table/experimental/row_operators.cuh +++ b/cpp/include/cudf/table/experimental/row_operators.cuh @@ -245,6 +245,16 @@ using optional_dremel_view = thrust::optional; * second letter in both words is the first non-equal letter, and `a < b`, thus * `aac < abb`. * + * @note The operator overloads in sub-class `element_comparator` are templated via the + * `type_dispatcher` to help select an overload instance for each column in a table. + * So, `cudf::is_nested` will return `true` if the table has nested-type columns, + * but it will be a runtime error if template parameter `has_nested_columns != true`. + * + * @tparam has_nested_columns compile-time optimization for primitive types. + * This template parameter is to be used by the developer by querying + * `cudf::detail::has_nested_columns(input)`. `true` compiles operator + * overloads for nested types, while `false` only compiles operator + * overloads for primitive types. * @tparam Nullate A cudf::nullate type describing whether to check for nulls. * @tparam PhysicalElementComparator A relational comparator functor that compares individual values * rather than logical elements, defaults to `NaN` aware relational comparator that evaluates `NaN` @@ -857,6 +867,16 @@ class self_comparator { * * `F(i,j)` returns true if and only if row `i` compares lexicographically less than row `j`. * + * @note The operator overloads in sub-class `element_comparator` are templated via the + * `type_dispatcher` to help select an overload instance for each column in a table. + * So, `cudf::is_nested` will return `true` if the table has nested-type columns, + * but it will be a runtime error if template parameter `has_nested_columns != true`. + * + * @tparam has_nested_columns compile-time optimization for primitive types. + * This template parameter is to be used by the developer by querying + * `cudf::detail::has_nested_columns(input)`. `true` compiles operator + * overloads for nested types, while `false` only compiles operator + * overloads for primitive types. * @tparam Nullate A cudf::nullate type describing whether to check for nulls. * @tparam PhysicalElementComparator A relational comparator functor that compares individual * values rather than logical elements, defaults to `NaN` aware relational comparator that @@ -1009,6 +1029,16 @@ class two_table_comparator { * only if row `i` of the right table compares lexicographically less than row * `j` of the left table. * + * @note The operator overloads in sub-class `element_comparator` are templated via the + * `type_dispatcher` to help select an overload instance for each column in a table. + * So, `cudf::is_nested` will return `true` if the table has nested-type columns, + * but it will be a runtime error if template parameter `has_nested_columns != true`. + * + * @tparam has_nested_columns compile-time optimization for primitive types. + * This template parameter is to be used by the developer by querying + * `cudf::detail::has_nested_columns(input)`. `true` compiles operator + * overloads for nested types, while `false` only compiles operator + * overloads for primitive types. * @tparam Nullate A cudf::nullate type describing whether to check for nulls. * @tparam PhysicalElementComparator A relational comparator functor that compares individual * values rather than logical elements, defaults to `NaN` aware relational comparator that @@ -1131,11 +1161,22 @@ struct nan_equal_physical_equality_comparator { * returns false, representing unequal rows. If the rows are compared without mismatched elements, * the rows are equal. * + * @note The operator overloads in sub-class `element_comparator` are templated via the + * `type_dispatcher` to help select an overload instance for each column in a table. + * So, `cudf::is_nested` will return `true` if the table has nested-type columns, + * but it will be a runtime error if template parameter `has_nested_columns != true`. + * + * @tparam has_nested_columns compile-time optimization for primitive types. + * This template parameter is to be used by the developer by querying + * `cudf::detail::has_nested_columns(input)`. `true` compiles operator + * overloads for nested types, while `false` only compiles operator + * overloads for primitive types. * @tparam Nullate A cudf::nullate type describing whether to check for nulls. * @tparam PhysicalEqualityComparator A equality comparator functor that compares individual values * rather than logical elements, defaults to a comparator for which `NaN == NaN`. */ -template class device_row_comparator { friend class self_comparator; ///< Allow self_comparator to access private members @@ -1246,14 +1287,14 @@ class device_row_comparator { template () and - not cudf::is_nested()), + (not has_nested_columns or not cudf::is_nested())), typename... Args> __device__ bool operator()(Args...) { CUDF_UNREACHABLE("Attempted to compare elements of uncomparable types."); } - template ())> + template ())> __device__ bool operator()(size_type const lhs_element_index, size_type const rhs_element_index) const noexcept { @@ -1437,6 +1478,16 @@ class self_comparator { * * `F(i,j)` returns true if and only if row `i` compares equal to row `j`. * + * @note The operator overloads in sub-class `element_comparator` are templated via the + * `type_dispatcher` to help select an overload instance for each column in a table. + * So, `cudf::is_nested` will return `true` if the table has nested-type columns, + * but it will be a runtime error if template parameter `has_nested_columns != true`. + * + * @tparam has_nested_columns compile-time optimization for primitive types. + * This template parameter is to be used by the developer by querying + * `cudf::detail::has_nested_columns(input)`. `true` compiles operator + * overloads for nested types, while `false` only compiles operator + * overloads for primitive types. * @tparam Nullate A cudf::nullate type describing whether to check for nulls. * @tparam PhysicalEqualityComparator A equality comparator functor that compares individual * values rather than logical elements, defaults to a comparator for which `NaN == NaN`. @@ -1445,13 +1496,15 @@ class self_comparator { * @param comparator Physical element equality comparison functor. * @return A binary callable object */ - template auto equal_to(Nullate nullate = {}, null_equality nulls_are_equal = null_equality::EQUAL, PhysicalEqualityComparator comparator = {}) const noexcept { - return device_row_comparator{nullate, *d_t, *d_t, nulls_are_equal, comparator}; + return device_row_comparator{ + nullate, *d_t, *d_t, nulls_are_equal, comparator}; } private: @@ -1539,6 +1592,16 @@ class two_table_comparator { * Similarly, `F(rhs_index_type i, lhs_index_type j)` returns true if and only if row `i` of the * right table compares equal to row `j` of the left table. * + * @note The operator overloads in sub-class `element_comparator` are templated via the + * `type_dispatcher` to help select an overload instance for each column in a table. + * So, `cudf::is_nested` will return `true` if the table has nested-type columns, + * but it will be a runtime error if template parameter `has_nested_columns != true`. + * + * @tparam has_nested_columns compile-time optimization for primitive types. + * This template parameter is to be used by the developer by querying + * `cudf::detail::has_nested_columns(input)`. `true` compiles operator + * overloads for nested types, while `false` only compiles operator + * overloads for primitive types. * @tparam Nullate A cudf::nullate type describing whether to check for nulls. * @tparam PhysicalEqualityComparator A equality comparator functor that compares individual * values rather than logical elements, defaults to a `NaN == NaN` equality comparator. @@ -1547,14 +1610,16 @@ class two_table_comparator { * @param comparator Physical element equality comparison functor. * @return A binary callable object */ - template auto equal_to(Nullate nullate = {}, null_equality nulls_are_equal = null_equality::EQUAL, PhysicalEqualityComparator comparator = {}) const noexcept { return strong_index_comparator_adapter{ - device_row_comparator(nullate, *d_left_table, *d_right_table, nulls_are_equal, comparator)}; + device_row_comparator( + nullate, *d_left_table, *d_right_table, nulls_are_equal, comparator)}; } private: diff --git a/cpp/src/binaryop/compiled/struct_binary_ops.cuh b/cpp/src/binaryop/compiled/struct_binary_ops.cuh index 2fcf1ce4e32..d167f0fe3c5 100644 --- a/cpp/src/binaryop/compiled/struct_binary_ops.cuh +++ b/cpp/src/binaryop/compiled/struct_binary_ops.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -106,6 +106,36 @@ void apply_struct_binary_op(mutable_column_view& out, } } +template +struct struct_equality_functor { + struct_equality_functor(OptionalIteratorType optional_iter, + DeviceComparatorType device_comparator, + bool is_lhs_scalar, + bool is_rhs_scalar, + bool preserve_output) + : _optional_iter(optional_iter), + _device_comparator(device_comparator), + _is_lhs_scalar(is_lhs_scalar), + _is_rhs_scalar(is_rhs_scalar), + _preserve_output(preserve_output) + { + } + + auto __device__ operator()(size_type i) const noexcept + { + auto const lhs = cudf::experimental::row::lhs_index_type{_is_lhs_scalar ? 0 : i}; + auto const rhs = cudf::experimental::row::rhs_index_type{_is_rhs_scalar ? 0 : i}; + return _optional_iter[i].has_value() and (_device_comparator(lhs, rhs) == _preserve_output); + } + + private: + OptionalIteratorType _optional_iter; + DeviceComparatorType _device_comparator; + bool _is_lhs_scalar; + bool _is_rhs_scalar; + bool _preserve_output; +}; + template void apply_struct_equality_op(mutable_column_view& out, @@ -125,26 +155,37 @@ void apply_struct_equality_op(mutable_column_view& out, auto trhs = table_view{{rhs}}; auto table_comparator = cudf::experimental::row::equality::two_table_comparator{tlhs, trhs, stream}; - auto device_comparator = - table_comparator.equal_to(nullate::DYNAMIC{has_nested_nulls(tlhs) || has_nested_nulls(trhs)}, - null_equality::EQUAL, - comparator); auto outd = column_device_view::create(out, stream); auto optional_iter = cudf::detail::make_optional_iterator(*outd, nullate::DYNAMIC{out.has_nulls()}); - thrust::tabulate(rmm::exec_policy(stream), - out.begin(), - out.end(), - [optional_iter, - is_lhs_scalar, - is_rhs_scalar, - preserve_output = (op != binary_operator::NOT_EQUAL), - device_comparator] __device__(size_type i) { - auto lhs = cudf::experimental::row::lhs_index_type{is_lhs_scalar ? 0 : i}; - auto rhs = cudf::experimental::row::rhs_index_type{is_rhs_scalar ? 0 : i}; - return optional_iter[i].has_value() and - (device_comparator(lhs, rhs) == preserve_output); - }); + + auto const comparator_helper = [&](auto const device_comparator) { + thrust::tabulate(rmm::exec_policy(stream), + out.begin(), + out.end(), + struct_equality_functor( + optional_iter, + device_comparator, + is_lhs_scalar, + is_rhs_scalar, + op != binary_operator::NOT_EQUAL)); + }; + + if (cudf::detail::has_nested_columns(tlhs) or cudf::detail::has_nested_columns(trhs)) { + auto device_comparator = table_comparator.equal_to( + nullate::DYNAMIC{has_nested_nulls(tlhs) || has_nested_nulls(trhs)}, + null_equality::EQUAL, + comparator); + + comparator_helper(device_comparator); + } else { + auto device_comparator = table_comparator.equal_to( + nullate::DYNAMIC{has_nested_nulls(tlhs) || has_nested_nulls(trhs)}, + null_equality::EQUAL, + comparator); + + comparator_helper(device_comparator); + } } } // namespace cudf::binops::compiled::detail diff --git a/cpp/src/groupby/hash/groupby.cu b/cpp/src/groupby/hash/groupby.cu index 50173d6a987..72ac6255549 100644 --- a/cpp/src/groupby/hash/groupby.cu +++ b/cpp/src/groupby/hash/groupby.cu @@ -68,12 +68,13 @@ namespace { // TODO: replace it with `cuco::static_map` // https://github.com/rapidsai/cudf/issues/10401 -using map_type = concurrent_unordered_map< - cudf::size_type, - cudf::size_type, - cudf::experimental::row::hash::device_row_hasher, - cudf::experimental::row::equality::device_row_comparator>; +template +using map_type = + concurrent_unordered_map, + ComparatorType>; /** * @brief List of aggregation operations that can be computed with a hash-based @@ -189,13 +190,14 @@ class groupby_simple_aggregations_collector final } }; +template class hash_compound_agg_finalizer final : public cudf::detail::aggregation_finalizer { column_view col; data_type result_type; cudf::detail::result_cache* sparse_results; cudf::detail::result_cache* dense_results; device_span gather_map; - map_type const& map; + map_type const& map; bitmask_type const* __restrict__ row_bitmask; rmm::cuda_stream_view stream; rmm::mr::device_memory_resource* mr; @@ -207,7 +209,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final cudf::detail::result_cache* sparse_results, cudf::detail::result_cache* dense_results, device_span gather_map, - map_type const& map, + map_type const& map, bitmask_type const* row_bitmask, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -336,7 +338,7 @@ class hash_compound_agg_finalizer final : public cudf::detail::aggregation_final rmm::exec_policy(stream), thrust::make_counting_iterator(0), col.size(), - ::cudf::detail::var_hash_functor{ + ::cudf::detail::var_hash_functor>{ map, row_bitmask, *var_result_view, *values_view, *sum_view, *count_view, agg._ddof}); sparse_results->add_result(col, agg, std::move(var_result)); dense_results->add_result(col, agg, to_dense_agg_result(agg)); @@ -394,12 +396,13 @@ flatten_single_pass_aggs(host_span requests) * * @see groupby_null_templated() */ +template void sparse_to_dense_results(table_view const& keys, host_span requests, cudf::detail::result_cache* sparse_results, cudf::detail::result_cache* dense_results, device_span gather_map, - map_type const& map, + map_type const& map, bool keys_have_nulls, null_policy include_null_keys, rmm::cuda_stream_view stream, @@ -461,10 +464,11 @@ auto create_sparse_results_table(table_view const& flattened_values, * @brief Computes all aggregations from `requests` that require a single pass * over the data and stores the results in `sparse_results` */ +template void compute_single_pass_aggs(table_view const& keys, host_span requests, cudf::detail::result_cache* sparse_results, - map_type& map, + map_type& map, bool keys_have_nulls, null_policy include_null_keys, rmm::cuda_stream_view stream) @@ -484,16 +488,16 @@ void compute_single_pass_aggs(table_view const& keys, auto row_bitmask = skip_key_rows_with_nulls ? cudf::detail::bitmask_and(keys, stream).first : rmm::device_buffer{}; - thrust::for_each_n( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - keys.num_rows(), - hash::compute_single_pass_aggs_fn{map, - *d_values, - *d_sparse_table, - d_aggs.data(), - static_cast(row_bitmask.data()), - skip_key_rows_with_nulls}); + thrust::for_each_n(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + keys.num_rows(), + hash::compute_single_pass_aggs_fn>{ + map, + *d_values, + *d_sparse_table, + d_aggs.data(), + static_cast(row_bitmask.data()), + skip_key_rows_with_nulls}); // Add results back to sparse_results cache auto sparse_result_cols = sparse_table.release(); for (size_t i = 0; i < aggs.size(); i++) { @@ -507,7 +511,8 @@ void compute_single_pass_aggs(table_view const& keys, * @brief Computes and returns a device vector containing all populated keys in * `map`. */ -rmm::device_uvector extract_populated_keys(map_type const& map, +template +rmm::device_uvector extract_populated_keys(map_type const& map, size_type num_keys, rmm::cuda_stream_view stream) { @@ -566,52 +571,60 @@ std::unique_ptr groupby(table_view const& keys, auto preprocessed_keys = cudf::experimental::row::hash::preprocessed_table::create(keys, stream); auto const comparator = cudf::experimental::row::equality::self_comparator{preprocessed_keys}; auto const row_hash = cudf::experimental::row::hash::row_hasher{std::move(preprocessed_keys)}; - auto const d_key_equal = comparator.equal_to(has_null, null_keys_are_equal); auto const d_row_hash = row_hash.device_hasher(has_null); size_type constexpr unused_key{std::numeric_limits::max()}; size_type constexpr unused_value{std::numeric_limits::max()}; - using allocator_type = typename map_type::allocator_type; - - auto map = map_type::create(compute_hash_table_size(num_keys), - stream, - unused_key, - unused_value, - d_row_hash, - d_key_equal, - allocator_type()); - // Cache of sparse results where the location of aggregate value in each // column is indexed by the hash map cudf::detail::result_cache sparse_results(requests.size()); - // Compute all single pass aggs first - compute_single_pass_aggs( - keys, requests, &sparse_results, *map, keys_have_nulls, include_null_keys, stream); - - // Extract the populated indices from the hash map and create a gather map. - // Gathering using this map from sparse results will give dense results. - auto gather_map = extract_populated_keys(*map, keys.num_rows(), stream); - - // Compact all results from sparse_results and insert into cache - sparse_to_dense_results(keys, - requests, - &sparse_results, - cache, - gather_map, - *map, - keys_have_nulls, - include_null_keys, - stream, - mr); - - return cudf::detail::gather(keys, - gather_map, - out_of_bounds_policy::DONT_CHECK, - cudf::detail::negative_index_policy::NOT_ALLOWED, - stream, - mr); + auto const comparator_helper = [&](auto const d_key_equal) { + using allocator_type = typename map_type::allocator_type; + + auto const map = map_type::create(compute_hash_table_size(num_keys), + stream, + unused_key, + unused_value, + d_row_hash, + d_key_equal, + allocator_type()); + // Compute all single pass aggs first + compute_single_pass_aggs( + keys, requests, &sparse_results, *map, keys_have_nulls, include_null_keys, stream); + + // Extract the populated indices from the hash map and create a gather map. + // Gathering using this map from sparse results will give dense results. + auto gather_map = extract_populated_keys(*map, keys.num_rows(), stream); + + // Compact all results from sparse_results and insert into cache + sparse_to_dense_results(keys, + requests, + &sparse_results, + cache, + gather_map, + *map, + keys_have_nulls, + include_null_keys, + stream, + mr); + + return cudf::detail::gather(keys, + gather_map, + out_of_bounds_policy::DONT_CHECK, + cudf::detail::negative_index_policy::NOT_ALLOWED, + stream, + mr); + }; + + if (cudf::detail::has_nested_columns(keys)) { + auto const d_key_equal = comparator.equal_to(has_null, null_keys_are_equal); + return comparator_helper(d_key_equal); + } else { + auto const d_key_equal = comparator.equal_to(has_null, null_keys_are_equal); + return comparator_helper(d_key_equal); + } } } // namespace diff --git a/cpp/src/groupby/sort/group_nunique.cu b/cpp/src/groupby/sort/group_nunique.cu index c411e654913..cf81253483e 100644 --- a/cpp/src/groupby/sort/group_nunique.cu +++ b/cpp/src/groupby/sort/group_nunique.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,10 +33,10 @@ namespace groupby { namespace detail { namespace { -template +template struct is_unique_iterator_fn { using comparator_type = - typename cudf::experimental::row::equality::device_row_comparator; + typename cudf::experimental::row::equality::device_row_comparator; Nullate nulls; column_device_view const v; @@ -91,24 +91,35 @@ std::unique_ptr group_nunique(column_view const& values, auto const values_view = table_view{{values}}; auto const comparator = cudf::experimental::row::equality::self_comparator{values_view, stream}; - auto const d_equal = comparator.equal_to( - cudf::nullate::DYNAMIC{cudf::has_nested_nulls(values_view)}, null_equality::EQUAL); auto const d_values_view = column_device_view::create(values, stream); - auto const is_unique_iterator = - thrust::make_transform_iterator(thrust::counting_iterator(0), - is_unique_iterator_fn{nullate::DYNAMIC{values.has_nulls()}, - *d_values_view, - d_equal, - null_handling, - group_offsets.data(), - group_labels.data()}); - thrust::reduce_by_key(rmm::exec_policy(stream), - group_labels.begin(), - group_labels.end(), - is_unique_iterator, - thrust::make_discard_iterator(), - result->mutable_view().begin()); + + auto const comparator_helper = [&](auto const d_equal) { + auto const is_unique_iterator = + thrust::make_transform_iterator(thrust::counting_iterator(0), + is_unique_iterator_fn{nullate::DYNAMIC{values.has_nulls()}, + *d_values_view, + d_equal, + null_handling, + group_offsets.data(), + group_labels.data()}); + thrust::reduce_by_key(rmm::exec_policy(stream), + group_labels.begin(), + group_labels.end(), + is_unique_iterator, + thrust::make_discard_iterator(), + result->mutable_view().begin()); + }; + + if (cudf::detail::has_nested_columns(values_view)) { + auto const d_equal = comparator.equal_to( + cudf::nullate::DYNAMIC{cudf::has_nested_nulls(values_view)}, null_equality::EQUAL); + comparator_helper(d_equal); + } else { + auto const d_equal = comparator.equal_to( + cudf::nullate::DYNAMIC{cudf::has_nested_nulls(values_view)}, null_equality::EQUAL); + comparator_helper(d_equal); + } return result; } diff --git a/cpp/src/groupby/sort/group_rank_scan.cu b/cpp/src/groupby/sort/group_rank_scan.cu index 149f026ffe6..479ce166724 100644 --- a/cpp/src/groupby/sort/group_rank_scan.cu +++ b/cpp/src/groupby/sort/group_rank_scan.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +41,38 @@ namespace groupby { namespace detail { namespace { +template +struct unique_identifier { + unique_identifier(size_type const* labels, + size_type const* offsets, + permuted_equal_t permuted_equal, + value_resolver resolver) + : _labels(labels), _offsets(offsets), _permuted_equal(permuted_equal), _resolver(resolver) + { + } + + auto __device__ operator()(size_type row_index) const noexcept + { + auto const group_start = _offsets[_labels[row_index]]; + if constexpr (forward) { + // First value of equal values is 1. + return _resolver(row_index == group_start || !_permuted_equal(row_index, row_index - 1), + row_index - group_start); + } else { + auto const group_end = _offsets[_labels[row_index] + 1]; + // Last value of equal values is 1. + return _resolver(row_index + 1 == group_end || !_permuted_equal(row_index, row_index + 1), + row_index - group_start); + } + } + + private: + size_type const* _labels; + size_type const* _offsets; + permuted_equal_t _permuted_equal; + value_resolver _resolver; +}; + /** * @brief generate grouped row ranks or dense ranks using a row comparison then scan the results * @@ -71,36 +103,34 @@ std::unique_ptr rank_generator(column_view const& grouped_values, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { + auto const grouped_values_view = table_view{{grouped_values}}; auto const comparator = - cudf::experimental::row::equality::self_comparator{table_view{{grouped_values}}, stream}; - auto const d_equal = comparator.equal_to(cudf::nullate::DYNAMIC{has_nulls}, null_equality::EQUAL); - auto const permuted_equal = - permuted_row_equality_comparator(d_equal, value_order.begin()); + cudf::experimental::row::equality::self_comparator{grouped_values_view, stream}; auto ranks = make_fixed_width_column( data_type{type_to_id()}, grouped_values.size(), mask_state::UNALLOCATED, stream, mr); auto mutable_ranks = ranks->mutable_view(); - auto unique_identifier = [labels = group_labels.begin(), - offsets = group_offsets.begin(), - permuted_equal, - resolver] __device__(size_type row_index) { - auto const group_start = offsets[labels[row_index]]; - if constexpr (forward) { - // First value of equal values is 1. - return resolver(row_index == group_start || !permuted_equal(row_index, row_index - 1), - row_index - group_start); - } else { - auto const group_end = offsets[labels[row_index] + 1]; - // Last value of equal values is 1. - return resolver(row_index + 1 == group_end || !permuted_equal(row_index, row_index + 1), - row_index - group_start); - } + auto const comparator_helper = [&](auto const d_equal) { + auto const permuted_equal = + permuted_row_equality_comparator(d_equal, value_order.begin()); + + thrust::tabulate(rmm::exec_policy(stream), + mutable_ranks.begin(), + mutable_ranks.end(), + unique_identifier( + group_labels.begin(), group_offsets.begin(), permuted_equal, resolver)); }; - thrust::tabulate(rmm::exec_policy(stream), - mutable_ranks.begin(), - mutable_ranks.end(), - unique_identifier); + + if (cudf::detail::has_nested_columns(grouped_values_view)) { + auto const d_equal = + comparator.equal_to(cudf::nullate::DYNAMIC{has_nulls}, null_equality::EQUAL); + comparator_helper(d_equal); + } else { + auto const d_equal = + comparator.equal_to(cudf::nullate::DYNAMIC{has_nulls}, null_equality::EQUAL); + comparator_helper(d_equal); + } auto [group_labels_begin, mutable_rank_begin] = [&]() { if constexpr (forward) { diff --git a/cpp/src/groupby/sort/sort_helper.cu b/cpp/src/groupby/sort/sort_helper.cu index 3be090159a7..b53955472b1 100644 --- a/cpp/src/groupby/sort/sort_helper.cu +++ b/cpp/src/groupby/sort/sort_helper.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -149,17 +149,28 @@ sort_groupby_helper::index_vector const& sort_groupby_helper::group_offsets( _group_offsets = std::make_unique(num_keys(stream) + 1, stream); - auto const comparator = cudf::experimental::row::equality::self_comparator{_keys, stream}; - auto const d_key_equal = comparator.equal_to( - cudf::nullate::DYNAMIC{cudf::has_nested_nulls(_keys)}, null_equality::EQUAL); + auto const comparator = cudf::experimental::row::equality::self_comparator{_keys, stream}; + auto const sorted_order = key_sort_order(stream).data(); decltype(_group_offsets->begin()) result_end; - result_end = thrust::unique_copy(rmm::exec_policy(stream), - thrust::counting_iterator(0), - thrust::counting_iterator(num_keys(stream)), - _group_offsets->begin(), - permuted_row_equality_comparator(d_key_equal, sorted_order)); + if (cudf::detail::has_nested_columns(_keys)) { + auto const d_key_equal = comparator.equal_to( + cudf::nullate::DYNAMIC{cudf::has_nested_nulls(_keys)}, null_equality::EQUAL); + result_end = thrust::unique_copy(rmm::exec_policy(stream), + thrust::counting_iterator(0), + thrust::counting_iterator(num_keys(stream)), + _group_offsets->begin(), + permuted_row_equality_comparator(d_key_equal, sorted_order)); + } else { + auto const d_key_equal = comparator.equal_to( + cudf::nullate::DYNAMIC{cudf::has_nested_nulls(_keys)}, null_equality::EQUAL); + result_end = thrust::unique_copy(rmm::exec_policy(stream), + thrust::counting_iterator(0), + thrust::counting_iterator(num_keys(stream)), + _group_offsets->begin(), + permuted_row_equality_comparator(d_key_equal, sorted_order)); + } size_type num_groups = thrust::distance(_group_offsets->begin(), result_end); _group_offsets->set_element(num_groups, num_keys(stream), stream); diff --git a/cpp/src/lists/contains.cu b/cpp/src/lists/contains.cu index 0142e736fd0..05fe82d1713 100644 --- a/cpp/src/lists/contains.cu +++ b/cpp/src/lists/contains.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -267,7 +267,7 @@ void index_of_nested_types(InputIterator input_it, auto const has_nulls = has_nested_nulls(child_tview) || has_nested_nulls(keys_tview); auto const comparator = cudf::experimental::row::equality::two_table_comparator(child_tview, keys_tview, stream); - auto const d_comp = comparator.equal_to(nullate::DYNAMIC{has_nulls}); + auto const d_comp = comparator.equal_to(nullate::DYNAMIC{has_nulls}); auto const do_search = [=](auto const key_validity_iter) { thrust::transform( diff --git a/cpp/src/reductions/scan/rank_scan.cu b/cpp/src/reductions/scan/rank_scan.cu index c6909bfd601..538763099d3 100644 --- a/cpp/src/reductions/scan/rank_scan.cu +++ b/cpp/src/reductions/scan/rank_scan.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,23 @@ namespace cudf { namespace detail { namespace { +template +struct rank_equality_functor { + rank_equality_functor(device_comparator_type comparator, value_resolver resolver) + : _comparator(comparator), _resolver(resolver) + { + } + + auto __device__ operator()(size_type row_index) const noexcept + { + return _resolver(row_index == 0 || !_comparator(row_index, row_index - 1), row_index); + } + + private: + device_comparator_type _comparator; + value_resolver _resolver; +}; + /** * @brief generate row ranks or dense ranks using a row comparison then scan the results * @@ -51,20 +68,30 @@ std::unique_ptr rank_generator(column_view const& order_by, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - auto comp = cudf::experimental::row::equality::self_comparator(table_view{{order_by}}, stream); - auto const device_comparator = - comp.equal_to(nullate::DYNAMIC{has_nested_nulls(table_view({order_by}))}); + auto const order_by_tview = table_view{{order_by}}; + auto comp = cudf::experimental::row::equality::self_comparator(order_by_tview, stream); + auto ranks = make_fixed_width_column( data_type{type_to_id()}, order_by.size(), mask_state::UNALLOCATED, stream, mr); auto mutable_ranks = ranks->mutable_view(); - thrust::tabulate(rmm::exec_policy(stream), - mutable_ranks.begin(), - mutable_ranks.end(), - [comparator = device_comparator, resolver] __device__(size_type row_index) { - return resolver(row_index == 0 || !comparator(row_index, row_index - 1), - row_index); - }); + auto const comparator_helper = [&](auto const device_comparator) { + thrust::tabulate(rmm::exec_policy(stream), + mutable_ranks.begin(), + mutable_ranks.end(), + rank_equality_functor( + device_comparator, resolver)); + }; + + if (cudf::detail::has_nested_columns(order_by_tview)) { + auto const device_comparator = + comp.equal_to(nullate::DYNAMIC{has_nested_nulls(table_view({order_by}))}); + comparator_helper(device_comparator); + } else { + auto const device_comparator = + comp.equal_to(nullate::DYNAMIC{has_nested_nulls(table_view({order_by}))}); + comparator_helper(device_comparator); + } thrust::inclusive_scan(rmm::exec_policy(stream), mutable_ranks.begin(), diff --git a/cpp/src/search/contains_scalar.cu b/cpp/src/search/contains_scalar.cu index 8c500e1e757..093a1f8f1ed 100644 --- a/cpp/src/search/contains_scalar.cu +++ b/cpp/src/search/contains_scalar.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -99,7 +99,6 @@ struct contains_scalar_dispatch { auto const comparator = cudf::experimental::row::equality::two_table_comparator(haystack_tv, needle_tv, stream); - auto const d_comp = comparator.equal_to(nullate::DYNAMIC{has_nulls}); auto const begin = cudf::experimental::row::lhs_iterator(0); auto const end = begin + haystack.size(); @@ -108,6 +107,7 @@ struct contains_scalar_dispatch { auto const check_nulls = haystack.has_nulls(); auto const haystack_cdv_ptr = column_device_view::create(haystack, stream); + auto const d_comp = comparator.equal_to(nullate::DYNAMIC{has_nulls}); return thrust::count_if( rmm::exec_policy(stream), begin, diff --git a/cpp/src/search/contains_table.cu b/cpp/src/search/contains_table.cu index 639dc503ce4..c1cc4659a19 100644 --- a/cpp/src/search/contains_table.cu +++ b/cpp/src/search/contains_table.cu @@ -204,27 +204,45 @@ rmm::device_uvector contains_with_lists_or_nans(table_view const& haystack auto const bitmask_buffer_and_ptr = build_row_bitmask(haystack, stream); auto const row_bitmask_ptr = bitmask_buffer_and_ptr.second; - // Insert only rows that do not have any null at any level. auto const insert_map = [&](auto const value_comp) { - auto const d_eqcomp = strong_index_comparator_adapter{ - comparator.equal_to(nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, value_comp)}; - map.insert_if(haystack_it, - haystack_it + haystack.num_rows(), - thrust::counting_iterator(0), // stencil - row_is_valid{row_bitmask_ptr}, - d_hasher, - d_eqcomp, - stream.value()); + if (cudf::detail::has_nested_columns(haystack)) { + auto const d_eqcomp = strong_index_comparator_adapter{comparator.equal_to( + nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, value_comp)}; + map.insert_if(haystack_it, + haystack_it + haystack.num_rows(), + thrust::counting_iterator(0), // stencil + row_is_valid{row_bitmask_ptr}, + d_hasher, + d_eqcomp, + stream.value()); + } else { + auto const d_eqcomp = strong_index_comparator_adapter{comparator.equal_to( + nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, value_comp)}; + map.insert_if(haystack_it, + haystack_it + haystack.num_rows(), + thrust::counting_iterator(0), // stencil + row_is_valid{row_bitmask_ptr}, + d_hasher, + d_eqcomp, + stream.value()); + } }; + // Insert only rows that do not have any null at any level. dispatch_nan_comparator(compare_nans, insert_map); - } else { // haystack_doesn't_have_nulls || compare_nulls == null_equality::EQUAL auto const insert_map = [&](auto const value_comp) { - auto const d_eqcomp = strong_index_comparator_adapter{ - comparator.equal_to(nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, value_comp)}; - map.insert( - haystack_it, haystack_it + haystack.num_rows(), d_hasher, d_eqcomp, stream.value()); + if (cudf::detail::has_nested_columns(haystack)) { + auto const d_eqcomp = strong_index_comparator_adapter{comparator.equal_to( + nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, value_comp)}; + map.insert( + haystack_it, haystack_it + haystack.num_rows(), d_hasher, d_eqcomp, stream.value()); + } else { + auto const d_eqcomp = strong_index_comparator_adapter{comparator.equal_to( + nullate::DYNAMIC{haystack_has_nulls}, compare_nulls, value_comp)}; + map.insert( + haystack_it, haystack_it + haystack.num_rows(), d_hasher, d_eqcomp, stream.value()); + } }; dispatch_nan_comparator(compare_nans, insert_map); @@ -247,14 +265,25 @@ rmm::device_uvector contains_with_lists_or_nans(table_view const& haystack cudf::experimental::row::equality::two_table_comparator(haystack, needles, stream); auto const check_contains = [&](auto const value_comp) { - auto const d_eqcomp = - comparator.equal_to(nullate::DYNAMIC{has_any_nulls}, compare_nulls, value_comp); - map.contains(needles_it, - needles_it + needles.num_rows(), - contained.begin(), - d_hasher, - d_eqcomp, - stream.value()); + if (cudf::detail::has_nested_columns(haystack) or cudf::detail::has_nested_columns(needles)) { + auto const d_eqcomp = + comparator.equal_to(nullate::DYNAMIC{has_any_nulls}, compare_nulls, value_comp); + map.contains(needles_it, + needles_it + needles.num_rows(), + contained.begin(), + d_hasher, + d_eqcomp, + stream.value()); + } else { + auto const d_eqcomp = + comparator.equal_to(nullate::DYNAMIC{has_any_nulls}, compare_nulls, value_comp); + map.contains(needles_it, + needles_it + needles.num_rows(), + contained.begin(), + d_hasher, + d_eqcomp, + stream.value()); + } }; dispatch_nan_comparator(compare_nans, check_contains); diff --git a/cpp/src/sort/rank.cu b/cpp/src/sort/rank.cu index 461e978643f..fd65e38d467 100644 --- a/cpp/src/sort/rank.cu +++ b/cpp/src/sort/rank.cu @@ -48,6 +48,24 @@ namespace cudf { namespace detail { namespace { +template +struct unique_functor { + unique_functor(PermutationIteratorType permute, DeviceComparatorType device_comparator) + : _permute(permute), _device_comparator(device_comparator) + { + } + + auto __device__ operator()(size_type index) const noexcept + { + return static_cast(index == 0 || + not _device_comparator(_permute[index], _permute[index - 1])); + } + + private: + PermutationIteratorType _permute; + DeviceComparatorType _device_comparator; +}; + // Assign rank from 1 to n unique values. Equal values get same rank value. rmm::device_uvector sorted_dense_rank(column_view input_col, column_view sorted_order_view, @@ -55,21 +73,37 @@ rmm::device_uvector sorted_dense_rank(column_view input_col, { auto const t_input = table_view{{input_col}}; auto const comparator = cudf::experimental::row::equality::self_comparator{t_input, stream}; - auto const device_comparator = comparator.equal_to(nullate::DYNAMIC{has_nested_nulls(t_input)}); auto const sorted_index_order = thrust::make_permutation_iterator( sorted_order_view.begin(), thrust::make_counting_iterator(0)); - auto conv = [permute = sorted_index_order, device_comparator] __device__(size_type index) { - return static_cast(index == 0 || - not device_comparator(permute[index], permute[index - 1])); - }; - auto const unique_it = cudf::detail::make_counting_transform_iterator(0, conv); auto const input_size = input_col.size(); rmm::device_uvector dense_rank_sorted(input_size, stream); - thrust::inclusive_scan( - rmm::exec_policy(stream), unique_it, unique_it + input_size, dense_rank_sorted.data()); + auto const comparator_helper = [&](auto const device_comparator) { + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(input_size), + dense_rank_sorted.data(), + unique_functor{ + sorted_index_order, device_comparator}); + }; + + if (cudf::detail::has_nested_columns(t_input)) { + auto const device_comparator = + comparator.equal_to(nullate::DYNAMIC{has_nested_nulls(t_input)}); + comparator_helper(device_comparator); + } else { + auto const device_comparator = + comparator.equal_to(nullate::DYNAMIC{has_nested_nulls(t_input)}); + comparator_helper(device_comparator); + } + + thrust::inclusive_scan(rmm::exec_policy(stream), + dense_rank_sorted.begin(), + dense_rank_sorted.end(), + dense_rank_sorted.data()); + return dense_rank_sorted; } diff --git a/cpp/src/stream_compaction/distinct.cu b/cpp/src/stream_compaction/distinct.cu index 8f462f58e4e..e15d54b4251 100644 --- a/cpp/src/stream_compaction/distinct.cu +++ b/cpp/src/stream_compaction/distinct.cu @@ -55,7 +55,8 @@ rmm::device_uvector get_distinct_indices(table_view const& input, auto const preprocessed_input = cudf::experimental::row::hash::preprocessed_table::create(input, stream); - auto const has_nulls = nullate::DYNAMIC{cudf::has_nested_nulls(input)}; + auto const has_nulls = nullate::DYNAMIC{cudf::has_nested_nulls(input)}; + auto const has_nested_columns = cudf::detail::has_nested_columns(input); auto const row_hasher = cudf::experimental::row::hash::row_hasher(preprocessed_input); auto const key_hasher = experimental::compaction_hash(row_hasher.device_hasher(has_nulls)); @@ -66,8 +67,13 @@ rmm::device_uvector get_distinct_indices(table_view const& input, size_type{0}, [] __device__(size_type const i) { return cuco::make_pair(i, i); }); auto const insert_keys = [&](auto const value_comp) { - auto const key_equal = row_comp.equal_to(has_nulls, nulls_equal, value_comp); - map.insert(pair_iter, pair_iter + input.num_rows(), key_hasher, key_equal, stream.value()); + if (has_nested_columns) { + auto const key_equal = row_comp.equal_to(has_nulls, nulls_equal, value_comp); + map.insert(pair_iter, pair_iter + input.num_rows(), key_hasher, key_equal, stream.value()); + } else { + auto const key_equal = row_comp.equal_to(has_nulls, nulls_equal, value_comp); + map.insert(pair_iter, pair_iter + input.num_rows(), key_hasher, key_equal, stream.value()); + } }; if (nans_equal == nan_equality::ALL_EQUAL) { @@ -92,6 +98,7 @@ rmm::device_uvector get_distinct_indices(table_view const& input, std::move(preprocessed_input), input.num_rows(), has_nulls, + has_nested_columns, keep, nulls_equal, nans_equal, diff --git a/cpp/src/stream_compaction/distinct_reduce.cu b/cpp/src/stream_compaction/distinct_reduce.cu index 468561273b3..020e6a495bc 100644 --- a/cpp/src/stream_compaction/distinct_reduce.cu +++ b/cpp/src/stream_compaction/distinct_reduce.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -93,6 +93,7 @@ rmm::device_uvector hash_reduce_by_row( std::shared_ptr const preprocessed_input, size_type num_rows, cudf::nullate::DYNAMIC has_nulls, + bool has_nested_columns, duplicate_keep_option keep, null_equality nulls_equal, nan_equality nans_equal, @@ -115,13 +116,23 @@ rmm::device_uvector hash_reduce_by_row( auto const row_comp = cudf::experimental::row::equality::self_comparator(preprocessed_input); auto const reduce_by_row = [&](auto const value_comp) { - auto const key_equal = row_comp.equal_to(has_nulls, nulls_equal, value_comp); - thrust::for_each( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(num_rows), - reduce_by_row_fn{ - map.get_device_view(), key_hasher, key_equal, keep, reduction_results.begin()}); + if (has_nested_columns) { + auto const key_equal = row_comp.equal_to(has_nulls, nulls_equal, value_comp); + thrust::for_each( + rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(num_rows), + reduce_by_row_fn{ + map.get_device_view(), key_hasher, key_equal, keep, reduction_results.begin()}); + } else { + auto const key_equal = row_comp.equal_to(has_nulls, nulls_equal, value_comp); + thrust::for_each( + rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(num_rows), + reduce_by_row_fn{ + map.get_device_view(), key_hasher, key_equal, keep, reduction_results.begin()}); + } }; if (nans_equal == nan_equality::ALL_EQUAL) { diff --git a/cpp/src/stream_compaction/distinct_reduce.cuh b/cpp/src/stream_compaction/distinct_reduce.cuh index c8a0c2869c8..e360d03280a 100644 --- a/cpp/src/stream_compaction/distinct_reduce.cuh +++ b/cpp/src/stream_compaction/distinct_reduce.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,6 +65,7 @@ auto constexpr reduction_init_value(duplicate_keep_option keep) * comparisons * @param num_rows The number of all input rows * @param has_nulls Indicate whether the input rows has any nulls at any nested levels + * @param has_nested_columns Indicates whether the input table has any nested columns * @param keep The parameter to determine what type of reduction to perform * @param nulls_equal Flag to specify whether null elements should be considered as equal * @param stream CUDA stream used for device memory operations and kernel launches @@ -76,6 +77,7 @@ rmm::device_uvector hash_reduce_by_row( std::shared_ptr const preprocessed_input, size_type num_rows, cudf::nullate::DYNAMIC has_nulls, + bool has_nested_columns, duplicate_keep_option keep, null_equality nulls_equal, nan_equality nans_equal, diff --git a/cpp/src/stream_compaction/unique.cu b/cpp/src/stream_compaction/unique.cu index 369b63995e3..511a7b7ae1c 100644 --- a/cpp/src/stream_compaction/unique.cu +++ b/cpp/src/stream_compaction/unique.cu @@ -65,28 +65,40 @@ std::unique_ptr
unique(table_view const& input, auto mutable_view = mutable_column_device_view::create(*unique_indices, stream); auto keys_view = input.select(keys); - auto comp = cudf::experimental::row::equality::self_comparator(keys_view, stream); - auto row_equal = comp.equal_to(nullate::DYNAMIC{has_nested_nulls(keys_view)}, nulls_equal); + auto comp = cudf::experimental::row::equality::self_comparator(keys_view, stream); - // get indices of unique rows - auto result_end = unique_copy(thrust::counting_iterator(0), - thrust::counting_iterator(num_rows), - mutable_view->begin(), - row_equal, - keep, - stream); - auto indices_view = - cudf::detail::slice(column_view(*unique_indices), - 0, - thrust::distance(mutable_view->begin(), result_end)); + auto const comparator_helper = [&](auto const row_equal) { + // get indices of unique rows + auto result_end = unique_copy(thrust::counting_iterator(0), + thrust::counting_iterator(num_rows), + mutable_view->begin(), + row_equal, + keep, + stream); - // gather unique rows and return - return detail::gather(input, - indices_view, - out_of_bounds_policy::DONT_CHECK, - detail::negative_index_policy::NOT_ALLOWED, - stream, - mr); + auto indices_view = + cudf::detail::slice(column_view(*unique_indices), + 0, + thrust::distance(mutable_view->begin(), result_end)); + + // gather unique rows and return + return detail::gather(input, + indices_view, + out_of_bounds_policy::DONT_CHECK, + detail::negative_index_policy::NOT_ALLOWED, + stream, + mr); + }; + + if (cudf::detail::has_nested_columns(keys_view)) { + auto row_equal = + comp.equal_to(nullate::DYNAMIC{has_nested_nulls(keys_view)}, nulls_equal); + return comparator_helper(row_equal); + } else { + auto row_equal = + comp.equal_to(nullate::DYNAMIC{has_nested_nulls(keys_view)}, nulls_equal); + return comparator_helper(row_equal); + } } } // namespace detail diff --git a/cpp/src/transform/one_hot_encode.cu b/cpp/src/transform/one_hot_encode.cu index 8f0a44585bf..3f3dd422f9d 100644 --- a/cpp/src/transform/one_hot_encode.cu +++ b/cpp/src/transform/one_hot_encode.cu @@ -36,6 +36,25 @@ namespace cudf { namespace detail { +template +struct ohe_equality_functor { + ohe_equality_functor(size_type input_size, DeviceComparatorType d_equal) + : _input_size(input_size), _d_equal(d_equal) + { + } + + auto __device__ operator()(size_type i) const noexcept + { + auto const element_index = cudf::experimental::row::lhs_index_type{i % _input_size}; + auto const category_index = cudf::experimental::row::rhs_index_type{i / _input_size}; + return _d_equal(element_index, category_index); + } + + private: + size_type _input_size; + DeviceComparatorType _d_equal; +}; + std::pair, table_view> one_hot_encode(column_view const& input, column_view const& categories, rmm::cuda_stream_view stream, @@ -59,19 +78,24 @@ std::pair, table_view> one_hot_encode(column_view const& auto const t_rhs = table_view{{categories}}; auto const comparator = cudf::experimental::row::equality::two_table_comparator{t_lhs, t_rhs, stream}; - auto const d_equal = - comparator.equal_to(nullate::DYNAMIC{has_nested_nulls(t_lhs) || has_nested_nulls(t_rhs)}); - - thrust::transform( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(total_size), - all_encodings->mutable_view().begin(), - [input_size = input.size(), d_equal] __device__(size_type i) { - auto const element_index = cudf::experimental::row::lhs_index_type{i % input_size}; - auto const category_index = cudf::experimental::row::rhs_index_type{i / input_size}; - return d_equal(element_index, category_index); - }); + + auto const comparator_helper = [&](auto const d_equal) { + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(total_size), + all_encodings->mutable_view().begin(), + ohe_equality_functor(input.size(), d_equal)); + }; + + if (cudf::detail::has_nested_columns(t_lhs) or cudf::detail::has_nested_columns(t_rhs)) { + auto const d_equal = comparator.equal_to( + nullate::DYNAMIC{has_nested_nulls(t_lhs) || has_nested_nulls(t_rhs)}); + comparator_helper(d_equal); + } else { + auto const d_equal = comparator.equal_to( + nullate::DYNAMIC{has_nested_nulls(t_lhs) || has_nested_nulls(t_rhs)}); + comparator_helper(d_equal); + } auto const split_iter = make_counting_transform_iterator(1, [width = input.size()](auto i) { return i * width; }); diff --git a/cpp/tests/table/experimental_row_operator_tests.cu b/cpp/tests/table/experimental_row_operator_tests.cu index ae55275aaec..1f3f7eefe79 100644 --- a/cpp/tests/table/experimental_row_operator_tests.cu +++ b/cpp/tests/table/experimental_row_operator_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -115,18 +115,32 @@ auto self_equality(cudf::table_view input, rmm::cuda_stream_view stream{cudf::get_default_stream()}; auto const table_comparator = cudf::experimental::row::equality::self_comparator{input, stream}; - auto const equal_comparator = - table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); auto output = cudf::make_numeric_column( cudf::data_type(cudf::type_id::BOOL8), input.num_rows(), cudf::mask_state::UNALLOCATED); - thrust::transform(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(input.num_rows()), - thrust::make_counting_iterator(0), - output->mutable_view().data(), - equal_comparator); + if (cudf::detail::has_nested_columns(input)) { + auto const equal_comparator = + table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); + + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(input.num_rows()), + thrust::make_counting_iterator(0), + output->mutable_view().data(), + equal_comparator); + } else { + auto const equal_comparator = + table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); + + thrust::transform(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(input.num_rows()), + thrust::make_counting_iterator(0), + output->mutable_view().data(), + equal_comparator); + } + return output; } @@ -140,20 +154,34 @@ auto two_table_equality(cudf::table_view lhs, auto const table_comparator = cudf::experimental::row::equality::two_table_comparator{lhs, rhs, stream}; - auto const equal_comparator = - table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); + auto const lhs_it = cudf::experimental::row::lhs_iterator(0); auto const rhs_it = cudf::experimental::row::rhs_iterator(0); auto output = cudf::make_numeric_column( cudf::data_type(cudf::type_id::BOOL8), lhs.num_rows(), cudf::mask_state::UNALLOCATED); - thrust::transform(rmm::exec_policy(stream), - lhs_it, - lhs_it + lhs.num_rows(), - rhs_it, - output->mutable_view().data(), - equal_comparator); + if (cudf::detail::has_nested_columns(lhs) or cudf::detail::has_nested_columns(rhs)) { + auto const equal_comparator = + table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); + + thrust::transform(rmm::exec_policy(stream), + lhs_it, + lhs_it + lhs.num_rows(), + rhs_it, + output->mutable_view().data(), + equal_comparator); + } else { + auto const equal_comparator = + table_comparator.equal_to(cudf::nullate::NO{}, cudf::null_equality::EQUAL, comparator); + + thrust::transform(rmm::exec_policy(stream), + lhs_it, + lhs_it + lhs.num_rows(), + rhs_it, + output->mutable_view().data(), + equal_comparator); + } return output; } From 506a479c210f97ed9d6e71ce884b6571cc65234c Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 16 Feb 2023 12:01:06 -0600 Subject: [PATCH 29/69] Merge `copy-on-write` feature branch into `branch-23.04` (#12619) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains changes from https://github.com/rapidsai/cudf/pull/11718 primarily that will enable Copy on write feature in cudf. This PR introduces `copy-on-write`. As the name suggests when `copy-on-write` is enabled, when there is a shallow copy of a column made, both the columns share the same memory and only when there is a write operation being performed on either the parent or any of it's copies a true copy will be triggered. Copy-on-write(`c-o-w`) can be enabled in two ways: 1. Setting `CUDF_COPY_ON_WRITE` environment variable to `1` will enable `c-o-w`, unsetting will disable `c-o-w`. 2. Setting `copy_on_write` option in `cudf` options by doing `cudf.set_option("copy_on_write", True)` to enable it and `cudf.set_option("copy_on_write", False)` to disable it. Note: Copy-on-write is not being enabled by default, it is being introduced as an opt-in. A valid performance comparison can be done only with `copy_on_write=OFF` + `.copy(deep=True)` vs `copy_on_write=ON` + `.copy(deep=False)`: ```python In [1]: import cudf In [2]: s = cudf.Series(range(0, 100000000)) # branch-23.02 : 1209MiB # This-PR : 1209MiB In [3]: s_copy = s.copy(deep=True) #branch-23.02 In [3]: s_copy = s.copy(deep=False) #This-PR # branch-23.02 : 1973MiB # This-PR : 1209MiB In [4]: s Out[4]: 0 0 1 1 2 2 3 3 4 4 ... 99999995 99999995 99999996 99999996 99999997 99999997 99999998 99999998 99999999 99999999 Length: 100000000, dtype: int64 In [5]: s_copy Out[5]: 0 0 1 1 2 2 3 3 4 4 ... 99999995 99999995 99999996 99999996 99999997 99999997 99999998 99999998 99999999 99999999 Length: 100000000, dtype: int64 In [6]: s[2] = 10001 # branch-23.02 : 3121MiB # This-PR : 3121MiB In [7]: s Out[7]: 0 0 1 1 2 10001 3 3 4 4 ... 99999995 99999995 99999996 99999996 99999997 99999997 99999998 99999998 99999999 99999999 Length: 100000000, dtype: int64 In [8]: s_copy Out[8]: 0 0 1 1 2 2 3 3 4 4 ... 99999995 99999995 99999996 99999996 99999997 99999997 99999998 99999998 99999999 99999999 Length: 100000000, dtype: int64 ``` Stats around the performance and memory gains : - [x] Memory usage of new copies will be 0 GPU memory additional overhead i.e., users will save 2x, 5x, 10x,...20x memory usage for making 2x, 5x, 10x,...20x deep copies respectively. **So, The more you copy the more you save 😉**(_as long as you don't write on all of them_) - [x] **copying times are now cut by 99%** for all dtypes when copy-on-write is enabled(`copy_on_write=OFF` + `.copy(deep=True)` vs `copy_on_write=ON` + `.copy(deep=False)`). ```python In [1]: import cudf In [2]: df = cudf.DataFrame({'a': range(0, 1000000)}) In [3]: df = cudf.DataFrame({'a': range(0, 100000000)}) In [4]: df['b'] = df.a.astype('str') # GPU memory usage # branch-23.02 : 2345MiB # This-PR : 2345MiB In [5]: df Out[5]: a b 0 0 0 1 1 1 2 2 2 3 3 3 4 4 4 ... ... ... 99999995 99999995 99999995 99999996 99999996 99999996 99999997 99999997 99999997 99999998 99999998 99999998 99999999 99999999 99999999 [100000000 rows x 2 columns] In [6]: def make_two_copies(df, deep): ...: return df.copy(deep=deep), df.copy(deep=deep) ...: In [7]: x, y = make_two_copies(df, deep=True) # branch-23.02 In [7]: x, y = make_two_copies(df, deep=False) # This PR # GPU memory usage # branch-23.02 : 6147MiB # This-PR : 2345MiB In [8]: %timeit make_two_copies(df, deep=True) # branch-23.02 In [8]: %timeit make_two_copies(df, deep=False) # This PR # Execution times # branch-23.02 : 135 ms ± 4.49 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) # This-PR : 100 µs ± 879 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each) ``` - [x] Even when `copy-on-write` is disabled, `string`, `list` & `struct` columns **deep copies are now 99% faster** ```python In [1]: import cudf In [2]: s = cudf.Series(range(0, 100000000), dtype='str') In [3]: %timeit s.copy(deep=True) # branch-23.02 : 28.3 ms ± 5.32 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) # This PR : 19.9 µs ± 93.5 ns per loop (mean ± std. dev. of 7 runs, 10,000 loops each) In [9]: s = cudf.Series([[1, 2], [2, 3], [3, 4], [4, 5], [6, 7]]* 10000000) In [10]: %timeit s.copy(deep=True) # branch-23.02 : 25.7 ms ± 5.99 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) # This-PR : 44.2 µs ± 1.61 µs per loop (mean ± std. dev. of 7 runs, 10,000 loops each) In [4]: df = cudf.DataFrame({'a': range(0, 100000000), 'b': range(0, 100000000)[::-1]}) In [5]: s = df.to_struct() In [6]: %timeit s.copy(deep=True) # branch-23.02 : 42.5 ms ± 3.5 ms per loop (mean ± std. dev. of 7 runs, 10 loops each) # This-PR : 89.7 µs ± 1.68 µs per loop (mean ± std. dev. of 7 runs, 10,000 loops each) ``` - [x] Add pytests - [x] Docs page explaining copy on write and how to enable/disable it. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Lawrence Mitchell (https://github.com/wence-) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/12619 --- .../source/developer_guide/library_design.md | 186 +++++++++ docs/cudf/source/user_guide/copy-on-write.md | 179 +++++++++ docs/cudf/source/user_guide/index.md | 1 + python/cudf/cudf/_lib/column.pyx | 14 +- python/cudf/cudf/core/buffer/__init__.py | 3 +- python/cudf/cudf/core/buffer/buffer.py | 37 +- python/cudf/cudf/core/buffer/cow_buffer.py | 170 ++++++++ .../cudf/cudf/core/buffer/spillable_buffer.py | 7 +- python/cudf/cudf/core/buffer/utils.py | 10 + python/cudf/cudf/core/column/categorical.py | 29 +- python/cudf/cudf/core/column/column.py | 84 +++- python/cudf/cudf/core/column/lists.py | 5 + python/cudf/cudf/core/column/numerical.py | 3 +- python/cudf/cudf/core/column/string.py | 5 + python/cudf/cudf/core/column/struct.py | 6 +- python/cudf/cudf/core/column_accessor.py | 6 +- python/cudf/cudf/core/dataframe.py | 8 +- python/cudf/cudf/core/frame.py | 9 +- python/cudf/cudf/core/multiindex.py | 4 +- python/cudf/cudf/core/reshape.py | 4 +- python/cudf/cudf/core/series.py | 31 +- python/cudf/cudf/options.py | 48 ++- python/cudf/cudf/testing/_utils.py | 5 + python/cudf/cudf/tests/test_copying.py | 379 +++++++++++++++++- python/cudf/cudf/tests/test_index.py | 19 +- python/cudf/cudf/tests/test_multiindex.py | 12 +- python/cudf/cudf/tests/test_series.py | 22 + python/cudf/cudf/tests/test_stats.py | 1 - python/cudf/cudf/utils/applyutils.py | 12 +- 29 files changed, 1209 insertions(+), 90 deletions(-) create mode 100644 docs/cudf/source/user_guide/copy-on-write.md create mode 100644 python/cudf/cudf/core/buffer/cow_buffer.py diff --git a/docs/cudf/source/developer_guide/library_design.md b/docs/cudf/source/developer_guide/library_design.md index bac5eae6b34..16b84476549 100644 --- a/docs/cudf/source/developer_guide/library_design.md +++ b/docs/cudf/source/developer_guide/library_design.md @@ -229,6 +229,7 @@ Additionally, parameters are: of `` in bytes. This introduces a modest overhead and is **disabled by default**. Furthermore, this is a *soft* limit. The memory usage might exceed the limit if too many buffers are unspillable. +(Buffer-design)= #### Design Spilling consists of two components: @@ -314,3 +315,188 @@ The pandas API also includes a number of helper objects, such as `GroupBy`, `Rol cuDF implements corresponding objects with the same APIs. Internally, these objects typically interact with cuDF objects at the Frame layer via composition. However, for performance reasons they frequently access internal attributes and methods of `Frame` and its subclasses. + + +(copy-on-write-dev-doc)= + +## Copy-on-write + +This section describes the internal implementation details of the copy-on-write feature. +It is recommended that developers familiarize themselves with [the user-facing documentation](copy-on-write-user-doc) of this functionality before reading through the internals +below. + +The core copy-on-write implementation relies on the `CopyOnWriteBuffer` class. +When the cudf option `"copy_on_write"` is `True`, `as_buffer` will always return a `CopyOnWriteBuffer`. +This subclass of `cudf.Buffer` contains all the mechanisms to enable copy-on-write behavior. +The class stores [weak references](https://docs.python.org/3/library/weakref.html) to every existing `CopyOnWriteBuffer` in `CopyOnWriteBuffer._instances`, a mapping from `ptr` keys to `WeakSet`s containing references to `CopyOnWriteBuffer` objects. +This means that all `CopyOnWriteBuffer`s that point to the same device memory are contained in the same `WeakSet` (corresponding to the same `ptr` key) in `CopyOnWriteBuffer._instances`. +This data structure is then used to determine whether or not to make a copy when a write operation is performed on a `Column` (see below). +If multiple buffers point to the same underlying memory, then a copy must be made whenever a modification is attempted. + + +### Eager copies when exposing to third-party libraries + +If a `Column`/`CopyOnWriteBuffer` is exposed to a third-party library via `__cuda_array_interface__`, we are no longer able to track whether or not modification of the buffer has occurred. Hence whenever +someone accesses data through the `__cuda_array_interface__`, we eagerly trigger the copy by calling +`_unlink_shared_buffers` which ensures a true copy of underlying device data is made and +unlinks the buffer from any shared "weak" references. Any future copy requests must also trigger a true physical copy (since we cannot track the lifetime of the third-party object). To handle this we also mark the `Column`/`CopyOnWriteBuffer` as +`obj._zero_copied=True` thus indicating that any future shallow-copy requests will trigger a true physical copy +rather than a copy-on-write shallow copy with weak references. + +### Obtaining a read-only object + +A read-only object can be quite useful for operations that will not +mutate the data. This can be achieved by calling `._get_cuda_array_interface(readonly=True)`, and creating a `SimpleNameSpace` object around it. +This will not trigger a deep copy even if the `CopyOnWriteBuffer` +has weak references. This API should only be used when the lifetime of the proxy object is restricted to cudf's internal code execution. Handing this out to external libraries or user-facing APIs will lead to untracked references and undefined copy-on-write behavior. We currently use this API for device to host +copies like in `ColumnBase.data_array_view(mode="read")` which is used for `Column.values_host`. + + +### Internal access to raw data pointers + +Since it is unsafe to access the raw pointer associated with a buffer when +copy-on-write is enabled, in addition to the readonly proxy object described above, +access to the pointer is gated through `Buffer.get_ptr`. This method accepts a mode +argument through which the caller indicates how they will access the data associated +with the buffer. If only read-only access is required (`mode="read"`), this indicates +that the caller has no intention of modifying the buffer through this pointer. +In this case, any shallow copies are not unlinked. In contrast, if modification is +required one may pass `mode="write"`, provoking unlinking of any shallow copies. + + +### Variable width data types +Weak references are implemented only for fixed-width data types as these are only column +types that can be mutated in place. +Requests for deep copies of variable width data types always return shallow copies of the Columns, because these +types don't support real in-place mutation of the data. +Internally, we mimic in-place mutations using `_mimic_inplace`, but the resulting data is always a deep copy of the underlying data. + + +### Examples + +When copy-on-write is enabled, taking a shallow copy of a `Series` or a `DataFrame` does not +eagerly create a copy of the data. Instead, it produces a view that will be lazily +copied when a write operation is performed on any of its copies. + +Let's create a series: + +```python +>>> import cudf +>>> cudf.set_option("copy_on_write", True) +>>> s1 = cudf.Series([1, 2, 3, 4]) +``` + +Make a copy of `s1`: +```python +>>> s2 = s1.copy(deep=False) +``` + +Make another copy, but of `s2`: +```python +>>> s3 = s2.copy(deep=False) +``` + +Viewing the data and memory addresses show that they all point to the same device memory: +```python +>>> s1 +0 1 +1 2 +2 3 +3 4 +dtype: int64 +>>> s2 +0 1 +1 2 +2 3 +3 4 +dtype: int64 +>>> s3 +0 1 +1 2 +2 3 +3 4 +dtype: int64 + +>>> s1.data._ptr +139796315897856 +>>> s2.data._ptr +139796315897856 +>>> s3.data._ptr +139796315897856 +``` + +Now, when we perform a write operation on one of them, say on `s2`, a new copy is created +for `s2` on device and then modified: + +```python +>>> s2[0:2] = 10 +>>> s2 +0 10 +1 10 +2 3 +3 4 +dtype: int64 +>>> s1 +0 1 +1 2 +2 3 +3 4 +dtype: int64 +>>> s3 +0 1 +1 2 +2 3 +3 4 +dtype: int64 +``` + +If we inspect the memory address of the data, `s1` and `s3` still share the same address but `s2` has a new one: + +```python +>>> s1.data._ptr +139796315897856 +>>> s3.data._ptr +139796315897856 +>>> s2.data._ptr +139796315899392 +``` + +Now, performing write operation on `s1` will trigger a new copy on device memory as there +is a weak reference being shared in `s3`: + +```python +>>> s1[0:2] = 11 +>>> s1 +0 11 +1 11 +2 3 +3 4 +dtype: int64 +>>> s2 +0 10 +1 10 +2 3 +3 4 +dtype: int64 +>>> s3 +0 1 +1 2 +2 3 +3 4 +dtype: int64 +``` + +If we inspect the memory address of the data, the addresses of `s2` and `s3` remain unchanged, but `s1`'s memory address has changed because of a copy operation performed during the writing: + +```python +>>> s2.data._ptr +139796315899392 +>>> s3.data._ptr +139796315897856 +>>> s1.data._ptr +139796315879723 +``` + +cuDF's copy-on-write implementation is motivated by the pandas proposals documented here: +1. [Google doc](https://docs.google.com/document/d/1ZCQ9mx3LBMy-nhwRl33_jgcvWo9IWdEfxDNQ2thyTb0/edit#heading=h.iexejdstiz8u) +2. [Github issue](https://github.com/pandas-dev/pandas/issues/36195) diff --git a/docs/cudf/source/user_guide/copy-on-write.md b/docs/cudf/source/user_guide/copy-on-write.md new file mode 100644 index 00000000000..fff8cafcd95 --- /dev/null +++ b/docs/cudf/source/user_guide/copy-on-write.md @@ -0,0 +1,179 @@ +(copy-on-write-user-doc)= + +# Copy-on-write + +Copy-on-write is a memory management strategy that allows multiple cuDF objects containing the same data to refer to the same memory address as long as neither of them modify the underlying data. +With this approach, any operation that generates an unmodified view of an object (such as copies, slices, or methods like `DataFrame.head`) returns a new object that points to the same memory as the original. +However, when either the existing or new object is _modified_, a copy of the data is made prior to the modification, ensuring that the changes do not propagate between the two objects. +This behavior is best understood by looking at the examples below. + +The default behaviour in cuDF is for copy-on-write to be disabled, so to use it, one must explicitly +opt in by setting a cuDF option. It is recommended to set the copy-on-write at the beginning of the +script execution, because when this setting is changed in middle of a script execution there will +be un-intended behavior where the objects created when copy-on-write is enabled will still have the +copy-on-write behavior whereas the objects created when copy-on-write is disabled will have +different behavior. + +## Enabling copy-on-write + +1. Use `cudf.set_option`: + + ```python + >>> import cudf + >>> cudf.set_option("copy_on_write", True) + ``` + +2. Set the environment variable ``CUDF_COPY_ON_WRITE`` to ``1`` prior to the +launch of the Python interpreter: + + ```bash + export CUDF_COPY_ON_WRITE="1" python -c "import cudf" + ``` + +## Disabling copy-on-write + + +Copy-on-write can be disabled by setting the ``copy_on_write`` option to ``False``: + +```python +>>> cudf.set_option("copy_on_write", False) +``` + +## Making copies + +There are no additional changes required in the code to make use of copy-on-write. + +```python +>>> series = cudf.Series([1, 2, 3, 4]) +``` + +Performing a shallow copy will create a new Series object pointing to the +same underlying device memory: + +```python +>>> copied_series = series.copy(deep=False) +>>> series +0 1 +1 2 +2 3 +3 4 +dtype: int64 +>>> copied_series +0 1 +1 2 +2 3 +3 4 +dtype: int64 +``` + +When a write operation is performed on either ``series`` or +``copied_series``, a true physical copy of the data is created: + +```python +>>> series[0:2] = 10 +>>> series +0 10 +1 10 +2 3 +3 4 +dtype: int64 +>>> copied_series +0 1 +1 2 +2 3 +3 4 +dtype: int64 +``` + + +## Notes + +When copy-on-write is enabled, there is no longer a concept of views when +slicing or indexing. In this sense, indexing behaves as one would expect for +built-in Python containers like `lists`, rather than indexing `numpy arrays`. +Modifying a "view" created by cuDF will always trigger a copy and will not +modify the original object. + +Copy-on-write produces much more consistent copy semantics. Since every object is a copy of the original, users no longer have to think about when modifications may unexpectedly happen in place. This will bring consistency across operations and bring cudf and pandas behavior into alignment when copy-on-write is enabled for both. Here is one example where pandas and cudf are currently inconsistent without copy-on-write enabled: + +```python + +>>> import pandas as pd +>>> s = pd.Series([1, 2, 3, 4, 5]) +>>> s1 = s[0:2] +>>> s1[0] = 10 +>>> s1 +0 10 +1 2 +dtype: int64 +>>> s +0 10 +1 2 +2 3 +3 4 +4 5 +dtype: int64 + +>>> import cudf +>>> s = cudf.Series([1, 2, 3, 4, 5]) +>>> s1 = s[0:2] +>>> s1[0] = 10 +>>> s1 +0 10 +1 2 +>>> s +0 1 +1 2 +2 3 +3 4 +4 5 +dtype: int64 +``` + +The above inconsistency is solved when copy-on-write is enabled: + +```python +>>> import pandas as pd +>>> pd.set_option("mode.copy_on_write", True) +>>> s = pd.Series([1, 2, 3, 4, 5]) +>>> s1 = s[0:2] +>>> s1[0] = 10 +>>> s1 +0 10 +1 2 +dtype: int64 +>>> s +0 1 +1 2 +2 3 +3 4 +4 5 +dtype: int64 + + +>>> import cudf +>>> cudf.set_option("copy_on_write", True) +>>> s = cudf.Series([1, 2, 3, 4, 5]) +>>> s1 = s[0:2] +>>> s1[0] = 10 +>>> s1 +0 10 +1 2 +dtype: int64 +>>> s +0 1 +1 2 +2 3 +3 4 +4 5 +dtype: int64 +``` + + +### Explicit deep and shallow copies comparison + + +| | Copy-on-Write enabled | Copy-on-Write disabled (default) | +|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------| +| `.copy(deep=True)` | A true copy is made and changes don't propagate to the original object. | A true copy is made and changes don't propagate to the original object. | +| `.copy(deep=False)` | Memory is shared between the two objects and but any write operation on one object will trigger a true physical copy before the write is performed. Hence changes will not propagate to the original object. | Memory is shared between the two objects and changes performed on one will propagate to the other object. | diff --git a/docs/cudf/source/user_guide/index.md b/docs/cudf/source/user_guide/index.md index 3b6ee5621af..d3ead13132f 100644 --- a/docs/cudf/source/user_guide/index.md +++ b/docs/cudf/source/user_guide/index.md @@ -13,4 +13,5 @@ guide-to-udfs cupy-interop options PandasCompat +copy-on-write ``` diff --git a/python/cudf/cudf/_lib/column.pyx b/python/cudf/cudf/_lib/column.pyx index 11b4a900896..2a163a795eb 100644 --- a/python/cudf/cudf/_lib/column.pyx +++ b/python/cudf/cudf/_lib/column.pyx @@ -1,5 +1,6 @@ # Copyright (c) 2020-2023, NVIDIA CORPORATION. + import cupy as cp import numpy as np @@ -10,6 +11,7 @@ import cudf._lib as libcudf from cudf.api.types import is_categorical_dtype from cudf.core.buffer import ( Buffer, + CopyOnWriteBuffer, SpillableBuffer, acquire_spill_lock, as_buffer, @@ -515,6 +517,13 @@ cdef class Column: rmm.DeviceBuffer(ptr=data_ptr, size=(size+offset) * dtype_itemsize) ) + elif column_owner and isinstance(data_owner, CopyOnWriteBuffer): + # TODO: In future, see if we can just pass on the + # CopyOnWriteBuffer reference to another column + # and still create a weak reference. + # With the current design that's not possible. + # https://github.com/rapidsai/cudf/issues/12734 + data = data_owner.copy(deep=False) elif ( # This is an optimization of the most common case where # from_column_view creates a "view" that is identical to @@ -538,7 +547,10 @@ cdef class Column: owner=data_owner, exposed=True, ) - if isinstance(data_owner, SpillableBuffer): + if isinstance(data_owner, CopyOnWriteBuffer): + data_owner.get_ptr(mode="write") + # accessing the pointer marks it exposed. + elif isinstance(data_owner, SpillableBuffer): if data_owner.is_spilled: raise ValueError( f"{data_owner} is spilled, which invalidates " diff --git a/python/cudf/cudf/core/buffer/__init__.py b/python/cudf/cudf/core/buffer/__init__.py index 49f2c57b17f..0d433509497 100644 --- a/python/cudf/cudf/core/buffer/__init__.py +++ b/python/cudf/cudf/core/buffer/__init__.py @@ -1,6 +1,7 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. from cudf.core.buffer.buffer import Buffer, cuda_array_interface_wrapper +from cudf.core.buffer.cow_buffer import CopyOnWriteBuffer from cudf.core.buffer.spillable_buffer import SpillableBuffer, SpillLock from cudf.core.buffer.utils import ( acquire_spill_lock, diff --git a/python/cudf/cudf/core/buffer/buffer.py b/python/cudf/cudf/core/buffer/buffer.py index 71f48e0ab0c..2262730d8a1 100644 --- a/python/cudf/cudf/core/buffer/buffer.py +++ b/python/cudf/cudf/core/buffer/buffer.py @@ -194,6 +194,28 @@ def __getitem__(self, key: slice) -> Buffer: raise ValueError("slice must be C-contiguous") return self._getitem(offset=start, size=stop - start) + def copy(self, deep: bool = True): + """ + Return a copy of Buffer. + + Parameters + ---------- + deep : bool, default True + If True, returns a deep copy of the underlying Buffer data. + If False, returns a shallow copy of the Buffer pointing to + the same underlying data. + + Returns + ------- + Buffer + """ + if deep: + return self._from_device_memory( + rmm.DeviceBuffer(ptr=self.get_ptr(mode="read"), size=self.size) + ) + else: + return self[:] + @property def size(self) -> int: """Size of the buffer in bytes.""" @@ -240,20 +262,6 @@ def _get_cuda_array_interface(self, readonly=False): "version": 0, } - @property - def _readonly_proxy_cai_obj(self): - """ - Returns a proxy object with a read-only CUDA Array Interface. - """ - return cuda_array_interface_wrapper( - ptr=self.get_ptr(mode="read"), - size=self.size, - owner=self, - readonly=True, - typestr="|u1", - version=0, - ) - def get_ptr(self, *, mode) -> int: """Device pointer to the start of the buffer. @@ -271,6 +279,7 @@ def get_ptr(self, *, mode) -> int: See Also -------- SpillableBuffer.get_ptr + CopyOnWriteBuffer.get_ptr """ return self._ptr diff --git a/python/cudf/cudf/core/buffer/cow_buffer.py b/python/cudf/cudf/core/buffer/cow_buffer.py new file mode 100644 index 00000000000..16a1e3942e7 --- /dev/null +++ b/python/cudf/cudf/core/buffer/cow_buffer.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +from __future__ import annotations + +import weakref +from collections import defaultdict +from typing import Any, DefaultDict, Tuple, Type, TypeVar +from weakref import WeakSet + +import rmm + +from cudf.core.buffer.buffer import Buffer + +T = TypeVar("T", bound="CopyOnWriteBuffer") + + +def _keys_cleanup(ptr): + weak_set_values = CopyOnWriteBuffer._instances[ptr] + if ( + len(weak_set_values) == 1 + and next(iter(weak_set_values.data))() is None + ): + # When the last remaining reference is being cleaned up we will still + # have a dead reference in `weak_set_values`. If that is the case, then + # we can safely clean up the key + del CopyOnWriteBuffer._instances[ptr] + + +class CopyOnWriteBuffer(Buffer): + """A copy-on-write buffer that implements Buffer. + + This buffer enables making copies of data only when there + is a write operation being performed. + + See more here :ref:`copy-on-write-dev-doc`. + + Use the factory function `as_buffer` to create a CopyOnWriteBuffer + instance. + """ + + _instances: DefaultDict[Tuple, WeakSet] = defaultdict(WeakSet) + """This dict keeps track of all instances that have the same `ptr` + and `size` attributes. Each key of the dict is a `(ptr, size)` + tuple and the corresponding value is a set of weak references to + instances with that `ptr` and `size`.""" + + # TODO: This is synonymous to SpillableBuffer._exposed attribute + # and has to be merged. + _zero_copied: bool + + def _finalize_init(self): + self.__class__._instances[self._ptr].add(self) + self._instances = self.__class__._instances[self._ptr] + self._zero_copied = False + weakref.finalize(self, _keys_cleanup, self._ptr) + + @classmethod + def _from_device_memory( + cls: Type[T], data: Any, *, exposed: bool = False + ) -> T: + """Create a Buffer from an object exposing `__cuda_array_interface__`. + + No data is being copied. + + Parameters + ---------- + data : device-buffer-like + An object implementing the CUDA Array Interface. + exposed : bool, optional + Mark the buffer as zero copied. + + Returns + ------- + Buffer + Buffer representing the same device memory as `data` + """ + + # Bypass `__init__` and initialize attributes manually + ret = super()._from_device_memory(data) + ret._finalize_init() + ret._zero_copied = exposed + return ret + + @classmethod + def _from_host_memory(cls: Type[T], data: Any) -> T: + ret = super()._from_host_memory(data) + ret._finalize_init() + return ret + + @property + def _is_shared(self): + """ + Return `True` if `self`'s memory is shared with other columns. + """ + return len(self._instances) > 1 + + def get_ptr(self, mode: str = "write") -> int: + """Device pointer to the start of the buffer. + + Parameters + ---------- + mode : str, default 'write' + Supported values are {"read", "write"} + If "write", when weak-references exist, they + are unlinked and the data pointed to may be modified + by the caller. If "read", the data pointed to + must not be modified by the caller. + Failure to fulfill this contract will cause + incorrect behavior. + + See Also + -------- + Buffer.get_ptr + SpillableBuffer.get_ptr + """ + if mode == "write": + self._unlink_shared_buffers() + elif mode != "read": + raise ValueError(f"Incorrect mode passed : {mode}") + return self._ptr + + def copy(self, deep: bool = True): + if deep or self._zero_copied: + return super().copy(deep=True) + else: + cls = type(self) + copied_buf = cls.__new__(cls) + copied_buf._ptr = self._ptr + copied_buf._size = self._size + copied_buf._owner = self._owner + copied_buf._finalize_init() + return copied_buf + + @property + def __cuda_array_interface__(self) -> dict: + # Unlink if there are any weak references. + # Mark the Buffer as ``zero_copied=True``, + # which will prevent any copy-on-write + # mechanism post this operation. + # This is done because we don't have any + # control over knowing if a third-party library + # has modified the data this Buffer is + # pointing to. + self._unlink_shared_buffers() + self._zero_copied = True + return self._get_cuda_array_interface(readonly=False) + + def _get_cuda_array_interface(self, readonly=False): + return { + "data": (self._ptr, readonly), + "shape": (self.size,), + "strides": None, + "typestr": "|u1", + "version": 0, + } + + def _unlink_shared_buffers(self): + """ + Unlinks a Buffer if it is shared with other buffers by + making a true deep-copy. + """ + if not self._zero_copied and self._is_shared: + # make a deep copy of existing DeviceBuffer + # and replace pointer to it. + current_buf = rmm.DeviceBuffer(ptr=self._ptr, size=self._size) + new_buf = current_buf.copy() + self._ptr = new_buf.ptr + self._size = new_buf.size + self._owner = new_buf + self._finalize_init() diff --git a/python/cudf/cudf/core/buffer/spillable_buffer.py b/python/cudf/cudf/core/buffer/spillable_buffer.py index 2064c1fd133..c71841a5a26 100644 --- a/python/cudf/cudf/core/buffer/spillable_buffer.py +++ b/python/cudf/cudf/core/buffer/spillable_buffer.py @@ -152,7 +152,7 @@ def __getitem__(self, i): class SpillableBuffer(Buffer): - """A spillable buffer that implements DeviceBufferLike. + """A Buffer that supports spilling memory off the GPU to avoid OOMs. This buffer supports spilling the represented data to host memory. Spilling can be done manually by calling `.spill(target="cpu")` but @@ -278,6 +278,11 @@ def _from_host_memory(cls: Type[T], data: Any) -> T: def is_spilled(self) -> bool: return self._ptr_desc["type"] != "gpu" + def copy(self, deep: bool = True): + spill_lock = SpillLock() + self.spill_lock(spill_lock=spill_lock) + return super().copy(deep=deep) + def spill(self, target: str = "cpu") -> None: """Spill or un-spill this buffer in-place diff --git a/python/cudf/cudf/core/buffer/utils.py b/python/cudf/cudf/core/buffer/utils.py index fac28f52b64..2fe332a12fe 100644 --- a/python/cudf/cudf/core/buffer/utils.py +++ b/python/cudf/cudf/core/buffer/utils.py @@ -7,8 +7,10 @@ from typing import Any, Dict, Optional, Tuple, Union from cudf.core.buffer.buffer import Buffer, cuda_array_interface_wrapper +from cudf.core.buffer.cow_buffer import CopyOnWriteBuffer from cudf.core.buffer.spill_manager import get_global_manager from cudf.core.buffer.spillable_buffer import SpillLock, as_spillable_buffer +from cudf.options import get_option def as_buffer( @@ -71,6 +73,14 @@ def as_buffer( "`data` is a buffer-like or array-like object" ) + if get_option("copy_on_write"): + if isinstance(data, Buffer) or hasattr( + data, "__cuda_array_interface__" + ): + return CopyOnWriteBuffer._from_device_memory(data, exposed=exposed) + if exposed: + raise ValueError("cannot created exposed host memory") + return CopyOnWriteBuffer._from_host_memory(data) if get_global_manager() is not None: return as_spillable_buffer(data, exposed=exposed) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index af21d7545ee..a1526d25512 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -733,7 +733,7 @@ def categories(self) -> ColumnBase: @categories.setter def categories(self, value): - self.dtype = CategoricalDtype( + self._dtype = CategoricalDtype( categories=value, ordered=self.dtype.ordered ) @@ -1276,31 +1276,12 @@ def _get_decategorized_column(self) -> ColumnBase: return out def copy(self, deep: bool = True) -> CategoricalColumn: + result_col = super().copy(deep=deep) if deep: - copied_col = libcudf.copying.copy_column(self) - copied_cat = libcudf.copying.copy_column(self.dtype._categories) - - return column.build_categorical_column( - categories=copied_cat, - codes=column.build_column( - copied_col.base_data, dtype=copied_col.dtype - ), - offset=copied_col.offset, - size=copied_col.size, - mask=copied_col.base_mask, - ordered=self.dtype.ordered, - ) - else: - return column.build_categorical_column( - categories=self.dtype.categories._values, - codes=column.build_column( - self.codes.base_data, dtype=self.codes.dtype - ), - mask=self.base_mask, - ordered=self.dtype.ordered, - offset=self.offset, - size=self.size, + result_col.categories = libcudf.copying.copy_column( + self.dtype._categories ) + return result_col @cached_property def memory_usage(self) -> int: diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 2f4d9e28314..965b413e84f 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -64,7 +64,12 @@ ) from cudf.core._compat import PANDAS_GE_150 from cudf.core.abc import Serializable -from cudf.core.buffer import Buffer, acquire_spill_lock, as_buffer +from cudf.core.buffer import ( + Buffer, + acquire_spill_lock, + as_buffer, + cuda_array_interface_wrapper, +) from cudf.core.dtypes import ( CategoricalDtype, IntervalDtype, @@ -137,7 +142,10 @@ def data_array_view( """ if self.data is not None: if mode == "read": - obj = self.data._readonly_proxy_cai_obj + obj = _proxy_cai_obj( + self.data._get_cuda_array_interface(readonly=True), + owner=self.data, + ) elif mode == "write": obj = self.data else: @@ -170,7 +178,10 @@ def mask_array_view( """ if self.mask is not None: if mode == "read": - obj = self.mask._readonly_proxy_cai_obj + obj = _proxy_cai_obj( + self.mask._get_cuda_array_interface(readonly=True), + owner=self.mask, + ) elif mode == "write": obj = self.mask else: @@ -418,24 +429,50 @@ def nullmask(self) -> Buffer: raise ValueError("Column has no null mask") return self.mask_array_view(mode="read") + def force_deep_copy(self: T) -> T: + """ + A method to create deep copy irrespective of whether + `copy-on-write` is enabled. + """ + result = libcudf.copying.copy_column(self) + return cast(T, result._with_type_metadata(self.dtype)) + def copy(self: T, deep: bool = True) -> T: - """Columns are immutable, so a deep copy produces a copy of the - underlying data and mask and a shallow copy creates a new column and - copies the references of the data and mask. + """ + Makes a copy of the Column. + + Parameters + ---------- + deep : bool, default True + If True, a true physical copy of the column + is made. + If False and `copy_on_write` is False, the same + memory is shared between the buffers of the Column + and changes made to one Column will propagate to + its copy and vice-versa. + If False and `copy_on_write` is True, the same + memory is shared between the buffers of the Column + until there is a write operation being performed on + them. """ if deep: - result = libcudf.copying.copy_column(self) - return cast(T, result._with_type_metadata(self.dtype)) + return self.force_deep_copy() else: return cast( T, build_column( - self.base_data, - self.dtype, - mask=self.base_mask, + data=self.base_data + if self.base_data is None + else self.base_data.copy(deep=False), + dtype=self.dtype, + mask=self.base_mask + if self.base_mask is None + else self.base_mask.copy(deep=False), size=self.size, offset=self.offset, - children=self.base_children, + children=tuple( + col.copy(deep=False) for col in self.base_children + ), ), ) @@ -1358,7 +1395,7 @@ def column_empty_like( data=None, dtype=dtype, mask=codes.base_mask, - children=(as_column(codes.base_data, dtype=codes.dtype),), + children=(codes,), size=codes.size, ) @@ -1887,6 +1924,8 @@ def as_column( arbitrary = cupy.ascontiguousarray(arbitrary) data = as_buffer(arbitrary) + if cudf.get_option("copy_on_write"): + data._zero_copied = True col = build_column(data, dtype=current_dtype, mask=mask) if dtype is not None: @@ -2561,3 +2600,22 @@ def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase: ) from e raise return col + + +def _proxy_cai_obj(cai, owner): + """ + Returns a proxy CAI SimpleNameSpace wrapped + with the provided `cai` as `__cuda_array_interface__` + and owner as `owner` to keep the object alive. + This is an internal utility for `data_array_view` + and `mask_array_view` where an object with + read-only CAI is required. + """ + return cuda_array_interface_wrapper( + ptr=cai["data"][0], + size=cai["shape"][0], + owner=owner, + readonly=cai["data"][1], + typestr=cai["typestr"], + version=cai["version"], + ) diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index 05ff4acde09..9b64f26f0fb 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -198,6 +198,11 @@ def _with_type_metadata( return self + def copy(self, deep: bool = True): + # Since list columns are immutable, both deep and shallow copies share + # the underlying device data and mask. + return super().copy(deep=False) + def leaves(self): if isinstance(self.elements, ListColumn): return self.elements.leaves() diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index 8ee3b6e15b6..87e73d212ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -13,7 +13,6 @@ cast, ) -import cupy import numpy as np import pandas as pd @@ -729,7 +728,7 @@ def to_pandas( pandas_array = pandas_nullable_dtype.__from_arrow__(arrow_array) pd_series = pd.Series(pandas_array, copy=False) elif str(self.dtype) in NUMERIC_TYPES and not self.has_nulls(): - pd_series = pd.Series(cupy.asnumpy(self.values), copy=False) + pd_series = pd.Series(self.values_host, copy=False) else: pd_series = self.to_arrow().to_pandas(**kwargs) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index ce8bc3da08b..8d6ffe48957 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5339,6 +5339,11 @@ def __init__( self._start_offset = None self._end_offset = None + def copy(self, deep: bool = True): + # Since string columns are immutable, both deep + # and shallow copies share the underlying device data and mask. + return super().copy(deep=False) + @property def start_offset(self) -> int: if self._start_offset is None: diff --git a/python/cudf/cudf/core/column/struct.py b/python/cudf/cudf/core/column/struct.py index 69d70cf427f..6838d711641 100644 --- a/python/cudf/cudf/core/column/struct.py +++ b/python/cudf/cudf/core/column/struct.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from __future__ import annotations from functools import cached_property @@ -95,7 +95,9 @@ def __setitem__(self, key, value): super().__setitem__(key, value) def copy(self, deep=True): - result = super().copy(deep=deep) + # Since struct columns are immutable, both deep and + # shallow copies share the underlying device data and mask. + result = super().copy(deep=False) if deep: result = result._rename_fields(self.dtype.fields.keys()) return result diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index e1d12807fa8..707eda3f5e6 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. from __future__ import annotations @@ -319,9 +319,9 @@ def copy(self, deep=False) -> ColumnAccessor: """ Make a copy of this ColumnAccessor. """ - if deep: + if deep or cudf.get_option("copy_on_write"): return self.__class__( - {k: v.copy(deep=True) for k, v in self._data.items()}, + {k: v.copy(deep=deep) for k, v in self._data.items()}, multiindex=self.multiindex, level_names=self.level_names, ) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 535fe2352aa..c32ab19fb69 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -6552,12 +6552,12 @@ def to_struct(self, name=None): field_names = [str(name) for name in self._data.names] col = cudf.core.column.build_struct_column( - names=field_names, children=self._data.columns, size=len(self) + names=field_names, + children=tuple(col.copy(deep=True) for col in self._data.columns), + size=len(self), ) return cudf.Series._from_data( - cudf.core.column_accessor.ColumnAccessor( - {name: col.copy(deep=True)} - ), + cudf.core.column_accessor.ColumnAccessor({name: col}), index=self.index, name=name, ) diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 8b508eac324..ea6a6de0b2b 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -29,6 +29,7 @@ from cudf import _lib as libcudf from cudf._typing import Dtype from cudf.api.types import is_dtype_equal, is_scalar +from cudf.core.buffer import acquire_spill_lock from cudf.core.column import ( ColumnBase, as_column, @@ -1701,9 +1702,10 @@ def _colwise_binop( # that nulls that are present in both left_column and # right_column are not filled. if left_column.nullable and right_column.nullable: - lmask = as_column(left_column.nullmask) - rmask = as_column(right_column.nullmask) - output_mask = (lmask | rmask).data + with acquire_spill_lock(): + lmask = as_column(left_column.nullmask) + rmask = as_column(right_column.nullmask) + output_mask = (lmask | rmask).data left_column = left_column.fillna(fill_value) right_column = right_column.fillna(fill_value) elif left_column.nullable: @@ -1738,6 +1740,7 @@ def _colwise_binop( def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): return _array_ufunc(self, ufunc, method, inputs, kwargs) + @acquire_spill_lock() def _apply_cupy_ufunc_to_operands( self, ufunc, cupy_func, operands, **kwargs ): diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index 7c4eac59eab..783c3996400 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. from __future__ import annotations @@ -156,7 +156,7 @@ def __init__( source_data = {} for i, (column_name, col) in enumerate(codes._data.items()): - if -1 in col.values: + if -1 in col: level = cudf.DataFrame( {column_name: [None] + list(levels[i])}, index=range(-1, len(levels[i])), diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 64622bde86e..df1a543c4aa 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -584,9 +584,7 @@ def _tile(A, reps): mdata[var_name] = cudf.Series( cudf.core.column.build_categorical_column( categories=value_vars, - codes=cudf.core.column.as_column( - temp._column.base_data, dtype=temp._column.dtype - ), + codes=temp._column, mask=temp._column.base_mask, size=temp._column.size, offset=temp._column.offset, diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 1c697a2d824..43b2f38ca46 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -366,6 +366,9 @@ class Series(SingleColumnFrame, IndexedFrame, Serializable): name : str, optional The name to give to the Series. + copy : bool, default False + Copy input data. Only affects Series or 1d ndarray input. + nan_as_null : bool, Default True If ``None``/``True``, converts ``np.nan`` values to ``null`` values. @@ -491,6 +494,7 @@ def __init__( index=None, dtype=None, name=None, + copy=False, nan_as_null=True, ): if isinstance(data, pd.Series): @@ -521,6 +525,8 @@ def __init__( if name is None: name = data.name data = data._column + if copy: + data = data.copy(deep=True) if dtype is not None: data = data.astype(dtype) @@ -539,7 +545,30 @@ def __init__( data = {} if not isinstance(data, ColumnBase): - data = column.as_column(data, nan_as_null=nan_as_null, dtype=dtype) + # Using `getattr_static` to check if + # `data` is on device memory and perform + # a deep copy later. This is different + # from `hasattr` because, it doesn't + # invoke the property we are looking + # for and the latter actually invokes + # the property, which in this case could + # be expensive or mark a buffer as + # unspillable. + has_cai = ( + type( + inspect.getattr_static( + data, "__cuda_array_interface__", None + ) + ) + is property + ) + data = column.as_column( + data, + nan_as_null=nan_as_null, + dtype=dtype, + ) + if copy and has_cai: + data = data.copy(deep=True) else: if dtype is not None: data = data.astype(dtype) diff --git a/python/cudf/cudf/options.py b/python/cudf/cudf/options.py index f80ad92879d..a375d8236d6 100644 --- a/python/cudf/cudf/options.py +++ b/python/cudf/cudf/options.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import os import textwrap @@ -150,6 +150,33 @@ def _validator(val): return _validator +def _cow_validator(val): + if get_option("spill") and val: + raise ValueError( + "Copy-on-write is not supported when spilling is enabled. " + "Please set `spill` to `False`" + ) + if val not in {False, True}: + raise ValueError( + f"{val} is not a valid option. Must be one of {{False, True}}." + ) + + +def _spill_validator(val): + try: + if get_option("copy_on_write") and val: + raise ValueError( + "Spilling is not supported when copy-on-write is enabled. " + "Please set `copy_on_write` to `False`" + ) + except KeyError: + pass + if val not in {False, True}: + raise ValueError( + f"{val} is not a valid option. Must be one of {{False, True}}." + ) + + def _integer_validator(val): try: int(val) @@ -205,7 +232,6 @@ def _integer_and_none_validator(val): _make_contains_validator([None, 32, 64]), ) - _register_option( "spill", _env_get_bool("CUDF_SPILL", False), @@ -215,9 +241,25 @@ def _integer_and_none_validator(val): \tValid values are True or False. Default is False. """ ), - _make_contains_validator([False, True]), + _spill_validator, ) + +_register_option( + "copy_on_write", + _env_get_bool("CUDF_COPY_ON_WRITE", False), + textwrap.dedent( + """ + If set to `False`, disables copy-on-write. + If set to `True`, enables copy-on-write. + Read more at: :ref:`copy-on-write-user-doc` + \tValid values are True or False. Default is False. + """ + ), + _cow_validator, +) + + _register_option( "spill_on_demand", _env_get_bool("CUDF_SPILL_ON_DEMAND", True), diff --git a/python/cudf/cudf/testing/_utils.py b/python/cudf/cudf/testing/_utils.py index fb4daba1209..061bad1d44b 100644 --- a/python/cudf/cudf/testing/_utils.py +++ b/python/cudf/cudf/testing/_utils.py @@ -346,6 +346,11 @@ def get_ptr(x) -> int: assert len(lhs.base_children) == len(rhs.base_children) for lhs_child, rhs_child in zip(lhs.base_children, rhs.base_children): assert_column_memory_eq(lhs_child, rhs_child) + if isinstance(lhs, cudf.core.column.CategoricalColumn) and isinstance( + rhs, cudf.core.column.CategoricalColumn + ): + assert_column_memory_eq(lhs.categories, rhs.categories) + assert_column_memory_eq(lhs.codes, rhs.codes) def assert_column_memory_ne( diff --git a/python/cudf/cudf/tests/test_copying.py b/python/cudf/cudf/tests/test_copying.py index 1d3d9e91ae2..d485912f7b7 100644 --- a/python/cudf/cudf/tests/test_copying.py +++ b/python/cudf/cudf/tests/test_copying.py @@ -1,5 +1,8 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. +import contextlib + +import cupy as cp import numpy as np import pandas as pd import pytest @@ -9,6 +12,17 @@ from cudf.testing._utils import NUMERIC_TYPES, OTHER_TYPES, assert_eq +# TODO: Make use of set_option context manager +# once https://github.com/rapidsai/cudf/issues/12736 +# is resolved. +@contextlib.contextmanager +def with_copy_on_write(on): + original_cow_setting = cudf.get_option("copy_on_write") + cudf.set_option("copy_on_write", on) + yield + cudf.set_option("copy_on_write", original_cow_setting) + + @pytest.mark.parametrize("dtype", NUMERIC_TYPES + OTHER_TYPES) def test_repeat(dtype): arr = np.random.rand(10) * 10 @@ -52,3 +66,366 @@ def test_null_copy(): col = Series(np.arange(2049)) col[:] = None assert len(col) == 2049 + + +@with_copy_on_write(on=True) +def test_series_setitem_cow_on(): + actual = cudf.Series([1, 2, 3, 4, 5]) + new_copy = actual.copy(deep=False) + + actual[1] = 100 + assert_eq(actual, cudf.Series([1, 100, 3, 4, 5])) + assert_eq(new_copy, cudf.Series([1, 2, 3, 4, 5])) + + +@with_copy_on_write(on=False) +def test_series_setitem_cow_off(): + actual = cudf.Series([1, 2, 3, 4, 5]) + new_copy = actual.copy(deep=False) + + actual[1] = 100 + assert_eq(actual, cudf.Series([1, 100, 3, 4, 5])) + assert_eq(new_copy, cudf.Series([1, 100, 3, 4, 5])) + + +@with_copy_on_write(on=True) +def test_series_setitem_both_slice_cow_on(): + actual = cudf.Series([1, 2, 3, 4, 5]) + new_copy = actual.copy(deep=False) + + actual[slice(0, 2, 1)] = 100 + assert_eq(actual, cudf.Series([100, 100, 3, 4, 5])) + assert_eq(new_copy, cudf.Series([1, 2, 3, 4, 5])) + + new_copy[slice(2, 4, 1)] = 300 + assert_eq(actual, cudf.Series([100, 100, 3, 4, 5])) + assert_eq(new_copy, cudf.Series([1, 2, 300, 300, 5])) + + +@with_copy_on_write(on=False) +def test_series_setitem_both_slice_cow_off(): + actual = cudf.Series([1, 2, 3, 4, 5]) + new_copy = actual.copy(deep=False) + + actual[slice(0, 2, 1)] = 100 + assert_eq(actual, cudf.Series([100, 100, 3, 4, 5])) + assert_eq(new_copy, cudf.Series([100, 100, 3, 4, 5])) + + new_copy[slice(2, 4, 1)] = 300 + assert_eq(actual, cudf.Series([100, 100, 300, 300, 5])) + assert_eq(new_copy, cudf.Series([100, 100, 300, 300, 5])) + + +@with_copy_on_write(on=True) +def test_series_setitem_partial_slice_cow_on(): + actual = cudf.Series([1, 2, 3, 4, 5]) + new_copy = actual.copy(deep=False) + + new_copy[slice(2, 4, 1)] = 300 + assert_eq(actual, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(new_copy, cudf.Series([1, 2, 300, 300, 5])) + + new_slice = actual[2:] + assert new_slice._column.base_data._ptr == actual._column.base_data._ptr + new_slice[0:2] = 10 + assert_eq(new_slice, cudf.Series([10, 10, 5], index=[2, 3, 4])) + assert_eq(actual, cudf.Series([1, 2, 3, 4, 5])) + + +@with_copy_on_write(on=False) +def test_series_setitem_partial_slice_cow_off(): + actual = cudf.Series([1, 2, 3, 4, 5]) + new_copy = actual.copy(deep=False) + + new_copy[slice(2, 4, 1)] = 300 + assert_eq(actual, cudf.Series([1, 2, 300, 300, 5])) + assert_eq(new_copy, cudf.Series([1, 2, 300, 300, 5])) + + new_slice = actual[2:] + assert new_slice._column.base_data._ptr == actual._column.base_data._ptr + new_slice[0:2] = 10 + assert_eq(new_slice, cudf.Series([10, 10, 5], index=[2, 3, 4])) + assert_eq(actual, cudf.Series([1, 2, 10, 10, 5])) + + +@with_copy_on_write(on=True) +def test_multiple_series_cow(): + # Verify constructing, modifying, deleting + # multiple copies of a series preserves + # the data appropriately when COW is enabled. + s = cudf.Series([10, 20, 30, 40, 50]) + s1 = s.copy(deep=False) + s2 = s.copy(deep=False) + s3 = s.copy(deep=False) + s4 = s2.copy(deep=False) + s5 = s4.copy(deep=False) + s6 = s3.copy(deep=False) + + s1[0:3] = 10000 + # s1 will be unlinked from actual data in s, + # and then modified. Rest all should + # contain the original data. + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + for ser in [s, s2, s3, s4, s5, s6]: + assert_eq(ser, cudf.Series([10, 20, 30, 40, 50])) + + s6[0:3] = 3000 + # s6 will be unlinked from actual data in s, + # and then modified. Rest all should + # contain the original data. + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + for ser in [s2, s3, s4, s5]: + assert_eq(ser, cudf.Series([10, 20, 30, 40, 50])) + + s2[1:4] = 4000 + # s2 will be unlinked from actual data in s, + # and then modified. Rest all should + # contain the original data. + assert_eq(s2, cudf.Series([10, 4000, 4000, 4000, 50])) + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + for ser in [s3, s4, s5]: + assert_eq(ser, cudf.Series([10, 20, 30, 40, 50])) + + s4[2:4] = 5000 + # s4 will be unlinked from actual data in s, + # and then modified. Rest all should + # contain the original data. + assert_eq(s4, cudf.Series([10, 20, 5000, 5000, 50])) + assert_eq(s2, cudf.Series([10, 4000, 4000, 4000, 50])) + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + for ser in [s3, s5]: + assert_eq(ser, cudf.Series([10, 20, 30, 40, 50])) + + s5[2:4] = 6000 + # s5 will be unlinked from actual data in s, + # and then modified. Rest all should + # contain the original data. + assert_eq(s5, cudf.Series([10, 20, 6000, 6000, 50])) + assert_eq(s4, cudf.Series([10, 20, 5000, 5000, 50])) + assert_eq(s2, cudf.Series([10, 4000, 4000, 4000, 50])) + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + for ser in [s3]: + assert_eq(ser, cudf.Series([10, 20, 30, 40, 50])) + + s7 = s5.copy(deep=False) + assert_eq(s7, cudf.Series([10, 20, 6000, 6000, 50])) + s7[1:3] = 55 + # Making a copy of s5, i.e., s7 and modifying shouldn't + # be touching/modifying data in other series. + assert_eq(s7, cudf.Series([10, 55, 55, 6000, 50])) + + assert_eq(s4, cudf.Series([10, 20, 5000, 5000, 50])) + assert_eq(s2, cudf.Series([10, 4000, 4000, 4000, 50])) + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + for ser in [s3]: + assert_eq(ser, cudf.Series([10, 20, 30, 40, 50])) + + # Deleting any of the following series objects + # shouldn't delete rest of the weekly referenced data + # elsewhere. + + del s2 + + assert_eq(s1, cudf.Series([10000, 10000, 10000, 40, 50])) + assert_eq(s3, cudf.Series([10, 20, 30, 40, 50])) + assert_eq(s4, cudf.Series([10, 20, 5000, 5000, 50])) + assert_eq(s5, cudf.Series([10, 20, 6000, 6000, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + assert_eq(s7, cudf.Series([10, 55, 55, 6000, 50])) + + del s4 + del s1 + + assert_eq(s3, cudf.Series([10, 20, 30, 40, 50])) + assert_eq(s5, cudf.Series([10, 20, 6000, 6000, 50])) + assert_eq(s6, cudf.Series([3000, 3000, 3000, 40, 50])) + assert_eq(s7, cudf.Series([10, 55, 55, 6000, 50])) + + del s + del s6 + + assert_eq(s3, cudf.Series([10, 20, 30, 40, 50])) + assert_eq(s5, cudf.Series([10, 20, 6000, 6000, 50])) + assert_eq(s7, cudf.Series([10, 55, 55, 6000, 50])) + + del s5 + + assert_eq(s3, cudf.Series([10, 20, 30, 40, 50])) + assert_eq(s7, cudf.Series([10, 55, 55, 6000, 50])) + + del s3 + assert_eq(s7, cudf.Series([10, 55, 55, 6000, 50])) + + +@with_copy_on_write(on=True) +def test_series_zero_copy_cow_on(): + s = cudf.Series([1, 2, 3, 4, 5]) + s1 = s.copy(deep=False) + cp_array = cp.asarray(s) + + # Ensure all original data & zero-copied + # data is same. + assert_eq(s, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(s1, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(cp_array, cp.array([1, 2, 3, 4, 5])) + + cp_array[0:3] = 10 + # Modifying a zero-copied array should only + # modify `s` and will leave rest of the copies + # untouched. + + assert_eq(s, cudf.Series([10, 10, 10, 4, 5])) + assert_eq(s1, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(cp_array, cp.array([10, 10, 10, 4, 5])) + + s2 = cudf.Series(cp_array) + assert_eq(s2, cudf.Series([10, 10, 10, 4, 5])) + + s3 = s2.copy(deep=False) + cp_array[0] = 20 + # Modifying a zero-copied array should modify + # `s2` and `s` only. Because `cp_array` + # is zero-copy shared with `s` & `s2`. + + assert_eq(s, cudf.Series([20, 10, 10, 4, 5])) + assert_eq(s1, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(cp_array, cp.array([20, 10, 10, 4, 5])) + assert_eq(s2, cudf.Series([20, 10, 10, 4, 5])) + assert_eq(s3, cudf.Series([10, 10, 10, 4, 5])) + + s4 = cudf.Series([10, 20, 30, 40, 50]) + s5 = cudf.Series(s4) + assert_eq(s5, cudf.Series([10, 20, 30, 40, 50])) + s5[0:2] = 1 + # Modifying `s5` should also modify `s4` + # because they are zero-copied. + assert_eq(s5, cudf.Series([1, 1, 30, 40, 50])) + assert_eq(s4, cudf.Series([1, 1, 30, 40, 50])) + + +@with_copy_on_write(on=False) +def test_series_zero_copy_cow_off(): + s = cudf.Series([1, 2, 3, 4, 5]) + s1 = s.copy(deep=False) + cp_array = cp.asarray(s) + + # Ensure all original data & zero-copied + # data is same. + assert_eq(s, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(s1, cudf.Series([1, 2, 3, 4, 5])) + assert_eq(cp_array, cp.array([1, 2, 3, 4, 5])) + + cp_array[0:3] = 10 + # When COW is off, modifying a zero-copied array + # will need to modify `s` & `s1` since they are + # shallow copied. + + assert_eq(s, cudf.Series([10, 10, 10, 4, 5])) + assert_eq(s1, cudf.Series([10, 10, 10, 4, 5])) + assert_eq(cp_array, cp.array([10, 10, 10, 4, 5])) + + s2 = cudf.Series(cp_array) + assert_eq(s2, cudf.Series([10, 10, 10, 4, 5])) + s3 = s2.copy(deep=False) + cp_array[0] = 20 + + # Modifying `cp_array`, will propagate the changes + # across all Series objects, because they are + # either shallow copied or zero-copied. + + assert_eq(s, cudf.Series([20, 10, 10, 4, 5])) + assert_eq(s1, cudf.Series([20, 10, 10, 4, 5])) + assert_eq(cp_array, cp.array([20, 10, 10, 4, 5])) + assert_eq(s2, cudf.Series([20, 10, 10, 4, 5])) + assert_eq(s3, cudf.Series([20, 10, 10, 4, 5])) + + s4 = cudf.Series([10, 20, 30, 40, 50]) + s5 = cudf.Series(s4) + assert_eq(s5, cudf.Series([10, 20, 30, 40, 50])) + s5[0:2] = 1 + + # Modifying `s5` should also modify `s4` + # because they are zero-copied. + assert_eq(s5, cudf.Series([1, 1, 30, 40, 50])) + assert_eq(s4, cudf.Series([1, 1, 30, 40, 50])) + + +@pytest.mark.parametrize("copy_on_write", [True, False]) +def test_series_str_copy(copy_on_write): + original_cow_setting = cudf.get_option("copy_on_write") + cudf.set_option("copy_on_write", copy_on_write) + s = cudf.Series(["a", "b", "c", "d", "e"]) + s1 = s.copy(deep=True) + s2 = s.copy(deep=True) + + assert_eq(s, cudf.Series(["a", "b", "c", "d", "e"])) + assert_eq(s1, cudf.Series(["a", "b", "c", "d", "e"])) + assert_eq(s2, cudf.Series(["a", "b", "c", "d", "e"])) + + s[0:3] = "abc" + + assert_eq(s, cudf.Series(["abc", "abc", "abc", "d", "e"])) + assert_eq(s1, cudf.Series(["a", "b", "c", "d", "e"])) + assert_eq(s2, cudf.Series(["a", "b", "c", "d", "e"])) + + s2[1:4] = "xyz" + + assert_eq(s, cudf.Series(["abc", "abc", "abc", "d", "e"])) + assert_eq(s1, cudf.Series(["a", "b", "c", "d", "e"])) + assert_eq(s2, cudf.Series(["a", "xyz", "xyz", "xyz", "e"])) + cudf.set_option("copy_on_write", original_cow_setting) + + +@pytest.mark.parametrize("copy_on_write", [True, False]) +def test_series_cat_copy(copy_on_write): + original_cow_setting = cudf.get_option("copy_on_write") + cudf.set_option("copy_on_write", copy_on_write) + s = cudf.Series([10, 20, 30, 40, 50], dtype="category") + s1 = s.copy(deep=True) + s2 = s1.copy(deep=True) + s3 = s1.copy(deep=True) + + s[0] = 50 + assert_eq(s, cudf.Series([50, 20, 30, 40, 50], dtype=s.dtype)) + assert_eq(s1, cudf.Series([10, 20, 30, 40, 50], dtype="category")) + assert_eq(s2, cudf.Series([10, 20, 30, 40, 50], dtype="category")) + assert_eq(s3, cudf.Series([10, 20, 30, 40, 50], dtype="category")) + + s2[3] = 10 + s3[2:5] = 20 + assert_eq(s, cudf.Series([50, 20, 30, 40, 50], dtype=s.dtype)) + assert_eq(s1, cudf.Series([10, 20, 30, 40, 50], dtype=s.dtype)) + assert_eq(s2, cudf.Series([10, 20, 30, 10, 50], dtype=s.dtype)) + assert_eq(s3, cudf.Series([10, 20, 20, 20, 20], dtype=s.dtype)) + cudf.set_option("copy_on_write", original_cow_setting) + + +@with_copy_on_write(on=True) +def test_dataframe_cow_slice_setitem(): + df = cudf.DataFrame({"a": [10, 11, 12, 13, 14], "b": [20, 30, 40, 50, 60]}) + slice_df = df[1:4] + + assert_eq( + slice_df, + cudf.DataFrame( + {"a": [11, 12, 13], "b": [30, 40, 50]}, index=[1, 2, 3] + ), + ) + + slice_df["a"][2] = 1111 + + assert_eq( + slice_df, + cudf.DataFrame( + {"a": [11, 1111, 13], "b": [30, 40, 50]}, index=[1, 2, 3] + ), + ) + assert_eq( + df, + cudf.DataFrame({"a": [10, 11, 12, 13, 14], "b": [20, 30, 40, 50, 60]}), + ) diff --git a/python/cudf/cudf/tests/test_index.py b/python/cudf/cudf/tests/test_index.py index 5b0eca1c635..f0b74ce70e7 100644 --- a/python/cudf/cudf/tests/test_index.py +++ b/python/cudf/cudf/tests/test_index.py @@ -392,6 +392,7 @@ def test_index_copy_category(name, dtype, deep=True): with pytest.warns(FutureWarning): cidx_copy = cidx.copy(name=name, deep=deep, dtype=dtype) + assert_column_memory_ne(cidx._values, cidx_copy._values) assert_eq(pidx_copy, cidx_copy) @@ -407,13 +408,27 @@ def test_index_copy_category(name, dtype, deep=True): cudf.CategoricalIndex(["a", "b", "c"]), ], ) -def test_index_copy_deep(idx, deep): +@pytest.mark.parametrize("copy_on_write", [True, False]) +def test_index_copy_deep(idx, deep, copy_on_write): """Test if deep copy creates a new instance for device data.""" idx_copy = idx.copy(deep=deep) - if not deep: + original_cow_setting = cudf.get_option("copy_on_write") + cudf.set_option("copy_on_write", copy_on_write) + if ( + isinstance(idx, cudf.StringIndex) + or not deep + or (cudf.get_option("copy_on_write") and not deep) + ): + # StringColumn is immutable hence, deep copies of a + # StringIndex will share the same StringColumn. + + # When `copy_on_write` is turned on, Index objects will + # have unique column object but they all point to same + # data pointers. assert_column_memory_eq(idx._values, idx_copy._values) else: assert_column_memory_ne(idx._values, idx_copy._values) + cudf.set_option("copy_on_write", original_cow_setting) @pytest.mark.parametrize("idx", [[1, None, 3, None, 5]]) diff --git a/python/cudf/cudf/tests/test_multiindex.py b/python/cudf/cudf/tests/test_multiindex.py index 3e1f001e7d1..14e3a2a1b9b 100644 --- a/python/cudf/cudf/tests/test_multiindex.py +++ b/python/cudf/cudf/tests/test_multiindex.py @@ -781,13 +781,15 @@ def test_multiindex_copy_sem(data, levels, codes, names): ), ], ) +@pytest.mark.parametrize("copy_on_write", [True, False]) @pytest.mark.parametrize("deep", [True, False]) -def test_multiindex_copy_deep(data, deep): +def test_multiindex_copy_deep(data, copy_on_write, deep): """Test memory identity for deep copy Case1: Constructed from GroupBy, StringColumns Case2: Constructed from MultiIndex, NumericColumns """ - same_ref = not deep + original_cow_setting = cudf.get_option("copy_on_write") + cudf.set_option("copy_on_write", copy_on_write) if isinstance(data, dict): import operator @@ -807,9 +809,12 @@ def test_multiindex_copy_deep(data, deep): lptrs = [child.base_data.get_ptr(mode="read") for child in lchildren] rptrs = [child.base_data.get_ptr(mode="read") for child in rchildren] - assert all((x == y) == same_ref for x, y in zip(lptrs, rptrs)) + assert all((x == y) for x, y in zip(lptrs, rptrs)) elif isinstance(data, cudf.MultiIndex): + same_ref = (not deep) or ( + cudf.get_option("copy_on_write") and not deep + ) mi1 = data mi2 = mi1.copy(deep=deep) @@ -846,6 +851,7 @@ def test_multiindex_copy_deep(data, deep): ] assert all((x == y) == same_ref for x, y in zip(lptrs, rptrs)) + cudf.set_option("copy_on_write", original_cow_setting) @pytest.mark.parametrize( diff --git a/python/cudf/cudf/tests/test_series.py b/python/cudf/cudf/tests/test_series.py index 6a1245a1582..b3c7c9ac9bb 100644 --- a/python/cudf/cudf/tests/test_series.py +++ b/python/cudf/cudf/tests/test_series.py @@ -2080,3 +2080,25 @@ def test_series_duplicated(data, index, keep): ps = gs.to_pandas() assert_eq(gs.duplicated(keep=keep), ps.duplicated(keep=keep)) + + +@pytest.mark.parametrize( + "data", + [ + [1, 2, 3, 4], + [10, 20, None, None], + ], +) +@pytest.mark.parametrize("copy", [True, False]) +def test_series_copy(data, copy): + psr = pd.Series(data) + gsr = cudf.from_pandas(psr) + + new_psr = pd.Series(psr, copy=copy) + new_gsr = cudf.Series(gsr, copy=copy) + + new_psr.iloc[0] = 999 + new_gsr.iloc[0] = 999 + + assert_eq(psr, gsr) + assert_eq(new_psr, new_gsr) diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index 715d17169a6..6478fbaad95 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -531,7 +531,6 @@ def test_nans_stats(data, ops, skipna): getattr(psr, ops)(skipna=skipna), getattr(gsr, ops)(skipna=skipna) ) - psr = _create_pandas_series(data) gsr = cudf.Series(data, nan_as_null=False) # Since there is no concept of `nan_as_null` in pandas, # nulls will be returned in the operations. So only diff --git a/python/cudf/cudf/utils/applyutils.py b/python/cudf/cudf/utils/applyutils.py index 7e998413642..933b98367b6 100644 --- a/python/cudf/cudf/utils/applyutils.py +++ b/python/cudf/cudf/utils/applyutils.py @@ -105,6 +105,7 @@ def apply_chunks( return applychunks.run(df, chunks=chunks, tpb=tpb) +@acquire_spill_lock() def make_aggregate_nullmask(df, columns=None, op="__and__"): out_mask = None @@ -112,17 +113,16 @@ def make_aggregate_nullmask(df, columns=None, op="__and__"): col = cudf.core.dataframe.extract_col(df, k) if not col.nullable: continue - nullmask = df[k].nullmask + nullmask = column.as_column(df[k]._column.nullmask) if out_mask is None: out_mask = column.as_column( nullmask.copy(), dtype=utils.mask_dtype ) - continue - - out_mask = libcudf.binaryop.binaryop( - column.as_column(nullmask), out_mask, op, out_mask.dtype - ) + else: + out_mask = libcudf.binaryop.binaryop( + nullmask, out_mask, op, out_mask.dtype + ) return out_mask From 7190e33fd7fcd1a2389668ae4cedf4f4ca8785ec Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Thu, 16 Feb 2023 14:43:59 -0500 Subject: [PATCH 30/69] Update `is_sorted` to use `experimental::row::lexicographic` (#12752) This PR is a part of https://github.com/rapidsai/cudf/issues/11844. Authors: - Divye Gala (https://github.com/divyegala) Approvers: - Nghia Truong (https://github.com/ttnghia) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/12752 --- cpp/src/sort/is_sorted.cu | 41 +++++++++------------ cpp/tests/sort/is_sorted_tests.cpp | 59 ++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 26 deletions(-) diff --git a/cpp/src/sort/is_sorted.cu b/cpp/src/sort/is_sorted.cu index 459dcf5467f..356c58b1c22 100644 --- a/cpp/src/sort/is_sorted.cu +++ b/cpp/src/sort/is_sorted.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include @@ -36,31 +36,27 @@ namespace detail { auto is_sorted(cudf::table_view const& in, std::vector const& column_order, - bool has_nulls, std::vector const& null_precedence, rmm::cuda_stream_view stream) { - // 0-table_view, 1-column_order, 2-null_precedence, 3-validity_columns - auto flattened = structs::detail::flatten_nested_columns(in, column_order, null_precedence); + auto const comparator = + experimental::row::lexicographic::self_comparator{in, column_order, null_precedence, stream}; - auto const d_input = table_device_view::create(flattened, stream); - auto const d_column_order = make_device_uvector_async(flattened.orders(), stream); - auto const d_null_precedence = has_nulls - ? make_device_uvector_async(flattened.null_orders(), stream) - : rmm::device_uvector(0, stream); + if (cudf::detail::has_nested_columns(in)) { + auto const device_comparator = comparator.less(has_nested_nulls(in)); - auto comparator = row_lexicographic_comparator(nullate::DYNAMIC{has_nulls}, - *d_input, - *d_input, - d_column_order.data(), - d_null_precedence.data()); + return thrust::is_sorted(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(in.num_rows()), + device_comparator); + } else { + auto const device_comparator = comparator.less(has_nested_nulls(in)); - auto sorted = thrust::is_sorted(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(in.num_rows()), - comparator); - - return sorted; + return thrust::is_sorted(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(in.num_rows()), + device_comparator); + } } } // namespace detail @@ -83,8 +79,7 @@ bool is_sorted(cudf::table_view const& in, "Number of columns in the table doesn't match the vector null_precedence's size .\n"); } - return detail::is_sorted( - in, column_order, has_nulls(in), null_precedence, cudf::get_default_stream()); + return detail::is_sorted(in, column_order, null_precedence, cudf::get_default_stream()); } } // namespace cudf diff --git a/cpp/tests/sort/is_sorted_tests.cpp b/cpp/tests/sort/is_sorted_tests.cpp index b47385d686c..eea8a21c67c 100644 --- a/cpp/tests/sort/is_sorted_tests.cpp +++ b/cpp/tests/sort/is_sorted_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -223,6 +224,58 @@ auto nulls_before() return cudf::test::structs_column_wrapper{{col1, col2}, {0, 1}}; } +using lcw = cudf::test::lists_column_wrapper; +using cudf::test::iterators::null_at; +/* +List +[ + [[[0]], [[0]], [[0]]], 0 + [[[0], [0], [0]]], 1 + [[[0, 0]], [[0, 0, 0, 0, 0, 0, 0, 0]], [[0]]] 2 + [[[0, 0, 0]]], 3 + [[[0, 0, 0]], [[0]], [[0]]], 4 +] +*/ + +template +std::enable_if_t, lcw> ascending() +{ + return lcw{lcw{lcw{lcw{0}}, lcw{lcw{0}}, lcw{lcw{0}}}, + lcw{lcw{lcw{0}, lcw{0}, lcw{0}}}, + lcw{lcw{lcw{0, 0}}, lcw{lcw{0, 0, 0, 0, 0, 0, 0, 0}}, lcw{lcw{0}}}, + lcw{lcw{lcw{0, 0, 0}}}, + lcw{lcw{lcw{0, 0, 0}}, lcw{lcw{0}}, lcw{lcw{0}}}}; +} + +template +std::enable_if_t, lcw> descending() +{ + return lcw{lcw{lcw{lcw{0, 0, 0}}, lcw{lcw{0}}, lcw{lcw{0}}}, + lcw{lcw{lcw{0, 0, 0}}}, + lcw{lcw{lcw{0, 0}}, lcw{lcw{0, 0, 0, 0, 0, 0, 0, 0}}, lcw{lcw{0}}}, + + lcw{lcw{lcw{0}, lcw{0}, lcw{0}}}, + lcw{lcw{lcw{0}}, lcw{lcw{0}}, lcw{lcw{0}}}}; +} + +template <> +auto empty() +{ + return lcw{}; +} + +template <> +auto nulls_after() +{ + return lcw{{{1}, {2, 2}, {0}}, null_at(2)}; +} + +template <> +auto nulls_before() +{ + return lcw{{{0}, {1}, {2, 2}}, null_at(0)}; +} + } // namespace testdata // ============================================================================= @@ -232,8 +285,8 @@ template struct IsSortedTest : public cudf::test::BaseFixture { }; -using SupportedTypes = - cudf::test::Concat>; +using SupportedTypes = cudf::test:: + Concat, cudf::test::ListTypes>; TYPED_TEST_SUITE(IsSortedTest, SupportedTypes); TYPED_TEST(IsSortedTest, NoColumns) From a00295a4db71c7c0c1396d2c0539eecee74a4ac5 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Fri, 17 Feb 2023 01:37:56 +0530 Subject: [PATCH 31/69] Fix bug in all-null list due to join_list_elements special handling (#12767) All-null list is considered as empty list in `join_list_elements`. So, nulls of children are replaced by `narep` before passing to `join_list_elements` API. Related https://github.com/rapidsai/cudf/issues/12766 Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12767 --- cpp/src/io/json/write_json.cu | 8 ++++++-- cpp/tests/io/json_writer.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/cpp/src/io/json/write_json.cu b/cpp/src/io/json/write_json.cu index 9849629015d..a7ae4d3bdd1 100644 --- a/cpp/src/io/json/write_json.cu +++ b/cpp/src/io/json/write_json.cu @@ -417,7 +417,9 @@ struct column_to_strings_fn { auto child_view = lists_column_view(column).get_sliced_child(stream_); auto constexpr child_index = lists_column_view::child_column_index; auto list_string = [&]() { - auto child_string = [&]() { + // nulls are replaced due to special handling of all-null lists as empty lists + // by join_list_elements + auto child_string_with_null = [&]() { if (child_view.type().id() == type_id::STRUCT) { return (*this).template operator()( child_view, @@ -431,7 +433,9 @@ struct column_to_strings_fn { } else { return cudf::type_dispatcher(child_view.type(), *this, child_view); } - }(); + }; + auto child_string = cudf::strings::detail::replace_nulls( + child_string_with_null()->view(), narep, stream_, rmm::mr::get_current_device_resource()); auto const list_child_string = column_view(column.type(), column.size(), diff --git a/cpp/tests/io/json_writer.cpp b/cpp/tests/io/json_writer.cpp index d129ed306e4..702315d6a97 100644 --- a/cpp/tests/io/json_writer.cpp +++ b/cpp/tests/io/json_writer.cpp @@ -362,4 +362,36 @@ TEST_F(JsonWriterTest, SpecialChars) EXPECT_EQ(expected, output_string); } +TEST_F(JsonWriterTest, NullList) +{ + std::string const data = R"( +{"a": [null], "b": [[1, 2, 3], [null], [null, null, null], [4, null, 5]]} +{"a": [2, null, null, 3] , "b": null} +{"a": [null, null, 4], "b": [[2, null], null]} +{"a": [5, null, null], "b": [null, [3, 4, 5]]} )"; + cudf::io::json_reader_options in_options = + cudf::io::json_reader_options::builder(cudf::io::source_info{data.data(), data.size()}) + .lines(true); + + cudf::io::table_with_metadata result = cudf::io::read_json(in_options); + cudf::table_view tbl_view = result.tbl->view(); + cudf::io::table_metadata mt{result.metadata}; + + std::vector out_buffer; + auto destination = cudf::io::sink_info(&out_buffer); + auto options_builder = cudf::io::json_writer_options_builder(destination, tbl_view) + .include_nulls(true) + .metadata(mt) + .lines(true) + .na_rep("null"); + + cudf::io::write_json(options_builder.build(), rmm::mr::get_current_device_resource()); + std::string const expected = R"({"a":[null],"b":[[1,2,3],[null],[null,null,null],[4,null,5]]} +{"a":[2,null,null,3],"b":null} +{"a":[null,null,4],"b":[[2,null],null]} +{"a":[5,null,null],"b":[null,[3,4,5]]} +)"; + EXPECT_EQ(expected, std::string(out_buffer.data(), out_buffer.size())); +} + CUDF_TEST_PROGRAM_MAIN() From cf82fb882270420eeb588416b4c761db8f8fa6e9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 16 Feb 2023 13:49:08 -0800 Subject: [PATCH 32/69] Tell cudf_kafka to use header-only fmt (#12796) The changes to spdlog/fmt packaging in rmm caused an undefined symbol issue in cudf_kafka. To unblock CI in https://github.com/rapidsai/cudf/pull/12783, I simply added fmt to the list of required libraries for the Python package (see https://github.com/rapidsai/cudf/pull/12783/commits/caf7adfb9a89e585bd4d3239c79643328449a242 and [the explanation](https://github.com/rapidsai/cudf/pull/12783#issuecomment-1431997720)). The underlying issue turns out to be that by default usage of fmt results in the dependent assuming that the headers are compiled in not header-only mode. When using CMake to manage the build, fmt relies on use of the appropriate target `fmt::fmt-header-only` to configure the build such that anything using fmt knows to use it in header-only mode, which sets the `FMT_HEADER_ONLY` preprocessor macro under the hood. Since the Python cudf_kafka package is not built using CMake, however, this information was not propagated to its build from its dependencies (libcudf/libcudf_kafka). As a result, it was compiled expecting an external definition of some fmt symbols rather than inlining them. This PR removes the undesirable library dependency on fmt introduced in https://github.com/rapidsai/cudf/pull/12783 in favor of properly telling fmt to expect inlined symbols. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) - Bradley Dice (https://github.com/bdice) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/12796 --- python/cudf_kafka/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index 499d1d4a84b..caadfcac8aa 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -79,9 +79,9 @@ CUDF_KAFKA_ROOT, ] ), - libraries=["cudf", "cudf_kafka", "fmt"], + libraries=["cudf", "cudf_kafka"], language="c++", - extra_compile_args=["-std=c++17"], + extra_compile_args=["-std=c++17", "-DFMT_HEADER_ONLY=1"], ) ] From 4e32bfe3aadf87716a91aed79048dec7dbe50e09 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Thu, 16 Feb 2023 17:55:04 -0500 Subject: [PATCH 33/69] Fix groupby gtests coded in namespace cudf::test (#12784) Fixes `GROUPBY_TEST` gtests source files coded in namespace `cudf::test` Additional creates a `groupby_test_util.cpp` for utilities rather than inlining the utilities into all 33 source files. The groupby tests for aggregations `std` and `var` are also not compiled for Debug builds. This changes those to compile but be runtime disabled to make them more noticeable in a Debug test run. Reference https://github.com/rapidsai/cudf/issues/11734 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/12784 --- cpp/tests/CMakeLists.txt | 1 + cpp/tests/groupby/argmax_tests.cpp | 194 ++++---- cpp/tests/groupby/argmin_tests.cpp | 190 ++++---- cpp/tests/groupby/collect_list_tests.cpp | 131 +++--- cpp/tests/groupby/collect_set_tests.cpp | 75 ++- cpp/tests/groupby/correlation_tests.cpp | 184 ++++---- cpp/tests/groupby/count_scan_tests.cpp | 124 +++-- cpp/tests/groupby/count_tests.cpp | 152 +++---- cpp/tests/groupby/covariance_tests.cpp | 199 ++++---- cpp/tests/groupby/groupby_test_util.cpp | 141 ++++++ cpp/tests/groupby/groupby_test_util.hpp | 179 ++------ cpp/tests/groupby/groups_tests.cpp | 84 ++-- cpp/tests/groupby/keys_tests.cpp | 246 +++++----- cpp/tests/groupby/lists_tests.cpp | 39 +- cpp/tests/groupby/max_scan_tests.cpp | 141 +++--- cpp/tests/groupby/max_tests.cpp | 256 ++++++----- cpp/tests/groupby/mean_tests.cpp | 109 +++-- cpp/tests/groupby/median_tests.cpp | 103 +++-- cpp/tests/groupby/min_scan_tests.cpp | 155 ++++--- cpp/tests/groupby/min_tests.cpp | 255 +++++------ cpp/tests/groupby/nth_element_tests.cpp | 295 ++++++------ cpp/tests/groupby/nunique_tests.cpp | 165 +++---- cpp/tests/groupby/product_tests.cpp | 97 ++-- cpp/tests/groupby/quantile_tests.cpp | 152 +++---- cpp/tests/groupby/rank_scan_tests.cpp | 315 +++++++------ cpp/tests/groupby/replace_nulls_tests.cpp | 220 ++++----- cpp/tests/groupby/shift_tests.cpp | 506 ++++++++++++--------- cpp/tests/groupby/std_tests.cpp | 132 +++--- cpp/tests/groupby/structs_tests.cpp | 134 +++--- cpp/tests/groupby/sum_of_squares_tests.cpp | 94 ++-- cpp/tests/groupby/sum_scan_tests.cpp | 67 ++- cpp/tests/groupby/sum_tests.cpp | 134 +++--- cpp/tests/groupby/tdigest_tests.cu | 242 +++++----- cpp/tests/groupby/var_tests.cpp | 143 +++--- cpp/tests/replace/replace_nulls_tests.cpp | 18 +- 35 files changed, 2899 insertions(+), 2773 deletions(-) create mode 100644 cpp/tests/groupby/groupby_test_util.cpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 83a1c14438b..13cb1739ae9 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -84,6 +84,7 @@ ConfigureTest( groupby/count_scan_tests.cpp groupby/count_tests.cpp groupby/covariance_tests.cpp + groupby/groupby_test_util.cpp groupby/groups_tests.cpp groupby/keys_tests.cpp groupby/lists_tests.cpp diff --git a/cpp/tests/groupby/argmax_tests.cpp b/cpp/tests/groupby/argmax_tests.cpp index 0b06c184b75..4796482e7c0 100644 --- a/cpp/tests/groupby/argmax_tests.cpp +++ b/cpp/tests/groupby/argmax_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,6 @@ #include -using namespace cudf::test::iterators; - -namespace cudf { -namespace test { template struct groupby_argmax_test : public cudf::test::BaseFixture { }; @@ -37,84 +33,85 @@ TYPED_TEST_SUITE(groupby_argmax_test, cudf::test::FixedWidthTypes); TYPED_TEST(groupby_argmax_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{0, 1, 2}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{0, 1, 2}; - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmax_aggregation(); + auto agg2 = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_argmax_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals({3, 4, 5}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}); - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmax_aggregation(); + auto agg2 = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_argmax_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, cudf::test::iterators::all_nulls()); - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmax_aggregation(); + auto agg2 = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_argmax_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper vals({9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 4}, - {0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 4}, + {0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0}); // {1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, + cudf::test::iterators::no_nulls()); // {6, 3, 5, 4, 0, 2, 1, -} - fixed_width_column_wrapper expect_vals({3, 4, 7, 0}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({3, 4, 7, 0}, {1, 1, 1, 0}); - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmax_aggregation(); + auto agg2 = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } @@ -123,37 +120,36 @@ struct groupby_argmax_string_test : public cudf::test::BaseFixture { TEST_F(groupby_argmax_string_test, basic) { - using V = string_view; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::strings_column_wrapper vals{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals({0, 4, 2}); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals({0, 4, 2}); - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmax_aggregation(); + auto agg2 = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TEST_F(groupby_argmax_string_test, zero_valid_values) { - using V = string_view; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::strings_column_wrapper vals({"año", "bit", "₹1"}, cudf::test::iterators::all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, cudf::test::iterators::all_nulls()); - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmax_aggregation(); + auto agg2 = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } @@ -163,22 +159,25 @@ struct groupby_dictionary_argmax_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_argmax_test, basic) { using V = std::string; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; - dictionary_column_wrapper vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); - fixed_width_column_wrapper expect_vals({ 0, 4, 2 }); + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; + cudf::test::dictionary_column_wrapper vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); + cudf::test::fixed_width_column_wrapper expect_vals({ 0, 4, 2 }); // clang-format on - test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_argmax_aggregation()); test_single_agg(keys, vals, expect_keys, expect_vals, - cudf::make_argmax_aggregation(), + cudf::make_argmax_aggregation()); + test_single_agg(keys, + vals, + expect_keys, + expect_vals, + cudf::make_argmax_aggregation(), force_use_sort_impl::YES); } @@ -187,74 +186,75 @@ struct groupby_argmax_struct_test : public cudf::test::BaseFixture { TEST_F(groupby_argmax_struct_test, basic) { - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = [] { - auto child1 = - strings_column_wrapper{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - auto child2 = fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; - auto const expect_indices = fixed_width_column_wrapper{0, 4, 2}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const expect_indices = cudf::test::fixed_width_column_wrapper{0, 4, 2}; - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_indices, std::move(agg)); } TEST_F(groupby_argmax_struct_test, slice_input) { constexpr int32_t dont_care{1}; - auto const keys_original = fixed_width_column_wrapper{ + auto const keys_original = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, dont_care}; auto const vals_original = [] { - auto child1 = strings_column_wrapper{"dont_care", - "dont_care", - "año", - "bit", - "₹1", - "aaa", - "zit", - "bat", - "aab", - "$1", - "€1", - "wut", - "dont_care"}; - auto child2 = fixed_width_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{"dont_care", + "dont_care", + "año", + "bit", + "₹1", + "aaa", + "zit", + "bat", + "aab", + "$1", + "€1", + "wut", + "dont_care"}; + auto child2 = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, dont_care}; - return structs_column_wrapper{{child1, child2}}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const keys = cudf::slice(keys_original, {2, 12})[0]; auto const vals = cudf::slice(vals_original, {2, 12})[0]; - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; - auto const expect_indices = fixed_width_column_wrapper{0, 4, 2}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const expect_indices = cudf::test::fixed_width_column_wrapper{0, 4, 2}; - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_indices, std::move(agg)); } TEST_F(groupby_argmax_struct_test, null_keys_and_values) { constexpr int32_t null{0}; - auto const keys = - fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; + auto const keys = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, cudf::test::iterators::null_at(7)}; auto const vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "bit", "₹1", "aaa", "zit", "" /*NULL*/, "" /*NULL*/, "$1", "€1", "wut", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, + cudf::test::iterators::nulls_at({5, 6, 10})}; }(); - auto const expect_keys = fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; - auto const expect_indices = fixed_width_column_wrapper{{0, 4, 2, null}, null_at(3)}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 4}, cudf::test::iterators::no_nulls()}; + auto const expect_indices = cudf::test::fixed_width_column_wrapper{ + {0, 4, 2, null}, cudf::test::iterators::null_at(3)}; - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_indices, std::move(agg)); } -} // namespace test -} // namespace cudf - CUDF_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/groupby/argmin_tests.cpp b/cpp/tests/groupby/argmin_tests.cpp index 67235a64066..3476e764d6a 100644 --- a/cpp/tests/groupby/argmin_tests.cpp +++ b/cpp/tests/groupby/argmin_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_argmin_test : public cudf::test::BaseFixture { }; @@ -37,85 +35,85 @@ TYPED_TEST_SUITE(groupby_argmin_test, cudf::test::FixedWidthTypes); TYPED_TEST(groupby_argmin_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{6, 9, 8}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{6, 9, 8}; - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmin_aggregation(); + auto agg2 = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_argmin_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals({3, 4, 5}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}); - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmin_aggregation(); + auto agg2 = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_argmin_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmin_aggregation(); + auto agg2 = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_argmin_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; if (std::is_same_v) return; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 4}, - {1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 4}, + {1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 9, 6, 8, 5, 0, 7, 1, -} - fixed_width_column_wrapper expect_vals({3, 9, 8, 0}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({3, 9, 8, 0}, {1, 1, 1, 0}); - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); // TODO: explore making this a gtest parameter - auto agg2 = cudf::make_argmin_aggregation(); + auto agg2 = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } @@ -124,37 +122,36 @@ struct groupby_argmin_string_test : public cudf::test::BaseFixture { TEST_F(groupby_argmin_string_test, basic) { - using V = string_view; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::strings_column_wrapper vals{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals({3, 5, 7}); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals({3, 5, 7}); - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmin_aggregation(); + auto agg2 = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TEST_F(groupby_argmin_string_test, zero_valid_values) { - using V = string_view; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_argmin_aggregation(); + auto agg2 = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } @@ -164,22 +161,25 @@ struct groupby_dictionary_argmin_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_argmin_test, basic) { using V = std::string; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; - dictionary_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); - fixed_width_column_wrapper expect_vals({ 3, 5, 7 }); + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; + cudf::test::dictionary_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); + cudf::test::fixed_width_column_wrapper expect_vals({ 3, 5, 7 }); // clang-format on - test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_argmin_aggregation()); test_single_agg(keys, vals, expect_keys, expect_vals, - cudf::make_argmin_aggregation(), + cudf::make_argmin_aggregation()); + test_single_agg(keys, + vals, + expect_keys, + expect_vals, + cudf::make_argmin_aggregation(), force_use_sort_impl::YES); } @@ -188,72 +188,72 @@ struct groupby_argmin_struct_test : public cudf::test::BaseFixture { TEST_F(groupby_argmin_struct_test, basic) { - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = [] { - auto child1 = - strings_column_wrapper{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - auto child2 = fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; - auto const expect_indices = fixed_width_column_wrapper{3, 5, 7}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const expect_indices = cudf::test::fixed_width_column_wrapper{3, 5, 7}; - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_indices, std::move(agg)); } TEST_F(groupby_argmin_struct_test, slice_input) { constexpr int32_t dont_care{1}; - auto const keys_original = fixed_width_column_wrapper{ + auto const keys_original = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, dont_care}; auto const vals_original = [] { - auto child1 = strings_column_wrapper{"dont_care", - "dont_care", - "año", - "bit", - "₹1", - "aaa", - "zit", - "bat", - "aab", - "$1", - "€1", - "wut", - "dont_care"}; - auto child2 = fixed_width_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{"dont_care", + "dont_care", + "año", + "bit", + "₹1", + "aaa", + "zit", + "bat", + "aab", + "$1", + "€1", + "wut", + "dont_care"}; + auto child2 = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, dont_care}; - return structs_column_wrapper{{child1, child2}}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const keys = cudf::slice(keys_original, {2, 12})[0]; auto const vals = cudf::slice(vals_original, {2, 12})[0]; - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; - auto const expect_indices = fixed_width_column_wrapper{3, 5, 7}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const expect_indices = cudf::test::fixed_width_column_wrapper{3, 5, 7}; - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_indices, std::move(agg)); } TEST_F(groupby_argmin_struct_test, null_keys_and_values) { constexpr int32_t null{0}; - auto const keys = - fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; + auto const keys = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; auto const vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "bit", "₹1", "aaa", "zit", "" /*NULL*/, "" /*NULL*/, "$1", "€1", "wut", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; }(); - auto const expect_keys = fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; - auto const expect_indices = fixed_width_column_wrapper{{3, 1, 8, null}, null_at(3)}; + auto const expect_keys = + cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; + auto const expect_indices = + cudf::test::fixed_width_column_wrapper{{3, 1, 8, null}, null_at(3)}; - auto agg = cudf::make_argmin_aggregation(); + auto agg = cudf::make_argmin_aggregation(); test_single_agg(keys, vals, expect_keys, expect_indices, std::move(agg)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/collect_list_tests.cpp b/cpp/tests/groupby/collect_list_tests.cpp index 8a724526dbf..4e4e6dab250 100644 --- a/cpp/tests/groupby/collect_list_tests.cpp +++ b/cpp/tests/groupby/collect_list_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,9 +22,6 @@ #include -namespace cudf { -namespace test { - template struct groupby_collect_list_test : public cudf::test::BaseFixture { }; @@ -39,13 +36,13 @@ TYPED_TEST(groupby_collect_list_test, CollectWithoutNulls) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys{1, 1, 1, 2, 2, 2}; - fixed_width_column_wrapper values{1, 2, 3, 4, 5, 6}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 1, 2, 2, 2}; + cudf::test::fixed_width_column_wrapper values{1, 2, 3, 4, 5, 6}; - fixed_width_column_wrapper expect_keys{1, 2}; - lists_column_wrapper expect_vals{{1, 2, 3}, {4, 5, 6}}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2}; + cudf::test::lists_column_wrapper expect_vals{{1, 2, 3}, {4, 5, 6}}; - auto agg = cudf::make_collect_list_aggregation(); + auto agg = cudf::make_collect_list_aggregation(); test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg)); } @@ -54,17 +51,17 @@ TYPED_TEST(groupby_collect_list_test, CollectWithNulls) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys{1, 1, 2, 2, 3, 3}; - fixed_width_column_wrapper values{{1, 2, 3, 4, 5, 6}, - {true, false, true, false, true, false}}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 2, 2, 3, 3}; + cudf::test::fixed_width_column_wrapper values{ + {1, 2, 3, 4, 5, 6}, {true, false, true, false, true, false}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; std::vector validity({true, false}); - lists_column_wrapper expect_vals{ + cudf::test::lists_column_wrapper expect_vals{ {{1, 2}, validity.begin()}, {{3, 4}, validity.begin()}, {{5, 6}, validity.begin()}}; - auto agg = cudf::make_collect_list_aggregation(); + auto agg = cudf::make_collect_list_aggregation(); test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg)); } @@ -73,16 +70,17 @@ TYPED_TEST(groupby_collect_list_test, CollectWithNullExclusion) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys{1, 1, 1, 2, 2, 3, 3, 4, 4}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 1, 2, 2, 3, 3, 4, 4}; - fixed_width_column_wrapper values{ + cudf::test::fixed_width_column_wrapper values{ {1, 2, 3, 4, 5, 6, 7, 8, 9}, {false, true, false, true, false, false, false, true, true}}; - fixed_width_column_wrapper expect_keys{1, 2, 3, 4}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3, 4}; - lists_column_wrapper expect_vals{{2}, {4}, {}, {8, 9}}; + cudf::test::lists_column_wrapper expect_vals{{2}, {4}, {}, {8, 9}}; - auto agg = cudf::make_collect_list_aggregation(null_policy::EXCLUDE); + auto agg = + cudf::make_collect_list_aggregation(cudf::null_policy::EXCLUDE); test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg)); } @@ -91,13 +89,14 @@ TYPED_TEST(groupby_collect_list_test, CollectOnEmptyInput) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper values{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper values{}; - fixed_width_column_wrapper expect_keys{}; - lists_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::lists_column_wrapper expect_vals{}; - auto agg = cudf::make_collect_list_aggregation(null_policy::EXCLUDE); + auto agg = + cudf::make_collect_list_aggregation(cudf::null_policy::EXCLUDE); test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg)); } @@ -108,15 +107,16 @@ TYPED_TEST(groupby_collect_list_test, CollectLists) using LCW = cudf::test::lists_column_wrapper; - fixed_width_column_wrapper keys{1, 1, 2, 2, 3, 3}; - lists_column_wrapper values{{1, 2}, {3, 4}, {5, 6, 7}, LCW{}, {9, 10}, {11}}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 2, 2, 3, 3}; + cudf::test::lists_column_wrapper values{ + {1, 2}, {3, 4}, {5, 6, 7}, LCW{}, {9, 10}, {11}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; - lists_column_wrapper expect_vals{ + cudf::test::lists_column_wrapper expect_vals{ {{1, 2}, {3, 4}}, {{5, 6, 7}, LCW{}}, {{9, 10}, {11}}}; - auto agg = cudf::make_collect_list_aggregation(); + auto agg = cudf::make_collect_list_aggregation(); test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg)); } @@ -127,15 +127,16 @@ TYPED_TEST(groupby_collect_list_test, CollectListsWithNullExclusion) using LCW = cudf::test::lists_column_wrapper; - fixed_width_column_wrapper keys{1, 1, 2, 2, 3, 3, 4, 4}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 2, 2, 3, 3, 4, 4}; const bool validity_mask[] = {true, false, false, true, true, true, false, false}; LCW values{{{1, 2}, {3, 4}, {5, 6, 7}, LCW{}, {9, 10}, {11}, {20, 30, 40}, LCW{}}, validity_mask}; - fixed_width_column_wrapper expect_keys{1, 2, 3, 4}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3, 4}; LCW expect_vals{{{1, 2}}, {LCW{}}, {{9, 10}, {11}}, {}}; - auto agg = cudf::make_collect_list_aggregation(null_policy::EXCLUDE); + auto agg = + cudf::make_collect_list_aggregation(cudf::null_policy::EXCLUDE); test_single_agg(keys, values, expect_keys, expect_vals, std::move(agg)); } @@ -146,19 +147,20 @@ TYPED_TEST(groupby_collect_list_test, CollectOnEmptyInputLists) using LCW = cudf::test::lists_column_wrapper; - auto offsets = data_type{type_to_id()}; + auto offsets = cudf::data_type{cudf::type_to_id()}; - fixed_width_column_wrapper keys{}; - auto values = cudf::make_lists_column(0, make_empty_column(offsets), LCW{}.release(), 0, {}); + cudf::test::fixed_width_column_wrapper keys{}; + auto values = + cudf::make_lists_column(0, cudf::make_empty_column(offsets), LCW{}.release(), 0, {}); - fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; auto expect_child = - cudf::make_lists_column(0, make_empty_column(offsets), LCW{}.release(), 0, {}); + cudf::make_lists_column(0, cudf::make_empty_column(offsets), LCW{}.release(), 0, {}); auto expect_values = - cudf::make_lists_column(0, make_empty_column(offsets), std::move(expect_child), 0, {}); + cudf::make_lists_column(0, cudf::make_empty_column(offsets), std::move(expect_child), 0, {}); - auto agg = cudf::make_collect_list_aggregation(); + auto agg = cudf::make_collect_list_aggregation(); test_single_agg(keys, values->view(), expect_keys, expect_values->view(), std::move(agg)); } @@ -169,24 +171,36 @@ TYPED_TEST(groupby_collect_list_test, CollectOnEmptyInputListsOfStructs) using LCW = cudf::test::lists_column_wrapper; - fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper keys{}; auto struct_child = LCW{}; - auto struct_column = structs_column_wrapper{{struct_child}}; + auto struct_column = cudf::test::structs_column_wrapper{{struct_child}}; - auto values = cudf::make_lists_column( - 0, make_empty_column(type_to_id()), struct_column.release(), 0, {}); + auto values = + cudf::make_lists_column(0, + cudf::make_empty_column(cudf::type_to_id()), + struct_column.release(), + 0, + {}); - fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; auto expect_struct_child = LCW{}; - auto expect_struct_column = structs_column_wrapper{{expect_struct_child}}; + auto expect_struct_column = cudf::test::structs_column_wrapper{{expect_struct_child}}; - auto expect_child = cudf::make_lists_column( - 0, make_empty_column(type_to_id()), expect_struct_column.release(), 0, {}); - auto expect_values = cudf::make_lists_column( - 0, make_empty_column(type_to_id()), std::move(expect_child), 0, {}); + auto expect_child = + cudf::make_lists_column(0, + cudf::make_empty_column(cudf::type_to_id()), + expect_struct_column.release(), + 0, + {}); + auto expect_values = + cudf::make_lists_column(0, + cudf::make_empty_column(cudf::type_to_id()), + std::move(expect_child), + 0, + {}); - auto agg = cudf::make_collect_list_aggregation(); + auto agg = cudf::make_collect_list_aggregation(); test_single_agg(keys, values->view(), expect_keys, expect_values->view(), std::move(agg)); } @@ -195,13 +209,13 @@ TYPED_TEST(groupby_collect_list_test, dictionary) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys{1, 1, 1, 2, 2, 2}; - dictionary_column_wrapper vals{1, 2, 3, 4, 5, 6}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 1, 2, 2, 2}; + cudf::test::dictionary_column_wrapper vals{1, 2, 3, 4, 5, 6}; - fixed_width_column_wrapper expect_keys{1, 2}; - lists_column_wrapper expect_vals_w{{1, 2, 3}, {4, 5, 6}}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2}; + cudf::test::lists_column_wrapper expect_vals_w{{1, 2, 3}, {4, 5, 6}}; - fixed_width_column_wrapper offsets({0, 3, 6}); + cudf::test::fixed_width_column_wrapper offsets({0, 3, 6}); auto expect_vals = cudf::make_lists_column(cudf::column_view(offsets).size() - 1, std::make_unique(offsets), std::make_unique(vals), @@ -212,8 +226,5 @@ TYPED_TEST(groupby_collect_list_test, dictionary) vals, expect_keys, expect_vals->view(), - cudf::make_collect_list_aggregation()); + cudf::make_collect_list_aggregation()); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/collect_set_tests.cpp b/cpp/tests/groupby/collect_set_tests.cpp index 818a4c63a1f..89a88560a2d 100644 --- a/cpp/tests/groupby/collect_set_tests.cpp +++ b/cpp/tests/groupby/collect_set_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,9 +25,6 @@ #include #include -namespace cudf { -namespace test { - namespace { constexpr cudf::test::debug_output_level verbosity{cudf::test::debug_output_level::FIRST_ERROR}; @@ -39,7 +36,7 @@ using validity_col = std::initializer_list; auto groupby_collect_set(cudf::column_view const& keys, cudf::column_view const& values, - std::unique_ptr&& agg) + std::unique_ptr&& agg) { std::vector requests; requests.emplace_back(cudf::groupby::aggregation_request()); @@ -78,13 +75,14 @@ struct CollectSetTest : public cudf::test::BaseFixture { static auto collect_set_null_unequal() { - return cudf::make_collect_set_aggregation(null_policy::INCLUDE, - null_equality::UNEQUAL); + return cudf::make_collect_set_aggregation( + cudf::null_policy::INCLUDE, cudf::null_equality::UNEQUAL); } static auto collect_set_null_exclude() { - return cudf::make_collect_set_aggregation(null_policy::EXCLUDE); + return cudf::make_collect_set_aggregation( + cudf::null_policy::EXCLUDE); } }; @@ -245,38 +243,38 @@ TEST_F(CollectSetTest, FloatsWithNaN) // null equal with nan unequal { - vals_expected = {{{-2.3e-5f, 1.0f, 2.3e5f, -NAN, -NAN, NAN, NAN, 0.0f}, + vals_expected = {{{-2.3e-5f, 1.0f, 2.3e5f, -NAN, -NAN, NAN, NAN, 0.0f}, validity_col{true, true, true, true, true, true, true, false}}}; - auto const [out_keys, out_lists] = - groupby_collect_set(keys, - vals, - cudf::make_collect_set_aggregation( - null_policy::INCLUDE, null_equality::EQUAL, nan_equality::UNEQUAL)); + auto const [out_keys, out_lists] = groupby_collect_set( + keys, + vals, + cudf::make_collect_set_aggregation( + cudf::null_policy::INCLUDE, cudf::null_equality::EQUAL, cudf::nan_equality::UNEQUAL)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(keys_expected, *out_keys, verbosity); CUDF_TEST_EXPECT_COLUMNS_EQUAL(vals_expected, *out_lists, verbosity); } // null unequal with nan unequal { - vals_expected = {{{-2.3e-5f, 1.0f, 2.3e5f, -NAN, -NAN, NAN, NAN, 0.0f, 0.0f}, + vals_expected = {{{-2.3e-5f, 1.0f, 2.3e5f, -NAN, -NAN, NAN, NAN, 0.0f, 0.0f}, validity_col{true, true, true, true, true, true, true, false, false}}}; - auto const [out_keys, out_lists] = - groupby_collect_set(keys, - vals, - cudf::make_collect_set_aggregation( - null_policy::INCLUDE, null_equality::UNEQUAL, nan_equality::UNEQUAL)); + auto const [out_keys, out_lists] = groupby_collect_set( + keys, + vals, + cudf::make_collect_set_aggregation( + cudf::null_policy::INCLUDE, cudf::null_equality::UNEQUAL, cudf::nan_equality::UNEQUAL)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(keys_expected, *out_keys, verbosity); CUDF_TEST_EXPECT_COLUMNS_EQUAL(vals_expected, *out_lists, verbosity); } // null exclude with nan unequal { - vals_expected = {{-2.3e-5f, 1.0f, 2.3e5f, -NAN, -NAN, NAN, NAN}}; - auto const [out_keys, out_lists] = - groupby_collect_set(keys, - vals, - cudf::make_collect_set_aggregation( - null_policy::EXCLUDE, null_equality::EQUAL, nan_equality::UNEQUAL)); + vals_expected = {{-2.3e-5f, 1.0f, 2.3e5f, -NAN, -NAN, NAN, NAN}}; + auto const [out_keys, out_lists] = groupby_collect_set( + keys, + vals, + cudf::make_collect_set_aggregation( + cudf::null_policy::EXCLUDE, cudf::null_equality::EQUAL, cudf::nan_equality::UNEQUAL)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(keys_expected, *out_keys, verbosity); CUDF_TEST_EXPECT_COLUMNS_EQUAL(vals_expected, *out_lists, verbosity); } @@ -285,24 +283,24 @@ TEST_F(CollectSetTest, FloatsWithNaN) { vals_expected = { {{-2.3e-5f, 1.0f, 2.3e5f, NAN, 0.0f}, validity_col{true, true, true, true, false}}}; - auto const [out_keys, out_lists] = - groupby_collect_set(keys, - vals, - cudf::make_collect_set_aggregation( - null_policy::INCLUDE, null_equality::EQUAL, nan_equality::ALL_EQUAL)); + auto const [out_keys, out_lists] = groupby_collect_set( + keys, + vals, + cudf::make_collect_set_aggregation( + cudf::null_policy::INCLUDE, cudf::null_equality::EQUAL, cudf::nan_equality::ALL_EQUAL)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(keys_expected, *out_keys, verbosity); CUDF_TEST_EXPECT_COLUMNS_EQUAL(vals_expected, *out_lists, verbosity); } // null unequal with nan equal { - vals_expected = {{{-2.3e-5f, 1.0f, 2.3e5f, -NAN, 0.0f, 0.0f}, + vals_expected = {{{-2.3e-5f, 1.0f, 2.3e5f, -NAN, 0.0f, 0.0f}, validity_col{true, true, true, true, false, false}}}; - auto const [out_keys, out_lists] = - groupby_collect_set(keys, - vals, - cudf::make_collect_set_aggregation( - null_policy::INCLUDE, null_equality::UNEQUAL, nan_equality::ALL_EQUAL)); + auto const [out_keys, out_lists] = groupby_collect_set( + keys, + vals, + cudf::make_collect_set_aggregation( + cudf::null_policy::INCLUDE, cudf::null_equality::UNEQUAL, cudf::nan_equality::ALL_EQUAL)); CUDF_TEST_EXPECT_COLUMNS_EQUAL(keys_expected, *out_keys, verbosity); CUDF_TEST_EXPECT_COLUMNS_EQUAL(vals_expected, *out_lists, verbosity); } @@ -400,6 +398,3 @@ TYPED_TEST(CollectSetTypedTest, CollectWithNulls) } } } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/correlation_tests.cpp b/cpp/tests/groupby/correlation_tests.cpp index 44db3eb859f..07407fac662 100644 --- a/cpp/tests/groupby/correlation_tests.cpp +++ b/cpp/tests/groupby/correlation_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ * limitations under the License. */ -#include -#include +#include #include #include @@ -23,23 +22,19 @@ #include #include -#include +#include #include #include using namespace cudf::test::iterators; -namespace cudf { -namespace test { - -constexpr auto nan = std::numeric_limits::quiet_NaN(); -using structs = structs_column_wrapper; template struct groupby_correlation_test : public cudf::test::BaseFixture { }; -using supported_types = RemoveIf>, cudf::test::NumericTypes>; +using supported_types = + cudf::test::RemoveIf>, cudf::test::NumericTypes>; TYPED_TEST_SUITE(groupby_correlation_test, supported_types); using K = int32_t; @@ -47,115 +42,121 @@ using K = int32_t; TYPED_TEST(groupby_correlation_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; + + constexpr auto nan = std::numeric_limits::quiet_NaN(); - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{{1.0, 0.6, nan}}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{{1.0, 0.6, nan}}; auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_correlation_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper member_0{}, member_1{}; - auto vals = structs{{member_0, member_1}}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper member_0{}, member_1{}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_correlation_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper member_0{3, 4, 5}, member_1{6, 7, 8}; - auto vals = structs{{member_0, member_1}}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper member_0{3, 4, 5}, member_1{6, 7, 8}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_correlation_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper member_0({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper member_1({3, 4, 5}, all_nulls()); - auto vals = structs{{member_0, member_1}}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper member_0({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper member_1({3, 4, 5}, all_nulls()); + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_correlation_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; + + constexpr auto nan = std::numeric_limits::quiet_NaN(); // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, + cudf::test::fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}); + cudf::test::fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}); // clang-format on - auto vals = structs{{val0, val1}}; + auto vals = cudf::test::structs_column_wrapper{{val0, val1}}; - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - fixed_width_column_wrapper expect_vals({1.0, 0.6, nan, 0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({1.0, 0.6, nan, 0.}, {1, 1, 1, 0}); auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_correlation_test, null_values_same) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; + + constexpr auto nan = std::numeric_limits::quiet_NaN(); // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, + cudf::test::fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); - fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}, + cudf::test::fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); // clang-format on - auto vals = structs{{val0, val1}}; + auto vals = cudf::test::structs_column_wrapper{{val0, val1}}; - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - fixed_width_column_wrapper expect_vals({1.0, 0.6, nan, 0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({1.0, 0.6, nan, 0.}, {1, 1, 1, 0}); auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } @@ -166,51 +167,55 @@ TYPED_TEST(groupby_correlation_test, null_values_same) TYPED_TEST(groupby_correlation_test, null_values_different) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; + + constexpr auto nan = std::numeric_limits::quiet_NaN(); // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, + cudf::test::fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, {0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper val1({1, 2, 1, 2,-1, 6, 3,-1, 0, 1, 2}, + cudf::test::fixed_width_column_wrapper val1({1, 2, 1, 2,-1, 6, 3,-1, 0, 1, 2}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); // clang-format on - auto vals = structs{{val0, val1}}; + auto vals = cudf::test::structs_column_wrapper{{val0, val1}}; - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - fixed_width_column_wrapper expect_vals({1.0, 0., nan, 0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({1.0, 0., nan, 0.}, {1, 1, 1, 0}); auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_correlation_test, min_periods) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + constexpr auto nan = std::numeric_limits::quiet_NaN(); - fixed_width_column_wrapper expect_keys{1, 2, 3}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_vals1{{1.0, 0.6, nan}}; - auto agg1 = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON, 3); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + + cudf::test::fixed_width_column_wrapper expect_vals1{{1.0, 0.6, nan}}; + auto agg1 = cudf::make_correlation_aggregation( + cudf::correlation_type::PEARSON, 3); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg1), force_use_sort_impl::YES); - fixed_width_column_wrapper expect_vals2{{1.0, 0.6, nan}, {0, 1, 0}}; - auto agg2 = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON, 4); + cudf::test::fixed_width_column_wrapper expect_vals2{{1.0, 0.6, nan}, {0, 1, 0}}; + auto agg2 = cudf::make_correlation_aggregation( + cudf::correlation_type::PEARSON, 4); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg2), force_use_sort_impl::YES); - fixed_width_column_wrapper expect_vals3{{1.0, 0.6, nan}, {0, 0, 0}}; - auto agg3 = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON, 5); + cudf::test::fixed_width_column_wrapper expect_vals3{{1.0, 0.6, nan}, {0, 0, 0}}; + auto agg3 = cudf::make_correlation_aggregation( + cudf::correlation_type::PEARSON, 5); test_single_agg(keys, vals, expect_keys, expect_vals3, std::move(agg3), force_use_sort_impl::YES); } @@ -220,20 +225,19 @@ struct groupby_dictionary_correlation_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_correlation_test, basic) { using V = int16_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; + + constexpr auto nan = std::numeric_limits::quiet_NaN(); - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = dictionary_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = dictionary_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::dictionary_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::dictionary_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{{1.0, 0.6, nan}}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{{1.0, 0.6, nan}}; auto agg = - cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); + cudf::make_correlation_aggregation(cudf::correlation_type::PEARSON); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/count_scan_tests.cpp b/cpp/tests/groupby/count_scan_tests.cpp index 54df690d307..e17d2466657 100644 --- a/cpp/tests/groupby/count_scan_tests.cpp +++ b/cpp/tests/groupby/count_scan_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,19 +23,14 @@ #include -using namespace cudf::test::iterators; - -namespace cudf { -namespace test { -using K = int32_t; -using key_wrapper = fixed_width_column_wrapper; +using key_wrapper = cudf::test::fixed_width_column_wrapper; template struct groupby_count_scan_test : public cudf::test::BaseFixture { using V = T; - using R = cudf::detail::target_type_t; - using value_wrapper = fixed_width_column_wrapper; - using result_wrapper = fixed_width_column_wrapper; + using R = cudf::detail::target_type_t; + using value_wrapper = cudf::test::fixed_width_column_wrapper; + using result_wrapper = cudf::test::fixed_width_column_wrapper; }; TYPED_TEST_SUITE(groupby_count_scan_test, cudf::test::AllTypes); @@ -53,12 +48,13 @@ TYPED_TEST(groupby_count_scan_test, basic) result_wrapper expect_vals{0, 1, 2, 0, 1, 2, 3, 0, 1, 2}; // clang-format on - // Count groupby aggregation is only supported with null_policy::EXCLUDE - auto agg1 = cudf::make_count_aggregation(); + // Count groupby aggregation is only supported with cudf::null_policy::EXCLUDE + auto agg1 = cudf::make_count_aggregation(); EXPECT_THROW(test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg1)), cudf::logic_error); - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -67,18 +63,16 @@ TYPED_TEST(groupby_count_scan_test, empty_cols) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off key_wrapper keys; value_wrapper vals; - key_wrapper expect_keys; result_wrapper expect_vals; - // clang-format on - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); EXPECT_NO_THROW(test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg1))); - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -87,15 +81,13 @@ TYPED_TEST(groupby_count_scan_test, zero_valid_keys) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys( {1, 2, 3}, all_nulls()); + key_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); value_wrapper vals{3, 4, 5}; - key_wrapper expect_keys{}; result_wrapper expect_vals{}; - // clang-format on - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -104,15 +96,13 @@ TYPED_TEST(groupby_count_scan_test, zero_valid_values) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys {1, 1, 1}; - value_wrapper vals({3, 4, 5}, all_nulls()); - + key_wrapper keys{1, 1, 1}; + value_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); key_wrapper expect_keys{1, 1, 1}; result_wrapper expect_vals{0, 1, 2}; - // clang-format on - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -126,12 +116,13 @@ TYPED_TEST(groupby_count_scan_test, null_keys_and_values) value_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // {1, 1, 1, 2, 2, 2, 2, 3, _, 3, 4} - key_wrapper expect_keys( {1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, no_nulls()); + key_wrapper expect_keys( {1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, cudf::test::iterators::no_nulls()); // {0, 3, 6, 1, 4, _, 9, 2, 7, 8, -} result_wrapper expect_vals{0, 1, 2, 0, 1, 2, 3, 0, 1, 0}; // clang-format on - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -141,56 +132,54 @@ struct groupby_count_scan_string_test : public cudf::test::BaseFixture { TEST_F(groupby_count_scan_string_test, basic) { using V = cudf::string_view; - using R = cudf::detail::target_type_t; - using result_wrapper = fixed_width_column_wrapper; + using R = cudf::detail::target_type_t; + using result_wrapper = cudf::test::fixed_width_column_wrapper; // clang-format off - key_wrapper keys { 1, 3, 3, 5, 5, 0}; - strings_column_wrapper vals{"1", "1", "1", "1", "1", "1"}; - + key_wrapper keys { 1, 3, 3, 5, 5, 0}; + cudf::test::strings_column_wrapper vals{"1", "1", "1", "1", "1", "1"}; key_wrapper expect_keys {0, 1, 3, 3, 5, 5}; result_wrapper expect_vals{0, 0, 0, 1, 0, 1}; // clang-format on - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupByCountScanFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupByCountScanFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupByCountScan) +TYPED_TEST(GroupByCountScanFixedPointTest, GroupByCountScan) { using namespace numeric; using decimalXX = TypeParam; using RepType = cudf::device_storage_type_t; - using fp_wrapper = fixed_point_column_wrapper; + using fp_wrapper = cudf::test::fixed_point_column_wrapper; using V = decimalXX; - using R = cudf::detail::target_type_t; - using result_wrapper = fixed_width_column_wrapper; - - auto const scale = scale_type{-1}; - // clang-format off - auto const keys = key_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + using R = cudf::detail::target_type_t; + using result_wrapper = cudf::test::fixed_width_column_wrapper; + auto const scale = scale_type{-1}; + auto const keys = key_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; auto const expect_keys = key_wrapper{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; auto const expect_vals = result_wrapper{0, 1, 2, 0, 1, 2, 3, 0, 1, 2}; - // clang-format on - // Count groupby aggregation is only supported with null_policy::EXCLUDE + // Count groupby aggregation is only supported with cudf::null_policy::EXCLUDE EXPECT_THROW(test_single_scan(keys, vals, expect_keys, expect_vals, - cudf::make_count_aggregation()), + cudf::make_count_aggregation()), cudf::logic_error); - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = + cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -199,27 +188,24 @@ struct groupby_dictionary_count_scan_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_count_scan_test, basic) { + using K = int32_t; using V = std::string; - using R = cudf::detail::target_type_t; - using result_wrapper = fixed_width_column_wrapper; + using R = cudf::detail::target_type_t; + using result_wrapper = cudf::test::fixed_width_column_wrapper; - // clang-format off - strings_column_wrapper keys{"1", "3", "3", "5", "5", "0"}; - dictionary_column_wrapper vals{1, 1, 1, 1, 1, 1}; - strings_column_wrapper expect_keys{"0", "1", "3", "3", "5", "5"}; + cudf::test::strings_column_wrapper keys{"1", "3", "3", "5", "5", "0"}; + cudf::test::dictionary_column_wrapper vals{1, 1, 1, 1, 1, 1}; + cudf::test::strings_column_wrapper expect_keys{"0", "1", "3", "3", "5", "5"}; result_wrapper expect_vals{0, 0, 0, 1, 0, 1}; - // clang-format on - // Count groupby aggregation is only supported with null_policy::EXCLUDE - auto agg1 = cudf::make_count_aggregation(); + // Count groupby aggregation is only supported with cudf::null_policy::EXCLUDE + auto agg1 = cudf::make_count_aggregation(); EXPECT_THROW(test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg1)), cudf::logic_error); - test_single_scan(keys, - vals, - expect_keys, - expect_vals, - cudf::make_count_aggregation(null_policy::INCLUDE)); + test_single_scan( + keys, + vals, + expect_keys, + expect_vals, + cudf::make_count_aggregation(cudf::null_policy::INCLUDE)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/count_tests.cpp b/cpp/tests/groupby/count_tests.cpp index c1cabe3fb08..d98c3735e66 100644 --- a/cpp/tests/groupby/count_tests.cpp +++ b/cpp/tests/groupby/count_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,10 +23,6 @@ #include -using namespace cudf::test::iterators; - -namespace cudf { -namespace test { template struct groupby_count_test : public cudf::test::BaseFixture { }; @@ -37,110 +33,110 @@ TYPED_TEST_SUITE(groupby_count_test, cudf::test::AllTypes); TYPED_TEST(groupby_count_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{3, 4, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{3, 4, 3}; - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2)); } TYPED_TEST(groupby_count_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals; - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); } TYPED_TEST(groupby_count_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2)); } TYPED_TEST(groupby_count_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals{0}; + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals{0}; - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); - fixed_width_column_wrapper expect_vals2{3}; - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + cudf::test::fixed_width_column_wrapper expect_vals2{3}; + auto agg2 = cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg2)); } TYPED_TEST(groupby_count_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // clang-format off - // {1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - // {3, 6, 1, 4, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals({2, 3, 2, 0}); + // {1, 1, 2, 2, 2, 3, 3, 4} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, cudf::test::iterators::no_nulls()); + // {3, 6, 1, 4, 9, 2, 8, -} + cudf::test::fixed_width_column_wrapper expect_vals({2, 3, 2, 0}); // clang-format on - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); - fixed_width_column_wrapper expect_vals2{3, 4, 2, 1}; - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + cudf::test::fixed_width_column_wrapper expect_vals2{3, 4, 2, 1}; + auto agg2 = cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg2)); } @@ -150,31 +146,31 @@ struct groupby_count_string_test : public cudf::test::BaseFixture { TEST_F(groupby_count_string_test, basic) { using V = cudf::string_view; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 3, 3, 5, 5, 0}; - strings_column_wrapper vals{"1", "1", "1", "1", "1", "1"}; + cudf::test::fixed_width_column_wrapper keys{1, 3, 3, 5, 5, 0}; + cudf::test::strings_column_wrapper vals{"1", "1", "1", "1", "1", "1"}; // clang-format on - fixed_width_column_wrapper expect_keys{0, 1, 3, 5}; - fixed_width_column_wrapper expect_vals{1, 1, 2, 2}; + cudf::test::fixed_width_column_wrapper expect_keys{0, 1, 3, 5}; + cudf::test::fixed_width_column_wrapper expect_vals{1, 1, 2, 2}; - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); } // clang-format on template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupByCountFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupByCountFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupByCount) +TYPED_TEST(GroupByCountFixedPointTest, GroupByCount) { using namespace numeric; using decimalXX = TypeParam; @@ -182,22 +178,22 @@ TYPED_TEST(FixedPointTestAllReps, GroupByCount) using fp_wrapper = cudf::test::fixed_point_column_wrapper; using V = decimalXX; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; auto const scale = scale_type{-1}; - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = fp_wrapper{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; - auto const expect_vals = fixed_width_column_wrapper{3, 4, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const expect_vals = cudf::test::fixed_width_column_wrapper{3, 4, 3}; - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg1 = cudf::make_count_aggregation(); + auto agg1 = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg1), force_use_sort_impl::YES); - auto agg2 = cudf::make_count_aggregation(null_policy::INCLUDE); + auto agg2 = cudf::make_count_aggregation(cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2)); } @@ -207,24 +203,24 @@ struct groupby_dictionary_count_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_count_test, basic) { using V = std::string; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - strings_column_wrapper keys{"1", "3", "3", "5", "5", "0"}; - dictionary_column_wrapper vals{1, 1, 1, 1, 1, 1}; - strings_column_wrapper expect_keys{"0", "1", "3", "5"}; - fixed_width_column_wrapper expect_vals{1, 1, 2, 2}; + cudf::test::strings_column_wrapper keys{"1", "3", "3", "5", "5", "0"}; + cudf::test::dictionary_column_wrapper vals{1, 1, 1, 1, 1, 1}; + cudf::test::strings_column_wrapper expect_keys{"0", "1", "3", "5"}; + cudf::test::fixed_width_column_wrapper expect_vals{1, 1, 2, 2}; // clang-format on - test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_count_aggregation()); test_single_agg(keys, vals, expect_keys, expect_vals, - cudf::make_count_aggregation(), + cudf::make_count_aggregation()); + test_single_agg(keys, + vals, + expect_keys, + expect_vals, + cudf::make_count_aggregation(), force_use_sort_impl::YES); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/covariance_tests.cpp b/cpp/tests/groupby/covariance_tests.cpp index b50d5f961fc..1f65bce4d4e 100644 --- a/cpp/tests/groupby/covariance_tests.cpp +++ b/cpp/tests/groupby/covariance_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ * limitations under the License. */ -#include -#include +#include #include #include @@ -23,22 +22,20 @@ #include #include -#include +#include +#include #include #include using namespace cudf::test::iterators; -namespace cudf { -namespace test { - -using structs = structs_column_wrapper; template struct groupby_covariance_test : public cudf::test::BaseFixture { }; -using supported_types = RemoveIf>, cudf::test::NumericTypes>; +using supported_types = + cudf::test::RemoveIf>, cudf::test::NumericTypes>; TYPED_TEST_SUITE(groupby_covariance_test, supported_types); using K = int32_t; @@ -47,14 +44,15 @@ TYPED_TEST(groupby_covariance_test, invalid_types) { using V = TypeParam; - auto keys = fixed_width_column_wrapper{{1, 2, 2, 1}}; - auto member_0 = fixed_width_column_wrapper{{1, 1, 1, 2}}; - // Covariance aggregations require all types are convertible to double, but + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 2, 1}}; + auto member_0 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2}}; + // Covariance cudf::aggregations require all types are convertible to double, but // duration_D cannot be converted to double. - auto member_1 = fixed_width_column_wrapper{{0, 0, 1, 1}}; - auto vals = structs{{member_0, member_1}}; + auto member_1 = + cudf::test::fixed_width_column_wrapper{{0, 0, 1, 1}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); EXPECT_THROW(test_single_agg(keys, vals, keys, vals, std::move(agg), force_use_sort_impl::YES), cudf::logic_error); } @@ -62,179 +60,179 @@ TYPED_TEST(groupby_covariance_test, invalid_types) TYPED_TEST(groupby_covariance_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{{1.0, 1.0, 0.0}}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{{1.0, 1.0, 0.0}}; - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper member_0{}, member_1{}; - auto vals = structs{{member_0, member_1}}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper member_0{}, member_1{}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper member_0{3, 4, 5}, member_1{6, 7, 8}; - auto vals = structs{{member_0, member_1}}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper member_0{3, 4, 5}, member_1{6, 7, 8}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper member_0({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper member_1({3, 4, 5}, all_nulls()); - auto vals = structs{{member_0, member_1}}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper member_0({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper member_1({3, 4, 5}, all_nulls()); + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, + cudf::test::fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}); + cudf::test::fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}); // clang-format on - auto vals = structs{{val0, val1}}; + auto vals = cudf::test::structs_column_wrapper{{val0, val1}}; - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - fixed_width_column_wrapper expect_vals({0.5, 1.0, 0.0, -0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({0.5, 1.0, 0.0, -0.}, {1, 1, 1, 0}); - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, null_values_same) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, + cudf::test::fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); - fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}, + cudf::test::fixed_width_column_wrapper val1({1, 1, 1, 2, 0, 3, 3,-1, 0, 2, 2}, {0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); // clang-format on - auto vals = structs{{val0, val1}}; + auto vals = cudf::test::structs_column_wrapper{{val0, val1}}; - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - fixed_width_column_wrapper expect_vals({0.5, 1.0, 0.0, -0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({0.5, 1.0, 0.0, -0.}, {1, 1, 1, 0}); - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, null_values_different) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, + cudf::test::fixed_width_column_wrapper val0({9, 1, 1, 2, 2, 3, 3,-1, 1, 4, 4}, {0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper val1({1, 2, 1, 2,-1, 3, 3,-1, 0, 4, 2}, + cudf::test::fixed_width_column_wrapper val1({1, 2, 1, 2,-1, 3, 3,-1, 0, 4, 2}, {0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0}); // clang-format on - auto vals = structs{{val0, val1}}; + auto vals = cudf::test::structs_column_wrapper{{val0, val1}}; - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - fixed_width_column_wrapper expect_vals( + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals( {std::numeric_limits::quiet_NaN(), 1.5, 0.0, -0.}, {0, 1, 1, 0}); - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, min_periods) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals1{{1.0, 1.0, 0.0}}; - auto agg1 = cudf::make_covariance_aggregation(3); + cudf::test::fixed_width_column_wrapper expect_vals1{{1.0, 1.0, 0.0}}; + auto agg1 = cudf::make_covariance_aggregation(3); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg1), force_use_sort_impl::YES); - fixed_width_column_wrapper expect_vals2{{1.0, 1.0, 0.0}, {0, 1, 0}}; - auto agg2 = cudf::make_covariance_aggregation(4); + cudf::test::fixed_width_column_wrapper expect_vals2{{1.0, 1.0, 0.0}, {0, 1, 0}}; + auto agg2 = cudf::make_covariance_aggregation(4); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg2), force_use_sort_impl::YES); - fixed_width_column_wrapper expect_vals3{{1.0, 1.0, 0.0}, {0, 0, 0}}; - auto agg3 = cudf::make_covariance_aggregation(5); + cudf::test::fixed_width_column_wrapper expect_vals3{{1.0, 1.0, 0.0}, {0, 0, 0}}; + auto agg3 = cudf::make_covariance_aggregation(5); test_single_agg(keys, vals, expect_keys, expect_vals3, std::move(agg3), force_use_sort_impl::YES); } TYPED_TEST(groupby_covariance_test, ddof) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::fixed_width_column_wrapper{{1, 1, 1, 2, 0, 3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals1{{2.0, 1.5, 0.0}}; - auto agg1 = cudf::make_covariance_aggregation(1, 2); + cudf::test::fixed_width_column_wrapper expect_vals1{{2.0, 1.5, 0.0}}; + auto agg1 = cudf::make_covariance_aggregation(1, 2); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg1), force_use_sort_impl::YES); auto const inf = std::numeric_limits::infinity(); - fixed_width_column_wrapper expect_vals2{{inf, 3.0, 0.0}, {0, 1, 0}}; - auto agg2 = cudf::make_covariance_aggregation(1, 3); + cudf::test::fixed_width_column_wrapper expect_vals2{{inf, 3.0, 0.0}, {0, 1, 0}}; + auto agg2 = cudf::make_covariance_aggregation(1, 3); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg2), force_use_sort_impl::YES); } @@ -244,19 +242,16 @@ struct groupby_dictionary_covariance_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_covariance_test, basic) { using V = int16_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - auto keys = fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; - auto member_0 = dictionary_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; - auto member_1 = dictionary_column_wrapper{{1, 1, 1, 2, 3, -3, 3, 1, 1, 2}}; - auto vals = structs{{member_0, member_1}}; + auto keys = cudf::test::fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}}; + auto member_0 = cudf::test::dictionary_column_wrapper{{1, 1, 1, 2, 2, 3, 3, 1, 1, 4}}; + auto member_1 = cudf::test::dictionary_column_wrapper{{1, 1, 1, 2, 3, -3, 3, 1, 1, 2}}; + auto vals = cudf::test::structs_column_wrapper{{member_0, member_1}}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{{1.0, -0.5, 0.0}}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{{1.0, -0.5, 0.0}}; - auto agg = cudf::make_covariance_aggregation(); + auto agg = cudf::make_covariance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/groupby_test_util.cpp b/cpp/tests/groupby/groupby_test_util.cpp new file mode 100644 index 00000000000..ef99e148a62 --- /dev/null +++ b/cpp/tests/groupby/groupby_test_util.cpp @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2020-2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "groupby_test_util.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +void test_single_agg(cudf::column_view const& keys, + cudf::column_view const& values, + cudf::column_view const& expect_keys, + cudf::column_view const& expect_vals, + std::unique_ptr&& agg, + force_use_sort_impl use_sort, + cudf::null_policy include_null_keys, + cudf::sorted keys_are_sorted, + std::vector const& column_order, + std::vector const& null_precedence, + cudf::sorted reference_keys_are_sorted) +{ + auto const [sorted_expect_keys, sorted_expect_vals] = [&]() { + if (reference_keys_are_sorted == cudf::sorted::NO) { + auto const sort_expect_order = + cudf::sorted_order(cudf::table_view{{expect_keys}}, column_order, null_precedence); + auto sorted_expect_keys = cudf::gather(cudf::table_view{{expect_keys}}, *sort_expect_order); + auto sorted_expect_vals = cudf::gather(cudf::table_view{{expect_vals}}, *sort_expect_order); + return std::make_pair(std::move(sorted_expect_keys), std::move(sorted_expect_vals)); + } else { + auto sorted_expect_keys = std::make_unique(cudf::table_view{{expect_keys}}); + auto sorted_expect_vals = std::make_unique(cudf::table_view{{expect_vals}}); + return std::make_pair(std::move(sorted_expect_keys), std::move(sorted_expect_vals)); + } + }(); + + std::vector requests; + requests.emplace_back(cudf::groupby::aggregation_request()); + requests[0].values = values; + + requests[0].aggregations.push_back(std::move(agg)); + + if (use_sort == force_use_sort_impl::YES) { + // WAR to force cudf::groupby to use sort implementation + requests[0].aggregations.push_back( + cudf::make_nth_element_aggregation(0)); + } + + // since the default behavior of cudf::groupby(...) for an empty null_precedence vector is + // null_order::AFTER whereas for cudf::sorted_order(...) it's null_order::BEFORE + auto const precedence = null_precedence.empty() + ? std::vector(1, cudf::null_order::BEFORE) + : null_precedence; + + cudf::groupby::groupby gb_obj( + cudf::table_view({keys}), include_null_keys, keys_are_sorted, column_order, precedence); + + auto result = gb_obj.aggregate(requests); + + if (use_sort == force_use_sort_impl::YES && keys_are_sorted == cudf::sorted::NO) { + CUDF_TEST_EXPECT_TABLES_EQUAL(*sorted_expect_keys, result.first->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(sorted_expect_vals->get_column(0), + *result.second[0].results[0]); + + } else { + auto const sort_order = cudf::sorted_order(result.first->view(), column_order, precedence); + auto const sorted_keys = cudf::gather(result.first->view(), *sort_order); + auto const sorted_vals = + cudf::gather(cudf::table_view({result.second[0].results[0]->view()}), *sort_order); + + CUDF_TEST_EXPECT_TABLES_EQUAL(*sorted_expect_keys, *sorted_keys); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(sorted_expect_vals->get_column(0), + sorted_vals->get_column(0)); + } +} + +void test_sum_agg(cudf::column_view const& keys, + cudf::column_view const& values, + cudf::column_view const& expected_keys, + cudf::column_view const& expected_values) +{ + auto const do_test = [&](auto const use_sort_option) { + test_single_agg(keys, + values, + expected_keys, + expected_values, + cudf::make_sum_aggregation(), + use_sort_option, + cudf::null_policy::INCLUDE); + }; + do_test(force_use_sort_impl::YES); + do_test(force_use_sort_impl::NO); +} + +void test_single_scan(cudf::column_view const& keys, + cudf::column_view const& values, + cudf::column_view const& expect_keys, + cudf::column_view const& expect_vals, + std::unique_ptr&& agg, + cudf::null_policy include_null_keys, + cudf::sorted keys_are_sorted, + std::vector const& column_order, + std::vector const& null_precedence) +{ + std::vector requests; + requests.emplace_back(cudf::groupby::scan_request()); + requests[0].values = values; + + requests[0].aggregations.push_back(std::move(agg)); + + cudf::groupby::groupby gb_obj( + cudf::table_view({keys}), include_null_keys, keys_are_sorted, column_order, null_precedence); + + // cudf::groupby scan uses sort implementation + auto result = gb_obj.scan(requests); + + CUDF_TEST_EXPECT_TABLES_EQUAL(cudf::table_view({expect_keys}), result.first->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expect_vals, *result.second[0].results[0]); +} diff --git a/cpp/tests/groupby/groupby_test_util.hpp b/cpp/tests/groupby/groupby_test_util.hpp index dff8655bf9b..4c4492f8151 100644 --- a/cpp/tests/groupby/groupby_test_util.hpp +++ b/cpp/tests/groupby/groupby_test_util.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,161 +16,36 @@ #pragma once -#include -#include -#include - #include -#include -#include #include #include -#include #include -#include - -#include -namespace cudf { -namespace test { enum class force_use_sort_impl : bool { NO, YES }; -inline void test_groups(column_view const& keys, - column_view const& expect_grouped_keys, - std::vector const& expect_group_offsets, - column_view const& values = {}, - column_view const& expect_grouped_values = {}) -{ - groupby::groupby gb(table_view({keys})); - groupby::groupby::groups gb_groups; - - if (values.size()) { - gb_groups = gb.get_groups(table_view({values})); - } else { - gb_groups = gb.get_groups(); - } - CUDF_TEST_EXPECT_TABLES_EQUAL(table_view({expect_grouped_keys}), gb_groups.keys->view()); - - auto got_offsets = gb_groups.offsets; - EXPECT_EQ(expect_group_offsets.size(), got_offsets.size()); - for (auto i = 0u; i != expect_group_offsets.size(); ++i) { - EXPECT_EQ(expect_group_offsets[i], got_offsets[i]); - } - - if (values.size()) { - CUDF_TEST_EXPECT_TABLES_EQUAL(table_view({expect_grouped_values}), gb_groups.values->view()); - } -} - -inline void test_single_agg(column_view const& keys, - column_view const& values, - column_view const& expect_keys, - column_view const& expect_vals, - std::unique_ptr&& agg, - force_use_sort_impl use_sort = force_use_sort_impl::NO, - null_policy include_null_keys = null_policy::EXCLUDE, - sorted keys_are_sorted = sorted::NO, - std::vector const& column_order = {}, - std::vector const& null_precedence = {}, - sorted reference_keys_are_sorted = sorted::NO) -{ - auto const [sorted_expect_keys, sorted_expect_vals] = [&]() { - if (reference_keys_are_sorted == sorted::NO) { - auto const sort_expect_order = - sorted_order(table_view{{expect_keys}}, column_order, null_precedence); - auto sorted_expect_keys = gather(table_view{{expect_keys}}, *sort_expect_order); - auto sorted_expect_vals = gather(table_view{{expect_vals}}, *sort_expect_order); - return std::make_pair(std::move(sorted_expect_keys), std::move(sorted_expect_vals)); - } else { - auto sorted_expect_keys = std::make_unique
(table_view{{expect_keys}}); - auto sorted_expect_vals = std::make_unique
(table_view{{expect_vals}}); - return std::make_pair(std::move(sorted_expect_keys), std::move(sorted_expect_vals)); - } - }(); - - std::vector requests; - requests.emplace_back(groupby::aggregation_request()); - requests[0].values = values; - - requests[0].aggregations.push_back(std::move(agg)); - - if (use_sort == force_use_sort_impl::YES) { - // WAR to force groupby to use sort implementation - requests[0].aggregations.push_back(make_nth_element_aggregation(0)); - } - - // since the default behavior of groupby(...) for an empty null_precedence vector is - // null_order::AFTER whereas for sorted_order(...) it's null_order::BEFORE - auto const precedence = - null_precedence.empty() ? std::vector(1, null_order::BEFORE) : null_precedence; - - groupby::groupby gb_obj( - table_view({keys}), include_null_keys, keys_are_sorted, column_order, precedence); - - auto result = gb_obj.aggregate(requests); - - if (use_sort == force_use_sort_impl::YES && keys_are_sorted == sorted::NO) { - CUDF_TEST_EXPECT_TABLES_EQUAL(*sorted_expect_keys, result.first->view()); - cudf::test::detail::expect_columns_equivalent(sorted_expect_vals->get_column(0), - *result.second[0].results[0], - debug_output_level::ALL_ERRORS); - - } else { - auto const sort_order = sorted_order(result.first->view(), column_order, precedence); - auto const sorted_keys = gather(result.first->view(), *sort_order); - auto const sorted_vals = gather(table_view({result.second[0].results[0]->view()}), *sort_order); - - CUDF_TEST_EXPECT_TABLES_EQUAL(*sorted_expect_keys, *sorted_keys); - cudf::test::detail::expect_columns_equivalent(sorted_expect_vals->get_column(0), - sorted_vals->get_column(0), - debug_output_level::ALL_ERRORS); - } -} - -inline void test_sum_agg(column_view const& keys, - column_view const& values, - column_view const& expected_keys, - column_view const& expected_values) -{ - auto const do_test = [&](auto const use_sort_option) { - test_single_agg(keys, - values, - expected_keys, - expected_values, - cudf::make_sum_aggregation(), - use_sort_option, - null_policy::INCLUDE); - }; - do_test(force_use_sort_impl::YES); - do_test(force_use_sort_impl::NO); -} - -inline void test_single_scan(column_view const& keys, - column_view const& values, - column_view const& expect_keys, - column_view const& expect_vals, - std::unique_ptr&& agg, - null_policy include_null_keys = null_policy::EXCLUDE, - sorted keys_are_sorted = sorted::NO, - std::vector const& column_order = {}, - std::vector const& null_precedence = {}) -{ - std::vector requests; - requests.emplace_back(groupby::scan_request()); - requests[0].values = values; - - requests[0].aggregations.push_back(std::move(agg)); - - groupby::groupby gb_obj( - table_view({keys}), include_null_keys, keys_are_sorted, column_order, null_precedence); - - // groupby scan uses sort implementation - auto result = gb_obj.scan(requests); - - CUDF_TEST_EXPECT_TABLES_EQUAL(table_view({expect_keys}), result.first->view()); - cudf::test::detail::expect_columns_equivalent( - expect_vals, *result.second[0].results[0], debug_output_level::ALL_ERRORS); -} - -} // namespace test -} // namespace cudf +void test_single_agg(cudf::column_view const& keys, + cudf::column_view const& values, + cudf::column_view const& expect_keys, + cudf::column_view const& expect_vals, + std::unique_ptr&& agg, + force_use_sort_impl use_sort = force_use_sort_impl::NO, + cudf::null_policy include_null_keys = cudf::null_policy::EXCLUDE, + cudf::sorted keys_are_sorted = cudf::sorted::NO, + std::vector const& column_order = {}, + std::vector const& null_precedence = {}, + cudf::sorted reference_keys_are_sorted = cudf::sorted::NO); + +void test_sum_agg(cudf::column_view const& keys, + cudf::column_view const& values, + cudf::column_view const& expected_keys, + cudf::column_view const& expected_values); + +void test_single_scan(cudf::column_view const& keys, + cudf::column_view const& values, + cudf::column_view const& expect_keys, + cudf::column_view const& expect_vals, + std::unique_ptr&& agg, + cudf::null_policy include_null_keys = cudf::null_policy::EXCLUDE, + cudf::sorted keys_are_sorted = cudf::sorted::NO, + std::vector const& column_order = {}, + std::vector const& null_precedence = {}); diff --git a/cpp/tests/groupby/groups_tests.cpp b/cpp/tests/groupby/groups_tests.cpp index 2ca359e0838..0681e23e10b 100644 --- a/cpp/tests/groupby/groups_tests.cpp +++ b/cpp/tests/groupby/groups_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,35 +14,60 @@ * limitations under the License. */ -#include - #include #include #include +#include #include +#include +#include #include -using namespace cudf::test::iterators; +void test_groups(cudf::column_view const& keys, + cudf::column_view const& expect_grouped_keys, + std::vector const& expect_group_offsets, + cudf::column_view const& values = {}, + cudf::column_view const& expect_grouped_values = {}) +{ + cudf::groupby::groupby gb(cudf::table_view({keys})); + cudf::groupby::groupby::groups gb_groups; + + if (values.size()) { + gb_groups = gb.get_groups(cudf::table_view({values})); + } else { + gb_groups = gb.get_groups(); + } + CUDF_TEST_EXPECT_TABLES_EQUAL(cudf::table_view({expect_grouped_keys}), gb_groups.keys->view()); + + auto got_offsets = gb_groups.offsets; + EXPECT_EQ(expect_group_offsets.size(), got_offsets.size()); + for (auto i = 0u; i != expect_group_offsets.size(); ++i) { + EXPECT_EQ(expect_group_offsets[i], got_offsets[i]); + } + + if (values.size()) { + CUDF_TEST_EXPECT_TABLES_EQUAL(cudf::table_view({expect_grouped_values}), + gb_groups.values->view()); + } +} -namespace cudf { -namespace test { -struct groupby_group_keys_test : public BaseFixture { +struct groupby_group_keys_test : public cudf::test::BaseFixture { }; template struct groupby_group_keys_and_values_test : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(groupby_group_keys_and_values_test, NumericTypes); +TYPED_TEST_SUITE(groupby_group_keys_and_values_test, cudf::test::NumericTypes); TEST_F(groupby_group_keys_test, basic) { using K = int32_t; - fixed_width_column_wrapper keys{1, 1, 2, 1, 2, 3}; - fixed_width_column_wrapper expect_grouped_keys{1, 1, 1, 2, 2, 3}; - std::vector expect_group_offsets = {0, 3, 5, 6}; + cudf::test::fixed_width_column_wrapper keys{1, 1, 2, 1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_grouped_keys{1, 1, 1, 2, 2, 3}; + std::vector expect_group_offsets = {0, 3, 5, 6}; test_groups(keys, expect_grouped_keys, expect_group_offsets); } @@ -50,9 +75,9 @@ TEST_F(groupby_group_keys_test, empty_keys) { using K = int32_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper expect_grouped_keys{}; - std::vector expect_group_offsets = {0}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper expect_grouped_keys{}; + std::vector expect_group_offsets = {0}; test_groups(keys, expect_grouped_keys, expect_group_offsets); } @@ -60,9 +85,10 @@ TEST_F(groupby_group_keys_test, all_null_keys) { using K = int32_t; - fixed_width_column_wrapper keys({1, 1, 2, 3, 1, 2}, all_nulls()); - fixed_width_column_wrapper expect_grouped_keys{}; - std::vector expect_group_offsets = {0}; + cudf::test::fixed_width_column_wrapper keys({1, 1, 2, 3, 1, 2}, + cudf::test::iterators::all_nulls()); + cudf::test::fixed_width_column_wrapper expect_grouped_keys{}; + std::vector expect_group_offsets = {0}; test_groups(keys, expect_grouped_keys, expect_group_offsets); } @@ -71,11 +97,11 @@ TYPED_TEST(groupby_group_keys_and_values_test, basic_with_values) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys({5, 4, 3, 2, 1, 0}); - fixed_width_column_wrapper expect_grouped_keys{0, 1, 2, 3, 4, 5}; - fixed_width_column_wrapper values({0, 0, 1, 1, 2, 2}); - fixed_width_column_wrapper expect_grouped_values{2, 2, 1, 1, 0, 0}; - std::vector expect_group_offsets = {0, 1, 2, 3, 4, 5, 6}; + cudf::test::fixed_width_column_wrapper keys({5, 4, 3, 2, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_grouped_keys{0, 1, 2, 3, 4, 5}; + cudf::test::fixed_width_column_wrapper values({0, 0, 1, 1, 2, 2}); + cudf::test::fixed_width_column_wrapper expect_grouped_values{2, 2, 1, 1, 0, 0}; + std::vector expect_group_offsets = {0, 1, 2, 3, 4, 5, 6}; test_groups(keys, expect_grouped_keys, expect_group_offsets, values, expect_grouped_values); } @@ -84,13 +110,11 @@ TYPED_TEST(groupby_group_keys_and_values_test, some_nulls) using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper keys({1, 1, 3, 2, 1, 2}, {1, 0, 1, 0, 0, 1}); - fixed_width_column_wrapper expect_grouped_keys({1, 2, 3}, no_nulls()); - fixed_width_column_wrapper values({1, 2, 3, 4, 5, 6}); - fixed_width_column_wrapper expect_grouped_values({1, 6, 3}); - std::vector expect_group_offsets = {0, 1, 2, 3}; + cudf::test::fixed_width_column_wrapper keys({1, 1, 3, 2, 1, 2}, {1, 0, 1, 0, 0, 1}); + cudf::test::fixed_width_column_wrapper expect_grouped_keys({1, 2, 3}, + cudf::test::iterators::no_nulls()); + cudf::test::fixed_width_column_wrapper values({1, 2, 3, 4, 5, 6}); + cudf::test::fixed_width_column_wrapper expect_grouped_values({1, 6, 3}); + std::vector expect_group_offsets = {0, 1, 2, 3}; test_groups(keys, expect_grouped_keys, expect_group_offsets, values, expect_grouped_values); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/keys_tests.cpp b/cpp/tests/groupby/keys_tests.cpp index efd0f52114e..d11b3786471 100644 --- a/cpp/tests/groupby/keys_tests.cpp +++ b/cpp/tests/groupby/keys_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,11 @@ #include #include +#include #include using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_keys_test : public cudf::test::BaseFixture { }; @@ -40,17 +39,17 @@ TYPED_TEST(groupby_keys_test, basic) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expect_keys { 1, 2, 3 }; - fixed_width_column_wrapper expect_vals { 3, 4, 3 }; + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3 }; + cudf::test::fixed_width_column_wrapper expect_vals { 3, 4, 3 }; // clang-format on - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -58,17 +57,17 @@ TYPED_TEST(groupby_keys_test, zero_valid_keys) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys ( { 1, 2, 3}, all_nulls() ); - fixed_width_column_wrapper vals { 3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys ( { 1, 2, 3}, all_nulls() ); + cudf::test::fixed_width_column_wrapper vals { 3, 4, 5}; - fixed_width_column_wrapper expect_keys { }; - fixed_width_column_wrapper expect_vals { }; + cudf::test::fixed_width_column_wrapper expect_keys { }; + cudf::test::fixed_width_column_wrapper expect_vals { }; // clang-format on - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -76,20 +75,20 @@ TYPED_TEST(groupby_keys_test, some_null_keys) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys( { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - { 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; - - // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4}, no_nulls() ); - // { 0, 3, 6, 1, 4, 5, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals { 3, 4, 2, 1}; + cudf::test::fixed_width_column_wrapper keys( { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + { 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; + + // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4} + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4}, no_nulls() ); + // { 0, 3, 6, 1, 4, 5, 9, 2, 8, -} + cudf::test::fixed_width_column_wrapper expect_vals { 3, 4, 2, 1}; // clang-format on - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -97,133 +96,133 @@ TYPED_TEST(groupby_keys_test, include_null_keys) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys( { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - { 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; - - // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, -} - fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4, 3}, - { 1, 1, 1, 1, 0}); - // { 0, 3, 6, 1, 4, 5, 9, 2, 8, -, -} - fixed_width_column_wrapper expect_vals { 9, 19, 10, 4, 7}; + cudf::test::fixed_width_column_wrapper keys( { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + { 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; + + // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4, -} + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4, 3}, + { 1, 1, 1, 1, 0}); + // { 0, 3, 6, 1, 4, 5, 9, 2, 8, -, -} + cudf::test::fixed_width_column_wrapper expect_vals { 9, 19, 10, 4, 7}; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::NO, - null_policy::INCLUDE); + cudf::null_policy::INCLUDE); } TYPED_TEST(groupby_keys_test, pre_sorted_keys) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4}; - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; + cudf::test::fixed_width_column_wrapper keys { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4}; + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; - fixed_width_column_wrapper expect_keys { 1, 2, 3, 4}; - fixed_width_column_wrapper expect_vals { 3, 18, 24, 4}; + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3, 4}; + cudf::test::fixed_width_column_wrapper expect_vals { 3, 18, 24, 4}; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES, - null_policy::EXCLUDE, - sorted::YES); + cudf::null_policy::EXCLUDE, + cudf::sorted::YES); } TYPED_TEST(groupby_keys_test, pre_sorted_keys_descending) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys { 4, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1}; - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; + cudf::test::fixed_width_column_wrapper keys { 4, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; - fixed_width_column_wrapper expect_keys { 4, 3, 2, 1 }; - fixed_width_column_wrapper expect_vals { 0, 6, 22, 21 }; + cudf::test::fixed_width_column_wrapper expect_keys { 4, 3, 2, 1 }; + cudf::test::fixed_width_column_wrapper expect_vals { 0, 6, 22, 21 }; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES, - null_policy::EXCLUDE, - sorted::YES, - {order::DESCENDING}); + cudf::null_policy::EXCLUDE, + cudf::sorted::YES, + {cudf::order::DESCENDING}); } TYPED_TEST(groupby_keys_test, pre_sorted_keys_nullable) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4}, - { 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; + cudf::test::fixed_width_column_wrapper keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4}, + { 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; - fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4}, no_nulls() ); - fixed_width_column_wrapper expect_vals { 3, 15, 17, 4}; + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4}, no_nulls() ); + cudf::test::fixed_width_column_wrapper expect_vals { 3, 15, 17, 4}; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES, - null_policy::EXCLUDE, - sorted::YES); + cudf::null_policy::EXCLUDE, + cudf::sorted::YES); } TYPED_TEST(groupby_keys_test, pre_sorted_keys_nulls_before_include_nulls) { using K = TypeParam; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4}, - { 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; - - // { 1, 1, 1, -, -, 2, 2, -, 3, 3, 4} - fixed_width_column_wrapper expect_keys({ 1, 2, 2, 3, 3, 4}, - { 1, 0, 1, 0, 1, 1}); - fixed_width_column_wrapper expect_vals { 3, 7, 11, 7, 17, 4}; + cudf::test::fixed_width_column_wrapper keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4}, + { 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}; + + // { 1, 1, 1, -, -, 2, 2, -, 3, 3, 4} + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 2, 3, 3, 4}, + { 1, 0, 1, 0, 1, 1}); + cudf::test::fixed_width_column_wrapper expect_vals { 3, 7, 11, 7, 17, 4}; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES, - null_policy::INCLUDE, - sorted::YES); + cudf::null_policy::INCLUDE, + cudf::sorted::YES); } TYPED_TEST(groupby_keys_test, mismatch_num_rows) @@ -231,13 +230,13 @@ TYPED_TEST(groupby_keys_test, mismatch_num_rows) using K = TypeParam; using V = int32_t; - fixed_width_column_wrapper keys{1, 2, 3}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4}; // Verify that scan throws an error when given data of mismatched sizes. - auto agg = cudf::make_count_aggregation(); + auto agg = cudf::make_count_aggregation(); EXPECT_THROW(test_single_agg(keys, vals, keys, vals, std::move(agg)), cudf::logic_error); - auto agg2 = cudf::make_count_aggregation(); + auto agg2 = cudf::make_count_aggregation(); EXPECT_THROW(test_single_scan(keys, vals, keys, vals, std::move(agg2)), cudf::logic_error); } @@ -248,7 +247,7 @@ TYPED_TEST(groupby_keys_test, structs) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; using STRINGS = cudf::test::strings_column_wrapper; using STRUCTS = cudf::test::structs_column_wrapper; @@ -292,7 +291,7 @@ TYPED_TEST(groupby_keys_test, structs) auto expect_keys = STRUCTS{{expected_s2, expected_col_c}, no_nulls()}; auto expect_vals = FWCW{6, 1, 8, 7}; - auto agg = cudf::make_argmax_aggregation(); + auto agg = cudf::make_argmax_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -301,7 +300,7 @@ using LCW = cudf::test::lists_column_wrapper; TYPED_TEST(groupby_keys_test, lists) { - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off auto keys = LCW { {1,1}, {2,2}, {3,3}, {1,1}, {2,2} }; @@ -311,7 +310,7 @@ TYPED_TEST(groupby_keys_test, lists) auto expected_values = FWCW { 3, 5, 2 }; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, values, expected_keys, expected_values, std::move(agg)); } @@ -321,17 +320,17 @@ struct groupby_string_keys_test : public cudf::test::BaseFixture { TEST_F(groupby_string_keys_test, basic) { using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - strings_column_wrapper keys { "aaa", "año", "₹1", "aaa", "año", "año", "aaa", "₹1", "₹1", "año"}; - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::strings_column_wrapper keys { "aaa", "año", "₹1", "aaa", "año", "año", "aaa", "₹1", "₹1", "año"}; + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - strings_column_wrapper expect_keys({ "aaa", "año", "₹1" }); - fixed_width_column_wrapper expect_vals { 9, 19, 17 }; + cudf::test::strings_column_wrapper expect_keys({ "aaa", "año", "₹1" }); + cudf::test::fixed_width_column_wrapper expect_vals { 9, 19, 17 }; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } // clang-format on @@ -343,85 +342,84 @@ TEST_F(groupby_dictionary_keys_test, basic) { using K = std::string; using V = int32_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - dictionary_column_wrapper keys { "aaa", "año", "₹1", "aaa", "año", "año", "aaa", "₹1", "₹1", "año"}; - fixed_width_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - dictionary_column_wrapperexpect_keys ({ "aaa", "año", "₹1" }); - fixed_width_column_wrapper expect_vals({ 9, 19, 17 }); + cudf::test::dictionary_column_wrapper keys { "aaa", "año", "₹1", "aaa", "año", "año", "aaa", "₹1", "₹1", "año"}; + cudf::test::fixed_width_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::dictionary_column_wrapperexpect_keys ({ "aaa", "año", "₹1" }); + cudf::test::fixed_width_column_wrapper expect_vals({ 9, 19, 17 }); // clang-format on test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation()); + keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation()); test_single_agg(keys, vals, expect_keys, expect_vals, - cudf::make_sum_aggregation(), + cudf::make_sum_aggregation(), force_use_sort_impl::YES); } struct groupby_cache_test : public cudf::test::BaseFixture { }; -// To check if the cache doesn't insert multiple times to cache for same aggregation on a column in -// same request. -// If this test fails, then insert happened and key stored in cache map becomes dangling reference. -// Any comparison with same aggregation as key will fail. +// To check if the cache doesn't insert multiple times to cache for the same aggregation on a +// column in the same request. If this test fails, then insert happened and the key stored in the +// cache map becomes a dangling reference. Any comparison with the same aggregation as the key will +// fail. TEST_F(groupby_cache_test, duplicate_agggregations) { using K = int32_t; using V = int32_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - groupby::groupby gb_obj(table_view({keys})); + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::groupby::groupby gb_obj(cudf::table_view({keys})); - std::vector requests; - requests.emplace_back(groupby::aggregation_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::aggregation_request()); requests[0].values = vals; - requests[0].aggregations.push_back(cudf::make_sum_aggregation()); - requests[0].aggregations.push_back(cudf::make_sum_aggregation()); + requests[0].aggregations.push_back(cudf::make_sum_aggregation()); + requests[0].aggregations.push_back(cudf::make_sum_aggregation()); // hash groupby EXPECT_NO_THROW(gb_obj.aggregate(requests)); // sort groupby // WAR to force groupby to use sort implementation - requests[0].aggregations.push_back(make_nth_element_aggregation(0)); + requests[0].aggregations.push_back( + cudf::make_nth_element_aggregation(0)); EXPECT_NO_THROW(gb_obj.aggregate(requests)); } -// To check if the cache doesn't insert multiple times to cache for same aggregation on same column -// but in different requests. -// If this test fails, then insert happened and key stored in cache map becomes dangling reference. -// Any comparison with same aggregation as key will fail. +// To check if the cache doesn't insert multiple times to cache for the same aggregation on the same +// column but in different requests. If this test fails, then insert happened and the key stored in +// the cache map becomes a dangling reference. Any comparison with the same aggregation as the key +// will fail. TEST_F(groupby_cache_test, duplicate_columns) { using K = int32_t; using V = int32_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - groupby::groupby gb_obj(table_view({keys})); + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::groupby::groupby gb_obj(cudf::table_view({keys})); - std::vector requests; - requests.emplace_back(groupby::aggregation_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::aggregation_request()); requests[0].values = vals; - requests[0].aggregations.push_back(cudf::make_sum_aggregation()); - requests.emplace_back(groupby::aggregation_request()); + requests[0].aggregations.push_back(cudf::make_sum_aggregation()); + requests.emplace_back(cudf::groupby::aggregation_request()); requests[1].values = vals; - requests[1].aggregations.push_back(cudf::make_sum_aggregation()); + requests[1].aggregations.push_back(cudf::make_sum_aggregation()); // hash groupby EXPECT_NO_THROW(gb_obj.aggregate(requests)); // sort groupby // WAR to force groupby to use sort implementation - requests[0].aggregations.push_back(make_nth_element_aggregation(0)); + requests[0].aggregations.push_back( + cudf::make_nth_element_aggregation(0)); EXPECT_NO_THROW(gb_obj.aggregate(requests)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/lists_tests.cpp b/cpp/tests/groupby/lists_tests.cpp index 96c1691ea36..3a57fdab58d 100644 --- a/cpp/tests/groupby/lists_tests.cpp +++ b/cpp/tests/groupby/lists_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,28 +14,14 @@ * limitations under the License. */ -#include "groupby_test_util.hpp" +#include #include #include #include #include -#include #include -#include -#include -#include - -#include - -#include -#include - -#include - -namespace cudf { -namespace test { template struct groupby_lists_test : public cudf::test::BaseFixture { @@ -45,9 +31,8 @@ TYPED_TEST_SUITE(groupby_lists_test, cudf::test::FixedWidthTypes); using namespace cudf::test::iterators; -using R = cudf::detail::target_type_t; // Type of aggregation result. -using strings = strings_column_wrapper; -using structs = structs_column_wrapper; +// Type of aggregation result. +using agg_result_t = cudf::detail::target_type_t; template using fwcw = cudf::test::fixed_width_column_wrapper; @@ -72,8 +57,8 @@ TYPED_TEST(groupby_lists_test, basic) auto keys = lcw { {1,1}, {2,2}, {3,3}, {1,1}, {2,2} }; auto values = fwcw { 0, 1, 2, 3, 4 }; - auto expected_keys = lcw { {1,1}, {2,2}, {3,3} }; - auto expected_values = fwcw { 3, 5, 2 }; + auto expected_keys = lcw { {1,1}, {2,2}, {3,3} }; + auto expected_values = fwcw{ 3, 5, 2 }; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -85,8 +70,8 @@ TYPED_TEST(groupby_lists_test, all_null_input) auto keys = lcw { {{1,1}, {2,2}, {3,3}, {1,1}, {2,2}}, all_nulls()}; auto values = fwcw { 0, 1, 2, 3, 4 }; - auto expected_keys = lcw { {{null,null}}, all_nulls()}; - auto expected_values = fwcw { 10 }; + auto expected_keys = lcw { {{null,null}}, all_nulls()}; + auto expected_values = fwcw{ 10 }; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -98,8 +83,8 @@ TYPED_TEST(groupby_lists_test, lists_with_nulls) auto keys = lcw { {{1,1}, {2,2}, {3,3}, {1,1}, {2,2}}, nulls_at({1,2,4})}; auto values = fwcw { 0, 1, 2, 3, 4 }; - auto expected_keys = lcw { {{null,null}, {1,1}}, null_at(0)}; - auto expected_values = fwcw { 7, 3 }; + auto expected_keys = lcw { {{null,null}, {1,1}}, null_at(0)}; + auto expected_values = fwcw{ 7, 3 }; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -117,9 +102,7 @@ TYPED_TEST(groupby_lists_test, lists_with_null_elements) auto expected_keys = lcw{ {{}, lcw{{{1, 2, 3}, {}, {4, 5}, {}, {6, 0}}, nulls_at({1, 3})}}, null_at(0)}; - auto expected_values = fwcw{9, 3}; + auto expected_values = fwcw{9, 3}; test_sum_agg(keys, values, expected_keys, expected_values); } -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/max_scan_tests.cpp b/cpp/tests/groupby/max_scan_tests.cpp index 19935dd4c91..522396e3591 100644 --- a/cpp/tests/groupby/max_scan_tests.cpp +++ b/cpp/tests/groupby/max_scan_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,17 +26,14 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { -using K = int32_t; -using key_wrapper = fixed_width_column_wrapper; +using key_wrapper = cudf::test::fixed_width_column_wrapper; template struct groupby_max_scan_test : public cudf::test::BaseFixture { using V = T; - using R = cudf::detail::target_type_t; - using value_wrapper = fixed_width_column_wrapper; - using result_wrapper = fixed_width_column_wrapper; + using R = cudf::detail::target_type_t; + using value_wrapper = cudf::test::fixed_width_column_wrapper; + using result_wrapper = cudf::test::fixed_width_column_wrapper; }; TYPED_TEST_SUITE(groupby_max_scan_test, cudf::test::FixedWidthTypesWithoutFixedPoint); @@ -55,7 +52,7 @@ TYPED_TEST(groupby_max_scan_test, basic) result_wrapper expect_vals({5, 8, 8, 6, 9, 9, 9, 7, 7, 7}); // clang-format on - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -72,9 +69,14 @@ TYPED_TEST(groupby_max_scan_test, pre_sorted) result_wrapper expect_vals({5, 8, 8, 6, 9, 9, 9, 7, 7, 7}); // clang-format on - auto agg = cudf::make_max_aggregation(); - test_single_scan( - keys, vals, expect_keys, expect_vals, std::move(agg), null_policy::EXCLUDE, sorted::YES); + auto agg = cudf::make_max_aggregation(); + test_single_scan(keys, + vals, + expect_keys, + expect_vals, + std::move(agg), + cudf::null_policy::EXCLUDE, + cudf::sorted::YES); } TYPED_TEST(groupby_max_scan_test, empty_cols) @@ -84,11 +86,10 @@ TYPED_TEST(groupby_max_scan_test, empty_cols) key_wrapper keys{}; value_wrapper vals{}; - key_wrapper expect_keys{}; result_wrapper expect_vals{}; - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -97,15 +98,12 @@ TYPED_TEST(groupby_max_scan_test, zero_valid_keys) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys( {1, 2, 3}, all_nulls()); + key_wrapper keys({1, 2, 3}, all_nulls()); value_wrapper vals({3, 4, 5}); - key_wrapper expect_keys{}; result_wrapper expect_vals{}; - // clang-format on - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -114,15 +112,12 @@ TYPED_TEST(groupby_max_scan_test, zero_valid_values) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys {1, 1, 1}; + key_wrapper keys{1, 1, 1}; value_wrapper vals({3, 4, 5}, all_nulls()); - - key_wrapper expect_keys {1, 1, 1}; + key_wrapper expect_keys{1, 1, 1}; result_wrapper expect_vals({-1, -1, -1}, all_nulls()); - // clang-format on - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -142,7 +137,7 @@ TYPED_TEST(groupby_max_scan_test, null_keys_and_values) { 0, 1, 1, 1, 1, 0, 1, 1, 1, 0}); // clang-format on - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -152,28 +147,29 @@ struct groupby_max_scan_string_test : public cudf::test::BaseFixture { TEST_F(groupby_max_scan_string_test, basic) { key_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; + cudf::test::strings_column_wrapper vals{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; key_wrapper expect_keys{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; - strings_column_wrapper expect_vals( + cudf::test::strings_column_wrapper expect_vals( {"año", "año", "año", "bit", "zit", "zit", "zit", "₹1", "₹1", "₹1"}); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupByMaxScanFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupByMaxScanFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupBySortMaxScanDecimalAsValue) +TYPED_TEST(GroupByMaxScanFixedPointTest, GroupBySortMaxScanDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; using RepType = cudf::device_storage_type_t; - using fp_wrapper = fixed_point_column_wrapper; + using fp_wrapper = cudf::test::fixed_point_column_wrapper; for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; @@ -181,12 +177,12 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortMaxScanDecimalAsValue) auto const keys = key_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = fp_wrapper{{5, 6, 7, 8, 9, 0, 1, 2, 3, 4}, scale}; - // {5, 8, 1, 6, 9, 0, 4, 7, 2, 3} + // {5, 8, 1, 6, 9, 0, 4, 7, 2, 3} auto const expect_keys = key_wrapper{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; auto const expect_vals_max = fp_wrapper{{5, 8, 8, 6, 9, 9, 9, 7, 7, 7}, scale}; // clang-format on - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals_max, std::move(agg)); } } @@ -198,21 +194,21 @@ TEST_F(groupby_max_scan_struct_test, basic) { auto const keys = key_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = [] { - auto child1 = - strings_column_wrapper{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - auto child2 = fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const expect_keys = key_wrapper{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; auto const expect_vals = [] { - auto child1 = - strings_column_wrapper{"año", "año", "año", "bit", "zit", "zit", "zit", "₹1", "₹1", "₹1"}; - auto child2 = fixed_width_column_wrapper{1, 1, 1, 2, 5, 5, 5, 3, 3, 3}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "año", "año", "bit", "zit", "zit", "zit", "₹1", "₹1", "₹1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 1, 1, 2, 5, 5, 5, 3, 3, 3}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -222,34 +218,34 @@ TEST_F(groupby_max_scan_struct_test, slice_input) auto const keys_original = key_wrapper{dont_care, dont_care, 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, dont_care}; auto const vals_original = [] { - auto child1 = strings_column_wrapper{"dont_care", - "dont_care", - "año", - "bit", - "₹1", - "aaa", - "zit", - "bat", - "aab", - "$1", - "€1", - "wut", - "dont_care"}; + auto child1 = cudf::test::strings_column_wrapper{"dont_care", + "dont_care", + "año", + "bit", + "₹1", + "aaa", + "zit", + "bat", + "aab", + "$1", + "€1", + "wut", + "dont_care"}; auto child2 = key_wrapper{dont_care, dont_care, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, dont_care}; - return structs_column_wrapper{{child1, child2}}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const keys = cudf::slice(keys_original, {2, 12})[0]; auto const vals = cudf::slice(vals_original, {2, 12})[0]; auto const expect_keys = key_wrapper{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; auto const expect_vals = [] { - auto child1 = - strings_column_wrapper{"año", "año", "año", "bit", "zit", "zit", "zit", "₹1", "₹1", "₹1"}; - auto child2 = fixed_width_column_wrapper{1, 1, 1, 2, 5, 5, 5, 3, 3, 3}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "año", "año", "bit", "zit", "zit", "zit", "₹1", "₹1", "₹1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 1, 1, 2, 5, 5, 5, 3, 3, 3}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -258,23 +254,22 @@ TEST_F(groupby_max_scan_struct_test, null_keys_and_values) constexpr int32_t null{0}; auto const keys = key_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; auto const vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "bit", "₹1", "aaa", "zit", "" /*NULL*/, "" /*NULL*/, "$1", "€1", "wut", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; }(); auto const expect_keys = key_wrapper{{1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, no_nulls()}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "año", "" /*NULL*/, "bit", "zit", "" /*NULL*/, "zit", "₹1", "₹1", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 9, null, 8, 5, null, 5, 7, 7, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({2, 5, 9})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 9, null, 8, 5, null, 5, 7, 7, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({2, 5, 9})}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/max_tests.cpp b/cpp/tests/groupby/max_tests.cpp index 1d2c8c489f3..bf5891226ca 100644 --- a/cpp/tests/groupby/max_tests.cpp +++ b/cpp/tests/groupby/max_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,6 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_max_test : public cudf::test::BaseFixture { }; @@ -40,13 +38,13 @@ TYPED_TEST_SUITE(groupby_max_test, cudf::test::FixedWidthTypesWithoutFixedPoint) TYPED_TEST(groupby_max_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals({6, 9, 8}); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals({6, 9, 8}); auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -58,13 +56,13 @@ TYPED_TEST(groupby_max_test, basic) TYPED_TEST(groupby_max_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -76,13 +74,13 @@ TYPED_TEST(groupby_max_test, empty_cols) TYPED_TEST(groupby_max_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals({3, 4, 5}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}); - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -94,13 +92,13 @@ TYPED_TEST(groupby_max_test, zero_valid_keys) TYPED_TEST(groupby_max_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -112,17 +110,17 @@ TYPED_TEST(groupby_max_test, zero_valid_values) TYPED_TEST(groupby_max_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 0, 3, 1, 4, 5, 2, 8, -} - fixed_width_column_wrapper expect_vals({3, 5, 8, 0}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({3, 5, 8, 0}, {1, 1, 1, 0}); auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -136,11 +134,12 @@ struct groupby_max_string_test : public cudf::test::BaseFixture { TEST_F(groupby_max_string_test, basic) { - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::strings_column_wrapper vals{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - strings_column_wrapper expect_vals({"año", "zit", "₹1"}); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::strings_column_wrapper expect_vals({"año", "zit", "₹1"}); auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -151,11 +150,11 @@ TEST_F(groupby_max_string_test, basic) TEST_F(groupby_max_string_test, zero_valid_values) { - fixed_width_column_wrapper keys{1, 1, 1}; - strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - strings_column_wrapper expect_vals({""}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::strings_column_wrapper expect_vals({""}, all_nulls()); auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -186,7 +185,7 @@ TEST_F(groupby_max_string_test, max_sorted_strings) {"06", "10", "14", "18", "22", "26", "30", "34", "38", "42", ""}, {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); - // fixed_width_column_wrapper expect_argmax( + // cudf::test::fixed_width_column_wrapper expect_argmax( // {6, 10, 14, 18, 22, 26, 30, 34, 38, 42, -1}, // {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); auto agg = cudf::make_max_aggregation(); @@ -196,8 +195,8 @@ TEST_F(groupby_max_string_test, max_sorted_strings) expect_vals, std::move(agg), force_use_sort_impl::NO, - null_policy::INCLUDE, - sorted::YES); + cudf::null_policy::INCLUDE, + cudf::sorted::YES); } struct groupby_dictionary_max_test : public cudf::test::BaseFixture { @@ -208,10 +207,10 @@ TEST_F(groupby_dictionary_max_test, basic) using V = std::string; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; - dictionary_column_wrapper vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; - fixed_width_column_wrapper expect_keys { 1, 2, 3 }; - dictionary_column_wrapper expect_vals_w({ "año", "zit", "₹1" }); + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; + cudf::test::dictionary_column_wrapper vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3 }; + cudf::test::dictionary_column_wrapper expect_vals_w({ "año", "zit", "₹1" }); // clang-format on auto expect_vals = cudf::dictionary::set_keys(expect_vals_w, vals.keys()); @@ -234,10 +233,10 @@ TEST_F(groupby_dictionary_max_test, fixed_width) using V = int64_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; - dictionary_column_wrapper vals{ 0xABC, 0xBBB, 0xF1, 0xAAA, 0xFFF, 0xBAA, 0xAAA, 0x01, 0xF1, 0xEEE}; - fixed_width_column_wrapper expect_keys { 1, 2, 3 }; - fixed_width_column_wrapper expect_vals_w({ 0xABC, 0xFFF, 0xF1 }); + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; + cudf::test::dictionary_column_wrapper vals{ 0xABC, 0xBBB, 0xF1, 0xAAA, 0xFFF, 0xBAA, 0xAAA, 0x01, 0xF1, 0xEEE}; + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3 }; + cudf::test::fixed_width_column_wrapper expect_vals_w({ 0xABC, 0xFFF, 0xF1 }); // clang-format on test_single_agg(keys, @@ -254,12 +253,12 @@ TEST_F(groupby_dictionary_max_test, fixed_width) } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupByMaxFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupByMaxFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupBySortMaxDecimalAsValue) +TYPED_TEST(GroupByMaxFixedPointTest, GroupBySortMaxDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; @@ -270,11 +269,11 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortMaxDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_max = fp_wrapper{{6, 9, 8}, scale}; auto agg3 = cudf::make_max_aggregation(); @@ -283,7 +282,7 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortMaxDecimalAsValue) } } -TYPED_TEST(FixedPointTestAllReps, GroupByHashMaxDecimalAsValue) +TYPED_TEST(GroupByMaxFixedPointTest, GroupByHashMaxDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; @@ -294,11 +293,11 @@ TYPED_TEST(FixedPointTestAllReps, GroupByHashMaxDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_max = fp_wrapper{{6, 9, 8}, scale}; auto agg7 = cudf::make_max_aggregation(); @@ -311,82 +310,84 @@ struct groupby_max_struct_test : public cudf::test::BaseFixture { TEST_F(groupby_max_struct_test, basic) { - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = [] { - auto child1 = - strings_column_wrapper{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - auto child2 = fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{"año", "zit", "₹1"}; - auto child2 = fixed_width_column_wrapper{1, 5, 3}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{"año", "zit", "₹1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 5, 3}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TEST_F(groupby_max_struct_test, slice_input) { constexpr int32_t dont_care{1}; - auto const keys_original = fixed_width_column_wrapper{ + auto const keys_original = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, dont_care}; auto const vals_original = [] { - auto child1 = strings_column_wrapper{"dont_care", - "dont_care", - "año", - "bit", - "₹1", - "aaa", - "zit", - "bat", - "aab", - "$1", - "€1", - "wut", - "dont_care"}; - auto child2 = fixed_width_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{"dont_care", + "dont_care", + "año", + "bit", + "₹1", + "aaa", + "zit", + "bat", + "aab", + "$1", + "€1", + "wut", + "dont_care"}; + auto child2 = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, dont_care}; - return structs_column_wrapper{{child1, child2}}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const keys = cudf::slice(keys_original, {2, 12})[0]; auto const vals = cudf::slice(vals_original, {2, 12})[0]; - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{"año", "zit", "₹1"}; - auto child2 = fixed_width_column_wrapper{1, 5, 3}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{"año", "zit", "₹1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 5, 3}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TEST_F(groupby_max_struct_test, null_keys_and_values) { constexpr int32_t null{0}; - auto const keys = - fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; + auto const keys = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; auto const vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "bit", "₹1", "aaa", "zit", "" /*NULL*/, "" /*NULL*/, "$1", "€1", "wut", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; }(); - auto const expect_keys = fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; + auto const expect_keys = + cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{"año", "zit", "₹1", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 5, 7, null}; - return structs_column_wrapper{{child1, child2}, null_at(3)}; + auto child1 = cudf::test::strings_column_wrapper{"año", "zit", "₹1", "" /*NULL*/}; + auto child2 = cudf::test::fixed_width_column_wrapper{9, 5, 7, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, null_at(3)}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -394,40 +395,40 @@ TEST_F(groupby_max_struct_test, values_with_null_child) { constexpr int32_t null{0}; { - auto const keys = fixed_width_column_wrapper{1, 1}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 1}; auto const vals = [] { - auto child1 = fixed_width_column_wrapper{1, 1}; - auto child2 = fixed_width_column_wrapper{{-1, null}, null_at(1)}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{1, 1}; + auto child2 = cudf::test::fixed_width_column_wrapper{{-1, null}, null_at(1)}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto const expect_keys = fixed_width_column_wrapper{1}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1}; auto const expect_vals = [] { - auto child1 = fixed_width_column_wrapper{1}; - auto child2 = fixed_width_column_wrapper{-1}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{1}; + auto child2 = cudf::test::fixed_width_column_wrapper{-1}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } { - auto const keys = fixed_width_column_wrapper{1, 1}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 1}; auto const vals = [] { - auto child1 = fixed_width_column_wrapper{{-1, null}, null_at(1)}; - auto child2 = fixed_width_column_wrapper{{null, null}, nulls_at({0, 1})}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{{-1, null}, null_at(1)}; + auto child2 = cudf::test::fixed_width_column_wrapper{{null, null}, nulls_at({0, 1})}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto const expect_keys = fixed_width_column_wrapper{1}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1}; auto const expect_vals = [] { - auto child1 = fixed_width_column_wrapper{-1}; - auto child2 = fixed_width_column_wrapper{{null}, null_at(0)}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{-1}; + auto child2 = cudf::test::fixed_width_column_wrapper{{null}, null_at(0)}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto agg = cudf::make_max_aggregation(); + auto agg = cudf::make_max_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } } @@ -441,8 +442,8 @@ TYPED_TEST_SUITE(groupby_max_floating_point_test, cudf::test::FloatingPointTypes TYPED_TEST(groupby_max_floating_point_test, values_with_infinity) { using T = TypeParam; - using int32s_col = fixed_width_column_wrapper; - using floats_col = fixed_width_column_wrapper; + using int32s_col = cudf::test::fixed_width_column_wrapper; + using floats_col = cudf::test::fixed_width_column_wrapper; auto constexpr inf = std::numeric_limits::infinity(); @@ -453,7 +454,7 @@ TYPED_TEST(groupby_max_floating_point_test, values_with_infinity) auto const expected_vals = floats_col{inf, static_cast(2)}; // Related issue: https://github.com/rapidsai/cudf/issues/11352 - // The issue only occurs in sort-based aggregation. + // The issue only occurs in sort-based cudf::aggregation. auto agg = cudf::make_max_aggregation(); test_single_agg( keys, vals, expected_keys, expected_vals, std::move(agg), force_use_sort_impl::YES); @@ -462,27 +463,24 @@ TYPED_TEST(groupby_max_floating_point_test, values_with_infinity) TYPED_TEST(groupby_max_floating_point_test, values_with_nan) { using T = TypeParam; - using int32s_col = fixed_width_column_wrapper; - using floats_col = fixed_width_column_wrapper; + using int32s_col = cudf::test::fixed_width_column_wrapper; + using floats_col = cudf::test::fixed_width_column_wrapper; auto constexpr nan = std::numeric_limits::quiet_NaN(); auto const keys = int32s_col{1, 1}; auto const vals = floats_col{nan, nan}; - std::vector requests; - requests.emplace_back(groupby::aggregation_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::aggregation_request()); requests[0].values = vals; requests[0].aggregations.emplace_back(cudf::make_max_aggregation()); // Without properly handling NaN, this will hang forever in hash-based aggregate (which is the // default back-end for min/max in groupby context). // This test is just to verify that the aggregate operation does not hang. - auto gb_obj = groupby::groupby(table_view({keys})); + auto gb_obj = cudf::groupby::groupby(cudf::table_view({keys})); auto const result = gb_obj.aggregate(requests); EXPECT_EQ(result.first->num_rows(), 1); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/mean_tests.cpp b/cpp/tests/groupby/mean_tests.cpp index ab794bf22df..16909dce96d 100644 --- a/cpp/tests/groupby/mean_tests.cpp +++ b/cpp/tests/groupby/mean_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ * limitations under the License. */ -#include -#include +#include #include #include @@ -23,17 +22,13 @@ #include #include -#include +#include #include -#include -#include #include using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_mean_test : public cudf::test::BaseFixture { }; @@ -54,88 +49,89 @@ using K = int32_t; TYPED_TEST(groupby_mean_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; using RT = typename std::conditional(), int, double>::type; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // clang-format off - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} std::vector expect_v = convert( {3., 19. / 4, 17. / 3}); - fixed_width_column_wrapper expect_vals(expect_v.cbegin(), expect_v.cend()); + cudf::test::fixed_width_column_wrapper expect_vals(expect_v.cbegin(), expect_v.cend()); // clang-format on - auto agg = cudf::make_mean_aggregation(); + auto agg = cudf::make_mean_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_mean_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_mean_aggregation(); + auto agg = cudf::make_mean_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_mean_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_mean_aggregation(); + auto agg = cudf::make_mean_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_mean_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_mean_aggregation(); + auto agg = cudf::make_mean_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_mean_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; using RT = typename std::conditional(), int, double>::type; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // clang-format off - // {1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - // {3, 6, 1, 4, 9, 2, 8, -} - std::vector expect_v = convert( {4.5, 14. / 3, 5., 0.}); - fixed_width_column_wrapper expect_vals(expect_v.cbegin(), expect_v.cend(), {1, 1, 1, 0}); + // {1, 1, 2, 2, 2, 3, 3, 4} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + // {3, 6, 1, 4, 9, 2, 8, -} + std::vector expect_v = convert( {4.5, 14. / 3, 5., 0.}); // clang-format on + cudf::test::fixed_width_column_wrapper expect_vals( + expect_v.cbegin(), expect_v.cend(), {1, 1, 1, 0}); - auto agg = cudf::make_mean_aggregation(); + auto agg = cudf::make_mean_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } // clang-format on @@ -146,18 +142,18 @@ struct groupby_dictionary_mean_test : public cudf::test::BaseFixture { TEST_F(groupby_dictionary_mean_test, basic) { using V = int16_t; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expect_keys( {1, 2, 3}); - fixed_width_column_wrapper expect_vals({9. / 3, 19. / 4, 17. / 3}); + cudf::test::fixed_width_column_wrapper expect_keys( {1, 2, 3}); + cudf::test::fixed_width_column_wrapper expect_vals({9. / 3, 19. / 4, 17. / 3}); // clang-format on test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_mean_aggregation()); + keys, vals, expect_keys, expect_vals, cudf::make_mean_aggregation()); } template @@ -176,11 +172,11 @@ TYPED_TEST(FixedPointTestBothReps, GroupBySortMeanDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_min = fp_wrapper{{3, 4, 5}, scale}; auto agg = cudf::make_mean_aggregation(); @@ -200,17 +196,14 @@ TYPED_TEST(FixedPointTestBothReps, GroupByHashMeanDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_min = fp_wrapper{{3, 4, 5}, scale}; auto agg = cudf::make_mean_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals_min, std::move(agg)); } } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/median_tests.cpp b/cpp/tests/groupby/median_tests.cpp index 087b104539e..3ba9e1afc1b 100644 --- a/cpp/tests/groupby/median_tests.cpp +++ b/cpp/tests/groupby/median_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,118 +25,121 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_median_test : public cudf::test::BaseFixture { }; -using K = int32_t; using supported_types = cudf::test::Types; TYPED_TEST_SUITE(groupby_median_test, supported_types); TYPED_TEST(groupby_median_test, basic) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // clang-format off - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({3., 4.5, 7.}, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({3., 4.5, 7.}, no_nulls()); // clang-format on - auto agg = cudf::make_median_aggregation(); + auto agg = cudf::make_median_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_median_test, empty_cols) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_median_aggregation(); + auto agg = cudf::make_median_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_median_test, zero_valid_keys) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_median_aggregation(); + auto agg = cudf::make_median_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_median_test, zero_valid_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_median_aggregation(); + auto agg = cudf::make_median_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_median_test, null_keys_and_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals({4.5, 4., 5., 0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({4.5, 4., 5., 0.}, {1, 1, 1, 0}); - auto agg = cudf::make_median_aggregation(); + auto agg = cudf::make_median_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_median_test, dictionary) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({1, 2, 3 }); - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({3., 4.5, 7. }, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3 }); + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({3., 4.5, 7. }, no_nulls()); // clang-format on - test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_median_aggregation()); + test_single_agg(keys, + vals, + expect_keys, + expect_vals, + cudf::make_median_aggregation()); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/min_scan_tests.cpp b/cpp/tests/groupby/min_scan_tests.cpp index c672209c7b0..fc5a395299d 100644 --- a/cpp/tests/groupby/min_scan_tests.cpp +++ b/cpp/tests/groupby/min_scan_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,19 +23,14 @@ #include -using namespace cudf::test::iterators; - -namespace cudf { -namespace test { -using K = int32_t; -using key_wrapper = fixed_width_column_wrapper; +using key_wrapper = cudf::test::fixed_width_column_wrapper; template struct groupby_min_scan_test : public cudf::test::BaseFixture { using V = T; - using R = cudf::detail::target_type_t; - using value_wrapper = fixed_width_column_wrapper; - using result_wrapper = fixed_width_column_wrapper; + using R = cudf::detail::target_type_t; + using value_wrapper = cudf::test::fixed_width_column_wrapper; + using result_wrapper = cudf::test::fixed_width_column_wrapper; }; TYPED_TEST_SUITE(groupby_min_scan_test, cudf::test::FixedWidthTypesWithoutFixedPoint); @@ -53,7 +48,7 @@ TYPED_TEST(groupby_min_scan_test, basic) result_wrapper expect_vals({5, 5, 1, 6, 6, 0, 0, 7, 2, 2}); // clang-format on - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -70,9 +65,14 @@ TYPED_TEST(groupby_min_scan_test, pre_sorted) result_wrapper expect_vals({5, 5, 1, 6, 6, 0, 0, 7, 2, 2}); // clang-format on - auto agg = cudf::make_min_aggregation(); - test_single_scan( - keys, vals, expect_keys, expect_vals, std::move(agg), null_policy::EXCLUDE, sorted::YES); + auto agg = cudf::make_min_aggregation(); + test_single_scan(keys, + vals, + expect_keys, + expect_vals, + std::move(agg), + cudf::null_policy::EXCLUDE, + cudf::sorted::YES); } TYPED_TEST(groupby_min_scan_test, empty_cols) @@ -82,11 +82,10 @@ TYPED_TEST(groupby_min_scan_test, empty_cols) key_wrapper keys{}; value_wrapper vals{}; - key_wrapper expect_keys{}; result_wrapper expect_vals{}; - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -95,15 +94,12 @@ TYPED_TEST(groupby_min_scan_test, zero_valid_keys) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys({1, 2, 3}, all_nulls()); + key_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); value_wrapper vals({3, 4, 5}); - key_wrapper expect_keys{}; result_wrapper expect_vals{}; - // clang-format on - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -112,15 +108,12 @@ TYPED_TEST(groupby_min_scan_test, zero_valid_values) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys {1, 1, 1}; - value_wrapper vals({3, 4, 5}, all_nulls()); - - key_wrapper expect_keys {1, 1, 1}; - result_wrapper expect_vals({-1, -1, -1}, all_nulls()); - // clang-format on + key_wrapper keys{1, 1, 1}; + value_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); + key_wrapper expect_keys{1, 1, 1}; + result_wrapper expect_vals({-1, -1, -1}, cudf::test::iterators::all_nulls()); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -134,13 +127,13 @@ TYPED_TEST(groupby_min_scan_test, null_keys_and_values) value_wrapper vals({5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 4}, {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 1, 2, 2, 2, 2, 3, _, 3, 4} - key_wrapper expect_keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, no_nulls()); + key_wrapper expect_keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, cudf::test::iterators::no_nulls()); // { _, 8, 1, 6, 9, _, 4, 7, 2, 3, _} result_wrapper expect_vals({-1, 8, 1, 6, 6, -1, 4, 7, 3, -1}, { 0, 1, 1, 1, 1, 0, 1, 1, 1, 0}); // clang-format on - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -150,28 +143,29 @@ struct groupby_min_scan_string_test : public cudf::test::BaseFixture { TEST_F(groupby_min_scan_string_test, basic) { key_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; + cudf::test::strings_column_wrapper vals{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; key_wrapper expect_keys{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; - strings_column_wrapper expect_vals( + cudf::test::strings_column_wrapper expect_vals( {"año", "aaa", "aaa", "bit", "bit", "bat", "bat", "₹1", "$1", "$1"}); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupByMinScanFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupByMinScanFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupBySortMinScanDecimalAsValue) +TYPED_TEST(GroupByMinScanFixedPointTest, GroupBySortMinScanDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; using RepType = cudf::device_storage_type_t; - using fp_wrapper = fixed_point_column_wrapper; + using fp_wrapper = cudf::test::fixed_point_column_wrapper; for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; @@ -185,7 +179,7 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortMinScanDecimalAsValue) auto const expect_vals_min = fp_wrapper{{5, 5, 1, 6, 6, 0, 0, 7, 2, 2}, scale}; // clang-format on - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals_min, std::move(agg)); } } @@ -197,21 +191,21 @@ TEST_F(groupby_min_scan_struct_test, basic) { auto const keys = key_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = [] { - auto child1 = - strings_column_wrapper{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - auto child2 = fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const expect_keys = key_wrapper{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; auto const expect_vals = [] { - auto child1 = - strings_column_wrapper{"año", "aaa", "aaa", "bit", "bit", "bat", "bat", "₹1", "$1", "$1"}; - auto child2 = fixed_width_column_wrapper{1, 4, 4, 2, 2, 6, 6, 3, 8, 8}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "aaa", "aaa", "bit", "bit", "bat", "bat", "₹1", "$1", "$1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 4, 4, 2, 2, 6, 6, 3, 8, 8}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -221,59 +215,62 @@ TEST_F(groupby_min_scan_struct_test, slice_input) auto const keys_original = key_wrapper{dont_care, dont_care, 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, dont_care}; auto const vals_original = [] { - auto child1 = strings_column_wrapper{"dont_care", - "dont_care", - "año", - "bit", - "₹1", - "aaa", - "zit", - "bat", - "aab", - "$1", - "€1", - "wut", - "dont_care"}; + auto child1 = cudf::test::strings_column_wrapper{"dont_care", + "dont_care", + "año", + "bit", + "₹1", + "aaa", + "zit", + "bat", + "aab", + "$1", + "€1", + "wut", + "dont_care"}; auto child2 = key_wrapper{dont_care, dont_care, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, dont_care}; - return structs_column_wrapper{{child1, child2}}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const keys = cudf::slice(keys_original, {2, 12})[0]; auto const vals = cudf::slice(vals_original, {2, 12})[0]; auto const expect_keys = key_wrapper{1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; auto const expect_vals = [] { - auto child1 = - strings_column_wrapper{"año", "aaa", "aaa", "bit", "bit", "bat", "bat", "₹1", "$1", "$1"}; - auto child2 = fixed_width_column_wrapper{1, 4, 4, 2, 2, 6, 6, 3, 8, 8}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "aaa", "aaa", "bit", "bit", "bat", "bat", "₹1", "$1", "$1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 4, 4, 2, 2, 6, 6, 3, 8, 8}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } TEST_F(groupby_min_scan_struct_test, null_keys_and_values) { constexpr int32_t null{0}; - auto const keys = key_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; + auto const keys = + key_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, cudf::test::iterators::null_at(7)}; auto const vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "bit", "₹1", "aaa", "zit", "" /*NULL*/, "" /*NULL*/, "$1", "€1", "wut", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, + cudf::test::iterators::nulls_at({5, 6, 10})}; }(); - auto const expect_keys = key_wrapper{{1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, no_nulls()}; + auto const expect_keys = + key_wrapper{{1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, cudf::test::iterators::no_nulls()}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "aaa", "" /*NULL*/, "bit", "bit", "" /*NULL*/, "bit", "₹1", "€1", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 6, null, 8, 8, null, 8, 7, 1, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({2, 5, 9})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 6, null, 8, 8, null, 8, 7, 1, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, + cudf::test::iterators::nulls_at({2, 5, 9})}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/min_tests.cpp b/cpp/tests/groupby/min_tests.cpp index 9606c8c55ee..c3a4f965c91 100644 --- a/cpp/tests/groupby/min_tests.cpp +++ b/cpp/tests/groupby/min_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,6 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_min_test : public cudf::test::BaseFixture { }; @@ -40,13 +38,13 @@ TYPED_TEST_SUITE(groupby_min_test, cudf::test::FixedWidthTypesWithoutFixedPoint) TYPED_TEST(groupby_min_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals({0, 1, 2}); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals({0, 1, 2}); auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -58,13 +56,13 @@ TYPED_TEST(groupby_min_test, basic) TYPED_TEST(groupby_min_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -76,13 +74,13 @@ TYPED_TEST(groupby_min_test, empty_cols) TYPED_TEST(groupby_min_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals({3, 4, 5}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}); - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -94,13 +92,13 @@ TYPED_TEST(groupby_min_test, zero_valid_keys) TYPED_TEST(groupby_min_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -112,17 +110,17 @@ TYPED_TEST(groupby_min_test, zero_valid_values) TYPED_TEST(groupby_min_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals({3, 1, 2, 0}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({3, 1, 2, 0}, {1, 1, 1, 0}); auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -136,11 +134,12 @@ struct groupby_min_string_test : public cudf::test::BaseFixture { TEST_F(groupby_min_string_test, basic) { - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::strings_column_wrapper vals{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - strings_column_wrapper expect_vals({"aaa", "bat", "$1"}); + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::strings_column_wrapper expect_vals({"aaa", "bat", "$1"}); auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -151,11 +150,11 @@ TEST_F(groupby_min_string_test, basic) TEST_F(groupby_min_string_test, zero_valid_values) { - fixed_width_column_wrapper keys{1, 1, 1}; - strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::strings_column_wrapper vals({"año", "bit", "₹1"}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - strings_column_wrapper expect_vals({""}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::strings_column_wrapper expect_vals({""}, all_nulls()); auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); @@ -186,9 +185,6 @@ TEST_F(groupby_min_string_test, min_sorted_strings) {"06", "10", "14", "18", "22", "26", "30", "34", "38", "42", ""}, {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); - // fixed_width_column_wrapper expect_argmin( - // {6, 10, 14, 18, 22, 26, 30, 34, 38, 42, -1}, - // {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0}); auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, @@ -196,8 +192,8 @@ TEST_F(groupby_min_string_test, min_sorted_strings) expect_vals, std::move(agg), force_use_sort_impl::NO, - null_policy::INCLUDE, - sorted::YES); + cudf::null_policy::INCLUDE, + cudf::sorted::YES); } struct groupby_dictionary_min_test : public cudf::test::BaseFixture { @@ -208,10 +204,10 @@ TEST_F(groupby_dictionary_min_test, basic) using V = std::string; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; - dictionary_column_wrapper vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; - fixed_width_column_wrapper expect_keys { 1, 2, 3 }; - dictionary_column_wrapper expect_vals_w({ "aaa", "bat", "$1" }); + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; + cudf::test::dictionary_column_wrapper vals{ "año", "bit", "₹1", "aaa", "zit", "bat", "aaa", "$1", "₹1", "wut"}; + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3 }; + cudf::test::dictionary_column_wrapper expect_vals_w({ "aaa", "bat", "$1" }); // clang-format on auto expect_vals = cudf::dictionary::set_keys(expect_vals_w, vals.keys()); @@ -234,10 +230,10 @@ TEST_F(groupby_dictionary_min_test, fixed_width) using V = int64_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; - dictionary_column_wrapper vals{ 0xABC, 0xBBB, 0xF1, 0xAAA, 0xFFF, 0xBAA, 0xAAA, 0x01, 0xF1, 0xEEE}; - fixed_width_column_wrapper expect_keys { 1, 2, 3 }; - fixed_width_column_wrapper expect_vals_w({ 0xAAA, 0xBAA, 0x01 }); + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }; + cudf::test::dictionary_column_wrapper vals{ 0xABC, 0xBBB, 0xF1, 0xAAA, 0xFFF, 0xBAA, 0xAAA, 0x01, 0xF1, 0xEEE}; + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3 }; + cudf::test::fixed_width_column_wrapper expect_vals_w({ 0xAAA, 0xBAA, 0x01 }); // clang-format on test_single_agg(keys, @@ -254,12 +250,12 @@ TEST_F(groupby_dictionary_min_test, fixed_width) } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupByMinFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupByMinFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupBySortMinDecimalAsValue) +TYPED_TEST(GroupByMinFixedPointTest, GroupBySortMinDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; @@ -269,11 +265,11 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortMinDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_min = fp_wrapper{{0, 1, 2}, scale}; auto agg2 = cudf::make_min_aggregation(); @@ -282,7 +278,7 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortMinDecimalAsValue) } } -TYPED_TEST(FixedPointTestAllReps, GroupByHashMinDecimalAsValue) +TYPED_TEST(GroupByMinFixedPointTest, GroupByHashMinDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; @@ -293,11 +289,11 @@ TYPED_TEST(FixedPointTestAllReps, GroupByHashMinDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_min = fp_wrapper{{0, 1, 2}, scale}; auto agg6 = cudf::make_min_aggregation(); @@ -310,82 +306,84 @@ struct groupby_min_struct_test : public cudf::test::BaseFixture { TEST_F(groupby_min_struct_test, basic) { - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto const vals = [] { - auto child1 = - strings_column_wrapper{"año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; - auto child2 = fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{ + "año", "bit", "₹1", "aaa", "zit", "bat", "aab", "$1", "€1", "wut"}; + auto child2 = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{"aaa", "bat", "$1"}; - auto child2 = fixed_width_column_wrapper{4, 6, 8}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{"aaa", "bat", "$1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{4, 6, 8}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TEST_F(groupby_min_struct_test, slice_input) { constexpr int32_t dont_care{1}; - auto const keys_original = fixed_width_column_wrapper{ + auto const keys_original = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, dont_care}; auto const vals_original = [] { - auto child1 = strings_column_wrapper{"dont_care", - "dont_care", - "año", - "bit", - "₹1", - "aaa", - "zit", - "bat", - "aab", - "$1", - "€1", - "wut", - "dont_care"}; - auto child2 = fixed_width_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{"dont_care", + "dont_care", + "año", + "bit", + "₹1", + "aaa", + "zit", + "bat", + "aab", + "$1", + "€1", + "wut", + "dont_care"}; + auto child2 = cudf::test::fixed_width_column_wrapper{ dont_care, dont_care, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, dont_care}; - return structs_column_wrapper{{child1, child2}}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); auto const keys = cudf::slice(keys_original, {2, 12})[0]; auto const vals = cudf::slice(vals_original, {2, 12})[0]; - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{"aaa", "bat", "$1"}; - auto child2 = fixed_width_column_wrapper{4, 6, 8}; - return structs_column_wrapper{{child1, child2}}; + auto child1 = cudf::test::strings_column_wrapper{"aaa", "bat", "$1"}; + auto child2 = cudf::test::fixed_width_column_wrapper{4, 6, 8}; + return cudf::test::structs_column_wrapper{{child1, child2}}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TEST_F(groupby_min_struct_test, null_keys_and_values) { constexpr int32_t null{0}; - auto const keys = - fixed_width_column_wrapper{{1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; + auto const keys = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 1, 2, 2, 1, null, 3, 2, 4}, null_at(7)}; auto const vals = [] { - auto child1 = strings_column_wrapper{ + auto child1 = cudf::test::strings_column_wrapper{ "año", "bit", "₹1", "aaa", "zit", "" /*NULL*/, "" /*NULL*/, "$1", "€1", "wut", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; - return structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; + auto child2 = + cudf::test::fixed_width_column_wrapper{9, 8, 7, 6, 5, null, null, 2, 1, 0, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, nulls_at({5, 6, 10})}; }(); - auto const expect_keys = fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; + auto const expect_keys = + cudf::test::fixed_width_column_wrapper{{1, 2, 3, 4}, no_nulls()}; auto const expect_vals = [] { - auto child1 = strings_column_wrapper{"aaa", "bit", "€1", "" /*NULL*/}; - auto child2 = fixed_width_column_wrapper{6, 8, 1, null}; - return structs_column_wrapper{{child1, child2}, null_at(3)}; + auto child1 = cudf::test::strings_column_wrapper{"aaa", "bit", "€1", "" /*NULL*/}; + auto child2 = cudf::test::fixed_width_column_wrapper{6, 8, 1, null}; + return cudf::test::structs_column_wrapper{{child1, child2}, null_at(3)}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -393,40 +391,40 @@ TEST_F(groupby_min_struct_test, values_with_null_child) { constexpr int32_t null{0}; { - auto const keys = fixed_width_column_wrapper{1, 1}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 1}; auto const vals = [] { - auto child1 = fixed_width_column_wrapper{1, 1}; - auto child2 = fixed_width_column_wrapper{{-1, null}, null_at(1)}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{1, 1}; + auto child2 = cudf::test::fixed_width_column_wrapper{{-1, null}, null_at(1)}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto const expect_keys = fixed_width_column_wrapper{1}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1}; auto const expect_vals = [] { - auto child1 = fixed_width_column_wrapper{1}; - auto child2 = fixed_width_column_wrapper{{null}, null_at(0)}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{1}; + auto child2 = cudf::test::fixed_width_column_wrapper{{null}, null_at(0)}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } { - auto const keys = fixed_width_column_wrapper{1, 1}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 1}; auto const vals = [] { - auto child1 = fixed_width_column_wrapper{{-1, null}, null_at(1)}; - auto child2 = fixed_width_column_wrapper{{null, null}, nulls_at({0, 1})}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{{-1, null}, null_at(1)}; + auto child2 = cudf::test::fixed_width_column_wrapper{{null, null}, nulls_at({0, 1})}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto const expect_keys = fixed_width_column_wrapper{1}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1}; auto const expect_vals = [] { - auto child1 = fixed_width_column_wrapper{{null}, null_at(0)}; - auto child2 = fixed_width_column_wrapper{{null}, null_at(0)}; - return structs_column_wrapper{child1, child2}; + auto child1 = cudf::test::fixed_width_column_wrapper{{null}, null_at(0)}; + auto child2 = cudf::test::fixed_width_column_wrapper{{null}, null_at(0)}; + return cudf::test::structs_column_wrapper{child1, child2}; }(); - auto agg = cudf::make_min_aggregation(); + auto agg = cudf::make_min_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } } @@ -440,8 +438,8 @@ TYPED_TEST_SUITE(groupby_min_floating_point_test, cudf::test::FloatingPointTypes TYPED_TEST(groupby_min_floating_point_test, values_with_infinity) { using T = TypeParam; - using int32s_col = fixed_width_column_wrapper; - using floats_col = fixed_width_column_wrapper; + using int32s_col = cudf::test::fixed_width_column_wrapper; + using floats_col = cudf::test::fixed_width_column_wrapper; auto constexpr inf = std::numeric_limits::infinity(); @@ -461,27 +459,24 @@ TYPED_TEST(groupby_min_floating_point_test, values_with_infinity) TYPED_TEST(groupby_min_floating_point_test, values_with_nan) { using T = TypeParam; - using int32s_col = fixed_width_column_wrapper; - using floats_col = fixed_width_column_wrapper; + using int32s_col = cudf::test::fixed_width_column_wrapper; + using floats_col = cudf::test::fixed_width_column_wrapper; auto constexpr nan = std::numeric_limits::quiet_NaN(); auto const keys = int32s_col{1, 1}; auto const vals = floats_col{nan, nan}; - std::vector requests; - requests.emplace_back(groupby::aggregation_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::aggregation_request()); requests[0].values = vals; requests[0].aggregations.emplace_back(cudf::make_min_aggregation()); // Without properly handling NaN, this will hang forever in hash-based aggregate (which is the // default back-end for min/max in groupby context). // This test is just to verify that the aggregate operation does not hang. - auto gb_obj = groupby::groupby(table_view({keys})); + auto gb_obj = cudf::groupby::groupby(cudf::table_view({keys})); auto const result = gb_obj.aggregate(requests); EXPECT_EQ(result.first->num_rows(), 1); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/nth_element_tests.cpp b/cpp/tests/groupby/nth_element_tests.cpp index 976064de344..8c721f0b795 100644 --- a/cpp/tests/groupby/nth_element_tests.cpp +++ b/cpp/tests/groupby/nth_element_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,9 +26,6 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { - template struct groupby_nth_element_test : public cudf::test::BaseFixture { }; @@ -40,26 +37,26 @@ TYPED_TEST(groupby_nth_element_test, basic) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); - //keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; - //vals {0, 3, 6, 1, 4, 5, 9, 2, 7, 8}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + //keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; + //vals {0, 3, 6, 1, 4, 5, 9, 2, 7, 8}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; //groupby.first() - auto agg = cudf::make_nth_element_aggregation(0); - fixed_width_column_wrapper expect_vals0({0, 1, 2}); + auto agg = cudf::make_nth_element_aggregation(0); + cudf::test::fixed_width_column_wrapper expect_vals0({0, 1, 2}); test_single_agg(keys, vals, expect_keys, expect_vals0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(1); - fixed_width_column_wrapper expect_vals1({3, 4, 7}); + agg = cudf::make_nth_element_aggregation(1); + cudf::test::fixed_width_column_wrapper expect_vals1({3, 4, 7}); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(2); - fixed_width_column_wrapper expect_vals2({6, 5, 8}); + agg = cudf::make_nth_element_aggregation(2); + cudf::test::fixed_width_column_wrapper expect_vals2({6, 5, 8}); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg)); } @@ -67,15 +64,15 @@ TYPED_TEST(groupby_nth_element_test, empty_cols) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_nth_element_aggregation(0); + auto agg = cudf::make_nth_element_aggregation(0); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -83,15 +80,15 @@ TYPED_TEST(groupby_nth_element_test, basic_out_of_bounds) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 3, 2, 2, 9}); + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 3, 2, 2, 9}); - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; - auto agg = cudf::make_nth_element_aggregation(3); - fixed_width_column_wrapper expect_vals({0, 9, 0}, {0, 1, 0}); + auto agg = cudf::make_nth_element_aggregation(3); + cudf::test::fixed_width_column_wrapper expect_vals({0, 9, 0}, {0, 1, 0}); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -99,26 +96,26 @@ TYPED_TEST(groupby_nth_element_test, negative) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); //keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; //vals {0, 3, 6, 1, 4, 5, 9, 2, 7, 8}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; //groupby.last() - auto agg = cudf::make_nth_element_aggregation(-1); - fixed_width_column_wrapper expect_vals0({6, 9, 8}); + auto agg = cudf::make_nth_element_aggregation(-1); + cudf::test::fixed_width_column_wrapper expect_vals0({6, 9, 8}); test_single_agg(keys, vals, expect_keys, expect_vals0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-2); - fixed_width_column_wrapper expect_vals1({3, 5, 7}); + agg = cudf::make_nth_element_aggregation(-2); + cudf::test::fixed_width_column_wrapper expect_vals1({3, 5, 7}); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-3); - fixed_width_column_wrapper expect_vals2({0, 4, 2}); + agg = cudf::make_nth_element_aggregation(-3); + cudf::test::fixed_width_column_wrapper expect_vals2({0, 4, 2}); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg)); } @@ -126,15 +123,15 @@ TYPED_TEST(groupby_nth_element_test, negative_out_of_bounds) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 3, 2, 2, 9}); + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 3, 2, 2, 9}); - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; - auto agg = cudf::make_nth_element_aggregation(-4); - fixed_width_column_wrapper expect_vals({0, 1, 0}, {0, 1, 0}); + auto agg = cudf::make_nth_element_aggregation(-4); + cudf::test::fixed_width_column_wrapper expect_vals({0, 1, 0}, {0, 1, 0}); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -142,15 +139,15 @@ TYPED_TEST(groupby_nth_element_test, zero_valid_keys) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals({3, 4, 5}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}); - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_nth_element_aggregation(0); + auto agg = cudf::make_nth_element_aggregation(0); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -158,15 +155,15 @@ TYPED_TEST(groupby_nth_element_test, zero_valid_values) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({3}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({3}, all_nulls()); - auto agg = cudf::make_nth_element_aggregation(0); + auto agg = cudf::make_nth_element_aggregation(0); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -174,19 +171,19 @@ TYPED_TEST(groupby_nth_element_test, null_keys_and_values) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); //keys {1, 1, 1 2,2,2,2 3, 3, 4} //vals {-,3,6, 1,4,-,9, 2,8, -} - fixed_width_column_wrapper expect_vals({-1, 1, 2, -1}, {0, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({-1, 1, 2, -1}, {0, 1, 1, 0}); - auto agg = cudf::make_nth_element_aggregation(0); + auto agg = cudf::make_nth_element_aggregation(0); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -194,19 +191,19 @@ TYPED_TEST(groupby_nth_element_test, null_keys_and_values_out_of_bounds) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // {1, 1, 1 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // {-,3,6, 1,4,-,9, 2,8, -} // value, null, out, out - fixed_width_column_wrapper expect_vals({6, -1, -1, -1}, {1, 0, 0, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({6, -1, -1, -1}, {1, 0, 0, 0}); - auto agg = cudf::make_nth_element_aggregation(2); + auto agg = cudf::make_nth_element_aggregation(2); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -214,41 +211,41 @@ TYPED_TEST(groupby_nth_element_test, exclude_nulls) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0}); - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); //keys {1, 1, 1 2, 2, 2, 2 3, 3, 3 4} //vals {-, 3, 6 1, 4, -, 9, - 2, 2, 8, 4,-} // 0 null, value, value, null // 1 value, value, value, null // 2 value, null, value, out //null_policy::INCLUDE - fixed_width_column_wrapper expect_nuls0({-1, 1, 2, 4}, {0, 1, 1, 1}); - fixed_width_column_wrapper expect_nuls1({3, 4, 2, -1}, {1, 1, 1, 0}); - fixed_width_column_wrapper expect_nuls2({6, -1, 8, -1}, {1, 0, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_nuls0({-1, 1, 2, 4}, {0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper expect_nuls1({3, 4, 2, -1}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_nuls2({6, -1, 8, -1}, {1, 0, 1, 0}); //null_policy::EXCLUDE - fixed_width_column_wrapper expect_vals0({3, 1, 2, 4}); - fixed_width_column_wrapper expect_vals1({6, 4, 2, -1}, {1, 1, 1, 0}); - fixed_width_column_wrapper expect_vals2({-1, 9, 8, -1}, {0, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals0({3, 1, 2, 4}); + cudf::test::fixed_width_column_wrapper expect_vals1({6, 4, 2, -1}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals2({-1, 9, 8, -1}, {0, 1, 1, 0}); - auto agg = cudf::make_nth_element_aggregation(0, cudf::null_policy::INCLUDE); + auto agg = cudf::make_nth_element_aggregation(0, cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_nuls0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(1, cudf::null_policy::INCLUDE); + agg = cudf::make_nth_element_aggregation(1, cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_nuls1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(2, cudf::null_policy::INCLUDE); + agg = cudf::make_nth_element_aggregation(2, cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_nuls2, std::move(agg)); - agg = cudf::make_nth_element_aggregation(0, cudf::null_policy::EXCLUDE); + agg = cudf::make_nth_element_aggregation(0, cudf::null_policy::EXCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(1, cudf::null_policy::EXCLUDE); + agg = cudf::make_nth_element_aggregation(1, cudf::null_policy::EXCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(2, cudf::null_policy::EXCLUDE); + agg = cudf::make_nth_element_aggregation(2, cudf::null_policy::EXCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg)); } @@ -256,14 +253,14 @@ TYPED_TEST(groupby_nth_element_test, exclude_nulls_negative_index) { using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0}); - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); //keys {1, 1, 1 2, 2, 2, 3, 3, 4} //vals {-, 3, 6 1, 4, -, 9, - 2, 2, 8, 4,-} // 0 null, value, value, value @@ -273,27 +270,27 @@ TYPED_TEST(groupby_nth_element_test, exclude_nulls_negative_index) // 4 out, null, out, out //null_policy::INCLUDE - fixed_width_column_wrapper expect_nuls0({6, -1, 8, -1}, {1, 0, 1, 0}); - fixed_width_column_wrapper expect_nuls1({3, 9, 2, 4}); - fixed_width_column_wrapper expect_nuls2({-1, -1, 2, -1}, {0, 0, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_nuls0({6, -1, 8, -1}, {1, 0, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_nuls1({3, 9, 2, 4}); + cudf::test::fixed_width_column_wrapper expect_nuls2({-1, -1, 2, -1}, {0, 0, 1, 0}); //null_policy::EXCLUDE - fixed_width_column_wrapper expect_vals0({6, 9, 8, 4}); - fixed_width_column_wrapper expect_vals1({3, 4, 2, -1}, {1, 1, 1, 0}); - fixed_width_column_wrapper expect_vals2({-1, 1, 2, -1}, {0, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals0({6, 9, 8, 4}); + cudf::test::fixed_width_column_wrapper expect_vals1({3, 4, 2, -1}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals2({-1, 1, 2, -1}, {0, 1, 1, 0}); - auto agg = cudf::make_nth_element_aggregation(-1, cudf::null_policy::INCLUDE); + auto agg = cudf::make_nth_element_aggregation(-1, cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_nuls0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-2, cudf::null_policy::INCLUDE); + agg = cudf::make_nth_element_aggregation(-2, cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_nuls1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-3, cudf::null_policy::INCLUDE); + agg = cudf::make_nth_element_aggregation(-3, cudf::null_policy::INCLUDE); test_single_agg(keys, vals, expect_keys, expect_nuls2, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-1, cudf::null_policy::EXCLUDE); + agg = cudf::make_nth_element_aggregation(-1, cudf::null_policy::EXCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-2, cudf::null_policy::EXCLUDE); + agg = cudf::make_nth_element_aggregation(-2, cudf::null_policy::EXCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-3, cudf::null_policy::EXCLUDE); + agg = cudf::make_nth_element_aggregation(-3, cudf::null_policy::EXCLUDE); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg)); } @@ -304,47 +301,45 @@ TEST_F(groupby_nth_element_string_test, basic_string) { using K = int32_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - strings_column_wrapper vals{"ABCD", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; - //keys {1, 1, 1, 2, 2, 2, 2, 3, 3, 3}; - //vals {A, 3, 6, 1, 4, 5, 9, 2, 7, 8}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::strings_column_wrapper vals{"ABCD", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; //groupby.first() - auto agg = cudf::make_nth_element_aggregation(0); - strings_column_wrapper expect_vals0{"ABCD", "1", "2"}; + auto agg = cudf::make_nth_element_aggregation(0); + cudf::test::strings_column_wrapper expect_vals0{"ABCD", "1", "2"}; test_single_agg(keys, vals, expect_keys, expect_vals0, std::move(agg)); - agg = cudf::make_nth_element_aggregation(1); - strings_column_wrapper expect_vals1{"3", "4", "7"}; + agg = cudf::make_nth_element_aggregation(1); + cudf::test::strings_column_wrapper expect_vals1{"3", "4", "7"}; test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg)); - agg = cudf::make_nth_element_aggregation(2); - strings_column_wrapper expect_vals2{"6", "5", "8"}; + agg = cudf::make_nth_element_aggregation(2); + cudf::test::strings_column_wrapper expect_vals2{"6", "5", "8"}; test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg)); //+ve out of bounds - agg = cudf::make_nth_element_aggregation(3); - strings_column_wrapper expect_vals3{{"", "9", ""}, {0, 1, 0}}; + agg = cudf::make_nth_element_aggregation(3); + cudf::test::strings_column_wrapper expect_vals3{{"", "9", ""}, {0, 1, 0}}; test_single_agg(keys, vals, expect_keys, expect_vals3, std::move(agg)); //groupby.last() - agg = cudf::make_nth_element_aggregation(-1); - strings_column_wrapper expect_vals4{"6", "9", "8"}; + agg = cudf::make_nth_element_aggregation(-1); + cudf::test::strings_column_wrapper expect_vals4{"6", "9", "8"}; test_single_agg(keys, vals, expect_keys, expect_vals4, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-2); - strings_column_wrapper expect_vals5{"3", "5", "7"}; + agg = cudf::make_nth_element_aggregation(-2); + cudf::test::strings_column_wrapper expect_vals5{"3", "5", "7"}; test_single_agg(keys, vals, expect_keys, expect_vals5, std::move(agg)); - agg = cudf::make_nth_element_aggregation(-3); - strings_column_wrapper expect_vals6{"ABCD", "4", "2"}; + agg = cudf::make_nth_element_aggregation(-3); + cudf::test::strings_column_wrapper expect_vals6{"ABCD", "4", "2"}; test_single_agg(keys, vals, expect_keys, expect_vals6, std::move(agg)); //-ve out of bounds - agg = cudf::make_nth_element_aggregation(-4); - strings_column_wrapper expect_vals7{{"", "1", ""}, {0, 1, 0}}; + agg = cudf::make_nth_element_aggregation(-4); + cudf::test::strings_column_wrapper expect_vals7{{"", "1", ""}, {0, 1, 0}}; test_single_agg(keys, vals, expect_keys, expect_vals7, std::move(agg)); } // clang-format on @@ -354,10 +349,10 @@ TEST_F(groupby_nth_element_string_test, dictionary) using K = int32_t; using V = std::string; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{"AB", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - dictionary_column_wrapper expect_vals_w{"6", "5", "8"}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{"AB", "1", "2", "3", "4", "5", "6", "7", "8", "9"}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::dictionary_column_wrapper expect_vals_w{"6", "5", "8"}; auto expect_vals = cudf::dictionary::set_keys(expect_vals_w, vals.keys()); @@ -365,14 +360,14 @@ TEST_F(groupby_nth_element_string_test, dictionary) vals, expect_keys, expect_vals->view(), - cudf::make_nth_element_aggregation(2)); + cudf::make_nth_element_aggregation(2)); } template -struct groupby_nth_element_lists_test : BaseFixture { +struct groupby_nth_element_lists_test : cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(groupby_nth_element_lists_test, FixedWidthTypesWithoutFixedPoint); +TYPED_TEST_SUITE(groupby_nth_element_lists_test, cudf::test::FixedWidthTypesWithoutFixedPoint); TYPED_TEST(groupby_nth_element_lists_test, Basics) { @@ -381,17 +376,17 @@ TYPED_TEST(groupby_nth_element_lists_test, Basics) using lists = cudf::test::lists_column_wrapper; - auto keys = fixed_width_column_wrapper{1, 1, 2, 2, 3, 3}; + auto keys = cudf::test::fixed_width_column_wrapper{1, 1, 2, 2, 3, 3}; auto values = lists{{1, 2}, {3, 4}, {5, 6, 7}, lists{}, {9, 10}, {11}}; - auto expected_keys = fixed_width_column_wrapper{1, 2, 3}; + auto expected_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto expected_values = lists{{1, 2}, {5, 6, 7}, {9, 10}}; test_single_agg(keys, values, expected_keys, expected_values, - cudf::make_nth_element_aggregation(0)); + cudf::make_nth_element_aggregation(0)); } TYPED_TEST(groupby_nth_element_lists_test, EmptyInput) @@ -401,20 +396,20 @@ TYPED_TEST(groupby_nth_element_lists_test, EmptyInput) using lists = cudf::test::lists_column_wrapper; - auto keys = fixed_width_column_wrapper{}; + auto keys = cudf::test::fixed_width_column_wrapper{}; auto values = lists{}; - auto expected_keys = fixed_width_column_wrapper{}; + auto expected_keys = cudf::test::fixed_width_column_wrapper{}; auto expected_values = lists{}; test_single_agg(keys, values, expected_keys, expected_values, - cudf::make_nth_element_aggregation(2)); + cudf::make_nth_element_aggregation(2)); } -struct groupby_nth_element_structs_test : BaseFixture { +struct groupby_nth_element_structs_test : cudf::test::BaseFixture { }; TEST_F(groupby_nth_element_structs_test, Basics) @@ -439,18 +434,19 @@ TEST_F(groupby_nth_element_structs_test, Basics) values, expected_keys, expected_values, - cudf::make_nth_element_aggregation(1)); + cudf::make_nth_element_aggregation(1)); expected_keys = ints{0, 1, 2, 3}; expected_ch0 = ints{0, 4, 6, 9}; expected_ch1 = doubles{0.1, 4.51, 6.3231, 9.999}; expected_ch2 = strings{"", "d", "f", "JJJ"}; expected_values = structs{{expected_ch0, expected_ch1, expected_ch2}, {1, 1, 1, 1}}; - test_single_agg(keys, - values, - expected_keys, - expected_values, - cudf::make_nth_element_aggregation(0, null_policy::EXCLUDE)); + test_single_agg( + keys, + values, + expected_keys, + expected_values, + cudf::make_nth_element_aggregation(0, cudf::null_policy::EXCLUDE)); } TEST_F(groupby_nth_element_structs_test, NestedStructs) @@ -479,7 +475,7 @@ TEST_F(groupby_nth_element_structs_test, NestedStructs) values, expected_keys, expected_values, - cudf::make_nth_element_aggregation(1)); + cudf::make_nth_element_aggregation(1)); expected_keys = ints{0, 1, 2, 3}; expected_ch0 = ints{0, 4, 6, 9}; @@ -488,11 +484,12 @@ TEST_F(groupby_nth_element_structs_test, NestedStructs) expected_ch1 = structs{expected_ch0_of_ch1, expected_ch1_of_ch1}; expected_ch2 = lists{{0}, {5, 6}, {}, {}}; expected_values = structs{{expected_ch0, expected_ch1, expected_ch2}, {1, 1, 1, 1}}; - test_single_agg(keys, - values, - expected_keys, - expected_values, - cudf::make_nth_element_aggregation(0, null_policy::EXCLUDE)); + test_single_agg( + keys, + values, + expected_keys, + expected_values, + cudf::make_nth_element_aggregation(0, cudf::null_policy::EXCLUDE)); } TEST_F(groupby_nth_element_structs_test, EmptyInput) @@ -517,7 +514,5 @@ TEST_F(groupby_nth_element_structs_test, EmptyInput) values, expected_keys, expected_values, - cudf::make_nth_element_aggregation(0)); + cudf::make_nth_element_aggregation(0)); } -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/nunique_tests.cpp b/cpp/tests/groupby/nunique_tests.cpp index b18fa59d706..2923aa2d8bf 100644 --- a/cpp/tests/groupby/nunique_tests.cpp +++ b/cpp/tests/groupby/nunique_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,33 +23,29 @@ #include -using namespace cudf::test::iterators; - -namespace cudf { -namespace test { template struct groupby_nunique_test : public cudf::test::BaseFixture { }; -using K = int32_t; TYPED_TEST_SUITE(groupby_nunique_test, cudf::test::AllTypes); TYPED_TEST(groupby_nunique_test, basic) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // clang-format off - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{3, 4, 3}; - fixed_width_column_wrapper expect_bool_vals{2, 1, 1}; + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{3, 4, 3}; + cudf::test::fixed_width_column_wrapper expect_bool_vals{2, 1, 1}; // clang-format on - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); if (std::is_same()) test_single_agg(keys, vals, expect_keys, expect_bool_vals, std::move(agg)); else @@ -58,32 +54,34 @@ TYPED_TEST(groupby_nunique_test, basic) TYPED_TEST(groupby_nunique_test, empty_cols) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_nunique_test, basic_duplicates) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 3, 2, 2, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 3, 2, 2, 9}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{2, 4, 1}; - fixed_width_column_wrapper expect_bool_vals{2, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{2, 4, 1}; + cudf::test::fixed_width_column_wrapper expect_bool_vals{2, 1, 1}; - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); if (std::is_same()) test_single_agg(keys, vals, expect_keys, expect_bool_vals, std::move(agg)); else @@ -92,51 +90,55 @@ TYPED_TEST(groupby_nunique_test, basic_duplicates) TYPED_TEST(groupby_nunique_test, zero_valid_keys) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals({3, 4, 5}); + cudf::test::fixed_width_column_wrapper keys({0, 0, 0}, cudf::test::iterators::all_nulls()); + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}); - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_nunique_test, zero_valid_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({0, 0, 0}, cudf::test::iterators::all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals{0}; + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals{0}; - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_nunique_test, null_keys_and_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // {1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, + cudf::test::iterators::no_nulls()); // all unique values only {3, 6, 1, 4, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals{2, 3, 2, 0}; - fixed_width_column_wrapper expect_bool_vals{1, 1, 1, 0}; + cudf::test::fixed_width_column_wrapper expect_vals{2, 3, 2, 0}; + cudf::test::fixed_width_column_wrapper expect_bool_vals{1, 1, 1, 0}; - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); if (std::is_same()) test_single_agg(keys, vals, expect_keys, expect_bool_vals, std::move(agg)); else @@ -145,22 +147,24 @@ TYPED_TEST(groupby_nunique_test, null_keys_and_values) TYPED_TEST(groupby_nunique_test, null_keys_and_values_with_duplicates) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, - {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, - {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, + {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, + {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, + cudf::test::iterators::no_nulls()); // { 3, 6,- 1, 4, 9,- 2*, 8, -*} // unique, with null, dup, dup null - fixed_width_column_wrapper expect_vals{2, 3, 2, 0}; - fixed_width_column_wrapper expect_bool_vals{1, 1, 1, 0}; + cudf::test::fixed_width_column_wrapper expect_vals{2, 3, 2, 0}; + cudf::test::fixed_width_column_wrapper expect_bool_vals{1, 1, 1, 0}; - auto agg = cudf::make_nunique_aggregation(); + auto agg = cudf::make_nunique_aggregation(); if (std::is_same()) test_single_agg(keys, vals, expect_keys, expect_bool_vals, std::move(agg)); else @@ -169,22 +173,24 @@ TYPED_TEST(groupby_nunique_test, null_keys_and_values_with_duplicates) TYPED_TEST(groupby_nunique_test, include_nulls) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, - {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, - {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 3, 3, 2, 4, 4, 2}, + {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 4, 4, 2}, + {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, + cudf::test::iterators::no_nulls()); // { 3, 6,- 1, 4, 9,- 2*, 8, -*} // unique, with null, dup, dup null - fixed_width_column_wrapper expect_vals{3, 4, 2, 1}; - fixed_width_column_wrapper expect_bool_vals{2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_vals{3, 4, 2, 1}; + cudf::test::fixed_width_column_wrapper expect_bool_vals{2, 2, 1, 1}; - auto agg = cudf::make_nunique_aggregation(null_policy::INCLUDE); + auto agg = cudf::make_nunique_aggregation(cudf::null_policy::INCLUDE); if (std::is_same()) test_single_agg(keys, vals, expect_keys, expect_bool_vals, std::move(agg)); else @@ -193,32 +199,31 @@ TYPED_TEST(groupby_nunique_test, include_nulls) TYPED_TEST(groupby_nunique_test, dictionary) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 0, 3, 2, 4, 4, 2}, + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 3, 1, 2, 2, 1, 0, 3, 2, 4, 4, 2}, {1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); - dictionary_column_wrapper vals({0, 1, 2, 2, 3, 4, 0, 6, 7, 8, 9, 0, 0, 0}, + cudf::test::dictionary_column_wrapper vals({0, 1, 2, 2, 3, 4, 0, 6, 7, 8, 9, 0, 0, 0}, {0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, cudf::test::iterators::no_nulls()); // { 3, 6,- 1, 4, 9,- 2*, 8, -*} // unique, with null, dup, dup null - fixed_width_column_wrapper expect_fixed_vals({3, 4, 2, 1}); - fixed_width_column_wrapper expect_bool_vals{2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_fixed_vals({3, 4, 2, 1}); + cudf::test::fixed_width_column_wrapper expect_bool_vals{2, 2, 1, 1}; // clang-format on cudf::column_view expect_vals = (std::is_same()) ? cudf::column_view{expect_bool_vals} : cudf::column_view{expect_fixed_vals}; - test_single_agg(keys, - vals, - expect_keys, - expect_vals, - cudf::make_nunique_aggregation(null_policy::INCLUDE)); + test_single_agg( + keys, + vals, + expect_keys, + expect_vals, + cudf::make_nunique_aggregation(cudf::null_policy::INCLUDE)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/product_tests.cpp b/cpp/tests/groupby/product_tests.cpp index 6f6263d90cc..a4173abde32 100644 --- a/cpp/tests/groupby/product_tests.cpp +++ b/cpp/tests/groupby/product_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,30 +25,28 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_product_test : public cudf::test::BaseFixture { }; -using K = int32_t; using supported_types = cudf::test::Types; TYPED_TEST_SUITE(groupby_product_test, supported_types); TYPED_TEST(groupby_product_test, basic) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys { 1, 2, 3 }; - // { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({ 0., 180., 112. }, no_nulls()); + // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys { 1, 2, 3 }; + // { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({ 0., 180., 112. }, no_nulls()); // clang-format on test_single_agg(keys, @@ -60,14 +58,15 @@ TYPED_TEST(groupby_product_test, basic) TYPED_TEST(groupby_product_test, empty_cols) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; test_single_agg(keys, vals, @@ -78,14 +77,15 @@ TYPED_TEST(groupby_product_test, empty_cols) TYPED_TEST(groupby_product_test, zero_valid_keys) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({0, 0, 0}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; test_single_agg(keys, vals, @@ -96,14 +96,15 @@ TYPED_TEST(groupby_product_test, zero_valid_keys) TYPED_TEST(groupby_product_test, zero_valid_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({0, 0, 0}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); test_single_agg(keys, vals, @@ -114,19 +115,20 @@ TYPED_TEST(groupby_product_test, zero_valid_values) TYPED_TEST(groupby_product_test, null_keys_and_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys( { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + cudf::test::fixed_width_column_wrapper keys( { 1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, { 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals( { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, + cudf::test::fixed_width_column_wrapper vals( { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, { 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3, 4}, no_nulls()); // { _, 3, 6, 1, 4, 9, 2, 8, _} - fixed_width_column_wrapper expect_vals({ 18., 36., 16., 3.}, + cudf::test::fixed_width_column_wrapper expect_vals({ 18., 36., 16., 3.}, { 1, 1, 1, 0}); // clang-format on @@ -139,17 +141,18 @@ TYPED_TEST(groupby_product_test, null_keys_and_values) TYPED_TEST(groupby_product_test, dictionary) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); - // { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({ 0., 180., 112. }, no_nulls()); + // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); + // { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({ 0., 180., 112. }, no_nulls()); // clang-format on test_single_agg(keys, @@ -161,18 +164,19 @@ TYPED_TEST(groupby_product_test, dictionary) TYPED_TEST(groupby_product_test, dictionary_with_nulls) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + cudf::test::fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, {1, 0, 0, 1, 1, 1, 1, 1, 1, 1}}; - // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); - // { 0, 3, 6, @, 4, 5, 9, @, 7, 8} - fixed_width_column_wrapper expect_vals({ 0., 180., 56. }, no_nulls()); + // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys({ 1, 2, 3 }); + // { 0, 3, 6, @, 4, 5, 9, @, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({ 0., 180., 56. }, no_nulls()); // clang-format on test_single_agg(keys, @@ -181,6 +185,3 @@ TYPED_TEST(groupby_product_test, dictionary_with_nulls) expect_vals, cudf::make_product_aggregation()); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/quantile_tests.cpp b/cpp/tests/groupby/quantile_tests.cpp index 0f7630d672b..d707b60ea05 100644 --- a/cpp/tests/groupby/quantile_tests.cpp +++ b/cpp/tests/groupby/quantile_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_quantile_test : public cudf::test::BaseFixture { }; @@ -39,151 +37,156 @@ TYPED_TEST_SUITE(groupby_quantile_test, supported_types); TYPED_TEST(groupby_quantile_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // clang-format on // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({3., 4.5, 7.}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({3., 4.5, 7.}, no_nulls()); // clang-format on - auto agg = cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR); + auto agg = + cudf::make_quantile_aggregation({0.5}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_quantile_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR); + auto agg = + cudf::make_quantile_aggregation({0.5}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_quantile_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR); + auto agg = + cudf::make_quantile_aggregation({0.5}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_quantile_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR); + auto agg = + cudf::make_quantile_aggregation({0.5}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_quantile_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals({4.5, 4., 5., 0.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({4.5, 4., 5., 0.}, {1, 1, 1, 0}); - auto agg = cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR); + auto agg = + cudf::make_quantile_aggregation({0.5}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_quantile_test, multiple_quantile) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // clang-format off // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({1.5, 4.5, 3.25, 6., 4.5, 7.5}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({1.5, 4.5, 3.25, 6., 4.5, 7.5}, no_nulls()); // clang-format on - auto agg = - cudf::make_quantile_aggregation({0.25, 0.75}, interpolation::LINEAR); + auto agg = cudf::make_quantile_aggregation( + {0.25, 0.75}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), force_use_sort_impl::YES, - null_policy::EXCLUDE, - sorted::NO, + cudf::null_policy::EXCLUDE, + cudf::sorted::NO, {}, {}, - sorted::YES); + cudf::sorted::YES); } TYPED_TEST(groupby_quantile_test, interpolation_types) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 9}; // clang-format off - // {1, 1, 1, 2, 2, 2, 2, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; + // {1, 1, 1, 2, 2, 2, 2, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; - // {0, 3, 6, 1, 4, 5, 9, 2, 7} - fixed_width_column_wrapper expect_vals1({2.4, 4.2, 4.}, no_nulls()); - auto agg1 = cudf::make_quantile_aggregation({0.4}, interpolation::LINEAR); + // {0, 3, 6, 1, 4, 5, 9, 2, 7} + cudf::test::fixed_width_column_wrapper expect_vals1({2.4, 4.2, 4.}, no_nulls()); + auto agg1 = cudf::make_quantile_aggregation({0.4}, cudf::interpolation::LINEAR); test_single_agg(keys, vals, expect_keys, expect_vals1, std::move(agg1)); - // {0, 3, 6, 1, 4, 5, 9, 2, 7} - fixed_width_column_wrapper expect_vals2({3, 4, 2}, no_nulls()); - auto agg2 = cudf::make_quantile_aggregation({0.4}, interpolation::NEAREST); + // {0, 3, 6, 1, 4, 5, 9, 2, 7} + cudf::test::fixed_width_column_wrapper expect_vals2({3, 4, 2}, no_nulls()); + auto agg2 = cudf::make_quantile_aggregation({0.4}, cudf::interpolation::NEAREST); test_single_agg(keys, vals, expect_keys, expect_vals2, std::move(agg2)); - // {0, 3, 6, 1, 4, 5, 9, 2, 7} - fixed_width_column_wrapper expect_vals3({0, 4, 2}, no_nulls()); - auto agg3 = cudf::make_quantile_aggregation({0.4}, interpolation::LOWER); + // {0, 3, 6, 1, 4, 5, 9, 2, 7} + cudf::test::fixed_width_column_wrapper expect_vals3({0, 4, 2}, no_nulls()); + auto agg3 = cudf::make_quantile_aggregation({0.4}, cudf::interpolation::LOWER); test_single_agg(keys, vals, expect_keys, expect_vals3, std::move(agg3)); - // {0, 3, 6, 1, 4, 5, 9, 2, 7} - fixed_width_column_wrapper expect_vals4({3, 5, 7}, no_nulls()); - auto agg4 = cudf::make_quantile_aggregation({0.4}, interpolation::HIGHER); + // {0, 3, 6, 1, 4, 5, 9, 2, 7} + cudf::test::fixed_width_column_wrapper expect_vals4({3, 5, 7}, no_nulls()); + auto agg4 = cudf::make_quantile_aggregation({0.4}, cudf::interpolation::HIGHER); test_single_agg(keys, vals, expect_keys, expect_vals4, std::move(agg4)); - // {0, 3, 6, 1, 4, 5, 9, 2, 7} - fixed_width_column_wrapper expect_vals5({1.5, 4.5, 4.5}, no_nulls()); - auto agg5 = cudf::make_quantile_aggregation({0.4}, interpolation::MIDPOINT); + // {0, 3, 6, 1, 4, 5, 9, 2, 7} + cudf::test::fixed_width_column_wrapper expect_vals5({1.5, 4.5, 4.5}, no_nulls()); + auto agg5 = cudf::make_quantile_aggregation({0.4}, cudf::interpolation::MIDPOINT); test_single_agg(keys, vals, expect_keys, expect_vals5, std::move(agg5)); // clang-format on } @@ -191,16 +194,16 @@ TYPED_TEST(groupby_quantile_test, interpolation_types) TYPED_TEST(groupby_quantile_test, dictionary) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({1, 2, 3}); - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({3., 4.5, 7.}, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3}); + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({3., 4.5, 7.}, no_nulls()); // clang-format on test_single_agg( @@ -208,8 +211,5 @@ TYPED_TEST(groupby_quantile_test, dictionary) vals, expect_keys, expect_vals, - cudf::make_quantile_aggregation({0.5}, interpolation::LINEAR)); + cudf::make_quantile_aggregation({0.5}, cudf::interpolation::LINEAR)); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/rank_scan_tests.cpp b/cpp/tests/groupby/rank_scan_tests.cpp index c9f31576aff..1b1b4fbd371 100644 --- a/cpp/tests/groupby/rank_scan_tests.cpp +++ b/cpp/tests/groupby/rank_scan_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,69 +19,72 @@ #include #include #include +#include #include #include -namespace cudf { -namespace test { - -using namespace iterators; +using namespace cudf::test::iterators; template -using input = fixed_width_column_wrapper; -using rank_result_col = fixed_width_column_wrapper; -using percent_result_col = fixed_width_column_wrapper; +using input = cudf::test::fixed_width_column_wrapper; +using rank_result_col = cudf::test::fixed_width_column_wrapper; +using percent_result_col = cudf::test::fixed_width_column_wrapper; using null_iter_t = decltype(nulls_at({})); auto constexpr X = int32_t{0}; // Placeholder for NULL rows. auto const all_valid = nulls_at({}); -inline void test_rank_scans(column_view const& keys, - column_view const& order, - column_view const& expected_dense, - column_view const& expected_rank, - column_view const& expected_percent_rank) +void test_rank_scans(cudf::column_view const& keys, + cudf::column_view const& order, + cudf::column_view const& expected_dense, + cudf::column_view const& expected_rank, + cudf::column_view const& expected_percent_rank) { - test_single_scan( - keys, - order, - keys, - expected_dense, - make_rank_aggregation(rank_method::DENSE, {}, null_policy::INCLUDE), - null_policy::INCLUDE, - sorted::YES); - test_single_scan( - keys, - order, - keys, - expected_rank, - make_rank_aggregation(rank_method::MIN, {}, null_policy::INCLUDE), - null_policy::INCLUDE, - sorted::YES); - test_single_scan( - keys, - order, - keys, - expected_percent_rank, - make_rank_aggregation( - rank_method::MIN, {}, null_policy::INCLUDE, {}, rank_percentage::ONE_NORMALIZED), - null_policy::INCLUDE, - sorted::YES); + test_single_scan(keys, + order, + keys, + expected_dense, + cudf::make_rank_aggregation( + cudf::rank_method::DENSE, {}, cudf::null_policy::INCLUDE), + cudf::null_policy::INCLUDE, + cudf::sorted::YES); + test_single_scan(keys, + order, + keys, + expected_rank, + cudf::make_rank_aggregation( + cudf::rank_method::MIN, {}, cudf::null_policy::INCLUDE), + cudf::null_policy::INCLUDE, + cudf::sorted::YES); + test_single_scan(keys, + order, + keys, + expected_percent_rank, + cudf::make_rank_aggregation( + cudf::rank_method::MIN, + {}, + cudf::null_policy::INCLUDE, + {}, + cudf::rank_percentage::ONE_NORMALIZED), + cudf::null_policy::INCLUDE, + cudf::sorted::YES); } -struct groupby_rank_scan_test : public BaseFixture { +struct groupby_rank_scan_test : public cudf::test::BaseFixture { }; -struct groupby_rank_scan_test_failures : public BaseFixture { +struct groupby_rank_scan_test_failures : public cudf::test::BaseFixture { }; template -struct typed_groupby_rank_scan_test : public BaseFixture { +struct typed_groupby_rank_scan_test : public cudf::test::BaseFixture { }; -using testing_type_set = - Concat; +using testing_type_set = cudf::test::Concat; TYPED_TEST_SUITE(typed_groupby_rank_scan_test, testing_type_set); @@ -91,7 +94,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, empty_cols) auto const keys = input{}; auto const order_by = input{}; - auto const order_by_struct = structs_column_wrapper{}; + auto const order_by_struct = cudf::test::structs_column_wrapper{}; auto const expected_dense = rank_result_col{}; auto const expected_rank = rank_result_col{}; @@ -110,7 +113,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, zero_valid_keys) auto const order_by_struct = [] { auto member_1 = input{{3, 3, 1}}; auto member_2 = input{{3, 3, 1}}; - return structs_column_wrapper{member_1, member_2}; + return cudf::test::structs_column_wrapper{member_1, member_2}; }(); auto const dense_rank_results = rank_result_col{1, 1, 2}; @@ -131,7 +134,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, zero_valid_orders) auto const make_struct_order_by = [&](null_iter_t const& null_iter = no_nulls()) { auto member1 = make_order_by(); auto member2 = make_order_by(); - return structs_column_wrapper{{member1, member2}, null_iter}; + return cudf::test::structs_column_wrapper{{member1, member2}, null_iter}; }; auto const order_by = make_order_by(); auto const order_by_struct = make_struct_order_by(); @@ -156,7 +159,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, basic) auto const order_by_struct = [&] { auto order2 = make_order_by(); auto order3 = make_order_by(); - return structs_column_wrapper{order2, order3}; + return cudf::test::structs_column_wrapper{order2, order3}; }(); auto const expected_dense = rank_result_col{1, 1, 1, 2, 2, 2, 3, 1, 2, 2, 3, 3}; @@ -180,7 +183,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, null_orders) auto const make_struct_order_by = [&](null_iter_t const& null_iter = all_valid) { auto member1 = make_order_by(); auto member2 = make_order_by(); - return structs_column_wrapper{{member1, member2}, null_iter}; + return cudf::test::structs_column_wrapper{{member1, member2}, null_iter}; }; auto const order_by = make_order_by(); auto const order_by_struct = make_struct_order_by(); @@ -209,7 +212,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, null_orders_and_keys) auto const make_struct_order_by = [&](null_iter_t const& null_iter = all_valid) { auto member1 = make_order_by(); auto member2 = make_order_by(); - return structs_column_wrapper{{member1, member2}, null_iter}; + return cudf::test::structs_column_wrapper{{member1, member2}, null_iter}; }; auto const order_by = make_order_by(); auto const order_by_struct = make_struct_order_by(); @@ -230,12 +233,12 @@ TYPED_TEST(typed_groupby_rank_scan_test, mixedStructs) { auto const struct_col = [] { auto nums = input{{0, 0, 7, 7, 7, X, 4, 4, 4, 9, 9, 9}, null_at(5)}; - auto strings = strings_column_wrapper{ + auto strings = cudf::test::strings_column_wrapper{ {"0a", "0a", "2a", "2a", "3b", "5", "6c", "6c", "XX", "9", "9", "10d"}, null_at(8)}; - return structs_column_wrapper{{nums, strings}, null_at(11)}.release(); + return cudf::test::structs_column_wrapper{{nums, strings}, null_at(11)}.release(); }(); - auto const keys = strings_column_wrapper{ + auto const keys = cudf::test::strings_column_wrapper{ {"0", "0", "0", "0", "0", "0", "1", "1", "1", "X", "X", "X"}, nulls_at({9, 10, 11})}; auto const expected_dense = rank_result_col{1, 1, 2, 2, 3, 4, 1, 1, 2, 1, 1, 2}; @@ -243,20 +246,25 @@ TYPED_TEST(typed_groupby_rank_scan_test, mixedStructs) auto const expected_percent = percent_result_col{ 0.0, 0.0, 2.0 / 5, 2.0 / 5, 4.0 / 5, 5.0 / 5, 0.0, 0.0, 2.0 / 2, 0.0, 0.0, 2.0 / 2}; - std::vector requests; - requests.emplace_back(groupby::scan_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::scan_request()); requests[0].values = *struct_col; - requests[0].aggregations.push_back( - make_rank_aggregation(rank_method::DENSE, {}, null_policy::INCLUDE)); - requests[0].aggregations.push_back( - make_rank_aggregation(rank_method::MIN, {}, null_policy::INCLUDE)); - requests[0].aggregations.push_back(make_rank_aggregation( - rank_method::MIN, {}, null_policy::INCLUDE, {}, rank_percentage::ONE_NORMALIZED)); - - groupby::groupby gb_obj(table_view({keys}), null_policy::INCLUDE, sorted::YES); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::DENSE, {}, cudf::null_policy::INCLUDE)); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, {}, cudf::null_policy::INCLUDE)); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, + {}, + cudf::null_policy::INCLUDE, + {}, + cudf::rank_percentage::ONE_NORMALIZED)); + + cudf::groupby::groupby gb_obj( + cudf::table_view({keys}), cudf::null_policy::INCLUDE, cudf::sorted::YES); auto [result_keys, agg_results] = gb_obj.scan(requests); - CUDF_TEST_EXPECT_TABLES_EQUAL(table_view({keys}), result_keys->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(cudf::table_view({keys}), result_keys->view()); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*agg_results[0].results[0], expected_dense); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*agg_results[0].results[1], expected_rank); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*agg_results[0].results[2], expected_percent); @@ -269,48 +277,57 @@ TYPED_TEST(typed_groupby_rank_scan_test, nestedStructs) auto nested_structs = [] { auto structs_member = [] { auto nums_member = input{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; - auto strings_member = strings_column_wrapper{ + auto strings_member = cudf::test::strings_column_wrapper{ {"0a", "0a", "2a", "2a", "3b", "5", "6c", "6c", "6c", "9", "9", "10d"}, null_at(8)}; - return structs_column_wrapper{nums_member, strings_member}; + return cudf::test::structs_column_wrapper{nums_member, strings_member}; }(); auto nums_member = input{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; - return structs_column_wrapper{structs_member, nums_member}.release(); + return cudf::test::structs_column_wrapper{structs_member, nums_member}.release(); }(); auto flat_struct = [] { auto nums_member = input{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; - auto strings_member = strings_column_wrapper{ + auto strings_member = cudf::test::strings_column_wrapper{ {"0a", "0a", "2a", "2a", "3b", "5", "6c", "6c", "6c", "9", "9", "10d"}, null_at(8)}; auto nuther_nums = - fixed_width_column_wrapper{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; - return structs_column_wrapper{nums_member, strings_member, nuther_nums}.release(); + cudf::test::fixed_width_column_wrapper{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; + return cudf::test::structs_column_wrapper{nums_member, strings_member, nuther_nums}.release(); }(); - auto const keys = strings_column_wrapper{ + auto const keys = cudf::test::strings_column_wrapper{ {"0", "0", "0", "0", "0", "0", "1", "1", "1", "1", "0", "1"}, nulls_at({9, 10, 11})}; - std::vector requests; - requests.emplace_back(groupby::scan_request()); - requests.emplace_back(groupby::scan_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::scan_request()); + requests.emplace_back(cudf::groupby::scan_request()); requests[0].values = *nested_structs; requests[0].aggregations.push_back( - make_rank_aggregation(rank_method::DENSE)); + cudf::make_rank_aggregation(cudf::rank_method::DENSE)); requests[0].aggregations.push_back( - make_rank_aggregation(rank_method::MIN)); - requests[0].aggregations.push_back(make_rank_aggregation( - rank_method::MIN, {}, null_policy::INCLUDE, {}, rank_percentage::ONE_NORMALIZED)); + cudf::make_rank_aggregation(cudf::rank_method::MIN)); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, + {}, + cudf::null_policy::INCLUDE, + {}, + cudf::rank_percentage::ONE_NORMALIZED)); requests[1].values = *flat_struct; requests[1].aggregations.push_back( - make_rank_aggregation(rank_method::DENSE)); + cudf::make_rank_aggregation(cudf::rank_method::DENSE)); requests[1].aggregations.push_back( - make_rank_aggregation(rank_method::MIN)); - requests[1].aggregations.push_back(make_rank_aggregation( - rank_method::MIN, {}, null_policy::INCLUDE, {}, rank_percentage::ONE_NORMALIZED)); - - groupby::groupby gb_obj(table_view({keys}), null_policy::INCLUDE, sorted::YES); + cudf::make_rank_aggregation(cudf::rank_method::MIN)); + requests[1].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, + {}, + cudf::null_policy::INCLUDE, + {}, + cudf::rank_percentage::ONE_NORMALIZED)); + + cudf::groupby::groupby gb_obj( + cudf::table_view({keys}), cudf::null_policy::INCLUDE, cudf::sorted::YES); auto [result_keys, agg_results] = gb_obj.scan(requests); - CUDF_TEST_EXPECT_TABLES_EQUAL(table_view({keys}), result_keys->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(cudf::table_view({keys}), result_keys->view()); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*agg_results[0].results[0], *agg_results[1].results[0]); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*agg_results[0].results[1], *agg_results[1].results[1]); CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(*agg_results[0].results[2], *agg_results[1].results[2]); @@ -324,10 +341,10 @@ TYPED_TEST(typed_groupby_rank_scan_test, structsWithNullPushdown) auto get_struct_column = [] { auto nums_member = - fixed_width_column_wrapper{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; - auto strings_member = strings_column_wrapper{ + cudf::test::fixed_width_column_wrapper{{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}, null_at(5)}; + auto strings_member = cudf::test::strings_column_wrapper{ {"0a", "0a", "2a", "2a", "3b", "5", "6c", "6c", "6c", "9", "9", "10d"}, null_at(8)}; - auto struct_column = structs_column_wrapper{nums_member, strings_member}.release(); + auto struct_column = cudf::test::structs_column_wrapper{nums_member, strings_member}.release(); // Reset null-mask, a posteriori. Nulls will not be pushed down to children. auto const null_iter = nulls_at({1, 2, 11}); struct_column->set_null_mask( @@ -339,32 +356,41 @@ TYPED_TEST(typed_groupby_rank_scan_test, structsWithNullPushdown) auto const definitely_null_structs = [&] { auto struct_column = get_struct_column(); - struct_column->set_null_mask(create_null_mask(num_rows, mask_state::ALL_NULL)); + struct_column->set_null_mask(cudf::create_null_mask(num_rows, cudf::mask_state::ALL_NULL)); return struct_column; }(); - strings_column_wrapper keys = {{"0", "0", "0", "0", "0", "0", "1", "1", "1", "X", "X", "X"}, - nulls_at({9, 10, 11})}; + cudf::test::strings_column_wrapper keys = { + {"0", "0", "0", "0", "0", "0", "1", "1", "1", "X", "X", "X"}, nulls_at({9, 10, 11})}; - std::vector requests; - requests.emplace_back(groupby::scan_request()); - requests.emplace_back(groupby::scan_request()); + std::vector requests; + requests.emplace_back(cudf::groupby::scan_request()); + requests.emplace_back(cudf::groupby::scan_request()); requests[0].values = *possibly_null_structs; - requests[0].aggregations.push_back( - make_rank_aggregation(rank_method::DENSE, {}, null_policy::INCLUDE)); - requests[0].aggregations.push_back( - make_rank_aggregation(rank_method::MIN, {}, null_policy::INCLUDE)); - requests[0].aggregations.push_back(make_rank_aggregation( - rank_method::MIN, {}, null_policy::INCLUDE, {}, rank_percentage::ONE_NORMALIZED)); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::DENSE, {}, cudf::null_policy::INCLUDE)); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, {}, cudf::null_policy::INCLUDE)); + requests[0].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, + {}, + cudf::null_policy::INCLUDE, + {}, + cudf::rank_percentage::ONE_NORMALIZED)); requests[1].values = *definitely_null_structs; - requests[1].aggregations.push_back( - make_rank_aggregation(rank_method::DENSE, {}, null_policy::INCLUDE)); - requests[1].aggregations.push_back( - make_rank_aggregation(rank_method::MIN, {}, null_policy::INCLUDE)); - requests[1].aggregations.push_back(make_rank_aggregation( - rank_method::MIN, {}, null_policy::INCLUDE, {}, rank_percentage::ONE_NORMALIZED)); - - groupby::groupby gb_obj(table_view({keys}), null_policy::INCLUDE, sorted::YES); + requests[1].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::DENSE, {}, cudf::null_policy::INCLUDE)); + requests[1].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, {}, cudf::null_policy::INCLUDE)); + requests[1].aggregations.push_back(cudf::make_rank_aggregation( + cudf::rank_method::MIN, + {}, + cudf::null_policy::INCLUDE, + {}, + cudf::rank_percentage::ONE_NORMALIZED)); + + cudf::groupby::groupby gb_obj( + cudf::table_view({keys}), cudf::null_policy::INCLUDE, cudf::sorted::YES); auto [result_keys, agg_results] = gb_obj.scan(requests); auto expected_dense = rank_result_col{1, 2, 2, 3, 4, 5, 1, 1, 2, 1, 1, 2}; @@ -385,7 +411,7 @@ TYPED_TEST(typed_groupby_rank_scan_test, structsWithNullPushdown) /* List support dependent on https://github.com/rapidsai/cudf/issues/8683 template -struct list_groupby_rank_scan_test : public BaseFixture { +struct list_groupby_rank_scan_test : public cudf::test::BaseFixture { }; using list_test_type_set = Concat element1{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}; - fixed_width_column_wrapper element2{0, 0, 2, 2, 3, 5, 6, 6, 6, 9, 9, 10}; - auto struct_col = structs_column_wrapper{element1, element2}.release(); + cudf::test::fixed_width_column_wrapper element1{0, 0, 7, 7, 7, 5, 4, 4, 4, 9, 9, 9}; + cudf::test::fixed_width_column_wrapper element2{0, 0, 2, 2, 3, 5, 6, 6, 6, 9, 9, 10}; + auto struct_col = cudf::test::structs_column_wrapper{element1, element2}.release(); - fixed_width_column_wrapper keys = {{0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1}, + cudf::test::fixed_width_column_wrapper keys = {{0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1}, {1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0}}; - std::vector requests; + std::vector requests; requests.emplace_back(groupby::aggregation_request()); requests.emplace_back(groupby::aggregation_request()); requests[0].values = list_col; - requests[0].aggregations.push_back(make_rank_aggregation(rank_method::DENSE)); - requests[0].aggregations.push_back(make_rank_aggregation(rank_method::MIN)); + requests[0].aggregations.push_back(make_rank_aggregation(rank_method::DENSE)); + requests[0].aggregations.push_back(make_rank_aggregation(rank_method::MIN)); requests[1].values = struct_col; - requests[1].aggregations.push_back(make_rank_aggregation(rank_method::DENSE)); - requests[1].aggregations.push_back(make_rank_aggregation(rank_method::MIN)); + requests[1].aggregations.push_back(make_rank_aggregation(rank_method::DENSE)); + requests[1].aggregations.push_back(make_rank_aggregation(rank_method::MIN)); - groupby::groupby gb_obj(table_view({keys}), null_policy::INCLUDE, sorted::YES); + cudf::groupby::groupby gb_obj(table_view({keys}), null_policy::INCLUDE, cudf::sorted::YES); auto result = gb_obj.scan(requests); CUDF_TEST_EXPECT_TABLES_EQUAL(table_view({keys}), result.first->view()); @@ -441,7 +467,7 @@ TYPED_TEST(list_groupby_rank_scan_test, lists) TEST(groupby_rank_scan_test, bools) { - using bools = fixed_width_column_wrapper; + using bools = cudf::test::fixed_width_column_wrapper; using null_iter_t = decltype(nulls_at({})); auto const keys = bools{{0, 0, 0, 0, 0, 0, 1, 1, 1, X, X, X}, nulls_at({9, 10, 11})}; @@ -450,7 +476,7 @@ TEST(groupby_rank_scan_test, bools) auto const make_structs = [&](null_iter_t const& null_iter = all_valid) { auto member_1 = make_order_by(); auto member_2 = make_order_by(); - return structs_column_wrapper{{member_1, member_2}, null_iter}; + return cudf::test::structs_column_wrapper{{member_1, member_2}, null_iter}; }; auto const order_by = make_order_by(); @@ -470,7 +496,7 @@ TEST(groupby_rank_scan_test, bools) TEST(groupby_rank_scan_test, strings) { - using strings = strings_column_wrapper; + using strings = cudf::test::strings_column_wrapper; using null_iter_t = decltype(nulls_at({})); auto const keys = @@ -483,7 +509,7 @@ TEST(groupby_rank_scan_test, strings) auto const make_structs = [&](null_iter_t const& null_iter = all_valid) { auto member_1 = make_order_by(); auto member_2 = make_order_by(); - return structs_column_wrapper{{member_1, member_2}, null_iter}; + return cudf::test::structs_column_wrapper{{member_1, member_2}, null_iter}; }; auto const order_by = make_order_by(); @@ -513,56 +539,59 @@ TEST_F(groupby_rank_scan_test_failures, DISABLED_test_exception_triggers) col, keys, col, - make_rank_aggregation(rank_method::DENSE), - null_policy::INCLUDE, - sorted::NO), + cudf::make_rank_aggregation( + cudf::rank_method::DENSE), + cudf::null_policy::INCLUDE, + cudf::sorted::NO), cudf::logic_error); EXPECT_THROW(test_single_scan(keys, col, keys, col, - make_rank_aggregation(rank_method::MIN), - null_policy::INCLUDE, - sorted::NO), + cudf::make_rank_aggregation( + cudf::rank_method::MIN), + cudf::null_policy::INCLUDE, + cudf::sorted::NO), cudf::logic_error); EXPECT_THROW(test_single_scan(keys, col, keys, col, - make_rank_aggregation(rank_method::DENSE), - null_policy::EXCLUDE, - sorted::YES), + cudf::make_rank_aggregation( + cudf::rank_method::DENSE), + cudf::null_policy::EXCLUDE, + cudf::sorted::YES), cudf::logic_error); EXPECT_THROW(test_single_scan(keys, col, keys, col, - make_rank_aggregation(rank_method::MIN), - null_policy::EXCLUDE, - sorted::YES), + cudf::make_rank_aggregation( + cudf::rank_method::MIN), + cudf::null_policy::EXCLUDE, + cudf::sorted::YES), cudf::logic_error); EXPECT_THROW(test_single_scan(keys, col, keys, col, - make_rank_aggregation(rank_method::DENSE), - null_policy::EXCLUDE, - sorted::NO), + cudf::make_rank_aggregation( + cudf::rank_method::DENSE), + cudf::null_policy::EXCLUDE, + cudf::sorted::NO), cudf::logic_error); EXPECT_THROW(test_single_scan(keys, col, keys, col, - make_rank_aggregation(rank_method::MIN), - null_policy::EXCLUDE, - sorted::NO), + cudf::make_rank_aggregation( + cudf::rank_method::MIN), + cudf::null_policy::EXCLUDE, + cudf::sorted::NO), cudf::logic_error); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/replace_nulls_tests.cpp b/cpp/tests/groupby/replace_nulls_tests.cpp index 7543050d0ef..89f738706ed 100644 --- a/cpp/tests/groupby/replace_nulls_tests.cpp +++ b/cpp/tests/groupby/replace_nulls_tests.cpp @@ -1,6 +1,5 @@ - /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,203 +14,209 @@ * limitations under the License. */ -#include - #include #include #include #include #include +#include +#include #include #include #include using namespace cudf::test::iterators; -namespace cudf { -namespace test { - using K = int32_t; template -struct GroupbyReplaceNullsFixedWidthTest : public BaseFixture { +struct GroupbyReplaceNullsFixedWidthTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(GroupbyReplaceNullsFixedWidthTest, FixedWidthTypes); +TYPED_TEST_SUITE(GroupbyReplaceNullsFixedWidthTest, cudf::test::FixedWidthTypes); template -void TestReplaceNullsGroupbySingle( - K const& key, V const& input, K const& expected_key, V const& expected_val, replace_policy policy) +void TestReplaceNullsGroupbySingle(K const& key, + V const& input, + K const& expected_key, + V const& expected_val, + cudf::replace_policy policy) { - groupby::groupby gb_obj(table_view({key})); - std::vector policies{policy}; - auto p = gb_obj.replace_nulls(table_view({input}), policies); + cudf::groupby::groupby gb_obj(cudf::table_view({key})); + std::vector policies{policy}; + auto p = gb_obj.replace_nulls(cudf::table_view({input}), policies); - CUDF_TEST_EXPECT_TABLES_EQUAL(*p.first, table_view({expected_key})); - CUDF_TEST_EXPECT_TABLES_EQUAL(*p.second, table_view({expected_val})); + CUDF_TEST_EXPECT_TABLES_EQUAL(*p.first, cudf::table_view({expected_key})); + CUDF_TEST_EXPECT_TABLES_EQUAL(*p.second, cudf::table_view({expected_val})); } TYPED_TEST(GroupbyReplaceNullsFixedWidthTest, PrecedingFill) { // Group 0 value: {42, 24, null} --> {42, 24, 24} // Group 1 value: {7, null, null} --> {7, 7, 7} - fixed_width_column_wrapper key{0, 1, 0, 1, 0, 1}; - fixed_width_column_wrapper val({42, 7, 24, 10, 1, 1000}, {1, 1, 1, 0, 0, 0}); + cudf::test::fixed_width_column_wrapper key{0, 1, 0, 1, 0, 1}; + cudf::test::fixed_width_column_wrapper val({42, 7, 24, 10, 1, 1000}, + {1, 1, 1, 0, 0, 0}); - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1}; - fixed_width_column_wrapper expect_val({42, 24, 24, 7, 7, 7}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_val({42, 24, 24, 7, 7, 7}, no_nulls()); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TYPED_TEST(GroupbyReplaceNullsFixedWidthTest, FollowingFill) { // Group 0 value: {2, null, 32} --> {2, 32, 32} // Group 1 value: {8, null, null, 128, 256} --> {8, 128, 128, 128, 256} - fixed_width_column_wrapper key{0, 0, 1, 1, 0, 1, 1, 1}; - fixed_width_column_wrapper val({2, 4, 8, 16, 32, 64, 128, 256}, - {1, 0, 1, 0, 1, 0, 1, 1}); + cudf::test::fixed_width_column_wrapper key{0, 0, 1, 1, 0, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper val({2, 4, 8, 16, 32, 64, 128, 256}, + {1, 0, 1, 0, 1, 0, 1, 1}); - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1, 1}; - fixed_width_column_wrapper expect_val({2, 32, 32, 8, 128, 128, 128, 256}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_val({2, 32, 32, 8, 128, 128, 128, 256}, + no_nulls()); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } TYPED_TEST(GroupbyReplaceNullsFixedWidthTest, PrecedingFillLeadingNulls) { // Group 0 value: {null, 24, null} --> {null, 24, 24} // Group 1 value: {null, null, null} --> {null, null, null} - fixed_width_column_wrapper key{0, 1, 0, 1, 0, 1}; - fixed_width_column_wrapper val({42, 7, 24, 10, 1, 1000}, {0, 0, 1, 0, 0, 0}); + cudf::test::fixed_width_column_wrapper key{0, 1, 0, 1, 0, 1}; + cudf::test::fixed_width_column_wrapper val({42, 7, 24, 10, 1, 1000}, + {0, 0, 1, 0, 0, 0}); - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1}; - fixed_width_column_wrapper expect_val({-1, 24, 24, -1, -1, -1}, {0, 1, 1, 0, 0, 0}); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_val({-1, 24, 24, -1, -1, -1}, + {0, 1, 1, 0, 0, 0}); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TYPED_TEST(GroupbyReplaceNullsFixedWidthTest, FollowingFillTrailingNulls) { // Group 0 value: {2, null, null} --> {2, null, null} // Group 1 value: {null, null, 64, null, null} --> {64, 64, 64, null, null} - fixed_width_column_wrapper key{0, 0, 1, 1, 0, 1, 1, 1}; - fixed_width_column_wrapper val({2, 4, 8, 16, 32, 64, 128, 256}, - {1, 0, 0, 0, 0, 1, 0, 0}); + cudf::test::fixed_width_column_wrapper key{0, 0, 1, 1, 0, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper val({2, 4, 8, 16, 32, 64, 128, 256}, + {1, 0, 0, 0, 0, 1, 0, 0}); - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1, 1}; - fixed_width_column_wrapper expect_val({2, -1, -1, 64, 64, 64, -1, -1}, - {1, 0, 0, 1, 1, 1, 0, 0}); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_val({2, -1, -1, 64, 64, 64, -1, -1}, + {1, 0, 0, 1, 1, 1, 0, 0}); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } -struct GroupbyReplaceNullsStringsTest : public BaseFixture { +struct GroupbyReplaceNullsStringsTest : public cudf::test::BaseFixture { }; TEST_F(GroupbyReplaceNullsStringsTest, PrecedingFill) { // Group 0 value: {"y" "42"} --> {"y", "42"} // Group 1 value: {"xx" @ "zzz" @ "one"} --> {"xx" "xx" "zzz" "zzz" "one"} - fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; - strings_column_wrapper val({"xx", "", "y", "zzz", "42", "", "one"}, - {true, false, true, true, true, false, true}); + cudf::test::fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; + cudf::test::strings_column_wrapper val({"xx", "", "y", "zzz", "42", "", "one"}, + {true, false, true, true, true, false, true}); - fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; - strings_column_wrapper expect_val({"y", "42", "xx", "xx", "zzz", "zzz", "one"}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; + cudf::test::strings_column_wrapper expect_val({"y", "42", "xx", "xx", "zzz", "zzz", "one"}, + no_nulls()); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TEST_F(GroupbyReplaceNullsStringsTest, FollowingFill) { // Group 0 value: {@ "42"} --> {"42", "42"} // Group 1 value: {"xx" @ "zzz" @ "one"} --> {"xx" "zzz" "zzz" "one" "one"} - fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; - strings_column_wrapper val({"xx", "", "", "zzz", "42", "", "one"}, - {true, false, false, true, true, false, true}); + cudf::test::fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; + cudf::test::strings_column_wrapper val({"xx", "", "", "zzz", "42", "", "one"}, + {true, false, false, true, true, false, true}); - fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; - strings_column_wrapper expect_val({"42", "42", "xx", "zzz", "zzz", "one", "one"}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; + cudf::test::strings_column_wrapper expect_val({"42", "42", "xx", "zzz", "zzz", "one", "one"}, + no_nulls()); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } TEST_F(GroupbyReplaceNullsStringsTest, PrecedingFillPrecedingNull) { // Group 0 value: {"y" "42"} --> {"y", "42"} // Group 1 value: {@ @ "zzz" "zzz" "zzz"} --> {@ @ "zzz" "zzz" "zzz"} - fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; - strings_column_wrapper val({"", "", "y", "zzz", "42", "", ""}, - {false, false, true, true, true, false, false}); + cudf::test::fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; + cudf::test::strings_column_wrapper val({"", "", "y", "zzz", "42", "", ""}, + {false, false, true, true, true, false, false}); - fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; - strings_column_wrapper expect_val({"y", "42", "", "", "zzz", "zzz", "zzz"}, - {true, true, false, false, true, true, true}); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; + cudf::test::strings_column_wrapper expect_val({"y", "42", "", "", "zzz", "zzz", "zzz"}, + {true, true, false, false, true, true, true}); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TEST_F(GroupbyReplaceNullsStringsTest, FollowingFillTrailingNull) { // Group 0 value: {@ "y"} --> {"y", "y"} // Group 1 value: {"xx" @ "zzz" @ @} --> {"xx" "zzz" "zzz" @ @} - fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; - strings_column_wrapper val({"xx", "", "", "zzz", "y", "", ""}, - {true, false, false, true, true, false, false}); + cudf::test::fixed_width_column_wrapper key{1, 1, 0, 1, 0, 1, 1}; + cudf::test::strings_column_wrapper val({"xx", "", "", "zzz", "y", "", ""}, + {true, false, false, true, true, false, false}); - fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; - strings_column_wrapper expect_val({"y", "y", "xx", "zzz", "zzz", "", ""}, - {true, true, true, true, true, false, false}); + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 1, 1, 1, 1, 1}; + cudf::test::strings_column_wrapper expect_val({"y", "y", "xx", "zzz", "zzz", "", ""}, + {true, true, true, true, true, false, false}); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } template -struct GroupbyReplaceNullsListsTest : public BaseFixture { +struct GroupbyReplaceNullsListsTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(GroupbyReplaceNullsListsTest, FixedWidthTypes); +TYPED_TEST_SUITE(GroupbyReplaceNullsListsTest, cudf::test::FixedWidthTypes); TYPED_TEST(GroupbyReplaceNullsListsTest, PrecedingFillNonNested) { - using LCW = lists_column_wrapper; + using LCW = cudf::test::lists_column_wrapper; // Group 0 value: {{1 2 3} @ {4 5} @} --> {{1 2 3} {1 2 3} {4 5} {4 5}}, w/o leading nulls // Group 1 value: {@ {} @} --> {@ {} {}}, w/ leading nulls - fixed_width_column_wrapper key{0, 1, 0, 0, 1, 1, 0}; + cudf::test::fixed_width_column_wrapper key{0, 1, 0, 0, 1, 1, 0}; - std::vector mask{1, 0, 0, 1, 1, 0, 0}; + std::vector mask{1, 0, 0, 1, 1, 0, 0}; LCW val({{1, 2, 3}, {}, {}, {4, 5}, {}, {}, {}}, mask.begin()); - fixed_width_column_wrapper expect_key{0, 0, 0, 0, 1, 1, 1}; - std::vector expected_mask{1, 1, 1, 1, 0, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 0, 1, 1, 1}; + std::vector expected_mask{1, 1, 1, 1, 0, 1, 1}; LCW expect_val({{1, 2, 3}, {1, 2, 3}, {4, 5}, {4, 5}, {-1}, {}, {}}, expected_mask.begin()); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TYPED_TEST(GroupbyReplaceNullsListsTest, FollowingFillNonNested) { - using LCW = lists_column_wrapper; + using LCW = cudf::test::lists_column_wrapper; // Group 0 value: {@ {5 6} @ {-1}} --> {{5 6} {5 6} {-1} {-1}}, w/o trailing nulls // Group 1 value: {@ {} @} --> {{} {} @}}, w/ trailing nulls - fixed_width_column_wrapper key{0, 1, 0, 0, 1, 1, 0}; + cudf::test::fixed_width_column_wrapper key{0, 1, 0, 0, 1, 1, 0}; - std::vector mask{0, 0, 1, 0, 1, 0, 1}; + std::vector mask{0, 0, 1, 0, 1, 0, 1}; LCW val({{}, {}, {5, 6}, {}, {}, {}, {-1}}, mask.begin()); - fixed_width_column_wrapper expect_key{0, 0, 0, 0, 1, 1, 1}; - std::vector expected_mask{1, 1, 1, 1, 1, 1, 0}; + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 0, 1, 1, 1}; + std::vector expected_mask{1, 1, 1, 1, 1, 1, 0}; LCW expect_val({{5, 6}, {5, 6}, {-1}, {-1}, {}, {}, {}}, expected_mask.begin()); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } TYPED_TEST(GroupbyReplaceNullsListsTest, PrecedingFillNested) { - using LCW = lists_column_wrapper; - using Mask_t = std::vector; + using LCW = cudf::test::lists_column_wrapper; + using Mask_t = std::vector; // Group 0 value: {{{1 @ 3} @} // @ // {{@} {}}}} --> @@ -227,7 +232,7 @@ TYPED_TEST(GroupbyReplaceNullsListsTest, PrecedingFillNested) // {@ {102 @}} // {{@ 202} {}}}}, w/ leading nulls // Only top level nulls are replaced. - fixed_width_column_wrapper key{1, 0, 1, 1, 0, 0, 1}; + cudf::test::fixed_width_column_wrapper key{1, 0, 1, 1, 0, 0, 1}; // clang-format off LCW val({{}, @@ -240,7 +245,7 @@ TYPED_TEST(GroupbyReplaceNullsListsTest, PrecedingFillNested) Mask_t{0, 1, 1, 0, 0, 1, 1}.begin()); // clang-format on - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; // clang-format off LCW expect_val({LCW({LCW({1, -1, 3}, Mask_t{1, 0, 1}.begin()), {}}, Mask_t{1, 0}.begin()), @@ -253,13 +258,13 @@ TYPED_TEST(GroupbyReplaceNullsListsTest, PrecedingFillNested) Mask_t{1, 1, 1, 0, 1 ,1 ,1}.begin()); // clang-format on - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TYPED_TEST(GroupbyReplaceNullsListsTest, FollowingFillNested) { - using LCW = lists_column_wrapper; - using Mask_t = std::vector; + using LCW = cudf::test::lists_column_wrapper; + using Mask_t = std::vector; // Group 0 value: {{{1 @ 3} @} // @ // {{@} {}}}} --> @@ -275,7 +280,7 @@ TYPED_TEST(GroupbyReplaceNullsListsTest, FollowingFillNested) // {{@ 202} {}}} // @}, w/ trailing nulls // Only top level nulls are replaced. - fixed_width_column_wrapper key{1, 0, 1, 1, 0, 0, 1}; + cudf::test::fixed_width_column_wrapper key{1, 0, 1, 1, 0, 0, 1}; // clang-format off LCW val({LCW({LCW{}, LCW({102, -1}, Mask_t{1, 0}.begin())}, Mask_t{0, 1}.begin()), @@ -288,7 +293,7 @@ TYPED_TEST(GroupbyReplaceNullsListsTest, FollowingFillNested) Mask_t{1, 1, 0, 1, 0, 1, 0}.begin()); // clang-format on - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; // clang-format off LCW expect_val({LCW({LCW({1, -1, 3}, Mask_t{1, 0, 1}.begin()), {}}, Mask_t{1, 0}.begin()), @@ -301,16 +306,16 @@ TYPED_TEST(GroupbyReplaceNullsListsTest, FollowingFillNested) Mask_t{1, 1, 1, 1, 1, 1, 0}.begin()); // clang-format on - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } -struct GroupbyReplaceNullsStructsTest : public BaseFixture { - using SCW = structs_column_wrapper; +struct GroupbyReplaceNullsStructsTest : public cudf::test::BaseFixture { + using SCW = cudf::test::structs_column_wrapper; - SCW data(fixed_width_column_wrapper field0, - strings_column_wrapper field1, - lists_column_wrapper field2, - std::initializer_list mask) + SCW data(cudf::test::fixed_width_column_wrapper field0, + cudf::test::strings_column_wrapper field1, + cudf::test::lists_column_wrapper field2, + std::initializer_list mask) { return SCW({field0, field1, field2}, mask.begin()); } @@ -318,9 +323,9 @@ struct GroupbyReplaceNullsStructsTest : public BaseFixture { TEST_F(GroupbyReplaceNullsStructsTest, PrecedingFill) { - using LCW = lists_column_wrapper; - using Mask_t = std::vector; - fixed_width_column_wrapper key{1, 0, 0, 1, 0, 1, 1}; + using LCW = cudf::test::lists_column_wrapper; + using Mask_t = std::vector; + cudf::test::fixed_width_column_wrapper key{1, 0, 0, 1, 0, 1, 1}; // Only null rows are replaced. @@ -330,7 +335,7 @@ TEST_F(GroupbyReplaceNullsStructsTest, PrecedingFill) LCW({{1, 2, 3}, {-1}, {}, {}, {42}, {}, {}}, Mask_t{1, 1, 0, 0, 1, 0, 0}.begin()), {1, 1, 0, 0, 1, 1, 0}); - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; SCW expect_val = this->data( {{-1, -1, -1, 1, 1, -1, -1}, {0, 0, 0, 1, 1, 0, 0}}, @@ -338,14 +343,14 @@ TEST_F(GroupbyReplaceNullsStructsTest, PrecedingFill) LCW({LCW{-1}, {-1}, {42}, {1, 2, 3}, {1, 2, 3}, {}, {}}, Mask_t{1, 1, 1, 1, 1, 0, 0}.begin()), {1, 1, 1, 1, 1, 1, 1}); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::PRECEDING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::PRECEDING); } TEST_F(GroupbyReplaceNullsStructsTest, FollowingFill) { - using LCW = lists_column_wrapper; - using Mask_t = std::vector; - fixed_width_column_wrapper key{1, 0, 0, 1, 0, 1, 1}; + using LCW = cudf::test::lists_column_wrapper; + using Mask_t = std::vector; + cudf::test::fixed_width_column_wrapper key{1, 0, 0, 1, 0, 1, 1}; // Only null rows are replaced. @@ -355,7 +360,7 @@ TEST_F(GroupbyReplaceNullsStructsTest, FollowingFill) LCW({{1, 2, 3}, {-1}, {}, {}, {42}, {}, {}}, Mask_t{1, 1, 0, 0, 1, 0, 0}.begin()), {1, 1, 0, 0, 1, 1, 0}); - fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; + cudf::test::fixed_width_column_wrapper expect_key{0, 0, 0, 1, 1, 1, 1}; SCW expect_val = this->data( {{-1, -1, -1, 1, -1, -1, -1}, {0, 0, 0, 1, 0, 0, 0}}, @@ -363,8 +368,5 @@ TEST_F(GroupbyReplaceNullsStructsTest, FollowingFill) LCW({LCW{-1}, {42}, {42}, {1, 2, 3}, {}, {}, {}}, Mask_t{1, 1, 1, 1, 0, 0, 0}.begin()), {1, 1, 1, 1, 1, 1, 0}); - TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, replace_policy::FOLLOWING); + TestReplaceNullsGroupbySingle(key, val, expect_key, expect_val, cudf::replace_policy::FOLLOWING); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/shift_tests.cpp b/cpp/tests/groupby/shift_tests.cpp index 3135ed8f033..6e0f425db0f 100644 --- a/cpp/tests/groupby/shift_tests.cpp +++ b/cpp/tests/groupby/shift_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,456 +24,510 @@ #include #include -namespace cudf { -namespace test { - -using K = int32_t; template -struct groupby_shift_fixed_width_test : public BaseFixture { +struct groupby_shift_fixed_width_test : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(groupby_shift_fixed_width_test, FixedWidthTypes); +TYPED_TEST_SUITE(groupby_shift_fixed_width_test, cudf::test::FixedWidthTypes); -template -void test_groupby_shift_fixed_width_single(fixed_width_column_wrapper const& key, - fixed_width_column_wrapper const& value, - size_type offset, - scalar const& fill_value, - fixed_width_column_wrapper const& expected) +template +void test_groupby_shift_fixed_width_single( + cudf::test::fixed_width_column_wrapper const& key, + cudf::test::fixed_width_column_wrapper const& value, + cudf::size_type offset, + cudf::scalar const& fill_value, + cudf::test::fixed_width_column_wrapper const& expected) { - groupby::groupby gb_obj(table_view({key})); - std::vector offsets{offset}; - auto got = gb_obj.shift(table_view{{value}}, offsets, {fill_value}); + cudf::groupby::groupby gb_obj(cudf::table_view({key})); + std::vector offsets{offset}; + auto got = gb_obj.shift(cudf::table_view{{value}}, offsets, {fill_value}); CUDF_TEST_EXPECT_COLUMNS_EQUAL((*got.second).view().column(0), expected); } TYPED_TEST(groupby_shift_fixed_width_test, ForwardShiftWithoutNull_NullScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expected({-1, -1, 3, 5, -1, -1, 4}, {0, 0, 1, 1, 0, 0, 1}); - size_type offset = 2; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper expected({-1, -1, 3, 5, -1, -1, 4}, + {0, 0, 1, 1, 0, 0, 1}); + cudf::size_type offset = 2; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, ForwardShiftWithNull_NullScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9}, {0, 0, 0, 1, 1, 1, 1}); - fixed_width_column_wrapper expected({-1, -1, -1, -1, -1, -1, -1}, {0, 0, 0, 0, 0, 0, 0}); - size_type offset = 2; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9}, {0, 0, 0, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper expected({-1, -1, -1, -1, -1, -1, -1}, + {0, 0, 0, 0, 0, 0, 0}); + cudf::size_type offset = 2; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, ForwardShiftWithoutNull_ValidScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}); - fixed_width_column_wrapper expected({42, 42, 42, 3, 5, 8, 9, 42, 42, 42, 4, 6, 7}); - size_type offset = 3; - auto slr = cudf::scalar_type_t(make_type_param_scalar(42), true); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}); + cudf::test::fixed_width_column_wrapper expected({42, 42, 42, 3, 5, 8, 9, 42, 42, 42, 4, 6, 7}); + cudf::size_type offset = 3; + auto slr = + cudf::scalar_type_t(cudf::test::make_type_param_scalar(42), true); - test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, ForwardShiftWithNull_ValidScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}, - {1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1}); - fixed_width_column_wrapper expected({42, 42, 42, 3, 5, -1, -1, 42, 42, 42, -1, -1, 7}, - {1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1}); - size_type offset = 3; - auto slr = cudf::scalar_type_t(make_type_param_scalar(42), true); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}, + {1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1}); + cudf::test::fixed_width_column_wrapper expected( + {42, 42, 42, 3, 5, -1, -1, 42, 42, 42, -1, -1, 7}, {1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1}); + cudf::size_type offset = 3; + auto slr = + cudf::scalar_type_t(cudf::test::make_type_param_scalar(42), true); - test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, BackwardShiftWithoutNull_NullScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expected({5, 8, 9, -1, 6, 7, -1}, {1, 1, 1, 0, 1, 1, 0}); - size_type offset = -1; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper expected({5, 8, 9, -1, 6, 7, -1}, + {1, 1, 1, 0, 1, 1, 0}); + cudf::size_type offset = -1; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, BackwardShiftWithNull_NullScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9}, {0, 0, 0, 1, 1, 1, 1}); - fixed_width_column_wrapper expected({-1, 8, 9, -1, 6, 7, -1}, {0, 1, 1, 0, 1, 1, 0}); - size_type offset = -1; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9}, {0, 0, 0, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper expected({-1, 8, 9, -1, 6, 7, -1}, + {0, 1, 1, 0, 1, 1, 0}); + cudf::size_type offset = -1; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, BackwardShiftWithoutNull_ValidScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; - fixed_width_column_wrapper expected({3, 5, 42, 42, 42, 42, 42, 4, 42, 42, 42, 42, 42}); - size_type offset = -5; - auto slr = cudf::scalar_type_t(make_type_param_scalar(42), true); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; + cudf::test::fixed_width_column_wrapper expected( + {3, 5, 42, 42, 42, 42, 42, 4, 42, 42, 42, 42, 42}); + cudf::size_type offset = -5; + auto slr = + cudf::scalar_type_t(cudf::test::make_type_param_scalar(42), true); - test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, BackwardShiftWithNull_ValidScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}, - {1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1}); - fixed_width_column_wrapper expected({5, -1, -1, -1, 3, 5, 42, -1, 7, 0, 2, -1, 42}, - {1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1}); - size_type offset = -1; - auto slr = cudf::scalar_type_t(make_type_param_scalar(42), true); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val({3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}, + {1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1}); + cudf::test::fixed_width_column_wrapper expected({5, -1, -1, -1, 3, 5, 42, -1, 7, 0, 2, -1, 42}, + {1, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1}); + cudf::size_type offset = -1; + auto slr = + cudf::scalar_type_t(cudf::test::make_type_param_scalar(42), true); - test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, ZeroShiftNullScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expected({3, 5, 8, 9, 4, 6, 7}); - size_type offset = 0; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper expected({3, 5, 8, 9, 4, 6, 7}); + cudf::size_type offset = 0; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, ZeroShiftValidScalar) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; - fixed_width_column_wrapper expected({3, 5, 8, 9, 1, 3, 5, 4, 6, 7, 0, 2, 4}); - size_type offset = 0; - auto slr = cudf::scalar_type_t(make_type_param_scalar(42), true); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; + cudf::test::fixed_width_column_wrapper expected({3, 5, 8, 9, 1, 3, 5, 4, 6, 7, 0, 2, 4}); + cudf::size_type offset = 0; + auto slr = + cudf::scalar_type_t(cudf::test::make_type_param_scalar(42), true); - test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, VeryLargeForwardOffset) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; - fixed_width_column_wrapper expected({-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); - size_type offset = 1024; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; + cudf::test::fixed_width_column_wrapper expected( + {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + cudf::size_type offset = 1024; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } TYPED_TEST(groupby_shift_fixed_width_test, VeryLargeBackwardOffset) { + using K = int32_t; using V = TypeParam; - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; - fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; - fixed_width_column_wrapper expected({-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); - size_type offset = -1024; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1, 2, 1, 2, 1, 2, 1}; + cudf::test::fixed_width_column_wrapper val{3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5}; + cudf::test::fixed_width_column_wrapper expected( + {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + cudf::size_type offset = -1024; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); - test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); + test_groupby_shift_fixed_width_single(key, val, offset, *slr, expected); } -struct groupby_shift_string_test : public BaseFixture { +struct groupby_shift_string_test : public cudf::test::BaseFixture { }; -void test_groupby_shift_string_single(fixed_width_column_wrapper const& key, - strings_column_wrapper const& value, - size_type offset, - scalar const& fill_value, - strings_column_wrapper const& expected) +template +void test_groupby_shift_string_single(cudf::test::fixed_width_column_wrapper const& key, + cudf::test::strings_column_wrapper const& value, + cudf::size_type offset, + cudf::scalar const& fill_value, + cudf::test::strings_column_wrapper const& expected) { - groupby::groupby gb_obj(table_view({key})); - std::vector offsets{offset}; - auto got = gb_obj.shift(table_view{{value}}, offsets, {fill_value}); + cudf::groupby::groupby gb_obj(cudf::table_view({key})); + std::vector offsets{offset}; + auto got = gb_obj.shift(cudf::table_view{{value}}, offsets, {fill_value}); CUDF_TEST_EXPECT_COLUMNS_EQUAL((*got.second).view().column(0), expected); } TEST_F(groupby_shift_string_test, ForwardShiftWithoutNull_NullScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"", "a", "cc", "f", "", "bb", "d"}, {0, 1, 1, 1, 0, 1, 1}); - size_type offset = 1; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"", "a", "cc", "f", "", "bb", "d"}, + {0, 1, 1, 1, 0, 1, 1}); + cudf::size_type offset = 1; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, ForwardShiftWithNull_NullScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, {1, 0, 1, 1, 0, 0, 0}); - strings_column_wrapper expected({"", "", "a", "cc", "", "", ""}, {0, 0, 1, 1, 0, 0, 0}); - size_type offset = 2; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, + {1, 0, 1, 1, 0, 0, 0}); + cudf::test::strings_column_wrapper expected({"", "", "a", "cc", "", "", ""}, + {0, 0, 1, 1, 0, 0, 0}); + cudf::size_type offset = 2; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, ForwardShiftWithoutNull_ValidScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"42", "42", "a", "cc", "42", "42", "bb"}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"42", "42", "a", "cc", "42", "42", "bb"}); - size_type offset = 2; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = 2; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, ForwardShiftWithNull_ValidScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, {1, 1, 0, 0, 1, 0, 1}); - strings_column_wrapper expected({"42", "a", "", "", "42", "bb", ""}, {1, 1, 0, 0, 1, 1, 0}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, + {1, 1, 0, 0, 1, 0, 1}); + cudf::test::strings_column_wrapper expected({"42", "a", "", "", "42", "bb", ""}, + {1, 1, 0, 0, 1, 1, 0}); - size_type offset = 1; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = 1; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, BackwardShiftWithoutNull_NullScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"gg", "", "", "", "", "", ""}, {1, 0, 0, 0, 0, 0, 0}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"gg", "", "", "", "", "", ""}, + {1, 0, 0, 0, 0, 0, 0}); - size_type offset = -3; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::size_type offset = -3; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, BackwardShiftWithNull_NullScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, {1, 0, 1, 1, 0, 0, 0}); - strings_column_wrapper expected({"cc", "", "", "", "d", "", ""}, {1, 0, 0, 0, 1, 0, 0}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, + {1, 0, 1, 1, 0, 0, 0}); + cudf::test::strings_column_wrapper expected({"cc", "", "", "", "d", "", ""}, + {1, 0, 0, 0, 1, 0, 0}); - size_type offset = -1; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::size_type offset = -1; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, BackwardShiftWithoutNull_ValidScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"42", "42", "42", "42", "42", "42", "42"}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"42", "42", "42", "42", "42", "42", "42"}); - size_type offset = -4; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = -4; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, BackwardShiftWithNull_ValidScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, {1, 1, 0, 0, 1, 0, 1}); - strings_column_wrapper expected({"", "gg", "42", "42", "eee", "42", "42"}, {0, 1, 1, 1, 1, 1, 1}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val({"a", "bb", "cc", "d", "eee", "f", "gg"}, + {1, 1, 0, 0, 1, 0, 1}); + cudf::test::strings_column_wrapper expected({"", "gg", "42", "42", "eee", "42", "42"}, + {0, 1, 1, 1, 1, 1, 1}); - size_type offset = -2; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = -2; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, ZeroShiftNullScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"a", "cc", "f", "gg", "bb", "d", "eee"}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"a", "cc", "f", "gg", "bb", "d", "eee"}); - size_type offset = 0; - auto slr = cudf::make_default_constructed_scalar(column_view(val).type()); + cudf::size_type offset = 0; + auto slr = cudf::make_default_constructed_scalar(cudf::column_view(val).type()); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, ZeroShiftValidScalar) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"a", "cc", "f", "gg", "bb", "d", "eee"}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"a", "cc", "f", "gg", "bb", "d", "eee"}); - size_type offset = 0; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = 0; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, VeryLargeForwardOffset) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"42", "42", "42", "42", "42", "42", "42"}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"42", "42", "42", "42", "42", "42", "42"}); - size_type offset = 1024; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = 1024; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } TEST_F(groupby_shift_string_test, VeryLargeBackwardOffset) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; - strings_column_wrapper expected({"42", "42", "42", "42", "42", "42", "42"}); + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper val{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::strings_column_wrapper expected({"42", "42", "42", "42", "42", "42", "42"}); - size_type offset = -1024; - auto slr = cudf::make_string_scalar("42"); + cudf::size_type offset = -1024; + auto slr = cudf::make_string_scalar("42"); test_groupby_shift_string_single(key, val, offset, *slr, expected); } template -struct groupby_shift_mixed_test : public BaseFixture { +struct groupby_shift_mixed_test : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(groupby_shift_mixed_test, FixedWidthTypes); +TYPED_TEST_SUITE(groupby_shift_mixed_test, cudf::test::FixedWidthTypes); -void test_groupby_shift_multi(fixed_width_column_wrapper const& key, - table_view const& value, - std::vector offsets, - std::vector> fill_values, - table_view const& expected) +template +void test_groupby_shift_multi(cudf::test::fixed_width_column_wrapper const& key, + cudf::table_view const& value, + std::vector offsets, + std::vector> fill_values, + cudf::table_view const& expected) { - groupby::groupby gb_obj(table_view({key})); + cudf::groupby::groupby gb_obj(cudf::table_view({key})); auto got = gb_obj.shift(value, offsets, fill_values); CUDF_TEST_EXPECT_TABLES_EQUAL((*got.second).view(), expected); } TYPED_TEST(groupby_shift_mixed_test, NoFill) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper v1{"a", "bb", "cc", "d", "eee", "f", "gg"}; - fixed_width_column_wrapper v2{1, 2, 3, 4, 5, 6, 7}; - table_view value{{v1, v2}}; + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper v1{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::fixed_width_column_wrapper v2{1, 2, 3, 4, 5, 6, 7}; + cudf::table_view value{{v1, v2}}; - strings_column_wrapper e1({"", "", "a", "cc", "", "", "bb"}, {0, 0, 1, 1, 0, 0, 1}); - fixed_width_column_wrapper e2({-1, 1, 3, 6, -1, 2, 4}, {0, 1, 1, 1, 0, 1, 1}); - table_view expected{{e1, e2}}; + cudf::test::strings_column_wrapper e1({"", "", "a", "cc", "", "", "bb"}, {0, 0, 1, 1, 0, 0, 1}); + cudf::test::fixed_width_column_wrapper e2({-1, 1, 3, 6, -1, 2, 4}, + {0, 1, 1, 1, 0, 1, 1}); + cudf::table_view expected{{e1, e2}}; - std::vector offset{2, 1}; - auto slr1 = cudf::make_default_constructed_scalar(column_view(v1).type()); - auto slr2 = cudf::make_default_constructed_scalar(column_view(v2).type()); - std::vector> fill_values{*slr1, *slr2}; + std::vector offset{2, 1}; + auto slr1 = cudf::make_default_constructed_scalar(cudf::column_view(v1).type()); + auto slr2 = cudf::make_default_constructed_scalar(cudf::column_view(v2).type()); + std::vector> fill_values{*slr1, *slr2}; test_groupby_shift_multi(key, value, offset, fill_values, expected); } TYPED_TEST(groupby_shift_mixed_test, Fill) { - fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; - strings_column_wrapper v1{"a", "bb", "cc", "d", "eee", "f", "gg"}; - fixed_width_column_wrapper v2{1, 2, 3, 4, 5, 6, 7}; - table_view value{{v1, v2}}; + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{1, 2, 1, 2, 2, 1, 1}; + cudf::test::strings_column_wrapper v1{"a", "bb", "cc", "d", "eee", "f", "gg"}; + cudf::test::fixed_width_column_wrapper v2{1, 2, 3, 4, 5, 6, 7}; + cudf::table_view value{{v1, v2}}; - strings_column_wrapper e1({"cc", "f", "gg", "42", "d", "eee", "42"}); - fixed_width_column_wrapper e2({6, 7, 42, 42, 5, 42, 42}); - table_view expected{{e1, e2}}; + cudf::test::strings_column_wrapper e1({"cc", "f", "gg", "42", "d", "eee", "42"}); + cudf::test::fixed_width_column_wrapper e2({6, 7, 42, 42, 5, 42, 42}); + cudf::table_view expected{{e1, e2}}; - std::vector offset{-1, -2}; + std::vector offset{-1, -2}; auto slr1 = cudf::make_string_scalar("42"); - auto slr2 = cudf::scalar_type_t(make_type_param_scalar(42), true); - std::vector> fill_values{*slr1, slr2}; + auto slr2 = + cudf::scalar_type_t(cudf::test::make_type_param_scalar(42), true); + std::vector> fill_values{*slr1, slr2}; test_groupby_shift_multi(key, value, offset, fill_values, expected); } -struct groupby_shift_fixed_point_type_test : public BaseFixture { +struct groupby_shift_fixed_point_type_test : public cudf::test::BaseFixture { }; TEST_F(groupby_shift_fixed_point_type_test, Matching) { - fixed_width_column_wrapper key{2, 3, 4, 4, 3, 2, 2, 4}; - fixed_point_column_wrapper v1{{10, 10, 40, 40, 20, 20, 30, 40}, numeric::scale_type{-1}}; - fixed_point_column_wrapper v2{{5, 5, 8, 8, 6, 7, 9, 7}, numeric::scale_type{3}}; - table_view value{{v1, v2}}; - - std::vector offset{-3, 1}; - auto slr1 = make_fixed_point_scalar(-42, numeric::scale_type{-1}); - auto slr2 = make_fixed_point_scalar(42, numeric::scale_type{3}); - std::vector> fill_values{*slr1, *slr2}; - - fixed_point_column_wrapper e1{{-42, -42, -42, -42, -42, -42, -42, -42}, - numeric::scale_type{-1}}; - fixed_point_column_wrapper e2{{42, 5, 7, 42, 5, 42, 8, 8}, numeric::scale_type{3}}; - table_view expected{{e1, e2}}; + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{2, 3, 4, 4, 3, 2, 2, 4}; + cudf::test::fixed_point_column_wrapper v1{{10, 10, 40, 40, 20, 20, 30, 40}, + numeric::scale_type{-1}}; + cudf::test::fixed_point_column_wrapper v2{{5, 5, 8, 8, 6, 7, 9, 7}, + numeric::scale_type{3}}; + cudf::table_view value{{v1, v2}}; + + std::vector offset{-3, 1}; + auto slr1 = cudf::make_fixed_point_scalar(-42, numeric::scale_type{-1}); + auto slr2 = cudf::make_fixed_point_scalar(42, numeric::scale_type{3}); + std::vector> fill_values{*slr1, *slr2}; + + cudf::test::fixed_point_column_wrapper e1{{-42, -42, -42, -42, -42, -42, -42, -42}, + numeric::scale_type{-1}}; + cudf::test::fixed_point_column_wrapper e2{{42, 5, 7, 42, 5, 42, 8, 8}, + numeric::scale_type{3}}; + cudf::table_view expected{{e1, e2}}; test_groupby_shift_multi(key, value, offset, fill_values, expected); } TEST_F(groupby_shift_fixed_point_type_test, MismatchScaleType) { - fixed_width_column_wrapper key{2, 3, 4, 4, 3, 2, 2, 4}; - fixed_point_column_wrapper v1{{10, 10, 40, 40, 20, 20, 30, 40}, numeric::scale_type{-1}}; + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{2, 3, 4, 4, 3, 2, 2, 4}; + cudf::test::fixed_point_column_wrapper v1{{10, 10, 40, 40, 20, 20, 30, 40}, + numeric::scale_type{-1}}; - std::vector offset{-3}; - auto slr1 = make_fixed_point_scalar(-42, numeric::scale_type{-4}); + std::vector offset{-3}; + auto slr1 = cudf::make_fixed_point_scalar(-42, numeric::scale_type{-4}); - fixed_point_column_wrapper stub{{-42, -42, -42, -42, -42, -42, -42, -42}, - numeric::scale_type{-1}}; + cudf::test::fixed_point_column_wrapper stub{{-42, -42, -42, -42, -42, -42, -42, -42}, + numeric::scale_type{-1}}; - EXPECT_THROW(test_groupby_shift_multi(key, table_view{{v1}}, offset, {*slr1}, table_view{{stub}}), - logic_error); + EXPECT_THROW(test_groupby_shift_multi( + key, cudf::table_view{{v1}}, offset, {*slr1}, cudf::table_view{{stub}}), + cudf::logic_error); } TEST_F(groupby_shift_fixed_point_type_test, MismatchRepType) { - fixed_width_column_wrapper key{2, 3, 4, 4, 3, 2, 2, 4}; - fixed_point_column_wrapper v1{{10, 10, 40, 40, 20, 20, 30, 40}, numeric::scale_type{-1}}; + using K = int32_t; + cudf::test::fixed_width_column_wrapper key{2, 3, 4, 4, 3, 2, 2, 4}; + cudf::test::fixed_point_column_wrapper v1{{10, 10, 40, 40, 20, 20, 30, 40}, + numeric::scale_type{-1}}; - std::vector offset{-3}; - auto slr1 = make_fixed_point_scalar(-42, numeric::scale_type{-1}); + std::vector offset{-3}; + auto slr1 = cudf::make_fixed_point_scalar(-42, numeric::scale_type{-1}); - fixed_point_column_wrapper stub{{-42, -42, -42, -42, -42, -42, -42, -42}, - numeric::scale_type{-1}}; + cudf::test::fixed_point_column_wrapper stub{{-42, -42, -42, -42, -42, -42, -42, -42}, + numeric::scale_type{-1}}; - EXPECT_THROW(test_groupby_shift_multi(key, table_view{{v1}}, offset, {*slr1}, table_view{{stub}}), - logic_error); + EXPECT_THROW(test_groupby_shift_multi( + key, cudf::table_view{{v1}}, offset, {*slr1}, cudf::table_view{{stub}}), + cudf::logic_error); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/std_tests.cpp b/cpp/tests/groupby/std_tests.cpp index 27f1deea844..fa3afeb30f8 100644 --- a/cpp/tests/groupby/std_tests.cpp +++ b/cpp/tests/groupby/std_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,6 @@ * limitations under the License. */ -#ifdef NDEBUG // currently groupby std tests are not supported. See groupstd.cu - #include #include @@ -27,140 +25,168 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_std_test : public cudf::test::BaseFixture { }; -using K = int32_t; using supported_types = cudf::test::Types; TYPED_TEST_SUITE(groupby_std_test, supported_types); +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, basic) +#else +TYPED_TEST(groupby_std_test, DISABLED_basic) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // clang-format off - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({3., sqrt(131./12), sqrt(31./3)}, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({3., sqrt(131./12), sqrt(31./3)}, no_nulls()); // clang-format on auto agg = cudf::make_std_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, empty_cols) +#else +TYPED_TEST(groupby_std_test, DISABLED_empty_cols) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_std_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, zero_valid_keys) +#else +TYPED_TEST(groupby_std_test, DISABLED_zero_valid_keys) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_std_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, zero_valid_values) +#else +TYPED_TEST(groupby_std_test, DISABLED_zero_valid_values) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); auto agg = cudf::make_std_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, null_keys_and_values) +#else +TYPED_TEST(groupby_std_test, DISABLED_null_keys_and_values) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, 3} - fixed_width_column_wrapper expect_vals({3 / sqrt(2), 7 / sqrt(3), 3 * sqrt(2), 0.}, - {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({3 / sqrt(2), 7 / sqrt(3), 3 * sqrt(2), 0.}, + {1, 1, 1, 0}); auto agg = cudf::make_std_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, ddof_non_default) +#else +TYPED_TEST(groupby_std_test, DISABLED_ddof_non_default) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, 3} - fixed_width_column_wrapper expect_vals({0., 7 * sqrt(2. / 3), 0., 0.}, {0, 1, 0, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({0., 7 * sqrt(2. / 3), 0., 0.}, + {0, 1, 0, 0}); auto agg = cudf::make_std_aggregation(2); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG // currently groupby std tests are not supported in debug TYPED_TEST(groupby_std_test, dictionary) +#else +TYPED_TEST(groupby_std_test, DISABLED_dictionary) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({1, 2, 3}); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3}); // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({3., sqrt(131./12), sqrt(31./3)}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({3., sqrt(131./12), sqrt(31./3)}, no_nulls()); // clang-format on test_single_agg( keys, vals, expect_keys, expect_vals, cudf::make_std_aggregation()); } - -} // namespace test -} // namespace cudf - -#endif // NDEBUG diff --git a/cpp/tests/groupby/structs_tests.cpp b/cpp/tests/groupby/structs_tests.cpp index c058b73b036..a7bc6016307 100644 --- a/cpp/tests/groupby/structs_tests.cpp +++ b/cpp/tests/groupby/structs_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,27 +25,17 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { - template struct groupby_structs_test : public cudf::test::BaseFixture { }; TYPED_TEST_SUITE(groupby_structs_test, cudf::test::FixedWidthTypes); -using V = int32_t; // Type of Aggregation Column. -using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. -using R = cudf::detail::target_type_t; // Type of aggregation result. -using offsets = std::vector; -using strings = strings_column_wrapper; -using structs = structs_column_wrapper; - template -using fwcw = fixed_width_column_wrapper; +using fwcw = cudf::test::fixed_width_column_wrapper; template -using lcw = lists_column_wrapper; +using lcw = cudf::test::lists_column_wrapper; namespace { static constexpr auto null = -1; // Signifies null value. @@ -57,22 +47,23 @@ static constexpr auto null = -1; // Signifies null value. // Set this to true to enable printing, for debugging. auto constexpr print_enabled = false; -void print_agg_results(column_view const& keys, column_view const& vals) +void print_agg_results(cudf::column_view const& keys, cudf::column_view const& vals) { if constexpr (print_enabled) { - auto requests = std::vector{}; - requests.push_back(groupby::aggregation_request{}); + auto requests = std::vector{}; + requests.push_back(cudf::groupby::aggregation_request{}); requests.back().values = vals; - requests.back().aggregations.push_back(cudf::make_sum_aggregation()); + requests.back().aggregations.push_back(cudf::make_sum_aggregation()); requests.back().aggregations.push_back( - cudf::make_nth_element_aggregation(0)); + cudf::make_nth_element_aggregation(0)); - auto gby = groupby::groupby{table_view({keys}), null_policy::INCLUDE, sorted::NO, {}, {}}; + auto gby = cudf::groupby::groupby{ + cudf::table_view({keys}), cudf::null_policy::INCLUDE, cudf::sorted::NO, {}, {}}; auto result = gby.aggregate(requests); std::cout << "Results: Keys: " << std::endl; - print(result.first->get_column(0).view()); + cudf::test::print(result.first->get_column(0).view()); std::cout << "Results: Values: " << std::endl; - print(result.second.front().results[0]->view()); + cudf::test::print(result.second.front().results[0]->view()); } } @@ -80,20 +71,23 @@ void print_agg_results(column_view const& keys, column_view const& vals) TYPED_TEST(groupby_structs_test, basic) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; auto member_0 = fwcw{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto member_1 = fwcw{ 11, 22, 33, 11, 22, 22, 11, 33, 33, 22}; - auto member_2 = strings {"11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; - auto keys = structs{member_0, member_1, member_2}; + auto member_2 = cudf::test::strings_column_wrapper {"11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; + auto keys = cudf::test::structs_column_wrapper{member_0, member_1, member_2}; auto expected_values = fwcw { 9, 19, 17 }; auto expected_member_0 = fwcw{ 1, 2, 3 }; auto expected_member_1 = fwcw{ 11, 22, 33 }; - auto expected_member_2 = strings {"11", "22", "33"}; - auto expected_keys = structs{expected_member_0, expected_member_1, expected_member_2}; + auto expected_member_2 = cudf::test::strings_column_wrapper {"11", "22", "33"}; + auto expected_keys = cudf::test::structs_column_wrapper{expected_member_0, expected_member_1, expected_member_2}; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -101,14 +95,17 @@ TYPED_TEST(groupby_structs_test, basic) TYPED_TEST(groupby_structs_test, structs_with_nulls_in_members) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; auto member_0 = fwcw{{ 1, null, 3, 1, 2, 2, 1, 3, 3, 2 }, null_at(1)}; auto member_1 = fwcw{{ 11, 22, 33, 11, 22, 22, 11, null, 33, 22 }, null_at(7)}; - auto member_2 = strings { "11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; - auto keys = structs{{member_0, member_1, member_2}}; + auto member_2 = cudf::test::strings_column_wrapper { "11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; + auto keys = cudf::test::structs_column_wrapper{{member_0, member_1, member_2}}; // clang-format on print_agg_results(keys, values); @@ -117,8 +114,8 @@ TYPED_TEST(groupby_structs_test, structs_with_nulls_in_members) auto expected_values = fwcw { 9, 18, 10, 7, 1 }; auto expected_member_0 = fwcw{ { 1, 2, 3, 3, null }, null_at(4)}; auto expected_member_1 = fwcw{ { 11, 22, 33, null, 22 }, null_at(3)}; - auto expected_member_2 = strings { "11", "22", "33", "33", "22" }; - auto expected_keys = structs{expected_member_0, expected_member_1, expected_member_2}; + auto expected_member_2 = cudf::test::strings_column_wrapper { "11", "22", "33", "33", "22" }; + auto expected_keys = cudf::test::structs_column_wrapper{expected_member_0, expected_member_1, expected_member_2}; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -126,20 +123,23 @@ TYPED_TEST(groupby_structs_test, structs_with_nulls_in_members) TYPED_TEST(groupby_structs_test, structs_with_null_rows) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; auto member_0 = fwcw{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto member_1 = fwcw{ 11, 22, 33, 11, 22, 22, 11, 33, 33, 22}; - auto member_2 = strings {"11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; - auto keys = structs{{member_0, member_1, member_2}, nulls_at({0, 3})}; + auto member_2 = cudf::test::strings_column_wrapper {"11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; + auto keys = cudf::test::structs_column_wrapper{{member_0, member_1, member_2}, nulls_at({0, 3})}; auto expected_values = fwcw { 6, 19, 17, 3 }; auto expected_member_0 = fwcw{ { 1, 2, 3, null }, null_at(3)}; auto expected_member_1 = fwcw{ { 11, 22, 33, null }, null_at(3)}; - auto expected_member_2 = strings { {"11", "22", "33", "null" }, null_at(3)}; - auto expected_keys = structs{{expected_member_0, expected_member_1, expected_member_2}, null_at(3)}; + auto expected_member_2 = cudf::test::strings_column_wrapper { {"11", "22", "33", "null" }, null_at(3)}; + auto expected_keys = cudf::test::structs_column_wrapper{{expected_member_0, expected_member_1, expected_member_2}, null_at(3)}; // clang-format on print_agg_results(keys, values); @@ -149,14 +149,17 @@ TYPED_TEST(groupby_structs_test, structs_with_null_rows) TYPED_TEST(groupby_structs_test, structs_with_nulls_in_rows_and_members) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; auto member_0 = fwcw{{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2 }, null_at(1)}; auto member_1 = fwcw{{ 11, 22, 33, 11, 22, 22, 11, 33, 33, 22 }, null_at(7)}; - auto member_2 = strings { "11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; - auto keys = structs{{member_0, member_1, member_2}, null_at(4)}; + auto member_2 = cudf::test::strings_column_wrapper { "11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; + auto keys = cudf::test::structs_column_wrapper{{member_0, member_1, member_2}, null_at(4)}; // clang-format on print_agg_results(keys, values); @@ -165,8 +168,8 @@ TYPED_TEST(groupby_structs_test, structs_with_nulls_in_rows_and_members) auto expected_values = fwcw { 9, 14, 10, 7, 1, 4 }; auto expected_member_0 = fwcw{{ 1, 2, 3, 3, null, null }, nulls_at({4,5})}; auto expected_member_1 = fwcw{{ 11, 22, 33, null, 22, null }, nulls_at({3,5})}; - auto expected_member_2 = strings {{ "11", "22", "33", "33", "22", "null" }, null_at(5)}; - auto expected_keys = structs{{expected_member_0, expected_member_1, expected_member_2}, null_at(5)}; + auto expected_member_2 = cudf::test::strings_column_wrapper {{ "11", "22", "33", "33", "22", "null" }, null_at(5)}; + auto expected_keys = cudf::test::structs_column_wrapper{{expected_member_0, expected_member_1, expected_member_2}, null_at(5)}; // clang-format on print_agg_results(keys, values); @@ -178,14 +181,17 @@ TYPED_TEST(groupby_structs_test, null_members_differ_from_null_structs) // This test specifically confirms that a non-null STRUCT row `{null, null, null}` is grouped // differently from a null STRUCT row (whose members are incidentally null). + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; auto member_0 = fwcw{{ 1, null, 3, 1, 2, 2, 1, 3, 3, 2 }, null_at(1)}; auto member_1 = fwcw{{ 11, null, 33, 11, 22, 22, 11, 33, 33, 22 }, null_at(1)}; - auto member_2 = strings {{ "11", "null", "33", "11", "22", "22", "11", "33", "33", "22"}, null_at(1)}; - auto keys = structs{{member_0, member_1, member_2}, null_at(4)}; + auto member_2 = cudf::test::strings_column_wrapper {{ "11", "null", "33", "11", "22", "22", "11", "33", "33", "22"}, null_at(1)}; + auto keys = cudf::test::structs_column_wrapper{{member_0, member_1, member_2}, null_at(4)}; // clang-format on print_agg_results(keys, values); @@ -197,8 +203,8 @@ TYPED_TEST(groupby_structs_test, null_members_differ_from_null_structs) auto expected_values = fwcw { 9, 14, 17, 1, 4 }; auto expected_member_0 = fwcw{ { 1, 2, 3, null, null }, nulls_at({3,4})}; auto expected_member_1 = fwcw{ { 11, 22, 33, null, null }, nulls_at({3,4})}; - auto expected_member_2 = strings { {"11", "22", "33", "null", "null" }, nulls_at({3,4})}; - auto expected_keys = structs{{expected_member_0, expected_member_1, expected_member_2}, null_at(4)}; + auto expected_member_2 = cudf::test::strings_column_wrapper { {"11", "22", "33", "null", "null" }, nulls_at({3,4})}; + auto expected_keys = cudf::test::structs_column_wrapper{{expected_member_0, expected_member_1, expected_member_2}, null_at(4)}; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -206,19 +212,24 @@ TYPED_TEST(groupby_structs_test, null_members_differ_from_null_structs) TYPED_TEST(groupby_structs_test, structs_of_structs) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; auto struct_0_member_0 = fwcw{{ 1, null, 3, 1, 2, 2, 1, 3, 3, 2 }, null_at(1)}; auto struct_0_member_1 = fwcw{{ 11, null, 33, 11, 22, 22, 11, 33, 33, 22 }, null_at(1)}; - auto struct_0_member_2 = strings {{ "11", "null", "33", "11", "22", "22", "11", "33", "33", "22"}, null_at(1)}; + auto struct_0_member_2 = cudf::test::strings_column_wrapper {{ "11", "null", "33", "11", "22", "22", "11", "33", "33", "22"}, null_at(1)}; // clang-format on - auto struct_0 = structs{{struct_0_member_0, struct_0_member_1, struct_0_member_2}, null_at(4)}; + auto struct_0 = cudf::test::structs_column_wrapper{ + {struct_0_member_0, struct_0_member_1, struct_0_member_2}, null_at(4)}; auto struct_1_member_1 = fwcw{8, 9, 6, 8, 0, 7, 8, 6, 6, 7}; - auto keys = structs{{struct_0, struct_1_member_1}}; // Struct of structs. + auto keys = cudf::test::structs_column_wrapper{ + {struct_0, struct_1_member_1}}; // Struct of cudf::test::structs_column_wrapper. print_agg_results(keys, values); @@ -226,10 +237,10 @@ TYPED_TEST(groupby_structs_test, structs_of_structs) auto expected_values = fwcw { 9, 14, 17, 1, 4 }; auto expected_member_0 = fwcw{ { 1, 2, 3, null, null }, nulls_at({3,4})}; auto expected_member_1 = fwcw{ { 11, 22, 33, null, null }, nulls_at({3,4})}; - auto expected_member_2 = strings { {"11", "22", "33", "null", "null" }, nulls_at({3,4})}; - auto expected_structs = structs{{expected_member_0, expected_member_1, expected_member_2}, null_at(4)}; + auto expected_member_2 = cudf::test::strings_column_wrapper { {"11", "22", "33", "null", "null" }, nulls_at({3,4})}; + auto expected_structs = cudf::test::structs_column_wrapper{{expected_member_0, expected_member_1, expected_member_2}, null_at(4)}; auto expected_struct_1_member_1 = fwcw{ 8, 7, 6, 9, 0 }; - auto expected_keys = structs{{expected_structs, expected_struct_1_member_1}}; + auto expected_keys = cudf::test::structs_column_wrapper{{expected_structs, expected_struct_1_member_1}}; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -237,20 +248,23 @@ TYPED_TEST(groupby_structs_test, structs_of_structs) TYPED_TEST(groupby_structs_test, empty_input) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw {}; auto member_0 = fwcw{}; auto member_1 = fwcw{}; - auto member_2 = strings {}; - auto keys = structs{member_0, member_1, member_2}; + auto member_2 = cudf::test::strings_column_wrapper {}; + auto keys = cudf::test::structs_column_wrapper{member_0, member_1, member_2}; auto expected_values = fwcw {}; auto expected_member_0 = fwcw{}; auto expected_member_1 = fwcw{}; - auto expected_member_2 = strings {}; - auto expected_keys = structs{expected_member_0, expected_member_1, expected_member_2}; + auto expected_member_2 = cudf::test::strings_column_wrapper {}; + auto expected_keys = cudf::test::structs_column_wrapper{expected_member_0, expected_member_1, expected_member_2}; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -258,20 +272,23 @@ TYPED_TEST(groupby_structs_test, empty_input) TYPED_TEST(groupby_structs_test, all_null_input) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. + using R = cudf::detail::target_type_t; // clang-format off auto values = fwcw { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; auto member_0 = fwcw{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; auto member_1 = fwcw{ 11, 22, 33, 11, 22, 22, 11, 33, 33, 22}; - auto member_2 = strings {"11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; - auto keys = structs{{member_0, member_1, member_2}, all_nulls()}; + auto member_2 = cudf::test::strings_column_wrapper {"11", "22", "33", "11", "22", "22", "11", "33", "33", "22"}; + auto keys = cudf::test::structs_column_wrapper{{member_0, member_1, member_2}, all_nulls()}; auto expected_values = fwcw { 45 }; auto expected_member_0 = fwcw{ null }; auto expected_member_1 = fwcw{ null }; - auto expected_member_2 = strings {"null"}; - auto expected_keys = structs{{expected_member_0, expected_member_1, expected_member_2}, all_nulls()}; + auto expected_member_2 = cudf::test::strings_column_wrapper {"null"}; + auto expected_keys = cudf::test::structs_column_wrapper{{expected_member_0, expected_member_1, expected_member_2}, all_nulls()}; // clang-format on test_sum_agg(keys, values, expected_keys, expected_values); @@ -279,6 +296,8 @@ TYPED_TEST(groupby_structs_test, all_null_input) TYPED_TEST(groupby_structs_test, lists_are_unsupported) { + using V = int32_t; // Type of Aggregation Column. + using M0 = int32_t; // Type of STRUCT's first (i.e. 0th) member. using M1 = TypeParam; // Type of STRUCT's second (i.e. 1th) member. // clang-format off @@ -286,10 +305,7 @@ TYPED_TEST(groupby_structs_test, lists_are_unsupported) auto member_0 = lcw { {1,1}, {2,2}, {3,3}, {1,1}, {2,2} }; auto member_1 = fwcw{ 1, 2, 3, 1, 2 }; // clang-format on - auto keys = structs{{member_0, member_1}}; + auto keys = cudf::test::structs_column_wrapper{{member_0, member_1}}; EXPECT_THROW(test_sum_agg(keys, values, keys, values), cudf::logic_error); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/sum_of_squares_tests.cpp b/cpp/tests/groupby/sum_of_squares_tests.cpp index 4f4d15be089..4a784539798 100644 --- a/cpp/tests/groupby/sum_of_squares_tests.cpp +++ b/cpp/tests/groupby/sum_of_squares_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,119 +25,119 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_sum_of_squares_test : public cudf::test::BaseFixture { }; using supported_types = cudf::test::Types; -using K = int32_t; TYPED_TEST_SUITE(groupby_sum_of_squares_test, supported_types); TYPED_TEST(groupby_sum_of_squares_test, basic) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; // { 1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; // { 0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({45., 123., 117.}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_vals({45., 123., 117.}, no_nulls()); - auto agg = cudf::make_sum_of_squares_aggregation(); + auto agg = cudf::make_sum_of_squares_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_sum_of_squares_test, empty_cols) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_sum_of_squares_aggregation(); + auto agg = cudf::make_sum_of_squares_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_sum_of_squares_test, zero_valid_keys) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_sum_of_squares_aggregation(); + auto agg = cudf::make_sum_of_squares_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_sum_of_squares_test, zero_valid_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); - auto agg = cudf::make_sum_of_squares_aggregation(); + auto agg = cudf::make_sum_of_squares_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_sum_of_squares_test, null_keys_and_values) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, 3} - fixed_width_column_wrapper expect_vals({45., 98., 68., 9.}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({45., 98., 68., 9.}, {1, 1, 1, 0}); - auto agg = cudf::make_sum_of_squares_aggregation(); + auto agg = cudf::make_sum_of_squares_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } TYPED_TEST(groupby_sum_of_squares_test, dictionary) { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({1, 2, 3 }); - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({45., 123., 117. }, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3 }); + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({45., 123., 117. }, no_nulls()); // clang-format on test_single_agg(keys, vals, expect_keys, expect_vals, - cudf::make_sum_of_squares_aggregation()); + cudf::make_sum_of_squares_aggregation()); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/sum_scan_tests.cpp b/cpp/tests/groupby/sum_scan_tests.cpp index f4ac3a94d19..32cba668467 100644 --- a/cpp/tests/groupby/sum_scan_tests.cpp +++ b/cpp/tests/groupby/sum_scan_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,19 +23,14 @@ #include -using namespace cudf::test::iterators; - -namespace cudf { -namespace test { -using K = int32_t; -using key_wrapper = fixed_width_column_wrapper; +using key_wrapper = cudf::test::fixed_width_column_wrapper; template struct groupby_sum_scan_test : public cudf::test::BaseFixture { using V = T; - using R = cudf::detail::target_type_t; - using value_wrapper = fixed_width_column_wrapper; - using result_wrapper = fixed_width_column_wrapper; + using R = cudf::detail::target_type_t; + using value_wrapper = cudf::test::fixed_width_column_wrapper; + using result_wrapper = cudf::test::fixed_width_column_wrapper; }; using supported_types = @@ -57,7 +52,7 @@ TYPED_TEST(groupby_sum_scan_test, basic) // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} result_wrapper expect_vals{0, 3, 9, 1, 5, 10, 19, 2, 9, 17}; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -74,9 +69,14 @@ TYPED_TEST(groupby_sum_scan_test, pre_sorted) result_wrapper expect_vals{0, 3, 9, 1, 5, 10, 19, 2, 9, 17}; // clang-format on - auto agg = cudf::make_sum_aggregation(); - test_single_scan( - keys, vals, expect_keys, expect_vals, std::move(agg), null_policy::EXCLUDE, sorted::YES); + auto agg = cudf::make_sum_aggregation(); + test_single_scan(keys, + vals, + expect_keys, + expect_vals, + std::move(agg), + cudf::null_policy::EXCLUDE, + cudf::sorted::YES); } TYPED_TEST(groupby_sum_scan_test, empty_cols) @@ -92,7 +92,7 @@ TYPED_TEST(groupby_sum_scan_test, empty_cols) result_wrapper expect_vals{}; // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -101,15 +101,12 @@ TYPED_TEST(groupby_sum_scan_test, zero_valid_keys) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys({1, 2, 3}, all_nulls()); + key_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); value_wrapper vals{3, 4, 5}; - key_wrapper expect_keys{}; result_wrapper expect_vals{}; - // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -118,15 +115,12 @@ TYPED_TEST(groupby_sum_scan_test, zero_valid_values) using value_wrapper = typename TestFixture::value_wrapper; using result_wrapper = typename TestFixture::result_wrapper; - // clang-format off - key_wrapper keys {1, 1, 1}; - value_wrapper vals({3, 4, 5}, all_nulls()); - - key_wrapper expect_keys {1, 1, 1}; - result_wrapper expect_vals({3, 4, 5}, all_nulls()); - // clang-format on + key_wrapper keys{1, 1, 1}; + value_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); + key_wrapper expect_keys{1, 1, 1}; + result_wrapper expect_vals({3, 4, 5}, cudf::test::iterators::all_nulls()); - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } @@ -140,28 +134,28 @@ TYPED_TEST(groupby_sum_scan_test, null_keys_and_values) value_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 1, 2, 2, 2, 2, 3, *, 3, 4}; - key_wrapper expect_keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, no_nulls()); + key_wrapper expect_keys( { 1, 1, 1, 2, 2, 2, 2, 3, 3, 4}, cudf::test::iterators::no_nulls()); // { -, 3, 6, 1, 4, -, 9, 2, _, 8, -} result_wrapper expect_vals({-1, 3, 9, 1, 5, -1, 14, 2, 10, -1}, { 0, 1, 1, 1, 1, 0, 1, 1, 1, 0}); // clang-format on - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals, std::move(agg)); } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupBySumScanFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupBySumScanFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupBySortSumScanDecimalAsValue) +TYPED_TEST(GroupBySumScanFixedPointTest, GroupBySortSumScanDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; using RepType = cudf::device_storage_type_t; - using fp_wrapper = fixed_point_column_wrapper; + using fp_wrapper = cudf::test::fixed_point_column_wrapper; for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; @@ -173,10 +167,7 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortSumScanDecimalAsValue) auto const expect_vals_sum = fp_wrapper{{0, 3, 9, 1, 5, 10, 19, 2, 9, 17}, scale}; // clang-format on - auto agg2 = cudf::make_sum_aggregation(); + auto agg2 = cudf::make_sum_aggregation(); test_single_scan(keys, vals, expect_keys, expect_vals_sum, std::move(agg2)); } } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/sum_tests.cpp b/cpp/tests/groupby/sum_tests.cpp index be7da4a784c..2369e6c1128 100644 --- a/cpp/tests/groupby/sum_tests.cpp +++ b/cpp/tests/groupby/sum_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,8 +25,6 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_sum_test : public cudf::test::BaseFixture { }; @@ -41,118 +39,115 @@ TYPED_TEST_SUITE(groupby_sum_test, supported_types); TYPED_TEST(groupby_sum_test, basic) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - fixed_width_column_wrapper expect_keys{1, 2, 3}; - fixed_width_column_wrapper expect_vals{9, 19, 17}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{9, 19, 17}; - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_sum_aggregation(); + auto agg2 = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_sum_test, empty_cols) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_sum_aggregation(); + auto agg2 = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_sum_test, zero_valid_keys) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, cudf::test::iterators::all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_sum_aggregation(); + auto agg2 = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_sum_test, zero_valid_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, cudf::test::iterators::all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, cudf::test::iterators::all_nulls()); - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_sum_aggregation(); + auto agg2 = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } TYPED_TEST(groupby_sum_test, null_keys_and_values) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 4}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0}); // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, + cudf::test::iterators::no_nulls()); // { 3, 6, 1, 4, 9, 2, 8, -} - fixed_width_column_wrapper expect_vals({9, 14, 10, 0}, {1, 1, 1, 0}); + cudf::test::fixed_width_column_wrapper expect_vals({9, 14, 10, 0}, {1, 1, 1, 0}); - auto agg = cudf::make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); - auto agg2 = cudf::make_sum_aggregation(); + auto agg2 = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg2), force_use_sort_impl::YES); } -// clang-format on TYPED_TEST(groupby_sum_test, dictionary) { using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - // clang-format off - fixed_width_column_wrapper keys{ 1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - - fixed_width_column_wrapper expect_keys{ 1, 2, 3 }; - fixed_width_column_wrapper expect_vals{ 9, 19, 17}; - // clang-format on + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + cudf::test::fixed_width_column_wrapper expect_vals{9, 19, 17}; test_single_agg( - keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation()); + keys, vals, expect_keys, expect_vals, cudf::make_sum_aggregation()); test_single_agg(keys, vals, expect_keys, expect_vals, - cudf::make_sum_aggregation(), + cudf::make_sum_aggregation(), force_use_sort_impl::YES); } @@ -160,8 +155,8 @@ struct overflow_test : public cudf::test::BaseFixture { }; TEST_F(overflow_test, overflow_integer) { - using int32_col = fixed_width_column_wrapper; - using int64_col = fixed_width_column_wrapper; + using int32_col = cudf::test::fixed_width_column_wrapper; + using int64_col = cudf::test::fixed_width_column_wrapper; auto const keys = int32_col{0, 0}; auto const vals = int32_col{-2147483648, -2147483648}; @@ -169,7 +164,7 @@ TEST_F(overflow_test, overflow_integer) auto const expect_vals = int64_col{-4294967296L}; auto test_sum = [&](auto const use_sort) { - auto agg = make_sum_aggregation(); + auto agg = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg), use_sort); }; @@ -178,12 +173,12 @@ TEST_F(overflow_test, overflow_integer) } template -struct FixedPointTestAllReps : public cudf::test::BaseFixture { +struct GroupBySumFixedPointTest : public cudf::test::BaseFixture { }; -TYPED_TEST_SUITE(FixedPointTestAllReps, cudf::test::FixedPointTypes); +TYPED_TEST_SUITE(GroupBySumFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(FixedPointTestAllReps, GroupBySortSumDecimalAsValue) +TYPED_TEST(GroupBySumFixedPointTest, GroupBySortSumDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; @@ -194,25 +189,25 @@ TYPED_TEST(FixedPointTestAllReps, GroupBySortSumDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_sum = fp_wrapper{{9, 19, 17}, scale}; - auto agg1 = cudf::make_sum_aggregation(); + auto agg1 = cudf::make_sum_aggregation(); test_single_agg( keys, vals, expect_keys, expect_vals_sum, std::move(agg1), force_use_sort_impl::YES); - auto agg4 = cudf::make_product_aggregation(); + auto agg4 = cudf::make_product_aggregation(); EXPECT_THROW( test_single_agg(keys, vals, expect_keys, {}, std::move(agg4), force_use_sort_impl::YES), cudf::logic_error); } } -TYPED_TEST(FixedPointTestAllReps, GroupByHashSumDecimalAsValue) +TYPED_TEST(GroupBySumFixedPointTest, GroupByHashSumDecimalAsValue) { using namespace numeric; using decimalXX = TypeParam; @@ -223,20 +218,17 @@ TYPED_TEST(FixedPointTestAllReps, GroupByHashSumDecimalAsValue) for (auto const i : {2, 1, 0, -1, -2}) { auto const scale = scale_type{i}; // clang-format off - auto const keys = fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto const keys = cudf::test::fixed_width_column_wrapper{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + auto const vals = fp_wrapper{ {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; // clang-format on - auto const expect_keys = fixed_width_column_wrapper{1, 2, 3}; + auto const expect_keys = cudf::test::fixed_width_column_wrapper{1, 2, 3}; auto const expect_vals_sum = fp_wrapper{{9, 19, 17}, scale}; - auto agg5 = cudf::make_sum_aggregation(); + auto agg5 = cudf::make_sum_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals_sum, std::move(agg5)); - auto agg8 = cudf::make_product_aggregation(); + auto agg8 = cudf::make_product_aggregation(); EXPECT_THROW(test_single_agg(keys, vals, expect_keys, {}, std::move(agg8)), cudf::logic_error); } } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/tdigest_tests.cu b/cpp/tests/groupby/tdigest_tests.cu index b757bf16c6f..d7446d4dabb 100644 --- a/cpp/tests/groupby/tdigest_tests.cu +++ b/cpp/tests/groupby/tdigest_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,27 +14,21 @@ * limitations under the License. */ -#include -#include -#include -#include -#include - #include #include #include #include #include +#include +#include +#include +#include + #include #include #include -namespace cudf { -namespace test { - -using namespace cudf; - /** * @brief Functor to generate a tdigest by key. * @@ -43,7 +37,9 @@ struct tdigest_gen_grouped { template < typename T, typename std::enable_if_t() || cudf::is_fixed_point()>* = nullptr> - std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta) + std::unique_ptr operator()(cudf::column_view const& keys, + cudf::column_view const& values, + int delta) { cudf::table_view t({keys}); cudf::groupby::groupby gb(t); @@ -58,7 +54,9 @@ struct tdigest_gen_grouped { template < typename T, typename std::enable_if_t() && !cudf::is_fixed_point()>* = nullptr> - std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta) + std::unique_ptr operator()(cudf::column_view const& keys, + cudf::column_view const& values, + int delta) { CUDF_FAIL("Invalid tdigest test type"); } @@ -69,11 +67,11 @@ struct tdigest_gen_grouped { * */ struct tdigest_groupby_simple_op { - std::unique_ptr operator()(column_view const& values, int delta) const + std::unique_ptr operator()(cudf::column_view const& values, int delta) const { // make a simple set of matching keys. auto keys = cudf::make_fixed_width_column( - data_type{type_id::INT32}, values.size(), mask_state::UNALLOCATED); + cudf::data_type{cudf::type_id::INT32}, values.size(), cudf::mask_state::UNALLOCATED); thrust::fill(rmm::exec_policy(cudf::get_default_stream()), keys->mutable_view().template begin(), keys->mutable_view().template end(), @@ -95,11 +93,12 @@ struct tdigest_groupby_simple_op { * */ struct tdigest_groupby_simple_merge_op { - std::unique_ptr operator()(column_view const& merge_values, int merge_delta) const + std::unique_ptr operator()(cudf::column_view const& merge_values, + int merge_delta) const { // make a simple set of matching keys. auto merge_keys = cudf::make_fixed_width_column( - data_type{type_id::INT32}, merge_values.size(), mask_state::UNALLOCATED); + cudf::data_type{cudf::type_id::INT32}, merge_values.size(), cudf::mask_state::UNALLOCATED); thrust::fill(rmm::exec_policy(cudf::get_default_stream()), merge_keys->mutable_view().template begin(), merge_keys->mutable_view().template end(), @@ -125,24 +124,25 @@ TYPED_TEST_SUITE(TDigestAllTypes, cudf::test::NumericTypes); TYPED_TEST(TDigestAllTypes, Simple) { using T = TypeParam; - tdigest_simple_aggregation(tdigest_groupby_simple_op{}); + cudf::test::tdigest_simple_aggregation(tdigest_groupby_simple_op{}); } TYPED_TEST(TDigestAllTypes, SimpleWithNulls) { using T = TypeParam; - tdigest_simple_with_nulls_aggregation(tdigest_groupby_simple_op{}); + cudf::test::tdigest_simple_with_nulls_aggregation(tdigest_groupby_simple_op{}); } TYPED_TEST(TDigestAllTypes, AllNull) { using T = TypeParam; - tdigest_simple_all_nulls_aggregation(tdigest_groupby_simple_op{}); + cudf::test::tdigest_simple_all_nulls_aggregation(tdigest_groupby_simple_op{}); } TYPED_TEST(TDigestAllTypes, LargeGroups) { - auto _values = generate_standardized_percentile_distribution(data_type{type_id::FLOAT64}); + auto _values = cudf::test::generate_standardized_percentile_distribution( + cudf::data_type{cudf::type_id::FLOAT64}); int const delta = 1000; // generate a random set of keys @@ -150,7 +150,7 @@ TYPED_TEST(TDigestAllTypes, LargeGroups) h_keys.reserve(_values->size()); auto iter = thrust::make_counting_iterator(0); std::transform(iter, iter + _values->size(), std::back_inserter(h_keys), [](int i) { - return static_cast(round(rand_range(0, 8))); + return static_cast(round(cudf::test::rand_range(0, 8))); }); cudf::test::fixed_width_column_wrapper _keys(h_keys.begin(), h_keys.end()); @@ -161,8 +161,8 @@ TYPED_TEST(TDigestAllTypes, LargeGroups) auto groups = setup_gb.get_groups(v); // slice it all up so we have keys/columns for everything. - std::vector keys; - std::vector values; + std::vector keys; + std::vector values; for (size_t idx = 0; idx < groups.offsets.size() - 1; idx++) { auto k = cudf::slice(groups.keys->get_column(0), {groups.offsets[idx], groups.offsets[idx + 1]}); @@ -174,7 +174,7 @@ TYPED_TEST(TDigestAllTypes, LargeGroups) } // generate a separate tdigest for each group - std::vector> parts; + std::vector> parts; std::transform( iter, iter + values.size(), std::back_inserter(parts), [&keys, &values, delta](int i) { cudf::table_view t({keys[i]}); @@ -186,11 +186,11 @@ TYPED_TEST(TDigestAllTypes, LargeGroups) auto result = gb.aggregate(requests); return std::move(result.second[0].results[0]); }); - std::vector part_views; + std::vector part_views; std::transform(parts.begin(), parts.end(), std::back_inserter(part_views), - [](std::unique_ptr const& col) { return col->view(); }); + [](std::unique_ptr const& col) { return col->view(); }); auto merged_parts = cudf::concatenate(part_views); // generate a tdigest on the whole input set @@ -224,30 +224,31 @@ TEST_F(TDigestTest, EmptyMixed) requests.push_back({values, std::move(aggregations)}); auto result = gb.aggregate(requests); - using FCW = cudf::test::fixed_width_column_wrapper; - auto expected = make_expected_tdigest_column({{FCW{}, FCW{}, 0, 0}, - {FCW{123456.78}, FCW{1.0}, 123456.78, 123456.78}, - {FCW{25.0}, FCW{1.0}, 25.0, 25.0}, - {FCW{}, FCW{}, 0, 0}, - {FCW{50.0, 60.0}, FCW{1.0, 1.0}, 50.0, 60.0}, - {FCW{}, FCW{}, 0, 0}}); + using FCW = cudf::test::fixed_width_column_wrapper; + auto expected = + cudf::test::make_expected_tdigest_column({{FCW{}, FCW{}, 0, 0}, + {FCW{123456.78}, FCW{1.0}, 123456.78, 123456.78}, + {FCW{25.0}, FCW{1.0}, 25.0, 25.0}, + {FCW{}, FCW{}, 0, 0}, + {FCW{50.0, 60.0}, FCW{1.0, 1.0}, 50.0, 60.0}, + {FCW{}, FCW{}, 0, 0}}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result.second[0].results[0], *expected); } TEST_F(TDigestTest, LargeInputDouble) { - tdigest_simple_large_input_double_aggregation(tdigest_groupby_simple_op{}); + cudf::test::tdigest_simple_large_input_double_aggregation(tdigest_groupby_simple_op{}); } TEST_F(TDigestTest, LargeInputInt) { - tdigest_simple_large_input_int_aggregation(tdigest_groupby_simple_op{}); + cudf::test::tdigest_simple_large_input_int_aggregation(tdigest_groupby_simple_op{}); } TEST_F(TDigestTest, LargeInputDecimal) { - tdigest_simple_large_input_decimal_aggregation(tdigest_groupby_simple_op{}); + cudf::test::tdigest_simple_large_input_decimal_aggregation(tdigest_groupby_simple_op{}); } struct TDigestMergeTest : public cudf::test::BaseFixture { @@ -257,19 +258,20 @@ struct TDigestMergeTest : public cudf::test::BaseFixture { // the same regardless of input. TEST_F(TDigestMergeTest, Simple) { - tdigest_merge_simple(tdigest_groupby_simple_op{}, tdigest_groupby_simple_merge_op{}); + cudf::test::tdigest_merge_simple(tdigest_groupby_simple_op{}, tdigest_groupby_simple_merge_op{}); } struct key_groups { - __device__ size_type operator()(size_type i) { return i < 250000 ? 0 : 1; } + __device__ cudf::size_type operator()(cudf::size_type i) { return i < 250000 ? 0 : 1; } }; TEST_F(TDigestMergeTest, Grouped) { - auto values = generate_standardized_percentile_distribution(data_type{type_id::FLOAT64}); + auto values = cudf::test::generate_standardized_percentile_distribution( + cudf::data_type{cudf::type_id::FLOAT64}); CUDF_EXPECTS(values->size() == 750000, "Unexpected distribution size"); // all in the same group auto keys = cudf::make_fixed_width_column( - data_type{type_id::INT32}, values->size(), mask_state::UNALLOCATED); + cudf::data_type{cudf::type_id::INT32}, values->size(), cudf::mask_state::UNALLOCATED); // 3 groups. 0-250000 in group 0. 250000-500000 in group 1 and 500000-750000 in group 1 auto key_iter = cudf::detail::make_counting_transform_iterator(0, key_groups{}); thrust::copy(rmm::exec_policy(cudf::get_default_stream()), @@ -284,7 +286,7 @@ TEST_F(TDigestMergeTest, Grouped) int const delta = 1000; // generate separate digests - std::vector> parts; + std::vector> parts; auto iter = thrust::make_counting_iterator(0); std::transform( iter, @@ -300,11 +302,11 @@ TEST_F(TDigestMergeTest, Grouped) auto result = gb.aggregate(requests); return std::move(result.second[0].results[0]); }); - std::vector part_views; + std::vector part_views; std::transform(parts.begin(), parts.end(), std::back_inserter(part_views), - [](std::unique_ptr const& col) { return col->view(); }); + [](std::unique_ptr const& col) { return col->view(); }); // merge delta = 1000 { @@ -326,38 +328,38 @@ TEST_F(TDigestMergeTest, Grouped) cudf::tdigest::tdigest_column_view tdv(*result.second[0].results[0]); // verify centroids - std::vector expected{// group 0 - {0, 0.00013945158577498588, 2}, - {10, 0.04804393446447509375, 50}, - {66, 2.10089484962640948851, 316}, - {139, 8.92977366346101852912, 601}, - {243, 23.89152910016953867967, 784}, - {366, 41.62636569363655780762, 586}, - {432, 47.73085102980330418632, 326}, - {460, 49.20637897385523018556, 196}, - {501, 49.99998311512171511595, 1}, - // group 1 - {502 + 0, 50.00022508669655252334, 2}, - {502 + 15, 50.05415694538910287292, 74}, - {502 + 70, 51.21421484112906341579, 334}, - {502 + 150, 55.19367617848146778670, 635}, - {502 + 260, 63.24605285552920008740, 783}, - {502 + 380, 76.99522005804017510400, 1289}, - {502 + 440, 84.22673817294192133431, 758}, - {502 + 490, 88.11787981529532487457, 784}, - {502 + 555, 93.02766411136053648079, 704}, - {502 + 618, 96.91486035315536184953, 516}, - {502 + 710, 99.87755861436669135855, 110}, - {502 + 733, 99.99970905482754801596, 1}}; - tdigest_sample_compare(tdv, expected); + std::vector expected{// group 0 + {0, 0.00013945158577498588, 2}, + {10, 0.04804393446447509375, 50}, + {66, 2.10089484962640948851, 316}, + {139, 8.92977366346101852912, 601}, + {243, 23.89152910016953867967, 784}, + {366, 41.62636569363655780762, 586}, + {432, 47.73085102980330418632, 326}, + {460, 49.20637897385523018556, 196}, + {501, 49.99998311512171511595, 1}, + // group 1 + {502 + 0, 50.00022508669655252334, 2}, + {502 + 15, 50.05415694538910287292, 74}, + {502 + 70, 51.21421484112906341579, 334}, + {502 + 150, 55.19367617848146778670, 635}, + {502 + 260, 63.24605285552920008740, 783}, + {502 + 380, 76.99522005804017510400, 1289}, + {502 + 440, 84.22673817294192133431, 758}, + {502 + 490, 88.11787981529532487457, 784}, + {502 + 555, 93.02766411136053648079, 704}, + {502 + 618, 96.91486035315536184953, 516}, + {502 + 710, 99.87755861436669135855, 110}, + {502 + 733, 99.99970905482754801596, 1}}; + cudf::test::tdigest_sample_compare(tdv, expected); // verify min/max auto split_results = cudf::split(*result.second[0].results[0], {1}); auto iter = thrust::make_counting_iterator(0); - std::for_each(iter, iter + split_results.size(), [&](size_type i) { - auto copied = std::make_unique(split_results[i]); - tdigest_minmax_compare(cudf::tdigest::tdigest_column_view(*copied), - grouped_split_values[i]); + std::for_each(iter, iter + split_results.size(), [&](cudf::size_type i) { + auto copied = std::make_unique(split_results[i]); + cudf::test::tdigest_minmax_compare(cudf::tdigest::tdigest_column_view(*copied), + grouped_split_values[i]); }); } @@ -381,30 +383,30 @@ TEST_F(TDigestMergeTest, Grouped) cudf::tdigest::tdigest_column_view tdv(*result.second[0].results[0]); // verify centroids - std::vector expected{// group 0 - {0, 0.02182479870203561656, 231}, - {3, 0.60625795002234528219, 1688}, - {13, 8.40462931740497687372, 5867}, - {27, 28.79997783486397722186, 7757}, - {35, 40.22391421196020644402, 6224}, - {45, 48.96506331299028857984, 2225}, - {50, 49.99979491345574444949, 4}, - // group 1 - {51 + 0, 50.02171921312970681583, 460}, - {51 + 5, 51.45308398121498072442, 5074}, - {51 + 11, 55.96880716301625113829, 10011}, - {51 + 22, 70.18029861315150697010, 15351}, - {51 + 38, 92.65943436519887654867, 10718}, - {51 + 47, 99.27745505225347244505, 3639}}; - tdigest_sample_compare(tdv, expected); + std::vector expected{// group 0 + {0, 0.02182479870203561656, 231}, + {3, 0.60625795002234528219, 1688}, + {13, 8.40462931740497687372, 5867}, + {27, 28.79997783486397722186, 7757}, + {35, 40.22391421196020644402, 6224}, + {45, 48.96506331299028857984, 2225}, + {50, 49.99979491345574444949, 4}, + // group 1 + {51 + 0, 50.02171921312970681583, 460}, + {51 + 5, 51.45308398121498072442, 5074}, + {51 + 11, 55.96880716301625113829, 10011}, + {51 + 22, 70.18029861315150697010, 15351}, + {51 + 38, 92.65943436519887654867, 10718}, + {51 + 47, 99.27745505225347244505, 3639}}; + cudf::test::tdigest_sample_compare(tdv, expected); // verify min/max auto split_results = cudf::split(*result.second[0].results[0], {1}); auto iter = thrust::make_counting_iterator(0); - std::for_each(iter, iter + split_results.size(), [&](size_type i) { - auto copied = std::make_unique(split_results[i]); - tdigest_minmax_compare(cudf::tdigest::tdigest_column_view(*copied), - grouped_split_values[i]); + std::for_each(iter, iter + split_results.size(), [&](cudf::size_type i) { + auto copied = std::make_unique(split_results[i]); + cudf::test::tdigest_minmax_compare(cudf::tdigest::tdigest_column_view(*copied), + grouped_split_values[i]); }); } @@ -428,34 +430,37 @@ TEST_F(TDigestMergeTest, Grouped) cudf::tdigest::tdigest_column_view tdv(*result.second[0].results[0]); // verify centroids - std::vector expected{// group 0 - {0, 2.34644806683495144028, 23623}, - {1, 10.95523693698660672169, 62290}, - {2, 24.90731657803452847588, 77208}, - {3, 38.88062495289155862110, 62658}, - {4, 47.56288303840698006297, 24217}, - {5, 49.99979491345574444949, 4}, - // group 1 - {6 + 0, 52.40174463129091719793, 47410}, - {6 + 1, 60.97025126481504031517, 124564}, - {6 + 2, 74.91722742839780835311, 154387}, - {6 + 3, 88.87559489177009197647, 124810}, - {6 + 4, 97.55823307073454486726, 48817}, - {6 + 5, 99.99901807905750672489, 12}}; - tdigest_sample_compare(tdv, expected); + std::vector expected{// group 0 + {0, 2.34644806683495144028, 23623}, + {1, 10.95523693698660672169, 62290}, + {2, 24.90731657803452847588, 77208}, + {3, 38.88062495289155862110, 62658}, + {4, 47.56288303840698006297, 24217}, + {5, 49.99979491345574444949, 4}, + // group 1 + {6 + 0, 52.40174463129091719793, 47410}, + {6 + 1, 60.97025126481504031517, 124564}, + {6 + 2, 74.91722742839780835311, 154387}, + {6 + 3, 88.87559489177009197647, 124810}, + {6 + 4, 97.55823307073454486726, 48817}, + {6 + 5, 99.99901807905750672489, 12}}; + cudf::test::tdigest_sample_compare(tdv, expected); // verify min/max auto split_results = cudf::split(*result.second[0].results[0], {1}); auto iter = thrust::make_counting_iterator(0); - std::for_each(iter, iter + split_results.size(), [&](size_type i) { - auto copied = std::make_unique(split_results[i]); - tdigest_minmax_compare(cudf::tdigest::tdigest_column_view(*copied), - grouped_split_values[i]); + std::for_each(iter, iter + split_results.size(), [&](cudf::size_type i) { + auto copied = std::make_unique(split_results[i]); + cudf::test::tdigest_minmax_compare(cudf::tdigest::tdigest_column_view(*copied), + grouped_split_values[i]); }); } } -TEST_F(TDigestMergeTest, Empty) { tdigest_merge_empty(tdigest_groupby_simple_merge_op{}); } +TEST_F(TDigestMergeTest, Empty) +{ + cudf::test::tdigest_merge_empty(tdigest_groupby_simple_merge_op{}); +} TEST_F(TDigestMergeTest, EmptyGroups) { @@ -468,13 +473,13 @@ TEST_F(TDigestMergeTest, EmptyGroups) auto a = cudf::tdigest::detail::make_empty_tdigest_column(cudf::get_default_stream()); auto b = cudf::type_dispatcher( - static_cast(values_b).type(), tdigest_gen_grouped{}, keys, values_b, delta); + static_cast(values_b).type(), tdigest_gen_grouped{}, keys, values_b, delta); auto c = cudf::tdigest::detail::make_empty_tdigest_column(cudf::get_default_stream()); auto d = cudf::type_dispatcher( - static_cast(values_d).type(), tdigest_gen_grouped{}, keys, values_d, delta); + static_cast(values_d).type(), tdigest_gen_grouped{}, keys, values_d, delta); auto e = cudf::tdigest::detail::make_empty_tdigest_column(cudf::get_default_stream()); - std::vector cols; + std::vector cols; cols.push_back(*a); cols.push_back(*b); cols.push_back(*c); @@ -496,11 +501,8 @@ TEST_F(TDigestMergeTest, EmptyGroups) cudf::test::fixed_width_column_wrapper expected_means{ 2, 55, 67, 99, 100, 126, 200, 300, 400, 500, 600}; cudf::test::fixed_width_column_wrapper expected_weights{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - auto expected = make_expected_tdigest_column( + auto expected = cudf::test::make_expected_tdigest_column( {{expected_means, expected_weights, 2, 600}, {FCW{}, FCW{}, 0, 0}, {FCW{}, FCW{}, 0, 0}}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*expected, *result.second[0].results[0]); } - -} // namespace test -} // namespace cudf diff --git a/cpp/tests/groupby/var_tests.cpp b/cpp/tests/groupby/var_tests.cpp index cc87ece1c65..2bffcd5c337 100644 --- a/cpp/tests/groupby/var_tests.cpp +++ b/cpp/tests/groupby/var_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,8 +14,6 @@ * limitations under the License. */ -#ifdef NDEBUG // currently groupby variance tests are not supported. See groupstd.cu - #include #include @@ -27,137 +25,169 @@ using namespace cudf::test::iterators; -namespace cudf { -namespace test { template struct groupby_var_test : public cudf::test::BaseFixture { }; -using K = int32_t; using supported_types = cudf::test::Types; TYPED_TEST_SUITE(groupby_var_test, supported_types); +#ifdef NDEBUG TYPED_TEST(groupby_var_test, basic) +#else +TYPED_TEST(groupby_var_test, DISABLED_basic) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys{1, 2, 3}; - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({9., 131. / 12, 31. / 3}, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({9., 131. / 12, 31. / 3}, no_nulls()); // clang-format on auto agg = cudf::make_variance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG TYPED_TEST(groupby_var_test, empty_cols) +#else +TYPED_TEST(groupby_var_test, DISABLED_empty_cols) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{}; - fixed_width_column_wrapper vals{}; + cudf::test::fixed_width_column_wrapper keys{}; + cudf::test::fixed_width_column_wrapper vals{}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_variance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG TYPED_TEST(groupby_var_test, zero_valid_keys) +#else +TYPED_TEST(groupby_var_test, DISABLED_zero_valid_keys) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); - fixed_width_column_wrapper vals{3, 4, 5}; + cudf::test::fixed_width_column_wrapper keys({1, 2, 3}, all_nulls()); + cudf::test::fixed_width_column_wrapper vals{3, 4, 5}; - fixed_width_column_wrapper expect_keys{}; - fixed_width_column_wrapper expect_vals{}; + cudf::test::fixed_width_column_wrapper expect_keys{}; + cudf::test::fixed_width_column_wrapper expect_vals{}; auto agg = cudf::make_variance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG TYPED_TEST(groupby_var_test, zero_valid_values) +#else +TYPED_TEST(groupby_var_test, DISABLED_zero_valid_values) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys{1, 1, 1}; - fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); + cudf::test::fixed_width_column_wrapper keys{1, 1, 1}; + cudf::test::fixed_width_column_wrapper vals({3, 4, 5}, all_nulls()); - fixed_width_column_wrapper expect_keys{1}; - fixed_width_column_wrapper expect_vals({0}, all_nulls()); + cudf::test::fixed_width_column_wrapper expect_keys{1}; + cudf::test::fixed_width_column_wrapper expect_vals({0}, all_nulls()); auto agg = cudf::make_variance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG TYPED_TEST(groupby_var_test, null_keys_and_values) +#else +TYPED_TEST(groupby_var_test, DISABLED_null_keys_and_values) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); // clang-format off - // {1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - // {3, 6, 1, 4, 9, 2, 8, 3} - fixed_width_column_wrapper expect_vals({4.5, 49. / 3, 18., 0.}, {1, 1, 1, 0}); + // {1, 1, 2, 2, 2, 3, 3, 4} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + // {3, 6, 1, 4, 9, 2, 8, 3} + cudf::test::fixed_width_column_wrapper expect_vals({4.5, 49. / 3, 18., 0.}, {1, 1, 1, 0}); // clang-format on auto agg = cudf::make_variance_aggregation(); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG TYPED_TEST(groupby_var_test, ddof_non_default) +#else +TYPED_TEST(groupby_var_test, DISABLED_ddof_non_default) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; - fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, - {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); - fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, - {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper keys({1, 2, 3, 1, 2, 2, 1, 3, 3, 2, 4}, + {1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1}); + cudf::test::fixed_width_column_wrapper vals({0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 3}, + {0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1}); // clang-format off - // { 1, 1, 2, 2, 2, 3, 3, 4} - fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); - // { 3, 6, 1, 4, 9, 2, 8, 3} - fixed_width_column_wrapper expect_vals({0., 98. / 3, 0., 0.}, - {0, 1, 0, 0}); + // { 1, 1, 2, 2, 2, 3, 3, 4} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3, 4}, no_nulls()); + // { 3, 6, 1, 4, 9, 2, 8, 3} + cudf::test::fixed_width_column_wrapper expect_vals({0., 98. / 3, 0., 0.}, + {0, 1, 0, 0}); // clang-format on auto agg = cudf::make_variance_aggregation(2); test_single_agg(keys, vals, expect_keys, expect_vals, std::move(agg)); } +#ifdef NDEBUG TYPED_TEST(groupby_var_test, dictionary) +#else +TYPED_TEST(groupby_var_test, DISABLED_dictionary) +#endif { + using K = int32_t; using V = TypeParam; - using R = cudf::detail::target_type_t; + using R = cudf::detail::target_type_t; // clang-format off - fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; - dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::dictionary_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} - fixed_width_column_wrapper expect_keys({1, 2, 3 }); - // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} - fixed_width_column_wrapper expect_vals({9., 131./12, 31./3 }, no_nulls()); + // {1, 1, 1, 2, 2, 2, 2, 3, 3, 3} + cudf::test::fixed_width_column_wrapper expect_keys({1, 2, 3 }); + // {0, 3, 6, 1, 4, 5, 9, 2, 7, 8} + cudf::test::fixed_width_column_wrapper expect_vals({9., 131./12, 31./3 }, no_nulls()); // clang-format on test_single_agg(keys, @@ -166,8 +196,3 @@ TYPED_TEST(groupby_var_test, dictionary) expect_vals, cudf::make_variance_aggregation()); } - -} // namespace test -} // namespace cudf - -#endif // NDEBUG diff --git a/cpp/tests/replace/replace_nulls_tests.cpp b/cpp/tests/replace/replace_nulls_tests.cpp index 616ba9d2f64..0cc953d37a1 100644 --- a/cpp/tests/replace/replace_nulls_tests.cpp +++ b/cpp/tests/replace/replace_nulls_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Copyright 2018 BlazingDB, Inc. * Copyright 2018 Alexander Ocsa @@ -17,22 +17,20 @@ * limitations under the License. */ -#include - -#include +#include +#include +#include +#include +#include +#include #include #include +#include #include #include #include #include -#include -#include -#include -#include -#include -#include #include #include From 79a924ab42fc4a04db6f48865007f631c45281f0 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Thu, 16 Feb 2023 23:27:25 -0500 Subject: [PATCH 34/69] Update `hash_partition` to use `experimental::row::row_hasher` (#12761) This PR is a part of https://github.com/rapidsai/cudf/issues/11844. Authors: - Divye Gala (https://github.com/divyegala) Approvers: - Nghia Truong (https://github.com/ttnghia) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/12761 --- cpp/src/partitioning/partitioning.cu | 14 ++-- .../partitioning/hash_partition_test.cpp | 79 ++++++++++++++++++- 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/cpp/src/partitioning/partitioning.cu b/cpp/src/partitioning/partitioning.cu index 876c8f136ae..edf5d6d6612 100644 --- a/cpp/src/partitioning/partitioning.cu +++ b/cpp/src/partitioning/partitioning.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #include @@ -489,9 +489,9 @@ std::pair, std::vector> hash_partition_table( auto row_partition_offset = cudf::detail::make_zeroed_device_uvector_async(num_rows, stream); - auto const device_input = table_device_view::create(table_to_hash, stream); - auto const hasher = row_hasher( - nullate::DYNAMIC{hash_has_nulls}, *device_input, seed); + auto const row_hasher = experimental::row::hash::row_hasher(table_to_hash, stream); + auto const hasher = + row_hasher.device_hasher(nullate::DYNAMIC{hash_has_nulls}, seed); // If the number of partitions is a power of two, we can compute the partition // number of each row more efficiently with bitwise operations @@ -578,7 +578,7 @@ std::pair, std::vector> hash_partition_table( mr); }); - if (has_nulls(input)) { + if (has_nested_nulls(input)) { // Use copy_block_partitions to compute a gather map auto gather_map = compute_gather_map(num_rows, num_partitions, @@ -730,7 +730,7 @@ std::pair, std::vector> hash_partition( return std::pair(empty_like(input), std::vector(num_partitions, 0)); } - if (has_nulls(table_to_hash)) { + if (has_nested_nulls(table_to_hash)) { return hash_partition_table( input, table_to_hash, num_partitions, seed, stream, mr); } else { diff --git a/cpp/tests/partitioning/hash_partition_test.cpp b/cpp/tests/partitioning/hash_partition_test.cpp index abf4095e4ec..7731ad815d2 100644 --- a/cpp/tests/partitioning/hash_partition_test.cpp +++ b/cpp/tests/partitioning/hash_partition_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,10 @@ using cudf::test::fixed_width_column_wrapper; using cudf::test::strings_column_wrapper; +using structs_col = cudf::test::structs_column_wrapper; + +using cudf::test::iterators::null_at; +using cudf::test::iterators::nulls_at; // Transform vector of column wrappers to vector of column views template @@ -361,4 +366,76 @@ TEST_F(HashPartition, FixedPointColumnsToHash) CUDF_TEST_EXPECT_COLUMNS_EQUAL(first_result->get_column(1).view(), first_input.column(1)); } +TEST_F(HashPartition, ListWithNulls) +{ + using lcw = cudf::test::lists_column_wrapper; + + lcw to_hash{{{1}, {2, 2}, {0}, {1}, {2, 2}, {0}}, nulls_at({2, 5})}; + fixed_width_column_wrapper first_col({7, 8, 9, 10, 11, 12}); + fixed_width_column_wrapper second_col({13, 14, 15, 16, 17, 18}); + auto const first_input = cudf::table_view({to_hash, first_col}); + auto const second_input = cudf::table_view({to_hash, second_col}); + + auto const columns_to_hash = std::vector({0}); + + cudf::size_type const num_partitions = 3; + auto const [first_result, first_offsets] = + cudf::hash_partition(first_input, columns_to_hash, num_partitions); + auto const [second_result, second_offsets] = + cudf::hash_partition(second_input, columns_to_hash, num_partitions); + + // Expect offsets to be equal and num_partitions in length + EXPECT_EQ(static_cast(num_partitions), first_offsets.size()); + EXPECT_TRUE(std::equal( + first_offsets.begin(), first_offsets.end(), second_offsets.begin(), second_offsets.end())); + + // Expect same result for the hashed columns + CUDF_TEST_EXPECT_COLUMNS_EQUAL(first_result->get_column(0).view(), + second_result->get_column(0).view()); +} + +TEST_F(HashPartition, StructofStructWithNulls) +{ + // +-----------------+ + // | s1{s2{a,b}, c} | + // +-----------------+ + // 0 | { {1, 1}, 5} | + // 1 | { {1, 1}, 5} | + // 2 | { {1, 2}, 4} | + // 3 | Null | + // 4 | { Null, 4} | + // 5 | { Null, 4} | + + auto const to_hash = [&] { + auto a = fixed_width_column_wrapper{1, 1, 1, 0, 0, 0}; + auto b = fixed_width_column_wrapper{1, 1, 2, 0, 0, 0}; + auto s2 = structs_col{{a, b}, nulls_at({4, 5})}; + + auto c = fixed_width_column_wrapper{5, 5, 4, 0, 4, 4}; + return structs_col({s2, c}, null_at(3)); + }(); + + fixed_width_column_wrapper first_col({7, 8, 9, 10, 11, 12}); + fixed_width_column_wrapper second_col({13, 14, 15, 16, 17, 18}); + auto const first_input = cudf::table_view({to_hash, first_col}); + auto const second_input = cudf::table_view({to_hash, second_col}); + + auto const columns_to_hash = std::vector({0}); + + cudf::size_type const num_partitions = 3; + auto const [first_result, first_offsets] = + cudf::hash_partition(first_input, columns_to_hash, num_partitions); + auto const [second_result, second_offsets] = + cudf::hash_partition(second_input, columns_to_hash, num_partitions); + + // Expect offsets to be equal and num_partitions in length + EXPECT_EQ(static_cast(num_partitions), first_offsets.size()); + EXPECT_TRUE(std::equal( + first_offsets.begin(), first_offsets.end(), second_offsets.begin(), second_offsets.end())); + + // Expect same result for the hashed columns + CUDF_TEST_EXPECT_COLUMNS_EQUAL(first_result->get_column(0).view(), + second_result->get_column(0).view()); +} + CUDF_TEST_PROGRAM_MAIN() From 2969b241c0654a11d1a61e29664bcaecd7bc4a15 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Fri, 17 Feb 2023 13:23:15 +0000 Subject: [PATCH 35/69] Produce useful guidance on overflow error in `to_csv` (#12705) Since writing to CSV files is implemented by converting all columns in a dataframe to strings, and then concatenating those columns, when we attempt to write a large dataframe to CSV without specifying a chunk size, we can easily overflow the maximum column size. Currently the error message is rather inscrutable: that the requested size of a string column exceeds the column size limit. To help the user, catch this error and provide a useful error message that points them towards setting the `chunksize` argument. So that we don't produce false positive advice, tighten the scope by only catching `OverflowError`, to do this, make partial progress towards resolving #10200 by throwing `std::overflow_error` when checking for overflow of string column lengths. Closes #12690. Authors: - Lawrence Mitchell (https://github.com/wence-) - Karthikeyan (https://github.com/karthikeyann) Approvers: - David Wendt (https://github.com/davidwendt) - Ashwin Srinath (https://github.com/shwina) - Nghia Truong (https://github.com/ttnghia) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/12705 --- cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh | 11 +++++++---- cpp/include/cudf/lists/filling.hpp | 4 +++- cpp/include/cudf/strings/detail/strings_children.cuh | 7 +++++-- cpp/src/lists/sequences.cu | 4 +++- cpp/src/strings/regex/utilities.cuh | 7 +++++-- cpp/src/text/ngrams_tokenize.cu | 5 ++++- cpp/tests/strings/repeat_strings_tests.cpp | 2 +- python/cudf/cudf/_lib/csv.pyx | 10 ++++++++-- python/cudf/cudf/_lib/json.pyx | 10 ++++++++-- python/cudf/cudf/utils/ioutils.py | 5 ++++- 10 files changed, 48 insertions(+), 17 deletions(-) diff --git a/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh b/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh index 013e74ff18c..eefc5718617 100644 --- a/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh +++ b/cpp/include/cudf/detail/sizes_to_offsets_iterator.cuh @@ -27,6 +27,8 @@ #include #include +#include + namespace cudf { namespace detail { @@ -242,7 +244,7 @@ static sizes_to_offsets_iterator make_sizes_to_offsets_i * auto const bytes = cudf::detail::sizes_to_offsets( * d_offsets, d_offsets + strings_count + 1, d_offsets, stream); * CUDF_EXPECTS(bytes <= static_cast(std::numeric_limits::max()), - * "Size of output exceeds column size limit"); + * "Size of output exceeds column size limit", std::overflow_error); * @endcode * * @tparam SizesIterator Iterator type for input of the scan using addition operation @@ -282,8 +284,8 @@ auto sizes_to_offsets(SizesIterator begin, * The return also includes the total number of elements -- the last element value from the * scan. * - * @throw cudf::logic_error if the total size of the scan (last element) greater than maximum value - * of `size_type` + * @throw std::overflow_error if the total size of the scan (last element) greater than maximum + * value of `size_type` * * @tparam InputIterator Used as input to scan to set the offset values * @param begin The beginning of the input sequence @@ -317,7 +319,8 @@ std::pair, size_type> make_offsets_child_column( auto const total_elements = sizes_to_offsets(input_itr, input_itr + count + 1, d_offsets, stream); CUDF_EXPECTS( total_elements <= static_cast(std::numeric_limits::max()), - "Size of output exceeds column size limit"); + "Size of output exceeds column size limit", + std::overflow_error); offsets_column->set_null_count(0); return std::pair(std::move(offsets_column), static_cast(total_elements)); diff --git a/cpp/include/cudf/lists/filling.hpp b/cpp/include/cudf/lists/filling.hpp index 4a071fc467f..059ed5ffd33 100644 --- a/cpp/include/cudf/lists/filling.hpp +++ b/cpp/include/cudf/lists/filling.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,6 +53,7 @@ namespace cudf::lists { * @throws cudf::logic_error if @p sizes column is not of integer types. * @throws cudf::logic_error if any input column has nulls. * @throws cudf::logic_error if @p starts and @p sizes columns do not have the same size. + * @throws std::overflow_error if the output column would exceed the column size limit. * * @param starts First values in the result sequences. * @param sizes Numbers of values in the result sequences. @@ -90,6 +91,7 @@ std::unique_ptr sequences( * @throws cudf::logic_error if any input column has nulls. * @throws cudf::logic_error if @p starts and @p steps columns have different types. * @throws cudf::logic_error if @p starts, @p steps, and @p sizes columns do not have the same size. + * @throws std::overflow_error if the output column would exceed the column size limit. * * @param starts First values in the result sequences. * @param steps Increment values for the result sequences. diff --git a/cpp/include/cudf/strings/detail/strings_children.cuh b/cpp/include/cudf/strings/detail/strings_children.cuh index 7f57984e88d..09e0f3bb079 100644 --- a/cpp/include/cudf/strings/detail/strings_children.cuh +++ b/cpp/include/cudf/strings/detail/strings_children.cuh @@ -27,6 +27,8 @@ #include #include +#include + namespace cudf { namespace strings { namespace detail { @@ -35,7 +37,7 @@ namespace detail { * @brief Creates child offsets and chars columns by applying the template function that * can be used for computing the output size of each string as well as create the output * - * @throws cudf::logic_error if the output strings column exceeds the column size limit + * @throws std::overflow_error if the output strings column exceeds the column size limit * * @tparam SizeAndExecuteFunction Function must accept an index and return a size. * It must also have members d_offsets and d_chars which are set to @@ -78,7 +80,8 @@ auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, auto const bytes = cudf::detail::sizes_to_offsets(d_offsets, d_offsets + strings_count + 1, d_offsets, stream); CUDF_EXPECTS(bytes <= static_cast(std::numeric_limits::max()), - "Size of output exceeds column size limit"); + "Size of output exceeds column size limit", + std::overflow_error); // Now build the chars column std::unique_ptr chars_column = diff --git a/cpp/src/lists/sequences.cu b/cpp/src/lists/sequences.cu index b3db8e6090b..ecdf81b9158 100644 --- a/cpp/src/lists/sequences.cu +++ b/cpp/src/lists/sequences.cu @@ -34,6 +34,7 @@ #include #include +#include namespace cudf::lists { namespace detail { @@ -169,7 +170,8 @@ std::unique_ptr sequences(column_view const& starts, auto const n_elements = cudf::detail::sizes_to_offsets( sizes_input_it, sizes_input_it + n_lists + 1, offsets_begin, stream); CUDF_EXPECTS(n_elements <= static_cast(std::numeric_limits::max()), - "Size of output exceeds column size limit"); + "Size of output exceeds column size limit", + std::overflow_error); auto child = type_dispatcher(starts.type(), sequences_dispatcher{}, diff --git a/cpp/src/strings/regex/utilities.cuh b/cpp/src/strings/regex/utilities.cuh index bc8e85bf99a..6bbd79166a8 100644 --- a/cpp/src/strings/regex/utilities.cuh +++ b/cpp/src/strings/regex/utilities.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,8 @@ #include +#include + namespace cudf { namespace strings { namespace detail { @@ -134,7 +136,8 @@ auto make_strings_children(SizeAndExecuteFunction size_and_exec_fn, auto const char_bytes = cudf::detail::sizes_to_offsets(d_offsets, d_offsets + strings_count + 1, d_offsets, stream); CUDF_EXPECTS(char_bytes <= static_cast(std::numeric_limits::max()), - "Size of output exceeds column size limit"); + "Size of output exceeds column size limit", + std::overflow_error); // Now build the chars column std::unique_ptr chars = diff --git a/cpp/src/text/ngrams_tokenize.cu b/cpp/src/text/ngrams_tokenize.cu index cf911f13a37..93757fa37e4 100644 --- a/cpp/src/text/ngrams_tokenize.cu +++ b/cpp/src/text/ngrams_tokenize.cu @@ -39,6 +39,8 @@ #include #include +#include + namespace nvtext { namespace detail { namespace { @@ -220,7 +222,8 @@ std::unique_ptr ngrams_tokenize(cudf::strings_column_view const& s chars_offsets.begin(), chars_offsets.end(), chars_offsets.begin(), stream); CUDF_EXPECTS( output_chars_size <= static_cast(std::numeric_limits::max()), - "Size of output exceeds column size limit"); + "Size of output exceeds column size limit", + std::overflow_error); // This will contain the size in bytes of each ngram to generate rmm::device_uvector ngram_sizes(total_ngrams, stream); diff --git a/cpp/tests/strings/repeat_strings_tests.cpp b/cpp/tests/strings/repeat_strings_tests.cpp index e75409d9f39..73009d167e8 100644 --- a/cpp/tests/strings/repeat_strings_tests.cpp +++ b/cpp/tests/strings/repeat_strings_tests.cpp @@ -229,7 +229,7 @@ TEST_F(RepeatStringsTest, StringsColumnWithColumnRepeatTimesOverflowOutput) auto const repeat_times = int32s_col{half_max, half_max, half_max, half_max, half_max, half_max, half_max}; - EXPECT_THROW(cudf::strings::repeat_strings(strs_cv, repeat_times), cudf::logic_error); + EXPECT_THROW(cudf::strings::repeat_strings(strs_cv, repeat_times), std::overflow_error); } TYPED_TEST(RepeatStringsTypedTest, StringsColumnNoNullWithScalarRepeatTimes) diff --git a/python/cudf/cudf/_lib/csv.pyx b/python/cudf/cudf/_lib/csv.pyx index eb6683aed31..09de1f1724e 100644 --- a/python/cudf/cudf/_lib/csv.pyx +++ b/python/cudf/cudf/_lib/csv.pyx @@ -533,8 +533,14 @@ def write_csv( .build() ) - with nogil: - cpp_write_csv(options) + try: + with nogil: + cpp_write_csv(options) + except OverflowError: + raise OverflowError( + f"Writing CSV file with chunksize={rows_per_chunk} failed. " + "Consider providing a smaller chunksize argument." + ) cdef data_type _get_cudf_data_type_from_dtype(object dtype) except +: diff --git a/python/cudf/cudf/_lib/json.pyx b/python/cudf/cudf/_lib/json.pyx index 2339b874ea0..21752062201 100644 --- a/python/cudf/cudf/_lib/json.pyx +++ b/python/cudf/cudf/_lib/json.pyx @@ -201,8 +201,14 @@ def write_json( .build() ) - with nogil: - libcudf_write_json(options) + try: + with nogil: + libcudf_write_json(options) + except OverflowError: + raise OverflowError( + f"Writing JSON file with rows_per_chunk={rows_per_chunk} failed. " + "Consider providing a smaller rows_per_chunk argument." + ) cdef schema_element _get_cudf_schema_element_from_dtype(object dtype) except +: diff --git a/python/cudf/cudf/utils/ioutils.py b/python/cudf/cudf/utils/ioutils.py index 56e2e539e01..924cc62fb15 100644 --- a/python/cudf/cudf/utils/ioutils.py +++ b/python/cudf/cudf/utils/ioutils.py @@ -1245,7 +1245,10 @@ Notes ----- - Follows the standard of Pandas csv.QUOTE_NONNUMERIC for all output. -- If `to_csv` leads to memory errors consider setting the `chunksize` argument. +- The default behaviour is to write all rows of the dataframe at once. + This can lead to memory or overflow errors for large tables. If this + happens, consider setting the ``chunksize`` argument to some + reasonable fraction of the total rows in the dataframe. Examples -------- From b1aa5d2027490de7a5e75d3b2375620a15ace515 Mon Sep 17 00:00:00 2001 From: "H. Thomson Comer" Date: Fri, 17 Feb 2023 11:24:57 -0600 Subject: [PATCH 36/69] Fix a bug with `num_keys` in `_scatter_by_slice` (#12749) This PR closes https://github.com/rapidsai/cudf/issues/12748 by changing the `num_keys` computation in `column._scatter_by_slice`. Authors: - H. Thomson Comer (https://github.com/thomcom) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12749 --- python/cudf/cudf/core/column/column.py | 2 +- python/cudf/cudf/tests/test_setitem.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 965b413e84f..fb1bcf6d673 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -607,7 +607,7 @@ def _scatter_by_slice( start, stop, step = key.indices(len(self)) if start >= stop: return None - num_keys = (stop - start) // step + num_keys = len(range(start, stop, step)) self._check_scatter_key_length(num_keys, value) diff --git a/python/cudf/cudf/tests/test_setitem.py b/python/cudf/cudf/tests/test_setitem.py index 6b2fb90e95b..d59226ee17a 100644 --- a/python/cudf/cudf/tests/test_setitem.py +++ b/python/cudf/cudf/tests/test_setitem.py @@ -347,3 +347,13 @@ def test_series_setitem_upcasting_string_value(): assert_eq(pd.Series([10, 0, 0], dtype=int), sr) with pytest.raises(ValueError): sr[0] = "non-integer" + + +def test_scatter_by_slice_with_start_and_step(): + source = pd.Series([1, 2, 3, 4, 5]) + csource = cudf.from_pandas(source) + target = pd.Series([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]) + ctarget = cudf.from_pandas(target) + target[1::2] = source + ctarget[1::2] = csource + assert_eq(target, ctarget) From ec8704a45c93915067f987e35bfb37bfec2e05ae Mon Sep 17 00:00:00 2001 From: "Robert (Bobby) Evans" Date: Fri, 17 Feb 2023 15:27:57 -0600 Subject: [PATCH 37/69] Fix a leak in a test and clarify some test names (#12781) Fix a small leak of host memory in a java unit test. Also updates some tests that verify that a double free is safe, but don't make it clear from the logs that the double free is expected to be there. This is so we don't have to spend too much time debugging if this double free is expected or not. Authors: - Robert (Bobby) Evans (https://github.com/revans2) Approvers: - Kuhu Shukla (https://github.com/kuhushukla) - Jim Brennan (https://github.com/jbrennan333) - Jason Lowe (https://github.com/jlowe) - Raza Jafri (https://github.com/razajafri) - Gera Shegalov (https://github.com/gerashegalov) URL: https://github.com/rapidsai/cudf/pull/12781 --- java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java | 4 ++-- java/src/test/java/ai/rapids/cudf/ScalarTest.java | 4 ++-- java/src/test/java/ai/rapids/cudf/TableTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index db64dcb08c7..937077c89c9 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -1045,8 +1045,8 @@ void decimal128Cv() { BigInteger bigInteger2 = new BigInteger("14"); BigInteger bigInteger3 = new BigInteger("152345742357340573405745"); final BigInteger[] bigInts = new BigInteger[] {bigInteger1, bigInteger2, bigInteger3}; - try (ColumnVector v = ColumnVector.decimalFromBigInt(-dec32Scale1, bigInts)) { - HostColumnVector hostColumnVector = v.copyToHost(); + try (ColumnVector v = ColumnVector.decimalFromBigInt(-dec32Scale1, bigInts); + HostColumnVector hostColumnVector = v.copyToHost()) { assertEquals(bigInteger1, hostColumnVector.getBigDecimal(0).unscaledValue()); assertEquals(bigInteger2, hostColumnVector.getBigDecimal(1).unscaledValue()); assertEquals(bigInteger3, hostColumnVector.getBigDecimal(2).unscaledValue()); diff --git a/java/src/test/java/ai/rapids/cudf/ScalarTest.java b/java/src/test/java/ai/rapids/cudf/ScalarTest.java index 86c340bb321..f4b652a7d03 100644 --- a/java/src/test/java/ai/rapids/cudf/ScalarTest.java +++ b/java/src/test/java/ai/rapids/cudf/ScalarTest.java @@ -1,6 +1,6 @@ /* * - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,7 +41,7 @@ public void testDoubleClose() { } @Test - public void testIncRef() { + public void testIncRefAndDoubleFree() { Scalar s = Scalar.fromNull(DType.INT32); try (Scalar ignored1 = s) { try (Scalar ignored2 = s.incRefCount()) { diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index 4f00bc7493d..c31bcf4f78d 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -244,7 +244,7 @@ void testOrderByWithNullsAndStrings() { } @Test - void testTableCreationIncreasesRefCount() { + void testTableCreationIncreasesRefCountWithDoubleFree() { //tests the Table increases the refcount on column vectors assertThrows(IllegalStateException.class, () -> { try (ColumnVector v1 = ColumnVector.build(DType.INT32, 5, Range.appendInts(5)); From 94bbc82a117a96979d1c5d7949ba213ade88c3be Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Sun, 19 Feb 2023 11:51:52 -0500 Subject: [PATCH 38/69] Add build metrics report as artifact to cpp-build workflow (#12750) Adds the Build Metrics Report into the gitHub actions workflow by uploading a link to the generated report to the CI downloads S3 and publishes a link to the report in the build output. A follow-on PR will provide a more direct mechanism for locating the report link. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12750 --- build.sh | 8 +++++--- ci/build_cpp.sh | 28 +++++++++++++++++++++++++++- ci/test_cpp.sh | 16 ---------------- conda/recipes/libcudf/meta.yaml | 1 + 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/build.sh b/build.sh index d9e088c765e..22a62df7182 100755 --- a/build.sh +++ b/build.sh @@ -315,9 +315,11 @@ if buildAll || hasArg libcudf; then LIBCUDF_FS=$(ls -lh ${LIB_BUILD_DIR}/libcudf.so | awk '{print $5}') MSG="${MSG}
libcudf.so size: $LIBCUDF_FS" fi - echo "$MSG" - python ${REPODIR}/cpp/scripts/sort_ninja_log.py ${LIB_BUILD_DIR}/.ninja_log --fmt html --msg "$MSG" > ${LIB_BUILD_DIR}/ninja_log.html - cp ${LIB_BUILD_DIR}/.ninja_log ${LIB_BUILD_DIR}/ninja.log + BMR_DIR=${RAPIDS_ARTIFACTS_DIR:-"${LIB_BUILD_DIR}"} + echo "Metrics output dir: [$BMR_DIR]" + mkdir -p ${BMR_DIR} + python ${REPODIR}/cpp/scripts/sort_ninja_log.py ${LIB_BUILD_DIR}/.ninja_log --fmt html --msg "$MSG" > ${BMR_DIR}/ninja_log.html + cp ${LIB_BUILD_DIR}/.ninja_log ${BMR_DIR}/ninja.log fi if [[ ${INSTALL_TARGET} != "" ]]; then diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 3b45b3ce2e7..b68c2bdbef6 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. set -euo pipefail @@ -14,3 +14,29 @@ rapids-logger "Begin cpp build" rapids-mamba-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp + +echo "++++++++++++++++++++++++++++++++++++++++++++" + +if [[ -d $RAPIDS_ARTIFACTS_DIR ]]; then + ls -l ${RAPIDS_ARTIFACTS_DIR} +fi + +echo "++++++++++++++++++++++++++++++++++++++++++++" + +FILE=${RAPIDS_ARTIFACTS_DIR}/ninja.log +if [[ -f $FILE ]]; then + echo -e "\x1B[33;1m\x1B[48;5;240m Ninja log for this build available at the following link \x1B[0m" + UPLOAD_NAME=cpp_cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch).ninja.log + rapids-upload-to-s3 "${UPLOAD_NAME}" "${FILE}" +fi + +echo "++++++++++++++++++++++++++++++++++++++++++++" + +FILE=${RAPIDS_ARTIFACTS_DIR}/ninja_log.html +if [[ -f $FILE ]]; then + echo -e "\x1B[33;1m\x1B[48;5;240m Build Metrics Report for this build available at the following link \x1B[0m" + UPLOAD_NAME=cpp_cuda${RAPIDS_CUDA_VERSION%%.*}_$(arch).BuildMetricsReport.html + rapids-upload-to-s3 "${UPLOAD_NAME}" "${FILE}" +fi + +echo "++++++++++++++++++++++++++++++++++++++++++++" diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 0be72486319..983a63d4ce9 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -66,21 +66,5 @@ for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do fi done -if [[ "${RAPIDS_BUILD_TYPE}" == "nightly" ]]; then - rapids-logger "Memcheck gtests with rmm_mode=cuda" - export GTEST_CUDF_RMM_MODE=cuda - COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" - for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do - test_name=$(basename ${gt}) - if [[ "$test_name" == "ERROR_TEST" ]]; then - continue - fi - echo "Running gtest $test_name" - ${COMPUTE_SANITIZER_CMD} ${gt} | tee "${RAPIDS_TESTS_DIR}${test_name}.cs.log" - done - unset GTEST_CUDF_RMM_MODE - # TODO: test-results/*.cs.log are processed in CI -fi - rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml index b0b86b427b7..fbfcf6e71a2 100644 --- a/conda/recipes/libcudf/meta.yaml +++ b/conda/recipes/libcudf/meta.yaml @@ -27,6 +27,7 @@ build: - SCCACHE_IDLE_TIMEOUT - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY + - RAPIDS_ARTIFACTS_DIR requirements: build: From c2f016122cf9fc769516dee785a9a68f44905693 Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Mon, 20 Feb 2023 23:02:52 -0500 Subject: [PATCH 39/69] Changing `cudf::io::source_info` to use `cudf::host_span` in a non-breaking form (#12730) Closes #12576 This change converts `cudf::io::source_info` to take a `host_span`. This version deprecates the original API, but leaves it intact to avoid breaking changes. After being deprecated for a few releases, they will be removed. Authors: - Mike Wilson (https://github.com/hyperbolic2346) - Vukasin Milovanovic (https://github.com/vuule) Approvers: - Nghia Truong (https://github.com/ttnghia) - Vukasin Milovanovic (https://github.com/vuule) URL: https://github.com/rapidsai/cudf/pull/12730 --- cpp/include/cudf/io/datasource.hpp | 10 ++++ cpp/include/cudf/io/types.hpp | 75 +++++++++++++++++++++++++++-- cpp/src/io/utilities/datasource.cpp | 8 ++- cpp/tests/io/parquet_test.cpp | 74 ++++++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 5 deletions(-) diff --git a/cpp/include/cudf/io/datasource.hpp b/cpp/include/cudf/io/datasource.hpp index fd4c049e2fc..a0ef2155f7d 100644 --- a/cpp/include/cudf/io/datasource.hpp +++ b/cpp/include/cudf/io/datasource.hpp @@ -112,11 +112,21 @@ class datasource { /** * @brief Creates a source from a host memory buffer. * + # @deprecated Since 23.04 + * * @param[in] buffer Host buffer object * @return Constructed datasource object */ static std::unique_ptr create(host_buffer const& buffer); + /** + * @brief Creates a source from a host memory buffer. + * + * @param[in] buffer Host buffer object + * @return Constructed datasource object + */ + static std::unique_ptr create(cudf::host_span buffer); + /** * @brief Creates a source from a device memory buffer. * diff --git a/cpp/include/cudf/io/types.hpp b/cpp/include/cudf/io/types.hpp index 06b52563e19..6f97eb768d9 100644 --- a/cpp/include/cudf/io/types.hpp +++ b/cpp/include/cudf/io/types.hpp @@ -150,6 +150,8 @@ struct table_with_metadata { /** * @brief Non-owning view of a host memory buffer * + * @deprecated Since 23.04 + * * Used to describe buffer input in `source_info` objects. */ struct host_buffer { @@ -166,6 +168,22 @@ struct host_buffer { host_buffer(const char* data, size_t size) : data(data), size(size) {} }; +/** + * @brief Returns `true` if the type is byte-like, meaning it is reasonable to pass as a pointer to + * bytes. + * + * @tparam T The representation type + * @return `true` if the type is considered a byte-like type + */ +template +constexpr inline auto is_byte_like_type() +{ + using non_cv_T = std::remove_cv_t; + return std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v || + std::is_same_v; +} + /** * @brief Source information for read interfaces */ @@ -191,21 +209,70 @@ struct source_info { /** * @brief Construct a new source info object for multiple buffers in host memory * + * @deprecated Since 23.04 + * * @param host_buffers Input buffers in host memory */ - explicit source_info(std::vector const& host_buffers) - : _type(io_type::HOST_BUFFER), _host_buffers(host_buffers) + explicit source_info(std::vector const& host_buffers) : _type(io_type::HOST_BUFFER) { + _host_buffers.reserve(host_buffers.size()); + std::transform(host_buffers.begin(), + host_buffers.end(), + std::back_inserter(_host_buffers), + [](auto const hb) { + return cudf::host_span{ + reinterpret_cast(hb.data), hb.size}; + }); } /** * @brief Construct a new source info object for a single buffer * + * @deprecated Since 23.04 + * * @param host_data Input buffer in host memory * @param size Size of the buffer */ explicit source_info(const char* host_data, size_t size) - : _type(io_type::HOST_BUFFER), _host_buffers({{host_data, size}}) + : _type(io_type::HOST_BUFFER), + _host_buffers( + {cudf::host_span(reinterpret_cast(host_data), size)}) + { + } + + /** + * @brief Construct a new source info object for multiple buffers in host memory + * + * @param host_buffers Input buffers in host memory + */ + template >())> + explicit source_info(cudf::host_span> const host_buffers) + : _type(io_type::HOST_BUFFER) + { + if constexpr (not std::is_same_v, std::byte>) { + _host_buffers.reserve(host_buffers.size()); + std::transform(host_buffers.begin(), + host_buffers.end(), + std::back_inserter(_host_buffers), + [](auto const s) { + return cudf::host_span{ + reinterpret_cast(s.data()), s.size()}; + }); + } else { + _host_buffers.assign(host_buffers.begin(), host_buffers.end()); + } + } + + /** + * @brief Construct a new source info object for a single buffer + * + * @param host_data Input buffer in host memory + */ + template >())> + explicit source_info(cudf::host_span host_data) + : _type(io_type::HOST_BUFFER), + _host_buffers{cudf::host_span( + reinterpret_cast(host_data.data()), host_data.size())} { } @@ -289,7 +356,7 @@ struct source_info { private: io_type _type = io_type::FILEPATH; std::vector _filepaths; - std::vector _host_buffers; + std::vector> _host_buffers; std::vector> _device_buffers; std::vector _user_sources; }; diff --git a/cpp/src/io/utilities/datasource.cpp b/cpp/src/io/utilities/datasource.cpp index c2f7b18d443..71d64900398 100644 --- a/cpp/src/io/utilities/datasource.cpp +++ b/cpp/src/io/utilities/datasource.cpp @@ -329,10 +329,16 @@ std::unique_ptr datasource::create(const std::string& filepath, } std::unique_ptr datasource::create(host_buffer const& buffer) +{ + return create( + cudf::host_span{reinterpret_cast(buffer.data), buffer.size}); +} + +std::unique_ptr datasource::create(cudf::host_span buffer) { // Use Arrow IO buffer class for zero-copy reads of host memory return std::make_unique(std::make_shared( - reinterpret_cast(buffer.data), buffer.size)); + reinterpret_cast(buffer.data()), buffer.size())); } std::unique_ptr datasource::create(cudf::device_span buffer) diff --git a/cpp/tests/io/parquet_test.cpp b/cpp/tests/io/parquet_test.cpp index 21752196430..48f69e3ecd3 100644 --- a/cpp/tests/io/parquet_test.cpp +++ b/cpp/tests/io/parquet_test.cpp @@ -357,6 +357,10 @@ struct ParquetWriterSchemaTest : public ParquetWriterTest { auto type() { return cudf::data_type{cudf::type_to_id()}; } }; +template +struct ParquetReaderSourceTest : public ParquetReaderTest { +}; + // Declare typed test cases // TODO: Replace with `NumericTypes` when unsigned support is added. Issue #5352 using SupportedTypes = cudf::test::Types; @@ -369,6 +373,8 @@ using SupportedTimestampTypes = cudf::test::Types; TYPED_TEST_SUITE(ParquetWriterTimestampTypeTest, SupportedTimestampTypes); TYPED_TEST_SUITE(ParquetWriterSchemaTest, cudf::test::AllTypes); +using ByteLikeTypes = cudf::test::Types; +TYPED_TEST_SUITE(ParquetReaderSourceTest, ByteLikeTypes); // Base test fixture for chunked writer tests struct ParquetChunkedWriterTest : public cudf::test::BaseFixture { @@ -5113,4 +5119,72 @@ TEST_P(ParquetSizedTest, DictionaryTest) EXPECT_EQ(nbits, GetParam()); } +TYPED_TEST(ParquetReaderSourceTest, BufferSourceTypes) +{ + using T = TypeParam; + + srand(31337); + auto table = create_random_fixed_table(5, 5, true); + + std::vector out_buffer; + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info(&out_buffer), *table); + cudf::io::write_parquet(out_opts); + + { + cudf::io::parquet_reader_options in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info( + cudf::host_span(reinterpret_cast(out_buffer.data()), out_buffer.size()))); + const auto result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(*table, result.tbl->view()); + } + + { + cudf::io::parquet_reader_options in_opts = + cudf::io::parquet_reader_options::builder(cudf::io::source_info(cudf::host_span( + reinterpret_cast(out_buffer.data()), out_buffer.size()))); + const auto result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(*table, result.tbl->view()); + } +} + +TYPED_TEST(ParquetReaderSourceTest, BufferSourceArrayTypes) +{ + using T = TypeParam; + + srand(31337); + auto table = create_random_fixed_table(5, 5, true); + + std::vector out_buffer; + cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info(&out_buffer), *table); + cudf::io::write_parquet(out_opts); + + auto full_table = cudf::concatenate(std::vector({*table, *table})); + + { + auto spans = std::vector>{ + cudf::host_span(reinterpret_cast(out_buffer.data()), out_buffer.size()), + cudf::host_span(reinterpret_cast(out_buffer.data()), out_buffer.size())}; + cudf::io::parquet_reader_options in_opts = cudf::io::parquet_reader_options::builder( + cudf::io::source_info(cudf::host_span>(spans.data(), spans.size()))); + const auto result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(*full_table, result.tbl->view()); + } + + { + auto spans = std::vector>{ + cudf::host_span(reinterpret_cast(out_buffer.data()), out_buffer.size()), + cudf::host_span(reinterpret_cast(out_buffer.data()), out_buffer.size())}; + cudf::io::parquet_reader_options in_opts = cudf::io::parquet_reader_options::builder( + cudf::io::source_info(cudf::host_span>(spans.data(), spans.size()))); + const auto result = cudf::io::read_parquet(in_opts); + + CUDF_TEST_EXPECT_TABLES_EQUAL(*full_table, result.tbl->view()); + } +} + CUDF_TEST_PROGRAM_MAIN() From 7da233b279bf84a501e9c2e3041cbc6fb335e610 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Tue, 21 Feb 2023 08:12:54 -0500 Subject: [PATCH 40/69] Rework logic in cudf::strings::split_record to improve performance (#12729) Updates the `cudf::strings::split_record` logic to match the more optimized code in `cudf::strings:split`. The optimized code performs much better for longer strings (>64 bytes) by parallelizing over the character bytes to find delimiters before determining split tokens. This led to refactoring the code so it both APIs can share the optimized code. Also fixes a bug found when using overlapped delimiters. Additional tests were added for multi-byte delimiters which can overlap and span multiple adjacent strings. Closes #12694 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Nghia Truong (https://github.com/ttnghia) - Yunsong Wang (https://github.com/PointKernel) - https://github.com/nvdbaranec URL: https://github.com/rapidsai/cudf/pull/12729 --- cpp/benchmarks/string/split.cpp | 14 +- cpp/src/strings/split/split.cu | 448 ++------------------------ cpp/src/strings/split/split.cuh | 403 +++++++++++++++++++++++ cpp/src/strings/split/split_record.cu | 168 +++------- cpp/tests/strings/split_tests.cpp | 78 ++++- 5 files changed, 565 insertions(+), 546 deletions(-) create mode 100644 cpp/src/strings/split/split.cuh diff --git a/cpp/benchmarks/string/split.cpp b/cpp/benchmarks/string/split.cpp index 0f005c462cc..1b3f4190680 100644 --- a/cpp/benchmarks/string/split.cpp +++ b/cpp/benchmarks/string/split.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,12 +57,12 @@ static void BM_split(benchmark::State& state, split_type rt) static void generate_bench_args(benchmark::internal::Benchmark* b) { - int const min_rows = 1 << 12; - int const max_rows = 1 << 24; - int const row_mult = 8; - int const min_rowlen = 1 << 5; - int const max_rowlen = 1 << 13; - int const len_mult = 4; + int constexpr min_rows = 1 << 12; + int constexpr max_rows = 1 << 24; + int constexpr row_mult = 8; + int constexpr min_rowlen = 1 << 5; + int constexpr max_rowlen = 1 << 13; + int constexpr len_mult = 2; for (int row_count = min_rows; row_count <= max_rows; row_count *= row_mult) { for (int rowlen = min_rowlen; rowlen <= max_rowlen; rowlen *= len_mult) { // avoid generating combinations that exceed the cudf column limit diff --git a/cpp/src/strings/split/split.cu b/cpp/src/strings/split/split.cu index c11d7ad47f9..18599fb568a 100644 --- a/cpp/src/strings/split/split.cu +++ b/cpp/src/strings/split/split.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "split.cuh" + #include #include #include @@ -31,14 +33,10 @@ #include #include -#include -#include -#include #include #include #include #include -#include #include #include @@ -46,321 +44,8 @@ namespace cudf { namespace strings { namespace detail { -using string_index_pair = thrust::pair; - namespace { -/** - * @brief Base class for delimiter-based tokenizers. - * - * These are common methods used by both split and rsplit tokenizer functors. - */ -struct base_split_tokenizer { - __device__ const char* get_base_ptr() const - { - return d_strings.child(strings_column_view::chars_column_index).data(); - } - - __device__ string_view const get_string(size_type idx) const - { - return d_strings.element(idx); - } - - __device__ bool is_valid(size_type idx) const { return d_strings.is_valid(idx); } - - /** - * @brief Initialize token elements for all strings. - * - * The process_tokens() only handles creating tokens for strings that contain - * delimiters. This function will initialize the output tokens for all - * strings by assigning null entries for null and empty strings and the - * string itself for strings with no delimiters. - * - * The tokens are placed in output order so that all tokens for each output - * column are stored consecutively in `d_all_tokens`. - * - * @param idx Index of string in column - * @param column_count Number of columns in output - * @param d_all_tokens Tokens vector for all strings - */ - __device__ void init_tokens(size_type idx, - size_type column_count, - string_index_pair* d_all_tokens) const - { - auto d_tokens = d_all_tokens + idx; - if (is_valid(idx)) { - auto d_str = get_string(idx); - *d_tokens = string_index_pair{d_str.data(), d_str.size_bytes()}; - --column_count; - d_tokens += d_strings.size(); - } - // this is like fill() but output needs to be strided - for (size_type col = 0; col < column_count; ++col) - d_tokens[d_strings.size() * col] = string_index_pair{nullptr, 0}; - } - - base_split_tokenizer(column_device_view const& d_strings, - string_view const& d_delimiter, - size_type max_tokens) - : d_strings(d_strings), d_delimiter(d_delimiter), max_tokens(max_tokens) - { - } - - protected: - column_device_view const d_strings; // strings to split - string_view const d_delimiter; // delimiter for split - size_type max_tokens; -}; - -/** - * @brief The tokenizer functions for split(). - * - * The methods here count delimiters, tokens, and output token elements - * for each string in a strings column. - */ -struct split_tokenizer_fn : base_split_tokenizer { - /** - * @brief This will create tokens around each delimiter honoring the string boundaries - * in which the delimiter resides. - * - * Each token is placed in `d_all_tokens` so they align consecutively - * with other tokens for the same output column. - * That is, `d_tokens[col * strings_count + string_index]` is the token at column `col` - * for string at `string_index`. - * - * @param idx Index of the delimiter in the chars column - * @param d_token_counts Token counts for each string - * @param d_positions The beginning byte position of each delimiter - * @param positions_count Number of delimiters - * @param d_indexes Indices of the strings for each delimiter - * @param d_all_tokens All output tokens for the strings column - */ - __device__ void process_tokens(size_type idx, - size_type const* d_token_counts, - size_type const* d_positions, - size_type positions_count, - size_type const* d_indexes, - string_index_pair* d_all_tokens) const - { - size_type str_idx = d_indexes[idx]; - if ((idx > 0) && d_indexes[idx - 1] == str_idx) - return; // the first delimiter for the string rules them all - --str_idx; // all of these are off by 1 from the upper_bound call - size_type token_count = d_token_counts[str_idx]; // max_tokens already included - const char* const base_ptr = get_base_ptr(); // d_positions values are based on this ptr - // this string's tokens output - auto d_tokens = d_all_tokens + str_idx; - // this string - const string_view d_str = get_string(str_idx); - const char* str_ptr = d_str.data(); // beginning of the string - const char* const str_end_ptr = str_ptr + d_str.size_bytes(); // end of the string - // build the index-pair of each token for this string - for (size_type col = 0; col < token_count; ++col) { - auto next_delim = ((idx + col) < positions_count) // boundary check for delims in last string - ? (base_ptr + d_positions[idx + col]) // start of next delimiter - : str_end_ptr; // or end of this string - auto eptr = (next_delim < str_end_ptr) // make sure delimiter is inside this string - && (col + 1 < token_count) // and this is not the last token - ? next_delim - : str_end_ptr; - // store the token into the output vector - d_tokens[col * d_strings.size()] = - string_index_pair{str_ptr, static_cast(eptr - str_ptr)}; - // point past this delimiter - str_ptr = eptr + d_delimiter.size_bytes(); - } - } - - /** - * @brief Returns `true` if the byte at `idx` is the start of the delimiter. - * - * @param idx Index of a byte in the chars column. - * @param d_offsets Offsets values to locate the chars ranges. - * @param chars_bytes Total number of characters to process. - * @return true if delimiter is found starting at position `idx` - */ - __device__ bool is_delimiter(size_type idx, // chars index - int32_t const* d_offsets, - size_type chars_bytes) const - { - auto d_chars = get_base_ptr() + d_offsets[0]; - if (idx + d_delimiter.size_bytes() > chars_bytes) return false; - return d_delimiter.compare(d_chars + idx, d_delimiter.size_bytes()) == 0; - } - - /** - * @brief This counts the tokens for strings that contain delimiters. - * - * @param idx Index of a delimiter - * @param d_positions Start positions of all the delimiters - * @param positions_count The number of delimiters - * @param d_indexes Indices of the strings for each delimiter - * @param d_counts The token counts for all the strings - */ - __device__ void count_tokens(size_type idx, // delimiter index - size_type const* d_positions, - size_type positions_count, - size_type const* d_indexes, - size_type* d_counts) const - { - size_type str_idx = d_indexes[idx]; - if ((idx > 0) && d_indexes[idx - 1] == str_idx) - return; // first delimiter found handles all of them for this string - auto const delim_length = d_delimiter.size_bytes(); - string_view const d_str = get_string(str_idx - 1); - const char* const base_ptr = get_base_ptr(); - size_type delim_count = 0; // re-count delimiters to compute the token-count - size_type last_pos = d_positions[idx] - delim_length; - while ((idx < positions_count) && (d_indexes[idx] == str_idx)) { - // make sure the whole delimiter is inside the string before counting it - auto d_pos = d_positions[idx]; - if (((base_ptr + d_pos + delim_length - 1) < (d_str.data() + d_str.size_bytes())) && - ((d_pos - last_pos) >= delim_length)) { - ++delim_count; // only count if the delimiter fits - last_pos = d_pos; // overlapping delimiters are ignored too - } - ++idx; - } - // the number of tokens is delim_count+1 but capped to max_tokens - d_counts[str_idx - 1] = - ((max_tokens > 0) && (delim_count + 1 > max_tokens)) ? max_tokens : delim_count + 1; - } - - split_tokenizer_fn(column_device_view const& d_strings, - string_view const& d_delimiter, - size_type max_tokens) - : base_split_tokenizer(d_strings, d_delimiter, max_tokens) - { - } -}; - -/** - * @brief The tokenizer functions for split(). - * - * The methods here count delimiters, tokens, and output token elements - * for each string in a strings column. - * - * Same as split_tokenizer_fn except tokens are counted from the end of each string. - */ -struct rsplit_tokenizer_fn : base_split_tokenizer { - /** - * @brief This will create tokens around each delimiter honoring the string boundaries - * in which the delimiter resides. - * - * The tokens are processed from the end of each string so the `max_tokens` - * is honored correctly. - * - * Each token is placed in `d_all_tokens` so they align consecutively - * with other tokens for the same output column. - * That is, `d_tokens[col * strings_count + string_index]` is the token at column `col` - * for string at `string_index`. - * - * @param idx Index of the delimiter in the chars column - * @param d_token_counts Token counts for each string - * @param d_positions The ending byte position of each delimiter - * @param positions_count Number of delimiters - * @param d_indexes Indices of the strings for each delimiter - * @param d_all_tokens All output tokens for the strings column - */ - __device__ void process_tokens(size_type idx, // delimiter position index - size_type const* d_token_counts, // token counts for each string - size_type const* d_positions, // end of each delimiter - size_type positions_count, // total number of delimiters - size_type const* d_indexes, // string indices for each delimiter - string_index_pair* d_all_tokens) const - { - size_type str_idx = d_indexes[idx]; - if ((idx + 1 < positions_count) && d_indexes[idx + 1] == str_idx) - return; // the last delimiter for the string rules them all - --str_idx; // all of these are off by 1 from the upper_bound call - size_type token_count = d_token_counts[str_idx]; // max_tokens already included - const char* const base_ptr = get_base_ptr(); // d_positions values are based on this ptr - // this string's tokens output - auto d_tokens = d_all_tokens + str_idx; - // this string - const string_view d_str = get_string(str_idx); - const char* const str_begin_ptr = d_str.data(); // beginning of the string - const char* str_ptr = str_begin_ptr + d_str.size_bytes(); // end of the string - // build the index-pair of each token for this string - for (size_type col = 0; col < token_count; ++col) { - auto prev_delim = (idx >= col) // boundary check for delims in first string - ? (base_ptr + d_positions[idx - col] + 1) // end of prev delimiter - : str_begin_ptr; // or the start of this string - auto sptr = (prev_delim > str_begin_ptr) // make sure delimiter is inside the string - && (col + 1 < token_count) // and this is not the last token - ? prev_delim - : str_begin_ptr; - // store the token into the output -- building the array backwards - d_tokens[d_strings.size() * (token_count - 1 - col)] = - string_index_pair{sptr, static_cast(str_ptr - sptr)}; - str_ptr = sptr - d_delimiter.size_bytes(); // get ready for the next prev token - } - } - - /** - * @brief Returns `true` if the byte at `idx` is the end of the delimiter. - * - * @param idx Index of a byte in the chars column. - * @param d_offsets Offsets values to locate the chars ranges. - * @return true if delimiter is found ending at position `idx` - */ - __device__ bool is_delimiter(size_type idx, int32_t const* d_offsets, size_type) const - { - auto delim_length = d_delimiter.size_bytes(); - if (idx < delim_length - 1) return false; - auto d_chars = get_base_ptr() + d_offsets[0]; - return d_delimiter.compare(d_chars + idx - (delim_length - 1), delim_length) == 0; - } - - /** - * @brief This counts the tokens for strings that contain delimiters. - * - * Token counting starts at the end of the string to honor the `max_tokens` - * appropriately. - * - * @param idx Index of a delimiter - * @param d_positions End positions of all the delimiters - * @param positions_count The number of delimiters - * @param d_indexes Indices of the strings for each delimiter - * @param d_counts The token counts for all the strings - */ - __device__ void count_tokens(size_type idx, - size_type const* d_positions, - size_type positions_count, - size_type const* d_indexes, - size_type* d_counts) const - { - size_type str_idx = d_indexes[idx]; // 1-based string index created by upper_bound() - if ((idx > 0) && d_indexes[idx - 1] == str_idx) - return; // first delimiter found handles all of them for this string - auto const delim_length = d_delimiter.size_bytes(); - const string_view d_str = get_string(str_idx - 1); // -1 for 0-based index - const char* const base_ptr = get_base_ptr(); - size_type delim_count = 0; - size_type last_pos = d_positions[idx] - delim_length; - while ((idx < positions_count) && (d_indexes[idx] == str_idx)) { - // make sure the whole delimiter is inside the string before counting it - auto d_pos = d_positions[idx]; - if (((base_ptr + d_pos + 1 - delim_length) >= d_str.data()) && - ((d_pos - last_pos) >= delim_length)) { - ++delim_count; // only count if the delimiter fits - last_pos = d_pos; // overlapping delimiters are also ignored - } - ++idx; - } - // the number of tokens is delim_count+1 but capped to max_tokens - d_counts[str_idx - 1] = - ((max_tokens > 0) && (delim_count + 1 > max_tokens)) ? max_tokens : delim_count + 1; - } - - rsplit_tokenizer_fn(column_device_view const& d_strings, - string_view const& d_delimiter, - size_type max_tokens) - : base_split_tokenizer(d_strings, d_delimiter, max_tokens) - { - } -}; - /** * @brief Generic split function called by split() and rsplit(). * @@ -423,125 +108,42 @@ struct rsplit_tokenizer_fn : base_split_tokenizer { * @return table of columns for the output of the split */ template -std::unique_ptr
split_fn(strings_column_view const& strings_column, +std::unique_ptr
split_fn(strings_column_view const& input, Tokenizer tokenizer, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { std::vector> results; - auto const strings_count = strings_column.size(); - if (strings_count == 0) { - results.push_back(make_empty_column(type_id::STRING)); + if (input.size() == input.null_count()) { + results.push_back(std::make_unique(input.parent(), stream, mr)); return std::make_unique
(std::move(results)); } - auto d_offsets = strings_column.offsets_begin(); - auto const chars_bytes = - cudf::detail::get_value( - strings_column.offsets(), strings_column.offset() + strings_count, stream) - - cudf::detail::get_value(strings_column.offsets(), strings_column.offset(), stream); + // builds the offsets and the vector of all tokens + auto [offsets, tokens] = split_helper(input, tokenizer, stream, mr); + auto const d_offsets = offsets->view().template data(); + auto const d_tokens = tokens.data(); - // count the number of delimiters in the entire column - auto const delimiter_count = - thrust::count_if(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(chars_bytes), - [tokenizer, d_offsets, chars_bytes] __device__(size_type idx) { - return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); - }); - - // create vector of every delimiter position in the chars column - rmm::device_uvector delimiter_positions(delimiter_count, stream); - auto d_positions = delimiter_positions.data(); - auto copy_end = thrust::copy_if(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(chars_bytes), - delimiter_positions.begin(), - [tokenizer, d_offsets, chars_bytes] __device__(size_type idx) { - return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); - }); - - // create vector of string indices for each delimiter - rmm::device_uvector string_indices(delimiter_count, stream); // these will - auto d_string_indices = string_indices.data(); // be strings that only contain delimiters - thrust::upper_bound(rmm::exec_policy(stream), - d_offsets, - d_offsets + strings_count, - delimiter_positions.begin(), - copy_end, - string_indices.begin()); - - // compute the number of tokens per string - rmm::device_uvector token_counts(strings_count, stream); - auto d_token_counts = token_counts.data(); - // first, initialize token counts for strings without delimiters in them - thrust::transform(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - thrust::make_counting_iterator(strings_count), - d_token_counts, - [tokenizer] __device__(size_type idx) { - // null are 0, all others 1 - return static_cast(tokenizer.is_valid(idx)); - }); - - // now compute the number of tokens in each string - thrust::for_each_n( + // compute the maximum number of tokens for any string + auto const columns_count = thrust::transform_reduce( rmm::exec_policy(stream), thrust::make_counting_iterator(0), - delimiter_count, - [tokenizer, d_positions, delimiter_count, d_string_indices, d_token_counts] __device__( - size_type idx) { - tokenizer.count_tokens(idx, d_positions, delimiter_count, d_string_indices, d_token_counts); - }); - - // the columns_count is the maximum number of tokens for any string - auto const columns_count = thrust::reduce( - rmm::exec_policy(stream), token_counts.begin(), token_counts.end(), 0, thrust::maximum{}); - // boundary case: if no columns, return one null column (custrings issue #119) - if (columns_count == 0) { - results.push_back(std::make_unique( - data_type{type_id::STRING}, - strings_count, - rmm::device_buffer{0, stream, mr}, // no data - cudf::detail::create_null_mask(strings_count, mask_state::ALL_NULL, stream, mr), - strings_count)); - } + thrust::make_counting_iterator(input.size()), + [d_offsets] __device__(auto idx) -> size_type { return d_offsets[idx + 1] - d_offsets[idx]; }, + 0, + thrust::maximum{}); - // create working area to hold all token positions - rmm::device_uvector tokens(columns_count * strings_count, stream); - string_index_pair* d_tokens = tokens.data(); - // initialize the token positions - // -- accounts for nulls, empty, and strings with no delimiter in them - thrust::for_each_n(rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - strings_count, - [tokenizer, columns_count, d_tokens] __device__(size_type idx) { - tokenizer.init_tokens(idx, columns_count, d_tokens); - }); - - // get the positions for every token using the delimiter positions - thrust::for_each_n( - rmm::exec_policy(stream), - thrust::make_counting_iterator(0), - delimiter_count, - [tokenizer, - d_token_counts, - d_positions, - delimiter_count, - d_string_indices, - d_tokens] __device__(size_type idx) { - tokenizer.process_tokens( - idx, d_token_counts, d_positions, delimiter_count, d_string_indices, d_tokens); - }); - - // Create each column. - // - Each pair points to the strings for that column for each row. - // - Create the strings column from the vector using the strings factory. + // build strings columns for each token position for (size_type col = 0; col < columns_count; ++col) { - auto column_tokens = d_tokens + (col * strings_count); - results.emplace_back( - make_strings_column(column_tokens, column_tokens + strings_count, stream, mr)); + auto itr = cudf::detail::make_counting_transform_iterator( + 0, [d_tokens, d_offsets, col] __device__(size_type idx) { + auto const offset = d_offsets[idx]; + auto const token_count = d_offsets[idx + 1] - offset; + return (col < token_count) ? d_tokens[offset + col] : string_index_pair{nullptr, 0}; + }); + results.emplace_back(make_strings_column(itr, itr + input.size(), stream, mr)); } + return std::make_unique
(std::move(results)); } diff --git a/cpp/src/strings/split/split.cuh b/cpp/src/strings/split/split.cuh new file mode 100644 index 00000000000..41213dac58b --- /dev/null +++ b/cpp/src/strings/split/split.cuh @@ -0,0 +1,403 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace cudf::strings::detail { + +/** + * @brief Base class for delimiter-based tokenizers + * + * These are common methods used by both split and rsplit tokenizer functors. + * + * The Derived class is required to implement the `process_tokens` function. + */ +template +struct base_split_tokenizer { + __device__ char const* get_base_ptr() const + { + return d_strings.child(strings_column_view::chars_column_index).data(); + } + + __device__ string_view const get_string(size_type idx) const + { + return d_strings.element(idx); + } + + __device__ bool is_valid(size_type idx) const { return d_strings.is_valid(idx); } + + /** + * @brief Returns `true` if the byte at `idx` is the start of the delimiter + * + * @param idx Index of a byte in the chars column + * @param d_offsets Offsets values to locate the chars ranges + * @param chars_bytes Total number of characters to process + * @return true if delimiter is found starting at position `idx` + */ + __device__ bool is_delimiter(size_type idx, + size_type const* d_offsets, + size_type chars_bytes) const + { + auto const d_chars = get_base_ptr() + d_offsets[0]; + if (idx + d_delimiter.size_bytes() > chars_bytes) { return false; } + return d_delimiter.compare(d_chars + idx, d_delimiter.size_bytes()) == 0; + } + + /** + * @brief This counts the tokens for strings that contain delimiters + * + * Counting tokens is the same regardless if counting from the left + * or from the right. This logic counts from the left which is simpler. + * The count will be truncated appropriately to the max_tokens value. + * + * @param idx Index of input string + * @param d_positions Start positions of all the delimiters + * @param d_delimiter_offsets Offsets per string to delimiters in d_positions + */ + __device__ size_type count_tokens(size_type idx, + size_type const* d_positions, + size_type const* d_delimiter_offsets) const + { + if (!is_valid(idx)) { return 0; } + + auto const delim_size = d_delimiter.size_bytes(); + auto const d_str = get_string(idx); + auto const d_str_end = d_str.data() + d_str.size_bytes(); + auto const base_ptr = get_base_ptr() + delim_size - 1; + auto const delimiters = + cudf::device_span(d_positions + d_delimiter_offsets[idx], + d_delimiter_offsets[idx + 1] - d_delimiter_offsets[idx]); + + size_type token_count = 1; // all strings will have at least one token + size_type last_pos = delimiters[0] - delim_size; + for (auto d_pos : delimiters) { + // delimiter must fit in string && overlapping delimiters are ignored + if (((base_ptr + d_pos) < d_str_end) && ((d_pos - last_pos) >= delim_size)) { + ++token_count; + last_pos = d_pos; + } + } + // number of tokens is capped to max_tokens + return ((max_tokens > 0) && (token_count > max_tokens)) ? max_tokens : token_count; + } + + /** + * @brief This will create tokens around each delimiter honoring the string boundaries + * in which the delimiter resides + * + * Each token is placed in `d_all_tokens` so they align consecutively + * with other tokens for the same output column. + * + * The actual token extraction is performed in the subclass process_tokens() function. + * + * @param idx Index of the string to tokenize + * @param d_tokens_offsets Token offsets for each string + * @param d_positions The beginning byte position of each delimiter + * @param d_delimiter_offsets Offsets to d_positions to each delimiter set per string + * @param d_all_tokens All output tokens for the strings column + */ + __device__ void get_tokens(size_type idx, + size_type const* d_tokens_offsets, + size_type const* d_positions, + size_type const* d_delimiter_offsets, + string_index_pair* d_all_tokens) const + { + auto const d_tokens = // this string's tokens output + cudf::device_span(d_all_tokens + d_tokens_offsets[idx], + d_tokens_offsets[idx + 1] - d_tokens_offsets[idx]); + + if (!is_valid(idx)) { return; } + + auto const d_str = get_string(idx); + + // max_tokens already included in token counts + if (d_tokens.size() == 1) { + d_tokens[0] = string_index_pair{d_str.data(), d_str.size_bytes()}; + return; + } + + auto const delimiters = + cudf::device_span(d_positions + d_delimiter_offsets[idx], + d_delimiter_offsets[idx + 1] - d_delimiter_offsets[idx]); + + auto& derived = static_cast(*this); + derived.process_tokens(d_str, delimiters, d_tokens); + } + + base_split_tokenizer(column_device_view const& d_strings, + string_view const& d_delimiter, + size_type max_tokens) + : d_strings(d_strings), d_delimiter(d_delimiter), max_tokens(max_tokens) + { + } + + protected: + column_device_view const d_strings; // strings to split + string_view const d_delimiter; // delimiter for split + size_type max_tokens; // maximum number of tokens to identify +}; + +/** + * @brief The tokenizer functions for forward splitting + */ +struct split_tokenizer_fn : base_split_tokenizer { + /** + * @brief This will create tokens around each delimiter honoring the string boundaries + * + * The tokens are processed from the beginning of each string ignoring overlapping + * delimiters and honoring the `max_tokens` value. + * + * @param d_str String to tokenize + * @param d_delimiters Positions of delimiters for this string + * @param d_tokens Output vector to store tokens for this string + */ + __device__ void process_tokens(string_view const d_str, + device_span d_delimiters, + device_span d_tokens) const + { + auto const base_ptr = get_base_ptr(); // d_positions values based on this + auto str_ptr = d_str.data(); + auto const str_end = str_ptr + d_str.size_bytes(); // end of the string + auto const token_count = static_cast(d_tokens.size()); + auto const delim_size = d_delimiter.size_bytes(); + + // build the index-pair of each token for this string + size_type token_idx = 0; + for (auto d_pos : d_delimiters) { + auto const next_delim = base_ptr + d_pos; + if (next_delim < str_ptr || ((next_delim + delim_size) > str_end)) { continue; } + auto const end_ptr = (token_idx + 1 < token_count) ? next_delim : str_end; + + // store the token into the output vector + d_tokens[token_idx++] = + string_index_pair{str_ptr, static_cast(thrust::distance(str_ptr, end_ptr))}; + + // setup for next token + str_ptr = end_ptr + delim_size; + } + // include anything leftover + if (token_idx < token_count) { + d_tokens[token_idx] = + string_index_pair{str_ptr, static_cast(thrust::distance(str_ptr, str_end))}; + } + } + + split_tokenizer_fn(column_device_view const& d_strings, + string_view const& d_delimiter, + size_type max_tokens) + : base_split_tokenizer(d_strings, d_delimiter, max_tokens) + { + } +}; + +/** + * @brief The tokenizer functions for backwards splitting + * + * Same as split_tokenizer_fn except delimiters are searched from the end of each string. + */ +struct rsplit_tokenizer_fn : base_split_tokenizer { + /** + * @brief This will create tokens around each delimiter honoring the string boundaries + * + * The tokens are processed from the end of each string ignoring overlapping + * delimiters and honoring the `max_tokens` value. + * + * @param d_str String to tokenize + * @param d_delimiters Positions of delimiters for this string + * @param d_tokens Output vector to store tokens for this string + */ + __device__ void process_tokens(string_view const d_str, + device_span d_delimiters, + device_span d_tokens) const + { + auto const base_ptr = get_base_ptr(); // d_positions values are based on this ptr + auto const str_begin = d_str.data(); // beginning of the string + auto const token_count = static_cast(d_tokens.size()); + auto const delim_count = static_cast(d_delimiters.size()); + auto const delim_size = d_delimiter.size_bytes(); + + // build the index-pair of each token for this string + auto str_ptr = str_begin + d_str.size_bytes(); + size_type token_idx = 0; + for (auto d = delim_count - 1; d >= 0; --d) { // read right-to-left + auto const prev_delim = base_ptr + d_delimiters[d] + delim_size; + if (prev_delim > str_ptr || ((prev_delim - delim_size) < str_begin)) { continue; } + auto const start_ptr = (token_idx + 1 < token_count) ? prev_delim : str_begin; + + // store the token into the output vector right-to-left + d_tokens[token_count - token_idx - 1] = + string_index_pair{start_ptr, static_cast(thrust::distance(start_ptr, str_ptr))}; + + // setup for next token + str_ptr = start_ptr - delim_size; + ++token_idx; + } + // include anything leftover (rightover?) + if (token_idx < token_count) { + d_tokens[0] = + string_index_pair{str_begin, static_cast(thrust::distance(str_begin, str_ptr))}; + } + } + + rsplit_tokenizer_fn(column_device_view const& d_strings, + string_view const& d_delimiter, + size_type max_tokens) + : base_split_tokenizer(d_strings, d_delimiter, max_tokens) + { + } +}; + +/** + * @brief Helper function used by split/rsplit and split_record/rsplit_record + * + * This function returns all the token/split positions within the input column as processed by + * the given tokenizer. It also returns the offsets for each set of tokens identified per string. + * + * @tparam Tokenizer Type of the tokenizer object + * + * @param input The input column of strings to split + * @param tokenizer Object used for counting and identifying delimiters and tokens + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned objects' device memory. + */ +template +std::pair, rmm::device_uvector> split_helper( + strings_column_view const& input, + Tokenizer tokenizer, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + auto const strings_count = input.size(); + auto const chars_bytes = + cudf::detail::get_value(input.offsets(), input.offset() + strings_count, stream) - + cudf::detail::get_value(input.offsets(), input.offset(), stream); + + auto d_offsets = input.offsets_begin(); + + // count the number of delimiters in the entire column + auto const delimiter_count = + thrust::count_if(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(chars_bytes), + [tokenizer, d_offsets, chars_bytes] __device__(size_type idx) { + return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); + }); + // Create a vector of every delimiter position in the chars column. + // These may include overlapping or otherwise out-of-bounds delimiters which + // will be resolved during token processing. + auto delimiter_positions = rmm::device_uvector(delimiter_count, stream); + auto d_positions = delimiter_positions.data(); + auto const copy_end = + thrust::copy_if(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(chars_bytes), + delimiter_positions.begin(), + [tokenizer, d_offsets, chars_bytes] __device__(size_type idx) { + return tokenizer.is_delimiter(idx, d_offsets, chars_bytes); + }); + + // create a vector of offsets to each string's delimiter set within delimiter_positions + auto const delimiter_offsets = [&] { + // first, create a vector of string indices for each delimiter + auto string_indices = rmm::device_uvector(delimiter_count, stream); + thrust::upper_bound(rmm::exec_policy(stream), + d_offsets, + d_offsets + strings_count, + delimiter_positions.begin(), + copy_end, + string_indices.begin()); + + // compute delimiter offsets per string + auto delimiter_offsets = rmm::device_uvector(strings_count + 1, stream); + auto d_delimiter_offsets = delimiter_offsets.data(); + + // memset to zero-out the delimiter counts for any null-entries or strings with no delimiters + CUDF_CUDA_TRY(cudaMemsetAsync( + d_delimiter_offsets, 0, delimiter_offsets.size() * sizeof(size_type), stream.value())); + + // next, count the number of delimiters per string + auto d_string_indices = string_indices.data(); // identifies strings with delimiters only + thrust::for_each_n(rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + delimiter_count, + [d_string_indices, d_delimiter_offsets] __device__(size_type idx) { + auto const str_idx = d_string_indices[idx] - 1; + atomicAdd(d_delimiter_offsets + str_idx, 1); + }); + // finally, convert the delimiter counts into offsets + thrust::exclusive_scan(rmm::exec_policy(stream), + delimiter_offsets.begin(), + delimiter_offsets.end(), + delimiter_offsets.begin()); + return delimiter_offsets; + }(); + auto const d_delimiter_offsets = delimiter_offsets.data(); + + // compute the number of tokens per string + auto token_counts = rmm::device_uvector(strings_count, stream); + thrust::transform( + rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + thrust::make_counting_iterator(strings_count), + token_counts.begin(), + [tokenizer, d_positions, d_delimiter_offsets] __device__(size_type idx) -> size_type { + return tokenizer.count_tokens(idx, d_positions, d_delimiter_offsets); + }); + + // create offsets from the counts for return to the caller + auto offsets = std::get<0>( + cudf::detail::make_offsets_child_column(token_counts.begin(), token_counts.end(), stream, mr)); + auto const total_tokens = + cudf::detail::get_value(offsets->view(), strings_count, stream); + auto const d_tokens_offsets = offsets->view().data(); + + // build a vector of all the token positions for all the strings + auto tokens = rmm::device_uvector(total_tokens, stream); + auto d_tokens = tokens.data(); + thrust::for_each_n( + rmm::exec_policy(stream), + thrust::make_counting_iterator(0), + strings_count, + [tokenizer, d_tokens_offsets, d_positions, d_delimiter_offsets, d_tokens] __device__( + size_type idx) { + tokenizer.get_tokens(idx, d_tokens_offsets, d_positions, d_delimiter_offsets, d_tokens); + }); + + return std::make_pair(std::move(offsets), std::move(tokens)); +} + +} // namespace cudf::strings::detail diff --git a/cpp/src/strings/split/split_record.cu b/cpp/src/strings/split/split_record.cu index d935ad0b1da..5b79fdefb5a 100644 --- a/cpp/src/strings/split/split_record.cu +++ b/cpp/src/strings/split/split_record.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "split.cuh" + #include #include #include @@ -23,14 +25,12 @@ #include #include #include -#include #include #include #include #include -#include #include #include @@ -38,108 +38,43 @@ namespace cudf { namespace strings { namespace detail { -using string_index_pair = thrust::pair; - namespace { -enum class Dir { FORWARD, BACKWARD }; - -/** - * @brief Compute the number of tokens for the `idx'th` string element of `d_strings`. - * - * The number of tokens is the same regardless if counting from the beginning - * or the end of the string. - */ -struct token_counter_fn { - column_device_view const d_strings; // strings to split - string_view const d_delimiter; // delimiter for split - size_type const max_tokens = std::numeric_limits::max(); - - __device__ size_type operator()(size_type idx) const - { - if (d_strings.is_null(idx)) { return 0; } - - auto const d_str = d_strings.element(idx); - size_type token_count = 0; - size_type start_pos = 0; - while (token_count < max_tokens - 1) { - auto const delimiter_pos = d_str.find(d_delimiter, start_pos); - if (delimiter_pos == string_view::npos) break; - token_count++; - start_pos = delimiter_pos + d_delimiter.length(); - } - return token_count + 1; // always at least one token - } -}; - -/** - * @brief Identify the tokens from the `idx'th` string element of `d_strings`. - */ -template -struct token_reader_fn { - column_device_view const d_strings; // strings to split - string_view const d_delimiter; // delimiter for split - int32_t* d_token_offsets{}; // for locating tokens in d_tokens - string_index_pair* d_tokens{}; - - __device__ string_index_pair resolve_token(string_view const& d_str, - size_type start_pos, - size_type end_pos, - size_type delimiter_pos) const - { - if (dir == Dir::FORWARD) { - auto const byte_offset = d_str.byte_offset(start_pos); - return string_index_pair{d_str.data() + byte_offset, - d_str.byte_offset(delimiter_pos) - byte_offset}; - } else { - auto const byte_offset = d_str.byte_offset(delimiter_pos + d_delimiter.length()); - return string_index_pair{d_str.data() + byte_offset, - d_str.byte_offset(end_pos) - byte_offset}; - } +template +std::unique_ptr split_record_fn(strings_column_view const& input, + Tokenizer tokenizer, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + if (input.is_empty()) { return make_empty_column(type_id::LIST); } + if (input.size() == input.null_count()) { + auto offsets = std::make_unique(input.offsets(), stream, mr); + auto results = make_empty_column(type_id::STRING); + return make_lists_column(input.size(), + std::move(offsets), + std::move(results), + input.null_count(), + copy_bitmask(input.parent(), stream, mr), + stream, + mr); } - __device__ void operator()(size_type idx) - { - if (d_strings.is_null(idx)) { return; } + // builds the offsets and the vector of all tokens + auto [offsets, tokens] = split_helper(input, tokenizer, stream, mr); - auto const token_offset = d_token_offsets[idx]; - auto const token_count = d_token_offsets[idx + 1] - token_offset; - auto d_result = d_tokens + token_offset; - auto const d_str = d_strings.element(idx); - if (d_str.empty()) { - // Pandas str.split("") for non-whitespace delimiter is an empty string - *d_result = string_index_pair{"", 0}; - return; - } + // build a strings column from the tokens + auto strings_child = make_strings_column(tokens.begin(), tokens.end(), stream, mr); - size_type token_idx = 0; - size_type start_pos = 0; // updates only if moving forward - size_type end_pos = d_str.length(); // updates only if moving backward - while (token_idx < token_count - 1) { - auto const delimiter_pos = dir == Dir::FORWARD ? d_str.find(d_delimiter, start_pos) - : d_str.rfind(d_delimiter, start_pos, end_pos); - if (delimiter_pos == string_view::npos) break; - auto const token = resolve_token(d_str, start_pos, end_pos, delimiter_pos); - if (dir == Dir::FORWARD) { - d_result[token_idx] = token; - start_pos = delimiter_pos + d_delimiter.length(); - } else { - d_result[token_count - 1 - token_idx] = token; - end_pos = delimiter_pos; - } - token_idx++; - } + return make_lists_column(input.size(), + std::move(offsets), + std::move(strings_child), + input.null_count(), + copy_bitmask(input.parent(), stream, mr), + stream, + mr); +} - // set last token to remainder of the string - if (dir == Dir::FORWARD) { - auto const offset_bytes = d_str.byte_offset(start_pos); - d_result[token_idx] = - string_index_pair{d_str.data() + offset_bytes, d_str.byte_offset(end_pos) - offset_bytes}; - } else { - d_result[0] = string_index_pair{d_str.data(), d_str.byte_offset(end_pos)}; - } - } -}; +enum class Dir { FORWARD, BACKWARD }; /** * @brief Compute the number of tokens for the `idx'th` string element of `d_strings`. @@ -196,7 +131,7 @@ struct whitespace_token_reader_fn { whitespace_string_tokenizer tokenizer(d_str, dir != Dir::FORWARD); size_type token_idx = 0; position_pair token{0, 0}; - if (dir == Dir::FORWARD) { + if constexpr (dir == Dir::FORWARD) { while (tokenizer.next_token() && (token_idx < token_count)) { token = tokenizer.get_token(); d_result[token_idx++] = @@ -224,11 +159,11 @@ struct whitespace_token_reader_fn { // The output is one list item per string template -std::unique_ptr split_record_fn(strings_column_view const& strings, - TokenCounter counter, - TokenReader reader, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) +std::unique_ptr whitespace_split_record_fn(strings_column_view const& strings, + TokenCounter counter, + TokenReader reader, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { // create offsets column by counting the number of tokens per string auto strings_count = strings.size(); @@ -244,7 +179,7 @@ std::unique_ptr split_record_fn(strings_column_view const& strings, rmm::exec_policy(stream), d_offsets, d_offsets + strings_count + 1, d_offsets); // last entry is the total number of tokens to be generated - auto total_tokens = cudf::detail::get_value(offsets->view(), strings_count, stream); + auto total_tokens = cudf::detail::get_value(offsets->view(), strings_count, stream); // split each string into an array of index-pair values rmm::device_uvector tokens(total_tokens, stream); reader.d_token_offsets = d_offsets; @@ -277,18 +212,21 @@ std::unique_ptr split_record(strings_column_view const& strings, auto d_strings_column_ptr = column_device_view::create(strings.parent(), stream); if (delimiter.size() == 0) { - return split_record_fn(strings, - whitespace_token_counter_fn{*d_strings_column_ptr, max_tokens}, - whitespace_token_reader_fn{*d_strings_column_ptr, max_tokens}, - stream, - mr); + return whitespace_split_record_fn( + strings, + whitespace_token_counter_fn{*d_strings_column_ptr, max_tokens}, + whitespace_token_reader_fn{*d_strings_column_ptr, max_tokens}, + stream, + mr); } else { string_view d_delimiter(delimiter.data(), delimiter.size()); - return split_record_fn(strings, - token_counter_fn{*d_strings_column_ptr, d_delimiter, max_tokens}, - token_reader_fn{*d_strings_column_ptr, d_delimiter}, - stream, - mr); + if (dir == Dir::FORWARD) { + return split_record_fn( + strings, split_tokenizer_fn{*d_strings_column_ptr, d_delimiter, max_tokens}, stream, mr); + } else { + return split_record_fn( + strings, rsplit_tokenizer_fn{*d_strings_column_ptr, d_delimiter, max_tokens}, stream, mr); + } } } diff --git a/cpp/tests/strings/split_tests.cpp b/cpp/tests/strings/split_tests.cpp index 73d5adab427..714c1ad416a 100644 --- a/cpp/tests/strings/split_tests.cpp +++ b/cpp/tests/strings/split_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -308,6 +308,82 @@ TEST_F(StringsSplitTest, SplitRecordWhitespaceWithMaxSplit) CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); } +TEST_F(StringsSplitTest, MultiByteDelimiters) +{ + // Overlapping delimiters + auto input = + cudf::test::strings_column_wrapper({"u::", "w:::x", "y::::z", "::a", ":::b", ":::c:::"}); + auto view = cudf::strings_column_view(input); + using LCW = cudf::test::lists_column_wrapper; + { + auto result = cudf::strings::split_record(view, cudf::string_scalar("::")); + auto expected_left = LCW({LCW{"u", ""}, + LCW{"w", ":x"}, + LCW{"y", "", "z"}, + LCW{"", "a"}, + LCW{"", ":b"}, + LCW{"", ":c", ":"}}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected_left); + result = cudf::strings::rsplit_record(view, cudf::string_scalar("::")); + auto expected_right = LCW({LCW{"u", ""}, + LCW{"w:", "x"}, + LCW{"y", "", "z"}, + LCW{"", "a"}, + LCW{":", "b"}, + LCW{":", "c:", ""}}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected_right); + } + { + auto result = cudf::strings::split(view, cudf::string_scalar("::")); + + auto c0 = cudf::test::strings_column_wrapper({"u", "w", "y", "", "", ""}); + auto c1 = cudf::test::strings_column_wrapper({"", ":x", "", "a", ":b", ":c"}); + auto c2 = cudf::test::strings_column_wrapper({"", "", "z", "", "", ":"}, {0, 0, 1, 0, 0, 1}); + std::vector> expected_columns; + expected_columns.push_back(c0.release()); + expected_columns.push_back(c1.release()); + expected_columns.push_back(c2.release()); + auto expected_left = std::make_unique(std::move(expected_columns)); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result, *expected_left); + + result = cudf::strings::rsplit(view, cudf::string_scalar("::")); + + c0 = cudf::test::strings_column_wrapper({"u", "w:", "y", "", ":", ":"}); + c1 = cudf::test::strings_column_wrapper({"", "x", "", "a", "b", "c:"}); + c2 = cudf::test::strings_column_wrapper({"", "", "z", "", "", ""}, {0, 0, 1, 0, 0, 1}); + expected_columns.push_back(c0.release()); + expected_columns.push_back(c1.release()); + expected_columns.push_back(c2.release()); + auto expected_right = std::make_unique(std::move(expected_columns)); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result, *expected_right); + } + + // Delimiters that span across adjacent strings + input = cudf::test::strings_column_wrapper({"{a=1}:{b=2}:", "{c=3}", ":{}:{}"}); + view = cudf::strings_column_view(input); + { + auto result = cudf::strings::split_record(view, cudf::string_scalar("}:{")); + auto expected = LCW({LCW{"{a=1", "b=2}:"}, LCW{"{c=3}"}, LCW{":{", "}"}}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + result = cudf::strings::rsplit_record(view, cudf::string_scalar("}:{")); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + } + { + auto result = cudf::strings::split(view, cudf::string_scalar("}:{")); + + auto c0 = cudf::test::strings_column_wrapper({"{a=1", "{c=3}", ":{"}); + auto c1 = cudf::test::strings_column_wrapper({"b=2}:", "", "}"}, {1, 0, 1}); + std::vector> expected_columns; + expected_columns.push_back(c0.release()); + expected_columns.push_back(c1.release()); + auto expected = std::make_unique(std::move(expected_columns)); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result, *expected); + + result = cudf::strings::rsplit(view, cudf::string_scalar("}:{")); + CUDF_TEST_EXPECT_TABLES_EQUIVALENT(*result, *expected); + } +} + TEST_F(StringsSplitTest, SplitRegex) { std::vector h_strings{" Héllo thesé", nullptr, "are some ", "tést String", ""}; From a308b24a478f4888ccce5d64b10a6107b97b7da9 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 21 Feb 2023 09:42:54 -0800 Subject: [PATCH 41/69] Remove now redundant cuda initialization (#12758) Prior to #11452 cuDF Python did not require CUDA for compilation. When libcudf was is found by CMake, however, it triggers a compilation of the C++ library, which does require CUDA for compilation. In order to support this behavior, we included some extra logic in cuDF's CMake to ensure that the appropriate CUDA architectures are compiled for (respecting the extra options like `RAPIDS` and `NATIVE` that `rapids-cmake` offers). However, with the merge of #11452 this conditional is now redundant because cuDF requires CUDA compilation unconditionally, so we can remove the extra code. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12758 --- python/cudf/CMakeLists.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 7457b770b13..7210d398c6b 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -72,15 +72,6 @@ endif() include(rapids-cython) if(NOT cudf_FOUND) - # TODO: This will not be necessary once we upgrade to CMake 3.22, which will pull in the required - # languages for the C++ project even if this project does not require those languages. - include(rapids-cuda) - rapids_cuda_init_architectures(cudf-python) - enable_language(CUDA) - # Since cudf only enables CUDA optionally we need to manually include the file that - # rapids_cuda_init_architectures relies on `project` including. - include("${CMAKE_PROJECT_cudf-python_INCLUDE}") - set(BUILD_TESTS OFF) set(BUILD_BENCHMARKS OFF) From 904b8c70fcb5359a787d71d460f54f1d2c16fbd8 Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Tue, 21 Feb 2023 21:32:55 -0500 Subject: [PATCH 42/69] Fixing parquet coalescing of reads (#12808) When digging through the new chunk writer for parquet pipelined reads, I noticed that the code changed and the read coalescing was no longer happening. Through some checking, I found that tests were issuing multiple smaller reads, specifically a read per rowgroup instead of a single read for multiple rowgroups. With this change, there is a single read per file. I believe this is an unintentional change and a departure from the previous behavior. Authors: - Mike Wilson (https://github.com/hyperbolic2346) Approvers: - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12808 --- cpp/src/io/parquet/reader_impl_preprocess.cu | 22 ++++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index 6577a1a3f0f..b1d013a96a3 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -685,7 +685,6 @@ void reader::impl::load_and_decompress_data(std::vector const& r auto const row_group_start = rg.start_row; auto const row_group_source = rg.source_index; auto const row_group_rows = std::min(remaining_rows, row_group.num_rows); - auto const io_chunk_idx = chunks.size(); // generate ColumnChunkDesc objects for everything to be decoded (all input columns) for (size_t i = 0; i < num_input_columns; ++i) { @@ -733,18 +732,19 @@ void reader::impl::load_and_decompress_data(std::vector const& r total_decompressed_size += col_meta.total_uncompressed_size; } } - // Read compressed chunk data to device memory - read_rowgroup_tasks.push_back(read_column_chunks_async(_sources, - raw_page_data, - chunks, - io_chunk_idx, - chunks.size(), - column_chunk_offsets, - chunk_source_map, - _stream)); - remaining_rows -= row_group.num_rows; } + + // Read compressed chunk data to device memory + read_rowgroup_tasks.push_back(read_column_chunks_async(_sources, + raw_page_data, + chunks, + 0, + chunks.size(), + column_chunk_offsets, + chunk_source_map, + _stream)); + for (auto& task : read_rowgroup_tasks) { task.wait(); } From f90ae52b52bf7aaa75925c45719da190f13711d8 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Wed, 22 Feb 2023 08:14:33 -0600 Subject: [PATCH 43/69] Move `strings_udf` code into cuDF (#12669) With the merge of https://github.com/rapidsai/cudf/pull/11452 we have the machinery to build and deploy PTX libraries of shim functions as part of cuDF's build process. With this there is no reason to keep the `strings_udf` code separate anymore. This PR removes the separate package and all of it's related CI plumbing as well as supports the strings feature by default, just like GroupBy. Authors: - https://github.com/brandon-b-miller - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12669 --- .gitattributes | 1 - .gitignore | 2 - build.sh | 10 +- ci/build_python.sh | 7 +- ci/release/update-version.sh | 2 - ci/test_python_other.sh | 36 - conda/recipes/cudf/meta.yaml | 1 + conda/recipes/strings_udf/build.sh | 4 - .../strings_udf/conda_build_config.yaml | 14 - conda/recipes/strings_udf/meta.yaml | 78 - .../source/user_guide/guide-to-udfs.ipynb | 192 +- python/cudf/CMakeLists.txt | 2 +- python/cudf/cudf/__init__.py | 5 +- python/cudf/cudf/_lib/CMakeLists.txt | 5 +- python/cudf/cudf/_lib/__init__.py | 3 +- .../cudf}/_lib/cpp/strings_udf.pxd | 5 +- .../cudf/_lib/strings_udf.pyx} | 33 +- python/cudf/cudf/core/dataframe.py | 7 +- python/cudf/cudf/core/indexed_frame.py | 2 - python/cudf/cudf/core/series.py | 7 +- python/cudf/cudf/core/udf/__init__.py | 86 +- python/cudf/cudf/core/udf/groupby_utils.py | 5 - python/cudf/cudf/core/udf/masked_lowering.py | 13 +- python/cudf/cudf/core/udf/masked_typing.py | 325 ++- python/cudf/cudf/core/udf/row_function.py | 27 +- python/cudf/cudf/core/udf/scalar_function.py | 7 +- python/cudf/cudf/core/udf/strings_lowering.py | 561 +++- python/cudf/cudf/core/udf/strings_typing.py | 313 ++- python/cudf/cudf/core/udf/utils.py | 188 +- .../cudf}/tests/test_string_udfs.py | 24 +- python/cudf/cudf/tests/test_udf_masked_ops.py | 5 +- python/cudf/setup.cfg | 3 +- .../cpp => cudf/udf_cpp}/CMakeLists.txt | 40 +- python/cudf/udf_cpp/groupby/CMakeLists.txt | 79 - python/cudf/udf_cpp/groupby/function.cu | 323 --- .../src/strings/udf => cudf/udf_cpp}/shim.cu | 310 ++- .../include/cudf/strings/udf/case.cuh | 2 +- .../include/cudf/strings/udf/char_types.cuh | 2 +- .../include/cudf/strings/udf/numeric.cuh | 2 +- .../strings}/include/cudf/strings/udf/pad.cuh | 2 +- .../include/cudf/strings/udf/replace.cuh | 2 +- .../include/cudf/strings/udf/search.cuh | 2 +- .../include/cudf/strings/udf/split.cuh | 2 +- .../include/cudf/strings/udf/starts_with.cuh | 2 +- .../include/cudf/strings/udf/strip.cuh | 2 +- .../include/cudf/strings/udf/udf_apis.hpp | 2 +- .../include/cudf/strings/udf/udf_string.cuh | 2 +- .../include/cudf/strings/udf/udf_string.hpp | 2 +- .../strings}/src/strings/udf/udf_apis.cu | 2 +- python/strings_udf/CMakeLists.txt | 46 - python/strings_udf/setup.cfg | 41 - python/strings_udf/setup.py | 82 - python/strings_udf/strings_udf/__init__.py | 44 - .../strings_udf/_lib/CMakeLists.txt | 21 - .../strings_udf/strings_udf/_lib/__init__.py | 0 .../strings_udf/_lib/cpp/__init__.pxd | 0 .../strings_udf/strings_udf/_lib/tables.pyx | 26 - python/strings_udf/strings_udf/_typing.py | 278 -- python/strings_udf/strings_udf/_version.py | 711 ------ python/strings_udf/strings_udf/lowering.py | 517 ---- python/strings_udf/versioneer.py | 2245 ----------------- 61 files changed, 1613 insertions(+), 5149 deletions(-) delete mode 100644 conda/recipes/strings_udf/build.sh delete mode 100644 conda/recipes/strings_udf/conda_build_config.yaml delete mode 100644 conda/recipes/strings_udf/meta.yaml rename python/{strings_udf/strings_udf => cudf/cudf}/_lib/cpp/strings_udf.pxd (96%) rename python/{strings_udf/strings_udf/_lib/cudf_jit_udf.pyx => cudf/cudf/_lib/strings_udf.pyx} (59%) rename python/{strings_udf/strings_udf => cudf/cudf}/tests/test_string_udfs.py (94%) rename python/{strings_udf/cpp => cudf/udf_cpp}/CMakeLists.txt (75%) delete mode 100644 python/cudf/udf_cpp/groupby/CMakeLists.txt delete mode 100644 python/cudf/udf_cpp/groupby/function.cu rename python/{strings_udf/cpp/src/strings/udf => cudf/udf_cpp}/shim.cu (55%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/case.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/char_types.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/numeric.cuh (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/pad.cuh (98%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/replace.cuh (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/search.cuh (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/split.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/starts_with.cuh (98%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/strip.cuh (98%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/udf_apis.hpp (97%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/udf_string.cuh (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/include/cudf/strings/udf/udf_string.hpp (99%) rename python/{strings_udf/cpp => cudf/udf_cpp/strings}/src/strings/udf/udf_apis.cu (98%) delete mode 100644 python/strings_udf/CMakeLists.txt delete mode 100644 python/strings_udf/setup.cfg delete mode 100644 python/strings_udf/setup.py delete mode 100644 python/strings_udf/strings_udf/__init__.py delete mode 100644 python/strings_udf/strings_udf/_lib/CMakeLists.txt delete mode 100644 python/strings_udf/strings_udf/_lib/__init__.py delete mode 100644 python/strings_udf/strings_udf/_lib/cpp/__init__.pxd delete mode 100644 python/strings_udf/strings_udf/_lib/tables.pyx delete mode 100644 python/strings_udf/strings_udf/_typing.py delete mode 100644 python/strings_udf/strings_udf/_version.py delete mode 100644 python/strings_udf/strings_udf/lowering.py delete mode 100644 python/strings_udf/versioneer.py diff --git a/.gitattributes b/.gitattributes index ed8e5e1425a..fbfe7434d50 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,4 @@ python/cudf/cudf/_version.py export-subst -python/strings_udf/strings_udf/_version.py export-subst python/cudf_kafka/cudf_kafka/_version.py export-subst python/custreamz/custreamz/_version.py export-subst python/dask_cudf/dask_cudf/_version.py export-subst diff --git a/.gitignore b/.gitignore index 0f81dcb6f2b..2d83aad7712 100644 --- a/.gitignore +++ b/.gitignore @@ -36,8 +36,6 @@ python/cudf_kafka/*/_lib/**/*.cpp python/cudf_kafka/*/_lib/**/*.h python/custreamz/*/_lib/**/*.cpp python/custreamz/*/_lib/**/*.h -python/strings_udf/strings_udf/_lib/*.cpp -python/strings_udf/strings_udf/*.ptx .Python env/ develop-eggs/ diff --git a/build.sh b/build.sh index 22a62df7182..bee66d819b4 100755 --- a/build.sh +++ b/build.sh @@ -17,7 +17,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz strings_udf -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" +VALIDARGS="clean libcudf cudf cudfjar dask_cudf benchmarks tests libcudf_kafka cudf_kafka custreamz -v -g -n -l --allgpuarch --disable_nvtx --opensource_nvcomp --show_depr_warn --ptds -h --build_metrics --incl_cache_stats" HELP="$0 [clean] [libcudf] [cudf] [cudfjar] [dask_cudf] [benchmarks] [tests] [libcudf_kafka] [cudf_kafka] [custreamz] [-v] [-g] [-n] [-h] [--cmake-args=\\\"\\\"] clean - remove all existing build artifacts and configuration (start over) @@ -337,14 +337,6 @@ if buildAll || hasArg cudf; then fi fi -if buildAll || hasArg strings_udf; then - - cd ${REPODIR}/python/strings_udf - python setup.py build_ext --inplace -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} -DCMAKE_CUDA_ARCHITECTURES=${CUDF_CMAKE_CUDA_ARCHITECTURES} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} - if [[ ${INSTALL_TARGET} != "" ]]; then - python setup.py install --single-version-externally-managed --record=record.txt -- -DCMAKE_PREFIX_PATH=${INSTALL_PREFIX} -DCMAKE_LIBRARY_PATH=${LIBCUDF_BUILD_DIR} ${EXTRA_CMAKE_ARGS} -- -j${PARALLEL_LEVEL:-1} - fi -fi # Build and install the dask_cudf Python package if buildAll || hasArg dask_cudf; then diff --git a/ci/build_python.sh b/ci/build_python.sh index 8756d759cc7..ec34d63b282 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. set -euo pipefail @@ -38,10 +38,5 @@ rapids-mamba-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/custreamz -rapids-mamba-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/strings_udf rapids-upload-conda-to-s3 python diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d2be7d5f222..15d81127450 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -40,8 +40,6 @@ sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' # Python update sed_runner 's/'"cudf_version .*)"'/'"cudf_version ${NEXT_FULL_TAG})"'/g' python/cudf/CMakeLists.txt -# Strings UDF update -sed_runner 's/'"strings_udf_version .*)"'/'"strings_udf_version ${NEXT_FULL_TAG})"'/g' python/strings_udf/CMakeLists.txt # cpp libcudf_kafka update sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/libcudf_kafka/CMakeLists.txt diff --git a/ci/test_python_other.sh b/ci/test_python_other.sh index b79cd44cdbe..25ed615df84 100755 --- a/ci/test_python_other.sh +++ b/ci/test_python_other.sh @@ -44,41 +44,5 @@ pytest \ custreamz popd -set -e -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - strings_udf -set +e - -rapids-logger "pytest strings_udf" -pushd python/strings_udf/strings_udf -pytest \ - --cache-clear \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-strings-udf.xml" \ - --numprocesses=8 \ - --dist=loadscope \ - --cov-config=.coveragerc \ - --cov=strings_udf \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/strings-udf-coverage.xml" \ - --cov-report=term \ - tests -popd - -rapids-logger "pytest cudf with strings_udf" -pushd python/cudf/cudf -pytest \ - --cache-clear \ - --ignore="benchmarks" \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-strings-udf.xml" \ - --numprocesses=8 \ - --dist=loadscope \ - --cov-config=../.coveragerc \ - --cov=cudf \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cudf-strings-udf-coverage.xml" \ - --cov-report=term \ - tests/test_udf_masked_ops.py -popd - rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml index 0d5b5d16e08..27073eb323b 100644 --- a/conda/recipes/cudf/meta.yaml +++ b/conda/recipes/cudf/meta.yaml @@ -72,6 +72,7 @@ requirements: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - nvtx >=0.2.1 - packaging + - ptxcompiler >=0.7.0 - cachetools - cubinlinker # CUDA enhanced compatibility. - cuda-python >=11.7.1,<12.0 diff --git a/conda/recipes/strings_udf/build.sh b/conda/recipes/strings_udf/build.sh deleted file mode 100644 index 2de1325347b..00000000000 --- a/conda/recipes/strings_udf/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh strings_udf diff --git a/conda/recipes/strings_udf/conda_build_config.yaml b/conda/recipes/strings_udf/conda_build_config.yaml deleted file mode 100644 index 4feac647e8c..00000000000 --- a/conda/recipes/strings_udf/conda_build_config.yaml +++ /dev/null @@ -1,14 +0,0 @@ -c_compiler_version: - - 9 - -cxx_compiler_version: - - 9 - -sysroot_version: - - "2.17" - -cmake_version: - - ">=3.23.1,!=3.25.0" - -cuda_compiler: - - nvcc diff --git a/conda/recipes/strings_udf/meta.yaml b/conda/recipes/strings_udf/meta.yaml deleted file mode 100644 index 93316a92c22..00000000000 --- a/conda/recipes/strings_udf/meta.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: strings_udf - version: {{ version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda_{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - PARALLEL_LEVEL - - CMAKE_GENERATOR - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - SCCACHE_S3_KEY_PREFIX=strings-udf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=strings-udf-linux64 # [linux64] - - SCCACHE_BUCKET - - SCCACHE_REGION - - SCCACHE_IDLE_TIMEOUT - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - ignore_run_exports: - # libcudf's run_exports pinning is looser than we would like - - libcudf - ignore_run_exports_from: - - {{ compiler('cuda') }} - -requirements: - build: - - cmake {{ cmake_version }} - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ compiler('cuda') }} {{ cuda_version }} - - ninja - - sysroot_{{ target_platform }} {{ sysroot_version }} - host: - - python - - cython >=0.29,<0.30 - - scikit-build >=0.13.1 - - setuptools - - numba >=0.54 - - libcudf ={{ version }} - - cudf ={{ version }} - - cudatoolkit ={{ cuda_version }} - run: - - python - - typing_extensions - - numba >=0.54 - - numpy - - libcudf ={{ version }} - - cudf ={{ version }} - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - cachetools - - ptxcompiler >=0.7.0 # CUDA enhanced compatibility. See https://github.com/rapidsai/ptxcompiler - -test: - requires: - - cudatoolkit ={{ cuda_version }} - imports: - - strings_udf - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: strings_udf library diff --git a/docs/cudf/source/user_guide/guide-to-udfs.ipynb b/docs/cudf/source/user_guide/guide-to-udfs.ipynb index e6ef5e86416..943fc980a31 100644 --- a/docs/cudf/source/user_guide/guide-to-udfs.ipynb +++ b/docs/cudf/source/user_guide/guide-to-udfs.ipynb @@ -336,30 +336,13 @@ "### String data" ] }, - { - "cell_type": "markdown", - "id": "c0980218", - "metadata": {}, - "source": [ - "Experimental support for a subset of string functionality is available for `apply` through the [strings_udf](https://anaconda.org/rapidsai-nightly/strings_udf) package, which is separately installable from RAPIDS conda channels. The following several cells show an example of its usage if present. If it is not present, this notebook may be restarted after a conda install of `strings_udf` to proceed through the example." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "b91a60f7", - "metadata": {}, - "outputs": [], - "source": [ - "string_udfs_enabled = cudf.core.udf._STRING_UDFS_ENABLED" - ] - }, { "cell_type": "markdown", "id": "81762aea", "metadata": {}, "source": [ - "Currently, the following string operations are provided through `strings_udf`:\",\n", + "Experimental support for a subset of string functionality is available for `apply`. The following string operations are currently supported:\n", + "\n", "- `str.count`\n", "- `str.startswith`\n", "- `str.endswith`\n", @@ -382,12 +365,12 @@ "- `upper`\n", "- `lower`\n", "- `+` (string concatenation)\n", - "- `replace" + "- `replace`" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "id": "d7d1abd7", "metadata": {}, "outputs": [], @@ -397,7 +380,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "id": "e8538ba0", "metadata": {}, "outputs": [], @@ -416,7 +399,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "id": "23524fd8", "metadata": {}, "outputs": [ @@ -432,10 +415,8 @@ } ], "source": [ - "# Requires strings_udf package, or will raise a compilation error\n", - "if string_udfs_enabled:\n", - " result = sr.apply(f)\n", - " print(result)" + "result = sr.apply(f)\n", + "print(result)" ] }, { @@ -456,14 +437,13 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "id": "b26ec6dc", "metadata": {}, "outputs": [], "source": [ - "if string_udfs_enabled:\n", - " from strings_udf import set_malloc_heap_size\n", - " set_malloc_heap_size(int(2e9))" + "from cudf.core.udf.utils import set_malloc_heap_size\n", + "set_malloc_heap_size(int(2e9))" ] }, { @@ -487,7 +467,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "id": "732434f6", "metadata": {}, "outputs": [], @@ -497,7 +477,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "id": "4f5997e5", "metadata": {}, "outputs": [], @@ -523,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "id": "ea6008a6", "metadata": {}, "outputs": [], @@ -543,7 +523,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "id": "183a82ed", "metadata": {}, "outputs": [ @@ -623,7 +603,7 @@ "4 979 982 1011 9790.0" ] }, - "execution_count": 20, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -672,7 +652,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "id": "73653918", "metadata": {}, "outputs": [], @@ -691,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 21, "id": "077feb75", "metadata": {}, "outputs": [ @@ -747,7 +727,7 @@ "2 3 6" ] }, - "execution_count": 22, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -770,7 +750,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 22, "id": "091e39e1", "metadata": {}, "outputs": [ @@ -783,7 +763,7 @@ "dtype: int64" ] }, - "execution_count": 23, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -802,7 +782,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 23, "id": "bd345fab", "metadata": {}, "outputs": [ @@ -815,7 +795,7 @@ "dtype: object" ] }, - "execution_count": 24, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -842,7 +822,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 24, "id": "b70f4b3b", "metadata": {}, "outputs": [ @@ -894,7 +874,7 @@ "2 3" ] }, - "execution_count": 25, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -913,7 +893,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "id": "0313c8df", "metadata": {}, "outputs": [ @@ -926,7 +906,7 @@ "dtype: int64" ] }, - "execution_count": 26, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -945,7 +925,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "id": "96a7952a", "metadata": {}, "outputs": [ @@ -1001,7 +981,7 @@ "2 3 1" ] }, - "execution_count": 27, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1024,7 +1004,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "id": "e0815f60", "metadata": {}, "outputs": [ @@ -1037,7 +1017,7 @@ "dtype: int64" ] }, - "execution_count": 28, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -1056,7 +1036,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 28, "id": "495efd14", "metadata": {}, "outputs": [ @@ -1112,7 +1092,7 @@ "2 3 3.14" ] }, - "execution_count": 29, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -1130,7 +1110,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 29, "id": "678b0b5a", "metadata": {}, "outputs": [ @@ -1143,7 +1123,7 @@ "dtype: float64" ] }, - "execution_count": 30, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -1175,7 +1155,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 30, "id": "acf48d56", "metadata": {}, "outputs": [ @@ -1227,7 +1207,7 @@ "2 5" ] }, - "execution_count": 31, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -1248,7 +1228,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 31, "id": "78a98172", "metadata": {}, "outputs": [ @@ -1261,7 +1241,7 @@ "dtype: float64" ] }, - "execution_count": 32, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -1280,7 +1260,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 32, "id": "142c30a9", "metadata": {}, "outputs": [ @@ -1348,7 +1328,7 @@ "2 3 6 4 8 6" ] }, - "execution_count": 33, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -1369,7 +1349,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 33, "id": "fee9198a", "metadata": {}, "outputs": [ @@ -1382,7 +1362,7 @@ "dtype: float64" ] }, - "execution_count": 34, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -1409,17 +1389,7 @@ }, { "cell_type": "code", - "execution_count": 35, - "id": "3bfb117a", - "metadata": {}, - "outputs": [], - "source": [ - "string_udfs_enabled = cudf.core.udf._STRING_UDFS_ENABLED" - ] - }, - { - "cell_type": "code", - "execution_count": 36, + "execution_count": 34, "id": "cccd59f7", "metadata": {}, "outputs": [ @@ -1475,7 +1445,7 @@ "2 Example 3" ] }, - "execution_count": 36, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1490,7 +1460,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 35, "id": "35737fd9", "metadata": {}, "outputs": [], @@ -1507,7 +1477,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 36, "id": "4ede4d5b", "metadata": {}, "outputs": [ @@ -1523,9 +1493,8 @@ } ], "source": [ - "if string_udfs_enabled:\n", - " result = str_df.apply(f, axis=1)\n", - " print(result)" + "result = str_df.apply(f, axis=1)\n", + "print(result)" ] }, { @@ -1548,7 +1517,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 37, "id": "90cbcd85", "metadata": {}, "outputs": [], @@ -1579,7 +1548,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 38, "id": "e782daff", "metadata": {}, "outputs": [ @@ -1651,7 +1620,7 @@ "2 3 6 4 8 6 9.0" ] }, - "execution_count": 40, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -1685,7 +1654,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 39, "id": "befd8333", "metadata": {}, "outputs": [ @@ -1759,7 +1728,7 @@ "4 979 982 1011" ] }, - "execution_count": 41, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1786,7 +1755,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 40, "id": "d1f3dcaf", "metadata": {}, "outputs": [ @@ -1866,7 +1835,7 @@ "4 979 982 1011 1961.0" ] }, - "execution_count": 42, + "execution_count": 40, "metadata": {}, "output_type": "execute_result" } @@ -1901,7 +1870,7 @@ }, { "cell_type": "code", - "execution_count": 43, + "execution_count": 41, "id": "6bc6aea3", "metadata": {}, "outputs": [ @@ -1917,7 +1886,7 @@ "dtype: float64" ] }, - "execution_count": 43, + "execution_count": 41, "metadata": {}, "output_type": "execute_result" } @@ -1929,7 +1898,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 42, "id": "a4c31df1", "metadata": {}, "outputs": [ @@ -1939,7 +1908,7 @@ "Rolling [window=3,min_periods=3,center=False]" ] }, - "execution_count": 44, + "execution_count": 42, "metadata": {}, "output_type": "execute_result" } @@ -1959,7 +1928,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 43, "id": "eb5a081b", "metadata": {}, "outputs": [], @@ -1985,7 +1954,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 44, "id": "ddec3263", "metadata": {}, "outputs": [ @@ -2001,7 +1970,7 @@ "dtype: float64" ] }, - "execution_count": 46, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -2020,7 +1989,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 45, "id": "8b61094a", "metadata": {}, "outputs": [ @@ -2088,7 +2057,7 @@ "4 59.0 59.0" ] }, - "execution_count": 47, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -2102,7 +2071,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 46, "id": "bb8c3019", "metadata": {}, "outputs": [ @@ -2200,7 +2169,7 @@ "9 100.0 100.0" ] }, - "execution_count": 48, + "execution_count": 46, "metadata": {}, "output_type": "execute_result" } @@ -2224,7 +2193,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 47, "id": "3dc272ab", "metadata": {}, "outputs": [ @@ -2304,7 +2273,7 @@ "4 -0.970850 False Sarah 0.342905" ] }, - "execution_count": 49, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } @@ -2316,7 +2285,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 48, "id": "c0578e0a", "metadata": {}, "outputs": [], @@ -2334,7 +2303,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 49, "id": "19f0f7fe", "metadata": {}, "outputs": [], @@ -2363,7 +2332,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 50, "id": "c43426c3", "metadata": {}, "outputs": [ @@ -2494,7 +2463,7 @@ "9 -0.725581 True George 0.405245 0.271319" ] }, - "execution_count": 52, + "execution_count": 50, "metadata": {}, "output_type": "execute_result" } @@ -2526,7 +2495,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 51, "id": "aa6a8509", "metadata": {}, "outputs": [ @@ -2536,7 +2505,7 @@ "array([ 1., 2., 3., 4., 10.])" ] }, - "execution_count": 53, + "execution_count": 51, "metadata": {}, "output_type": "execute_result" } @@ -2559,7 +2528,7 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 52, "id": "0bb8bf93", "metadata": {}, "outputs": [ @@ -2574,7 +2543,7 @@ "dtype: int32" ] }, - "execution_count": 54, + "execution_count": 52, "metadata": {}, "output_type": "execute_result" } @@ -2601,7 +2570,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 53, "id": "ce60b639", "metadata": {}, "outputs": [ @@ -2611,7 +2580,7 @@ "array([ 5., 10., 15., 20., 50.])" ] }, - "execution_count": 55, + "execution_count": 53, "metadata": {}, "output_type": "execute_result" } @@ -2655,6 +2624,7 @@ "- CuPy NDArrays\n", "- Numba DeviceNDArrays\n", "- Generalized NA UDFs\n", + "- String UDFs\n", "\n", "\n", "For more information please see the [cuDF](https://docs.rapids.ai/api/cudf/nightly/), [Numba.cuda](https://numba.readthedocs.io/en/stable/cuda/index.html), and [CuPy](https://docs-cupy.chainer.org/en/stable/) documentation." @@ -2677,7 +2647,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.15" + "version": "3.10.8" } }, "nbformat": 4, diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 7210d398c6b..c528eb69575 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -111,7 +111,7 @@ endif() rapids_cython_init() add_subdirectory(cudf/_lib) -add_subdirectory(udf_cpp/groupby) +add_subdirectory(udf_cpp) include(cmake/Modules/ProtobufHelpers.cmake) codegen_protoc(cudf/utils/metadata/orc_column_statistics.proto) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index b86fb72d955..05f61ee4f5a 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -91,10 +91,9 @@ # cuDF requires a stronger set of conditions than what is # checked by patch_numba_linker_if_needed due to the PTX # files needed for JIT Groupby Apply and string UDFs - from cudf.core.udf.groupby_utils import dev_func_ptx - from cudf.core.udf.utils import _setup_numba_linker + from cudf.core.udf.utils import _PTX_FILE, _setup_numba_linker - _setup_numba_linker(dev_func_ptx) + _setup_numba_linker(_PTX_FILE) del patch_numba_linker_if_needed diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 90d2d59cc66..4b785563484 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -46,6 +46,7 @@ set(cython_sources sort.pyx stream_compaction.pyx string_casting.pyx + strings_udf.pyx text.pyx transform.pyx transpose.pyx @@ -61,6 +62,8 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf ) +target_link_libraries(strings_udf cudf_strings_udf) + # TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was # fixed in https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7410 and will be available in # CMake 3.24, so we can remove the Development component once we upgrade to CMake 3.24. diff --git a/python/cudf/cudf/_lib/__init__.py b/python/cudf/cudf/_lib/__init__.py index 8ecb9a57426..b101db9a744 100644 --- a/python/cudf/cudf/_lib/__init__.py +++ b/python/cudf/cudf/_lib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import numpy as np from . import ( @@ -33,6 +33,7 @@ stream_compaction, string_casting, strings, + strings_udf, text, transpose, unary, diff --git a/python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd b/python/cudf/cudf/_lib/cpp/strings_udf.pxd similarity index 96% rename from python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd rename to python/cudf/cudf/_lib/cpp/strings_udf.pxd index b3bf6465db6..7d45bc858f5 100644 --- a/python/strings_udf/strings_udf/_lib/cpp/strings_udf.pxd +++ b/python/cudf/cudf/_lib/cpp/strings_udf.pxd @@ -1,14 +1,15 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. from libc.stdint cimport uint8_t, uint16_t from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.vector cimport vector +from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer + from cudf._lib.cpp.column.column cimport column from cudf._lib.cpp.column.column_view cimport column_view from cudf._lib.cpp.types cimport size_type -from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer cdef extern from "cudf/strings/udf/udf_string.hpp" namespace \ diff --git a/python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx b/python/cudf/cudf/_lib/strings_udf.pyx similarity index 59% rename from python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx rename to python/cudf/cudf/_lib/strings_udf.pyx index bf459f22c16..3d465f9172b 100644 --- a/python/strings_udf/strings_udf/_lib/cudf_jit_udf.pyx +++ b/python/cudf/cudf/_lib/strings_udf.pyx @@ -1,15 +1,25 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +from libc.stdint cimport uint8_t, uint16_t, uintptr_t + +from cudf._lib.cpp.strings_udf cimport ( + get_character_cases_table as cpp_get_character_cases_table, + get_character_flags_table as cpp_get_character_flags_table, + get_special_case_mapping_table as cpp_get_special_case_mapping_table, +) + +import numpy as np from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from cudf.core.buffer import as_buffer -from cudf._lib.column cimport Column -from cudf._lib.cpp.column.column cimport column, column_view from rmm._lib.device_buffer cimport DeviceBuffer, device_buffer -from strings_udf._lib.cpp.strings_udf cimport ( +from cudf._lib.column cimport Column +from cudf._lib.cpp.column.column cimport column, column_view +from cudf._lib.cpp.strings_udf cimport ( column_from_udf_string_array as cpp_column_from_udf_string_array, free_udf_string_array as cpp_free_udf_string_array, to_string_view_array as cpp_to_string_view_array, @@ -39,3 +49,18 @@ def column_from_udf_string_array(DeviceBuffer d_buffer): result = Column.from_unique_ptr(move(c_result)) return result + + +def get_character_flags_table_ptr(): + cdef const uint8_t* tbl_ptr = cpp_get_character_flags_table() + return np.uintp(tbl_ptr) + + +def get_character_cases_table_ptr(): + cdef const uint16_t* tbl_ptr = cpp_get_character_cases_table() + return np.uintp(tbl_ptr) + + +def get_special_case_mapping_table_ptr(): + cdef const void* tbl_ptr = cpp_get_special_case_mapping_table() + return np.uintp(tbl_ptr) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index c32ab19fb69..d43621d3d36 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -4192,11 +4192,8 @@ def apply( For more information, see the `cuDF guide to user defined functions `__. - Support for use of string data within UDFs is provided through the - `strings_udf `__ - RAPIDS library. Supported operations on strings include the subset of - functions and string methods that expect an input string but do not - return a string. Refer to caveats in the UDF guide referenced above. + Some string functions and methods are supported. Refer to the guide + to UDFs for details. Parameters ---------- diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 25c89c34040..43277fb55ff 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2112,7 +2112,6 @@ def _apply(self, func, kernel_getter, *args, **kwargs): """Apply `func` across the rows of the frame.""" if kwargs: raise ValueError("UDFs using **kwargs are not yet supported.") - try: kernel, retty = _compile_or_get( self, func, args, kernel_getter=kernel_getter @@ -2130,7 +2129,6 @@ def _apply(self, func, kernel_getter, *args, **kwargs): output_args = [(ans_col, ans_mask), len(self)] input_args = _get_input_args_from_frame(self) launch_args = output_args + input_args + list(args) - try: kernel.forall(len(self))(*launch_args) except Exception as e: diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 43b2f38ca46..60655c5a6f9 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2327,11 +2327,8 @@ def apply(self, func, convert_dtype=True, args=(), **kwargs): For more information, see the `cuDF guide to user defined functions `__. - Support for use of string data within UDFs is provided through the - `strings_udf `__ - RAPIDS library. Supported operations on strings include the subset of - functions and string methods that expect an input string but do not - return a string. Refer to caveats in the UDF guide referenced above. + Some string functions and methods are supported. Refer to the guide + to UDFs for details. Parameters ---------- diff --git a/python/cudf/cudf/core/udf/__init__.py b/python/cudf/cudf/core/udf/__init__.py index 06ceecf0a35..85d454652b7 100644 --- a/python/cudf/cudf/core/udf/__init__.py +++ b/python/cudf/cudf/core/udf/__init__.py @@ -1,81 +1,9 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. - -from functools import lru_cache - -from numba import types -from numba.cuda.cudaimpl import lower as cuda_lower - -from cudf.core.dtypes import dtype -from cudf.core.udf import api, row_function, utils -from cudf.utils.dtypes import STRING_TYPES - -from . import groupby_lowering, groupby_typing, masked_lowering, masked_typing - -_units = ["ns", "ms", "us", "s"] -_datetime_cases = {types.NPDatetime(u) for u in _units} -_timedelta_cases = {types.NPTimedelta(u) for u in _units} -_supported_masked_types = ( - types.integer_domain - | types.real_domain - | _datetime_cases - | _timedelta_cases - | {types.boolean} +from . import ( + groupby_lowering, + groupby_typing, + masked_lowering, + masked_typing, + strings_lowering, + strings_typing, ) -_STRING_UDFS_ENABLED = False -cudf_str_dtype = dtype(str) - - -try: - import strings_udf - from strings_udf import ptxpath - - if ptxpath: - utils.ptx_files.append(ptxpath) - - from strings_udf._lib.cudf_jit_udf import ( - column_from_udf_string_array, - column_to_string_view_array, - ) - from strings_udf._typing import ( - str_view_arg_handler, - string_view, - udf_string, - ) - - from . import strings_typing # isort: skip - from . import strings_lowering # isort: skip - - cuda_lower(api.Masked, string_view, types.boolean)( - masked_lowering.masked_constructor - ) - utils.JIT_SUPPORTED_TYPES |= STRING_TYPES - _supported_masked_types |= {string_view, udf_string} - - @lru_cache(maxsize=None) - def set_initial_malloc_heap_size(): - strings_udf.set_malloc_heap_size() - - def column_to_string_view_array_init_heap(col): - # lazily allocate heap only when a string needs to be returned - set_initial_malloc_heap_size() - return column_to_string_view_array(col) - - utils.launch_arg_getters[ - cudf_str_dtype - ] = column_to_string_view_array_init_heap - utils.output_col_getters[cudf_str_dtype] = column_from_udf_string_array - utils.masked_array_types[cudf_str_dtype] = string_view - row_function.itemsizes[cudf_str_dtype] = string_view.size_bytes - - utils.arg_handlers.append(str_view_arg_handler) - - masked_typing.MASKED_INIT_MAP[udf_string] = udf_string - - _STRING_UDFS_ENABLED = True - -except ImportError as e: - # allow cuDF to work without strings_udf - pass - -masked_typing._register_masked_constructor_typing(_supported_masked_types) -masked_lowering._register_masked_constructor_lowering(_supported_masked_types) diff --git a/python/cudf/cudf/core/udf/groupby_utils.py b/python/cudf/cudf/core/udf/groupby_utils.py index a1174835db9..dc31cf43292 100644 --- a/python/cudf/cudf/core/udf/groupby_utils.py +++ b/python/cudf/cudf/core/udf/groupby_utils.py @@ -1,6 +1,5 @@ # Copyright (c) 2022-2023, NVIDIA CORPORATION. -import os import cupy as cp import numpy as np @@ -22,16 +21,12 @@ from cudf.core.udf.utils import ( _get_extensionty_size, _get_kernel, - _get_ptx_file, _get_udf_return_type, _supported_cols_from_frame, _supported_dtypes_from_frame, ) from cudf.utils.utils import _cudf_nvtx_annotate -dev_func_ptx = _get_ptx_file(os.path.dirname(__file__), "function_") -cudf.core.udf.utils.ptx_files.append(dev_func_ptx) - def _get_frame_groupby_type(dtype, index_dtype): """ diff --git a/python/cudf/cudf/core/udf/masked_lowering.py b/python/cudf/cudf/core/udf/masked_lowering.py index 37f3117e756..74b414ce36a 100644 --- a/python/cudf/cudf/core/udf/masked_lowering.py +++ b/python/cudf/cudf/core/udf/masked_lowering.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import operator @@ -18,7 +18,11 @@ comparison_ops, unary_ops, ) -from cudf.core.udf.masked_typing import MaskedType, NAType +from cudf.core.udf.masked_typing import ( + MaskedType, + NAType, + _supported_masked_types, +) @cuda_lowering_registry.lower_constant(NAType) @@ -381,9 +385,8 @@ def masked_constructor(context, builder, sig, args): return masked._getvalue() -def _register_masked_constructor_lowering(supported_masked_types): - for ty in supported_masked_types: - lower_builtin(api.Masked, ty, types.boolean)(masked_constructor) +for ty in _supported_masked_types: + lower_builtin(api.Masked, ty, types.boolean)(masked_constructor) # Allows us to make an instance of MaskedType a global variable diff --git a/python/cudf/cudf/core/udf/masked_typing.py b/python/cudf/cudf/core/udf/masked_typing.py index 7baf2d585e2..30caa641701 100644 --- a/python/cudf/cudf/core/udf/masked_typing.py +++ b/python/cudf/cudf/core/udf/masked_typing.py @@ -1,7 +1,6 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import operator -from typing import Any, Dict from numba import types from numba.core.extending import ( @@ -27,6 +26,18 @@ comparison_ops, unary_ops, ) +from cudf.core.udf.strings_typing import ( + StringView, + UDFString, + bool_binary_funcs, + id_unary_funcs, + int_binary_funcs, + size_type, + string_return_attrs, + string_unary_funcs, + string_view, + udf_string, +) from cudf.utils.dtypes import ( DATETIME_TYPES, NUMERIC_TYPES, @@ -34,19 +45,33 @@ TIMEDELTA_TYPES, ) +SUPPORTED_NUMPY_TYPES = ( + NUMERIC_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES | STRING_TYPES +) +supported_type_str = "\n".join(sorted(list(SUPPORTED_NUMPY_TYPES) + ["bool"])) + +_units = ["ns", "ms", "us", "s"] +_datetime_cases = {types.NPDatetime(u) for u in _units} +_timedelta_cases = {types.NPTimedelta(u) for u in _units} +_supported_masked_types = ( + types.integer_domain + | types.real_domain + | _datetime_cases + | _timedelta_cases + | {types.boolean} + | {string_view, udf_string} +) + + SUPPORTED_NUMBA_TYPES = ( types.Number, types.Boolean, types.NPDatetime, types.NPTimedelta, + StringView, + UDFString, ) -SUPPORTED_NUMPY_TYPES = ( - NUMERIC_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES | STRING_TYPES -) -supported_type_str = "\n".join(sorted(list(SUPPORTED_NUMPY_TYPES) + ["bool"])) -MASKED_INIT_MAP: Dict[Any, Any] = {} - def _format_error_string(err): """ @@ -56,31 +81,19 @@ def _format_error_string(err): def _type_to_masked_type(t): - result = MASKED_INIT_MAP.get(t) - if result is None: - if isinstance(t, SUPPORTED_NUMBA_TYPES): - return t - else: - # Unsupported Dtype. Numba tends to print out the type info - # for whatever operands and operation failed to type and then - # output its own error message. Putting the message in the repr - # then is one way of getting the true cause to the user - err = _format_error_string( - "Unsupported MaskedType. This is usually caused by " - "attempting to use a column of unsupported dtype in a UDF. " - f"Supported dtypes are:\n{supported_type_str}" - ) - return types.Poison(err) + if isinstance(t, SUPPORTED_NUMBA_TYPES): + return t else: - return result - - -MASKED_INIT_MAP[types.pyobject] = types.Poison( - _format_error_string( - "strings_udf library required for usage of string dtypes " - "inside user defined functions." - ) -) + # Unsupported Dtype. Numba tends to print out the type info + # for whatever operands and operation failed to type and then + # output its own error message. Putting the message in the repr + # then is one way of getting the true cause to the user + err = _format_error_string( + "Unsupported MaskedType. This is usually caused by " + "attempting to use a column of unsupported dtype in a UDF. " + f"Supported dtypes are:\n{supported_type_str}" + ) + return types.Poison(err) # Masked scalars of all types @@ -169,30 +182,30 @@ def typeof_masked(val, c): # Implemented typing for Masked(value, valid) - the construction of a Masked # type in a kernel. -def _register_masked_constructor_typing(supported_masked_types): - class MaskedConstructor(ConcreteTemplate): - key = api.Masked - cases = [ - nb_signature(MaskedType(t), t, types.boolean) - for t in supported_masked_types - ] - - cuda_decl_registry.register(MaskedConstructor) - - # Typing for `api.Masked` - @cuda_decl_registry.register_attr - class ClassesTemplate(AttributeTemplate): - key = types.Module(api) - - def resolve_Masked(self, mod): - return types.Function(MaskedConstructor) - - # Registration of the global is also needed for Numba to type api.Masked - cuda_decl_registry.register_global(api, types.Module(api)) - # For typing bare Masked (as in `from .api import Masked` - cuda_decl_registry.register_global( - api.Masked, types.Function(MaskedConstructor) - ) +@cuda_decl_registry.register +class MaskedConstructor(ConcreteTemplate): + key = api.Masked + cases = [ + nb_signature(MaskedType(t), t, types.boolean) + for t in _supported_masked_types + ] + + +# Typing for `api.Masked` +@cuda_decl_registry.register_attr +class ClassesTemplate(AttributeTemplate): + key = types.Module(api) + + def resolve_Masked(self, mod): + return types.Function(MaskedConstructor) + + +# Registration of the global is also needed for Numba to type api.Masked +cuda_decl_registry.register_global(api, types.Module(api)) +# For typing bare Masked (as in `from .api import Masked` +cuda_decl_registry.register_global( + api.Masked, types.Function(MaskedConstructor) +) # Provide access to `m.value` and `m.valid` in a kernel for a Masked `m`. @@ -423,3 +436,203 @@ def generic(self, args, kws): for unary_op in unary_ops: cuda_decl_registry.register_global(unary_op)(MaskedScalarUnaryOp) + + +# Strings functions and utilities +def _is_valid_string_arg(ty): + return ( + isinstance(ty, MaskedType) and isinstance(ty.value_type, StringView) + ) or isinstance(ty, types.StringLiteral) + + +def register_masked_string_function(func): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to associate a signature with a function or + operator to be overloaded. + """ + + def deco(generic): + class MaskedStringFunction(AbstractTemplate): + pass + + MaskedStringFunction.generic = generic + cuda_decl_registry.register_global(func)(MaskedStringFunction) + + return deco + + +@register_masked_string_function(len) +def len_typing(self, args, kws): + if isinstance(args[0], MaskedType) and isinstance( + args[0].value_type, StringView + ): + return nb_signature(MaskedType(size_type), args[0]) + elif isinstance(args[0], types.StringLiteral) and len(args) == 1: + return nb_signature(size_type, args[0]) + + +@register_masked_string_function(operator.add) +def concat_typing(self, args, kws): + if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): + return nb_signature( + MaskedType(udf_string), + MaskedType(string_view), + MaskedType(string_view), + ) + + +@register_masked_string_function(operator.contains) +def contains_typing(self, args, kws): + if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): + return nb_signature( + MaskedType(types.boolean), + MaskedType(string_view), + MaskedType(string_view), + ) + + +class MaskedStringViewCmpOp(AbstractTemplate): + """ + return the boolean result of `cmpop` between to strings + since the typing is the same for every comparison operator, + we can reuse this class for all of them. + """ + + def generic(self, args, kws): + if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): + return nb_signature( + MaskedType(types.boolean), + MaskedType(string_view), + MaskedType(string_view), + ) + + +for op in comparison_ops: + cuda_decl_registry.register_global(op)(MaskedStringViewCmpOp) + + +def create_masked_binary_attr(attrname, retty): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to register a binary function of two masked + string objects as an attribute of one, e.g. `string.func(other)`. + """ + + class MaskedStringViewBinaryAttr(AbstractTemplate): + key = attrname + + def generic(self, args, kws): + return nb_signature( + MaskedType(retty), MaskedType(string_view), recvr=self.this + ) + + def attr(self, mod): + return types.BoundFunction( + MaskedStringViewBinaryAttr, + MaskedType(string_view), + ) + + return attr + + +def create_masked_unary_attr(attrname, retty): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to register a unary function of a masked + string object as an attribute, e.g. `string.func()`. + """ + + class MaskedStringViewIdentifierAttr(AbstractTemplate): + key = attrname + + def generic(self, args, kws): + return nb_signature(MaskedType(retty), recvr=self.this) + + def attr(self, mod): + return types.BoundFunction( + MaskedStringViewIdentifierAttr, + MaskedType(string_view), + ) + + return attr + + +class MaskedStringViewCount(AbstractTemplate): + key = "MaskedType.count" + + def generic(self, args, kws): + return nb_signature( + MaskedType(size_type), MaskedType(string_view), recvr=self.this + ) + + +class MaskedStringViewReplace(AbstractTemplate): + key = "MaskedType.replace" + + def generic(self, args, kws): + return nb_signature( + MaskedType(udf_string), + MaskedType(string_view), + MaskedType(string_view), + recvr=self.this, + ) + + +class MaskedStringViewAttrs(AttributeTemplate): + key = MaskedType(string_view) + + def resolve_replace(self, mod): + return types.BoundFunction( + MaskedStringViewReplace, MaskedType(string_view) + ) + + def resolve_count(self, mod): + return types.BoundFunction( + MaskedStringViewCount, MaskedType(string_view) + ) + + def resolve_value(self, mod): + return string_view + + def resolve_valid(self, mod): + return types.boolean + + +# Build attributes for `MaskedType(string_view)` +for func in bool_binary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_binary_attr(f"MaskedType.{func}", types.boolean), + ) + +for func in int_binary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_binary_attr(f"MaskedType.{func}", size_type), + ) + +for func in string_return_attrs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_binary_attr(f"MaskedType.{func}", udf_string), + ) + +for func in id_unary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_unary_attr(f"MaskedType.{func}", types.boolean), + ) + +for func in string_unary_funcs: + setattr( + MaskedStringViewAttrs, + f"resolve_{func}", + create_masked_unary_attr(f"MaskedType.{func}", udf_string), + ) + +cuda_decl_registry.register_attr(MaskedStringViewAttrs) diff --git a/python/cudf/cudf/core/udf/row_function.py b/python/cudf/cudf/core/udf/row_function.py index 8d887a37706..bfb04e38e7d 100644 --- a/python/cudf/cudf/core/udf/row_function.py +++ b/python/cudf/cudf/core/udf/row_function.py @@ -1,6 +1,5 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import math -from typing import Any, Dict import numpy as np from numba import cuda @@ -9,6 +8,7 @@ from cudf.core.udf.api import Masked, pack_return from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import string_view from cudf.core.udf.templates import ( masked_input_initializer_template, row_initializer_template, @@ -18,6 +18,7 @@ from cudf.core.udf.utils import ( _all_dtypes_from_frame, _construct_signature, + _get_extensionty_size, _get_kernel, _get_udf_return_type, _mask_get, @@ -25,8 +26,6 @@ _supported_dtypes_from_frame, ) -itemsizes: Dict[Any, int] = {} - def _get_frame_row_type(dtype): """ @@ -47,8 +46,12 @@ def _get_frame_row_type(dtype): offset = 0 sizes = [ - itemsizes.get(val[0], val[0].itemsize) for val in dtype.fields.values() + _get_extensionty_size(string_view) + if val[0] == np.dtype("O") + else val[0].itemsize + for val in dtype.fields.values() ] + for i, (name, info) in enumerate(dtype.fields.items()): # *info* consists of the element dtype, its offset from the beginning # of the record, and an optional "title" containing metadata. @@ -56,7 +59,13 @@ def _get_frame_row_type(dtype): # instead, we compute the correct offset based on the masked type. elemdtype = info[0] title = info[2] if len(info) == 3 else None - ty = numpy_support.from_dtype(elemdtype) + + ty = ( + # columns of dtype string start life as string_view + string_view + if elemdtype == np.dtype("O") + else numpy_support.from_dtype(elemdtype) + ) infos = { "type": MaskedType(ty), "offset": offset, @@ -65,7 +74,11 @@ def _get_frame_row_type(dtype): fields.append((name, infos)) # increment offset by itemsize plus one byte for validity - itemsize = itemsizes.get(elemdtype, elemdtype.itemsize) + itemsize = ( + _get_extensionty_size(string_view) + if elemdtype == np.dtype("O") + else elemdtype.itemsize + ) offset += itemsize + 1 # Align the next member of the struct to be a multiple of the diff --git a/python/cudf/cudf/core/udf/scalar_function.py b/python/cudf/cudf/core/udf/scalar_function.py index 31599f4151e..ff7fad3fb82 100644 --- a/python/cudf/cudf/core/udf/scalar_function.py +++ b/python/cudf/cudf/core/udf/scalar_function.py @@ -1,10 +1,11 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from numba import cuda from numba.np import numpy_support from cudf.core.udf.api import Masked, pack_return from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import string_view from cudf.core.udf.templates import ( masked_input_initializer_template, scalar_kernel_template, @@ -48,7 +49,9 @@ def f(x, c, k): def _get_scalar_kernel(sr, func, args): - sr_type = MaskedType(numpy_support.from_dtype(sr.dtype)) + sr_type = MaskedType( + string_view if sr.dtype == "O" else numpy_support.from_dtype(sr.dtype) + ) scalar_return_type = _get_udf_return_type(sr_type, func, args) sig = _construct_signature(sr, scalar_return_type, args=args) diff --git a/python/cudf/cudf/core/udf/strings_lowering.py b/python/cudf/cudf/core/udf/strings_lowering.py index ec956cdd65d..00905f72cda 100644 --- a/python/cudf/cudf/core/udf/strings_lowering.py +++ b/python/cudf/cudf/core/udf/strings_lowering.py @@ -1,38 +1,521 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import operator +from functools import partial -from numba import types +from numba import cuda, types from numba.core import cgutils +from numba.core.datamodel import default_manager from numba.core.typing import signature as nb_signature -from numba.cuda.cudaimpl import lower as cuda_lower - -from strings_udf._typing import size_type, string_view, udf_string -from strings_udf.lowering import ( - contains_impl, - count_impl, - endswith_impl, - find_impl, - isalnum_impl, - isalpha_impl, - isdecimal_impl, - isdigit_impl, - islower_impl, - isspace_impl, - istitle_impl, - isupper_impl, - len_impl, - lower_impl, - lstrip_impl, - replace_impl, - rfind_impl, - rstrip_impl, - startswith_impl, - strip_impl, - upper_impl, +from numba.cuda.cudadrv import nvvm +from numba.cuda.cudaimpl import ( + lower as cuda_lower, + registry as cuda_lowering_registry, ) +from cudf._lib.strings_udf import ( + get_character_cases_table_ptr, + get_character_flags_table_ptr, + get_special_case_mapping_table_ptr, +) from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import size_type, string_view, udf_string + +_STR_VIEW_PTR = types.CPointer(string_view) +_UDF_STRING_PTR = types.CPointer(udf_string) + + +# CUDA function declarations +# read-only (input is a string_view, output is a fixed with type) +_string_view_len = cuda.declare_device("len", size_type(_STR_VIEW_PTR)) +_concat_string_view = cuda.declare_device( + "concat", types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) +) + +_string_view_replace = cuda.declare_device( + "replace", + types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), +) + + +def _declare_binary_func(lhs, rhs, out, name): + # Declare a binary function + return cuda.declare_device( + name, + out(lhs, rhs), + ) + + +def _declare_strip_func(name): + return cuda.declare_device( + name, size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) + ) + + +# A binary function of the form f(string, string) -> bool +_declare_bool_str_str_func = partial( + _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, types.boolean +) + +_declare_size_type_str_str_func = partial( + _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, size_type +) + +_string_view_contains = _declare_bool_str_str_func("contains") +_string_view_eq = _declare_bool_str_str_func("eq") +_string_view_ne = _declare_bool_str_str_func("ne") +_string_view_ge = _declare_bool_str_str_func("ge") +_string_view_le = _declare_bool_str_str_func("le") +_string_view_gt = _declare_bool_str_str_func("gt") +_string_view_lt = _declare_bool_str_str_func("lt") +_string_view_startswith = _declare_bool_str_str_func("startswith") +_string_view_endswith = _declare_bool_str_str_func("endswith") +_string_view_find = _declare_size_type_str_str_func("find") +_string_view_rfind = _declare_size_type_str_str_func("rfind") +_string_view_contains = _declare_bool_str_str_func("contains") +_string_view_strip = _declare_strip_func("strip") +_string_view_lstrip = _declare_strip_func("lstrip") +_string_view_rstrip = _declare_strip_func("rstrip") + + +# A binary function of the form f(string, int) -> bool +_declare_bool_str_int_func = partial( + _declare_binary_func, _STR_VIEW_PTR, types.int64, types.boolean +) + + +def _declare_upper_or_lower(func): + return cuda.declare_device( + func, + types.void( + _UDF_STRING_PTR, + _STR_VIEW_PTR, + types.uintp, + types.uintp, + types.uintp, + ), + ) + + +_string_view_isdigit = _declare_bool_str_int_func("pyisdigit") +_string_view_isalnum = _declare_bool_str_int_func("pyisalnum") +_string_view_isalpha = _declare_bool_str_int_func("pyisalpha") +_string_view_isdecimal = _declare_bool_str_int_func("pyisdecimal") +_string_view_isnumeric = _declare_bool_str_int_func("pyisnumeric") +_string_view_isspace = _declare_bool_str_int_func("pyisspace") +_string_view_isupper = _declare_bool_str_int_func("pyisupper") +_string_view_islower = _declare_bool_str_int_func("pyislower") +_string_view_istitle = _declare_bool_str_int_func("pyistitle") +_string_view_upper = _declare_upper_or_lower("upper") +_string_view_lower = _declare_upper_or_lower("lower") + + +_string_view_count = cuda.declare_device( + "pycount", + size_type(_STR_VIEW_PTR, _STR_VIEW_PTR), +) + + +# casts +@cuda_lowering_registry.lower_cast(types.StringLiteral, string_view) +def cast_string_literal_to_string_view(context, builder, fromty, toty, val): + """ + Cast a literal to a string_view + """ + # create an empty string_view + sv = cgutils.create_struct_proxy(string_view)(context, builder) + + # set the empty strview data pointer to point to the literal value + s = context.insert_const_string(builder.module, fromty.literal_value) + sv.data = context.insert_addrspace_conv( + builder, s, nvvm.ADDRSPACE_CONSTANT + ) + sv.length = context.get_constant(size_type, len(fromty.literal_value)) + sv.bytes = context.get_constant( + size_type, len(fromty.literal_value.encode("UTF-8")) + ) + + return sv._getvalue() + + +@cuda_lowering_registry.lower_cast(string_view, udf_string) +def cast_string_view_to_udf_string(context, builder, fromty, toty, val): + sv_ptr = builder.alloca(default_manager[fromty].get_value_type()) + udf_str_ptr = builder.alloca(default_manager[toty].get_value_type()) + builder.store(val, sv_ptr) + _ = context.compile_internal( + builder, + call_create_udf_string_from_string_view, + nb_signature(types.void, _STR_VIEW_PTR, types.CPointer(udf_string)), + (sv_ptr, udf_str_ptr), + ) + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + + return result._getvalue() + + +# utilities +_create_udf_string_from_string_view = cuda.declare_device( + "udf_string_from_string_view", + types.void(types.CPointer(string_view), types.CPointer(udf_string)), +) + + +def call_create_udf_string_from_string_view(sv, udf_str): + _create_udf_string_from_string_view(sv, udf_str) + + +# String function implementations +def call_len_string_view(st): + return _string_view_len(st) + + +@cuda_lower(len, string_view) +def len_impl(context, builder, sig, args): + sv_ptr = builder.alloca(args[0].type) + builder.store(args[0], sv_ptr) + result = context.compile_internal( + builder, + call_len_string_view, + nb_signature(size_type, _STR_VIEW_PTR), + (sv_ptr,), + ) + + return result + + +def call_concat_string_view(result, lhs, rhs): + return _concat_string_view(result, lhs, rhs) + + +@cuda_lower(operator.add, string_view, string_view) +def concat_impl(context, builder, sig, args): + lhs_ptr = builder.alloca(args[0].type) + rhs_ptr = builder.alloca(args[1].type) + builder.store(args[0], lhs_ptr) + builder.store(args[1], rhs_ptr) + + udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) + _ = context.compile_internal( + builder, + call_concat_string_view, + types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), + (udf_str_ptr, lhs_ptr, rhs_ptr), + ) + + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + +def call_string_view_replace(result, src, to_replace, replacement): + return _string_view_replace(result, src, to_replace, replacement) + + +@cuda_lower("StringView.replace", string_view, string_view, string_view) +def replace_impl(context, builder, sig, args): + src_ptr = builder.alloca(args[0].type) + to_replace_ptr = builder.alloca(args[1].type) + replacement_ptr = builder.alloca(args[2].type) + + builder.store(args[0], src_ptr) + builder.store(args[1], to_replace_ptr), + builder.store(args[2], replacement_ptr) + + udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) + + _ = context.compile_internal( + builder, + call_string_view_replace, + types.void( + _UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR + ), + (udf_str_ptr, src_ptr, to_replace_ptr, replacement_ptr), + ) + + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + +def create_binary_string_func(binary_func, retty): + """ + Provide a wrapper around numba's low-level extension API which + produces the boilerplate needed to implement a binary function + of two strings. + """ + + def deco(cuda_func): + @cuda_lower(binary_func, string_view, string_view) + def binary_func_impl(context, builder, sig, args): + lhs_ptr = builder.alloca(args[0].type) + rhs_ptr = builder.alloca(args[1].type) + builder.store(args[0], lhs_ptr) + builder.store(args[1], rhs_ptr) + + # these conditional statements should compile out + if retty != udf_string: + # binary function of two strings yielding a fixed-width type + # example: str.startswith(other) -> bool + # shim functions can return the value through nb_retval + result = context.compile_internal( + builder, + cuda_func, + nb_signature(retty, _STR_VIEW_PTR, _STR_VIEW_PTR), + (lhs_ptr, rhs_ptr), + ) + return result + else: + # binary function of two strings yielding a new string + # example: str.strip(other) -> str + # shim functions can not return a struct due to C linkage + # so we create a new udf_string and pass a pointer to it + # for the shim function to write the output to. The return + # value of compile_internal is therefore discarded (although + # this may change in the future if we need to return error + # codes, for instance). + udf_str_ptr = builder.alloca( + default_manager[udf_string].get_value_type() + ) + _ = context.compile_internal( + builder, + cuda_func, + size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), + (udf_str_ptr, lhs_ptr, rhs_ptr), + ) + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + return binary_func_impl + + return deco + + +@create_binary_string_func(operator.contains, types.boolean) +def contains_impl(st, substr): + return _string_view_contains(st, substr) + + +@create_binary_string_func(operator.eq, types.boolean) +def eq_impl(st, rhs): + return _string_view_eq(st, rhs) + + +@create_binary_string_func(operator.ne, types.boolean) +def ne_impl(st, rhs): + return _string_view_ne(st, rhs) + + +@create_binary_string_func(operator.ge, types.boolean) +def ge_impl(st, rhs): + return _string_view_ge(st, rhs) + + +@create_binary_string_func(operator.le, types.boolean) +def le_impl(st, rhs): + return _string_view_le(st, rhs) + + +@create_binary_string_func(operator.gt, types.boolean) +def gt_impl(st, rhs): + return _string_view_gt(st, rhs) + + +@create_binary_string_func(operator.lt, types.boolean) +def lt_impl(st, rhs): + return _string_view_lt(st, rhs) + + +@create_binary_string_func("StringView.strip", udf_string) +def strip_impl(result, to_strip, strip_char): + return _string_view_strip(result, to_strip, strip_char) + + +@create_binary_string_func("StringView.lstrip", udf_string) +def lstrip_impl(result, to_strip, strip_char): + return _string_view_lstrip(result, to_strip, strip_char) + + +@create_binary_string_func("StringView.rstrip", udf_string) +def rstrip_impl(result, to_strip, strip_char): + return _string_view_rstrip(result, to_strip, strip_char) + + +@create_binary_string_func("StringView.startswith", types.boolean) +def startswith_impl(sv, substr): + return _string_view_startswith(sv, substr) + + +@create_binary_string_func("StringView.endswith", types.boolean) +def endswith_impl(sv, substr): + return _string_view_endswith(sv, substr) + + +@create_binary_string_func("StringView.count", size_type) +def count_impl(st, substr): + return _string_view_count(st, substr) + + +@create_binary_string_func("StringView.find", size_type) +def find_impl(sv, substr): + return _string_view_find(sv, substr) + + +@create_binary_string_func("StringView.rfind", size_type) +def rfind_impl(sv, substr): + return _string_view_rfind(sv, substr) + + +def create_unary_identifier_func(id_func): + """ + Provide a wrapper around numba's low-level extension API which + produces the boilerplate needed to implement a unary function + of a string. + """ + + def deco(cuda_func): + @cuda_lower(id_func, string_view) + def id_func_impl(context, builder, sig, args): + str_ptr = builder.alloca(args[0].type) + builder.store(args[0], str_ptr) + + # Lookup table required for conversion functions + # must be resolved at runtime after context initialization, + # therefore cannot be a global variable + tbl_ptr = context.get_constant( + types.uintp, get_character_flags_table_ptr() + ) + result = context.compile_internal( + builder, + cuda_func, + nb_signature(types.boolean, _STR_VIEW_PTR, types.uintp), + (str_ptr, tbl_ptr), + ) + + return result + + return id_func_impl + + return deco + + +def create_upper_or_lower(id_func): + """ + Provide a wrapper around numba's low-level extension API which + produces the boilerplate needed to implement either the upper + or lower attrs of a string view. + """ + + def deco(cuda_func): + @cuda_lower(id_func, string_view) + def id_func_impl(context, builder, sig, args): + str_ptr = builder.alloca(args[0].type) + builder.store(args[0], str_ptr) + + # Lookup table required for conversion functions + # must be resolved at runtime after context initialization, + # therefore cannot be a global variable + flags_tbl_ptr = context.get_constant( + types.uintp, get_character_flags_table_ptr() + ) + cases_tbl_ptr = context.get_constant( + types.uintp, get_character_cases_table_ptr() + ) + special_tbl_ptr = context.get_constant( + types.uintp, get_special_case_mapping_table_ptr() + ) + udf_str_ptr = builder.alloca( + default_manager[udf_string].get_value_type() + ) + + _ = context.compile_internal( + builder, + cuda_func, + types.void( + _UDF_STRING_PTR, + _STR_VIEW_PTR, + types.uintp, + types.uintp, + types.uintp, + ), + ( + udf_str_ptr, + str_ptr, + flags_tbl_ptr, + cases_tbl_ptr, + special_tbl_ptr, + ), + ) + + result = cgutils.create_struct_proxy(udf_string)( + context, builder, value=builder.load(udf_str_ptr) + ) + return result._getvalue() + + return id_func_impl + + return deco + + +@create_upper_or_lower("StringView.upper") +def upper_impl(result, st, flags, cases, special): + return _string_view_upper(result, st, flags, cases, special) + + +@create_upper_or_lower("StringView.lower") +def lower_impl(result, st, flags, cases, special): + return _string_view_lower(result, st, flags, cases, special) + + +@create_unary_identifier_func("StringView.isdigit") +def isdigit_impl(st, tbl): + return _string_view_isdigit(st, tbl) + + +@create_unary_identifier_func("StringView.isalnum") +def isalnum_impl(st, tbl): + return _string_view_isalnum(st, tbl) + + +@create_unary_identifier_func("StringView.isalpha") +def isalpha_impl(st, tbl): + return _string_view_isalpha(st, tbl) + + +@create_unary_identifier_func("StringView.isnumeric") +def isnumeric_impl(st, tbl): + return _string_view_isnumeric(st, tbl) + + +@create_unary_identifier_func("StringView.isdecimal") +def isdecimal_impl(st, tbl): + return _string_view_isdecimal(st, tbl) + + +@create_unary_identifier_func("StringView.isspace") +def isspace_impl(st, tbl): + return _string_view_isspace(st, tbl) + + +@create_unary_identifier_func("StringView.isupper") +def isupper_impl(st, tbl): + return _string_view_isupper(st, tbl) + + +@create_unary_identifier_func("StringView.islower") +def islower_impl(st, tbl): + return _string_view_islower(st, tbl) + + +@create_unary_identifier_func("StringView.istitle") +def istitle_impl(st, tbl): + return _string_view_istitle(st, tbl) @cuda_lower(len, MaskedType(string_view)) @@ -85,7 +568,7 @@ def masked_string_view_replace_impl(context, builder, sig, args): return ret._getvalue() -def create_binary_string_func(op, cuda_func, retty): +def create_masked_binary_string_func(op, cuda_func, retty): """ Provide a wrapper around numba's low-level extension API which produces the boilerplate needed to implement a binary function @@ -165,19 +648,23 @@ def upper_or_lower_impl(context, builder, sig, args): cuda_lower(op, MaskedType(string_view))(upper_or_lower_impl) -create_binary_string_func("MaskedType.strip", strip_impl, udf_string) -create_binary_string_func("MaskedType.lstrip", lstrip_impl, udf_string) -create_binary_string_func("MaskedType.rstrip", rstrip_impl, udf_string) -create_binary_string_func( +create_masked_binary_string_func("MaskedType.strip", strip_impl, udf_string) +create_masked_binary_string_func("MaskedType.lstrip", lstrip_impl, udf_string) +create_masked_binary_string_func("MaskedType.rstrip", rstrip_impl, udf_string) +create_masked_binary_string_func( "MaskedType.startswith", startswith_impl, types.boolean, ) -create_binary_string_func("MaskedType.endswith", endswith_impl, types.boolean) -create_binary_string_func("MaskedType.find", find_impl, size_type) -create_binary_string_func("MaskedType.rfind", rfind_impl, size_type) -create_binary_string_func("MaskedType.count", count_impl, size_type) -create_binary_string_func(operator.contains, contains_impl, types.boolean) +create_masked_binary_string_func( + "MaskedType.endswith", endswith_impl, types.boolean +) +create_masked_binary_string_func("MaskedType.find", find_impl, size_type) +create_masked_binary_string_func("MaskedType.rfind", rfind_impl, size_type) +create_masked_binary_string_func("MaskedType.count", count_impl, size_type) +create_masked_binary_string_func( + operator.contains, contains_impl, types.boolean +) create_masked_unary_identifier_func("MaskedType.isalnum", isalnum_impl) diff --git a/python/cudf/cudf/core/udf/strings_typing.py b/python/cudf/cudf/core/udf/strings_typing.py index e373b8b018d..7c8429d9997 100644 --- a/python/cudf/cudf/core/udf/strings_typing.py +++ b/python/cudf/cudf/core/udf/strings_typing.py @@ -1,226 +1,273 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import operator +import numpy as np from numba import types +from numba.core.extending import models, register_model from numba.core.typing import signature as nb_signature from numba.core.typing.templates import AbstractTemplate, AttributeTemplate from numba.cuda.cudadecl import registry as cuda_decl_registry -from strings_udf._typing import ( - StringView, - bool_binary_funcs, - id_unary_funcs, - int_binary_funcs, - size_type, - string_return_attrs, - string_unary_funcs, - string_view, - udf_string, -) +import rmm -from cudf.core.udf import masked_typing -from cudf.core.udf._ops import comparison_ops -from cudf.core.udf.masked_typing import MaskedType +# libcudf size_type +size_type = types.int32 -masked_typing.MASKED_INIT_MAP[types.pyobject] = string_view -masked_typing.MASKED_INIT_MAP[string_view] = string_view +# String object definitions +class UDFString(types.Type): -def _is_valid_string_arg(ty): - return ( - isinstance(ty, MaskedType) and isinstance(ty.value_type, StringView) - ) or isinstance(ty, types.StringLiteral) + np_dtype = np.dtype("object") + def __init__(self): + super().__init__(name="udf_string") -def register_string_function(func): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to associate a signature with a function or - operator to be overloaded. - """ + @property + def return_type(self): + return self - def deco(generic): - class MaskedStringFunction(AbstractTemplate): - pass - MaskedStringFunction.generic = generic - cuda_decl_registry.register_global(func)(MaskedStringFunction) +class StringView(types.Type): - return deco + np_dtype = np.dtype("object") + def __init__(self): + super().__init__(name="string_view") -@register_string_function(len) -def len_typing(self, args, kws): - if isinstance(args[0], MaskedType) and isinstance( - args[0].value_type, StringView - ): - return nb_signature(MaskedType(size_type), args[0]) - elif isinstance(args[0], types.StringLiteral) and len(args) == 1: - return nb_signature(size_type, args[0]) + @property + def return_type(self): + return UDFString() -@register_string_function(operator.add) -def concat_typing(self, args, kws): - if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): - return nb_signature( - MaskedType(udf_string), - MaskedType(string_view), - MaskedType(string_view), - ) +@register_model(StringView) +class stringview_model(models.StructModel): + # from string_view.hpp: + _members = ( + # const char* _data{} + # Pointer to device memory contain char array for this string + ("data", types.CPointer(types.char)), + # size_type _bytes{}; + # Number of bytes in _data for this string + ("bytes", size_type), + # mutable size_type _length{}; + # Number of characters in this string (computed) + ("length", size_type), + ) + def __init__(self, dmm, fe_type): + super().__init__(dmm, fe_type, self._members) -@register_string_function(operator.contains) -def contains_typing(self, args, kws): - if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): - return nb_signature( - MaskedType(types.boolean), - MaskedType(string_view), - MaskedType(string_view), - ) + +@register_model(UDFString) +class udf_string_model(models.StructModel): + # from udf_string.hpp: + # private: + # char* m_data{}; + # cudf::size_type m_bytes{}; + # cudf::size_type m_size{}; + + _members = ( + ("m_data", types.CPointer(types.char)), + ("m_bytes", size_type), + ("m_size", size_type), + ) + + def __init__(self, dmm, fe_type): + super().__init__(dmm, fe_type, self._members) -class MaskedStringViewCmpOp(AbstractTemplate): +any_string_ty = (StringView, UDFString, types.StringLiteral) +string_view = StringView() +udf_string = UDFString() + + +class StrViewArgHandler: """ - return the boolean result of `cmpop` between to strings - since the typing is the same for every comparison operator, - we can reuse this class for all of them. + As part of Numba's preprocessing step, incoming function arguments are + modified based on the associated type for that argument that was used + to JIT the kernel. However it only knows how to handle built in array + types natively. With string UDFs, the jitted type is string_view*, + which numba does not know how to handle. + + This class converts string_view* to raw pointer arguments, which Numba + knows how to use. + + See numba.cuda.compiler._prepare_args for details. + """ + + def prepare_args(self, ty, val, **kwargs): + if isinstance(ty, types.CPointer) and isinstance( + ty.dtype, (StringView, UDFString) + ): + return types.uint64, val.ptr if isinstance( + val, rmm._lib.device_buffer.DeviceBuffer + ) else val.get_ptr(mode="read") + else: + return ty, val + + +str_view_arg_handler = StrViewArgHandler() + + +# String functions +@cuda_decl_registry.register_global(len) +class StringLength(AbstractTemplate): + """ + provide the length of a cudf::string_view like struct """ def generic(self, args, kws): - if _is_valid_string_arg(args[0]) and _is_valid_string_arg(args[1]): - return nb_signature( - MaskedType(types.boolean), - MaskedType(string_view), - MaskedType(string_view), - ) + if isinstance(args[0], any_string_ty) and len(args) == 1: + # length: + # string_view -> int32 + # udf_string -> int32 + # literal -> int32 + return nb_signature(size_type, args[0]) -for op in comparison_ops: - cuda_decl_registry.register_global(op)(MaskedStringViewCmpOp) +def register_stringview_binaryop(op, retty): + """ + Helper function wrapping numba's low level extension API. Provides + the boilerplate needed to associate a signature with a function or + operator expecting a string. + """ + + class StringViewBinaryOp(AbstractTemplate): + def generic(self, args, kws): + if isinstance(args[0], any_string_ty) and isinstance( + args[1], any_string_ty + ): + return nb_signature(retty, string_view, string_view) + cuda_decl_registry.register_global(op)(StringViewBinaryOp) -def create_masked_binary_attr(attrname, retty): + +def create_binary_attr(attrname, retty): """ Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a binary function of two masked - string objects as an attribute of one, e.g. `string.func(other)`. + the boilerplate needed to register a binary function of two string + objects as an attribute of one, e.g. `string.func(other)`. """ - class MaskedStringViewBinaryAttr(AbstractTemplate): - key = attrname + class StringViewBinaryAttr(AbstractTemplate): + key = f"StringView.{attrname}" def generic(self, args, kws): - return nb_signature( - MaskedType(retty), MaskedType(string_view), recvr=self.this - ) + return nb_signature(retty, string_view, recvr=self.this) def attr(self, mod): - return types.BoundFunction( - MaskedStringViewBinaryAttr, - MaskedType(string_view), - ) + return types.BoundFunction(StringViewBinaryAttr, string_view) return attr -def create_masked_unary_attr(attrname, retty): +def create_identifier_attr(attrname, retty): """ Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a unary function of a masked - string object as an attribute, e.g. `string.func()`. + the boilerplate needed to register a unary function of a string + object as an attribute, e.g. `string.func()`. """ - class MaskedStringViewIdentifierAttr(AbstractTemplate): - key = attrname + class StringViewIdentifierAttr(AbstractTemplate): + key = f"StringView.{attrname}" def generic(self, args, kws): - return nb_signature(MaskedType(retty), recvr=self.this) + return nb_signature(retty, recvr=self.this) def attr(self, mod): - return types.BoundFunction( - MaskedStringViewIdentifierAttr, - MaskedType(string_view), - ) + return types.BoundFunction(StringViewIdentifierAttr, string_view) return attr -class MaskedStringViewCount(AbstractTemplate): - key = "MaskedType.count" +class StringViewCount(AbstractTemplate): + key = "StringView.count" def generic(self, args, kws): - return nb_signature( - MaskedType(size_type), MaskedType(string_view), recvr=self.this - ) + return nb_signature(size_type, string_view, recvr=self.this) -class MaskedStringViewReplace(AbstractTemplate): - key = "MaskedType.replace" +class StringViewReplace(AbstractTemplate): + key = "StringView.replace" def generic(self, args, kws): return nb_signature( - MaskedType(udf_string), - MaskedType(string_view), - MaskedType(string_view), - recvr=self.this, + udf_string, string_view, string_view, recvr=self.this ) -class MaskedStringViewAttrs(AttributeTemplate): - key = MaskedType(string_view) - - def resolve_replace(self, mod): - return types.BoundFunction( - MaskedStringViewReplace, MaskedType(string_view) - ) +class StringViewAttrs(AttributeTemplate): + key = string_view def resolve_count(self, mod): - return types.BoundFunction( - MaskedStringViewCount, MaskedType(string_view) - ) - - def resolve_value(self, mod): - return string_view - - def resolve_valid(self, mod): - return types.boolean + return types.BoundFunction(StringViewCount, string_view) + def resolve_replace(self, mod): + return types.BoundFunction(StringViewReplace, string_view) + + +bool_binary_funcs = ["startswith", "endswith"] +int_binary_funcs = ["find", "rfind"] +id_unary_funcs = [ + "isalpha", + "isalnum", + "isdecimal", + "isdigit", + "isupper", + "islower", + "isspace", + "isnumeric", + "istitle", +] +string_unary_funcs = ["upper", "lower"] +string_return_attrs = ["strip", "lstrip", "rstrip"] -# Build attributes for `MaskedType(string_view)` for func in bool_binary_funcs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_binary_attr(f"MaskedType.{func}", types.boolean), + create_binary_attr(func, types.boolean), ) -for func in int_binary_funcs: +for func in string_return_attrs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_binary_attr(f"MaskedType.{func}", size_type), + create_binary_attr(func, udf_string), ) -for func in string_return_attrs: + +for func in int_binary_funcs: setattr( - MaskedStringViewAttrs, - f"resolve_{func}", - create_masked_binary_attr(f"MaskedType.{func}", udf_string), + StringViewAttrs, f"resolve_{func}", create_binary_attr(func, size_type) ) for func in id_unary_funcs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_unary_attr(f"MaskedType.{func}", types.boolean), + create_identifier_attr(func, types.boolean), ) for func in string_unary_funcs: setattr( - MaskedStringViewAttrs, + StringViewAttrs, f"resolve_{func}", - create_masked_unary_attr(f"MaskedType.{func}", udf_string), + create_identifier_attr(func, udf_string), ) -cuda_decl_registry.register_attr(MaskedStringViewAttrs) +cuda_decl_registry.register_attr(StringViewAttrs) + +register_stringview_binaryop(operator.eq, types.boolean) +register_stringview_binaryop(operator.ne, types.boolean) +register_stringview_binaryop(operator.lt, types.boolean) +register_stringview_binaryop(operator.gt, types.boolean) +register_stringview_binaryop(operator.le, types.boolean) +register_stringview_binaryop(operator.ge, types.boolean) + +# st in other +register_stringview_binaryop(operator.contains, types.boolean) + +# st + other +register_stringview_binaryop(operator.add, udf_string) diff --git a/python/cudf/cudf/core/udf/utils.py b/python/cudf/cudf/core/udf/utils.py index 3ee1d8edcbd..edc1a16353f 100644 --- a/python/cudf/cudf/core/udf/utils.py +++ b/python/cudf/cudf/core/udf/utils.py @@ -2,13 +2,14 @@ import glob import os -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict import cachetools import cupy as cp import llvmlite.binding as ll import numpy as np from cubinlinker.patch import _numba_version_ok, get_logger, new_patched_linker +from cuda import cudart from numba import cuda, typeof from numba.core.datamodel import default_manager from numba.core.errors import TypingError @@ -19,32 +20,105 @@ import rmm +from cudf._lib.strings_udf import ( + column_from_udf_string_array, + column_to_string_view_array, +) from cudf.core.column.column import as_column +from cudf.core.dtypes import dtype from cudf.core.udf.masked_typing import MaskedType +from cudf.core.udf.strings_typing import ( + str_view_arg_handler, + string_view, + udf_string, +) from cudf.utils import cudautils from cudf.utils.dtypes import ( BOOL_TYPES, DATETIME_TYPES, NUMERIC_TYPES, + STRING_TYPES, TIMEDELTA_TYPES, ) -from cudf.utils.utils import _cudf_nvtx_annotate +from cudf.utils.utils import _cudf_nvtx_annotate, initfunc + +# Maximum size of a string column is 2 GiB +_STRINGS_UDF_DEFAULT_HEAP_SIZE = os.environ.get( + "STRINGS_UDF_HEAP_SIZE", 2**31 +) +_heap_size = 0 +_cudf_str_dtype = dtype(str) + logger = get_logger() JIT_SUPPORTED_TYPES = ( - NUMERIC_TYPES | BOOL_TYPES | DATETIME_TYPES | TIMEDELTA_TYPES + NUMERIC_TYPES + | BOOL_TYPES + | DATETIME_TYPES + | TIMEDELTA_TYPES + | STRING_TYPES ) libcudf_bitmask_type = numpy_support.from_dtype(np.dtype("int32")) MASK_BITSIZE = np.dtype("int32").itemsize * 8 precompiled: cachetools.LRUCache = cachetools.LRUCache(maxsize=32) -arg_handlers: List[Any] = [] -ptx_files: List[Any] = [] -masked_array_types: Dict[Any, Any] = {} launch_arg_getters: Dict[Any, Any] = {} -output_col_getters: Dict[Any, Any] = {} + + +def _get_best_ptx_file(archs, max_compute_capability): + """ + Determine of the available PTX files which one is + the most recent up to and including the device cc + """ + filtered_archs = [x for x in archs if x[0] <= max_compute_capability] + if filtered_archs: + return max(filtered_archs, key=lambda y: y[0]) + else: + return None + + +def _get_ptx_file(path, prefix): + if "RAPIDS_NO_INITIALIZE" in os.environ: + # cc=60 ptx is always built + cc = int(os.environ.get("STRINGS_UDF_CC", "60")) + else: + dev = cuda.get_current_device() + + # Load the highest compute capability file available that is less than + # the current device's. + cc = int("".join(str(x) for x in dev.compute_capability)) + files = glob.glob(os.path.join(path, f"{prefix}*.ptx")) + if len(files) == 0: + raise RuntimeError(f"Missing PTX files for cc={cc}") + regular_sms = [] + + for f in files: + file_name = os.path.basename(f) + sm_number = file_name.rstrip(".ptx").lstrip(prefix) + if sm_number.endswith("a"): + processed_sm_number = int(sm_number.rstrip("a")) + if processed_sm_number == cc: + return f + else: + regular_sms.append((int(sm_number), f)) + + regular_result = None + + if regular_sms: + regular_result = _get_best_ptx_file(regular_sms, cc) + + if regular_result is None: + raise RuntimeError( + "This cuDF installation is missing the necessary PTX " + f"files that are <={cc}." + ) + else: + return regular_result[1] + + +_PTX_FILE = _get_ptx_file(os.path.dirname(__file__), "shim_") @_cudf_nvtx_annotate @@ -129,9 +203,8 @@ def _masked_array_type_from_col(col): array of bools representing a mask. """ - col_type = masked_array_types.get(col.dtype) - if col_type: - col_type = CPointer(col_type) + if col.dtype == _cudf_str_dtype: + col_type = CPointer(string_view) else: nb_scalar_ty = numpy_support.from_dtype(col.dtype) col_type = nb_scalar_ty[::1] @@ -239,7 +312,9 @@ def _get_kernel(kernel_string, globals_, sig, func): globals_["f_"] = f_ exec(kernel_string, globals_) _kernel = globals_["_kernel"] - kernel = cuda.jit(sig, link=ptx_files, extensions=arg_handlers)(_kernel) + kernel = cuda.jit( + sig, link=[_PTX_FILE], extensions=[str_view_arg_handler] + )(_kernel) return kernel @@ -248,9 +323,8 @@ def _get_input_args_from_frame(fr): args = [] offsets = [] for col in _supported_cols_from_frame(fr).values(): - getter = launch_arg_getters.get(col.dtype) - if getter: - data = getter(col) + if col.dtype == _cudf_str_dtype: + data = column_to_string_view_array_init_heap(col) else: data = col.data if col.mask is not None: @@ -264,69 +338,18 @@ def _get_input_args_from_frame(fr): return args + offsets -def _return_arr_from_dtype(dt, size): - if extensionty := masked_array_types.get(dt): - return rmm.DeviceBuffer(size=size * extensionty.return_type.size_bytes) - return cp.empty(size, dtype=dt) +def _return_arr_from_dtype(dtype, size): + if dtype == _cudf_str_dtype: + return rmm.DeviceBuffer(size=size * _get_extensionty_size(udf_string)) + return cp.empty(size, dtype=dtype) def _post_process_output_col(col, retty): - if getter := output_col_getters.get(retty): - col = getter(col) + if retty == _cudf_str_dtype: + return column_from_udf_string_array(col) return as_column(col, retty) -def _get_best_ptx_file(archs, max_compute_capability): - """ - Determine of the available PTX files which one is - the most recent up to and including the device cc - """ - filtered_archs = [x for x in archs if x[0] <= max_compute_capability] - if filtered_archs: - return max(filtered_archs, key=lambda y: y[0]) - else: - return None - - -def _get_ptx_file(path, prefix): - if "RAPIDS_NO_INITIALIZE" in os.environ: - # cc=60 ptx is always built - cc = int(os.environ.get("STRINGS_UDF_CC", "60")) - else: - dev = cuda.get_current_device() - - # Load the highest compute capability file available that is less than - # the current device's. - cc = int("".join(str(x) for x in dev.compute_capability)) - files = glob.glob(os.path.join(path, f"{prefix}*.ptx")) - if len(files) == 0: - raise RuntimeError(f"Missing PTX files for cc={cc}") - regular_sms = [] - - for f in files: - file_name = os.path.basename(f) - sm_number = file_name.rstrip(".ptx").lstrip(prefix) - if sm_number.endswith("a"): - processed_sm_number = int(sm_number.rstrip("a")) - if processed_sm_number == cc: - return f - else: - regular_sms.append((int(sm_number), f)) - - regular_result = None - - if regular_sms: - regular_result = _get_best_ptx_file(regular_sms, cc) - - if regular_result is None: - raise RuntimeError( - "This cuDF installation is missing the necessary PTX " - f"files that are <={cc}." - ) - else: - return regular_result[1] - - def _get_extensionty_size(ty): """ Return the size of an extension type in bytes @@ -420,3 +443,26 @@ def maybe_patch_numba_linker( Linker.new = new_patched_linker else: logger.debug("Cannot patch Numba Linker - unsupported version") + + +@initfunc +def set_malloc_heap_size(size=None): + """ + Heap size control for strings_udf, size in bytes. + """ + global _heap_size + if size is None: + size = _STRINGS_UDF_DEFAULT_HEAP_SIZE + if size != _heap_size: + (ret,) = cudart.cudaDeviceSetLimit( + cudart.cudaLimit.cudaLimitMallocHeapSize, size + ) + if ret.value != 0: + raise RuntimeError("Unable to set cudaMalloc heap size") + + _heap_size = size + + +def column_to_string_view_array_init_heap(col): + # lazily allocate heap only when a string needs to be returned + return column_to_string_view_array(col) diff --git a/python/strings_udf/strings_udf/tests/test_string_udfs.py b/python/cudf/cudf/tests/test_string_udfs.py similarity index 94% rename from python/strings_udf/strings_udf/tests/test_string_udfs.py rename to python/cudf/cudf/tests/test_string_udfs.py index b8de821e101..a8de63be0f5 100644 --- a/python/strings_udf/strings_udf/tests/test_string_udfs.py +++ b/python/cudf/cudf/tests/test_string_udfs.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. import numba import numpy as np @@ -8,16 +8,20 @@ from numba.core.typing import signature as nb_signature from numba.types import CPointer, void -import cudf import rmm -from cudf.testing._utils import assert_eq -import strings_udf -from strings_udf._lib.cudf_jit_udf import ( +import cudf +from cudf._lib.strings_udf import ( column_from_udf_string_array, column_to_string_view_array, ) -from strings_udf._typing import str_view_arg_handler, string_view, udf_string +from cudf.core.udf.strings_typing import ( + str_view_arg_handler, + string_view, + udf_string, +) +from cudf.core.udf.utils import _PTX_FILE, _get_extensionty_size +from cudf.testing._utils import assert_eq def get_kernel(func, dtype, size): @@ -36,9 +40,7 @@ def get_kernel(func, dtype, size): outty = numba.np.numpy_support.from_dtype(dtype)[::1] sig = nb_signature(void, CPointer(string_view), outty) - @cuda.jit( - sig, link=[strings_udf.ptxpath], extensions=[str_view_arg_handler] - ) + @cuda.jit(sig, link=[_PTX_FILE], extensions=[str_view_arg_handler]) def kernel(input_strings, output_col): id = cuda.grid(1) if id < size: @@ -59,7 +61,9 @@ def run_udf_test(data, func, dtype): comparing it with the equivalent pandas result """ if dtype == "str": - output = rmm.DeviceBuffer(size=len(data) * udf_string.size_bytes) + output = rmm.DeviceBuffer( + size=len(data) * _get_extensionty_size(udf_string) + ) else: dtype = np.dtype(dtype) output = cudf.core.column.column_empty(len(data), dtype=dtype) diff --git a/python/cudf/cudf/tests/test_udf_masked_ops.py b/python/cudf/cudf/tests/test_udf_masked_ops.py index 091f44176ab..4d54c3181b2 100644 --- a/python/cudf/cudf/tests/test_udf_masked_ops.py +++ b/python/cudf/cudf/tests/test_udf_masked_ops.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import math import operator @@ -8,7 +8,6 @@ import cudf from cudf.core.missing import NA -from cudf.core.udf import _STRING_UDFS_ENABLED from cudf.core.udf._ops import ( arith_ops, bitwise_ops, @@ -757,8 +756,6 @@ def func(x): run_masked_udf_series(func, data, check_dtype=False) -# only run string udf tests if library exists and is enabled -@pytest.mark.skipif(not _STRING_UDFS_ENABLED, reason="String UDFs not enabled") class TestStringUDFs: def test_string_udf_len(self, str_udf_data): def func(row): diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index 8a648097ac8..59dd5d0179e 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. # See the docstring in versioneer.py for instructions. Note that you must # re-run 'versioneer.py setup' after changing this section, and commit the @@ -25,7 +25,6 @@ known_dask= dask_cuda known_rapids= rmm - strings_udf known_first_party= cudf default_section=THIRDPARTY diff --git a/python/strings_udf/cpp/CMakeLists.txt b/python/cudf/udf_cpp/CMakeLists.txt similarity index 75% rename from python/strings_udf/cpp/CMakeLists.txt rename to python/cudf/udf_cpp/CMakeLists.txt index 3e58d10d6e2..0c07236682f 100644 --- a/python/strings_udf/cpp/CMakeLists.txt +++ b/python/cudf/udf_cpp/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -16,38 +16,28 @@ cmake_minimum_required(VERSION 3.23.1) include(rapids-cmake) include(rapids-cpm) -include(rapids-cuda) include(rapids-find) -rapids_cuda_init_architectures(strings-udf-cpp) - -# Create a project so that we can enable CUDA architectures in this file. -project( - strings-udf-cpp - VERSION ${strings_udf_version} - LANGUAGES CUDA -) - rapids_cpm_init() rapids_find_package( CUDAToolkit REQUIRED - BUILD_EXPORT_SET strings-udf-exports - INSTALL_EXPORT_SET strings-udf-exports + BUILD_EXPORT_SET udf-exports + INSTALL_EXPORT_SET udf-exports ) include(${rapids-cmake-dir}/cpm/libcudacxx.cmake) -rapids_cpm_libcudacxx(BUILD_EXPORT_SET strings-udf-exports INSTALL_EXPORT_SET strings-udf-exports) +rapids_cpm_libcudacxx(BUILD_EXPORT_SET udf-exports INSTALL_EXPORT_SET udf-exports) -add_library(cudf_strings_udf SHARED src/strings/udf/udf_apis.cu) +add_library(cudf_strings_udf SHARED strings/src/strings/udf/udf_apis.cu) target_include_directories( - cudf_strings_udf PUBLIC "$" + cudf_strings_udf PUBLIC "$" ) set_target_properties( cudf_strings_udf - PROPERTIES BUILD_RPATH "\$ORIGIN" - INSTALL_RPATH "\$ORIGIN" + PROPERTIES BUILD_RPATH "\$ORIGIN/../" + INSTALL_RPATH "\$ORIGIN/../" CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON CUDA_STANDARD 17 @@ -62,8 +52,8 @@ target_compile_options( cudf_strings_udf PRIVATE "$<$:${UDF_CXX_FLAGS}>" "$<$:${UDF_CUDA_FLAGS}>" ) -target_link_libraries(cudf_strings_udf PUBLIC cudf::cudf CUDA::nvrtc) -install(TARGETS cudf_strings_udf DESTINATION ./strings_udf/_lib/) +target_link_libraries(cudf_strings_udf PUBLIC cudf::cudf) +install(TARGETS cudf_strings_udf DESTINATION ./cudf/_lib/) # This function will copy the generated PTX file from its generator-specific location in the build # tree into a specified location in the build tree from which we can install it. @@ -104,18 +94,20 @@ list(REMOVE_DUPLICATES CMAKE_CUDA_ARCHITECTURES) foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) set(tgt shim_${arch}) - add_library(${tgt} OBJECT src/strings/udf/shim.cu) + add_library(${tgt} OBJECT shim.cu) set_target_properties(${tgt} PROPERTIES CUDA_ARCHITECTURES ${arch} CUDA_PTX_COMPILATION ON) - target_include_directories(${tgt} PUBLIC include) + target_include_directories( + ${tgt} PUBLIC "$" + ) target_compile_options(${tgt} PRIVATE "$<$:${SHIM_CUDA_FLAGS}>") target_link_libraries(${tgt} PUBLIC cudf::cudf) - copy_ptx_to_location(${tgt} "${CMAKE_CURRENT_BINARY_DIR}/../strings_udf" ${tgt}.ptx) + copy_ptx_to_location(${tgt} "${CMAKE_CURRENT_BINARY_DIR}/../udf" ${tgt}.ptx) install( FILES $ - DESTINATION ./strings_udf + DESTINATION ./cudf/core/udf/ RENAME ${tgt}.ptx ) endforeach() diff --git a/python/cudf/udf_cpp/groupby/CMakeLists.txt b/python/cudf/udf_cpp/groupby/CMakeLists.txt deleted file mode 100644 index 043ab28f362..00000000000 --- a/python/cudf/udf_cpp/groupby/CMakeLists.txt +++ /dev/null @@ -1,79 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.23.1) - -include(rapids-find) - -# This function will copy the generated PTX file from its generator-specific location in the build -# tree into a specified location in the build tree from which we can install it. -function(copy_ptx_to_location target destination) - set(cmake_generated_file - "${CMAKE_CURRENT_BINARY_DIR}/cmake/cp_${target}_$>_ptx.cmake" - ) - file( - GENERATE - OUTPUT "${cmake_generated_file}" - CONTENT - " -set(ptx_paths \"$\") -file(COPY_FILE \${ptx_paths} \"${destination}/${target}.ptx\")" - ) - - add_custom_target( - ${target}_cp_ptx ALL - COMMAND ${CMAKE_COMMAND} -P "${cmake_generated_file}" - DEPENDS $ - COMMENT "Copying PTX files to '${destination}'" - ) -endfunction() - -# Create the shim library for each architecture. -set(GROUPBY_FUNCTION_CUDA_FLAGS --expt-relaxed-constexpr) - -# always build a default PTX file in case RAPIDS_NO_INITIALIZE is set and the device cc can't be -# safely queried through a context -list(INSERT CMAKE_CUDA_ARCHITECTURES 0 "60") - -list(TRANSFORM CMAKE_CUDA_ARCHITECTURES REPLACE "-real" "") -list(TRANSFORM CMAKE_CUDA_ARCHITECTURES REPLACE "-virtual" "") -list(SORT CMAKE_CUDA_ARCHITECTURES) -list(REMOVE_DUPLICATES CMAKE_CUDA_ARCHITECTURES) - -foreach(arch IN LISTS CMAKE_CUDA_ARCHITECTURES) - set(tgt function_${arch}) - - add_library(${tgt} OBJECT function.cu) - set_target_properties( - ${tgt} - PROPERTIES CUDA_STANDARD 17 - CUDA_STANDARD_REQUIRED ON - CUDA_ARCHITECTURES ${arch} - CUDA_PTX_COMPILATION ON - CUDA_SEPARABLE_COMPILATION ON - ) - - target_include_directories(${tgt} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) - target_compile_options( - ${tgt} PRIVATE "$<$:${GROUPBY_FUNCTION_CUDA_FLAGS}>" - ) - target_link_libraries(${tgt} PUBLIC cudf::cudf) - - copy_ptx_to_location(${tgt} "${CMAKE_CURRENT_BINARY_DIR}/") - install( - FILES $ - DESTINATION ./cudf/core/udf/ - RENAME ${tgt}.ptx - ) -endforeach() diff --git a/python/cudf/udf_cpp/groupby/function.cu b/python/cudf/udf_cpp/groupby/function.cu deleted file mode 100644 index 782371b8a44..00000000000 --- a/python/cudf/udf_cpp/groupby/function.cu +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (c) 2022-2023, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include - -#include - -#include -#include - -template -__device__ bool are_all_nans(cooperative_groups::thread_block const& block, - T const* data, - int64_t size) -{ - // TODO: to be refactored with CG vote functions once - // block size is known at build time - __shared__ int64_t count; - - if (block.thread_rank() == 0) { count = 0; } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - if (not std::isnan(data[idx])) { - cuda::atomic_ref ref{count}; - ref.fetch_add(1, cuda::std::memory_order_relaxed); - break; - } - } - - block.sync(); - return count == 0; -} - -template -__device__ void device_sum(cooperative_groups::thread_block const& block, - T const* data, - int64_t size, - T* sum) -{ - T local_sum = 0; - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - local_sum += data[idx]; - } - - cuda::atomic_ref ref{*sum}; - ref.fetch_add(local_sum, cuda::std::memory_order_relaxed); - - block.sync(); -} - -template -__device__ T BlockSum(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - if constexpr (std::is_floating_point_v) { - if (are_all_nans(block, data, size)) { return 0; } - } - - __shared__ T block_sum; - if (block.thread_rank() == 0) { block_sum = 0; } - block.sync(); - - device_sum(block, data, size, &block_sum); - return block_sum; -} - -template -__device__ double BlockMean(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ T block_sum; - if (block.thread_rank() == 0) { block_sum = 0; } - block.sync(); - - device_sum(block, data, size, &block_sum); - return static_cast(block_sum) / static_cast(size); -} - -template -__device__ double BlockVar(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ double block_var; - __shared__ T block_sum; - if (block.thread_rank() == 0) { - block_var = 0; - block_sum = 0; - } - block.sync(); - - T local_sum = 0; - double local_var = 0; - - device_sum(block, data, size, &block_sum); - - auto const mean = static_cast(block_sum) / static_cast(size); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - auto const delta = static_cast(data[idx]) - mean; - local_var += delta * delta; - } - - cuda::atomic_ref ref{block_var}; - ref.fetch_add(local_var, cuda::std::memory_order_relaxed); - block.sync(); - - if (block.thread_rank() == 0) { block_var = block_var / static_cast(size - 1); } - block.sync(); - return block_var; -} - -template -__device__ double BlockStd(T const* data, int64_t size) -{ - auto const var = BlockVar(data, size); - return sqrt(var); -} - -template -__device__ T BlockMax(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - if constexpr (std::is_floating_point_v) { - if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } - } - - auto local_max = cudf::DeviceMax::identity(); - __shared__ T block_max; - if (block.thread_rank() == 0) { block_max = local_max; } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - local_max = max(local_max, data[idx]); - } - - cuda::atomic_ref ref{block_max}; - ref.fetch_max(local_max, cuda::std::memory_order_relaxed); - - block.sync(); - - return block_max; -} - -template -__device__ T BlockMin(T const* data, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - if constexpr (std::is_floating_point_v) { - if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } - } - - auto local_min = cudf::DeviceMin::identity(); - - __shared__ T block_min; - if (block.thread_rank() == 0) { block_min = local_min; } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - local_min = min(local_min, data[idx]); - } - - cuda::atomic_ref ref{block_min}; - ref.fetch_min(local_min, cuda::std::memory_order_relaxed); - - block.sync(); - - return block_min; -} - -template -__device__ int64_t BlockIdxMax(T const* data, int64_t* index, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ T block_max; - __shared__ int64_t block_idx_max; - __shared__ bool found_max; - - auto local_max = cudf::DeviceMax::identity(); - auto local_idx_max = cudf::DeviceMin::identity(); - - if (block.thread_rank() == 0) { - block_max = local_max; - block_idx_max = local_idx_max; - found_max = false; - } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - auto const current_data = data[idx]; - if (current_data > local_max) { - local_max = current_data; - local_idx_max = index[idx]; - found_max = true; - } - } - - cuda::atomic_ref ref{block_max}; - ref.fetch_max(local_max, cuda::std::memory_order_relaxed); - block.sync(); - - if (found_max) { - if (local_max == block_max) { - cuda::atomic_ref ref_idx{block_idx_max}; - ref_idx.fetch_min(local_idx_max, cuda::std::memory_order_relaxed); - } - } else { - if (block.thread_rank() == 0) { block_idx_max = index[0]; } - } - block.sync(); - - return block_idx_max; -} - -template -__device__ int64_t BlockIdxMin(T const* data, int64_t* index, int64_t size) -{ - auto block = cooperative_groups::this_thread_block(); - - __shared__ T block_min; - __shared__ int64_t block_idx_min; - __shared__ bool found_min; - - auto local_min = cudf::DeviceMin::identity(); - auto local_idx_min = cudf::DeviceMin::identity(); - - if (block.thread_rank() == 0) { - block_min = local_min; - block_idx_min = local_idx_min; - found_min = false; - } - block.sync(); - - for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { - auto const current_data = data[idx]; - if (current_data < local_min) { - local_min = current_data; - local_idx_min = index[idx]; - found_min = true; - } - } - - cuda::atomic_ref ref{block_min}; - ref.fetch_min(local_min, cuda::std::memory_order_relaxed); - block.sync(); - - if (found_min) { - if (local_min == block_min) { - cuda::atomic_ref ref_idx{block_idx_min}; - ref_idx.fetch_min(local_idx_min, cuda::std::memory_order_relaxed); - } - } else { - if (block.thread_rank() == 0) { block_idx_min = index[0]; } - } - block.sync(); - - return block_idx_min; -} - -extern "C" { -#define make_definition(name, cname, type, return_type) \ - __device__ int name##_##cname(return_type* numba_return_value, type* const data, int64_t size) \ - { \ - return_type const res = name(data, size); \ - *numba_return_value = res; \ - __syncthreads(); \ - return 0; \ - } - -make_definition(BlockSum, int64, int64_t, int64_t); -make_definition(BlockSum, float64, double, double); -make_definition(BlockMean, int64, int64_t, double); -make_definition(BlockMean, float64, double, double); -make_definition(BlockStd, int64, int64_t, double); -make_definition(BlockStd, float64, double, double); -make_definition(BlockVar, int64, int64_t, double); -make_definition(BlockVar, float64, double, double); -make_definition(BlockMin, int64, int64_t, int64_t); -make_definition(BlockMin, float64, double, double); -make_definition(BlockMax, int64, int64_t, int64_t); -make_definition(BlockMax, float64, double, double); -#undef make_definition -} - -extern "C" { -#define make_definition_idx(name, cname, type) \ - __device__ int name##_##cname( \ - int64_t* numba_return_value, type* const data, int64_t* index, int64_t size) \ - { \ - auto const res = name(data, index, size); \ - *numba_return_value = res; \ - __syncthreads(); \ - return 0; \ - } - -make_definition_idx(BlockIdxMin, int64, int64_t); -make_definition_idx(BlockIdxMin, float64, double); -make_definition_idx(BlockIdxMax, int64, int64_t); -make_definition_idx(BlockIdxMax, float64, double); -#undef make_definition_idx -} diff --git a/python/strings_udf/cpp/src/strings/udf/shim.cu b/python/cudf/udf_cpp/shim.cu similarity index 55% rename from python/strings_udf/cpp/src/strings/udf/shim.cu rename to python/cudf/udf_cpp/shim.cu index d10cc635209..9223a54654e 100644 --- a/python/strings_udf/cpp/src/strings/udf/shim.cu +++ b/python/cudf/udf_cpp/shim.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,7 @@ * limitations under the License. */ +#include #include #include #include @@ -22,6 +23,13 @@ #include #include +#include + +#include + +#include +#include + using namespace cudf::strings::udf; extern "C" __device__ int len(int* nb_retval, void const* str) @@ -343,3 +351,303 @@ extern "C" __device__ int replace( return 0; } + +// Groupby Shim Functions +template +__device__ bool are_all_nans(cooperative_groups::thread_block const& block, + T const* data, + int64_t size) +{ + // TODO: to be refactored with CG vote functions once + // block size is known at build time + __shared__ int64_t count; + + if (block.thread_rank() == 0) { count = 0; } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + if (not std::isnan(data[idx])) { + cuda::atomic_ref ref{count}; + ref.fetch_add(1, cuda::std::memory_order_relaxed); + break; + } + } + + block.sync(); + return count == 0; +} + +template +__device__ void device_sum(cooperative_groups::thread_block const& block, + T const* data, + int64_t size, + T* sum) +{ + T local_sum = 0; + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + local_sum += data[idx]; + } + + cuda::atomic_ref ref{*sum}; + ref.fetch_add(local_sum, cuda::std::memory_order_relaxed); + + block.sync(); +} + +template +__device__ T BlockSum(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + if constexpr (std::is_floating_point_v) { + if (are_all_nans(block, data, size)) { return 0; } + } + + __shared__ T block_sum; + if (block.thread_rank() == 0) { block_sum = 0; } + block.sync(); + + device_sum(block, data, size, &block_sum); + return block_sum; +} + +template +__device__ double BlockMean(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ T block_sum; + if (block.thread_rank() == 0) { block_sum = 0; } + block.sync(); + + device_sum(block, data, size, &block_sum); + return static_cast(block_sum) / static_cast(size); +} + +template +__device__ double BlockVar(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ double block_var; + __shared__ T block_sum; + if (block.thread_rank() == 0) { + block_var = 0; + block_sum = 0; + } + block.sync(); + + T local_sum = 0; + double local_var = 0; + + device_sum(block, data, size, &block_sum); + + auto const mean = static_cast(block_sum) / static_cast(size); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + auto const delta = static_cast(data[idx]) - mean; + local_var += delta * delta; + } + + cuda::atomic_ref ref{block_var}; + ref.fetch_add(local_var, cuda::std::memory_order_relaxed); + block.sync(); + + if (block.thread_rank() == 0) { block_var = block_var / static_cast(size - 1); } + block.sync(); + return block_var; +} + +template +__device__ double BlockStd(T const* data, int64_t size) +{ + auto const var = BlockVar(data, size); + return sqrt(var); +} + +template +__device__ T BlockMax(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + if constexpr (std::is_floating_point_v) { + if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } + } + + auto local_max = cudf::DeviceMax::identity(); + __shared__ T block_max; + if (block.thread_rank() == 0) { block_max = local_max; } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + local_max = max(local_max, data[idx]); + } + + cuda::atomic_ref ref{block_max}; + ref.fetch_max(local_max, cuda::std::memory_order_relaxed); + + block.sync(); + + return block_max; +} + +template +__device__ T BlockMin(T const* data, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + if constexpr (std::is_floating_point_v) { + if (are_all_nans(block, data, size)) { return std::numeric_limits::quiet_NaN(); } + } + + auto local_min = cudf::DeviceMin::identity(); + + __shared__ T block_min; + if (block.thread_rank() == 0) { block_min = local_min; } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + local_min = min(local_min, data[idx]); + } + + cuda::atomic_ref ref{block_min}; + ref.fetch_min(local_min, cuda::std::memory_order_relaxed); + + block.sync(); + + return block_min; +} + +template +__device__ int64_t BlockIdxMax(T const* data, int64_t* index, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ T block_max; + __shared__ int64_t block_idx_max; + __shared__ bool found_max; + + auto local_max = cudf::DeviceMax::identity(); + auto local_idx_max = cudf::DeviceMin::identity(); + + if (block.thread_rank() == 0) { + block_max = local_max; + block_idx_max = local_idx_max; + found_max = false; + } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + auto const current_data = data[idx]; + if (current_data > local_max) { + local_max = current_data; + local_idx_max = index[idx]; + found_max = true; + } + } + + cuda::atomic_ref ref{block_max}; + ref.fetch_max(local_max, cuda::std::memory_order_relaxed); + block.sync(); + + if (found_max) { + if (local_max == block_max) { + cuda::atomic_ref ref_idx{block_idx_max}; + ref_idx.fetch_min(local_idx_max, cuda::std::memory_order_relaxed); + } + } else { + if (block.thread_rank() == 0) { block_idx_max = index[0]; } + } + block.sync(); + + return block_idx_max; +} + +template +__device__ int64_t BlockIdxMin(T const* data, int64_t* index, int64_t size) +{ + auto block = cooperative_groups::this_thread_block(); + + __shared__ T block_min; + __shared__ int64_t block_idx_min; + __shared__ bool found_min; + + auto local_min = cudf::DeviceMin::identity(); + auto local_idx_min = cudf::DeviceMin::identity(); + + if (block.thread_rank() == 0) { + block_min = local_min; + block_idx_min = local_idx_min; + found_min = false; + } + block.sync(); + + for (int64_t idx = block.thread_rank(); idx < size; idx += block.size()) { + auto const current_data = data[idx]; + if (current_data < local_min) { + local_min = current_data; + local_idx_min = index[idx]; + found_min = true; + } + } + + cuda::atomic_ref ref{block_min}; + ref.fetch_min(local_min, cuda::std::memory_order_relaxed); + block.sync(); + + if (found_min) { + if (local_min == block_min) { + cuda::atomic_ref ref_idx{block_idx_min}; + ref_idx.fetch_min(local_idx_min, cuda::std::memory_order_relaxed); + } + } else { + if (block.thread_rank() == 0) { block_idx_min = index[0]; } + } + block.sync(); + + return block_idx_min; +} + +extern "C" { +#define make_definition(name, cname, type, return_type) \ + __device__ int name##_##cname(return_type* numba_return_value, type* const data, int64_t size) \ + { \ + return_type const res = name(data, size); \ + *numba_return_value = res; \ + __syncthreads(); \ + return 0; \ + } + +make_definition(BlockSum, int64, int64_t, int64_t); +make_definition(BlockSum, float64, double, double); +make_definition(BlockMean, int64, int64_t, double); +make_definition(BlockMean, float64, double, double); +make_definition(BlockStd, int64, int64_t, double); +make_definition(BlockStd, float64, double, double); +make_definition(BlockVar, int64, int64_t, double); +make_definition(BlockVar, float64, double, double); +make_definition(BlockMin, int64, int64_t, int64_t); +make_definition(BlockMin, float64, double, double); +make_definition(BlockMax, int64, int64_t, int64_t); +make_definition(BlockMax, float64, double, double); +#undef make_definition +} + +extern "C" { +#define make_definition_idx(name, cname, type) \ + __device__ int name##_##cname( \ + int64_t* numba_return_value, type* const data, int64_t* index, int64_t size) \ + { \ + auto const res = name(data, index, size); \ + *numba_return_value = res; \ + __syncthreads(); \ + return 0; \ + } + +make_definition_idx(BlockIdxMin, int64, int64_t); +make_definition_idx(BlockIdxMin, float64, double); +make_definition_idx(BlockIdxMax, int64, int64_t); +make_definition_idx(BlockIdxMax, float64, double); +#undef make_definition_idx +} diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/case.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/case.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/case.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/case.cuh index 472101959a6..76477087e71 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/case.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/case.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/char_types.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/char_types.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/char_types.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/char_types.cuh index 9320686442b..d03de2c4290 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/char_types.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/char_types.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/numeric.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/numeric.cuh similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/numeric.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/numeric.cuh index c8c9f6e46f4..ecd952a2307 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/numeric.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/numeric.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/pad.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/pad.cuh similarity index 98% rename from python/strings_udf/cpp/include/cudf/strings/udf/pad.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/pad.cuh index d6d4ed637e9..2f3b4fe0298 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/pad.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/pad.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/replace.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/replace.cuh similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/replace.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/replace.cuh index c1f0cdc94c5..6d2135291fd 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/replace.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/replace.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/search.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/search.cuh similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/search.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/search.cuh index ef15886f1f5..3300c859dba 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/search.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/search.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/split.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/split.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/split.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/split.cuh index ca31425aa62..d3131464121 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/split.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/split.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/starts_with.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/starts_with.cuh similarity index 98% rename from python/strings_udf/cpp/include/cudf/strings/udf/starts_with.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/starts_with.cuh index 38c609ae505..1c7beac71f5 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/starts_with.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/starts_with.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/strip.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/strip.cuh similarity index 98% rename from python/strings_udf/cpp/include/cudf/strings/udf/strip.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/strip.cuh index f2db3073460..521b578bc52 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/strip.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/strip.cuh @@ -1,6 +1,6 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/udf_apis.hpp b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_apis.hpp similarity index 97% rename from python/strings_udf/cpp/include/cudf/strings/udf/udf_apis.hpp rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_apis.hpp index 68834afa082..219dbe27682 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/udf_apis.hpp +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_apis.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.cuh b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.cuh similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/udf_string.cuh rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.cuh index 724116143d8..a8970c04b74 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.cuh +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.hpp b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.hpp similarity index 99% rename from python/strings_udf/cpp/include/cudf/strings/udf/udf_string.hpp rename to python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.hpp index 2bbda357cee..59cb35df419 100644 --- a/python/strings_udf/cpp/include/cudf/strings/udf/udf_string.hpp +++ b/python/cudf/udf_cpp/strings/include/cudf/strings/udf/udf_string.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/cpp/src/strings/udf/udf_apis.cu b/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu similarity index 98% rename from python/strings_udf/cpp/src/strings/udf/udf_apis.cu rename to python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu index 3e6491e32e7..bedaa8e8fff 100644 --- a/python/strings_udf/cpp/src/strings/udf/udf_apis.cu +++ b/python/cudf/udf_cpp/strings/src/strings/udf/udf_apis.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/python/strings_udf/CMakeLists.txt b/python/strings_udf/CMakeLists.txt deleted file mode 100644 index 2507116f957..00000000000 --- a/python/strings_udf/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -cmake_minimum_required(VERSION 3.23.1 FATAL_ERROR) - -set(strings_udf_version 23.04.00) - -include(../../fetch_rapids.cmake) - -include(rapids-cuda) -rapids_cuda_init_architectures(strings-udf-python) - -project( - strings-udf-python - VERSION ${strings_udf_version} - LANGUAGES CXX - # TODO: Building Python extension modules via the python_extension_module requires the C - # language to be enabled here. The test project that is built in scikit-build to verify - # various linking options for the python library is hardcoded to build with C, so until - # that is fixed we need to keep C. - C - # TODO: Enabling CUDA will not be necessary once we upgrade to CMake 3.22, which will - # pull in the required languages for the C++ project even if this project does not - # require those languages. - CUDA -) - -find_package(cudf ${strings_udf_version} REQUIRED) - -add_subdirectory(cpp) - -include(rapids-cython) -rapids_cython_init() - -add_subdirectory(strings_udf/_lib) diff --git a/python/strings_udf/setup.cfg b/python/strings_udf/setup.cfg deleted file mode 100644 index 9f29b26b5e0..00000000000 --- a/python/strings_udf/setup.cfg +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = strings_udf/_version.py -versionfile_build = strings_udf/_version.py -tag_prefix = v -parentdir_prefix = strings_udf- - -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda -known_rapids= - rmm - cudf -known_first_party= - strings_udf -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - __init__.py diff --git a/python/strings_udf/setup.py b/python/strings_udf/setup.py deleted file mode 100644 index 5b4c503e032..00000000000 --- a/python/strings_udf/setup.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -import os -import re -import shutil - -import versioneer -from setuptools import find_packages -from skbuild import setup - -install_requires = ["numba>=0.53.1", "numpy", "cudf"] - -extras_require = { - "test": [ - "pytest", - ] -} - - -def get_cuda_version_from_header(cuda_include_dir, delimiter=""): - - cuda_version = None - - with open(os.path.join(cuda_include_dir, "cuda.h"), encoding="utf-8") as f: - for line in f.readlines(): - if re.search(r"#define CUDA_VERSION ", line) is not None: - cuda_version = line - break - - if cuda_version is None: - raise TypeError("CUDA_VERSION not found in cuda.h") - cuda_version = int(cuda_version.split()[2]) - return "%d%s%d" % ( - cuda_version // 1000, - delimiter, - (cuda_version % 1000) // 10, - ) - - -CUDA_HOME = os.environ.get("CUDA_HOME", False) -if not CUDA_HOME: - path_to_cuda_gdb = shutil.which("cuda-gdb") - if path_to_cuda_gdb is None: - raise OSError( - "Could not locate CUDA. " - "Please set the environment variable " - "CUDA_HOME to the path to the CUDA installation " - "and try again." - ) - CUDA_HOME = os.path.dirname(os.path.dirname(path_to_cuda_gdb)) - -if not os.path.isdir(CUDA_HOME): - raise OSError(f"Invalid CUDA_HOME: directory does not exist: {CUDA_HOME}") - -cuda_include_dir = os.path.join(CUDA_HOME, "include") - -setup( - name="strings_udf", - version=versioneer.get_version(), - description="Strings UDF Library", - url="https://github.com/rapidsai/cudf", - author="NVIDIA Corporation", - license="Apache 2.0", - classifiers=[ - "Intended Audience :: Developers", - "Topic :: Database", - "Topic :: Scientific/Engineering", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - ], - packages=find_packages(include=["strings_udf", "strings_udf.*"]), - package_data={ - key: ["*.pxd"] for key in find_packages(include=["strings_udf._lib*"]) - }, - cmdclass=versioneer.get_cmdclass(), - install_requires=install_requires, - extras_require=extras_require, - zip_safe=False, -) diff --git a/python/strings_udf/strings_udf/__init__.py b/python/strings_udf/strings_udf/__init__.py deleted file mode 100644 index 66c037125e6..00000000000 --- a/python/strings_udf/strings_udf/__init__.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -import os - -from cuda import cudart -from ptxcompiler.patch import NO_DRIVER, safe_get_versions - -from cudf.core.udf.utils import _get_cuda_version_from_ptx_file, _get_ptx_file - -from . import _version - -__version__ = _version.get_versions()["version"] - - -path = os.path.dirname(__file__) - - -# Maximum size of a string column is 2 GiB -_STRINGS_UDF_DEFAULT_HEAP_SIZE = os.environ.get( - "STRINGS_UDF_HEAP_SIZE", 2**31 -) -heap_size = 0 - - -def set_malloc_heap_size(size=None): - """ - Heap size control for strings_udf, size in bytes. - """ - global heap_size - if size is None: - size = _STRINGS_UDF_DEFAULT_HEAP_SIZE - if size != heap_size: - (ret,) = cudart.cudaDeviceSetLimit( - cudart.cudaLimit.cudaLimitMallocHeapSize, size - ) - if ret.value != 0: - raise RuntimeError("Unable to set cudaMalloc heap size") - - heap_size = size - - -ptxpath = None -versions = safe_get_versions() -if versions != NO_DRIVER: - ptxpath = _get_ptx_file(path, "shim_") diff --git a/python/strings_udf/strings_udf/_lib/CMakeLists.txt b/python/strings_udf/strings_udf/_lib/CMakeLists.txt deleted file mode 100644 index 55a33a050e0..00000000000 --- a/python/strings_udf/strings_udf/_lib/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -set(cython_sources cudf_jit_udf.pyx tables.pyx) -set(linked_libraries cudf::cudf cudf_strings_udf) -rapids_cython_create_modules( - CXX - SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" -) diff --git a/python/strings_udf/strings_udf/_lib/__init__.py b/python/strings_udf/strings_udf/_lib/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/strings_udf/strings_udf/_lib/cpp/__init__.pxd b/python/strings_udf/strings_udf/_lib/cpp/__init__.pxd deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/python/strings_udf/strings_udf/_lib/tables.pyx b/python/strings_udf/strings_udf/_lib/tables.pyx deleted file mode 100644 index 6442a34f63f..00000000000 --- a/python/strings_udf/strings_udf/_lib/tables.pyx +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -from libc.stdint cimport uint8_t, uint16_t, uintptr_t - -from strings_udf._lib.cpp.strings_udf cimport ( - get_character_cases_table as cpp_get_character_cases_table, - get_character_flags_table as cpp_get_character_flags_table, - get_special_case_mapping_table as cpp_get_special_case_mapping_table, -) - -import numpy as np - - -def get_character_flags_table_ptr(): - cdef const uint8_t* tbl_ptr = cpp_get_character_flags_table() - return np.uintp(tbl_ptr) - - -def get_character_cases_table_ptr(): - cdef const uint16_t* tbl_ptr = cpp_get_character_cases_table() - return np.uintp(tbl_ptr) - - -def get_special_case_mapping_table_ptr(): - cdef const void* tbl_ptr = cpp_get_special_case_mapping_table() - return np.uintp(tbl_ptr) diff --git a/python/strings_udf/strings_udf/_typing.py b/python/strings_udf/strings_udf/_typing.py deleted file mode 100644 index fa87ad63dc2..00000000000 --- a/python/strings_udf/strings_udf/_typing.py +++ /dev/null @@ -1,278 +0,0 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. - -import operator - -import numpy as np -from numba import types -from numba.core.extending import models, register_model -from numba.core.typing import signature as nb_signature -from numba.core.typing.templates import AbstractTemplate, AttributeTemplate -from numba.cuda.cudadecl import registry as cuda_decl_registry - -import rmm -from cudf.core.udf.utils import _get_extensionty_size - -# libcudf size_type -size_type = types.int32 - - -# String object definitions -class UDFString(types.Type): - - np_dtype = np.dtype("object") - - def __init__(self): - super().__init__(name="udf_string") - self.size_bytes = _get_extensionty_size(self) - - @property - def return_type(self): - return self - - -class StringView(types.Type): - - np_dtype = np.dtype("object") - - def __init__(self): - super().__init__(name="string_view") - self.size_bytes = _get_extensionty_size(self) - - @property - def return_type(self): - return UDFString() - - -@register_model(StringView) -class stringview_model(models.StructModel): - # from string_view.hpp: - _members = ( - # const char* _data{} - # Pointer to device memory contain char array for this string - ("data", types.CPointer(types.char)), - # size_type _bytes{}; - # Number of bytes in _data for this string - ("bytes", size_type), - # mutable size_type _length{}; - # Number of characters in this string (computed) - ("length", size_type), - ) - - def __init__(self, dmm, fe_type): - super().__init__(dmm, fe_type, self._members) - - -@register_model(UDFString) -class udf_string_model(models.StructModel): - # from udf_string.hpp: - # private: - # char* m_data{}; - # cudf::size_type m_bytes{}; - # cudf::size_type m_size{}; - - _members = ( - ("m_data", types.CPointer(types.char)), - ("m_bytes", size_type), - ("m_size", size_type), - ) - - def __init__(self, dmm, fe_type): - super().__init__(dmm, fe_type, self._members) - - -any_string_ty = (StringView, UDFString, types.StringLiteral) -string_view = StringView() -udf_string = UDFString() - - -class StrViewArgHandler: - """ - As part of Numba's preprocessing step, incoming function arguments are - modified based on the associated type for that argument that was used - to JIT the kernel. However it only knows how to handle built in array - types natively. With string UDFs, the jitted type is string_view*, - which numba does not know how to handle. - - This class converts string_view* to raw pointer arguments, which Numba - knows how to use. - - See numba.cuda.compiler._prepare_args for details. - """ - - def prepare_args(self, ty, val, **kwargs): - if isinstance(ty, types.CPointer) and isinstance( - ty.dtype, (StringView, UDFString) - ): - return types.uint64, val.ptr if isinstance( - val, rmm._lib.device_buffer.DeviceBuffer - ) else val.get_ptr(mode="read") - else: - return ty, val - - -str_view_arg_handler = StrViewArgHandler() - - -# String functions -@cuda_decl_registry.register_global(len) -class StringLength(AbstractTemplate): - """ - provide the length of a cudf::string_view like struct - """ - - def generic(self, args, kws): - if isinstance(args[0], any_string_ty) and len(args) == 1: - # length: - # string_view -> int32 - # udf_string -> int32 - # literal -> int32 - return nb_signature(size_type, args[0]) - - -def register_stringview_binaryop(op, retty): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to associate a signature with a function or - operator expecting a string. - """ - - class StringViewBinaryOp(AbstractTemplate): - def generic(self, args, kws): - if isinstance(args[0], any_string_ty) and isinstance( - args[1], any_string_ty - ): - return nb_signature(retty, string_view, string_view) - - cuda_decl_registry.register_global(op)(StringViewBinaryOp) - - -def create_binary_attr(attrname, retty): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a binary function of two string - objects as an attribute of one, e.g. `string.func(other)`. - """ - - class StringViewBinaryAttr(AbstractTemplate): - key = f"StringView.{attrname}" - - def generic(self, args, kws): - return nb_signature(retty, string_view, recvr=self.this) - - def attr(self, mod): - return types.BoundFunction(StringViewBinaryAttr, string_view) - - return attr - - -def create_identifier_attr(attrname, retty): - """ - Helper function wrapping numba's low level extension API. Provides - the boilerplate needed to register a unary function of a string - object as an attribute, e.g. `string.func()`. - """ - - class StringViewIdentifierAttr(AbstractTemplate): - key = f"StringView.{attrname}" - - def generic(self, args, kws): - return nb_signature(retty, recvr=self.this) - - def attr(self, mod): - return types.BoundFunction(StringViewIdentifierAttr, string_view) - - return attr - - -class StringViewCount(AbstractTemplate): - key = "StringView.count" - - def generic(self, args, kws): - return nb_signature(size_type, string_view, recvr=self.this) - - -class StringViewReplace(AbstractTemplate): - key = "StringView.replace" - - def generic(self, args, kws): - return nb_signature( - udf_string, string_view, string_view, recvr=self.this - ) - - -@cuda_decl_registry.register_attr -class StringViewAttrs(AttributeTemplate): - key = string_view - - def resolve_count(self, mod): - return types.BoundFunction(StringViewCount, string_view) - - def resolve_replace(self, mod): - return types.BoundFunction(StringViewReplace, string_view) - - -# Build attributes for `MaskedType(string_view)` -bool_binary_funcs = ["startswith", "endswith"] -int_binary_funcs = ["find", "rfind"] -id_unary_funcs = [ - "isalpha", - "isalnum", - "isdecimal", - "isdigit", - "isupper", - "islower", - "isspace", - "isnumeric", - "istitle", -] -string_unary_funcs = ["upper", "lower"] -string_return_attrs = ["strip", "lstrip", "rstrip"] - -for func in bool_binary_funcs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_binary_attr(func, types.boolean), - ) - -for func in string_return_attrs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_binary_attr(func, udf_string), - ) - - -for func in int_binary_funcs: - setattr( - StringViewAttrs, f"resolve_{func}", create_binary_attr(func, size_type) - ) - -for func in id_unary_funcs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_identifier_attr(func, types.boolean), - ) - -for func in string_unary_funcs: - setattr( - StringViewAttrs, - f"resolve_{func}", - create_identifier_attr(func, udf_string), - ) - -cuda_decl_registry.register_attr(StringViewAttrs) - -register_stringview_binaryop(operator.eq, types.boolean) -register_stringview_binaryop(operator.ne, types.boolean) -register_stringview_binaryop(operator.lt, types.boolean) -register_stringview_binaryop(operator.gt, types.boolean) -register_stringview_binaryop(operator.le, types.boolean) -register_stringview_binaryop(operator.ge, types.boolean) - -# st in other -register_stringview_binaryop(operator.contains, types.boolean) - -# st + other -register_stringview_binaryop(operator.add, udf_string) diff --git a/python/strings_udf/strings_udf/_version.py b/python/strings_udf/strings_udf/_version.py deleted file mode 100644 index 14ff9ec314d..00000000000 --- a/python/strings_udf/strings_udf/_version.py +++ /dev/null @@ -1,711 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.23 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import functools -import os -import re -import subprocess -import sys -from typing import Callable, Dict - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "strings_udf-" - cfg.versionfile_source = "strings_udf/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen( - [command] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - **popen_kwargs, - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r"\d", r): - continue - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - f"{tag_prefix}[[:digit:]]*", - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner( - GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root - ) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post( - pieces["closest-tag"] - ) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % ( - post_version + 1, - pieces["distance"], - ) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/python/strings_udf/strings_udf/lowering.py b/python/strings_udf/strings_udf/lowering.py deleted file mode 100644 index afc96a8380a..00000000000 --- a/python/strings_udf/strings_udf/lowering.py +++ /dev/null @@ -1,517 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -import operator -from functools import partial - -from numba import cuda, types -from numba.core import cgutils -from numba.core.datamodel import default_manager -from numba.core.typing import signature as nb_signature -from numba.cuda.cudadrv import nvvm -from numba.cuda.cudaimpl import ( - lower as cuda_lower, - registry as cuda_lowering_registry, -) - -from strings_udf._lib.tables import ( - get_character_cases_table_ptr, - get_character_flags_table_ptr, - get_special_case_mapping_table_ptr, -) -from strings_udf._typing import size_type, string_view, udf_string - -_STR_VIEW_PTR = types.CPointer(string_view) -_UDF_STRING_PTR = types.CPointer(udf_string) - - -# CUDA function declarations -# read-only (input is a string_view, output is a fixed with type) -_string_view_len = cuda.declare_device("len", size_type(_STR_VIEW_PTR)) -_concat_string_view = cuda.declare_device( - "concat", types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) -) - -_string_view_replace = cuda.declare_device( - "replace", - types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), -) - - -def _declare_binary_func(lhs, rhs, out, name): - # Declare a binary function - return cuda.declare_device( - name, - out(lhs, rhs), - ) - - -def _declare_strip_func(name): - return cuda.declare_device( - name, size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR) - ) - - -# A binary function of the form f(string, string) -> bool -_declare_bool_str_str_func = partial( - _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, types.boolean -) - -_declare_size_type_str_str_func = partial( - _declare_binary_func, _STR_VIEW_PTR, _STR_VIEW_PTR, size_type -) - -_string_view_contains = _declare_bool_str_str_func("contains") -_string_view_eq = _declare_bool_str_str_func("eq") -_string_view_ne = _declare_bool_str_str_func("ne") -_string_view_ge = _declare_bool_str_str_func("ge") -_string_view_le = _declare_bool_str_str_func("le") -_string_view_gt = _declare_bool_str_str_func("gt") -_string_view_lt = _declare_bool_str_str_func("lt") -_string_view_startswith = _declare_bool_str_str_func("startswith") -_string_view_endswith = _declare_bool_str_str_func("endswith") -_string_view_find = _declare_size_type_str_str_func("find") -_string_view_rfind = _declare_size_type_str_str_func("rfind") -_string_view_contains = _declare_bool_str_str_func("contains") -_string_view_strip = _declare_strip_func("strip") -_string_view_lstrip = _declare_strip_func("lstrip") -_string_view_rstrip = _declare_strip_func("rstrip") - - -# A binary function of the form f(string, int) -> bool -_declare_bool_str_int_func = partial( - _declare_binary_func, _STR_VIEW_PTR, types.int64, types.boolean -) - - -def _declare_upper_or_lower(func): - return cuda.declare_device( - func, - types.void( - _UDF_STRING_PTR, - _STR_VIEW_PTR, - types.uintp, - types.uintp, - types.uintp, - ), - ) - - -_string_view_isdigit = _declare_bool_str_int_func("pyisdigit") -_string_view_isalnum = _declare_bool_str_int_func("pyisalnum") -_string_view_isalpha = _declare_bool_str_int_func("pyisalpha") -_string_view_isdecimal = _declare_bool_str_int_func("pyisdecimal") -_string_view_isnumeric = _declare_bool_str_int_func("pyisnumeric") -_string_view_isspace = _declare_bool_str_int_func("pyisspace") -_string_view_isupper = _declare_bool_str_int_func("pyisupper") -_string_view_islower = _declare_bool_str_int_func("pyislower") -_string_view_istitle = _declare_bool_str_int_func("pyistitle") -_string_view_upper = _declare_upper_or_lower("upper") -_string_view_lower = _declare_upper_or_lower("lower") - - -_string_view_count = cuda.declare_device( - "pycount", - size_type(_STR_VIEW_PTR, _STR_VIEW_PTR), -) - - -# casts -@cuda_lowering_registry.lower_cast(types.StringLiteral, string_view) -def cast_string_literal_to_string_view(context, builder, fromty, toty, val): - """ - Cast a literal to a string_view - """ - # create an empty string_view - sv = cgutils.create_struct_proxy(string_view)(context, builder) - - # set the empty strview data pointer to point to the literal value - s = context.insert_const_string(builder.module, fromty.literal_value) - sv.data = context.insert_addrspace_conv( - builder, s, nvvm.ADDRSPACE_CONSTANT - ) - sv.length = context.get_constant(size_type, len(fromty.literal_value)) - sv.bytes = context.get_constant( - size_type, len(fromty.literal_value.encode("UTF-8")) - ) - - return sv._getvalue() - - -@cuda_lowering_registry.lower_cast(string_view, udf_string) -def cast_string_view_to_udf_string(context, builder, fromty, toty, val): - sv_ptr = builder.alloca(default_manager[fromty].get_value_type()) - udf_str_ptr = builder.alloca(default_manager[toty].get_value_type()) - builder.store(val, sv_ptr) - _ = context.compile_internal( - builder, - call_create_udf_string_from_string_view, - nb_signature(types.void, _STR_VIEW_PTR, types.CPointer(udf_string)), - (sv_ptr, udf_str_ptr), - ) - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - - return result._getvalue() - - -# utilities -_create_udf_string_from_string_view = cuda.declare_device( - "udf_string_from_string_view", - types.void(types.CPointer(string_view), types.CPointer(udf_string)), -) - - -def call_create_udf_string_from_string_view(sv, udf_str): - _create_udf_string_from_string_view(sv, udf_str) - - -# String function implementations -def call_len_string_view(st): - return _string_view_len(st) - - -@cuda_lower(len, string_view) -def len_impl(context, builder, sig, args): - sv_ptr = builder.alloca(args[0].type) - builder.store(args[0], sv_ptr) - result = context.compile_internal( - builder, - call_len_string_view, - nb_signature(size_type, _STR_VIEW_PTR), - (sv_ptr,), - ) - - return result - - -def call_concat_string_view(result, lhs, rhs): - return _concat_string_view(result, lhs, rhs) - - -@cuda_lower(operator.add, string_view, string_view) -def concat_impl(context, builder, sig, args): - lhs_ptr = builder.alloca(args[0].type) - rhs_ptr = builder.alloca(args[1].type) - builder.store(args[0], lhs_ptr) - builder.store(args[1], rhs_ptr) - - udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) - _ = context.compile_internal( - builder, - call_concat_string_view, - types.void(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), - (udf_str_ptr, lhs_ptr, rhs_ptr), - ) - - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - -def call_string_view_replace(result, src, to_replace, replacement): - return _string_view_replace(result, src, to_replace, replacement) - - -@cuda_lower("StringView.replace", string_view, string_view, string_view) -def replace_impl(context, builder, sig, args): - src_ptr = builder.alloca(args[0].type) - to_replace_ptr = builder.alloca(args[1].type) - replacement_ptr = builder.alloca(args[2].type) - - builder.store(args[0], src_ptr) - builder.store(args[1], to_replace_ptr), - builder.store(args[2], replacement_ptr) - - udf_str_ptr = builder.alloca(default_manager[udf_string].get_value_type()) - - _ = context.compile_internal( - builder, - call_string_view_replace, - types.void( - _UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR - ), - (udf_str_ptr, src_ptr, to_replace_ptr, replacement_ptr), - ) - - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - -def create_binary_string_func(binary_func, retty): - """ - Provide a wrapper around numba's low-level extension API which - produces the boilerplate needed to implement a binary function - of two strings. - """ - - def deco(cuda_func): - @cuda_lower(binary_func, string_view, string_view) - def binary_func_impl(context, builder, sig, args): - lhs_ptr = builder.alloca(args[0].type) - rhs_ptr = builder.alloca(args[1].type) - builder.store(args[0], lhs_ptr) - builder.store(args[1], rhs_ptr) - - # these conditional statements should compile out - if retty != udf_string: - # binary function of two strings yielding a fixed-width type - # example: str.startswith(other) -> bool - # shim functions can return the value through nb_retval - result = context.compile_internal( - builder, - cuda_func, - nb_signature(retty, _STR_VIEW_PTR, _STR_VIEW_PTR), - (lhs_ptr, rhs_ptr), - ) - return result - else: - # binary function of two strings yielding a new string - # example: str.strip(other) -> str - # shim functions can not return a struct due to C linkage - # so we create a new udf_string and pass a pointer to it - # for the shim function to write the output to. The return - # value of compile_internal is therefore discarded (although - # this may change in the future if we need to return error - # codes, for instance). - udf_str_ptr = builder.alloca( - default_manager[udf_string].get_value_type() - ) - _ = context.compile_internal( - builder, - cuda_func, - size_type(_UDF_STRING_PTR, _STR_VIEW_PTR, _STR_VIEW_PTR), - (udf_str_ptr, lhs_ptr, rhs_ptr), - ) - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - return binary_func_impl - - return deco - - -@create_binary_string_func(operator.contains, types.boolean) -def contains_impl(st, substr): - return _string_view_contains(st, substr) - - -@create_binary_string_func(operator.eq, types.boolean) -def eq_impl(st, rhs): - return _string_view_eq(st, rhs) - - -@create_binary_string_func(operator.ne, types.boolean) -def ne_impl(st, rhs): - return _string_view_ne(st, rhs) - - -@create_binary_string_func(operator.ge, types.boolean) -def ge_impl(st, rhs): - return _string_view_ge(st, rhs) - - -@create_binary_string_func(operator.le, types.boolean) -def le_impl(st, rhs): - return _string_view_le(st, rhs) - - -@create_binary_string_func(operator.gt, types.boolean) -def gt_impl(st, rhs): - return _string_view_gt(st, rhs) - - -@create_binary_string_func(operator.lt, types.boolean) -def lt_impl(st, rhs): - return _string_view_lt(st, rhs) - - -@create_binary_string_func("StringView.strip", udf_string) -def strip_impl(result, to_strip, strip_char): - return _string_view_strip(result, to_strip, strip_char) - - -@create_binary_string_func("StringView.lstrip", udf_string) -def lstrip_impl(result, to_strip, strip_char): - return _string_view_lstrip(result, to_strip, strip_char) - - -@create_binary_string_func("StringView.rstrip", udf_string) -def rstrip_impl(result, to_strip, strip_char): - return _string_view_rstrip(result, to_strip, strip_char) - - -@create_binary_string_func("StringView.startswith", types.boolean) -def startswith_impl(sv, substr): - return _string_view_startswith(sv, substr) - - -@create_binary_string_func("StringView.endswith", types.boolean) -def endswith_impl(sv, substr): - return _string_view_endswith(sv, substr) - - -@create_binary_string_func("StringView.count", size_type) -def count_impl(st, substr): - return _string_view_count(st, substr) - - -@create_binary_string_func("StringView.find", size_type) -def find_impl(sv, substr): - return _string_view_find(sv, substr) - - -@create_binary_string_func("StringView.rfind", size_type) -def rfind_impl(sv, substr): - return _string_view_rfind(sv, substr) - - -def create_unary_identifier_func(id_func): - """ - Provide a wrapper around numba's low-level extension API which - produces the boilerplate needed to implement a unary function - of a string. - """ - - def deco(cuda_func): - @cuda_lower(id_func, string_view) - def id_func_impl(context, builder, sig, args): - str_ptr = builder.alloca(args[0].type) - builder.store(args[0], str_ptr) - - # Lookup table required for conversion functions - # must be resolved at runtime after context initialization, - # therefore cannot be a global variable - tbl_ptr = context.get_constant( - types.uintp, get_character_flags_table_ptr() - ) - result = context.compile_internal( - builder, - cuda_func, - nb_signature(types.boolean, _STR_VIEW_PTR, types.uintp), - (str_ptr, tbl_ptr), - ) - - return result - - return id_func_impl - - return deco - - -def create_upper_or_lower(id_func): - """ - Provide a wrapper around numba's low-level extension API which - produces the boilerplate needed to implement either the upper - or lower attrs of a string view. - """ - - def deco(cuda_func): - @cuda_lower(id_func, string_view) - def id_func_impl(context, builder, sig, args): - str_ptr = builder.alloca(args[0].type) - builder.store(args[0], str_ptr) - - # Lookup table required for conversion functions - # must be resolved at runtime after context initialization, - # therefore cannot be a global variable - flags_tbl_ptr = context.get_constant( - types.uintp, get_character_flags_table_ptr() - ) - cases_tbl_ptr = context.get_constant( - types.uintp, get_character_cases_table_ptr() - ) - special_tbl_ptr = context.get_constant( - types.uintp, get_special_case_mapping_table_ptr() - ) - udf_str_ptr = builder.alloca( - default_manager[udf_string].get_value_type() - ) - - _ = context.compile_internal( - builder, - cuda_func, - types.void( - _UDF_STRING_PTR, - _STR_VIEW_PTR, - types.uintp, - types.uintp, - types.uintp, - ), - ( - udf_str_ptr, - str_ptr, - flags_tbl_ptr, - cases_tbl_ptr, - special_tbl_ptr, - ), - ) - - result = cgutils.create_struct_proxy(udf_string)( - context, builder, value=builder.load(udf_str_ptr) - ) - return result._getvalue() - - return id_func_impl - - return deco - - -@create_upper_or_lower("StringView.upper") -def upper_impl(result, st, flags, cases, special): - return _string_view_upper(result, st, flags, cases, special) - - -@create_upper_or_lower("StringView.lower") -def lower_impl(result, st, flags, cases, special): - return _string_view_lower(result, st, flags, cases, special) - - -@create_unary_identifier_func("StringView.isdigit") -def isdigit_impl(st, tbl): - return _string_view_isdigit(st, tbl) - - -@create_unary_identifier_func("StringView.isalnum") -def isalnum_impl(st, tbl): - return _string_view_isalnum(st, tbl) - - -@create_unary_identifier_func("StringView.isalpha") -def isalpha_impl(st, tbl): - return _string_view_isalpha(st, tbl) - - -@create_unary_identifier_func("StringView.isnumeric") -def isnumeric_impl(st, tbl): - return _string_view_isnumeric(st, tbl) - - -@create_unary_identifier_func("StringView.isdecimal") -def isdecimal_impl(st, tbl): - return _string_view_isdecimal(st, tbl) - - -@create_unary_identifier_func("StringView.isspace") -def isspace_impl(st, tbl): - return _string_view_isspace(st, tbl) - - -@create_unary_identifier_func("StringView.isupper") -def isupper_impl(st, tbl): - return _string_view_isupper(st, tbl) - - -@create_unary_identifier_func("StringView.islower") -def islower_impl(st, tbl): - return _string_view_islower(st, tbl) - - -@create_unary_identifier_func("StringView.istitle") -def istitle_impl(st, tbl): - return _string_view_istitle(st, tbl) diff --git a/python/strings_udf/versioneer.py b/python/strings_udf/versioneer.py deleted file mode 100644 index 6194b6a5698..00000000000 --- a/python/strings_udf/versioneer.py +++ /dev/null @@ -1,2245 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -# Version: 0.23 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/python-versioneer/python-versioneer -* Brian Warner -* License: Public Domain (CC0-1.0) -* Compatible with: Python 3.7, 3.8, 3.9, 3.10 and pypy3 -* [![Latest Version][pypi-image]][pypi-url] -* [![Build Status][travis-image]][travis-url] - -This is a tool for managing a recorded version number in -distutils/setuptools-based python projects. The goal is to -remove the tedious and error-prone "update the embedded version string" -step from your release process. Making a new release should be as easy -as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere in your $PATH -* add a `[versioneer]` section to your setup.cfg (see [Install](INSTALL.md)) -* run `versioneer install` in your source tree, commit the results -* Verify version information with `python setup.py version` - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes). - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/python-versioneer/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other languages) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/python-versioneer/python-versioneer/issues/38) -is tracking this issue. The discussion in -[PR #61](https://github.com/python-versioneer/python-versioneer/pull/61) -describes the issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/python-versioneer/python-versioneer/issues/83) -describes this one, but upgrading to a newer version of setuptools should -probably resolve it. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - -## Similar projects - -* [setuptools_scm](https://github.com/pypa/setuptools_scm/) - a non-vendored - build-time dependency -* [minver](https://github.com/jbweston/miniver) - a lightweight - reimplementation of versioneer -* [versioningit](https://github.com/jwodder/versioningit) - a PEP 518-based - setuptools plugin - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -[pypi-image]: https://img.shields.io/pypi/v/versioneer.svg -[pypi-url]: https://pypi.python.org/pypi/versioneer/ -[travis-image]: -https://img.shields.io/travis/com/python-versioneer/python-versioneer.svg -[travis-url]: https://travis-ci.com/github/python-versioneer/python-versioneer - -""" -# pylint:disable=invalid-name,import-outside-toplevel,missing-function-docstring -# pylint:disable=missing-class-docstring,too-many-branches,too-many-statements -# pylint:disable=raise-missing-from,too-many-lines,too-many-locals,import-error -# pylint:disable=too-few-public-methods,redefined-outer-name,consider-using-with -# pylint:disable=attribute-defined-outside-init,too-many-arguments - -import configparser -import errno -import functools -import json -import os -import re -import subprocess -import sys -from typing import Callable, Dict - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - my_path = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(my_path)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(my_path), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise OSError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.ConfigParser() - with open(setup_cfg, "r") as cfg_file: - parser.read_file(cfg_file) - VCS = parser.get("versioneer", "VCS") # mandatory - - # Dict-like interface for non-mandatory entries - section = parser["versioneer"] - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = section.get("style", "") - cfg.versionfile_source = section.get("versionfile_source") - cfg.versionfile_build = section.get("versionfile_build") - cfg.tag_prefix = section.get("tag_prefix") - if cfg.tag_prefix in ("''", '""', None): - cfg.tag_prefix = "" - cfg.parentdir_prefix = section.get("parentdir_prefix") - cfg.verbose = section.get("verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - HANDLERS.setdefault(vcs, {})[method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen( - [command] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - **popen_kwargs, - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, process.returncode - return stdout, process.returncode - - -LONG_VERSION_PY[ - "git" -] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.23 (https://github.com/python-versioneer/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys -from typing import Callable, Dict -import functools - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY: Dict[str, str] = {} -HANDLERS: Dict[str, Dict[str, Callable]] = {} - - -def register_vcs_handler(vcs, method): # decorator - """Create decorator to mark a method as the handler of a VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - process = None - - popen_kwargs = {} - if sys.platform == "win32": - # This hides the console window if pythonw.exe is used - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - popen_kwargs["startupinfo"] = startupinfo - - for command in commands: - try: - dispcmd = str([command] + args) - # remember shell=False, so use git.cmd on windows, not just git - process = subprocess.Popen([command] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None), **popen_kwargs) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = process.communicate()[0].strip().decode() - if process.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, process.returncode - return stdout, process.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG):] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r'\d', r)} - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r'\d', r): - continue - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner(GITS, [ - "describe", "--tags", "--dirty", "--always", "--long", - "--match", f"{tag_prefix}[[:digit:]]*" - ], cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post( - pieces["closest-tag"] - ) - rendered = tag_version - if post_version is not None: - rendered += ".post%%d.dev%%d" %% ( - post_version + 1, pieces["distance"] - ) - else: - rendered += ".post0.dev%%d" %% (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for _ in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - with open(versionfile_abs, "r") as fobj: - for line in fobj: - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if "refnames" not in keywords: - raise NotThisMethod("Short version file found") - date = keywords.get("date") - if date is not None: - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - # Filter out refs that exactly match prefix or that don't start - # with a number once the prefix is stripped (mostly a concern - # when prefix is '') - if not re.match(r"\d", r): - continue - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - # GIT_DIR can interfere with correct operation of Versioneer. - # It may be intended to be passed to the Versioneer-versioned project, - # but that should not change where we get our version from. - env = os.environ.copy() - env.pop("GIT_DIR", None) - runner = functools.partial(runner, env=env) - - _, rc = runner( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = runner( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - f"{tag_prefix}[[:digit:]]*", - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - branch_name, rc = runner( - GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root - ) - # --abbrev-ref was added in git-1.6.3 - if rc != 0 or branch_name is None: - raise NotThisMethod("'git rev-parse --abbrev-ref' returned error") - branch_name = branch_name.strip() - - if branch_name == "HEAD": - # If we aren't exactly on a branch, pick a branch which represents - # the current commit. If all else fails, we are on a branchless - # commit. - branches, rc = runner(GITS, ["branch", "--contains"], cwd=root) - # --contains was added in git-1.5.4 - if rc != 0 or branches is None: - raise NotThisMethod("'git branch --contains' returned error") - branches = branches.split("\n") - - # Remove the first line if we're running detached - if "(" in branches[0]: - branches.pop(0) - - # Strip off the leading "* " from the list of branches. - branches = [branch[2:] for branch in branches] - if "master" in branches: - branch_name = "master" - elif not branches: - branch_name = None - else: - # Pick the first branch that is returned. Good or bad. - branch_name = branches[0] - - pieces["branch"] = branch_name - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparsable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root) - pieces["distance"] = len(out.split()) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - # Use only the last line. Previous lines may contain GPG signature - # information. - date = date.splitlines()[-1] - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [versionfile_source] - if ipy: - files.append(ipy) - try: - my_path = __file__ - if my_path.endswith(".pyc") or my_path.endswith(".pyo"): - my_path = os.path.splitext(my_path)[0] + ".py" - versioneer_file = os.path.relpath(my_path) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - with open(".gitattributes", "r") as fobj: - for line in fobj: - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - break - except OSError: - pass - if not present: - with open(".gitattributes", "a+") as fobj: - fobj.write(f"{versionfile_source} export-subst\n") - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for _ in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.23) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except OSError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_branch(pieces): - """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] . - - The ".dev0" means not master branch. Note that .dev0 sorts backwards - (a feature branch will appear "older" than the master branch). - - Exceptions: - 1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0" - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def pep440_split_post(ver): - """Split pep440 version string at the post-release segment. - - Returns the release segments before the post-release and the - post-release version number (or -1 if no post-release segment is present). - """ - vc = str.split(ver, ".post") - return vc[0], int(vc[1] or 0) if len(vc) == 2 else None - - -def render_pep440_pre(pieces): - """TAG[.postN.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post0.devDISTANCE - """ - if pieces["closest-tag"]: - if pieces["distance"]: - # update the post release segment - tag_version, post_version = pep440_split_post( - pieces["closest-tag"] - ) - rendered = tag_version - if post_version is not None: - rendered += ".post%d.dev%d" % ( - post_version + 1, - pieces["distance"], - ) - else: - rendered += ".post0.dev%d" % (pieces["distance"]) - else: - # no commits, use the tag as the version - rendered = pieces["closest-tag"] - else: - # exception #1 - rendered = "0.post0.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_post_branch(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] . - - The ".dev0" means not master branch. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["branch"] != "master": - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-branch": - rendered = render_pep440_branch(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-post-branch": - rendered = render_pep440_post_branch(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(cmdclass=None): - """Get the custom setuptools subclasses used by Versioneer. - - If the package uses a different cmdclass (e.g. one from numpy), it - should be provide as an argument. - """ - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see - # https://github.com/python-versioneer/python-versioneer/issues/52 - - cmds = {} if cmdclass is None else cmdclass.copy() - - # we add "version" to setuptools - from setuptools import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # pip install -e . and setuptool/editable_wheel will invoke build_py - # but the build_py command is not expected to copy any files. - - # we override different "build_py" commands for both environments - if "build_py" in cmds: - _build_py = cmds["build_py"] - else: - from setuptools.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - if getattr(self, "editable_mode", False): - # During editable installs `.py` and data files are - # not copied to build_lib - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "build_ext" in cmds: - _build_ext = cmds["build_ext"] - else: - from setuptools.command.build_ext import build_ext as _build_ext - - class cmd_build_ext(_build_ext): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_ext.run(self) - if self.inplace: - # build_ext --inplace will only build extensions in - # build/lib<..> dir with no _version.py to write to. - # As in place builds will already have a _version.py - # in the module dir, we do not need to write one. - return - # now locate _version.py in the new build/ directory and replace - # it with an updated value - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - if not os.path.exists(target_versionfile): - print( - f"Warning: {target_versionfile} does not exist, skipping " - "version update. This can happen if you are running " - "build_ext without first running build_py." - ) - return - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_ext"] = cmd_build_ext - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - from py2exe.distutils_buildexe import py2exe as _py2exe - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # sdist farms its file list building out to egg_info - if "egg_info" in cmds: - _sdist = cmds["egg_info"] - else: - from setuptools.command.egg_info import egg_info as _egg_info - - class cmd_egg_info(_egg_info): - def find_sources(self): - # egg_info.find_sources builds the manifest list and writes it - # in one shot - super().find_sources() - - # Modify the filelist and normalize it - root = get_root() - cfg = get_config_from_root(root) - self.filelist.append("versioneer.py") - if cfg.versionfile_source: - # There are rare cases where versionfile_source might not be - # included by default, so we must be explicit - self.filelist.append(cfg.versionfile_source) - self.filelist.sort() - self.filelist.remove_duplicates() - - # The write method is hidden in the manifest_maker instance that - # generated the filelist and was thrown away - # We will instead replicate their final normalization (to unicode, - # and POSIX-style paths) - from setuptools import unicode_utils - - normalized = [ - unicode_utils.filesys_decode(f).replace(os.sep, "/") - for f in self.filelist.files - ] - - manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") - with open(manifest_filename, "w") as fobj: - fobj.write("\n".join(normalized)) - - cmds["egg_info"] = cmd_egg_info - - # we override different "sdist" commands for both environments - if "sdist" in cmds: - _sdist = cmds["sdist"] - else: - from setuptools.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -OLD_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - -INIT_PY_SNIPPET = """ -from . import {0} -__version__ = {0}.get_versions()['version'] -""" - - -def do_setup(): - """Do main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - OSError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (OSError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except OSError: - old = "" - module = os.path.splitext(os.path.basename(cfg.versionfile_source))[0] - snippet = INIT_PY_SNIPPET.format(module) - if OLD_SNIPPET in old: - print(" replacing boilerplate in %s" % ipy) - with open(ipy, "w") as f: - f.write(old.replace(OLD_SNIPPET, snippet)) - elif snippet not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(snippet) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) From d441f51ad0d6860e0d3c25837d8ecc55076a1be5 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 22 Feb 2023 09:46:25 -0500 Subject: [PATCH 44/69] Enable groupby std and variance aggregation types in libcudf Debug build (#12799) Re-enable groupby with `std` and `var` aggregations that were disabled for Debug builds due to a runtime issue. Retesting with nvcc 11.5, the error is no longer present so the code and the gtests have been re-enabled. Found while working on PR #12784 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Bradley Dice (https://github.com/bdice) - Mike Wilson (https://github.com/hyperbolic2346) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/12799 --- cpp/src/groupby/sort/group_std.cu | 8 +------- cpp/tests/groupby/std_tests.cpp | 28 ---------------------------- cpp/tests/groupby/var_tests.cpp | 28 ---------------------------- 3 files changed, 1 insertion(+), 63 deletions(-) diff --git a/cpp/src/groupby/sort/group_std.cu b/cpp/src/groupby/sort/group_std.cu index 87fd9f7e843..a3efc1f172a 100644 --- a/cpp/src/groupby/sort/group_std.cu +++ b/cpp/src/groupby/sort/group_std.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -99,9 +99,6 @@ struct var_functor { rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { -// Running this in debug build causes a runtime error: -// `reduce_by_key failed on 2nd step: invalid device function` -#if !defined(__CUDACC_DEBUG__) using ResultType = cudf::detail::target_type_t; std::unique_ptr result = make_numeric_column(data_type(type_to_id()), @@ -141,9 +138,6 @@ struct var_functor { }); return result; -#else - CUDF_FAIL("Groupby std/var supported in debug build"); -#endif } template diff --git a/cpp/tests/groupby/std_tests.cpp b/cpp/tests/groupby/std_tests.cpp index fa3afeb30f8..56ddce1554f 100644 --- a/cpp/tests/groupby/std_tests.cpp +++ b/cpp/tests/groupby/std_tests.cpp @@ -33,11 +33,7 @@ using supported_types = cudf::test::Types Date: Wed, 22 Feb 2023 09:42:46 -0800 Subject: [PATCH 45/69] Variable fragment sizes for Parquet writer (#12685) Fixes #12613 This PR adds the ability for columns to have different fragment sizes. This allows a large fragment size for narrow columns, but allows for finer grained fragments for very wide columns. This change should make wide columns fit (approximately) within page size constraints, and should help with compressors that rely on pages being under a certain threshold (i.e. Zstandard). Authors: - Ed Seidl (https://github.com/etseidl) - Vukasin Milovanovic (https://github.com/vuule) - Mike Wilson (https://github.com/hyperbolic2346) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Mike Wilson (https://github.com/hyperbolic2346) URL: https://github.com/rapidsai/cudf/pull/12685 --- cpp/include/cudf/io/parquet.hpp | 4 +- cpp/src/io/parquet/page_enc.cu | 251 ++++++++++-------- cpp/src/io/parquet/parquet_gpu.hpp | 43 ++- cpp/src/io/parquet/writer_impl.cu | 265 ++++++++++++++----- cpp/src/io/parquet/writer_impl.hpp | 42 +-- cpp/tests/io/parquet_chunked_reader_test.cpp | 3 +- cpp/tests/io/parquet_test.cpp | 46 +++- 7 files changed, 449 insertions(+), 205 deletions(-) diff --git a/cpp/include/cudf/io/parquet.hpp b/cpp/include/cudf/io/parquet.hpp index f4fb4d91f58..92b69deb671 100644 --- a/cpp/include/cudf/io/parquet.hpp +++ b/cpp/include/cudf/io/parquet.hpp @@ -494,7 +494,7 @@ class parquet_writer_options { // Maximum size of column chunk dictionary (in bytes) size_t _max_dictionary_size = default_max_dictionary_size; // Maximum number of rows in a page fragment - size_type _max_page_fragment_size = default_max_page_fragment_size; + std::optional _max_page_fragment_size; /** * @brief Constructor from sink and table. @@ -1076,7 +1076,7 @@ class chunked_parquet_writer_options { // Maximum size of column chunk dictionary (in bytes) size_t _max_dictionary_size = default_max_dictionary_size; // Maximum number of rows in a page fragment - size_type _max_page_fragment_size = default_max_page_fragment_size; + std::optional _max_page_fragment_size; /** * @brief Constructor from sink. diff --git a/cpp/src/io/parquet/page_enc.cu b/cpp/src/io/parquet/page_enc.cu index 9f8f42702cd..5a12acec2a3 100644 --- a/cpp/src/io/parquet/page_enc.cu +++ b/cpp/src/io/parquet/page_enc.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,129 +126,164 @@ constexpr size_t underflow_safe_subtract(size_t a, size_t b) return a - b; } +void __device__ init_frag_state(frag_init_state_s* const s, + uint32_t fragment_size, + int part_end_row) +{ + // frag.num_rows = fragment_size except for the last fragment in partition which can be + // smaller. num_rows is fixed but fragment size could be larger if the data is strings or + // nested. + s->frag.num_rows = min(fragment_size, part_end_row - s->frag.start_row); + s->frag.num_dict_vals = 0; + s->frag.fragment_data_size = 0; + s->frag.dict_data_size = 0; + + s->frag.start_value_idx = row_to_value_idx(s->frag.start_row, s->col); + auto const end_value_idx = row_to_value_idx(s->frag.start_row + s->frag.num_rows, s->col); + s->frag.num_leaf_values = end_value_idx - s->frag.start_value_idx; + + if (s->col.level_offsets != nullptr) { + // For nested schemas, the number of values in a fragment is not directly related to the + // number of encoded data elements or the number of rows. It is simply the number of + // repetition/definition values which together encode validity and nesting information. + auto const first_level_val_idx = s->col.level_offsets[s->frag.start_row]; + auto const last_level_val_idx = s->col.level_offsets[s->frag.start_row + s->frag.num_rows]; + s->frag.num_values = last_level_val_idx - first_level_val_idx; + } else { + s->frag.num_values = s->frag.num_rows; + } +} + +template +void __device__ calculate_frag_size(frag_init_state_s* const s, int t) +{ + using block_reduce = cub::BlockReduce; + __shared__ typename block_reduce::TempStorage reduce_storage; + + auto const physical_type = s->col.physical_type; + auto const leaf_type = s->col.leaf_column->type().id(); + auto const dtype_len = physical_type_len(physical_type, leaf_type); + auto const nvals = s->frag.num_leaf_values; + auto const start_value_idx = s->frag.start_value_idx; + + for (uint32_t i = 0; i < nvals; i += block_size) { + auto const val_idx = start_value_idx + i + t; + auto const is_valid = i + t < nvals && val_idx < s->col.leaf_column->size() && + s->col.leaf_column->is_valid(val_idx); + uint32_t len; + if (is_valid) { + len = dtype_len; + if (physical_type == BYTE_ARRAY) { + switch (leaf_type) { + case type_id::STRING: { + auto str = s->col.leaf_column->element(val_idx); + len += str.size_bytes(); + } break; + case type_id::LIST: { + auto list_element = + get_element(*s->col.leaf_column, val_idx); + len += list_element.size_bytes(); + } break; + default: CUDF_UNREACHABLE("Unsupported data type for leaf column"); + } + } + } else { + len = 0; + } + + len = block_reduce(reduce_storage).Sum(len); + if (t == 0) { s->frag.fragment_data_size += len; } + __syncthreads(); + // page fragment size must fit in a 32-bit signed integer + if (s->frag.fragment_data_size > std::numeric_limits::max()) { + CUDF_UNREACHABLE("page fragment size exceeds maximum for i32"); + } + } +} + } // anonymous namespace // blockDim {512,1,1} template __global__ void __launch_bounds__(block_size) - gpuInitPageFragments(device_2dspan frag, - device_span col_desc, - device_span partitions, - device_span part_frag_offset, - uint32_t fragment_size) + gpuInitRowGroupFragments(device_2dspan frag, + device_span col_desc, + device_span partitions, + device_span part_frag_offset, + uint32_t fragment_size) { __shared__ __align__(16) frag_init_state_s state_g; - using block_reduce = cub::BlockReduce; - __shared__ typename block_reduce::TempStorage reduce_storage; - frag_init_state_s* const s = &state_g; uint32_t const t = threadIdx.x; - auto const physical_type = col_desc[blockIdx.x].physical_type; uint32_t const num_fragments_per_column = frag.size().second; if (t == 0) { s->col = col_desc[blockIdx.x]; } __syncthreads(); - auto const leaf_type = s->col.leaf_column->type().id(); - auto const dtype_len = physical_type_len(physical_type, leaf_type); - for (uint32_t frag_y = blockIdx.y; frag_y < num_fragments_per_column; frag_y += gridDim.y) { if (t == 0) { // Find which partition this fragment came from auto it = thrust::upper_bound(thrust::seq, part_frag_offset.begin(), part_frag_offset.end(), frag_y); - int p = it - part_frag_offset.begin() - 1; - int part_end_row = partitions[p].start_row + partitions[p].num_rows; + int const p = it - part_frag_offset.begin() - 1; + int const part_end_row = partitions[p].start_row + partitions[p].num_rows; s->frag.start_row = (frag_y - part_frag_offset[p]) * fragment_size + partitions[p].start_row; - - // frag.num_rows = fragment_size except for the last fragment in partition which can be - // smaller. num_rows is fixed but fragment size could be larger if the data is strings or - // nested. - s->frag.num_rows = min(fragment_size, part_end_row - s->frag.start_row); - s->frag.num_dict_vals = 0; - s->frag.fragment_data_size = 0; - s->frag.dict_data_size = 0; - - s->frag.start_value_idx = row_to_value_idx(s->frag.start_row, s->col); - size_type end_value_idx = row_to_value_idx(s->frag.start_row + s->frag.num_rows, s->col); - s->frag.num_leaf_values = end_value_idx - s->frag.start_value_idx; - - if (s->col.level_offsets != nullptr) { - // For nested schemas, the number of values in a fragment is not directly related to the - // number of encoded data elements or the number of rows. It is simply the number of - // repetition/definition values which together encode validity and nesting information. - size_type first_level_val_idx = s->col.level_offsets[s->frag.start_row]; - size_type last_level_val_idx = s->col.level_offsets[s->frag.start_row + s->frag.num_rows]; - s->frag.num_values = last_level_val_idx - first_level_val_idx; - } else { - s->frag.num_values = s->frag.num_rows; - } + s->frag.chunk = frag[blockIdx.x][frag_y].chunk; + init_frag_state(s, fragment_size, part_end_row); } __syncthreads(); - size_type nvals = s->frag.num_leaf_values; - size_type start_value_idx = s->frag.start_value_idx; - - for (uint32_t i = 0; i < nvals; i += block_size) { - uint32_t val_idx = start_value_idx + i + t; - uint32_t is_valid = (i + t < nvals && val_idx < s->col.leaf_column->size()) - ? s->col.leaf_column->is_valid(val_idx) - : 0; - uint32_t len; - if (is_valid) { - len = dtype_len; - if (physical_type == BYTE_ARRAY) { - switch (leaf_type) { - case type_id::STRING: { - auto str = s->col.leaf_column->element(val_idx); - len += str.size_bytes(); - } break; - case type_id::LIST: { - auto list_element = - get_element(*s->col.leaf_column, val_idx); - len += list_element.size_bytes(); - } break; - default: CUDF_UNREACHABLE("Unsupported data type for leaf column"); - } - } - } else { - len = 0; - } - - len = block_reduce(reduce_storage).Sum(len); - if (t == 0) { s->frag.fragment_data_size += len; } - __syncthreads(); - // page fragment size must fit in a 32-bit signed integer - if (s->frag.fragment_data_size > std::numeric_limits::max()) { - CUDF_UNREACHABLE("page fragment size exceeds maximum for i32"); - } - } + calculate_frag_size(s, t); __syncthreads(); if (t == 0) { frag[blockIdx.x][frag_y] = s->frag; } } } +// blockDim {512,1,1} +template +__global__ void __launch_bounds__(block_size) + gpuCalculatePageFragments(device_span frag, + device_span column_frag_sizes) +{ + __shared__ __align__(16) frag_init_state_s state_g; + + EncColumnChunk* const ck_g = frag[blockIdx.x].chunk; + frag_init_state_s* const s = &state_g; + uint32_t const t = threadIdx.x; + auto const fragment_size = column_frag_sizes[ck_g->col_desc_id]; + + if (t == 0) { s->col = *ck_g->col_desc; } + __syncthreads(); + + if (t == 0) { + int const part_end_row = ck_g->start_row + ck_g->num_rows; + s->frag.start_row = ck_g->start_row + (blockIdx.x - ck_g->first_fragment) * fragment_size; + s->frag.chunk = ck_g; + init_frag_state(s, fragment_size, part_end_row); + } + __syncthreads(); + + calculate_frag_size(s, t); + if (t == 0) { frag[blockIdx.x] = s->frag; } +} + // blockDim {128,1,1} __global__ void __launch_bounds__(128) - gpuInitFragmentStats(device_2dspan groups, - device_2dspan fragments, - device_span col_desc) + gpuInitFragmentStats(device_span groups, + device_span fragments) { - uint32_t const lane_id = threadIdx.x & WARP_MASK; - uint32_t const column_id = blockIdx.x; - uint32_t const num_fragments_per_column = fragments.size().second; - - uint32_t frag_id = blockIdx.y * 4 + (threadIdx.x / cudf::detail::warp_size); - while (frag_id < num_fragments_per_column) { + uint32_t const lane_id = threadIdx.x & WARP_MASK; + uint32_t const frag_id = blockIdx.x * 4 + (threadIdx.x / cudf::detail::warp_size); + if (frag_id < fragments.size()) { if (lane_id == 0) { statistics_group g; - g.col = &col_desc[column_id]; - g.start_row = fragments[column_id][frag_id].start_value_idx; - g.num_rows = fragments[column_id][frag_id].num_leaf_values; - groups[column_id][frag_id] = g; + auto* const ck_g = fragments[frag_id].chunk; + g.col = ck_g->col_desc; + g.start_row = fragments[frag_id].start_value_idx; + g.num_rows = fragments[frag_id].num_leaf_values; + groups[frag_id] = g; } - frag_id += gridDim.y * 4; } } @@ -389,7 +424,7 @@ __global__ void __launch_bounds__(128) if (num_rows >= ck_g.num_rows || (values_in_page > 0 && (page_size + fragment_data_size > this_max_page_size)) || - rows_in_page >= max_page_size_rows) { + rows_in_page + frag_g.num_rows > max_page_size_rows) { if (ck_g.use_dictionary) { // Additional byte to store entry bit width page_size = 1 + max_RLE_page_size(ck_g.dict_rle_bits, values_in_page); @@ -2057,33 +2092,35 @@ __global__ void __launch_bounds__(1) ck_g->column_index_size = static_cast(col_idx_end - ck_g->column_index_blob); } -void InitPageFragments(device_2dspan frag, - device_span col_desc, - device_span partitions, - device_span part_frag_offset, - uint32_t fragment_size, - rmm::cuda_stream_view stream) +void InitRowGroupFragments(device_2dspan frag, + device_span col_desc, + device_span partitions, + device_span part_frag_offset, + uint32_t fragment_size, + rmm::cuda_stream_view stream) { auto const num_columns = frag.size().first; auto const num_fragments_per_column = frag.size().second; auto const grid_y = std::min(static_cast(num_fragments_per_column), MAX_GRID_Y_SIZE); dim3 const dim_grid(num_columns, grid_y); // 1 threadblock per fragment - gpuInitPageFragments<512><<>>( + gpuInitRowGroupFragments<512><<>>( frag, col_desc, partitions, part_frag_offset, fragment_size); } -void InitFragmentStatistics(device_2dspan groups, - device_2dspan fragments, - device_span col_desc, +void CalculatePageFragments(device_span frag, + device_span column_frag_sizes, + rmm::cuda_stream_view stream) +{ + gpuCalculatePageFragments<512><<>>(frag, column_frag_sizes); +} + +void InitFragmentStatistics(device_span groups, + device_span fragments, rmm::cuda_stream_view stream) { - int const num_columns = col_desc.size(); - int const num_fragments_per_column = fragments.size().second; - auto const y_dim = - util::div_rounding_up_safe(num_fragments_per_column, 128 / cudf::detail::warp_size); - auto const grid_y = std::min(static_cast(y_dim), MAX_GRID_Y_SIZE); - dim3 const dim_grid(num_columns, grid_y); // 1 warp per fragment - gpuInitFragmentStats<<>>(groups, fragments, col_desc); + int const num_fragments = fragments.size(); + int const dim = util::div_rounding_up_safe(num_fragments, 128 / cudf::detail::warp_size); + gpuInitFragmentStats<<>>(groups, fragments); } void InitEncoderPages(device_2dspan chunks, diff --git a/cpp/src/io/parquet/parquet_gpu.hpp b/cpp/src/io/parquet/parquet_gpu.hpp index 9b156745e41..c91f182c4f4 100644 --- a/cpp/src/io/parquet/parquet_gpu.hpp +++ b/cpp/src/io/parquet/parquet_gpu.hpp @@ -480,8 +480,9 @@ void DecodePageData(hostdevice_vector& pages, rmm::cuda_stream_view stream); /** - * @brief Launches kernel for initializing encoder page fragments + * @brief Launches kernel for initializing encoder row group fragments * + * These fragments are used to calculate row group boundaries. * Based on the number of rows in each fragment, populates the value count, the size of data in the * fragment, the number of unique values, and the data size of unique values. * @@ -492,24 +493,38 @@ void DecodePageData(hostdevice_vector& pages, * @param[in] fragment_size Number of rows per fragment * @param[in] stream CUDA stream to use */ -void InitPageFragments(cudf::detail::device_2dspan frag, - device_span col_desc, - device_span partitions, - device_span first_frag_in_part, - uint32_t fragment_size, - rmm::cuda_stream_view stream); +void InitRowGroupFragments(cudf::detail::device_2dspan frag, + device_span col_desc, + device_span partitions, + device_span first_frag_in_part, + uint32_t fragment_size, + rmm::cuda_stream_view stream); + +/** + * @brief Launches kernel for calculating encoder page fragments with variable fragment sizes + * + * Based on the number of rows in each fragment, populates the value count, the size of data in the + * fragment, the number of unique values, and the data size of unique values. + * + * This assumes an initial call to InitRowGroupFragments has been made. + * + * @param[out] frag Fragment array [fragment_id] + * @param[in] column_frag_sizes Number of rows per fragment per column [column_id] + * @param[in] stream CUDA stream to use + */ +void CalculatePageFragments(device_span frag, + device_span column_frag_sizes, + rmm::cuda_stream_view stream); /** - * @brief Launches kernel for initializing fragment statistics groups + * @brief Launches kernel for initializing fragment statistics groups with variable fragment sizes * - * @param[out] groups Statistics groups [num_columns x num_fragments] - * @param[in] fragments Page fragments [num_columns x num_fragments] - * @param[in] col_desc Column description [num_columns] + * @param[out] groups Statistics groups [total_fragments] + * @param[in] fragments Page fragments [total_fragments] * @param[in] stream CUDA stream to use */ -void InitFragmentStatistics(cudf::detail::device_2dspan groups, - cudf::detail::device_2dspan fragments, - device_span col_desc, +void InitFragmentStatistics(device_span groups, + device_span fragments, rmm::cuda_stream_view stream); /** diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 13ec2d652a6..88176ee1901 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include #include +#include #include #include @@ -85,6 +87,44 @@ parquet::Compression to_parquet_compression(compression_type compression) } } +size_type column_size(column_view const& column, rmm::cuda_stream_view stream) +{ + if (column.size() == 0) { return 0; } + + if (is_fixed_width(column.type())) { + return size_of(column.type()) * column.size(); + } else if (column.type().id() == type_id::STRING) { + auto const scol = strings_column_view(column); + return cudf::detail::get_value(scol.offsets(), column.size(), stream) - + cudf::detail::get_value(scol.offsets(), 0, stream); + } else if (column.type().id() == type_id::STRUCT) { + auto const scol = structs_column_view(column); + size_type ret = 0; + for (int i = 0; i < scol.num_children(); i++) { + ret += column_size(scol.get_sliced_child(i), stream); + } + return ret; + } else if (column.type().id() == type_id::LIST) { + auto const lcol = lists_column_view(column); + return column_size(lcol.get_sliced_child(stream), stream); + } + + CUDF_FAIL("Unexpected compound type"); +} + +// checks to see if the given column has a fixed size. This doesn't +// check every row, so assumes string and list columns are not fixed, even +// if each row is the same width. +// TODO: update this if FIXED_LEN_BYTE_ARRAY is ever supported for writes. +bool is_col_fixed_width(column_view const& column) +{ + if (column.type().id() == type_id::STRUCT) { + return std::all_of(column.child_begin(), column.child_end(), is_col_fixed_width); + } + + return is_fixed_width(column.type()); +} + } // namespace struct aggregate_writer_metadata { @@ -886,34 +926,33 @@ gpu::parquet_column_device_view parquet_column_view::get_device_view( return desc; } -void writer::impl::init_page_fragments(cudf::detail::hostdevice_2dvector& frag, - device_span col_desc, - host_span partitions, - device_span part_frag_offset, - uint32_t fragment_size) +void writer::impl::init_row_group_fragments( + cudf::detail::hostdevice_2dvector& frag, + device_span col_desc, + host_span partitions, + device_span part_frag_offset, + uint32_t fragment_size) { auto d_partitions = cudf::detail::make_device_uvector_async(partitions, stream); - gpu::InitPageFragments(frag, col_desc, d_partitions, part_frag_offset, fragment_size, stream); + gpu::InitRowGroupFragments(frag, col_desc, d_partitions, part_frag_offset, fragment_size, stream); frag.device_to_host(stream, true); } -void writer::impl::gather_fragment_statistics( - device_2dspan frag_stats_chunk, - device_2dspan frag, - device_span col_desc, - uint32_t num_fragments) +void writer::impl::calculate_page_fragments(device_span frag, + host_span frag_sizes) { - auto num_columns = col_desc.size(); - rmm::device_uvector frag_stats_group(num_fragments * num_columns, stream); - auto frag_stats_group_2dview = - device_2dspan(frag_stats_group.data(), num_columns, num_fragments); - - gpu::InitFragmentStatistics(frag_stats_group_2dview, frag, col_desc, stream); - detail::calculate_group_statistics(frag_stats_chunk.data(), - frag_stats_group.data(), - num_fragments * num_columns, - stream, - int96_timestamps); + auto d_frag_sz = cudf::detail::make_device_uvector_async(frag_sizes, stream); + gpu::CalculatePageFragments(frag, d_frag_sz, stream); +} + +void writer::impl::gather_fragment_statistics(device_span frag_stats, + device_span frags) +{ + rmm::device_uvector frag_stats_group(frag_stats.size(), stream); + + gpu::InitFragmentStatistics(frag_stats_group, frags, stream); + detail::calculate_group_statistics( + frag_stats.data(), frag_stats_group.data(), frag_stats.size(), stream, int96_timestamps); stream.synchronize(); } @@ -1407,23 +1446,63 @@ void writer::impl::write(table_view const& table, std::vector co }); // Init page fragments - // 5000 is good enough for up to ~200-character strings. Longer strings will start producing - // fragments larger than the desired page size -> TODO: keep track of the max fragment size, and - // iteratively reduce this value if the largest fragment exceeds the max page size limit (we - // ideally want the page size to be below 1MB so as to have enough pages to get good - // compression/decompression performance). - // If using the default fragment size, scale it up or down depending on the requested page size. - if (max_page_fragment_size_ == cudf::io::default_max_page_fragment_size) { - max_page_fragment_size_ = (cudf::io::default_max_page_fragment_size * max_page_size_bytes) / - cudf::io::default_max_page_size_bytes; + // 5000 is good enough for up to ~200-character strings. Longer strings and deeply nested columns + // will start producing fragments larger than the desired page size, so calculate fragment sizes + // for each leaf column. Skip if the fragment size is not the default. + auto max_page_fragment_size = max_page_fragment_size_.value_or(default_max_page_fragment_size); + + std::vector column_frag_size(num_columns, max_page_fragment_size); + + if (table.num_rows() > 0 && not max_page_fragment_size_.has_value()) { + std::vector column_sizes; + std::transform(single_streams_table.begin(), + single_streams_table.end(), + std::back_inserter(column_sizes), + [this](auto const& column) { return column_size(column, stream); }); + + // adjust global fragment size if a single fragment will overrun a rowgroup + auto const table_size = std::reduce(column_sizes.begin(), column_sizes.end()); + auto const avg_row_len = util::div_rounding_up_safe(table_size, table.num_rows()); + if (avg_row_len > 0) { + auto const rg_frag_size = util::div_rounding_up_safe(max_row_group_size, avg_row_len); + max_page_fragment_size = std::min(rg_frag_size, max_page_fragment_size); + } + + // dividing page size by average row length will tend to overshoot the desired + // page size when there's high variability in the row lengths. instead, shoot + // for multiple fragments per page to smooth things out. using 2 was too + // unbalanced in final page sizes, so using 4 which seems to be a good + // compromise at smoothing things out without getting fragment sizes too small. + auto frag_size_fn = [&](auto const& col, size_type col_size) { + const int target_frags_per_page = is_col_fixed_width(col) ? 1 : 4; + auto const avg_len = + target_frags_per_page * util::div_rounding_up_safe(col_size, table.num_rows()); + if (avg_len > 0) { + auto const frag_size = util::div_rounding_up_safe(max_page_size_bytes, avg_len); + return std::min(max_page_fragment_size, frag_size); + } else { + return max_page_fragment_size; + } + }; + + std::transform(single_streams_table.begin(), + single_streams_table.end(), + column_sizes.begin(), + column_frag_size.begin(), + frag_size_fn); } + // Fragments are calculated in two passes. In the first pass, a uniform number of fragments + // per column is used. This is done to satisfy the requirement that each column chunk within + // a row group has the same number of rows. After the row group (and thus column chunk) + // boundaries are known, a second pass is done to calculate fragments to be used in determining + // page boundaries within each column chunk. std::vector num_frag_in_part; std::transform(partitions.begin(), partitions.end(), std::back_inserter(num_frag_in_part), - [this](auto const& part) { - return util::div_rounding_up_unsafe(part.num_rows, max_page_fragment_size_); + [this, max_page_fragment_size](auto const& part) { + return util::div_rounding_up_unsafe(part.num_rows, max_page_fragment_size); }); size_type num_fragments = std::reduce(num_frag_in_part.begin(), num_frag_in_part.end()); @@ -1434,7 +1513,7 @@ void writer::impl::write(table_view const& table, std::vector co part_frag_offset.push_back(part_frag_offset.back() + num_frag_in_part.back()); auto d_part_frag_offset = cudf::detail::make_device_uvector_async(part_frag_offset, stream); - cudf::detail::hostdevice_2dvector fragments( + cudf::detail::hostdevice_2dvector row_group_fragments( num_columns, num_fragments, stream); if (num_fragments != 0) { @@ -1443,8 +1522,8 @@ void writer::impl::write(table_view const& table, std::vector co leaf_column_views = create_leaf_column_device_views( col_desc, *parent_column_table_device_view, stream); - init_page_fragments( - fragments, col_desc, partitions, d_part_frag_offset, max_page_fragment_size_); + init_row_group_fragments( + row_group_fragments, col_desc, partitions, d_part_frag_offset, max_page_fragment_size); } std::vector const global_rowgroup_base = md->num_row_groups_per_file(); @@ -1461,9 +1540,9 @@ void writer::impl::write(table_view const& table, std::vector co for (auto f = first_frag_in_rg; f <= last_frag_in_part; ++f) { size_t fragment_data_size = 0; for (auto c = 0; c < num_columns; c++) { - fragment_data_size += fragments[c][f].fragment_data_size; + fragment_data_size += row_group_fragments[c][f].fragment_data_size; } - size_type fragment_num_rows = fragments[0][f].num_rows; + size_type fragment_num_rows = row_group_fragments[0][f].num_rows; // If the fragment size gets larger than rg limit then break off a rg if (f > first_frag_in_rg && // There has to be at least one fragment in row group @@ -1490,17 +1569,6 @@ void writer::impl::write(table_view const& table, std::vector co } } - // Allocate column chunks and gather fragment statistics - rmm::device_uvector frag_stats(0, stream); - if (stats_granularity_ != statistics_freq::STATISTICS_NONE) { - frag_stats.resize(num_fragments * num_columns, stream); - if (not frag_stats.is_empty()) { - auto frag_stats_2dview = - device_2dspan(frag_stats.data(), num_columns, num_fragments); - gather_fragment_statistics(frag_stats_2dview, fragments, col_desc, num_fragments); - } - } - std::vector first_rg_in_part; std::exclusive_scan( num_rg_in_part.begin(), num_rg_in_part.end(), std::back_inserter(first_rg_in_part), 0); @@ -1509,6 +1577,9 @@ void writer::impl::write(table_view const& table, std::vector co auto const num_chunks = num_rowgroups * num_columns; hostdevice_2dvector chunks(num_rowgroups, num_columns, stream); + // total fragments per column (in case they are non-uniform) + std::vector frags_per_column(num_columns, 0); + for (size_t p = 0; p < partitions.size(); ++p) { int f = part_frag_offset[p]; size_type start_row = partitions[p].start_row; @@ -1516,22 +1587,21 @@ void writer::impl::write(table_view const& table, std::vector co size_t global_r = global_rowgroup_base[p] + r; // Number of rowgroups already in file/part auto& row_group = md->file(p).row_groups[global_r]; uint32_t fragments_in_chunk = - util::div_rounding_up_unsafe(row_group.num_rows, max_page_fragment_size_); + util::div_rounding_up_unsafe(row_group.num_rows, max_page_fragment_size); row_group.total_byte_size = 0; row_group.columns.resize(num_columns); for (int c = 0; c < num_columns; c++) { gpu::EncColumnChunk& ck = chunks[r + first_rg_in_part[p]][c]; - ck = {}; - ck.col_desc = col_desc.device_ptr() + c; - ck.col_desc_id = c; - ck.fragments = &fragments.device_view()[c][f]; - ck.stats = - (not frag_stats.is_empty()) ? frag_stats.data() + c * num_fragments + f : nullptr; + ck = {}; + ck.col_desc = col_desc.device_ptr() + c; + ck.col_desc_id = c; + ck.fragments = &row_group_fragments.device_view()[c][f]; + ck.stats = nullptr; ck.start_row = start_row; ck.num_rows = (uint32_t)row_group.num_rows; ck.first_fragment = c * num_fragments + f; - auto chunk_fragments = fragments[c].subspan(f, fragments_in_chunk); + auto chunk_fragments = row_group_fragments[c].subspan(f, fragments_in_chunk); // In fragment struct, add a pointer to the chunk it belongs to // In each fragment in chunk_fragments, update the chunk pointer here. for (auto& frag : chunk_fragments) { @@ -1551,15 +1621,23 @@ void writer::impl::write(table_view const& table, std::vector co column_chunk_meta.path_in_schema = parquet_columns[c].get_path_in_schema(); column_chunk_meta.codec = UNCOMPRESSED; column_chunk_meta.num_values = ck.num_values; + + frags_per_column[c] += util::div_rounding_up_unsafe( + row_group.num_rows, std::min(column_frag_size[c], max_page_fragment_size)); } f += fragments_in_chunk; start_row += (uint32_t)row_group.num_rows; } } - fragments.host_to_device(stream); - auto dict_info_owner = build_chunk_dictionaries( - chunks, col_desc, fragments, compression_, dict_policy_, max_dictionary_size_, stream); + row_group_fragments.host_to_device(stream); + auto dict_info_owner = build_chunk_dictionaries(chunks, + col_desc, + row_group_fragments, + compression_, + dict_policy_, + max_dictionary_size_, + stream); for (size_t p = 0; p < partitions.size(); p++) { for (int rg = 0; rg < num_rg_in_part[p]; rg++) { size_t global_rg = global_rowgroup_base[p] + rg; @@ -1572,7 +1650,72 @@ void writer::impl::write(table_view const& table, std::vector co } } - // Build chunk dictionaries and count pages + // The code preceding this used a uniform fragment size for all columns. Now recompute + // fragments with a (potentially) varying number of fragments per column. + + // first figure out the total number of fragments and calculate the start offset for each column + std::vector frag_offsets; + size_type const total_frags = [&]() { + if (frags_per_column.size() > 0) { + std::exclusive_scan(frags_per_column.data(), + frags_per_column.data() + num_columns + 1, + std::back_inserter(frag_offsets), + 0); + return frag_offsets[num_columns]; + } else { + return 0; + } + }(); + + rmm::device_uvector frag_stats(0, stream); + hostdevice_vector page_fragments(total_frags, stream); + + // update fragments and/or prepare for fragment statistics calculation if necessary + if (total_frags != 0) { + if (stats_granularity_ != statistics_freq::STATISTICS_NONE) { + frag_stats.resize(total_frags, stream); + } + + for (int c = 0; c < num_columns; c++) { + auto frag_offset = frag_offsets[c]; + auto const frag_size = column_frag_size[c]; + + for (size_t p = 0; p < partitions.size(); ++p) { + for (int r = 0; r < num_rg_in_part[p]; r++) { + auto const global_r = global_rowgroup_base[p] + r; + auto const& row_group = md->file(p).row_groups[global_r]; + uint32_t const fragments_in_chunk = + util::div_rounding_up_unsafe(row_group.num_rows, frag_size); + gpu::EncColumnChunk& ck = chunks[r + first_rg_in_part[p]][c]; + ck.fragments = page_fragments.device_ptr(frag_offset); + ck.first_fragment = frag_offset; + + // update the chunk pointer here for each fragment in chunk.fragments + for (uint32_t i = 0; i < fragments_in_chunk; i++) { + page_fragments[frag_offset + i].chunk = + &chunks.device_view()[r + first_rg_in_part[p]][c]; + } + + if (not frag_stats.is_empty()) { ck.stats = frag_stats.data() + frag_offset; } + frag_offset += fragments_in_chunk; + } + } + } + + chunks.host_to_device(stream); + + // re-initialize page fragments + page_fragments.host_to_device(stream); + calculate_page_fragments(page_fragments, column_frag_size); + + // and gather fragment statistics + if (not frag_stats.is_empty()) { + gather_fragment_statistics(frag_stats, + {page_fragments.device_ptr(), static_cast(total_frags)}); + } + } + + // Build chunk dictionaries and count pages. Sends chunks to device. hostdevice_vector comp_page_sizes = init_page_sizes( chunks, col_desc, num_columns, max_page_size_bytes, max_page_size_rows, compression_, stream); diff --git a/cpp/src/io/parquet/writer_impl.hpp b/cpp/src/io/parquet/writer_impl.hpp index 3569281fb47..24c35455ff7 100644 --- a/cpp/src/io/parquet/writer_impl.hpp +++ b/cpp/src/io/parquet/writer_impl.hpp @@ -122,32 +122,42 @@ class writer::impl { private: /** - * @brief Gather page fragments + * @brief Gather row group fragments * - * @param frag Destination page fragments + * This calculates fragments to be used in determining row group boundariesa. + * + * @param frag Destination row group fragments * @param col_desc column description array * @param[in] partitions Information about partitioning of table * @param[in] part_frag_offset A Partition's offset into fragment array * @param fragment_size Number of rows per fragment */ - void init_page_fragments(hostdevice_2dvector& frag, - device_span col_desc, - host_span partitions, - device_span part_frag_offset, - uint32_t fragment_size); + void init_row_group_fragments(hostdevice_2dvector& frag, + device_span col_desc, + host_span partitions, + device_span part_frag_offset, + uint32_t fragment_size); + + /** + * @brief Recalculate page fragments + * + * This calculates fragments to be used to determine page boundaries within + * column chunks. + * + * @param frag Destination page fragments + * @param frag_sizes Array of fragment sizes for each column + */ + void calculate_page_fragments(device_span frag, + host_span frag_sizes); /** * @brief Gather per-fragment statistics * - * @param dst_stats output statistics - * @param frag Input page fragments - * @param col_desc column description array - * @param num_fragments Total number of fragments per column + * @param frag_stats output statistics + * @param frags Input page fragments */ - void gather_fragment_statistics(device_2dspan dst_stats, - device_2dspan frag, - device_span col_desc, - uint32_t num_fragments); + void gather_fragment_statistics(device_span frag_stats, + device_span frags); /** * @brief Initialize encoder pages @@ -220,9 +230,9 @@ class writer::impl { statistics_freq stats_granularity_ = statistics_freq::STATISTICS_NONE; dictionary_policy dict_policy_ = dictionary_policy::ALWAYS; size_t max_dictionary_size_ = default_max_dictionary_size; - size_type max_page_fragment_size_ = default_max_page_fragment_size; bool int96_timestamps = false; int32_t column_index_truncate_length = default_column_index_truncate_length; + std::optional max_page_fragment_size_; // Overall file metadata. Filled in during the process and written during write_chunked_end() std::unique_ptr md; // File footer key-value metadata. Written during write_chunked_end() diff --git a/cpp/tests/io/parquet_chunked_reader_test.cpp b/cpp/tests/io/parquet_chunked_reader_test.cpp index 0cecf62cc9d..ed95d8381d9 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cpp +++ b/cpp/tests/io/parquet_chunked_reader_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -93,6 +93,7 @@ auto write_file(std::vector>& input_columns, cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, *input_table) .max_page_size_bytes(max_page_size_bytes) .max_page_size_rows(max_page_size_rows) + .max_page_fragment_size(cudf::io::default_max_page_fragment_size) .build(); cudf::io::write_parquet(write_opts); diff --git a/cpp/tests/io/parquet_test.cpp b/cpp/tests/io/parquet_test.cpp index 48f69e3ecd3..141c06733a6 100644 --- a/cpp/tests/io/parquet_test.cpp +++ b/cpp/tests/io/parquet_test.cpp @@ -3705,6 +3705,42 @@ TEST_F(ParquetWriterTest, CheckPageRows) EXPECT_EQ(ph.data_page_header.num_values, page_rows); } +TEST_F(ParquetWriterTest, CheckPageRowsAdjusted) +{ + // enough for a few pages with the default 20'000 rows/page + constexpr auto rows_per_page = 20'000; + constexpr auto num_rows = 3 * rows_per_page; + const std::string s1(32, 'a'); + auto col0_elements = + cudf::detail::make_counting_transform_iterator(0, [&](auto i) { return s1; }); + auto col0 = cudf::test::strings_column_wrapper(col0_elements, col0_elements + num_rows); + + auto const expected = table_view{{col0}}; + + auto const filepath = temp_env->get_temp_filepath("CheckPageRowsAdjusted.parquet"); + const cudf::io::parquet_writer_options out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) + .max_page_size_rows(rows_per_page); + cudf::io::write_parquet(out_opts); + + // check first page header and make sure it has only page_rows values + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::FileMetaData fmd; + + read_footer(source, &fmd); + CUDF_EXPECTS(fmd.row_groups.size() > 0, "No row groups found"); + CUDF_EXPECTS(fmd.row_groups[0].columns.size() == 1, "Invalid number of columns"); + auto const& first_chunk = fmd.row_groups[0].columns[0].meta_data; + CUDF_EXPECTS(first_chunk.data_page_offset > 0, "Invalid location for first data page"); + + // read first data page header. sizeof(PageHeader) is not exact, but the thrift encoded + // version should be smaller than size of the struct. + auto const ph = read_page_header( + source, {first_chunk.data_page_offset, sizeof(cudf::io::parquet::PageHeader), 0}); + + EXPECT_LE(ph.data_page_header.num_values, rows_per_page); +} + TEST_F(ParquetWriterTest, Decimal128Stats) { // check that decimal128 min and max statistics are written in network byte order @@ -4046,14 +4082,14 @@ int32_t compare_binary(const std::vector& v1, TEST_F(ParquetWriterTest, LargeColumnIndex) { // create a file large enough to be written in 2 batches (currently 1GB per batch) + // pick fragment size that num_rows is divisible by, so we'll get equal sized row groups const std::string s1(1000, 'a'); const std::string s2(1000, 'b'); - constexpr auto num_rows = 512 * 1024; + constexpr auto num_rows = 512 * 1024; + constexpr auto frag_size = num_rows / 128; - // TODO(ets) need dictionary_policy set to NEVER from #12211. Then - // we don't need to append a number to make the strings unique. auto col0_elements = cudf::detail::make_counting_transform_iterator( - 0, [&](auto i) { return ((i < num_rows) ? s1 : s2) + std::to_string(i); }); + 0, [&](auto i) { return (i < num_rows) ? s1 : s2; }); auto col0 = cudf::test::strings_column_wrapper(col0_elements, col0_elements + 2 * num_rows); auto const expected = table_view{{col0, col0}}; @@ -4063,6 +4099,8 @@ TEST_F(ParquetWriterTest, LargeColumnIndex) cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected) .stats_level(cudf::io::statistics_freq::STATISTICS_COLUMN) .compression(cudf::io::compression_type::NONE) + .dictionary_policy(cudf::io::dictionary_policy::NEVER) + .max_page_fragment_size(frag_size) .row_group_size_bytes(1024 * 1024 * 1024) .row_group_size_rows(num_rows); cudf::io::write_parquet(out_opts); From a96b1508cf96207eef5e26b330d94d20f6bc5054 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 22 Feb 2023 17:12:37 -0800 Subject: [PATCH 46/69] Stop using versioneer to manage versions (#12741) This PR replaces usage of versioneer with hard-coded version numbers in setup.py and __init__.py. Since cudf needs to manage versions across a wide range of file types (CMake, C++, Sphinx and doxygen docs, etc), versioneer cannot be relied on as a single source of truth and therefore does not allow us to single-source our versioning to the Git repo as is intended. Additionally, since the primary means of installing cudf is via conda packages (or now, pip packages), information from the package manager tends to be far more informative than the version strings for troubleshooting and debugging purposes. Conversely, the nonstandard version strings that it produces tend to be problematic for other tools, which at best will ignore such versions but at worst will simply fail. This PR also replaces usage of an environment variable to set the package name for wheels in setup.py, instead moving the renaming logic into the same sed script used to update package versions. This change makes setup.py essentially static, paving the way for migration to pyproject.toml. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12741 --- .gitattributes | 4 - .github/workflows/build.yaml | 2 + .github/workflows/pr.yaml | 2 + .pre-commit-config.yaml | 3 +- ci/checks/copyright.py | 3 +- ci/release/apply_wheel_modifications.sh | 33 + ci/release/update-version.sh | 27 +- python/cudf/_custom_build/backend.py | 37 - python/cudf/cudf/__init__.py | 4 +- python/cudf/cudf/_version.py | 566 ------- python/cudf/pyproject.toml | 6 +- python/cudf/setup.cfg | 12 - python/cudf/setup.py | 28 +- python/cudf/versioneer.py | 1904 ---------------------- python/cudf_kafka/cudf_kafka/_version.py | 566 ------- python/cudf_kafka/setup.cfg | 10 +- python/cudf_kafka/setup.py | 4 +- python/cudf_kafka/versioneer.py | 1904 ---------------------- python/custreamz/custreamz/_version.py | 566 ------- python/custreamz/setup.cfg | 10 +- python/custreamz/setup.py | 4 +- python/custreamz/versioneer.py | 1904 ---------------------- python/dask_cudf/dask_cudf/__init__.py | 6 +- python/dask_cudf/dask_cudf/_version.py | 566 ------- python/dask_cudf/setup.cfg | 10 +- python/dask_cudf/setup.py | 24 +- python/dask_cudf/versioneer.py | 1904 ---------------------- setup.cfg | 7 +- 28 files changed, 79 insertions(+), 10037 deletions(-) delete mode 100644 .gitattributes create mode 100755 ci/release/apply_wheel_modifications.sh delete mode 100644 python/cudf/_custom_build/backend.py delete mode 100644 python/cudf/cudf/_version.py delete mode 100644 python/cudf/versioneer.py delete mode 100644 python/cudf_kafka/cudf_kafka/_version.py delete mode 100644 python/cudf_kafka/versioneer.py delete mode 100644 python/custreamz/custreamz/_version.py delete mode 100644 python/custreamz/versioneer.py delete mode 100644 python/dask_cudf/dask_cudf/_version.py delete mode 100644 python/dask_cudf/versioneer.py diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index fbfe7434d50..00000000000 --- a/.gitattributes +++ /dev/null @@ -1,4 +0,0 @@ -python/cudf/cudf/_version.py export-subst -python/cudf_kafka/cudf_kafka/_version.py export-subst -python/custreamz/custreamz/_version.py export-subst -python/dask_cudf/dask_cudf/_version.py export-subst diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 26d07515f70..fa6704ef04e 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -64,6 +64,7 @@ jobs: package-name: cudf package-dir: python/cudf skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" + uses-setup-env-vars: false wheel-publish-cudf: needs: wheel-build-cudf secrets: inherit @@ -85,6 +86,7 @@ jobs: date: ${{ inputs.date }} package-name: dask_cudf package-dir: python/dask_cudf + uses-setup-env-vars: false wheel-publish-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f33fc15c52f..73df2de20c2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -91,6 +91,7 @@ jobs: package-name: cudf package-dir: python/cudf skbuild-configure-options: "-DCUDF_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF" + uses-setup-env-vars: false wheel-tests-cudf: needs: wheel-build-cudf secrets: inherit @@ -112,6 +113,7 @@ jobs: package-name: dask_cudf package-dir: python/dask_cudf before-wheel: "RAPIDS_PY_WHEEL_NAME=cudf_cu11 rapids-download-wheels-from-s3 ./local-cudf && pip install --no-deps ./local-cudf/cudf*.whl" + uses-setup-env-vars: false wheel-tests-dask-cudf: needs: wheel-build-dask-cudf secrets: inherit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a13b4ca10f1..244fc0d3872 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -144,8 +144,7 @@ repos: exclude: | (?x)^( .*test.*| - ^CHANGELOG.md$| - ^.*versioneer.py$ + ^CHANGELOG.md$ ) default_language_version: diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py index 0f2540c440c..e76d9524c76 100644 --- a/ci/checks/copyright.py +++ b/ci/checks/copyright.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2022, NVIDIA CORPORATION. +# Copyright (c) 2019-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,7 +31,6 @@ ] ExemptFiles = [ re.compile(r"cpp/include/cudf_test/cxxopts.hpp"), - re.compile(r"versioneer[.]py"), ] # this will break starting at year 10000, which is probably OK :) diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh new file mode 100755 index 00000000000..e017b24be6e --- /dev/null +++ b/ci/release/apply_wheel_modifications.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Usage: bash apply_wheel_modifications.sh + +VERSION=${1} +CUDA_SUFFIX=${2} + +# __init__.py versions +sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/cudf/cudf/__init__.py +sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/dask_cudf/dask_cudf/__init__.py +sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/cudf_kafka/cudf_kafka/__init__.py +sed -i "s/__version__ = .*/__version__ = \"${VERSION}\"/g" python/custreamz/custreamz/__init__.py + +# setup.py versions +sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/cudf/setup.py +sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/dask_cudf/setup.py +sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/cudf_kafka/setup.py +sed -i "s/version=.*,/version=\"${VERSION}\",/g" python/custreamz/setup.py + +# cudf setup.py cuda suffixes +sed -i "s/name=\"cudf\"/name=\"cudf${CUDA_SUFFIX}\"/g" python/cudf/setup.py +sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/setup.py +sed -i "s/ptxcompiler/ptxcompiler${CUDA_SUFFIX}/g" python/cudf/setup.py +sed -i "s/cubinlinker/cubinlinker${CUDA_SUFFIX}/g" python/cudf/setup.py + +# cudf pyproject.toml cuda suffixes +sed -i "s/rmm/rmm${CUDA_SUFFIX}/g" python/cudf/pyproject.toml + +# dask_cudf setup.py cuda suffixes +sed -i "s/name=\"dask-cudf\"/name=\"dask-cudf${CUDA_SUFFIX}\"/g" python/dask_cudf/setup.py +# Need to provide the == to avoid modifying the URL +sed -i "s/\"cudf==/\"cudf${CUDA_SUFFIX}==/g" python/dask_cudf/setup.py diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 15d81127450..c8875fda641 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -34,19 +34,27 @@ function sed_runner() { # cpp update sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/CMakeLists.txt -# cpp stream testing update -sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/tests/utilities/identify_stream_usage/CMakeLists.txt - -# Python update +# Python CMakeLists updates sed_runner 's/'"cudf_version .*)"'/'"cudf_version ${NEXT_FULL_TAG})"'/g' python/cudf/CMakeLists.txt - # cpp libcudf_kafka update sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' cpp/libcudf_kafka/CMakeLists.txt # cpp cudf_jni update sed_runner 's/'"VERSION ${CURRENT_SHORT_TAG}.*"'/'"VERSION ${NEXT_FULL_TAG}"'/g' java/src/main/native/CMakeLists.txt +# Python __init__.py updates +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cudf/cudf/__init__.py +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/dask_cudf/dask_cudf/__init__.py +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cudf_kafka/cudf_kafka/__init__.py +sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/custreamz/custreamz/__init__.py + +# Python setup.py updates +sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/cudf/setup.py +sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/dask_cudf/setup.py +sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/cudf_kafka/setup.py +sed_runner "s/version=.*,/version=\"${NEXT_FULL_TAG}\",/g" python/custreamz/setup.py + # rapids-cmake version sed_runner 's/'"branch-.*\/RAPIDS.cmake"'/'"branch-${NEXT_SHORT_TAG}\/RAPIDS.cmake"'/g' fetch_rapids.cmake @@ -81,9 +89,12 @@ sed_runner "s/CUDF_TAG branch-${CURRENT_SHORT_TAG}/CUDF_TAG branch-${NEXT_SHORT_ # Need to distutils-normalize the original version NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))") -# Wheel builds install intra-RAPIDS dependencies from same release -sed_runner "s/rmm{cuda_suffix}.*\",/rmm{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/setup.py -sed_runner "s/cudf{cuda_suffix}==.*\",/cudf{cuda_suffix}==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/setup.py +# Dependency versions in setup.py +sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/setup.py +sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/setup.py + +# Dependency versions in pyproject.toml +sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pyproject.toml for FILE in .github/workflows/*.yaml; do sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" diff --git a/python/cudf/_custom_build/backend.py b/python/cudf/_custom_build/backend.py deleted file mode 100644 index 37b7edf2432..00000000000 --- a/python/cudf/_custom_build/backend.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. - -"""Custom build backend for cudf to get versioned requirements. - -Based on https://setuptools.pypa.io/en/latest/build_meta.html -""" -import os -from functools import wraps - -from setuptools import build_meta as _orig - -# Alias the required bits -build_wheel = _orig.build_wheel -build_sdist = _orig.build_sdist - - -def replace_requirements(func): - @wraps(func) - def wrapper(config_settings=None): - orig_list = getattr(_orig, func.__name__)(config_settings) - append_list = [ - f"rmm{os.getenv('RAPIDS_PY_WHEEL_CUDA_SUFFIX', default='')}" - ] - return orig_list + append_list - - return wrapper - - -get_requires_for_build_wheel = replace_requirements( - _orig.get_requires_for_build_wheel -) -get_requires_for_build_sdist = replace_requirements( - _orig.get_requires_for_build_sdist -) -get_requires_for_build_editable = replace_requirements( - _orig.get_requires_for_build_editable -) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index 05f61ee4f5a..04b64e18594 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -10,7 +10,6 @@ import rmm from cudf import api, core, datasets, testing -from cudf._version import get_versions from cudf.api.extensions import ( register_dataframe_accessor, register_index_accessor, @@ -112,8 +111,7 @@ rmm.register_reinitialize_hook(clear_cache) -__version__ = get_versions()["version"] -del get_versions +__version__ = "23.04.00" __all__ = [ "BaseIndex", diff --git a/python/cudf/cudf/_version.py b/python/cudf/cudf/_version.py deleted file mode 100644 index 60a2afed39b..00000000000 --- a/python/cudf/cudf/_version.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "cudf-" - cfg.versionfile_source = "cudf/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print(f"unable to find command, tried {commands}") - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs) - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces[ - "error" - ] = f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'" - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 57464e83282..49c4d83245f 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -1,7 +1,7 @@ # Copyright (c) 2021-2023, NVIDIA CORPORATION. [build-system] - +build-backend = "setuptools.build_meta" requires = [ "wheel", "setuptools", @@ -13,7 +13,5 @@ requires = [ # Hard pin the patch version used during the build. "pyarrow==10.0.1", "protoc-wheel", - "versioneer", + "rmm==23.4.*", ] -build-backend = "backend" -backend-path = ["_custom_build"] diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg index 59dd5d0179e..8380da371f9 100644 --- a/python/cudf/setup.cfg +++ b/python/cudf/setup.cfg @@ -1,17 +1,5 @@ # Copyright (c) 2018-2023, NVIDIA CORPORATION. -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = cudf/_version.py -versionfile_build = cudf/_version.py -tag_prefix = v -parentdir_prefix = cudf- - [isort] line_length=79 multi_line_output=3 diff --git a/python/cudf/setup.py b/python/cudf/setup.py index 88bc2cfae28..0150c4fe715 100644 --- a/python/cudf/setup.py +++ b/python/cudf/setup.py @@ -1,13 +1,8 @@ # Copyright (c) 2018-2023, NVIDIA CORPORATION. -import os - -import versioneer from setuptools import find_packages from skbuild import setup -cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") - install_requires = [ "cachetools", "cuda-python>=11.7.1,<12.0", @@ -21,9 +16,9 @@ "typing_extensions", # Allow floating minor versions for Arrow. "pyarrow==10", - f"rmm{cuda_suffix}==23.4.*", - f"ptxcompiler{cuda_suffix}", - f"cubinlinker{cuda_suffix}", + "rmm==23.4.*", + "ptxcompiler", + "cubinlinker", "cupy-cuda11x", ] @@ -43,21 +38,9 @@ ] } -if "RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE" in os.environ: - orig_get_versions = versioneer.get_versions - - version_override = os.environ["RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE"] - - def get_versions(): - data = orig_get_versions() - data["version"] = version_override - return data - - versioneer.get_versions = get_versions - setup( - name=f"cudf{cuda_suffix}", - version=versioneer.get_version(), + name="cudf", + version="23.04.00", description="cuDF - GPU Dataframe", url="https://github.com/rapidsai/cudf", author="NVIDIA Corporation", @@ -72,7 +55,6 @@ def get_versions(): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], - cmdclass=versioneer.get_cmdclass(), include_package_data=True, packages=find_packages(include=["cudf", "cudf.*"]), package_data={ diff --git a/python/cudf/versioneer.py b/python/cudf/versioneer.py deleted file mode 100644 index a6537a34ede..00000000000 --- a/python/cudf/versioneer.py +++ /dev/null @@ -1,1904 +0,0 @@ -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from cudf._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -import errno -import json -import os -import re -import subprocess -import sys - -try: - import configparser -except ImportError: - import ConfigParser as configparser - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from cudf._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) diff --git a/python/cudf_kafka/cudf_kafka/_version.py b/python/cudf_kafka/cudf_kafka/_version.py deleted file mode 100644 index 3c1d113fd47..00000000000 --- a/python/cudf_kafka/cudf_kafka/_version.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "cudf_kafka-" - cfg.versionfile_source = "cudf_kafka/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print(f"unable to find command, tried {commands}") - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs) - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces[ - "error" - ] = f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'" - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/python/cudf_kafka/setup.cfg b/python/cudf_kafka/setup.cfg index f884e67908b..ee0d783b184 100644 --- a/python/cudf_kafka/setup.cfg +++ b/python/cudf_kafka/setup.cfg @@ -1,12 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = cudf_kafka/_version.py -versionfile_build = cudf_kafka/_version.py -tag_prefix = v -parentdir_prefix = cudf_kafka- +# Copyright (c) 2020-2023, NVIDIA CORPORATION. [isort] line_length=79 diff --git a/python/cudf_kafka/setup.py b/python/cudf_kafka/setup.py index caadfcac8aa..c39b65cdb55 100644 --- a/python/cudf_kafka/setup.py +++ b/python/cudf_kafka/setup.py @@ -6,7 +6,6 @@ import numpy as np import pyarrow as pa -import versioneer from Cython.Build import cythonize from setuptools import find_packages, setup from setuptools.extension import Extension @@ -87,7 +86,7 @@ setup( name="cudf_kafka", - version=versioneer.get_version(), + version="23.04.00", description="cuDF Kafka Datasource", url="https://github.com/rapidsai/cudf", author="NVIDIA Corporation", @@ -116,7 +115,6 @@ find_packages(include=["cudf_kafka._lib*"]), ["*.pxd"], ), - cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, extras_require=extras_require, zip_safe=False, diff --git a/python/cudf_kafka/versioneer.py b/python/cudf_kafka/versioneer.py deleted file mode 100644 index dbddb6e0fd0..00000000000 --- a/python/cudf_kafka/versioneer.py +++ /dev/null @@ -1,1904 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from cudf_kafka._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - - -import errno -import json -import os -import re -import subprocess -import sys - -try: - import configparser -except ImportError: - import ConfigParser as configparser - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg) as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print(f"unable to find command, tried {commands}") - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs) - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except OSError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except OSError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set {} to '{}'".format(filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print(f"got version from file {versionfile_abs} {ver}") - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from cudf_kafka._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - OSError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy) as f: - old = f.read() - except OSError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in) as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except OSError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) diff --git a/python/custreamz/custreamz/_version.py b/python/custreamz/custreamz/_version.py deleted file mode 100644 index a017486df32..00000000000 --- a/python/custreamz/custreamz/_version.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "v" - cfg.parentdir_prefix = "custreamz-" - cfg.versionfile_source = "custreamz/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print(f"unable to find command, tried {commands}") - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs) - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces[ - "error" - ] = f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'" - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/python/custreamz/setup.cfg b/python/custreamz/setup.cfg index 2ce4eaa82f0..8c038db9349 100644 --- a/python/custreamz/setup.cfg +++ b/python/custreamz/setup.cfg @@ -1,12 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = custreamz/_version.py -versionfile_build = custreamz/_version.py -tag_prefix = v -parentdir_prefix = custreamz- +# Copyright (c) 2020-2023, NVIDIA CORPORATION. [isort] line_length=79 diff --git a/python/custreamz/setup.py b/python/custreamz/setup.py index 2fe12a54855..65a7aac6395 100644 --- a/python/custreamz/setup.py +++ b/python/custreamz/setup.py @@ -1,6 +1,5 @@ # Copyright (c) 2020-2023, NVIDIA CORPORATION. -import versioneer from setuptools import find_packages, setup install_requires = ["cudf_kafka", "cudf"] @@ -9,7 +8,7 @@ setup( name="custreamz", - version=versioneer.get_version(), + version="23.04.00", description="cuStreamz - GPU Accelerated Streaming", url="https://github.com/rapidsai/cudf", author="NVIDIA Corporation", @@ -26,7 +25,6 @@ "Programming Language :: Python :: 3.10", ], packages=find_packages(include=["custreamz", "custreamz.*"]), - cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, extras_require=extras_require, zip_safe=False, diff --git a/python/custreamz/versioneer.py b/python/custreamz/versioneer.py deleted file mode 100644 index 9c9ddae7340..00000000000 --- a/python/custreamz/versioneer.py +++ /dev/null @@ -1,1904 +0,0 @@ -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from custreamz._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -import errno -import json -import os -import re -import subprocess -import sys - -try: - import configparser -except ImportError: - import ConfigParser as configparser - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from custreamz._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) diff --git a/python/dask_cudf/dask_cudf/__init__.py b/python/dask_cudf/dask_cudf/__init__.py index 5e3a9342c25..010e4a104b2 100644 --- a/python/dask_cudf/dask_cudf/__init__.py +++ b/python/dask_cudf/dask_cudf/__init__.py @@ -1,9 +1,8 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. +# Copyright (c) 2018-2023, NVIDIA CORPORATION. from dask.dataframe import from_delayed import cudf -from cudf._version import get_versions from . import backends from .core import DataFrame, Series, concat, from_cudf, from_dask_dataframe @@ -15,8 +14,7 @@ except ImportError: pass -__version__ = get_versions()["version"] -del get_versions +__version__ = "23.04.00" __all__ = [ "DataFrame", diff --git a/python/dask_cudf/dask_cudf/_version.py b/python/dask_cudf/dask_cudf/_version.py deleted file mode 100644 index f0dbcac0017..00000000000 --- a/python/dask_cudf/dask_cudf/_version.py +++ /dev/null @@ -1,566 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "$Format:%d$" - git_full = "$Format:%H$" - git_date = "$Format:%ci$" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "pep440" - cfg.tag_prefix = "" - cfg.parentdir_prefix = "dask_cudf-" - cfg.versionfile_source = "dask_cudf/_version.py" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except OSError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print(f"unable to find command, tried {commands}") - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs) - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except OSError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = {r.strip() for r in refnames.strip("()").split(",")} - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = {r for r in refs if re.search(r"\d", r)} - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces[ - "error" - ] = f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'" - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords( - get_keywords(), cfg.tag_prefix, verbose - ) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split("/"): - root = os.path.dirname(root) - except NameError: - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None, - } - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } diff --git a/python/dask_cudf/setup.cfg b/python/dask_cudf/setup.cfg index f45bdf00430..66f4b8891d0 100644 --- a/python/dask_cudf/setup.cfg +++ b/python/dask_cudf/setup.cfg @@ -1,12 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -[versioneer] -VCS = git -style = pep440 -versionfile_source = dask_cudf/_version.py -versionfile_build = dask_cudf/_version.py -tag_prefix = -parentdir_prefix = dask_cudf- +# Copyright (c) 2020-2023, NVIDIA CORPORATION. [isort] line_length=79 diff --git a/python/dask_cudf/setup.py b/python/dask_cudf/setup.py index 04145d23978..8611f2379f7 100644 --- a/python/dask_cudf/setup.py +++ b/python/dask_cudf/setup.py @@ -1,19 +1,14 @@ # Copyright (c) 2019-2023, NVIDIA CORPORATION. -import os - -import versioneer from setuptools import find_packages, setup -cuda_suffix = os.getenv("RAPIDS_PY_WHEEL_CUDA_SUFFIX", default="") - install_requires = [ "dask>=2023.1.1", "distributed>=2023.1.1", "fsspec>=0.6.0", "numpy", "pandas>=1.0,<1.6.0dev0", - f"cudf{cuda_suffix}==23.4.*", + "cudf==23.4.*", "cupy-cuda11x", ] @@ -27,21 +22,9 @@ ] } -if "RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE" in os.environ: - orig_get_versions = versioneer.get_versions - - version_override = os.environ["RAPIDS_PY_WHEEL_VERSIONEER_OVERRIDE"] - - def get_versions(): - data = orig_get_versions() - data["version"] = version_override - return data - - versioneer.get_versions = get_versions - setup( - name=f"dask-cudf{cuda_suffix}", - version=versioneer.get_version(), + name="dask-cudf", + version="23.04.00", description="Utilities for Dask and cuDF interactions", url="https://github.com/rapidsai/cudf", author="NVIDIA Corporation", @@ -57,7 +40,6 @@ def get_versions(): "Programming Language :: Python :: 3.10", ], packages=find_packages(exclude=["tests", "tests.*"]), - cmdclass=versioneer.get_cmdclass(), install_requires=install_requires, extras_require=extras_require, ) diff --git a/python/dask_cudf/versioneer.py b/python/dask_cudf/versioneer.py deleted file mode 100644 index a560f2e8797..00000000000 --- a/python/dask_cudf/versioneer.py +++ /dev/null @@ -1,1904 +0,0 @@ -# Version: 0.18 - -"""The Versioneer - like a rocketeer, but for versions. - -The Versioneer -============== - -* like a rocketeer, but for versions! -* https://github.com/warner/python-versioneer -* Brian Warner -* License: Public Domain -* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy -* [![Latest Version] -(https://pypip.in/version/versioneer/badge.svg?style=flat) -](https://pypi.python.org/pypi/versioneer/) -* [![Build Status] -(https://travis-ci.org/warner/python-versioneer.png?branch=master) -](https://travis-ci.org/warner/python-versioneer) - -This is a tool for managing a recorded version number in distutils-based -python projects. The goal is to remove the tedious and error-prone "update -the embedded version string" step from your release process. Making a new -release should be as easy as recording a new tag in your version-control -system, and maybe making new tarballs. - - -## Quick Install - -* `pip install versioneer` to somewhere to your $PATH -* add a `[versioneer]` section to your setup.cfg (see below) -* run `versioneer install` in your source tree, commit the results - -## Version Identifiers - -Source trees come from a variety of places: - -* a version-control system checkout (mostly used by developers) -* a nightly tarball, produced by build automation -* a snapshot tarball, produced by a web-based VCS browser, like github's - "tarball from tag" feature -* a release tarball, produced by "setup.py sdist", distributed through PyPI - -Within each source tree, the version identifier (either a string or a number, -this tool is format-agnostic) can come from a variety of places: - -* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows - about recent "tags" and an absolute revision-id -* the name of the directory into which the tarball was unpacked -* an expanded VCS keyword ($Id$, etc) -* a `_version.py` created by some earlier build step - -For released software, the version identifier is closely related to a VCS -tag. Some projects use tag names that include more than just the version -string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool -needs to strip the tag prefix to extract the version identifier. For -unreleased software (between tags), the version identifier should provide -enough information to help developers recreate the same tree, while also -giving them an idea of roughly how old the tree is (after version 1.2, before -version 1.3). Many VCS systems can report a description that captures this, -for example `git describe --tags --dirty --always` reports things like -"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the -0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has -uncommitted changes. - -The version identifier is used for multiple purposes: - -* to allow the module to self-identify its version: `myproject.__version__` -* to choose a name and prefix for a 'setup.py sdist' tarball - -## Theory of Operation - -Versioneer works by adding a special `_version.py` file into your source -tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. - -`_version.py` also contains `$Revision$` markers, and the installation -process marks `_version.py` to have this marker rewritten with a tag name -during the `git archive` command. As a result, generated tarballs will -contain enough information to get the proper version. - -To allow `setup.py` to compute a version too, a `versioneer.py` is added to -the top level of your source tree, next to `setup.py` and the `setup.cfg` -that configures it. This overrides several distutils/setuptools commands to -compute the version when invoked, and changes `setup.py build` and `setup.py -sdist` to replace `_version.py` with a small static file that contains just -the generated version data. - -## Installation - -See [INSTALL.md](./INSTALL.md) for detailed installation instructions. - -## Version-String Flavors - -Code which uses Versioneer can learn about its version string at runtime by -importing `_version` from your main `__init__.py` file and running the -`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can -import the top-level `versioneer.py` and run `get_versions()`. - -Both functions return a dictionary with different flavors of version -information: - -* `['version']`: A condensed version string, rendered using the selected - style. This is the most commonly used value for the project's version - string. The default "pep440" style yields strings like `0.11`, - `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section - below for alternative styles. - -* `['full-revisionid']`: detailed revision identifier. For Git, this is the - full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". - -* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the - commit date in ISO 8601 format. This will be None if the date is not - available. - -* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that - this is only accurate if run in a VCS checkout, otherwise it is likely to - be False or None - -* `['error']`: if the version string could not be computed, this will be set - to a string describing the problem, otherwise it will be None. It may be - useful to throw an exception in setup.py if this is set, to avoid e.g. - creating tarballs with a version string of "unknown". - -Some variants are more useful than others. Including `full-revisionid` in a -bug report should allow developers to reconstruct the exact code being tested -(or indicate the presence of local changes that should be shared with the -developers). `version` is suitable for display in an "about" box or a CLI -`--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. - -The installer adds the following text to your `__init__.py` to place a basic -version in `YOURPROJECT.__version__`: - - from ._version import get_versions - __version__ = get_versions()['version'] - del get_versions - -## Styles - -The setup.cfg `style=` configuration controls how the VCS information is -rendered into a version string. - -The default style, "pep440", produces a PEP440-compliant string, equal to the -un-prefixed tag name for actual releases, and containing an additional "local -version" section with more detail for in-between builds. For Git, this is -TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags ---dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the -tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and -that this commit is two revisions ("+2") beyond the "0.11" tag. For released -software (exactly equal to a known tag), the identifier will only contain the -stripped tag, e.g. "0.11". - -Other styles are available. See [details.md](details.md) in the Versioneer -source tree for descriptions. - -## Debugging - -Versioneer tries to avoid fatal errors: if something goes wrong, it will tend -to return a version of "0+unknown". To investigate the problem, run `setup.py -version`, which will run the version-lookup code in a verbose mode, and will -display the full contents of `get_versions()` (including the `error` string, -which may help identify what went wrong). - -## Known Limitations - -Some situations are known to cause problems for Versioneer. This details the -most significant ones. More can be found on Github -[issues page](https://github.com/warner/python-versioneer/issues). - -### Subprojects - -Versioneer has limited support for source trees in which `setup.py` is not in -the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are -two common reasons why `setup.py` might not be in the root: - -* Source trees which contain multiple subprojects, such as - [Buildbot](https://github.com/buildbot/buildbot), which contains both - "master" and "slave" subprojects, each with their own `setup.py`, - `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI - distributions (and upload multiple independently-installable tarballs). -* Source trees whose main purpose is to contain a C library, but which also - provide bindings to Python (and perhaps other langauges) in subdirectories. - -Versioneer will look for `.git` in parent directories, and most operations -should get the right version string. However `pip` and `setuptools` have bugs -and implementation details which frequently cause `pip install .` from a -subproject directory to fail to find a correct version string (so it usually -defaults to `0+unknown`). - -`pip install --editable .` should work correctly. `setup.py install` might -work too. - -Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in -some later version. - -[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking -this issue. The discussion in -[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the -issue from the Versioneer side in more detail. -[pip PR#3176](https://github.com/pypa/pip/pull/3176) and -[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve -pip to let Versioneer work correctly. - -Versioneer-0.16 and earlier only looked for a `.git` directory next to the -`setup.cfg`, so subprojects were completely unsupported with those releases. - -### Editable installs with setuptools <= 18.5 - -`setup.py develop` and `pip install --editable .` allow you to install a -project into a virtualenv once, then continue editing the source code (and -test) without re-installing after every change. - -"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a -convenient way to specify executable scripts that should be installed along -with the python package. - -These both work as expected when using modern setuptools. When using -setuptools-18.5 or earlier, however, certain operations will cause -`pkg_resources.DistributionNotFound` errors when running the entrypoint -script, which must be resolved by re-installing the package. This happens -when the install happens with one version, then the egg_info data is -regenerated while a different version is checked out. Many setup.py commands -cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into -a different virtualenv), so this can be surprising. - -[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes -this one, but upgrading to a newer version of setuptools should probably -resolve it. - -### Unicode version strings - -While Versioneer works (and is continually tested) with both Python 2 and -Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. -Newer releases probably generate unicode version strings on py2. It's not -clear that this is wrong, but it may be surprising for applications when then -write these strings to a network connection or include them in bytes-oriented -APIs like cryptographic checksums. - -[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates -this question. - - -## Updating Versioneer - -To upgrade your project to a new release of Versioneer, do the following: - -* install the new Versioneer (`pip install -U versioneer` or equivalent) -* edit `setup.cfg`, if necessary, to include any new configuration settings - indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. -* re-run `versioneer install` in your source tree, to replace - `SRC/_version.py` -* commit any changed files - -## Future Directions - -This tool is designed to make it easily extended to other version-control -systems: all VCS-specific components are in separate directories like -src/git/ . The top-level `versioneer.py` script is assembled from these -components by running make-versioneer.py . In the future, make-versioneer.py -will take a VCS name as an argument, and will construct a version of -`versioneer.py` that is specific to the given VCS. It might also take the -configuration arguments that are currently provided manually during -installation by editing setup.py . Alternatively, it might go the other -direction and include code from all supported VCS systems, reducing the -number of intermediate scripts. - - -## License - -To make Versioneer easier to embed, all its code is dedicated to the public -domain. The `_version.py` that it creates is also in the public domain. -Specifically, both are released under the Creative Commons "Public Domain -Dedication" license (CC0-1.0), as described in -https://creativecommons.org/publicdomain/zero/1.0/ . - -""" - -from __future__ import print_function - -import errno -import json -import os -import re -import subprocess -import sys - -try: - import configparser -except ImportError: - import ConfigParser as configparser - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_root(): - """Get the project root directory. - - We require that all commands are run from the project root, i.e. the - directory that contains setup.py, setup.cfg, and versioneer.py . - """ - root = os.path.realpath(os.path.abspath(os.getcwd())) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - # allow 'python path/to/setup.py COMMAND' - root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) - setup_py = os.path.join(root, "setup.py") - versioneer_py = os.path.join(root, "versioneer.py") - if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): - err = ( - "Versioneer was unable to run the project root directory. " - "Versioneer requires setup.py to be executed from " - "its immediate directory (like 'python setup.py COMMAND'), " - "or in a way that lets it use sys.argv[0] to find the root " - "(like 'python path/to/setup.py COMMAND')." - ) - raise VersioneerBadRootError(err) - try: - # Certain runtime workflows (setup.py install/develop in a setuptools - # tree) execute all dependencies in a single python process, so - # "versioneer" may be imported multiple times, and python's shared - # module-import table will cache the first one. So we can't use - # os.path.dirname(__file__), as that will find whichever - # versioneer.py was first imported, even in later projects. - me = os.path.realpath(os.path.abspath(__file__)) - me_dir = os.path.normcase(os.path.splitext(me)[0]) - vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) - if me_dir != vsr_dir: - print( - "Warning: build in %s is using versioneer.py from %s" - % (os.path.dirname(me), versioneer_py) - ) - except NameError: - pass - return root - - -def get_config_from_root(root): - """Read the project setup.cfg file to determine Versioneer config.""" - # This might raise EnvironmentError (if setup.cfg is missing), or - # configparser.NoSectionError (if it lacks a [versioneer] section), or - # configparser.NoOptionError (if it lacks "VCS="). See the docstring at - # the top of versioneer.py for instructions on writing your setup.cfg . - setup_cfg = os.path.join(root, "setup.cfg") - parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: - parser.readfp(f) - VCS = parser.get("versioneer", "VCS") # mandatory - - def get(parser, name): - if parser.has_option("versioneer", name): - return parser.get("versioneer", name) - return None - - cfg = VersioneerConfig() - cfg.VCS = VCS - cfg.style = get(parser, "style") or "" - cfg.versionfile_source = get(parser, "versionfile_source") - cfg.versionfile_build = get(parser, "versionfile_build") - cfg.tag_prefix = get(parser, "tag_prefix") - if cfg.tag_prefix in ("''", '""'): - cfg.tag_prefix = "" - cfg.parentdir_prefix = get(parser, "parentdir_prefix") - cfg.verbose = get(parser, "verbose") - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -# these dictionaries contain VCS-specific tools -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - - return decorate - - -def run_command( - commands, args, cwd=None, verbose=False, hide_stderr=False, env=None -): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen( - [c] + args, - cwd=cwd, - env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr else None), - ) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % dispcmd) - print("stdout was %s" % stdout) - return None, p.returncode - return stdout, p.returncode - - -LONG_VERSION_PY[ - "git" -] = r''' -# This file helps to compute a version number in source trees obtained from -# git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (built by setup.py sdist) and build -# directories (produced by setup.py build) will contain a much shorter file -# that just contains the computed version number. - -# This file is released into the public domain. Generated by -# versioneer-0.18 (https://github.com/warner/python-versioneer) - -"""Git implementation of _version.py.""" - -import errno -import os -import re -import subprocess -import sys - - -def get_keywords(): - """Get the keywords needed to look up the version information.""" - # these strings will be replaced by git during git-archive. - # setup.py/versioneer.py will grep for the variable names, so they must - # each be defined on a line of their own. _version.py will just call - # get_keywords(). - git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" - git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" - git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" - keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} - return keywords - - -class VersioneerConfig: - """Container for Versioneer configuration parameters.""" - - -def get_config(): - """Create, populate and return the VersioneerConfig() object.""" - # these strings are filled in when 'setup.py versioneer' creates - # _version.py - cfg = VersioneerConfig() - cfg.VCS = "git" - cfg.style = "%(STYLE)s" - cfg.tag_prefix = "%(TAG_PREFIX)s" - cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" - cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" - cfg.verbose = False - return cfg - - -class NotThisMethod(Exception): - """Exception raised if a method is not valid for the current scenario.""" - - -LONG_VERSION_PY = {} -HANDLERS = {} - - -def register_vcs_handler(vcs, method): # decorator - """Decorator to mark a method as the handler for a particular VCS.""" - def decorate(f): - """Store f in HANDLERS[vcs][method].""" - if vcs not in HANDLERS: - HANDLERS[vcs] = {} - HANDLERS[vcs][method] = f - return f - return decorate - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): - """Call the given command(s).""" - assert isinstance(commands, list) - p = None - for c in commands: - try: - dispcmd = str([c] + args) - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %%s" %% dispcmd) - print(e) - return None, None - else: - if verbose: - print("unable to find command, tried %%s" %% (commands,)) - return None, None - stdout = p.communicate()[0].strip() - if sys.version_info[0] >= 3: - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %%s (error)" %% dispcmd) - print("stdout was %%s" %% stdout) - return None, p.returncode - return stdout, p.returncode - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print("Tried directories %%s but none started with prefix %%s" %% - (str(rootdirs), parentdir_prefix)) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %%d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) - if verbose: - print("discarding '%%s', no digits" %% ",".join(refs - tags)) - if verbose: - print("likely tags: %%s" %% ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] - if verbose: - print("picking %%s" %% r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) - if rc != 0: - if verbose: - print("Directory %%s not under git control" %% root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%%s*" %% tag_prefix], - cwd=root) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%%s'" - %% describe_out) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%%s' doesn't start with prefix '%%s'" - print(fmt %% (full_tag, tag_prefix)) - pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" - %% (full_tag, tag_prefix)) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], - cwd=root)[0].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], - pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%%d" %% pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%%d" %% pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%%s" %% pieces["short"] - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%%s" %% pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%%d" %% pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%%s'" %% style) - - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} - - -def get_versions(): - """Get version information or return default if unable to do so.""" - # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have - # __file__, we can work backwards from there to the root. Some - # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded keywords. - - cfg = get_config() - verbose = cfg.verbose - - try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) - except NotThisMethod: - pass - - try: - root = os.path.realpath(__file__) - # versionfile_source is the relative path from the top of the source - # tree (where the .git directory might live) to this file. Invert - # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): - root = os.path.dirname(root) - except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} - - try: - pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) - return render(pieces, cfg.style) - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - except NotThisMethod: - pass - - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} -''' - - -@register_vcs_handler("git", "get_keywords") -def git_get_keywords(versionfile_abs): - """Extract version information from the given file.""" - # the code embedded in _version.py can just fetch the value of these - # keywords. When used from setup.py, we don't want to import _version.py, - # so we do it with a regexp instead. This function is not used from - # _version.py. - keywords = {} - try: - f = open(versionfile_abs, "r") - for line in f.readlines(): - if line.strip().startswith("git_refnames ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["refnames"] = mo.group(1) - if line.strip().startswith("git_full ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["full"] = mo.group(1) - if line.strip().startswith("git_date ="): - mo = re.search(r'=\s*"(.*)"', line) - if mo: - keywords["date"] = mo.group(1) - f.close() - except EnvironmentError: - pass - return keywords - - -@register_vcs_handler("git", "keywords") -def git_versions_from_keywords(keywords, tag_prefix, verbose): - """Get version information from git keywords.""" - if not keywords: - raise NotThisMethod("no keywords at all, weird") - date = keywords.get("date") - if date is not None: - # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant - # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 - # -like" string, which we must then edit to make compliant), because - # it's been around since git-1.5.3, and it's too difficult to - # discover which version we're using, or to work around using an - # older one. - date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - refnames = keywords["refnames"].strip() - if refnames.startswith("$Format"): - if verbose: - print("keywords are unexpanded, not using") - raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) - # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of - # just "foo-1.0". If we see a "tag: " prefix, prefer those. - TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) - if not tags: - # Either we're using git < 1.8.3, or there really are no tags. We use - # a heuristic: assume all version tags have a digit. The old git %d - # expansion behaves like git log --decorate=short and strips out the - # refs/heads/ and refs/tags/ prefixes that would let us distinguish - # between branches and tags. By ignoring refnames without digits, we - # filter out many common branch names like "release" and - # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) - if verbose: - print("discarding '%s', no digits" % ",".join(refs - tags)) - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - for ref in sorted(tags): - # sorting will prefer e.g. "2.0" over "2.0rc1" - if ref.startswith(tag_prefix): - r = ref[len(tag_prefix) :] - if verbose: - print("picking %s" % r) - return { - "version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": None, - "date": date, - } - # no suitable tags, so version is "0+unknown", but full hex is still there - if verbose: - print("no suitable tags, using unknown + full revision id") - return { - "version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, - "error": "no suitable tags", - "date": None, - } - - -@register_vcs_handler("git", "pieces_from_vcs") -def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): - """Get version from 'git describe' in the root of the source tree. - - This only gets called if the git-archive 'subst' keywords were *not* - expanded, and _version.py hasn't already been rewritten with a short - version string, meaning we're inside a checked out source tree. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - - out, rc = run_command( - GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True - ) - if rc != 0: - if verbose: - print("Directory %s not under git control" % root) - raise NotThisMethod("'git rev-parse --git-dir' returned error") - - # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] - # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command( - GITS, - [ - "describe", - "--tags", - "--dirty", - "--always", - "--long", - "--match", - "%s*" % tag_prefix, - ], - cwd=root, - ) - # --long was added in git-1.5.5 - if describe_out is None: - raise NotThisMethod("'git describe' failed") - describe_out = describe_out.strip() - full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if full_out is None: - raise NotThisMethod("'git rev-parse' failed") - full_out = full_out.strip() - - pieces = {} - pieces["long"] = full_out - pieces["short"] = full_out[:7] # maybe improved later - pieces["error"] = None - - # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] - # TAG might have hyphens. - git_describe = describe_out - - # look for -dirty suffix - dirty = git_describe.endswith("-dirty") - pieces["dirty"] = dirty - if dirty: - git_describe = git_describe[: git_describe.rindex("-dirty")] - - # now we have TAG-NUM-gHEX or HEX - - if "-" in git_describe: - # TAG-NUM-gHEX - mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) - if not mo: - # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ( - "unable to parse git-describe output: '%s'" % describe_out - ) - return pieces - - # tag - full_tag = mo.group(1) - if not full_tag.startswith(tag_prefix): - if verbose: - fmt = "tag '%s' doesn't start with prefix '%s'" - print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( - full_tag, - tag_prefix, - ) - return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix) :] - - # distance: number of commits since tag - pieces["distance"] = int(mo.group(2)) - - # commit: short hex revision ID - pieces["short"] = mo.group(3) - - else: - # HEX: no tags - pieces["closest-tag"] = None - count_out, rc = run_command( - GITS, ["rev-list", "HEAD", "--count"], cwd=root - ) - pieces["distance"] = int(count_out) # total number of commits - - # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ - 0 - ].strip() - pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) - - return pieces - - -def do_vcs_install(manifest_in, versionfile_source, ipy): - """Git-specific installation logic for Versioneer. - - For Git, this means creating/changing .gitattributes to mark _version.py - for export-subst keyword substitution. - """ - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source] - if ipy: - files.append(ipy) - try: - me = __file__ - if me.endswith(".pyc") or me.endswith(".pyo"): - me = os.path.splitext(me)[0] + ".py" - versioneer_file = os.path.relpath(me) - except NameError: - versioneer_file = "versioneer.py" - files.append(versioneer_file) - present = False - try: - f = open(".gitattributes", "r") - for line in f.readlines(): - if line.strip().startswith(versionfile_source): - if "export-subst" in line.strip().split()[1:]: - present = True - f.close() - except EnvironmentError: - pass - if not present: - f = open(".gitattributes", "a+") - f.write("%s export-subst\n" % versionfile_source) - f.close() - files.append(".gitattributes") - run_command(GITS, ["add", "--"] + files) - - -def versions_from_parentdir(parentdir_prefix, root, verbose): - """Try to determine the version from the parent directory name. - - Source tarballs conventionally unpack into a directory that includes both - the project name and a version string. We will also support searching up - two directory levels for an appropriately named parent directory - """ - rootdirs = [] - - for i in range(3): - dirname = os.path.basename(root) - if dirname.startswith(parentdir_prefix): - return { - "version": dirname[len(parentdir_prefix) :], - "full-revisionid": None, - "dirty": False, - "error": None, - "date": None, - } - else: - rootdirs.append(root) - root = os.path.dirname(root) # up a level - - if verbose: - print( - "Tried directories %s but none started with prefix %s" - % (str(rootdirs), parentdir_prefix) - ) - raise NotThisMethod("rootdir doesn't start with parentdir_prefix") - - -SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.18) from -# revision-control system data, or from the parent directory name of an -# unpacked source archive. Distribution tarballs contain a pre-generated copy -# of this file. - -import json - -version_json = ''' -%s -''' # END VERSION_JSON - - -def get_versions(): - return json.loads(version_json) -""" - - -def versions_from_file(filename): - """Try to determine the version from _version.py if present.""" - try: - with open(filename) as f: - contents = f.read() - except EnvironmentError: - raise NotThisMethod("unable to read _version.py") - mo = re.search( - r"version_json = '''\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - mo = re.search( - r"version_json = '''\r\n(.*)''' # END VERSION_JSON", - contents, - re.M | re.S, - ) - if not mo: - raise NotThisMethod("no version_json in _version.py") - return json.loads(mo.group(1)) - - -def write_to_version_file(filename, versions): - """Write the given version number to the given _version.py file.""" - os.unlink(filename) - contents = json.dumps( - versions, sort_keys=True, indent=1, separators=(",", ": ") - ) - with open(filename, "w") as f: - f.write(SHORT_VERSION_PY % contents) - - print("set %s to '%s'" % (filename, versions["version"])) - - -def plus_or_dot(pieces): - """Return a + if we don't already have one, else return a .""" - if "+" in pieces.get("closest-tag", ""): - return "." - return "+" - - -def render_pep440(pieces): - """Build up version string, with post-release "local version identifier". - - Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you - get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty - - Exceptions: - 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += plus_or_dot(pieces) - rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - else: - # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) - if pieces["dirty"]: - rendered += ".dirty" - return rendered - - -def render_pep440_pre(pieces): - """TAG[.post.devDISTANCE] -- No -dirty. - - Exceptions: - 1: no tags. 0.post.devDISTANCE - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += ".post.dev%d" % pieces["distance"] - else: - # exception #1 - rendered = "0.post.dev%d" % pieces["distance"] - return rendered - - -def render_pep440_post(pieces): - """TAG[.postDISTANCE[.dev0]+gHEX] . - - The ".dev0" means dirty. Note that .dev0 sorts backwards - (a dirty tree will appear "older" than the corresponding clean one), - but you shouldn't be releasing software with -dirty anyways. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += plus_or_dot(pieces) - rendered += "g%s" % pieces["short"] - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - rendered += "+g%s" % pieces["short"] - return rendered - - -def render_pep440_old(pieces): - """TAG[.postDISTANCE[.dev0]] . - - The ".dev0" means dirty. - - Exceptions: - 1: no tags. 0.postDISTANCE[.dev0] - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"] or pieces["dirty"]: - rendered += ".post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - else: - # exception #1 - rendered = "0.post%d" % pieces["distance"] - if pieces["dirty"]: - rendered += ".dev0" - return rendered - - -def render_git_describe(pieces): - """TAG[-DISTANCE-gHEX][-dirty]. - - Like 'git describe --tags --dirty --always'. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - if pieces["distance"]: - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render_git_describe_long(pieces): - """TAG-DISTANCE-gHEX[-dirty]. - - Like 'git describe --tags --dirty --always -long'. - The distance/hash is unconditional. - - Exceptions: - 1: no tags. HEX[-dirty] (note: no 'g' prefix) - """ - if pieces["closest-tag"]: - rendered = pieces["closest-tag"] - rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) - else: - # exception #1 - rendered = pieces["short"] - if pieces["dirty"]: - rendered += "-dirty" - return rendered - - -def render(pieces, style): - """Render the given version pieces into the requested style.""" - if pieces["error"]: - return { - "version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None, - } - - if not style or style == "default": - style = "pep440" # the default - - if style == "pep440": - rendered = render_pep440(pieces) - elif style == "pep440-pre": - rendered = render_pep440_pre(pieces) - elif style == "pep440-post": - rendered = render_pep440_post(pieces) - elif style == "pep440-old": - rendered = render_pep440_old(pieces) - elif style == "git-describe": - rendered = render_git_describe(pieces) - elif style == "git-describe-long": - rendered = render_git_describe_long(pieces) - else: - raise ValueError("unknown style '%s'" % style) - - return { - "version": rendered, - "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], - "error": None, - "date": pieces.get("date"), - } - - -class VersioneerBadRootError(Exception): - """The project root directory is unknown or missing key files.""" - - -def get_versions(verbose=False): - """Get the project version from whatever source is available. - - Returns dict with two keys: 'version' and 'full'. - """ - if "versioneer" in sys.modules: - # see the discussion in cmdclass.py:get_cmdclass() - del sys.modules["versioneer"] - - root = get_root() - cfg = get_config_from_root(root) - - assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" - handlers = HANDLERS.get(cfg.VCS) - assert handlers, "unrecognized VCS '%s'" % cfg.VCS - verbose = verbose or cfg.verbose - assert ( - cfg.versionfile_source is not None - ), "please set versioneer.versionfile_source" - assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" - - versionfile_abs = os.path.join(root, cfg.versionfile_source) - - # extract version from first of: _version.py, VCS command (e.g. 'git - # describe'), parentdir. This is meant to work for developers using a - # source checkout, for users of a tarball created by 'setup.py sdist', - # and for users of a tarball/zipball created by 'git archive' or github's - # download-from-tag feature or the equivalent in other VCSes. - - get_keywords_f = handlers.get("get_keywords") - from_keywords_f = handlers.get("keywords") - if get_keywords_f and from_keywords_f: - try: - keywords = get_keywords_f(versionfile_abs) - ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) - if verbose: - print("got version from expanded keyword %s" % ver) - return ver - except NotThisMethod: - pass - - try: - ver = versions_from_file(versionfile_abs) - if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) - return ver - except NotThisMethod: - pass - - from_vcs_f = handlers.get("pieces_from_vcs") - if from_vcs_f: - try: - pieces = from_vcs_f(cfg.tag_prefix, root, verbose) - ver = render(pieces, cfg.style) - if verbose: - print("got version from VCS %s" % ver) - return ver - except NotThisMethod: - pass - - try: - if cfg.parentdir_prefix: - ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) - if verbose: - print("got version from parentdir %s" % ver) - return ver - except NotThisMethod: - pass - - if verbose: - print("unable to compute version") - - return { - "version": "0+unknown", - "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", - "date": None, - } - - -def get_version(): - """Get the short version string for this project.""" - return get_versions()["version"] - - -def get_cmdclass(): - """Get the custom setuptools/distutils subclasses used by Versioneer.""" - if "versioneer" in sys.modules: - del sys.modules["versioneer"] - # this fixes the "python setup.py develop" case (also 'install' and - # 'easy_install .'), in which subdependencies of the main project are - # built (using setup.py bdist_egg) in the same python process. Assume - # a main project A and a dependency B, which use different versions - # of Versioneer. A's setup.py imports A's Versioneer, leaving it in - # sys.modules by the time B's setup.py is executed, causing B to run - # with the wrong versioneer. Setuptools wraps the sub-dep builds in a - # sandbox that restores sys.modules to it's pre-build state, so the - # parent is protected against the child's "import versioneer". By - # removing ourselves from sys.modules here, before the child build - # happens, we protect the child from the parent's versioneer too. - # Also see https://github.com/warner/python-versioneer/issues/52 - - cmds = {} - - # we add "version" to both distutils and setuptools - from distutils.core import Command - - class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - vers = get_versions(verbose=True) - print("Version: %s" % vers["version"]) - print(" full-revisionid: %s" % vers.get("full-revisionid")) - print(" dirty: %s" % vers.get("dirty")) - print(" date: %s" % vers.get("date")) - if vers["error"]: - print(" error: %s" % vers["error"]) - - cmds["version"] = cmd_version - - # we override "build_py" in both distutils and setuptools - # - # most invocation pathways end up running build_py: - # distutils/build -> build_py - # distutils/install -> distutils/build ->.. - # setuptools/bdist_wheel -> distutils/install ->.. - # setuptools/bdist_egg -> distutils/install_lib -> build_py - # setuptools/install -> bdist_egg ->.. - # setuptools/develop -> ? - # pip install: - # copies source tree to a tempdir before running egg_info/etc - # if .git isn't copied too, 'git describe' will fail - # then does setup.py bdist_wheel, or sometimes setup.py install - # setup.py egg_info -> ? - - # we override different "build_py" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.build_py import build_py as _build_py - else: - from distutils.command.build_py import build_py as _build_py - - class cmd_build_py(_build_py): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - _build_py.run(self) - # now locate _version.py in the new build/ directory and replace - # it with an updated value - if cfg.versionfile_build: - target_versionfile = os.path.join( - self.build_lib, cfg.versionfile_build - ) - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - cmds["build_py"] = cmd_build_py - - if "cx_Freeze" in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe - - # nczeczulin reports that py2exe won't like the pep440-style string - # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. - # setup(console=[{ - # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION - # "product_version": versioneer.get_version(), - # ... - - class cmd_build_exe(_build_exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _build_exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["build_exe"] = cmd_build_exe - del cmds["build_py"] - - if "py2exe" in sys.modules: # py2exe enabled? - try: - from py2exe.distutils_buildexe import py2exe as _py2exe # py3 - except ImportError: - from py2exe.build_exe import py2exe as _py2exe # py2 - - class cmd_py2exe(_py2exe): - def run(self): - root = get_root() - cfg = get_config_from_root(root) - versions = get_versions() - target_versionfile = cfg.versionfile_source - print("UPDATING %s" % target_versionfile) - write_to_version_file(target_versionfile, versions) - - _py2exe.run(self) - os.unlink(target_versionfile) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - cmds["py2exe"] = cmd_py2exe - - # we override different "sdist" commands for both environments - if "setuptools" in sys.modules: - from setuptools.command.sdist import sdist as _sdist - else: - from distutils.command.sdist import sdist as _sdist - - class cmd_sdist(_sdist): - def run(self): - versions = get_versions() - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old - # version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - root = get_root() - cfg = get_config_from_root(root) - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory - # (remembering that it may be a hardlink) and replace it with an - # updated value - target_versionfile = os.path.join(base_dir, cfg.versionfile_source) - print("UPDATING %s" % target_versionfile) - write_to_version_file( - target_versionfile, self._versioneer_generated_versions - ) - - cmds["sdist"] = cmd_sdist - - return cmds - - -CONFIG_ERROR = """ -setup.cfg is missing the necessary Versioneer configuration. You need -a section like: - - [versioneer] - VCS = git - style = pep440 - versionfile_source = src/myproject/_version.py - versionfile_build = myproject/_version.py - tag_prefix = - parentdir_prefix = myproject- - -You will also need to edit your setup.py to use the results: - - import versioneer - setup(version=versioneer.get_version(), - cmdclass=versioneer.get_cmdclass(), ...) - -Please read the docstring in ./versioneer.py for configuration instructions, -edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. -""" - -SAMPLE_CONFIG = """ -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. - -[versioneer] -#VCS = git -#style = pep440 -#versionfile_source = -#versionfile_build = -#tag_prefix = -#parentdir_prefix = - -""" - -INIT_PY_SNIPPET = """ -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions -""" - - -def do_setup(): - """Main VCS-independent setup function for installing Versioneer.""" - root = get_root() - try: - cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: - if isinstance(e, (EnvironmentError, configparser.NoSectionError)): - print( - "Adding sample versioneer config to setup.cfg", file=sys.stderr - ) - with open(os.path.join(root, "setup.cfg"), "a") as f: - f.write(SAMPLE_CONFIG) - print(CONFIG_ERROR, file=sys.stderr) - return 1 - - print(" creating %s" % cfg.versionfile_source) - with open(cfg.versionfile_source, "w") as f: - LONG = LONG_VERSION_PY[cfg.VCS] - f.write( - LONG - % { - "DOLLAR": "$", - "STYLE": cfg.style, - "TAG_PREFIX": cfg.tag_prefix, - "PARENTDIR_PREFIX": cfg.parentdir_prefix, - "VERSIONFILE_SOURCE": cfg.versionfile_source, - } - ) - - ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") - if os.path.exists(ipy): - try: - with open(ipy, "r") as f: - old = f.read() - except EnvironmentError: - old = "" - if INIT_PY_SNIPPET not in old: - print(" appending to %s" % ipy) - with open(ipy, "a") as f: - f.write(INIT_PY_SNIPPET) - else: - print(" %s unmodified" % ipy) - else: - print(" %s doesn't exist, ok" % ipy) - ipy = None - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(root, "MANIFEST.in") - simple_includes = set() - try: - with open(manifest_in, "r") as f: - for line in f: - if line.startswith("include "): - for include in line.split()[1:]: - simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - with open(manifest_in, "a") as f: - f.write("include versioneer.py\n") - else: - print(" 'versioneer.py' already in MANIFEST.in") - if cfg.versionfile_source not in simple_includes: - print( - " appending versionfile_source ('%s') to MANIFEST.in" - % cfg.versionfile_source - ) - with open(manifest_in, "a") as f: - f.write("include %s\n" % cfg.versionfile_source) - else: - print(" versionfile_source already in MANIFEST.in") - - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-subst keyword - # substitution. - do_vcs_install(manifest_in, cfg.versionfile_source, ipy) - return 0 - - -def scan_setup_py(): - """Validate the contents of setup.py against Versioneer's expectations.""" - found = set() - setters = False - errors = 0 - with open("setup.py", "r") as f: - for line in f.readlines(): - if "import versioneer" in line: - found.add("import") - if "versioneer.get_cmdclass()" in line: - found.add("cmdclass") - if "versioneer.get_version()" in line: - found.add("get_version") - if "versioneer.VCS" in line: - setters = True - if "versioneer.versionfile_source" in line: - setters = True - if len(found) != 3: - print("") - print("Your setup.py appears to be missing some important items") - print("(but I might be wrong). Please make sure it has something") - print("roughly like the following:") - print("") - print(" import versioneer") - print(" setup( version=versioneer.get_version(),") - print(" cmdclass=versioneer.get_cmdclass(), ...)") - print("") - errors += 1 - if setters: - print("You should remove lines like 'versioneer.VCS = ' and") - print("'versioneer.versionfile_source = ' . This configuration") - print("now lives in setup.cfg, and should be removed from setup.py") - print("") - errors += 1 - return errors - - -if __name__ == "__main__": - cmd = sys.argv[1] - if cmd == "setup": - errors = do_setup() - errors += scan_setup_py() - if errors: - sys.exit(1) diff --git a/setup.cfg b/setup.cfg index 3f8fa7e8406..962b7d73bbe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,4 @@ -# Copyright (c) 2017-2022, NVIDIA CORPORATION. +# Copyright (c) 2017-2023, NVIDIA CORPORATION. [flake8] filename = *.py, *.pyx, *.pxd, *.pxi @@ -44,8 +44,7 @@ ignore_missing_imports = True # they are imported by a checked file. follow_imports = skip exclude = (?x)( - (cudf|custreamz|cudf_kafka|dask_cudf)/_version\.py - | cudf/_lib/ + cudf/_lib/ | cudf/cudf/benchmarks/ | cudf/cudf/tests/ | cudf/cudf/utils/metadata/orc_column_statistics_pb2.py @@ -57,7 +56,7 @@ exclude = (?x)( [codespell] # note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - # this is only to allow you to run codespell interactively -skip = ./.git,./.github,./cpp/build,.*egg-info.*,versioneer.py,./.mypy_cache,./cpp/tests,./python/cudf/cudf/tests,./java/src/test,./cpp/include/cudf_test/cxxopts.hpp +skip = ./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,./cpp/tests,./python/cudf/cudf/tests,./java/src/test,./cpp/include/cudf_test/cxxopts.hpp # ignore short words, and typename parameters like OffsetT ignore-regex = \b(.{1,4}|[A-Z]\w*T)\b ignore-words-list = inout,unparseable From f076905f53f3f2d44ada244c0754afc459b306a1 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 22 Feb 2023 20:52:02 -0500 Subject: [PATCH 47/69] Add segmented reduction support for fixed-point types (#12680) Depends on #12573 Adds additional support for fixed-point types in `cudf::segmented_reduce` for simple aggregations: sum, product, and sum-of-squares. Reference: #10432 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Mike Wilson (https://github.com/hyperbolic2346) - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12680 --- cpp/CMakeLists.txt | 1 + cpp/src/reductions/segmented/compound.cuh | 29 +- cpp/src/reductions/segmented/counts.cu | 54 ++++ cpp/src/reductions/segmented/counts.hpp | 55 ++++ cpp/src/reductions/segmented/simple.cuh | 91 ++++-- .../reductions/segmented_reduction_tests.cpp | 305 +++++++++--------- 6 files changed, 341 insertions(+), 194 deletions(-) create mode 100644 cpp/src/reductions/segmented/counts.cu create mode 100644 cpp/src/reductions/segmented/counts.hpp diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d402a47628c..96524b7c55f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -455,6 +455,7 @@ add_library( src/reductions/scan/scan_inclusive.cu src/reductions/segmented/all.cu src/reductions/segmented/any.cu + src/reductions/segmented/counts.cu src/reductions/segmented/max.cu src/reductions/segmented/mean.cu src/reductions/segmented/min.cu diff --git a/cpp/src/reductions/segmented/compound.cuh b/cpp/src/reductions/segmented/compound.cuh index dc8a995d1b0..e8abd32cf61 100644 --- a/cpp/src/reductions/segmented/compound.cuh +++ b/cpp/src/reductions/segmented/compound.cuh @@ -16,6 +16,7 @@ #pragma once +#include "counts.hpp" #include "update_validity.hpp" #include @@ -63,34 +64,26 @@ std::unique_ptr compound_segmented_reduction(column_view const& col, data_type{type_to_id()}, num_segments, mask_state::UNALLOCATED, stream, mr); auto out_itr = result->mutable_view().template begin(); - // Compute valid counts - rmm::device_uvector valid_counts(num_segments, stream); - if (col.has_nulls() && (null_handling == null_policy::EXCLUDE)) { - auto valid_fn = [] __device__(auto p) -> size_type { return static_cast(p.second); }; - auto itr = thrust::make_transform_iterator(d_col->pair_begin(), valid_fn); - cudf::reduction::detail::segmented_reduce(itr, - offsets.begin(), - offsets.end(), - valid_counts.data(), - thrust::plus{}, - 0, - stream); - } else { - thrust::adjacent_difference( - rmm::exec_policy(stream), offsets.begin() + 1, offsets.end(), valid_counts.begin()); - } + // Compute counts + rmm::device_uvector counts = + cudf::reduction::detail::segmented_counts(col.null_mask(), + col.has_nulls(), + offsets, + null_handling, + stream, + rmm::mr::get_current_device_resource()); // Run segmented reduction if (col.has_nulls()) { auto nrt = compound_op.template get_null_replacing_element_transformer(); auto itr = thrust::make_transform_iterator(d_col->pair_begin(), nrt); cudf::reduction::detail::segmented_reduce( - itr, offsets.begin(), offsets.end(), out_itr, compound_op, ddof, valid_counts.data(), stream); + itr, offsets.begin(), offsets.end(), out_itr, compound_op, ddof, counts.data(), stream); } else { auto et = compound_op.template get_element_transformer(); auto itr = thrust::make_transform_iterator(d_col->begin(), et); cudf::reduction::detail::segmented_reduce( - itr, offsets.begin(), offsets.end(), out_itr, compound_op, ddof, valid_counts.data(), stream); + itr, offsets.begin(), offsets.end(), out_itr, compound_op, ddof, counts.data(), stream); } // Compute the output null mask diff --git a/cpp/src/reductions/segmented/counts.cu b/cpp/src/reductions/segmented/counts.cu new file mode 100644 index 00000000000..b9064ad3ffe --- /dev/null +++ b/cpp/src/reductions/segmented/counts.cu @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "counts.hpp" + +#include + +#include + +namespace cudf { +namespace reduction { +namespace detail { + +rmm::device_uvector segmented_counts(bitmask_type const* null_mask, + bool has_nulls, + device_span offsets, + null_policy null_handling, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + auto const num_segments = offsets.size() - 1; + + if (has_nulls && (null_handling == null_policy::EXCLUDE)) { + return cudf::detail::segmented_count_bits(null_mask, + offsets.begin(), + offsets.end() - 1, + offsets.begin() + 1, + cudf::detail::count_bits_policy::SET_BITS, + stream, + mr); + } + + rmm::device_uvector valid_counts(num_segments, stream, mr); + thrust::adjacent_difference( + rmm::exec_policy(stream), offsets.begin() + 1, offsets.end(), valid_counts.begin()); + return valid_counts; +} + +} // namespace detail +} // namespace reduction +} // namespace cudf diff --git a/cpp/src/reductions/segmented/counts.hpp b/cpp/src/reductions/segmented/counts.hpp new file mode 100644 index 00000000000..c5ee1fadae7 --- /dev/null +++ b/cpp/src/reductions/segmented/counts.hpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include +#include + +namespace cudf { +class column_device_view; + +namespace reduction { +namespace detail { + +/** + * @brief Compute the number of elements per segment + * + * If `null_handling == null_policy::EXCLUDE`, the count for each + * segment omits any null entries. Otherwise, this returns the number + * of elements in each segment. + * + * @param null_mask Null values over which the segment offsets apply + * @param has_nulls True if d_col contains any nulls + * @param offsets Indices to segment boundaries + * @param null_handling How null entries are processed within each segment + * @param stream Used for device memory operations and kernel launches + * @param mr Device memory resource used to allocate the returned column's device memory + * @return The number of elements in each segment + */ +rmm::device_uvector segmented_counts(bitmask_type const* null_mask, + bool has_nulls, + device_span offsets, + null_policy null_handling, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr); + +} // namespace detail +} // namespace reduction +} // namespace cudf diff --git a/cpp/src/reductions/segmented/simple.cuh b/cpp/src/reductions/segmented/simple.cuh index fb080ebf67c..0c22848fd89 100644 --- a/cpp/src/reductions/segmented/simple.cuh +++ b/cpp/src/reductions/segmented/simple.cuh @@ -16,6 +16,7 @@ #pragma once +#include "counts.hpp" #include "update_validity.hpp" #include @@ -36,6 +37,7 @@ #include #include #include +#include #include #include @@ -188,7 +190,7 @@ std::unique_ptr string_segmented_reduction(column_view const& col, } /** - * @brief Fixed point segmented reduction for 'min', 'max'. + * @brief Specialization for fixed-point segmented reduction * * @tparam InputType the input column data-type * @tparam Op the operator of cudf::reduction::op:: @@ -200,11 +202,7 @@ std::unique_ptr string_segmented_reduction(column_view const& col, * @param mr Device memory resource used to allocate the returned column's device memory * @return Output column in device memory */ - -template || - std::is_same_v)> +template std::unique_ptr fixed_point_segmented_reduction( column_view const& col, device_span offsets, @@ -214,23 +212,55 @@ std::unique_ptr fixed_point_segmented_reduction( rmm::mr::device_memory_resource* mr) { using RepType = device_storage_type_t; - return simple_segmented_reduction( - col, offsets, null_handling, init, stream, mr); -} + auto result = + simple_segmented_reduction(col, offsets, null_handling, init, stream, mr); + auto const scale = [&] { + if constexpr (std::is_same_v) { + // The product aggregation requires updating the scale of the fixed-point output column. + // The output scale needs to be the maximum count of all segments multiplied by + // the input scale value. + rmm::device_uvector const counts = + cudf::reduction::detail::segmented_counts(col.null_mask(), + col.has_nulls(), + offsets, + null_policy::EXCLUDE, // do not count nulls + stream, + rmm::mr::get_current_device_resource()); + + auto const max_count = thrust::reduce(rmm::exec_policy(stream), + counts.begin(), + counts.end(), + size_type{0}, + thrust::maximum{}); + + auto const new_scale = numeric::scale_type{col.type().scale() * max_count}; + + // adjust values in each segment to match the new scale + auto const d_col = column_device_view::create(col, stream); + thrust::transform(rmm::exec_policy(stream), + d_col->begin(), + d_col->end(), + d_col->begin(), + [new_scale] __device__(auto fp) { return fp.rescaled(new_scale); }); + return new_scale; + } -template () && - !std::is_same_v())> -std::unique_ptr fixed_point_segmented_reduction( - column_view const& col, - device_span offsets, - null_policy null_handling, - std::optional>, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) -{ - CUDF_FAIL("Segmented reduction on fixed point column only supports min and max reduction."); + if constexpr (std::is_same_v) { + return numeric::scale_type{col.type().scale() * 2}; + } + + return numeric::scale_type{col.type().scale()}; + }(); + + auto const size = result->size(); // get these before + auto const null_count = result->null_count(); // release() is called + auto contents = result->release(); + + return std::make_unique(data_type{type_to_id(), scale}, + size, + std::move(*(contents.data.release())), + std::move(*(contents.null_mask.release())), + null_count); } /** @@ -431,8 +461,23 @@ struct column_type_dispatcher { return reduce_numeric(col, offsets, output_type, null_handling, init, stream, mr); } + template ()>* = nullptr> + std::unique_ptr operator()(column_view const& col, + device_span offsets, + data_type const output_type, + null_policy null_handling, + std::optional> init, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) + { + CUDF_EXPECTS(output_type == col.type(), "Output type must be same as input column type."); + return fixed_point_segmented_reduction( + col, offsets, null_handling, init, stream, mr); + } + template ()>* = nullptr> + std::enable_if_t() and + not cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const&, device_span, data_type const, diff --git a/cpp/tests/reductions/segmented_reduction_tests.cpp b/cpp/tests/reductions/segmented_reduction_tests.cpp index b4873a14509..74c5e7fb504 100644 --- a/cpp/tests/reductions/segmented_reduction_tests.cpp +++ b/cpp/tests/reductions/segmented_reduction_tests.cpp @@ -1094,223 +1094,222 @@ struct SegmentedReductionFixedPointTest : public cudf::test::BaseFixture { TYPED_TEST_SUITE(SegmentedReductionFixedPointTest, cudf::test::FixedPointTypes); -TYPED_TEST(SegmentedReductionFixedPointTest, MaxIncludeNulls) +TYPED_TEST(SegmentedReductionFixedPointTest, MaxWithNulls) { - // scale: -2, 0, 5 - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] - // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} - // offsets: {0, 3, 6, 7, 8, 10, 10} - // nullmask: {1, 1, 1, 1, 0, 1, 1, 0, 0, 0} - // outputs: {3, XXX, 1, XXX, XXX, XXX} - // output nullmask: {1, 0, 1, 0, 0, 0} - using RepType = cudf::device_storage_type_t; + auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_max_aggregation(); + for (auto scale : {-2, 0, 5}) { auto const input = cudf::test::fixed_point_column_wrapper({1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX}, {1, 1, 1, 1, 0, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); - auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; - auto const d_offsets = - cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); - auto out_type = cudf::column_view(input).type(); - auto const expect = cudf::test::fixed_point_column_wrapper( + auto out_type = cudf::column_view(input).type(); + auto expect = cudf::test::fixed_point_column_wrapper( {3, XXX, 1, XXX, XXX, XXX}, {1, 0, 1, 0, 0, 0}, numeric::scale_type{scale}); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - auto res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_max_aggregation(), - out_type, - cudf::null_policy::INCLUDE); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*res, expect); + expect = cudf::test::fixed_point_column_wrapper( + {3, 3, 1, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); } } -TYPED_TEST(SegmentedReductionFixedPointTest, MaxExcludeNulls) +TYPED_TEST(SegmentedReductionFixedPointTest, MinWithNulls) { - // scale: -2, 0, 5 - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] - // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} - // offsets: {0, 3, 6, 7, 8, 10, 10} - // nullmask: {1, 1, 1, 1, 0, 1, 1, 0, 0, 0} - // outputs: {3, 3, 1, XXX, XXX, XXX} - // output nullmask: {1, 1, 1, 0, 0, 0} - using RepType = cudf::device_storage_type_t; + auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_min_aggregation(); + for (auto scale : {-2, 0, 5}) { auto const input = cudf::test::fixed_point_column_wrapper({1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX}, {1, 1, 1, 1, 0, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); - auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; - auto const d_offsets = - cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto out_type = cudf::column_view(input).type(); + auto expect = cudf::test::fixed_point_column_wrapper( + {1, XXX, 1, XXX, XXX, XXX}, {1, 0, 1, 0, 0, 0}, numeric::scale_type{scale}); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + + expect = cudf::test::fixed_point_column_wrapper( + {1, 1, 1, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + } +} + +TYPED_TEST(SegmentedReductionFixedPointTest, MaxNonNullableInput) +{ + using RepType = cudf::device_storage_type_t; + + auto const offsets = std::vector{0, 3, 4, 4}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_max_aggregation(); + + for (auto scale : {-2, 0, 5}) { + auto const input = + cudf::test::fixed_point_column_wrapper({1, 2, 3, 1}, numeric::scale_type{scale}); auto out_type = cudf::column_view(input).type(); auto const expect = cudf::test::fixed_point_column_wrapper( - {3, 3, 1, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); + {3, 1, XXX}, {1, 1, 0}, numeric::scale_type{scale}); - auto res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_max_aggregation(), - out_type, - cudf::null_policy::EXCLUDE); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*res, expect); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); } } -TYPED_TEST(SegmentedReductionFixedPointTest, MinIncludeNulls) +TYPED_TEST(SegmentedReductionFixedPointTest, MinNonNullableInput) { - // scale: -2, 0, 5 - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] - // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} - // offsets: {0, 3, 6, 7, 8, 10, 10} - // nullmask: {1, 1, 1, 1, 0, 1, 1, 0, 0, 0} - // outputs: {1, XXX, 1, XXX, XXX, XXX} - // output nullmask: {1, 0, 1, 0, 0, 0} - using RepType = cudf::device_storage_type_t; + auto const offsets = std::vector{0, 3, 4, 4}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_min_aggregation(); + for (auto scale : {-2, 0, 5}) { auto const input = - cudf::test::fixed_point_column_wrapper({1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX}, - {1, 1, 1, 1, 0, 1, 1, 0, 0, 0}, - numeric::scale_type{scale}); - auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; - auto const d_offsets = - cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + cudf::test::fixed_point_column_wrapper({1, 2, 3, 1}, numeric::scale_type{scale}); auto out_type = cudf::column_view(input).type(); auto const expect = cudf::test::fixed_point_column_wrapper( - {1, XXX, 1, XXX, XXX, XXX}, {1, 0, 1, 0, 0, 0}, numeric::scale_type{scale}); + {1, 1, XXX}, {1, 1, 0}, numeric::scale_type{scale}); - auto res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_min_aggregation(), - out_type, - cudf::null_policy::INCLUDE); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*res, expect); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); } } -TYPED_TEST(SegmentedReductionFixedPointTest, MinExcludeNulls) +TYPED_TEST(SegmentedReductionFixedPointTest, Sum) { - // scale: -2, 0, 5 - // [1, 2, 3], [1, null, 3], [1], [null], [null, null], [] - // values: {1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX} - // offsets: {0, 3, 6, 7, 8, 10, 10} - // nullmask: {1, 1, 1, 1, 0, 1, 1, 0, 0, 0} - // outputs: {1, 1, 1, XXX, XXX, XXX} - // output nullmask: {1, 1, 1, 0, 0, 0} - using RepType = cudf::device_storage_type_t; + auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_sum_aggregation(); + for (auto scale : {-2, 0, 5}) { - auto const input = - cudf::test::fixed_point_column_wrapper({1, 2, 3, 1, XXX, 3, 1, XXX, XXX, XXX}, + auto input = + cudf::test::fixed_point_column_wrapper({-10, 0, 33, 100, XXX, 53, 11, XXX, XXX, XXX}, {1, 1, 1, 1, 0, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); - auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; - auto const d_offsets = - cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); - auto out_type = cudf::column_view(input).type(); - auto const expect = cudf::test::fixed_point_column_wrapper( - {1, 1, 1, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); + auto const out_type = cudf::column_view(input).type(); + + auto expect = cudf::test::fixed_point_column_wrapper( + {23, XXX, 11, XXX, XXX, XXX}, {1, 0, 1, 0, 0, 0}, numeric::scale_type{scale}); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - auto res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_min_aggregation(), - out_type, - cudf::null_policy::EXCLUDE); + expect = cudf::test::fixed_point_column_wrapper( + {23, 153, 11, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*res, expect); + input = cudf::test::fixed_point_column_wrapper( + {-10, 0, 33, 100, 123, 53, 11, 0, -120, 88}, numeric::scale_type{scale}); + expect = cudf::test::fixed_point_column_wrapper( + {23, 276, 11, 0, -32, XXX}, {1, 1, 1, 1, 1, 0}, numeric::scale_type{scale}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); } } -TYPED_TEST(SegmentedReductionFixedPointTest, MaxNonNullableInput) +TYPED_TEST(SegmentedReductionFixedPointTest, Product) { - // scale: -2, 0, 5 - // [1, 2, 3], [1], [] - // values: {1, 2, 3, 1} - // offsets: {0, 3, 4} - // outputs: {3, 1, XXX} - // output nullmask: {1, 1, 0} - using RepType = cudf::device_storage_type_t; + auto const offsets = std::vector{0, 3, 6, 7, 8, 12, 12}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_product_aggregation(); + for (auto scale : {-2, 0, 5}) { - auto const input = - cudf::test::fixed_point_column_wrapper({1, 2, 3, 1}, numeric::scale_type{scale}); - auto const offsets = std::vector{0, 3, 4, 4}; - auto const d_offsets = - cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); - auto out_type = cudf::column_view(input).type(); - auto const expect = cudf::test::fixed_point_column_wrapper( - {3, 1, XXX}, {1, 1, 0}, numeric::scale_type{scale}); + auto input = cudf::test::fixed_point_column_wrapper( + {-10, 1, 33, 40, XXX, 50, 11000, XXX, XXX, XXX, XXX, XXX}, + {1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0}, + numeric::scale_type{scale}); + auto const out_type = cudf::column_view(input).type(); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + auto expect = cudf::test::fixed_point_column_wrapper( + {-330, XXX, 11000, XXX, XXX, XXX}, {1, 0, 1, 0, 0, 0}, numeric::scale_type{scale * 3}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + expect = cudf::test::fixed_point_column_wrapper( + {-330, 2000, 11000, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale * 3}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - auto include_null_res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_max_aggregation(), - out_type, - cudf::null_policy::INCLUDE); - - auto exclude_null_res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_max_aggregation(), - out_type, - cudf::null_policy::EXCLUDE); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*include_null_res, expect); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*exclude_null_res, expect); + input = cudf::test::fixed_point_column_wrapper( + {-10, 1, 33, 3, 40, 50, 11000, 0, -3, 50, 10, 4}, numeric::scale_type{scale}); + expect = cudf::test::fixed_point_column_wrapper( + {-330, 6000, 11000, 0, -6000, XXX}, {1, 1, 1, 1, 1, 0}, numeric::scale_type{scale * 4}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); } } -TYPED_TEST(SegmentedReductionFixedPointTest, MinNonNullableInput) +TYPED_TEST(SegmentedReductionFixedPointTest, SumOfSquares) { - // scale: -2, 0, 5 - // [1, 2, 3], [1], [] - // values: {1, 2, 3, 1} - // offsets: {0, 3, 4} - // outputs: {1, 1, XXX} - // output nullmask: {1, 1, 0} - using RepType = cudf::device_storage_type_t; + auto const offsets = std::vector{0, 3, 6, 7, 8, 10, 10}; + auto const d_offsets = + cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); + auto const agg = cudf::make_sum_of_squares_aggregation(); + for (auto scale : {-2, 0, 5}) { - auto const input = - cudf::test::fixed_point_column_wrapper({1, 2, 3, 1}, numeric::scale_type{scale}); - auto const offsets = std::vector{0, 3, 4, 4}; - auto const d_offsets = - cudf::detail::make_device_uvector_async(offsets, cudf::get_default_stream()); - auto out_type = cudf::column_view(input).type(); - auto const expect = cudf::test::fixed_point_column_wrapper( - {1, 1, XXX}, {1, 1, 0}, numeric::scale_type{scale}); + auto input = + cudf::test::fixed_point_column_wrapper({-10, 0, 33, 100, XXX, 53, 11, XXX, XXX, XXX}, + {1, 1, 1, 1, 0, 1, 1, 0, 0, 0}, + numeric::scale_type{scale}); + auto const out_type = cudf::column_view(input).type(); + + auto expect = cudf::test::fixed_point_column_wrapper( + {1189, XXX, 121, XXX, XXX, XXX}, {1, 0, 1, 0, 0, 0}, numeric::scale_type{scale * 2}); + auto result = + cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); - auto include_null_res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_min_aggregation(), - out_type, - cudf::null_policy::INCLUDE); - - auto exclude_null_res = - cudf::segmented_reduce(input, - d_offsets, - *cudf::make_min_aggregation(), - out_type, - cudf::null_policy::EXCLUDE); - - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*include_null_res, expect); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*exclude_null_res, expect); + expect = cudf::test::fixed_point_column_wrapper( + {1189, 12809, 121, XXX, XXX, XXX}, {1, 1, 1, 0, 0, 0}, numeric::scale_type{scale * 2}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + + input = cudf::test::fixed_point_column_wrapper( + {-10, 0, 33, 100, 123, 53, 11, 0, -120, 88}, numeric::scale_type{scale}); + expect = cudf::test::fixed_point_column_wrapper( + {1189, 27938, 121, 0, 22144, XXX}, {1, 1, 1, 1, 1, 0}, numeric::scale_type{scale * 2}); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::INCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); + result = cudf::segmented_reduce(input, d_offsets, *agg, out_type, cudf::null_policy::EXCLUDE); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*result, expect); } } From fffdc0cdac913d402c91880919ee391877f47514 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Wed, 22 Feb 2023 20:52:42 -0500 Subject: [PATCH 48/69] Add compute-sanitizer github workflow action to nightly tests (#12800) Adds github workflow action to the nightly tests for running `compute-sanitizer` on the libcudf gtests. Reference: #12530 Authors: - David Wendt (https://github.com/davidwendt) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12800 --- .github/workflows/test.yaml | 12 +++++++ ci/test_cpp.sh | 30 +---------------- ci/test_cpp_common.sh | 32 +++++++++++++++++++ ci/test_cpp_memcheck.sh | 25 +++++++++++++++ .../all_cuda-118_arch-x86_64.yaml | 1 + dependencies.yaml | 12 +++++++ 6 files changed, 83 insertions(+), 29 deletions(-) create mode 100644 ci/test_cpp_common.sh create mode 100755 ci/test_cpp_memcheck.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ff19d51f8ef..4d9e97a7b28 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,18 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + conda-cpp-memcheck-tests: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/test_cpp_memcheck.sh" conda-python-cudf-tests: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@branch-23.04 diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 983a63d4ce9..bd7a82afbea 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -1,35 +1,7 @@ #!/bin/bash # Copyright (c) 2022-2023, NVIDIA CORPORATION. -set -euo pipefail - -. /opt/conda/etc/profile.d/conda.sh - -rapids-logger "Generate C++ testing dependencies" -rapids-dependency-file-generator \ - --output conda \ - --file_key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml - -rapids-mamba-retry env create --force -f env.yaml -n test - -# Temporarily allow unbound variables for conda activation. -set +u -conda activate test -set -u - -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ -mkdir -p "${RAPIDS_TESTS_DIR}" - -rapids-print-env - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - libcudf libcudf_kafka libcudf-tests - -rapids-logger "Check GPU usage" -nvidia-smi +source "$(dirname "$0")/test_cpp_common.sh" EXITCODE=0 trap "EXITCODE=1" ERR diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh new file mode 100644 index 00000000000..c7c095dc4df --- /dev/null +++ b/ci/test_cpp_common.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright (c) 2022-2023, NVIDIA CORPORATION. + +set -euo pipefail + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate C++ testing dependencies" +rapids-dependency-file-generator \ + --output conda \ + --file_key test_cpp \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n test + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate test +set -u + +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ +mkdir -p "${RAPIDS_TESTS_DIR}" + +rapids-print-env + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + libcudf libcudf_kafka libcudf-tests + +rapids-logger "Check GPU usage" +nvidia-smi diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh new file mode 100755 index 00000000000..0cad4fc3a3f --- /dev/null +++ b/ci/test_cpp_memcheck.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +source "$(dirname "$0")/test_cpp_common.sh" + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +# Run gtests with compute-sanitizer +rapids-logger "Memcheck gtests with rmm_mode=cuda" +export GTEST_CUDF_RMM_MODE=cuda +COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" +for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do + test_name=$(basename ${gt}) + if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then + continue + fi + echo "Running compute-sanitizer on $test_name" + ${COMPUTE_SANITIZER_CMD} ${gt} --gtest_output=xml:"${RAPIDS_TESTS_DIR}${test_name}.xml" +done +unset GTEST_CUDF_RMM_MODE + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 675df3891c3..44d6be65574 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -16,6 +16,7 @@ dependencies: - cmake>=3.23.1,!=3.25.0 - cubinlinker - cuda-python>=11.7.1,<12.0 +- cuda-sanitizer-api=11.8.86 - cudatoolkit=11.8 - cupy>=9.5.0,<12.0.0a0 - cxx-compiler diff --git a/dependencies.yaml b/dependencies.yaml index ae8eac4ea30..de9795b4b39 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,12 +13,14 @@ files: - notebooks - py_version - run + - test_cpp - test_python test_cpp: output: none includes: - cudatoolkit - libidentify_stream_usage_build + - test_cpp test_python: output: none includes: @@ -266,6 +268,16 @@ dependencies: arch: aarch64 packages: - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. + test_cpp: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "11.8" + packages: + - cuda-sanitizer-api=11.8.86 + - matrix: + packages: test_java: common: - output_types: conda From 5719463d2db460e5cbbeb266512904dc1b475c2c Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Thu, 23 Feb 2023 09:12:50 -0600 Subject: [PATCH 49/69] Add docs build job (#12592) The PR adds a `docs_build` process to the PR and Build workflows for this repository. The generated docs are synced to s3 for only the build workflows. Authors: - Jake Awe (https://github.com/AyodeAwe) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12592 --- .github/workflows/build.yaml | 11 +++++++++ .github/workflows/pr.yaml | 11 +++++++++ .gitignore | 3 +++ ci/build_docs.sh | 47 ++++++++++++++++++++++++++++++++++++ dependencies.yaml | 13 +++++++--- docs/cudf/source/conf.py | 6 ----- 6 files changed, 82 insertions(+), 9 deletions(-) create mode 100755 ci/build_docs.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fa6704ef04e..024eb828e3c 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -53,6 +53,17 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} skip_upload_pkgs: libcudf-example + docs-build: + if: github.ref_type == 'branch' && github.event_name == 'push' + needs: python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 + with: + build_type: branch + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-cudf: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.04 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 73df2de20c2..952b58abda5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,6 +20,7 @@ jobs: - conda-python-other-tests - conda-java-tests - conda-notebook-tests + - docs-build - wheel-build-cudf - wheel-tests-cudf - wheel-build-dask-cudf @@ -82,6 +83,16 @@ jobs: arch: "amd64" container_image: "rapidsai/ci:latest" run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/custom-job.yaml@branch-23.04 + with: + build_type: pull-request + node_type: "gpu-latest-1" + arch: "amd64" + container_image: "rapidsai/ci:latest" + run_script: "ci/build_docs.sh" wheel-build-cudf: needs: checks secrets: inherit diff --git a/.gitignore b/.gitignore index 2d83aad7712..fb5c301fe3f 100644 --- a/.gitignore +++ b/.gitignore @@ -166,6 +166,9 @@ docs/cudf/source/api_docs/generated/* docs/cudf/source/api_docs/api/* docs/cudf/source/user_guide/example_output/* docs/cudf/source/user_guide/cudf.*Dtype.*.rst +_html +_text +jupyter_execute # cibuildwheel /wheelhouse diff --git a/ci/build_docs.sh b/ci/build_docs.sh new file mode 100755 index 00000000000..9551d98e9fe --- /dev/null +++ b/ci/build_docs.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create test conda environment" +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file_key docs \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --force -f env.yaml -n docs +conda activate docs + +rapids-print-env + +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +VERSION_NUMBER=$(rapids-get-rapids-version-from-git) + +rapids-mamba-retry install \ + --channel "${CPP_CHANNEL}" \ + --channel "${PYTHON_CHANNEL}" \ + libcudf cudf dask-cudf + + +rapids-logger "Build Doxygen docs" +pushd cpp/doxygen +aws s3 cp s3://rapidsai-docs/librmm/${VERSION_NUMBER}/html/rmm.tag . || echo "Failed to download rmm Doxygen tag" +doxygen Doxyfile +popd + +rapids-logger "Build Sphinx docs" +pushd docs/cudf +sphinx-build -b dirhtml source _html +sphinx-build -b text source _text +popd + + +if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then + aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" + aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" + aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" +fi diff --git a/dependencies.yaml b/dependencies.yaml index de9795b4b39..ba6d240e069 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -9,7 +9,7 @@ files: - build - cudatoolkit - develop - - doc + - docs - notebooks - py_version - run @@ -43,6 +43,12 @@ files: includes: - develop - py_version + docs: + output: none + includes: + - cudatoolkit + - docs + - py_version channels: - rapidsai - rapidsai-nightly @@ -123,10 +129,11 @@ dependencies: - output_types: conda packages: - doxygen=1.8.20 # pre-commit hook needs a specific version. - doc: + docs: common: - - output_types: [conda, requirements] + - output_types: [conda] packages: + - doxygen=1.8.20 - myst-nb - nbsphinx - numpydoc diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 371a8b4e1c1..3d92d955263 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright (c) 2018-2023, NVIDIA CORPORATION. # # cudf documentation build configuration file, created by @@ -23,11 +22,6 @@ from docutils.nodes import Text from sphinx.addnodes import pending_xref -import cudf - -sys.path.insert(0, os.path.abspath(cudf.__path__[0])) -sys.path.insert(0, os.path.abspath(".")) -sys.path.insert(0, os.path.abspath("../..")) sys.path.append(os.path.abspath("./_ext")) # -- General configuration ------------------------------------------------ From 430d91e9acb0a3db7c45df214e5158750311e626 Mon Sep 17 00:00:00 2001 From: Mike Wilson Date: Thu, 23 Feb 2023 13:56:32 -0500 Subject: [PATCH 50/69] Shuffling read into a sub function in parquet read (#12809) This change is the first step toward the pipelined parquet reader and moves the chunk creation and file reads into another function. Right now, the operation is the same, but this change will allow for smaller groups to be read at a time for pipelining. Authors: - Mike Wilson (https://github.com/hyperbolic2346) Approvers: - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12809 --- cpp/src/io/parquet/reader_impl.hpp | 15 ++++++- cpp/src/io/parquet/reader_impl_preprocess.cu | 43 ++++++++++++-------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index fcfea35f50c..8b86412ae63 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -130,10 +130,21 @@ class reader::impl { bool uses_custom_row_bounds, host_span const> row_group_indices); + /** + * @brief Create chunk information and start file reads + * + * @param row_groups_info vector of information about row groups to read + * @param num_rows Maximum number of rows to read + * @return pair of boolean indicating if compressed chunks were found and a vector of futures for + * read completion + */ + std::pair>> create_and_read_column_chunks( + cudf::host_span const row_groups_info, size_type num_rows); + /** * @brief Load and decompress the input file(s) into memory. */ - void load_and_decompress_data(std::vector const& row_groups_info, + void load_and_decompress_data(cudf::host_span const row_groups_info, size_type num_rows); /** diff --git a/cpp/src/io/parquet/reader_impl_preprocess.cu b/cpp/src/io/parquet/reader_impl_preprocess.cu index b1d013a96a3..0f55cd6e400 100644 --- a/cpp/src/io/parquet/reader_impl_preprocess.cu +++ b/cpp/src/io/parquet/reader_impl_preprocess.cu @@ -651,16 +651,11 @@ void reader::impl::allocate_nesting_info() page_nesting_decode_info.host_to_device(_stream); } -void reader::impl::load_and_decompress_data(std::vector const& row_groups_info, - size_type num_rows) +std::pair>> reader::impl::create_and_read_column_chunks( + cudf::host_span const row_groups_info, size_type num_rows) { - // This function should never be called if `num_rows == 0`. - CUDF_EXPECTS(num_rows > 0, "Number of reading rows must not be zero."); - - auto& raw_page_data = _file_itm_data.raw_page_data; - auto& decomp_page_data = _file_itm_data.decomp_page_data; - auto& chunks = _file_itm_data.chunks; - auto& pages_info = _file_itm_data.pages_info; + auto& raw_page_data = _file_itm_data.raw_page_data; + auto& chunks = _file_itm_data.chunks; // Descriptors for all the chunks that make up the selected columns const auto num_input_columns = _input_columns.size(); @@ -732,7 +727,7 @@ void reader::impl::load_and_decompress_data(std::vector const& r total_decompressed_size += col_meta.total_uncompressed_size; } } - remaining_rows -= row_group.num_rows; + remaining_rows -= row_group_rows; } // Read compressed chunk data to device memory @@ -745,12 +740,29 @@ void reader::impl::load_and_decompress_data(std::vector const& r chunk_source_map, _stream)); + CUDF_EXPECTS(remaining_rows == 0, "All rows data must be read."); + + return {total_decompressed_size > 0, std::move(read_rowgroup_tasks)}; +} + +void reader::impl::load_and_decompress_data( + cudf::host_span const row_groups_info, size_type num_rows) +{ + // This function should never be called if `num_rows == 0`. + CUDF_EXPECTS(num_rows > 0, "Number of reading rows must not be zero."); + + auto& raw_page_data = _file_itm_data.raw_page_data; + auto& decomp_page_data = _file_itm_data.decomp_page_data; + auto& chunks = _file_itm_data.chunks; + auto& pages_info = _file_itm_data.pages_info; + + auto const [has_compressed_data, read_rowgroup_tasks] = + create_and_read_column_chunks(row_groups_info, num_rows); + for (auto& task : read_rowgroup_tasks) { task.wait(); } - CUDF_EXPECTS(remaining_rows <= 0, "All rows data must be read."); - // Process dataset chunk pages into output columns auto const total_pages = count_page_headers(chunks, _stream); pages_info = hostdevice_vector(total_pages, total_pages, _stream); @@ -758,14 +770,11 @@ void reader::impl::load_and_decompress_data(std::vector const& r if (total_pages > 0) { // decoding of column/page information decode_page_headers(chunks, pages_info, _stream); - if (total_decompressed_size > 0) { + if (has_compressed_data) { decomp_page_data = decompress_page_data(chunks, pages_info, _stream); // Free compressed data for (size_t c = 0; c < chunks.size(); c++) { - if (chunks[c].codec != parquet::Compression::UNCOMPRESSED) { - raw_page_data[c].reset(); - // TODO: Check if this is called - } + if (chunks[c].codec != parquet::Compression::UNCOMPRESSED) { raw_page_data[c].reset(); } } } From e64e26eda09f8508b7760ddba9f742c4f4e827cb Mon Sep 17 00:00:00 2001 From: Ayush Dattagupta Date: Thu, 23 Feb 2023 18:07:56 -0800 Subject: [PATCH 51/69] Expose seed argument to hash_values (#12795) This PR exposes the `seed` param to `hash_values` that is already supported by libcudf's `hash` method. Closes #12775 Authors: - Ayush Dattagupta (https://github.com/ayushdg) Approvers: - https://github.com/brandon-b-miller URL: https://github.com/rapidsai/cudf/pull/12795 --- python/cudf/cudf/core/indexed_frame.py | 24 +++++++++++++-- python/cudf/cudf/tests/test_dataframe.py | 39 +++++++++++++++++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 43277fb55ff..2992cb005e5 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -1629,7 +1629,7 @@ def memory_usage(self, index=True, deep=False): """ raise NotImplementedError - def hash_values(self, method="murmur3"): + def hash_values(self, method="murmur3", seed=None): """Compute the hash of values in this column. Parameters @@ -1639,6 +1639,12 @@ def hash_values(self, method="murmur3"): * murmur3: MurmurHash3 hash function. * md5: MD5 hash function. + seed : int, optional + Seed value to use for the hash function. + Note - This only has effect for the following supported + hash functions: + * murmur3: MurmurHash3 hash function. + Returns ------- Series @@ -1665,6 +1671,11 @@ def hash_values(self, method="murmur3"): 1 947ca8d2c5f0f27437f156cfbfab0969 2 d0580ef52d27c043c8e341fd5039b166 dtype: object + >>> series.hash_values(method="murmur3", seed=42) + 0 2364453205 + 1 422621911 + 2 3353449140 + dtype: uint32 **DataFrame** @@ -1686,11 +1697,20 @@ def hash_values(self, method="murmur3"): 2 fe061786ea286a515b772d91b0dfcd70 dtype: object """ + seed_hash_methods = {"murmur3"} + if seed is None: + seed = 0 + elif method not in seed_hash_methods: + warnings.warn( + "Provided seed value has no effect for hash method" + f" `{method}`. Refer to the docstring for information" + " on hash methods that support the `seed` param" + ) # Note that both Series and DataFrame return Series objects from this # calculation, necessitating the unfortunate circular reference to the # child class here. return cudf.Series._from_data( - {None: libcudf.hash.hash([*self._columns], method)}, + {None: libcudf.hash.hash([*self._columns], method, seed)}, index=self.index, ) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 09b9f57356c..13f312f6f0c 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -38,6 +38,7 @@ NUMERIC_TYPES, assert_eq, assert_exceptions_equal, + assert_neq, does_not_raise, expect_warning_if, gen_rand, @@ -1323,9 +1324,10 @@ def test_assign(): @pytest.mark.parametrize("nrows", [1, 8, 100, 1000]) @pytest.mark.parametrize("method", ["murmur3", "md5"]) -def test_dataframe_hash_values(nrows, method): +@pytest.mark.parametrize("seed", [None, 42]) +def test_dataframe_hash_values(nrows, method, seed): gdf = cudf.DataFrame() - data = np.asarray(range(nrows)) + data = np.arange(nrows) data[0] = data[-1] # make first and last the same gdf["a"] = data gdf["b"] = gdf.a + 100 @@ -1334,12 +1336,41 @@ def test_dataframe_hash_values(nrows, method): assert len(out) == nrows assert out.dtype == np.uint32 + warning_expected = ( + True if seed is not None and method not in {"murmur3"} else False + ) # Check single column - out_one = gdf[["a"]].hash_values(method=method) + if warning_expected: + with pytest.warns( + UserWarning, match="Provided seed value has no effect*" + ): + out_one = gdf[["a"]].hash_values(method=method, seed=seed) + else: + out_one = gdf[["a"]].hash_values(method=method, seed=seed) # First matches last assert out_one.iloc[0] == out_one.iloc[-1] # Equivalent to the cudf.Series.hash_values() - assert_eq(gdf["a"].hash_values(method=method), out_one) + if warning_expected: + with pytest.warns( + UserWarning, match="Provided seed value has no effect*" + ): + assert_eq(gdf["a"].hash_values(method=method, seed=seed), out_one) + else: + assert_eq(gdf["a"].hash_values(method=method, seed=seed), out_one) + + +@pytest.mark.parametrize("method", ["murmur3"]) +def test_dataframe_hash_values_seed(method): + gdf = cudf.DataFrame() + data = np.arange(10) + data[0] = data[-1] # make first and last the same + gdf["a"] = data + gdf["b"] = gdf.a + 100 + out_one = gdf.hash_values(method=method, seed=0) + out_two = gdf.hash_values(method=method, seed=1) + assert out_one.iloc[0] == out_one.iloc[-1] + assert out_two.iloc[0] == out_two.iloc[-1] + assert_neq(out_one, out_two) @pytest.mark.parametrize("nrows", [3, 10, 100, 1000]) From 2e80eba6f75b03f039517c947f386ede65842a4c Mon Sep 17 00:00:00 2001 From: "Richard (Rick) Zamora" Date: Fri, 24 Feb 2023 10:28:25 -0600 Subject: [PATCH 52/69] Fix parquet `RangeIndex` bug (#12838) Possible fix for https://github.com/rapidsai/cudf/issues/12837 Avoids dropping RangeIndex when `columns` argument is passed to `read_parquet` (unless `columns=[]`). Authors: - Richard (Rick) Zamora (https://github.com/rjzamora) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12838 --- python/cudf/cudf/_lib/parquet.pyx | 2 +- python/cudf/cudf/tests/test_parquet.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/_lib/parquet.pyx b/python/cudf/cudf/_lib/parquet.pyx index e5520ae1987..464d9243408 100644 --- a/python/cudf/cudf/_lib/parquet.pyx +++ b/python/cudf/cudf/_lib/parquet.pyx @@ -170,7 +170,7 @@ cpdef read_parquet(filepaths_or_buffers, columns=None, row_groups=None, allow_range_index = True if columns is not None: cpp_columns.reserve(len(columns)) - allow_range_index = False + allow_range_index = len(columns) > 0 for col in columns: cpp_columns.push_back(str(col).encode()) args.set_columns(cpp_columns) diff --git a/python/cudf/cudf/tests/test_parquet.py b/python/cudf/cudf/tests/test_parquet.py index ccd62729a9d..661497e4650 100644 --- a/python/cudf/cudf/tests/test_parquet.py +++ b/python/cudf/cudf/tests/test_parquet.py @@ -2650,6 +2650,20 @@ def test_parquet_columns_and_index_param(index, columns): assert_eq(expected, got, check_index_type=True) +@pytest.mark.parametrize("columns", [None, ["b", "a"]]) +def test_parquet_columns_and_range_index(columns): + buffer = BytesIO() + df = cudf.DataFrame( + {"a": [1, 2, 3], "b": ["a", "b", "c"]}, index=pd.RangeIndex(2, 5) + ) + df.to_parquet(buffer) + + expected = pd.read_parquet(buffer, columns=columns) + got = cudf.read_parquet(buffer, columns=columns) + + assert_eq(expected, got, check_index_type=True) + + def test_parquet_nested_struct_list(): buffer = BytesIO() data = { From 0e4e6dd567964404934d96a1fe8fc14b1d25a526 Mon Sep 17 00:00:00 2001 From: Divye Gala Date: Fri, 24 Feb 2023 12:07:51 -0500 Subject: [PATCH 53/69] Add `always_nullable` flag to Dremel encoding (#12727) Closes #12389 by fixing the bug describe here https://github.com/rapidsai/cudf/issues/12389#issuecomment-1419949751. This flag, when `always_nullable=true`, generates `definition levels` in the Dremel encoding such that it considers every nested column and child to be `nullable`, even if they actually are not. In the context of `two_table_comparators`, this helps us with producing consistently mapped `definition levels` in case there are some nested columns or children that are not nullable in either one or both of the tables. This PR now exposes two APIs: 1. `cudf::detail::get_dremel_data(...)` : This API is consistent with standard Dremel encoding 2. `cudf::detail::get_comparator_data(...)` : This API modifies the definition levels in Dremel encoding to produce the effect described above Authors: - Divye Gala (https://github.com/divyegala) - Nghia Truong (https://github.com/ttnghia) Approvers: - Nghia Truong (https://github.com/ttnghia) - Yunsong Wang (https://github.com/PointKernel) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12727 --- cpp/include/cudf/lists/detail/dremel.hpp | 30 +++++++-- .../cudf/table/experimental/row_operators.cuh | 3 +- cpp/src/lists/dremel.cu | 48 ++++++++++---- cpp/src/table/row_operators.cu | 2 +- cpp/tests/search/search_list_test.cpp | 64 ++++++++++++++++++- 5 files changed, 124 insertions(+), 23 deletions(-) diff --git a/cpp/include/cudf/lists/detail/dremel.hpp b/cpp/include/cudf/lists/detail/dremel.hpp index 4e3aeec2499..d36a4091947 100644 --- a/cpp/include/cudf/lists/detail/dremel.hpp +++ b/cpp/include/cudf/lists/detail/dremel.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -183,16 +183,34 @@ struct dremel_data { * - | - | -- | --- * ``` * - * @param col Column of LIST type - * @param level_nullability Pre-determined nullability at each list level. Empty means infer from - * `col` + * @param input Column of LIST type + * @param nullability Pre-determined nullability at each list level. Empty means infer from + * `input` + * @param output_as_byte_array if `true`, then any nested list level that has a child of type + * `uint8_t` will be considered as the last level * @param stream CUDA stream used for device memory operations and kernel launches. - * * @return A struct containing dremel data */ -dremel_data get_dremel_data(column_view h_col, +dremel_data get_dremel_data(column_view input, std::vector nullability, bool output_as_byte_array, rmm::cuda_stream_view stream); +/** + * @brief Get Dremel offsets, repetition levels, and modified definition levels to be used for + * lexicographical comparators. The modified definition levels are produced by treating + * each nested column in the input as nullable + * + * @param input Column of LIST type + * @param nullability Pre-determined nullability at each list level. Empty means infer from + * `input` + * @param output_as_byte_array if `true`, then any nested list level that has a child of type + * `uint8_t` will be considered as the last level + * @param stream CUDA stream used for device memory operations and kernel launches. + * @return A struct containing dremel data + */ +dremel_data get_comparator_data(column_view input, + std::vector nullability, + bool output_as_byte_array, + rmm::cuda_stream_view stream); } // namespace cudf::detail diff --git a/cpp/include/cudf/table/experimental/row_operators.cuh b/cpp/include/cudf/table/experimental/row_operators.cuh index f9ffbfcdf7b..2a207d2a5c4 100644 --- a/cpp/include/cudf/table/experimental/row_operators.cuh +++ b/cpp/include/cudf/table/experimental/row_operators.cuh @@ -487,7 +487,8 @@ class device_row_comparator { // element_index because either both rows have a deeply nested NULL at the // same position, and we'll "continue" in our iteration, or we will early // exit if only one of the rows has a deeply nested NULL - if (lcol.nullable() and l_def_levels[l_dremel_index] == l_max_def_level - 1) { + if ((lcol.nullable() and l_def_levels[l_dremel_index] == l_max_def_level - 1) or + (rcol.nullable() and r_def_levels[r_dremel_index] == r_max_def_level - 1)) { ++element_index; } if (l_def_level == r_def_level) { continue; } diff --git a/cpp/src/lists/dremel.cu b/cpp/src/lists/dremel.cu index 26988622aee..c96a21df905 100644 --- a/cpp/src/lists/dremel.cu +++ b/cpp/src/lists/dremel.cu @@ -35,7 +35,7 @@ #include namespace cudf::detail { - +namespace { /** * @brief Functor to get definition level value for a nested struct column until the leaf level or * the first list level. @@ -46,6 +46,7 @@ struct def_level_fn { uint8_t const* d_nullability; uint8_t sub_level_start; uint8_t curr_def_level; + bool always_nullable; __device__ uint32_t operator()(size_type i) { @@ -55,7 +56,7 @@ struct def_level_fn { auto col = *parent_col; do { // If col not nullable then it does not contribute to def levels - if (d_nullability[l]) { + if (always_nullable or d_nullability[l]) { if (not col.nullable() or bit_is_set(col.null_mask(), i)) { ++def; } else { // We have found the shallowest level at which this row is null @@ -72,10 +73,11 @@ struct def_level_fn { } }; -dremel_data get_dremel_data(column_view h_col, - std::vector nullability, - bool output_as_byte_array, - rmm::cuda_stream_view stream) +dremel_data get_encoding(column_view h_col, + std::vector nullability, + bool output_as_byte_array, + bool always_nullable, + rmm::cuda_stream_view stream) { auto get_list_level = [](column_view col) { while (col.type().id() == type_id::STRUCT) { @@ -173,14 +175,14 @@ dremel_data get_dremel_data(column_view h_col, uint32_t def = 0; start_at_sub_level.push_back(curr_nesting_level_idx); while (col.type().id() == type_id::STRUCT) { - def += (nullability[curr_nesting_level_idx]) ? 1 : 0; + def += (always_nullable or nullability[curr_nesting_level_idx]) ? 1 : 0; col = col.child(0); ++curr_nesting_level_idx; } // At the end of all those structs is either a list column or the leaf. List column contributes // at least one def level. Leaf contributes 1 level only if it is nullable. - def += - (col.type().id() == type_id::LIST ? 1 : 0) + (nullability[curr_nesting_level_idx] ? 1 : 0); + def += (col.type().id() == type_id::LIST ? 1 : 0) + + (always_nullable or nullability[curr_nesting_level_idx] ? 1 : 0); def_at_level.push_back(def); ++curr_nesting_level_idx; }; @@ -209,7 +211,7 @@ dremel_data get_dremel_data(column_view h_col, } } - auto [device_view_owners, d_nesting_levels] = + [[maybe_unused]] auto [device_view_owners, d_nesting_levels] = contiguous_copy_column_device_views(nesting_levels, stream); auto max_def_level = def_at_level.back(); @@ -297,7 +299,8 @@ dremel_data get_dremel_data(column_view h_col, def_level_fn{d_nesting_levels + level, d_nullability.data(), start_at_sub_level[level], - def_at_level[level]}); + def_at_level[level], + always_nullable}); // `nesting_levels.size()` == no of list levels + leaf. Max repetition level = no of list levels auto input_child_rep_it = thrust::make_constant_iterator(nesting_levels.size() - 1); @@ -306,7 +309,8 @@ dremel_data get_dremel_data(column_view h_col, def_level_fn{d_nesting_levels + level + 1, d_nullability.data(), start_at_sub_level[level + 1], - def_at_level[level + 1]}); + def_at_level[level + 1], + always_nullable}); // Zip the input and output value iterators so that merge operation is done only once auto input_parent_zip_it = @@ -389,7 +393,8 @@ dremel_data get_dremel_data(column_view h_col, def_level_fn{d_nesting_levels + level, d_nullability.data(), start_at_sub_level[level], - def_at_level[level]}); + def_at_level[level], + always_nullable}); // Zip the input and output value iterators so that merge operation is done only once auto input_parent_zip_it = @@ -459,5 +464,22 @@ dremel_data get_dremel_data(column_view h_col, leaf_data_size, max_def_level}; } +} // namespace + +dremel_data get_dremel_data(column_view h_col, + std::vector nullability, + bool output_as_byte_array, + rmm::cuda_stream_view stream) +{ + return get_encoding(h_col, nullability, output_as_byte_array, false, stream); +} + +dremel_data get_comparator_data(column_view h_col, + std::vector nullability, + bool output_as_byte_array, + rmm::cuda_stream_view stream) +{ + return get_encoding(h_col, nullability, output_as_byte_array, true, stream); +} } // namespace cudf::detail diff --git a/cpp/src/table/row_operators.cu b/cpp/src/table/row_operators.cu index 766a1b63905..8a63a6f6411 100644 --- a/cpp/src/table/row_operators.cu +++ b/cpp/src/table/row_operators.cu @@ -264,7 +264,7 @@ auto list_lex_preprocess(table_view table, rmm::cuda_stream_view stream) std::vector dremel_device_views; for (auto const& col : table) { if (col.type().id() == type_id::LIST) { - dremel_data.push_back(detail::get_dremel_data(col, {}, false, stream)); + dremel_data.push_back(detail::get_comparator_data(col, {}, false, stream)); dremel_device_views.push_back(dremel_data.back()); } } diff --git a/cpp/tests/search/search_list_test.cpp b/cpp/tests/search/search_list_test.cpp index 1393095037d..1e97933fa4d 100644 --- a/cpp/tests/search/search_list_test.cpp +++ b/cpp/tests/search/search_list_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, NVIDIA CORPORATION. + * Copyright (c) 2022-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,8 @@ #include #include -using namespace cudf::test::iterators; +using cudf::test::iterators::null_at; +using cudf::test::iterators::nulls_at; using bools_col = cudf::test::fixed_width_column_wrapper; using int32s_col = cudf::test::fixed_width_column_wrapper; @@ -347,3 +348,62 @@ TYPED_TEST(TypedListContainsTestColumnNeedles, ListsOfStructs) auto const result = cudf::contains(*haystack, *needles); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected, *result, verbosity); } + +struct ListLowerBound : public cudf::test::BaseFixture { +}; + +TEST_F(ListLowerBound, ListWithNulls) +{ + { + using lcw = cudf::test::lists_column_wrapper; + auto const haystack = lcw{ + lcw{-3.45967821e+12}, // 0 + lcw{-3.6912186e-32}, // 1 + lcw{9.721175}, // 2 + }; + + auto const needles = lcw{ + lcw{{0, 4.22671e+32}, null_at(0)}, + }; + + auto const expect = int32s_col{0}; + auto const result = cudf::lower_bound(cudf::table_view{{haystack}}, + cudf::table_view{{needles}}, + {cudf::order::ASCENDING}, + {cudf::null_order::BEFORE}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect, *result); + } + + { + using lcw = cudf::test::lists_column_wrapper; + auto const col1 = lcw{ + lcw{{0}, null_at(0)}, // 0 + lcw{-80}, // 1 + lcw{-17}, // 2 + }; + + auto const col2 = lcw{ + lcw{27}, // 0 + lcw{{0}, null_at(0)}, // 1 + lcw{}, // 2 + }; + + auto const val1 = lcw{ + lcw{87}, + }; + + auto const val2 = lcw{ + lcw{}, + }; + + cudf::table_view input{{col1, col2}}; + cudf::table_view values{{val1, val2}}; + std::vector column_order{cudf::order::ASCENDING, cudf::order::DESCENDING}; + std::vector null_order_flags{cudf::null_order::BEFORE, + cudf::null_order::BEFORE}; + + auto const expect = int32s_col{3}; + auto const result = cudf::lower_bound(input, values, column_order, null_order_flags); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(expect, *result); + } +} From 8a7fb2f14a73937d31f648a65f57bc47751e97c1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 24 Feb 2023 12:25:49 -0600 Subject: [PATCH 54/69] Deprecate `inplace` parameters in categorical methods (#12824) To get ready for pandas-2.0 compatibility, this PR deprecates `inplace` in the following APIs: - [x] `as_ordered` - [x] `as_unordered` - [x] `add_categories` - [x] `remove_categories` - [x] `set_categories` - [x] `reorder_categories` Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - https://github.com/brandon-b-miller URL: https://github.com/rapidsai/cudf/pull/12824 --- python/cudf/cudf/core/column/categorical.py | 78 ++++++++++++++++++++- python/cudf/cudf/tests/test_categorical.py | 19 +++-- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/core/column/categorical.py b/python/cudf/cudf/core/column/categorical.py index a1526d25512..52f7c0b957f 100644 --- a/python/cudf/cudf/core/column/categorical.py +++ b/python/cudf/cudf/core/column/categorical.py @@ -141,6 +141,13 @@ def as_ordered(self, inplace: bool = False) -> Optional[SeriesOrIndex]: or return a copy of this categorical with added categories. + .. deprecated:: 23.02 + + The `inplace` parameter is is deprecated and + will be removed in a future version of cudf. + Setting categories as ordered will always + return a new Categorical object. + Returns ------- Categorical @@ -204,6 +211,13 @@ def as_unordered(self, inplace: bool = False) -> Optional[SeriesOrIndex]: in-place or return a copy of this categorical with ordered set to False. + .. deprecated:: 23.02 + + The `inplace` parameter is is deprecated and + will be removed in a future version of cudf. + Setting categories as unordered will always + return a new Categorical object. + Returns ------- Categorical @@ -286,6 +300,13 @@ def add_categories( or return a copy of this categorical with added categories. + .. deprecated:: 23.04 + + The `inplace` parameter is is deprecated and + will be removed in a future version of cudf. + Adding categories will always return a + new Categorical object. + Returns ------- cat @@ -318,7 +339,14 @@ def add_categories( dtype: category Categories (5, int64): [1, 2, 0, 3, 4] """ - + if inplace: + warnings.warn( + "The `inplace` parameter in cudf.Series.cat.add_categories " + "is deprecated and will be removed in a future version of " + "cudf. Adding categories will always return a new " + "Categorical object.", + FutureWarning, + ) old_categories = self._column.categories new_categories = column.as_column( new_categories, @@ -371,6 +399,13 @@ def remove_categories( inplace or return a copy of this categorical with removed categories. + .. deprecated:: 23.04 + + The `inplace` parameter is is deprecated and + will be removed in a future version of cudf. + Removing categories will always return a + new Categorical object. + Returns ------- cat @@ -423,6 +458,16 @@ def remove_categories( dtype: category Categories (2, int64): [1, 2] """ + if inplace: + warnings.warn( + "The `inplace` parameter in " + "cudf.Series.cat.remove_categories is deprecated and " + "will be removed in a future version of cudf. " + "Removing categories will always return a new " + "Categorical object.", + FutureWarning, + ) + cats = self.categories.to_series() removals = cudf.Series(removals, dtype=cats.dtype) removals_mask = removals.isin(cats) @@ -485,6 +530,13 @@ def set_categories( or return a copy of this categorical with reordered categories. + .. deprecated:: 23.04 + + The `inplace` parameter is is deprecated and + will be removed in a future version of cudf. + Setting categories will always return a + new Categorical object. + Returns ------- cat @@ -524,6 +576,14 @@ def set_categories( dtype: category Categories (2, int64): [1, 10] """ + if inplace: + warnings.warn( + "The `inplace` parameter in cudf.Series.cat.set_categories is " + "deprecated and will be removed in a future version of cudf. " + "Setting categories will always return a new Categorical " + "object.", + FutureWarning, + ) return self._return_or_inplace( self._column.set_categories( new_categories=new_categories, ordered=ordered, rename=rename @@ -556,6 +616,13 @@ def reorder_categories( inplace or return a copy of this categorical with reordered categories. + .. deprecated:: 23.04 + + The `inplace` parameter is is deprecated and + will be removed in a future version of cudf. + Reordering categories will always return a + new Categorical object. + Returns ------- cat @@ -597,6 +664,15 @@ def reorder_categories( ValueError: items in new_categories are not the same as in old categories """ + if inplace: + warnings.warn( + "The `inplace` parameter in " + "cudf.Series.cat.reorder_categories is deprecated " + "and will be removed in a future version of cudf. " + "Reordering categories will always return a new " + "Categorical object.", + FutureWarning, + ) return self._return_or_inplace( self._column.reorder_categories(new_categories, ordered=ordered), inplace=inplace, diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index fa8981cf7e3..496039ca2f8 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -443,10 +443,13 @@ def test_categorical_reorder_categories( "reorder_categories" ): pd_sr_1 = pd_sr.cat.reorder_categories(list("cba"), **kwargs) - cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) if inplace: + with pytest.warns(FutureWarning): + cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) pd_sr_1 = pd_sr cd_sr_1 = cd_sr + else: + cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) assert_eq(pd_sr_1, cd_sr_1) @@ -479,10 +482,14 @@ def test_categorical_add_categories(pd_str_cat, inplace): "add_categories" ): pd_sr_1 = pd_sr.cat.add_categories(["d"], inplace=inplace) - cd_sr_1 = cd_sr.cat.add_categories(["d"], inplace=inplace) + if inplace: + with pytest.warns(FutureWarning): + cd_sr_1 = cd_sr.cat.add_categories(["d"], inplace=inplace) pd_sr_1 = pd_sr cd_sr_1 = cd_sr + else: + cd_sr_1 = cd_sr.cat.add_categories(["d"], inplace=inplace) assert "d" in pd_sr_1.cat.categories.to_list() assert "d" in cd_sr_1.cat.categories.to_pandas().to_list() @@ -516,10 +523,14 @@ def test_categorical_remove_categories(pd_str_cat, inplace): "remove_categories" ): pd_sr_1 = pd_sr.cat.remove_categories(["a"], inplace=inplace) - cd_sr_1 = cd_sr.cat.remove_categories(["a"], inplace=inplace) + if inplace: + with pytest.warns(FutureWarning): + cd_sr_1 = cd_sr.cat.remove_categories(["a"], inplace=inplace) pd_sr_1 = pd_sr cd_sr_1 = cd_sr + else: + cd_sr_1 = cd_sr.cat.remove_categories(["a"], inplace=inplace) assert "a" not in pd_sr_1.cat.categories.to_list() assert "a" not in cd_sr_1.cat.categories.to_pandas().to_list() @@ -529,7 +540,7 @@ def test_categorical_remove_categories(pd_str_cat, inplace): # test using ordered operators with _hide_deprecated_pandas_categorical_inplace_warnings( "remove_categories" - ): + ) as _, pytest.warns(FutureWarning) as _: assert_exceptions_equal( lfunc=cd_sr.to_pandas().cat.remove_categories, rfunc=cd_sr.cat.remove_categories, From 54ee14e36157fe63d0eb58ed7ac8bafc2b1e4932 Mon Sep 17 00:00:00 2001 From: Jordan Jacobelli Date: Fri, 24 Feb 2023 19:37:29 +0100 Subject: [PATCH 55/69] Update datasets download URL (#12840) Update datasets download URL to reduce latency and costs Authors: - Jordan Jacobelli (https://github.com/jjacobelli) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12840 --- python/cudf/cudf/benchmarks/get_datasets.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/benchmarks/get_datasets.py b/python/cudf/cudf/benchmarks/get_datasets.py index f3b66eda512..7090539bcb0 100644 --- a/python/cudf/cudf/benchmarks/get_datasets.py +++ b/python/cudf/cudf/benchmarks/get_datasets.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import argparse import os @@ -9,10 +9,7 @@ Dataset = namedtuple("Dataset", ["url", "dir"]) datasets = { "cuio_dataset": Dataset( - ( - "https://rapidsai-data.s3.us-east-2.amazonaws.com/cudf/" - "benchmark/avro_json_datasets.zip" - ), + "https://data.rapids.ai/cudf/benchmark/avro_json_datasets.zip", "cudf/benchmarks/cuio_data/", ), } From 12e4501c49daac3d0e3837a3f65078e63e20b904 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 24 Feb 2023 13:42:49 -0500 Subject: [PATCH 56/69] Remove KAFKA_HOST_TEST from compute-sanitizer check (#12831) Removes the `KAFKA_HOST_TEST` from the compute-sanitizer memcheck nighly runs. The following error occurs when running this host test. ``` Running compute-sanitizer on KAFKA_HOST_TEST ========= COMPUTE-SANITIZER Running main() from gmock_main.cc [==========] Running 2 tests from 1 test suite. [----------] Global test environment set-up. [----------] 2 tests from KafkaDatasourceTest [ RUN ] KafkaDatasourceTest.MissingGroupID [ OK ] KafkaDatasourceTest.MissingGroupID (0 ms) [ RUN ] KafkaDatasourceTest.InvalidConfigValues [ OK ] KafkaDatasourceTest.InvalidConfigValues (0 ms) [----------] 2 tests from KafkaDatasourceTest (0 ms total) [----------] Global test environment tear-down [==========] 2 tests from 1 test suite ran. (0 ms total) [ PASSED ] 2 tests. ========= Error: Target application terminated before first instrumented API call ========= Tracking kernels launched by child processes requires the --target-processes all option. ``` Adding the `--target-processes all` option gives the same error. Disabling the check of this test since it is a host test that checks error conditions and does not appear to make any device calls. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12831 --- ci/test_cpp_memcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_cpp_memcheck.sh b/ci/test_cpp_memcheck.sh index 0cad4fc3a3f..db9ce143d51 100755 --- a/ci/test_cpp_memcheck.sh +++ b/ci/test_cpp_memcheck.sh @@ -11,7 +11,7 @@ set +e rapids-logger "Memcheck gtests with rmm_mode=cuda" export GTEST_CUDF_RMM_MODE=cuda COMPUTE_SANITIZER_CMD="compute-sanitizer --tool memcheck" -for gt in "$CONDA_PREFIX"/bin/gtests/{libcudf,libcudf_kafka}/* ; do +for gt in "$CONDA_PREFIX"/bin/gtests/libcudf/* ; do test_name=$(basename ${gt}) if [[ "$test_name" == "ERROR_TEST" ]] || [[ "$test_name" == "STREAM_IDENTIFICATION_TEST" ]]; then continue From 77c2e03ec572527b5c5c7a3f7a48b0cabd29abde Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 24 Feb 2023 10:47:44 -0800 Subject: [PATCH 57/69] Consolidate linter configs into pyproject.toml (#12834) This consolidation allows us to get rid of now unnecessary setup.cfg files (thanks to removing versioneer in #12741). It also allows us to move towards a fully pyproject.toml-driven build. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) - David Wendt (https://github.com/davidwendt) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12834 --- .flake8 | 24 +++++++ .pre-commit-config.yaml | 12 ++-- ci/release/update-version.sh | 2 +- cpp/benchmarks/common/generate_input.cu | 4 +- cpp/benchmarks/common/generate_input.hpp | 6 +- .../developer_guide/contributing_guide.md | 8 +-- pyproject.toml | 38 +++++++++++ python/cudf/cudf/_lib/utils.pyx | 4 +- python/cudf/pyproject.toml | 43 +++++++++++++ python/cudf/setup.cfg | 32 ---------- python/cudf_kafka/pyproject.toml | 46 +++++++++++++ python/cudf_kafka/setup.cfg | 35 ---------- python/custreamz/pyproject.toml | 45 +++++++++++++ python/custreamz/setup.cfg | 34 ---------- python/dask_cudf/pyproject.toml | 45 +++++++++++++ python/dask_cudf/setup.cfg | 31 --------- setup.cfg | 64 ------------------- 17 files changed, 261 insertions(+), 212 deletions(-) create mode 100644 .flake8 delete mode 100644 python/cudf/setup.cfg delete mode 100644 python/cudf_kafka/setup.cfg delete mode 100644 python/custreamz/setup.cfg delete mode 100644 setup.cfg diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000000..e80e3afc443 --- /dev/null +++ b/.flake8 @@ -0,0 +1,24 @@ +# Copyright (c) 2017-2023, NVIDIA CORPORATION. + +[flake8] +filename = *.py, *.pyx, *.pxd, *.pxi +exclude = __init__.py, *.egg, build, docs, .git +force-check = True +ignore = + # line break before binary operator + W503, + # whitespace before : + E203 +per-file-ignores = + # Rules ignored only in Cython: + # E211: whitespace before '(' (used in multi-line imports) + # E225: Missing whitespace around operators (breaks cython casting syntax like ) + # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) + # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) + # E275: Missing whitespace after keyword (Doesn't work with Cython except?) + # E402: invalid syntax (works for Python, not Cython) + # E999: invalid syntax (works for Python, not Cython) + # W504: line break after binary operator (breaks lines that end with a pointer) + *.pyx: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxd: E211, E225, E226, E227, E275, E402, E999, W504 + *.pxi: E211, E225, E226, E227, E275, E402, E999, W504 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 244fc0d3872..e252af717ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: rev: 5.0.4 hooks: - id: flake8 - args: ["--config=setup.cfg"] + args: ["--config=.flake8"] files: python/.*$ types: [file] types_or: [python, cython] @@ -48,7 +48,7 @@ repos: hooks: - id: mypy additional_dependencies: [types-cachetools] - args: ["--config-file=setup.cfg", + args: ["--config-file=pyproject.toml", "python/cudf/cudf", "python/custreamz/custreamz", "python/cudf_kafka/cudf_kafka", @@ -58,7 +58,9 @@ repos: rev: 6.1.1 hooks: - id: pydocstyle - args: ["--config=setup.cfg"] + # https://github.com/PyCQA/pydocstyle/issues/603 + additional_dependencies: [toml] + args: ["--config=pyproject.toml"] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v11.1.0 hooks: @@ -138,9 +140,11 @@ repos: pass_filenames: false verbose: false - repo: https://github.com/codespell-project/codespell - rev: v2.1.0 + rev: v2.2.2 hooks: - id: codespell + additional_dependencies: [tomli] + args: ["--toml", "pyproject.toml"] exclude: | (?x)^( .*test.*| diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index c8875fda641..831b91bb2a6 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -94,7 +94,7 @@ sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/setup sed_runner "s/cudf==.*\",/cudf==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/dask_cudf/setup.py # Dependency versions in pyproject.toml -sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/pyproject.toml +sed_runner "s/rmm==.*\",/rmm==${NEXT_SHORT_TAG_PEP440}.*\",/g" python/cudf/pyproject.toml for FILE in .github/workflows/*.yaml; do sed_runner "/shared-action-workflows/ s/@.*/@branch-${NEXT_SHORT_TAG}/g" "${FILE}" diff --git a/cpp/benchmarks/common/generate_input.cu b/cpp/benchmarks/common/generate_input.cu index dee7e2b8586..2829d14070c 100644 --- a/cpp/benchmarks/common/generate_input.cu +++ b/cpp/benchmarks/common/generate_input.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -501,7 +501,7 @@ std::unique_ptr create_random_utf8_string_column(data_profile cons rmm::device_uvector offsets(num_rows + 1, cudf::get_default_stream()); thrust::exclusive_scan( thrust::device, valid_lengths, valid_lengths + lengths.size(), offsets.begin()); - // offfsets are ready. + // offsets are ready. auto chars_length = *thrust::device_pointer_cast(offsets.end() - 1); rmm::device_uvector chars(chars_length, cudf::get_default_stream()); thrust::for_each_n(thrust::device, diff --git a/cpp/benchmarks/common/generate_input.hpp b/cpp/benchmarks/common/generate_input.hpp index f8ea194f0c4..e65aa69763b 100644 --- a/cpp/benchmarks/common/generate_input.hpp +++ b/cpp/benchmarks/common/generate_input.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -373,13 +373,13 @@ class data_profile { void set_bool_probability_true(double p) { - CUDF_EXPECTS(p >= 0. and p <= 1., "probablity must be in range [0...1]"); + CUDF_EXPECTS(p >= 0. and p <= 1., "probability must be in range [0...1]"); bool_probability_true = p; } void set_null_probability(std::optional p) { CUDF_EXPECTS(p.value_or(0.) >= 0. and p.value_or(0.) <= 1., - "probablity must be in range [0...1]"); + "probability must be in range [0...1]"); null_probability = p; } void set_cardinality(cudf::size_type c) { cardinality = c; } diff --git a/docs/cudf/source/developer_guide/contributing_guide.md b/docs/cudf/source/developer_guide/contributing_guide.md index 34071f44914..bb3479cf4c1 100644 --- a/docs/cudf/source/developer_guide/contributing_guide.md +++ b/docs/cudf/source/developer_guide/contributing_guide.md @@ -22,16 +22,16 @@ Specifically, cuDF uses the following tools: In conjunction with [type hints](https://docs.python.org/3/library/typing.html), `mypy` can help catch various bugs that are otherwise difficult to find. - [`pydocstyle`](https://github.com/PyCQA/pydocstyle/) lints docstring style. +- [`codespell`](https://github.com/codespell-project/codespell) finds spelling errors. Linter config data is stored in a number of files. -We generally use `pyproject.toml` over `setup.cfg` and avoid project-specific files (e.g. `setup.cfg` > `python/cudf/setup.cfg`). +We generally use `pyproject.toml` over `setup.cfg` and avoid project-specific files (e.g. `pyproject.toml` > `python/cudf/pyproject.toml`). However, differences between tools and the different packages in the repo result in the following caveats: -- `flake8` has no plans to support `pyproject.toml`, so it must live in `setup.cfg`. +- `flake8` has no plans to support `pyproject.toml`, so it must live in `.flake8`. - `isort` must be configured per project to set which project is the "first party" project. -Additionally, our use of `versioneer` means that each project must have a `setup.cfg`. -As a result, we currently maintain both root and project-level `pyproject.toml` and `setup.cfg` files. +As a result, we currently maintain both root and project-level `pyproject.toml` files as well as a `.flake8` file. For more information on how to use pre-commit hooks, see the code formatting section of the [overall contributing guide](https://github.com/rapidsai/cudf/blob/main/CONTRIBUTING.md#python--pre-commit-hooks). diff --git a/pyproject.toml b/pyproject.toml index dfd22f33785..3940d9119ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,41 @@ force-exclude = ''' dist )/ ''' + +[tool.pydocstyle] +# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather +# than include using match-dir. Note that as discussed in +# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle, +# unlike the match option above this match-dir will have no effect when +# pydocstyle is invoked from pre-commit. Therefore this exclusion list must +# also be maintained in the pre-commit config file. +match-dir = "^(?!(ci|cpp|conda|docs|java|notebooks)).*$" +# Allow missing docstrings for docutils +ignore-decorators = ".*(docutils|doc_apply|copy_docstring).*" +select = "D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418" + # Would like to enable the following rules in the future: + # D200, D202, D205, D400 + +[tool.mypy] +ignore_missing_imports = true +# If we don't specify this, then mypy will check excluded files if +# they are imported by a checked file. +follow_imports = "skip" +exclude = [ + "cudf/_lib/", + "cudf/cudf/benchmarks/", + "cudf/cudf/tests/", + "cudf/cudf/utils/metadata/orc_column_statistics_pb2.py", + "custreamz/custreamz/tests/", + "dask_cudf/dask_cudf/tests/", + ] + +[tool.codespell] +# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - +# this is only to allow you to run codespell interactively +skip = "./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,./cpp/tests,./python/cudf/cudf/tests,./java/src/test,./cpp/include/cudf_test/cxxopts.hpp" +# ignore short words, and typename parameters like OffsetT +ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" +ignore-words-list = "inout,unparseable,falsy" +builtin = "clear" +quiet-level = 3 diff --git a/python/cudf/cudf/_lib/utils.pyx b/python/cudf/cudf/_lib/utils.pyx index 5f4d3e17fbc..56918799cca 100644 --- a/python/cudf/cudf/_lib/utils.pyx +++ b/python/cudf/cudf/_lib/utils.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. import numpy as np import pyarrow as pa @@ -315,7 +315,7 @@ cdef columns_from_table_view( object owners, ): """ - Given a ``cudf::table_view``, construsts a list of columns from it, + Given a ``cudf::table_view``, constructs a list of columns from it, along with referencing an owner Python object that owns the memory lifetime. owner must be either None or a list of column. If owner is a list of columns, the owner of the `i`th ``cudf::column_view`` diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 49c4d83245f..305e8822030 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -15,3 +15,46 @@ requires = [ "protoc-wheel", "rmm==23.4.*", ] + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_dask = [ + "dask", + "distributed", + "dask_cuda", +] +known_rapids = [ + "rmm", +] +known_first_party = [ + "cudf", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] diff --git a/python/cudf/setup.cfg b/python/cudf/setup.cfg deleted file mode 100644 index 8380da371f9..00000000000 --- a/python/cudf/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2018-2023, NVIDIA CORPORATION. - -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda -known_rapids= - rmm -known_first_party= - cudf -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - __init__.py diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 0924fc90352..308a7869bc0 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -7,3 +7,49 @@ requires = [ "setuptools", "cython>=0.29,<0.30", ] + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_dask = [ + "dask", + "distributed", + "dask_cuda", + "streamz", +] +known_rapids = [ + "rmm", + "cudf", + "dask_cudf", +] +known_first_party = [ + "cudf_kafka", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] diff --git a/python/cudf_kafka/setup.cfg b/python/cudf_kafka/setup.cfg deleted file mode 100644 index ee0d783b184..00000000000 --- a/python/cudf_kafka/setup.cfg +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda - streamz -known_rapids= - rmm - cudf - dask_cudf -known_first_party= - cudf_kafka -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - __init__.py diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 806848c356e..d5c41945482 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -6,3 +6,48 @@ requires = [ "wheel", "setuptools", ] + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_dask = [ + "dask", + "distributed", + "dask_cuda", +] +known_rapids = [ + "rmm", + "cudf", + "dask_cudf", +] +known_first_party = [ + "streamz", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] diff --git a/python/custreamz/setup.cfg b/python/custreamz/setup.cfg deleted file mode 100644 index 8c038db9349..00000000000 --- a/python/custreamz/setup.cfg +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda -known_rapids= - rmm - cudf - dask_cudf -known_first_party= - streamz -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - __init__.py diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 806848c356e..8cf823d4291 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -6,3 +6,48 @@ requires = [ "wheel", "setuptools", ] + +[tool.isort] +line_length = 79 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true + +known_dask = [ + "dask", + "distributed", + "dask_cuda", +] +known_rapids = [ + "rmm", + "cudf", +] +known_first_party = [ + "dask_cudf", +] + +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "DASK", + "RAPIDS", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", +] diff --git a/python/dask_cudf/setup.cfg b/python/dask_cudf/setup.cfg index 66f4b8891d0..8139b3c7dc6 100644 --- a/python/dask_cudf/setup.cfg +++ b/python/dask_cudf/setup.cfg @@ -1,36 +1,5 @@ # Copyright (c) 2020-2023, NVIDIA CORPORATION. -[isort] -line_length=79 -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -combine_as_imports=True -order_by_type=True -known_dask= - dask - distributed - dask_cuda -known_rapids= - rmm - cudf -known_first_party= - dask_cudf -default_section=THIRDPARTY -sections=FUTURE,STDLIB,THIRDPARTY,DASK,RAPIDS,FIRSTPARTY,LOCALFOLDER -skip= - thirdparty - .eggs - .git - .hg - .mypy_cache - .tox - .venv - _build - buck-out - build - dist - [options.entry_points] dask.dataframe.backends = cudf = dask_cudf.backends:CudfBackendEntrypoint diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 962b7d73bbe..00000000000 --- a/setup.cfg +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2017-2023, NVIDIA CORPORATION. - -[flake8] -filename = *.py, *.pyx, *.pxd, *.pxi -exclude = __init__.py, *.egg, build, docs, .git -force-check = True -ignore = - # line break before binary operator - W503, - # whitespace before : - E203 -per-file-ignores = - # Rules ignored only in Cython: - # E211: whitespace before '(' (used in multi-line imports) - # E225: Missing whitespace around operators (breaks cython casting syntax like ) - # E226: Missing whitespace around arithmetic operators (breaks cython pointer syntax like int*) - # E227: Missing whitespace around bitwise or shift operator (Can also break casting syntax) - # E275: Missing whitespace after keyword (Doesn't work with Cython except?) - # E402: invalid syntax (works for Python, not Cython) - # E999: invalid syntax (works for Python, not Cython) - # W504: line break after binary operator (breaks lines that end with a pointer) - *.pyx: E211, E225, E226, E227, E275, E402, E999, W504 - *.pxd: E211, E225, E226, E227, E275, E402, E999, W504 - *.pxi: E211, E225, E226, E227, E275, E402, E999, W504 - -[pydocstyle] -# Due to https://github.com/PyCQA/pydocstyle/issues/363, we must exclude rather -# than include using match-dir. Note that as discussed in -# https://stackoverflow.com/questions/65478393/how-to-filter-directories-using-the-match-dir-flag-for-pydocstyle, -# unlike the match option above this match-dir will have no effect when -# pydocstyle is invoked from pre-commit. Therefore this exclusion list must -# also be maintained in the pre-commit config file. -match-dir = ^(?!(ci|cpp|conda|docs|java|notebooks)).*$ -# Allow missing docstrings for docutils -ignore-decorators = .*(docutils|doc_apply|copy_docstring).* -select = - D201, D204, D206, D207, D208, D209, D210, D211, D214, D215, D300, D301, D302, D403, D405, D406, D407, D408, D409, D410, D411, D412, D414, D418 - # Would like to enable the following rules in the future: - # D200, D202, D205, D400 - -[mypy] -ignore_missing_imports = True -# If we don't specify this, then mypy will check excluded files if -# they are imported by a checked file. -follow_imports = skip -exclude = (?x)( - cudf/_lib/ - | cudf/cudf/benchmarks/ - | cudf/cudf/tests/ - | cudf/cudf/utils/metadata/orc_column_statistics_pb2.py - | custreamz/custreamz/tests/ - | dask_cudf/dask_cudf/tests/ - # This close paren cannot be in column zero otherwise the config parser barfs - ) - -[codespell] -# note: pre-commit passes explicit lists of files here, which this skip file list doesn't override - -# this is only to allow you to run codespell interactively -skip = ./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,./cpp/tests,./python/cudf/cudf/tests,./java/src/test,./cpp/include/cudf_test/cxxopts.hpp -# ignore short words, and typename parameters like OffsetT -ignore-regex = \b(.{1,4}|[A-Z]\w*T)\b -ignore-words-list = inout,unparseable -builtin = clear -quiet-level = 3 From 4f2f37987fbd66de0cc9116734d2094ca4a39948 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 24 Feb 2023 17:04:59 -0600 Subject: [PATCH 58/69] Enable nbqa pre-commit hooks for isort and black. (#12848) This enables `black` and `isort` linters for ipynb notebooks via [nbqa](https://github.com/nbQA-dev/nbQA). I propose this change to avoid manually linting notebooks like https://github.com/rapidsai/cudf/pull/12595. cc: @galipremsagar Authors: - Bradley Dice (https://github.com/bdice) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12848 --- .pre-commit-config.yaml | 10 ++ docs/cudf/source/user_guide/10min.ipynb | 1 + .../cudf/source/user_guide/cupy-interop.ipynb | 34 ++-- .../source/user_guide/guide-to-udfs.ipynb | 149 +++++++++--------- .../cudf/source/user_guide/missing-data.ipynb | 56 ++++--- 5 files changed, 141 insertions(+), 109 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e252af717ce..a030f3bd25b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,6 +61,16 @@ repos: # https://github.com/PyCQA/pydocstyle/issues/603 additional_dependencies: [toml] args: ["--config=pyproject.toml"] + - repo: https://github.com/nbQA-dev/nbQA + rev: 1.6.3 + hooks: + - id: nbqa-isort + # Use the cudf_kafka isort orderings in notebooks so that dask + # and RAPIDS packages have their own sections. + args: ["--settings-file=python/cudf_kafka/pyproject.toml"] + - id: nbqa-black + # Explicitly specify the pyproject.toml at the repo root, not per-project. + args: ["--config=pyproject.toml"] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v11.1.0 hooks: diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index af938b79a29..0352c624e04 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -35,6 +35,7 @@ "\n", "import cupy as cp\n", "import pandas as pd\n", + "\n", "import cudf\n", "import dask_cudf\n", "\n", diff --git a/docs/cudf/source/user_guide/cupy-interop.ipynb b/docs/cudf/source/user_guide/cupy-interop.ipynb index 3e169984ace..c98a4ddea23 100644 --- a/docs/cudf/source/user_guide/cupy-interop.ipynb +++ b/docs/cudf/source/user_guide/cupy-interop.ipynb @@ -18,9 +18,10 @@ "outputs": [], "source": [ "import timeit\n", - "from packaging import version\n", "\n", "import cupy as cp\n", + "from packaging import version\n", + "\n", "import cudf\n", "\n", "if version.parse(cp.__version__) >= version.parse(\"10.0.0\"):\n", @@ -63,10 +64,13 @@ ], "source": [ "nelem = 10000\n", - "df = cudf.DataFrame({'a':range(nelem),\n", - " 'b':range(500, nelem + 500),\n", - " 'c':range(1000, nelem + 1000)}\n", - " )\n", + "df = cudf.DataFrame(\n", + " {\n", + " \"a\": range(nelem),\n", + " \"b\": range(500, nelem + 500),\n", + " \"c\": range(1000, nelem + 1000),\n", + " }\n", + ")\n", "\n", "%timeit arr_cupy = cupy_from_dlpack(df.to_dlpack())\n", "%timeit arr_cupy = df.values\n", @@ -138,7 +142,7 @@ } ], "source": [ - "col = 'a'\n", + "col = \"a\"\n", "\n", "%timeit cola_cupy = cp.asarray(df[col])\n", "%timeit cola_cupy = cupy_from_dlpack(df[col].to_dlpack())\n", @@ -1088,14 +1092,16 @@ "metadata": {}, "outputs": [], "source": [ - "def cudf_to_cupy_sparse_matrix(data, sparseformat='column'):\n", - " \"\"\"Converts a cuDF object to a CuPy Sparse Column matrix.\n", - " \"\"\"\n", - " if sparseformat not in ('row', 'column',):\n", + "def cudf_to_cupy_sparse_matrix(data, sparseformat=\"column\"):\n", + " \"\"\"Converts a cuDF object to a CuPy Sparse Column matrix.\"\"\"\n", + " if sparseformat not in (\n", + " \"row\",\n", + " \"column\",\n", + " ):\n", " raise ValueError(\"Let's focus on column and row formats for now.\")\n", - " \n", + "\n", " _sparse_constructor = cp.sparse.csc_matrix\n", - " if sparseformat == 'row':\n", + " if sparseformat == \"row\":\n", " _sparse_constructor = cp.sparse.csr_matrix\n", "\n", " return _sparse_constructor(cupy_from_dlpack(data.to_dlpack()))" @@ -1121,8 +1127,8 @@ "nonzero = 1000\n", "for i in range(20):\n", " arr = cp.random.normal(5, 5, nelem)\n", - " arr[cp.random.choice(arr.shape[0], nelem-nonzero, replace=False)] = 0\n", - " df['a' + str(i)] = arr" + " arr[cp.random.choice(arr.shape[0], nelem - nonzero, replace=False)] = 0\n", + " df[\"a\" + str(i)] = arr" ] }, { diff --git a/docs/cudf/source/user_guide/guide-to-udfs.ipynb b/docs/cudf/source/user_guide/guide-to-udfs.ipynb index 943fc980a31..ba8c65784d2 100644 --- a/docs/cudf/source/user_guide/guide-to-udfs.ipynb +++ b/docs/cudf/source/user_guide/guide-to-udfs.ipynb @@ -15,9 +15,10 @@ "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", + "\n", "import cudf\n", - "from cudf.datasets import randomdata\n", - "import numpy as np" + "from cudf.datasets import randomdata" ] }, { @@ -375,7 +376,7 @@ "metadata": {}, "outputs": [], "source": [ - "sr = cudf.Series(['', 'abc', 'some_example'])" + "sr = cudf.Series([\"\", \"abc\", \"some_example\"])" ] }, { @@ -387,9 +388,9 @@ "source": [ "def f(st):\n", " if len(st) > 0:\n", - " if st.startswith('a'):\n", + " if st.startswith(\"a\"):\n", " return 1\n", - " elif 'example' in st:\n", + " elif \"example\" in st:\n", " return 2\n", " else:\n", " return -1\n", @@ -443,6 +444,7 @@ "outputs": [], "source": [ "from cudf.core.udf.utils import set_malloc_heap_size\n", + "\n", "set_malloc_heap_size(int(2e9))" ] }, @@ -472,7 +474,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = randomdata(nrows=5, dtypes={'a':int, 'b':int, 'c':int}, seed=12)" + "df = randomdata(nrows=5, dtypes={\"a\": int, \"b\": int, \"c\": int}, seed=12)" ] }, { @@ -484,10 +486,11 @@ "source": [ "from numba import cuda\n", "\n", + "\n", "@cuda.jit\n", "def multiply(in_col, out_col, multiplier):\n", " i = cuda.grid(1)\n", - " if i < in_col.size: # boundary guard\n", + " if i < in_col.size: # boundary guard\n", " out_col[i] = in_col[i] * multiplier" ] }, @@ -508,9 +511,9 @@ "metadata": {}, "outputs": [], "source": [ - "size = len(df['a'])\n", - "df['e'] = 0.0\n", - "multiply.forall(size)(df['a'], df['e'], 10.0)" + "size = len(df[\"a\"])\n", + "df[\"e\"] = 0.0\n", + "multiply.forall(size)(df[\"a\"], df[\"e\"], 10.0)" ] }, { @@ -658,7 +661,7 @@ "outputs": [], "source": [ "def f(row):\n", - " return row['A'] + row['B']" + " return row[\"A\"] + row[\"B\"]" ] }, { @@ -733,10 +736,7 @@ } ], "source": [ - "df = cudf.DataFrame({\n", - " 'A': [1,2,3],\n", - " 'B': [4,cudf.NA,6]\n", - "})\n", + "df = cudf.DataFrame({\"A\": [1, 2, 3], \"B\": [4, cudf.NA, 6]})\n", "df" ] }, @@ -881,13 +881,14 @@ ], "source": [ "def f(row):\n", - " x = row['a']\n", + " x = row[\"a\"]\n", " if x is cudf.NA:\n", " return 0\n", " else:\n", " return x + 1\n", "\n", - "df = cudf.DataFrame({'a': [1, cudf.NA, 3]})\n", + "\n", + "df = cudf.DataFrame({\"a\": [1, cudf.NA, 3]})\n", "df" ] }, @@ -988,17 +989,15 @@ ], "source": [ "def f(row):\n", - " x = row['a']\n", - " y = row['b']\n", + " x = row[\"a\"]\n", + " y = row[\"b\"]\n", " if x + y > 3:\n", " return cudf.NA\n", " else:\n", " return x + y\n", "\n", - "df = cudf.DataFrame({\n", - " 'a': [1, 2, 3], \n", - " 'b': [2, 1, 1]\n", - "})\n", + "\n", + "df = cudf.DataFrame({\"a\": [1, 2, 3], \"b\": [2, 1, 1]})\n", "df" ] }, @@ -1099,12 +1098,10 @@ ], "source": [ "def f(row):\n", - " return row['a'] + row['b']\n", + " return row[\"a\"] + row[\"b\"]\n", + "\n", "\n", - "df = cudf.DataFrame({\n", - " 'a': [1, 2, 3], \n", - " 'b': [0.5, cudf.NA, 3.14]\n", - "})\n", + "df = cudf.DataFrame({\"a\": [1, 2, 3], \"b\": [0.5, cudf.NA, 3.14]})\n", "df" ] }, @@ -1214,15 +1211,14 @@ ], "source": [ "def f(row):\n", - " x = row['a']\n", + " x = row[\"a\"]\n", " if x > 3:\n", - " return x\n", + " return x\n", " else:\n", - " return 1.5\n", + " return 1.5\n", + "\n", "\n", - "df = cudf.DataFrame({\n", - " 'a': [1, 3, 5]\n", - "})\n", + "df = cudf.DataFrame({\"a\": [1, 3, 5]})\n", "df" ] }, @@ -1335,15 +1331,18 @@ ], "source": [ "def f(row):\n", - " return row['a'] + (row['b'] - (row['c'] / row['d'])) % row['e']\n", + " return row[\"a\"] + (row[\"b\"] - (row[\"c\"] / row[\"d\"])) % row[\"e\"]\n", "\n", - "df = cudf.DataFrame({\n", - " 'a': [1, 2, 3],\n", - " 'b': [4, 5, 6],\n", - " 'c': [cudf.NA, 4, 4],\n", - " 'd': [8, 7, 8],\n", - " 'e': [7, 1, 6]\n", - "})\n", + "\n", + "df = cudf.DataFrame(\n", + " {\n", + " \"a\": [1, 2, 3],\n", + " \"b\": [4, 5, 6],\n", + " \"c\": [cudf.NA, 4, 4],\n", + " \"d\": [8, 7, 8],\n", + " \"e\": [7, 1, 6],\n", + " }\n", + ")\n", "df" ] }, @@ -1451,10 +1450,9 @@ } ], "source": [ - "str_df = cudf.DataFrame({\n", - " 'str_col': ['abc', 'ABC', 'Example'],\n", - " 'scale': [1, 2, 3]\n", - "})\n", + "str_df = cudf.DataFrame(\n", + " {\"str_col\": [\"abc\", \"ABC\", \"Example\"], \"scale\": [1, 2, 3]}\n", + ")\n", "str_df" ] }, @@ -1466,9 +1464,9 @@ "outputs": [], "source": [ "def f(row):\n", - " st = row['str_col']\n", - " scale = row['scale']\n", - " \n", + " st = row[\"str_col\"]\n", + " scale = row[\"scale\"]\n", + "\n", " if len(st) > 5:\n", " return len(st) + scale\n", " else:\n", @@ -1626,11 +1624,12 @@ } ], "source": [ - "df = df.apply_rows(conditional_add, \n", - " incols={'a':'x', 'e':'y'},\n", - " outcols={'out': np.float64},\n", - " kwargs={}\n", - " )\n", + "df = df.apply_rows(\n", + " conditional_add,\n", + " incols={\"a\": \"x\", \"e\": \"y\"},\n", + " outcols={\"out\": np.float64},\n", + " kwargs={},\n", + ")\n", "df.head()" ] }, @@ -1738,10 +1737,11 @@ " for i, (x, y) in enumerate(zip(a, b)):\n", " out[i] = x + y\n", "\n", - "df = randomdata(nrows=5, dtypes={'a':int, 'b':int, 'c':int}, seed=12)\n", - "df.loc[2, 'a'] = None\n", - "df.loc[3, 'b'] = None\n", - "df.loc[1, 'c'] = None\n", + "\n", + "df = randomdata(nrows=5, dtypes={\"a\": int, \"b\": int, \"c\": int}, seed=12)\n", + "df.loc[2, \"a\"] = None\n", + "df.loc[3, \"b\"] = None\n", + "df.loc[1, \"c\"] = None\n", "df.head()" ] }, @@ -1841,10 +1841,9 @@ } ], "source": [ - "df = df.apply_rows(gpu_add, \n", - " incols=['a', 'b'],\n", - " outcols={'out':np.float64},\n", - " kwargs={})\n", + "df = df.apply_rows(\n", + " gpu_add, incols=[\"a\", \"b\"], outcols={\"out\": np.float64}, kwargs={}\n", + ")\n", "df.head()" ] }, @@ -1892,7 +1891,7 @@ } ], "source": [ - "ser = cudf.Series([16, 25, 36, 49, 64, 81], dtype='float64')\n", + "ser = cudf.Series([16, 25, 36, 49, 64, 81], dtype=\"float64\")\n", "ser" ] }, @@ -1935,12 +1934,13 @@ "source": [ "import math\n", "\n", + "\n", "def example_func(window):\n", " b = 0\n", " for a in window:\n", " b = max(b, math.sqrt(a))\n", " if b == 8:\n", - " return 100 \n", + " return 100\n", " return b" ] }, @@ -2064,8 +2064,8 @@ ], "source": [ "df2 = cudf.DataFrame()\n", - "df2['a'] = np.arange(55, 65, dtype='float64')\n", - "df2['b'] = np.arange(55, 65, dtype='float64')\n", + "df2[\"a\"] = np.arange(55, 65, dtype=\"float64\")\n", + "df2[\"b\"] = np.arange(55, 65, dtype=\"float64\")\n", "df2.head()" ] }, @@ -2279,7 +2279,9 @@ } ], "source": [ - "df = randomdata(nrows=10, dtypes={'a':float, 'b':bool, 'c':str, 'e': float}, seed=12)\n", + "df = randomdata(\n", + " nrows=10, dtypes={\"a\": float, \"b\": bool, \"c\": str, \"e\": float}, seed=12\n", + ")\n", "df.head()" ] }, @@ -2290,7 +2292,7 @@ "metadata": {}, "outputs": [], "source": [ - "grouped = df.groupby(['b'])" + "grouped = df.groupby([\"b\"])" ] }, { @@ -2469,9 +2471,9 @@ } ], "source": [ - "results = grouped.apply_grouped(rolling_avg,\n", - " incols=['e'],\n", - " outcols=dict(rolling_avg_e=np.float64))\n", + "results = grouped.apply_grouped(\n", + " rolling_avg, incols=[\"e\"], outcols=dict(rolling_avg_e=np.float64)\n", + ")\n", "results" ] }, @@ -2554,8 +2556,9 @@ " i = cuda.grid(1)\n", " if i < x.size:\n", " out[i] = x[i] * 5\n", - " \n", - "out = cudf.Series(cp.zeros(len(s), dtype='int32'))\n", + "\n", + "\n", + "out = cudf.Series(cp.zeros(len(s), dtype=\"int32\"))\n", "multiply_by_5.forall(s.shape[0])(s, out)\n", "out" ] diff --git a/docs/cudf/source/user_guide/missing-data.ipynb b/docs/cudf/source/user_guide/missing-data.ipynb index ac5bddd34cf..f1404ce0b77 100644 --- a/docs/cudf/source/user_guide/missing-data.ipynb +++ b/docs/cudf/source/user_guide/missing-data.ipynb @@ -39,8 +39,9 @@ "metadata": {}, "outputs": [], "source": [ - "import cudf\n", - "import numpy as np" + "import numpy as np\n", + "\n", + "import cudf" ] }, { @@ -50,7 +51,7 @@ "metadata": {}, "outputs": [], "source": [ - "df = cudf.DataFrame({'a': [1, 2, None, 4], 'b':[0.1, None, 2.3, 17.17]})" + "df = cudf.DataFrame({\"a\": [1, 2, None, 4], \"b\": [0.1, None, 2.3, 17.17]})" ] }, { @@ -221,7 +222,7 @@ } ], "source": [ - "df['a'].notna()" + "df[\"a\"].notna()" ] }, { @@ -304,7 +305,7 @@ } ], "source": [ - "df['b'] == np.nan" + "df[\"b\"] == np.nan" ] }, { @@ -535,7 +536,10 @@ ], "source": [ "import pandas as pd\n", - "datetime_series = cudf.Series([pd.Timestamp(\"20120101\"), pd.NaT, pd.Timestamp(\"20120101\")])\n", + "\n", + "datetime_series = cudf.Series(\n", + " [pd.Timestamp(\"20120101\"), pd.NaT, pd.Timestamp(\"20120101\")]\n", + ")\n", "datetime_series" ] }, @@ -618,7 +622,12 @@ "metadata": {}, "outputs": [], "source": [ - "df1 = cudf.DataFrame({'a':[1, None, 2, 3, None], 'b':cudf.Series([np.nan, 2, 3.2, 0.1, 1], nan_as_null=False)})" + "df1 = cudf.DataFrame(\n", + " {\n", + " \"a\": [1, None, 2, 3, None],\n", + " \"b\": cudf.Series([np.nan, 2, 3.2, 0.1, 1], nan_as_null=False),\n", + " }\n", + ")" ] }, { @@ -628,7 +637,9 @@ "metadata": {}, "outputs": [], "source": [ - "df2 = cudf.DataFrame({'a':[1, 11, 2, 34, 10], 'b':cudf.Series([0.23, 22, 3.2, None, 1])})" + "df2 = cudf.DataFrame(\n", + " {\"a\": [1, 11, 2, 34, 10], \"b\": cudf.Series([0.23, 22, 3.2, None, 1])}\n", + ")" ] }, { @@ -899,7 +910,7 @@ } ], "source": [ - "df1['a']" + "df1[\"a\"]" ] }, { @@ -920,7 +931,7 @@ } ], "source": [ - "df1['a'].sum()" + "df1[\"a\"].sum()" ] }, { @@ -949,7 +960,7 @@ } ], "source": [ - "df1['a'].mean()" + "df1[\"a\"].mean()" ] }, { @@ -980,7 +991,7 @@ } ], "source": [ - "df1['a'].sum(skipna=False)" + "df1[\"a\"].sum(skipna=False)" ] }, { @@ -1001,7 +1012,7 @@ } ], "source": [ - "df1['a'].mean(skipna=False)" + "df1[\"a\"].mean(skipna=False)" ] }, { @@ -1035,7 +1046,7 @@ } ], "source": [ - "df1['a'].cumsum()" + "df1[\"a\"].cumsum()" ] }, { @@ -1069,7 +1080,7 @@ } ], "source": [ - "df1['a'].cumsum(skipna=False)" + "df1[\"a\"].cumsum(skipna=False)" ] }, { @@ -1148,7 +1159,7 @@ } ], "source": [ - "cudf.Series([], dtype='float64').sum()" + "cudf.Series([], dtype=\"float64\").sum()" ] }, { @@ -1219,7 +1230,7 @@ } ], "source": [ - "cudf.Series([], dtype='float64').prod()" + "cudf.Series([], dtype=\"float64\").prod()" ] }, { @@ -1382,7 +1393,7 @@ } ], "source": [ - "df1.groupby('a').mean()" + "df1.groupby(\"a\").mean()" ] }, { @@ -1463,7 +1474,7 @@ } ], "source": [ - "df1.groupby('a', dropna=False).mean()" + "df1.groupby(\"a\", dropna=False).mean()" ] }, { @@ -1670,7 +1681,7 @@ } ], "source": [ - "df1['b'].fillna(10)" + "df1[\"b\"].fillna(10)" ] }, { @@ -1697,7 +1708,8 @@ "outputs": [], "source": [ "import cupy as cp\n", - "dff = cudf.DataFrame(cp.random.randn(10, 3), columns=list('ABC'))" + "\n", + "dff = cudf.DataFrame(cp.random.randn(10, 3), columns=list(\"ABC\"))" ] }, { @@ -2339,7 +2351,7 @@ } ], "source": [ - "df1['a'].dropna()" + "df1[\"a\"].dropna()" ] }, { From d14d980b63402a779a3f75cc64cb3a5a0be7898d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 24 Feb 2023 15:30:07 -0800 Subject: [PATCH 59/69] Add dfg as a pre-commit hook (#12819) This change allows local and remote runs to handle calls to dfg identically, and removes the need for a separate CI check. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12819 --- .github/workflows/pr.yaml | 2 ++ .pre-commit-config.yaml | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 952b58abda5..3a80139e333 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -30,6 +30,8 @@ jobs: checks: secrets: inherit uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@branch-23.04 + with: + enable_check_generated_files: false conda-cpp-build: needs: checks secrets: inherit diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a030f3bd25b..1eb2c508db9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -160,6 +160,11 @@ repos: .*test.*| ^CHANGELOG.md$ ) + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.4.0 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] default_language_version: python: python3 From eb4da9345f172c3911f78c5e851757ec2ec222b9 Mon Sep 17 00:00:00 2001 From: Carl Simon Adorf Date: Sat, 25 Feb 2023 01:13:34 +0100 Subject: [PATCH 60/69] CI: Remove specification of manual stage for check_style.sh script. (#12803) Do not explicitly specify to run the "manual" stage when running pre-commits as part of the ci/check_style.sh script. Authors: - Carl Simon Adorf (https://github.com/csadorf) - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12803 --- ci/check_style.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/check_style.sh b/ci/check_style.sh index 020143095ce..f9bfea7b47c 100755 --- a/ci/check_style.sh +++ b/ci/check_style.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. set -euo pipefail @@ -20,4 +20,4 @@ mkdir -p $(dirname ${RAPIDS_CMAKE_FORMAT_FILE}) wget -O ${RAPIDS_CMAKE_FORMAT_FILE} ${FORMAT_FILE_URL} # Run pre-commit checks -pre-commit run --hook-stage manual --all-files --show-diff-on-failure +pre-commit run --all-files --show-diff-on-failure From 173459e99fa8fce4202e2613f1f2c89a016cd350 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 24 Feb 2023 21:12:33 -0800 Subject: [PATCH 61/69] Replace message parsing with throwing more specific exceptions (#12426) This PR identifies places where cuDF Python is parsing exception messages from libcudf in order to throw an appropriate Python exception and modifies the associated libcudf error macros (`CUDF_EXPECTS` or `CUDF_FAIL`) to throw a more descriptive exception. In order to fully support this behavior with arbitrary libcudf exceptions, this PR also introduces a custom Cython exception handler that can be extended to catch and handle any new type of exception thrown by libcudf. To cases where issues with the dtype of an argument is treated as a TypeError rather than a ValueError in pandas, a new exception type `dtype_error : public logic_error` is defined and mapped to a TypeError in Python. This PR does not aim to exhaustively improve the choice of thrown exceptions throughout libcudf, only in places that are immediately relevant to removing message parsing in cudf Python. Resolves #10632 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Robert (Bobby) Evans (https://github.com/revans2) - Ashwin Srinath (https://github.com/shwina) - Matthew Roeschke (https://github.com/mroeschke) - David Wendt (https://github.com/davidwendt) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12426 --- cpp/include/cudf/binaryop.hpp | 5 +- cpp/include/cudf/concatenate.hpp | 10 +-- cpp/include/cudf/lists/combine.hpp | 8 +- cpp/include/cudf/lists/contains.hpp | 9 +-- cpp/include/cudf/lists/gather.hpp | 4 +- cpp/include/cudf/strings/json.hpp | 4 +- cpp/include/cudf/utilities/error.hpp | 25 +++++- cpp/src/binaryop/binaryop.cpp | 2 +- cpp/src/binaryop/compiled/equality_ops.cu | 5 +- .../binaryop/compiled/struct_binary_ops.cuh | 3 +- cpp/src/copying/concatenate.cu | 11 ++- .../combine/concatenate_list_elements.cu | 8 +- cpp/src/lists/contains.cu | 3 +- cpp/src/lists/copying/segmented_gather.cu | 4 +- cpp/src/strings/json/json_path.cu | 7 +- .../binop-compiled-fixed_point-test.cpp | 2 +- cpp/tests/copying/concatenate_tests.cu | 25 +++--- .../copying/segmented_gather_list_tests.cpp | 6 +- .../concatenate_list_elements_tests.cpp | 8 +- cpp/tests/lists/contains_tests.cpp | 33 +++++--- cpp/tests/strings/json_tests.cpp | 10 ++- .../developer_guide/contributing_guide.md | 16 ++-- java/src/main/native/include/jni_utils.hpp | 6 +- python/cudf/cudf/_lib/CMakeLists.txt | 5 ++ python/cudf/cudf/_lib/cpp/copying.pxd | 5 +- python/cudf/cudf/_lib/cpp/lists/contains.pxd | 9 ++- python/cudf/cudf/_lib/exception_handler.hpp | 80 +++++++++++++++++++ python/cudf/cudf/_lib/exception_handler.pxd | 5 ++ python/cudf/cudf/core/column/column.py | 17 +--- python/cudf/cudf/core/column/decimal.py | 48 +++++------ python/cudf/cudf/core/column/lists.py | 67 ++++------------ python/cudf/cudf/core/column/string.py | 33 +++----- python/cudf/cudf/tests/test_column.py | 5 +- 33 files changed, 284 insertions(+), 204 deletions(-) create mode 100644 python/cudf/cudf/_lib/exception_handler.hpp create mode 100644 python/cudf/cudf/_lib/exception_handler.pxd diff --git a/cpp/include/cudf/binaryop.hpp b/cpp/include/cudf/binaryop.hpp index 6371cb6c82b..77d6a4d1e89 100644 --- a/cpp/include/cudf/binaryop.hpp +++ b/cpp/include/cudf/binaryop.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -108,6 +108,7 @@ enum class binary_operator : int32_t { * @throw cudf::logic_error if @p output_type dtype isn't fixed-width * @throw cudf::logic_error if @p output_type dtype isn't boolean for comparison and logical * operations. + * @throw cudf::data_type_error if the operation is not supported for the types of @p lhs and @p rhs */ std::unique_ptr binary_operation( scalar const& lhs, @@ -136,6 +137,7 @@ std::unique_ptr binary_operation( * @throw cudf::logic_error if @p output_type dtype isn't fixed-width * @throw cudf::logic_error if @p output_type dtype isn't boolean for comparison and logical * operations. + * @throw cudf::data_type_error if the operation is not supported for the types of @p lhs and @p rhs */ std::unique_ptr binary_operation( column_view const& lhs, @@ -163,6 +165,7 @@ std::unique_ptr binary_operation( * @throw cudf::logic_error if @p output_type dtype isn't boolean for comparison and logical * operations. * @throw cudf::logic_error if @p output_type dtype isn't fixed-width + * @throw cudf::data_type_error if the operation is not supported for the types of @p lhs and @p rhs */ std::unique_ptr binary_operation( column_view const& lhs, diff --git a/cpp/include/cudf/concatenate.hpp b/cpp/include/cudf/concatenate.hpp index 1f8ce65ad93..b20c97b3c31 100644 --- a/cpp/include/cudf/concatenate.hpp +++ b/cpp/include/cudf/concatenate.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,8 +50,8 @@ rmm::device_buffer concatenate_masks( /** * @brief Concatenates multiple columns into a single column. * - * @throws cudf::logic_error - * If types of the input columns mismatch + * @throws cudf::logic_error If types of the input columns mismatch + * @throws std::overflow_error If the the total number of output rows exceeds cudf::size_type * * @param columns_to_concat host_span of column views to be concatenated into a single column * @param mr Device memory resource used to allocate the returned column's device memory @@ -80,8 +80,8 @@ std::unique_ptr concatenate( * column_view tc1 = (t->view()).column(1); //Contains {0,1,2,3,4,5,6,7} * ``` * - * @throws cudf::logic_error - * If number of columns mismatch + * @throws cudf::logic_error If number of columns mismatch + * @throws std::overflow_error If the the total number of output rows exceeds cudf::size_type * * @param tables_to_concat host_span of table views to be concatenated into a single table * @param mr Device memory resource used to allocate the returned table's device memory diff --git a/cpp/include/cudf/lists/combine.hpp b/cpp/include/cudf/lists/combine.hpp index 4f211e87cc7..531396e940e 100644 --- a/cpp/include/cudf/lists/combine.hpp +++ b/cpp/include/cudf/lists/combine.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -80,10 +80,10 @@ std::unique_ptr concatenate_rows( * r is [ {1, 2, 3, 4, 5}, {6, 7, 8, 9} ] * @endcode * - * @throws cudf::logic_error if the input column is not at least two-level depth lists column (i.e., - * each row must be a list of list). + * @throws std::invalid_argument if the input column is not at least two-level depth lists column + * (i.e., each row must be a list of list). * @throws cudf::logic_error if the input lists column contains nested typed entries that are not - * lists. + * lists. * * @param input The lists column containing lists of list elements to concatenate. * @param null_policy The parameter to specify whether a null list element will be ignored from diff --git a/cpp/include/cudf/lists/contains.hpp b/cpp/include/cudf/lists/contains.hpp index d2b4d59dfba..a9f06bf399c 100644 --- a/cpp/include/cudf/lists/contains.hpp +++ b/cpp/include/cudf/lists/contains.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,9 +126,7 @@ enum class duplicate_find_option : int32_t { * @param mr Device memory resource used to allocate the returned column's device memory. * @return std::unique_ptr INT32 column of `n` rows with the location of the `search_key` * - * @throw cudf::logic_error If `search_key` type does not match the element type in `lists` - * @throw cudf::logic_error If `search_key` is of a nested type, or `lists` contains nested - * elements (LIST, STRUCT) + * @throw cudf::data_type_error If `search_keys` type does not match the element type in `lists` */ std::unique_ptr index_of( cudf::lists_column_view const& lists, @@ -163,8 +161,7 @@ std::unique_ptr index_of( * @return std::unique_ptr INT32 column of `n` rows with the location of the `search_key` * * @throw cudf::logic_error If `search_keys` does not match `lists` in its number of rows - * @throw cudf::logic_error If `search_keys` type does not match the element type in `lists` - * @throw cudf::logic_error If `lists` or `search_keys` contains nested elements (LIST, STRUCT) + * @throw cudf::data_type_error If `search_keys` type does not match the element type in `lists` */ std::unique_ptr index_of( cudf::lists_column_view const& lists, diff --git a/cpp/include/cudf/lists/gather.hpp b/cpp/include/cudf/lists/gather.hpp index f91ce29a7cb..38bed9ede43 100644 --- a/cpp/include/cudf/lists/gather.hpp +++ b/cpp/include/cudf/lists/gather.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ namespace lists { * @endcode * * @throws cudf::logic_error if `gather_map_list` size is not same as `source_column` size. - * @throws cudf::logic_error if gather_map contains null values. + * @throws std::invalid_argument if gather_map contains null values. * @throws cudf::logic_error if gather_map is not list column of an index type. * * If indices in `gather_map_list` are outside the range `[-n, n)`, where `n` is the number of diff --git a/cpp/include/cudf/strings/json.hpp b/cpp/include/cudf/strings/json.hpp index 11e8daa9855..8fabee6b9a5 100644 --- a/cpp/include/cudf/strings/json.hpp +++ b/cpp/include/cudf/strings/json.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -160,6 +160,8 @@ class get_json_object_options { * @param options Options for controlling the behavior of the function * @param mr Resource for allocating device memory. * @return New strings column containing the retrieved json object strings + * + * @throw std::invalid_argument if provided an invalid operator or an empty name */ std::unique_ptr get_json_object( cudf::strings_column_view const& col, diff --git a/cpp/include/cudf/utilities/error.hpp b/cpp/include/cudf/utilities/error.hpp index 38ca0f2651e..f70ef4e5f07 100644 --- a/cpp/include/cudf/utilities/error.hpp +++ b/cpp/include/cudf/utilities/error.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -84,6 +84,29 @@ struct cuda_error : public std::runtime_error { struct fatal_cuda_error : public cuda_error { using cuda_error::cuda_error; // Inherit constructors }; + +/** + * @brief Exception thrown when an operation is attempted on an unsupported dtype. + * + * This exception should be thrown when an operation is attempted on an + * unsupported data_type. This exception should not be thrown directly and is + * instead thrown by the CUDF_EXPECTS or CUDF_FAIL macros. + */ +struct data_type_error : public std::invalid_argument { + /** + * @brief Constructs a data_type_error with the error message. + * + * @param message Message to be associated with the exception + */ + data_type_error(char const* const message) : std::invalid_argument(message) {} + + /** + * @brief Construct a new data_type_error object with error message + * + * @param message Message to be associated with the exception + */ + data_type_error(std::string const& message) : std::invalid_argument(message) {} +}; /** @} */ } // namespace cudf diff --git a/cpp/src/binaryop/binaryop.cpp b/cpp/src/binaryop/binaryop.cpp index b23c1fc9fe1..f81f0dcc311 100644 --- a/cpp/src/binaryop/binaryop.cpp +++ b/cpp/src/binaryop/binaryop.cpp @@ -203,7 +203,7 @@ std::unique_ptr binary_operation(LhsType const& lhs, return cudf::binops::compiled::string_null_min_max(lhs, rhs, op, output_type, stream, mr); if (not cudf::binops::compiled::is_supported_operation(output_type, lhs.type(), rhs.type(), op)) - CUDF_FAIL("Unsupported operator for these types"); + CUDF_FAIL("Unsupported operator for these types", cudf::data_type_error); if (cudf::is_fixed_point(lhs.type()) or cudf::is_fixed_point(rhs.type())) { cudf::binops::compiled::fixed_point_binary_operation_validation( diff --git a/cpp/src/binaryop/compiled/equality_ops.cu b/cpp/src/binaryop/compiled/equality_ops.cu index 61f02252a26..041fca76494 100644 --- a/cpp/src/binaryop/compiled/equality_ops.cu +++ b/cpp/src/binaryop/compiled/equality_ops.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,8 @@ void dispatch_equality_op(mutable_column_view& out, rmm::cuda_stream_view stream) { CUDF_EXPECTS(op == binary_operator::EQUAL || op == binary_operator::NOT_EQUAL, - "Unsupported operator for these types"); + "Unsupported operator for these types", + cudf::data_type_error); auto common_dtype = get_common_type(out.type(), lhs.type(), rhs.type()); auto outd = mutable_column_device_view::create(out, stream); auto lhsd = column_device_view::create(lhs, stream); diff --git a/cpp/src/binaryop/compiled/struct_binary_ops.cuh b/cpp/src/binaryop/compiled/struct_binary_ops.cuh index d167f0fe3c5..8418493318f 100644 --- a/cpp/src/binaryop/compiled/struct_binary_ops.cuh +++ b/cpp/src/binaryop/compiled/struct_binary_ops.cuh @@ -149,7 +149,8 @@ void apply_struct_equality_op(mutable_column_view& out, { CUDF_EXPECTS(op == binary_operator::EQUAL || op == binary_operator::NOT_EQUAL || op == binary_operator::NULL_EQUALS, - "Unsupported operator for these types"); + "Unsupported operator for these types", + cudf::data_type_error); auto tlhs = table_view{{lhs}}; auto trhs = table_view{{rhs}}; diff --git a/cpp/src/copying/concatenate.cu b/cpp/src/copying/concatenate.cu index 577d6427b19..5d36d70696c 100644 --- a/cpp/src/copying/concatenate.cu +++ b/cpp/src/copying/concatenate.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -228,7 +228,8 @@ std::unique_ptr fused_concatenate(host_span views, auto const output_size = std::get<3>(device_views); CUDF_EXPECTS(output_size <= static_cast(std::numeric_limits::max()), - "Total number of concatenated rows exceeds size_type range"); + "Total number of concatenated rows exceeds size_type range", + std::overflow_error); // Allocate output auto const policy = has_nulls ? mask_policy::ALWAYS : mask_policy::NEVER; @@ -398,7 +399,8 @@ void traverse_children::operator()(host_span(std::numeric_limits::max()), - "Total number of concatenated chars exceeds size_type range"); + "Total number of concatenated chars exceeds size_type range", + std::overflow_error); } template <> @@ -469,7 +471,8 @@ void bounds_and_type_check(host_span cols, rmm::cuda_stream_v }); // note: output text must include "exceeds size_type range" for python error handling CUDF_EXPECTS(total_row_count <= static_cast(std::numeric_limits::max()), - "Total number of concatenated rows exceeds size_type range"); + "Total number of concatenated rows exceeds size_type range", + std::overflow_error); // traverse children cudf::type_dispatcher(cols.front().type(), traverse_children{}, cols, stream); diff --git a/cpp/src/lists/combine/concatenate_list_elements.cu b/cpp/src/lists/combine/concatenate_list_elements.cu index 496d9ee670a..257b0aed82f 100644 --- a/cpp/src/lists/combine/concatenate_list_elements.cu +++ b/cpp/src/lists/combine/concatenate_list_elements.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -257,11 +257,13 @@ std::unique_ptr concatenate_list_elements(column_view const& input, rmm::mr::device_memory_resource* mr) { auto type = input.type(); // Column that is lists of lists. - CUDF_EXPECTS(type.id() == type_id::LIST, "Input column must be a lists column."); + CUDF_EXPECTS( + type.id() == type_id::LIST, "Input column must be a lists column.", std::invalid_argument); auto col = lists_column_view(input).child(); // Rows, which are lists. type = col.type(); - CUDF_EXPECTS(type.id() == type_id::LIST, "Rows of the input column must be lists."); + CUDF_EXPECTS( + type.id() == type_id::LIST, "Rows of the input column must be lists.", std::invalid_argument); col = lists_column_view(col).child(); // The last level entries what we need to check. type = col.type(); diff --git a/cpp/src/lists/contains.cu b/cpp/src/lists/contains.cu index 05fe82d1713..a3293e36825 100644 --- a/cpp/src/lists/contains.cu +++ b/cpp/src/lists/contains.cu @@ -309,7 +309,8 @@ struct dispatch_index_of { auto const child = lists.child(); CUDF_EXPECTS(child.type() == search_keys.type(), - "Type/Scale of search key does not match list column element type."); + "Type/Scale of search key does not match list column element type.", + cudf::data_type_error); CUDF_EXPECTS(search_keys.type().id() != type_id::EMPTY, "Type cannot be empty."); auto constexpr search_key_is_scalar = std::is_same_v; diff --git a/cpp/src/lists/copying/segmented_gather.cu b/cpp/src/lists/copying/segmented_gather.cu index 2c12e09bcd9..79d33e7c17d 100644 --- a/cpp/src/lists/copying/segmented_gather.cu +++ b/cpp/src/lists/copying/segmented_gather.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ std::unique_ptr segmented_gather(lists_column_view const& value_column, { CUDF_EXPECTS(is_index_type(gather_map.child().type()), "Gather map should be list column of index type"); - CUDF_EXPECTS(!gather_map.has_nulls(), "Gather map contains nulls"); + CUDF_EXPECTS(!gather_map.has_nulls(), "Gather map contains nulls", std::invalid_argument); CUDF_EXPECTS(value_column.size() == gather_map.size(), "Gather map and list column should be same size"); diff --git a/cpp/src/strings/json/json_path.cu b/cpp/src/strings/json/json_path.cu index afe16518036..c6ea47ec0f3 100644 --- a/cpp/src/strings/json/json_path.cu +++ b/cpp/src/strings/json/json_path.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -588,7 +588,7 @@ class path_state : private parser { return path_operator{path_operator_type::CHILD_WILDCARD}; } break; - default: CUDF_FAIL("Unrecognized JSONPath operator"); break; + default: CUDF_FAIL("Unrecognized JSONPath operator", std::invalid_argument); break; } return {path_operator_type::ERROR}; } @@ -624,7 +624,8 @@ class path_state : private parser { } // an empty name is not valid - CUDF_EXPECTS(name.size_bytes() > 0, "Invalid empty name in JSONPath query string"); + CUDF_EXPECTS( + name.size_bytes() > 0, "Invalid empty name in JSONPath query string", std::invalid_argument); return true; } diff --git a/cpp/tests/binaryop/binop-compiled-fixed_point-test.cpp b/cpp/tests/binaryop/binop-compiled-fixed_point-test.cpp index 8170fe4f490..bdd0003b86b 100644 --- a/cpp/tests/binaryop/binop-compiled-fixed_point-test.cpp +++ b/cpp/tests/binaryop/binop-compiled-fixed_point-test.cpp @@ -678,7 +678,7 @@ TYPED_TEST(FixedPointCompiledTest, FixedPointBinaryOpThrows) auto const col = fp_wrapper{{100, 300, 500, 700}, scale_type{-2}}; auto const non_bool_type = cudf::data_type{cudf::type_to_id(), -2}; EXPECT_THROW(cudf::binary_operation(col, col, cudf::binary_operator::LESS, non_bool_type), - cudf::logic_error); + cudf::data_type_error); } TYPED_TEST(FixedPointCompiledTest, FixedPointBinaryOpModSimple) diff --git a/cpp/tests/copying/concatenate_tests.cu b/cpp/tests/copying/concatenate_tests.cu index 79ec2293455..ca343b963d7 100644 --- a/cpp/tests/copying/concatenate_tests.cu +++ b/cpp/tests/copying/concatenate_tests.cu @@ -36,6 +36,7 @@ #include #include +#include #include template @@ -369,7 +370,7 @@ TEST_F(OverflowTest, OverflowTest) cudf::table_view tbl({*many_chars}); EXPECT_THROW(cudf::concatenate(std::vector({tbl, tbl, tbl, tbl, tbl, tbl})), - cudf::logic_error); + std::overflow_error); } // string column, overflow on chars @@ -384,7 +385,7 @@ TEST_F(OverflowTest, OverflowTest) cudf::table_view tbl({*col}); EXPECT_THROW(cudf::concatenate(std::vector({tbl, tbl, tbl, tbl, tbl, tbl})), - cudf::logic_error); + std::overflow_error); } // string column, overflow on offsets (rows) @@ -400,7 +401,7 @@ TEST_F(OverflowTest, OverflowTest) cudf::table_view tbl({*col}); EXPECT_THROW(cudf::concatenate(std::vector({tbl, tbl, tbl, tbl, tbl, tbl})), - cudf::logic_error); + std::overflow_error); } // list, structs too long @@ -425,7 +426,7 @@ TEST_F(OverflowTest, OverflowTest) cudf::table_view tbl({*col}); auto tables = std::vector({tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl}); - EXPECT_THROW(cudf::concatenate(tables), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(tables), std::overflow_error); } // struct, list child too long @@ -450,7 +451,7 @@ TEST_F(OverflowTest, OverflowTest) cudf::table_view tbl({*col}); auto tables = std::vector({tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl, tbl}); - EXPECT_THROW(cudf::concatenate(tables), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(tables), std::overflow_error); } } @@ -470,7 +471,8 @@ TEST_F(OverflowTest, Presliced) // 513 * 1024 * 1024, should fail cudf::table_view b({sliced[1]}); - EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), + std::overflow_error); } // struct column @@ -490,7 +492,8 @@ TEST_F(OverflowTest, Presliced) // 513 * 1024 * 1024, should fail cudf::table_view b({sliced[1]}); - EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), + std::overflow_error); } // strings, overflow on chars @@ -516,7 +519,8 @@ TEST_F(OverflowTest, Presliced) // (num_rows / 2) + 1 should fail cudf::table_view b({sliced[1]}); - EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), + std::overflow_error); } // strings, overflow on offsets @@ -589,7 +593,7 @@ TEST_F(OverflowTest, Presliced) auto sliced = cudf::split(*col, {2}); cudf::table_view tbl({sliced[1]}); auto tables = std::vector({tbl, tbl, tbl, tbl}); - EXPECT_THROW(cudf::concatenate(tables), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(tables), std::overflow_error); } // list, overflow on offsets @@ -674,7 +678,8 @@ TEST_F(OverflowTest, Presliced) cudf::concatenate(std::vector({a, a, a, a})); cudf::table_view b({sliced[1]}); - EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), cudf::logic_error); + EXPECT_THROW(cudf::concatenate(std::vector({b, b, b, b})), + std::overflow_error); } } diff --git a/cpp/tests/copying/segmented_gather_list_tests.cpp b/cpp/tests/copying/segmented_gather_list_tests.cpp index deeebc641c2..fc21af2087b 100644 --- a/cpp/tests/copying/segmented_gather_list_tests.cpp +++ b/cpp/tests/copying/segmented_gather_list_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022, NVIDIA CORPORATION. + * Copyright (c) 2020-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,8 @@ #include #include +#include + template class SegmentedGatherTest : public cudf::test::BaseFixture { }; @@ -611,7 +613,7 @@ TEST_F(SegmentedGatherTestFloat, Fails) // Nulls are not supported in the gather map. EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, cudf::lists_column_view{nulls_map}), - cudf::logic_error); + std::invalid_argument); // Gather map and list column sizes must be the same. EXPECT_THROW(cudf::lists::segmented_gather(cudf::lists_column_view{list}, diff --git a/cpp/tests/lists/combine/concatenate_list_elements_tests.cpp b/cpp/tests/lists/combine/concatenate_list_elements_tests.cpp index ca25560141c..77e8f904d01 100644 --- a/cpp/tests/lists/combine/concatenate_list_elements_tests.cpp +++ b/cpp/tests/lists/combine/concatenate_list_elements_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,8 @@ #include +#include + using namespace cudf::test::iterators; namespace { @@ -47,13 +49,13 @@ TEST_F(ConcatenateListElementsTest, InvalidInput) // Input lists is not a 2-level depth lists column. { auto const col = IntCol{}; - EXPECT_THROW(cudf::lists::concatenate_list_elements(col), cudf::logic_error); + EXPECT_THROW(cudf::lists::concatenate_list_elements(col), std::invalid_argument); } // Input lists is not at least 2-level depth lists column. { auto const col = IntListsCol{1, 2, 3}; - EXPECT_THROW(cudf::lists::concatenate_list_elements(col), cudf::logic_error); + EXPECT_THROW(cudf::lists::concatenate_list_elements(col), std::invalid_argument); } } diff --git a/cpp/tests/lists/contains_tests.cpp b/cpp/tests/lists/contains_tests.cpp index 2139103500a..f592819dacb 100644 --- a/cpp/tests/lists/contains_tests.cpp +++ b/cpp/tests/lists/contains_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -503,10 +503,11 @@ TEST_F(ContainsTest, ScalarTypeRelatedExceptions) {{1, 2, 3}, {4, 5, 6}}}.release(); auto skey = create_scalar_search_key(10); - EXPECT_THROW(cudf::lists::contains(list_of_lists->view(), *skey), cudf::logic_error); + EXPECT_THROW(cudf::lists::contains(list_of_lists->view(), *skey), cudf::data_type_error); EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), *skey, FIND_FIRST), - cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), *skey, FIND_LAST), cudf::logic_error); + cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), *skey, FIND_LAST), + cudf::data_type_error); } { // Search key must match list elements in type. @@ -517,9 +518,11 @@ TEST_F(ContainsTest, ScalarTypeRelatedExceptions) } .release(); auto skey = create_scalar_search_key("Hello, World!"); - EXPECT_THROW(cudf::lists::contains(list_of_ints->view(), *skey), cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), *skey, FIND_FIRST), cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), *skey, FIND_LAST), cudf::logic_error); + EXPECT_THROW(cudf::lists::contains(list_of_ints->view(), *skey), cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), *skey, FIND_FIRST), + cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), *skey, FIND_LAST), + cudf::data_type_error); } } @@ -813,9 +816,11 @@ TEST_F(ContainsTest, VectorTypeRelatedExceptions) {{1, 2, 3}, {4, 5, 6}}}.release(); auto skey = cudf::test::fixed_width_column_wrapper{0, 1, 2}; - EXPECT_THROW(cudf::lists::contains(list_of_lists->view(), skey), cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), skey, FIND_FIRST), cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), skey, FIND_LAST), cudf::logic_error); + EXPECT_THROW(cudf::lists::contains(list_of_lists->view(), skey), cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), skey, FIND_FIRST), + cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_lists->view(), skey, FIND_LAST), + cudf::data_type_error); } { // Search key must match list elements in type. @@ -826,9 +831,11 @@ TEST_F(ContainsTest, VectorTypeRelatedExceptions) } .release(); auto skey = cudf::test::strings_column_wrapper{"Hello", "World"}; - EXPECT_THROW(cudf::lists::contains(list_of_ints->view(), skey), cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), skey, FIND_FIRST), cudf::logic_error); - EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), skey, FIND_LAST), cudf::logic_error); + EXPECT_THROW(cudf::lists::contains(list_of_ints->view(), skey), cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), skey, FIND_FIRST), + cudf::data_type_error); + EXPECT_THROW(cudf::lists::index_of(list_of_ints->view(), skey, FIND_LAST), + cudf::data_type_error); } { // Search key column size must match lists column size. diff --git a/cpp/tests/strings/json_tests.cpp b/cpp/tests/strings/json_tests.cpp index 1924d809743..4a485de2f2a 100644 --- a/cpp/tests/strings/json_tests.cpp +++ b/cpp/tests/strings/json_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * Copyright (c) 2021-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,6 +22,8 @@ #include #include +#include + // reference: https://jsonpath.herokuapp.com/ // clang-format off @@ -566,7 +568,7 @@ TEST_F(JsonPathTests, GetJsonObjectIllegalQuery) auto query = [&]() { auto result = cudf::strings::get_json_object(cudf::strings_column_view(input), json_path); }; - EXPECT_THROW(query(), cudf::logic_error); + EXPECT_THROW(query(), std::invalid_argument); } { @@ -575,7 +577,7 @@ TEST_F(JsonPathTests, GetJsonObjectIllegalQuery) auto query = [&]() { auto result = cudf::strings::get_json_object(cudf::strings_column_view(input), json_path); }; - EXPECT_THROW(query(), cudf::logic_error); + EXPECT_THROW(query(), std::invalid_argument); } { @@ -584,7 +586,7 @@ TEST_F(JsonPathTests, GetJsonObjectIllegalQuery) auto query = [&]() { auto result = cudf::strings::get_json_object(cudf::strings_column_view(input), json_path); }; - EXPECT_THROW(query(), cudf::logic_error); + EXPECT_THROW(query(), std::invalid_argument); } } diff --git a/docs/cudf/source/developer_guide/contributing_guide.md b/docs/cudf/source/developer_guide/contributing_guide.md index bb3479cf4c1..b5ea9519842 100644 --- a/docs/cudf/source/developer_guide/contributing_guide.md +++ b/docs/cudf/source/developer_guide/contributing_guide.md @@ -123,19 +123,13 @@ There is no need to mention when the argument will be supported in the future. ### Handling libcudf Exceptions -Currently libcudf raises `cudf::logic_error` and `cudf::cuda_error`. -These error types are mapped to `RuntimeError` in python. -Several APIs use the exception payload `what()` message to determine the exception type raised by libcudf. - -Determining error type based on exception payload is brittle since libcudf does not maintain API stability on exception messages. -This is a compromise due to libcudf only raising a limited number of error types. -Only adopt this strategy when necessary. - -The projected roadmap is to diversify the exception types raised by libcudf. Standard C++ natively supports various [exception types](https://en.cppreference.com/w/cpp/error/exception), which Cython maps to [these Python exception types](https://docs.cython.org/en/latest/src/userguide/wrapping_CPlusPlus.html#exceptions). -In the future, libcudf may employ custom C++ exception types. -If that occurs, this section will be updated to reflect how these may be mapped to desired Python exception types. +In addition to built-in exceptions, libcudf also raises a few additional types of exceptions. +cuDF extends Cython's default mapping to account for these exception types. +When a new libcudf exception type is added, a suitable except clause should be added to cuDF's +[exception handler](https://github.com/rapidsai/cudf/blob/main/python/cudf/cudf/_lib/cpp/exception_handler.hpp). +If no built-in Python exception seems like a good match, a new Python exception should be created. ### Raising warnings diff --git a/java/src/main/native/include/jni_utils.hpp b/java/src/main/native/include/jni_utils.hpp index 78239b86ae2..ee2325cc76f 100644 --- a/java/src/main/native/include/jni_utils.hpp +++ b/java/src/main/native/include/jni_utils.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +32,7 @@ constexpr jint MINIMUM_JNI_VERSION = JNI_VERSION_1_6; constexpr char const *CUDA_ERROR_CLASS = "ai/rapids/cudf/CudaException"; constexpr char const *CUDA_FATAL_ERROR_CLASS = "ai/rapids/cudf/CudaFatalException"; constexpr char const *CUDF_ERROR_CLASS = "ai/rapids/cudf/CudfException"; +constexpr char const *CUDF_DTYPE_ERROR_CLASS = "ai/rapids/cudf/CudfException"; constexpr char const *INDEX_OOB_CLASS = "java/lang/ArrayIndexOutOfBoundsException"; constexpr char const *ILLEGAL_ARG_CLASS = "java/lang/IllegalArgumentException"; constexpr char const *NPE_CLASS = "java/lang/NullPointerException"; @@ -861,6 +862,9 @@ inline void jni_cuda_check(JNIEnv *const env, cudaError_t cuda_status) { catch (const cudf::cuda_error &e) { \ JNI_CHECK_CUDA_ERROR(env, cudf::jni::CUDA_ERROR_CLASS, e, ret_val); \ } \ + catch (const cudf::data_type_error &e) { \ + JNI_CHECK_THROW_NEW(env, cudf::jni::CUDF_DTYPE_ERROR_CLASS, e.what(), ret_val); \ + } \ catch (const std::exception &e) { \ /* Double check whether the thrown exception is unrecoverable CUDA error or not. */ \ /* Like cudf::detail::throw_cuda_error, it is nearly certain that a fatal error */ \ diff --git a/python/cudf/cudf/_lib/CMakeLists.txt b/python/cudf/cudf/_lib/CMakeLists.txt index 4b785563484..f7d4f12ad81 100644 --- a/python/cudf/cudf/_lib/CMakeLists.txt +++ b/python/cudf/cudf/_lib/CMakeLists.txt @@ -62,6 +62,11 @@ rapids_cython_create_modules( LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS cudf ) +# All modules need to include the header containing the exception handler. +foreach(target IN LISTS RAPIDS_CYTHON_CREATED_TARGETS) + target_include_directories(${target} PRIVATE ${CMAKE_CURRENT_LIST_DIR}) +endforeach() + target_link_libraries(strings_udf cudf_strings_udf) # TODO: Finding NumPy currently requires finding Development due to a bug in CMake. This bug was diff --git a/python/cudf/cudf/_lib/cpp/copying.pxd b/python/cudf/cudf/_lib/cpp/copying.pxd index bc89d364004..09e8538ebb7 100644 --- a/python/cudf/cudf/_lib/cpp/copying.pxd +++ b/python/cudf/cudf/_lib/cpp/copying.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. +# Copyright (c) 2020-2023, NVIDIA CORPORATION. from libc.stdint cimport int32_t, int64_t, uint8_t from libcpp cimport bool @@ -14,6 +14,7 @@ from cudf._lib.cpp.scalar.scalar cimport scalar from cudf._lib.cpp.table.table cimport table from cudf._lib.cpp.table.table_view cimport table_view from cudf._lib.cpp.types cimport size_type +from cudf._lib.exception_handler cimport cudf_exception_handler ctypedef const scalar constscalar @@ -32,7 +33,7 @@ cdef extern from "cudf/copying.hpp" namespace "cudf" nogil: const table_view& source_table, const column_view& gather_map, out_of_bounds_policy policy - ) except + + ) except +cudf_exception_handler cdef unique_ptr[column] shift( const column_view& input, diff --git a/python/cudf/cudf/_lib/cpp/lists/contains.pxd b/python/cudf/cudf/_lib/cpp/lists/contains.pxd index e3cb01721a0..e86c73deed2 100644 --- a/python/cudf/cudf/_lib/cpp/lists/contains.pxd +++ b/python/cudf/cudf/_lib/cpp/lists/contains.pxd @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. from libcpp.memory cimport unique_ptr @@ -6,20 +6,21 @@ from cudf._lib.cpp.column.column cimport column from cudf._lib.cpp.column.column_view cimport column_view from cudf._lib.cpp.lists.lists_column_view cimport lists_column_view from cudf._lib.cpp.scalar.scalar cimport scalar +from cudf._lib.exception_handler cimport cudf_exception_handler cdef extern from "cudf/lists/contains.hpp" namespace "cudf::lists" nogil: cdef unique_ptr[column] contains( lists_column_view lists, scalar search_key, - ) except + + ) except +cudf_exception_handler cdef unique_ptr[column] index_of( lists_column_view lists, scalar search_key, - ) except + + ) except +cudf_exception_handler cdef unique_ptr[column] index_of( lists_column_view lists, column_view search_keys, - ) except + + ) except +cudf_exception_handler diff --git a/python/cudf/cudf/_lib/exception_handler.hpp b/python/cudf/cudf/_lib/exception_handler.hpp new file mode 100644 index 00000000000..8daffddd7bd --- /dev/null +++ b/python/cudf/cudf/_lib/exception_handler.hpp @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +namespace cudf_python { +namespace exceptions { + +/** + * @brief Exception handler to map C++ exceptions to Python ones in Cython + * + * This exception handler extends the base exception handler provided by + * Cython (https://github.com/cython/cython/blob/master/Cython/Utility/CppSupport.cpp#L9). + * In addition to the exceptions that Cython itself supports, this file adds support + * for additional exceptions thrown by libcudf that need to be mapped to specific Python + * exceptions. + * + * Since this function interoperates with Python's exception state, it does not throw + * any C++ exceptions. + */ +void cudf_exception_handler() +{ + // Catch a handful of different errors here and turn them into the + // equivalent Python errors. + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const cudf::data_type_error& exn) { + // Have to catch data_type_error before invalid_argument because it is a subclass + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + // Unfortunately, in standard C++ we have no way of distinguishing EOF + // from other errors here; be careful with the exception mask + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + // Change out_of_range to IndexError + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + // The below is the default catch-all case. + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } catch (...) { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} + +} // namespace exceptions +} // namespace cudf_python diff --git a/python/cudf/cudf/_lib/exception_handler.pxd b/python/cudf/cudf/_lib/exception_handler.pxd new file mode 100644 index 00000000000..14ac3bb1d40 --- /dev/null +++ b/python/cudf/cudf/_lib/exception_handler.pxd @@ -0,0 +1,5 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. + + +cdef extern from "exception_handler.hpp" namespace "cudf_python::exceptions": + cdef void cudf_exception_handler() diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index fb1bcf6d673..b5f36aa3594 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -2587,19 +2587,10 @@ def concat_columns(objs: "MutableSequence[ColumnBase]") -> ColumnBase: f"size > {libcudf.MAX_COLUMN_SIZE_STR}" ) elif newsize == 0: - col = column_empty(0, head.dtype, masked=True) - else: - # Filter out inputs that have 0 length, then concatenate. - objs = [o for o in objs if len(o)] - try: - col = libcudf.concat.concat_columns(objs) - except RuntimeError as e: - if "exceeds size_type range" in str(e): - raise OverflowError( - "total size of output is too large for a cudf column" - ) from e - raise - return col + return column_empty(0, head.dtype, masked=True) + + # Filter out inputs that have 0 length, then concatenate. + return libcudf.concat.concat_columns([o for o in objs if len(o)]) def _proxy_cai_obj(cai, owner): diff --git a/python/cudf/cudf/core/column/decimal.py b/python/cudf/cudf/core/column/decimal.py index 157bc1f4291..96b8002e2a1 100644 --- a/python/cudf/cudf/core/column/decimal.py +++ b/python/cudf/cudf/core/column/decimal.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. +# Copyright (c) 2021-2023, NVIDIA CORPORATION. import warnings from decimal import Decimal @@ -79,32 +79,26 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str): # Binary Arithmetics between decimal columns. `Scale` and `precision` # are computed outside of libcudf - unsupported_msg = ( - f"{op} not supported for the following dtypes: " - f"{self.dtype}, {other.dtype}" - ) - try: - if op in {"__add__", "__sub__", "__mul__", "__div__"}: - output_type = _get_decimal_type(lhs.dtype, rhs.dtype, op) - result = libcudf.binaryop.binaryop(lhs, rhs, op, output_type) - # TODO: Why is this necessary? Why isn't the result's - # precision already set correctly based on output_type? - result.dtype.precision = output_type.precision - elif op in { - "__eq__", - "__ne__", - "__lt__", - "__gt__", - "__le__", - "__ge__", - }: - result = libcudf.binaryop.binaryop(lhs, rhs, op, bool) - else: - raise TypeError(unsupported_msg) - except RuntimeError as e: - if "Unsupported operator for these types" in str(e): - raise TypeError(unsupported_msg) from e - raise + if op in {"__add__", "__sub__", "__mul__", "__div__"}: + output_type = _get_decimal_type(lhs.dtype, rhs.dtype, op) + result = libcudf.binaryop.binaryop(lhs, rhs, op, output_type) + # TODO: Why is this necessary? Why isn't the result's + # precision already set correctly based on output_type? + result.dtype.precision = output_type.precision + elif op in { + "__eq__", + "__ne__", + "__lt__", + "__gt__", + "__le__", + "__ge__", + }: + result = libcudf.binaryop.binaryop(lhs, rhs, op, bool) + else: + raise TypeError( + f"{op} not supported for the following dtypes: " + f"{self.dtype}, {other.dtype}" + ) return result diff --git a/python/cudf/cudf/core/column/lists.py b/python/cudf/cudf/core/column/lists.py index 9b64f26f0fb..4eea64c00d3 100644 --- a/python/cudf/cudf/core/column/lists.py +++ b/python/cudf/cudf/core/column/lists.py @@ -406,19 +406,9 @@ def contains(self, search_key: ScalarLike) -> ParentType: Series([False, True, True]) dtype: bool """ - search_key = cudf.Scalar(search_key) - try: - res = self._return_or_inplace( - contains_scalar(self._column, search_key) - ) - except RuntimeError as e: - if ( - "Type/Scale of search key does not " - "match list column element type." in str(e) - ): - raise TypeError(str(e)) from e - raise - return res + return self._return_or_inplace( + contains_scalar(self._column, cudf.Scalar(search_key)) + ) def index(self, search_key: Union[ScalarLike, ColumnLike]) -> ParentType: """ @@ -465,23 +455,14 @@ def index(self, search_key: Union[ScalarLike, ColumnLike]) -> ParentType: dtype: int32 """ - try: - if is_scalar(search_key): - return self._return_or_inplace( - index_of_scalar(self._column, cudf.Scalar(search_key)) - ) - else: - return self._return_or_inplace( - index_of_column(self._column, as_column(search_key)) - ) - - except RuntimeError as e: - if ( - "Type/Scale of search key does not " - "match list column element type." in str(e) - ): - raise TypeError(str(e)) from e - raise + if is_scalar(search_key): + return self._return_or_inplace( + index_of_scalar(self._column, cudf.Scalar(search_key)) + ) + else: + return self._return_or_inplace( + index_of_column(self._column, as_column(search_key)) + ) @property def leaves(self) -> ParentType: @@ -577,16 +558,9 @@ def take(self, lists_indices: ColumnLike) -> ParentType: "lists_indices should be column of values of index types." ) - try: - res = self._return_or_inplace( - segmented_gather(self._column, lists_indices_col) - ) - except RuntimeError as e: - if "contains nulls" in str(e): - raise ValueError("lists_indices contains null.") from e - raise - else: - return res + return self._return_or_inplace( + segmented_gather(self._column, lists_indices_col) + ) def unique(self) -> ParentType: """ @@ -720,16 +694,9 @@ def concat(self, dropna=True) -> ParentType: 1 [6.0, nan, 7.0, 8.0, 9.0] dtype: list """ - try: - result = concatenate_list_elements(self._column, dropna=dropna) - except RuntimeError as e: - if "Rows of the input column must be lists." in str(e): - raise ValueError( - "list.concat() can only be called on " - "list columns with at least one level " - "of nesting" - ) - return self._return_or_inplace(result) + return self._return_or_inplace( + concatenate_list_elements(self._column, dropna=dropna) + ) def astype(self, dtype): """ diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 8d6ffe48957..d5ef5fb5d11 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -2379,29 +2379,18 @@ def get_json_object( dtype: object """ - try: - options = libstrings.GetJsonObjectOptions( - allow_single_quotes=allow_single_quotes, - strip_quotes_from_single_strings=( - strip_quotes_from_single_strings - ), - missing_fields_as_nulls=missing_fields_as_nulls, - ) - res = self._return_or_inplace( - libstrings.get_json_object( - self._column, cudf.Scalar(json_path, "str"), options - ) - ) - except RuntimeError as e: - matches = ( - "Unrecognized JSONPath operator", - "Invalid empty name in JSONPath query string", + options = libstrings.GetJsonObjectOptions( + allow_single_quotes=allow_single_quotes, + strip_quotes_from_single_strings=( + strip_quotes_from_single_strings + ), + missing_fields_as_nulls=missing_fields_as_nulls, + ) + return self._return_or_inplace( + libstrings.get_json_object( + self._column, cudf.Scalar(json_path, "str"), options ) - if any(match in str(e) for match in matches): - raise ValueError("JSONPath value not found") from e - raise - else: - return res + ) def split( self, diff --git a/python/cudf/cudf/tests/test_column.py b/python/cudf/cudf/tests/test_column.py index 7d113bbb9e2..a15afa727c0 100644 --- a/python/cudf/cudf/tests/test_column.py +++ b/python/cudf/cudf/tests/test_column.py @@ -520,10 +520,7 @@ def test_concatenate_large_column_strings(): s_1 = cudf.Series(["very long string " * string_scale_f] * num_strings) s_2 = cudf.Series(["very long string " * string_scale_f] * num_strings) - with pytest.raises( - OverflowError, - match="total size of output is too large for a cudf column", - ): + with pytest.raises(OverflowError): cudf.concat([s_1, s_2]) From ac1cac67839ceb2272e1eba151782cfaa744270a Mon Sep 17 00:00:00 2001 From: Raza Jafri Date: Sun, 26 Feb 2023 20:29:11 -0800 Subject: [PATCH 62/69] Add JNI methods for detecting and purging non-empty nulls from LIST and STRUCT (#12742) This PR adds methods for detecting and purging non-empty nulls. Authors: - Raza Jafri (https://github.com/razajafri) - Nghia Truong (https://github.com/ttnghia) - AJ Schmidt (https://github.com/ajschmidt8) Approvers: - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/12742 --- .../main/java/ai/rapids/cudf/ColumnView.java | 37 +++++++++++ java/src/main/native/src/ColumnViewJni.cpp | 22 +++++++ .../java/ai/rapids/cudf/ColumnVectorTest.java | 62 +++++++++++++++++++ 3 files changed, 121 insertions(+) diff --git a/java/src/main/java/ai/rapids/cudf/ColumnView.java b/java/src/main/java/ai/rapids/cudf/ColumnView.java index 0cb9ed37d9f..84183819854 100644 --- a/java/src/main/java/ai/rapids/cudf/ColumnView.java +++ b/java/src/main/java/ai/rapids/cudf/ColumnView.java @@ -4639,6 +4639,10 @@ static native long makeCudfColumnView(int type, int scale, long data, long dataS static native long applyBooleanMask(long arrayColumnView, long booleanMaskHandle) throws CudfException; + static native boolean hasNonEmptyNulls(long handle) throws CudfException; + + static native long purgeNonEmptyNulls(long handle) throws CudfException; + /** * A utility class to create column vector like objects without refcounts and other APIs when * creating the device side vector from host side nested vectors. Eventually this can go away or @@ -4997,4 +5001,37 @@ public HostColumnVector copyToHost() { } } } + + /** + * Exact check if a column or its descendants have non-empty null rows + * + * @return Whether the column or its descendants have non-empty null rows + */ + public boolean hasNonEmptyNulls() { + return hasNonEmptyNulls(viewHandle); + } + + /** + * Copies this column into output while purging any non-empty null rows in the column or its + * descendants. + * + * If this column is not of compound type (LIST/STRING/STRUCT/DICTIONARY), the output will be + * the same as input. + * + * The purge operation only applies directly to LIST and STRING columns, but it applies indirectly + * to STRUCT/DICTIONARY columns as well, since these columns may have child columns that + * are LIST or STRING. + * + * Examples: + * lists = data: [{{0,1}, {2,3}, {4,5}} validity: {true, false, true}] + * lists[1] is null, but the list's child column still stores `{2,3}`. + * + * After purging the contents of the list's null rows, the column's contents will be: + * lists = [data: {{0,1}, {4,5}} validity: {true, false, true}] + * + * @return A new column with equivalent contents to `input`, but with null rows purged + */ + public ColumnVector purgeNonEmptyNulls() { + return new ColumnVector(purgeNonEmptyNulls(viewHandle)); + } } diff --git a/java/src/main/native/src/ColumnViewJni.cpp b/java/src/main/native/src/ColumnViewJni.cpp index c42cc430560..f2c361c5e8c 100644 --- a/java/src/main/native/src/ColumnViewJni.cpp +++ b/java/src/main/native/src/ColumnViewJni.cpp @@ -2457,4 +2457,26 @@ JNIEXPORT jlong JNICALL Java_ai_rapids_cudf_ColumnView_applyBooleanMask( CATCH_STD(env, 0); } +JNIEXPORT jboolean JNICALL +Java_ai_rapids_cudf_ColumnView_hasNonEmptyNulls(JNIEnv *env, jclass, jlong column_view_handle) { + JNI_NULL_CHECK(env, column_view_handle, "column_view handle is null", 0); + try { + cudf::jni::auto_set_device(env); + auto const *cv = reinterpret_cast(column_view_handle); + return cudf::has_nonempty_nulls(*cv); + } + CATCH_STD(env, 0); +} + +JNIEXPORT jlong JNICALL +Java_ai_rapids_cudf_ColumnView_purgeNonEmptyNulls(JNIEnv *env, jclass, jlong column_view_handle) { + JNI_NULL_CHECK(env, column_view_handle, "column_view handle is null", 0); + try { + cudf::jni::auto_set_device(env); + auto const *cv = reinterpret_cast(column_view_handle); + return release_as_jlong(cudf::purge_nonempty_nulls(*cv)); + } + CATCH_STD(env, 0); +} + } // extern "C" diff --git a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java index 937077c89c9..7848807dab8 100644 --- a/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java +++ b/java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -6691,4 +6692,65 @@ void testApplyBooleanMaskFromListOfStructure() { assertColumnsAreEqual(expectedCv, actualCv); } } + + /** + * The caller needs to make sure to close the returned ColumnView + */ + private ColumnView[] getColumnViewWithNonEmptyNulls() { + List list0 = Arrays.asList(1, 2, 3); + List list1 = Arrays.asList(4, 5, null); + List list2 = Arrays.asList(7, 8, 9); + List list3 = null; + ColumnVector input = makeListsColumn(DType.INT32, list0, list1, list2, list3); + // Modify the validity buffer + BaseDeviceMemoryBuffer dmb = input.getDeviceBufferFor(BufferType.VALIDITY); + try (HostMemoryBuffer newValidity = HostMemoryBuffer.allocate(64)) { + newValidity.copyFromDeviceBuffer(dmb); + BitVectorHelper.setNullAt(newValidity, 1); + dmb.copyFromHostBuffer(newValidity); + } + try (HostColumnVector hostColumnVector = input.copyToHost()) { + assert (hostColumnVector.isNull(1)); + assert (hostColumnVector.isNull(3)); + } + try (ColumnVector expectedOffsetsBeforePurge = ColumnVector.fromInts(0, 3, 6, 9, 9)) { + ColumnView offsetsCvBeforePurge = input.getListOffsetsView(); + assertColumnsAreEqual(expectedOffsetsBeforePurge, offsetsCvBeforePurge); + } + ColumnView colWithNonEmptyNulls = new ColumnView(input.type, input.rows, Optional.of(2L), dmb, + input.getDeviceBufferFor(BufferType.OFFSET), input.getChildColumnViews()); + assertEquals(2, colWithNonEmptyNulls.nullCount); + return new ColumnView[]{input, colWithNonEmptyNulls}; + } + + @Test + void testPurgeNonEmptyNullsList() { + ColumnView[] values = getColumnViewWithNonEmptyNulls(); + try (ColumnView colWithNonEmptyNulls = values[1]; + ColumnView input = values[0]; + // purge non-empty nulls + ColumnView colWithEmptyNulls = colWithNonEmptyNulls.purgeNonEmptyNulls(); + ColumnVector expectedOffsetsAfterPurge = ColumnVector.fromInts(0, 3, 3, 6, 6); + ColumnView offsetsCvAfterPurge = colWithEmptyNulls.getListOffsetsView()) { + assertTrue(colWithNonEmptyNulls.hasNonEmptyNulls()); + assertColumnsAreEqual(expectedOffsetsAfterPurge, offsetsCvAfterPurge); + assertFalse(colWithEmptyNulls.hasNonEmptyNulls()); + } + } + + @Test + void testPurgeNonEmptyNullsStruct() { + ColumnView[] values = getColumnViewWithNonEmptyNulls(); + try (ColumnView listCol = values[1]; + ColumnView input = values[0]; + ColumnView stringsCol = ColumnVector.fromStrings("A", "col", "of", "Strings"); + ColumnView structView = ColumnView.makeStructView(stringsCol, listCol); + ColumnView structWithEmptyNulls = structView.purgeNonEmptyNulls(); + ColumnView newListChild = structWithEmptyNulls.getChildColumnView(1); + ColumnVector expectedOffsetsAfterPurge = ColumnVector.fromInts(0, 3, 3, 6, 6); + ColumnView offsetsCvAfterPurge = newListChild.getListOffsetsView()) { + assertColumnsAreEqual(expectedOffsetsAfterPurge, offsetsCvAfterPurge); + assertFalse(newListChild.hasNonEmptyNulls()); + } + } } From 202578307c81b1f7aad8fbe6f71ca9c7f3fa5c4a Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 27 Feb 2023 13:46:45 -0500 Subject: [PATCH 63/69] Improve performance for cudf::strings::count_characters for long strings (#12779) Adds more efficient counting algorithm specifically for columns with long strings--greater than 64 bytes on average. The internal detail method will be used to help improve performance in other strings functions. Authors: - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/12779 --- cpp/benchmarks/CMakeLists.txt | 2 +- cpp/benchmarks/string/lengths.cpp | 56 +++++++++++++++++++ cpp/src/strings/attributes.cu | 91 +++++++++++++++++++++++++++++-- cpp/tests/strings/attrs_tests.cpp | 22 ++++++-- 4 files changed, 160 insertions(+), 11 deletions(-) create mode 100644 cpp/benchmarks/string/lengths.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index c5ae3345da5..11da30f108a 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -295,7 +295,7 @@ ConfigureBench( string/url_decode.cu ) -ConfigureNVBench(STRINGS_NVBENCH string/like.cpp string/reverse.cpp) +ConfigureNVBench(STRINGS_NVBENCH string/like.cpp string/reverse.cpp string/lengths.cpp) # ################################################################################################## # * json benchmark ------------------------------------------------------------------- diff --git a/cpp/benchmarks/string/lengths.cpp b/cpp/benchmarks/string/lengths.cpp new file mode 100644 index 00000000000..4540e4a8f42 --- /dev/null +++ b/cpp/benchmarks/string/lengths.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include + +static void bench_lengths(nvbench::state& state) +{ + auto const num_rows = static_cast(state.get_int64("num_rows")); + auto const row_width = static_cast(state.get_int64("row_width")); + + if (static_cast(num_rows) * static_cast(row_width) >= + static_cast(std::numeric_limits::max())) { + state.skip("Skip benchmarks greater than size_type limit"); + } + + data_profile const table_profile = data_profile_builder().distribution( + cudf::type_id::STRING, distribution_id::NORMAL, 0, row_width); + auto const table = + create_random_table({cudf::type_id::STRING}, row_count{num_rows}, table_profile); + cudf::strings_column_view input(table->view().column(0)); + + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + // gather some throughput statistics as well + auto chars_size = input.chars_size(); + state.add_global_memory_reads(chars_size); // all bytes are read; + state.add_global_memory_writes(num_rows); // output is an integer per row + + state.exec(nvbench::exec_tag::sync, [&](nvbench::launch& launch) { + auto result = cudf::strings::count_characters(input); + }); +} + +NVBENCH_BENCH(bench_lengths) + .set_name("strings_lengths") + .add_int64_axis("num_rows", {4096, 32768, 262144, 2097152, 16777216}) + .add_int64_axis("row_width", {32, 64, 128, 256, 512, 1024, 2048, 4096}); diff --git a/cpp/src/strings/attributes.cu b/cpp/src/strings/attributes.cu index 127d3aa8fe7..66288c7d14d 100644 --- a/cpp/src/strings/attributes.cu +++ b/cpp/src/strings/attributes.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,7 +19,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -29,6 +31,7 @@ #include #include +#include #include #include #include @@ -37,10 +40,24 @@ #include #include +#include + namespace cudf { namespace strings { namespace detail { namespace { + +/** + * @brief Threshold to decide on using string or warp parallel functions. + * + * If the average byte length of a string in a column exceeds this value then + * the warp-parallel function is used. + * Otherwise, a regular string-parallel function is used. + * + * This value was found using the strings_lengths benchmark results. + */ +constexpr size_type AVG_CHAR_BYTES_THRESHOLD = 64; + /** * @brief Returns a numeric column containing lengths of each string in * based on the provided unary function. @@ -85,21 +102,85 @@ std::unique_ptr counts_fn(strings_column_view const& strings, return results; } +/** + * @brief Count characters using a warp per string + * + * @param d_strings Column with strings to count + * @param d_lengths Results of the counts per string + */ +__global__ void count_characters_parallel_fn(column_device_view const d_strings, + size_type* d_lengths) +{ + size_type const idx = static_cast(threadIdx.x + blockIdx.x * blockDim.x); + using warp_reduce = cub::WarpReduce; + __shared__ typename warp_reduce::TempStorage temp_storage; + + if (idx >= (d_strings.size() * cudf::detail::warp_size)) { return; } + + auto const str_idx = idx / cudf::detail::warp_size; + auto const lane_idx = idx % cudf::detail::warp_size; + if (d_strings.is_null(str_idx)) { + d_lengths[str_idx] = 0; + return; + } + auto const d_str = d_strings.element(str_idx); + auto const str_ptr = d_str.data(); + + auto count = 0; + for (auto i = lane_idx; i < d_str.size_bytes(); i += cudf::detail::warp_size) { + count += static_cast(is_begin_utf8_char(str_ptr[i])); + } + auto const char_count = warp_reduce(temp_storage).Sum(count); + if (lane_idx == 0) { d_lengths[str_idx] = char_count; } +} + +std::unique_ptr count_characters_parallel(strings_column_view const& input, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) +{ + // create output column + auto results = make_numeric_column(data_type{type_to_id()}, + input.size(), + cudf::detail::copy_bitmask(input.parent(), stream, mr), + input.null_count(), + stream, + mr); + + auto const d_lengths = results->mutable_view().data(); + auto const d_strings = cudf::column_device_view::create(input.parent(), stream); + + // fill in the lengths + constexpr int block_size = 256; + cudf::detail::grid_1d grid{input.size() * cudf::detail::warp_size, block_size}; + count_characters_parallel_fn<<>>( + *d_strings, d_lengths); + + // reset null count after call to mutable_view() + results->set_null_count(input.null_count()); + + return results; +} + } // namespace -std::unique_ptr count_characters(strings_column_view const& strings, +std::unique_ptr count_characters(strings_column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - auto ufn = [] __device__(const string_view& d_str) { return d_str.length(); }; - return counts_fn(strings, ufn, stream, mr); + if ((input.size() == input.null_count()) || + ((input.chars_size() / (input.size() - input.null_count())) < AVG_CHAR_BYTES_THRESHOLD)) { + auto ufn = [] __device__(string_view const& d_str) { return d_str.length(); }; + return counts_fn(input, ufn, stream, mr); + } + + return count_characters_parallel(input, stream, mr); } std::unique_ptr count_bytes(strings_column_view const& strings, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { - auto ufn = [] __device__(const string_view& d_str) { return d_str.size_bytes(); }; + auto ufn = [] __device__(string_view const& d_str) { return d_str.size_bytes(); }; return counts_fn(strings, ufn, stream, mr); } diff --git a/cpp/tests/strings/attrs_tests.cpp b/cpp/tests/strings/attrs_tests.cpp index 9ff2c55ed81..eff992604a6 100644 --- a/cpp/tests/strings/attrs_tests.cpp +++ b/cpp/tests/strings/attrs_tests.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2023, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -65,7 +65,7 @@ TEST_F(StringsAttributesTest, ZeroSizeStringsColumn) TEST_F(StringsAttributesTest, StringsLengths) { std::vector h_strings{ - "eee", "bb", nullptr, "", "aa", "ééé", " something a bit longer "}; + "eee", "bb", nullptr, "", "aa", "ééé", "something a bit longer than 32 bytes"}; cudf::test::strings_column_wrapper strings( h_strings.begin(), h_strings.end(), @@ -74,17 +74,16 @@ TEST_F(StringsAttributesTest, StringsLengths) { auto results = cudf::strings::count_characters(strings_view); - std::vector h_expected{3, 2, 0, 0, 2, 3, 24}; + std::vector h_expected{3, 2, 0, 0, 2, 3, 36}; cudf::test::fixed_width_column_wrapper expected( h_expected.begin(), h_expected.end(), thrust::make_transform_iterator(h_strings.begin(), [](auto str) { return str != nullptr; })); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); } { auto results = cudf::strings::count_bytes(strings_view); - std::vector h_expected{3, 2, 0, 0, 2, 6, 24}; + std::vector h_expected{3, 2, 0, 0, 2, 6, 36}; cudf::test::fixed_width_column_wrapper expected( h_expected.begin(), h_expected.end(), @@ -93,3 +92,16 @@ TEST_F(StringsAttributesTest, StringsLengths) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); } } + +TEST_F(StringsAttributesTest, StringsLengthsLong) +{ + std::vector h_strings( + 40000, "something a bit longer than 32 bytes ééé ééé ééé ééé ééé ééé ééé"); + cudf::test::strings_column_wrapper strings(h_strings.begin(), h_strings.end()); + auto strings_view = cudf::strings_column_view(strings); + + auto results = cudf::strings::count_characters(strings_view); + std::vector h_expected(h_strings.size(), 64); + cudf::test::fixed_width_column_wrapper expected(h_expected.begin(), h_expected.end()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +} From 9a91270b59f846cd4b0d6a1577183c6b333cdf57 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 27 Feb 2023 16:11:22 -0800 Subject: [PATCH 64/69] Remove tokenizers pre-install pinning. (#12854) We pinned tokenizers during early wheel development because at the time the latest version on PyPI was only a source distribution without wheels available for arm and we didn't want to have to compile the package. More recent versions appear to be providing aarch binaries. Conversely, the pinned version (0.10.2) predated the existence of Python 3.10 wheels, which is resulting in nightly CI failing because Python 3.10 runs no longer find a wheel and attempt to compile from source. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12854 --- .github/workflows/pr.yaml | 3 +-- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3a80139e333..c36c539a102 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -113,8 +113,7 @@ jobs: build_type: pull-request package-name: cudf # Install cupy-cuda11x for arm from a special index url - # Install tokenizers last binary wheel to avoid a Rust compile from the latest sdist - test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64" + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests" test-smoketest: "python ./ci/wheel_smoke_test_cudf.py" wheel-build-dask-cudf: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4d9e97a7b28..5dc04ece919 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -86,7 +86,7 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} package-name: cudf - test-before-arm64: "pip install tokenizers==0.10.2 cupy-cuda11x -f https://pip.cupy.dev/aarch64" + test-before-arm64: "pip install cupy-cuda11x -f https://pip.cupy.dev/aarch64" test-unittest: "pytest -v -n 8 ./python/cudf/cudf/tests" wheel-tests-dask-cudf: secrets: inherit From 8a45ae266641d96f30c9336abcd656d5290beaa1 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 28 Feb 2023 09:10:18 -0600 Subject: [PATCH 65/69] Make docs builds less verbose (#12836) This PR adds the `--no-progress` flag to reduce verbose output from the `s3 sync` commands. Authors: - Jake Awe (https://github.com/AyodeAwe) Approvers: - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12836 --- ci/build_docs.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 9551d98e9fe..6daedb59733 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -41,7 +41,8 @@ popd if [[ ${RAPIDS_BUILD_TYPE} == "branch" ]]; then - aws s3 sync --delete cpp/doxygen/html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" - aws s3 sync --delete docs/cudf/_html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" - aws s3 sync --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" + rapids-logger "Upload Docs to S3" + aws s3 sync --no-progress --delete cpp/doxygen/html "s3://rapidsai-docs/libcudf/${VERSION_NUMBER}/html" + aws s3 sync --no-progress --delete docs/cudf/_html "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/html" + aws s3 sync --no-progress --delete docs/cudf/_text "s3://rapidsai-docs/cudf/${VERSION_NUMBER}/txt" fi From a7e50920f35c667d041678f6a82d975355cdfd8f Mon Sep 17 00:00:00 2001 From: Peter Andreas Entschev Date: Tue, 28 Feb 2023 19:35:20 +0100 Subject: [PATCH 66/69] Update RMM allocators (#12861) Authors: - Peter Andreas Entschev (https://github.com/pentschev) Approvers: - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/12861 --- python/cudf/cudf/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/cudf/cudf/__init__.py b/python/cudf/cudf/__init__.py index 04b64e18594..7bab131a85a 100644 --- a/python/cudf/cudf/__init__.py +++ b/python/cudf/cudf/__init__.py @@ -8,6 +8,8 @@ from numba import config as numba_config, cuda import rmm +from rmm.allocators.cupy import rmm_cupy_allocator +from rmm.allocators.numba import RMMNumbaManager from cudf import api, core, datasets, testing from cudf.api.extensions import ( @@ -96,8 +98,8 @@ del patch_numba_linker_if_needed -cuda.set_memory_manager(rmm.RMMNumbaManager) -cupy.cuda.set_allocator(rmm.rmm_cupy_allocator) +cuda.set_memory_manager(RMMNumbaManager) +cupy.cuda.set_allocator(rmm_cupy_allocator) try: # Numba 0.54: Disable low occupancy warnings From afdb51bb90969e8f2b7ec75dd79d8fb6c437d461 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Thu, 2 Mar 2023 02:05:35 +0530 Subject: [PATCH 67/69] Adds JSON reader, writer io benchmark (#12753) - Add JSON writer benchmark. This benchmark is modeled after CSV writer. - Add JSON reader benchmark with file data source ([NESTED_JSON](https://github.com/rapidsai/cudf/blob/branch-23.04/cpp/benchmarks/io/json/nested_json.cpp?rgh-link-date=2023-02-08T22%3A43%3A38Z) only does parsing and only on device buffers). This benchmark is modeled after BM_csv_read_io fixes part of https://github.com/rapidsai/cudf/issues/12739 Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/12753 --- cpp/benchmarks/CMakeLists.txt | 3 +- cpp/benchmarks/io/json/json_reader_input.cpp | 128 +++++++++++++++++++ cpp/benchmarks/io/json/json_writer.cpp | 125 ++++++++++++++++++ cpp/src/io/json/write_json.cu | 4 + 4 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 cpp/benchmarks/io/json/json_reader_input.cpp create mode 100644 cpp/benchmarks/io/json/json_writer.cpp diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 11da30f108a..e6b59c0b9f0 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -301,7 +301,8 @@ ConfigureNVBench(STRINGS_NVBENCH string/like.cpp string/reverse.cpp string/lengt # * json benchmark ------------------------------------------------------------------- ConfigureBench(JSON_BENCH string/json.cu) ConfigureNVBench(FST_NVBENCH io/fst.cu) -ConfigureNVBench(NESTED_JSON_NVBENCH io/json/nested_json.cpp) +ConfigureNVBench(JSON_READER_NVBENCH io/json/nested_json.cpp io/json/json_reader_input.cpp) +ConfigureNVBench(JSON_WRITER_NVBENCH io/json/json_writer.cpp) # ################################################################################################## # * io benchmark --------------------------------------------------------------------- diff --git a/cpp/benchmarks/io/json/json_reader_input.cpp b/cpp/benchmarks/io/json/json_reader_input.cpp new file mode 100644 index 00000000000..55614d040d5 --- /dev/null +++ b/cpp/benchmarks/io/json/json_reader_input.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include + +// Size of the data in the the benchmark dataframe; chosen to be low enough to allow benchmarks to +// run on most GPUs, but large enough to allow highest throughput +constexpr size_t data_size = 512 << 20; +constexpr cudf::size_type num_cols = 64; + +void json_read_common(cudf::io::json_writer_options const& write_opts, + cuio_source_sink_pair& source_sink, + nvbench::state& state) +{ + cudf::io::write_json(write_opts); + + cudf::io::json_reader_options read_opts = + cudf::io::json_reader_options::builder(source_sink.make_source_info()); + + auto mem_stats_logger = cudf::memory_stats_logger(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer, + [&](nvbench::launch& launch, auto& timer) { + try_drop_l3_cache(); + + timer.start(); + cudf::io::read_json(read_opts); + timer.stop(); + }); + + auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); + state.add_buffer_size( + mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); + state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); +} + +template +void BM_json_read_io(nvbench::state& state, nvbench::type_list>) +{ + auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL), + static_cast(data_type::FLOAT), + static_cast(data_type::DECIMAL), + static_cast(data_type::TIMESTAMP), + static_cast(data_type::DURATION), + static_cast(data_type::STRING), + static_cast(data_type::LIST), + static_cast(data_type::STRUCT)}); + + auto const source_type = IO; + + auto const tbl = create_random_table( + cycle_dtypes(d_type, num_cols), table_size_bytes{data_size}, data_profile_builder()); + auto const view = tbl->view(); + + cuio_source_sink_pair source_sink(source_type); + cudf::io::json_writer_options const write_opts = + cudf::io::json_writer_options::builder(source_sink.make_sink_info(), view).na_rep("null"); + + json_read_common(write_opts, source_sink, state); +} + +template +void BM_json_read_data_type( + nvbench::state& state, nvbench::type_list, nvbench::enum_type>) +{ + auto const d_type = get_type_or_group(static_cast(DataType)); + auto const source_type = IO; + + auto const tbl = create_random_table( + cycle_dtypes(d_type, num_cols), table_size_bytes{data_size}, data_profile_builder()); + auto const view = tbl->view(); + + cuio_source_sink_pair source_sink(source_type); + cudf::io::json_writer_options const write_opts = + cudf::io::json_writer_options::builder(source_sink.make_sink_info(), view).na_rep("null"); + + json_read_common(write_opts, source_sink, state); +} + +using d_type_list = nvbench::enum_type_list; + +using io_list = nvbench::enum_type_list; + +using compression_list = + nvbench::enum_type_list; + +NVBENCH_BENCH_TYPES(BM_json_read_data_type, + NVBENCH_TYPE_AXES(d_type_list, + nvbench::enum_type_list)) + .set_name("json_read_data_type") + .set_type_axes_names({"data_type", "io"}) + .set_min_samples(4); + +NVBENCH_BENCH_TYPES(BM_json_read_io, NVBENCH_TYPE_AXES(io_list)) + .set_name("json_read_io") + .set_type_axes_names({"io"}) + .set_min_samples(4); diff --git a/cpp/benchmarks/io/json/json_writer.cpp b/cpp/benchmarks/io/json/json_writer.cpp new file mode 100644 index 00000000000..ee183b327fe --- /dev/null +++ b/cpp/benchmarks/io/json/json_writer.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2023, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include + +// Size of the data in the the benchmark dataframe; chosen to be low enough to allow benchmarks to +// run on most GPUs, but large enough to allow highest throughput +constexpr size_t data_size = 512 << 20; +constexpr cudf::size_type num_cols = 64; + +void json_write_common(cudf::io::json_writer_options const& write_opts, + cuio_source_sink_pair& source_sink, + size_t const data_size, + nvbench::state& state) +{ + auto mem_stats_logger = cudf::memory_stats_logger(); + state.set_cuda_stream(nvbench::make_cuda_stream_view(cudf::get_default_stream().value())); + state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer, + [&](nvbench::launch& launch, auto& timer) { + try_drop_l3_cache(); + + timer.start(); + cudf::io::write_json(write_opts); + timer.stop(); + }); + + auto const time = state.get_summary("nv/cold/time/gpu/mean").get_float64("value"); + state.add_element_count(static_cast(data_size) / time, "bytes_per_second"); + state.add_buffer_size( + mem_stats_logger.peak_memory_usage(), "peak_memory_usage", "peak_memory_usage"); + state.add_buffer_size(source_sink.size(), "encoded_file_size", "encoded_file_size"); +} + +template +void BM_json_write_io(nvbench::state& state, nvbench::type_list>) +{ + auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL), + static_cast(data_type::FLOAT), + static_cast(data_type::DECIMAL), + static_cast(data_type::TIMESTAMP), + static_cast(data_type::DURATION), + static_cast(data_type::STRING), + static_cast(data_type::LIST), + static_cast(data_type::STRUCT)}); + + auto const source_type = IO; + + auto const tbl = create_random_table( + cycle_dtypes(d_type, num_cols), table_size_bytes{data_size}, data_profile_builder()); + auto const view = tbl->view(); + + cuio_source_sink_pair source_sink(source_type); + cudf::io::json_writer_options write_opts = + cudf::io::json_writer_options::builder(source_sink.make_sink_info(), view).na_rep("null"); + + json_write_common(write_opts, source_sink, data_size, state); +} + +void BM_json_writer_options(nvbench::state& state) +{ + auto const d_type = get_type_or_group({static_cast(data_type::INTEGRAL), + static_cast(data_type::FLOAT), + static_cast(data_type::DECIMAL), + static_cast(data_type::TIMESTAMP), + static_cast(data_type::DURATION), + static_cast(data_type::STRING), + static_cast(data_type::LIST), + static_cast(data_type::STRUCT)}); + + auto const source_type = io_type::HOST_BUFFER; + bool const json_lines = state.get_int64("json_lines"); + bool const include_nulls = state.get_int64("include_nulls"); + auto const rows_per_chunk = state.get_int64("rows_per_chunk"); + + auto const tbl = create_random_table( + cycle_dtypes(d_type, num_cols), table_size_bytes{data_size}, data_profile_builder()); + auto const view = tbl->view(); + + cuio_source_sink_pair source_sink(source_type); + cudf::io::json_writer_options write_opts = + cudf::io::json_writer_options::builder(source_sink.make_sink_info(), view) + .na_rep("null") + .lines(json_lines) + .include_nulls(include_nulls) + .rows_per_chunk(rows_per_chunk); + + json_write_common(write_opts, source_sink, data_size, state); +} + +using io_list = nvbench::enum_type_list; + +NVBENCH_BENCH_TYPES(BM_json_write_io, NVBENCH_TYPE_AXES(io_list)) + .set_name("json_write_io") + .set_type_axes_names({"io"}) + .set_min_samples(4); + +NVBENCH_BENCH(BM_json_writer_options) + .set_name("json_write_options") + .set_min_samples(4) + .add_int64_axis("json_lines", {false, true}) + .add_int64_axis("include_nulls", {false, true}) + .add_int64_power_of_two_axis("rows_per_chunk", nvbench::range(10, 20, 2)); diff --git a/cpp/src/io/json/write_json.cu b/cpp/src/io/json/write_json.cu index a7ae4d3bdd1..b4bcb5548de 100644 --- a/cpp/src/io/json/write_json.cu +++ b/cpp/src/io/json/write_json.cu @@ -240,6 +240,7 @@ std::unique_ptr struct_to_strings(table_view const& strings_columns, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { + CUDF_FUNC_RANGE(); CUDF_EXPECTS(column_names.type().id() == type_id::STRING, "Column names must be of type string"); auto const num_columns = strings_columns.num_columns(); CUDF_EXPECTS(num_columns == column_names.size(), @@ -481,6 +482,7 @@ struct column_to_strings_fn { column_iterator column_end, host_span children_names) const { + CUDF_FUNC_RANGE(); auto const num_columns = std::distance(column_begin, column_end); auto column_names = make_column_names_column(children_names, num_columns, stream_); auto column_names_view = column_names->view(); @@ -590,6 +592,7 @@ void write_chunked(data_sink* out_sink, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { + CUDF_FUNC_RANGE(); CUDF_EXPECTS(str_column_view.size() > 0, "Unexpected empty strings column."); auto p_str_col_w_nl = cudf::strings::detail::join_strings(str_column_view, @@ -620,6 +623,7 @@ void write_json(data_sink* out_sink, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { + CUDF_FUNC_RANGE(); std::vector user_column_names = [&]() { auto const& metadata = options.get_metadata(); if (metadata.has_value() and not metadata->schema_info.empty()) { From 195e2f7b531ce4c5ff80ff5985dd82b7eae43134 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Wed, 1 Mar 2023 15:00:13 -0600 Subject: [PATCH 68/69] Fix docs build to be `pydata-sphinx-theme=0.13.0` compatible (#12874) This PR fixes cudf docs build to be compatible with `pydata-sphinx-theme=0.13.0` by adding an empty `icon_links` entry. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Bradley Dice (https://github.com/bdice) - AJ Schmidt (https://github.com/ajschmidt8) URL: https://github.com/rapidsai/cudf/pull/12874 --- docs/cudf/source/conf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/cudf/source/conf.py b/docs/cudf/source/conf.py index 3d92d955263..b97879b0ae4 100644 --- a/docs/cudf/source/conf.py +++ b/docs/cudf/source/conf.py @@ -45,8 +45,8 @@ "myst_nb", ] -jupyter_execute_notebooks = "force" -execution_timeout = 300 +nb_execution_mode = "force" +nb_execution_timeout = 300 copybutton_prompt_text = ">>> " autosummary_generate = True @@ -103,6 +103,8 @@ html_theme_options = { "external_links": [], + # https://github.com/pydata/pydata-sphinx-theme/issues/1220 + "icon_links": [], "github_url": "https://github.com/rapidsai/cudf", "twitter_url": "https://twitter.com/rapidsai", "show_toc_level": 1, From 40e56c94e8450603a169546faad36397f2aef8aa Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Wed, 1 Mar 2023 15:32:54 -0800 Subject: [PATCH 69/69] Parquet writer column_size() should return a size_t (#12870) Fixes #12867. Bug introduced in #12685. A calculation of total bytes in a column was returned in a 32-bit `size_type` rather than 64-bit `size_t` leading to overflow for tables with many millions of rows. Authors: - Ed Seidl (https://github.com/etseidl) - Vukasin Milovanovic (https://github.com/vuule) - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vukasin Milovanovic (https://github.com/vuule) - Karthikeyan (https://github.com/karthikeyann) - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/12870 --- cpp/src/io/parquet/writer_impl.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/io/parquet/writer_impl.cu b/cpp/src/io/parquet/writer_impl.cu index 88176ee1901..2c9bff33a14 100644 --- a/cpp/src/io/parquet/writer_impl.cu +++ b/cpp/src/io/parquet/writer_impl.cu @@ -87,7 +87,7 @@ parquet::Compression to_parquet_compression(compression_type compression) } } -size_type column_size(column_view const& column, rmm::cuda_stream_view stream) +size_t column_size(column_view const& column, rmm::cuda_stream_view stream) { if (column.size() == 0) { return 0; } @@ -99,7 +99,7 @@ size_type column_size(column_view const& column, rmm::cuda_stream_view stream) cudf::detail::get_value(scol.offsets(), 0, stream); } else if (column.type().id() == type_id::STRUCT) { auto const scol = structs_column_view(column); - size_type ret = 0; + size_t ret = 0; for (int i = 0; i < scol.num_children(); i++) { ret += column_size(scol.get_sliced_child(i), stream); }