From 2a2e3f0a5223d15b43f35a104f7f9b23471e4f41 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 23 Feb 2022 17:02:43 -0600 Subject: [PATCH 01/11] Fix warnings in test_binops.py. (#10327) This PR catches or silences warnings in `test_binops.py`. (I am working through one test file at a time so we can enable `-Werr` in the future.) Most of the warnings come from deprecated binops inside cudf. Once the deprecated code is removed, these tests will fail until the workaround is removed. Another warning comes from numpy, `DeprecationWarning: elementwise comparison failed; this will raise an error in the future.`. This warning can be circumvented by comparing with `series == scalar` instead of `scalar == series`. I believe that seeing this warning is expected when comparing `scalar == series`, if both numpy and pandas/cudf implement `__eq__` that accept one another's array-like types, but I'm not certain. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/10327 --- python/cudf/cudf/tests/test_binops.py | 32 +++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index f688cc3b642..c98568d53a5 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -4,6 +4,7 @@ import decimal import operator import random +from contextlib import contextmanager from itertools import combinations_with_replacement, product import cupy as cp @@ -26,6 +27,24 @@ STRING_TYPES = {"str"} + +@contextmanager +def _hide_deprecated_ops_warnings(func, lhs, rhs): + if func in { + cudf.logical_and, + cudf.logical_or, + cudf.remainder, + } and isinstance(lhs, cudf.Series): + name = func.__name__ + with pytest.warns( + FutureWarning, + match=f"Series.{name} is deprecated and will be removed.", + ): + yield + else: + yield + + _binops = [ operator.add, operator.sub, @@ -170,7 +189,8 @@ def test_series_logical_binop(lhstype, rhstype, binop, cubinop): arr2 = arr2 * (np.random.random(10) * 100).astype(rhstype) sr2 = Series(arr2) - result = cubinop(sr1, sr2) + with _hide_deprecated_ops_warnings(cubinop, sr1, sr2): + result = cubinop(sr1, sr2) expect = binop(arr1, arr2) utils.assert_eq(result, expect) @@ -955,7 +975,9 @@ def test_ufunc_ops(lhs, rhs, ops): curhs = rhs expect = np_op(lhs, rhs) - got = cu_op(culhs, curhs) + with _hide_deprecated_ops_warnings(cu_op, culhs, curhs): + got = cu_op(culhs, curhs) + if np.isscalar(expect): assert got == expect else: @@ -1741,8 +1763,10 @@ def test_binops_with_lhs_numpy_scalar(frame, dtype): else: val = cudf.dtype(dtype).type(4) - expected = val == data.to_pandas() - got = val == data + # Compare equality with series on left side to dispatch to the pandas/cudf + # __eq__ operator and avoid a DeprecationWarning from numpy. + expected = data.to_pandas() == val + got = data == val # In case of index, expected would be a numpy array if isinstance(data, cudf.BaseIndex): From aa746ae3d2035809c516ab6a2100e4336e1cfa21 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 23 Feb 2022 17:52:32 -0800 Subject: [PATCH 02/11] Remove internal columns usage (#10315) This PR uses the feature introduced in #10263 to remove the usage of DataFrame.columns where possible. Currently this removes a large number of internal uses but not all of them to verify that CI does indeed fail as expected since some changes were made on #10263 after the last test of CI by reviewer request. I will convert this from a draft once I see CI fail and once all the necessary changes have been made for tests to pass. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Michael Wang (https://github.com/isVoid) URL: https://github.com/rapidsai/cudf/pull/10315 --- python/cudf/cudf/core/_base_index.py | 4 +- python/cudf/cudf/core/_internals/where.py | 4 +- python/cudf/cudf/core/dataframe.py | 165 ++++++++++++--------- python/cudf/cudf/core/df_protocol.py | 4 +- python/cudf/cudf/core/frame.py | 2 +- python/cudf/cudf/core/groupby/groupby.py | 12 +- python/cudf/cudf/core/indexed_frame.py | 25 ++-- python/cudf/cudf/core/multiindex.py | 9 +- python/cudf/cudf/core/reshape.py | 23 +-- python/cudf/cudf/core/window/rolling.py | 30 ++-- python/cudf/cudf/io/parquet.py | 2 +- python/cudf/cudf/testing/testing.py | 12 +- python/cudf/cudf/tests/test_df_protocol.py | 2 +- 13 files changed, 161 insertions(+), 133 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 7a9a17631a9..16fb0cf99c1 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -957,7 +957,9 @@ def _union(self, other, sort=None): self_df["order"] = self_df.index other_df["order"] = other_df.index res = self_df.merge(other_df, on=[0], how="outer") - res = res.sort_values(by=res.columns[1:], ignore_index=True) + res = res.sort_values( + by=res._data.to_pandas_index()[1:], ignore_index=True + ) union_result = cudf.core.index._index_from_data({0: res._data[0]}) if sort is None and len(other): diff --git a/python/cudf/cudf/core/_internals/where.py b/python/cudf/cudf/core/_internals/where.py index 6c94a84fd37..8bfcad4c8f4 100644 --- a/python/cudf/cudf/core/_internals/where.py +++ b/python/cudf/cudf/core/_internals/where.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. import warnings from typing import Any, Optional, Tuple, Union, cast @@ -264,7 +264,7 @@ def where( ) # Setting `frame` column names to `cond` # as `cond` has no column names. - cond.columns = frame.columns + cond._set_column_names_like(frame) (source_df, others,) = _normalize_columns_and_scalars_type( frame, other diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 1c672aacd86..6b5f3809c98 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -75,7 +75,7 @@ min_scalar_type, numeric_normalize_types, ) -from cudf.utils.utils import GetAttrGetItemMixin +from cudf.utils.utils import GetAttrGetItemMixin, _external_only_api T = TypeVar("T", bound="DataFrame") @@ -94,8 +94,9 @@ class _DataFrameIndexer(_FrameIndexer): def __getitem__(self, arg): - if isinstance(self._frame.index, MultiIndex) or isinstance( - self._frame.columns, MultiIndex + if ( + isinstance(self._frame.index, MultiIndex) + or self._frame._data.multiindex ): # This try/except block allows the use of pandas-like # tuple arguments into MultiIndex dataframes. @@ -164,7 +165,7 @@ def _downcast_to_series(self, df, arg): # determine the axis along which the Series is taken: if nrows == 1 and ncols == 1: if is_scalar(arg[0]) and is_scalar(arg[1]): - return df[df.columns[0]].iloc[0] + return df[df._column_names[0]].iloc[0] elif not is_scalar(arg[0]): axis = 1 else: @@ -285,8 +286,9 @@ def _getitem_tuple_arg(self, arg): @annotate("LOC_SETITEM", color="blue", domain="cudf_python") def _setitem_tuple_arg(self, key, value): - if isinstance(self._frame.index, MultiIndex) or isinstance( - self._frame.columns, pd.MultiIndex + if ( + isinstance(self._frame.index, MultiIndex) + or self._frame._data.multiindex ): raise NotImplementedError( "Setting values using df.loc[] not supported on " @@ -546,7 +548,6 @@ def __init__( ) else: self._data = data._data - self.columns = data.columns elif isinstance(data, (cudf.Series, pd.Series)): if isinstance(data, pd.Series): data = cudf.Series.from_pandas(data, nan_as_null=nan_as_null) @@ -595,7 +596,6 @@ def __init__( self._data = new_df._data self.index = new_df._index - self.columns = new_df.columns elif hasattr(data, "__array_interface__"): arr_interface = data.__array_interface__ if len(arr_interface["descr"]) == 1: @@ -605,7 +605,6 @@ def __init__( new_df = self.from_records(data, index=index, columns=columns) self._data = new_df._data self.index = new_df._index - self.columns = new_df.columns else: if is_list_like(data): if len(data) > 0 and is_scalar(data[0]): @@ -617,7 +616,6 @@ def __init__( self._data = new_df._data self.index = new_df._index - self.columns = new_df.columns elif len(data) > 0 and isinstance(data[0], Series): self._init_from_series_list( data=data, columns=columns, index=index @@ -719,8 +717,9 @@ def _init_from_series_list(self, data, columns, index): else: concat_df = cudf.concat(data, axis=1) - if concat_df.columns.dtype == "object": - concat_df.columns = concat_df.columns.astype("str") + cols = concat_df._data.to_pandas_index() + if cols.dtype == "object": + concat_df.columns = cols.astype("str") transpose = concat_df.T @@ -972,7 +971,9 @@ def __dir__(self): o = set(dir(type(self))) o.update(self.__dict__) o.update( - c for c in self.columns if isinstance(c, str) and c.isidentifier() + c + for c in self._column_names + if isinstance(c, str) and c.isidentifier() ) return list(o) @@ -983,7 +984,6 @@ def __setattr__(self, key, col): # properties, and we must call object.__getattribute__ to bypass # the `__getitem__` behavior inherited from `GetAttrGetItemMixin`. object.__getattribute__(self, key) - super().__setattr__(key, col) except AttributeError: if key not in self._PROTECTED_KEYS: try: @@ -998,6 +998,17 @@ def __setattr__(self, key, col): # Set a new attribute that is not already a column. super().__setattr__(key, col) + except RuntimeError as e: + # TODO: This allows setting properties that are marked as forbidden + # for internal usage. It is necesary because the __getattribute__ + # call in the try block will trigger the error. We should see if + # setting these variables can also always be disabled + if "External-only API" not in str(e): + raise + super().__setattr__(key, col) + else: + super().__setattr__(key, col) + @annotate("DATAFRAME_GETITEM", color="blue", domain="cudf_python") def __getitem__(self, arg): """ @@ -1304,7 +1315,7 @@ def _slice(self: T, arg: slice) -> T: # Adding index of type RangeIndex back to # result result.index = self.index[start:stop] - result.columns = self.columns + result._set_column_names_like(self) return result @annotate("DATAFRAME_MEMORY_USAGE", color="blue", domain="cudf_python") @@ -1606,8 +1617,8 @@ def _concat( ) # Reassign index and column names - if isinstance(objs[0].columns, pd.MultiIndex): - out.columns = objs[0].columns + if objs[0]._data.multiindex: + out._set_column_names_like(objs[0]) else: out.columns = names if not ignore_index: @@ -1930,12 +1941,14 @@ def _prep_for_binop( not can_reindex and fn in cudf.utils.utils._EQUALITY_OPS and ( - not lhs.columns.equals(rhs.columns) + not lhs._data.to_pandas_index().equals( + rhs._data.to_pandas_index() + ) or not lhs.index.equals(rhs.index) ) ): raise ValueError( - "Can only compare identically-labeled " "DataFrame objects" + "Can only compare identically-labeled DataFrame objects" ) lhs, rhs = _align_indices(lhs, rhs) @@ -2074,8 +2087,9 @@ def update( if not isinstance(other, DataFrame): other = DataFrame(other) - if not self.columns.equals(other.columns): - other = other.reindex(self.columns, axis=1) + self_cols = self._data.to_pandas_index() + if not self_cols.equals(other._data.to_pandas_index()): + other = other.reindex(self_cols, axis=1) if not self.index.equals(other.index): other = other.reindex(self.index, axis=0) @@ -2104,7 +2118,7 @@ def update( @annotate("DATAFRAME_ITER", color="blue", domain="cudf_python") def __iter__(self): - return iter(self.columns) + return iter(self._column_names) @annotate("DATAFRAME_ITERITEMS", color="blue", domain="cudf_python") def iteritems(self): @@ -2150,6 +2164,11 @@ def at(self): return self.loc @property # type: ignore + @_external_only_api( + "Use _column_names instead, or _data.to_pandas_index() if a pandas " + "index is absolutely necessary. For checking if the columns are a " + "MultiIndex, use _data.multiindex." + ) @annotate("DATAFRAME_COLUMNS_GETTER", color="yellow", domain="cudf_python") def columns(self): """Returns a tuple of columns""" @@ -2177,12 +2196,20 @@ def columns(self, columns): f"got {len(columns)} elements" ) - data = dict(zip(columns, self._data.columns)) - if len(columns) != len(data): + self._set_column_names(columns, is_multiindex, columns.names) + + def _set_column_names(self, names, multiindex=False, level_names=None): + data = dict(zip(names, self._data.columns)) + if len(names) != len(data): raise ValueError("Duplicate column names are not allowed") self._data = ColumnAccessor( - data, multiindex=is_multiindex, level_names=columns.names, + data, multiindex=multiindex, level_names=level_names, + ) + + def _set_column_names_like(self, other): + self._set_column_names( + other._data.names, other._data.multiindex, other._data.level_names ) @annotate("DATAFRAME_REINDEX_INTERNAL", color="blue", domain="cudf_python") @@ -2451,7 +2478,7 @@ def set_index( for col in keys: # Is column label if is_scalar(col) or isinstance(col, tuple): - if col in self.columns: + if col in self._column_names: columns_to_add.append(self[col]) names.append(col) if drop: @@ -2747,7 +2774,7 @@ def diff(self, periods=1, axis=0): df = cudf.DataFrame._from_data( { name: column_empty(len(self), dtype=dtype, masked=True) - for name, dtype in zip(self.columns, self.dtypes) + for name, dtype in zip(self._column_names, self.dtypes) } ) return df @@ -3460,7 +3487,7 @@ def transpose(self): # Never transpose a MultiIndex - remove the existing columns and # replace with a RangeIndex. Afterward, reassign. columns = self.index.copy(deep=False) - index = self.columns.copy(deep=False) + index = self._data.to_pandas_index() if self._num_columns == 0 or self._num_rows == 0: return DataFrame(index=index, columns=columns) # Set the old column names as the new index @@ -4313,13 +4340,13 @@ def info( ) lines.append(index_summary) - if len(self.columns) == 0: + if len(self._data) == 0: lines.append(f"Empty {type(self).__name__}") cudf.utils.ioutils.buffer_write_lines(buf, lines) return - cols = self.columns - col_count = len(self.columns) + cols = self._column_names + col_count = len(cols) if max_cols is None: max_cols = pd.options.display.max_info_columns @@ -4337,7 +4364,7 @@ def _put_str(s, space): return str(s)[:space].ljust(space) def _verbose_repr(): - lines.append(f"Data columns (total {len(self.columns)} columns):") + lines.append(f"Data columns (total {col_count} columns):") id_head = " # " column_head = "Column" @@ -4357,10 +4384,10 @@ def _verbose_repr(): ) if show_counts: counts = self.count().to_pandas().tolist() - if len(cols) != len(counts): + if col_count != len(counts): raise AssertionError( f"Columns must equal " - f"counts ({len(cols)} != {len(counts)})" + f"counts ({col_count} != {len(counts)})" ) count_header = "Non-Null Count" len_count = len(count_header) @@ -4393,7 +4420,7 @@ def _verbose_repr(): + _put_str("-" * len_dtype, space_dtype).rstrip() ) - for i, col in enumerate(self.columns): + for i, col in enumerate(self._column_names): dtype = self.dtypes.iloc[i] col = pprint_thing(col) @@ -4410,13 +4437,11 @@ def _verbose_repr(): ) def _non_verbose_repr(): - if len(self.columns) > 0: - entries_summary = f", {self.columns[0]} to {self.columns[-1]}" + if col_count > 0: + entries_summary = f", {cols[0]} to {cols[-1]}" else: entries_summary = "" - columns_summary = ( - f"Columns: {len(self.columns)} entries{entries_summary}" - ) + columns_summary = f"Columns: {col_count} entries{entries_summary}" lines.append(columns_summary) def _sizeof_fmt(num, size_qualifier): @@ -4479,7 +4504,7 @@ def describe( if datetime_is_numeric: default_include.append("datetime") data_to_describe = self.select_dtypes(include=default_include) - if len(data_to_describe.columns) == 0: + if data_to_describe._num_columns == 0: data_to_describe = self elif include == "all": @@ -4497,7 +4522,7 @@ def describe( describe_series_list = [ data_to_describe[col].describe(percentiles=percentiles) - for col in data_to_describe.columns + for col in data_to_describe._column_names ] if len(describe_series_list) == 1: return describe_series_list[0].to_frame() @@ -4587,26 +4612,14 @@ def to_pandas(self, nullable=False, **kwargs): out_data = {} out_index = self.index.to_pandas() - if not isinstance(self.columns, pd.Index): - out_columns = self.columns.to_pandas() - else: - out_columns = self.columns - for i, col_key in enumerate(self._data): out_data[i] = self._data[col_key].to_pandas( index=out_index, nullable=nullable ) - if isinstance(self.columns, BaseIndex): - out_columns = self.columns.to_pandas() - if isinstance(self.columns, MultiIndex): - if self.columns.names is not None: - out_columns.names = self.columns.names - else: - out_columns.name = self.columns.name - out_df = pd.DataFrame(out_data, index=out_index) - out_df.columns = out_columns + out_df.columns = self._data.to_pandas_index() + return out_df @classmethod @@ -5105,7 +5118,7 @@ def quantile( if isinstance(q, numbers.Number) and numeric_only: result = result.fillna(np.nan) result = result.iloc[0] - result.index = as_index(data_df.columns) + result.index = data_df._data.to_pandas_index() result.name = q return result else: @@ -5285,6 +5298,7 @@ def make_false_column_like_self(): f"'{type(values).__name__}'" ) + # TODO: Update this logic to properly preserve MultiIndex columns. return DataFrame._from_data(result, self.index) # @@ -5521,7 +5535,7 @@ def mode(self, axis=0, numeric_only=False, dropna=True): if isinstance(df, Series): df = df.to_frame() - df.columns = data_df.columns + df._set_column_names_like(data_df) return df @@ -5651,7 +5665,7 @@ def _apply_cupy_method_axis_1(self, method, *args, **kwargs): return Series(result, index=self.index, dtype=result_dtype,) else: result_df = DataFrame(result).set_index(self.index) - result_df.columns = prepared.columns + result_df._set_column_names_like(prepared) return result_df @annotate("DATAFRAME_COLUMNS_VIEW", color="green", domain="cudf_python") @@ -5926,16 +5940,18 @@ def cov(self, **kwargs): cov : DataFrame """ cov = cupy.cov(self.values, rowvar=False) - df = DataFrame(cupy.asfortranarray(cov)).set_index(self.columns) - df.columns = self.columns + cols = self._data.to_pandas_index() + df = DataFrame(cupy.asfortranarray(cov)).set_index(cols) + df._set_column_names_like(self) return df @annotate("DATAFRAME_CORR", color="green", domain="cudf_python") def corr(self): """Compute the correlation matrix of a DataFrame.""" corr = cupy.corrcoef(self.values, rowvar=False) - df = DataFrame(cupy.asfortranarray(corr)).set_index(self.columns) - df.columns = self.columns + cols = self._data.to_pandas_index() + df = DataFrame(cupy.asfortranarray(corr)).set_index(cols) + df._set_column_names_like(self) return df @annotate("DATAFRAME_TO_STRUCT", color="green", domain="cudf_python") @@ -6001,7 +6017,7 @@ def keys(self): >>> df.keys() Int64Index([0, 1, 2, 3], dtype='int64') """ - return self.columns + return self._data.to_pandas_index() def itertuples(self, index=True, name="Pandas"): raise TypeError( @@ -6134,7 +6150,7 @@ def append( "or if the Series has a name" ) - current_cols = self.columns + current_cols = self._data.to_pandas_index() combined_columns = other.index.to_pandas() if len(current_cols): @@ -6162,8 +6178,11 @@ def append( pass elif not isinstance(other[0], DataFrame): other = DataFrame(other) - if (self.columns.get_indexer(other.columns) >= 0).all(): - other = other.reindex(columns=self.columns) + cols = self._data.to_pandas_index() + if ( + cols.get_indexer(other._data.to_pandas_index()) >= 0 + ).all(): + other = other.reindex(columns=cols) if is_list_like(other): to_concat = [self, *other] @@ -6587,10 +6606,10 @@ def _align_indices(lhs, rhs): df = df.sort_index() lhs_out = DataFrame(index=df.index) rhs_out = DataFrame(index=df.index) - common = set(lhs.columns) & set(rhs.columns) + common = set(lhs._column_names) & set(rhs._column_names) common_x = {f"{x}_x" for x in common} common_y = {f"{x}_y" for x in common} - for col in df.columns: + for col in df._column_names: if col in common_x: lhs_out[col[:-2]] = df[col] elif col in common_y: @@ -6620,9 +6639,9 @@ def _setitem_with_dataframe( """ if input_cols is None: - input_cols = input_df.columns + input_cols = input_df._column_names - if len(input_cols) != len(replace_df.columns): + if len(input_cols) != len(replace_df._column_names): raise ValueError( "Number of Input Columns must be same replacement Dataframe" ) @@ -6634,8 +6653,8 @@ def _setitem_with_dataframe( ): replace_df = replace_df.reindex(input_df.index) - for col_1, col_2 in zip(input_cols, replace_df.columns): - if col_1 in input_df.columns: + for col_1, col_2 in zip(input_cols, replace_df._column_names): + if col_1 in input_df._column_names: if mask is not None: input_df._data[col_1][mask] = column.as_column( replace_df[col_2] diff --git a/python/cudf/cudf/core/df_protocol.py b/python/cudf/cudf/core/df_protocol.py index 0f0cce283f3..8f00289afcb 100644 --- a/python/cudf/cudf/core/df_protocol.py +++ b/python/cudf/cudf/core/df_protocol.py @@ -537,7 +537,7 @@ def metadata(self): return {"cudf.index": self._df.index} def num_columns(self) -> int: - return len(self._df.columns) + return len(self._df._column_names) def num_rows(self) -> int: return len(self._df) @@ -546,7 +546,7 @@ def num_chunks(self) -> int: return 1 def column_names(self) -> Iterable[str]: - return self._df.columns.tolist() + return self._df._column_names def get_column(self, i: int) -> _CuDFColumn: return _CuDFColumn( diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 9a2841cb402..91d81a4c6de 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -6822,7 +6822,7 @@ def _drop_rows_by_labels( return obj.__class__._from_data( join_res.iloc[:, idx_nlv:]._data, index=midx, - columns=obj.columns, + columns=obj._data.to_pandas_index(), ) else: diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index b90f857ce84..0ad6a9dc35a 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -7,7 +7,6 @@ from functools import cached_property import numpy as np -import pandas as pd from nvtx import annotate import cudf @@ -905,7 +904,7 @@ def corr(self, method="pearson", min_periods=1): # create expanded dataframe consisting all combinations of the # struct columns-pairs to be correlated # i.e (('col1', 'col1'), ('col1', 'col2'), ('col2', 'col2')) - _cols = self.grouping.values.columns.tolist() + _cols = self.grouping.values._data.to_pandas_index().tolist() len_cols = len(_cols) new_df_data = {} @@ -1048,7 +1047,7 @@ def cov(self, min_periods=0, ddof=1): # create expanded dataframe consisting all combinations of the # struct columns-pairs used in the covariance calculation # i.e. (('col1', 'col1'), ('col1', 'col2'), ('col2', 'col2')) - column_names = self.grouping.values.columns.tolist() + column_names = self.grouping.values._column_names num_cols = len(column_names) column_pair_structs = {} @@ -1616,11 +1615,8 @@ def agg(self, func): return result.iloc[:, 0] # drop the first level if we have a multiindex - if ( - isinstance(result.columns, pd.MultiIndex) - and result.columns.nlevels > 1 - ): - result.columns = result.columns.droplevel(0) + if result._data.nlevels > 1: + result.columns = result._data.to_pandas_index().droplevel(0) return result diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index bc7337d0a42..8ff3e39519c 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -444,12 +444,11 @@ def sort_index( out = self._gather(inds) # TODO: frame factory function should handle multilevel column # names - if isinstance( - self, cudf.core.dataframe.DataFrame - ) and isinstance( - self.columns, pd.core.indexes.multi.MultiIndex + if ( + isinstance(self, cudf.core.dataframe.DataFrame) + and self._data.multiindex ): - out.columns = self.columns + out._set_column_names_like(self) elif (ascending and idx.is_monotonic_increasing) or ( not ascending and idx.is_monotonic_decreasing ): @@ -459,12 +458,11 @@ def sort_index( ascending=ascending, na_position=na_position ) out = self._gather(inds) - if isinstance( - self, cudf.core.dataframe.DataFrame - ) and isinstance( - self.columns, pd.core.indexes.multi.MultiIndex + if ( + isinstance(self, cudf.core.dataframe.DataFrame) + and self._data.multiindex ): - out.columns = self.columns + out._set_column_names_like(self) else: labels = sorted(self._data.names, reverse=not ascending) out = self[labels] @@ -938,10 +936,11 @@ def sort_values( ), keep_index=not ignore_index, ) - if isinstance(self, cudf.core.dataframe.DataFrame) and isinstance( - self.columns, pd.core.indexes.multi.MultiIndex + if ( + isinstance(self, cudf.core.dataframe.DataFrame) + and self._data.multiindex ): - out.columns = self.columns + out.columns = self._data.to_pandas_index() return out def _n_largest_or_smallest(self, largest, n, columns, keep): diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index b934c6e7038..b09a2d39c14 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -110,7 +110,7 @@ def __init__( levels = [cudf.Series(level) for level in levels] - if len(levels) != len(codes.columns): + if len(levels) != len(codes._data): raise ValueError( "MultiIndex has unequal number of levels and " "codes and is inconsistent!" @@ -960,12 +960,12 @@ def _concat(cls, objs): # TODO: Verify if this is really necessary or if we can rely on # DataFrame._concat. if len(source_data) > 1: - colnames = source_data[0].columns + colnames = source_data[0]._data.to_pandas_index() for obj in source_data[1:]: obj.columns = colnames source_data = cudf.DataFrame._concat(source_data) - names = [None for x in source_data.columns] + names = [None] * source_data._num_columns objs = list(filter(lambda o: o.names is not None, objs)) for o in range(len(objs)): for i, name in enumerate(objs[o].names): @@ -1682,7 +1682,8 @@ def _union(self, other, sort=None): result_df = self_df.merge(other_df, on=col_names, how="outer") result_df = result_df.sort_values( - by=result_df.columns[self.nlevels :], ignore_index=True + by=result_df._data.to_pandas_index()[self.nlevels :], + ignore_index=True, ) midx = MultiIndex.from_frame(result_df.iloc[:, : self.nlevels]) diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 43c7d0343fd..5aa7f616e35 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. import itertools from typing import Dict, Optional @@ -334,8 +334,10 @@ def concat(objs, axis=0, join="outer", ignore_index=False, sort=None): else: df[name] = col - result_columns = objs[0].columns.append( - [obj.columns for obj in objs[1:]] + result_columns = ( + objs[0] + ._data.to_pandas_index() + .append([obj._data.to_pandas_index() for obj in objs[1:]]) ) if ignore_index: @@ -492,7 +494,7 @@ def melt( if not isinstance(id_vars, collections.abc.Sequence): id_vars = [id_vars] id_vars = list(id_vars) - missing = set(id_vars) - set(frame.columns) + missing = set(id_vars) - set(frame._column_names) if not len(missing) == 0: raise KeyError( f"The following 'id_vars' are not present" @@ -506,7 +508,7 @@ def melt( if not isinstance(value_vars, collections.abc.Sequence): value_vars = [value_vars] value_vars = list(value_vars) - missing = set(value_vars) - set(frame.columns) + missing = set(value_vars) - set(frame._column_names) if not len(missing) == 0: raise KeyError( f"The following 'value_vars' are not present" @@ -514,8 +516,7 @@ def melt( ) else: # then all remaining columns in frame - value_vars = frame.columns.drop(id_vars) - value_vars = list(value_vars) + value_vars = list(set(frame._column_names) - set(id_vars)) # Error for unimplemented support for datatype dtypes = [frame[col].dtype for col in id_vars + value_vars] @@ -689,7 +690,9 @@ def get_dummies( encode_fallback_dtypes = ["object", "category"] if columns is None or len(columns) == 0: - columns = df.select_dtypes(include=encode_fallback_dtypes).columns + columns = df.select_dtypes( + include=encode_fallback_dtypes + )._column_names _length_check_params(prefix, columns, "prefix") _length_check_params(prefix_sep, columns, "prefix_sep") @@ -1060,7 +1063,9 @@ def unstack(df, level, fill_value=None): ) res = df.T.stack(dropna=False) # Result's index is a multiindex - res.index.names = tuple(df.columns.names) + df.index.names + res.index.names = ( + tuple(df._data.to_pandas_index().names) + df.index.names + ) return res else: columns = df.index._poplevels(level) diff --git a/python/cudf/cudf/core/window/rolling.py b/python/cudf/cudf/core/window/rolling.py index 8ffd75b1d76..2282a435ed3 100644 --- a/python/cudf/cudf/core/window/rolling.py +++ b/python/cudf/cudf/core/window/rolling.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION +# Copyright (c) 2020-2022, NVIDIA CORPORATION import itertools @@ -198,8 +198,7 @@ def __getitem__(self, arg): center=self.center, ) - def _apply_agg_series(self, sr, agg_name): - source_column = sr._column + def _apply_agg_column(self, source_column, agg_name): min_periods = self.min_periods or 1 if isinstance(self.window, int): preceding_window = None @@ -230,7 +229,7 @@ def _apply_agg_series(self, sr, agg_name): ) window = None - result_col = libcudf.rolling.rolling( + return libcudf.rolling.rolling( source_column=source_column, pre_column_window=preceding_window, fwd_column_window=following_window, @@ -240,19 +239,26 @@ def _apply_agg_series(self, sr, agg_name): op=agg_name, agg_params=self.agg_params, ) - return sr._from_data({sr.name: result_col}, sr._index) def _apply_agg_dataframe(self, df, agg_name): - result_df = cudf.DataFrame({}) - for i, col_name in enumerate(df.columns): - result_col = self._apply_agg_series(df[col_name], agg_name) - result_df.insert(i, col_name, result_col) - result_df.index = df.index - return result_df + return cudf.DataFrame._from_data( + { + col_name: self._apply_agg_column(col, agg_name) + for col_name, col in df._data.items() + }, + index=df.index, + ) def _apply_agg(self, agg_name): if isinstance(self.obj, cudf.Series): - return self._apply_agg_series(self.obj, agg_name) + return cudf.Series._from_data( + { + self.obj.name: self._apply_agg_column( + self.obj._column, agg_name + ) + }, + index=self.obj.index, + ) else: return self._apply_agg_dataframe(self.obj, agg_name) diff --git a/python/cudf/cudf/io/parquet.py b/python/cudf/cudf/io/parquet.py index 948428de4f0..6e4e104df4d 100644 --- a/python/cudf/cudf/io/parquet.py +++ b/python/cudf/cudf/io/parquet.py @@ -565,7 +565,7 @@ def to_parquet( if engine == "cudf": # Ensure that no columns dtype is 'category' - for col in df.columns: + for col in df._column_names: if partition_cols is None or col not in partition_cols: if df[col].dtype.name == "category": raise ValueError( diff --git a/python/cudf/cudf/testing/testing.py b/python/cudf/cudf/testing/testing.py index b3e30fac7d5..5f7616cc75e 100644 --- a/python/cudf/cudf/testing/testing.py +++ b/python/cudf/cudf/testing/testing.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. from __future__ import annotations @@ -696,8 +696,8 @@ def assert_frame_equal( if PANDAS_GE_110: pd.testing.assert_index_equal( - left.columns, - right.columns, + left._data.to_pandas_index(), + right._data.to_pandas_index(), exact=check_column_type, check_names=check_names, check_exact=check_exact, @@ -708,8 +708,8 @@ def assert_frame_equal( ) else: pd.testing.assert_index_equal( - left.columns, - right.columns, + left._data.to_pandas_index(), + right._data.to_pandas_index(), exact=check_column_type, check_names=check_names, check_exact=check_exact, @@ -717,7 +717,7 @@ def assert_frame_equal( obj=f"{obj}.columns", ) - for col in left.columns: + for col in left._column_names: assert_column_equal( left._data[col], right._data[col], diff --git a/python/cudf/cudf/tests/test_df_protocol.py b/python/cudf/cudf/tests/test_df_protocol.py index 2f851e805b4..c67fc199710 100644 --- a/python/cudf/cudf/tests/test_df_protocol.py +++ b/python/cudf/cudf/tests/test_df_protocol.py @@ -82,7 +82,7 @@ def assert_dataframe_equal(dfo: DataFrameObject, df: cudf.DataFrame): assert dfo.num_columns() == len(df.columns) assert dfo.num_rows() == len(df) assert dfo.num_chunks() == 1 - assert dfo.column_names() == list(df.columns) + assert dfo.column_names() == tuple(df.columns) for col in df.columns: assert_column_equal(dfo.get_column_by_name(col), df[col]._column) From 3a1dbe8c8b5428e2da7f4cdcb15a68f22e9af9e1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 24 Feb 2022 10:20:59 -0600 Subject: [PATCH 03/11] Enable proper `Index` round-tripping in `orc` reader and writer (#10170) Fixes: #10010 This PR: - [x] Fixes `to_orc` by enabling writing of dataframe metadata to ORC file being created. - [x] Fixes `read_orc` to correctly read and assign `Index` objects that exist in `metadata` - [x] Note: This change is not backward compatible with files already written in previous versions of `cudf`. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Ram (Ramakrishna Prabhu) (https://github.com/rgsl888prabhu) URL: https://github.com/rapidsai/cudf/pull/10170 --- python/cudf/cudf/_lib/orc.pyx | 83 +++++++++++++++++++++++++++++- python/cudf/cudf/tests/test_orc.py | 12 +++-- 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/_lib/orc.pyx b/python/cudf/cudf/_lib/orc.pyx index ce4f183e795..127e3a612dc 100644 --- a/python/cudf/cudf/_lib/orc.pyx +++ b/python/cudf/cudf/_lib/orc.pyx @@ -8,6 +8,12 @@ from libcpp.memory cimport make_unique, unique_ptr from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector +from collections import OrderedDict + +try: + import ujson as json +except ImportError: + import json cimport cudf._lib.cpp.io.types as cudf_io_types from cudf._lib.column cimport Column @@ -123,8 +129,22 @@ cpdef read_orc(object filepaths_or_buffers, c_result = move(libcudf_read_orc(c_orc_reader_options)) names = [name.decode() for name in c_result.metadata.column_names] + actual_index_names, names, is_range_index, reset_index_name, range_idx = \ + _get_index_from_metadata(c_result.metadata.user_data, + names, + skip_rows, + num_rows) + + data, index = data_from_unique_ptr( + move(c_result.tbl), + names, + actual_index_names + ) - data, index = data_from_unique_ptr(move(c_result.tbl), names) + if is_range_index: + index = range_idx + elif reset_index_name: + index.names = [None] * len(index.names) data = { name: update_column_struct_field_names( @@ -144,6 +164,60 @@ cdef compression_type _get_comp_type(object compression): else: raise ValueError(f"Unsupported `compression` type {compression}") +cdef tuple _get_index_from_metadata( + map[string, string] user_data, + object names, + object skip_rows, + object num_rows): + json_str = user_data[b'pandas'].decode('utf-8') + meta = None + index_col = None + is_range_index = False + reset_index_name = False + range_idx = None + if json_str != "": + meta = json.loads(json_str) + + if 'index_columns' in meta and len(meta['index_columns']) > 0: + index_col = meta['index_columns'] + if isinstance(index_col[0], dict) and \ + index_col[0]['kind'] == 'range': + is_range_index = True + else: + index_col_names = OrderedDict() + for idx_col in index_col: + for c in meta['columns']: + if c['field_name'] == idx_col: + index_col_names[idx_col] = \ + c['name'] or c['field_name'] + if c['name'] is None: + reset_index_name = True + + actual_index_names = None + if index_col is not None and len(index_col) > 0: + if is_range_index: + range_index_meta = index_col[0] + range_idx = cudf.RangeIndex( + start=range_index_meta['start'], + stop=range_index_meta['stop'], + step=range_index_meta['step'], + name=range_index_meta['name'] + ) + if skip_rows is not None: + range_idx = range_idx[skip_rows:] + if num_rows is not None: + range_idx = range_idx[:num_rows] + else: + actual_index_names = list(index_col_names.values()) + names = names[len(actual_index_names):] + + return ( + actual_index_names, + names, + is_range_index, + reset_index_name, + range_idx + ) cdef cudf_io_types.statistics_freq _get_orc_stat_freq(object statistics): """ @@ -180,6 +254,10 @@ cpdef write_orc(table, cdef unique_ptr[data_sink] data_sink_c cdef sink_info sink_info_c = make_sink_info(path_or_buf, data_sink_c) cdef unique_ptr[table_input_metadata] tbl_meta + cdef map[string, string] user_data + user_data[str.encode("pandas")] = str.encode(generate_pandas_metadata( + table, None) + ) if not isinstance(table._index, cudf.RangeIndex): tv = table_view_from_table(table) @@ -204,8 +282,9 @@ cpdef write_orc(table, cdef orc_writer_options c_orc_writer_options = move( orc_writer_options.builder( - sink_info_c, table_view_from_table(table, ignore_index=True) + sink_info_c, tv ).metadata(tbl_meta.get()) + .key_value_metadata(move(user_data)) .compression(compression_) .enable_statistics(_get_orc_stat_freq(statistics)) .build() diff --git a/python/cudf/cudf/tests/test_orc.py b/python/cudf/cudf/tests/test_orc.py index 623098741a9..bd7335c577c 100644 --- a/python/cudf/cudf/tests/test_orc.py +++ b/python/cudf/cudf/tests/test_orc.py @@ -552,7 +552,7 @@ def test_orc_writer_sliced(tmpdir): df_select = df.iloc[1:3] df_select.to_orc(cudf_path) - assert_eq(cudf.read_orc(cudf_path), df_select.reset_index(drop=True)) + assert_eq(cudf.read_orc(cudf_path), df_select) @pytest.mark.parametrize( @@ -794,7 +794,8 @@ def test_orc_bool_encode_fail(): # Also validate data pdf = pa.orc.ORCFile(buffer).read().to_pandas() - assert_eq(okay_df, pdf) + + assert_eq(okay_df.to_pandas(nullable=True), pdf) def test_nanoseconds_overflow(): @@ -840,7 +841,12 @@ def test_empty_string_columns(data): got_df = cudf.read_orc(buffer) assert_eq(expected, got_df) - assert_eq(expected_pdf, got_df) + assert_eq( + expected_pdf, + got_df.to_pandas(nullable=True) + if expected_pdf["string"].dtype == pd.StringDtype() + else got_df, + ) @pytest.mark.parametrize("scale", [-3, 0, 3]) From df646b22ea35aab0a4a9e23d949af4704e92cc42 Mon Sep 17 00:00:00 2001 From: Nghia Truong Date: Thu, 24 Feb 2022 19:06:56 -0700 Subject: [PATCH 04/11] Fix `std::bad_alloc` exception due to JIT reserving a huge buffer (#10317) In file `jit/cache.cpp`, a program cache always internally reserves a `std::unordered_map` using a size set by an environment variable `LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS`. If that environment variable does not exist, a default value (`std::numeric_limit::max`) is used. Such default value is huge, leading to allocating a huge (impossible) size of memory chunk that crashes the system. This PR changes that default value from `std::numeric_limit::max` to `1024^2`. This is essentially a reverse of the PR https://github.com/rapidsai/cudf/issues/10312 but set the default value to `1024` instead of `100`. Note that `1024^2` is just some random number, not based on any specific calculation. Closes #10312 and closes #9362. Authors: - Nghia Truong (https://github.com/ttnghia) Approvers: - Mark Harris (https://github.com/harrism) - Karthikeyan (https://github.com/karthikeyann) URL: https://github.com/rapidsai/cudf/pull/10317 --- cpp/src/jit/cache.cpp | 45 ++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/cpp/src/jit/cache.cpp b/cpp/src/jit/cache.cpp index 37b5f58da22..159681eaffc 100644 --- a/cpp/src/jit/cache.cpp +++ b/cpp/src/jit/cache.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -104,13 +104,10 @@ std::string get_program_cache_dir() #endif } -void try_parse_numeric_env_var(std::size_t& result, char const* const env_name) +std::size_t try_parse_numeric_env_var(char const* const env_name, std::size_t default_val) { - auto value = std::getenv(env_name); - - if (value != nullptr) { - result = std::stoull(value); // fails if env var contains invalid value. - } + auto const value = std::getenv(env_name); + return value != nullptr ? std::stoull(value) : default_val; } jitify2::ProgramCache<>& get_program_cache(jitify2::PreprocessedProgramData preprog) @@ -123,27 +120,19 @@ jitify2::ProgramCache<>& get_program_cache(jitify2::PreprocessedProgramData prep auto existing_cache = caches.find(preprog.name()); if (existing_cache == caches.end()) { - std::size_t kernel_limit_proc = std::numeric_limits::max(); - std::size_t kernel_limit_disk = std::numeric_limits::max(); - try_parse_numeric_env_var(kernel_limit_proc, "LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS"); - try_parse_numeric_env_var(kernel_limit_disk, "LIBCUDF_KERNEL_CACHE_LIMIT_DISK"); - - auto cache_dir = get_program_cache_dir(); - - if (kernel_limit_disk == 0) { - // if kernel_limit_disk is zero, jitify will assign it the value of kernel_limit_proc. - // to avoid this, we treat zero as "disable disk caching" by not providing the cache dir. - cache_dir = {}; - } - - auto res = caches.insert({preprog.name(), - std::make_unique>( // - kernel_limit_proc, - preprog, - nullptr, - cache_dir, - kernel_limit_disk)}); - + auto const kernel_limit_proc = + try_parse_numeric_env_var("LIBCUDF_KERNEL_CACHE_LIMIT_PER_PROCESS", 10'000); + auto const kernel_limit_disk = + try_parse_numeric_env_var("LIBCUDF_KERNEL_CACHE_LIMIT_DISK", 100'000); + + // if kernel_limit_disk is zero, jitify will assign it the value of kernel_limit_proc. + // to avoid this, we treat zero as "disable disk caching" by not providing the cache dir. + auto const cache_dir = kernel_limit_disk == 0 ? std::string{} : get_program_cache_dir(); + + auto const res = + caches.insert({preprog.name(), + std::make_unique>( + kernel_limit_proc, preprog, nullptr, cache_dir, kernel_limit_disk)}); existing_cache = res.first; } From eaae94b533a3cf0830efda7f528c94d439e4c037 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Fri, 25 Feb 2022 07:48:46 +0530 Subject: [PATCH 05/11] Add device create_sequence_table for benchmarks (#10300) addresses parts of https://github.com/rapidsai/cudf/issues/5773 - Add `create_sequence_table` which creates sequences in device (only numeric types supported) with/without nulls. - Add `create_random_null_mask` to create random null mask with given probability. (0.0-1.0 null probability) ~- add gnu++17 to generate_input.cu (temporarily for int128 STL support).~ - renamed `repeat_dtypes` to `cycle_dtypes` and moved out of create_* methods - updated ast bench, search, scatter , binary ops bench Splitting PR https://github.com/rapidsai/cudf/pull/10109 for review Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - Conor Hoekstra (https://github.com/codereport) - Nghia Truong (https://github.com/ttnghia) - Robert Maynard (https://github.com/robertmaynard) - MithunR (https://github.com/mythrocks) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/10300 --- cpp/benchmarks/CMakeLists.txt | 2 +- cpp/benchmarks/ast/transform.cpp | 73 ++++-------- cpp/benchmarks/binaryop/binaryop.cpp | 53 +++------ cpp/benchmarks/binaryop/compiled_binaryop.cpp | 21 ++-- cpp/benchmarks/common/generate_input.cpp | 39 +++++-- cpp/benchmarks/common/generate_input.hpp | 58 +++++++--- cpp/benchmarks/common/generate_nullmask.cu | 59 ++++++++++ .../common/random_distribution_factory.hpp | 8 +- cpp/benchmarks/copying/copy_if_else.cpp | 4 +- cpp/benchmarks/copying/scatter.cu | 75 ++++-------- cpp/benchmarks/groupby/group_struct.cu | 17 +-- cpp/benchmarks/hashing/hash.cpp | 4 +- cpp/benchmarks/io/csv/csv_reader.cpp | 7 +- cpp/benchmarks/io/csv/csv_writer.cpp | 7 +- cpp/benchmarks/io/orc/orc_reader.cpp | 8 +- cpp/benchmarks/io/orc/orc_writer.cpp | 6 +- cpp/benchmarks/io/parquet/parquet_reader.cpp | 8 +- cpp/benchmarks/io/parquet/parquet_writer.cpp | 8 +- .../io/parquet/parquet_writer_chunks.cpp | 9 +- cpp/benchmarks/io/text/multibyte_split.cpp | 1 - cpp/benchmarks/join/join_common.hpp | 4 +- cpp/benchmarks/reduction/scan.cpp | 4 +- cpp/benchmarks/replace/clamp.cpp | 4 +- cpp/benchmarks/replace/nans.cpp | 4 +- cpp/benchmarks/search/search.cpp | 108 +++++++----------- cpp/benchmarks/sort/sort_strings.cpp | 4 +- cpp/benchmarks/string/case.cpp | 4 +- cpp/benchmarks/string/combine.cpp | 6 +- cpp/benchmarks/string/contains.cpp | 4 +- cpp/benchmarks/string/convert_datetime.cpp | 4 +- cpp/benchmarks/string/convert_fixed_point.cpp | 4 +- cpp/benchmarks/string/convert_numerics.cpp | 4 +- cpp/benchmarks/string/copy.cu | 4 +- cpp/benchmarks/string/factory.cu | 7 +- cpp/benchmarks/string/filter.cpp | 5 +- cpp/benchmarks/string/find.cpp | 5 +- cpp/benchmarks/string/repeat_strings.cpp | 4 +- cpp/benchmarks/string/replace.cpp | 5 +- cpp/benchmarks/string/replace_re.cpp | 5 +- cpp/benchmarks/string/split.cpp | 5 +- cpp/benchmarks/string/substring.cpp | 5 +- cpp/benchmarks/string/translate.cpp | 5 +- cpp/benchmarks/text/ngrams.cpp | 5 +- cpp/benchmarks/text/normalize.cpp | 5 +- cpp/benchmarks/text/normalize_spaces.cpp | 5 +- cpp/benchmarks/text/tokenize.cpp | 5 +- 46 files changed, 339 insertions(+), 352 deletions(-) create mode 100644 cpp/benchmarks/common/generate_nullmask.cu diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 11eef015364..99aeff0df93 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -14,7 +14,7 @@ find_package(Threads REQUIRED) -add_library(cudf_datagen STATIC common/generate_input.cpp) +add_library(cudf_datagen STATIC common/generate_input.cpp common/generate_nullmask.cu) target_compile_features(cudf_datagen PUBLIC cxx_std_17 cuda_std_17) target_compile_options( diff --git a/cpp/benchmarks/ast/transform.cpp b/cpp/benchmarks/ast/transform.cpp index c17c288a6d3..de0429f74ad 100644 --- a/cpp/benchmarks/ast/transform.cpp +++ b/cpp/benchmarks/ast/transform.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,26 +14,18 @@ * limitations under the License. */ -#include -#include -#include +#include +#include +#include + #include #include -#include - -#include - -#include -#include -#include -#include #include #include #include -#include -#include +#include #include enum class TreeType { @@ -41,45 +33,23 @@ enum class TreeType { // child column reference }; +template class AST : public cudf::benchmark { }; template static void BM_ast_transform(benchmark::State& state) { - const cudf::size_type table_size{(cudf::size_type)state.range(0)}; - const cudf::size_type tree_levels = (cudf::size_type)state.range(1); + auto const table_size{static_cast(state.range(0))}; + auto const tree_levels{static_cast(state.range(1))}; // Create table data - auto n_cols = reuse_columns ? 1 : tree_levels + 1; - auto column_wrappers = std::vector>(n_cols); - auto columns = std::vector(n_cols); - - auto data_iterator = thrust::make_counting_iterator(0); - - if constexpr (Nullable) { - auto validities = std::vector(table_size); - std::random_device rd; - std::mt19937 gen(rd()); - - std::generate( - validities.begin(), validities.end(), [&]() { return gen() > (0.5 * gen.max()); }); - std::generate_n(column_wrappers.begin(), n_cols, [=]() { - return cudf::test::fixed_width_column_wrapper( - data_iterator, data_iterator + table_size, validities.begin()); - }); - } else { - std::generate_n(column_wrappers.begin(), n_cols, [=]() { - return cudf::test::fixed_width_column_wrapper(data_iterator, - data_iterator + table_size); - }); - } - std::transform( - column_wrappers.begin(), column_wrappers.end(), columns.begin(), [](auto const& col) { - return static_cast(col); - }); - - cudf::table_view table{columns}; + auto const n_cols = reuse_columns ? 1 : tree_levels + 1; + auto const source_table = + create_sequence_table(cycle_dtypes({cudf::type_to_id()}, n_cols), + row_count{table_size}, + Nullable ? 0.5 : -1.0); + auto table = source_table->view(); // Create column references auto column_refs = std::vector(); @@ -138,10 +108,15 @@ static void CustomRanges(benchmark::internal::Benchmark* b) } } -#define AST_TRANSFORM_BENCHMARK_DEFINE(name, key_type, tree_type, reuse_columns, nullable) \ - TEMPLATED_BENCHMARK_F(AST, BM_ast_transform, key_type, tree_type, reuse_columns, nullable) \ - ->Apply(CustomRanges) \ - ->Unit(benchmark::kMillisecond) \ +#define AST_TRANSFORM_BENCHMARK_DEFINE(name, key_type, tree_type, reuse_columns, nullable) \ + BENCHMARK_TEMPLATE_DEFINE_F(AST, name, key_type, tree_type, reuse_columns, nullable) \ + (::benchmark::State & st) \ + { \ + BM_ast_transform(st); \ + } \ + BENCHMARK_REGISTER_F(AST, name) \ + ->Apply(CustomRanges) \ + ->Unit(benchmark::kMillisecond) \ ->UseManualTime(); AST_TRANSFORM_BENCHMARK_DEFINE( diff --git a/cpp/benchmarks/binaryop/binaryop.cpp b/cpp/benchmarks/binaryop/binaryop.cpp index 314d657679b..e5bde94f1f9 100644 --- a/cpp/benchmarks/binaryop/binaryop.cpp +++ b/cpp/benchmarks/binaryop/binaryop.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,23 +14,15 @@ * limitations under the License. */ +#include +#include +#include + #include -#include -#include #include #include -#include - -#include - -#include -#include -#include - -#include #include -#include #include // This set of benchmarks is designed to be a comparison for the AST benchmarks @@ -47,40 +39,29 @@ class BINARYOP : public cudf::benchmark { template static void BM_binaryop_transform(benchmark::State& state) { - const cudf::size_type table_size{(cudf::size_type)state.range(0)}; - const cudf::size_type tree_levels = (cudf::size_type)state.range(1); + auto const table_size{static_cast(state.range(0))}; + auto const tree_levels{static_cast(state.range(1))}; // Create table data - auto n_cols = reuse_columns ? 1 : tree_levels + 1; - auto column_wrappers = std::vector>(); - auto columns = std::vector(n_cols); - - auto data_iterator = thrust::make_counting_iterator(0); - std::generate_n(std::back_inserter(column_wrappers), n_cols, [=]() { - return cudf::test::fixed_width_column_wrapper(data_iterator, - data_iterator + table_size); - }); - std::transform( - column_wrappers.begin(), column_wrappers.end(), columns.begin(), [](auto const& col) { - return static_cast(col); - }); - - cudf::table_view table{columns}; + auto const n_cols = reuse_columns ? 1 : tree_levels + 1; + auto const source_table = create_sequence_table( + cycle_dtypes({cudf::type_to_id()}, n_cols), row_count{table_size}); + cudf::table_view table{*source_table}; // Execute benchmark for (auto _ : state) { cuda_event_timer raii(state, true); // flush_l2_cache = true, stream = 0 // Execute tree that chains additions like (((a + b) + c) + d) - auto const op = cudf::binary_operator::ADD; - auto result_data_type = cudf::data_type(cudf::type_to_id()); + auto const op = cudf::binary_operator::ADD; + auto const result_data_type = cudf::data_type(cudf::type_to_id()); if (reuse_columns) { - auto result = cudf::binary_operation(columns.at(0), columns.at(0), op, result_data_type); + auto result = cudf::binary_operation(table.column(0), table.column(0), op, result_data_type); for (cudf::size_type i = 0; i < tree_levels - 1; i++) { - result = cudf::binary_operation(result->view(), columns.at(0), op, result_data_type); + result = cudf::binary_operation(result->view(), table.column(0), op, result_data_type); } } else { - auto result = cudf::binary_operation(columns.at(0), columns.at(1), op, result_data_type); - std::for_each(std::next(columns.cbegin(), 2), columns.cend(), [&](auto const& col) { + auto result = cudf::binary_operation(table.column(0), table.column(1), op, result_data_type); + std::for_each(std::next(table.begin(), 2), table.end(), [&](auto const& col) { result = cudf::binary_operation(result->view(), col, op, result_data_type); }); } diff --git a/cpp/benchmarks/binaryop/compiled_binaryop.cpp b/cpp/benchmarks/binaryop/compiled_binaryop.cpp index f8226c7387a..50cd0b7b8d5 100644 --- a/cpp/benchmarks/binaryop/compiled_binaryop.cpp +++ b/cpp/benchmarks/binaryop/compiled_binaryop.cpp @@ -14,30 +14,25 @@ * limitations under the License. */ -#include -#include -#include - -#include +#include +#include +#include #include -#include - class COMPILED_BINARYOP : public cudf::benchmark { }; template void BM_compiled_binaryop(benchmark::State& state, cudf::binary_operator binop) { - const cudf::size_type column_size{(cudf::size_type)state.range(0)}; + auto const column_size{static_cast(state.range(0))}; - auto data_it = thrust::make_counting_iterator(0); - cudf::test::fixed_width_column_wrapper input1(data_it, data_it + column_size); - cudf::test::fixed_width_column_wrapper input2(data_it, data_it + column_size); + auto const source_table = create_sequence_table( + {cudf::type_to_id(), cudf::type_to_id()}, row_count{column_size}); - auto lhs = cudf::column_view(input1); - auto rhs = cudf::column_view(input2); + auto lhs = cudf::column_view(source_table->get_column(0)); + auto rhs = cudf::column_view(source_table->get_column(1)); auto output_dtype = cudf::data_type(cudf::type_to_id()); // Call once for hot cache. diff --git a/cpp/benchmarks/common/generate_input.cpp b/cpp/benchmarks/common/generate_input.cpp index 68eabd3f1cc..d6564428a2e 100644 --- a/cpp/benchmarks/common/generate_input.cpp +++ b/cpp/benchmarks/common/generate_input.cpp @@ -19,6 +19,8 @@ #include #include +#include +#include #include #include @@ -571,11 +573,11 @@ columns_vector create_random_columns(data_profile const& profile, } /** - * @brief Repeats the input data types in round-robin order to fill a vector of @ref num_cols + * @brief Repeats the input data types cyclically order to fill a vector of @ref num_cols * elements. */ -std::vector repeat_dtypes(std::vector const& dtype_ids, - cudf::size_type num_cols) +std::vector cycle_dtypes(std::vector const& dtype_ids, + cudf::size_type num_cols) { if (dtype_ids.size() == static_cast(num_cols)) { return dtype_ids; } std::vector out_dtypes; @@ -586,29 +588,26 @@ std::vector repeat_dtypes(std::vector const& dtype } std::unique_ptr create_random_table(std::vector const& dtype_ids, - cudf::size_type num_cols, table_size_bytes table_bytes, data_profile const& profile, unsigned seed) { - auto const out_dtype_ids = repeat_dtypes(dtype_ids, num_cols); size_t const avg_row_bytes = - std::accumulate(out_dtype_ids.begin(), out_dtype_ids.end(), 0ul, [&](size_t sum, auto tid) { + std::accumulate(dtype_ids.begin(), dtype_ids.end(), 0ul, [&](size_t sum, auto tid) { return sum + avg_element_size(profile, cudf::data_type(tid)); }); cudf::size_type const num_rows = table_bytes.size / avg_row_bytes; - return create_random_table(out_dtype_ids, num_cols, row_count{num_rows}, profile, seed); + return create_random_table(dtype_ids, row_count{num_rows}, profile, seed); } std::unique_ptr create_random_table(std::vector const& dtype_ids, - cudf::size_type num_cols, row_count num_rows, data_profile const& profile, unsigned seed) { - auto const out_dtype_ids = repeat_dtypes(dtype_ids, num_cols); - auto seed_engine = deterministic_engine(seed); + cudf::size_type const num_cols = dtype_ids.size(); + auto seed_engine = deterministic_engine(seed); auto const processor_count = std::thread::hardware_concurrency(); cudf::size_type const cols_per_thread = (num_cols + processor_count - 1) / processor_count; @@ -619,8 +618,8 @@ std::unique_ptr create_random_table(std::vector cons for (unsigned int i = 0; i < processor_count && next_col < num_cols; ++i) { auto thread_engine = deterministic_engine(seed_dist(seed_engine)); auto const thread_num_cols = std::min(num_cols - next_col, cols_per_thread); - std::vector thread_types(out_dtype_ids.begin() + next_col, - out_dtype_ids.begin() + next_col + thread_num_cols); + std::vector thread_types(dtype_ids.begin() + next_col, + dtype_ids.begin() + next_col + thread_num_cols); col_futures.emplace_back(std::async(std::launch::async, create_random_columns, std::cref(profile), @@ -642,6 +641,22 @@ std::unique_ptr create_random_table(std::vector cons return std::make_unique(std::move(output_columns)); } +std::unique_ptr create_sequence_table(std::vector const& dtype_ids, + row_count num_rows, + float null_probability, + unsigned seed) +{ + auto columns = std::vector>(dtype_ids.size()); + std::transform(dtype_ids.begin(), dtype_ids.end(), columns.begin(), [&](auto dtype) mutable { + auto init = cudf::make_default_constructed_scalar(cudf::data_type{dtype}); + auto col = cudf::sequence(num_rows.count, *init); + auto [mask, count] = create_random_null_mask(num_rows.count, null_probability, seed++); + col->set_null_mask(std::move(mask), count); + return col; + }); + return std::make_unique(std::move(columns)); +} + std::vector get_type_or_group(int32_t id) { // identity transformation when passing a concrete type_id diff --git a/cpp/benchmarks/common/generate_input.hpp b/cpp/benchmarks/common/generate_input.hpp index 1999ccb8ec3..17bd650e722 100644 --- a/cpp/benchmarks/common/generate_input.hpp +++ b/cpp/benchmarks/common/generate_input.hpp @@ -19,6 +19,7 @@ #include #include +#include #include /** @@ -223,9 +224,9 @@ class data_profile { cudf::size_type avg_run_length = 4; public: - template < - typename T, - typename std::enable_if_t && std::is_integral_v, T>* = nullptr> + template && cuda::std::is_integral_v, T>* = + nullptr> distribution_params get_distribution_params() const { auto it = int_params.find(cudf::type_to_id()); @@ -306,7 +307,7 @@ class data_profile { // discrete distributions (integers, strings, lists). Otherwise the call with have no effect. template , T>* = nullptr> + typename std::enable_if_t, T>* = nullptr> void set_distribution_params(Type_enum type_or_group, distribution_id dist, T lower_bound, @@ -369,18 +370,13 @@ struct row_count { /** * @brief Deterministically generates a table filled with data with the given parameters. * - * If the number of passed types is smaller than the number of requested column, the columns types - * with be repeated in round-robin order to fill the table. - * * @param dtype_ids Vector of requested column types - * @param num_cols Number of columns in the output table * @param table_bytes Target size of the output table, in bytes. Some type may not produce columns * of exact size * @param data_params optional, set of data parameters describing the data profile for each type * @param seed optional, seed for the pseudo-random engine */ std::unique_ptr create_random_table(std::vector const& dtype_ids, - cudf::size_type num_cols, table_size_bytes table_bytes, data_profile const& data_params = data_profile{}, unsigned seed = 1); @@ -388,17 +384,51 @@ std::unique_ptr create_random_table(std::vector cons /** * @brief Deterministically generates a table filled with data with the given parameters. * - * If the number of passed types is smaller than the number of requested column, the columns types - * with be repeated in round-robin order to fill the table. - * * @param dtype_ids Vector of requested column types - * @param num_cols Number of columns in the output table * @param num_rows Number of rows in the output table * @param data_params optional, set of data parameters describing the data profile for each type * @param seed optional, seed for the pseudo-random engine */ std::unique_ptr create_random_table(std::vector const& dtype_ids, - cudf::size_type num_cols, row_count num_rows, data_profile const& data_params = data_profile{}, unsigned seed = 1); + +/** + * @brief Generate sequence columns starting with value 0 in first row and increasing by 1 in + * subsequent rows. + * + * @param dtype_ids Vector of requested column types + * @param num_rows Number of rows in the output table + * @param null_probability optional, probability of a null value + * <0 implies no null mask, =0 implies all valids, >=1 implies all nulls + * @param seed optional, seed for the pseudo-random engine + * @return A table with the sequence columns. + */ +std::unique_ptr create_sequence_table(std::vector const& dtype_ids, + row_count num_rows, + float null_probability = -1.0, + unsigned seed = 1); + +/** + * @brief Repeats the input data types cyclically to fill a vector of @ref num_cols + * elements. + * + * @param dtype_ids Vector of requested column types + * @param num_cols Number of types in the output vector + * @return A vector of type_ids + */ +std::vector cycle_dtypes(std::vector const& dtype_ids, + cudf::size_type num_cols); +/** + * @brief Create a random null mask object + * + * @param size number of rows + * @param null_probability probability of a null value + * <0 implies no null mask, =0 implies all valids, >=1 implies all nulls + * @param seed optional, seed for the pseudo-random engine + * @return null mask device buffer with random null mask data and null count + */ +std::pair create_random_null_mask(cudf::size_type size, + float null_probability, + unsigned seed = 1); diff --git a/cpp/benchmarks/common/generate_nullmask.cu b/cpp/benchmarks/common/generate_nullmask.cu new file mode 100644 index 00000000000..502af95a971 --- /dev/null +++ b/cpp/benchmarks/common/generate_nullmask.cu @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "generate_input.hpp" + +#include +#include + +#include + +/** + * @brief bool generator with given probability [0.0 - 1.0] of returning true. + * + */ +struct bool_generator { + thrust::minstd_rand engine; + thrust::uniform_real_distribution dist; + float probability_true; + bool_generator(unsigned seed, float probability_true) + : engine(seed), dist{0, 1}, probability_true{probability_true} + { + } + + __device__ bool operator()(size_t n) + { + engine.discard(n); + return dist(engine) < probability_true; + } +}; + +std::pair create_random_null_mask(cudf::size_type size, + float null_probability, + unsigned seed) +{ + if (null_probability < 0.0f) { + return {rmm::device_buffer{}, 0}; + } else if (null_probability == 0.0f) { + return {cudf::create_null_mask(size, cudf::mask_state::ALL_NULL), size}; + } else if (null_probability >= 1.0f) { + return {cudf::create_null_mask(size, cudf::mask_state::ALL_VALID), 0}; + } else { + return cudf::detail::valid_if(thrust::make_counting_iterator(0), + thrust::make_counting_iterator(size), + bool_generator{seed, 1.0f - null_probability}); + } +}; diff --git a/cpp/benchmarks/common/random_distribution_factory.hpp b/cpp/benchmarks/common/random_distribution_factory.hpp index 3289c6f40ab..df2b6e0a754 100644 --- a/cpp/benchmarks/common/random_distribution_factory.hpp +++ b/cpp/benchmarks/common/random_distribution_factory.hpp @@ -24,7 +24,7 @@ /** * @brief Generates a normal(binomial) distribution between zero and upper_bound. */ -template , T>* = nullptr> +template , T>* = nullptr> auto make_normal_dist(T upper_bound) { using uT = typename std::make_unsigned::type; @@ -42,7 +42,7 @@ auto make_normal_dist(T upper_bound) return std::normal_distribution(mean, stddev); } -template , T>* = nullptr> +template , T>* = nullptr> auto make_uniform_dist(T range_start, T range_end) { return std::uniform_int_distribution(range_start, range_end); @@ -62,7 +62,7 @@ double geometric_dist_p(T range_size) return p ? p : std::numeric_limits::epsilon(); } -template , T>* = nullptr> +template , T>* = nullptr> auto make_geometric_dist(T range_start, T range_end) { using uT = typename std::make_unsigned::type; @@ -82,7 +82,7 @@ auto make_geometric_dist(T range_start, T range_end) template using distribution_fn = std::function; -template , T>* = nullptr> +template , T>* = nullptr> distribution_fn make_distribution(distribution_id did, T lower_bound, T upper_bound) { switch (did) { diff --git a/cpp/benchmarks/copying/copy_if_else.cpp b/cpp/benchmarks/copying/copy_if_else.cpp index 6f3ba34e373..6f094aba680 100644 --- a/cpp/benchmarks/copying/copy_if_else.cpp +++ b/cpp/benchmarks/copying/copy_if_else.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,7 +32,7 @@ static void BM_copy_if_else(benchmark::State& state, bool nulls) cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; auto input_type = cudf::type_to_id(); auto bool_type = cudf::type_id::BOOL8; - auto const input = create_random_table({input_type, input_type, bool_type}, 3, row_count{n_rows}); + auto const input = create_random_table({input_type, input_type, bool_type}, row_count{n_rows}); if (!nulls) { input->get_column(2).set_null_mask(rmm::device_buffer{}, 0); diff --git a/cpp/benchmarks/copying/scatter.cu b/cpp/benchmarks/copying/scatter.cu index a9ab376c8c3..977937beaa2 100644 --- a/cpp/benchmarks/copying/scatter.cu +++ b/cpp/benchmarks/copying/scatter.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,24 +14,15 @@ * limitations under the License. */ -#include +#include +#include +#include #include - -#include -#include -// #include -#include -#include -#include - #include -#include -#include - -#include "../fixture/benchmark_fixture.hpp" -#include "../synchronization/synchronization.hpp" +#include +#include class Scatter : public cudf::benchmark { }; @@ -39,53 +30,33 @@ class Scatter : public cudf::benchmark { template void BM_scatter(benchmark::State& state) { - const cudf::size_type source_size{(cudf::size_type)state.range(0)}; - const auto n_cols = (cudf::size_type)state.range(1); - - // Every element is valid - auto data = cudf::detail::make_counting_transform_iterator(0, [](auto i) { return i; }); + auto const source_size{static_cast(state.range(0))}; + auto const n_cols{static_cast(state.range(1))}; // Gather indices - std::vector host_map_data(source_size); - std::iota(host_map_data.begin(), host_map_data.end(), 0); + auto scatter_map_table = + create_sequence_table({cudf::type_to_id()}, row_count{source_size}); + auto scatter_map = scatter_map_table->get_column(0).mutable_view(); if (coalesce) { - std::reverse(host_map_data.begin(), host_map_data.end()); + thrust::reverse( + thrust::device, scatter_map.begin(), scatter_map.end()); } else { - std::random_shuffle(host_map_data.begin(), host_map_data.end()); + thrust::shuffle(thrust::device, + scatter_map.begin(), + scatter_map.end(), + thrust::default_random_engine()); } - cudf::test::fixed_width_column_wrapper scatter_map(host_map_data.begin(), - host_map_data.end()); - - std::vector> source_column_wrappers; - std::vector source_columns(n_cols); - - std::vector> target_column_wrappers; - std::vector target_columns(n_cols); - - std::generate_n(std::back_inserter(source_column_wrappers), n_cols, [=]() { - return cudf::test::fixed_width_column_wrapper(data, data + source_size); - }); - std::transform(source_column_wrappers.begin(), - source_column_wrappers.end(), - source_columns.begin(), - [](auto const& col) { return static_cast(col); }); - - std::generate_n(std::back_inserter(target_column_wrappers), n_cols, [=]() { - return cudf::test::fixed_width_column_wrapper(data, data + source_size); - }); - std::transform(target_column_wrappers.begin(), - target_column_wrappers.end(), - target_columns.begin(), - [](auto const& col) { return static_cast(col); }); - - cudf::table_view source_table{source_columns}; - cudf::table_view target_table{target_columns}; + // Every element is valid + auto source_table = create_sequence_table(cycle_dtypes({cudf::type_to_id()}, n_cols), + row_count{source_size}); + auto target_table = create_sequence_table(cycle_dtypes({cudf::type_to_id()}, n_cols), + row_count{source_size}); for (auto _ : state) { cuda_event_timer raii(state, true); // flush_l2_cache = true, stream = 0 - cudf::scatter(source_table, scatter_map, target_table); + cudf::scatter(*source_table, scatter_map, *target_table); } state.SetBytesProcessed(static_cast(state.iterations()) * state.range(0) * n_cols * 2 * diff --git a/cpp/benchmarks/groupby/group_struct.cu b/cpp/benchmarks/groupby/group_struct.cu index 355c7cbab6c..34f2d1adc75 100644 --- a/cpp/benchmarks/groupby/group_struct.cu +++ b/cpp/benchmarks/groupby/group_struct.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,18 +41,11 @@ static auto create_data_table(cudf::size_type n_rows) // The first two struct members are int32 and string. // The first column is also used as keys in groupby. - auto col_ids = std::vector{cudf::type_id::INT32, cudf::type_id::STRING}; - // The subsequent struct members are int32 and string again. - for (cudf::size_type i = 3; i <= num_struct_members; ++i) { - if (i % 2) { - col_ids.push_back(cudf::type_id::INT32); - } else { - col_ids.push_back(cudf::type_id::STRING); - } - } - - return create_random_table(col_ids, num_struct_members, row_count{n_rows}, table_profile); + return create_random_table( + cycle_dtypes({cudf::type_id::INT32, cudf::type_id::STRING}, num_struct_members), + row_count{n_rows}, + table_profile); } // Max aggregation/scan technically has the same performance as min. diff --git a/cpp/benchmarks/hashing/hash.cpp b/cpp/benchmarks/hashing/hash.cpp index e2ad38230a2..fe22795bb6b 100644 --- a/cpp/benchmarks/hashing/hash.cpp +++ b/cpp/benchmarks/hashing/hash.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ enum contains_nulls { no_nulls, nulls }; static void BM_hash(benchmark::State& state, cudf::hash_id hid, contains_nulls has_nulls) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; - auto const data = create_random_table({cudf::type_id::INT64}, 1, row_count{n_rows}); + auto const data = create_random_table({cudf::type_id::INT64}, row_count{n_rows}); if (has_nulls == contains_nulls::no_nulls) data->get_column(0).set_null_mask(rmm::device_buffer{}, 0); diff --git a/cpp/benchmarks/io/csv/csv_reader.cpp b/cpp/benchmarks/io/csv/csv_reader.cpp index 241ba4d5954..c50f5220200 100644 --- a/cpp/benchmarks/io/csv/csv_reader.cpp +++ b/cpp/benchmarks/io/csv/csv_reader.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,8 @@ void BM_csv_read_varying_input(benchmark::State& state) auto const data_types = get_type_or_group(state.range(0)); auto const source_type = static_cast(state.range(1)); - auto const tbl = create_random_table(data_types, num_cols, table_size_bytes{data_size}); + auto const tbl = + create_random_table(cycle_dtypes(data_types, num_cols), table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(source_type); @@ -75,7 +76,7 @@ void BM_csv_read_varying_options(benchmark::State& state) col_sel); auto const cols_to_read = select_column_indexes(data_types.size(), col_sel); - auto const tbl = create_random_table(data_types, data_types.size(), table_size_bytes{data_size}); + auto const tbl = create_random_table(data_types, table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(io_type::HOST_BUFFER); diff --git a/cpp/benchmarks/io/csv/csv_writer.cpp b/cpp/benchmarks/io/csv/csv_writer.cpp index 413a269bcb2..65aa31c68dc 100644 --- a/cpp/benchmarks/io/csv/csv_writer.cpp +++ b/cpp/benchmarks/io/csv/csv_writer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,8 @@ void BM_csv_write_varying_inout(benchmark::State& state) auto const data_types = get_type_or_group(state.range(0)); auto const sink_type = static_cast(state.range(1)); - auto const tbl = create_random_table(data_types, num_cols, table_size_bytes{data_size}); + auto const tbl = + create_random_table(cycle_dtypes(data_types, num_cols), table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(sink_type); @@ -66,7 +67,7 @@ void BM_csv_write_varying_options(benchmark::State& state) int32_t(type_group_id::TIMESTAMP), int32_t(cudf::type_id::STRING)}); - auto const tbl = create_random_table(data_types, data_types.size(), table_size_bytes{data_size}); + auto const tbl = create_random_table(data_types, table_size_bytes{data_size}); auto const view = tbl->view(); std::string const na_per(na_per_len, '#'); diff --git a/cpp/benchmarks/io/orc/orc_reader.cpp b/cpp/benchmarks/io/orc/orc_reader.cpp index e15513275ee..29d4860a0e5 100644 --- a/cpp/benchmarks/io/orc/orc_reader.cpp +++ b/cpp/benchmarks/io/orc/orc_reader.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,8 +45,8 @@ void BM_orc_read_varying_input(benchmark::State& state) data_profile table_data_profile; table_data_profile.set_cardinality(cardinality); table_data_profile.set_avg_run_length(run_length); - auto const tbl = - create_random_table(data_types, num_cols, table_size_bytes{data_size}, table_data_profile); + auto const tbl = create_random_table( + cycle_dtypes(data_types, num_cols), table_size_bytes{data_size}, table_data_profile); auto const view = tbl->view(); cuio_source_sink_pair source_sink(source_type); @@ -96,7 +96,7 @@ void BM_orc_read_varying_options(benchmark::State& state) int32_t(type_group_id::TIMESTAMP), int32_t(cudf::type_id::STRING)}), col_sel); - auto const tbl = create_random_table(data_types, data_types.size(), table_size_bytes{data_size}); + auto const tbl = create_random_table(data_types, table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(io_type::HOST_BUFFER); diff --git a/cpp/benchmarks/io/orc/orc_writer.cpp b/cpp/benchmarks/io/orc/orc_writer.cpp index 50ae76e867c..e24ca7f749d 100644 --- a/cpp/benchmarks/io/orc/orc_writer.cpp +++ b/cpp/benchmarks/io/orc/orc_writer.cpp @@ -46,8 +46,8 @@ void BM_orc_write_varying_inout(benchmark::State& state) data_profile table_data_profile; table_data_profile.set_cardinality(cardinality); table_data_profile.set_avg_run_length(run_length); - auto const tbl = - create_random_table(data_types, num_cols, table_size_bytes{data_size}, table_data_profile); + auto const tbl = create_random_table( + cycle_dtypes(data_types, num_cols), table_size_bytes{data_size}, table_data_profile); auto const view = tbl->view(); cuio_source_sink_pair source_sink(sink_type); @@ -83,7 +83,7 @@ void BM_orc_write_varying_options(benchmark::State& state) int32_t(cudf::type_id::STRING), int32_t(cudf::type_id::LIST)}); - auto const tbl = create_random_table(data_types, data_types.size(), table_size_bytes{data_size}); + auto const tbl = create_random_table(data_types, table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(io_type::FILEPATH); diff --git a/cpp/benchmarks/io/parquet/parquet_reader.cpp b/cpp/benchmarks/io/parquet/parquet_reader.cpp index 09194931498..74613e50158 100644 --- a/cpp/benchmarks/io/parquet/parquet_reader.cpp +++ b/cpp/benchmarks/io/parquet/parquet_reader.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,8 +45,8 @@ void BM_parq_read_varying_input(benchmark::State& state) data_profile table_data_profile; table_data_profile.set_cardinality(cardinality); table_data_profile.set_avg_run_length(run_length); - auto const tbl = - create_random_table(data_types, num_cols, table_size_bytes{data_size}, table_data_profile); + auto const tbl = create_random_table( + cycle_dtypes(data_types, num_cols), table_size_bytes{data_size}, table_data_profile); auto const view = tbl->view(); cuio_source_sink_pair source_sink(source_type); @@ -96,7 +96,7 @@ void BM_parq_read_varying_options(benchmark::State& state) static_cast(type_group_id::TIMESTAMP), static_cast(cudf::type_id::STRING)}), col_sel); - auto const tbl = create_random_table(data_types, data_types.size(), table_size_bytes{data_size}); + auto const tbl = create_random_table(data_types, table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(io_type::HOST_BUFFER); diff --git a/cpp/benchmarks/io/parquet/parquet_writer.cpp b/cpp/benchmarks/io/parquet/parquet_writer.cpp index 8287c27f804..d203f0d27c8 100644 --- a/cpp/benchmarks/io/parquet/parquet_writer.cpp +++ b/cpp/benchmarks/io/parquet/parquet_writer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,8 +45,8 @@ void BM_parq_write_varying_inout(benchmark::State& state) data_profile table_data_profile; table_data_profile.set_cardinality(cardinality); table_data_profile.set_avg_run_length(run_length); - auto const tbl = - create_random_table(data_types, num_cols, table_size_bytes{data_size}, table_data_profile); + auto const tbl = create_random_table( + cycle_dtypes(data_types, num_cols), table_size_bytes{data_size}, table_data_profile); auto const view = tbl->view(); cuio_source_sink_pair source_sink(sink_type); @@ -77,7 +77,7 @@ void BM_parq_write_varying_options(benchmark::State& state) int32_t(cudf::type_id::STRING), int32_t(cudf::type_id::LIST)}); - auto const tbl = create_random_table(data_types, data_types.size(), table_size_bytes{data_size}); + auto const tbl = create_random_table(data_types, table_size_bytes{data_size}); auto const view = tbl->view(); cuio_source_sink_pair source_sink(io_type::FILEPATH); diff --git a/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp b/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp index 98eaba213e5..30ed245ed9a 100644 --- a/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp +++ b/cpp/benchmarks/io/parquet/parquet_writer_chunks.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,8 @@ void PQ_write(benchmark::State& state) { cudf::size_type num_cols = state.range(0); - auto tbl = create_random_table({cudf::type_id::INT32}, num_cols, table_size_bytes{data_size}); + auto tbl = create_random_table(cycle_dtypes({cudf::type_id::INT32}, num_cols), + table_size_bytes{data_size}); cudf::table_view view = tbl->view(); auto mem_stats_logger = cudf::memory_stats_logger(); @@ -69,8 +70,8 @@ void PQ_write_chunked(benchmark::State& state) std::vector> tables; for (cudf::size_type idx = 0; idx < num_tables; idx++) { - tables.push_back(create_random_table( - {cudf::type_id::INT32}, num_cols, table_size_bytes{size_t(data_size / num_tables)})); + tables.push_back(create_random_table(cycle_dtypes({cudf::type_id::INT32}, num_cols), + table_size_bytes{size_t(data_size / num_tables)})); } auto mem_stats_logger = cudf::memory_stats_logger(); diff --git a/cpp/benchmarks/io/text/multibyte_split.cpp b/cpp/benchmarks/io/text/multibyte_split.cpp index b13835c15bb..8c4b10d928d 100644 --- a/cpp/benchmarks/io/text/multibyte_split.cpp +++ b/cpp/benchmarks/io/text/multibyte_split.cpp @@ -70,7 +70,6 @@ static cudf::string_scalar create_random_input(int32_t num_chars, auto const values_table = create_random_table( // {cudf::type_id::STRING}, - 1, row_count{num_rows}, table_profile); diff --git a/cpp/benchmarks/join/join_common.hpp b/cpp/benchmarks/join/join_common.hpp index f2b9cb1bdb9..c1957db7929 100644 --- a/cpp/benchmarks/join/join_common.hpp +++ b/cpp/benchmarks/join/join_common.hpp @@ -147,8 +147,8 @@ static void BM_join(state_type& state, Join JoinFunc) // Benchmark conditional join if constexpr (std::is_same_v and is_conditional) { // Common column references. - const auto col_ref_left_0 = cudf::ast::column_reference(0); - const auto col_ref_right_0 = cudf::ast::column_reference(0, cudf::ast::table_reference::RIGHT); + auto const col_ref_left_0 = cudf::ast::column_reference(0); + auto const col_ref_right_0 = cudf::ast::column_reference(0, cudf::ast::table_reference::RIGHT); auto left_zero_eq_right_zero = cudf::ast::operation(cudf::ast::ast_operator::EQUAL, col_ref_left_0, col_ref_right_0); diff --git a/cpp/benchmarks/reduction/scan.cpp b/cpp/benchmarks/reduction/scan.cpp index 05c15a4fcb5..7a0d3f9515f 100644 --- a/cpp/benchmarks/reduction/scan.cpp +++ b/cpp/benchmarks/reduction/scan.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ static void BM_reduction_scan(benchmark::State& state, bool include_nulls) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; auto const dtype = cudf::type_to_id(); - auto const table = create_random_table({dtype}, 1, row_count{n_rows}); + auto const table = create_random_table({dtype}, row_count{n_rows}); if (!include_nulls) table->get_column(0).set_null_mask(rmm::device_buffer{}, 0); cudf::column_view input(table->view().column(0)); diff --git a/cpp/benchmarks/replace/clamp.cpp b/cpp/benchmarks/replace/clamp.cpp index dd8b06227bc..d3a7415a478 100644 --- a/cpp/benchmarks/replace/clamp.cpp +++ b/cpp/benchmarks/replace/clamp.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ static void BM_clamp(benchmark::State& state, bool include_nulls) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; auto const dtype = cudf::type_to_id(); - auto const table = create_random_table({dtype}, 1, row_count{n_rows}); + auto const table = create_random_table({dtype}, row_count{n_rows}); if (!include_nulls) { table->get_column(0).set_null_mask(rmm::device_buffer{}, 0); } cudf::column_view input(table->view().column(0)); diff --git a/cpp/benchmarks/replace/nans.cpp b/cpp/benchmarks/replace/nans.cpp index 3faf217956b..e1b05bbc337 100644 --- a/cpp/benchmarks/replace/nans.cpp +++ b/cpp/benchmarks/replace/nans.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ static void BM_replace_nans(benchmark::State& state, bool include_nulls) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; auto const dtype = cudf::type_to_id(); - auto const table = create_random_table({dtype}, 1, row_count{n_rows}); + auto const table = create_random_table({dtype}, row_count{n_rows}); if (!include_nulls) { table->get_column(0).set_null_mask(rmm::device_buffer{}, 0); } cudf::column_view input(table->view().column(0)); diff --git a/cpp/benchmarks/search/search.cpp b/cpp/benchmarks/search/search.cpp index c3529c7e79c..0bccbbaff54 100644 --- a/cpp/benchmarks/search/search.cpp +++ b/cpp/benchmarks/search/search.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,63 +14,47 @@ * limitations under the License. */ +#include +#include +#include + +#include #include +#include +#include #include #include #include -#include - #include -#include -#include -#include - class Search : public cudf::benchmark { }; -auto make_validity_iter() -{ - static constexpr int r_min = 1; - static constexpr int r_max = 10; - - cudf::test::UniformRandomGenerator rand_gen(r_min, r_max); - uint8_t mod_base = rand_gen.generate(); - return cudf::detail::make_counting_transform_iterator( - 0, [mod_base](auto row) { return (row % mod_base) > 0; }); -} - void BM_column(benchmark::State& state, bool nulls) { - const cudf::size_type column_size{(cudf::size_type)state.range(0)}; - const cudf::size_type values_size = column_size; - - auto col_data_it = cudf::detail::make_counting_transform_iterator( - 0, [=](cudf::size_type row) { return static_cast(row); }); - auto val_data_it = cudf::detail::make_counting_transform_iterator( - 0, [=](cudf::size_type row) { return static_cast(values_size - row); }); - - auto column = [&]() { - return nulls ? cudf::test::fixed_width_column_wrapper( - col_data_it, col_data_it + column_size, make_validity_iter()) - : cudf::test::fixed_width_column_wrapper(col_data_it, - col_data_it + column_size); - }(); - auto values = [&]() { - return nulls ? cudf::test::fixed_width_column_wrapper( - val_data_it, val_data_it + values_size, make_validity_iter()) - : cudf::test::fixed_width_column_wrapper(val_data_it, - val_data_it + values_size); - }(); - - auto data_table = cudf::sort(cudf::table_view({column})); + auto const column_size{static_cast(state.range(0))}; + auto const values_size = column_size; + + auto init_data = cudf::make_fixed_width_scalar(static_cast(0)); + auto init_value = cudf::make_fixed_width_scalar(static_cast(values_size)); + auto step = cudf::make_fixed_width_scalar(static_cast(-1)); + auto column = cudf::sequence(column_size, *init_data); + auto values = cudf::sequence(values_size, *init_value, *step); + if (nulls) { + auto [column_null_mask, column_null_count] = create_random_null_mask(column->size(), 0.1, 1); + column->set_null_mask(std::move(column_null_mask), column_null_count); + auto [values_null_mask, values_null_count] = create_random_null_mask(values->size(), 0.1, 2); + values->set_null_mask(std::move(values_null_mask), values_null_count); + } + + auto data_table = cudf::sort(cudf::table_view({*column})); for (auto _ : state) { cuda_event_timer timer(state, true); auto col = cudf::upper_bound(data_table->view(), - cudf::table_view({values}), + cudf::table_view({*values}), {cudf::order::ASCENDING}, {cudf::null_order::BEFORE}); } @@ -93,9 +77,9 @@ void BM_table(benchmark::State& state) { using wrapper = cudf::test::fixed_width_column_wrapper; - const cudf::size_type num_columns{(cudf::size_type)state.range(0)}; - const cudf::size_type column_size{(cudf::size_type)state.range(1)}; - const cudf::size_type values_size = column_size; + auto const num_columns{static_cast(state.range(0))}; + auto const column_size{static_cast(state.range(1))}; + auto const values_size = column_size; auto make_table = [&](cudf::size_type col_size) { cudf::test::UniformRandomGenerator random_gen(0, 100); @@ -142,30 +126,24 @@ BENCHMARK_REGISTER_F(Search, Table) void BM_contains(benchmark::State& state, bool nulls) { - const cudf::size_type column_size{(cudf::size_type)state.range(0)}; - const cudf::size_type values_size = column_size; - - auto col_data_it = cudf::detail::make_counting_transform_iterator( - 0, [=](cudf::size_type row) { return static_cast(row); }); - auto val_data_it = cudf::detail::make_counting_transform_iterator( - 0, [=](cudf::size_type row) { return static_cast(values_size - row); }); - - auto column = [&]() { - return nulls ? cudf::test::fixed_width_column_wrapper( - col_data_it, col_data_it + column_size, make_validity_iter()) - : cudf::test::fixed_width_column_wrapper(col_data_it, - col_data_it + column_size); - }(); - auto values = [&]() { - return nulls ? cudf::test::fixed_width_column_wrapper( - val_data_it, val_data_it + values_size, make_validity_iter()) - : cudf::test::fixed_width_column_wrapper(val_data_it, - val_data_it + values_size); - }(); + auto const column_size{static_cast(state.range(0))}; + auto const values_size = column_size; + + auto init_data = cudf::make_fixed_width_scalar(static_cast(0)); + auto init_value = cudf::make_fixed_width_scalar(static_cast(values_size)); + auto step = cudf::make_fixed_width_scalar(static_cast(-1)); + auto column = cudf::sequence(column_size, *init_data); + auto values = cudf::sequence(values_size, *init_value, *step); + if (nulls) { + auto [column_null_mask, column_null_count] = create_random_null_mask(column->size(), 0.1, 1); + column->set_null_mask(std::move(column_null_mask), column_null_count); + auto [values_null_mask, values_null_count] = create_random_null_mask(values->size(), 0.1, 2); + values->set_null_mask(std::move(values_null_mask), values_null_count); + } for (auto _ : state) { cuda_event_timer timer(state, true); - auto col = cudf::contains(column, values); + auto col = cudf::contains(*column, *values); } } diff --git a/cpp/benchmarks/sort/sort_strings.cpp b/cpp/benchmarks/sort/sort_strings.cpp index 8adeef21a79..30a7aee043b 100644 --- a/cpp/benchmarks/sort/sort_strings.cpp +++ b/cpp/benchmarks/sort/sort_strings.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ static void BM_sort(benchmark::State& state) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; - auto const table = create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}); for (auto _ : state) { cuda_event_timer raii(state, true, rmm::cuda_stream_default); diff --git a/cpp/benchmarks/string/case.cpp b/cpp/benchmarks/string/case.cpp index 0f1653af2c6..0d74d0a6b7c 100644 --- a/cpp/benchmarks/string/case.cpp +++ b/cpp/benchmarks/string/case.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ class StringCase : public cudf::benchmark { static void BM_case(benchmark::State& state) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; - auto const table = create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}); cudf::strings_column_view input(table->view().column(0)); for (auto _ : state) { diff --git a/cpp/benchmarks/string/combine.cpp b/cpp/benchmarks/string/combine.cpp index 8983646b6f1..a0cfcd15fe8 100644 --- a/cpp/benchmarks/string/combine.cpp +++ b/cpp/benchmarks/string/combine.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,8 +36,8 @@ static void BM_combine(benchmark::State& state) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 2, row_count{n_rows}, table_profile); + auto const table = create_random_table( + {cudf::type_id::STRING, cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input1(table->view().column(0)); cudf::strings_column_view input2(table->view().column(1)); cudf::string_scalar separator("+"); diff --git a/cpp/benchmarks/string/contains.cpp b/cpp/benchmarks/string/contains.cpp index fbcfabb4532..8c536372359 100644 --- a/cpp/benchmarks/string/contains.cpp +++ b/cpp/benchmarks/string/contains.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ enum contains_type { contains, count, findall }; static void BM_contains(benchmark::State& state, contains_type ct) { cudf::size_type const n_rows{(cudf::size_type)state.range(0)}; - auto const table = create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}); cudf::strings_column_view input(table->view().column(0)); for (auto _ : state) { diff --git a/cpp/benchmarks/string/convert_datetime.cpp b/cpp/benchmarks/string/convert_datetime.cpp index af51b504ee8..3782fea1e36 100644 --- a/cpp/benchmarks/string/convert_datetime.cpp +++ b/cpp/benchmarks/string/convert_datetime.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ void BM_convert_datetime(benchmark::State& state, direction dir) auto const n_rows = static_cast(state.range(0)); auto const data_type = cudf::data_type(cudf::type_to_id()); - auto const table = create_random_table({data_type.id()}, 1, row_count{n_rows}); + auto const table = create_random_table({data_type.id()}, row_count{n_rows}); cudf::column_view input(table->view().column(0)); auto source = dir == direction::to ? cudf::strings::from_timestamps(input, "%Y-%m-%d %H:%M:%S") diff --git a/cpp/benchmarks/string/convert_fixed_point.cpp b/cpp/benchmarks/string/convert_fixed_point.cpp index 5c050592c7b..05b87906eca 100644 --- a/cpp/benchmarks/string/convert_fixed_point.cpp +++ b/cpp/benchmarks/string/convert_fixed_point.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ namespace { std::unique_ptr get_strings_column(cudf::size_type rows) { std::unique_ptr result = - create_random_table({cudf::type_id::FLOAT32}, 1, row_count{static_cast(rows)}); + create_random_table({cudf::type_id::FLOAT32}, row_count{static_cast(rows)}); return cudf::strings::from_floats(result->release().front()->view()); } diff --git a/cpp/benchmarks/string/convert_numerics.cpp b/cpp/benchmarks/string/convert_numerics.cpp index 02ccb17e74a..71a23c76829 100644 --- a/cpp/benchmarks/string/convert_numerics.cpp +++ b/cpp/benchmarks/string/convert_numerics.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ template std::unique_ptr get_numerics_column(cudf::size_type rows) { std::unique_ptr result = - create_random_table({cudf::type_to_id()}, 1, row_count{rows}); + create_random_table({cudf::type_to_id()}, row_count{rows}); return std::move(result->release().front()); } diff --git a/cpp/benchmarks/string/copy.cu b/cpp/benchmarks/string/copy.cu index 2f064e71c44..00eb818256c 100644 --- a/cpp/benchmarks/string/copy.cu +++ b/cpp/benchmarks/string/copy.cu @@ -40,9 +40,9 @@ static void BM_copy(benchmark::State& state, copy_type ct) cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); auto const source = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); auto const target = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); // scatter indices auto index_map_col = make_numeric_column( diff --git a/cpp/benchmarks/string/factory.cu b/cpp/benchmarks/string/factory.cu index 2a88def1871..47356af129e 100644 --- a/cpp/benchmarks/string/factory.cu +++ b/cpp/benchmarks/string/factory.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,9 +53,8 @@ static void BM_factory(benchmark::State& state) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); - auto d_column = cudf::column_device_view::create(table->view().column(0)); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); + auto d_column = cudf::column_device_view::create(table->view().column(0)); rmm::device_uvector pairs(d_column->size(), rmm::cuda_stream_default); thrust::transform(thrust::device, d_column->pair_begin(), diff --git a/cpp/benchmarks/string/filter.cpp b/cpp/benchmarks/string/filter.cpp index fb030c2ccc2..b39cf25bc91 100644 --- a/cpp/benchmarks/string/filter.cpp +++ b/cpp/benchmarks/string/filter.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,8 +41,7 @@ static void BM_filter_chars(benchmark::State& state, FilterAPI api) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); auto const types = cudf::strings::string_character_types::SPACE; diff --git a/cpp/benchmarks/string/find.cpp b/cpp/benchmarks/string/find.cpp index 167e9bc1348..55eb52c9b30 100644 --- a/cpp/benchmarks/string/find.cpp +++ b/cpp/benchmarks/string/find.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,8 +39,7 @@ static void BM_find_scalar(benchmark::State& state, FindAPI find_api) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); cudf::string_scalar target("+"); cudf::test::strings_column_wrapper targets({"+", "-"}); diff --git a/cpp/benchmarks/string/repeat_strings.cpp b/cpp/benchmarks/string/repeat_strings.cpp index 86b8525023f..9044db18522 100644 --- a/cpp/benchmarks/string/repeat_strings.cpp +++ b/cpp/benchmarks/string/repeat_strings.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ static std::unique_ptr create_data_table(cudf::size_type n_cols, cudf::type_id::INT32, distribution_id::NORMAL, min_repeat_times, max_repeat_times); } - return create_random_table(dtype_ids, n_cols, row_count{n_rows}, table_profile); + return create_random_table(dtype_ids, row_count{n_rows}, table_profile); } static void BM_repeat_strings_scalar_times(benchmark::State& state) diff --git a/cpp/benchmarks/string/replace.cpp b/cpp/benchmarks/string/replace.cpp index 9be2e3a8627..0a3607c64f0 100644 --- a/cpp/benchmarks/string/replace.cpp +++ b/cpp/benchmarks/string/replace.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,8 +40,7 @@ static void BM_replace(benchmark::State& state, replace_type rt) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); cudf::string_scalar target("+"); cudf::string_scalar repl(""); diff --git a/cpp/benchmarks/string/replace_re.cpp b/cpp/benchmarks/string/replace_re.cpp index c106953bf69..b9d04630837 100644 --- a/cpp/benchmarks/string/replace_re.cpp +++ b/cpp/benchmarks/string/replace_re.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,8 +37,7 @@ static void BM_replace(benchmark::State& state, replace_type rt) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); cudf::test::strings_column_wrapper repls({"#", ""}); diff --git a/cpp/benchmarks/string/split.cpp b/cpp/benchmarks/string/split.cpp index fc879d1d0eb..ad25cfe54de 100644 --- a/cpp/benchmarks/string/split.cpp +++ b/cpp/benchmarks/string/split.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,8 +38,7 @@ static void BM_split(benchmark::State& state, split_type rt) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); cudf::string_scalar target("+"); diff --git a/cpp/benchmarks/string/substring.cpp b/cpp/benchmarks/string/substring.cpp index 8864fffc40b..2195cc56515 100644 --- a/cpp/benchmarks/string/substring.cpp +++ b/cpp/benchmarks/string/substring.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,8 +43,7 @@ static void BM_substring(benchmark::State& state, substring_type rt) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); auto starts_itr = thrust::constant_iterator(1); auto stops_itr = thrust::constant_iterator(max_str_length / 2); diff --git a/cpp/benchmarks/string/translate.cpp b/cpp/benchmarks/string/translate.cpp index 98688fa14fc..38c6ff9c701 100644 --- a/cpp/benchmarks/string/translate.cpp +++ b/cpp/benchmarks/string/translate.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,8 +41,7 @@ static void BM_translate(benchmark::State& state, int entry_count) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); std::vector entries(entry_count); diff --git a/cpp/benchmarks/text/ngrams.cpp b/cpp/benchmarks/text/ngrams.cpp index 7c39ebbb1bb..157c27ae48a 100644 --- a/cpp/benchmarks/text/ngrams.cpp +++ b/cpp/benchmarks/text/ngrams.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,8 +38,7 @@ static void BM_ngrams(benchmark::State& state, ngrams_type nt) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); for (auto _ : state) { diff --git a/cpp/benchmarks/text/normalize.cpp b/cpp/benchmarks/text/normalize.cpp index ac8e92b3376..2cc083f4ae8 100644 --- a/cpp/benchmarks/text/normalize.cpp +++ b/cpp/benchmarks/text/normalize.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,8 +36,7 @@ static void BM_normalize(benchmark::State& state, bool to_lower) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); for (auto _ : state) { diff --git a/cpp/benchmarks/text/normalize_spaces.cpp b/cpp/benchmarks/text/normalize_spaces.cpp index 34749b579b9..3bd636d4aa9 100644 --- a/cpp/benchmarks/text/normalize_spaces.cpp +++ b/cpp/benchmarks/text/normalize_spaces.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,8 +37,7 @@ static void BM_normalize(benchmark::State& state) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); for (auto _ : state) { diff --git a/cpp/benchmarks/text/tokenize.cpp b/cpp/benchmarks/text/tokenize.cpp index fa3f816db59..4cb9c9e5271 100644 --- a/cpp/benchmarks/text/tokenize.cpp +++ b/cpp/benchmarks/text/tokenize.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,8 +40,7 @@ static void BM_tokenize(benchmark::State& state, tokenize_type tt) data_profile table_profile; table_profile.set_distribution_params( cudf::type_id::STRING, distribution_id::NORMAL, 0, max_str_length); - auto const table = - create_random_table({cudf::type_id::STRING}, 1, row_count{n_rows}, table_profile); + auto const table = create_random_table({cudf::type_id::STRING}, row_count{n_rows}, table_profile); cudf::strings_column_view input(table->view().column(0)); cudf::test::strings_column_wrapper delimiters({" ", "+", "-"}); From 044922d1bcc9dc17dd3da05dc1d24820a514a474 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 25 Feb 2022 07:43:25 -0600 Subject: [PATCH 06/11] Add cleanup of python artifacts (#10355) This PR removes `__pycache__`, `.so` & `.pyc` files. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/10355 --- build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.sh b/build.sh index 765a1b5325f..5fb957c80a6 100755 --- a/build.sh +++ b/build.sh @@ -168,6 +168,10 @@ if hasArg clean; then rmdir ${bd} || true fi done + + # Cleaning up python artifacts + find ${REPODIR}/python/ | grep -E "(__pycache__|\.pyc|\.pyo|\.so$)" | xargs rm -rf + fi From 3f175ced5728f8e5a20d8cfc5dda21286d62d3df Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 25 Feb 2022 08:49:46 -0600 Subject: [PATCH 07/11] Fix warnings in test_categorical.py. (#10354) This PR catches or silences warnings in `test_categorical.py`. (I am working through one test file at a time so we can enable `-Werr` in the future.) Most of the warnings come from deprecated `inplace` arguments to pandas' categorical functions. The `inplace` argument will be removed in pandas 2.0. Until then, we should just hide the warning. Additionally, I refactored some `inplace` behavior to make the expected behavior of the test clearer. Authors: - Bradley Dice (https://github.com/bdice) Approvers: - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/10354 --- python/cudf/cudf/core/_compat.py | 3 +- python/cudf/cudf/tests/test_categorical.py | 199 ++++++++++++--------- 2 files changed, 119 insertions(+), 83 deletions(-) diff --git a/python/cudf/cudf/core/_compat.py b/python/cudf/cudf/core/_compat.py index 2cf579ce3f1..70162c7afc6 100644 --- a/python/cudf/cudf/core/_compat.py +++ b/python/cudf/cudf/core/_compat.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. import pandas as pd from packaging import version @@ -9,4 +9,5 @@ PANDAS_GE_120 = PANDAS_VERSION >= version.parse("1.2") PANDAS_LE_122 = PANDAS_VERSION <= version.parse("1.2.2") PANDAS_GE_130 = PANDAS_VERSION >= version.parse("1.3.0") +PANDAS_GE_134 = PANDAS_VERSION >= version.parse("1.3.4") PANDAS_LT_140 = PANDAS_VERSION < version.parse("1.4.0") diff --git a/python/cudf/cudf/tests/test_categorical.py b/python/cudf/cudf/tests/test_categorical.py index bc3ae721554..19a5cd4a49d 100644 --- a/python/cudf/cudf/tests/test_categorical.py +++ b/python/cudf/cudf/tests/test_categorical.py @@ -1,14 +1,17 @@ -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. import operator import string +import warnings +from contextlib import contextmanager +from textwrap import dedent import numpy as np import pandas as pd import pytest import cudf -from cudf.core._compat import PANDAS_GE_110 +from cudf.core._compat import PANDAS_GE_110, PANDAS_GE_134 from cudf.testing._utils import ( NUMERIC_TYPES, assert_eq, @@ -16,6 +19,30 @@ ) +@contextmanager +def _hide_deprecated_pandas_categorical_inplace_warnings(function_name): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + ( + "The `inplace` parameter in " + f"pandas.Categorical.{function_name} is deprecated and will " + "be removed in a future version." + ), + category=FutureWarning, + ) + yield + + +@contextmanager +def _hide_cudf_safe_casting_warning(): + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", "Can't safely cast column", category=UserWarning, + ) + yield + + @pytest.fixture def pd_str_cat(): categories = list("abc") @@ -51,9 +78,8 @@ def test_categorical_basic(): assert_eq(cat.codes, cudf_cat.codes.to_numpy()) +@pytest.mark.skipif(not PANDAS_GE_110, reason="requires pandas>=1.1.0") def test_categorical_integer(): - if not PANDAS_GE_110: - pytest.xfail(reason="pandas >=1.1 required") cat = pd.Categorical(["a", "_", "_", "c", "a"], categories=["a", "b", "c"]) pdsr = pd.Series(cat) sr = cudf.Series(cat) @@ -67,17 +93,17 @@ def test_categorical_integer(): sr.cat.codes.astype(pdsr.cat.codes.dtype).fillna(-1).to_numpy(), ) - string = str(sr) - expect_str = """ -0 a -1 -2 -3 c -4 a -dtype: category -Categories (3, object): ['a', 'b', 'c'] -""" - assert string.split() == expect_str.split() + expect_str = dedent( + """\ + 0 a + 1 + 2 + 3 c + 4 a + dtype: category + Categories (3, object): ['a', 'b', 'c']""" + ) + assert str(sr) == expect_str def test_categorical_compare_unordered(): @@ -152,23 +178,9 @@ def test_categorical_binary_add(): rfunc=operator.add, lfunc_args_and_kwargs=([pdsr, pdsr],), rfunc_args_and_kwargs=([sr, sr],), - expected_error_message="Series of dtype `category` cannot perform " - "the operation: add", - ) - - -def test_categorical_unary_ceil(): - cat = pd.Categorical(["a", "a", "b", "c", "a"], categories=["a", "b", "c"]) - pdsr = pd.Series(cat) - sr = cudf.Series(cat) - - assert_exceptions_equal( - lfunc=getattr, - rfunc=sr.ceil, - lfunc_args_and_kwargs=([pdsr, "ceil"],), - check_exception_type=False, - expected_error_message="Series of dtype `category` cannot " - "perform the operation: ceil", + expected_error_message=( + "Series of dtype `category` cannot perform the operation: add" + ), ) @@ -238,26 +250,25 @@ def test_cat_series_binop_error(): df["a"] = pd.Categorical(list("aababcabbc"), categories=list("abc")) df["b"] = np.arange(len(df)) - dfa = df["a"] - dfb = df["b"] + pdf = df.to_pandas() - # lhs is a categorical + # lhs is categorical assert_exceptions_equal( lfunc=operator.add, rfunc=operator.add, - lfunc_args_and_kwargs=([dfa, dfb],), - rfunc_args_and_kwargs=([dfa, dfb],), - check_exception_type=False, - expected_error_message="Series of dtype `category` cannot " - "perform the operation: add", + lfunc_args_and_kwargs=([pdf["a"], pdf["b"]],), + rfunc_args_and_kwargs=([df["a"], df["b"]],), + expected_error_message=( + "Series of dtype `category` cannot perform the operation: add" + ), ) - # if lhs is a numerical + + # lhs is numerical assert_exceptions_equal( lfunc=operator.add, rfunc=operator.add, - lfunc_args_and_kwargs=([dfb, dfa],), - rfunc_args_and_kwargs=([dfb, dfa],), - check_exception_type=False, + lfunc_args_and_kwargs=([pdf["b"], pdf["a"]],), + rfunc_args_and_kwargs=([df["b"], df["a"]],), expected_error_message="'add' operator not supported", ) @@ -367,8 +378,9 @@ def test_categorical_as_ordered(pd_str_cat, inplace): pd_sr_1 = pd_sr.cat.as_ordered(inplace=inplace) cd_sr_1 = cd_sr.cat.as_ordered(inplace=inplace) - pd_sr_1 = pd_sr if pd_sr_1 is None else pd_sr_1 - cd_sr_1 = cd_sr if cd_sr_1 is None else cd_sr_1 + if inplace: + pd_sr_1 = pd_sr + cd_sr_1 = cd_sr assert cd_sr_1.cat.ordered is True assert cd_sr_1.cat.ordered == pd_sr_1.cat.ordered @@ -386,8 +398,9 @@ def test_categorical_as_unordered(pd_str_cat, inplace): pd_sr_1 = pd_sr.cat.as_unordered(inplace=inplace) cd_sr_1 = cd_sr.cat.as_unordered(inplace=inplace) - pd_sr_1 = pd_sr if pd_sr_1 is None else pd_sr_1 - cd_sr_1 = cd_sr if cd_sr_1 is None else cd_sr_1 + if inplace: + pd_sr_1 = pd_sr + cd_sr_1 = cd_sr assert cd_sr_1.cat.ordered is False assert cd_sr_1.cat.ordered == pd_sr_1.cat.ordered @@ -401,8 +414,9 @@ def test_categorical_as_unordered(pd_str_cat, inplace): [ pytest.param( True, - marks=pytest.mark.xfail( - reason="https://github.com/pandas-dev/pandas/issues/43232" + marks=pytest.mark.skipif( + not PANDAS_GE_134, + reason="https://github.com/pandas-dev/pandas/issues/43232", ), ), False, @@ -421,10 +435,14 @@ def test_categorical_reorder_categories( kwargs = dict(ordered=to_ordered, inplace=inplace) - pd_sr_1 = pd_sr.cat.reorder_categories(list("cba"), **kwargs) + with _hide_deprecated_pandas_categorical_inplace_warnings( + "reorder_categories" + ): + pd_sr_1 = pd_sr.cat.reorder_categories(list("cba"), **kwargs) cd_sr_1 = cd_sr.cat.reorder_categories(list("cba"), **kwargs) - pd_sr_1 = pd_sr if pd_sr_1 is None else pd_sr_1 - cd_sr_1 = cd_sr if cd_sr_1 is None else cd_sr_1 + if inplace: + pd_sr_1 = pd_sr + cd_sr_1 = cd_sr assert_eq(pd_sr_1, cd_sr_1) @@ -436,8 +454,9 @@ def test_categorical_reorder_categories( [ pytest.param( True, - marks=pytest.mark.xfail( - reason="https://github.com/pandas-dev/pandas/issues/43232" + marks=pytest.mark.skipif( + not PANDAS_GE_134, + reason="https://github.com/pandas-dev/pandas/issues/43232", ), ), False, @@ -452,10 +471,14 @@ def test_categorical_add_categories(pd_str_cat, inplace): assert str(pd_sr) == str(cd_sr) - pd_sr_1 = pd_sr.cat.add_categories(["d"], inplace=inplace) + with _hide_deprecated_pandas_categorical_inplace_warnings( + "add_categories" + ): + pd_sr_1 = pd_sr.cat.add_categories(["d"], inplace=inplace) cd_sr_1 = cd_sr.cat.add_categories(["d"], inplace=inplace) - pd_sr_1 = pd_sr if pd_sr_1 is None else pd_sr_1 - cd_sr_1 = cd_sr if cd_sr_1 is None else cd_sr_1 + if inplace: + pd_sr_1 = pd_sr + cd_sr_1 = cd_sr assert "d" in pd_sr_1.cat.categories.to_list() assert "d" in cd_sr_1.cat.categories.to_pandas().to_list() @@ -468,8 +491,9 @@ def test_categorical_add_categories(pd_str_cat, inplace): [ pytest.param( True, - marks=pytest.mark.xfail( - reason="https://github.com/pandas-dev/pandas/issues/43232" + marks=pytest.mark.skipif( + not PANDAS_GE_134, + reason="https://github.com/pandas-dev/pandas/issues/43232", ), ), False, @@ -484,10 +508,14 @@ def test_categorical_remove_categories(pd_str_cat, inplace): assert str(pd_sr) == str(cd_sr) - pd_sr_1 = pd_sr.cat.remove_categories(["a"], inplace=inplace) + with _hide_deprecated_pandas_categorical_inplace_warnings( + "remove_categories" + ): + pd_sr_1 = pd_sr.cat.remove_categories(["a"], inplace=inplace) cd_sr_1 = cd_sr.cat.remove_categories(["a"], inplace=inplace) - pd_sr_1 = pd_sr if pd_sr_1 is None else pd_sr_1 - cd_sr_1 = cd_sr if cd_sr_1 is None else cd_sr_1 + if inplace: + pd_sr_1 = pd_sr + cd_sr_1 = cd_sr assert "a" not in pd_sr_1.cat.categories.to_list() assert "a" not in cd_sr_1.cat.categories.to_pandas().to_list() @@ -495,13 +523,16 @@ def test_categorical_remove_categories(pd_str_cat, inplace): assert_eq(pd_sr_1, cd_sr_1) # test using ordered operators - assert_exceptions_equal( - lfunc=cd_sr.to_pandas().cat.remove_categories, - rfunc=cd_sr.cat.remove_categories, - lfunc_args_and_kwargs=([["a", "d"]], {"inplace": inplace}), - rfunc_args_and_kwargs=([["a", "d"]], {"inplace": inplace}), - expected_error_message="removals must all be in old categories", - ) + with _hide_deprecated_pandas_categorical_inplace_warnings( + "remove_categories" + ): + assert_exceptions_equal( + lfunc=cd_sr.to_pandas().cat.remove_categories, + rfunc=cd_sr.cat.remove_categories, + lfunc_args_and_kwargs=([["a", "d"]], {"inplace": inplace}), + rfunc_args_and_kwargs=([["a", "d"]], {"inplace": inplace}), + expected_error_message="removals must all be in old categories", + ) def test_categorical_dataframe_slice_copy(): @@ -583,19 +614,21 @@ def test_categorical_set_categories_categoricals(data, new_categories): pd_data = data.copy().astype("category") gd_data = cudf.from_pandas(pd_data) - assert_eq( - pd_data.cat.set_categories(new_categories=new_categories), - gd_data.cat.set_categories(new_categories=new_categories), - ) + expected = pd_data.cat.set_categories(new_categories=new_categories) + with _hide_cudf_safe_casting_warning(): + actual = gd_data.cat.set_categories(new_categories=new_categories) - assert_eq( - pd_data.cat.set_categories( - new_categories=pd.Series(new_categories, dtype="category") - ), - gd_data.cat.set_categories( - new_categories=cudf.Series(new_categories, dtype="category") - ), + assert_eq(expected, actual) + + expected = pd_data.cat.set_categories( + new_categories=pd.Series(new_categories, dtype="category") ) + with _hide_cudf_safe_casting_warning(): + actual = gd_data.cat.set_categories( + new_categories=cudf.Series(new_categories, dtype="category") + ) + + assert_eq(expected, actual) @pytest.mark.parametrize( @@ -703,7 +736,9 @@ def test_add_categories(data, add): gds = cudf.Series(data, dtype="category") expected = pds.cat.add_categories(add) - actual = gds.cat.add_categories(add) + with _hide_cudf_safe_casting_warning(): + actual = gds.cat.add_categories(add) + assert_eq( expected.cat.codes, actual.cat.codes.astype(expected.cat.codes.dtype) ) From e0af727a091c3ee98a873193ad3303abb4b3f240 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Feb 2022 11:14:52 -0800 Subject: [PATCH 08/11] Refactor array_ufunc for Index and unify across all classes (#10346) This PR builds on #10217 and #10287 to bring full ufunc support for Index types, expanding well beyond the small set previously supported in the `cudf.core.ops` namespace. By using most of the machinery introduced for IndexedFrame in the prior two PRs we avoid duplicating much logic so that all ufunc dispatches flow through a relatively standard path of known methods prior to a common cupy dispatch. With this change we are also able to deprecate the various ufunc operations defined in cudf/core/ops.py that exist only for this purpose as well as a number of Frame methods that are not defined for the corresponding pandas types. Users of those APIs are recommended to calling the corresponding numpy/cupy ufuncs instead to leverage the new dispatch. This PR also fixes a bug where index binary operations that output booleans would previously return instances of GenericIndex, whereas those pandas operations would return numpy arrays. cudf now returns cupy arrays in those cases. Resolves #9083. Contributes to #9038. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - GALI PREM SAGAR (https://github.com/galipremsagar) URL: https://github.com/rapidsai/cudf/pull/10346 --- python/cudf/cudf/core/_base_index.py | 8 - python/cudf/cudf/core/dataframe.py | 55 +++--- python/cudf/cudf/core/frame.py | 170 ++++++++++++++++- python/cudf/cudf/core/index.py | 55 +++++- python/cudf/cudf/core/indexed_frame.py | 186 ++++++------------- python/cudf/cudf/core/ops.py | 82 +++++++- python/cudf/cudf/core/series.py | 51 ++--- python/cudf/cudf/core/single_column_frame.py | 16 +- python/cudf/cudf/tests/test_array_ufunc.py | 89 ++++++++- python/cudf/cudf/tests/test_binops.py | 6 +- python/cudf/cudf/tests/test_dataframe.py | 8 - python/cudf/cudf/tests/test_pickling.py | 6 +- 12 files changed, 498 insertions(+), 234 deletions(-) diff --git a/python/cudf/cudf/core/_base_index.py b/python/cudf/cudf/core/_base_index.py index 16fb0cf99c1..b7b61e4d332 100644 --- a/python/cudf/cudf/core/_base_index.py +++ b/python/cudf/cudf/core/_base_index.py @@ -41,14 +41,6 @@ class BaseIndex(Serializable): _accessors: Set[Any] = set() _data: ColumnAccessor - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - - if method == "__call__" and hasattr(cudf, ufunc.__name__): - func = getattr(cudf, ufunc.__name__) - return func(*inputs) - else: - return NotImplemented - @cached_property def _values(self) -> ColumnBase: raise NotImplementedError diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 6b5f3809c98..4062f811975 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -11,7 +11,17 @@ import warnings from collections import defaultdict from collections.abc import Iterable, Sequence -from typing import Any, MutableMapping, Optional, Set, TypeVar +from typing import ( + Any, + Dict, + MutableMapping, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, +) import cupy import numpy as np @@ -44,6 +54,7 @@ from cudf.core.abc import Serializable from cudf.core.column import ( CategoricalColumn, + ColumnBase, as_column, build_categorical_column, build_column, @@ -1909,7 +1920,7 @@ def _get_columns_by_label(self, labels, downcast=False): ) return out - def _prep_for_binop( + def _make_operands_and_index_for_binop( self, other: Any, fn: str, @@ -1918,7 +1929,13 @@ def _prep_for_binop( can_reindex: bool = False, *args, **kwargs, - ): + ) -> Tuple[ + Union[ + Dict[Optional[str], Tuple[ColumnBase, Any, bool, Any]], + Type[NotImplemented], + ], + Optional[BaseIndex], + ]: lhs, rhs = self, other if _is_scalar_or_zero_d_array(rhs): @@ -1999,28 +2016,6 @@ def _prep_for_binop( return operands, lhs._index - @annotate("DATAFRAME_BINARYOP", color="blue", domain="cudf_python") - def _binaryop( - self, - other: Any, - fn: str, - fill_value: Any = None, - reflect: bool = False, - can_reindex: bool = False, - *args, - **kwargs, - ): - operands, out_index = self._prep_for_binop( - other, fn, fill_value, reflect, can_reindex - ) - if operands is NotImplemented: - return NotImplemented - - return self._from_data( - ColumnAccessor(type(self)._colwise_binop(operands, fn)), - index=out_index, - ) - @annotate("DATAFRAME_UPDATE", color="blue", domain="cudf_python") def update( self, @@ -2183,9 +2178,7 @@ def columns(self, columns): columns = pd.Index(range(len(self._data.columns))) is_multiindex = isinstance(columns, pd.MultiIndex) - if isinstance( - columns, (Series, cudf.Index, cudf.core.column.ColumnBase) - ): + if isinstance(columns, (Series, cudf.Index, ColumnBase)): columns = pd.Index(columns.to_numpy(), tupleize_cols=is_multiindex) elif not isinstance(columns, pd.Index): columns = pd.Index(columns, tupleize_cols=is_multiindex) @@ -6626,7 +6619,7 @@ def _setitem_with_dataframe( input_df: DataFrame, replace_df: DataFrame, input_cols: Any = None, - mask: Optional[cudf.core.column.ColumnBase] = None, + mask: Optional[ColumnBase] = None, ignore_index: bool = False, ): """ @@ -6717,9 +6710,7 @@ def _get_union_of_series_names(series_list): def _get_host_unique(array): - if isinstance( - array, (cudf.Series, cudf.Index, cudf.core.column.ColumnBase) - ): + if isinstance(array, (cudf.Series, cudf.Index, ColumnBase)): return array.unique.to_pandas() elif isinstance(array, (str, numbers.Number)): return [array] diff --git a/python/cudf/cudf/core/frame.py b/python/cudf/cudf/core/frame.py index 91d81a4c6de..0fd7848c7d1 100644 --- a/python/cudf/cudf/core/frame.py +++ b/python/cudf/cudf/core/frame.py @@ -54,6 +54,49 @@ T = TypeVar("T", bound="Frame") +# Mapping from ufuncs to the corresponding binary operators. +_ufunc_binary_operations = { + # Arithmetic binary operations. + "add": "add", + "subtract": "sub", + "multiply": "mul", + "matmul": "matmul", + "divide": "truediv", + "true_divide": "truediv", + "floor_divide": "floordiv", + "power": "pow", + "float_power": "pow", + "remainder": "mod", + "mod": "mod", + "fmod": "mod", + # Bitwise binary operations. + "bitwise_and": "and", + "bitwise_or": "or", + "bitwise_xor": "xor", + # Comparison binary operators + "greater": "gt", + "greater_equal": "ge", + "less": "lt", + "less_equal": "le", + "not_equal": "ne", + "equal": "eq", +} + +# These operators need to be mapped to their inverses when performing a +# reflected ufunc operation because no reflected version of the operators +# themselves exist. When these operators are invoked directly (not via +# __array_ufunc__) Python takes care of calling the inverse operation. +_ops_without_reflection = { + "gt": "lt", + "ge": "le", + "lt": "gt", + "le": "ge", + # ne and eq are symmetric, so they are their own inverse op + "ne": "ne", + "eq": "eq", +} + + class Frame: """A collection of Column objects with an optional index. @@ -2752,6 +2795,11 @@ def sin(self): 0.8011526357338306, 0.8939966636005579], dtype='float64') """ + warnings.warn( + "sin is deprecated and will be removed. Use numpy.sin instead", + FutureWarning, + ) + return self._unaryop("sin") @annotate("FRAME_COS", color="green", domain="cudf_python") @@ -2814,6 +2862,11 @@ def cos(self): -0.5984600690578581, -0.4480736161291701], dtype='float64') """ + warnings.warn( + "cos is deprecated and will be removed. Use numpy.cos instead", + FutureWarning, + ) + return self._unaryop("cos") @annotate("FRAME_TAN", color="green", domain="cudf_python") @@ -2876,6 +2929,11 @@ def tan(self): -1.3386902103511544, -1.995200412208242], dtype='float64') """ + warnings.warn( + "tan is deprecated and will be removed. Use numpy.tan instead", + FutureWarning, + ) + return self._unaryop("tan") @annotate("FRAME_ASIN", color="green", domain="cudf_python") @@ -2927,6 +2985,11 @@ def asin(self): 1.5707963267948966, 0.3046926540153975], dtype='float64') """ + warnings.warn( + "asin is deprecated and will be removed in the future", + FutureWarning, + ) + return self._unaryop("asin") @annotate("FRAME_ACOS", color="green", domain="cudf_python") @@ -2978,6 +3041,11 @@ def acos(self): 1.5707963267948966, 1.266103672779499], dtype='float64') """ + warnings.warn( + "acos is deprecated and will be removed. Use numpy.acos instead", + FutureWarning, + ) + result = self.copy(deep=False) for col in result._data: min_float_dtype = cudf.utils.dtypes.get_min_float_dtype( @@ -3047,6 +3115,11 @@ def atan(self): 0.2914567944778671], dtype='float64') """ + warnings.warn( + "atan is deprecated and will be removed. Use numpy.atan instead", + FutureWarning, + ) + return self._unaryop("atan") @annotate("FRAME_EXP", color="green", domain="cudf_python") @@ -3110,6 +3183,11 @@ def exp(self): 2.718281828459045, 1.0, 1.3498588075760032], dtype='float64') """ + warnings.warn( + "exp is deprecated and will be removed. Use numpy.exp instead", + FutureWarning, + ) + return self._unaryop("exp") @annotate("FRAME_LOG", color="green", domain="cudf_python") @@ -3172,6 +3250,11 @@ def log(self): Float64Index([2.302585092994046, 2.3978952727983707, 6.214608098422191], dtype='float64') """ + warnings.warn( + "log is deprecated and will be removed. Use numpy.log instead", + FutureWarning, + ) + return self._unaryop("log") @annotate("FRAME_SQRT", color="green", domain="cudf_python") @@ -3228,6 +3311,11 @@ def sqrt(self): >>> index.sqrt() Float64Index([nan, 10.0, 25.0], dtype='float64') """ + warnings.warn( + "sqrt is deprecated and will be removed. Use numpy.sqrt instead", + FutureWarning, + ) + return self._unaryop("sqrt") @annotate("FRAME_ABS", color="green", domain="cudf_python") @@ -3496,7 +3584,9 @@ def _binaryop( Frame A new instance containing the result of the operation. """ - raise NotImplementedError + raise NotImplementedError( + f"Binary operations are not supported for {self.__class__}" + ) @classmethod @annotate("FRAME_COLWISE_BINOP", color="green", domain="cudf_python") @@ -3658,6 +3748,84 @@ def _colwise_binop( return output + # For more detail on this function and how it should work, see + # https://numpy.org/doc/stable/reference/ufuncs.html + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + # We don't currently support reduction, accumulation, etc. We also + # don't support any special kwargs or higher arity ufuncs than binary. + if method != "__call__" or kwargs or ufunc.nin > 2: + return NotImplemented + + fname = ufunc.__name__ + if fname in _ufunc_binary_operations: + reflect = self is not inputs[0] + other = inputs[0] if reflect else inputs[1] + + op = _ufunc_binary_operations[fname] + if reflect and op in _ops_without_reflection: + op = _ops_without_reflection[op] + reflect = False + op = f"__{'r' if reflect else ''}{op}__" + + # Float_power returns float irrespective of the input type. + if fname == "float_power": + return getattr(self, op)(other).astype(float) + return getattr(self, op)(other) + + # Special handling for various unary operations. + if fname == "negative": + return self * -1 + if fname == "positive": + return self.copy(deep=True) + if fname == "invert": + return ~self + if fname == "absolute": + return self.abs() + if fname == "fabs": + return self.abs().astype(np.float64) + + # None is a sentinel used by subclasses to trigger cupy dispatch. + return None + + def _apply_cupy_ufunc_to_operands( + self, ufunc, cupy_func, operands, **kwargs + ): + # Note: There are some operations that may be supported by libcudf but + # are not supported by pandas APIs. In particular, libcudf binary + # operations support logical and/or operations as well as + # trigonometric, but those operations are not defined on + # pd.Series/DataFrame. For now those operations will dispatch to cupy, + # but if ufuncs are ever a bottleneck we could add special handling to + # dispatch those (or any other) functions that we could implement + # without cupy. + + mask = None + data = [{} for _ in range(ufunc.nout)] + for name, (left, right, _, _) in operands.items(): + cupy_inputs = [] + for inp in (left, right) if ufunc.nin == 2 else (left,): + if isinstance(inp, ColumnBase) and inp.has_nulls(): + new_mask = as_column(inp.nullmask) + + # TODO: This is a hackish way to perform a bitwise and + # of bitmasks. Once we expose + # cudf::detail::bitwise_and, then we can use that + # instead. + mask = new_mask if mask is None else (mask & new_mask) + + # Arbitrarily fill with zeros. For ufuncs, we assume + # that the end result propagates nulls via a bitwise + # and, so these elements are irrelevant. + inp = inp.fillna(0) + cupy_inputs.append(cupy.asarray(inp)) + + cp_output = cupy_func(*cupy_inputs, **kwargs) + if ufunc.nout == 1: + cp_output = (cp_output,) + for i, out in enumerate(cp_output): + data[i][name] = as_column(out).set_mask(mask) + return data + @annotate("FRAME_DOT", color="green", domain="cudf_python") def dot(self, other, reflect=False): """ diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 5b60e8dbd1c..5aab834d452 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -520,6 +520,11 @@ def _as_int64(self): # that are not defined directly on RangeIndex. return Int64Index._from_data(self._data) + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + return self._as_int64().__array_ufunc__( + ufunc, method, *inputs, **kwargs + ) + def __getattr__(self, key): # For methods that are not defined for RangeIndex we attempt to operate # on the corresponding integer index if possible. @@ -773,6 +778,41 @@ def __init__(self, data, **kwargs): name = kwargs.get("name") super().__init__({name: data}) + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + ret = super().__array_ufunc__(ufunc, method, *inputs, **kwargs) + + if ret is not None: + return ret + + # Attempt to dispatch all other functions to cupy. + cupy_func = getattr(cupy, ufunc.__name__) + if cupy_func: + if ufunc.nin == 2: + other = inputs[self is inputs[0]] + inputs = self._make_operands_for_binop(other) + else: + inputs = { + name: (col, None, False, None) + for name, col in self._data.items() + } + + data = self._apply_cupy_ufunc_to_operands( + ufunc, cupy_func, inputs, **kwargs + ) + + out = [_index_from_data(out) for out in data] + + # pandas returns numpy arrays when the outputs are boolean. + for i, o in enumerate(out): + # We explicitly _do not_ use isinstance here: we want only + # boolean GenericIndexes, not dtype-specific subclasses. + if type(o) is GenericIndex and o.dtype.kind == "b": + out[i] = o.values + + return out[0] if ufunc.nout == 1 else tuple(out) + + return NotImplemented + def _binaryop( self, other: T, @@ -784,11 +824,16 @@ def _binaryop( ) -> SingleColumnFrame: # Specialize binops to generate the appropriate output index type. operands = self._make_operands_for_binop(other, fill_value, reflect) - return ( - _index_from_data(data=self._colwise_binop(operands, fn),) - if operands is not NotImplemented - else NotImplemented - ) + if operands is NotImplemented: + return NotImplemented + ret = _index_from_data(self._colwise_binop(operands, fn)) + + # pandas returns numpy arrays when the outputs are boolean. We + # explicitly _do not_ use isinstance here: we want only boolean + # GenericIndexes, not dtype-specific subclasses. + if type(ret) is GenericIndex and ret.dtype.kind == "b": + return ret.values + return ret def _copy_type_metadata( self, other: Frame, include_index: bool = True diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8ff3e39519c..3ae0a838873 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -7,7 +7,7 @@ import warnings from collections import Counter, abc from functools import cached_property -from typing import Callable, Type, TypeVar +from typing import Any, Callable, Dict, Optional, Tuple, Type, TypeVar, Union from uuid import uuid4 import cupy as cp @@ -1694,150 +1694,86 @@ def last(self, offset): slice_func=lambda i: self.iloc[i:], ) - # For more detail on this function and how it should work, see - # https://numpy.org/doc/stable/reference/ufuncs.html - def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): - # We don't currently support reduction, accumulation, etc. We also - # don't support any special kwargs or higher arity ufuncs than binary. - if method != "__call__" or kwargs or ufunc.nin > 2: + def _binaryop( + self, + other: Any, + fn: str, + fill_value: Any = None, + reflect: bool = False, + can_reindex: bool = False, + *args, + **kwargs, + ): + operands, out_index = self._make_operands_and_index_for_binop( + other, fn, fill_value, reflect, can_reindex + ) + if operands is NotImplemented: return NotImplemented - # Binary operations - binary_operations = { - # Arithmetic binary operations. - "add": "add", - "subtract": "sub", - "multiply": "mul", - "matmul": "matmul", - "divide": "truediv", - "true_divide": "truediv", - "floor_divide": "floordiv", - "power": "pow", - "float_power": "pow", - "remainder": "mod", - "mod": "mod", - "fmod": "mod", - # Bitwise binary operations. - "bitwise_and": "and", - "bitwise_or": "or", - "bitwise_xor": "xor", - # Comparison binary operators - "greater": "gt", - "greater_equal": "ge", - "less": "lt", - "less_equal": "le", - "not_equal": "ne", - "equal": "eq", - } + return self._from_data( + ColumnAccessor(type(self)._colwise_binop(operands, fn)), + index=out_index, + ) + + def _make_operands_and_index_for_binop( + self, + other: Any, + fn: str, + fill_value: Any = None, + reflect: bool = False, + can_reindex: bool = False, + *args, + **kwargs, + ) -> Tuple[ + Union[ + Dict[ + Optional[str], + Tuple[cudf.core.column.ColumnBase, Any, bool, Any], + ], + Type[NotImplemented], + ], + Optional[cudf.BaseIndex], + ]: + raise NotImplementedError( + "Binary operations are not supported for {self.__class__}" + ) - # First look for methods of the class. + def __array_ufunc__(self, ufunc, method, *inputs, **kwargs): + ret = super().__array_ufunc__(ufunc, method, *inputs, **kwargs) fname = ufunc.__name__ - if fname in binary_operations: - reflect = self is not inputs[0] - other = inputs[0] if reflect else inputs[1] - - # These operators need to be mapped to their inverses when - # performing a reflected operation because no reflected version of - # the operators themselves exist. - ops_without_reflection = { - "gt": "lt", - "ge": "le", - "lt": "gt", - "le": "ge", - # ne and eq are symmetric, so they are their own inverse op - "ne": "ne", - "eq": "eq", - } - - op = binary_operations[fname] - if reflect and op in ops_without_reflection: - op = ops_without_reflection[op] - reflect = False - op = f"__{'r' if reflect else ''}{op}__" + if ret is not None: # pandas bitwise operations return bools if indexes are misaligned. - if ( - "bitwise" in fname - and isinstance(other, IndexedFrame) - and not self.index.equals(other.index) - ): - return getattr(self, op)(other).astype(bool) - # Float_power returns float irrespective of the input type. - if fname == "float_power": - return getattr(self, op)(other).astype(float) - return getattr(self, op)(other) - - # Special handling for unary operations. - if fname == "negative": - return self * -1 - if fname == "positive": - return self.copy(deep=True) - if fname == "invert": - return ~self - if fname == "absolute": - return self.abs() - if fname == "fabs": - return self.abs().astype(np.float64) - - # Note: There are some operations that may be supported by libcudf but - # are not supported by pandas APIs. In particular, libcudf binary - # operations support logical and/or operations, but those operations - # are not defined on pd.Series/DataFrame. For now those operations will - # dispatch to cupy, but if ufuncs are ever a bottleneck we could add - # special handling to dispatch those (or any other) functions that we - # could implement without cupy. + if "bitwise" in fname: + reflect = self is not inputs[0] + other = inputs[0] if reflect else inputs[1] + if isinstance(other, self.__class__) and not self.index.equals( + other.index + ): + ret = ret.astype(bool) + return ret # Attempt to dispatch all other functions to cupy. cupy_func = getattr(cp, fname) if cupy_func: - # Indices must be aligned before converting to arrays. if ufunc.nin == 2: other = inputs[self is inputs[0]] - inputs, index = self._prep_for_binop(other, fname) + inputs, index = self._make_operands_and_index_for_binop( + other, fname + ) else: + # This works for Index too inputs = { name: (col, None, False, None) for name, col in self._data.items() } index = self._index - mask = None - data = [{} for _ in range(ufunc.nout)] - for name, (left, right, _, _) in inputs.items(): - cupy_inputs = [] - # TODO: I'm jumping through multiple hoops to get the unary - # behavior to match up with the binary. I should see if there - # are better patterns to employ here. - for inp in (left, right) if ufunc.nin == 2 else (left,): - if ( - isinstance(inp, cudf.core.column.ColumnBase) - and inp.has_nulls() - ): - new_mask = cudf.core.column.as_column(inp.nullmask) - - # TODO: This is a hackish way to perform a bitwise and - # of bitmasks. Once we expose - # cudf::detail::bitwise_and, then we can use that - # instead. - mask = new_mask if mask is None else (mask & new_mask) - - # Arbitrarily fill with zeros. For ufuncs, we assume - # that the end result propagates nulls via a bitwise - # and, so these elements are irrelevant. - inp = inp.fillna(0) - cupy_inputs.append(cp.asarray(inp)) - - cp_output = cupy_func(*cupy_inputs, **kwargs) - if ufunc.nout == 1: - cp_output = (cp_output,) - for i, out in enumerate(cp_output): - data[i][name] = cudf.core.column.as_column(out).set_mask( - mask - ) - - out = tuple( - self.__class__._from_data(out, index=index) for out in data + data = self._apply_cupy_ufunc_to_operands( + ufunc, cupy_func, inputs, **kwargs ) + + out = tuple(self._from_data(out, index=index) for out in data) return out[0] if ufunc.nout == 1 else out return NotImplemented diff --git a/python/cudf/cudf/core/ops.py b/python/cudf/cudf/core/ops.py index fe9e012f406..c2a8c0e72fb 100644 --- a/python/cudf/cudf/core/ops.py +++ b/python/cudf/cudf/core/ops.py @@ -1,4 +1,5 @@ -# Copyright (c) 2019-2020, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. +import warnings from numbers import Number import numpy as np @@ -10,6 +11,10 @@ def sin(arbitrary): + warnings.warn( + "sin is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(arbitrary, Number): return np.sin(arbitrary) else: @@ -17,6 +22,10 @@ def sin(arbitrary): def cos(arbitrary): + warnings.warn( + "cos is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(arbitrary, Number): return np.cos(arbitrary) else: @@ -24,6 +33,10 @@ def cos(arbitrary): def tan(arbitrary): + warnings.warn( + "tan is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(arbitrary, Number): return np.tan(arbitrary) else: @@ -31,6 +44,11 @@ def tan(arbitrary): def arcsin(arbitrary): + warnings.warn( + "arcsin is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(arbitrary, Number): return np.arcsin(arbitrary) else: @@ -38,6 +56,11 @@ def arcsin(arbitrary): def arccos(arbitrary): + warnings.warn( + "arcsin is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(arbitrary, Number): return np.arccos(arbitrary) else: @@ -45,6 +68,11 @@ def arccos(arbitrary): def arctan(arbitrary): + warnings.warn( + "arctan is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(arbitrary, Number): return np.arctan(arbitrary) else: @@ -52,6 +80,10 @@ def arctan(arbitrary): def exp(arbitrary): + warnings.warn( + "exp is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(arbitrary, Number): return np.exp(arbitrary) else: @@ -59,6 +91,10 @@ def exp(arbitrary): def log(arbitrary): + warnings.warn( + "log is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(arbitrary, Number): return np.log(arbitrary) else: @@ -66,6 +102,10 @@ def log(arbitrary): def sqrt(arbitrary): + warnings.warn( + "sqrt is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(arbitrary, Number): return np.sqrt(arbitrary) else: @@ -73,6 +113,11 @@ def sqrt(arbitrary): def logical_not(arbitrary): + warnings.warn( + "logical_not is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(arbitrary, Number): return np.logical_not(arbitrary) else: @@ -80,6 +125,11 @@ def logical_not(arbitrary): def logical_and(lhs, rhs): + warnings.warn( + "logical_and is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.logical_and(lhs, rhs) else: @@ -87,6 +137,11 @@ def logical_and(lhs, rhs): def logical_or(lhs, rhs): + warnings.warn( + "logical_or is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.logical_or(lhs, rhs) else: @@ -94,6 +149,11 @@ def logical_or(lhs, rhs): def remainder(lhs, rhs): + warnings.warn( + "remainder is deprecated and will be removed in the future", + FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.mod(lhs, rhs) elif isinstance(lhs, Frame): @@ -103,6 +163,10 @@ def remainder(lhs, rhs): def floor_divide(lhs, rhs): + warnings.warn( + "sin is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.floor_divide(lhs, rhs) elif isinstance(lhs, Frame): @@ -112,6 +176,10 @@ def floor_divide(lhs, rhs): def subtract(lhs, rhs): + warnings.warn( + "sin is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.subtract(lhs, rhs) elif isinstance(lhs, Frame): @@ -121,6 +189,10 @@ def subtract(lhs, rhs): def add(lhs, rhs): + warnings.warn( + "sin is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.add(lhs, rhs) elif isinstance(rhs, Frame): @@ -130,6 +202,10 @@ def add(lhs, rhs): def true_divide(lhs, rhs): + warnings.warn( + "sin is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.true_divide(lhs, rhs) elif isinstance(rhs, Frame): @@ -139,6 +215,10 @@ def true_divide(lhs, rhs): def multiply(lhs, rhs): + warnings.warn( + "sin is deprecated and will be removed in the future", FutureWarning, + ) + if isinstance(lhs, Number) and isinstance(rhs, Number): return np.multiply(lhs, rhs) elif isinstance(rhs, Frame): diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 8574a152c44..45a44016449 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -9,7 +9,7 @@ from collections import abc as abc from numbers import Number from shutil import get_terminal_size -from typing import Any, MutableMapping, Optional, Set, Union +from typing import Any, Dict, MutableMapping, Optional, Set, Tuple, Type, Union import cupy import numpy as np @@ -39,6 +39,7 @@ ) from cudf.core.abc import Serializable from cudf.core.column import ( + ColumnBase, DatetimeColumn, TimeDeltaColumn, arange, @@ -435,7 +436,7 @@ def __init__( else: data = {} - if not isinstance(data, column.ColumnBase): + if not isinstance(data, ColumnBase): data = column.as_column(data, nan_as_null=nan_as_null, dtype=dtype) else: if dtype is not None: @@ -444,7 +445,7 @@ def __init__( if index is not None and not isinstance(index, BaseIndex): index = as_index(index) - assert isinstance(data, column.ColumnBase) + assert isinstance(data, ColumnBase) super().__init__({name: data}) self._index = RangeIndex(len(data)) if index is None else index @@ -1206,7 +1207,7 @@ def __repr__(self): lines.append(category_memory) return "\n".join(lines) - def _prep_for_binop( + def _make_operands_and_index_for_binop( self, other: Any, fn: str, @@ -1215,22 +1216,19 @@ def _prep_for_binop( can_reindex: bool = False, *args, **kwargs, - ): + ) -> Tuple[ + Union[ + Dict[Optional[str], Tuple[ColumnBase, Any, bool, Any]], + Type[NotImplemented], + ], + Optional[BaseIndex], + ]: # Specialize binops to align indices. - if isinstance(other, SingleColumnFrame): + if isinstance(other, Series): if ( - # TODO: The can_reindex logic also needs to be applied for - # DataFrame (the methods that need it just don't exist yet). not can_reindex and fn in cudf.utils.utils._EQUALITY_OPS - and ( - isinstance(other, Series) - # TODO: mypy doesn't like this line because the index - # property is not defined on SingleColumnFrame (or Index, - # for that matter). Ignoring is the easy solution for now, - # a cleaner fix requires reworking the type hierarchy. - and not self.index.equals(other.index) # type: ignore - ) + and not self.index.equals(other.index) ): raise ValueError( "Can only compare identically-labeled Series objects" @@ -1242,27 +1240,6 @@ def _prep_for_binop( operands = lhs._make_operands_for_binop(other, fill_value, reflect) return operands, lhs._index - def _binaryop( - self, - other: Frame, - fn: str, - fill_value: Any = None, - reflect: bool = False, - can_reindex: bool = False, - *args, - **kwargs, - ): - operands, out_index = self._prep_for_binop( - other, fn, fill_value, reflect, can_reindex - ) - return ( - self._from_data( - data=self._colwise_binop(operands, fn), index=out_index, - ) - if operands is not NotImplemented - else NotImplemented - ) - def logical_and(self, other): warnings.warn( "Series.logical_and is deprecated and will be removed.", diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index 50b206d3388..f02e3b9f959 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -4,7 +4,16 @@ from __future__ import annotations import builtins -from typing import Any, Dict, MutableMapping, Optional, Tuple, TypeVar, Union +from typing import ( + Any, + Dict, + MutableMapping, + Optional, + Tuple, + Type, + TypeVar, + Union, +) import cupy import numpy as np @@ -279,7 +288,10 @@ def _make_operands_for_binop( reflect: bool = False, *args, **kwargs, - ) -> Dict[Optional[str], Tuple[ColumnBase, Any, bool, Any]]: + ) -> Union[ + Dict[Optional[str], Tuple[ColumnBase, Any, bool, Any]], + Type[NotImplemented], + ]: """Generate the dictionary of operands used for a binary operation. Parameters diff --git a/python/cudf/cudf/tests/test_array_ufunc.py b/python/cudf/cudf/tests/test_array_ufunc.py index e4b4d5020ea..9d762f26ebd 100644 --- a/python/cudf/cudf/tests/test_array_ufunc.py +++ b/python/cudf/cudf/tests/test_array_ufunc.py @@ -50,6 +50,83 @@ def _hide_ufunc_warnings(ufunc): yield +@pytest.mark.parametrize("ufunc", _UFUNCS) +def test_ufunc_index(ufunc): + # Note: This test assumes that all ufuncs are unary or binary. + fname = ufunc.__name__ + + N = 100 + # Avoid zeros in either array to skip division by 0 errors. Also limit the + # scale to avoid issues with overflow, etc. We use ints because some + # operations (like bitwise ops) are not defined for floats. + pandas_args = args = [ + cudf.Index(cp.random.randint(low=1, high=10, size=N),) + for _ in range(ufunc.nin) + ] + + try: + got = ufunc(*args) + except AttributeError as e: + # We xfail if we don't have an explicit dispatch and cupy doesn't have + # the method so that we can easily identify these methods. As of this + # writing, the only missing methods are isnat and heaviside. + if "module 'cupy' has no attribute" in str(e): + pytest.xfail(reason="Operation not supported by cupy") + raise + + expect = ufunc(*(arg.to_pandas() for arg in pandas_args)) + + try: + if ufunc.nout > 1: + for g, e in zip(got, expect): + assert_eq(g, e, check_exact=False) + else: + assert_eq(got, expect, check_exact=False) + except AssertionError: + # TODO: This branch can be removed when + # https://github.com/rapidsai/cudf/issues/10178 is resolved + if fname in ("power", "float_power"): + if (got - expect).abs().max() == 1: + pytest.xfail("https://github.com/rapidsai/cudf/issues/10178") + raise + + +@pytest.mark.parametrize( + "ufunc", [np.add, np.greater, np.greater_equal, np.logical_and] +) +@pytest.mark.parametrize("type_", ["cupy", "numpy", "list"]) +@pytest.mark.parametrize("reflect", [True, False]) +def test_binary_ufunc_index_array(ufunc, type_, reflect): + N = 100 + # Avoid zeros in either array to skip division by 0 errors. Also limit the + # scale to avoid issues with overflow, etc. We use ints because some + # operations (like bitwise ops) are not defined for floats. + args = [cudf.Index(cp.random.rand(N)) for _ in range(ufunc.nin)] + + arg1 = args[1].to_cupy() if type_ == "cupy" else args[1].to_numpy() + if type_ == "list": + arg1 = arg1.tolist() + + if reflect: + got = ufunc(arg1, args[0]) + expect = ufunc(args[1].to_numpy(), args[0].to_pandas()) + else: + got = ufunc(args[0], arg1) + expect = ufunc(args[0].to_pandas(), args[1].to_numpy()) + + if ufunc.nout > 1: + for g, e in zip(got, expect): + if type_ == "cupy" and reflect: + assert (cp.asnumpy(g) == e).all() + else: + assert_eq(g, e, check_exact=False) + else: + if type_ == "cupy" and reflect: + assert (cp.asnumpy(got) == expect).all() + else: + assert_eq(got, expect, check_exact=False) + + @pytest.mark.parametrize("ufunc", _UFUNCS) @pytest.mark.parametrize("has_nulls", [True, False]) @pytest.mark.parametrize("indexed", [True, False]) @@ -117,11 +194,11 @@ def test_ufunc_series(ufunc, has_nulls, indexed): for g, e in zip(got, expect): if has_nulls: e[mask] = np.nan - assert_eq(g, e) + assert_eq(g, e, check_exact=False) else: if has_nulls: expect[mask] = np.nan - assert_eq(got, expect) + assert_eq(got, expect, check_exact=False) except AssertionError: # TODO: This branch can be removed when # https://github.com/rapidsai/cudf/issues/10178 is resolved @@ -195,14 +272,14 @@ def test_binary_ufunc_series_array(ufunc, has_nulls, indexed, type_, reflect): if type_ == "cupy" and reflect: assert (cp.asnumpy(g) == e).all() else: - assert_eq(g, e) + assert_eq(g, e, check_exact=False) else: if has_nulls: expect[mask] = np.nan if type_ == "cupy" and reflect: assert (cp.asnumpy(got) == expect).all() else: - assert_eq(got, expect) + assert_eq(got, expect, check_exact=False) @pytest.mark.parametrize( @@ -298,11 +375,11 @@ def test_ufunc_dataframe(ufunc, has_nulls, indexed): for g, e in zip(got, expect): if has_nulls: e[mask] = np.nan - assert_eq(g, e) + assert_eq(g, e, check_exact=False) else: if has_nulls: expect[mask] = np.nan - assert_eq(got, expect) + assert_eq(got, expect, check_exact=False) except AssertionError: # TODO: This branch can be removed when # https://github.com/rapidsai/cudf/issues/10178 is resolved diff --git a/python/cudf/cudf/tests/test_binops.py b/python/cudf/cudf/tests/test_binops.py index c98568d53a5..db12743ac17 100644 --- a/python/cudf/cudf/tests/test_binops.py +++ b/python/cudf/cudf/tests/test_binops.py @@ -1768,10 +1768,6 @@ def test_binops_with_lhs_numpy_scalar(frame, dtype): expected = data.to_pandas() == val got = data == val - # In case of index, expected would be a numpy array - if isinstance(data, cudf.BaseIndex): - expected = pd.Index(expected) - utils.assert_eq(expected, got) @@ -2969,7 +2965,7 @@ def test_binops_non_cudf_types(obj_class, binop, other_type): data = range(1, 100) lhs = obj_class(data) rhs = other_type(data) - assert cp.all((binop(lhs, rhs) == binop(lhs, lhs)).values) + assert (binop(lhs, rhs) == binop(lhs, lhs)).all() @pytest.mark.parametrize("binop", _binops + _binops_compare) diff --git a/python/cudf/cudf/tests/test_dataframe.py b/python/cudf/cudf/tests/test_dataframe.py index 1db91633c5e..d7a5d07a5fc 100644 --- a/python/cudf/cudf/tests/test_dataframe.py +++ b/python/cudf/cudf/tests/test_dataframe.py @@ -3342,14 +3342,6 @@ def test_select_dtype_datetime_with_frequency(): ) -def test_array_ufunc(): - gdf = cudf.DataFrame({"x": [2, 3, 4.0], "y": [9.0, 2.5, 1.1]}) - pdf = gdf.to_pandas() - - assert_eq(np.sqrt(gdf), np.sqrt(pdf)) - assert_eq(np.sqrt(gdf.x), np.sqrt(pdf.x)) - - def test_dataframe_describe_exclude(): np.random.seed(12) data_length = 10000 diff --git a/python/cudf/cudf/tests/test_pickling.py b/python/cudf/cudf/tests/test_pickling.py index 8d504edd669..57b297004bf 100644 --- a/python/cudf/cudf/tests/test_pickling.py +++ b/python/cudf/cudf/tests/test_pickling.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. import sys @@ -90,9 +90,7 @@ def test_pickle_index(): idx = GenericIndex(np.arange(nelem), name="a") pickled = pickle.dumps(idx) out = pickle.loads(pickled) - # TODO: Once operations like `all` are supported on Index objects, we can - # just use that without calling values first. - assert (idx == out).values.all() + assert (idx == out).all() def test_pickle_buffer(): From 21325e8348f33b28e434d08d687a28f251c38f67 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 25 Feb 2022 12:09:55 -0800 Subject: [PATCH 09/11] Implement a mixin for reductions (#9925) This PR implements a factory for mixins classes based on the common pattern in cuDF of categories of similar functions all calling a common method implementing some standard pre/post-processing before calling a lower-level API of either one of its members (e.g. `Frames` calling `Column` methods) or the C++ libcudf library. When added to another class, these mixins support customization of which methods are exposed via a class member set of method names. Documentation for these methods is generated by formatting the docstring for the common internal method, e.g. `_reduce` for reductions. As a first pass, this PR generates a single mixin for reductions and applies it to all the relevant classes. Future PRs will use this to generate classes for scans, binary operations, and unary operations, and perhaps other similar categories as they are uncovered. This approach assumes a great deal of API homogeneity between the different methods in a category. `Frame` violates this assumption because similar operations often support slightly different parameters (for instance, some reductions support a `min_count` parameter), so for now `Frame` was not made `Reducible`. That decision could be revisited if 1) the degree of homogeneity of these function signatures increases over time, or 2) we can introduce greater customization into these mixins without adding too much complexity. A first attempt of (2) can be seen in [this branch](https://github.com/vyasr/cudf/tree/refactor/reductions_extended), but the degree of additional complexity just to support `Frame` isn't really justifiable at this stage, so unless we can come up with a simpler solution I recommend leaving `Frame` as is for now. Authors: - Vyas Ramasubramani (https://github.com/vyasr) - Ashwin Srinath (https://github.com/shwina) Approvers: - Michael Wang (https://github.com/isVoid) - Ashwin Srinath (https://github.com/shwina) URL: https://github.com/rapidsai/cudf/pull/9925 --- python/cudf/cudf/core/column/column.py | 78 ++---- python/cudf/cudf/core/column/datetime.py | 20 +- .../cudf/cudf/core/column/numerical_base.py | 88 +++--- python/cudf/cudf/core/column/string.py | 2 +- python/cudf/cudf/core/column/timedelta.py | 21 +- python/cudf/cudf/core/dataframe.py | 11 +- python/cudf/cudf/core/groupby/groupby.py | 109 ++++---- python/cudf/cudf/core/mixins/__init__.py | 5 + python/cudf/cudf/core/mixins/mixin_factory.py | 259 ++++++++++++++++++ python/cudf/cudf/core/mixins/reductions.py | 35 +++ python/cudf/cudf/core/mixins/reductions.pyi | 70 +++++ python/cudf/cudf/core/series.py | 15 +- python/cudf/cudf/core/single_column_frame.py | 5 +- python/cudf/cudf/core/window/rolling.py | 32 ++- python/cudf/cudf/tests/test_stats.py | 11 +- 15 files changed, 562 insertions(+), 199 deletions(-) create mode 100644 python/cudf/cudf/core/mixins/__init__.py create mode 100644 python/cudf/cudf/core/mixins/mixin_factory.py create mode 100644 python/cudf/cudf/core/mixins/reductions.py create mode 100644 python/cudf/cudf/core/mixins/reductions.pyi diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 2788ac6a600..775bc365cee 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -69,6 +69,7 @@ ListDtype, StructDtype, ) +from cudf.core.mixins import Reducible from cudf.utils import utils from cudf.utils.dtypes import ( cudf_dtype_from_pa_type, @@ -86,7 +87,14 @@ Slice = TypeVar("Slice", bound=slice) -class ColumnBase(Column, Serializable, NotIterable): +class ColumnBase(Column, Serializable, Reducible, NotIterable): + _VALID_REDUCTIONS = { + "any", + "all", + "max", + "min", + } + def as_frame(self) -> "cudf.core.frame.Frame": """ Converts a Column to Frame @@ -674,16 +682,10 @@ def append(self, other: ColumnBase) -> ColumnBase: return concat_columns([self, as_column(other)]) def quantile( - self, - q: Union[float, Sequence[float]], - interpolation: builtins.str, - exact: bool, + self, q: Union[float, Sequence[float]], interpolation: str, exact: bool ) -> ColumnBase: raise TypeError(f"cannot perform quantile with type {self.dtype}") - def median(self, skipna: bool = None) -> ScalarLike: - raise TypeError(f"cannot perform median with type {self.dtype}") - def take( self: T, indices: ColumnBase, nullify: bool = False, check_bounds=True ) -> T: @@ -1162,53 +1164,23 @@ def _minmax(self, skipna: bool = None): return libcudf.reduce.minmax(result_col) return result_col - def min(self, skipna: bool = None, dtype: Dtype = None): - result_col = self._process_for_reduction(skipna=skipna) - if isinstance(result_col, ColumnBase): - return libcudf.reduce.reduce("min", result_col, dtype=dtype) - return result_col - - def max(self, skipna: bool = None, dtype: Dtype = None): - result_col = self._process_for_reduction(skipna=skipna) - if isinstance(result_col, ColumnBase): - return libcudf.reduce.reduce("max", result_col, dtype=dtype) - return result_col - - def sum( - self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0 - ): - raise TypeError(f"cannot perform sum with type {self.dtype}") - - def product( - self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0 - ): - raise TypeError(f"cannot perform product with type {self.dtype}") - - def mean(self, skipna: bool = None, dtype: Dtype = None): - raise TypeError(f"cannot perform mean with type {self.dtype}") - - def std(self, skipna: bool = None, ddof=1, dtype: Dtype = np.float64): - raise TypeError(f"cannot perform std with type {self.dtype}") - - def var(self, skipna: bool = None, ddof=1, dtype: Dtype = np.float64): - raise TypeError(f"cannot perform var with type {self.dtype}") - - def kurtosis(self, skipna: bool = None): - raise TypeError(f"cannot perform kurtosis with type {self.dtype}") + def _reduce( + self, op: str, skipna: bool = None, min_count: int = 0, *args, **kwargs + ) -> ScalarLike: + """Compute {op} of column values. - def skew(self, skipna: bool = None): - raise TypeError(f"cannot perform skew with type {self.dtype}") - - def cov(self, other: ColumnBase): - raise TypeError( - f"cannot perform covarience with types {self.dtype}, " - f"{other.dtype}" - ) - - def corr(self, other: ColumnBase): - raise TypeError( - f"cannot perform corr with types {self.dtype}, {other.dtype}" + skipna : bool + Whether or not na values must be skipped. + min_count : int, default 0 + The minimum number of entries for the reduction, otherwise the + reduction returns NaN. + """ + preprocessed = self._process_for_reduction( + skipna=skipna, min_count=min_count ) + if isinstance(preprocessed, ColumnBase): + return libcudf.reduce.reduce(op, preprocessed, **kwargs) + return preprocessed @property def contains_na_entries(self) -> bool: diff --git a/python/cudf/cudf/core/column/datetime.py b/python/cudf/cudf/core/column/datetime.py index c72fb66addc..44e5c6f62f1 100644 --- a/python/cudf/cudf/core/column/datetime.py +++ b/python/cudf/cudf/core/column/datetime.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. from __future__ import annotations @@ -346,17 +346,27 @@ def as_string_column( column.column_empty(0, dtype="object", masked=False), ) - def mean(self, skipna=None, dtype=np.float64) -> ScalarLike: + def mean( + self, skipna=None, min_count: int = 0, dtype=np.float64 + ) -> ScalarLike: return pd.Timestamp( - self.as_numerical.mean(skipna=skipna, dtype=dtype), + self.as_numerical.mean( + skipna=skipna, min_count=min_count, dtype=dtype + ), unit=self.time_unit, ) def std( - self, skipna: bool = None, ddof: int = 1, dtype: Dtype = np.float64 + self, + skipna: bool = None, + min_count: int = 0, + dtype: Dtype = np.float64, + ddof: int = 1, ) -> pd.Timedelta: return pd.Timedelta( - self.as_numerical.std(skipna=skipna, ddof=ddof, dtype=dtype) + self.as_numerical.std( + skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof + ) * _numpy_to_pandas_conversion[self.time_unit], ) diff --git a/python/cudf/cudf/core/column/numerical_base.py b/python/cudf/cudf/core/column/numerical_base.py index 1f84cb88e37..db333328692 100644 --- a/python/cudf/cudf/core/column/numerical_base.py +++ b/python/cudf/cudf/core/column/numerical_base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. """Define an interface for columns that can perform numerical operations.""" from __future__ import annotations @@ -10,7 +10,7 @@ import cudf from cudf import _lib as libcudf -from cudf._typing import Dtype, ScalarLike +from cudf._typing import ScalarLike from cudf.core.column import ColumnBase @@ -23,59 +23,14 @@ class NumericalBaseColumn(ColumnBase): point, should be encoded here. """ - def reduce( - self, op: str, skipna: bool = None, min_count: int = 0, **kwargs - ) -> ScalarLike: - """Perform a reduction operation. - - op : str - The operation to perform. - skipna : bool - Whether or not na values must be - """ - preprocessed = self._process_for_reduction( - skipna=skipna, min_count=min_count - ) - if isinstance(preprocessed, ColumnBase): - return libcudf.reduce.reduce(op, preprocessed, **kwargs) - else: - return preprocessed - - def sum( - self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0 - ) -> ScalarLike: - return self.reduce( - "sum", skipna=skipna, dtype=dtype, min_count=min_count - ) - - def product( - self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0 - ) -> ScalarLike: - return self.reduce( - "product", skipna=skipna, dtype=dtype, min_count=min_count - ) - - def mean( - self, skipna: bool = None, dtype: Dtype = np.float64 - ) -> ScalarLike: - return self.reduce("mean", skipna=skipna, dtype=dtype) - - def var( - self, skipna: bool = None, ddof: int = 1, dtype: Dtype = np.float64 - ) -> ScalarLike: - return self.reduce("var", skipna=skipna, dtype=dtype, ddof=ddof) - - def std( - self, skipna: bool = None, ddof: int = 1, dtype: Dtype = np.float64 - ) -> ScalarLike: - return self.reduce("std", skipna=skipna, dtype=dtype, ddof=ddof) - - def sum_of_squares( - self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0 - ) -> ScalarLike: - return self.reduce( - "sum_of_squares", skipna=skipna, dtype=dtype, min_count=min_count - ) + _VALID_REDUCTIONS = { + "sum", + "product", + "sum_of_squares", + "mean", + "var", + "std", + } def _can_return_nan(self, skipna: bool = None) -> bool: return not skipna and self.has_nulls() @@ -148,6 +103,25 @@ def quantile( ) return result + def mean(self, skipna: bool = None, min_count: int = 0, dtype=np.float64): + return self._reduce( + "mean", skipna=skipna, min_count=min_count, dtype=dtype + ) + + def var( + self, skipna: bool = None, min_count: int = 0, dtype=np.float64, ddof=1 + ): + return self._reduce( + "var", skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof + ) + + def std( + self, skipna: bool = None, min_count: int = 0, dtype=np.float64, ddof=1 + ): + return self._reduce( + "std", skipna=skipna, min_count=min_count, dtype=dtype, ddof=ddof + ) + def median(self, skipna: bool = None) -> NumericalBaseColumn: skipna = True if skipna is None else skipna @@ -171,7 +145,7 @@ def _numeric_quantile( self, quant, interpolation, sorted_indices, exact ) - def cov(self, other: ColumnBase) -> float: + def cov(self, other: NumericalBaseColumn) -> float: if ( len(self) == 0 or len(other) == 0 @@ -183,7 +157,7 @@ def cov(self, other: ColumnBase) -> float: cov_sample = result.sum() / (len(self) - 1) return cov_sample - def corr(self, other: ColumnBase) -> float: + def corr(self, other: NumericalBaseColumn) -> float: if len(self) == 0 or len(other) == 0: return cudf.utils.dtypes._get_nan_for_dtype(self.dtype) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 982ac098bbf..396e36a803a 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -5154,7 +5154,7 @@ def to_arrow(self) -> pa.Array: return super().to_arrow() def sum( - self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0 + self, skipna: bool = None, dtype: Dtype = None, min_count: int = 0, ): result_col = self._process_for_reduction( skipna=skipna, min_count=min_count diff --git a/python/cudf/cudf/core/column/timedelta.py b/python/cudf/cudf/core/column/timedelta.py index 6c8c904e13c..7a5f777e88e 100644 --- a/python/cudf/cudf/core/column/timedelta.py +++ b/python/cudf/cudf/core/column/timedelta.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. from __future__ import annotations @@ -385,20 +385,29 @@ def quantile( return result.astype(self.dtype) def sum( - self, skipna: bool = None, dtype: Dtype = None, min_count=0 + self, skipna: bool = None, min_count: int = 0, dtype: Dtype = None, ) -> pd.Timedelta: return pd.Timedelta( - self.as_numerical.sum( - skipna=skipna, dtype=dtype, min_count=min_count + # Since sum isn't overriden in Numerical[Base]Column, mypy only + # sees the signature from Reducible (which doesn't have the extra + # parameters from ColumnBase._reduce) so we have to ignore this. + self.as_numerical.sum( # type: ignore + skipna=skipna, min_count=min_count, dtype=dtype ), unit=self.time_unit, ) def std( - self, skipna: bool = None, ddof: int = 1, dtype: Dtype = np.float64 + self, + skipna: bool = None, + min_count: int = 0, + dtype: Dtype = np.float64, + ddof: int = 1, ) -> pd.Timedelta: return pd.Timedelta( - self.as_numerical.std(skipna=skipna, ddof=ddof, dtype=dtype), + self.as_numerical.std( + skipna=skipna, min_count=min_count, ddof=ddof, dtype=dtype + ), unit=self.time_unit, ) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 4062f811975..287fd3796f4 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5410,10 +5410,13 @@ def _reduce( axis = self._get_axis_from_axis_arg(axis) if axis == 0: - result = [ - getattr(self._data[col], op)(**kwargs) - for col in self._data.names - ] + try: + result = [ + getattr(self._data[col], op)(**kwargs) + for col in self._data.names + ] + except AttributeError: + raise TypeError(f"cannot perform {op} with type {self.dtype}") return Series._from_data( {None: result}, as_index(self._data.names) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 0ad6a9dc35a..b76f5dcc261 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -16,6 +16,7 @@ from cudf.api.types import is_list_like from cudf.core.abc import Serializable from cudf.core.column.column import arange, as_column +from cudf.core.mixins import Reducible from cudf.core.multiindex import MultiIndex from cudf.utils.utils import GetAttrGetItemMixin @@ -35,7 +36,23 @@ def _quantile_75(x): return x.quantile(0.75) -class GroupBy(Serializable): +class GroupBy(Serializable, Reducible): + + _VALID_REDUCTIONS = { + "sum", + "prod", + "idxmin", + "idxmax", + "min", + "max", + "mean", + "median", + "nunique", + "first", + "last", + "var", + "std", + } _MAX_GROUPS_BEFORE_WARN = 100 @@ -296,6 +313,46 @@ def agg(self, func): return result + def _reduce( + self, + op: str, + numeric_only: bool = False, + min_count: int = 0, + *args, + **kwargs, + ): + """Compute {op} of group values. + + Parameters + ---------- + numeric_only : bool, default None + Include only float, int, boolean columns. If None, will attempt to + use everything, then use only numeric data. + min_count : int, default 0 + The required number of valid values to perform the operation. If + fewer than ``min_count`` non-NA values are present the result will + be NA. + + Returns + ------- + Series or DataFrame + Computed {op} of values within each group. + + Notes + ----- + Difference from pandas: + * Not supporting: numeric_only, min_count + """ + if numeric_only: + raise NotImplementedError( + "numeric_only parameter is not implemented yet" + ) + if min_count != 0: + raise NotImplementedError( + "min_count parameter is not implemented yet" + ) + return self.agg(op) + aggregate = agg def nth(self, n): @@ -812,38 +869,6 @@ def describe(self, include=None, exclude=None): ) return res - def sum(self): - """Compute the column-wise sum of the values in each group.""" - return self.agg("sum") - - def prod(self): - """Compute the column-wise product of the values in each group.""" - return self.agg("prod") - - def idxmin(self): - """Get the column-wise index of the minimum value in each group.""" - return self.agg("idxmin") - - def idxmax(self): - """Get the column-wise index of the maximum value in each group.""" - return self.agg("idxmax") - - def min(self): - """Get the column-wise minimum value in each group.""" - return self.agg("min") - - def max(self): - """Get the column-wise maximum value in each group.""" - return self.agg("max") - - def mean(self): - """Compute the column-wise mean of the values in each group.""" - return self.agg("mean") - - def median(self): - """Get the column-wise median of the values in each group.""" - return self.agg("median") - def corr(self, method="pearson", min_periods=1): """ Compute pairwise correlation of columns, excluding NA/null values. @@ -1177,10 +1202,6 @@ def func(x): return self.agg(func) - def nunique(self): - """Compute the number of unique values in each column in each group.""" - return self.agg("nunique") - def collect(self): """Get a list of all the values for each column in each group.""" return self.agg("collect") @@ -1202,14 +1223,6 @@ def cummax(self): """Get the column-wise cumulative maximum value in each group.""" return self.agg("cummax") - def first(self): - """Get the first non-null value in each group.""" - return self.agg("first") - - def last(self): - """Get the last non-null value in each group.""" - return self.agg("last") - def diff(self, periods=1, axis=0): """Get the difference between the values in each group. @@ -1539,12 +1552,6 @@ def __getitem__(self, key): by=self.grouping.keys, dropna=self._dropna, sort=self._sort ) - def nunique(self): - """ - Return the number of unique values per group - """ - return self.agg("nunique") - class SeriesGroupBy(GroupBy): """ diff --git a/python/cudf/cudf/core/mixins/__init__.py b/python/cudf/cudf/core/mixins/__init__.py new file mode 100644 index 00000000000..cecf4c1c7ed --- /dev/null +++ b/python/cudf/cudf/core/mixins/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +from .reductions import Reducible + +__all__ = ["Reducible"] diff --git a/python/cudf/cudf/core/mixins/mixin_factory.py b/python/cudf/cudf/core/mixins/mixin_factory.py new file mode 100644 index 00000000000..ecb18f61830 --- /dev/null +++ b/python/cudf/cudf/core/mixins/mixin_factory.py @@ -0,0 +1,259 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +import inspect + + +# `functools.partialmethod` does not allow setting attributes such as +# __doc__ on the resulting method. So we use a simple alternative to +# it here: +def _partialmethod(method, *args1, **kwargs1): + def wrapper(self, *args2, **kwargs2): + return method(self, *args1, *args2, **kwargs1, **kwargs2) + + return wrapper + + +class Operation: + """Descriptor used to define operations for delegating mixins. + + This class is designed to be assigned to the attributes (the delegating + methods) defined by the OperationMixin. This class will create the method + and mimic all the expected attributes for that method to appear as though + it was originally designed on the class. The use of the descriptor pattern + ensures that the method is only created the first time it is invoked, after + which all further calls use the callable generated on the first invocation. + + Parameters + ---------- + name : str + The name of the operation. + docstring_format_args : str + The attribute of the owning class from which to pull format parameters + for this operation's docstring. + base_operation : str + The underlying operation function to be invoked when operation `name` + is called on the owning class. + """ + + def __init__(self, name, docstring_format_args, base_operation): + self._name = name + self._docstring_format_args = docstring_format_args + self._base_operation = base_operation + + def __get__(self, obj, owner=None): + retfunc = _partialmethod(self._base_operation, op=self._name) + + # Required attributes that will exist. + retfunc.__name__ = self._name + retfunc.__qualname__ = ".".join([owner.__name__, self._name]) + retfunc.__module__ = self._base_operation.__module__ + + if self._base_operation.__doc__ is not None: + retfunc.__doc__ = self._base_operation.__doc__.format( + cls=owner.__name__, + op=self._name, + **self._docstring_format_args, + ) + + retfunc.__annotations__ = self._base_operation.__annotations__.copy() + retfunc.__annotations__.pop("op", None) + retfunc_params = [ + v + for k, v in inspect.signature( + self._base_operation + ).parameters.items() + if k != "op" + ] + retfunc.__signature__ = inspect.Signature(retfunc_params) + + setattr(owner, self._name, retfunc) + + if obj is None: + return getattr(owner, self._name) + else: + return getattr(obj, self._name) + + +def _should_define_operation(cls, operation, base_operation_name): + if operation not in dir(cls): + return True + + # If the class doesn't override the base operation we stick to whatever + # parent implementation exists. + if base_operation_name not in cls.__dict__: + return False + + # At this point we know that the class has the operation defined but it + # also overrides the base operation. Since this function is called before + # the operation is defined on the current class, we know that it inherited + # the operation from a parent. We therefore have two possibilities: + # 1. A parent class manually defined the operation. That override takes + # precedence even if the current class defined the base operation. + # 2. A parent class has an auto-generated operation, i.e. it is of type + # Operation and was created by OperationMixin.__init_subclass__. The + # current class must override it so that its base operation is used + # rather than the parent's base operation. + for base_cls in cls.__mro__: + # The first attribute in the MRO is the one that will be used. + if operation in base_cls.__dict__: + return isinstance(base_cls.__dict__[operation], Operation) + + # This line should be unreachable since we know the attribute exists + # somewhere in the MRO if the for loop was entered. + assert False, "Operation attribute not found in hierarchy." + + +def _create_delegating_mixin( + mixin_name, + docstring, + category_name, + base_operation_name, + supported_operations, +): + """Factory for mixins defining collections of delegated operations. + + This function generates mixins based on two common paradigms in cuDF: + + 1. libcudf groups many operations into categories using a common API. These + APIs usually accept an enum to delineate the specific operation to + perform, e.g. binary operations use the `binary_operator` enum when + calling the `binary_operation` function. cuDF Python mimics this + structure by having operations within a category delegate to a common + internal function (e.g. DataFrame.__add__ calls DataFrame._binaryop). + 2. Many cuDF classes implement similar operations (e.g. `sum`) via + delegation to lower-level APIs before reaching a libcudf C++ function + call. As a result, many API function calls actually involve multiple + delegations to lower-level APIs that can look essentially identical. An + example of such a sequence would be DataFrame.sum -> DataFrame._reduce + -> Column.sum -> Column._reduce -> libcudf. + + This factory creates mixins for a category of operations implemented by via + this delegator pattern. The resulting mixins make it easy to share common + functions across various classes while also providing a common entrypoint + for implementing the centralized logic for a given category of operations. + Its usage is best demonstrated by example below. + + Parameters + ---------- + mixin_name : str + The name of the class. This argument should be the same as the object + that this function's output is assigned to, e.g. + :code:`Baz = _create_delegating_mixin("Baz", ...)`. + docstring : str + The documentation string for the mixin class. + category_name : str + The category of operations for which a mixin is being created. This + name will be used to define or access the following attributes as shown + in the example below: + - f'_{category_name}_DOCSTRINGS' + - f'_VALID_{category_name}S' # The subset of ops a subclass allows + - f'_SUPPORTED_{category_name}S' # The ops supported by the mixin + base_operation_name : str + The name given to the core function implementing this category of + operations. The corresponding function is the entrypoint for child + classes. + supported_ops : List[str] + The list of valid operations that subclasses of the resulting mixin may + request to be implemented. + + Examples + -------- + >>> # The class below: + >>> class Person: + ... def _greet(self, op): + ... print(op) + ... + ... def hello(self): + ... self._greet("hello") + ... + ... def goodbye(self): + ... self._greet("goodbye") + >>> # can be rewritten using a delegating mixin as follows: + >>> Greeter = _create_delegating_mixin( + ... "Greeter", "", "GREETING", "_greet", {"hello", "goodbye", "hey"} + ... ) + >>> # The `hello` and `goodbye` methods will now be automatically generated + >>> # for the Person class below. + >>> class Person(Greeter): + ... _VALID_GREETINGS = {"hello", "goodbye"} + ... + ... def _greet(self, op: str): + ... '''Say {op}.''' + ... print(op) + >>> mom = Person() + >>> mom.hello() + hello + >>> # The Greeter class could also enable the `hey` method, but Person did + >>> # not include it in the _VALID_GREETINGS set so it will not exist. + >>> mom.hey() + Traceback (most recent call last): + ... + AttributeError: 'Person' object has no attribute 'hey' + >>> # The docstrings for each method are generated by formatting the _greet + >>> # docstring with the operation name as well as any additional keys + >>> # provided via the _GREETING_DOCSTRINGS parameter. + >>> print(mom.hello.__doc__) + Say hello. + """ + # The first two attributes may be defined on subclasses of the generated + # OperationMixin to indicate valid attributes and parameters to use when + # formatting docstrings. The supported_attr will be defined on the + # OperationMixin itself to indicate what operations its subclass may + # inherit from it. + validity_attr = f"_VALID_{category_name}S" + docstring_attr = f"_{category_name}_DOCSTRINGS" + supported_attr = f"_SUPPORTED_{category_name}S" + + class OperationMixin: + @classmethod + def __init_subclass__(cls): + # Support composition of various OperationMixins. Note that since + # this __init_subclass__ is defined on mixins, it does not prohibit + # classes that inherit it from implementing this method as well as + # long as those implementations also include this super call. + super().__init_subclass__() + + # Only add the valid set of operations for a particular class. + valid_operations = set() + for base_cls in cls.__mro__: + valid_operations |= getattr(base_cls, validity_attr, set()) + + invalid_operations = valid_operations - supported_operations + assert ( + len(invalid_operations) == 0 + ), f"Invalid requested operations: {invalid_operations}" + + base_operation = getattr(cls, base_operation_name) + for operation in valid_operations: + if _should_define_operation( + cls, operation, base_operation_name + ): + docstring_format_args = getattr( + cls, docstring_attr, {} + ).get(operation, {}) + op_attr = Operation( + operation, docstring_format_args, base_operation + ) + setattr(cls, operation, op_attr) + + OperationMixin.__name__ = mixin_name + OperationMixin.__qualname__ = mixin_name + OperationMixin.__doc__ = docstring + + def _operation(self, op: str, *args, **kwargs): + raise NotImplementedError + + _operation.__name__ = base_operation_name + _operation.__qualname__ = ".".join([mixin_name, base_operation_name]) + _operation.__doc__ = ( + f"The core {category_name.lower()} function. Must be overridden by " + "subclasses, the default implementation raises a NotImplementedError." + ) + + setattr(OperationMixin, base_operation_name, _operation) + # This attribute is set in case lookup is convenient at a later point, but + # it is not strictly necessary since `supported_operations` is part of the + # closure associated with the class's creation. + setattr(OperationMixin, supported_attr, supported_operations) + + return OperationMixin diff --git a/python/cudf/cudf/core/mixins/reductions.py b/python/cudf/cudf/core/mixins/reductions.py new file mode 100644 index 00000000000..f73f0e8fbc6 --- /dev/null +++ b/python/cudf/cudf/core/mixins/reductions.py @@ -0,0 +1,35 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +from .mixin_factory import _create_delegating_mixin + +Reducible = _create_delegating_mixin( + "Reducible", + "Mixin encapsulating reduction operations.", + "REDUCTION", + "_reduce", + { + "sum", + "product", + "min", + "max", + "count", + "any", + "all", + "sum_of_squares", + "mean", + "var", + "std", + "median", + "argmax", + "argmin", + "nunique", + "nth", + "collect", + "unique", + "prod", + "idxmin", + "idxmax", + "first", + "last", + }, +) diff --git a/python/cudf/cudf/core/mixins/reductions.pyi b/python/cudf/cudf/core/mixins/reductions.pyi new file mode 100644 index 00000000000..600f30e9372 --- /dev/null +++ b/python/cudf/cudf/core/mixins/reductions.pyi @@ -0,0 +1,70 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +from __future__ import annotations + +class Reducible: + def sum(self): + ... + + def product(self): + ... + + def min(self): + ... + + def max(self): + ... + + def count(self): + ... + + def any(self): + ... + + def all(self): + ... + + def sum_of_squares(self): + ... + + def mean(self): + ... + + def var(self): + ... + + def std(self): + ... + + def median(self): + ... + + def argmax(self): + ... + + def argmin(self): + ... + + def nunique(self): + ... + + def nth(self): + ... + + def collect(self): + ... + + def prod(self): + ... + + def idxmin(self): + ... + + def idxmax(self): + ... + + def first(self): + ... + + def last(self): + ... diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 45a44016449..ec87fcdb066 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2516,7 +2516,13 @@ def cov(self, other, min_periods=None): lhs, rhs = _align_indices([lhs, rhs], how="inner") - return lhs._column.cov(rhs._column) + try: + return lhs._column.cov(rhs._column) + except AttributeError: + raise TypeError( + f"cannot perform covariance with types {self.dtype}, " + f"{other.dtype}" + ) def transpose(self): """Return the transpose, which is by definition self. @@ -2552,7 +2558,12 @@ def corr(self, other, method="pearson", min_periods=None): rhs = other.nans_to_nulls().dropna() lhs, rhs = _align_indices([lhs, rhs], how="inner") - return lhs._column.corr(rhs._column) + try: + return lhs._column.corr(rhs._column) + except AttributeError: + raise TypeError( + f"cannot perform corr with types {self.dtype}, {other.dtype}" + ) def autocorr(self, lag=1): """Compute the lag-N autocorrelation. This method computes the Pearson diff --git a/python/cudf/cudf/core/single_column_frame.py b/python/cudf/cudf/core/single_column_frame.py index f02e3b9f959..508bbfb3a9a 100644 --- a/python/cudf/cudf/core/single_column_frame.py +++ b/python/cudf/cudf/core/single_column_frame.py @@ -55,7 +55,10 @@ def _reduce( raise NotImplementedError( "numeric_only parameter is not implemented yet" ) - return getattr(self._column, op)(**kwargs) + try: + return getattr(self._column, op)(**kwargs) + except AttributeError: + raise TypeError(f"cannot perform {op} with type {self.dtype}") def _scan(self, op, axis=None, *args, **kwargs): if axis not in (None, 0): diff --git a/python/cudf/cudf/core/window/rolling.py b/python/cudf/cudf/core/window/rolling.py index 2282a435ed3..fa482d52104 100644 --- a/python/cudf/cudf/core/window/rolling.py +++ b/python/cudf/cudf/core/window/rolling.py @@ -11,11 +11,12 @@ from cudf.api.types import is_integer, is_number from cudf.core import column from cudf.core.column.column import as_column +from cudf.core.mixins import Reducible from cudf.utils import cudautils from cudf.utils.utils import GetAttrGetItemMixin -class Rolling(GetAttrGetItemMixin): +class Rolling(GetAttrGetItemMixin, Reducible): """ Rolling window calculations. @@ -163,6 +164,15 @@ class Rolling(GetAttrGetItemMixin): _time_window = False + _VALID_REDUCTIONS = { + "sum", + "min", + "max", + "mean", + "var", + "std", + } + def __init__( self, obj, @@ -262,17 +272,17 @@ def _apply_agg(self, agg_name): else: return self._apply_agg_dataframe(self.obj, agg_name) - def sum(self): - return self._apply_agg("sum") - - def min(self): - return self._apply_agg("min") - - def max(self): - return self._apply_agg("max") + def _reduce( + self, op: str, *args, **kwargs, + ): + """Calculate the rolling {op}. - def mean(self): - return self._apply_agg("mean") + Returns + ------- + Series or DataFrame + Return type is the same as the original object. + """ + return self._apply_agg(op) def var(self, ddof=1): self.agg_params["ddof"] = ddof diff --git a/python/cudf/cudf/tests/test_stats.py b/python/cudf/cudf/tests/test_stats.py index cb3a369d067..7bf339d6ab7 100644 --- a/python/cudf/cudf/tests/test_stats.py +++ b/python/cudf/cudf/tests/test_stats.py @@ -1,6 +1,5 @@ -# Copyright (c) 2018-2021, NVIDIA CORPORATION. +# Copyright (c) 2018-2022, NVIDIA CORPORATION. -import re from concurrent.futures import ThreadPoolExecutor import numpy as np @@ -513,9 +512,7 @@ def test_cov_corr_invalid_dtypes(gsr): rfunc=gsr.corr, lfunc_args_and_kwargs=([psr],), rfunc_args_and_kwargs=([gsr],), - expected_error_message=re.escape( - f"cannot perform corr with types {gsr.dtype}, {gsr.dtype}" - ), + compare_error_message=False, ) assert_exceptions_equal( @@ -523,7 +520,5 @@ def test_cov_corr_invalid_dtypes(gsr): rfunc=gsr.cov, lfunc_args_and_kwargs=([psr],), rfunc_args_and_kwargs=([gsr],), - expected_error_message=re.escape( - f"cannot perform covarience with types {gsr.dtype}, {gsr.dtype}" - ), + compare_error_message=False, ) From 3e33453008a2031d6c19ff3b783a08fcf527c1a2 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Sat, 26 Feb 2022 10:45:41 +0530 Subject: [PATCH 10/11] move input generation for json benchmark to device (#10281) - Use device functions to move input generation to device in json benchmark. - Use raw string literal for readability - replace BENCHMARK_CAPTURE with benchmark fixture without template. Splitting PR https://github.com/rapidsai/cudf/pull/10109 for review Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - https://github.com/nvdbaranec - Robert Maynard (https://github.com/robertmaynard) URL: https://github.com/rapidsai/cudf/pull/10281 --- cpp/benchmarks/CMakeLists.txt | 2 +- cpp/benchmarks/string/json.cpp | 140 -------------------- cpp/benchmarks/string/json.cu | 228 +++++++++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 141 deletions(-) delete mode 100644 cpp/benchmarks/string/json.cpp create mode 100644 cpp/benchmarks/string/json.cu diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt index 99aeff0df93..054410c3265 100644 --- a/cpp/benchmarks/CMakeLists.txt +++ b/cpp/benchmarks/CMakeLists.txt @@ -276,7 +276,7 @@ ConfigureBench( # ################################################################################################## # * json benchmark ------------------------------------------------------------------- -ConfigureBench(JSON_BENCH string/json.cpp) +ConfigureBench(JSON_BENCH string/json.cu) # ################################################################################################## # * io benchmark --------------------------------------------------------------------- diff --git a/cpp/benchmarks/string/json.cpp b/cpp/benchmarks/string/json.cpp deleted file mode 100644 index 1ade4d01e1e..00000000000 --- a/cpp/benchmarks/string/json.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2021, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include -#include - -#include -#include - -class JsonPath : public cudf::benchmark { -}; - -float frand() { return static_cast(rand()) / static_cast(RAND_MAX); } - -int rand_range(int min, int max) { return min + static_cast(frand() * (max - min)); } - -std::vector Books{ - "{\n\"category\": \"reference\",\n\"author\": \"Nigel Rees\",\n\"title\": \"Sayings of the " - "Century\",\n\"price\": 8.95\n}", - "{\n\"category\": \"fiction\",\n\"author\": \"Evelyn Waugh\",\n\"title\": \"Sword of " - "Honour\",\n\"price\": 12.99\n}", - "{\n\"category\": \"fiction\",\n\"author\": \"Herman Melville\",\n\"title\": \"Moby " - "Dick\",\n\"isbn\": \"0-553-21311-3\",\n\"price\": 8.99\n}", - "{\n\"category\": \"fiction\",\n\"author\": \"J. R. R. Tolkien\",\n\"title\": \"The Lord of the " - "Rings\",\n\"isbn\": \"0-395-19395-8\",\n\"price\": 22.99\n}"}; -constexpr int Approx_book_size = 110; -std::vector Bicycles{ - "{\"color\": \"red\", \"price\": 9.95}", - "{\"color\": \"green\", \"price\": 29.95}", - "{\"color\": \"blue\", \"price\": 399.95}", - "{\"color\": \"yellow\", \"price\": 99.95}", - "{\"color\": \"mauve\", \"price\": 199.95}", -}; -constexpr int Approx_bicycle_size = 33; -std::string Misc{"\n\"expensive\": 10\n"}; -std::string generate_field(std::vector const& values, int num_values) -{ - std::string res; - for (int idx = 0; idx < num_values; idx++) { - if (idx > 0) { res += std::string(",\n"); } - int vindex = std::min(static_cast(floor(frand() * values.size())), - static_cast(values.size() - 1)); - res += values[vindex]; - } - return res; -} - -std::string build_row(int desired_bytes) -{ - // always have at least 2 books and 2 bikes - int num_books = 2; - int num_bicycles = 2; - int remaining_bytes = - desired_bytes - ((num_books * Approx_book_size) + (num_bicycles * Approx_bicycle_size)); - - // divide up the remainder between books and bikes - float book_pct = frand(); - float bicycle_pct = 1.0f - book_pct; - num_books += (remaining_bytes * book_pct) / Approx_book_size; - num_bicycles += (remaining_bytes * bicycle_pct) / Approx_bicycle_size; - - std::string books = "\"book\": [\n" + generate_field(Books, num_books) + "]\n"; - std::string bicycles = "\"bicycle\": [\n" + generate_field(Bicycles, num_bicycles) + "]\n"; - - std::string store = "\"store\": {\n"; - if (frand() <= 0.5f) { - store += books + std::string(",\n") + bicycles; - } else { - store += bicycles + std::string(",\n") + books; - } - store += std::string("}\n"); - - std::string row = std::string("{\n"); - if (frand() <= 0.5f) { - row += store + std::string(",\n") + Misc; - } else { - row += Misc + std::string(",\n") + store; - } - row += std::string("}\n"); - return row; -} - -template -static void BM_case(benchmark::State& state, QueryArg&&... query_arg) -{ - srand(5236); - auto iter = thrust::make_transform_iterator( - thrust::make_counting_iterator(0), - [desired_bytes = state.range(1)](int index) { return build_row(desired_bytes); }); - int num_rows = state.range(0); - cudf::test::strings_column_wrapper input(iter, iter + num_rows); - cudf::strings_column_view scv(input); - size_t num_chars = scv.chars().size(); - - std::string json_path(query_arg...); - - for (auto _ : state) { - cuda_event_timer raii(state, true); - auto result = cudf::strings::get_json_object(scv, json_path); - cudaStreamSynchronize(0); - } - - // this isn't strictly 100% accurate. a given query isn't necessarily - // going to visit every single incoming character. but in spirit it does. - state.SetBytesProcessed(state.iterations() * num_chars); -} - -#define JSON_BENCHMARK_DEFINE(name, query) \ - BENCHMARK_CAPTURE(BM_case, name, query) \ - ->ArgsProduct({{100, 1000, 100000, 400000}, {300, 600, 4096}}) \ - ->UseManualTime() \ - ->Unit(benchmark::kMillisecond); - -JSON_BENCHMARK_DEFINE(query0, "$"); -JSON_BENCHMARK_DEFINE(query1, "$.store"); -JSON_BENCHMARK_DEFINE(query2, "$.store.book"); -JSON_BENCHMARK_DEFINE(query3, "$.store.*"); -JSON_BENCHMARK_DEFINE(query4, "$.store.book[*]"); -JSON_BENCHMARK_DEFINE(query5, "$.store.book[*].category"); -JSON_BENCHMARK_DEFINE(query6, "$.store['bicycle']"); -JSON_BENCHMARK_DEFINE(query7, "$.store.book[*]['isbn']"); -JSON_BENCHMARK_DEFINE(query8, "$.store.bicycle[1]"); diff --git a/cpp/benchmarks/string/json.cu b/cpp/benchmarks/string/json.cu new file mode 100644 index 00000000000..1a986aff83a --- /dev/null +++ b/cpp/benchmarks/string/json.cu @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include + +class JsonPath : public cudf::benchmark { +}; + +const std::vector Books{ + R"json({ +"category": "reference", +"author": "Nigel Rees", +"title": "Sayings of the Century", +"price": 8.95 +})json", + R"json({ +"category": "fiction", +"author": "Evelyn Waugh", +"title": "Sword of Honour", +"price": 12.99 +})json", + R"json({ +"category": "fiction", +"author": "Herman Melville", +"title": "Moby Dick", +"isbn": "0-553-21311-3", +"price": 8.99 +})json", + R"json({ +"category": "fiction", +"author": "J. R. R. Tolkien", +"title": "The Lord of the Rings", +"isbn": "0-395-19395-8", +"price": 22.99 +})json"}; +constexpr int Approx_book_size = 110; +const std::vector Bicycles{ + R"json({"color": "red", "price": 9.95})json", + R"json({"color": "green", "price": 29.95})json", + R"json({"color": "blue", "price": 399.95})json", + R"json({"color": "yellow", "price": 99.95})json", + R"json({"color": "mauve", "price": 199.95})json", +}; +constexpr int Approx_bicycle_size = 33; +std::string Misc{"\n\"expensive\": 10\n"}; + +struct json_benchmark_row_builder { + int const desired_bytes; + cudf::size_type const num_rows; + cudf::column_device_view const d_books_bicycles[2]; // Books, Bicycles strings + cudf::column_device_view const d_book_pct; // Book percentage + cudf::column_device_view const d_misc_order; // Misc-Store order + cudf::column_device_view const d_store_order; // Books-Bicycles order + int32_t* d_offsets{}; + char* d_chars{}; + thrust::minstd_rand rng{5236}; + thrust::uniform_int_distribution dist{}; + + // internal data structure for {bytes, out_ptr} with operator+= + struct bytes_and_ptr { + cudf::size_type bytes; + char* ptr; + __device__ bytes_and_ptr& operator+=(cudf::string_view const& str_append) + { + bytes += str_append.size_bytes(); + if (ptr) { ptr = cudf::strings::detail::copy_string(ptr, str_append); } + return *this; + } + }; + + __device__ inline void copy_items(int this_idx, + cudf::size_type num_items, + bytes_and_ptr& output_str) + { + using param_type = thrust::uniform_int_distribution::param_type; + dist.param(param_type{0, d_books_bicycles[this_idx].size() - 1}); + cudf::string_view comma(",\n", 2); + for (int i = 0; i < num_items; i++) { + if (i > 0) { output_str += comma; } + int idx = dist(rng); + auto item = d_books_bicycles[this_idx].element(idx); + output_str += item; + } + } + + __device__ void operator()(cudf::size_type idx) + { + int num_books = 2; + int num_bicycles = 2; + int remaining_bytes = max( + 0, desired_bytes - ((num_books * Approx_book_size) + (num_bicycles * Approx_bicycle_size))); + + // divide up the remainder between books and bikes + auto book_pct = d_book_pct.element(idx); + // {Misc, store} OR {store, Misc} + // store: {books, bicycles} OR store: {bicycles, books} + float bicycle_pct = 1.0f - book_pct; + num_books += (remaining_bytes * book_pct) / Approx_book_size; + num_bicycles += (remaining_bytes * bicycle_pct) / Approx_bicycle_size; + + char* out_ptr = d_chars ? d_chars + d_offsets[idx] : nullptr; + bytes_and_ptr output_str{0, out_ptr}; + // + cudf::string_view comma(",\n", 2); + cudf::string_view brace1("{\n", 2); + cudf::string_view store_member_start[2]{{"\"book\": [\n", 10}, {"\"bicycle\": [\n", 13}}; + cudf::string_view store("\"store\": {\n", 11); + cudf::string_view Misc{"\"expensive\": 10", 15}; + cudf::string_view brace2("\n}", 2); + cudf::string_view square2{"\n]", 2}; + + output_str += brace1; + if (d_misc_order.element(idx)) { // Misc. first. + output_str += Misc; + output_str += comma; + } + output_str += store; + for (int store_order = 0; store_order < 2; store_order++) { + if (store_order > 0) { output_str += comma; } + int this_idx = (d_store_order.element(idx) == store_order); + auto& mem_start = store_member_start[this_idx]; + output_str += mem_start; + copy_items(this_idx, this_idx == 0 ? num_books : num_bicycles, output_str); + output_str += square2; + } + output_str += brace2; + if (!d_misc_order.element(idx)) { // Misc, if not first. + output_str += comma; + output_str += Misc; + } + output_str += brace2; + if (!output_str.ptr) d_offsets[idx] = output_str.bytes; + } +}; + +auto build_json_string_column(int desired_bytes, int num_rows) +{ + data_profile profile; + profile.set_cardinality(0); + profile.set_null_frequency(-0.1); + profile.set_distribution_params( + cudf::type_id::FLOAT32, distribution_id::UNIFORM, 0.0, 1.0); + auto float_2bool_columns = + create_random_table({cudf::type_id::FLOAT32, cudf::type_id::BOOL8, cudf::type_id::BOOL8}, + 3, + row_count{num_rows}, + profile); + + cudf::test::strings_column_wrapper books(Books.begin(), Books.end()); + cudf::test::strings_column_wrapper bicycles(Bicycles.begin(), Bicycles.end()); + auto d_books = cudf::column_device_view::create(books); + auto d_bicycles = cudf::column_device_view::create(bicycles); + auto d_book_pct = cudf::column_device_view::create(float_2bool_columns->get_column(0)); + auto d_misc_order = cudf::column_device_view::create(float_2bool_columns->get_column(1)); + auto d_store_order = cudf::column_device_view::create(float_2bool_columns->get_column(2)); + json_benchmark_row_builder jb{ + desired_bytes, num_rows, {*d_books, *d_bicycles}, *d_book_pct, *d_misc_order, *d_store_order}; + auto children = cudf::strings::detail::make_strings_children(jb, num_rows); + return cudf::make_strings_column( + num_rows, std::move(children.first), std::move(children.second), 0, {}); +} + +void BM_case(benchmark::State& state, std::string query_arg) +{ + srand(5236); + int num_rows = state.range(0); + int desired_bytes = state.range(1); + auto input = build_json_string_column(desired_bytes, num_rows); + cudf::strings_column_view scv(input->view()); + size_t num_chars = scv.chars().size(); + + std::string json_path(query_arg); + + for (auto _ : state) { + cuda_event_timer raii(state, true); + auto result = cudf::strings::get_json_object(scv, json_path); + cudaStreamSynchronize(0); + } + + // this isn't strictly 100% accurate. a given query isn't necessarily + // going to visit every single incoming character. but in spirit it does. + state.SetBytesProcessed(state.iterations() * num_chars); +} + +#define JSON_BENCHMARK_DEFINE(name, query) \ + BENCHMARK_DEFINE_F(JsonPath, name)(::benchmark::State & state) { BM_case(state, query); } \ + BENCHMARK_REGISTER_F(JsonPath, name) \ + ->ArgsProduct({{100, 1000, 100000, 400000}, {300, 600, 4096}}) \ + ->UseManualTime() \ + ->Unit(benchmark::kMillisecond); + +JSON_BENCHMARK_DEFINE(query0, "$"); +JSON_BENCHMARK_DEFINE(query1, "$.store"); +JSON_BENCHMARK_DEFINE(query2, "$.store.book"); +JSON_BENCHMARK_DEFINE(query3, "$.store.*"); +JSON_BENCHMARK_DEFINE(query4, "$.store.book[*]"); +JSON_BENCHMARK_DEFINE(query5, "$.store.book[*].category"); +JSON_BENCHMARK_DEFINE(query6, "$.store['bicycle']"); +JSON_BENCHMARK_DEFINE(query7, "$.store.book[*]['isbn']"); +JSON_BENCHMARK_DEFINE(query8, "$.store.bicycle[1]"); From 4c9ef5161268e2486938546deef00f7fc84c9a95 Mon Sep 17 00:00:00 2001 From: Karthikeyan <6488848+karthikeyann@users.noreply.github.com> Date: Sat, 26 Feb 2022 19:18:41 +0530 Subject: [PATCH 11/11] C++17 cleanup: traits replace std::enable_if<>::type with std::enable_if_t (#10343) Authors: - Karthikeyan (https://github.com/karthikeyann) Approvers: - Bradley Dice (https://github.com/bdice) - David Wendt (https://github.com/davidwendt) - Robert Maynard (https://github.com/robertmaynard) - Nghia Truong (https://github.com/ttnghia) URL: https://github.com/rapidsai/cudf/pull/10343 --- cpp/benchmarks/common/generate_input.cpp | 12 +- cpp/benchmarks/common/generate_input.hpp | 27 ++-- .../common/random_distribution_factory.hpp | 8 +- cpp/benchmarks/string/json.cu | 1 - .../type_dispatcher/type_dispatcher.cu | 2 +- .../detail/calendrical_month_sequence.cuh | 18 +-- cpp/include/cudf/detail/reduction.cuh | 10 +- .../cudf/detail/utilities/device_atomics.cuh | 12 +- .../detail/utilities/device_operators.cuh | 53 ++++---- cpp/include/cudf/utilities/span.hpp | 18 +-- cpp/include/cudf/utilities/traits.hpp | 8 +- cpp/include/cudf_test/column_utilities.hpp | 6 +- cpp/include/cudf_test/column_wrapper.hpp | 36 ++--- cpp/include/cudf_test/type_lists.hpp | 11 +- cpp/src/binaryop/compiled/binary_ops.cu | 18 +-- cpp/src/column/column_factories.cpp | 9 +- cpp/src/datetime/datetime_ops.cu | 20 +-- cpp/src/dictionary/detail/concatenate.cu | 6 +- cpp/src/dictionary/set_keys.cu | 6 +- cpp/src/filling/sequence.cu | 12 +- cpp/src/groupby/sort/group_nunique.cu | 22 ++-- cpp/src/groupby/sort/group_tdigest.cu | 14 +- cpp/src/io/csv/csv_gpu.cu | 16 +-- cpp/src/io/json/json_gpu.cu | 9 +- cpp/src/io/orc/orc.h | 52 ++++---- cpp/src/io/utilities/parsing_utils.cuh | 4 +- cpp/src/reductions/scan/scan_exclusive.cu | 2 +- cpp/src/reductions/scan/scan_inclusive.cu | 4 +- cpp/src/reductions/simple.cuh | 21 ++- cpp/src/replace/nulls.cu | 5 +- cpp/src/reshape/interleave_columns.cu | 10 +- cpp/src/rolling/rolling_detail.cuh | 14 +- cpp/src/round/round.cu | 36 ++--- cpp/src/scalar/scalar_factories.cpp | 18 ++- cpp/src/sort/sort.cu | 6 +- cpp/src/sort/sort_column.cu | 12 +- cpp/src/sort/stable_sort_column.cu | 12 +- cpp/src/unary/cast_ops.cu | 123 +++++++++--------- cpp/src/unary/math_ops.cu | 26 ++-- cpp/tests/binaryop/binop-fixture.hpp | 6 +- cpp/tests/binaryop/util/operation.h | 79 ++++++----- cpp/tests/copying/copy_tests.cpp | 10 +- .../device_atomics/device_atomics_test.cu | 8 +- cpp/tests/groupby/tdigest_tests.cu | 12 +- cpp/tests/io/csv_test.cpp | 4 +- cpp/tests/quantiles/percentile_approx_test.cu | 35 +++-- cpp/tests/reductions/reduction_tests.cpp | 4 +- cpp/tests/reductions/scan_tests.hpp | 14 +- cpp/tests/sort/is_sorted_tests.cpp | 27 ++-- cpp/tests/utilities/column_utilities.cu | 21 ++- cpp/tests/wrappers/timestamps_test.cu | 6 +- 51 files changed, 437 insertions(+), 488 deletions(-) diff --git a/cpp/benchmarks/common/generate_input.cpp b/cpp/benchmarks/common/generate_input.cpp index d6564428a2e..6330beda54c 100644 --- a/cpp/benchmarks/common/generate_input.cpp +++ b/cpp/benchmarks/common/generate_input.cpp @@ -122,7 +122,7 @@ struct random_value_fn; * @brief Creates an random timestamp/duration value */ template -struct random_value_fn()>> { +struct random_value_fn()>> { std::function seconds_gen; std::function nanoseconds_gen; @@ -164,7 +164,7 @@ struct random_value_fn()>> { * @brief Creates an random fixed_point value. Not implemented yet. */ template -struct random_value_fn()>> { +struct random_value_fn()>> { using rep = typename T::rep; rep const lower_bound; rep const upper_bound; @@ -194,9 +194,7 @@ struct random_value_fn()>> * @brief Creates an random numeric value with the given distribution. */ template -struct random_value_fn< - T, - typename std::enable_if_t && cudf::is_numeric()>> { +struct random_value_fn && cudf::is_numeric()>> { T const lower_bound; T const upper_bound; distribution_fn dist; @@ -219,7 +217,7 @@ struct random_value_fn< * @brief Creates an boolean value with given probability of returning `true`. */ template -struct random_value_fn>> { +struct random_value_fn>> { std::bernoulli_distribution b_dist; random_value_fn(distribution_params const& desc) : b_dist{desc.probability_true} {} @@ -260,7 +258,7 @@ struct stored_as { // Use `int8_t` for bools because that's how they're stored in columns template -struct stored_as>> { +struct stored_as>> { using type = int8_t; }; diff --git a/cpp/benchmarks/common/generate_input.hpp b/cpp/benchmarks/common/generate_input.hpp index 17bd650e722..43fee5c50a7 100644 --- a/cpp/benchmarks/common/generate_input.hpp +++ b/cpp/benchmarks/common/generate_input.hpp @@ -128,9 +128,7 @@ struct distribution_params; * @brief Numeric values are parameterized with a distribution type and bounds of the same type. */ template -struct distribution_params< - T, - typename std::enable_if_t && cudf::is_numeric()>> { +struct distribution_params && cudf::is_numeric()>> { distribution_id id; T lower_bound; T upper_bound; @@ -140,7 +138,7 @@ struct distribution_params< * @brief Booleans are parameterized with the probability of getting `true` value. */ template -struct distribution_params>> { +struct distribution_params>> { double probability_true; }; @@ -148,7 +146,7 @@ struct distribution_params> * @brief Timestamps and durations are parameterized with a distribution type and int64_t bounds. */ template -struct distribution_params()>> { +struct distribution_params()>> { distribution_id id; int64_t lower_bound; int64_t upper_bound; @@ -158,7 +156,7 @@ struct distribution_params()>> { * @brief Strings are parameterized by the distribution of their length, as an integral value. */ template -struct distribution_params>> { +struct distribution_params>> { distribution_params length_params; }; @@ -167,7 +165,7 @@ struct distribution_params -struct distribution_params>> { +struct distribution_params>> { cudf::type_id element_type; distribution_params length_params; cudf::size_type max_depth; @@ -175,7 +173,7 @@ struct distribution_params -struct distribution_params()>> { +struct distribution_params()>> { }; /** @@ -225,8 +223,7 @@ class data_profile { public: template && cuda::std::is_integral_v, T>* = - nullptr> + std::enable_if_t && cuda::std::is_integral_v, T>* = nullptr> distribution_params get_distribution_params() const { auto it = int_params.find(cudf::type_to_id()); @@ -239,7 +236,7 @@ class data_profile { } } - template , T>* = nullptr> + template , T>* = nullptr> distribution_params get_distribution_params() const { auto it = float_params.find(cudf::type_to_id()); @@ -258,7 +255,7 @@ class data_profile { return distribution_params{bool_probability}; } - template ()>* = nullptr> + template ()>* = nullptr> distribution_params get_distribution_params() const { auto it = int_params.find(cudf::type_to_id()); @@ -284,7 +281,7 @@ class data_profile { return list_dist_desc; } - template ()>* = nullptr> + template ()>* = nullptr> distribution_params get_distribution_params() const { using rep = typename T::rep; @@ -307,7 +304,7 @@ class data_profile { // discrete distributions (integers, strings, lists). Otherwise the call with have no effect. template , T>* = nullptr> + std::enable_if_t, T>* = nullptr> void set_distribution_params(Type_enum type_or_group, distribution_id dist, T lower_bound, @@ -331,7 +328,7 @@ class data_profile { // have continuous distributions (floating point types). Otherwise the call with have no effect. template , T>* = nullptr> + std::enable_if_t, T>* = nullptr> void set_distribution_params(Type_enum type_or_group, distribution_id dist, T lower_bound, diff --git a/cpp/benchmarks/common/random_distribution_factory.hpp b/cpp/benchmarks/common/random_distribution_factory.hpp index df2b6e0a754..f2f3833f15d 100644 --- a/cpp/benchmarks/common/random_distribution_factory.hpp +++ b/cpp/benchmarks/common/random_distribution_factory.hpp @@ -24,7 +24,7 @@ /** * @brief Generates a normal(binomial) distribution between zero and upper_bound. */ -template , T>* = nullptr> +template , T>* = nullptr> auto make_normal_dist(T upper_bound) { using uT = typename std::make_unsigned::type; @@ -42,7 +42,7 @@ auto make_normal_dist(T upper_bound) return std::normal_distribution(mean, stddev); } -template , T>* = nullptr> +template , T>* = nullptr> auto make_uniform_dist(T range_start, T range_end) { return std::uniform_int_distribution(range_start, range_end); @@ -62,7 +62,7 @@ double geometric_dist_p(T range_size) return p ? p : std::numeric_limits::epsilon(); } -template , T>* = nullptr> +template , T>* = nullptr> auto make_geometric_dist(T range_start, T range_end) { using uT = typename std::make_unsigned::type; @@ -82,7 +82,7 @@ auto make_geometric_dist(T range_start, T range_end) template using distribution_fn = std::function; -template , T>* = nullptr> +template , T>* = nullptr> distribution_fn make_distribution(distribution_id did, T lower_bound, T upper_bound) { switch (did) { diff --git a/cpp/benchmarks/string/json.cu b/cpp/benchmarks/string/json.cu index 1a986aff83a..69c42f97d7f 100644 --- a/cpp/benchmarks/string/json.cu +++ b/cpp/benchmarks/string/json.cu @@ -170,7 +170,6 @@ auto build_json_string_column(int desired_bytes, int num_rows) cudf::type_id::FLOAT32, distribution_id::UNIFORM, 0.0, 1.0); auto float_2bool_columns = create_random_table({cudf::type_id::FLOAT32, cudf::type_id::BOOL8, cudf::type_id::BOOL8}, - 3, row_count{num_rows}, profile); diff --git a/cpp/benchmarks/type_dispatcher/type_dispatcher.cu b/cpp/benchmarks/type_dispatcher/type_dispatcher.cu index 48b31e5dae7..3be599e8c41 100644 --- a/cpp/benchmarks/type_dispatcher/type_dispatcher.cu +++ b/cpp/benchmarks/type_dispatcher/type_dispatcher.cu @@ -42,7 +42,7 @@ struct Functor { }; template -struct Functor>> { +struct Functor>> { static __device__ Float f(Float x) { if (ft == BANDWIDTH_BOUND) { diff --git a/cpp/include/cudf/detail/calendrical_month_sequence.cuh b/cpp/include/cudf/detail/calendrical_month_sequence.cuh index 00742db7982..321cc3d19ef 100644 --- a/cpp/include/cudf/detail/calendrical_month_sequence.cuh +++ b/cpp/include/cudf/detail/calendrical_month_sequence.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,12 +30,12 @@ namespace cudf { namespace detail { struct calendrical_month_sequence_functor { template - typename std::enable_if_t::value, std::unique_ptr> - operator()(size_type n, - scalar const& input, - size_type months, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) + std::enable_if_t::value, std::unique_ptr> operator()( + size_type n, + scalar const& input, + size_type months, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr = rmm::mr::get_current_device_resource()) { // Return empty column if n = 0 if (n == 0) return cudf::make_empty_column(input.type()); @@ -59,8 +59,8 @@ struct calendrical_month_sequence_functor { } template - typename std::enable_if_t::value, std::unique_ptr> - operator()(Args&&...) + std::enable_if_t::value, std::unique_ptr> operator()( + Args&&...) { CUDF_FAIL("Cannot make a date_range of a non-datetime type"); } diff --git a/cpp/include/cudf/detail/reduction.cuh b/cpp/include/cudf/detail/reduction.cuh index 76825285745..e176529ed6d 100644 --- a/cpp/include/cudf/detail/reduction.cuh +++ b/cpp/include/cudf/detail/reduction.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,8 +49,8 @@ namespace detail { template ::type, - typename std::enable_if_t() && - not cudf::is_fixed_point()>* = nullptr> + std::enable_if_t() && + not cudf::is_fixed_point()>* = nullptr> std::unique_ptr reduce(InputIterator d_in, cudf::size_type num_items, op::simple_op sop, @@ -92,7 +92,7 @@ std::unique_ptr reduce(InputIterator d_in, template ::type, - typename std::enable_if_t()>* = nullptr> + std::enable_if_t()>* = nullptr> std::unique_ptr reduce(InputIterator d_in, cudf::size_type num_items, op::simple_op sop, @@ -109,7 +109,7 @@ std::unique_ptr reduce(InputIterator d_in, template ::type, - typename std::enable_if_t>* = nullptr> + std::enable_if_t>* = nullptr> std::unique_ptr reduce(InputIterator d_in, cudf::size_type num_items, op::simple_op sop, diff --git a/cpp/include/cudf/detail/utilities/device_atomics.cuh b/cpp/include/cudf/detail/utilities/device_atomics.cuh index 221e90a9816..f985135064f 100644 --- a/cpp/include/cudf/detail/utilities/device_atomics.cuh +++ b/cpp/include/cudf/detail/utilities/device_atomics.cuh @@ -426,7 +426,7 @@ struct typesAtomicCASImpl { * @returns The old value at `address` */ template -typename std::enable_if_t(), T> __forceinline__ __device__ +std::enable_if_t(), T> __forceinline__ __device__ genericAtomicOperation(T* address, T const& update_value, BinaryOp op) { auto fun = cudf::detail::genericAtomicOperationImpl{}; @@ -435,7 +435,7 @@ genericAtomicOperation(T* address, T const& update_value, BinaryOp op) // specialization for cudf::detail::timestamp types template -typename std::enable_if_t(), T> __forceinline__ __device__ +std::enable_if_t(), T> __forceinline__ __device__ genericAtomicOperation(T* address, T const& update_value, BinaryOp op) { using R = typename T::rep; @@ -448,7 +448,7 @@ genericAtomicOperation(T* address, T const& update_value, BinaryOp op) // specialization for cudf::detail::duration types template -typename std::enable_if_t(), T> __forceinline__ __device__ +std::enable_if_t(), T> __forceinline__ __device__ genericAtomicOperation(T* address, T const& update_value, BinaryOp op) { using R = typename T::rep; @@ -616,7 +616,7 @@ __forceinline__ __device__ T atomicCAS(T* address, T compare, T val) * * @returns The old value at `address` */ -template , T>* = nullptr> +template , T>* = nullptr> __forceinline__ __device__ T atomicAnd(T* address, T val) { return cudf::genericAtomicOperation(address, val, cudf::DeviceAnd{}); @@ -637,7 +637,7 @@ __forceinline__ __device__ T atomicAnd(T* address, T val) * * @returns The old value at `address` */ -template , T>* = nullptr> +template , T>* = nullptr> __forceinline__ __device__ T atomicOr(T* address, T val) { return cudf::genericAtomicOperation(address, val, cudf::DeviceOr{}); @@ -658,7 +658,7 @@ __forceinline__ __device__ T atomicOr(T* address, T val) * * @returns The old value at `address` */ -template , T>* = nullptr> +template , T>* = nullptr> __forceinline__ __device__ T atomicXor(T* address, T val) { return cudf::genericAtomicOperation(address, val, cudf::DeviceXor{}); diff --git a/cpp/include/cudf/detail/utilities/device_operators.cuh b/cpp/include/cudf/detail/utilities/device_operators.cuh index 9423cb6b998..87fef5bc187 100644 --- a/cpp/include/cudf/detail/utilities/device_operators.cuh +++ b/cpp/include/cudf/detail/utilities/device_operators.cuh @@ -61,27 +61,26 @@ CUDF_HOST_DEVICE inline auto max(LHS const& lhs, RHS const& rhs) * @brief Binary `sum` operator */ struct DeviceSum { - template ()>* = nullptr> + template ()>* = nullptr> CUDF_HOST_DEVICE inline auto operator()(const T& lhs, const T& rhs) -> decltype(lhs + rhs) { return lhs + rhs; } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { return T{typename T::duration{0}}; } - template < - typename T, - typename std::enable_if_t() && !cudf::is_fixed_point()>* = nullptr> + template () && !cudf::is_fixed_point()>* = nullptr> static constexpr T identity() { return T{0}; } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { CUDF_FAIL("fixed_point does not yet support device operator identity"); @@ -93,13 +92,13 @@ struct DeviceSum { * @brief `count` operator - used in rolling windows */ struct DeviceCount { - template ()>* = nullptr> + template ()>* = nullptr> CUDF_HOST_DEVICE inline T operator()(const T& lhs, const T& rhs) { return T{DeviceCount{}(lhs.time_since_epoch(), rhs.time_since_epoch())}; } - template ()>* = nullptr> + template ()>* = nullptr> CUDF_HOST_DEVICE inline T operator()(const T&, const T& rhs) { return rhs + T{1}; @@ -123,10 +122,9 @@ struct DeviceMin { return numeric::detail::min(lhs, rhs); } - template < - typename T, - typename std::enable_if_t && !cudf::is_dictionary() && - !cudf::is_fixed_point()>* = nullptr> + template && !cudf::is_dictionary() && + !cudf::is_fixed_point()>* = nullptr> static constexpr T identity() { // chrono types do not have std::numeric_limits specializations and should use T::max() @@ -135,7 +133,7 @@ struct DeviceMin { return cuda::std::numeric_limits::max(); } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { CUDF_FAIL("fixed_point does not yet support DeviceMin identity"); @@ -143,13 +141,13 @@ struct DeviceMin { } // @brief identity specialized for string_view - template >* = nullptr> + template >* = nullptr> CUDF_HOST_DEVICE inline static constexpr T identity() { return string_view::max(); } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { return static_cast(T::max_value()); @@ -167,10 +165,9 @@ struct DeviceMax { return numeric::detail::max(lhs, rhs); } - template < - typename T, - typename std::enable_if_t && !cudf::is_dictionary() && - !cudf::is_fixed_point()>* = nullptr> + template && !cudf::is_dictionary() && + !cudf::is_fixed_point()>* = nullptr> static constexpr T identity() { // chrono types do not have std::numeric_limits specializations and should use T::min() @@ -179,20 +176,20 @@ struct DeviceMax { return cuda::std::numeric_limits::lowest(); } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { CUDF_FAIL("fixed_point does not yet support DeviceMax identity"); return cuda::std::numeric_limits::lowest(); } - template >* = nullptr> + template >* = nullptr> CUDF_HOST_DEVICE inline static constexpr T identity() { return string_view::min(); } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { return static_cast(T::lowest_value()); @@ -203,19 +200,19 @@ struct DeviceMax { * @brief binary `product` operator */ struct DeviceProduct { - template ()>* = nullptr> + template ()>* = nullptr> CUDF_HOST_DEVICE inline auto operator()(const T& lhs, const T& rhs) -> decltype(lhs * rhs) { return lhs * rhs; } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { return T{1}; } - template ()>* = nullptr> + template ()>* = nullptr> static constexpr T identity() { CUDF_FAIL("fixed_point does not yet support DeviceProduct identity"); @@ -227,7 +224,7 @@ struct DeviceProduct { * @brief binary `and` operator */ struct DeviceAnd { - template >* = nullptr> + template >* = nullptr> CUDF_HOST_DEVICE inline auto operator()(const T& lhs, const T& rhs) -> decltype(lhs & rhs) { return (lhs & rhs); @@ -238,7 +235,7 @@ struct DeviceAnd { * @brief binary `or` operator */ struct DeviceOr { - template >* = nullptr> + template >* = nullptr> CUDF_HOST_DEVICE inline auto operator()(const T& lhs, const T& rhs) -> decltype(lhs | rhs) { return (lhs | rhs); @@ -249,7 +246,7 @@ struct DeviceOr { * @brief binary `xor` operator */ struct DeviceXor { - template >* = nullptr> + template >* = nullptr> CUDF_HOST_DEVICE inline auto operator()(const T& lhs, const T& rhs) -> decltype(lhs ^ rhs) { return (lhs ^ rhs); diff --git a/cpp/include/cudf/utilities/span.hpp b/cpp/include/cudf/utilities/span.hpp index 9ccd4d21682..1172a5a68cd 100644 --- a/cpp/include/cudf/utilities/span.hpp +++ b/cpp/include/cudf/utilities/span.hpp @@ -159,9 +159,9 @@ struct host_span : public cudf::detail::span_base, - void>::type* = nullptr> + std::enable_if_t<(Extent == OtherExtent || Extent == dynamic_extent) && + std::is_convertible_v, + void>* = nullptr> constexpr host_span(const host_span& other) noexcept : base(other.data(), other.size()) { @@ -220,9 +220,9 @@ struct device_span : public cudf::detail::span_base, - void>::type* = nullptr> + std::enable_if_t<(Extent == OtherExtent || Extent == dynamic_extent) && + std::is_convertible_v, + void>* = nullptr> constexpr device_span(const device_span& other) noexcept : base(other.data(), other.size()) { @@ -283,9 +283,9 @@ class base_2dspan { template typename OtherRowType, - typename std::enable_if, - RowType>, - void>::type* = nullptr> + std::enable_if_t, + RowType>, + void>* = nullptr> constexpr base_2dspan(base_2dspan const& other) noexcept : _data{other.data()}, _size{other.size()} { diff --git a/cpp/include/cudf/utilities/traits.hpp b/cpp/include/cudf/utilities/traits.hpp index f1ad11a9030..504ec6de405 100644 --- a/cpp/include/cudf/utilities/traits.hpp +++ b/cpp/include/cudf/utilities/traits.hpp @@ -676,13 +676,13 @@ constexpr inline bool is_nested(data_type type) template struct is_bit_castable_to_impl { - template ()>* = nullptr> + template ()>* = nullptr> constexpr bool operator()() { return false; } - template ()>* = nullptr> + template ()>* = nullptr> constexpr bool operator()() { if (not cuda::std::is_trivially_copyable_v || @@ -696,13 +696,13 @@ struct is_bit_castable_to_impl { }; struct is_bit_castable_from_impl { - template ()>* = nullptr> + template ()>* = nullptr> constexpr bool operator()(data_type) { return false; } - template ()>* = nullptr> + template ()>* = nullptr> constexpr bool operator()(data_type to) { return cudf::type_dispatcher(to, is_bit_castable_to_impl{}); diff --git a/cpp/include/cudf_test/column_utilities.hpp b/cpp/include/cudf_test/column_utilities.hpp index aa77686fee4..cd96748f081 100644 --- a/cpp/include/cudf_test/column_utilities.hpp +++ b/cpp/include/cudf_test/column_utilities.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -178,7 +178,7 @@ bool validate_host_masks(std::vector const& expected_mask, * @return std::pair, std::vector> first is the * `column_view`'s data, and second is the column's bitmask. */ -template ()>* = nullptr> +template ()>* = nullptr> std::pair, std::vector> to_host(column_view c) { thrust::host_vector host_data(c.size()); @@ -197,7 +197,7 @@ std::pair, std::vector> to_host(column_view * @return std::pair, std::vector> first is the * `column_view`'s data, and second is the column's bitmask. */ -template ()>* = nullptr> +template ()>* = nullptr> std::pair, std::vector> to_host(column_view c) { using namespace numeric; diff --git a/cpp/include/cudf_test/column_wrapper.hpp b/cpp/include/cudf_test/column_wrapper.hpp index c190105e292..4005a4f9adc 100644 --- a/cpp/include/cudf_test/column_wrapper.hpp +++ b/cpp/include/cudf_test/column_wrapper.hpp @@ -93,31 +93,31 @@ class column_wrapper { template struct fixed_width_type_converter { // Are the types same - simply copy elements from [begin, end) to out - template , void>::type* = nullptr> + template , void>* = nullptr> constexpr ToT operator()(FromT element) const { return element; } // Are the types convertible or can target be constructed from source? - template && - (cudf::is_convertible::value || - std::is_constructible_v), - void>::type* = nullptr> + template < + typename FromT = From, + typename ToT = To, + std::enable_if_t && (cudf::is_convertible::value || + std::is_constructible_v), + void>* = nullptr> constexpr ToT operator()(FromT element) const { return static_cast(element); } // Convert integral values to timestamps - template && cudf::is_timestamp(), - void>::type* = nullptr> + template < + typename FromT = From, + typename ToT = To, + std::enable_if_t && cudf::is_timestamp(), void>* = nullptr> constexpr ToT operator()(FromT element) const { return ToT{typename ToT::duration{element}}; @@ -137,7 +137,7 @@ struct fixed_width_type_converter { template ()>* = nullptr> + std::enable_if_t()>* = nullptr> rmm::device_buffer make_elements(InputIterator begin, InputIterator end) { static_assert(cudf::is_fixed_width(), "Unexpected non-fixed width type."); @@ -162,8 +162,8 @@ rmm::device_buffer make_elements(InputIterator begin, InputIterator end) template () and - cudf::is_fixed_point()>* = nullptr> + std::enable_if_t() and + cudf::is_fixed_point()>* = nullptr> rmm::device_buffer make_elements(InputIterator begin, InputIterator end) { using RepType = typename ElementTo::rep; @@ -187,8 +187,8 @@ rmm::device_buffer make_elements(InputIterator begin, InputIterator end) template () and - cudf::is_fixed_point()>* = nullptr> + std::enable_if_t() and + cudf::is_fixed_point()>* = nullptr> rmm::device_buffer make_elements(InputIterator begin, InputIterator end) { using namespace numeric; diff --git a/cpp/include/cudf_test/type_lists.hpp b/cpp/include/cudf_test/type_lists.hpp index e84417c91d6..ac2892a0f34 100644 --- a/cpp/include/cudf_test/type_lists.hpp +++ b/cpp/include/cudf_test/type_lists.hpp @@ -80,9 +80,8 @@ constexpr auto types_to_ids() * @return Vector of TypeParam with the values specified */ template -typename std::enable_if() && - !cudf::is_timestamp_t::value, - thrust::host_vector>::type +std::enable_if_t() && !cudf::is_timestamp_t::value, + thrust::host_vector> make_type_param_vector(std::initializer_list const& init_list) { thrust::host_vector vec(init_list.size()); @@ -100,8 +99,7 @@ make_type_param_vector(std::initializer_list const& init_list) * @return Vector of TypeParam with the values specified */ template -typename std::enable_if::value, - thrust::host_vector>::type +std::enable_if_t::value, thrust::host_vector> make_type_param_vector(std::initializer_list const& init_list) { thrust::host_vector vec(init_list.size()); @@ -119,8 +117,7 @@ make_type_param_vector(std::initializer_list const& init_list) */ template -typename std::enable_if, - thrust::host_vector>::type +std::enable_if_t, thrust::host_vector> make_type_param_vector(std::initializer_list const& init_list) { thrust::host_vector vec(init_list.size()); diff --git a/cpp/src/binaryop/compiled/binary_ops.cu b/cpp/src/binaryop/compiled/binary_ops.cu index 995c6702cf8..c4538379836 100644 --- a/cpp/src/binaryop/compiled/binary_ops.cu +++ b/cpp/src/binaryop/compiled/binary_ops.cu @@ -119,9 +119,9 @@ struct compare_functor { // This is used to compare a scalar and a column value template - __device__ inline typename std::enable_if_t && - !std::is_same_v, - OutT> + __device__ inline std::enable_if_t && + !std::is_same_v, + OutT> operator()(cudf::size_type i) const { return cfunc_(lhs_dev_view_.is_valid(i), @@ -133,9 +133,9 @@ struct compare_functor { // This is used to compare a scalar and a column value template - __device__ inline typename std::enable_if_t && - std::is_same_v, - OutT> + __device__ inline std::enable_if_t && + std::is_same_v, + OutT> operator()(cudf::size_type i) const { return cfunc_(lhs_dev_view_.is_valid(), @@ -147,9 +147,9 @@ struct compare_functor { // This is used to compare 2 column values template - __device__ inline typename std::enable_if_t && - std::is_same_v, - OutT> + __device__ inline std::enable_if_t && + std::is_same_v, + OutT> operator()(cudf::size_type i) const { return cfunc_(lhs_dev_view_.is_valid(i), diff --git a/cpp/src/column/column_factories.cpp b/cpp/src/column/column_factories.cpp index fefe0b3c862..118a08ab26d 100644 --- a/cpp/src/column/column_factories.cpp +++ b/cpp/src/column/column_factories.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,21 +31,20 @@ namespace cudf { namespace { struct size_of_helper { cudf::data_type type; - template ()>* = nullptr> + template ()>* = nullptr> constexpr int operator()() const { CUDF_FAIL("Invalid, non fixed-width element type."); return 0; } - template () && not is_fixed_point()>* = nullptr> + template () && not is_fixed_point()>* = nullptr> constexpr int operator()() const noexcept { return sizeof(T); } - template ()>* = nullptr> + template ()>* = nullptr> constexpr int operator()() const noexcept { // Only want the sizeof fixed_point::Rep as fixed_point::scale is stored in data_type diff --git a/cpp/src/datetime/datetime_ops.cu b/cpp/src/datetime/datetime_ops.cu index 122ad4a9752..4dbe9faaa47 100644 --- a/cpp/src/datetime/datetime_ops.cu +++ b/cpp/src/datetime/datetime_ops.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -278,14 +278,14 @@ struct launch_functor { launch_functor(column_view inp, mutable_column_view out) : input(inp), output(out) {} template - typename std::enable_if_t::value, void> operator()( + std::enable_if_t::value, void> operator()( rmm::cuda_stream_view stream) const { CUDF_FAIL("Cannot extract datetime component from non-timestamp column."); } template - typename std::enable_if_t::value, void> operator()( + std::enable_if_t::value, void> operator()( rmm::cuda_stream_view stream) const { thrust::transform(rmm::exec_policy(stream), @@ -326,18 +326,18 @@ std::unique_ptr apply_datetime_op(column_view const& column, struct add_calendrical_months_functor { template - typename std::enable_if_t::value, std::unique_ptr> - operator()(Args&&...) const + std::enable_if_t::value, std::unique_ptr> operator()( + Args&&...) const { CUDF_FAIL("Cannot extract datetime component from non-timestamp column."); } template - typename std::enable_if_t::value, std::unique_ptr> - operator()(column_view timestamp_column, - MonthIterator months_begin, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) const + std::enable_if_t::value, std::unique_ptr> operator()( + column_view timestamp_column, + MonthIterator months_begin, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) const { auto size = timestamp_column.size(); auto output_col_type = timestamp_column.type(); diff --git a/cpp/src/dictionary/detail/concatenate.cu b/cpp/src/dictionary/detail/concatenate.cu index 301338fa1a8..871a36f7d62 100644 --- a/cpp/src/dictionary/detail/concatenate.cu +++ b/cpp/src/dictionary/detail/concatenate.cu @@ -122,8 +122,7 @@ struct compute_children_offsets_fn { */ struct dispatch_compute_indices { template - typename std::enable_if_t(), - std::unique_ptr> + std::enable_if_t(), std::unique_ptr> operator()(column_view const& all_keys, column_view const& all_indices, column_view const& new_keys, @@ -184,8 +183,7 @@ struct dispatch_compute_indices { } template - typename std::enable_if_t(), - std::unique_ptr> + std::enable_if_t(), std::unique_ptr> operator()(Args&&...) { CUDF_FAIL("dictionary concatenate not supported for this column type"); diff --git a/cpp/src/dictionary/set_keys.cu b/cpp/src/dictionary/set_keys.cu index c1fb1fa2180..7783e5f8daf 100644 --- a/cpp/src/dictionary/set_keys.cu +++ b/cpp/src/dictionary/set_keys.cu @@ -50,8 +50,7 @@ namespace { */ struct dispatch_compute_indices { template - typename std::enable_if_t(), - std::unique_ptr> + std::enable_if_t(), std::unique_ptr> operator()(dictionary_column_view const& input, column_view const& new_keys, rmm::cuda_stream_view stream, @@ -100,8 +99,7 @@ struct dispatch_compute_indices { } template - typename std::enable_if_t(), - std::unique_ptr> + std::enable_if_t(), std::unique_ptr> operator()(Args&&...) { CUDF_FAIL("dictionary set_keys not supported for this column type"); diff --git a/cpp/src/filling/sequence.cu b/cpp/src/filling/sequence.cu index c49142f91f9..e5bffcf21c1 100644 --- a/cpp/src/filling/sequence.cu +++ b/cpp/src/filling/sequence.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,9 +55,8 @@ struct const_tabulator { * by init and step. */ struct sequence_functor { - template < - typename T, - typename std::enable_if_t() and not cudf::is_boolean()>* = nullptr> + template () and not cudf::is_boolean()>* = nullptr> std::unique_ptr operator()(size_type size, scalar const& init, scalar const& step, @@ -83,9 +82,8 @@ struct sequence_functor { return result; } - template < - typename T, - typename std::enable_if_t() and not cudf::is_boolean()>* = nullptr> + template () and not cudf::is_boolean()>* = nullptr> std::unique_ptr operator()(size_type size, scalar const& init, rmm::cuda_stream_view stream, diff --git a/cpp/src/groupby/sort/group_nunique.cu b/cpp/src/groupby/sort/group_nunique.cu index 5154c867095..37d13d5aea3 100644 --- a/cpp/src/groupby/sort/group_nunique.cu +++ b/cpp/src/groupby/sort/group_nunique.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,14 +33,14 @@ namespace detail { namespace { struct nunique_functor { template - typename std::enable_if_t(), std::unique_ptr> - operator()(column_view const& values, - cudf::device_span group_labels, - size_type const num_groups, - cudf::device_span group_offsets, - null_policy null_handling, - rmm::cuda_stream_view stream, - rmm::mr::device_memory_resource* mr) + std::enable_if_t(), std::unique_ptr> operator()( + column_view const& values, + cudf::device_span group_labels, + size_type const num_groups, + cudf::device_span group_offsets, + null_policy null_handling, + rmm::cuda_stream_view stream, + rmm::mr::device_memory_resource* mr) { auto result = make_numeric_column( data_type(type_to_id()), num_groups, mask_state::UNALLOCATED, stream, mr); @@ -94,8 +94,8 @@ struct nunique_functor { } template - typename std::enable_if_t(), std::unique_ptr> - operator()(Args&&...) + std::enable_if_t(), std::unique_ptr> operator()( + Args&&...) { CUDF_FAIL("list_view group_nunique not supported yet"); } diff --git a/cpp/src/groupby/sort/group_tdigest.cu b/cpp/src/groupby/sort/group_tdigest.cu index f48ab852f24..f726de9bf3c 100644 --- a/cpp/src/groupby/sort/group_tdigest.cu +++ b/cpp/src/groupby/sort/group_tdigest.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -707,9 +707,8 @@ struct get_scalar_minmax { }; struct typed_group_tdigest { - template < - typename T, - typename std::enable_if_t() || cudf::is_fixed_point()>* = nullptr> + template () || cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& col, cudf::device_span group_offsets, cudf::device_span group_labels, @@ -766,10 +765,9 @@ struct typed_group_tdigest { mr); } - template < - typename T, - typename... Args, - typename std::enable_if_t() && !cudf::is_fixed_point()>* = nullptr> + template () && !cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(Args&&...) { CUDF_FAIL("Non-numeric type in group_tdigest"); diff --git a/cpp/src/io/csv/csv_gpu.cu b/cpp/src/io/csv/csv_gpu.cu index 13f5a57ac1f..e2e478af9ef 100644 --- a/cpp/src/io/csv/csv_gpu.cu +++ b/cpp/src/io/csv/csv_gpu.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -375,8 +375,8 @@ struct decode_op { * @return bool Whether the parsed value is valid. */ template and !std::is_same_v and - !cudf::is_fixed_point()>* = nullptr> + std::enable_if_t and !std::is_same_v and + !cudf::is_fixed_point()>* = nullptr> __host__ __device__ __forceinline__ bool operator()(void* out_buffer, size_t row, const data_type, @@ -402,7 +402,7 @@ struct decode_op { * * @return bool Whether the parsed value is valid. */ - template ()>* = nullptr> + template ()>* = nullptr> __host__ __device__ __forceinline__ bool operator()(void* out_buffer, size_t row, const data_type output_type, @@ -423,7 +423,7 @@ struct decode_op { /** * @brief Dispatch for boolean type types. */ - template >* = nullptr> + template >* = nullptr> __host__ __device__ __forceinline__ bool operator()(void* out_buffer, size_t row, const data_type, @@ -447,7 +447,7 @@ struct decode_op { * @brief Dispatch for floating points, which are set to NaN if the input * is not valid. In such case, the validity mask is set to zero too. */ - template >* = nullptr> + template >* = nullptr> __host__ __device__ __forceinline__ bool operator()(void* out_buffer, size_t row, const data_type, @@ -466,8 +466,8 @@ struct decode_op { * @brief Dispatch for all other types. */ template and !std::is_floating_point_v and - !cudf::is_fixed_point()>* = nullptr> + std::enable_if_t and !std::is_floating_point_v and + !cudf::is_fixed_point()>* = nullptr> __host__ __device__ __forceinline__ bool operator()(void* out_buffer, size_t row, const data_type, diff --git a/cpp/src/io/json/json_gpu.cu b/cpp/src/io/json/json_gpu.cu index 5cf0b03a6f1..21455e3ab93 100644 --- a/cpp/src/io/json/json_gpu.cu +++ b/cpp/src/io/json/json_gpu.cu @@ -216,7 +216,7 @@ struct ConvertFunctor { * It is handled here rather than within convertStrToValue() as that function * is used by other types (ex. timestamp) that aren't 'booleable'. */ - template >* = nullptr> + template >* = nullptr> __host__ __device__ __forceinline__ bool operator()(char const* begin, char const* end, void* output_column, @@ -240,7 +240,7 @@ struct ConvertFunctor { * @brief Dispatch for floating points, which are set to NaN if the input * is not valid. In such case, the validity mask is set to zero too. */ - template >* = nullptr> + template >* = nullptr> __host__ __device__ __forceinline__ bool operator()(char const* begin, char const* end, void* out_buffer, @@ -257,9 +257,8 @@ struct ConvertFunctor { * @brief Default template operator() dispatch specialization all data types * (including wrapper types) that is not covered by above. */ - template < - typename T, - typename std::enable_if_t and !std::is_integral_v>* = nullptr> + template and !std::is_integral_v>* = nullptr> __host__ __device__ __forceinline__ bool operator()(char const* begin, char const* end, void* output_column, diff --git a/cpp/src/io/orc/orc.h b/cpp/src/io/orc/orc.h index 386e3d8d73a..47020023419 100644 --- a/cpp/src/io/orc/orc.h +++ b/cpp/src/io/orc/orc.h @@ -137,56 +137,51 @@ int inline constexpr encode_field_number(int field_number, ProtofType field_type } namespace { -template < - typename base_t, - typename std::enable_if_t and !std::is_enum_v>* = nullptr> +template and !std::is_enum_v>* = nullptr> int static constexpr encode_field_number_base(int field_number) noexcept { return encode_field_number(field_number, ProtofType::FIXEDLEN); } -template < - typename base_t, - typename std::enable_if_t or std::is_enum_v>* = nullptr> +template or std::is_enum_v>* = nullptr> int static constexpr encode_field_number_base(int field_number) noexcept { return encode_field_number(field_number, ProtofType::VARINT); } -template >* = nullptr> +template >* = nullptr> int static constexpr encode_field_number_base(int field_number) noexcept { return encode_field_number(field_number, ProtofType::FIXED32); } -template >* = nullptr> +template >* = nullptr> int static constexpr encode_field_number_base(int field_number) noexcept { return encode_field_number(field_number, ProtofType::FIXED64); } }; // namespace -template < - typename T, - typename std::enable_if_t or std::is_same_v>* = nullptr> +template or std::is_same_v>* = nullptr> int constexpr encode_field_number(int field_number) noexcept { return encode_field_number_base(field_number); } // containters change the field number encoding -template < - typename T, - typename std::enable_if_t>>* = nullptr> +template >>* = nullptr> int constexpr encode_field_number(int field_number) noexcept { return encode_field_number_base(field_number); } // optional fields don't change the field number encoding -template < - typename T, - typename std::enable_if_t>>* = nullptr> +template >>* = nullptr> int constexpr encode_field_number(int field_number) noexcept { return encode_field_number_base(field_number); @@ -244,19 +239,19 @@ class ProtobufReader { uint32_t read_field_size(const uint8_t* end); - template >* = nullptr> + template >* = nullptr> void read_field(T& value, const uint8_t* end) { value = get(); } - template >* = nullptr> + template >* = nullptr> void read_field(T& value, const uint8_t* end) { value = static_cast(get()); } - template >* = nullptr> + template >* = nullptr> void read_field(T& value, const uint8_t* end) { auto const size = read_field_size(end); @@ -264,8 +259,7 @@ class ProtobufReader { m_cur += size; } - template >>* = nullptr> + template >>* = nullptr> void read_field(T& value, const uint8_t* end) { auto const size = read_field_size(end); @@ -273,10 +267,9 @@ class ProtobufReader { m_cur += size; } - template < - typename T, - typename std::enable_if_t> and - !std::is_same_v>* = nullptr> + template > and + !std::is_same_v>* = nullptr> void read_field(T& value, const uint8_t* end) { auto const size = read_field_size(end); @@ -284,9 +277,8 @@ class ProtobufReader { read(value.back(), size); } - template < - typename T, - typename std::enable_if_t>>* = nullptr> + template >>* = nullptr> void read_field(T& value, const uint8_t* end) { typename T::value_type contained_value; @@ -301,7 +293,7 @@ class ProtobufReader { read(value, size); } - template >* = nullptr> + template >* = nullptr> void read_field(T& value, const uint8_t* end) { memcpy(&value, m_cur, sizeof(T)); diff --git a/cpp/src/io/utilities/parsing_utils.cuh b/cpp/src/io/utilities/parsing_utils.cuh index d1b2e2862c6..74b98eff010 100644 --- a/cpp/src/io/utilities/parsing_utils.cuh +++ b/cpp/src/io/utilities/parsing_utils.cuh @@ -98,7 +98,7 @@ struct parse_options { * * @return uint8_t Numeric value of the character, or `0` */ -template >* = nullptr> +template >* = nullptr> constexpr uint8_t decode_digit(char c, bool* valid_flag) { if (c >= '0' && c <= '9') return c - '0'; @@ -119,7 +119,7 @@ constexpr uint8_t decode_digit(char c, bool* valid_flag) * * @return uint8_t Numeric value of the character, or `0` */ -template >* = nullptr> +template >* = nullptr> constexpr uint8_t decode_digit(char c, bool* valid_flag) { if (c >= '0' && c <= '9') return c - '0'; diff --git a/cpp/src/reductions/scan/scan_exclusive.cu b/cpp/src/reductions/scan/scan_exclusive.cu index bf9b06a3602..9811a986224 100644 --- a/cpp/src/reductions/scan/scan_exclusive.cu +++ b/cpp/src/reductions/scan/scan_exclusive.cu @@ -50,7 +50,7 @@ struct scan_dispatcher { * @param mr Device memory resource used to allocate the returned column's device memory * @return Output column with scan results */ - template >* = nullptr> + template >* = nullptr> std::unique_ptr operator()(column_view const& input, null_policy, rmm::cuda_stream_view stream, diff --git a/cpp/src/reductions/scan/scan_inclusive.cu b/cpp/src/reductions/scan/scan_inclusive.cu index bf2c83b5b8d..ee865f09f2e 100644 --- a/cpp/src/reductions/scan/scan_inclusive.cu +++ b/cpp/src/reductions/scan/scan_inclusive.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -225,7 +225,7 @@ struct scan_dispatcher { * * @tparam T type of input column */ - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(column_view const& input, null_policy, rmm::cuda_stream_view stream, diff --git a/cpp/src/reductions/simple.cuh b/cpp/src/reductions/simple.cuh index 7dc8e6cb2c4..e5303246452 100644 --- a/cpp/src/reductions/simple.cuh +++ b/cpp/src/reductions/simple.cuh @@ -260,8 +260,7 @@ struct same_element_type_dispatcher { return !(cudf::is_dictionary() || std::is_same_v); } - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr resolve_key(column_view const& keys, scalar const& keys_index, rmm::cuda_stream_view stream, @@ -271,8 +270,7 @@ struct same_element_type_dispatcher { return cudf::detail::get_element(keys, index.value(stream), stream, mr); } - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr resolve_key(column_view const&, scalar const&, rmm::cuda_stream_view, @@ -353,7 +351,7 @@ struct element_type_dispatcher { * @brief Specialization for reducing floating-point column types to any output type. */ template >* = nullptr> + std::enable_if_t>* = nullptr> std::unique_ptr reduce_numeric(column_view const& col, data_type const output_type, rmm::cuda_stream_view stream, @@ -375,8 +373,7 @@ struct element_type_dispatcher { /** * @brief Specialization for reducing integer column types to any output type. */ - template >* = nullptr> + template >* = nullptr> std::unique_ptr reduce_numeric(column_view const& col, data_type const output_type, rmm::cuda_stream_view stream, @@ -405,8 +402,7 @@ struct element_type_dispatcher { * @param stream CUDA stream used for device memory operations and kernel launches. * @param mr Device memory resource used to allocate the returned scalar's device memory */ - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(column_view const& col, data_type const output_type, rmm::cuda_stream_view stream, @@ -423,8 +419,7 @@ struct element_type_dispatcher { /** * @brief Specialization for reducing fixed_point column types to fixed_point number */ - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(column_view const& col, data_type const output_type, rmm::cuda_stream_view stream, @@ -436,8 +431,8 @@ struct element_type_dispatcher { } template () and - not cudf::is_fixed_point()>* = nullptr> + std::enable_if_t() and + not cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const&, data_type const, rmm::cuda_stream_view, diff --git a/cpp/src/replace/nulls.cu b/cpp/src/replace/nulls.cu index 93bc6cf5ae5..d41bdb6ca5a 100644 --- a/cpp/src/replace/nulls.cu +++ b/cpp/src/replace/nulls.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -297,8 +297,7 @@ struct replace_nulls_functor { * `replace_nulls` with the appropriate data types. */ struct replace_nulls_scalar_kernel_forwarder { - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, cudf::scalar const& replacement, rmm::cuda_stream_view stream, diff --git a/cpp/src/reshape/interleave_columns.cu b/cpp/src/reshape/interleave_columns.cu index 0e3ead3fd99..cd66cad392e 100644 --- a/cpp/src/reshape/interleave_columns.cu +++ b/cpp/src/reshape/interleave_columns.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,7 +53,7 @@ struct interleave_columns_functor { }; template -struct interleave_columns_impl>> { +struct interleave_columns_impl>> { std::unique_ptr operator()(table_view const& lists_columns, bool create_mask, rmm::cuda_stream_view stream, @@ -64,7 +64,7 @@ struct interleave_columns_impl -struct interleave_columns_impl>> { +struct interleave_columns_impl>> { std::unique_ptr operator()(table_view const& structs_columns, bool create_mask, rmm::cuda_stream_view stream, @@ -131,7 +131,7 @@ struct interleave_columns_impl -struct interleave_columns_impl>> { +struct interleave_columns_impl>> { std::unique_ptr operator()(table_view const& strings_columns, bool create_mask, rmm::cuda_stream_view stream, @@ -214,7 +214,7 @@ struct interleave_columns_impl -struct interleave_columns_impl()>> { +struct interleave_columns_impl()>> { std::unique_ptr operator()(table_view const& input, bool create_mask, rmm::cuda_stream_view stream, diff --git a/cpp/src/rolling/rolling_detail.cuh b/cpp/src/rolling/rolling_detail.cuh index 7c52856b147..958da04e57c 100644 --- a/cpp/src/rolling/rolling_detail.cuh +++ b/cpp/src/rolling/rolling_detail.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021, NVIDIA CORPORATION. + * Copyright (c) 2020-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -91,14 +91,14 @@ struct DeviceRolling { // operations we do support template - DeviceRolling(size_type _min_periods, typename std::enable_if_t()>* = nullptr) + DeviceRolling(size_type _min_periods, std::enable_if_t()>* = nullptr) : min_periods(_min_periods) { } // operations we don't support template - DeviceRolling(size_type _min_periods, typename std::enable_if_t()>* = nullptr) + DeviceRolling(size_type _min_periods, std::enable_if_t()>* = nullptr) : min_periods(_min_periods) { CUDF_FAIL("Invalid aggregation/type pair"); @@ -441,12 +441,12 @@ struct DeviceRollingLead { return cudf::is_fixed_width(); } - template ()>* = nullptr> + template ()>* = nullptr> DeviceRollingLead(size_type _row_offset) : row_offset(_row_offset) { } - template ()>* = nullptr> + template ()>* = nullptr> DeviceRollingLead(size_type _row_offset) : row_offset(_row_offset) { CUDF_FAIL("Invalid aggregation/type pair"); @@ -497,12 +497,12 @@ struct DeviceRollingLag { return cudf::is_fixed_width(); } - template ()>* = nullptr> + template ()>* = nullptr> DeviceRollingLag(size_type _row_offset) : row_offset(_row_offset) { } - template ()>* = nullptr> + template ()>* = nullptr> DeviceRollingLag(size_type _row_offset) : row_offset(_row_offset) { CUDF_FAIL("Invalid aggregation/type pair"); diff --git a/cpp/src/round/round.cu b/cpp/src/round/round.cu index 81bf03f7c0a..9a2b1002997 100644 --- a/cpp/src/round/round.cu +++ b/cpp/src/round/round.cu @@ -49,26 +49,26 @@ inline double __device__ generic_round_half_even(double d) { return rint(d); } inline float __device__ generic_modf(float a, float* b) { return modff(a, b); } inline double __device__ generic_modf(double a, double* b) { return modf(a, b); } -template >* = nullptr> +template >* = nullptr> T __device__ generic_abs(T value) { return numeric::detail::abs(value); } -template >* = nullptr> +template >* = nullptr> T __device__ generic_abs(T value) { return value; } -template >* = nullptr> +template >* = nullptr> int16_t __device__ generic_sign(T value) { return value < 0 ? -1 : 1; } // this is needed to suppress warning: pointless comparison of unsigned integer with zero -template >* = nullptr> +template >* = nullptr> int16_t __device__ generic_sign(T) { return 1; @@ -83,13 +83,13 @@ constexpr inline auto is_supported_round_type() template struct half_up_zero { T n; // unused in the decimal_places = 0 case - template ()>* = nullptr> + template ()>* = nullptr> __device__ U operator()(U e) { return generic_round(e); } - template >* = nullptr> + template >* = nullptr> __device__ U operator()(U) { assert(false); // Should never get here. Just for compilation @@ -100,7 +100,7 @@ struct half_up_zero { template struct half_up_positive { T n; - template ()>* = nullptr> + template ()>* = nullptr> __device__ U operator()(U e) { T integer_part; @@ -108,7 +108,7 @@ struct half_up_positive { return integer_part + generic_round(fractional_part * n) / n; } - template >* = nullptr> + template >* = nullptr> __device__ U operator()(U) { assert(false); // Should never get here. Just for compilation @@ -119,13 +119,13 @@ struct half_up_positive { template struct half_up_negative { T n; - template ()>* = nullptr> + template ()>* = nullptr> __device__ U operator()(U e) { return generic_round(e / n) * n; } - template >* = nullptr> + template >* = nullptr> __device__ U operator()(U e) { auto const down = (e / n) * n; // result from rounding down @@ -136,13 +136,13 @@ struct half_up_negative { template struct half_even_zero { T n; // unused in the decimal_places = 0 case - template ()>* = nullptr> + template ()>* = nullptr> __device__ U operator()(U e) { return generic_round_half_even(e); } - template >* = nullptr> + template >* = nullptr> __device__ U operator()(U) { assert(false); // Should never get here. Just for compilation @@ -153,7 +153,7 @@ struct half_even_zero { template struct half_even_positive { T n; - template ()>* = nullptr> + template ()>* = nullptr> __device__ U operator()(U e) { T integer_part; @@ -161,7 +161,7 @@ struct half_even_positive { return integer_part + generic_round_half_even(fractional_part * n) / n; } - template >* = nullptr> + template >* = nullptr> __device__ U operator()(U) { assert(false); // Should never get here. Just for compilation @@ -172,13 +172,13 @@ struct half_even_positive { template struct half_even_negative { T n; - template ()>* = nullptr> + template ()>* = nullptr> __device__ U operator()(U e) { return generic_round_half_even(e / n) * n; } - template >* = nullptr> + template >* = nullptr> __device__ U operator()(U e) { auto const down_over_n = e / n; // use this to determine HALF_EVEN case @@ -205,7 +205,7 @@ struct half_even_fixed_point { template typename RoundFunctor, - typename std::enable_if_t()>* = nullptr> + std::enable_if_t()>* = nullptr> std::unique_ptr round_with(column_view const& input, int32_t decimal_places, rmm::cuda_stream_view stream, @@ -231,7 +231,7 @@ std::unique_ptr round_with(column_view const& input, template typename RoundFunctor, - typename std::enable_if_t()>* = nullptr> + std::enable_if_t()>* = nullptr> std::unique_ptr round_with(column_view const& input, int32_t decimal_places, rmm::cuda_stream_view stream, diff --git a/cpp/src/scalar/scalar_factories.cpp b/cpp/src/scalar/scalar_factories.cpp index c18b57d220f..3a2920f8f1a 100644 --- a/cpp/src/scalar/scalar_factories.cpp +++ b/cpp/src/scalar/scalar_factories.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,8 +28,8 @@ namespace cudf { namespace { struct scalar_construction_helper { template , - typename std::enable_if_t() and not is_fixed_point()>* = nullptr> + typename ScalarType = scalar_type_t, + std::enable_if_t() and not is_fixed_point()>* = nullptr> std::unique_ptr operator()(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const { @@ -39,8 +39,8 @@ struct scalar_construction_helper { } template , - typename std::enable_if_t()>* = nullptr> + typename ScalarType = scalar_type_t, + std::enable_if_t()>* = nullptr> std::unique_ptr operator()(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) const { @@ -49,9 +49,7 @@ struct scalar_construction_helper { return std::unique_ptr(s); } - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(Args... args) const { CUDF_FAIL("Invalid type."); @@ -124,14 +122,14 @@ namespace { struct default_scalar_functor { data_type type; - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { return make_fixed_width_scalar(data_type(type_to_id()), stream, mr); } - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) { diff --git a/cpp/src/sort/sort.cu b/cpp/src/sort/sort.cu index 42b57bdb47a..5ce82cd3740 100644 --- a/cpp/src/sort/sort.cu +++ b/cpp/src/sort/sort.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ std::unique_ptr sort_by_key(table_view const& values, } struct inplace_column_sort_fn { - template ()>* = nullptr> + template ()>* = nullptr> void operator()(mutable_column_view& col, bool ascending, rmm::cuda_stream_view stream) const { CUDF_EXPECTS(!col.has_nulls(), "Nulls not supported for in-place sort"); @@ -68,7 +68,7 @@ struct inplace_column_sort_fn { } } - template ()>* = nullptr> + template ()>* = nullptr> void operator()(mutable_column_view&, bool, rmm::cuda_stream_view) const { CUDF_FAIL("Column type must be relationally comparable and fixed-width"); diff --git a/cpp/src/sort/sort_column.cu b/cpp/src/sort/sort_column.cu index 74c796e7962..7a4072cf8ae 100644 --- a/cpp/src/sort/sort_column.cu +++ b/cpp/src/sort/sort_column.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ struct column_sorted_order_fn { * @param ascending True if sort order is ascending * @param stream CUDA stream used for device memory operations and kernel launches */ - template ()>* = nullptr> + template ()>* = nullptr> void radix_sort(column_view const& input, mutable_column_view& indices, bool ascending, @@ -68,7 +68,7 @@ struct column_sorted_order_fn { thrust::greater()); } } - template ()>* = nullptr> + template ()>* = nullptr> void radix_sort(column_view const&, mutable_column_view&, bool, rmm::cuda_stream_view) { CUDF_FAIL("Only fixed-width types are suitable for faster sorting"); @@ -85,8 +85,7 @@ struct column_sorted_order_fn { * @param null_precedence How null rows are to be ordered * @param stream CUDA stream used for device memory operations and kernel launches */ - template ()>* = nullptr> + template ()>* = nullptr> void operator()(column_view const& input, mutable_column_view& indices, bool ascending, @@ -105,8 +104,7 @@ struct column_sorted_order_fn { } } - template ()>* = nullptr> + template ()>* = nullptr> void operator()(column_view const&, mutable_column_view&, bool, null_order, rmm::cuda_stream_view) { CUDF_FAIL("Column type must be relationally comparable"); diff --git a/cpp/src/sort/stable_sort_column.cu b/cpp/src/sort/stable_sort_column.cu index 49aecf52625..d79a691a580 100644 --- a/cpp/src/sort/stable_sort_column.cu +++ b/cpp/src/sort/stable_sort_column.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ struct column_stable_sorted_order_fn { * @param indices Output sorted indices * @param stream CUDA stream used for device memory operations and kernel launches */ - template ()>* = nullptr> + template ()>* = nullptr> void faster_stable_sort(column_view const& input, mutable_column_view& indices, rmm::cuda_stream_view stream) @@ -38,7 +38,7 @@ struct column_stable_sorted_order_fn { thrust::stable_sort_by_key( rmm::exec_policy(stream), d_col.begin(), d_col.end(), indices.begin()); } - template ()>* = nullptr> + template ()>* = nullptr> void faster_stable_sort(column_view const&, mutable_column_view&, rmm::cuda_stream_view) { CUDF_FAIL("Only fixed-width types are suitable for faster stable sorting"); @@ -55,8 +55,7 @@ struct column_stable_sorted_order_fn { * @param null_precedence How null rows are to be ordered * @param stream CUDA stream used for device memory operations and kernel launches */ - template ()>* = nullptr> + template ()>* = nullptr> void operator()(column_view const& input, mutable_column_view& indices, bool ascending, @@ -74,8 +73,7 @@ struct column_stable_sorted_order_fn { faster_stable_sort(input, indices, stream); } } - template ()>* = nullptr> + template ()>* = nullptr> void operator()(column_view const&, mutable_column_view&, bool, null_order, rmm::cuda_stream_view) { CUDF_FAIL("Column type must be relationally comparable"); diff --git a/cpp/src/unary/cast_ops.cu b/cpp/src/unary/cast_ops.cu index 5cc4ce5f6c9..f77ab7aa3d9 100644 --- a/cpp/src/unary/cast_ops.cu +++ b/cpp/src/unary/cast_ops.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,19 +34,19 @@ namespace detail { namespace { // anonymous namespace template struct unary_cast { - template () && - cudf::is_numeric())>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_numeric() && cudf::is_numeric())>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { return static_cast(element); } - template () && - cudf::is_timestamp())>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_timestamp() && cudf::is_timestamp())>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { // Convert source tick counts into target tick counts without blindly truncating them @@ -55,46 +55,46 @@ struct unary_cast { return TargetT{cuda::std::chrono::floor(element.time_since_epoch())}; } - template () && - cudf::is_duration())>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_duration() && cudf::is_duration())>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { return TargetT{cuda::std::chrono::floor(element)}; } - template () && - cudf::is_duration()>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t() && cudf::is_duration()>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { return TargetT{static_cast(element)}; } - template () && - cudf::is_duration())>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_timestamp() && cudf::is_duration())>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { return TargetT{cuda::std::chrono::floor(element.time_since_epoch())}; } - template () && - cudf::is_numeric()>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t() && cudf::is_numeric()>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { return static_cast(element.count()); } - template () && - cudf::is_timestamp())>* = nullptr> + template < + typename SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_duration() && cudf::is_timestamp())>* = nullptr> __device__ inline TargetT operator()(SourceT const element) { return TargetT{cuda::std::chrono::floor(element)}; @@ -107,20 +107,20 @@ struct fixed_point_unary_cast { using FixedPointT = std::conditional_t(), _SourceT, _TargetT>; using DeviceT = device_storage_type_t; - template () && - cudf::is_numeric())>* = nullptr> + template < + typename SourceT = _SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_fixed_point<_SourceT>() && cudf::is_numeric())>* = nullptr> __device__ inline TargetT operator()(DeviceT const element) { auto const fp = SourceT{numeric::scaled_integer{element, scale}}; return static_cast(fp); } - template () && - cudf::is_fixed_point())>* = nullptr> + template < + typename SourceT = _SourceT, + typename TargetT = _TargetT, + std::enable_if_t<(cudf::is_numeric<_SourceT>() && cudf::is_fixed_point())>* = nullptr> __device__ inline DeviceT operator()(SourceT const element) { return TargetT{element, scale}.value(); @@ -169,7 +169,7 @@ struct device_cast { * * @return std::unique_ptr Returned column with new @p scale */ -template ()>* = nullptr> +template ()>* = nullptr> std::unique_ptr rescale(column_view input, numeric::scale_type scale, rmm::cuda_stream_view stream, @@ -207,10 +207,9 @@ struct dispatch_unary_cast_to { dispatch_unary_cast_to(column_view inp) : input(inp) {} - template < - typename TargetT, - typename SourceT = _SourceT, - typename std::enable_if_t()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(data_type type, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -234,10 +233,10 @@ struct dispatch_unary_cast_to { return output; } - template () && - cudf::is_numeric()>* = nullptr> + template < + typename TargetT, + typename SourceT = _SourceT, + std::enable_if_t() && cudf::is_numeric()>* = nullptr> std::unique_ptr operator()(data_type type, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -264,10 +263,10 @@ struct dispatch_unary_cast_to { return output; } - template () && - cudf::is_fixed_point()>* = nullptr> + template < + typename TargetT, + typename SourceT = _SourceT, + std::enable_if_t() && cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(data_type type, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -294,11 +293,10 @@ struct dispatch_unary_cast_to { return output; } - template < - typename TargetT, - typename SourceT = _SourceT, - typename std::enable_if_t() && cudf::is_fixed_point() && - std::is_same_v>* = nullptr> + template () && cudf::is_fixed_point() && + std::is_same_v>* = nullptr> std::unique_ptr operator()(data_type type, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -310,11 +308,10 @@ struct dispatch_unary_cast_to { return detail::rescale(input, numeric::scale_type{type.scale()}, stream, mr); } - template < - typename TargetT, - typename SourceT = _SourceT, - typename std::enable_if_t() && cudf::is_fixed_point() && - not std::is_same_v>* = nullptr> + template () && cudf::is_fixed_point() && + not std::is_same_v>* = nullptr> std::unique_ptr operator()(data_type type, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -356,8 +353,8 @@ struct dispatch_unary_cast_to { } template ()>* = nullptr> + typename SourceT = _SourceT, + std::enable_if_t()>* = nullptr> std::unique_ptr operator()(data_type, rmm::cuda_stream_view, rmm::mr::device_memory_resource*) @@ -379,7 +376,7 @@ struct dispatch_unary_cast_from { dispatch_unary_cast_from(column_view inp) : input(inp) {} - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(data_type type, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) diff --git a/cpp/src/unary/math_ops.cu b/cpp/src/unary/math_ops.cu index 474c7b76ddc..e92d5a1ca7e 100644 --- a/cpp/src/unary/math_ops.cu +++ b/cpp/src/unary/math_ops.cu @@ -348,7 +348,7 @@ std::unique_ptr transform_fn(cudf::dictionary_column_view const& i template struct MathOpDispatcher { - template >* = nullptr> + template >* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -362,7 +362,7 @@ struct MathOpDispatcher { } struct dictionary_dispatch { - template >* = nullptr> + template >* = nullptr> std::unique_ptr operator()(cudf::dictionary_column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -377,9 +377,9 @@ struct MathOpDispatcher { } }; - template and - std::is_same_v>* = nullptr> + template < + typename T, + std::enable_if_t and std::is_same_v>* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -401,7 +401,7 @@ struct MathOpDispatcher { template struct BitwiseOpDispatcher { - template >* = nullptr> + template >* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -415,7 +415,7 @@ struct BitwiseOpDispatcher { } struct dictionary_dispatch { - template >* = nullptr> + template >* = nullptr> std::unique_ptr operator()(cudf::dictionary_column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -431,8 +431,7 @@ struct BitwiseOpDispatcher { }; template and std::is_same_v>* = - nullptr> + std::enable_if_t and std::is_same_v>* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -462,7 +461,7 @@ struct LogicalOpDispatcher { } public: - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -477,7 +476,7 @@ struct LogicalOpDispatcher { } struct dictionary_dispatch { - template ()>* = nullptr> + template ()>* = nullptr> std::unique_ptr operator()(cudf::dictionary_column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) @@ -499,9 +498,8 @@ struct LogicalOpDispatcher { } }; - template < - typename T, - typename std::enable_if_t() and std::is_same_v>* = nullptr> + template () and std::is_same_v>* = nullptr> std::unique_ptr operator()(cudf::column_view const& input, rmm::cuda_stream_view stream, rmm::mr::device_memory_resource* mr) diff --git a/cpp/tests/binaryop/binop-fixture.hpp b/cpp/tests/binaryop/binop-fixture.hpp index 65243b1ae2e..2ba5561826e 100644 --- a/cpp/tests/binaryop/binop-fixture.hpp +++ b/cpp/tests/binaryop/binop-fixture.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Copyright 2018-2019 BlazingDB, Inc. * Copyright 2018 Christian Noboa Mardini @@ -61,14 +61,14 @@ struct BinaryOperationTest : public cudf::test::BaseFixture { return cudf::test::fixed_width_column_wrapper(data_iter, data_iter + size, validity_iter); } - template >* = nullptr> + template >* = nullptr> auto make_random_wrapped_scalar() { cudf::test::UniformRandomGenerator rand_gen(r_min, r_max); return cudf::scalar_type_t(rand_gen.generate()); } - template >* = nullptr> + template >* = nullptr> auto make_random_wrapped_scalar() { cudf::test::UniformRandomGenerator rand_gen(r_min, r_max); diff --git a/cpp/tests/binaryop/util/operation.h b/cpp/tests/binaryop/util/operation.h index d78ad8938c4..93a84a7bc49 100644 --- a/cpp/tests/binaryop/util/operation.h +++ b/cpp/tests/binaryop/util/operation.h @@ -32,19 +32,19 @@ template struct Add { // Allow sum between chronos only when both input and output types // are chronos. Unsupported combinations will fail to compile - template () && cudf::is_chrono() && - cudf::is_chrono(), - void>::type* = nullptr> + template () && cudf::is_chrono() && + cudf::is_chrono(), + void>* = nullptr> OutT operator()(TypeLhs lhs, TypeRhs rhs) const { return lhs + rhs; } - template () || !cudf::is_chrono() || - !cudf::is_chrono(), - void>::type* = nullptr> + template () || !cudf::is_chrono() || + !cudf::is_chrono(), + void>* = nullptr> OutT operator()(TypeLhs lhs, TypeRhs rhs) const { using TypeCommon = typename std::common_type::type; @@ -56,19 +56,19 @@ template struct Sub { // Allow difference between chronos only when both input and output types // are chronos. Unsupported combinations will fail to compile - template () && cudf::is_chrono() && - cudf::is_chrono(), - void>::type* = nullptr> + template () && cudf::is_chrono() && + cudf::is_chrono(), + void>* = nullptr> OutT operator()(TypeLhs lhs, TypeRhs rhs) const { return lhs - rhs; } - template () || !cudf::is_chrono() || - !cudf::is_chrono(), - void>::type* = nullptr> + template () || !cudf::is_chrono() || + !cudf::is_chrono(), + void>* = nullptr> OutT operator()(TypeLhs lhs, TypeRhs rhs) const { using TypeCommon = typename std::common_type::type; @@ -78,28 +78,27 @@ struct Sub { template struct Mul { - template ::value, void>::type* = nullptr> + template ::value, void>* = nullptr> TypeOut operator()(TypeLhs lhs, TypeRhs rhs) const { using TypeCommon = typename std::common_type::type; return static_cast(static_cast(lhs) * static_cast(rhs)); } - template ::value, void>::type* = nullptr> + template ::value, void>* = nullptr> TypeOut operator()(TypeLhs x, TypeRhs y) const { return DurationProduct(x, y); } - template < - typename OutT, - typename LhsT, - typename RhsT, - typename std::enable_if<(cudf::is_duration_t::value && std::is_integral_v) || - (cudf::is_duration_t::value && std::is_integral_v), - void>::type* = nullptr> + template ::value && std::is_integral_v) || + (cudf::is_duration_t::value && std::is_integral_v), + void>* = nullptr> OutT DurationProduct(LhsT x, RhsT y) const { return x * y; @@ -108,26 +107,26 @@ struct Mul { template struct Div { - template ::value, void>::type* = nullptr> + template ::value, void>* = nullptr> TypeOut operator()(TypeLhs lhs, TypeRhs rhs) { using TypeCommon = typename std::common_type::type; return static_cast(static_cast(lhs) / static_cast(rhs)); } - template ::value, void>::type* = nullptr> + template ::value, void>* = nullptr> TypeOut operator()(TypeLhs x, TypeRhs y) const { return DurationDivide(x, y); } - template || cudf::is_duration()), - void>::type* = nullptr> + template < + typename OutT, + typename LhsT, + typename RhsT, + std::enable_if_t<(std::is_integral_v || cudf::is_duration()), void>* = nullptr> OutT DurationDivide(LhsT x, RhsT y) const { return x / y; @@ -185,10 +184,10 @@ struct Mod { } // Mod with duration types - duration % (integral or a duration) = duration - template ::value && - cudf::is_duration_t::value>* = nullptr> + template ::value && + cudf::is_duration_t::value>* = nullptr> TypeOut operator()(TypeLhs lhs, TypeRhs rhs) { return lhs % rhs; diff --git a/cpp/tests/copying/copy_tests.cpp b/cpp/tests/copying/copy_tests.cpp index 4254794bf19..62f1300c284 100644 --- a/cpp/tests/copying/copy_tests.cpp +++ b/cpp/tests/copying/copy_tests.cpp @@ -378,18 +378,16 @@ TYPED_TEST(CopyTestNumeric, CopyIfElseTestScalarScalar) template struct create_chrono_scalar { template - typename std::enable_if_t< - std::is_same_v::type, std::true_type>, - cudf::timestamp_scalar> + std::enable_if_t::type, std::true_type>, + cudf::timestamp_scalar> operator()(Args&&... args) const { return cudf::timestamp_scalar(std::forward(args)...); } template - typename std::enable_if_t< - std::is_same_v::type, std::true_type>, - cudf::duration_scalar> + std::enable_if_t::type, std::true_type>, + cudf::duration_scalar> operator()(Args&&... args) const { return cudf::duration_scalar(std::forward(args)...); diff --git a/cpp/tests/device_atomics/device_atomics_test.cu b/cpp/tests/device_atomics/device_atomics_test.cu index fd065249c4e..31174d3fd72 100644 --- a/cpp/tests/device_atomics/device_atomics_test.cu +++ b/cpp/tests/device_atomics/device_atomics_test.cu @@ -51,7 +51,7 @@ constexpr inline bool is_timestamp_sum() // Disable SUM of TIMESTAMP types template ()>* = nullptr> + std::enable_if_t()>* = nullptr> __device__ T atomic_op(T* addr, T const& value, BinaryOp op) { return {}; @@ -59,7 +59,7 @@ __device__ T atomic_op(T* addr, T const& value, BinaryOp op) template ()>* = nullptr> + std::enable_if_t()>* = nullptr> __device__ T atomic_op(T* addr, T const& value, BinaryOp op) { T old_value = *addr; @@ -92,13 +92,13 @@ __global__ void gpu_atomicCAS_test(T* result, T* data, size_t size) } template -typename std::enable_if_t(), T> accumulate(cudf::host_span xs) +std::enable_if_t(), T> accumulate(cudf::host_span xs) { return std::accumulate(xs.begin(), xs.end(), T{0}); } template -typename std::enable_if_t(), T> accumulate(cudf::host_span xs) +std::enable_if_t(), T> accumulate(cudf::host_span xs) { auto ys = std::vector(xs.size()); std::transform( diff --git a/cpp/tests/groupby/tdigest_tests.cu b/cpp/tests/groupby/tdigest_tests.cu index 2591f395914..b0ce22bae7c 100644 --- a/cpp/tests/groupby/tdigest_tests.cu +++ b/cpp/tests/groupby/tdigest_tests.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,9 +60,8 @@ struct column_max { }; struct tdigest_gen { - template < - typename T, - typename std::enable_if_t() || cudf::is_fixed_point()>* = nullptr> + template () || cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta) { cudf::table_view t({keys}); @@ -75,9 +74,8 @@ struct tdigest_gen { return std::move(result.second[0].results[0]); } - template < - typename T, - typename std::enable_if_t() && !cudf::is_fixed_point()>* = nullptr> + template () && !cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta) { CUDF_FAIL("Invalid tdigest test type"); diff --git a/cpp/tests/io/csv_test.cpp b/cpp/tests/io/csv_test.cpp index 08cdbb10359..e5e44b1aa6e 100644 --- a/cpp/tests/io/csv_test.cpp +++ b/cpp/tests/io/csv_test.cpp @@ -262,7 +262,7 @@ void check_string_column(cudf::column_view const& col_lhs, } // Helper function to compare two floating-point column contents -template >* = nullptr> +template >* = nullptr> void expect_column_data_equal(std::vector const& lhs, cudf::column_view const& rhs) { EXPECT_THAT(cudf::test::to_host(rhs).first, @@ -270,7 +270,7 @@ void expect_column_data_equal(std::vector const& lhs, cudf::column_view const } // Helper function to compare two column contents -template >* = nullptr> +template >* = nullptr> void expect_column_data_equal(std::vector const& lhs, cudf::column_view const& rhs) { EXPECT_THAT(cudf::test::to_host(rhs).first, ::testing::ElementsAreArray(lhs)); diff --git a/cpp/tests/quantiles/percentile_approx_test.cu b/cpp/tests/quantiles/percentile_approx_test.cu index 2f4d5a7a604..035cd664aa2 100644 --- a/cpp/tests/quantiles/percentile_approx_test.cu +++ b/cpp/tests/quantiles/percentile_approx_test.cu @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include @@ -21,9 +36,8 @@ using namespace cudf; using namespace cudf::tdigest; struct tdigest_gen { - template < - typename T, - typename std::enable_if_t() || cudf::is_fixed_point()>* = nullptr> + template () || cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta) { cudf::table_view t({keys}); @@ -36,9 +50,8 @@ struct tdigest_gen { return std::move(result.second[0].results[0]); } - template < - typename T, - typename std::enable_if_t() && !cudf::is_fixed_point()>* = nullptr> + template () && !cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta) { CUDF_FAIL("Invalid tdigest test type"); @@ -89,9 +102,8 @@ std::unique_ptr arrow_percentile_approx(column_view const& _values, } struct percentile_approx_dispatch { - template < - typename T, - typename std::enable_if_t() || cudf::is_fixed_point()>* = nullptr> + template () || cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta, @@ -127,9 +139,8 @@ struct percentile_approx_dispatch { return result; } - template < - typename T, - typename std::enable_if_t() && !cudf::is_fixed_point()>* = nullptr> + template () && !cudf::is_fixed_point()>* = nullptr> std::unique_ptr operator()(column_view const& keys, column_view const& values, int delta, diff --git a/cpp/tests/reductions/reduction_tests.cpp b/cpp/tests/reductions/reduction_tests.cpp index fcecc124978..276b244dac6 100644 --- a/cpp/tests/reductions/reduction_tests.cpp +++ b/cpp/tests/reductions/reduction_tests.cpp @@ -38,7 +38,7 @@ using aggregation = cudf::aggregation; template -typename std::enable_if::value, std::vector>::type convert_values( +std::enable_if_t::value, std::vector> convert_values( std::vector const& int_values) { std::vector v(int_values.size()); @@ -50,7 +50,7 @@ typename std::enable_if::value, std::vector>::type c } template -typename std::enable_if::value, std::vector>::type convert_values( +std::enable_if_t::value, std::vector> convert_values( std::vector const& int_values) { std::vector v(int_values.size()); diff --git a/cpp/tests/reductions/scan_tests.hpp b/cpp/tests/reductions/scan_tests.hpp index 346103de85b..858697d8ef5 100644 --- a/cpp/tests/reductions/scan_tests.hpp +++ b/cpp/tests/reductions/scan_tests.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, NVIDIA CORPORATION. + * Copyright (c) 2021-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,25 +58,23 @@ struct TypeParam_to_host_type { }; template -typename std::enable_if, - thrust::host_vector>::type +std::enable_if_t, thrust::host_vector> make_vector(std::initializer_list const& init) { return cudf::test::make_type_param_vector(init); } template -typename std::enable_if(), - thrust::host_vector>::type +std::enable_if_t(), thrust::host_vector> make_vector(std::initializer_list const& init) { return cudf::test::make_type_param_vector(init); } template -typename std::enable_if || - cudf::is_fixed_point()), - thrust::host_vector>::type +std::enable_if_t || + cudf::is_fixed_point()), + thrust::host_vector> make_vector(std::initializer_list const& init) { return cudf::test::make_type_param_vector(init); diff --git a/cpp/tests/sort/is_sorted_tests.cpp b/cpp/tests/sort/is_sorted_tests.cpp index 7d277059ef7..44fa83204ee 100644 --- a/cpp/tests/sort/is_sorted_tests.cpp +++ b/cpp/tests/sort/is_sorted_tests.cpp @@ -36,8 +36,7 @@ namespace testdata { // ----- most numerics template -typename std::enable_if && !std::is_same_v, - fixed_width_column_wrapper>::type +std::enable_if_t && !std::is_same_v, fixed_width_column_wrapper> ascending() { return std::is_signed_v ? fixed_width_column_wrapper({std::numeric_limits::lowest(), @@ -58,8 +57,7 @@ ascending() } template -typename std::enable_if && !std::is_same_v, - fixed_width_column_wrapper>::type +std::enable_if_t && !std::is_same_v, fixed_width_column_wrapper> descending() { return std::is_signed_v ? fixed_width_column_wrapper({std::numeric_limits::max(), @@ -100,14 +98,13 @@ auto nulls_before() // ----- bool template -typename std::enable_if, fixed_width_column_wrapper>::type ascending() +std::enable_if_t, fixed_width_column_wrapper> ascending() { return fixed_width_column_wrapper({false, false, true, true}); } template -typename std::enable_if, fixed_width_column_wrapper>::type -descending() +std::enable_if_t, fixed_width_column_wrapper> descending() { return fixed_width_column_wrapper({true, true, false, false}); } @@ -115,13 +112,13 @@ descending() // ----- chrono types template -typename std::enable_if(), fixed_width_column_wrapper>::type ascending() +std::enable_if_t(), fixed_width_column_wrapper> ascending() { return fixed_width_column_wrapper({T::min(), T::max()}); } template -typename std::enable_if(), fixed_width_column_wrapper>::type descending() +std::enable_if_t(), fixed_width_column_wrapper> descending() { return fixed_width_column_wrapper({T::max(), T::min()}); } @@ -129,15 +126,13 @@ typename std::enable_if(), fixed_width_column_wrapper>::ty // ----- string_view template -typename std::enable_if, strings_column_wrapper>::type -ascending() +std::enable_if_t, strings_column_wrapper> ascending() { return strings_column_wrapper({"A", "B"}); } template -typename std::enable_if, strings_column_wrapper>::type -descending() +std::enable_if_t, strings_column_wrapper> descending() { return strings_column_wrapper({"B", "A"}); } @@ -163,8 +158,7 @@ auto nulls_before() // ----- struct_view {"nestedInt" : {"Int" : 0 }, "float" : 1} template -typename std::enable_if, structs_column_wrapper>::type -ascending() +std::enable_if_t, structs_column_wrapper> ascending() { using T1 = int32_t; auto int_col = fixed_width_column_wrapper({std::numeric_limits::lowest(), @@ -182,8 +176,7 @@ ascending() } template -typename std::enable_if, structs_column_wrapper>::type -descending() +std::enable_if_t, structs_column_wrapper> descending() { using T1 = int32_t; auto int_col = fixed_width_column_wrapper({std::numeric_limits::max(), diff --git a/cpp/tests/utilities/column_utilities.cu b/cpp/tests/utilities/column_utilities.cu index 5403d56318e..9daf70227f8 100644 --- a/cpp/tests/utilities/column_utilities.cu +++ b/cpp/tests/utilities/column_utilities.cu @@ -836,13 +836,13 @@ std::vector bitmask_to_host(cudf::column_view const& c) namespace { -template >* = nullptr> +template >* = nullptr> static auto numeric_to_string_precise(T value) { return std::to_string(value); } -template >* = nullptr> +template >* = nullptr> static auto numeric_to_string_precise(T value) { std::ostringstream o; @@ -915,7 +915,7 @@ std::string nested_offsets_to_string(NestedColumnView const& c, std::string cons } struct column_view_printer { - template ()>* = nullptr> + template ()>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const&) { auto h_data = cudf::test::to_host(col); @@ -939,7 +939,7 @@ struct column_view_printer { } } - template ()>* = nullptr> + template ()>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const& indent) @@ -965,7 +965,7 @@ struct column_view_printer { this->template operator()(*col_as_strings, out, indent); } - template ()>* = nullptr> + template ()>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const&) { auto const h_data = cudf::test::to_host(col); @@ -987,7 +987,7 @@ struct column_view_printer { } template >* = nullptr> + std::enable_if_t>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const&) { // @@ -1008,7 +1008,7 @@ struct column_view_printer { } template >* = nullptr> + std::enable_if_t>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const&) { cudf::dictionary_column_view dictionary(col); @@ -1029,7 +1029,7 @@ struct column_view_printer { } // Print the tick counts with the units - template ()>* = nullptr> + template ()>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const&) { auto h_data = cudf::test::to_host(col); @@ -1054,8 +1054,7 @@ struct column_view_printer { } } - template >* = nullptr> + template >* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const& indent) @@ -1084,7 +1083,7 @@ struct column_view_printer { } template >* = nullptr> + std::enable_if_t>* = nullptr> void operator()(cudf::column_view const& col, std::vector& out, std::string const& indent) diff --git a/cpp/tests/wrappers/timestamps_test.cu b/cpp/tests/wrappers/timestamps_test.cu index 097b786aefe..48500c84942 100644 --- a/cpp/tests/wrappers/timestamps_test.cu +++ b/cpp/tests/wrappers/timestamps_test.cu @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020, NVIDIA CORPORATION. + * Copyright (c) 2019-2022, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ struct compare_chrono_elements_to_primitive_representation { { } - template ()>* = nullptr> + template ()>* = nullptr> __host__ __device__ bool operator()(const int32_t element_index) { using Primitive = typename ChronoT::rep; @@ -59,7 +59,7 @@ struct compare_chrono_elements_to_primitive_representation { return primitive == timestamp.time_since_epoch().count(); } - template ()>* = nullptr> + template ()>* = nullptr> __host__ __device__ bool operator()(const int32_t element_index) { using Primitive = typename ChronoT::rep;