From acdbebbb2aa624ebd9e7b3c7f32bfd31e3e87b37 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:35:28 -0700 Subject: [PATCH 01/27] Add all benchmark files. --- python/cudf/benchmarks/API/bench_dataframe.py | 104 +++++++ .../benchmarks/API/bench_frame_or_index.py | 86 ++++++ python/cudf/benchmarks/API/bench_functions.py | 50 ++++ .../benchmarks/API/bench_functions_cases.py | 134 +++++++++ python/cudf/benchmarks/API/bench_index.py | 15 + .../benchmarks/API/bench_indexed_frame.py | 28 ++ .../cudf/benchmarks/API/bench_multiindex.py | 38 +++ python/cudf/benchmarks/API/bench_series.py | 21 ++ python/cudf/benchmarks/README.md | 6 + python/cudf/benchmarks/common/config.py | 66 +++++ python/cudf/benchmarks/common/utils.py | 263 ++++++++++++++++++ python/cudf/benchmarks/conftest.py | 219 +++++++++++++++ .../cudf/benchmarks/internal/bench_column.py | 93 +++++++ .../internal/bench_dataframe_internal.py | 7 + .../internal/bench_rangeindex_internal.py | 6 + python/cudf/benchmarks/internal/conftest.py | 47 ++++ python/cudf/benchmarks/pytest.ini | 6 + 17 files changed, 1189 insertions(+) create mode 100644 python/cudf/benchmarks/API/bench_dataframe.py create mode 100644 python/cudf/benchmarks/API/bench_frame_or_index.py create mode 100644 python/cudf/benchmarks/API/bench_functions.py create mode 100644 python/cudf/benchmarks/API/bench_functions_cases.py create mode 100644 python/cudf/benchmarks/API/bench_index.py create mode 100644 python/cudf/benchmarks/API/bench_indexed_frame.py create mode 100644 python/cudf/benchmarks/API/bench_multiindex.py create mode 100644 python/cudf/benchmarks/API/bench_series.py create mode 100644 python/cudf/benchmarks/README.md create mode 100644 python/cudf/benchmarks/common/config.py create mode 100644 python/cudf/benchmarks/common/utils.py create mode 100644 python/cudf/benchmarks/conftest.py create mode 100644 python/cudf/benchmarks/internal/bench_column.py create mode 100644 python/cudf/benchmarks/internal/bench_dataframe_internal.py create mode 100644 python/cudf/benchmarks/internal/bench_rangeindex_internal.py create mode 100644 python/cudf/benchmarks/internal/conftest.py create mode 100644 python/cudf/benchmarks/pytest.ini diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py new file mode 100644 index 00000000000..00b83280533 --- /dev/null +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -0,0 +1,104 @@ +import string + +import numpy +import pytest +from config import cudf, cupy +from utils import accepts_cudf_fixture + + +@pytest.mark.parametrize("N", [100, 1_000_000]) +def bench_construction(benchmark, N): + benchmark(cudf.DataFrame, {None: cupy.random.rand(N)}) + + +@accepts_cudf_fixture(cls="dataframe", dtype="float", cols=6) +@pytest.mark.parametrize( + "expr", ["a+b", "a+b+c+d+e", "a / (sin(a) + cos(b)) * tanh(d*e*f)"] +) +def bench_eval_func(benchmark, expr, dataframe): + benchmark(dataframe.eval, expr) + + +@accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6, name="df") +@pytest.mark.parametrize( + "nkey_cols", + [2, 3, 4], +) +def bench_merge(benchmark, df, nkey_cols): + benchmark(df.merge, df, on=list(df.columns[:nkey_cols])) + + +# TODO: Some of these cases could be generalized to an IndexedFrame benchmark +# instead of a DataFrame benchmark. +@accepts_cudf_fixture(cls="dataframe", dtype="int") +@pytest.mark.parametrize( + "values", + [ + range(1000), + {f"key{i}": range(1000) for i in range(10)}, + cudf.DataFrame({f"key{i}": range(1000) for i in range(10)}), + cudf.Series(range(1000)), + ], +) +def bench_isin(benchmark, dataframe, values): + benchmark(dataframe.isin, values) + + +@pytest.fixture( + params=[0, numpy.random.RandomState, cupy.random.RandomState], + ids=["Seed", "NumpyRandomState", "CupyRandomState"], +) +def random_state(request): + rs = request.param + return rs if isinstance(rs, int) else rs(seed=42) + + +@accepts_cudf_fixture(cls="dataframe", dtype="int") +@pytest.mark.parametrize("frac", [0.5]) +def bench_sample(benchmark, dataframe, axis, frac, random_state): + if axis == 1 and isinstance(random_state, cupy.random.RandomState): + pytest.skip("Unsupported params.") + benchmark(dataframe.sample, frac=frac, axis=axis, random_state=random_state) + + +@accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) +@pytest.mark.parametrize( + "nkey_cols", + [2, 3, 4], +) +def bench_groupby(benchmark, dataframe, nkey_cols): + benchmark(dataframe.groupby, by=list(dataframe.columns[:nkey_cols])) + + +@accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) +@pytest.mark.parametrize( + "agg", + [ + "sum", + ["sum", "mean"], + {f"{string.ascii_lowercase[i]}": ["sum", "mean", "count"] for i in range(6)}, + ], +) +@pytest.mark.parametrize( + "nkey_cols", + [2, 3, 4], +) +@pytest.mark.parametrize("as_index", [True, False]) +@pytest.mark.parametrize("sort", [True, False]) +def bench_groupby_agg(benchmark, dataframe, agg, nkey_cols, as_index, sort): + by = list(dataframe.columns[:nkey_cols]) + benchmark(dataframe.groupby(by=by, as_index=as_index, sort=sort).agg, agg) + + +@accepts_cudf_fixture(cls="dataframe", dtype="int") +@pytest.mark.parametrize("ncol_sort", [1]) +def bench_sort_values(benchmark, dataframe, ncol_sort): + benchmark(dataframe.sort_values, list(dataframe.columns[:ncol_sort])) + + +@accepts_cudf_fixture(cls="dataframe", dtype="int") +@pytest.mark.parametrize("ncol_sort", [1]) +@pytest.mark.parametrize("n", [10]) +def bench_nsmallest(benchmark, dataframe, ncol_sort, n): + by = list(dataframe.columns[:ncol_sort]) + benchmark(dataframe.nsmallest, n, by) diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py new file mode 100644 index 00000000000..7ce31572bef --- /dev/null +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -0,0 +1,86 @@ +"""Benchmarks of methods that exist for both Frame and BaseIndex.""" + +import operator + +import numpy as np +import pytest +from utils import accepts_cudf_fixture, make_gather_map + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@pytest.mark.parametrize("gather_how", ["sequence", "reverse", "random"]) +@pytest.mark.parametrize("fraction", [0.4]) +def bench_take(benchmark, gather_how, fraction, frame_or_index): + nr = len(frame_or_index) + gather_map = make_gather_map(nr * fraction, nr, gather_how) + benchmark(frame_or_index.take, gather_map) + + +@pytest.mark.pandas_incompatible # Series/Index work, but not DataFrame +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_argsort(benchmark, frame_or_index): + benchmark(frame_or_index.argsort) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_min(benchmark, frame_or_index): + benchmark(frame_or_index.min) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_where(benchmark, frame_or_index): + cond = frame_or_index % 2 == 0 + benchmark(frame_or_index.where, cond, 0) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +@pytest.mark.pandas_incompatible +def bench_values_host(benchmark, frame_or_index): + benchmark(lambda: frame_or_index.values_host) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +def bench_values(benchmark, frame_or_index): + benchmark(lambda: frame_or_index.values) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_nunique(benchmark, frame_or_index): + benchmark(frame_or_index.nunique) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +def bench_to_numpy(benchmark, frame_or_index): + benchmark(frame_or_index.to_numpy) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +@pytest.mark.pandas_incompatible +def bench_to_cupy(benchmark, frame_or_index): + benchmark(frame_or_index.to_cupy) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@pytest.mark.pandas_incompatible +def bench_to_arrow(benchmark, frame_or_index): + benchmark(frame_or_index.to_arrow) + + +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_astype(benchmark, frame_or_index): + benchmark(frame_or_index.astype, float) + + +@pytest.mark.parametrize("ufunc", [np.add, np.logical_and]) +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_ufunc_series_binary(benchmark, frame_or_index, ufunc): + benchmark(ufunc, frame_or_index, frame_or_index) + + +@pytest.mark.parametrize( + "op", + [operator.add, operator.mul, operator.eq], +) +@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +def bench_binops(benchmark, op, frame_or_index): + benchmark(lambda: op(frame_or_index, frame_or_index)) diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py new file mode 100644 index 00000000000..2da4749d4ad --- /dev/null +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -0,0 +1,50 @@ +"""Benchmarks of free functions that accept cudf objects.""" + +import pytest +import pytest_cases +from config import cudf, cupy + + +@pytest_cases.parametrize_with_cases("objs", prefix="concat") +@pytest.mark.parametrize( + "axis", + [ + 1, + ], +) +@pytest.mark.parametrize("join", ["inner", "outer"]) +@pytest.mark.parametrize("ignore_index", [True, False]) +def bench_concat_axis_1(benchmark, objs, axis, join, ignore_index): + benchmark(cudf.concat, objs=objs, axis=axis, join=join, ignore_index=ignore_index) + + +@pytest.mark.parametrize("size", [10_000, 100_000]) +@pytest.mark.parametrize("cardinality", [10, 100, 1000]) +@pytest.mark.parametrize("dtype", [cupy.bool_, cupy.float64]) +def bench_get_dummies_high_cardinality(benchmark, size, cardinality, dtype): + """This test is mean to test the performance of get_dummies given the + cardinality of column to encode is high. + """ + df = cudf.DataFrame( + { + "col": cudf.Series( + cupy.random.randint(low=0, high=cardinality, size=size) + ).astype("category") + } + ) + benchmark(cudf.get_dummies, df, columns=["col"], dtype=dtype) + + +@pytest.mark.parametrize("prefix", [None, "pre"]) +def bench_get_dummies_simple(benchmark, prefix): + """This test provides a small input to get_dummies to test the efficiency + of the API itself. + """ + df = cudf.DataFrame( + { + "col1": list(range(10)), + "col2": list("abcdefghij"), + "col3": cudf.Series(list(range(100, 110)), dtype="category"), + } + ) + benchmark(cudf.get_dummies, df, columns=["col1", "col2", "col3"], prefix=prefix) diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py new file mode 100644 index 00000000000..01ca99a5240 --- /dev/null +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -0,0 +1,134 @@ +import pytest_cases +from config import NUM_ROWS, cudf, cupy + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_default_index(nr): + return [ + cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr)}), + cudf.DataFrame({"b": cupy.tile([4, 5, 7], nr)}), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_contiguous_indexes(nr): + return [ + cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr)}), + cudf.DataFrame( + {"b": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), + ), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_contiguous_indexes_different_cols(nr): + return [ + cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"c": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), + ), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_string_index(nr): + return [ + cudf.DataFrame( + {"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=0, stop=nr * 3).astype("str"), + ), + cudf.DataFrame( + {"c": [4, 5, 7] * nr}, + index=cudf.RangeIndex(start=0, stop=nr * 3).astype("str"), + ), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_contiguous_string_index_different_col(nr): + return [ + cudf.DataFrame( + {"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=0, stop=nr * 3).astype("str"), + ), + cudf.DataFrame( + {"c": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3).astype("str"), + ), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_complex_string_index(nr): + return [ + cudf.DataFrame( + {"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=0, stop=nr * 3).astype("str"), + ), + cudf.DataFrame( + {"c": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3).astype("str"), + ), + cudf.DataFrame( + {"d": cupy.tile([1, 2, 3], nr), "e": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=0, stop=nr * 3).astype("str"), + ), + cudf.DataFrame( + {"f": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3).astype("str"), + ), + cudf.DataFrame( + {"g": cupy.tile([1, 2, 3], nr), "h": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=0, stop=nr * 3).astype("str"), + ), + cudf.DataFrame( + {"i": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3).astype("str"), + ), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_unique_columns(nr): + # To avoid any edge case bugs, always use at least 10 rows per DataFrame. + nr_actual = max(10, nr // 20) + return [ + cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr_actual)}), + cudf.DataFrame({"b": cupy.tile([4, 5, 7], nr_actual)}), + cudf.DataFrame({"c": cupy.tile([1, 2, 3], nr_actual)}), + cudf.DataFrame({"d": cupy.tile([4, 5, 7], nr_actual)}), + cudf.DataFrame({"e": cupy.tile([1, 2, 3], nr_actual)}), + cudf.DataFrame({"f": cupy.tile([4, 5, 7], nr_actual)}), + cudf.DataFrame({"g": cupy.tile([1, 2, 3], nr_actual)}), + cudf.DataFrame({"h": cupy.tile([4, 5, 7], nr_actual)}), + cudf.DataFrame({"i": cupy.tile([1, 2, 3], nr_actual)}), + cudf.DataFrame({"j": cupy.tile([4, 5, 7], nr_actual)}), + ] + + +@pytest_cases.parametrize("nr", NUM_ROWS) +def concat_case_unique_columns_with_different_range_index(nr): + return [ + cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"c": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), + ), + cudf.DataFrame({"d": cupy.tile([1, 2, 3], nr), "e": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"f": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), + ), + cudf.DataFrame({"g": cupy.tile([1, 2, 3], nr), "h": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"i": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), + ), + cudf.DataFrame({"j": cupy.tile([1, 2, 3], nr), "k": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"l": cupy.tile([4, 5, 7], nr)}, + index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), + ), + ] diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py new file mode 100644 index 00000000000..72d37a5abd7 --- /dev/null +++ b/python/cudf/benchmarks/API/bench_index.py @@ -0,0 +1,15 @@ +"""Benchmarks of Index methods.""" + +import pytest +from config import cudf, cupy +from utils import accepts_cudf_fixture + + +@pytest.mark.parametrize("N", [100, 1_000_000]) +def bench_construction(benchmark, N): + benchmark(cudf.Index, cupy.random.rand(N)) + + +@accepts_cudf_fixture(cls="index", dtype="int", nulls=False) +def bench_sort_values(benchmark, index): + benchmark(index.sort_values) diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py new file mode 100644 index 00000000000..f33f71b5644 --- /dev/null +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -0,0 +1,28 @@ +"""Benchmarks of IndexedFrame methods.""" + +import pytest +from utils import accepts_cudf_fixture + + +@accepts_cudf_fixture(cls="indexedframe", dtype="int") +@pytest.mark.parametrize("op", ["cumsum", "cumprod", "cummax"]) +def bench_scans(benchmark, op, indexedframe): + benchmark(getattr(indexedframe, op)) + + +@accepts_cudf_fixture(cls="indexedframe", dtype="int") +@pytest.mark.parametrize("op", ["sum", "product", "mean"]) +def bench_reductions(benchmark, op, indexedframe): + benchmark(getattr(indexedframe, op)) + + +@accepts_cudf_fixture(cls="indexedframe", dtype="int") +def bench_drop_duplicates(benchmark, indexedframe): + benchmark(indexedframe.drop_duplicates) + + +@accepts_cudf_fixture(cls="indexedframe", dtype="int") +def bench_rangeindex_replace(benchmark, indexedframe): + # TODO: Consider adding more DataFrame-specific benchmarks for different + # types of valid inputs (dicts, etc). + benchmark(indexedframe.replace, 0, 2) diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py new file mode 100644 index 00000000000..568d5c767b9 --- /dev/null +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -0,0 +1,38 @@ +import numpy as np +import pandas as pd +import pytest +from config import cudf + + +@pytest.fixture +def pidx(): + num_elements = int(1e3) + a = np.random.randint(0, num_elements // 10, num_elements) + b = np.random.randint(0, num_elements // 10, num_elements) + return pd.MultiIndex.from_arrays([a, b], names=("a", "b")) + + +@pytest.fixture +def midx(pidx): + num_elements = int(1e3) + a = np.random.randint(0, num_elements // 10, num_elements) + b = np.random.randint(0, num_elements // 10, num_elements) + df = cudf.DataFrame({"a": a, "b": b}) + return cudf.MultiIndex.from_frame(df) + + +@pytest.mark.pandas_incompatible +def bench_from_pandas(benchmark, pidx): + benchmark(cudf.MultiIndex.from_pandas, pidx) + + +def bench_constructor(benchmark, midx): + benchmark(cudf.MultiIndex, codes=midx.codes, levels=midx.levels, names=midx.names) + + +def bench_from_frame(benchmark, midx): + benchmark(cudf.MultiIndex.from_frame, midx.to_frame(index=False)) + + +def bench_copy(benchmark, midx): + benchmark(midx.copy, deep=False) diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py new file mode 100644 index 00000000000..5caab20ea8b --- /dev/null +++ b/python/cudf/benchmarks/API/bench_series.py @@ -0,0 +1,21 @@ +"""Benchmarks of Series methods.""" + +import pytest +from config import cudf, cupy +from utils import accepts_cudf_fixture + + +@pytest.mark.parametrize("N", [100, 1_000_000]) +def bench_construction(benchmark, N): + benchmark(cudf.Series, cupy.random.rand(N)) + + +@accepts_cudf_fixture(cls="series", dtype="int") +def bench_sort_values(benchmark, series): + benchmark(series.sort_values) + + +@accepts_cudf_fixture(cls="series", dtype="int") +@pytest.mark.parametrize("n", [10]) +def bench_series_nsmallest(benchmark, series, n): + benchmark(series.nsmallest, n) diff --git a/python/cudf/benchmarks/README.md b/python/cudf/benchmarks/README.md new file mode 100644 index 00000000000..1441069c57d --- /dev/null +++ b/python/cudf/benchmarks/README.md @@ -0,0 +1,6 @@ +# About + +Benchmarks of cuDF. + +Benchmarks of public APIs are contained in the API subdirectory. +Benchmarks of non-public internals are contained in the internal subdirectory. diff --git a/python/cudf/benchmarks/common/config.py b/python/cudf/benchmarks/common/config.py new file mode 100644 index 00000000000..51f17f8eff5 --- /dev/null +++ b/python/cudf/benchmarks/common/config.py @@ -0,0 +1,66 @@ +"""Module used for global configuration of benchmarks. + +This file contains global definitions that are important for configuring all +benchmarks such as fixture sizes. In addition, this file supports the following +features: + - Defining the CUDF_BENCHMARKS_USE_PANDAS environment variable will change + all benchmarks to run with pandas instead of cudf (and numpy instead of + cupy). This feature enables easy comparisons of benchmarks between cudf + and pandas. All common modules (cudf, cupy) should be imported from here + by benchmark modules to allow configuration if needed. + - Defining CUDF_BENCHMARKS_TEST_ONLY will set global configuration + variables to avoid running large benchmarks, instead using minimal values + to simply ensure that benchmarks are functional. + +This file is also where standard pytest hooks should be overridden. While these +definitions typically belong in conftest.py, since any of the above environment +variables could affect test collection or other properties, we must define them +in this file and import them in conftest.py to ensure that they are handled +appropriately. +""" +import os +import sys + +# Environment variable-based configuration of benchmarking pandas or cudf. +collect_ignore = [] +if "CUDF_BENCHMARKS_USE_PANDAS" in os.environ: + import numpy as cupy + import pandas as cudf + + # cudf internals offer no pandas compatibility guarantees, and we also + # never need to compare those benchmarks to pandas. + collect_ignore.append("internal/") + + # Also filter out benchmarks of APIs that are not compatible with pandas. + def is_pandas_compatible(item): + return all(m.name != "pandas_incompatible" for m in item.own_markers) + + def pytest_collection_modifyitems(session, config, items): + items[:] = list(filter(is_pandas_compatible, items)) + +else: + import cudf # noqa: W0611, F401 + import cupy # noqa: W0611, F401 + + def pytest_collection_modifyitems(session, config, items): + pass + + +def pytest_sessionstart(session): + """Add the common files to the path for all tests to import.""" + sys.path.insert(0, os.path.join(os.getcwd(), "common")) + + +def pytest_sessionfinish(session, exitstatus): + """Clean up sys.path after exit.""" + if "common" in sys.path[0]: + del sys.path[0] + + +# Constants used to define benchmarking standards. +if "CUDF_BENCHMARKS_TEST_ONLY" in os.environ: + NUM_ROWS = [10, 20] + NUM_COLS = [1, 6] +else: + NUM_ROWS = [100, 10_000, 1_000_000] + NUM_COLS = [1, 6] diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py new file mode 100644 index 00000000000..24799bdfeef --- /dev/null +++ b/python/cudf/benchmarks/common/utils.py @@ -0,0 +1,263 @@ +"""Common utilities for fixture creation and benchmarking.""" + +import inspect +import re +import textwrap +from collections.abc import MutableSet +from itertools import groupby +from numbers import Real + +import pytest_cases +from config import NUM_COLS, NUM_ROWS, cudf, cupy + + +def make_gather_map(len_gather_map: Real, len_column: Real, how: str): + """Create a gather map based on "how" you'd like to gather from input. + - sequence: gather the first `len_gather_map` rows, the first thread + collects the first element + - reverse: gather the last `len_gather_map` rows, the first thread + collects the last element + - random: create a pseudorandom gather map + + `len_gather_map`, `len_column` gets rounded to integer. + """ + len_gather_map = round(len_gather_map) + len_column = round(len_column) + + rstate = cupy.random.RandomState(seed=0) + if how == "sequence": + return cudf.Series(cupy.arange(0, len_gather_map)) + elif how == "reverse": + return cudf.Series( + cupy.arange(len_column - 1, len_column - len_gather_map - 1, -1) + ) + elif how == "random": + return cudf.Series(rstate.randint(0, len_column, len_gather_map)) + + +def make_boolean_mask_column(size): + rstate = cupy.random.RandomState(seed=0) + return cudf.core.column.as_column(rstate.randint(0, 2, size).astype(bool)) + + +def flatten(xs): + for x in xs: + if not isinstance(x, str): + yield from x + else: + yield x + + +def accepts_cudf_fixture( + cls, *, dtype="int", nulls=None, cols=None, rows=None, name=None +): + """Pass "standard" cudf fixtures to functions without renaming parameters. + + The fixture generation logic in conftest.py provides a plethora of useful + fixtures to allow developers to easily select an appropriate cross-section + of the space of objects to apply a particular benchmark to. However, the + usage of this fixtures is cumbersome because creating them in a principled + fashion results in long names and very specific naming schemes. This + decorator abstracts that naming logic away from the developer, allowing + them to instead focus on defining the fixture semantically by describing + its properties. + + Parameters + ---------- + cls : Union[str, Type] + The class of object to test. May either be specified as the type + itself, or using the name (as a string). If a string, the case is + irrelevant as the string will be converted to all lowercase. + dtype : Union[str, Iterable[str]], default 'int' + The dtype or set of dtypes to use. + nulls : Optional[bool], default None + Whether to test nullable or non-nullable data. If None, both nullable + and non-nullable data are included. + cols : Optional[int], None + The number of columns. Only valid if cls == 'dataframe'. If None, use + all possible numbers of columns. Specifying multiple values is + unsupported. + rows : Optional[int], None + The number of rows. If None, use all possible numbers of rows. + Specifying multiple values is unsupported. + fixture : str, default None + The name of the fixture as used in the decorated test. If None, + defaults to `cls.lower()` if cls is a string, otherwise + `cls.__name__.lower()`. Use of this name allows the decorated function + to masquerade as a pytest receiving a fixture while substituting the + real fixture (with a much longer name). + + Raises + ------ + AssertionError + If any of the parameters do not correspond to extant fixtures. + + Examples + -------- + # Note: As an internal function, this example is not meant for doctesting. + + @accepts_cudf_fixture("dataframe", dtype="int", nulls=False, name="df") + def bench_columns(benchmark, df): + benchmark(df.columns) + """ + if inspect.isclass(cls): + cls = cls.__name__ + cls = cls.lower() + + supported_classes = ( + "column", + "series", + "index", + "dataframe", + "indexedframe", + "frame_or_index", + ) + assert ( + cls in supported_classes + ), f"cls {cls} is invalid, choose from {', '.join(c for c in supported_classes)}" + + name = name or cls + + if not isinstance(dtype, list): + dtype = [dtype] + assert all( + dt in column_generators for dt in dtype + ), f"The only supported dtypes are {', '.join(dt for dt in column_generators)}" + dtype_str = "_dtype_" + "_or_".join(dtype) + + null_str = "" + if nulls is not None: + null_str = f"_nulls_{nulls}".lower() + + col_str = "" + if cols is not None: + assert cols in NUM_COLS, ( + f"You have requested a DataFrame with {cols} columns but fixtures " + f"only exist for the values {', '.join(c for c in NUM_COLS)}" + ) + col_str = f"_cols_{cols}" + + row_str = "" + if rows is not None: + assert rows in NUM_ROWS, ( + f"You have requested a {cls} with {rows} rows but fixtures " + f"only exist for the values {', '.join(c for c in NUM_ROWS)}" + ) + row_str = f"_rows_{rows}" + + fixture_name = f"{cls}{dtype_str}{null_str}{col_str}{row_str}" + + def deco(bm): + # pytests's test collection process relies on parsing the globals dict + # to find test functions and identify their parameters for the purpose + # of fixtures and parameters. Therefore, the primary purpose of this + # decorator is to define a new benchmark function with a signature + # identical to that of the decorated benchmark except with the user's + # fixture name replaced by the true fixture name based on the arguments + # to accepts_cudf_fixture. + parameters = inspect.signature(bm).parameters + + # Note: This logic assumes that any benchmark using this fixture has at + # least two parameters since they must be using both the + # pytest-benchmark `benchmark` fixture and the cudf object. + params_str = ", ".join(f"{p}" for p in parameters if p != name) + arg_str = ", ".join(f"{p}={p}" for p in parameters if p != name) + + if params_str: + params_str += ", " + if arg_str: + arg_str += ", " + + params_str += f"{fixture_name}" + arg_str += f"{name}={fixture_name}" + + src = textwrap.dedent( + f""" + def wrapped_bm({params_str}): + return bm({arg_str}) + """ + ) + globals_ = {"bm": bm} + exec(src, globals_) + wrapped_bm = globals_["wrapped_bm"] + # In case marks were applied to the original benchmark, copy them over. + if marks := getattr(bm, "pytestmark", None): + wrapped_bm.pytestmark = marks + return wrapped_bm + + return deco + + +class OrderedSet(MutableSet): + """A minimal OrderedSet implementation built on a dict. + + This implementation exploits the fact that dicts are ordered as of Python + 3.7. It is not intended to be performant, so only the minimal set of + methods are implemented. We need this class to ensure that fixture names + are constructed deterministically, otherwise pytest-xdist will complain if + different threads have seemingly different tests. + """ + + def __init__(self, args=None): + args = args or [] + self._data = {value: None for value in args} + + def __contains__(self, key): + return key in self._data + + def __iter__(self): + return iter(self._data) + + def __len__(self): + return len(self._data) + + def __repr__(self): + # Helpful for debugging. + data = ", ".join(str(i) for i in self._data) + return f"{self.__class__.__name__}({data})" + + def add(self, value): + self._data[value] = None + + def discard(self, value): + self._data.pop(value, None) + + +def make_fixture(name, func, globals_, fixtures): + """Create a named fixture in `globals_` and save its name in `fixtures`. + + https://github.com/pytest-dev/pytest/issues/2424#issuecomment-333387206 + explains why this hack is necessary. Essentially, dynamically generated + fixtures must exist in globals() to be found by pytest. + """ + globals_[name] = pytest_cases.fixture(name=name)(func) + fixtures.add(name) + + +def collapse_fixtures(fixtures, pattern, repl, globals_, idfunc=None): + """Create unions of fixtures based on specific name mappings. + + `fixtures` are grouped into unions according the regex replacement + `re.sub(pattern, repl)` and placed into `new_fixtures`. + """ + + def collapser(n): + return re.sub(pattern, repl, n) + + # Note: sorted creates a new list, not a view, so it's OK to modify the + # list of fixtures while iterating over the sorted result. + for name, group in groupby(sorted(fixtures, key=collapser), key=collapser): + group = list(group) + if len(group) > 1 and name not in fixtures: + pytest_cases.fixture_union(name=name, fixtures=group, ids=idfunc) + # Need to assign back to the parent scope's globals. + globals_[name] = globals()[name] + fixtures.add(name) + + +# A dictionary of callables that create a column of a specified length +random_state = cupy.random.RandomState(42) +column_generators = { + "int": (lambda nr: random_state.randint(low=0, high=100, size=nr)), + "float": (lambda nr: random_state.rand(nr)), +} diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py new file mode 100644 index 00000000000..4a391502673 --- /dev/null +++ b/python/cudf/benchmarks/conftest.py @@ -0,0 +1,219 @@ +"""Standard location for shared fixtures and code across tests. + +Most fixtures defined in this file represent one of the primary classes in the +cuDF ecosystem such as DataFrame, Series, or Index. These fixtures may in turn +be broken up into two categories: base fixtures and fixture unions. Each base +fixture represents a specific type of object as well as certain of its +properties crucial for benchmarking. Specifically, fixtures must account for +the following different parameters: + - Class of object (DataFrame, Series, Index) + - Dtype + - Nullability + - Size (rows for all, rows/columns for DataFrame) + +One such fixture is a series of nullable integer data. Given that we generally +want data across different sizes, we parametrize all fixtures across different +numbers of rows rather than generating separate fixtures for each different +possible number of rows. The number of columns is only relevant for DataFrame. + +While this core set of fixtures means that any benchmark can be run for any +combination of these parameters, it also means that we would effectively have +to parametrize our benchmarks with many fixtures. Not only is parametrizing +tests with fixtures in this manner unsupported by pytest, it is also an +inelegant solution leading to cumbersome parameter lists that must be +maintained across all tests. Instead we make use of the +`pytest_cases _` pytest plugin, +which supports the creation of fixture unions: fixtures that result from +combining other fixtures together. The result is a set of well-defined fixtures +that allow us to write benchmarks that naturally express the set of objects for +which they are valid, e.g. `def bench_sort_values(frame_or_index)`. + +The generated fixtures are named according to the following convention: +`{classname}_dtype_{dtype}[_nulls_{true|false}][[_cols_{num_cols}]_rows_{num_rows}]` +where classname is one of the following: index, series, dataframe, +indexedframe, frame, frame_or_index. Note that in the case of indexes, to match +Series/DataFrame we simply set `classname=index` and rely on the +`dtype_{dtype}` component to delineate which index class is actually in use. + +In addition to the above fixtures, we also provide the following more +specialized fixtures: + - rangeindex: Since RangeIndex always holds int64 data we cannot conflate + it with index_dtype_int64 (a true Int64Index), and it cannot hold nulls. + As a result, it is provided as a separate fixture. +""" + + +import os +import string +import sys + +import pytest_cases + +# TODO: Rather than doing this path hacking (including the sessionstart and +# sessionfinish hooks), we could just make the benchmarks a (sub)package to +# enable relative imports. A minor change to consider when these are ported +# into the main repo. +sys.path.insert(0, os.path.join(os.getcwd(), "common")) + +from config import cudf # noqa: W0611, E402, F401 +from utils import ( # noqa: E402 + OrderedSet, + collapse_fixtures, + column_generators, + make_fixture, +) + +# Turn off isort until we upgrade to 5.8.0 +# https://github.com/pycqa/isort/issues/1594 +# isort: off +from config import ( # noqa: W0611, E402, F401 + NUM_COLS, + NUM_ROWS, + collect_ignore, + pytest_collection_modifyitems, + pytest_sessionfinish, + pytest_sessionstart, +) + +# isort: on + + +@pytest_cases.fixture(params=[0, 1], ids=["AxisIndex", "AxisColumn"]) +def axis(request): + return request.param + + +# First generate all the base fixtures. +fixtures = OrderedSet() +for dtype, column_generator in column_generators.items(): + + def make_dataframe(nr, nc, column_generator=column_generator): + assert nc <= len(string.ascii_lowercase) + return cudf.DataFrame( + {f"{string.ascii_lowercase[i]}": column_generator(nr) for i in range(nc)} + ) + + for nr in NUM_ROWS: + # TODO: pytest_cases.fixture doesn't appear to support lambdas where + # pytest does. https://github.com/smarie/python-pytest-cases/issues/278 + # Once that is fixed we could use lambdas here. + # TODO: pytest_cases has a bug where the first argument being a + # defaulted kwarg e.g. (nr=nr, nc=nc) raises errors. + # https://github.com/smarie/python-pytest-cases/issues/278 + # Once that is fixed we could remove all the extraneous `request` + # fixtures in these fixtures. + def series_nulls_false(request, nr=nr, column_generator=column_generator): + return cudf.Series(column_generator(nr)) + + make_fixture( + f"series_dtype_{dtype}_nulls_false_rows_{nr}", + series_nulls_false, + globals(), + fixtures, + ) + + def series_nulls_true(request, nr=nr, column_generator=column_generator): + s = cudf.Series(column_generator(nr)) + s.iloc[::2] = None + return s + + make_fixture( + f"series_dtype_{dtype}_nulls_true_rows_{nr}", + series_nulls_true, + globals(), + fixtures, + ) + + # For now, not bothering to include a nullable index fixture. + def index_nulls_false(request, nr=nr, column_generator=column_generator): + return cudf.Index(column_generator(nr)) + + make_fixture( + f"index_dtype_{dtype}_nulls_false_rows_{nr}", + index_nulls_false, + globals(), + fixtures, + ) + + for nc in NUM_COLS: + + def dataframe_nulls_false( + request, nr=nr, nc=nc, make_dataframe=make_dataframe + ): + return make_dataframe(nr, nc) + + make_fixture( + f"dataframe_dtype_{dtype}_nulls_false_cols_{nc}_rows_{nr}", + dataframe_nulls_false, + globals(), + fixtures, + ) + + def dataframe_nulls_true( + request, nr=nr, nc=nc, make_dataframe=make_dataframe + ): + df = make_dataframe(nr, nc) + df.iloc[::2, :] = None + return df + + make_fixture( + f"dataframe_dtype_{dtype}_nulls_true_cols_{nc}_rows_{nr}", + dataframe_nulls_true, + globals(), + fixtures, + ) + + +# We define some custom naming functions for use in the creation of fixture +# unions to create more readable test function names that don't contain the +# entire union, which quickly becomes intractably long. +def unique_union_id(val): + return val.alternative_name + + +def default_union_id(val): + return f"alt{val.get_alternative_idx()}" + + +# Label the first level differently from others since there's no redundancy. +idfunc = unique_union_id +num_new_fixtures = len(fixtures) + +# Keep trying to merge existing fixtures until no new fixtures are added. +while num_new_fixtures > 0: + num_fixtures = len(fixtures) + + # Note: If we start also introducing unions across dtypes, most likely + # those will take the form `*int_and_float*` or similar since we won't want + # to union _all_ dtypes. In that case, the regexes will need to use + # suitable lookaheads etc to avoid infinite loops here. + for pat, repl in [ + ("_nulls_(true|false)", ""), + ("series|dataframe", "indexedframe"), + ("indexedframe|index", "frame_or_index"), + (r"_rows_\d+", ""), + (r"_cols_\d+", ""), + ]: + + collapse_fixtures(fixtures, pat, repl, globals(), idfunc) + + num_new_fixtures = len(fixtures) - num_fixtures + # All subsequent levels get the same (collapsed) labels. + idfunc = default_union_id + + +for dtype, column_generator in column_generators.items(): + # We have to manually add this one because we aren't including nullable + # indexes but we want to be able to run some benchmarks on Series/DataFrame + # that may or may not be nullable as well as Index objects. + pytest_cases.fixture_union( + name=f"frame_or_index_dtype_{dtype}", + fixtures=(f"indexedframe_dtype_{dtype}", f"index_dtype_{dtype}_nulls_false"), + ids=["", f"index_dtype_{dtype}_nulls_false"], + ) + + +# TODO: Decide where to incorporate RangeIndex and MultiIndex fixtures. +@pytest_cases.fixture(params=NUM_ROWS) +def rangeindex(request): + return cudf.RangeIndex(request.param) diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py new file mode 100644 index 00000000000..22c7f9f634f --- /dev/null +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -0,0 +1,93 @@ +import pytest +import pytest_cases +from utils import accepts_cudf_fixture, make_boolean_mask_column, make_gather_map + + +@accepts_cudf_fixture(cls="column", dtype="float") +def bench_apply_boolean_mask(benchmark, column): + mask = make_boolean_mask_column(column.size) + benchmark(column.apply_boolean_mask, mask) + + +@accepts_cudf_fixture(cls="column", dtype="float") +@pytest.mark.parametrize("dropnan", [True, False]) +def bench_dropna(benchmark, column, dropnan): + benchmark(column.dropna, drop_nan=dropnan) + + +@accepts_cudf_fixture(cls="column", dtype="float") +def bench_unique_single_column(benchmark, column): + benchmark(column.unique) + + +@accepts_cudf_fixture(cls="column", dtype="float") +@pytest.mark.parametrize("nullify", [True, False]) +@pytest.mark.parametrize("gather_how", ["sequence", "reverse", "random"]) +def bench_take(benchmark, column, gather_how, nullify): + gather_map = make_gather_map(column.size * 0.4, column.size, gather_how)._column + benchmark(column.take, gather_map, nullify=nullify) + + +# TODO: Due to https://github.com/smarie/python-pytest-cases/issues/280 we +# cannot use the accepts_cudf_fixture decorator for cases. If and when that is +# resolved, we can change all of the cases below to use that instead of +# hardcoding the fixture name. +def setitem_case_stride_1_slice_scalar(column_dtype_int_nulls_false): + return column_dtype_int_nulls_false, slice(None, None, 1), 42 + + +def setitem_case_stride_2_slice_scalar(column_dtype_int_nulls_false): + return column_dtype_int_nulls_false, slice(None, None, 2), 42 + + +def setitem_case_boolean_column_scalar(column_dtype_int_nulls_false): + column = column_dtype_int_nulls_false + return column, [True, False] * (len(column) // 2), 42 + + +def setitem_case_int_column_scalar(column_dtype_int_nulls_false): + column = column_dtype_int_nulls_false + return column, list(range(len(column))), 42 + + +def setitem_case_stride_1_slice_align_to_key_size(column_dtype_int_nulls_false): + column = column_dtype_int_nulls_false + key = slice(None, None, 1) + start, stop, stride = key.indices(len(column)) + materialized_key_size = len(column.slice(start, stop, stride)) + return column, key, [42] * materialized_key_size + + +def setitem_case_stride_2_slice_align_to_key_size(column_dtype_int_nulls_false): + column = column_dtype_int_nulls_false + key = slice(None, None, 2) + start, stop, stride = key.indices(len(column)) + materialized_key_size = len(column.slice(start, stop, stride)) + return column, key, [42] * materialized_key_size + + +def setitem_case_boolean_column_align_to_col_size(column_dtype_int_nulls_false): + column = column_dtype_int_nulls_false + size = len(column) + return column, [True, False] * (size // 2), [42] * size + + +def setitem_case_int_column_align_to_col_size(column_dtype_int_nulls_false): + column = column_dtype_int_nulls_false + size = len(column) + return column, list(range(size)), [42] * size + + +# Benchmark Grid +# key: slice == 1 (fill or copy_range shortcut), +# slice != 1 (scatter), +# column(bool) (boolean_mask_scatter), +# column(int) (scatter) +# value: scalar, +# column (len(val) == len(key)), +# column (len(val) != len(key) & len == num_true) + + +@pytest_cases.parametrize_with_cases("column,key,value", cases=".", prefix="setitem") +def bench_setitem(benchmark, column, key, value): + benchmark(column.__setitem__, key, value) diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py new file mode 100644 index 00000000000..7cc2c0ffbf3 --- /dev/null +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -0,0 +1,7 @@ +from utils import accepts_cudf_fixture, make_boolean_mask_column + + +@accepts_cudf_fixture(cls="dataframe", dtype="int") +def bench_apply_boolean_mask(benchmark, dataframe): + mask = make_boolean_mask_column(len(dataframe)) + benchmark(dataframe._apply_boolean_mask, mask) diff --git a/python/cudf/benchmarks/internal/bench_rangeindex_internal.py b/python/cudf/benchmarks/internal/bench_rangeindex_internal.py new file mode 100644 index 00000000000..8e7a6304e99 --- /dev/null +++ b/python/cudf/benchmarks/internal/bench_rangeindex_internal.py @@ -0,0 +1,6 @@ +def bench_column(benchmark, rangeindex): + benchmark(lambda: rangeindex._column) + + +def bench_columns(benchmark, rangeindex): + benchmark(lambda: rangeindex._columns) diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py new file mode 100644 index 00000000000..8438beb45e0 --- /dev/null +++ b/python/cudf/benchmarks/internal/conftest.py @@ -0,0 +1,47 @@ +from config import NUM_ROWS, cudf # noqa: E402 +from utils import OrderedSet, collapse_fixtures, column_generators, make_fixture + +fixtures = OrderedSet() +for dtype, column_generator in column_generators.items(): + for nr in NUM_ROWS: + + def column_nulls_false(request, nr=nr): + return cudf.core.column.as_column(column_generator(nr)) + + make_fixture( + f"column_dtype_{dtype}_nulls_false_rows_{nr}", + column_nulls_false, + globals(), + fixtures, + ) + + def column_nulls_true(request, nr=nr): + c = cudf.core.column.as_column(column_generator(nr)) + c[::2] = None + return c + + make_fixture( + f"column_dtype_{dtype}_nulls_true_rows_{nr}", + column_nulls_true, + globals(), + fixtures, + ) + +num_new_fixtures = len(fixtures) + +# Keep trying to merge existing fixtures until no new fixtures are added. +while num_new_fixtures > 0: + num_fixtures = len(fixtures) + + # Note: If we start also introducing unions across dtypes, most likely + # those will take the form `*int_and_float*` or similar since we won't want + # to union _all_ dtypes. In that case, the regexes will need to use + # suitable lookaheads etc to avoid infinite loops here. + for pat, repl in [ + ("_nulls_(true|false)", ""), + (r"_rows_\d+", ""), + ]: + + collapse_fixtures(fixtures, pat, repl, globals()) + + num_new_fixtures = len(fixtures) - num_fixtures diff --git a/python/cudf/benchmarks/pytest.ini b/python/cudf/benchmarks/pytest.ini new file mode 100644 index 00000000000..75e90cb21bf --- /dev/null +++ b/python/cudf/benchmarks/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +python_files = bench_*.py +python_classes = Bench +python_functions = bench_* +markers = + pandas_incompatible: mark a benchmark that cannot be run with pandas From 28ed6a8318c22c8185852ff14ab22e363dd6082e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:38:08 -0700 Subject: [PATCH 02/27] Fix most linter issues. --- python/cudf/benchmarks/API/bench_dataframe.py | 13 +++++++--- python/cudf/benchmarks/API/bench_functions.py | 8 ++++-- .../benchmarks/API/bench_functions_cases.py | 20 ++++++++++---- .../cudf/benchmarks/API/bench_multiindex.py | 4 ++- python/cudf/benchmarks/common/config.py | 3 ++- python/cudf/benchmarks/conftest.py | 22 ++++++++++++---- .../cudf/benchmarks/internal/bench_column.py | 26 ++++++++++++++----- python/cudf/benchmarks/internal/conftest.py | 7 ++++- 8 files changed, 79 insertions(+), 24 deletions(-) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 00b83280533..7fec5873977 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -19,7 +19,9 @@ def bench_eval_func(benchmark, expr, dataframe): benchmark(dataframe.eval, expr) -@accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6, name="df") +@accepts_cudf_fixture( + cls="dataframe", dtype="int", nulls=False, cols=6, name="df" +) @pytest.mark.parametrize( "nkey_cols", [2, 3, 4], @@ -58,7 +60,9 @@ def random_state(request): def bench_sample(benchmark, dataframe, axis, frac, random_state): if axis == 1 and isinstance(random_state, cupy.random.RandomState): pytest.skip("Unsupported params.") - benchmark(dataframe.sample, frac=frac, axis=axis, random_state=random_state) + benchmark( + dataframe.sample, frac=frac, axis=axis, random_state=random_state + ) @accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) @@ -76,7 +80,10 @@ def bench_groupby(benchmark, dataframe, nkey_cols): [ "sum", ["sum", "mean"], - {f"{string.ascii_lowercase[i]}": ["sum", "mean", "count"] for i in range(6)}, + { + f"{string.ascii_lowercase[i]}": ["sum", "mean", "count"] + for i in range(6) + }, ], ) @pytest.mark.parametrize( diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index 2da4749d4ad..4c22956ae3b 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -15,7 +15,9 @@ @pytest.mark.parametrize("join", ["inner", "outer"]) @pytest.mark.parametrize("ignore_index", [True, False]) def bench_concat_axis_1(benchmark, objs, axis, join, ignore_index): - benchmark(cudf.concat, objs=objs, axis=axis, join=join, ignore_index=ignore_index) + benchmark( + cudf.concat, objs=objs, axis=axis, join=join, ignore_index=ignore_index + ) @pytest.mark.parametrize("size", [10_000, 100_000]) @@ -47,4 +49,6 @@ def bench_get_dummies_simple(benchmark, prefix): "col3": cudf.Series(list(range(100, 110)), dtype="category"), } ) - benchmark(cudf.get_dummies, df, columns=["col1", "col2", "col3"], prefix=prefix) + benchmark( + cudf.get_dummies, df, columns=["col1", "col2", "col3"], prefix=prefix + ) diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index 01ca99a5240..f8093cc8f3e 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -24,7 +24,9 @@ def concat_case_contiguous_indexes(nr): @pytest_cases.parametrize("nr", NUM_ROWS) def concat_case_contiguous_indexes_different_cols(nr): return [ - cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)} + ), cudf.DataFrame( {"c": cupy.tile([4, 5, 7], nr)}, index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), @@ -111,22 +113,30 @@ def concat_case_unique_columns(nr): @pytest_cases.parametrize("nr", NUM_ROWS) def concat_case_unique_columns_with_different_range_index(nr): return [ - cudf.DataFrame({"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"a": cupy.tile([1, 2, 3], nr), "b": cupy.tile([4, 5, 7], nr)} + ), cudf.DataFrame( {"c": cupy.tile([4, 5, 7], nr)}, index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), ), - cudf.DataFrame({"d": cupy.tile([1, 2, 3], nr), "e": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"d": cupy.tile([1, 2, 3], nr), "e": cupy.tile([4, 5, 7], nr)} + ), cudf.DataFrame( {"f": cupy.tile([4, 5, 7], nr)}, index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), ), - cudf.DataFrame({"g": cupy.tile([1, 2, 3], nr), "h": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"g": cupy.tile([1, 2, 3], nr), "h": cupy.tile([4, 5, 7], nr)} + ), cudf.DataFrame( {"i": cupy.tile([4, 5, 7], nr)}, index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), ), - cudf.DataFrame({"j": cupy.tile([1, 2, 3], nr), "k": cupy.tile([4, 5, 7], nr)}), + cudf.DataFrame( + {"j": cupy.tile([1, 2, 3], nr), "k": cupy.tile([4, 5, 7], nr)} + ), cudf.DataFrame( {"l": cupy.tile([4, 5, 7], nr)}, index=cudf.RangeIndex(start=nr * 3, stop=nr * 2 * 3), diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index 568d5c767b9..54ec70934e3 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -27,7 +27,9 @@ def bench_from_pandas(benchmark, pidx): def bench_constructor(benchmark, midx): - benchmark(cudf.MultiIndex, codes=midx.codes, levels=midx.levels, names=midx.names) + benchmark( + cudf.MultiIndex, codes=midx.codes, levels=midx.levels, names=midx.names + ) def bench_from_frame(benchmark, midx): diff --git a/python/cudf/benchmarks/common/config.py b/python/cudf/benchmarks/common/config.py index 51f17f8eff5..fd1cf84896a 100644 --- a/python/cudf/benchmarks/common/config.py +++ b/python/cudf/benchmarks/common/config.py @@ -39,9 +39,10 @@ def pytest_collection_modifyitems(session, config, items): items[:] = list(filter(is_pandas_compatible, items)) else: - import cudf # noqa: W0611, F401 import cupy # noqa: W0611, F401 + import cudf # noqa: W0611, F401 + def pytest_collection_modifyitems(session, config, items): pass diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index 4a391502673..c8665346359 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -90,7 +90,10 @@ def axis(request): def make_dataframe(nr, nc, column_generator=column_generator): assert nc <= len(string.ascii_lowercase) return cudf.DataFrame( - {f"{string.ascii_lowercase[i]}": column_generator(nr) for i in range(nc)} + { + f"{string.ascii_lowercase[i]}": column_generator(nr) + for i in range(nc) + } ) for nr in NUM_ROWS: @@ -102,7 +105,9 @@ def make_dataframe(nr, nc, column_generator=column_generator): # https://github.com/smarie/python-pytest-cases/issues/278 # Once that is fixed we could remove all the extraneous `request` # fixtures in these fixtures. - def series_nulls_false(request, nr=nr, column_generator=column_generator): + def series_nulls_false( + request, nr=nr, column_generator=column_generator + ): return cudf.Series(column_generator(nr)) make_fixture( @@ -112,7 +117,9 @@ def series_nulls_false(request, nr=nr, column_generator=column_generator): fixtures, ) - def series_nulls_true(request, nr=nr, column_generator=column_generator): + def series_nulls_true( + request, nr=nr, column_generator=column_generator + ): s = cudf.Series(column_generator(nr)) s.iloc[::2] = None return s @@ -125,7 +132,9 @@ def series_nulls_true(request, nr=nr, column_generator=column_generator): ) # For now, not bothering to include a nullable index fixture. - def index_nulls_false(request, nr=nr, column_generator=column_generator): + def index_nulls_false( + request, nr=nr, column_generator=column_generator + ): return cudf.Index(column_generator(nr)) make_fixture( @@ -208,7 +217,10 @@ def default_union_id(val): # that may or may not be nullable as well as Index objects. pytest_cases.fixture_union( name=f"frame_or_index_dtype_{dtype}", - fixtures=(f"indexedframe_dtype_{dtype}", f"index_dtype_{dtype}_nulls_false"), + fixtures=( + f"indexedframe_dtype_{dtype}", + f"index_dtype_{dtype}_nulls_false", + ), ids=["", f"index_dtype_{dtype}_nulls_false"], ) diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index 22c7f9f634f..f228a54e4ff 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -1,6 +1,10 @@ import pytest import pytest_cases -from utils import accepts_cudf_fixture, make_boolean_mask_column, make_gather_map +from utils import ( + accepts_cudf_fixture, + make_boolean_mask_column, + make_gather_map, +) @accepts_cudf_fixture(cls="column", dtype="float") @@ -24,7 +28,9 @@ def bench_unique_single_column(benchmark, column): @pytest.mark.parametrize("nullify", [True, False]) @pytest.mark.parametrize("gather_how", ["sequence", "reverse", "random"]) def bench_take(benchmark, column, gather_how, nullify): - gather_map = make_gather_map(column.size * 0.4, column.size, gather_how)._column + gather_map = make_gather_map( + column.size * 0.4, column.size, gather_how + )._column benchmark(column.take, gather_map, nullify=nullify) @@ -50,7 +56,9 @@ def setitem_case_int_column_scalar(column_dtype_int_nulls_false): return column, list(range(len(column))), 42 -def setitem_case_stride_1_slice_align_to_key_size(column_dtype_int_nulls_false): +def setitem_case_stride_1_slice_align_to_key_size( + column_dtype_int_nulls_false, +): column = column_dtype_int_nulls_false key = slice(None, None, 1) start, stop, stride = key.indices(len(column)) @@ -58,7 +66,9 @@ def setitem_case_stride_1_slice_align_to_key_size(column_dtype_int_nulls_false): return column, key, [42] * materialized_key_size -def setitem_case_stride_2_slice_align_to_key_size(column_dtype_int_nulls_false): +def setitem_case_stride_2_slice_align_to_key_size( + column_dtype_int_nulls_false, +): column = column_dtype_int_nulls_false key = slice(None, None, 2) start, stop, stride = key.indices(len(column)) @@ -66,7 +76,9 @@ def setitem_case_stride_2_slice_align_to_key_size(column_dtype_int_nulls_false): return column, key, [42] * materialized_key_size -def setitem_case_boolean_column_align_to_col_size(column_dtype_int_nulls_false): +def setitem_case_boolean_column_align_to_col_size( + column_dtype_int_nulls_false, +): column = column_dtype_int_nulls_false size = len(column) return column, [True, False] * (size // 2), [42] * size @@ -88,6 +100,8 @@ def setitem_case_int_column_align_to_col_size(column_dtype_int_nulls_false): # column (len(val) != len(key) & len == num_true) -@pytest_cases.parametrize_with_cases("column,key,value", cases=".", prefix="setitem") +@pytest_cases.parametrize_with_cases( + "column,key,value", cases=".", prefix="setitem" +) def bench_setitem(benchmark, column, key, value): benchmark(column.__setitem__, key, value) diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 8438beb45e0..40bd056fa65 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -1,5 +1,10 @@ from config import NUM_ROWS, cudf # noqa: E402 -from utils import OrderedSet, collapse_fixtures, column_generators, make_fixture +from utils import ( + OrderedSet, + collapse_fixtures, + column_generators, + make_fixture, +) fixtures = OrderedSet() for dtype, column_generator in column_generators.items(): From 7e3db91791608589f6318e031cfb64259f4c0b06 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:39:37 -0700 Subject: [PATCH 03/27] Add copyrights. --- python/cudf/benchmarks/API/bench_dataframe.py | 2 ++ python/cudf/benchmarks/API/bench_frame_or_index.py | 2 ++ python/cudf/benchmarks/API/bench_functions.py | 2 ++ python/cudf/benchmarks/API/bench_functions_cases.py | 2 ++ python/cudf/benchmarks/API/bench_index.py | 2 ++ python/cudf/benchmarks/API/bench_indexed_frame.py | 2 ++ python/cudf/benchmarks/API/bench_multiindex.py | 2 ++ python/cudf/benchmarks/API/bench_series.py | 2 ++ python/cudf/benchmarks/common/config.py | 2 ++ python/cudf/benchmarks/common/utils.py | 2 ++ python/cudf/benchmarks/conftest.py | 2 ++ python/cudf/benchmarks/internal/bench_column.py | 2 ++ python/cudf/benchmarks/internal/bench_dataframe_internal.py | 2 ++ python/cudf/benchmarks/internal/bench_rangeindex_internal.py | 3 +++ python/cudf/benchmarks/internal/conftest.py | 2 ++ python/cudf/benchmarks/pytest.ini | 2 ++ 16 files changed, 33 insertions(+) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 7fec5873977..4d6e7a79f83 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + import string import numpy diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py index 7ce31572bef..1ff3afd496a 100644 --- a/python/cudf/benchmarks/API/bench_frame_or_index.py +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Benchmarks of methods that exist for both Frame and BaseIndex.""" import operator diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index 4c22956ae3b..756c609d86a 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Benchmarks of free functions that accept cudf objects.""" import pytest diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index f8093cc8f3e..d938d506f69 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + import pytest_cases from config import NUM_ROWS, cudf, cupy diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py index 72d37a5abd7..e3796857b9d 100644 --- a/python/cudf/benchmarks/API/bench_index.py +++ b/python/cudf/benchmarks/API/bench_index.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Benchmarks of Index methods.""" import pytest diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py index f33f71b5644..bca974e3814 100644 --- a/python/cudf/benchmarks/API/bench_indexed_frame.py +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Benchmarks of IndexedFrame methods.""" import pytest diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index 54ec70934e3..68765870070 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + import numpy as np import pandas as pd import pytest diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py index 5caab20ea8b..7747210b72f 100644 --- a/python/cudf/benchmarks/API/bench_series.py +++ b/python/cudf/benchmarks/API/bench_series.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Benchmarks of Series methods.""" import pytest diff --git a/python/cudf/benchmarks/common/config.py b/python/cudf/benchmarks/common/config.py index fd1cf84896a..ef75dd50a07 100644 --- a/python/cudf/benchmarks/common/config.py +++ b/python/cudf/benchmarks/common/config.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Module used for global configuration of benchmarks. This file contains global definitions that are important for configuring all diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index 24799bdfeef..7d6be1bdf7c 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Common utilities for fixture creation and benchmarking.""" import inspect diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index c8665346359..43b0c2cf45f 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + """Standard location for shared fixtures and code across tests. Most fixtures defined in this file represent one of the primary classes in the diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index f228a54e4ff..6ed390810b2 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + import pytest import pytest_cases from utils import ( diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py index 7cc2c0ffbf3..c47baeade27 100644 --- a/python/cudf/benchmarks/internal/bench_dataframe_internal.py +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + from utils import accepts_cudf_fixture, make_boolean_mask_column diff --git a/python/cudf/benchmarks/internal/bench_rangeindex_internal.py b/python/cudf/benchmarks/internal/bench_rangeindex_internal.py index 8e7a6304e99..4098ec43a54 100644 --- a/python/cudf/benchmarks/internal/bench_rangeindex_internal.py +++ b/python/cudf/benchmarks/internal/bench_rangeindex_internal.py @@ -1,3 +1,6 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + + def bench_column(benchmark, rangeindex): benchmark(lambda: rangeindex._column) diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 40bd056fa65..20686c8aa2b 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + from config import NUM_ROWS, cudf # noqa: E402 from utils import ( OrderedSet, diff --git a/python/cudf/benchmarks/pytest.ini b/python/cudf/benchmarks/pytest.ini index 75e90cb21bf..db24415ef9e 100644 --- a/python/cudf/benchmarks/pytest.ini +++ b/python/cudf/benchmarks/pytest.ini @@ -1,3 +1,5 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + [pytest] python_files = bench_*.py python_classes = Bench From af6882f1da27026d04f4fff8979e4ea2cec640eb Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:42:23 -0700 Subject: [PATCH 04/27] Add missing module docstrings. --- python/cudf/benchmarks/API/bench_dataframe.py | 2 ++ python/cudf/benchmarks/API/bench_functions_cases.py | 2 ++ python/cudf/benchmarks/conftest.py | 3 +-- python/cudf/benchmarks/internal/bench_column.py | 2 ++ python/cudf/benchmarks/internal/bench_dataframe_internal.py | 2 ++ python/cudf/benchmarks/internal/bench_rangeindex_internal.py | 2 ++ python/cudf/benchmarks/internal/conftest.py | 2 ++ 7 files changed, 13 insertions(+), 2 deletions(-) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 4d6e7a79f83..9f5cb3b95ce 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Benchmarks of DataFrame methods.""" + import string import numpy diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index d938d506f69..c81f8f20f80 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Test cases for benchmarks in bench_functions.py.""" + import pytest_cases from config import NUM_ROWS, cudf, cupy diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index 43b0c2cf45f..83734d17614 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -1,6 +1,6 @@ # Copyright (c) 2022, NVIDIA CORPORATION. -"""Standard location for shared fixtures and code across tests. +"""Defines pytest fixtures for all benchmarks. Most fixtures defined in this file represent one of the primary classes in the cuDF ecosystem such as DataFrame, Series, or Index. These fixtures may in turn @@ -44,7 +44,6 @@ As a result, it is provided as a separate fixture. """ - import os import string import sys diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index 6ed390810b2..933da9a7d4b 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Benchmarks of Column methods.""" + import pytest import pytest_cases from utils import ( diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py index c47baeade27..c0f22c1efb2 100644 --- a/python/cudf/benchmarks/internal/bench_dataframe_internal.py +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Benchmarks of internal DataFrame methods.""" + from utils import accepts_cudf_fixture, make_boolean_mask_column diff --git a/python/cudf/benchmarks/internal/bench_rangeindex_internal.py b/python/cudf/benchmarks/internal/bench_rangeindex_internal.py index 4098ec43a54..c4cf1de8ab9 100644 --- a/python/cudf/benchmarks/internal/bench_rangeindex_internal.py +++ b/python/cudf/benchmarks/internal/bench_rangeindex_internal.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Benchmarks of internal RangeIndex methods.""" + def bench_column(benchmark, rangeindex): benchmark(lambda: rangeindex._column) diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 20686c8aa2b..50f57973007 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Defines pytest fixtures for internal benchmarks.""" + from config import NUM_ROWS, cudf # noqa: E402 from utils import ( OrderedSet, From 97c601c0ec8f13c2d1f7161750e534de5c1044ec Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:44:23 -0700 Subject: [PATCH 05/27] Fix long lines for flake8. --- python/cudf/benchmarks/common/utils.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index 7d6be1bdf7c..56a7dad98d7 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -114,17 +114,20 @@ def bench_columns(benchmark, df): "indexedframe", "frame_or_index", ) - assert ( - cls in supported_classes - ), f"cls {cls} is invalid, choose from {', '.join(c for c in supported_classes)}" + assert cls in supported_classes, ( + f"cls {cls} is invalid, choose from " + f"{', '.join(c for c in supported_classes)}" + ) name = name or cls if not isinstance(dtype, list): dtype = [dtype] - assert all( - dt in column_generators for dt in dtype - ), f"The only supported dtypes are {', '.join(dt for dt in column_generators)}" + assert all(dt in column_generators for dt in dtype), ( + f"The only supported dtypes are " + f"{', '.join(dt for dt in column_generators)}" + ) + dtype_str = "_dtype_" + "_or_".join(dtype) null_str = "" From 55434afc594d2d2e2adbcc9f89439b2ff476bdea Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:44:55 -0700 Subject: [PATCH 06/27] Remove unnecessary README. --- python/cudf/benchmarks/README.md | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 python/cudf/benchmarks/README.md diff --git a/python/cudf/benchmarks/README.md b/python/cudf/benchmarks/README.md deleted file mode 100644 index 1441069c57d..00000000000 --- a/python/cudf/benchmarks/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# About - -Benchmarks of cuDF. - -Benchmarks of public APIs are contained in the API subdirectory. -Benchmarks of non-public internals are contained in the internal subdirectory. From 97bbe3e879d9571bf4f4d455d4c7755686651aff Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:49:19 -0700 Subject: [PATCH 07/27] Run benchmarks as tests in CI. --- ci/gpu/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index a1311d489eb..b720fa99633 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -235,6 +235,11 @@ cd "$WORKSPACE/python/cudf/cudf" gpuci_logger "Python py.test for cuDF" py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope tests +# Run benchmarks with both cudf and pandas to ensure compatibility is maintained. +cd "$WORKSPACE/python/cudf" +CUDF_BENCHMARKS_TEST_ONLY=ON pytest -n 8 benchmarks +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_TEST_ONLY=ON pytest -n 8 benchmarks + cd "$WORKSPACE/python/dask_cudf" gpuci_logger "Python py.test for dask-cudf" py.test -n 8 --cache-clear --basetemp="$WORKSPACE/dask-cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-dask-cudf.xml" -v --cov-config=.coveragerc --cov=dask_cudf --cov-report=xml:"$WORKSPACE/python/dask_cudf/dask-cudf-coverage.xml" --cov-report term dask_cudf From 0dd018b06c44813c29389e4a1c58eba24a661308 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 17 Jun 2022 14:53:36 -0700 Subject: [PATCH 08/27] Update environment. --- conda/environments/cudf_dev_cuda11.5.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index b2e77f9c9fb..fd2879c1902 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -28,6 +28,7 @@ dependencies: - cython>=0.29,<0.30 - fsspec>=0.6.0 - pytest + - pytest-cases - pytest-benchmark - pytest-xdist - sphinx From 7c370f4de4199637c3cb76de036a6e837c7d0973 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 21 Jun 2022 09:54:10 -0700 Subject: [PATCH 09/27] Enable usage of accepts_cudf_fixture with cases. --- python/cudf/benchmarks/common/utils.py | 3 ++ python/cudf/benchmarks/conftest.py | 2 +- .../cudf/benchmarks/internal/bench_column.py | 44 ++++++++++--------- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index 56a7dad98d7..c04ac2584c7 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -188,6 +188,9 @@ def wrapped_bm({params_str}): # In case marks were applied to the original benchmark, copy them over. if marks := getattr(bm, "pytestmark", None): wrapped_bm.pytestmark = marks + wrapped_bm.__name__ = bm.__name__ + wrapped_bm.__module__ = bm.__module__ + wrapped_bm.place_as = bm return wrapped_bm return deco diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index 83734d17614..be0f4e441b0 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -54,7 +54,7 @@ # sessionfinish hooks), we could just make the benchmarks a (sub)package to # enable relative imports. A minor change to consider when these are ported # into the main repo. -sys.path.insert(0, os.path.join(os.getcwd(), "common")) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) from config import cudf # noqa: W0611, E402, F401 from utils import ( # noqa: E402 diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index 933da9a7d4b..fea556da3e6 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -38,58 +38,62 @@ def bench_take(benchmark, column, gather_how, nullify): benchmark(column.take, gather_map, nullify=nullify) -# TODO: Due to https://github.com/smarie/python-pytest-cases/issues/280 we -# cannot use the accepts_cudf_fixture decorator for cases. If and when that is -# resolved, we can change all of the cases below to use that instead of -# hardcoding the fixture name. -def setitem_case_stride_1_slice_scalar(column_dtype_int_nulls_false): - return column_dtype_int_nulls_false, slice(None, None, 1), 42 +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +def setitem_case_stride_1_slice_scalar(column): + return column, slice(None, None, 1), 42 -def setitem_case_stride_2_slice_scalar(column_dtype_int_nulls_false): - return column_dtype_int_nulls_false, slice(None, None, 2), 42 +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +def setitem_case_stride_2_slice_scalar(column): + return column, slice(None, None, 2), 42 -def setitem_case_boolean_column_scalar(column_dtype_int_nulls_false): - column = column_dtype_int_nulls_false +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +def setitem_case_boolean_column_scalar(column): + column = column return column, [True, False] * (len(column) // 2), 42 -def setitem_case_int_column_scalar(column_dtype_int_nulls_false): - column = column_dtype_int_nulls_false +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +def setitem_case_int_column_scalar(column): + column = column return column, list(range(len(column))), 42 +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) def setitem_case_stride_1_slice_align_to_key_size( - column_dtype_int_nulls_false, + column, ): - column = column_dtype_int_nulls_false + column = column key = slice(None, None, 1) start, stop, stride = key.indices(len(column)) materialized_key_size = len(column.slice(start, stop, stride)) return column, key, [42] * materialized_key_size +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) def setitem_case_stride_2_slice_align_to_key_size( - column_dtype_int_nulls_false, + column, ): - column = column_dtype_int_nulls_false + column = column key = slice(None, None, 2) start, stop, stride = key.indices(len(column)) materialized_key_size = len(column.slice(start, stop, stride)) return column, key, [42] * materialized_key_size +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) def setitem_case_boolean_column_align_to_col_size( - column_dtype_int_nulls_false, + column, ): - column = column_dtype_int_nulls_false + column = column size = len(column) return column, [True, False] * (size // 2), [42] * size -def setitem_case_int_column_align_to_col_size(column_dtype_int_nulls_false): - column = column_dtype_int_nulls_false +@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +def setitem_case_int_column_align_to_col_size(column): + column = column size = len(column) return column, list(range(size)), [42] * size From 83ee6b7d1895fd7925f092f0ba9deaa912cc3aa6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 21 Jun 2022 09:58:19 -0700 Subject: [PATCH 10/27] Use makefun to create the wrapper more thoroughly. --- python/cudf/benchmarks/common/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index c04ac2584c7..d1690ff9184 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -178,6 +178,12 @@ def deco(bm): src = textwrap.dedent( f""" + import makefun + @makefun.wraps( + bm, + remove_args=("{name}",), + prepend_args=("{fixture_name}",) + ) def wrapped_bm({params_str}): return bm({arg_str}) """ @@ -188,8 +194,6 @@ def wrapped_bm({params_str}): # In case marks were applied to the original benchmark, copy them over. if marks := getattr(bm, "pytestmark", None): wrapped_bm.pytestmark = marks - wrapped_bm.__name__ = bm.__name__ - wrapped_bm.__module__ = bm.__module__ wrapped_bm.place_as = bm return wrapped_bm From c61ba9f3c27ed7680b2ca3aced6ac83d11c81edf Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 21 Jun 2022 10:13:00 -0700 Subject: [PATCH 11/27] Address a couple minor parts of @isVoid review. --- python/cudf/benchmarks/common/utils.py | 8 -------- python/cudf/benchmarks/conftest.py | 4 +++- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index d1690ff9184..b17211d1357 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -42,14 +42,6 @@ def make_boolean_mask_column(size): return cudf.core.column.as_column(rstate.randint(0, 2, size).astype(bool)) -def flatten(xs): - for x in xs: - if not isinstance(x, str): - yield from x - else: - yield x - - def accepts_cudf_fixture( cls, *, dtype="int", nulls=None, cols=None, rows=None, name=None ): diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index be0f4e441b0..fe97fd5d36f 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -89,7 +89,9 @@ def axis(request): for dtype, column_generator in column_generators.items(): def make_dataframe(nr, nc, column_generator=column_generator): - assert nc <= len(string.ascii_lowercase) + assert nc <= len( + string.ascii_lowercase + ), "make_dataframe only supports a maximum of 26 columns" return cudf.DataFrame( { f"{string.ascii_lowercase[i]}": column_generator(nr) From 690577fc4b44eaa9c814ceb3d52a7a7c8947647b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 21 Jun 2022 10:14:21 -0700 Subject: [PATCH 12/27] Rename CUDF_BENCHMARKS_TEST_ONLY to CUDF_BENCHMARKS_DEBUG_ONLY. --- ci/gpu/build.sh | 4 ++-- python/cudf/benchmarks/common/config.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index b720fa99633..a2ed68222d0 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -237,8 +237,8 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORK # Run benchmarks with both cudf and pandas to ensure compatibility is maintained. cd "$WORKSPACE/python/cudf" -CUDF_BENCHMARKS_TEST_ONLY=ON pytest -n 8 benchmarks -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_TEST_ONLY=ON pytest -n 8 benchmarks +CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 benchmarks +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 benchmarks cd "$WORKSPACE/python/dask_cudf" gpuci_logger "Python py.test for dask-cudf" diff --git a/python/cudf/benchmarks/common/config.py b/python/cudf/benchmarks/common/config.py index ef75dd50a07..305a21d0a29 100644 --- a/python/cudf/benchmarks/common/config.py +++ b/python/cudf/benchmarks/common/config.py @@ -10,7 +10,7 @@ cupy). This feature enables easy comparisons of benchmarks between cudf and pandas. All common modules (cudf, cupy) should be imported from here by benchmark modules to allow configuration if needed. - - Defining CUDF_BENCHMARKS_TEST_ONLY will set global configuration + - Defining CUDF_BENCHMARKS_DEBUG_ONLY will set global configuration variables to avoid running large benchmarks, instead using minimal values to simply ensure that benchmarks are functional. @@ -61,7 +61,7 @@ def pytest_sessionfinish(session, exitstatus): # Constants used to define benchmarking standards. -if "CUDF_BENCHMARKS_TEST_ONLY" in os.environ: +if "CUDF_BENCHMARKS_DEBUG_ONLY" in os.environ: NUM_ROWS = [10, 20] NUM_COLS = [1, 6] else: From 1025be4ddb86cbc46fe75d8da3b72bfce910df56 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 10:02:29 -0700 Subject: [PATCH 13/27] Address most review comments. --- conda/environments/cudf_dev_cuda11.5.yml | 2 +- python/cudf/benchmarks/API/bench_dataframe.py | 32 ++++++++++--------- python/cudf/benchmarks/API/bench_functions.py | 8 ++--- python/cudf/benchmarks/common/utils.py | 16 ++++------ python/cudf/benchmarks/conftest.py | 2 +- python/cudf/benchmarks/internal/conftest.py | 2 +- 6 files changed, 29 insertions(+), 33 deletions(-) diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml index fd2879c1902..76c242d672d 100644 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ b/conda/environments/cudf_dev_cuda11.5.yml @@ -28,8 +28,8 @@ dependencies: - cython>=0.29,<0.30 - fsspec>=0.6.0 - pytest - - pytest-cases - pytest-benchmark + - pytest-cases - pytest-xdist - sphinx - sphinxcontrib-websupport diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 9f5cb3b95ce..4124d4b1af1 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -27,11 +27,11 @@ def bench_eval_func(benchmark, expr, dataframe): cls="dataframe", dtype="int", nulls=False, cols=6, name="df" ) @pytest.mark.parametrize( - "nkey_cols", + "num_key_cols", [2, 3, 4], ) -def bench_merge(benchmark, df, nkey_cols): - benchmark(df.merge, df, on=list(df.columns[:nkey_cols])) +def bench_merge(benchmark, df, num_key_cols): + benchmark(df.merge, df, on=list(df.columns[:num_key_cols])) # TODO: Some of these cases could be generalized to an IndexedFrame benchmark @@ -71,11 +71,11 @@ def bench_sample(benchmark, dataframe, axis, frac, random_state): @accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) @pytest.mark.parametrize( - "nkey_cols", + "num_key_cols", [2, 3, 4], ) -def bench_groupby(benchmark, dataframe, nkey_cols): - benchmark(dataframe.groupby, by=list(dataframe.columns[:nkey_cols])) +def bench_groupby(benchmark, dataframe, num_key_cols): + benchmark(dataframe.groupby, by=list(dataframe.columns[:num_key_cols])) @accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) @@ -91,25 +91,27 @@ def bench_groupby(benchmark, dataframe, nkey_cols): ], ) @pytest.mark.parametrize( - "nkey_cols", + "num_key_cols", [2, 3, 4], ) @pytest.mark.parametrize("as_index", [True, False]) @pytest.mark.parametrize("sort", [True, False]) -def bench_groupby_agg(benchmark, dataframe, agg, nkey_cols, as_index, sort): - by = list(dataframe.columns[:nkey_cols]) +def bench_groupby_agg(benchmark, dataframe, agg, num_key_cols, as_index, sort): + by = list(dataframe.columns[:num_key_cols]) benchmark(dataframe.groupby(by=by, as_index=as_index, sort=sort).agg, agg) @accepts_cudf_fixture(cls="dataframe", dtype="int") -@pytest.mark.parametrize("ncol_sort", [1]) -def bench_sort_values(benchmark, dataframe, ncol_sort): - benchmark(dataframe.sort_values, list(dataframe.columns[:ncol_sort])) +@pytest.mark.parametrize("num_cols_to_sort", [1]) +def bench_sort_values(benchmark, dataframe, num_cols_to_sort): + benchmark( + dataframe.sort_values, list(dataframe.columns[:num_cols_to_sort]) + ) @accepts_cudf_fixture(cls="dataframe", dtype="int") -@pytest.mark.parametrize("ncol_sort", [1]) +@pytest.mark.parametrize("num_cols_to_sort", [1]) @pytest.mark.parametrize("n", [10]) -def bench_nsmallest(benchmark, dataframe, ncol_sort, n): - by = list(dataframe.columns[:ncol_sort]) +def bench_nsmallest(benchmark, dataframe, num_cols_to_sort, n): + by = list(dataframe.columns[:num_cols_to_sort]) benchmark(dataframe.nsmallest, n, by) diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index 756c609d86a..a166317a46b 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -26,9 +26,7 @@ def bench_concat_axis_1(benchmark, objs, axis, join, ignore_index): @pytest.mark.parametrize("cardinality", [10, 100, 1000]) @pytest.mark.parametrize("dtype", [cupy.bool_, cupy.float64]) def bench_get_dummies_high_cardinality(benchmark, size, cardinality, dtype): - """This test is mean to test the performance of get_dummies given the - cardinality of column to encode is high. - """ + """Benchmark when the cardinality of column to encode is high.""" df = cudf.DataFrame( { "col": cudf.Series( @@ -41,9 +39,7 @@ def bench_get_dummies_high_cardinality(benchmark, size, cardinality, dtype): @pytest.mark.parametrize("prefix", [None, "pre"]) def bench_get_dummies_simple(benchmark, prefix): - """This test provides a small input to get_dummies to test the efficiency - of the API itself. - """ + """Benchmark with small input to test the efficiency of the API itself.""" df = cudf.DataFrame( { "col1": list(range(10)), diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index b17211d1357..bfd765ea913 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -50,7 +50,7 @@ def accepts_cudf_fixture( The fixture generation logic in conftest.py provides a plethora of useful fixtures to allow developers to easily select an appropriate cross-section of the space of objects to apply a particular benchmark to. However, the - usage of this fixtures is cumbersome because creating them in a principled + usage of these fixtures is cumbersome because creating them in a principled fashion results in long names and very specific naming schemes. This decorator abstracts that naming logic away from the developer, allowing them to instead focus on defining the fixture semantically by describing @@ -74,7 +74,7 @@ def accepts_cudf_fixture( rows : Optional[int], None The number of rows. If None, use all possible numbers of rows. Specifying multiple values is unsupported. - fixture : str, default None + name : str, default None The name of the fixture as used in the decorated test. If None, defaults to `cls.lower()` if cls is a string, otherwise `cls.__name__.lower()`. Use of this name allows the decorated function @@ -107,8 +107,7 @@ def bench_columns(benchmark, df): "frame_or_index", ) assert cls in supported_classes, ( - f"cls {cls} is invalid, choose from " - f"{', '.join(c for c in supported_classes)}" + f"cls {cls} is invalid, choose from " f"{', '.join(supported_classes)}" ) name = name or cls @@ -116,8 +115,7 @@ def bench_columns(benchmark, df): if not isinstance(dtype, list): dtype = [dtype] assert all(dt in column_generators for dt in dtype), ( - f"The only supported dtypes are " - f"{', '.join(dt for dt in column_generators)}" + f"The only supported dtypes are " f"{', '.join(column_generators)}" ) dtype_str = "_dtype_" + "_or_".join(dtype) @@ -130,7 +128,7 @@ def bench_columns(benchmark, df): if cols is not None: assert cols in NUM_COLS, ( f"You have requested a DataFrame with {cols} columns but fixtures " - f"only exist for the values {', '.join(c for c in NUM_COLS)}" + f"only exist for the values {', '.join(NUM_COLS)}" ) col_str = f"_cols_{cols}" @@ -138,14 +136,14 @@ def bench_columns(benchmark, df): if rows is not None: assert rows in NUM_ROWS, ( f"You have requested a {cls} with {rows} rows but fixtures " - f"only exist for the values {', '.join(c for c in NUM_ROWS)}" + f"only exist for the values {', '.join(NUM_ROWS)}" ) row_str = f"_rows_{rows}" fixture_name = f"{cls}{dtype_str}{null_str}{col_str}{row_str}" def deco(bm): - # pytests's test collection process relies on parsing the globals dict + # pytest's test collection process relies on parsing the globals dict # to find test functions and identify their parameters for the purpose # of fixtures and parameters. Therefore, the primary purpose of this # decorator is to define a new benchmark function with a signature diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index fe97fd5d36f..4f2bb96061f 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -31,7 +31,7 @@ which they are valid, e.g. `def bench_sort_values(frame_or_index)`. The generated fixtures are named according to the following convention: -`{classname}_dtype_{dtype}[_nulls_{true|false}][[_cols_{num_cols}]_rows_{num_rows}]` +`{classname}_dtype_{dtype}[_nulls_{true|false}][_cols_{num_cols}][_rows_{num_rows}]` where classname is one of the following: index, series, dataframe, indexedframe, frame, frame_or_index. Note that in the case of indexes, to match Series/DataFrame we simply set `classname=index` and rely on the diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 50f57973007..7351f1d1427 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -2,7 +2,7 @@ """Defines pytest fixtures for internal benchmarks.""" -from config import NUM_ROWS, cudf # noqa: E402 +from config import NUM_ROWS, cudf from utils import ( OrderedSet, collapse_fixtures, From 5db4589c2df62a1fc77cce64d24700adc32e69b7 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 10:27:50 -0700 Subject: [PATCH 14/27] Convert the benchmarks into a package so that we can use relative imports. --- python/cudf/benchmarks/API/__init__.py | 1 + python/cudf/benchmarks/API/bench_dataframe.py | 5 +-- .../benchmarks/API/bench_frame_or_index.py | 3 +- python/cudf/benchmarks/API/bench_functions.py | 3 +- .../benchmarks/API/bench_functions_cases.py | 3 +- python/cudf/benchmarks/API/bench_index.py | 5 +-- .../benchmarks/API/bench_indexed_frame.py | 3 +- .../cudf/benchmarks/API/bench_multiindex.py | 3 +- python/cudf/benchmarks/API/bench_series.py | 5 +-- python/cudf/benchmarks/__init__.py | 1 + python/cudf/benchmarks/common/config.py | 12 ------- python/cudf/benchmarks/common/utils.py | 3 +- python/cudf/benchmarks/conftest.py | 32 +++++-------------- python/cudf/benchmarks/internal/__init__.py | 1 + .../cudf/benchmarks/internal/bench_column.py | 3 +- .../internal/bench_dataframe_internal.py | 2 +- python/cudf/benchmarks/internal/conftest.py | 4 +-- 17 files changed, 37 insertions(+), 52 deletions(-) create mode 100644 python/cudf/benchmarks/API/__init__.py create mode 100644 python/cudf/benchmarks/__init__.py create mode 100644 python/cudf/benchmarks/internal/__init__.py diff --git a/python/cudf/benchmarks/API/__init__.py b/python/cudf/benchmarks/API/__init__.py new file mode 100644 index 00000000000..6feb3c33ee1 --- /dev/null +++ b/python/cudf/benchmarks/API/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 4124d4b1af1..799527b1eec 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -6,8 +6,9 @@ import numpy import pytest -from config import cudf, cupy -from utils import accepts_cudf_fixture + +from ..common.config import cudf, cupy +from ..common.utils import accepts_cudf_fixture @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py index 1ff3afd496a..b00180d46d7 100644 --- a/python/cudf/benchmarks/API/bench_frame_or_index.py +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -6,7 +6,8 @@ import numpy as np import pytest -from utils import accepts_cudf_fixture, make_gather_map + +from ..common.utils import accepts_cudf_fixture, make_gather_map @accepts_cudf_fixture(cls="frame_or_index", dtype="int") diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index a166317a46b..2dabd957e60 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -4,7 +4,8 @@ import pytest import pytest_cases -from config import cudf, cupy + +from ..common.config import cudf, cupy @pytest_cases.parametrize_with_cases("objs", prefix="concat") diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index c81f8f20f80..1562527241d 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -3,7 +3,8 @@ """Test cases for benchmarks in bench_functions.py.""" import pytest_cases -from config import NUM_ROWS, cudf, cupy + +from ..common.config import NUM_ROWS, cudf, cupy @pytest_cases.parametrize("nr", NUM_ROWS) diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py index e3796857b9d..f51a38ef87b 100644 --- a/python/cudf/benchmarks/API/bench_index.py +++ b/python/cudf/benchmarks/API/bench_index.py @@ -3,8 +3,9 @@ """Benchmarks of Index methods.""" import pytest -from config import cudf, cupy -from utils import accepts_cudf_fixture + +from ..common.config import cudf, cupy +from ..common.utils import accepts_cudf_fixture @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py index bca974e3814..7502ec2b509 100644 --- a/python/cudf/benchmarks/API/bench_indexed_frame.py +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -3,7 +3,8 @@ """Benchmarks of IndexedFrame methods.""" import pytest -from utils import accepts_cudf_fixture + +from ..common.utils import accepts_cudf_fixture @accepts_cudf_fixture(cls="indexedframe", dtype="int") diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index 68765870070..75bde5f9d63 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -3,7 +3,8 @@ import numpy as np import pandas as pd import pytest -from config import cudf + +from ..common.config import cudf @pytest.fixture diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py index 7747210b72f..0bedbf5ba77 100644 --- a/python/cudf/benchmarks/API/bench_series.py +++ b/python/cudf/benchmarks/API/bench_series.py @@ -3,8 +3,9 @@ """Benchmarks of Series methods.""" import pytest -from config import cudf, cupy -from utils import accepts_cudf_fixture + +from ..common.config import cudf, cupy +from ..common.utils import accepts_cudf_fixture @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/__init__.py b/python/cudf/benchmarks/__init__.py new file mode 100644 index 00000000000..6feb3c33ee1 --- /dev/null +++ b/python/cudf/benchmarks/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. diff --git a/python/cudf/benchmarks/common/config.py b/python/cudf/benchmarks/common/config.py index 305a21d0a29..49e80653848 100644 --- a/python/cudf/benchmarks/common/config.py +++ b/python/cudf/benchmarks/common/config.py @@ -21,7 +21,6 @@ appropriately. """ import os -import sys # Environment variable-based configuration of benchmarking pandas or cudf. collect_ignore = [] @@ -49,17 +48,6 @@ def pytest_collection_modifyitems(session, config, items): pass -def pytest_sessionstart(session): - """Add the common files to the path for all tests to import.""" - sys.path.insert(0, os.path.join(os.getcwd(), "common")) - - -def pytest_sessionfinish(session, exitstatus): - """Clean up sys.path after exit.""" - if "common" in sys.path[0]: - del sys.path[0] - - # Constants used to define benchmarking standards. if "CUDF_BENCHMARKS_DEBUG_ONLY" in os.environ: NUM_ROWS = [10, 20] diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index bfd765ea913..d2d14a8902a 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -10,7 +10,8 @@ from numbers import Real import pytest_cases -from config import NUM_COLS, NUM_ROWS, cudf, cupy + +from .config import NUM_COLS, NUM_ROWS, cudf, cupy def make_gather_map(len_gather_map: Real, len_column: Real, how: str): diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index 4f2bb96061f..c8e04dd352f 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -44,39 +44,23 @@ As a result, it is provided as a separate fixture. """ -import os import string -import sys import pytest_cases -# TODO: Rather than doing this path hacking (including the sessionstart and -# sessionfinish hooks), we could just make the benchmarks a (sub)package to -# enable relative imports. A minor change to consider when these are ported -# into the main repo. -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) - -from config import cudf # noqa: W0611, E402, F401 -from utils import ( # noqa: E402 - OrderedSet, - collapse_fixtures, - column_generators, - make_fixture, -) - -# Turn off isort until we upgrade to 5.8.0 -# https://github.com/pycqa/isort/issues/1594 -# isort: off -from config import ( # noqa: W0611, E402, F401 +from .common.config import ( # noqa: W0611, F401 NUM_COLS, NUM_ROWS, collect_ignore, + cudf, pytest_collection_modifyitems, - pytest_sessionfinish, - pytest_sessionstart, ) - -# isort: on +from .common.utils import ( + OrderedSet, + collapse_fixtures, + column_generators, + make_fixture, +) @pytest_cases.fixture(params=[0, 1], ids=["AxisIndex", "AxisColumn"]) diff --git a/python/cudf/benchmarks/internal/__init__.py b/python/cudf/benchmarks/internal/__init__.py new file mode 100644 index 00000000000..6feb3c33ee1 --- /dev/null +++ b/python/cudf/benchmarks/internal/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index fea556da3e6..8dce68828e2 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -4,7 +4,8 @@ import pytest import pytest_cases -from utils import ( + +from ..common.utils import ( accepts_cudf_fixture, make_boolean_mask_column, make_gather_map, diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py index c0f22c1efb2..af5dc6a377e 100644 --- a/python/cudf/benchmarks/internal/bench_dataframe_internal.py +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -2,7 +2,7 @@ """Benchmarks of internal DataFrame methods.""" -from utils import accepts_cudf_fixture, make_boolean_mask_column +from ..common.utils import accepts_cudf_fixture, make_boolean_mask_column @accepts_cudf_fixture(cls="dataframe", dtype="int") diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 7351f1d1427..001796c781c 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -2,8 +2,8 @@ """Defines pytest fixtures for internal benchmarks.""" -from config import NUM_ROWS, cudf -from utils import ( +from ..common.config import NUM_ROWS, cudf +from ..common.utils import ( OrderedSet, collapse_fixtures, column_generators, From 6d57c10a9e787a9695d0d27211b6e68b4cd8e596 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 11:35:05 -0700 Subject: [PATCH 15/27] Rename accepts_cudf_fixture to benchmark_with_object. --- python/cudf/benchmarks/API/bench_dataframe.py | 18 ++++++------ .../benchmarks/API/bench_frame_or_index.py | 28 +++++++++---------- python/cudf/benchmarks/API/bench_index.py | 4 +-- .../benchmarks/API/bench_indexed_frame.py | 10 +++---- python/cudf/benchmarks/API/bench_series.py | 6 ++-- python/cudf/benchmarks/common/utils.py | 6 ++-- .../cudf/benchmarks/internal/bench_column.py | 26 ++++++++--------- .../internal/bench_dataframe_internal.py | 4 +-- 8 files changed, 51 insertions(+), 51 deletions(-) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 799527b1eec..0e6c36fa12e 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -8,7 +8,7 @@ import pytest from ..common.config import cudf, cupy -from ..common.utils import accepts_cudf_fixture +from ..common.utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) @@ -16,7 +16,7 @@ def bench_construction(benchmark, N): benchmark(cudf.DataFrame, {None: cupy.random.rand(N)}) -@accepts_cudf_fixture(cls="dataframe", dtype="float", cols=6) +@benchmark_with_object(cls="dataframe", dtype="float", cols=6) @pytest.mark.parametrize( "expr", ["a+b", "a+b+c+d+e", "a / (sin(a) + cos(b)) * tanh(d*e*f)"] ) @@ -24,7 +24,7 @@ def bench_eval_func(benchmark, expr, dataframe): benchmark(dataframe.eval, expr) -@accepts_cudf_fixture( +@benchmark_with_object( cls="dataframe", dtype="int", nulls=False, cols=6, name="df" ) @pytest.mark.parametrize( @@ -37,7 +37,7 @@ def bench_merge(benchmark, df, num_key_cols): # TODO: Some of these cases could be generalized to an IndexedFrame benchmark # instead of a DataFrame benchmark. -@accepts_cudf_fixture(cls="dataframe", dtype="int") +@benchmark_with_object(cls="dataframe", dtype="int") @pytest.mark.parametrize( "values", [ @@ -60,7 +60,7 @@ def random_state(request): return rs if isinstance(rs, int) else rs(seed=42) -@accepts_cudf_fixture(cls="dataframe", dtype="int") +@benchmark_with_object(cls="dataframe", dtype="int") @pytest.mark.parametrize("frac", [0.5]) def bench_sample(benchmark, dataframe, axis, frac, random_state): if axis == 1 and isinstance(random_state, cupy.random.RandomState): @@ -70,7 +70,7 @@ def bench_sample(benchmark, dataframe, axis, frac, random_state): ) -@accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) +@benchmark_with_object(cls="dataframe", dtype="int", nulls=False, cols=6) @pytest.mark.parametrize( "num_key_cols", [2, 3, 4], @@ -79,7 +79,7 @@ def bench_groupby(benchmark, dataframe, num_key_cols): benchmark(dataframe.groupby, by=list(dataframe.columns[:num_key_cols])) -@accepts_cudf_fixture(cls="dataframe", dtype="int", nulls=False, cols=6) +@benchmark_with_object(cls="dataframe", dtype="int", nulls=False, cols=6) @pytest.mark.parametrize( "agg", [ @@ -102,7 +102,7 @@ def bench_groupby_agg(benchmark, dataframe, agg, num_key_cols, as_index, sort): benchmark(dataframe.groupby(by=by, as_index=as_index, sort=sort).agg, agg) -@accepts_cudf_fixture(cls="dataframe", dtype="int") +@benchmark_with_object(cls="dataframe", dtype="int") @pytest.mark.parametrize("num_cols_to_sort", [1]) def bench_sort_values(benchmark, dataframe, num_cols_to_sort): benchmark( @@ -110,7 +110,7 @@ def bench_sort_values(benchmark, dataframe, num_cols_to_sort): ) -@accepts_cudf_fixture(cls="dataframe", dtype="int") +@benchmark_with_object(cls="dataframe", dtype="int") @pytest.mark.parametrize("num_cols_to_sort", [1]) @pytest.mark.parametrize("n", [10]) def bench_nsmallest(benchmark, dataframe, num_cols_to_sort, n): diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py index b00180d46d7..8243f582f83 100644 --- a/python/cudf/benchmarks/API/bench_frame_or_index.py +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -7,10 +7,10 @@ import numpy as np import pytest -from ..common.utils import accepts_cudf_fixture, make_gather_map +from ..common.utils import benchmark_with_object, make_gather_map -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") @pytest.mark.parametrize("gather_how", ["sequence", "reverse", "random"]) @pytest.mark.parametrize("fraction", [0.4]) def bench_take(benchmark, gather_how, fraction, frame_or_index): @@ -20,62 +20,62 @@ def bench_take(benchmark, gather_how, fraction, frame_or_index): @pytest.mark.pandas_incompatible # Series/Index work, but not DataFrame -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_argsort(benchmark, frame_or_index): benchmark(frame_or_index.argsort) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_min(benchmark, frame_or_index): benchmark(frame_or_index.min) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_where(benchmark, frame_or_index): cond = frame_or_index % 2 == 0 benchmark(frame_or_index.where, cond, 0) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +@benchmark_with_object(cls="frame_or_index", dtype="int", nulls=False) @pytest.mark.pandas_incompatible def bench_values_host(benchmark, frame_or_index): benchmark(lambda: frame_or_index.values_host) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +@benchmark_with_object(cls="frame_or_index", dtype="int", nulls=False) def bench_values(benchmark, frame_or_index): benchmark(lambda: frame_or_index.values) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_nunique(benchmark, frame_or_index): benchmark(frame_or_index.nunique) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +@benchmark_with_object(cls="frame_or_index", dtype="int", nulls=False) def bench_to_numpy(benchmark, frame_or_index): benchmark(frame_or_index.to_numpy) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int", nulls=False) +@benchmark_with_object(cls="frame_or_index", dtype="int", nulls=False) @pytest.mark.pandas_incompatible def bench_to_cupy(benchmark, frame_or_index): benchmark(frame_or_index.to_cupy) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") @pytest.mark.pandas_incompatible def bench_to_arrow(benchmark, frame_or_index): benchmark(frame_or_index.to_arrow) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_astype(benchmark, frame_or_index): benchmark(frame_or_index.astype, float) @pytest.mark.parametrize("ufunc", [np.add, np.logical_and]) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_ufunc_series_binary(benchmark, frame_or_index, ufunc): benchmark(ufunc, frame_or_index, frame_or_index) @@ -84,6 +84,6 @@ def bench_ufunc_series_binary(benchmark, frame_or_index, ufunc): "op", [operator.add, operator.mul, operator.eq], ) -@accepts_cudf_fixture(cls="frame_or_index", dtype="int") +@benchmark_with_object(cls="frame_or_index", dtype="int") def bench_binops(benchmark, op, frame_or_index): benchmark(lambda: op(frame_or_index, frame_or_index)) diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py index f51a38ef87b..ef8335ad056 100644 --- a/python/cudf/benchmarks/API/bench_index.py +++ b/python/cudf/benchmarks/API/bench_index.py @@ -5,7 +5,7 @@ import pytest from ..common.config import cudf, cupy -from ..common.utils import accepts_cudf_fixture +from ..common.utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) @@ -13,6 +13,6 @@ def bench_construction(benchmark, N): benchmark(cudf.Index, cupy.random.rand(N)) -@accepts_cudf_fixture(cls="index", dtype="int", nulls=False) +@benchmark_with_object(cls="index", dtype="int", nulls=False) def bench_sort_values(benchmark, index): benchmark(index.sort_values) diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py index 7502ec2b509..d19be394802 100644 --- a/python/cudf/benchmarks/API/bench_indexed_frame.py +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -4,27 +4,27 @@ import pytest -from ..common.utils import accepts_cudf_fixture +from ..common.utils import benchmark_with_object -@accepts_cudf_fixture(cls="indexedframe", dtype="int") +@benchmark_with_object(cls="indexedframe", dtype="int") @pytest.mark.parametrize("op", ["cumsum", "cumprod", "cummax"]) def bench_scans(benchmark, op, indexedframe): benchmark(getattr(indexedframe, op)) -@accepts_cudf_fixture(cls="indexedframe", dtype="int") +@benchmark_with_object(cls="indexedframe", dtype="int") @pytest.mark.parametrize("op", ["sum", "product", "mean"]) def bench_reductions(benchmark, op, indexedframe): benchmark(getattr(indexedframe, op)) -@accepts_cudf_fixture(cls="indexedframe", dtype="int") +@benchmark_with_object(cls="indexedframe", dtype="int") def bench_drop_duplicates(benchmark, indexedframe): benchmark(indexedframe.drop_duplicates) -@accepts_cudf_fixture(cls="indexedframe", dtype="int") +@benchmark_with_object(cls="indexedframe", dtype="int") def bench_rangeindex_replace(benchmark, indexedframe): # TODO: Consider adding more DataFrame-specific benchmarks for different # types of valid inputs (dicts, etc). diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py index 0bedbf5ba77..51f2e316c3e 100644 --- a/python/cudf/benchmarks/API/bench_series.py +++ b/python/cudf/benchmarks/API/bench_series.py @@ -5,7 +5,7 @@ import pytest from ..common.config import cudf, cupy -from ..common.utils import accepts_cudf_fixture +from ..common.utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) @@ -13,12 +13,12 @@ def bench_construction(benchmark, N): benchmark(cudf.Series, cupy.random.rand(N)) -@accepts_cudf_fixture(cls="series", dtype="int") +@benchmark_with_object(cls="series", dtype="int") def bench_sort_values(benchmark, series): benchmark(series.sort_values) -@accepts_cudf_fixture(cls="series", dtype="int") +@benchmark_with_object(cls="series", dtype="int") @pytest.mark.parametrize("n", [10]) def bench_series_nsmallest(benchmark, series, n): benchmark(series.nsmallest, n) diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index d2d14a8902a..eea7c41b9b0 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -43,7 +43,7 @@ def make_boolean_mask_column(size): return cudf.core.column.as_column(rstate.randint(0, 2, size).astype(bool)) -def accepts_cudf_fixture( +def benchmark_with_object( cls, *, dtype="int", nulls=None, cols=None, rows=None, name=None ): """Pass "standard" cudf fixtures to functions without renaming parameters. @@ -91,7 +91,7 @@ def accepts_cudf_fixture( -------- # Note: As an internal function, this example is not meant for doctesting. - @accepts_cudf_fixture("dataframe", dtype="int", nulls=False, name="df") + @benchmark_with_object("dataframe", dtype="int", nulls=False, name="df") def bench_columns(benchmark, df): benchmark(df.columns) """ @@ -150,7 +150,7 @@ def deco(bm): # decorator is to define a new benchmark function with a signature # identical to that of the decorated benchmark except with the user's # fixture name replaced by the true fixture name based on the arguments - # to accepts_cudf_fixture. + # to benchmark_with_object. parameters = inspect.signature(bm).parameters # Note: This logic assumes that any benchmark using this fixture has at diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index 8dce68828e2..9f08a888696 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -6,30 +6,30 @@ import pytest_cases from ..common.utils import ( - accepts_cudf_fixture, + benchmark_with_object, make_boolean_mask_column, make_gather_map, ) -@accepts_cudf_fixture(cls="column", dtype="float") +@benchmark_with_object(cls="column", dtype="float") def bench_apply_boolean_mask(benchmark, column): mask = make_boolean_mask_column(column.size) benchmark(column.apply_boolean_mask, mask) -@accepts_cudf_fixture(cls="column", dtype="float") +@benchmark_with_object(cls="column", dtype="float") @pytest.mark.parametrize("dropnan", [True, False]) def bench_dropna(benchmark, column, dropnan): benchmark(column.dropna, drop_nan=dropnan) -@accepts_cudf_fixture(cls="column", dtype="float") +@benchmark_with_object(cls="column", dtype="float") def bench_unique_single_column(benchmark, column): benchmark(column.unique) -@accepts_cudf_fixture(cls="column", dtype="float") +@benchmark_with_object(cls="column", dtype="float") @pytest.mark.parametrize("nullify", [True, False]) @pytest.mark.parametrize("gather_how", ["sequence", "reverse", "random"]) def bench_take(benchmark, column, gather_how, nullify): @@ -39,29 +39,29 @@ def bench_take(benchmark, column, gather_how, nullify): benchmark(column.take, gather_map, nullify=nullify) -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_stride_1_slice_scalar(column): return column, slice(None, None, 1), 42 -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_stride_2_slice_scalar(column): return column, slice(None, None, 2), 42 -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_boolean_column_scalar(column): column = column return column, [True, False] * (len(column) // 2), 42 -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_int_column_scalar(column): column = column return column, list(range(len(column))), 42 -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_stride_1_slice_align_to_key_size( column, ): @@ -72,7 +72,7 @@ def setitem_case_stride_1_slice_align_to_key_size( return column, key, [42] * materialized_key_size -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_stride_2_slice_align_to_key_size( column, ): @@ -83,7 +83,7 @@ def setitem_case_stride_2_slice_align_to_key_size( return column, key, [42] * materialized_key_size -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_boolean_column_align_to_col_size( column, ): @@ -92,7 +92,7 @@ def setitem_case_boolean_column_align_to_col_size( return column, [True, False] * (size // 2), [42] * size -@accepts_cudf_fixture(cls="column", dtype="int", nulls=False) +@benchmark_with_object(cls="column", dtype="int", nulls=False) def setitem_case_int_column_align_to_col_size(column): column = column size = len(column) diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py index af5dc6a377e..4e8afa32437 100644 --- a/python/cudf/benchmarks/internal/bench_dataframe_internal.py +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -2,10 +2,10 @@ """Benchmarks of internal DataFrame methods.""" -from ..common.utils import accepts_cudf_fixture, make_boolean_mask_column +from ..common.utils import benchmark_with_object, make_boolean_mask_column -@accepts_cudf_fixture(cls="dataframe", dtype="int") +@benchmark_with_object(cls="dataframe", dtype="int") def bench_apply_boolean_mask(benchmark, dataframe): mask = make_boolean_mask_column(len(dataframe)) benchmark(dataframe._apply_boolean_mask, mask) From 0521038a583357a632eafd3779b9eb2bffd988d5 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 11:40:04 -0700 Subject: [PATCH 16/27] Remove the name parameter to benchmark_with_object. --- python/cudf/benchmarks/API/bench_dataframe.py | 10 +++++----- python/cudf/benchmarks/common/utils.py | 20 ++++++------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 0e6c36fa12e..f862e039ce1 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -24,15 +24,15 @@ def bench_eval_func(benchmark, expr, dataframe): benchmark(dataframe.eval, expr) -@benchmark_with_object( - cls="dataframe", dtype="int", nulls=False, cols=6, name="df" -) +@benchmark_with_object(cls="dataframe", dtype="int", nulls=False, cols=6) @pytest.mark.parametrize( "num_key_cols", [2, 3, 4], ) -def bench_merge(benchmark, df, num_key_cols): - benchmark(df.merge, df, on=list(df.columns[:num_key_cols])) +def bench_merge(benchmark, dataframe, num_key_cols): + benchmark( + dataframe.merge, dataframe, on=list(dataframe.columns[:num_key_cols]) + ) # TODO: Some of these cases could be generalized to an IndexedFrame benchmark diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index eea7c41b9b0..f448f689e7a 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -44,7 +44,7 @@ def make_boolean_mask_column(size): def benchmark_with_object( - cls, *, dtype="int", nulls=None, cols=None, rows=None, name=None + cls, *, dtype="int", nulls=None, cols=None, rows=None ): """Pass "standard" cudf fixtures to functions without renaming parameters. @@ -75,12 +75,6 @@ def benchmark_with_object( rows : Optional[int], None The number of rows. If None, use all possible numbers of rows. Specifying multiple values is unsupported. - name : str, default None - The name of the fixture as used in the decorated test. If None, - defaults to `cls.lower()` if cls is a string, otherwise - `cls.__name__.lower()`. Use of this name allows the decorated function - to masquerade as a pytest receiving a fixture while substituting the - real fixture (with a much longer name). Raises ------ @@ -91,7 +85,7 @@ def benchmark_with_object( -------- # Note: As an internal function, this example is not meant for doctesting. - @benchmark_with_object("dataframe", dtype="int", nulls=False, name="df") + @benchmark_with_object("dataframe", dtype="int", nulls=False) def bench_columns(benchmark, df): benchmark(df.columns) """ @@ -111,8 +105,6 @@ def bench_columns(benchmark, df): f"cls {cls} is invalid, choose from " f"{', '.join(supported_classes)}" ) - name = name or cls - if not isinstance(dtype, list): dtype = [dtype] assert all(dt in column_generators for dt in dtype), ( @@ -156,8 +148,8 @@ def deco(bm): # Note: This logic assumes that any benchmark using this fixture has at # least two parameters since they must be using both the # pytest-benchmark `benchmark` fixture and the cudf object. - params_str = ", ".join(f"{p}" for p in parameters if p != name) - arg_str = ", ".join(f"{p}={p}" for p in parameters if p != name) + params_str = ", ".join(f"{p}" for p in parameters if p != cls) + arg_str = ", ".join(f"{p}={p}" for p in parameters if p != cls) if params_str: params_str += ", " @@ -165,14 +157,14 @@ def deco(bm): arg_str += ", " params_str += f"{fixture_name}" - arg_str += f"{name}={fixture_name}" + arg_str += f"{cls}={fixture_name}" src = textwrap.dedent( f""" import makefun @makefun.wraps( bm, - remove_args=("{name}",), + remove_args=("{cls}",), prepend_args=("{fixture_name}",) ) def wrapped_bm({params_str}): From 0d69a1f7d8fb367a4e89638c74201d08ecc579d6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 13:40:10 -0700 Subject: [PATCH 17/27] Run tests from inside the benchmarks directory to find the correct cudf. --- ci/gpu/build.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index a2ed68222d0..a799c258eb4 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -236,9 +236,12 @@ gpuci_logger "Python py.test for cuDF" py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope tests # Run benchmarks with both cudf and pandas to ensure compatibility is maintained. -cd "$WORKSPACE/python/cudf" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 benchmarks -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 benchmarks +# Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. +# Therefore, these runs only verify that benchmarks are valid. +# They do not generate meaningful performance measurements. +cd "$WORKSPACE/python/cudf/benchmarks/" +CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . cd "$WORKSPACE/python/dask_cudf" gpuci_logger "Python py.test for dask-cudf" From b060ec7e531808ef4554c3068b1e0f9bf01dab13 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 13:42:42 -0700 Subject: [PATCH 18/27] Address PR comments. --- ci/gpu/build.sh | 16 ++++++++-------- python/cudf/benchmarks/API/bench_multiindex.py | 2 ++ python/cudf/benchmarks/internal/bench_column.py | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index a799c258eb4..d260ae265d5 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -235,14 +235,6 @@ cd "$WORKSPACE/python/cudf/cudf" gpuci_logger "Python py.test for cuDF" py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --ignore="$WORKSPACE/python/cudf/cudf/benchmarks" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope tests -# Run benchmarks with both cudf and pandas to ensure compatibility is maintained. -# Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. -# Therefore, these runs only verify that benchmarks are valid. -# They do not generate meaningful performance measurements. -cd "$WORKSPACE/python/cudf/benchmarks/" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . - cd "$WORKSPACE/python/dask_cudf" gpuci_logger "Python py.test for dask-cudf" py.test -n 8 --cache-clear --basetemp="$WORKSPACE/dask-cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-dask-cudf.xml" -v --cov-config=.coveragerc --cov=dask_cudf --cov-report=xml:"$WORKSPACE/python/dask_cudf/dask-cudf-coverage.xml" --cov-report term dask_cudf @@ -251,6 +243,14 @@ cd "$WORKSPACE/python/custreamz" gpuci_logger "Python py.test for cuStreamz" py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml="$WORKSPACE/junit-custreamz.xml" -v --cov-config=.coveragerc --cov=custreamz --cov-report=xml:"$WORKSPACE/python/custreamz/custreamz-coverage.xml" --cov-report term custreamz +# Run benchmarks with both cudf and pandas to ensure compatibility is maintained. +# Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. +# Therefore, these runs only verify that benchmarks are valid. +# They do not generate meaningful performance measurements. +cd "$WORKSPACE/python/cudf/benchmarks/" +CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . + gpuci_logger "Test notebooks" "$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log python "$WORKSPACE/ci/utils/nbtestlog2junitxml.py" nbtest.log diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index 75bde5f9d63..a77cdd4607f 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -1,5 +1,7 @@ # Copyright (c) 2022, NVIDIA CORPORATION. +"""Benchmarks of MultiIndex methods.""" + import numpy as np import pandas as pd import pytest diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index 9f08a888696..ed278f488ca 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -106,7 +106,7 @@ def setitem_case_int_column_align_to_col_size(column): # column(int) (scatter) # value: scalar, # column (len(val) == len(key)), -# column (len(val) != len(key) & len == num_true) +# column (len(val) != len(key) and len == num_true) @pytest_cases.parametrize_with_cases( From ec4eaa7772719de12d6481f9548026166846fa05 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 23 Jun 2022 16:42:14 -0700 Subject: [PATCH 19/27] Try running from the root. --- ci/gpu/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index d260ae265d5..3ca6070b0e1 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -247,9 +247,9 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml # Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. # Therefore, these runs only verify that benchmarks are valid. # They do not generate meaningful performance measurements. -cd "$WORKSPACE/python/cudf/benchmarks/" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 . +cd "$WORKSPACE" +CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/cudf/benchmarks +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/cudf/benchmarks gpuci_logger "Test notebooks" "$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log From fe2a3a0327aa67b946c3d95b9533915f3558d6c4 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 24 Jun 2022 13:09:15 -0700 Subject: [PATCH 20/27] Fix path to benchmarks. --- ci/gpu/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 3ca6070b0e1..23e7dd7c733 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -248,8 +248,8 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml # Therefore, these runs only verify that benchmarks are valid. # They do not generate meaningful performance measurements. cd "$WORKSPACE" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/cudf/benchmarks -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/cudf/benchmarks +CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/benchmarks +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/benchmarks gpuci_logger "Test notebooks" "$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log From dd01dd46d01cb1592b7ea133a2cce66273c80870 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 24 Jun 2022 13:25:30 -0700 Subject: [PATCH 21/27] Switch to absolute imports. --- python/cudf/benchmarks/API/bench_dataframe.py | 4 ++-- python/cudf/benchmarks/API/bench_frame_or_index.py | 2 +- python/cudf/benchmarks/API/bench_functions.py | 2 +- python/cudf/benchmarks/API/bench_functions_cases.py | 2 +- python/cudf/benchmarks/API/bench_index.py | 4 ++-- python/cudf/benchmarks/API/bench_indexed_frame.py | 2 +- python/cudf/benchmarks/API/bench_multiindex.py | 2 +- python/cudf/benchmarks/API/bench_series.py | 4 ++-- python/cudf/benchmarks/internal/bench_column.py | 2 +- python/cudf/benchmarks/internal/bench_dataframe_internal.py | 5 ++++- python/cudf/benchmarks/internal/conftest.py | 4 ++-- 11 files changed, 18 insertions(+), 15 deletions(-) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index f862e039ce1..fc0d0cd8f59 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -7,8 +7,8 @@ import numpy import pytest -from ..common.config import cudf, cupy -from ..common.utils import benchmark_with_object +from benchmarks.common.config import cudf, cupy +from benchmarks.common.utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py index 8243f582f83..6eec17e1af1 100644 --- a/python/cudf/benchmarks/API/bench_frame_or_index.py +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from ..common.utils import benchmark_with_object, make_gather_map +from benchmarks.common.utils import benchmark_with_object, make_gather_map @benchmark_with_object(cls="frame_or_index", dtype="int") diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index 2dabd957e60..48100af430f 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -5,7 +5,7 @@ import pytest import pytest_cases -from ..common.config import cudf, cupy +from benchmarks.common.config import cudf, cupy @pytest_cases.parametrize_with_cases("objs", prefix="concat") diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index 1562527241d..a4ba30d2a64 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -4,7 +4,7 @@ import pytest_cases -from ..common.config import NUM_ROWS, cudf, cupy +from benchmarks.common.config import NUM_ROWS, cudf, cupy @pytest_cases.parametrize("nr", NUM_ROWS) diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py index ef8335ad056..d084196de69 100644 --- a/python/cudf/benchmarks/API/bench_index.py +++ b/python/cudf/benchmarks/API/bench_index.py @@ -4,8 +4,8 @@ import pytest -from ..common.config import cudf, cupy -from ..common.utils import benchmark_with_object +from benchmarks.common.config import cudf, cupy +from benchmarks.common.utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py index d19be394802..ad3b92e5d05 100644 --- a/python/cudf/benchmarks/API/bench_indexed_frame.py +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -4,7 +4,7 @@ import pytest -from ..common.utils import benchmark_with_object +from benchmarks.common.utils import benchmark_with_object @benchmark_with_object(cls="indexedframe", dtype="int") diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index a77cdd4607f..c6318d66435 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -6,7 +6,7 @@ import pandas as pd import pytest -from ..common.config import cudf +from benchmarks.common.config import cudf @pytest.fixture diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py index 51f2e316c3e..5cf57d79293 100644 --- a/python/cudf/benchmarks/API/bench_series.py +++ b/python/cudf/benchmarks/API/bench_series.py @@ -4,8 +4,8 @@ import pytest -from ..common.config import cudf, cupy -from ..common.utils import benchmark_with_object +from benchmarks.common.config import cudf, cupy +from benchmarks.common.utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index ed278f488ca..14ae658613f 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -5,7 +5,7 @@ import pytest import pytest_cases -from ..common.utils import ( +from benchmarks.common.utils import ( benchmark_with_object, make_boolean_mask_column, make_gather_map, diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py index 4e8afa32437..ddbc931c880 100644 --- a/python/cudf/benchmarks/internal/bench_dataframe_internal.py +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -2,7 +2,10 @@ """Benchmarks of internal DataFrame methods.""" -from ..common.utils import benchmark_with_object, make_boolean_mask_column +from benchmarks.common.utils import ( + benchmark_with_object, + make_boolean_mask_column, +) @benchmark_with_object(cls="dataframe", dtype="int") diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 001796c781c..1362619f6bd 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -2,8 +2,8 @@ """Defines pytest fixtures for internal benchmarks.""" -from ..common.config import NUM_ROWS, cudf -from ..common.utils import ( +from benchmarks.common.config import NUM_ROWS, cudf +from benchmarks.common.utils import ( OrderedSet, collapse_fixtures, column_generators, From 0183a510ea0df4262156a16989d539fb72c09073 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 24 Jun 2022 13:25:53 -0700 Subject: [PATCH 22/27] Add gpuci_logger line. --- ci/gpu/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 23e7dd7c733..c17160e8b39 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -248,6 +248,7 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml # Therefore, these runs only verify that benchmarks are valid. # They do not generate meaningful performance measurements. cd "$WORKSPACE" +gpuci_logger "Python pytest for cuDF benchmarks" CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/benchmarks CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/benchmarks From 28a6ea6d316bdf043565e588a14466f14a482dcd Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 25 Jun 2022 06:32:27 -0700 Subject: [PATCH 23/27] Try using py.test instead of pytest --- ci/gpu/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index c17160e8b39..e32ea2ed0e9 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -249,8 +249,9 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml # They do not generate meaningful performance measurements. cd "$WORKSPACE" gpuci_logger "Python pytest for cuDF benchmarks" -CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/benchmarks -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 python/cudf/benchmarks +gpuci_logger "The path is ${PYTHONPATH}" +CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 python/cudf/benchmarks +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 python/cudf/benchmarks gpuci_logger "Test notebooks" "$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log From a9ecdf3370c71c96776a147e224125a88379339c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 25 Jun 2022 08:28:52 -0700 Subject: [PATCH 24/27] Try using identical commands as for testing. --- ci/gpu/build.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index e32ea2ed0e9..1bcca978425 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -247,11 +247,10 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml # Benchmarks are run in DEBUG_ONLY mode, meaning that only small data sizes are used. # Therefore, these runs only verify that benchmarks are valid. # They do not generate meaningful performance measurements. -cd "$WORKSPACE" +cd "$WORKSPACE/python/cudf" gpuci_logger "Python pytest for cuDF benchmarks" -gpuci_logger "The path is ${PYTHONPATH}" -CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 python/cudf/benchmarks -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 python/cudf/benchmarks +CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope benchmarks +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope benchmarks gpuci_logger "Test notebooks" "$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log From 5e5e3d4ae7e56d53e18e03018c5c4329abe71aaf Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 25 Jun 2022 10:18:26 -0700 Subject: [PATCH 25/27] Revert "Convert the benchmarks into a package so that we can use relative imports." This reverts commit 5db4589c2df62a1fc77cce64d24700adc32e69b7. --- python/cudf/benchmarks/API/__init__.py | 1 - python/cudf/benchmarks/API/bench_dataframe.py | 4 +-- .../benchmarks/API/bench_frame_or_index.py | 2 +- python/cudf/benchmarks/API/bench_functions.py | 2 +- .../benchmarks/API/bench_functions_cases.py | 2 +- python/cudf/benchmarks/API/bench_index.py | 4 +-- .../benchmarks/API/bench_indexed_frame.py | 2 +- .../cudf/benchmarks/API/bench_multiindex.py | 2 +- python/cudf/benchmarks/API/bench_series.py | 4 +-- python/cudf/benchmarks/__init__.py | 1 - python/cudf/benchmarks/common/config.py | 12 +++++++ python/cudf/benchmarks/common/utils.py | 3 +- python/cudf/benchmarks/conftest.py | 32 ++++++++++++++----- python/cudf/benchmarks/internal/__init__.py | 1 - .../cudf/benchmarks/internal/bench_column.py | 2 +- .../internal/bench_dataframe_internal.py | 5 +-- python/cudf/benchmarks/internal/conftest.py | 4 +-- 17 files changed, 52 insertions(+), 31 deletions(-) delete mode 100644 python/cudf/benchmarks/API/__init__.py delete mode 100644 python/cudf/benchmarks/__init__.py delete mode 100644 python/cudf/benchmarks/internal/__init__.py diff --git a/python/cudf/benchmarks/API/__init__.py b/python/cudf/benchmarks/API/__init__.py deleted file mode 100644 index 6feb3c33ee1..00000000000 --- a/python/cudf/benchmarks/API/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index fc0d0cd8f59..1e32de1f7e0 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -7,8 +7,8 @@ import numpy import pytest -from benchmarks.common.config import cudf, cupy -from benchmarks.common.utils import benchmark_with_object +from config import cudf, cupy +from utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py index 6eec17e1af1..61de25cba66 100644 --- a/python/cudf/benchmarks/API/bench_frame_or_index.py +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -7,7 +7,7 @@ import numpy as np import pytest -from benchmarks.common.utils import benchmark_with_object, make_gather_map +from utils import benchmark_with_object, make_gather_map @benchmark_with_object(cls="frame_or_index", dtype="int") diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index 48100af430f..e2e149346ee 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -5,7 +5,7 @@ import pytest import pytest_cases -from benchmarks.common.config import cudf, cupy +from config import cudf, cupy @pytest_cases.parametrize_with_cases("objs", prefix="concat") diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index a4ba30d2a64..c9059405948 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -4,7 +4,7 @@ import pytest_cases -from benchmarks.common.config import NUM_ROWS, cudf, cupy +from config import NUM_ROWS, cudf, cupy @pytest_cases.parametrize("nr", NUM_ROWS) diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py index d084196de69..98c9b722f2e 100644 --- a/python/cudf/benchmarks/API/bench_index.py +++ b/python/cudf/benchmarks/API/bench_index.py @@ -4,8 +4,8 @@ import pytest -from benchmarks.common.config import cudf, cupy -from benchmarks.common.utils import benchmark_with_object +from config import cudf, cupy +from utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py index ad3b92e5d05..5be5d6209a2 100644 --- a/python/cudf/benchmarks/API/bench_indexed_frame.py +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -4,7 +4,7 @@ import pytest -from benchmarks.common.utils import benchmark_with_object +from utils import benchmark_with_object @benchmark_with_object(cls="indexedframe", dtype="int") diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index c6318d66435..8e72d0b551a 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -6,7 +6,7 @@ import pandas as pd import pytest -from benchmarks.common.config import cudf +from config import cudf @pytest.fixture diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py index 5cf57d79293..25dcd1dff0f 100644 --- a/python/cudf/benchmarks/API/bench_series.py +++ b/python/cudf/benchmarks/API/bench_series.py @@ -4,8 +4,8 @@ import pytest -from benchmarks.common.config import cudf, cupy -from benchmarks.common.utils import benchmark_with_object +from config import cudf, cupy +from utils import benchmark_with_object @pytest.mark.parametrize("N", [100, 1_000_000]) diff --git a/python/cudf/benchmarks/__init__.py b/python/cudf/benchmarks/__init__.py deleted file mode 100644 index 6feb3c33ee1..00000000000 --- a/python/cudf/benchmarks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. diff --git a/python/cudf/benchmarks/common/config.py b/python/cudf/benchmarks/common/config.py index 49e80653848..305a21d0a29 100644 --- a/python/cudf/benchmarks/common/config.py +++ b/python/cudf/benchmarks/common/config.py @@ -21,6 +21,7 @@ appropriately. """ import os +import sys # Environment variable-based configuration of benchmarking pandas or cudf. collect_ignore = [] @@ -48,6 +49,17 @@ def pytest_collection_modifyitems(session, config, items): pass +def pytest_sessionstart(session): + """Add the common files to the path for all tests to import.""" + sys.path.insert(0, os.path.join(os.getcwd(), "common")) + + +def pytest_sessionfinish(session, exitstatus): + """Clean up sys.path after exit.""" + if "common" in sys.path[0]: + del sys.path[0] + + # Constants used to define benchmarking standards. if "CUDF_BENCHMARKS_DEBUG_ONLY" in os.environ: NUM_ROWS = [10, 20] diff --git a/python/cudf/benchmarks/common/utils.py b/python/cudf/benchmarks/common/utils.py index f448f689e7a..363316f0930 100644 --- a/python/cudf/benchmarks/common/utils.py +++ b/python/cudf/benchmarks/common/utils.py @@ -10,8 +10,7 @@ from numbers import Real import pytest_cases - -from .config import NUM_COLS, NUM_ROWS, cudf, cupy +from config import NUM_COLS, NUM_ROWS, cudf, cupy def make_gather_map(len_gather_map: Real, len_column: Real, how: str): diff --git a/python/cudf/benchmarks/conftest.py b/python/cudf/benchmarks/conftest.py index c8e04dd352f..4f2bb96061f 100644 --- a/python/cudf/benchmarks/conftest.py +++ b/python/cudf/benchmarks/conftest.py @@ -44,24 +44,40 @@ As a result, it is provided as a separate fixture. """ +import os import string +import sys import pytest_cases -from .common.config import ( # noqa: W0611, F401 - NUM_COLS, - NUM_ROWS, - collect_ignore, - cudf, - pytest_collection_modifyitems, -) -from .common.utils import ( +# TODO: Rather than doing this path hacking (including the sessionstart and +# sessionfinish hooks), we could just make the benchmarks a (sub)package to +# enable relative imports. A minor change to consider when these are ported +# into the main repo. +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "common")) + +from config import cudf # noqa: W0611, E402, F401 +from utils import ( # noqa: E402 OrderedSet, collapse_fixtures, column_generators, make_fixture, ) +# Turn off isort until we upgrade to 5.8.0 +# https://github.com/pycqa/isort/issues/1594 +# isort: off +from config import ( # noqa: W0611, E402, F401 + NUM_COLS, + NUM_ROWS, + collect_ignore, + pytest_collection_modifyitems, + pytest_sessionfinish, + pytest_sessionstart, +) + +# isort: on + @pytest_cases.fixture(params=[0, 1], ids=["AxisIndex", "AxisColumn"]) def axis(request): diff --git a/python/cudf/benchmarks/internal/__init__.py b/python/cudf/benchmarks/internal/__init__.py deleted file mode 100644 index 6feb3c33ee1..00000000000 --- a/python/cudf/benchmarks/internal/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index 14ae658613f..cd42c2548ae 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -5,7 +5,7 @@ import pytest import pytest_cases -from benchmarks.common.utils import ( +from utils import ( benchmark_with_object, make_boolean_mask_column, make_gather_map, diff --git a/python/cudf/benchmarks/internal/bench_dataframe_internal.py b/python/cudf/benchmarks/internal/bench_dataframe_internal.py index ddbc931c880..5204d4fb65d 100644 --- a/python/cudf/benchmarks/internal/bench_dataframe_internal.py +++ b/python/cudf/benchmarks/internal/bench_dataframe_internal.py @@ -2,10 +2,7 @@ """Benchmarks of internal DataFrame methods.""" -from benchmarks.common.utils import ( - benchmark_with_object, - make_boolean_mask_column, -) +from utils import benchmark_with_object, make_boolean_mask_column @benchmark_with_object(cls="dataframe", dtype="int") diff --git a/python/cudf/benchmarks/internal/conftest.py b/python/cudf/benchmarks/internal/conftest.py index 1362619f6bd..7351f1d1427 100644 --- a/python/cudf/benchmarks/internal/conftest.py +++ b/python/cudf/benchmarks/internal/conftest.py @@ -2,8 +2,8 @@ """Defines pytest fixtures for internal benchmarks.""" -from benchmarks.common.config import NUM_ROWS, cudf -from benchmarks.common.utils import ( +from config import NUM_ROWS, cudf +from utils import ( OrderedSet, collapse_fixtures, column_generators, From b57eee21c04947c50ba78e7a04f727ae93c783df Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 25 Jun 2022 10:56:35 -0700 Subject: [PATCH 26/27] Fix style. --- python/cudf/benchmarks/API/bench_dataframe.py | 1 - python/cudf/benchmarks/API/bench_frame_or_index.py | 1 - python/cudf/benchmarks/API/bench_functions.py | 1 - python/cudf/benchmarks/API/bench_functions_cases.py | 1 - python/cudf/benchmarks/API/bench_index.py | 1 - python/cudf/benchmarks/API/bench_indexed_frame.py | 1 - python/cudf/benchmarks/API/bench_multiindex.py | 1 - python/cudf/benchmarks/API/bench_series.py | 1 - python/cudf/benchmarks/internal/bench_column.py | 1 - 9 files changed, 9 deletions(-) diff --git a/python/cudf/benchmarks/API/bench_dataframe.py b/python/cudf/benchmarks/API/bench_dataframe.py index 1e32de1f7e0..d5e175ff85c 100644 --- a/python/cudf/benchmarks/API/bench_dataframe.py +++ b/python/cudf/benchmarks/API/bench_dataframe.py @@ -6,7 +6,6 @@ import numpy import pytest - from config import cudf, cupy from utils import benchmark_with_object diff --git a/python/cudf/benchmarks/API/bench_frame_or_index.py b/python/cudf/benchmarks/API/bench_frame_or_index.py index 61de25cba66..14b29e8ef75 100644 --- a/python/cudf/benchmarks/API/bench_frame_or_index.py +++ b/python/cudf/benchmarks/API/bench_frame_or_index.py @@ -6,7 +6,6 @@ import numpy as np import pytest - from utils import benchmark_with_object, make_gather_map diff --git a/python/cudf/benchmarks/API/bench_functions.py b/python/cudf/benchmarks/API/bench_functions.py index e2e149346ee..a166317a46b 100644 --- a/python/cudf/benchmarks/API/bench_functions.py +++ b/python/cudf/benchmarks/API/bench_functions.py @@ -4,7 +4,6 @@ import pytest import pytest_cases - from config import cudf, cupy diff --git a/python/cudf/benchmarks/API/bench_functions_cases.py b/python/cudf/benchmarks/API/bench_functions_cases.py index c9059405948..c81f8f20f80 100644 --- a/python/cudf/benchmarks/API/bench_functions_cases.py +++ b/python/cudf/benchmarks/API/bench_functions_cases.py @@ -3,7 +3,6 @@ """Test cases for benchmarks in bench_functions.py.""" import pytest_cases - from config import NUM_ROWS, cudf, cupy diff --git a/python/cudf/benchmarks/API/bench_index.py b/python/cudf/benchmarks/API/bench_index.py index 98c9b722f2e..53e617141b6 100644 --- a/python/cudf/benchmarks/API/bench_index.py +++ b/python/cudf/benchmarks/API/bench_index.py @@ -3,7 +3,6 @@ """Benchmarks of Index methods.""" import pytest - from config import cudf, cupy from utils import benchmark_with_object diff --git a/python/cudf/benchmarks/API/bench_indexed_frame.py b/python/cudf/benchmarks/API/bench_indexed_frame.py index 5be5d6209a2..6969121b0da 100644 --- a/python/cudf/benchmarks/API/bench_indexed_frame.py +++ b/python/cudf/benchmarks/API/bench_indexed_frame.py @@ -3,7 +3,6 @@ """Benchmarks of IndexedFrame methods.""" import pytest - from utils import benchmark_with_object diff --git a/python/cudf/benchmarks/API/bench_multiindex.py b/python/cudf/benchmarks/API/bench_multiindex.py index 8e72d0b551a..6268bcc4267 100644 --- a/python/cudf/benchmarks/API/bench_multiindex.py +++ b/python/cudf/benchmarks/API/bench_multiindex.py @@ -5,7 +5,6 @@ import numpy as np import pandas as pd import pytest - from config import cudf diff --git a/python/cudf/benchmarks/API/bench_series.py b/python/cudf/benchmarks/API/bench_series.py index 25dcd1dff0f..92032da4a2e 100644 --- a/python/cudf/benchmarks/API/bench_series.py +++ b/python/cudf/benchmarks/API/bench_series.py @@ -3,7 +3,6 @@ """Benchmarks of Series methods.""" import pytest - from config import cudf, cupy from utils import benchmark_with_object diff --git a/python/cudf/benchmarks/internal/bench_column.py b/python/cudf/benchmarks/internal/bench_column.py index cd42c2548ae..d4969b39f7f 100644 --- a/python/cudf/benchmarks/internal/bench_column.py +++ b/python/cudf/benchmarks/internal/bench_column.py @@ -4,7 +4,6 @@ import pytest import pytest_cases - from utils import ( benchmark_with_object, make_boolean_mask_column, From 5f800619a68141666db3eb3ce3c26be5f464fe23 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Sat, 25 Jun 2022 12:42:22 -0700 Subject: [PATCH 27/27] Remove unnecessary coverage parameters for benchmarks. --- ci/gpu/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 1bcca978425..1a5073b2f24 100755 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -249,8 +249,10 @@ py.test -n 8 --cache-clear --basetemp="$WORKSPACE/custreamz-cuda-tmp" --junitxml # They do not generate meaningful performance measurements. cd "$WORKSPACE/python/cudf" gpuci_logger "Python pytest for cuDF benchmarks" -CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope benchmarks -CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON py.test -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" --junitxml="$WORKSPACE/junit-cudf.xml" -v --cov-config="$WORKSPACE/python/cudf/.coveragerc" --cov=cudf --cov-report=xml:"$WORKSPACE/python/cudf/cudf-coverage.xml" --cov-report term --dist=loadscope benchmarks +CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" -v --dist=loadscope benchmarks + +gpuci_logger "Python pytest for cuDF benchmarks using pandas" +CUDF_BENCHMARKS_USE_PANDAS=ON CUDF_BENCHMARKS_DEBUG_ONLY=ON pytest -n 8 --cache-clear --basetemp="$WORKSPACE/cudf-cuda-tmp" -v --dist=loadscope benchmarks gpuci_logger "Test notebooks" "$WORKSPACE/ci/gpu/test-notebooks.sh" 2>&1 | tee nbtest.log