From acc05228de488a4155f3017caef169bee92d4d27 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 7 Aug 2024 09:41:48 -0600 Subject: [PATCH 1/6] Expand groupby_reduce property tests --- tests/test_properties.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index c032f074..4934b3ec 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -61,7 +61,11 @@ def not_overflowing_array(array: np.ndarray[Any, Any]) -> bool: return result -@given(data=st.data(), array=numeric_arrays, func=func_st) +@given( + data=st.data(), + array=st.one_of(numeric_arrays, chunked_arrays(arrays=numeric_arrays)), + func=func_st, +) def test_groupby_reduce(data, array, func: str) -> None: # overflow behaviour differs between bincount and sum (for example) assume(not_overflowing_array(array)) From 805b8d313750232756cc9bf970e8cd472e01995a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 13 Aug 2024 22:20:23 -0600 Subject: [PATCH 2/6] Add back var, std --- tests/strategies.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/strategies.py b/tests/strategies.py index a2f95da1..c91e5c9b 100644 --- a/tests/strategies.py +++ b/tests/strategies.py @@ -32,11 +32,7 @@ def supported_dtypes() -> st.SearchStrategy[np.dtype]: NON_NUMPY_FUNCS = ["first", "last", "nanfirst", "nanlast", "count", "any", "all"] + list( SCIPY_STATS_FUNCS ) -SKIPPED_FUNCS = ["var", "std", "nanvar", "nanstd"] - -func_st = st.sampled_from( - [f for f in ALL_FUNCS if f not in NON_NUMPY_FUNCS and f not in SKIPPED_FUNCS] -) +func_st = st.sampled_from([f for f in ALL_FUNCS if f not in NON_NUMPY_FUNCS]) numeric_arrays = npst.arrays( elements={"allow_subnormal": False}, shape=npst.array_shapes(), dtype=array_dtype_st ) From 216c71fe83d493bc0ea94a7408a3d401af0ae906 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 13 Aug 2024 22:22:09 -0600 Subject: [PATCH 3/6] cast quantile result --- tests/test_properties.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_properties.py b/tests/test_properties.py index 4934b3ec..3c86dc34 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -97,7 +97,13 @@ def test_groupby_reduce(data, array, func: str) -> None: # numpy-groupies always does the calculation in float64 if ( - ("var" in func or "std" in func or "sum" in func or "mean" in func) + ( + "var" in func + or "std" in func + or "sum" in func + or "mean" in func + or "quantile" in func + ) and array.dtype.kind == "f" and array.dtype.itemsize != 8 ): From fcb04ccfe28767aefcb94c7433fd6fe09596965c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 13 Aug 2024 22:22:27 -0600 Subject: [PATCH 4/6] Revert "Add back var, std" This reverts commit 805b8d313750232756cc9bf970e8cd472e01995a. --- tests/strategies.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/strategies.py b/tests/strategies.py index c91e5c9b..a2f95da1 100644 --- a/tests/strategies.py +++ b/tests/strategies.py @@ -32,7 +32,11 @@ def supported_dtypes() -> st.SearchStrategy[np.dtype]: NON_NUMPY_FUNCS = ["first", "last", "nanfirst", "nanlast", "count", "any", "all"] + list( SCIPY_STATS_FUNCS ) -func_st = st.sampled_from([f for f in ALL_FUNCS if f not in NON_NUMPY_FUNCS]) +SKIPPED_FUNCS = ["var", "std", "nanvar", "nanstd"] + +func_st = st.sampled_from( + [f for f in ALL_FUNCS if f not in NON_NUMPY_FUNCS and f not in SKIPPED_FUNCS] +) numeric_arrays = npst.arrays( elements={"allow_subnormal": False}, shape=npst.array_shapes(), dtype=array_dtype_st ) From a818902bcfd713a926ae361e67f0f577323eaf64 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 14 Aug 2024 10:13:18 -0600 Subject: [PATCH 5/6] pin numpy in benchmark env --- ci/benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/benchmark.yml b/ci/benchmark.yml index e443eca9..2a877c37 100644 --- a/ci/benchmark.yml +++ b/ci/benchmark.yml @@ -6,7 +6,7 @@ dependencies: - build - cachey - dask-core - - numpy>=1.22 + - numpy<2 - mamba - pip - python=3.10 From 54e5dc5597c5be1d915c968f4393ce6265076f6e Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 14 Aug 2024 12:59:41 -0600 Subject: [PATCH 6/6] Add benchmarks as test --- flox/aggregate_flox.py | 2 +- tests/test_asv.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/test_asv.py diff --git a/flox/aggregate_flox.py b/flox/aggregate_flox.py index c23c5471..68482236 100644 --- a/flox/aggregate_flox.py +++ b/flox/aggregate_flox.py @@ -160,7 +160,7 @@ def _np_grouped_op( def _nan_grouped_op(group_idx, array, func, fillna, *args, **kwargs): if fillna in [dtypes.INF, dtypes.NINF]: - fillna = dtypes._get_fill_value(kwargs.get("dtype", array.dtype), fillna) + fillna = dtypes._get_fill_value(kwargs.get("dtype", None) or array.dtype, fillna) result = func(group_idx, np.where(isnull(array), fillna, array), *args, **kwargs) # np.nanmax([np.nan, np.nan]) = np.nan # To recover this behaviour, we need to search for the fillna value diff --git a/tests/test_asv.py b/tests/test_asv.py new file mode 100644 index 00000000..c6f9525b --- /dev/null +++ b/tests/test_asv.py @@ -0,0 +1,24 @@ +# Run asv benchmarks as tests + +import pytest + +pytest.importorskip("dask") + +from asv_bench.benchmarks import reduce + + +@pytest.mark.parametrize( + "problem", [reduce.ChunkReduce1D, reduce.ChunkReduce2D, reduce.ChunkReduce2DAllAxes] +) +def test_reduce(problem) -> None: + testcase = problem() + testcase.setup() + for args in zip(*testcase.time_reduce.params): + testcase.time_reduce(*args) + + +def test_reduce_bare() -> None: + testcase = reduce.ChunkReduce1D() + testcase.setup() + for args in zip(*testcase.time_reduce_bare.params): + testcase.time_reduce_bare(*args)