Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Return nan when one variable to be correlated has zero variance in JIT GroupBy Apply #13884

Merged
merged 3 commits into from
Aug 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions python/cudf/cudf/tests/test_groupby.py
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,21 @@ def func(group):
run_groupby_apply_jit_test(groupby_jit_data, func, keys)


@pytest.mark.parametrize("dtype", ["int32", "int64"])
def test_groupby_apply_jit_correlation_zero_variance(dtype):
# pearson correlation is undefined when the variance of either
# variable is zero. This test ensures that the jit implementation
# returns the same result as pandas in this case.
data = DataFrame(
{"a": [0, 0, 0, 0, 0], "b": [1, 1, 1, 1, 1], "c": [2, 2, 2, 2, 2]}
)

def func(group):
return group["b"].corr(group["c"])

run_groupby_apply_jit_test(data, func, ["a"])


@pytest.mark.parametrize("dtype", ["float64"])
@pytest.mark.parametrize("func", ["min", "max", "sum", "mean", "var", "std"])
@pytest.mark.parametrize("special_val", [np.nan, np.inf, -np.inf])
Expand Down
3 changes: 1 addition & 2 deletions python/cudf/udf_cpp/shim.cu
Original file line number Diff line number Diff line change
Expand Up @@ -643,9 +643,8 @@ __device__ double BlockCorr(T* const lhs_ptr, T* const rhs_ptr, int64_t size)
{
auto numerator = BlockCoVar(lhs_ptr, rhs_ptr, size);
auto denominator = BlockStd(lhs_ptr, size) * BlockStd<T>(rhs_ptr, size);

if (denominator == 0.0) {
return 0.0;
return std::numeric_limits<double>::quiet_NaN();
galipremsagar marked this conversation as resolved.
Show resolved Hide resolved
} else {
return numerator / denominator;
}
Expand Down