Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

For testing DataFrame API #2

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

For testing DataFrame API #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dcb7c51
Enable column tests
anmyachev Nov 29, 2023
6e659b7
changes for dataframe tests
anmyachev Nov 30, 2023
7579521
fixes
anmyachev Nov 30, 2023
f212220
fixes
anmyachev Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion modin/core/dataframe/algebra/binary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"""Module houses builder class for Binary operator."""

from typing import Optional
import warnings

import numpy as np
import pandas
Expand Down Expand Up @@ -126,7 +127,9 @@ def maybe_compute_dtypes_common_cast(
df2 = pandas.DataFrame([[1] * len(common_columns)]).astype(
{i: dtypes_second[col] for i, col in enumerate(common_columns)}
)
dtypes = func(df1, df2).dtypes.set_axis(common_columns)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=pandas.errors.PerformanceWarning)
dtypes = func(df1, df2).dtypes.set_axis(common_columns)
# it sometimes doesn't work correctly with strings, so falling back to
# the "common_cast" method in this case
except TypeError:
Expand Down
11 changes: 10 additions & 1 deletion modin/core/execution/ray/generic/partitioning/partition_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,16 @@ def to_numpy(cls, partitions, **kwargs):
"""
if partitions.shape[1] == 1:
parts = cls.get_objects_from_partitions(partitions.flatten())
parts = [part.to_numpy(**kwargs) for part in parts]
# breakpoint()
# inconsistency: parts[0].to_numpy(dtype="int64", copy=False, na_value=np.nan)
# *** ValueError: cannot convert float NaN to integer
# vs
# parts[0]["result"].to_numpy(dtype="int64", copy=False, na_value=np.nan)
# array([1577840521123543000, 1577934062321654000, 1578027849987321000], dtype=int64)
# breakpoint()
parts = [(part.squeeze(axis=1) if len(part.columns) == 1 else part).to_numpy(**kwargs) for part in parts]
# reshape to 2d numpy array
parts = [part.reshape(len(part), 1) if len(part.shape) == 1 else part for part in parts]
else:
parts = RayWrapper.materialize(
[
Expand Down
4 changes: 2 additions & 2 deletions modin/pandas/test/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -663,7 +663,7 @@
break


def df_equals(df1, df2, check_dtypes=True):
def df_equals(df1, df2, check_dtypes=True, check_exact=False):

Check notice

Code scanning / CodeQL

Explicit returns mixed with implicit (fall through) returns Note test

Mixing implicit and explicit returns may indicate an error as implicit returns always return None.
"""Tests if df1 and df2 are equal.

Args:
Expand Down Expand Up @@ -732,7 +732,7 @@
elif isinstance(df1, pandas.Index) and isinstance(df2, pandas.Index):
assert_index_equal(df1, df2)
elif isinstance(df1, pandas.Series) and isinstance(df2, pandas.Series):
assert_series_equal(df1, df2, check_dtype=False, check_series_type=False)
assert_series_equal(df1, df2, check_dtype=False, check_series_type=False, check_exact=check_exact)
elif (
hasattr(df1, "dtype")
and hasattr(df2, "dtype")
Expand Down
Loading