From 0cf4a8680005b6dd42a5af178bc1d6e9153df2bb Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 28 Jul 2020 15:51:29 +0100 Subject: [PATCH 1/6] BUG: tuple-of-tuples indexing results in NumPy VisibleDeprecationWarning --- pandas/core/common.py | 10 ++++++++-- pandas/tests/indexing/test_indexing.py | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index e7260a9923ee0..5c6581527511a 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -138,7 +138,10 @@ def is_bool_indexer(key: Any) -> bool: return True elif isinstance(key, list): try: - arr = np.asarray(key) + # https://github.com/pandas-dev/pandas/issues/35434 + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=np.VisibleDeprecationWarning) + arr = np.asarray(key) return arr.dtype == np.bool_ and len(arr) == len(key) except TypeError: # pragma: no cover return False @@ -225,7 +228,10 @@ def asarray_tuplesafe(values, dtype=None): if isinstance(values, list) and dtype in [np.object_, object]: return construct_1d_object_array_from_listlike(values) - result = np.asarray(values, dtype=dtype) + # https://github.com/pandas-dev/pandas/issues/35434 + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=np.VisibleDeprecationWarning) + result = np.asarray(values, dtype=dtype) if issubclass(result.dtype.type, str): result = np.asarray(values, dtype=object) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 5b7f013d5de31..9b4066c394461 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1110,3 +1110,11 @@ def test_setitem_categorical(): {"h": pd.Categorical(["m", "n"]).reorder_categories(["n", "m"])} ) tm.assert_frame_equal(df, expected) + + +def test_nested_tuple_no_warning(): + # https://github.com/pandas-dev/pandas/issues/35434 + tup = "A", ("B", 2) + ser = pd.Series([42], index=[tup]) + with tm.assert_produces_warning(None): + ser[[tup]] From c87fcabefd25b78948797deb936e522f0c575f2b Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 28 Jul 2020 16:13:30 +0100 Subject: [PATCH 2/6] black fixup --- pandas/core/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 5c6581527511a..40bebb2d0694c 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -140,7 +140,7 @@ def is_bool_indexer(key: Any) -> bool: try: # https://github.com/pandas-dev/pandas/issues/35434 with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=np.VisibleDeprecationWarning) + warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) arr = np.asarray(key) return arr.dtype == np.bool_ and len(arr) == len(key) except TypeError: # pragma: no cover @@ -230,7 +230,7 @@ def asarray_tuplesafe(values, dtype=None): # https://github.com/pandas-dev/pandas/issues/35434 with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=np.VisibleDeprecationWarning) + warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) result = np.asarray(values, dtype=dtype) if issubclass(result.dtype.type, str): From 3a6828515a4a8445d93223294ea638c605f25403 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 28 Jul 2020 16:24:41 +0100 Subject: [PATCH 3/6] catch ValueError: setting an array element with a sequence in is_bool_indexer --- pandas/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 40bebb2d0694c..8449c029a8564 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -143,7 +143,7 @@ def is_bool_indexer(key: Any) -> bool: warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) arr = np.asarray(key) return arr.dtype == np.bool_ and len(arr) == len(key) - except TypeError: # pragma: no cover + except (TypeError, ValueError): # pragma: no cover return False return False From b2ea85ec80fd726722040d300538a4f7a89e34c2 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 28 Jul 2020 17:12:00 +0100 Subject: [PATCH 4/6] catch ValueError: setting an array element with a sequence in asarray_tuplesafe --- pandas/core/common.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 8449c029a8564..1d150f5d744df 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -228,10 +228,15 @@ def asarray_tuplesafe(values, dtype=None): if isinstance(values, list) and dtype in [np.object_, object]: return construct_1d_object_array_from_listlike(values) - # https://github.com/pandas-dev/pandas/issues/35434 - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) - result = np.asarray(values, dtype=dtype) + try: + # https://github.com/pandas-dev/pandas/issues/35434 + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=np.VisibleDeprecationWarning) + result = np.asarray(values, dtype=dtype) + except ValueError: + # we get here with a list-like of nested values if dtype=None + # for numpy < 1.18 + return construct_1d_object_array_from_listlike(values) if issubclass(result.dtype.type, str): result = np.asarray(values, dtype=object) From 5c02e3af0368a4686ec290398a185b06857197dc Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 28 Jul 2020 18:02:35 +0100 Subject: [PATCH 5/6] extend test --- pandas/tests/indexing/test_indexing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/tests/indexing/test_indexing.py b/pandas/tests/indexing/test_indexing.py index 9b4066c394461..9e87faadf446b 100644 --- a/pandas/tests/indexing/test_indexing.py +++ b/pandas/tests/indexing/test_indexing.py @@ -1117,4 +1117,5 @@ def test_nested_tuple_no_warning(): tup = "A", ("B", 2) ser = pd.Series([42], index=[tup]) with tm.assert_produces_warning(None): - ser[[tup]] + result = ser[[tup]] + tm.assert_series_equal(result, ser) From 273985844a0d9bee3224c420eea3bc8e5a757461 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Tue, 28 Jul 2020 20:42:16 +0100 Subject: [PATCH 6/6] wip - ci testing --- pandas/core/indexers.py | 35 +++++++++++++++++++++++++++++++++++ pandas/core/series.py | 17 +++++++++++------ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/pandas/core/indexers.py b/pandas/core/indexers.py index d9aa02db3e42a..8fa89ef2f475a 100644 --- a/pandas/core/indexers.py +++ b/pandas/core/indexers.py @@ -17,6 +17,41 @@ ) from pandas.core.dtypes.generic import ABCIndexClass, ABCSeries +from pandas.core.dtypes.common import is_scalar, is_iterator +import pandas.core.common as com + + +class Indexer: + _is_iterator = None + _is_bool_indexer = None + + def __init__(self, key): + if isinstance(key, (list, tuple)): + key = unpack_1tuple(key) + self.key = key + + @property + def is_scalar(self): + return is_scalar(self.key) + + @property + def is_bool_indexer(self): + is_bool_indexer = self._is_bool_indexer + if is_bool_indexer is not None: + return is_bool_indexer + + key = self.key + if self._is_iterator is None: + if is_iterator(key): + key = list(key) + self.key = key + self._is_iterator = False + + is_bool_indexer = com.is_bool_indexer(key) + self._is_bool_indexer = is_bool_indexer + return is_bool_indexer + + # ----------------------------------------------------------- # Indexer Identification diff --git a/pandas/core/series.py b/pandas/core/series.py index ef3be854bc3bb..c5a7506611287 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -50,7 +50,6 @@ is_dict_like, is_extension_array_dtype, is_integer, - is_iterator, is_list_like, is_object_dtype, is_scalar, @@ -78,7 +77,7 @@ sanitize_array, ) from pandas.core.generic import NDFrame -from pandas.core.indexers import deprecate_ndim_indexing, unpack_1tuple +from pandas.core.indexers import deprecate_ndim_indexing, unpack_1tuple, Indexer from pandas.core.indexes.accessors import CombinedDatetimelikeProperties from pandas.core.indexes.api import Float64Index, Index, MultiIndex, ensure_index import pandas.core.indexes.base as ibase @@ -897,17 +896,23 @@ def __getitem__(self, key): # in the first level of our MultiIndex return self._get_values_tuple(key) - if is_iterator(key): - key = list(key) + # if is_iterator(key): + # key = list(key) + + _key = Indexer(key) - if com.is_bool_indexer(key): + if _key.is_bool_indexer: key = check_bool_indexer(self.index, key) key = np.asarray(key, dtype=bool) return self._get_values(key) - return self._get_with(key) + return self._get_with(_key) def _get_with(self, key): + # breakpoint() + _key = key + key = _key.key + # other: fancy integer or otherwise if isinstance(key, slice): # _convert_slice_indexer to determine if this slice is positional