From 9aa0159e9f1f3830400728d04af5199a050631ad Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Fri, 18 Aug 2017 16:28:36 -0400 Subject: [PATCH 1/7] Treating na values as none for clips --- pandas/core/generic.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 5a7f37bba91aa..d65938c419c67 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -4660,9 +4660,6 @@ def _clip_with_one_bound(self, threshold, method, axis, inplace): if axis is not None: axis = self._get_axis_number(axis) - if np.any(isna(threshold)): - raise ValueError("Cannot use an NA value as a clip threshold") - # method is self.le for upper bound and self.ge for lower bound if is_scalar(threshold) and is_number(threshold): if method.__name__ == 'le': @@ -4742,6 +4739,12 @@ def clip(self, lower=None, upper=None, axis=None, inplace=False, axis = nv.validate_clip_with_axis(axis, args, kwargs) + # GH 17276 + if np.any(pd.isnull(lower)): + lower = None + if np.any(pd.isnull(upper)): + upper = None + # GH 2747 (arguments were reversed) if lower is not None and upper is not None: if is_scalar(lower) and is_scalar(upper): @@ -4758,7 +4761,6 @@ def clip(self, lower=None, upper=None, axis=None, inplace=False, if upper is not None: if inplace: result = self - result = result.clip_upper(upper, axis, inplace=inplace) return result From d9627fe4c715402996ecc3d657680715d6f2c2b1 Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Fri, 18 Aug 2017 16:28:46 -0400 Subject: [PATCH 2/7] adding clip tests --- pandas/tests/frame/test_analytics.py | 40 ++++++++++++++++----------- pandas/tests/series/test_analytics.py | 9 ++++++ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 484a09f11b58a..79e6aec2185ea 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1931,22 +1931,30 @@ def test_clip_against_frame(self, axis): tm.assert_frame_equal(clipped_df[ub_mask], ub[ub_mask]) tm.assert_frame_equal(clipped_df[mask], df[mask]) - def test_clip_na(self): - msg = "Cannot use an NA" - with tm.assert_raises_regex(ValueError, msg): - self.frame.clip(lower=np.nan) - - with tm.assert_raises_regex(ValueError, msg): - self.frame.clip(lower=[np.nan]) - - with tm.assert_raises_regex(ValueError, msg): - self.frame.clip(upper=np.nan) - - with tm.assert_raises_regex(ValueError, msg): - self.frame.clip(upper=[np.nan]) - - with tm.assert_raises_regex(ValueError, msg): - self.frame.clip(lower=np.nan, upper=np.nan) + # def test_clip_na(self): + # msg = "Cannot use an NA" + # with tm.assert_raises_regex(ValueError, msg): + # self.frame.clip(lower=np.nan) + + # with tm.assert_raises_regex(ValueError, msg): + # self.frame.clip(lower=[np.nan]) + + # with tm.assert_raises_regex(ValueError, msg): + # self.frame.clip(upper=np.nan) + + # with tm.assert_raises_regex(ValueError, msg): + # self.frame.clip(upper=[np.nan]) + + # with tm.assert_raises_regex(ValueError, msg): + # self.frame.clip(lower=np.nan, upper=np.nan) + + def test_clip_with_na_args(self): + """Should process np.nan argument as None """ + # GH # 17276 + self.frame.clip(np.nan) + self.frame.clip(upper=[1,2,np.nan]) + self.frame.clip(lower=[1,np.nan,3]) + self.frame.clip(upper=np.nan, lower=np.nan) # Matrix-like diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index 44da0968d7024..7a9ffb2a62411 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -1000,6 +1000,15 @@ def test_clip_types_and_nulls(self): assert list(isna(s)) == list(isna(l)) assert list(isna(s)) == list(isna(u)) + def test_clip_with_na_args(self): + """Should process np.nan argument as None """ + # GH # 17276 + s = Series([1,2,3]) + s.clip(np.nan) + s.clip(upper=[1,2,np.nan]) + s.clip(lower=[1,np.nan,3]) + s.clip(upper=np.nan, lower=np.nan) + def test_clip_against_series(self): # GH #6966 From 781ea724a19c9e6623e91f033139f21b570b8a72 Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Fri, 18 Aug 2017 16:53:55 -0400 Subject: [PATCH 3/7] whatsnew entry --- doc/source/whatsnew/v0.21.0.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 4f55c6388c728..2dd48f2bdc6de 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -128,6 +128,7 @@ Other Enhancements - :func:`DataFrame.add_prefix` and :func:`DataFrame.add_suffix` now accept strings containing the '%' character. (:issue:`17151`) - `read_*` methods can now infer compression from non-string paths, such as ``pathlib.Path`` objects (:issue:`17206`). - :func:`pd.read_sas()` now recognizes much more of the most frequently used date (datetime) formats in SAS7BDAT files (:issue:`15871`). +- :func:`Series.clip()` and `DataFrame.clip()` now treat NA values for upper and lower arguments as None instead of raising `ValueError` (:issue:`17276`). .. _whatsnew_0210.api_breaking: From 7e2367879083fac81d3587fa7bade1c4e9d929d1 Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Fri, 18 Aug 2017 16:55:14 -0400 Subject: [PATCH 4/7] formatting updates --- pandas/tests/frame/test_analytics.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 79e6aec2185ea..8ba9cc74e285d 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1931,22 +1931,6 @@ def test_clip_against_frame(self, axis): tm.assert_frame_equal(clipped_df[ub_mask], ub[ub_mask]) tm.assert_frame_equal(clipped_df[mask], df[mask]) - # def test_clip_na(self): - # msg = "Cannot use an NA" - # with tm.assert_raises_regex(ValueError, msg): - # self.frame.clip(lower=np.nan) - - # with tm.assert_raises_regex(ValueError, msg): - # self.frame.clip(lower=[np.nan]) - - # with tm.assert_raises_regex(ValueError, msg): - # self.frame.clip(upper=np.nan) - - # with tm.assert_raises_regex(ValueError, msg): - # self.frame.clip(upper=[np.nan]) - - # with tm.assert_raises_regex(ValueError, msg): - # self.frame.clip(lower=np.nan, upper=np.nan) def test_clip_with_na_args(self): """Should process np.nan argument as None """ From c44204080ee9001604cc6da1a650a8395b31d36e Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Fri, 18 Aug 2017 17:06:51 -0400 Subject: [PATCH 5/7] formatting fixes --- pandas/tests/frame/test_analytics.py | 5 ++--- pandas/tests/series/test_analytics.py | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index 8ba9cc74e285d..c202a07c9ef8a 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1931,13 +1931,12 @@ def test_clip_against_frame(self, axis): tm.assert_frame_equal(clipped_df[ub_mask], ub[ub_mask]) tm.assert_frame_equal(clipped_df[mask], df[mask]) - def test_clip_with_na_args(self): """Should process np.nan argument as None """ # GH # 17276 self.frame.clip(np.nan) - self.frame.clip(upper=[1,2,np.nan]) - self.frame.clip(lower=[1,np.nan,3]) + self.frame.clip(upper=[1, 2, np.nan]) + self.frame.clip(lower=[1, np.nan, 3]) self.frame.clip(upper=np.nan, lower=np.nan) # Matrix-like diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index 7a9ffb2a62411..13900536227e0 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -1003,10 +1003,11 @@ def test_clip_types_and_nulls(self): def test_clip_with_na_args(self): """Should process np.nan argument as None """ # GH # 17276 - s = Series([1,2,3]) + s = Series([1, 2, 3]) + s.clip(np.nan) - s.clip(upper=[1,2,np.nan]) - s.clip(lower=[1,np.nan,3]) + s.clip(upper=[1, 2, np.nan]) + s.clip(lower=[1, np.nan, 3]) s.clip(upper=np.nan, lower=np.nan) def test_clip_against_series(self): From 4e0464eaf2f4bd2dda7b05e3c3468a318d3e138e Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Mon, 21 Aug 2017 09:39:39 -0400 Subject: [PATCH 6/7] fixing whatsnew text --- doc/source/whatsnew/v0.21.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 2dd48f2bdc6de..66aa1d61ab4ec 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -128,7 +128,7 @@ Other Enhancements - :func:`DataFrame.add_prefix` and :func:`DataFrame.add_suffix` now accept strings containing the '%' character. (:issue:`17151`) - `read_*` methods can now infer compression from non-string paths, such as ``pathlib.Path`` objects (:issue:`17206`). - :func:`pd.read_sas()` now recognizes much more of the most frequently used date (datetime) formats in SAS7BDAT files (:issue:`15871`). -- :func:`Series.clip()` and `DataFrame.clip()` now treat NA values for upper and lower arguments as None instead of raising `ValueError` (:issue:`17276`). +- :func:`Series.clip()` and :func:`DataFrame.clip()` now treat NA values for upper and lower arguments as None instead of raising `ValueError` (:issue:`17276`). .. _whatsnew_0210.api_breaking: From 9333952c2300f97facb23734547cd5c90a096063 Mon Sep 17 00:00:00 2001 From: Michael Gasvoda Date: Mon, 21 Aug 2017 09:53:09 -0400 Subject: [PATCH 7/7] Checking output of tests --- pandas/tests/frame/test_analytics.py | 11 +++++++---- pandas/tests/series/test_analytics.py | 9 +++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pandas/tests/frame/test_analytics.py b/pandas/tests/frame/test_analytics.py index c202a07c9ef8a..93514a8a42215 100644 --- a/pandas/tests/frame/test_analytics.py +++ b/pandas/tests/frame/test_analytics.py @@ -1934,10 +1934,13 @@ def test_clip_against_frame(self, axis): def test_clip_with_na_args(self): """Should process np.nan argument as None """ # GH # 17276 - self.frame.clip(np.nan) - self.frame.clip(upper=[1, 2, np.nan]) - self.frame.clip(lower=[1, np.nan, 3]) - self.frame.clip(upper=np.nan, lower=np.nan) + tm.assert_frame_equal(self.frame.clip(np.nan), self.frame) + tm.assert_frame_equal(self.frame.clip(upper=[1, 2, np.nan]), + self.frame) + tm.assert_frame_equal(self.frame.clip(lower=[1, np.nan, 3]), + self.frame) + tm.assert_frame_equal(self.frame.clip(upper=np.nan, lower=np.nan), + self.frame) # Matrix-like diff --git a/pandas/tests/series/test_analytics.py b/pandas/tests/series/test_analytics.py index 13900536227e0..f1d044f7a1132 100644 --- a/pandas/tests/series/test_analytics.py +++ b/pandas/tests/series/test_analytics.py @@ -1005,10 +1005,11 @@ def test_clip_with_na_args(self): # GH # 17276 s = Series([1, 2, 3]) - s.clip(np.nan) - s.clip(upper=[1, 2, np.nan]) - s.clip(lower=[1, np.nan, 3]) - s.clip(upper=np.nan, lower=np.nan) + assert_series_equal(s.clip(np.nan), Series([1, 2, 3])) + assert_series_equal(s.clip(upper=[1, 1, np.nan]), Series([1, 2, 3])) + assert_series_equal(s.clip(lower=[1, np.nan, 1]), Series([1, 2, 3])) + assert_series_equal(s.clip(upper=np.nan, lower=np.nan), + Series([1, 2, 3])) def test_clip_against_series(self): # GH #6966