From d9c1918d9e87d37d38d7cfac350a61c27577c1b8 Mon Sep 17 00:00:00 2001 From: fwx Date: Wed, 28 Nov 2018 15:40:42 +0530 Subject: [PATCH 1/6] Raising correct exceptions for to_datetime() #23830 --- pandas/_libs/tslib.pyx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index c5bc969ede3c9..388a673e54fc8 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -691,9 +691,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', continue elif require_iso8601: if is_raise: - raise ValueError("time data {val} doesn't " - "match format specified" - .format(val=val)) + continue return values, tz_out raise From f6c1efa04ee6bd339a69575502bab0cf843ffb89 Mon Sep 17 00:00:00 2001 From: fwx Date: Wed, 28 Nov 2018 16:10:45 +0530 Subject: [PATCH 2/6] Adding test case for checking that the correct exception is raised by to_datetime #23830 --- pandas/tests/indexes/datetimes/test_tools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index c24c1025ea63c..6607a3e78f7d8 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -947,6 +947,9 @@ def test_to_datetime_barely_out_of_bounds(self): with pytest.raises(OutOfBoundsDatetime): to_datetime(arr) + with pytest.raises(OutOfBoundsDatetime): + to_datetime(arr, format="%Y-%m-%d %H:%M:%S.%f") + @pytest.mark.parametrize('cache', [True, False]) def test_to_datetime_iso8601(self, cache): result = to_datetime(["2012-01-01 00:00:00"], cache=cache) From cefb1c18a9939b34954f5348e08846a1da3a8328 Mon Sep 17 00:00:00 2001 From: fwx Date: Thu, 29 Nov 2018 21:55:26 +0530 Subject: [PATCH 3/6] Fixes to correct caught in CI --- pandas/_libs/tslib.pyx | 6 ++---- pandas/tests/indexes/datetimes/test_tools.py | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 388a673e54fc8..15a1a287ccf32 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -623,9 +623,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', iresult[i] = NPY_NAT continue elif is_raise: - raise ValueError("time data {val} doesn't match " - "format specified" - .format(val=val)) + raise return values, tz_out try: @@ -691,7 +689,7 @@ cpdef array_to_datetime(ndarray[object] values, errors='raise', continue elif require_iso8601: if is_raise: - continue + raise return values, tz_out raise diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index 6607a3e78f7d8..ee15f74bb8ae9 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -947,6 +947,11 @@ def test_to_datetime_barely_out_of_bounds(self): with pytest.raises(OutOfBoundsDatetime): to_datetime(arr) + def test_to_datetime(self): + # GH#23830 raise the correct exception when + # the format argument is passed. + arr = np.array(['2262-04-11 23:47:16.854775808'], dtype=object) + with pytest.raises(OutOfBoundsDatetime): to_datetime(arr, format="%Y-%m-%d %H:%M:%S.%f") From b3fadc661b0076a86e19dc8b91a4e37a9a015d16 Mon Sep 17 00:00:00 2001 From: fwx Date: Thu, 29 Nov 2018 23:51:35 +0530 Subject: [PATCH 4/6] Checking the error with match inside pytest --- pandas/tests/indexes/datetimes/test_tools.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_tools.py b/pandas/tests/indexes/datetimes/test_tools.py index ee15f74bb8ae9..c041f2efac03a 100644 --- a/pandas/tests/indexes/datetimes/test_tools.py +++ b/pandas/tests/indexes/datetimes/test_tools.py @@ -947,13 +947,16 @@ def test_to_datetime_barely_out_of_bounds(self): with pytest.raises(OutOfBoundsDatetime): to_datetime(arr) - def test_to_datetime(self): + def test_to_datetime_out_of_bounds_with_format_arg(self): # GH#23830 raise the correct exception when # the format argument is passed. - arr = np.array(['2262-04-11 23:47:16.854775808'], dtype=object) - - with pytest.raises(OutOfBoundsDatetime): - to_datetime(arr, format="%Y-%m-%d %H:%M:%S.%f") + msg = r"Out of bounds nanosecond timestamp" + with pytest.raises( + OutOfBoundsDatetime, + match=msg + ): + to_datetime("2417-10-27 00:00:00", + format="%Y-%m-%d %H:%M:%S") @pytest.mark.parametrize('cache', [True, False]) def test_to_datetime_iso8601(self, cache): From 399becacfe7810f23d1528dceaebcb98d48e3f89 Mon Sep 17 00:00:00 2001 From: fwx Date: Sat, 1 Dec 2018 12:14:50 +0530 Subject: [PATCH 5/6] Added a whatsnew note --- doc/source/whatsnew/v0.24.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index d0dddb19f4c93..8277543cb2a73 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -1215,6 +1215,7 @@ Datetimelike - Bug in :class:`DatetimeIndex` where calling ``np.array(dtindex, dtype=object)`` would incorrectly return an array of ``long`` objects (:issue:`23524`) - Bug in :class:`Index` where passing a timezone-aware :class:`DatetimeIndex` and `dtype=object` would incorrectly raise a ``ValueError`` (:issue:`23524`) - Bug in :class:`Index` where calling ``np.array(dtindex, dtype=object)`` on a timezone-naive :class:`DatetimeIndex` would return an array of ``datetime`` objects instead of :class:`Timestamp` objects, potentially losing nanosecond portions of the timestamps (:issue:`23524`) +- Bug in :func:`to_datetime` which would raise an (incorrect) ``ValueError`` when called with a date far into the future and the ``format`` argument specified instead of raising ``OutOfBoundsDatetime`` (:issue:`23830`) Timedelta ^^^^^^^^^ From 8d8b893d6cb541ac87f28dee607f12e972684219 Mon Sep 17 00:00:00 2001 From: fwx Date: Sun, 9 Dec 2018 00:54:18 +0530 Subject: [PATCH 6/6] Fixing issues caught in CI --- pandas/_libs/tslib.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 7751e75fea75e..8a14bc89e910b 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -598,7 +598,7 @@ cpdef array_to_datetime(ndarray[object] values, str errors='raise', # other formats if is_coerce: iresult[i] = NPY_NAT - continue + continue elif is_raise: raise return values, tz_out @@ -659,7 +659,7 @@ cpdef array_to_datetime(ndarray[object] values, str errors='raise', iresult[i] = NPY_NAT else: raise TypeError("{typ} is not convertible to datetime" - .format(typ=type(val))) + .format(typ=type(val))) except OutOfBoundsDatetime: if is_coerce: @@ -670,7 +670,7 @@ cpdef array_to_datetime(ndarray[object] values, str errors='raise', # dateutil parser will return incorrect result because # it will ignore nanoseconds if is_raise: - raise + raise assert is_ignore return values, tz_out raise