From 67088cf141f122eed537d92e50b764158c995fab Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Mon, 25 Jul 2022 22:58:37 +0530 Subject: [PATCH 01/12] Revert "addressing reviews CL#1" This reverts commit 69dc2d43875c5ffc307b829397612a66b0549316. --- pandas/_libs/tslibs/parsing.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index 5cb11436f6f45..266ab63d957e0 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -282,7 +282,7 @@ def parse_datetime_string( pass try: - dt = du_parse(date_string, default=_DEFAULT_DATETIME, + dt = du_parse(date_string, default=_DEFAULT_DATETIME, dayfirst=dayfirst, yearfirst=yearfirst, **kwargs) except TypeError: # following may be raised from dateutil From d25d7fb80c88cd5396d0c689b6e87ec231c7fd73 Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Tue, 26 Jul 2022 15:25:36 +0530 Subject: [PATCH 02/12] Added improvements in to_datetime Error reporting message - incorrect field now/today shown as error --- pandas/_libs/tslib.pyx | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 9c7f35d240f96..84ea8b27d3654 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -798,6 +798,7 @@ cdef _array_to_datetime_object( # We return an object array and only attempt to parse: # 1) NaT or NaT-like values # 2) datetime strings, which we return as datetime.datetime + # 3) special strings - "now" & "today" for i in range(n): val = values[i] if checknull_with_nat_and_na(val) or PyDateTime_Check(val): @@ -812,10 +813,16 @@ cdef _array_to_datetime_object( oresult[i] = 'NaT' continue try: - oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, + # Handling special case strings today & now + if val == "today": + oresult[i] = datetime.today() + elif val == "now": + oresult[i] = datetime.now() + else: + oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, yearfirst=yearfirst) - pydatetime_to_dt64(oresult[i], &dts) - check_dts_bounds(&dts) + pydatetime_to_dt64(oresult[i], &dts) + check_dts_bounds(&dts) except (ValueError, OverflowError): if is_coerce: oresult[i] = NaT From 5582fd8ac638e70c62f6eb41974d5c5080a02110 Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Tue, 26 Jul 2022 15:30:26 +0530 Subject: [PATCH 03/12] Added improvements in to_datetime Error reporting message - incorrect field now/today shown as error --- pandas/_libs/tslibs/parsing.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index 266ab63d957e0..5cb11436f6f45 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -282,7 +282,7 @@ def parse_datetime_string( pass try: - dt = du_parse(date_string, default=_DEFAULT_DATETIME, + dt = du_parse(date_string, default=_DEFAULT_DATETIME, dayfirst=dayfirst, yearfirst=yearfirst, **kwargs) except TypeError: # following may be raised from dateutil From 3c2a8331756e0ce5f6b99f7ed41ce7ab01af2519 Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Tue, 2 Aug 2022 07:02:20 +0530 Subject: [PATCH 04/12] added testcase --- pandas/tests/tools/test_to_datetime.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index b128838318ac0..5e0a86a6064b4 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -2591,6 +2591,11 @@ def test_invalid_origins_tzinfo(self): with pytest.raises(ValueError, match="must be tz-naive"): to_datetime(1, unit="D", origin=datetime(2000, 1, 1, tzinfo=pytz.utc)) + def test_incorrect_value_exception(self): + # GH47495 + with pytest.raises(ValueError, match="Unknown string format: today"): + to_datetime(["today", "yesterday"]) + @pytest.mark.parametrize("format", [None, "%Y-%m-%d %H:%M:%S"]) def test_to_datetime_out_of_bounds_with_format_arg(self, format): # see gh-23830 From 387851df6cd20817f6a4ad9a4c8d343890cce79d Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Tue, 2 Aug 2022 07:19:23 +0530 Subject: [PATCH 05/12] adding position info & updated testcase --- pandas/_libs/tslib.pyx | 3 ++- pandas/tests/tools/test_to_datetime.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 909d124923a84..36c8f90560183 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -824,7 +824,8 @@ cdef _array_to_datetime_object( yearfirst=yearfirst) pydatetime_to_dt64(oresult[i], &dts) check_dts_bounds(&dts) - except (ValueError, OverflowError): + except (ValueError, OverflowError) as ex: + ex.args = (str(ex) + f" present at position {i}", ) if is_coerce: oresult[i] = NaT continue diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index 5e0a86a6064b4..3e742935416ca 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -2593,7 +2593,7 @@ def test_invalid_origins_tzinfo(self): def test_incorrect_value_exception(self): # GH47495 - with pytest.raises(ValueError, match="Unknown string format: today"): + with pytest.raises(ValueError, match="Unknown string format: yesterday present at position 1"): to_datetime(["today", "yesterday"]) @pytest.mark.parametrize("format", [None, "%Y-%m-%d %H:%M:%S"]) From 0c70ceefb7168e27e83bc0a7451c9edfebf35d5a Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Tue, 2 Aug 2022 07:20:17 +0530 Subject: [PATCH 06/12] precommit format changes --- pandas/tests/tools/test_to_datetime.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index 3e742935416ca..f3824d0bf3ea2 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -2593,7 +2593,9 @@ def test_invalid_origins_tzinfo(self): def test_incorrect_value_exception(self): # GH47495 - with pytest.raises(ValueError, match="Unknown string format: yesterday present at position 1"): + with pytest.raises( + ValueError, match="Unknown string format: yesterday present at position 1" + ): to_datetime(["today", "yesterday"]) @pytest.mark.parametrize("format", [None, "%Y-%m-%d %H:%M:%S"]) From cf6ceb7f18194fa98b1cd259bf8ad3319cc0897e Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Fri, 12 Aug 2022 12:39:06 +0530 Subject: [PATCH 07/12] addressing review comments --- pandas/_libs/tslib.pyx | 15 +++++---------- pandas/_libs/tslibs/parsing.pyx | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 36c8f90560183..73a37c30876ef 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -815,17 +815,12 @@ cdef _array_to_datetime_object( continue try: # Handling special case strings today & now - if val == "today": - oresult[i] = datetime.today() - elif val == "now": - oresult[i] = datetime.now() - else: - oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, - yearfirst=yearfirst) - pydatetime_to_dt64(oresult[i], &dts) - check_dts_bounds(&dts) + oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, + yearfirst=yearfirst) + pydatetime_to_dt64(oresult[i], &dts) + check_dts_bounds(&dts) except (ValueError, OverflowError) as ex: - ex.args = (str(ex) + f" present at position {i}", ) + ex.args = (f"{ex} present at position {i}", ) if is_coerce: oresult[i] = NaT continue diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index 97a8f81094a8f..f7aaa3efe3aa6 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -63,6 +63,7 @@ from pandas._libs.tslibs.util cimport ( get_c_string_buf_and_size, is_array, ) +from pandas._libs.tslibs.timestamps import Timestamp cdef extern from "../src/headers/portable.h": @@ -298,6 +299,11 @@ def parse_datetime_string( if dt is not None: return dt + # Handling special case strings today & now + dt = _parse_today_now(date_string) + if dt is not None: + return dt + try: dt, _ = _parse_dateabbr_string(date_string, _DEFAULT_DATETIME, freq=None) return dt @@ -1210,3 +1216,13 @@ cpdef str get_rule_month(str source): return "DEC" else: return source.split("-")[1] + +cdef inline object _parse_today_now(str date_string): + cdef: + object res = None + + if date_string == "now": + res = Timestamp.utcnow() + elif date_string == "today": + res = Timestamp.today() + return res From 438a60e3852d1b3dceb105e86b9bc44945596260 Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Fri, 12 Aug 2022 12:40:29 +0530 Subject: [PATCH 08/12] addressing review comments --- pandas/_libs/tslib.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 73a37c30876ef..6ee64cefc1eef 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -814,7 +814,6 @@ cdef _array_to_datetime_object( oresult[i] = 'NaT' continue try: - # Handling special case strings today & now oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, yearfirst=yearfirst) pydatetime_to_dt64(oresult[i], &dts) From 201b5dd4f90016ae1cea4d9f604e2d693da03e7a Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Fri, 12 Aug 2022 12:41:20 +0530 Subject: [PATCH 09/12] addressing review comments --- pandas/_libs/tslib.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 6ee64cefc1eef..fe4078f611f7e 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -815,7 +815,7 @@ cdef _array_to_datetime_object( continue try: oresult[i] = parse_datetime_string(val, dayfirst=dayfirst, - yearfirst=yearfirst) + yearfirst=yearfirst) pydatetime_to_dt64(oresult[i], &dts) check_dts_bounds(&dts) except (ValueError, OverflowError) as ex: From 38e8eb203909b399b8b61120a2a2efb4f032119b Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Fri, 12 Aug 2022 12:46:21 +0530 Subject: [PATCH 10/12] addressing review comments --- pandas/_libs/tslibs/parsing.pyx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index f7aaa3efe3aa6..e66d5da5f9b5a 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -1218,6 +1218,18 @@ cpdef str get_rule_month(str source): return source.split("-")[1] cdef inline object _parse_today_now(str date_string): + """ + Parse special case date inputs - "today", "now" + and if present return corresponding datetime object + + Parameters + ---------- + date_string : str + + Returns: + -------- + datetime or None + """ cdef: object res = None From 5ce48c2beb70b21f133cb4b0d2bc268ac5cdaba5 Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Fri, 12 Aug 2022 23:29:27 +0530 Subject: [PATCH 11/12] addressing review comments #2 --- pandas/_libs/tslibs/parsing.pyx | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index e66d5da5f9b5a..871c15383d5ab 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -63,7 +63,6 @@ from pandas._libs.tslibs.util cimport ( get_c_string_buf_and_size, is_array, ) -from pandas._libs.tslibs.timestamps import Timestamp cdef extern from "../src/headers/portable.h": @@ -300,7 +299,10 @@ def parse_datetime_string( return dt # Handling special case strings today & now - dt = _parse_today_now(date_string) + if date_string == "now": + dt = datetime.now() + elif date_string == "today": + dt = datetime.today() if dt is not None: return dt @@ -1216,25 +1218,3 @@ cpdef str get_rule_month(str source): return "DEC" else: return source.split("-")[1] - -cdef inline object _parse_today_now(str date_string): - """ - Parse special case date inputs - "today", "now" - and if present return corresponding datetime object - - Parameters - ---------- - date_string : str - - Returns: - -------- - datetime or None - """ - cdef: - object res = None - - if date_string == "now": - res = Timestamp.utcnow() - elif date_string == "today": - res = Timestamp.today() - return res From 65c44639dd50f7f56df897121235a1d018dc5c8b Mon Sep 17 00:00:00 2001 From: dannyi96 Date: Sat, 13 Aug 2022 11:10:35 +0530 Subject: [PATCH 12/12] addressing review comments --- pandas/_libs/tslibs/parsing.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/parsing.pyx b/pandas/_libs/tslibs/parsing.pyx index 871c15383d5ab..1d5bbf87090eb 100644 --- a/pandas/_libs/tslibs/parsing.pyx +++ b/pandas/_libs/tslibs/parsing.pyx @@ -301,9 +301,9 @@ def parse_datetime_string( # Handling special case strings today & now if date_string == "now": dt = datetime.now() + return dt elif date_string == "today": dt = datetime.today() - if dt is not None: return dt try: