From 651a55f793c51006d6748f50e5503510527b79df Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Mon, 20 Jan 2020 18:08:20 +0300 Subject: [PATCH 01/25] BUG: nonexistent Timestamp pre-summer/winter DST change with dateutil timezone --- doc/source/whatsnew/v1.1.0.rst | 3 ++- pandas/_libs/tslibs/nattype.pyx | 2 +- pandas/_libs/tslibs/timedeltas.pyx | 6 ++++-- pandas/tests/scalar/timestamp/test_timestamp.py | 12 ++++++++++++ 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/doc/source/whatsnew/v1.1.0.rst b/doc/source/whatsnew/v1.1.0.rst index 40c02eb495f67..907cb8cf26940 100644 --- a/doc/source/whatsnew/v1.1.0.rst +++ b/doc/source/whatsnew/v1.1.0.rst @@ -59,9 +59,10 @@ Categorical Datetimelike ^^^^^^^^^^^^ + - Bug in :class:`Timestamp` where constructing :class:`Timestamp` from ambiguous epoch time and calling constructor again changed :meth:`Timestamp.value` property (:issue:`24329`) - :meth:`DatetimeArray.searchsorted`, :meth:`TimedeltaArray.searchsorted`, :meth:`PeriodArray.searchsorted` not recognizing non-pandas scalars and incorrectly raising ``ValueError`` instead of ``TypeError`` (:issue:`30950`) -- +- Bug in :class:`Timestamp` where constructing :class:`Timestamp` with dateutil timezone less than 128 nanoseconds before daylight saving time switch from winter to summer would result in nonexistent time (:issue:`31043`) Timedelta ^^^^^^^^^ diff --git a/pandas/_libs/tslibs/nattype.pyx b/pandas/_libs/tslibs/nattype.pyx index 67c0f0cc33ab8..61b2721696bad 100644 --- a/pandas/_libs/tslibs/nattype.pyx +++ b/pandas/_libs/tslibs/nattype.pyx @@ -278,7 +278,7 @@ cdef class _NaT(datetime): def total_seconds(self): """ - Total duration of timedelta in seconds (to ns precision). + Total duration of timedelta in seconds (to microsecond precision). """ # GH#10939 return np.nan diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 0a773b8a215ed..c21969338cc52 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -861,9 +861,11 @@ cdef class _Timedelta(timedelta): def total_seconds(self): """ - Total duration of timedelta in seconds (to ns precision). + Total duration of timedelta in seconds (to microsecond precision). """ - return self.value / 1e9 + # GH 31043 + # Round down to microseconds to avoid confusing tzinfo.utcoffset + return (self.value - self.value % 1000) / 1e9 def view(self, dtype): """ diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index c60406fdbc8a6..b542269741dc9 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1092,3 +1092,15 @@ def test_constructor_ambigous_dst(): expected = ts.value result = Timestamp(ts).value assert result == expected + + +def test_constructor_before_dst_switch(): + # GH 31043 + # Make sure that calling Timestamp constructor + # on time just before DST switch doesn't lead to + # nonexistent time + epoch = 1552211999999999872 + ts = Timestamp(epoch, tz="dateutil/US/Pacific") + expected = timedelta(seconds=0) + result = ts.tz.dst(ts) + assert result == expected From c9a87bde41e6c6fef9a12cde3879f4d586aec4e4 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Mon, 20 Jan 2020 20:23:36 +0300 Subject: [PATCH 02/25] switch from scientific number notation --- pandas/_libs/tslibs/timedeltas.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index c21969338cc52..b09fbac8801d3 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Round down to microseconds to avoid confusing tzinfo.utcoffset - return (self.value - self.value % 1000) / 1e9 + return (self.value - self.value % 1000) / 1000000000 def view(self, dtype): """ From ca34eed99d00bc9ac8124b2a12f32a25c7cbe3b5 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Mon, 20 Jan 2020 21:54:10 +0300 Subject: [PATCH 03/25] move back to scientific notation to reset tests --- pandas/_libs/tslibs/timedeltas.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index b09fbac8801d3..c21969338cc52 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Round down to microseconds to avoid confusing tzinfo.utcoffset - return (self.value - self.value % 1000) / 1000000000 + return (self.value - self.value % 1000) / 1e9 def view(self, dtype): """ From 65b3bb81db587943f0e0a96d515e67ad67dfbff3 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Mon, 20 Jan 2020 21:54:28 +0300 Subject: [PATCH 04/25] move away from scientific notation to reset tests --- pandas/_libs/tslibs/timedeltas.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index c21969338cc52..b09fbac8801d3 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Round down to microseconds to avoid confusing tzinfo.utcoffset - return (self.value - self.value % 1000) / 1e9 + return (self.value - self.value % 1000) / 1000000000 def view(self, dtype): """ From 1eb9500671f24d666ad69c6153b91a7450fd87f7 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Mon, 20 Jan 2020 23:13:25 +0300 Subject: [PATCH 05/25] add casts to force correct division --- pandas/_libs/tslibs/timedeltas.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index b09fbac8801d3..8abdb7b66d42b 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,8 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Round down to microseconds to avoid confusing tzinfo.utcoffset - return (self.value - self.value % 1000) / 1000000000 + # use casts to make sure precision isn't lost + return np.float64(int(self.value - self.value % 1000) / 1000000000) def view(self, dtype): """ From 2f3850e7e5927c3024aac47c6578753f8dc67ba1 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Mon, 20 Jan 2020 23:49:33 +0300 Subject: [PATCH 06/25] switch to dateutil implementation --- pandas/_libs/tslibs/timedeltas.pyx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 8abdb7b66d42b..c5a67d969912a 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -864,9 +864,9 @@ cdef class _Timedelta(timedelta): Total duration of timedelta in seconds (to microsecond precision). """ # GH 31043 - # Round down to microseconds to avoid confusing tzinfo.utcoffset - # use casts to make sure precision isn't lost - return np.float64(int(self.value - self.value % 1000) / 1000000000) + # Microseconds precision to avoid confusing tzinfo.utcoffset + return ((self.days * 86400 + self.seconds) * 10**6 + + self.microseconds) / 10**6 def view(self, dtype): """ From 6c87f1b3e79f5ae20916cec9c56e220ff5a9eeb2 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 00:50:53 +0300 Subject: [PATCH 07/25] TST: expand test for debugging --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index b542269741dc9..4465edb02ea64 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1101,6 +1101,8 @@ def test_constructor_before_dst_switch(): # nonexistent time epoch = 1552211999999999872 ts = Timestamp(epoch, tz="dateutil/US/Pacific") + delta = ts - datetime.utcfromtimestamp(0) + assert delta.total_seconds() < 1552183200 expected = timedelta(seconds=0) result = ts.tz.dst(ts) assert result == expected From 43f6645990ad28c0d6e9bf308774bf5c37724089 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 00:53:27 +0300 Subject: [PATCH 08/25] TST: fix test expansion --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 4465edb02ea64..a5f0ea93cbd32 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1101,7 +1101,7 @@ def test_constructor_before_dst_switch(): # nonexistent time epoch = 1552211999999999872 ts = Timestamp(epoch, tz="dateutil/US/Pacific") - delta = ts - datetime.utcfromtimestamp(0) + delta = ts.replace(tzinfo=None) - datetime.utcfromtimestamp(0) assert delta.total_seconds() < 1552183200 expected = timedelta(seconds=0) result = ts.tz.dst(ts) From b1defde85c63a3a9fd79215589fbf35d42de6d25 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 01:02:44 +0300 Subject: [PATCH 09/25] go back to previous implementation --- pandas/_libs/tslibs/timedeltas.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index c5a67d969912a..461479b7ace3f 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,8 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Microseconds precision to avoid confusing tzinfo.utcoffset - return ((self.days * 86400 + self.seconds) * 10**6 + - self.microseconds) / 10**6 + return (self.value - self.value % 1000) / 1000000000 def view(self, dtype): """ From e46c7740cdb7f21b77e0924d21424c7cedb23c2e Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 09:13:34 +0300 Subject: [PATCH 10/25] hopefully a more robust implementation --- pandas/_libs/tslibs/timedeltas.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 461479b7ace3f..c02848c15e99e 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Microseconds precision to avoid confusing tzinfo.utcoffset - return (self.value - self.value % 1000) / 1000000000 + return float(self.value) / 1e9 - float(self.value % 1000) / 1e9 def view(self, dtype): """ From 4f8b4900eb4299a0772a7ec748444da055121b09 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 09:38:10 +0300 Subject: [PATCH 11/25] add rounding --- pandas/_libs/tslibs/timedeltas.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index c02848c15e99e..01391eb516deb 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Microseconds precision to avoid confusing tzinfo.utcoffset - return float(self.value) / 1e9 - float(self.value % 1000) / 1e9 + return np.round((self.value - self.value % 1000) / 1e9, 6) def view(self, dtype): """ From f8dfb369422d1aaf4e724648452516c7fdb1316c Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 10:02:49 +0300 Subject: [PATCH 12/25] TST: remove exception from another test Remove exception from test_dti_construction_onexistent_endpoint --- pandas/tests/indexes/datetimes/test_timezones.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index df64820777f3f..cd8e8c3542cce 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -581,12 +581,7 @@ def test_dti_construction_ambiguous_endpoint(self, tz): ["US/Pacific", "shift_forward", "2019-03-10 03:00"], ["dateutil/US/Pacific", "shift_forward", "2019-03-10 03:00"], ["US/Pacific", "shift_backward", "2019-03-10 01:00"], - pytest.param( - "dateutil/US/Pacific", - "shift_backward", - "2019-03-10 01:00", - marks=pytest.mark.xfail(reason="GH 24329"), - ), + ["dateutil/US/Pacific", "shift_backward", "2019-03-10 01:00"], ["US/Pacific", timedelta(hours=1), "2019-03-10 03:00"], ], ) From 3ad32125021b975d5b3a0df9f85f1ecc4fc9351c Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 10:10:22 +0300 Subject: [PATCH 13/25] add more to the test --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index a5f0ea93cbd32..5dad0bd67de07 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1102,7 +1102,7 @@ def test_constructor_before_dst_switch(): epoch = 1552211999999999872 ts = Timestamp(epoch, tz="dateutil/US/Pacific") delta = ts.replace(tzinfo=None) - datetime.utcfromtimestamp(0) - assert delta.total_seconds() < 1552183200 + assert 1552183199.99 < delta.total_seconds() < 1552183200 expected = timedelta(seconds=0) result = ts.tz.dst(ts) assert result == expected From 2174ca0f86fe5b79225b4ec75317af9242a4a8ac Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 11:16:46 +0300 Subject: [PATCH 14/25] try removing rounding, see if it breaks --- pandas/_libs/tslibs/timedeltas.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/tslibs/timedeltas.pyx b/pandas/_libs/tslibs/timedeltas.pyx index 01391eb516deb..8f78ae6a1322e 100644 --- a/pandas/_libs/tslibs/timedeltas.pyx +++ b/pandas/_libs/tslibs/timedeltas.pyx @@ -865,7 +865,7 @@ cdef class _Timedelta(timedelta): """ # GH 31043 # Microseconds precision to avoid confusing tzinfo.utcoffset - return np.round((self.value - self.value % 1000) / 1e9, 6) + return (self.value - self.value % 1000) / 1e9 def view(self, dtype): """ From 0cf53c1eb4be1e0ee3fd1db441ea7e6fb6625292 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 15:11:08 +0300 Subject: [PATCH 15/25] add value stability and skip condition to the test --- .../tests/scalar/timestamp/test_timestamp.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 5dad0bd67de07..db002a88b0ac5 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -8,6 +8,7 @@ import dateutil from dateutil.tz import tzutc import numpy as np +from packaging.version import parse as parse_version import pytest import pytz from pytz import timezone, utc @@ -1093,16 +1094,21 @@ def test_constructor_ambigous_dst(): result = Timestamp(ts).value assert result == expected - -def test_constructor_before_dst_switch(): +@pytest.mark.skipif( + parse_version(dateutil.__version__) < parse_version('2.7.0'), + reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", +) +@pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) +def test_constructor_before_dst_switch(epoch): # GH 31043 # Make sure that calling Timestamp constructor # on time just before DST switch doesn't lead to - # nonexistent time - epoch = 1552211999999999872 + # nonexistent time or value change + # Works only with dateutil >= 2.7.0 as dateutil overrid + # pandas.Timedelta.total_seconds with + # datetime.timedelta.total_seconds before ts = Timestamp(epoch, tz="dateutil/US/Pacific") - delta = ts.replace(tzinfo=None) - datetime.utcfromtimestamp(0) - assert 1552183199.99 < delta.total_seconds() < 1552183200 - expected = timedelta(seconds=0) result = ts.tz.dst(ts) + expected = timedelta(seconds=0) + assert ts.value == epoch assert result == expected From 3f79c3d91b66b08adfaa2302bae356346888e997 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 15:24:13 +0300 Subject: [PATCH 16/25] TST: add xfails for dateutil version < 2.7.0 --- .../tests/indexes/datetimes/test_timezones.py | 32 +++++++++++++++++++ .../tests/scalar/timestamp/test_timestamp.py | 5 +-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index cd8e8c3542cce..09b99987aaa8b 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -6,6 +6,7 @@ import dateutil from dateutil.tz import gettz, tzlocal import numpy as np +from packaging.version import parse as parse_version import pytest import pytz @@ -598,6 +599,37 @@ def test_dti_construction_nonexistent_endpoint(self, tz, option, expected): ) assert times[-1] == Timestamp(expected, tz=tz, freq="H") + @pytest.mark.parametrize( + "tz, option, expected", + [ + ["US/Pacific", "shift_forward", "2019-03-10 03:00"], + ["dateutil/US/Pacific", "shift_forward", "2019-03-10 03:00"], + ["US/Pacific", "shift_backward", "2019-03-10 01:00"], + pytest.param( + "dateutil/US/Pacific", + "shift_backward", + "2019-03-10 01:00", + marks=pytest.mark.xfail( + parse_version(dateutil.__version__) < parse_version("2.7.0"), + reason="GH 31043", + ), + ), + ["US/Pacific", timedelta(hours=1), "2019-03-10 03:00"], + ], + ) + def test_dti_construction_nonexistent_endpoint(self, tz, option, expected): + # construction with an nonexistent end-point + + with pytest.raises(pytz.NonExistentTimeError): + date_range( + "2019-03-10 00:00", "2019-03-10 02:00", tz="US/Pacific", freq="H" + ) + + times = date_range( + "2019-03-10 00:00", "2019-03-10 02:00", freq="H", tz=tz, nonexistent=option + ) + assert times[-1] == Timestamp(expected, tz=tz, freq="H") + def test_dti_tz_localize_bdate_range(self): dr = pd.bdate_range("1/1/2009", "1/1/2010") dr_utc = pd.bdate_range("1/1/2009", "1/1/2010", tz=pytz.utc) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index db002a88b0ac5..b54e4595d0cc2 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1094,8 +1094,9 @@ def test_constructor_ambigous_dst(): result = Timestamp(ts).value assert result == expected -@pytest.mark.skipif( - parse_version(dateutil.__version__) < parse_version('2.7.0'), + +@pytest.mark.xfail( + parse_version(dateutil.__version__) < parse_version("2.7.0"), reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", ) @pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) From 1c147d33ccc85e983263b3d23004493f564f1d97 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 15:32:55 +0300 Subject: [PATCH 17/25] TST: remove duplicate test --- .../tests/indexes/datetimes/test_timezones.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index 09b99987aaa8b..933a92b067f6f 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -576,29 +576,6 @@ def test_dti_construction_ambiguous_endpoint(self, tz): assert times[0] == Timestamp("2013-10-26 23:00", tz=tz, freq="H") assert times[-1] == Timestamp("2013-10-27 01:00:00+0000", tz=tz, freq="H") - @pytest.mark.parametrize( - "tz, option, expected", - [ - ["US/Pacific", "shift_forward", "2019-03-10 03:00"], - ["dateutil/US/Pacific", "shift_forward", "2019-03-10 03:00"], - ["US/Pacific", "shift_backward", "2019-03-10 01:00"], - ["dateutil/US/Pacific", "shift_backward", "2019-03-10 01:00"], - ["US/Pacific", timedelta(hours=1), "2019-03-10 03:00"], - ], - ) - def test_dti_construction_nonexistent_endpoint(self, tz, option, expected): - # construction with an nonexistent end-point - - with pytest.raises(pytz.NonExistentTimeError): - date_range( - "2019-03-10 00:00", "2019-03-10 02:00", tz="US/Pacific", freq="H" - ) - - times = date_range( - "2019-03-10 00:00", "2019-03-10 02:00", freq="H", tz=tz, nonexistent=option - ) - assert times[-1] == Timestamp(expected, tz=tz, freq="H") - @pytest.mark.parametrize( "tz, option, expected", [ From f03e7c9fd61068a516c4c24a8878dbd29d8751ee Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 15:59:38 +0300 Subject: [PATCH 18/25] TST: remove packaging import to find the error --- pandas/tests/indexes/datetimes/test_timezones.py | 5 ----- pandas/tests/scalar/timestamp/test_timestamp.py | 5 ----- 2 files changed, 10 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index 933a92b067f6f..34e48dc449a5b 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -6,7 +6,6 @@ import dateutil from dateutil.tz import gettz, tzlocal import numpy as np -from packaging.version import parse as parse_version import pytest import pytz @@ -586,10 +585,6 @@ def test_dti_construction_ambiguous_endpoint(self, tz): "dateutil/US/Pacific", "shift_backward", "2019-03-10 01:00", - marks=pytest.mark.xfail( - parse_version(dateutil.__version__) < parse_version("2.7.0"), - reason="GH 31043", - ), ), ["US/Pacific", timedelta(hours=1), "2019-03-10 03:00"], ], diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index b54e4595d0cc2..5d9d27b96e42d 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -8,7 +8,6 @@ import dateutil from dateutil.tz import tzutc import numpy as np -from packaging.version import parse as parse_version import pytest import pytz from pytz import timezone, utc @@ -1095,10 +1094,6 @@ def test_constructor_ambigous_dst(): assert result == expected -@pytest.mark.xfail( - parse_version(dateutil.__version__) < parse_version("2.7.0"), - reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", -) @pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) def test_constructor_before_dst_switch(epoch): # GH 31043 From c88e3544cf9525c8f392ad03ca6ce61aaf26dee3 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 16:31:36 +0300 Subject: [PATCH 19/25] TST: switch to LooseVersion for dateutil version check --- pandas/tests/indexes/datetimes/test_timezones.py | 5 +++++ pandas/tests/scalar/timestamp/test_timestamp.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index 34e48dc449a5b..ac94f1db54631 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -2,6 +2,7 @@ Tests for DatetimeIndex timezone-related methods """ from datetime import date, datetime, time, timedelta, tzinfo +from distutils.version import LooseVersion import dateutil from dateutil.tz import gettz, tzlocal @@ -585,6 +586,10 @@ def test_dti_construction_ambiguous_endpoint(self, tz): "dateutil/US/Pacific", "shift_backward", "2019-03-10 01:00", + marks=pytest.mark.xfail( + LooseVersion(dateutil.__version__) < LooseVersion("2.7.0"), + reason="GH 31043", + ), ), ["US/Pacific", timedelta(hours=1), "2019-03-10 03:00"], ], diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 5d9d27b96e42d..d57716bf2647d 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -2,6 +2,7 @@ import calendar from datetime import datetime, timedelta +from distutils.version import LooseVersion import locale import unicodedata @@ -1094,6 +1095,10 @@ def test_constructor_ambigous_dst(): assert result == expected +@pytest.mark.xfail( + LooseVersion(dateutil.__version__) < LooseVersion("2.7.0"), + reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", +) @pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) def test_constructor_before_dst_switch(epoch): # GH 31043 From 109a285ac48415070208a6eb173544e34deaf04e Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Tue, 21 Jan 2020 18:38:53 +0300 Subject: [PATCH 20/25] TST: use get_distribution to correct mypy error --- pandas/tests/indexes/datetimes/test_timezones.py | 4 +++- pandas/tests/scalar/timestamp/test_timestamp.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index ac94f1db54631..faca288d205ba 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -7,6 +7,7 @@ import dateutil from dateutil.tz import gettz, tzlocal import numpy as np +from pkg_resources import get_distribution import pytest import pytz @@ -587,7 +588,8 @@ def test_dti_construction_ambiguous_endpoint(self, tz): "shift_backward", "2019-03-10 01:00", marks=pytest.mark.xfail( - LooseVersion(dateutil.__version__) < LooseVersion("2.7.0"), + LooseVersion(get_distribution("python-dateutil").version) + < LooseVersion("2.7.0"), reason="GH 31043", ), ), diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index d57716bf2647d..fcd001cd59275 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -9,6 +9,7 @@ import dateutil from dateutil.tz import tzutc import numpy as np +from pkg_resources import get_distribution import pytest import pytz from pytz import timezone, utc @@ -1096,7 +1097,7 @@ def test_constructor_ambigous_dst(): @pytest.mark.xfail( - LooseVersion(dateutil.__version__) < LooseVersion("2.7.0"), + LooseVersion(get_distribution("python-dateutil").version) < LooseVersion("2.7.0"), reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", ) @pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) From 8e754f741e38a9644c20a9c7131d3a6f2ce2d89c Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Wed, 22 Jan 2020 11:10:45 +0300 Subject: [PATCH 21/25] TST: fix value stability test --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index fcd001cd59275..8e87e8cfbf162 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1112,5 +1112,5 @@ def test_constructor_before_dst_switch(epoch): ts = Timestamp(epoch, tz="dateutil/US/Pacific") result = ts.tz.dst(ts) expected = timedelta(seconds=0) - assert ts.value == epoch + assert Timestamp(ts).value == epoch assert result == expected From 894ed16851d002bcae06826b15edef80569af35b Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Wed, 22 Jan 2020 13:28:45 +0300 Subject: [PATCH 22/25] Revert "TST: fix value stability test" This reverts commit 8e754f741e38a9644c20a9c7131d3a6f2ce2d89c. --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 8e87e8cfbf162..fcd001cd59275 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1112,5 +1112,5 @@ def test_constructor_before_dst_switch(epoch): ts = Timestamp(epoch, tz="dateutil/US/Pacific") result = ts.tz.dst(ts) expected = timedelta(seconds=0) - assert Timestamp(ts).value == epoch + assert ts.value == epoch assert result == expected From 614176f357b8a47ad3d3158c45589409f961a951 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Wed, 22 Jan 2020 13:29:18 +0300 Subject: [PATCH 23/25] Revert Revert "TST: fix value stability test" This reverts commit 894ed16851d002bcae06826b15edef80569af35b. --- pandas/tests/scalar/timestamp/test_timestamp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index fcd001cd59275..8e87e8cfbf162 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -1112,5 +1112,5 @@ def test_constructor_before_dst_switch(epoch): ts = Timestamp(epoch, tz="dateutil/US/Pacific") result = ts.tz.dst(ts) expected = timedelta(seconds=0) - assert ts.value == epoch + assert Timestamp(ts).value == epoch assert result == expected From 56a3e711e22e62fb8d8efeac71e60b7be0b74931 Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Wed, 22 Jan 2020 16:25:30 +0300 Subject: [PATCH 24/25] TST: move to compat._optional._get_version --- pandas/tests/indexes/datetimes/test_timezones.py | 5 ++--- pandas/tests/scalar/timestamp/test_timestamp.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pandas/tests/indexes/datetimes/test_timezones.py b/pandas/tests/indexes/datetimes/test_timezones.py index faca288d205ba..c785eb67e5184 100644 --- a/pandas/tests/indexes/datetimes/test_timezones.py +++ b/pandas/tests/indexes/datetimes/test_timezones.py @@ -7,11 +7,11 @@ import dateutil from dateutil.tz import gettz, tzlocal import numpy as np -from pkg_resources import get_distribution import pytest import pytz from pandas._libs.tslibs import conversion, timezones +from pandas.compat._optional import _get_version import pandas.util._test_decorators as td import pandas as pd @@ -588,8 +588,7 @@ def test_dti_construction_ambiguous_endpoint(self, tz): "shift_backward", "2019-03-10 01:00", marks=pytest.mark.xfail( - LooseVersion(get_distribution("python-dateutil").version) - < LooseVersion("2.7.0"), + LooseVersion(_get_version(dateutil)) < LooseVersion("2.7.0"), reason="GH 31043", ), ), diff --git a/pandas/tests/scalar/timestamp/test_timestamp.py b/pandas/tests/scalar/timestamp/test_timestamp.py index 8e87e8cfbf162..b70948318e39e 100644 --- a/pandas/tests/scalar/timestamp/test_timestamp.py +++ b/pandas/tests/scalar/timestamp/test_timestamp.py @@ -9,7 +9,6 @@ import dateutil from dateutil.tz import tzutc import numpy as np -from pkg_resources import get_distribution import pytest import pytz from pytz import timezone, utc @@ -1097,7 +1096,7 @@ def test_constructor_ambigous_dst(): @pytest.mark.xfail( - LooseVersion(get_distribution("python-dateutil").version) < LooseVersion("2.7.0"), + LooseVersion(compat._optional._get_version(dateutil)) < LooseVersion("2.7.0"), reason="dateutil moved to Timedelta.total_seconds() in 2.7.0", ) @pytest.mark.parametrize("epoch", [1552211999999999872, 1552211999999999999]) From 9d21a81715ee0f8f48831fb3884f32aac3caadff Mon Sep 17 00:00:00 2001 From: Alexander Kirko Date: Wed, 22 Jan 2020 21:03:47 +0300 Subject: [PATCH 25/25] restart checks