From 74f4b8bfa080b4eab7eac610dc1df10cd28dd47b Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 00:17:44 +0200 Subject: [PATCH 01/17] Issue #30520 --- pandas/core/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 9017584171850..461870470c648 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -84,7 +84,7 @@ def maybe_box(indexer, values, obj, key): def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed - if isinstance(value, (np.datetime64, datetime)): + if isinstance(value, (np.datetime64, datetime)) and not isinstance(value, tslibs.Timestamp): value = tslibs.Timestamp(value) elif isinstance(value, (np.timedelta64, timedelta)): value = tslibs.Timedelta(value) From dcb45ed63f1ff04a6369eade55feaf24dcbbd566 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 00:50:25 +0200 Subject: [PATCH 02/17] Issue #30520 --- pandas/core/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 461870470c648..e5db0fe38f916 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -84,7 +84,8 @@ def maybe_box(indexer, values, obj, key): def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed - if isinstance(value, (np.datetime64, datetime)) and not isinstance(value, tslibs.Timestamp): + if isinstance(value, (np.datetime64, datetime)) \ + and not isinstance(value, tslibs.Timestamp): value = tslibs.Timestamp(value) elif isinstance(value, (np.timedelta64, timedelta)): value = tslibs.Timedelta(value) From 6a2511c14f32726cd9d0ab0d0fecf1a5d4cfba52 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 01:01:14 +0200 Subject: [PATCH 03/17] Issue #30520 works for timedeltas as well --- pandas/core/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index e5db0fe38f916..eaccc516408c1 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -87,7 +87,8 @@ def maybe_box_datetimelike(value): if isinstance(value, (np.datetime64, datetime)) \ and not isinstance(value, tslibs.Timestamp): value = tslibs.Timestamp(value) - elif isinstance(value, (np.timedelta64, timedelta)): + elif isinstance(value, (np.timedelta64, timedelta)) \ + and not isinstance(value, tslibs.Timedelta): value = tslibs.Timedelta(value) return value From 54cb25e63fb66474490bf4f0159cbba5be3beda6 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 03:28:44 +0200 Subject: [PATCH 04/17] Issue #30520 check for NaT values --- pandas/core/common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index eaccc516408c1..445548aa565e5 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -85,10 +85,12 @@ def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed if isinstance(value, (np.datetime64, datetime)) \ - and not isinstance(value, tslibs.Timestamp): + and not isinstance(value, tslibs.Timestamp) \ + and notnull(value): value = tslibs.Timestamp(value) elif isinstance(value, (np.timedelta64, timedelta)) \ - and not isinstance(value, tslibs.Timedelta): + and not isinstance(value, tslibs.Timedelta) \ + and notnull(value): value = tslibs.Timedelta(value) return value From 9d751b95dda35b18de68082f419701f2c5d9a505 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 12:18:15 +0200 Subject: [PATCH 05/17] new lines using parentheses --- pandas/core/common.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 445548aa565e5..4834b9ae365c3 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -72,7 +72,6 @@ def consensus_name_attr(objs): def maybe_box(indexer, values, obj, key): - # if we have multiples coming back, box em if isinstance(values, np.ndarray): return obj[indexer.get_loc(key)] @@ -84,13 +83,13 @@ def maybe_box(indexer, values, obj, key): def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed - if isinstance(value, (np.datetime64, datetime)) \ - and not isinstance(value, tslibs.Timestamp) \ - and notnull(value): + if isinstance(value, (np.datetime64, datetime)) and ( + not isinstance(value, tslibs.Timestamp)) and ( + notnull(value)): value = tslibs.Timestamp(value) - elif isinstance(value, (np.timedelta64, timedelta)) \ - and not isinstance(value, tslibs.Timedelta) \ - and notnull(value): + elif isinstance(value, (np.timedelta64, timedelta)) and ( + not isinstance(value, tslibs.Timedelta)) and ( + notnull(value)): value = tslibs.Timedelta(value) return value @@ -123,7 +122,7 @@ def is_bool_indexer(key: Any) -> bool: """ na_msg = "cannot index with vector containing NA / NaN values" if isinstance(key, (ABCSeries, np.ndarray, ABCIndex)) or ( - is_array_like(key) and is_extension_array_dtype(key.dtype) + is_array_like(key) and is_extension_array_dtype(key.dtype) ): if key.dtype == np.object_: key = np.asarray(values_from_object(key)) @@ -219,7 +218,6 @@ def try_sort(iterable): def asarray_tuplesafe(values, dtype=None): - if not (isinstance(values, (list, tuple)) or hasattr(values, "__array__")): values = list(values) elif isinstance(values, ABCIndexClass): @@ -288,10 +286,10 @@ def is_null_slice(obj): We have a null slice. """ return ( - isinstance(obj, slice) - and obj.start is None - and obj.stop is None - and obj.step is None + isinstance(obj, slice) + and obj.start is None + and obj.stop is None + and obj.step is None ) @@ -308,7 +306,7 @@ def is_full_slice(obj, l): We have a full length slice. """ return ( - isinstance(obj, slice) and obj.start == 0 and obj.stop == l and obj.step is None + isinstance(obj, slice) and obj.start == 0 and obj.stop == l and obj.step is None ) From 524150c059dd0c244c9f65d8c16987fbe19e7f36 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 22:20:41 +0200 Subject: [PATCH 06/17] Issue #30520 more readable code + specifically check for NaT --- pandas/core/common.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index 4834b9ae365c3..a4fe96b8fec7d 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -83,13 +83,12 @@ def maybe_box(indexer, values, obj, key): def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed - if isinstance(value, (np.datetime64, datetime)) and ( - not isinstance(value, tslibs.Timestamp)) and ( - notnull(value)): + if isinstance(value, (tslibs.Timedelta, tslibs.Timestamp)) or value is tslibs.NaT: + return value + + if isinstance(value, (np.datetime64, datetime)): value = tslibs.Timestamp(value) - elif isinstance(value, (np.timedelta64, timedelta)) and ( - not isinstance(value, tslibs.Timedelta)) and ( - notnull(value)): + elif isinstance(value, (np.timedelta64, timedelta)): value = tslibs.Timedelta(value) return value From c9699d9be93e1bcbbd05802e7fa4a597b221310c Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 22:21:39 +0200 Subject: [PATCH 07/17] code formatting --- pandas/core/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index a4fe96b8fec7d..f835bd639cac8 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -83,7 +83,8 @@ def maybe_box(indexer, values, obj, key): def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed - if isinstance(value, (tslibs.Timedelta, tslibs.Timestamp)) or value is tslibs.NaT: + if isinstance(value, (tslibs.Timedelta, tslibs.Timestamp)) or ( + value is tslibs.NaT): return value if isinstance(value, (np.datetime64, datetime)): From 65ab96456dba2eed4458ef6905ea2bb0de23070c Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Sun, 29 Dec 2019 22:44:01 +0200 Subject: [PATCH 08/17] code formatting --- pandas/core/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pandas/core/common.py b/pandas/core/common.py index f835bd639cac8..98194acae1346 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -84,7 +84,8 @@ def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed if isinstance(value, (tslibs.Timedelta, tslibs.Timestamp)) or ( - value is tslibs.NaT): + value is tslibs.NaT + ): return value if isinstance(value, (np.datetime64, datetime)): From 48cc580710ef44607766537198a77ba5ba883a91 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Mon, 30 Dec 2019 13:42:25 +0200 Subject: [PATCH 09/17] first attempt at ASV performance test --- asv_bench/benchmarks/inference.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index e85b3bd2c7687..dfa043bf7b74f 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -1,10 +1,15 @@ +import datetime + import numpy as np +from pandas._libs.tslibs.timestamps import Timestamp from pandas import DataFrame, Series, to_numeric import pandas.util.testing as tm from .pandas_vb_common import lib, numeric_dtypes +from pandas.core.common import maybe_box_datetimelike + class NumericInferOps: # from GH 7332 @@ -52,7 +57,6 @@ def time_add_timedeltas(self, df): class ToNumeric: - params = ["ignore", "coerce"] param_names = ["errors"] @@ -73,7 +77,6 @@ def time_from_str(self, errors): class ToNumericDowncast: - param_names = ["dtype", "downcast"] params = [ [ @@ -121,4 +124,16 @@ def time_convert(self, data): lib.maybe_convert_numeric(data, set(), coerce_numeric=False) +class MaybeBoxDatetimelike: + def setup(self): + self.pd_timestamp = Timestamp.now() + self.py_timestamp = datetime.datetime.now() + + def pd_datetime_box(self): + maybe_box_datetimelike(self.pd_timestamp) + + def py_datetime_box(self): + maybe_box_datetimelike(self.py_timestamp) + + from .pandas_vb_common import setup # noqa: F401 isort:skip From 81d66b0fee270d751c68efd7c6aa1fa67ba90e0c Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Mon, 30 Dec 2019 13:59:21 +0200 Subject: [PATCH 10/17] import sorting fix --- asv_bench/benchmarks/inference.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index dfa043bf7b74f..25bae6acd32cd 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -3,12 +3,10 @@ import numpy as np from pandas._libs.tslibs.timestamps import Timestamp -from pandas import DataFrame, Series, to_numeric import pandas.util.testing as tm - -from .pandas_vb_common import lib, numeric_dtypes - +from pandas import DataFrame, Series, to_numeric from pandas.core.common import maybe_box_datetimelike +from .pandas_vb_common import lib, numeric_dtypes class NumericInferOps: @@ -136,4 +134,3 @@ def py_datetime_box(self): maybe_box_datetimelike(self.py_timestamp) -from .pandas_vb_common import setup # noqa: F401 isort:skip From 17470980a074e4c35a68ab64553738a53bf76358 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Mon, 30 Dec 2019 14:15:56 +0200 Subject: [PATCH 11/17] inference py blank lines at the end of the file removed --- asv_bench/benchmarks/inference.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index 25bae6acd32cd..ad5ebe6c1ff0f 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -131,6 +131,4 @@ def pd_datetime_box(self): maybe_box_datetimelike(self.pd_timestamp) def py_datetime_box(self): - maybe_box_datetimelike(self.py_timestamp) - - + maybe_box_datetimelike(self.py_timestamp) \ No newline at end of file From 894ee24c973aeaff2ee5337b45cfa8702c845f74 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Mon, 30 Dec 2019 14:18:12 +0200 Subject: [PATCH 12/17] inference.py imports fix --- asv_bench/benchmarks/inference.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index ad5ebe6c1ff0f..d3e7bc238312a 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -1,10 +1,9 @@ import datetime import numpy as np -from pandas._libs.tslibs.timestamps import Timestamp import pandas.util.testing as tm -from pandas import DataFrame, Series, to_numeric +from pandas import DataFrame, Series, Timestamp, to_numeric from pandas.core.common import maybe_box_datetimelike from .pandas_vb_common import lib, numeric_dtypes @@ -131,4 +130,4 @@ def pd_datetime_box(self): maybe_box_datetimelike(self.pd_timestamp) def py_datetime_box(self): - maybe_box_datetimelike(self.py_timestamp) \ No newline at end of file + maybe_box_datetimelike(self.py_timestamp) From bdc8eb4d94b24fc980e3049140dc90598f181a32 Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Mon, 30 Dec 2019 14:38:03 +0200 Subject: [PATCH 13/17] imports sorted with isort --- asv_bench/benchmarks/inference.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index d3e7bc238312a..273c5d5ffb361 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -2,9 +2,10 @@ import numpy as np -import pandas.util.testing as tm from pandas import DataFrame, Series, Timestamp, to_numeric from pandas.core.common import maybe_box_datetimelike +import pandas.util.testing as tm + from .pandas_vb_common import lib, numeric_dtypes From b4de215972d16990fdb47a2967c3add361dfb98d Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Wed, 1 Jan 2020 22:36:13 +0200 Subject: [PATCH 14/17] formatting and import fixes --- asv_bench/benchmarks/inference.py | 1 + pandas/core/common.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index 273c5d5ffb361..c6104a7a8f9f2 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -7,6 +7,7 @@ import pandas.util.testing as tm from .pandas_vb_common import lib, numeric_dtypes +from .pandas_vb_common import setup # noqa: F401 isort:skip class NumericInferOps: diff --git a/pandas/core/common.py b/pandas/core/common.py index 98194acae1346..7fcf55a1e3c35 100644 --- a/pandas/core/common.py +++ b/pandas/core/common.py @@ -84,7 +84,7 @@ def maybe_box_datetimelike(value): # turn a datetime like into a Timestamp/timedelta as needed if isinstance(value, (tslibs.Timedelta, tslibs.Timestamp)) or ( - value is tslibs.NaT + value is tslibs.NaT ): return value @@ -123,7 +123,7 @@ def is_bool_indexer(key: Any) -> bool: """ na_msg = "cannot index with vector containing NA / NaN values" if isinstance(key, (ABCSeries, np.ndarray, ABCIndex)) or ( - is_array_like(key) and is_extension_array_dtype(key.dtype) + is_array_like(key) and is_extension_array_dtype(key.dtype) ): if key.dtype == np.object_: key = np.asarray(values_from_object(key)) @@ -287,10 +287,10 @@ def is_null_slice(obj): We have a null slice. """ return ( - isinstance(obj, slice) - and obj.start is None - and obj.stop is None - and obj.step is None + isinstance(obj, slice) + and obj.start is None + and obj.stop is None + and obj.step is None ) @@ -307,7 +307,7 @@ def is_full_slice(obj, l): We have a full length slice. """ return ( - isinstance(obj, slice) and obj.start == 0 and obj.stop == l and obj.step is None + isinstance(obj, slice) and obj.start == 0 and obj.stop == l and obj.step is None ) From 995326d9e44685755775769f3a18d1d55fe5b24c Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Wed, 1 Jan 2020 22:57:32 +0200 Subject: [PATCH 15/17] imports formatting fix --- asv_bench/benchmarks/inference.py | 1 + 1 file changed, 1 insertion(+) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index c6104a7a8f9f2..3153ef13374f8 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -7,6 +7,7 @@ import pandas.util.testing as tm from .pandas_vb_common import lib, numeric_dtypes + from .pandas_vb_common import setup # noqa: F401 isort:skip From 961ebb7c88fed9963a8a1b56f45bcf5d20aa152e Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Thu, 2 Jan 2020 01:08:50 +0200 Subject: [PATCH 16/17] import reordering --- asv_bench/benchmarks/inference.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index 3153ef13374f8..db2904ba620a3 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -8,8 +8,6 @@ from .pandas_vb_common import lib, numeric_dtypes -from .pandas_vb_common import setup # noqa: F401 isort:skip - class NumericInferOps: # from GH 7332 @@ -134,3 +132,6 @@ def pd_datetime_box(self): def py_datetime_box(self): maybe_box_datetimelike(self.py_timestamp) + + +from .pandas_vb_common import setup # noqa: F401 isort:skip \ No newline at end of file From 4071dde86e33434e1bee8304fa62074949f813cc Mon Sep 17 00:00:00 2001 From: Ivan Vasilev Date: Thu, 2 Jan 2020 01:39:53 +0200 Subject: [PATCH 17/17] new line at the end of the file --- asv_bench/benchmarks/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asv_bench/benchmarks/inference.py b/asv_bench/benchmarks/inference.py index db2904ba620a3..2455cfbbb21df 100644 --- a/asv_bench/benchmarks/inference.py +++ b/asv_bench/benchmarks/inference.py @@ -134,4 +134,4 @@ def py_datetime_box(self): maybe_box_datetimelike(self.py_timestamp) -from .pandas_vb_common import setup # noqa: F401 isort:skip \ No newline at end of file +from .pandas_vb_common import setup # noqa: F401 isort:skip