diff --git a/doc/source/whatsnew/v2.0.3.rst b/doc/source/whatsnew/v2.0.3.rst index d8d152f2bd779..5590066d9513c 100644 --- a/doc/source/whatsnew/v2.0.3.rst +++ b/doc/source/whatsnew/v2.0.3.rst @@ -23,6 +23,7 @@ Fixed regressions Bug fixes ~~~~~~~~~ - Bug in :func:`RangeIndex.union` when using ``sort=True`` with another :class:`RangeIndex` (:issue:`53490`) +- Bug in :func:`Series.reindex` when expanding a non-nanosecond datetime or timedelta :class:`Series` would not fill with ``NaT`` correctly (:issue:`53497`) - Bug in :func:`read_csv` when defining ``dtype`` with ``bool[pyarrow]`` for the ``"c"`` and ``"python"`` engines (:issue:`53390`) - Bug in :meth:`Series.str.split` and :meth:`Series.str.rsplit` with ``expand=True`` for :class:`ArrowDtype` with ``pyarrow.string`` (:issue:`53532`) - Bug in indexing methods (e.g. :meth:`DataFrame.__getitem__`) where taking the entire :class:`DataFrame`/:class:`Series` would raise an ``OverflowError`` when Copy on Write was enabled and the length of the array was over the maximum size a 32-bit integer can hold (:issue:`53616`) diff --git a/pandas/core/dtypes/cast.py b/pandas/core/dtypes/cast.py index 831b368f58225..be23911165138 100644 --- a/pandas/core/dtypes/cast.py +++ b/pandas/core/dtypes/cast.py @@ -562,9 +562,16 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan): If fill_value is a non-scalar and dtype is not object. """ orig = fill_value + orig_is_nat = False if checknull(fill_value): # https://github.com/pandas-dev/pandas/pull/39692#issuecomment-1441051740 # avoid cache misses with NaN/NaT values that are not singletons + if fill_value is not NA: + try: + orig_is_nat = np.isnat(fill_value) + except TypeError: + pass + fill_value = _canonical_nans.get(type(fill_value), fill_value) # for performance, we are using a cached version of the actual implementation @@ -580,8 +587,10 @@ def maybe_promote(dtype: np.dtype, fill_value=np.nan): # if fill_value is not hashable (required for caching) dtype, fill_value = _maybe_promote(dtype, fill_value) - if dtype == _dtype_obj and orig is not None: - # GH#51592 restore our potentially non-canonical fill_value + if (dtype == _dtype_obj and orig is not None) or ( + orig_is_nat and np.datetime_data(orig)[0] != "ns" + ): + # GH#51592,53497 restore our potentially non-canonical fill_value fill_value = orig return dtype, fill_value diff --git a/pandas/tests/series/methods/test_reindex.py b/pandas/tests/series/methods/test_reindex.py index 16fc04d588a52..614002d755b6d 100644 --- a/pandas/tests/series/methods/test_reindex.py +++ b/pandas/tests/series/methods/test_reindex.py @@ -12,6 +12,7 @@ NaT, Period, PeriodIndex, + RangeIndex, Series, Timedelta, Timestamp, @@ -422,3 +423,14 @@ def test_reindexing_with_float64_NA_log(): result_log = np.log(s_reindex) expected_log = Series([0, np.NaN, np.NaN], dtype=Float64Dtype()) tm.assert_series_equal(result_log, expected_log) + + +@pytest.mark.parametrize("dtype", ["timedelta64", "datetime64"]) +def test_reindex_expand_nonnano_nat(dtype): + # GH 53497 + ser = Series(np.array([1], dtype=f"{dtype}[s]")) + result = ser.reindex(RangeIndex(2)) + expected = Series( + np.array([1, getattr(np, dtype)("nat", "s")], dtype=f"{dtype}[s]") + ) + tm.assert_series_equal(result, expected)