diff --git a/doc/source/whatsnew/v1.0.4.rst b/doc/source/whatsnew/v1.0.4.rst
index 80f8976826bc8..2852b3b4fe92c 100644
--- a/doc/source/whatsnew/v1.0.4.rst
+++ b/doc/source/whatsnew/v1.0.4.rst
@@ -21,6 +21,7 @@ Fixed regressions
 - Fix performance regression in ``memory_usage(deep=True)`` for object dtype (:issue:`33012`)
 - Bug where :meth:`Categorical.replace` would replace with ``NaN`` whenever the new value and replacement value were equal (:issue:`33288`)
 - Bug where an ordered :class:`Categorical` containing only ``NaN`` values would raise rather than returning ``NaN`` when taking the minimum or maximum  (:issue:`33450`)
+- Fix to preserve the ability to index with the "nearest" method with xarray's CFTimeIndex, an :class:`Index` subclass (`pydata/xarray#3751 <https://github.com/pydata/xarray/issues/3751>`_, :issue:`32905`).
 - 
 
 .. _whatsnew_104.bug_fixes:
diff --git a/environment.yml b/environment.yml
index 3c7aed32af3f8..ea15bf1bffe1f 100644
--- a/environment.yml
+++ b/environment.yml
@@ -102,6 +102,7 @@ dependencies:
   - s3fs  # pandas.read_csv... when using 's3://...' path
   - sqlalchemy  # pandas.read_sql, DataFrame.to_sql
   - xarray  # DataFrame.to_xarray
+  - cftime  # Needed for downstream xarray.CFTimeIndex test
   - pyreadstat  # pandas.read_spss
   - tabulate>=0.8.3  # DataFrame.to_markdown
   - pip:
diff --git a/pandas/core/indexes/base.py b/pandas/core/indexes/base.py
index 38f8521bfa2d4..5ff8f590e78aa 100644
--- a/pandas/core/indexes/base.py
+++ b/pandas/core/indexes/base.py
@@ -2817,8 +2817,9 @@ def _get_nearest_indexer(self, target, limit, tolerance):
         left_indexer = self.get_indexer(target, "pad", limit=limit)
         right_indexer = self.get_indexer(target, "backfill", limit=limit)
 
-        left_distances = np.abs(self[left_indexer] - target)
-        right_distances = np.abs(self[right_indexer] - target)
+        target_values = target._values
+        left_distances = np.abs(self._values[left_indexer] - target_values)
+        right_distances = np.abs(self._values[right_indexer] - target_values)
 
         op = operator.lt if self.is_monotonic_increasing else operator.le
         indexer = np.where(
@@ -2827,11 +2828,11 @@ def _get_nearest_indexer(self, target, limit, tolerance):
             right_indexer,
         )
         if tolerance is not None:
-            indexer = self._filter_indexer_tolerance(target, indexer, tolerance)
+            indexer = self._filter_indexer_tolerance(target_values, indexer, tolerance)
         return indexer
 
     def _filter_indexer_tolerance(self, target, indexer, tolerance):
-        distance = abs(self.values[indexer] - target)
+        distance = abs(self._values[indexer] - target)
         indexer = np.where(distance <= tolerance, indexer, -1)
         return indexer
 
diff --git a/pandas/tests/test_downstream.py b/pandas/tests/test_downstream.py
index d4d7803a724e2..9f473d1ad4df6 100644
--- a/pandas/tests/test_downstream.py
+++ b/pandas/tests/test_downstream.py
@@ -8,6 +8,8 @@
 import numpy as np  # noqa
 import pytest
 
+import pandas.util._test_decorators as td
+
 from pandas import DataFrame, Series
 import pandas._testing as tm
 
@@ -47,6 +49,19 @@ def test_xarray(df):
     assert df.to_xarray() is not None
 
 
+@td.skip_if_no("cftime")
+@td.skip_if_no("xarray", "0.10.4")
+def test_xarray_cftimeindex_nearest():
+    # https://github.com/pydata/xarray/issues/3751
+    import cftime
+    import xarray
+
+    times = xarray.cftime_range("0001", periods=2)
+    result = times.get_loc(cftime.DatetimeGregorian(2000, 1, 1), method="nearest")
+    expected = 1
+    assert result == expected
+
+
 def test_oo_optimizable():
     # GH 21071
     subprocess.check_call([sys.executable, "-OO", "-c", "import pandas"])
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 3089b96d26780..0b8236c4fd1d2 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -69,6 +69,7 @@ tables>=3.4.2
 s3fs
 sqlalchemy
 xarray
+cftime
 pyreadstat
 tabulate>=0.8.3
 git+https://github.com/pandas-dev/pydata-sphinx-theme.git@master
\ No newline at end of file