diff --git a/pandas/_libs/internals.pyx b/pandas/_libs/internals.pyx
index 7a9a3b84fd69f..3b1a6bc7436c3 100644
--- a/pandas/_libs/internals.pyx
+++ b/pandas/_libs/internals.pyx
@@ -897,6 +897,11 @@ cdef class BlockValuesRefs:
         else:
             self.referenced_blocks = []
 
+    def _clear_dead_references(self) -> None:
+        self.referenced_blocks = [
+            ref for ref in self.referenced_blocks if ref() is not None
+        ]
+
     def add_reference(self, blk: Block) -> None:
         """Adds a new reference to our reference collection.
 
@@ -905,6 +910,7 @@ cdef class BlockValuesRefs:
         blk : Block
             The block that the new references should point to.
         """
+        self._clear_dead_references()
         self.referenced_blocks.append(weakref.ref(blk))
 
     def add_index_reference(self, index: object) -> None:
@@ -915,6 +921,7 @@ cdef class BlockValuesRefs:
         index : Index
             The index that the new reference should point to.
         """
+        self._clear_dead_references()
         self.referenced_blocks.append(weakref.ref(index))
 
     def has_reference(self) -> bool:
@@ -927,8 +934,6 @@ cdef class BlockValuesRefs:
         -------
         bool
         """
-        self.referenced_blocks = [
-            ref for ref in self.referenced_blocks if ref() is not None
-        ]
+        self._clear_dead_references()
         # Checking for more references than block pointing to itself
         return len(self.referenced_blocks) > 1