Do not keep going if there are 5 back-to-back background update failu…

…res. (#12781)
matrix-org · May 18, 2022 · 635f0d9 · 635f0d9
1 parent df49635
commit 635f0d9
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/changelog.d/12781.misc b/changelog.d/12781.misc
@@ -0,0 +1 @@
+Do not keep going if there are 5 back-to-back background update failures.
diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py
@@ -282,12 +282,20 @@ async def run_background_updates(self, sleep: bool) -> None:
 
         self._running = True
 
+        back_to_back_failures = 0
+
         try:
             logger.info("Starting background schema updates")
             while self.enabled:
                 try:
                     result = await self.do_next_background_update(sleep)
+                    back_to_back_failures = 0
                 except Exception:
+                    back_to_back_failures += 1
+                    if back_to_back_failures >= 5:
+                        raise RuntimeError(
+                            "5 back-to-back background update failures; aborting."
+                        )
                     logger.exception("Error doing update")
                 else:
                     if result:
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Do not keep going if there are 5 back-to-back background update failures.