limit the number of exception lines posted to Slack

allenai · Jan 6, 2025 · 03a597a · 03a597a
1 parent c634066
commit 03a597a
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/src/olmo_core/train/callbacks/slack_notifier.py b/src/olmo_core/train/callbacks/slack_notifier.py
@@ -12,6 +12,7 @@
 
 SLACK_WEBHOOK_URL_ENV_VAR = "SLACK_WEBHOOK_URL"
 BEAKER_JOB_ID_ENV_VAR = "BEAKER_JOB_ID"
+EXC_LINE_LIMIT = 30
 
 
 class SlackNotificationSetting(StrEnum):
@@ -98,7 +99,12 @@ def on_error(self, exc: BaseException):
             SlackNotificationSetting.end_only,
             SlackNotificationSetting.failure_only,
         ):
-            self._post_message(f"failed with error:\n{exc}")
+            exc_lines = str(exc).rstrip("\n").split("\n")
+            if len(exc_lines) > EXC_LINE_LIMIT:
+                exc_lines = exc_lines[:EXC_LINE_LIMIT]
+                exc_lines.append("...")
+            exc_str = "\n".join(exc_lines)
+            self._post_message(f"failed with error:\n```\n{exc_str}\n```")
 
     def _post_message(self, msg: str):
         webhook_url = self.webhook_url or os.environ.get(SLACK_WEBHOOK_URL_ENV_VAR)