WIP

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.0
+    rev: v0.8.3
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

requirements/main.in

@@ -23,6 +23,7 @@ pyvo<1.6
 pyyaml
 rubin-nublado-client>=8.0.3
 safir>=6.5.1
+sentry-sdk
 shortuuid
 structlog
 websockets

src/mobu/main.py

@@ -15,6 +15,7 @@
 from datetime import timedelta
 from importlib.metadata import metadata, version
 
+import sentry_sdk
 import structlog
 from fastapi import FastAPI
 from fastapi.openapi.utils import get_openapi
@@ -35,6 +36,8 @@
 from .handlers.internal import internal_router
 from .status import post_status
 
+sentry_sdk.init(traces_sample_rate=0)
+
 __all__ = ["create_app", "lifespan"]
 
 

src/mobu/services/business/notebookrunner.py

@@ -11,10 +11,12 @@
 import shutil
 from collections.abc import AsyncIterator
 from contextlib import asynccontextmanager
+from dataclasses import asdict
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any
 
+import sentry_sdk
 import yaml
 from httpx import AsyncClient
 from rubin.nublado.client import JupyterLabSession
@@ -23,7 +25,7 @@
 
 from ...constants import GITHUB_REPO_CONFIG_PATH
 from ...dependencies.config import config_dependency
-from ...exceptions import NotebookRepositoryError, RepositoryConfigError
+from ...exceptions import NotebookRepositoryError
 from ...models.business.notebookrunner import (
     ListNotebookRunnerOptions,
     NotebookFilterResults,
@@ -108,18 +110,22 @@ async def initialize(self) -> None:
 
         repo_config_path = self._repo_dir / GITHUB_REPO_CONFIG_PATH
         if repo_config_path.exists():
-            try:
-                repo_config = RepoConfig.model_validate(
-                    yaml.safe_load(repo_config_path.read_text())
-                )
-            except Exception as err:
-                raise RepositoryConfigError(
-                    err=err,
-                    user=self.user.username,
-                    config_file=GITHUB_REPO_CONFIG_PATH,
-                    repo_url=self.options.repo_url,
-                    repo_ref=self.options.repo_ref,
-                ) from err
+            sentry_sdk.set_context(
+                "repo_info",
+                {
+                    "repo_url": self.options.repo_url,
+                    "repo_ref": self.options.repo_ref,
+                },
+            )
+            sentry_sdk.set_tag("repo_url", self.options.repo_url)
+            sentry_sdk.set_tag("repo_ref", self.options.repo_ref)
+            sentry_sdk.set_context(
+                "repo_info", {"config_file": GITHUB_REPO_CONFIG_PATH}
+            )
+
+            repo_config = RepoConfig.model_validate(
+                yaml.safe_load(repo_config_path.read_text())
+            )
         else:
             repo_config = RepoConfig()
 
@@ -290,6 +296,7 @@ async def execute_code(self, session: JupyterLabSession) -> None:
                 return
 
             self._notebook = self.next_notebook()
+            sentry_sdk.set_tag("notebook", self._notebook.name)
 
             iteration = f"{count + 1}/{num_executions}"
             msg = f"Notebook {self._notebook.name} iteration {iteration}"
@@ -305,6 +312,8 @@ async def execute_code(self, session: JupyterLabSession) -> None:
                     cell_number=f"#{cell['_index']}",
                     cell_source=code,
                 )
+                sentry_sdk.set_tag("cell", cell_id)
+                sentry_sdk.set_context("cell_info", asdict(ctx))
                 await self.execute_cell(session, code, cell_id, ctx)
                 if not await self.execution_idle():
                     break

src/mobu/services/business/tap.py

@@ -9,11 +9,12 @@
 
 import pyvo
 import requests
+import sentry_sdk
 from httpx import AsyncClient
 from structlog.stdlib import BoundLogger
 
 from ...dependencies.config import config_dependency
-from ...exceptions import CodeExecutionError, TAPClientError
+from ...exceptions import TAPClientError
 from ...models.business.tap import TAPBusinessData, TAPBusinessOptions
 from ...models.user import AuthenticatedUser
 from .base import Business
@@ -73,21 +74,16 @@ async def execute(self) -> None:
         query = self.get_next_query()
         with self.timings.start("execute_query", {"query": query}) as sw:
             self._running_query = query
-
-            try:
-                if self.options.sync:
-                    await self.run_sync_query(query)
-                else:
-                    await self.run_async_query(query)
-            except Exception as e:
-                raise CodeExecutionError(
-                    user=self.user.username,
-                    code=query,
-                    code_type="TAP query",
-                    event="execute_query",
-                    started_at=sw.start_time,
-                    error=f"{type(e).__name__}: {e!s}",
-                ) from e
+            sentry_sdk.set_tag("event", "execute_query")
+            sentry_sdk.set_tag("sync", self.options.sync)
+            sentry_sdk.set_context(
+                "query_info", {"query": query, "started_at": sw.start_time}
+            )
+
+            if self.options.sync:
+                await self.run_sync_query(query)
+            else:
+                await self.run_async_query(query)
 
             self._running_query = None
             elapsed = sw.elapsed.total_seconds()

src/mobu/services/monkey.py

@@ -6,12 +6,11 @@
 import sys
 from tempfile import NamedTemporaryFile, _TemporaryFileWrapper
 
+import sentry_sdk
 import structlog
 from aiojobs import Job, Scheduler
 from httpx import AsyncClient
-from safir.datetime import current_datetime, format_datetime_for_logging
 from safir.logging import Profile
-from safir.slack.blockkit import SlackException, SlackMessage, SlackTextField
 from safir.slack.webhook import SlackWebhookClient
 from structlog.stdlib import BoundLogger
 
@@ -135,36 +134,7 @@ async def alert(self, exc: Exception) -> None:
             state = self._state.name
             self._logger.info(f"Not sending alert because state is {state}")
             return
-        if not self._slack:
-            self._logger.info("Alert hook isn't set, so not sending to Slack")
-            return
-        monkey = f"{self._flock}/{self._name}" if self._flock else self._name
-        if isinstance(exc, MobuMixin):
-            # Add the monkey info if it is not already set.
-            if not exc.monkey:
-                exc.monkey = monkey
-        if isinstance(exc, SlackException):
-            # Avoid post_exception here since it adds the application name,
-            # but mobu (unusually) uses a dedicated web hook and therefore
-            # doesn't need to label its alerts.
-            await self._slack.post(exc.to_slack())
-        else:
-            now = current_datetime(microseconds=True)
-            date = format_datetime_for_logging(now)
-            name = type(exc).__name__
-            error = f"{name}: {exc!s}"
-            message = SlackMessage(
-                message=f"Unexpected exception {error}",
-                fields=[
-                    SlackTextField(heading="Exception type", text=name),
-                    SlackTextField(heading="Failed at", text=date),
-                    SlackTextField(heading="Monkey", text=monkey),
-                    SlackTextField(heading="User", text=self._user.username),
-                ],
-            )
-            await self._slack.post(message)
-
-        self._global_logger.info("Sent alert to Slack")
+        sentry_sdk.capture_exception(exc)
 
     def logfile(self) -> str:
         """Get the log file for a monkey's log."""
@@ -205,32 +175,39 @@ async def _runner(self) -> None:
         run = True
 
         while run:
-            try:
-                self._state = MonkeyState.RUNNING
-                await self.business.run()
-                run = False
-            except Exception as e:
-                msg = "Exception thrown while doing monkey business"
-                if self._flock:
-                    monkey = f"{self._flock}/{self._name}"
-                else:
-                    monkey = self._name
-                if isinstance(e, MobuMixin):
-                    e.monkey = monkey
-
-                await self.alert(e)
-                self._logger.exception(msg)
-
-                run = self._restart and self._state == MonkeyState.RUNNING
-                if run:
-                    self._state = MonkeyState.ERROR
-                    await self.business.error_idle()
-                    if self._state == MonkeyState.STOPPING:
-                        run = False
-                else:
-                    self._state = MonkeyState.STOPPING
-                    msg = "Shutting down monkey due to error"
-                    self._global_logger.warning(msg)
+            with sentry_sdk.isolation_scope() as scope:
+                scope.clear_breadcrumbs()
+                scope.set_tag("flock", self._flock)
+                scope.set_user({"username": self._user.username})
+                scope.set_tag("monkey", self._name)
+                scope.set_tag("business", self.business.__class__.__name__)
+
+                try:
+                    self._state = MonkeyState.RUNNING
+                    await self.business.run()
+                    run = False
+                except Exception as e:
+                    msg = "Exception thrown while doing monkey business"
+                    if self._flock:
+                        monkey = f"{self._flock}/{self._name}"
+                    else:
+                        monkey = self._name
+                    if isinstance(e, MobuMixin):
+                        e.monkey = monkey
+
+                    await self.alert(e)
+                    self._logger.exception(msg)
+
+                    run = self._restart and self._state == MonkeyState.RUNNING
+                    if run:
+                        self._state = MonkeyState.ERROR
+                        await self.business.error_idle()
+                        if self._state == MonkeyState.STOPPING:
+                            run = False
+                    else:
+                        self._state = MonkeyState.STOPPING
+                        msg = "Shutting down monkey due to error"
+                        self._global_logger.warning(msg)
 
         await self.business.close()
         self.state = MonkeyState.FINISHED