Feat: Sentry error monitoring

.env.sample

@@ -1,2 +1,7 @@
 FLASK_APP=eligibility_server/app.py
+
+# this sample uses an odd relative path because the value is
+# used by code under the eligibility_server directory
+# and thus the config folder is one level up
+
 ELIGIBILITY_SERVER_SETTINGS=../config/sample.py

docs/configuration/settings.md

@@ -6,6 +6,10 @@ The sections below outline in more detail the settings that you either must set
 
 ## App settings
 
+### `AGENCY_NAME`
+
+The name of the agency that this server is deployed for
+
 ### `APP_NAME`
 
 The name set on the Flask app
@@ -133,3 +137,37 @@ These are the possible values for the `CSV_QUOTING` variable:
 - `csv.QUOTE_ALL`: 1 - To be used when all the values in the CSV file are present inside quotation marks
 - `csv.QUOTE_NONNUMERIC`: 2 - To be used when the CSV file uses quotes around non-numeric entries
 - `csv.QUOTE_NONE`: 3 - To be used when the CSV file does not use quotes around entries
+
+## Sentry
+
+### `SENTRY_DSN`
+
+Cal-ITP's Sentry instance collects both [errors ("Issues")](https://sentry.calitp.org/organizations/sentry/issues/?project=4) and app [performance info](https://sentry.calitp.org/organizations/sentry/performance/?project=4).
+
+[Alerts are sent to #benefits-notify in Slack.](https://sentry.calitp.org/organizations/sentry/alerts/rules/eligibility-server/10/details/) [Others can be configured.](https://sentry.calitp.org/organizations/sentry/alerts/rules/)
+
+You can troubleshoot Sentry itself by [turning on debug mode](#debug_mode) and visiting `/error/`.
+
+!!! tldr "Sentry docs"
+
+    [Data Source Name (DSN)](https://docs.sentry.io/product/sentry-basics/dsn-explainer/)
+
+Enables sending events to Sentry.
+
+### `SENTRY_ENVIRONMENT`
+
+!!! tldr "Sentry docs"
+
+    [`environment` config value](https://docs.sentry.io/platforms/python/configuration/options/#environment)
+
+Segments errors by which deployment they occur in. This defaults to `local`, and can be set to match one of the environment names.
+
+### `SENTRY_TRACES_SAMPLE_RATE`
+
+!!! tldr "Sentry docs"
+
+    [`traces_sample_rate` config value](https://docs.sentry.io/platforms/python/configuration/options/#traces-sample-rate)
+
+Control the volume of transactions sent to Sentry. Value must be a float in the range `[0.0, 1.0]`.
+
+The default is `0.0` (i.e. no transactions are tracked).

eligibility_server/app.py

@@ -7,7 +7,7 @@
 from flask_restful import Api
 from flask.logging import default_handler
 
-from eligibility_server import db
+from eligibility_server import db, sentry
 from eligibility_server.keypair import get_server_public_key
 from eligibility_server.settings import Configuration
 from eligibility_server.verify import Verify
@@ -23,6 +23,7 @@
 
 # use an app context for access to config settings
 with app.app_context():
+    sentry.configure(config)
     # configure root logger first, to prevent duplicate log entries from Flask's logger
     logging.basicConfig(level=config.log_level, format=format_string)
     # configure Flask's logger
@@ -37,6 +38,14 @@ def TextResponse(content):
     return response
 
 
+with app.app_context():
+    if config.debug_mode:
+
+        @app.route("/error")
+        def trigger_error():
+            raise ValueError("testing Sentry for eligibility-server")
+
+
 @app.route("/healthcheck")
 def healthcheck():
     app.logger.info("Request healthcheck")

eligibility_server/sentry.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import subprocess
+
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+from sentry_sdk.scrubber import EventScrubber, DEFAULT_DENYLIST
+
+from eligibility_server.settings import Configuration
+
+logger = logging.getLogger(__name__)
+
+
+SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT", "local")
+
+
+# https://stackoverflow.com/a/21901260/358804
+def get_git_revision_hash():
+    return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip()
+
+
+def get_release() -> str:
+    return get_git_revision_hash()
+
+
+def get_denylist():
+    # custom denylist
+    denylist = DEFAULT_DENYLIST + ["sub", "name"]
+    return denylist
+
+
+def get_traces_sample_rate(config: Configuration):
+    rate = config.sentry_traces_sample_rate
+    if rate < 0.0 or rate > 1.0:
+        logger.warning("SENTRY_TRACES_SAMPLE_RATE was not in the range [0.0, 1.0], defaulting to 0.0")
+        rate = 0.0
+    else:
+        logger.info(f"SENTRY_TRACES_SAMPLE_RATE set to: {rate}")
+
+    return rate
+
+
+def configure(config: Configuration):
+    SENTRY_DSN = config.sentry_dsn
+    if SENTRY_DSN:
+        release = get_release()
+        logger.info(f"Enabling Sentry for environment '{SENTRY_ENVIRONMENT}', release '{release}'...")
+
+        sentry_sdk.init(
+            dsn=SENTRY_DSN,
+            integrations=[
+                FlaskIntegration(),
+            ],
+            traces_sample_rate=get_traces_sample_rate(config),
+            environment=SENTRY_ENVIRONMENT,
+            release=release,
+            in_app_include=["eligibility_server"],
+            # send_default_pii must be False (the default) for a custom EventScrubber/denylist
+            # https://docs.sentry.io/platforms/python/data-management/sensitive-data/#event_scrubber
+            send_default_pii=False,
+            event_scrubber=EventScrubber(denylist=get_denylist()),
+        )
+
+        sentry_sdk.set_tag("agency_name", config.agency_name)
+    else:
+        logger.warning("SENTRY_DSN not set, so won't send events")

eligibility_server/settings.py

@@ -5,6 +5,7 @@
 
 # App settings
 
+AGENCY_NAME = "[unset]"
 APP_NAME = "eligibility_server.app"
 DEBUG_MODE = True
 HOST = "0.0.0.0"  # nosec
@@ -43,10 +44,19 @@
 CSV_QUOTING = 3
 CSV_QUOTECHAR = '"'
 
+# Sentry
+
+SENTRY_DSN = None
+SENTRY_TRACES_SAMPLE_RATE = 0.0
+
 
 class Configuration:
     # App settings
 
+    @property
+    def agency_name(self):
+        return str(current_app.config["AGENCY_NAME"])
+
     @property
     def app_name(self):
         return str(current_app.config["APP_NAME"])
@@ -134,3 +144,14 @@ def csv_quoting(self):
     @property
     def csv_quotechar(self):
         return str(current_app.config["CSV_QUOTECHAR"])
+
+    # Sentry
+
+    @property
+    def sentry_dsn(self):
+        sentry_dsn = current_app.config["SENTRY_DSN"]
+        return str(sentry_dsn) if sentry_dsn else None
+
+    @property
+    def sentry_traces_sample_rate(self):
+        return float(current_app.config["SENTRY_TRACES_SAMPLE_RATE"])

pyproject.toml

@@ -11,7 +11,8 @@ dependencies = [
     "Flask==2.3.3",
     "Flask-RESTful==0.3.10",
     "Flask-SQLAlchemy==3.1.1",
-    "requests==2.31.0"
+    "requests==2.31.0",
+    "sentry-sdk[flask]==1.19.1"
 ]
 
 [project.optional-dependencies]

terraform/app_service.tf

@@ -56,6 +56,9 @@ resource "azurerm_linux_web_app" "main" {
     "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = "false"
     "WEBSITES_PORT"                       = "8000"
     "WEBSITES_CONTAINER_START_TIME_LIMIT" = "1800"
+
+    # Sentry
+    "SENTRY_ENVIRONMENT" = "${local.env_name}"
   }
 
   identity {

tests/test_settings.py

@@ -225,3 +225,14 @@ def test_configuration_csv_quotechar(mocker, configuration: Configuration):
     mocker.patch.dict("eligibility_server.settings.current_app.config", mocked_config)
 
     assert configuration.csv_quotechar == new_value
+
+
+@pytest.mark.usefixtures("flask")
+def test_configuration_sentry_dsn(mocker, configuration: Configuration):
+    assert configuration.sentry_dsn is None
+
+    new_value = "https://sentry.example.com"
+    mocked_config = {"SENTRY_DSN": new_value}
+    mocker.patch.dict("eligibility_server.settings.current_app.config", mocked_config)
+
+    assert configuration.sentry_dsn == new_value