Merge pull request #1657 from cal-itp/rt-logging

RT archiver logging improvements

kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/archiver-channel-vars.yaml

@@ -4,4 +4,4 @@ metadata:
   name: archiver-channel-vars
 data:
   AIRFLOW_ENV: development
-  HUEY_CONSUMER_WORKERS: "4"
+  HUEY_CONSUMER_WORKERS: "8"

kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/kustomization.yaml

@@ -10,3 +10,8 @@ resources:
 patches:
 - consumer.patch.yaml
 - ticker.patch.yaml
+
+images:
+- name: 'gtfs-rt-archiver'
+  newName: 'ghcr.io/cal-itp/data-infra/gtfs-rt-archiver'
+  newTag: '3.1.1'

kubernetes/apps/overlays/gtfs-rt-archiver-v3-prod/kustomization.yaml

@@ -6,3 +6,8 @@ resources:
 - ns.yaml
 - archiver-channel-vars.yaml
 - ../gtfs-rt-archiver-v3-release
+
+images:
+- name: 'gtfs-rt-archiver'
+  newName: 'ghcr.io/cal-itp/data-infra/gtfs-rt-archiver'
+  newTag: '3.1.1'

kubernetes/apps/overlays/gtfs-rt-archiver-v3-release/kustomization.yaml

@@ -3,8 +3,3 @@ kind: Kustomization
 
 resources:
 - '../../manifests/gtfs-rt-archiver-v3'
-
-images:
-- name: 'gtfs-rt-archiver'
-  newName: 'ghcr.io/cal-itp/data-infra/gtfs-rt-archiver'
-  newTag: '3.0'

services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/tasks.py

@@ -1,6 +1,7 @@
 import os
 from datetime import datetime
 
+import humanize
 import orjson
 import pendulum
 import structlog
@@ -10,7 +11,7 @@
 from huey import RedisExpireHuey
 from huey.registry import Message
 from huey.serializer import Serializer
-from requests import HTTPError
+from requests import HTTPError, RequestException
 
 from .metrics import (
     FETCH_PROCESSING_TIME,
@@ -46,6 +47,7 @@ def _deserialize(self, data: bytes) -> Message:
 
 client = storage.Client()
 
+structlog.configure(processors=[structlog.processors.JSONRenderer()])
 base_logger = structlog.get_logger()
 
 
@@ -56,7 +58,7 @@ def instrument_signals(signal, task, exc=None):
         record_uri=task.kwargs["record"].uri,
         record_feed_type=task.kwargs["record"].data,
         signal=signal,
-        exc_type=type(exc) if exc else "",
+        exc_type=type(exc).__name__ if exc else "",
     ).inc()
 
 
@@ -81,10 +83,12 @@ def load_auth_dict():
 @huey.task(expires=5)
 def fetch(tick: datetime, record: AirtableGTFSDataRecord):
     labels = dict(
-        record_name=record.name, record_uri=record.uri, record_feed_type=record.data
+        record_name=record.name,
+        record_uri=record.uri,
+        record_feed_type=record.data,
     )
     logger = base_logger.bind(
-        tick=tick,
+        tick=tick.isoformat(),
         **labels,
     )
     slippage = (pendulum.now() - tick).total_seconds()
@@ -95,18 +99,28 @@ def fetch(tick: datetime, record: AirtableGTFSDataRecord):
             extract, content = download_feed(record, ts=tick, auth_dict=auth_dict)
         except HTTPError as e:
             logger.error(
-                "http error occurred while downloading feed",
+                "unexpected HTTP response code from feed request",
                 code=e.response.status_code,
-                exc_type=type(e),
+                content=e.response.text,
+                exc_type=type(e).__name__,
+            )
+            raise
+        except RequestException as e:
+            logger.error(
+                "request exception occurred from feed request",
+                exc_type=type(e).__name__,
             )
             raise
         except Exception as e:
             logger.error(
-                "other exception occurred while downloading feed", exc_type=type(e)
+                "other non-request exception occurred during download_feed",
+                exc_type=type(e).__name__,
             )
             raise
 
-        typer.secho(f"saving {len(content)} bytes from {record.uri} to {extract.path}")
+        typer.secho(
+            f"saving {humanize.naturalsize(len(content))} from {record.uri} to {extract.path}"
+        )
         extract.save_content(content=content, client=client)
         FETCH_PROCESSED_BYTES.labels(
             **labels,

services/gtfs-rt-archiver-v3/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gtfs-rt-archiver"
-version = "3.0"
+version = "3.1.1"
 description = ""
 authors = ["Andrew Vaccaro <[email protected]>"]