diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/archiver-channel-vars.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/archiver-channel-vars.yaml index 204c993948..23cc70a840 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/archiver-channel-vars.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/archiver-channel-vars.yaml @@ -4,4 +4,4 @@ metadata: name: archiver-channel-vars data: AIRFLOW_ENV: development - HUEY_CONSUMER_WORKERS: "4" + HUEY_CONSUMER_WORKERS: "8" diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/kustomization.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/kustomization.yaml index 1288caf64e..39246a0535 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/kustomization.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-dev/kustomization.yaml @@ -10,3 +10,8 @@ resources: patches: - consumer.patch.yaml - ticker.patch.yaml + +images: +- name: 'gtfs-rt-archiver' + newName: 'ghcr.io/cal-itp/data-infra/gtfs-rt-archiver' + newTag: '3.1.1' diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-prod/kustomization.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-prod/kustomization.yaml index 88e70f5032..81d470fb2f 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-prod/kustomization.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-prod/kustomization.yaml @@ -6,3 +6,8 @@ resources: - ns.yaml - archiver-channel-vars.yaml - ../gtfs-rt-archiver-v3-release + +images: +- name: 'gtfs-rt-archiver' + newName: 'ghcr.io/cal-itp/data-infra/gtfs-rt-archiver' + newTag: '3.1.1' diff --git a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-release/kustomization.yaml b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-release/kustomization.yaml index 0790b2131b..9ace0ed667 100644 --- a/kubernetes/apps/overlays/gtfs-rt-archiver-v3-release/kustomization.yaml +++ b/kubernetes/apps/overlays/gtfs-rt-archiver-v3-release/kustomization.yaml @@ -3,8 +3,3 @@ kind: Kustomization resources: - '../../manifests/gtfs-rt-archiver-v3' - -images: -- name: 'gtfs-rt-archiver' - newName: 'ghcr.io/cal-itp/data-infra/gtfs-rt-archiver' - newTag: '3.0' diff --git a/services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/tasks.py b/services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/tasks.py index 964dfac444..2961e48947 100644 --- a/services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/tasks.py +++ b/services/gtfs-rt-archiver-v3/gtfs_rt_archiver_v3/tasks.py @@ -1,6 +1,7 @@ import os from datetime import datetime +import humanize import orjson import pendulum import structlog @@ -10,7 +11,7 @@ from huey import RedisExpireHuey from huey.registry import Message from huey.serializer import Serializer -from requests import HTTPError +from requests import HTTPError, RequestException from .metrics import ( FETCH_PROCESSING_TIME, @@ -46,6 +47,7 @@ def _deserialize(self, data: bytes) -> Message: client = storage.Client() +structlog.configure(processors=[structlog.processors.JSONRenderer()]) base_logger = structlog.get_logger() @@ -56,7 +58,7 @@ def instrument_signals(signal, task, exc=None): record_uri=task.kwargs["record"].uri, record_feed_type=task.kwargs["record"].data, signal=signal, - exc_type=type(exc) if exc else "", + exc_type=type(exc).__name__ if exc else "", ).inc() @@ -81,10 +83,12 @@ def load_auth_dict(): @huey.task(expires=5) def fetch(tick: datetime, record: AirtableGTFSDataRecord): labels = dict( - record_name=record.name, record_uri=record.uri, record_feed_type=record.data + record_name=record.name, + record_uri=record.uri, + record_feed_type=record.data, ) logger = base_logger.bind( - tick=tick, + tick=tick.isoformat(), **labels, ) slippage = (pendulum.now() - tick).total_seconds() @@ -95,18 +99,28 @@ def fetch(tick: datetime, record: AirtableGTFSDataRecord): extract, content = download_feed(record, ts=tick, auth_dict=auth_dict) except HTTPError as e: logger.error( - "http error occurred while downloading feed", + "unexpected HTTP response code from feed request", code=e.response.status_code, - exc_type=type(e), + content=e.response.text, + exc_type=type(e).__name__, + ) + raise + except RequestException as e: + logger.error( + "request exception occurred from feed request", + exc_type=type(e).__name__, ) raise except Exception as e: logger.error( - "other exception occurred while downloading feed", exc_type=type(e) + "other non-request exception occurred during download_feed", + exc_type=type(e).__name__, ) raise - typer.secho(f"saving {len(content)} bytes from {record.uri} to {extract.path}") + typer.secho( + f"saving {humanize.naturalsize(len(content))} from {record.uri} to {extract.path}" + ) extract.save_content(content=content, client=client) FETCH_PROCESSED_BYTES.labels( **labels, diff --git a/services/gtfs-rt-archiver-v3/pyproject.toml b/services/gtfs-rt-archiver-v3/pyproject.toml index 37be1f9f2f..b9c1a25f04 100644 --- a/services/gtfs-rt-archiver-v3/pyproject.toml +++ b/services/gtfs-rt-archiver-v3/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gtfs-rt-archiver" -version = "3.0" +version = "3.1.1" description = "" authors = ["Andrew Vaccaro "]