diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index e606d29fc2..194821cbdc 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -94,6 +94,18 @@ Message published for each changed tracked object. The first message is publishe } ``` +### `frigate/tracked_object_update` + +Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description. + +```json +{ + "type": "description", + "id": "1607123955.475377-mxklsc", + "description": "The car is a red sedan moving away from the camera." +} +``` + ### `frigate/reviews` Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published. diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py index 1f480fa9c5..2bddc97a53 100644 --- a/frigate/comms/dispatcher.py +++ b/frigate/comms/dispatcher.py @@ -22,7 +22,7 @@ ) from frigate.models import Event, Previews, Recordings, ReviewSegment from frigate.ptz.onvif import OnvifCommandEnum, OnvifController -from frigate.types import ModelStatusTypesEnum +from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum from frigate.util.object import get_camera_regions_grid from frigate.util.services import restart_frigate @@ -137,8 +137,14 @@ def handle_update_event_description(): event.data["description"] = payload["description"] event.save() self.publish( - "event_update", - json.dumps({"id": event.id, "description": event.data["description"]}), + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.description, + "id": event.id, + "description": event.data["description"], + } + ), ) def handle_update_model_state(): diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 12c8bac725..dde8f8df42 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -24,6 +24,7 @@ from frigate.events.types import EventTypeEnum from frigate.genai import get_genai_client from frigate.models import Event +from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import serialize from frigate.util.image import SharedMemoryFrameManager, calculate_region @@ -287,7 +288,11 @@ def _embed_description(self, event: Event, thumbnails: list[bytes]) -> None: # fire and forget description update self.requestor.send_data( UPDATE_EVENT_DESCRIPTION, - {"id": event.id, "description": description}, + { + "type": TrackedObjectUpdateTypesEnum.description, + "id": event.id, + "description": description, + }, ) # Embed the description diff --git a/frigate/types.py b/frigate/types.py index 3e6ad46ccf..11ab31238c 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -19,3 +19,7 @@ class ModelStatusTypesEnum(str, Enum): downloading = "downloading" downloaded = "downloaded" error = "error" + + +class TrackedObjectUpdateTypesEnum(str, Enum): + description = "description" diff --git a/web/src/api/ws.tsx b/web/src/api/ws.tsx index c7bb74095b..9b8924d1b0 100644 --- a/web/src/api/ws.tsx +++ b/web/src/api/ws.tsx @@ -407,9 +407,9 @@ export function useImproveContrast(camera: string): { return { payload: payload as ToggleableSetting, send }; } -export function useEventUpdate(): { payload: string } { +export function useTrackedObjectUpdate(): { payload: string } { const { value: { payload }, - } = useWs("event_update", ""); + } = useWs("tracked_object_update", ""); return useDeepMemo(JSON.parse(payload as string)); } diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index 5eca9a934b..f7af31606e 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -309,7 +309,7 @@ function ObjectDetailsTab({ return undefined; } - if (search.sub_label) { + if (search.sub_label && search.data?.sub_label_score) { return Math.round((search.data?.sub_label_score ?? 0) * 100); } else { return undefined; diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index 711666807f..2bf2bb0223 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -1,6 +1,6 @@ import { useEmbeddingsReindexProgress, - useEventUpdate, + useTrackedObjectUpdate, useModelState, } from "@/api/ws"; import ActivityIndicator from "@/components/indicators/activity-indicator"; @@ -227,15 +227,15 @@ export default function Explore() { // mutation and revalidation - const eventUpdate = useEventUpdate(); + const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { - if (eventUpdate) { + if (trackedObjectUpdate) { mutate(); } // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps - }, [eventUpdate]); + }, [trackedObjectUpdate]); // embeddings reindex progress diff --git a/web/src/views/explore/ExploreView.tsx b/web/src/views/explore/ExploreView.tsx index f37c374536..ea9c3cbeff 100644 --- a/web/src/views/explore/ExploreView.tsx +++ b/web/src/views/explore/ExploreView.tsx @@ -15,7 +15,7 @@ import { SearchResult } from "@/types/search"; import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator"; import useImageLoaded from "@/hooks/use-image-loaded"; import ActivityIndicator from "@/components/indicators/activity-indicator"; -import { useEventUpdate } from "@/api/ws"; +import { useTrackedObjectUpdate } from "@/api/ws"; import { isEqual } from "lodash"; import TimeAgo from "@/components/dynamic/TimeAgo"; import SearchResultActions from "@/components/menu/SearchResultActions"; @@ -72,13 +72,13 @@ export default function ExploreView({ }, {}); }, [events]); - const eventUpdate = useEventUpdate(); + const trackedObjectUpdate = useTrackedObjectUpdate(); useEffect(() => { mutate(); // mutate / revalidate when event description updates come in // eslint-disable-next-line react-hooks/exhaustive-deps - }, [eventUpdate]); + }, [trackedObjectUpdate]); // update search detail when results change