Tracked object metadata changes (blakeblackshear#15055)

* add enum and change topic name

* frontend renaming

* docs

* only display sublabel score if it it exists

* remove debug print

docs/docs/integrations/mqtt.md

@@ -94,6 +94,18 @@ Message published for each changed tracked object. The first message is publishe
 }
 ```
 
+### `frigate/tracked_object_update`
+
+Message published for updates to tracked object metadata, for example when GenAI runs and returns a tracked object description.
+
+```json
+{
+  "type": "description",
+  "id": "1607123955.475377-mxklsc",
+  "description": "The car is a red sedan moving away from the camera."
+}
+```
+
 ### `frigate/reviews`
 
 Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. When additional objects are detected or when a zone change occurs, it will publish a, `update` message with the same id. When the review activity has ended a final `end` message is published.

frigate/comms/dispatcher.py

@@ -22,7 +22,7 @@
 )
 from frigate.models import Event, Previews, Recordings, ReviewSegment
 from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
-from frigate.types import ModelStatusTypesEnum
+from frigate.types import ModelStatusTypesEnum, TrackedObjectUpdateTypesEnum
 from frigate.util.object import get_camera_regions_grid
 from frigate.util.services import restart_frigate
 
@@ -137,8 +137,14 @@ def handle_update_event_description():
             event.data["description"] = payload["description"]
             event.save()
             self.publish(
-                "event_update",
-                json.dumps({"id": event.id, "description": event.data["description"]}),
+                "tracked_object_update",
+                json.dumps(
+                    {
+                        "type": TrackedObjectUpdateTypesEnum.description,
+                        "id": event.id,
+                        "description": event.data["description"],
+                    }
+                ),
             )
 
         def handle_update_model_state():

frigate/embeddings/maintainer.py

@@ -24,6 +24,7 @@
 from frigate.events.types import EventTypeEnum
 from frigate.genai import get_genai_client
 from frigate.models import Event
+from frigate.types import TrackedObjectUpdateTypesEnum
 from frigate.util.builtin import serialize
 from frigate.util.image import SharedMemoryFrameManager, calculate_region
 
@@ -287,7 +288,11 @@ def _embed_description(self, event: Event, thumbnails: list[bytes]) -> None:
         # fire and forget description update
         self.requestor.send_data(
             UPDATE_EVENT_DESCRIPTION,
-            {"id": event.id, "description": description},
+            {
+                "type": TrackedObjectUpdateTypesEnum.description,
+                "id": event.id,
+                "description": description,
+            },
         )
 
         # Embed the description

frigate/types.py

@@ -19,3 +19,7 @@ class ModelStatusTypesEnum(str, Enum):
     downloading = "downloading"
     downloaded = "downloaded"
     error = "error"
+
+
+class TrackedObjectUpdateTypesEnum(str, Enum):
+    description = "description"

web/src/api/ws.tsx

@@ -407,9 +407,9 @@ export function useImproveContrast(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
-export function useEventUpdate(): { payload: string } {
+export function useTrackedObjectUpdate(): { payload: string } {
   const {
     value: { payload },
-  } = useWs("event_update", "");
+  } = useWs("tracked_object_update", "");
   return useDeepMemo(JSON.parse(payload as string));
 }

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -309,7 +309,7 @@ function ObjectDetailsTab({
       return undefined;
     }
 
-    if (search.sub_label) {
+    if (search.sub_label && search.data?.sub_label_score) {
       return Math.round((search.data?.sub_label_score ?? 0) * 100);
     } else {
       return undefined;

web/src/pages/Explore.tsx

@@ -1,6 +1,6 @@
 import {
   useEmbeddingsReindexProgress,
-  useEventUpdate,
+  useTrackedObjectUpdate,
   useModelState,
 } from "@/api/ws";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
@@ -227,15 +227,15 @@ export default function Explore() {
 
   // mutation and revalidation
 
-  const eventUpdate = useEventUpdate();
+  const trackedObjectUpdate = useTrackedObjectUpdate();
 
   useEffect(() => {
-    if (eventUpdate) {
+    if (trackedObjectUpdate) {
       mutate();
     }
     // mutate / revalidate when event description updates come in
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [eventUpdate]);
+  }, [trackedObjectUpdate]);
 
   // embeddings reindex progress
 

web/src/views/explore/ExploreView.tsx

@@ -15,7 +15,7 @@ import { SearchResult } from "@/types/search";
 import ImageLoadingIndicator from "@/components/indicators/ImageLoadingIndicator";
 import useImageLoaded from "@/hooks/use-image-loaded";
 import ActivityIndicator from "@/components/indicators/activity-indicator";
-import { useEventUpdate } from "@/api/ws";
+import { useTrackedObjectUpdate } from "@/api/ws";
 import { isEqual } from "lodash";
 import TimeAgo from "@/components/dynamic/TimeAgo";
 import SearchResultActions from "@/components/menu/SearchResultActions";
@@ -72,13 +72,13 @@ export default function ExploreView({
     }, {});
   }, [events]);
 
-  const eventUpdate = useEventUpdate();
+  const trackedObjectUpdate = useTrackedObjectUpdate();
 
   useEffect(() => {
     mutate();
     // mutate / revalidate when event description updates come in
     // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, [eventUpdate]);
+  }, [trackedObjectUpdate]);
 
   // update search detail when results change