Find similar footage from review item snapshots (#13662)

* Find similar footage from review item snapshots * Include confidence score for similarity search
blakeblackshear · Sep 10, 2024 · e016bd6 · e016bd6
1 parent d2588d9
commit e016bd6
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 6 deletions.
diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx
@@ -25,14 +25,15 @@ import { cn } from "@/lib/utils";
 import { FrigatePlusDialog } from "../dialog/FrigatePlusDialog";
 import ObjectLifecycle from "./ObjectLifecycle";
 import Chip from "@/components/indicators/Chip";
-import { FaDownload } from "react-icons/fa";
+import { FaDownload, FaImages } from "react-icons/fa";
 import FrigatePlusIcon from "@/components/icons/FrigatePlusIcon";
 import { FaArrowsRotate } from "react-icons/fa6";
 import {
   Tooltip,
   TooltipContent,
   TooltipTrigger,
 } from "@/components/ui/tooltip";
+import { useNavigate } from "react-router-dom";
 
 type ReviewDetailDialogProps = {
   review?: ReviewSegment;
@@ -234,6 +235,8 @@ function EventItem({
 
   const [hovered, setHovered] = useState(isMobile);
 
+  const navigate = useNavigate();
+
   return (
     <>
       <div
@@ -328,6 +331,27 @@ function EventItem({
                   <TooltipContent>View Object Lifecycle</TooltipContent>
                 </Tooltip>
               )}
+
+              {event.has_snapshot && config?.semantic_search.enabled && (
+                <Tooltip>
+                  <TooltipTrigger>
+                    <Chip
+                      className="cursor-pointer rounded-md bg-gray-500 bg-gradient-to-br from-gray-400 to-gray-500"
+                      onClick={() => {
+                        const similaritySearchParams = new URLSearchParams({
+                          search_type: "similarity",
+                          event_id: event.id,
+                        }).toString();
+
+                        navigate(`/search?${similaritySearchParams}`);
+                      }}
+                    >
+                      <FaImages className="size-4 text-white" />
+                    </Chip>
+                  </TooltipTrigger>
+                  <TooltipContent>Find Similar</TooltipContent>
+                </Tooltip>
+              )}
             </div>
           </div>
         )}

diff --git a/web/src/pages/Search.tsx b/web/src/pages/Search.tsx
@@ -3,7 +3,11 @@ import { useCameraPreviews } from "@/hooks/use-camera-previews";
 import { useOverlayState } from "@/hooks/use-overlay-state";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { RecordingStartingPoint } from "@/types/record";
-import { SearchFilter, SearchResult } from "@/types/search";
+import {
+  PartialSearchResult,
+  SearchFilter,
+  SearchResult,
+} from "@/types/search";
 import { TimeRange } from "@/types/timeline";
 import { RecordingView } from "@/views/recording/RecordingView";
 import SearchView from "@/views/search/SearchView";
@@ -38,7 +42,27 @@ export default function Search() {
 
   // search api
 
-  const [similaritySearch, setSimilaritySearch] = useState<SearchResult>();
+  const [similaritySearch, setSimilaritySearch] =
+    useState<PartialSearchResult>();
+
+  useEffect(() => {
+    if (
+      config?.semantic_search.enabled &&
+      searchSearchParams["search_type"] == "similarity" &&
+      searchSearchParams["event_id"]?.length != 0 &&
+      searchFilter
+    ) {
+      setSimilaritySearch({
+        id: searchSearchParams["event_id"],
+      });
+
+      // remove event id from url params
+      const { event_id: _event_id, ...newFilter } = searchFilter;
+      setSearchFilter(newFilter);
+    }
+    // only run similarity search with event_id in the url when coming from review
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
 
   useEffect(() => {
     if (similaritySearch) {

diff --git a/web/src/types/search.ts b/web/src/types/search.ts
@@ -25,6 +25,9 @@ export type SearchResult = {
   };
 };
 
+
+export type PartialSearchResult = Partial<SearchResult> & { id: string };
+
 export type SearchFilter = {
   cameras?: string[];
   labels?: string[];
@@ -33,4 +36,5 @@ export type SearchFilter = {
   before?: number;
   after?: number;
   search_type?: SearchSource[];
+  event_id?: string;
 };
diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx
@@ -13,7 +13,11 @@ import {
 import { cn } from "@/lib/utils";
 import { FrigateConfig } from "@/types/frigateConfig";
 import { Preview } from "@/types/preview";
-import { SearchFilter, SearchResult } from "@/types/search";
+import {
+  PartialSearchResult,
+  SearchFilter,
+  SearchResult,
+} from "@/types/search";
 import { useCallback, useMemo, useState } from "react";
 import { isMobileOnly } from "react-device-detect";
 import { LuImage, LuSearchX, LuText, LuXCircle } from "react-icons/lu";
@@ -26,7 +30,7 @@ type SearchViewProps = {
   searchResults?: SearchResult[];
   allPreviews?: Preview[];
   isLoading: boolean;
-  similaritySearch?: SearchResult;
+  similaritySearch?: PartialSearchResult;
   setSearch: (search: string) => void;
   setSimilaritySearch: (search: SearchResult) => void;
   onUpdateFilter: (filter: SearchFilter) => void;
@@ -186,7 +190,7 @@ export default function SearchView({
                       scrollLock={false}
                       onClick={onSelectSearch}
                     />
-                    {searchTerm && (
+                    {(searchTerm || similaritySearch) && (
                       <div className={cn("absolute right-2 top-2 z-40")}>
                         <Tooltip>
                           <TooltipTrigger>