Implement folder specific search for datasets

CHANGELOG.unreleased.md

@@ -12,6 +12,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 
 ### Added
 - Added a new datasets tab to the dashboard which supports managing datasets in folders. Folders can be organized hierarchically and datasets can be moved into these folders. Selecting a dataset will show dataset details in a sidebar. [#6591](https://github.com/scalableminds/webknossos/pull/6591)
+- Added the option to search a specific folder in the new datasets tab. [#6677](https://github.com/scalableminds/webknossos/pull/6677)
 
 ### Changed
 - webKnossos is now able to recover from a lost webGL context. [#6663](https://github.com/scalableminds/webknossos/pull/6663)

app/controllers/DataSetController.scala

@@ -178,6 +178,10 @@ class DataSetController @Inject()(userService: UserService,
       uploaderId: Option[String],
       @ApiParam(value = "Optional filtering: List only datasets in the folder with this id")
       folderId: Option[String],
+      @ApiParam(
+        value =
+          "Optional filtering: If a folderId was specified, this parameter controls whether subfolders should be considered, too (default: false)")
+      recursive: Option[Boolean],
       @ApiParam(value = "Optional filtering: List only datasets with names matching this search query")
       searchQuery: Option[String],
       @ApiParam(value = "Optional limit, return only the first n matching datasets.")
@@ -210,7 +214,8 @@ class DataSetController @Inject()(userService: UserService,
     ) { filter =>
       for {
         folderIdValidated <- Fox.runOptional(folderId)(ObjectId.fromString)
-        dataSets <- dataSetDAO.findAllWithSearch(folderIdValidated, searchQuery) ?~> "dataSet.list.failed"
+        dataSets <- dataSetDAO
+          .findAllWithSearch(folderIdValidated, searchQuery, recursive.getOrElse(false)) ?~> "dataSet.list.failed"
         filtered <- filter.applyOn(dataSets)
         limited = limit.map(l => filtered.take(l)).getOrElse(filtered)
         js <- listGrouped(limited, request.identity) ?~> "dataSet.list.failed"

app/models/binary/DataSet.scala

@@ -163,11 +163,18 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
       parsed <- parseFirst(r, id)
     } yield parsed
 
-  def findAllWithSearch(folderIdOpt: Option[ObjectId], searchQuery: Option[String])(
+  def findAllWithSearch(folderIdOpt: Option[ObjectId], searchQuery: Option[String], recursive: Boolean = false)(
       implicit ctx: DBAccessContext): Fox[List[DataSet]] =
     for {
       accessQuery <- readAccessQuery
-      folderPredicate = folderIdOpt.map(folderId => s"_folder = '$folderId'").getOrElse("true")
+      folderPredicate = folderIdOpt
+        .map(
+          folderId =>
+            if (recursive)
+              s"_folder in (select _descendant FROM webknossos.folder_paths fp WHERE fp._ancestor = '$folderId')"
+            else
+              s"_folder = '$folderId'")
+        .getOrElse("true")
       searchPredicate = buildSearchPredicate(searchQuery)
       r <- run(sql"""SELECT #$columns
               FROM #$existingCollectionName

conf/webknossos.latest.routes

@@ -66,7 +66,7 @@ GET           /teams/:id/progressOverview
 # DataSets
 POST          /datasets/:organizationName/:dataSetName/createExplorational              controllers.AnnotationController.createExplorational(organizationName: String, dataSetName: String)
 GET           /datasets/:organizationName/:dataSetName/sandbox/:typ                     controllers.AnnotationController.getSandbox(organizationName: String, dataSetName: String, typ: String, sharingToken: Option[String])
-GET           /datasets                                                                 controllers.DataSetController.list(isActive: Option[Boolean], isUnreported: Option[Boolean], isEditable: Option[Boolean], organizationName: Option[String], onlyMyOrganization: Option[Boolean], uploaderId: Option[String], folderId: Option[String], searchQuery: Option[String], limit: Option[Int])
+GET           /datasets                                                                 controllers.DataSetController.list(isActive: Option[Boolean], isUnreported: Option[Boolean], isEditable: Option[Boolean], organizationName: Option[String], onlyMyOrganization: Option[Boolean], uploaderId: Option[String], folderId: Option[String], recursive: Option[Boolean], searchQuery: Option[String], limit: Option[Int])
 POST          /datasets                                                                 controllers.DataSetController.create(typ: String)
 POST          /datasets/exploreRemote                                                   controllers.DataSetController.exploreRemoteDataset
 GET           /datasets/disambiguate/:dataSetName/toNew                                 controllers.DataSetController.getOrganizationForDataSet(dataSetName: String)

frontend/javascripts/admin/admin_rest_api.ts

@@ -1025,13 +1025,14 @@ export async function getDatasets(
   isUnreported: boolean | null | undefined = null,
   folderId: string | null = null,
   searchQuery: string | null = null,
+  searchRecursively: boolean | null = null,
   limit: number | null = null,
 ): Promise<Array<APIMaybeUnimportedDataset>> {
   const params = new URLSearchParams();
   if (isUnreported != null) {
     params.append("isUnreported", String(isUnreported));
   }
-  if (folderId != null) {
+  if (folderId != null && folderId !== "") {
     params.append("folderId", folderId);
   }
   if (searchQuery != null) {
@@ -1040,6 +1041,9 @@ export async function getDatasets(
   if (limit != null) {
     params.append("limit", String(limit));
   }
+  if (searchRecursively != null) {
+    params.append("recursive", searchRecursively ? "true" : "false");
+  }
 
   const datasets = await Request.receiveJSON(`/api/datasets?${params}`);
   assertResponseLimit(datasets);

frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx

@@ -36,7 +36,11 @@ import { ContextMenuContext, GenericContextMenuContainer } from "oxalis/view/con
 import Shortcut from "libs/shortcut_component";
 import { MINIMUM_SEARCH_QUERY_LENGTH } from "dashboard/dataset/queries";
 import { useSelector } from "react-redux";
+import { DatasetCacheContextValue } from "dashboard/dataset/dataset_cache_provider";
+import { DatasetCollectionContextValue } from "dashboard/dataset/dataset_collection_context";
+import { Unicode } from "oxalis/constants";
 
+const { ThinSpace } = Unicode;
 const { Column } = Table;
 const typeHint: APIMaybeUnimportedDataset[] = [];
 const useLruRank = true;
@@ -58,6 +62,7 @@ type Props = {
   onSelectDataset?: (dataset: APIMaybeUnimportedDataset | null) => void;
   selectedDataset?: APIMaybeUnimportedDataset | null | undefined;
   hideDetailsColumns?: boolean;
+  context: DatasetCacheContextValue | DatasetCollectionContextValue;
 };
 type State = {
   prevSearchQuery: string;
@@ -489,7 +494,12 @@ class DatasetTable extends React.PureComponent<Props, State> {
                   {dataset.name}
                 </Link>
                 <br />
-                <Tag color={stringToColor(dataset.dataStore.name)}>{dataset.dataStore.name}</Tag>
+
+                {"getBreadcrumbs" in this.props.context ? (
+                  <BreadcrumbsTag parts={this.props.context.getBreadcrumbs(dataset)} />
+                ) : (
+                  <Tag color={stringToColor(dataset.dataStore.name)}>{dataset.dataStore.name}</Tag>
+                )}
               </>
             )}
           />
@@ -725,4 +735,26 @@ export function TeamTags({
   );
 }
 
+function formatPath(parts: string[]) {
+  return parts.join(`${ThinSpace}/${ThinSpace}`);
+}
+
+function BreadcrumbsTag({ parts: allParts }: { parts: string[] | null }) {
+  if (allParts == null) {
+    return null;
+  }
+  let parts;
+  if (allParts.length <= 4) {
+    parts = allParts;
+  } else {
+    parts = [...allParts.slice(0, 2), "...", ...allParts.slice(-2)];
+  }
+
+  return (
+    <Tooltip title={`This folder is located in ${formatPath(allParts)}.`}>
+      <Tag>{formatPath(parts)}</Tag>
+    </Tooltip>
+  );
+}
+
 export default DatasetTable;

frontend/javascripts/dashboard/dataset/dataset_collection_context.tsx

@@ -5,7 +5,7 @@ import { getDatastores, triggerDatasetCheck } from "admin/admin_rest_api";
 import UserLocalStorage from "libs/user_local_storage";
 import _ from "lodash";
 import {
-  useFolderTreeQuery,
+  useFolderHierarchyQuery,
   useDatasetsInFolderQuery,
   useDatasetSearchQuery,
   useCreateFolderMutation,
@@ -38,8 +38,13 @@ export type DatasetCollectionContextValue = {
   supportsFolders: true;
   globalSearchQuery: string | null;
   setGlobalSearchQuery: (val: string | null) => void;
+  folderIdForSearch: string | null;
+  setFolderIdForSearch: (val: string | null) => void;
+  searchRecursively: boolean;
+  setSearchRecursively: (val: boolean) => void;
+  getBreadcrumbs: (dataset: APIMaybeUnimportedDataset) => string[] | null;
   queries: {
-    folderTreeQuery: ReturnType<typeof useFolderTreeQuery>;
+    folderHierarchyQuery: ReturnType<typeof useFolderHierarchyQuery>;
     datasetsInFolderQuery: ReturnType<typeof useDatasetsInFolderQuery>;
     datasetSearchQuery: ReturnType<typeof useDatasetSearchQuery>;
     createFolderMutation: ReturnType<typeof useCreateFolderMutation>;
@@ -78,6 +83,7 @@ export default function DatasetCollectionContextProvider({
   const { data: folder } = useFolderQuery(activeFolderId);
 
   const [globalSearchQuery, setGlobalSearchQueryInner] = useState<string | null>(null);
+  const [folderIdForSearch, setFolderIdForSearch] = useState<string | null>(null);
   const setGlobalSearchQuery = useCallback(
     (value: string | null) => {
       // Empty string should be handled as null
@@ -88,6 +94,7 @@ export default function DatasetCollectionContextProvider({
     },
     [setGlobalSearchQueryInner, setActiveFolderId],
   );
+  const [searchRecursively, setSearchRecursively] = useState<boolean>(true);
 
   // Clear search query if active folder changes.
   useEffect(() => {
@@ -97,7 +104,14 @@ export default function DatasetCollectionContextProvider({
   }, [activeFolderId]);
 
   // Keep url GET parameters in sync with search and active folder
-  useManagedUrlParams(setGlobalSearchQuery, setActiveFolderId, globalSearchQuery, activeFolderId);
+  useManagedUrlParams(
+    setGlobalSearchQuery,
+    setActiveFolderId,
+    globalSearchQuery,
+    activeFolderId,
+    folderIdForSearch,
+    setFolderIdForSearch,
+  );
 
   useEffect(() => {
     // Persist last active folder to localStorage. We
@@ -109,9 +123,13 @@ export default function DatasetCollectionContextProvider({
     }
   }, [folder, activeFolderId]);
 
-  const folderTreeQuery = useFolderTreeQuery();
+  const folderHierarchyQuery = useFolderHierarchyQuery();
   const datasetsInFolderQuery = useDatasetsInFolderQuery(activeFolderId);
-  const datasetSearchQuery = useDatasetSearchQuery(globalSearchQuery);
+  const datasetSearchQuery = useDatasetSearchQuery(
+    globalSearchQuery,
+    folderIdForSearch,
+    searchRecursively,
+  );
   const createFolderMutation = useCreateFolderMutation();
   const deleteFolderMutation = useDeleteFolderMutation();
   const updateFolderMutation = useUpdateFolderMutation();
@@ -134,10 +152,26 @@ export default function DatasetCollectionContextProvider({
     updateDatasetMutation.mutateAsync([dataset, dataset.folderId]);
   }
 
+  const getBreadcrumbs = (dataset: APIMaybeUnimportedDataset) => {
+    if (folderHierarchyQuery.data?.itemById == null) {
+      return null;
+    }
+    const { itemById } = folderHierarchyQuery.data;
+
+    let currentFolder = itemById[dataset.folderId];
+    const breadcrumbs = [currentFolder.title];
+    while (currentFolder?.parent != null) {
+      currentFolder = itemById[currentFolder.parent];
+      breadcrumbs.unshift(currentFolder.title);
+    }
+
+    return breadcrumbs;
+  };
+
   const isLoading =
     (globalSearchQuery
       ? datasetSearchQuery.isFetching
-      : folderTreeQuery.isLoading ||
+      : folderHierarchyQuery.isLoading ||
         datasetsInFolderQuery.isFetching ||
         datasetsInFolderQuery.isRefetching) || isMutating;
 
@@ -152,6 +186,7 @@ export default function DatasetCollectionContextProvider({
       activeFolderId,
       setActiveFolderId,
       isChecking,
+      getBreadcrumbs,
       checkDatasets: async () => {
         if (isChecking) {
           console.warn("Ignore second rechecking request, since a recheck is already in progress");
@@ -175,8 +210,12 @@ export default function DatasetCollectionContextProvider({
       },
       globalSearchQuery,
       setGlobalSearchQuery,
+      folderIdForSearch,
+      setFolderIdForSearch,
+      searchRecursively,
+      setSearchRecursively,
       queries: {
-        folderTreeQuery,
+        folderHierarchyQuery,
         datasetsInFolderQuery,
         datasetSearchQuery,
         createFolderMutation,
@@ -195,14 +234,19 @@ export default function DatasetCollectionContextProvider({
       updateCachedDataset,
       activeFolderId,
       setActiveFolderId,
-      folderTreeQuery,
+      folderHierarchyQuery,
       datasetsInFolderQuery,
       datasetSearchQuery,
+      searchRecursively,
+      setSearchRecursively,
       createFolderMutation,
       deleteFolderMutation,
       updateFolderMutation,
       moveFolderMutation,
       updateDatasetMutation,
+      globalSearchQuery,
+      folderIdForSearch,
+      setFolderIdForSearch,
     ],
   );
 
@@ -216,6 +260,8 @@ function useManagedUrlParams(
   setActiveFolderId: React.Dispatch<React.SetStateAction<string | null>>,
   globalSearchQuery: string | null,
   activeFolderId: string | null,
+  folderIdForSearch: string | null,
+  setFolderIdForSearch: React.Dispatch<React.SetStateAction<string | null>>,
 ) {
   const { data: folder } = useFolderQuery(activeFolderId);
 
@@ -226,6 +272,10 @@ function useManagedUrlParams(
     if (query) {
       setGlobalSearchQuery(query);
     }
+    const folderId = params.get("folderId");
+    if (folderId) {
+      setFolderIdForSearch(folderId);
+    }
 
     const folderSpecifier = _.last(location.pathname.split("/"));
 
@@ -238,14 +288,19 @@ function useManagedUrlParams(
     }
   }, []);
 
-  // Update query
+  // Update query and folderIdForSearch
   useEffect(() => {
     const params = new URLSearchParams(location.search);
     if (globalSearchQuery) {
       params.set("query", globalSearchQuery);
     } else {
       params.delete("query");
     }
+    if (globalSearchQuery && folderIdForSearch) {
+      params.set("folderId", folderIdForSearch);
+    } else {
+      params.delete("folderId");
+    }
     const paramStr = params.toString();
 
     // Don't use useHistory because this would lose the input search
@@ -255,7 +310,7 @@ function useManagedUrlParams(
       "",
       `${location.pathname}${paramStr === "" ? "" : "?"}${paramStr}`,
     );
-  }, [globalSearchQuery]);
+  }, [globalSearchQuery, folderIdForSearch]);
 
   // Update folderId
   useEffect(() => {