Merge branch 'master' of github.com:scalableminds/webknossos into pri…

…cing * 'master' of github.com:scalableminds/webknossos: fixes infinite loop in getLineCount (#6689) Catch Malformed Json Exceptions (#6691) Implement folder-specific search for datasets (#6677) Fix s3fs region access, s3 url styles (#6679) Swagger annotation for shortLinkByKey (#6682) Improve layout of dashboard Provide valid JSON schema (#6642)
scalableminds · Dec 12, 2022 · 4805fbe · 4805fbe
2 parents c9d9cb3 + 5f04119
commit 4805fbe
Show file tree

Hide file tree

Showing 25 changed files with 530 additions and 249 deletions.
diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md
@@ -13,6 +13,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 ### Added
 - Added sign in via OIDC. [#6534](https://github.com/scalableminds/webknossos/pull/6534)
 - Added a new datasets tab to the dashboard which supports managing datasets in folders. Folders can be organized hierarchically and datasets can be moved into these folders. Selecting a dataset will show dataset details in a sidebar. [#6591](https://github.com/scalableminds/webknossos/pull/6591)
+- Added the option to search a specific folder in the new datasets tab. [#6677](https://github.com/scalableminds/webknossos/pull/6677)
 
 ### Changed
 - The log viewer in the Voxelytics workflow reporting now uses a virtualized list. [#6579](https://github.com/scalableminds/webknossos/pull/6579)
@@ -31,6 +32,10 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Fixed a bug where it was possible to create invalid an state by deleting teams that are referenced elsewhere. [6664](https://github.com/scalableminds/webknossos/pull/6664)
 - Miscellaneous fixes for the new folder UI. [#6674](https://github.com/scalableminds/webknossos/pull/6674)
 - Fixed import of remote datasets with multiple layers and differing resolution pyramid. #[6670](https://github.com/scalableminds/webknossos/pull/6670)
+- Fixed broken Get-new-Task button in task dashboard. [#6677](https://github.com/scalableminds/webknossos/pull/6677)
+- Fixed access of remote datasets using the Amazon S3 protocol [#6679](https://github.com/scalableminds/webknossos/pull/6679)
+- Fixed a bug in line measurement that would lead to an infinite loop. [#6689](https://github.com/scalableminds/webknossos/pull/6689)
+- Fixed a bug where malformed json files could lead to uncaught exceptions.[#6691](https://github.com/scalableminds/webknossos/pull/6691)
 
 ### Removed
 

diff --git a/app/controllers/DataSetController.scala b/app/controllers/DataSetController.scala
@@ -178,6 +178,10 @@ class DataSetController @Inject()(userService: UserService,
       uploaderId: Option[String],
       @ApiParam(value = "Optional filtering: List only datasets in the folder with this id")
       folderId: Option[String],
+      @ApiParam(
+        value =
+          "Optional filtering: If a folderId was specified, this parameter controls whether subfolders should be considered, too (default: false)")
+      recursive: Option[Boolean],
       @ApiParam(value = "Optional filtering: List only datasets with names matching this search query")
       searchQuery: Option[String],
       @ApiParam(value = "Optional limit, return only the first n matching datasets.")
@@ -210,7 +214,8 @@ class DataSetController @Inject()(userService: UserService,
     ) { filter =>
       for {
         folderIdValidated <- Fox.runOptional(folderId)(ObjectId.fromString)
-        dataSets <- dataSetDAO.findAllWithSearch(folderIdValidated, searchQuery) ?~> "dataSet.list.failed"
+        dataSets <- dataSetDAO
+          .findAllWithSearch(folderIdValidated, searchQuery, recursive.getOrElse(false)) ?~> "dataSet.list.failed"
         filtered <- filter.applyOn(dataSets)
         limited = limit.map(l => filtered.take(l)).getOrElse(filtered)
         js <- listGrouped(limited, request.identity) ?~> "dataSet.list.failed"

diff --git a/app/controllers/ShortLinkController.scala b/app/controllers/ShortLinkController.scala
@@ -2,6 +2,7 @@ package controllers
 
 import com.mohiva.play.silhouette.api.Silhouette
 import com.scalableminds.util.tools.FoxImplicits
+import io.swagger.annotations.{Api, ApiOperation, ApiParam}
 import models.shortlinks.{ShortLink, ShortLinkDAO}
 import oxalis.security.{RandomIDGenerator, WkEnv}
 import play.api.libs.json.Json
@@ -11,12 +12,14 @@ import utils.{ObjectId, WkConf}
 import javax.inject.Inject
 import scala.concurrent.ExecutionContext
 
+@Api
 class ShortLinkController @Inject()(shortLinkDAO: ShortLinkDAO, sil: Silhouette[WkEnv], wkConf: WkConf)(
     implicit ec: ExecutionContext,
     val bodyParsers: PlayBodyParsers)
     extends Controller
     with FoxImplicits {
 
+  @ApiOperation(hidden = true, value = "")
   def create: Action[String] = sil.SecuredAction.async(validateJson[String]) { implicit request =>
     val longLink = request.body
     val _id = ObjectId.generate
@@ -28,7 +31,12 @@ class ShortLinkController @Inject()(shortLinkDAO: ShortLinkDAO, sil: Silhouette[
     } yield Ok(Json.toJson(inserted))
   }
 
-  def getByKey(key: String): Action[AnyContent] = Action.async { implicit request =>
+  @ApiOperation(value = "Information about a short link, including the original long link.",
+                nickname = "shortLinkByKey")
+  def getByKey(
+      @ApiParam(value = "key of the shortLink, this is the short random string identifying the link.",
+                example = "aU7yv5Aja99T0829")
+      key: String): Action[AnyContent] = Action.async { implicit request =>
     for {
       shortLink <- shortLinkDAO.findOneByKey(key)
     } yield Ok(Json.toJson(shortLink))

diff --git a/app/models/binary/DataSet.scala b/app/models/binary/DataSet.scala
@@ -163,11 +163,16 @@ class DataSetDAO @Inject()(sqlClient: SQLClient,
       parsed <- parseFirst(r, id)
     } yield parsed
 
-  def findAllWithSearch(folderIdOpt: Option[ObjectId], searchQuery: Option[String])(
+  def findAllWithSearch(folderIdOpt: Option[ObjectId], searchQuery: Option[String], includeSubfolders: Boolean = false)(
       implicit ctx: DBAccessContext): Fox[List[DataSet]] =
     for {
       accessQuery <- readAccessQuery
-      folderPredicate = folderIdOpt.map(folderId => s"_folder = '$folderId'").getOrElse("true")
+      folderPredicate = folderIdOpt match {
+        case Some(folderId) if (includeSubfolders) =>
+          s"_folder IN (select _descendant FROM webknossos.folder_paths fp WHERE fp._ancestor = '$folderId')"
+        case Some(folderId) => s"_folder = '$folderId'"
+        case None           => "true"
+      }
       searchPredicate = buildSearchPredicate(searchQuery)
       r <- run(sql"""SELECT #$columns
               FROM #$existingCollectionName

diff --git a/app/models/binary/explore/RemoteLayerExplorer.scala b/app/models/binary/explore/RemoteLayerExplorer.scala
@@ -26,7 +26,7 @@ trait RemoteLayerExplorer extends FoxImplicits {
   protected def parseJsonFromPath[T: Reads](path: Path): Fox[T] =
     for {
       fileAsString <- tryo(new String(Files.readAllBytes(path), StandardCharsets.UTF_8)).toFox ?~> "Failed to read remote file"
-      parsed <- JsonHelper.parseAndValidateJson[T](fileAsString) ?~> "Failed to validate json against data schema"
+      parsed <- JsonHelper.parseAndValidateJson[T](fileAsString) ?~> "Failed to parse or validate json against data schema"
     } yield parsed
 
   protected def looksLikeSegmentationLayer(layerName: String, elementClass: ElementClass.Value): Boolean =

diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes
@@ -69,7 +69,7 @@ GET           /teams/:id/progressOverview
 # DataSets
 POST          /datasets/:organizationName/:dataSetName/createExplorational              controllers.AnnotationController.createExplorational(organizationName: String, dataSetName: String)
 GET           /datasets/:organizationName/:dataSetName/sandbox/:typ                     controllers.AnnotationController.getSandbox(organizationName: String, dataSetName: String, typ: String, sharingToken: Option[String])
-GET           /datasets                                                                 controllers.DataSetController.list(isActive: Option[Boolean], isUnreported: Option[Boolean], isEditable: Option[Boolean], organizationName: Option[String], onlyMyOrganization: Option[Boolean], uploaderId: Option[String], folderId: Option[String], searchQuery: Option[String], limit: Option[Int])
+GET           /datasets                                                                 controllers.DataSetController.list(isActive: Option[Boolean], isUnreported: Option[Boolean], isEditable: Option[Boolean], organizationName: Option[String], onlyMyOrganization: Option[Boolean], uploaderId: Option[String], folderId: Option[String], includeSubfolders: Option[Boolean], searchQuery: Option[String], limit: Option[Int])
 POST          /datasets                                                                 controllers.DataSetController.create(typ: String)
 POST          /datasets/exploreRemote                                                   controllers.DataSetController.exploreRemoteDataset
 GET           /datasets/disambiguate/:dataSetName/toNew                                 controllers.DataSetController.getOrganizationForDataSet(dataSetName: String)

diff --git a/docs/data_formats.md b/docs/data_formats.md
@@ -349,25 +349,23 @@ my_dataset                      # Dataset root
 All segment IDs belonging to the same super-voxel need to be listed in an array:  
 ```
 {
-  {
-    "name": "astrocytes",
-    "classes": [
-      [
-        69381,
-        69445,
-        138248
-      ],
-      [
-        138307,
-        343831
-      ],
-      [
-        348348,
-        132432,
-        387433,
-        338330
-      ]
+  "name": "astrocytes",
+  "classes": [
+    [
+      69381,
+      69445,
+      138248
+    ],
+    [
+      138307,
+      343831
+    ],
+    [
+      348348,
+      132432,
+      387433,
+      338330
     ]
-  }
+  ]
 }
 ```
diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts
@@ -1026,13 +1026,14 @@ export async function getDatasets(
   isUnreported: boolean | null | undefined = null,
   folderId: string | null = null,
   searchQuery: string | null = null,
+  includeSubfolders: boolean | null = null,
   limit: number | null = null,
 ): Promise<Array<APIMaybeUnimportedDataset>> {
   const params = new URLSearchParams();
   if (isUnreported != null) {
     params.append("isUnreported", String(isUnreported));
   }
-  if (folderId != null) {
+  if (folderId != null && folderId !== "") {
     params.append("folderId", folderId);
   }
   if (searchQuery != null) {
@@ -1041,6 +1042,9 @@ export async function getDatasets(
   if (limit != null) {
     params.append("limit", String(limit));
   }
+  if (includeSubfolders != null) {
+    params.append("includeSubfolders", includeSubfolders ? "true" : "false");
+  }
 
   const datasets = await Request.receiveJSON(`/api/datasets?${params}`);
   assertResponseLimit(datasets);

diff --git a/frontend/javascripts/admin/voxelytics/log_tab.tsx b/frontend/javascripts/admin/voxelytics/log_tab.tsx
@@ -53,18 +53,20 @@ function findBreakableCharFromRight(str: string, position: number): number {
   }
   return -1;
 }
+
 function getLineCount(str: string, wrapLength: number): number {
   // Inspired by https://stackoverflow.com/a/857770
-  const trimmedStr = str.trim();
-  if (trimmedStr.length <= wrapLength) {
-    return 1;
-  } else {
+  let trimmedStr = str.trim();
+  let counter = 1;
+  while (trimmedStr.length > wrapLength) {
     let splitIdx = findBreakableCharFromRight(trimmedStr, wrapLength);
-    if (splitIdx === -1) {
+    if (splitIdx < 1) {
       splitIdx = wrapLength;
     }
-    return 1 + getLineCount(trimmedStr.substring(splitIdx), wrapLength);
+    trimmedStr = trimmedStr.substring(splitIdx).trim();
+    counter++;
   }
+  return counter;
 }
 
 function LogContent({

diff --git a/frontend/javascripts/admin/welcome_ui.tsx b/frontend/javascripts/admin/welcome_ui.tsx
@@ -6,7 +6,7 @@ import {
   PlusCircleOutlined,
   RightOutlined,
 } from "@ant-design/icons";
-import { Tooltip } from "antd";
+import { Button, Tooltip } from "antd";
 import * as React from "react";
 import { Link } from "react-router-dom";
 import { isUserAdminOrTeamManager, isUserAdminOrDatasetManager } from "libs/utils";
@@ -73,7 +73,10 @@ export const WhatsNextHeader = ({ activeUser, onDismiss }: WhatsNextHeaderProps)
         }}
       >
         <Tooltip title="Don't show this again" placement="left">
-          <CloseOutlined onClick={onDismiss} />
+          <Button type="text" onClick={onDismiss}>
+            Close
+            <CloseOutlined />
+          </Button>
         </Tooltip>
       </div>
       <div className="welcome-header-content">

diff --git a/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx b/frontend/javascripts/dashboard/advanced_dataset/dataset_table.tsx
@@ -1,4 +1,4 @@
-import { PlusOutlined, WarningOutlined } from "@ant-design/icons";
+import { FolderOpenOutlined, PlusOutlined, WarningOutlined } from "@ant-design/icons";
 import { Link } from "react-router-dom";
 import { Dropdown, Table, Tag, Tooltip } from "antd";
 import type {
@@ -36,7 +36,11 @@ import { ContextMenuContext, GenericContextMenuContainer } from "oxalis/view/con
 import Shortcut from "libs/shortcut_component";
 import { MINIMUM_SEARCH_QUERY_LENGTH } from "dashboard/dataset/queries";
 import { useSelector } from "react-redux";
+import { DatasetCacheContextValue } from "dashboard/dataset/dataset_cache_provider";
+import { DatasetCollectionContextValue } from "dashboard/dataset/dataset_collection_context";
+import { Unicode } from "oxalis/constants";
 
+const { ThinSpace } = Unicode;
 const { Column } = Table;
 const typeHint: APIMaybeUnimportedDataset[] = [];
 const useLruRank = true;
@@ -58,6 +62,7 @@ type Props = {
   onSelectDataset?: (dataset: APIMaybeUnimportedDataset | null) => void;
   selectedDataset?: APIMaybeUnimportedDataset | null | undefined;
   hideDetailsColumns?: boolean;
+  context: DatasetCacheContextValue | DatasetCollectionContextValue;
 };
 type State = {
   prevSearchQuery: string;
@@ -489,7 +494,12 @@ class DatasetTable extends React.PureComponent<Props, State> {
                   {dataset.name}
                 </Link>
                 <br />
-                <Tag color={stringToColor(dataset.dataStore.name)}>{dataset.dataStore.name}</Tag>
+
+                {"getBreadcrumbs" in this.props.context ? (
+                  <BreadcrumbsTag parts={this.props.context.getBreadcrumbs(dataset)} />
+                ) : (
+                  <Tag color={stringToColor(dataset.dataStore.name)}>{dataset.dataStore.name}</Tag>
+                )}
               </>
             )}
           />
@@ -725,4 +735,29 @@ export function TeamTags({
   );
 }
 
+function formatPath(parts: string[]) {
+  return parts.join(`${ThinSpace}/${ThinSpace}`);
+}
+
+function BreadcrumbsTag({ parts: allParts }: { parts: string[] | null }) {
+  if (allParts == null) {
+    return null;
+  }
+  let parts;
+  if (allParts.length <= 4) {
+    parts = allParts;
+  } else {
+    parts = [...allParts.slice(0, 2), "...", ...allParts.slice(-2)];
+  }
+
+  return (
+    <Tooltip title={`This dataset is located in ${formatPath(allParts)}.`}>
+      <Tag>
+        <FolderOpenOutlined />
+        {formatPath(parts)}
+      </Tag>
+    </Tooltip>
+  );
+}
+
 export default DatasetTable;
diff --git a/frontend/javascripts/dashboard/dashboard_task_list_view.tsx b/frontend/javascripts/dashboard/dashboard_task_list_view.tsx
@@ -529,7 +529,7 @@ class DashboardTaskListView extends React.PureComponent<Props, State> {
     return (
       <div>
         <TopBar
-          confirmGetNewTask={this.confirmGetNewTask}
+          confirmGetNewTask={() => this.confirmGetNewTask()}
           isAdminView={this.props.isAdminView}
           userId={this.props.userId}
           toggleShowFinished={this.toggleShowFinished}
@@ -589,7 +589,7 @@ function TopBar({
       <AsyncButton
         type="primary"
         icon={<UserAddOutlined />}
-        onClick={() => confirmGetNewTask()}
+        onClick={confirmGetNewTask}
         disabled={isAdminView && userId != null}
       >
         Get a New Task

diff --git a/frontend/javascripts/dashboard/dashboard_view.tsx b/frontend/javascripts/dashboard/dashboard_view.tsx
@@ -295,7 +295,7 @@ class DashboardView extends PureComponent<PropsWithRouter, State> {
     return (
       <NmlUploadZoneContainer onImport={this.uploadNmls} isUpdateAllowed>
         {whatsNextBanner}
-        <div className="container propagate-flex-height">
+        <div className="container propagate-flex-height" style={{ minHeight: "66vh" }}>
           {pricingPlanWarnings}
           {userHeader}
           <DatasetCacheProvider>