Allow Exploring Datasets on Local File System

CHANGELOG.unreleased.md

@@ -15,6 +15,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - Added social media link previews for links to datasets and annotations (only if they are public or if the links contain sharing tokens). [#7331](https://github.com/scalableminds/webknossos/pull/7331)
 - Loading sharded zarr3 datasets is now significantly faster. [#7363](https://github.com/scalableminds/webknossos/pull/7363) and [#7370](https://github.com/scalableminds/webknossos/pull/7370)
 - Higher-dimension coordinates (e.g., for the t axis) are now encoded in the URL, too, so that reloading the page will keep you at your current position. Only relevant for 4D datasets. [#7328](https://github.com/scalableminds/webknossos/pull/7328)
+- WEBKNOSSOS can now also explore datasets on the local file system if enabled in the new config key `datastore.localFolderWhitelist`. [#7389](https://github.com/scalableminds/webknossos/pull/7389)
 
 ### Changed
 - Updated backend code to Scala 2.13, with upgraded Dependencies for optimized performance. [#7327](https://github.com/scalableminds/webknossos/pull/7327)

app/models/dataset/credential/CredentialService.scala

@@ -43,6 +43,8 @@ class CredentialService @Inject()(credentialDAO: CredentialDAO) {
           secret <- credentialSecret
           secretJson <- tryo(Json.parse(secret)).toOption
         } yield GoogleServiceAccountCredential(uri.toString, secretJson, userId.toString, organizationId.toString)
+      case _ =>
+        None
     }
 
   def insertOne(credential: DataVaultCredential)(implicit ec: ExecutionContext): Fox[ObjectId] = {

app/models/dataset/explore/ExploreRemoteLayerService.scala

@@ -19,15 +19,15 @@ import com.scalableminds.webknossos.datastore.models.datasource._
 import com.scalableminds.webknossos.datastore.rpc.RPC
 import com.scalableminds.webknossos.datastore.storage.{DataVaultService, RemoteSourceDescriptor}
 import com.typesafe.scalalogging.LazyLogging
-import models.dataset.{DatasetService, DataStoreDAO, WKRemoteDataStoreClient}
+import models.dataset.{DataStoreDAO, DatasetService, WKRemoteDataStoreClient}
 import models.dataset.credential.CredentialService
 import models.organization.OrganizationDAO
 import models.user.User
 import net.liftweb.common.{Empty, Failure, Full}
 import net.liftweb.util.Helpers.tryo
 import play.api.libs.json.{Json, OFormat}
 import security.{WkEnv, WkSilhouetteEnvironment}
-import utils.ObjectId
+import utils.{ObjectId, WkConf}
 
 import java.net.URI
 import javax.inject.Inject
@@ -57,7 +57,8 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
                                           dataStoreDAO: DataStoreDAO,
                                           datasetService: DatasetService,
                                           wkSilhouetteEnvironment: WkSilhouetteEnvironment,
-                                          rpc: RPC)
+                                          rpc: RPC,
+                                          wkConf: WkConf)
     extends FoxImplicits
     with LazyLogging {
 
@@ -152,7 +153,7 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
 
   private def isPowerOfTwo(x: Double): Boolean = {
     val epsilon = 0.0001
-    val l = (math.log(x) / math.log(2))
+    val l = math.log(x) / math.log(2)
     math.abs(l - l.round.toDouble) < epsilon
   }
 
@@ -265,7 +266,9 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       reportMutable: ListBuffer[String],
       requestingUser: User)(implicit ec: ExecutionContext): Fox[List[(DataLayer, Vec3Double)]] =
     for {
-      uri <- tryo(new URI(normalizeUri(layerUri))) ?~> s"Received invalid URI: $layerUri"
+      uri <- tryo(new URI(removeHeaderFileNamesFromUriSuffix(layerUri))) ?~> s"Received invalid URI: $layerUri"
+      _ <- bool2Fox(uri.getScheme != null) ?~> s"Received invalid URI: $layerUri"
+      _ <- assertLocalPathInWhitelist(uri)
       credentialOpt = credentialService.createCredentialOpt(uri,
                                                             credentialIdentifier,
                                                             credentialSecret,
@@ -290,7 +293,12 @@ class ExploreRemoteLayerService @Inject()(credentialService: CredentialService,
       )
     } yield layersWithVoxelSizes
 
-  private def normalizeUri(uri: String): String =
+  private def assertLocalPathInWhitelist(uri: URI)(implicit ec: ExecutionContext): Fox[Unit] =
+    if (uri.getScheme == DataVaultService.schemeFile) {
+      bool2Fox(wkConf.Datastore.localFolderWhitelist.exists(whitelistEntry => uri.getPath.startsWith(whitelistEntry))) ?~> s"Absolute path ${uri.getPath} in local file system is not in path whitelist. Consider adding it to datastore.pathWhitelist"
+    } else Fox.successful(())
+
+  private def removeHeaderFileNamesFromUriSuffix(uri: String): String =
     if (uri.endsWith(N5Header.FILENAME_ATTRIBUTES_JSON)) uri.dropRight(N5Header.FILENAME_ATTRIBUTES_JSON.length)
     else if (uri.endsWith(ZarrHeader.FILENAME_DOT_ZARRAY)) uri.dropRight(ZarrHeader.FILENAME_DOT_ZARRAY.length)
     else if (uri.endsWith(NgffMetadata.FILENAME_DOT_ZATTRS)) uri.dropRight(NgffMetadata.FILENAME_DOT_ZATTRS.length)

app/utils/WkConf.scala

@@ -126,6 +126,7 @@ class WkConf @Inject()(configuration: Configuration) extends ConfigReader with L
     val key: String = get[String]("datastore.key")
     val name: String = get[String]("datastore.name")
     val publicUri: Option[String] = getOptional[String]("datastore.publicUri")
+    val localFolderWhitelist: List[String] = getList[String]("datastore.localFolderWhitelist")
   }
 
   object Tracingstore {

conf/application.conf

@@ -178,6 +178,7 @@ datastore {
     pingInterval = 10 minutes
   }
   baseFolder = "binaryData"
+  localFolderWhitelist = [] # list of local absolute directory paths where image data may be explored and served from
   watchFileSystem {
     enabled = true
     interval = 1 minute

webknossos-datastore/app/com/scalableminds/webknossos/datastore/DataStoreConfig.scala

@@ -21,6 +21,7 @@ class DataStoreConfig @Inject()(configuration: Configuration) extends ConfigRead
       val pingInterval: FiniteDuration = get[FiniteDuration]("datastore.webKnossos.pingInterval")
     }
     val baseFolder: String = get[String]("datastore.baseFolder")
+    val localFolderWhitelist: List[String] = getList[String]("datastore.localFolderWhitelist")
     object WatchFileSystem {
       val enabled: Boolean = get[Boolean]("datastore.watchFileSystem.enabled")
       val interval: FiniteDuration = get[FiniteDuration]("datastore.watchFileSystem.interval")

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/BucketProvider.scala

@@ -1,7 +1,6 @@
 package com.scalableminds.webknossos.datastore.dataformats
 
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
-import com.scalableminds.webknossos.datastore.datavault.{FileSystemVaultPath, VaultPath}
 import com.scalableminds.webknossos.datastore.models.BucketPosition
 import com.scalableminds.webknossos.datastore.models.requests.DataReadInstruction
 import com.scalableminds.webknossos.datastore.storage.{DataCubeCache, RemoteSourceDescriptorService}
@@ -15,19 +14,19 @@ trait BucketProvider extends FoxImplicits with LazyLogging {
   def remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService]
 
   // To be defined in subclass.
-  def loadFromUnderlying(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext): Fox[DataCubeHandle] =
+  def openShardOrArrayHandle(readInstruction: DataReadInstruction)(implicit ec: ExecutionContext): Fox[DataCubeHandle] =
     Empty
 
   def load(readInstruction: DataReadInstruction, cache: DataCubeCache)(
       implicit ec: ExecutionContext): Fox[Array[Byte]] =
-    cache.withCache(readInstruction)(loadFromUnderlyingWithTimeout)(
+    cache.withCache(readInstruction)(openShardOrArrayHandleWithTimeout)(
       _.cutOutBucket(readInstruction.bucket, readInstruction.dataLayer))
 
-  private def loadFromUnderlyingWithTimeout(readInstruction: DataReadInstruction)(
+  private def openShardOrArrayHandleWithTimeout(readInstruction: DataReadInstruction)(
       implicit ec: ExecutionContext): Fox[DataCubeHandle] = {
     val t = System.currentTimeMillis
     for {
-      result <- loadFromUnderlying(readInstruction).futureBox
+      result <- openShardOrArrayHandle(readInstruction).futureBox
       duration = System.currentTimeMillis - t
       _ = if (duration > 500) {
         val className = this.getClass.getName.split("\\.").last
@@ -45,24 +44,4 @@ trait BucketProvider extends FoxImplicits with LazyLogging {
   def bucketStream(version: Option[Long] = None): Iterator[(BucketPosition, Array[Byte])] =
     Iterator.empty
 
-  protected def localPathFrom(readInstruction: DataReadInstruction, relativeMagPath: String)(
-      implicit ec: ExecutionContext): Fox[VaultPath] = {
-    val magPathRelativeToDataset = FileSystemVaultPath.fromPath(
-      readInstruction.baseDir
-        .resolve(readInstruction.dataSource.id.team)
-        .resolve(readInstruction.dataSource.id.name)
-        .resolve(relativeMagPath))
-    val magPathRelativeToLayer = FileSystemVaultPath.fromPath(
-      readInstruction.baseDir
-        .resolve(readInstruction.dataSource.id.team)
-        .resolve(readInstruction.dataSource.id.name)
-        .resolve(readInstruction.dataLayer.name)
-        .resolve(relativeMagPath))
-    if (magPathRelativeToDataset.exists) {
-      Fox.successful(magPathRelativeToDataset)
-    } else if (magPathRelativeToLayer.exists) {
-      Fox.successful(magPathRelativeToLayer)
-    } else Fox.empty
-  }
-
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/MagLocator.scala

@@ -3,22 +3,15 @@ package com.scalableminds.webknossos.datastore.dataformats
 import com.scalableminds.util.geometry.Vec3Int
 import com.scalableminds.webknossos.datastore.datareaders.AxisOrder
 import com.scalableminds.webknossos.datastore.models.datasource.ResolutionFormatHelper
-import com.scalableminds.webknossos.datastore.storage.{DataVaultService, LegacyDataVaultCredential}
+import com.scalableminds.webknossos.datastore.storage.LegacyDataVaultCredential
 import play.api.libs.json.{Json, OFormat}
 
-import java.net.URI
-
 case class MagLocator(mag: Vec3Int,
                       path: Option[String],
                       credentials: Option[LegacyDataVaultCredential],
                       axisOrder: Option[AxisOrder],
                       channelIndex: Option[Int],
-                      credentialId: Option[String]) {
-
-  lazy val pathWithFallback: String = path.getOrElse(mag.toMagLiteral(allowScalar = true))
-  lazy val uri: URI = new URI(pathWithFallback)
-  lazy val isRemote: Boolean = DataVaultService.isSupportedRemoteScheme(uri.getScheme)
-}
+                      credentialId: Option[String])
 
 object MagLocator extends ResolutionFormatHelper {
   implicit val jsonFormat: OFormat[MagLocator] = Json.format[MagLocator]

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/n5/N5BucketProvider.scala

@@ -36,23 +36,29 @@ class N5BucketProvider(layer: N5Layer,
     extends BucketProvider
     with LazyLogging {
 
-  override def loadFromUnderlying(readInstruction: DataReadInstruction)(
+  override def openShardOrArrayHandle(readInstruction: DataReadInstruction)(
       implicit ec: ExecutionContext): Fox[N5CubeHandle] = {
-    val n5MagOpt: Option[MagLocator] =
+    val magLocatorOpt: Option[MagLocator] =
       layer.mags.find(_.mag == readInstruction.bucket.mag)
 
-    n5MagOpt match {
+    magLocatorOpt match {
       case None => Fox.empty
-      case Some(n5Mag) =>
+      case Some(magLocator) =>
         remoteSourceDescriptorServiceOpt match {
           case Some(remoteSourceDescriptorService: RemoteSourceDescriptorService) =>
             for {
-              magPath: VaultPath <- if (n5Mag.isRemote) {
-                remoteSourceDescriptorService.vaultPathFor(n5Mag)
-              } else localPathFrom(readInstruction, n5Mag.pathWithFallback)
+              magPath: VaultPath <- remoteSourceDescriptorService.vaultPathFor(readInstruction.baseDir,
+                                                                               readInstruction.dataSource.id,
+                                                                               readInstruction.dataLayer.name,
+                                                                               magLocator)
               chunkContentsCache <- sharedChunkContentsCache.toFox
               cubeHandle <- N5Array
-                .open(magPath, dataSourceId, layer.name, n5Mag.axisOrder, n5Mag.channelIndex, chunkContentsCache)
+                .open(magPath,
+                      dataSourceId,
+                      layer.name,
+                      magLocator.axisOrder,
+                      magLocator.channelIndex,
+                      chunkContentsCache)
                 .map(new N5CubeHandle(_))
             } yield cubeHandle
           case None => Empty

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/precomputed/PrecomputedBucketProvider.scala

@@ -36,27 +36,28 @@ class PrecomputedBucketProvider(layer: PrecomputedLayer,
     extends BucketProvider
     with LazyLogging {
 
-  override def loadFromUnderlying(readInstruction: DataReadInstruction)(
+  override def openShardOrArrayHandle(readInstruction: DataReadInstruction)(
       implicit ec: ExecutionContext): Fox[PrecomputedCubeHandle] = {
-    val precomputedMagOpt: Option[MagLocator] =
+    val magLocatorOpt: Option[MagLocator] =
       layer.mags.find(_.mag == readInstruction.bucket.mag)
 
-    precomputedMagOpt match {
+    magLocatorOpt match {
       case None => Fox.empty
-      case Some(precomputedMag) =>
+      case Some(magLocator) =>
         remoteSourceDescriptorServiceOpt match {
           case Some(remoteSourceDescriptorService: RemoteSourceDescriptorService) =>
             for {
-              magPath: VaultPath <- if (precomputedMag.isRemote) {
-                remoteSourceDescriptorService.vaultPathFor(precomputedMag)
-              } else localPathFrom(readInstruction, precomputedMag.pathWithFallback)
+              magPath: VaultPath <- remoteSourceDescriptorService.vaultPathFor(readInstruction.baseDir,
+                                                                               readInstruction.dataSource.id,
+                                                                               readInstruction.dataLayer.name,
+                                                                               magLocator)
               chunkContentsCache <- sharedChunkContentsCache.toFox
               cubeHandle <- PrecomputedArray
                 .open(magPath,
                       dataSourceId,
                       layer.name,
-                      precomputedMag.axisOrder,
-                      precomputedMag.channelIndex,
+                      magLocator.axisOrder,
+                      magLocator.channelIndex,
                       chunkContentsCache)
                 .map(new PrecomputedCubeHandle(_))
             } yield cubeHandle

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/wkw/WKWBucketProvider.scala

@@ -37,7 +37,7 @@ class WKWBucketProvider(layer: WKWLayer) extends BucketProvider with WKWDataForm
 
   override def remoteSourceDescriptorServiceOpt: Option[RemoteSourceDescriptorService] = None
 
-  override def loadFromUnderlying(readInstruction: DataReadInstruction)(
+  override def openShardOrArrayHandle(readInstruction: DataReadInstruction)(
       implicit ec: ExecutionContext): Fox[WKWCubeHandle] = {
     val wkwFile = wkwFilePath(
       readInstruction.cube,

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr/ZarrBucketProvider.scala

@@ -41,23 +41,29 @@ class ZarrBucketProvider(layer: ZarrLayer,
     extends BucketProvider
     with LazyLogging {
 
-  override def loadFromUnderlying(readInstruction: DataReadInstruction)(
+  override def openShardOrArrayHandle(readInstruction: DataReadInstruction)(
       implicit ec: ExecutionContext): Fox[ZarrCubeHandle] = {
-    val zarrMagOpt: Option[MagLocator] =
+    val magLocatorOpt: Option[MagLocator] =
       layer.mags.find(_.mag == readInstruction.bucket.mag)
 
-    zarrMagOpt match {
+    magLocatorOpt match {
       case None => Fox.empty
-      case Some(zarrMag) =>
+      case Some(magLocator) =>
         remoteSourceDescriptorServiceOpt match {
           case Some(remoteSourceDescriptorService: RemoteSourceDescriptorService) =>
             for {
-              magPath: VaultPath <- if (zarrMag.isRemote) {
-                remoteSourceDescriptorService.vaultPathFor(zarrMag)
-              } else localPathFrom(readInstruction, zarrMag.pathWithFallback)
+              magPath: VaultPath <- remoteSourceDescriptorService.vaultPathFor(readInstruction.baseDir,
+                                                                               readInstruction.dataSource.id,
+                                                                               readInstruction.dataLayer.name,
+                                                                               magLocator)
               chunkContentsCache <- sharedChunkContentsCache.toFox
               cubeHandle <- ZarrArray
-                .open(magPath, dataSourceId, layer.name, zarrMag.axisOrder, zarrMag.channelIndex, chunkContentsCache)
+                .open(magPath,
+                      dataSourceId,
+                      layer.name,
+                      magLocator.axisOrder,
+                      magLocator.channelIndex,
+                      chunkContentsCache)
                 .map(new ZarrCubeHandle(_))
             } yield cubeHandle
           case None => Empty

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/zarr3/Zarr3BucketProvider.scala

@@ -36,23 +36,29 @@ class Zarr3BucketProvider(layer: Zarr3Layer,
     extends BucketProvider
     with LazyLogging {
 
-  override def loadFromUnderlying(readInstruction: DataReadInstruction)(
+  override def openShardOrArrayHandle(readInstruction: DataReadInstruction)(
       implicit ec: ExecutionContext): Fox[ZarrCubeHandle] = {
-    val zarrMagOpt: Option[MagLocator] =
+    val magLocatorOpt: Option[MagLocator] =
       layer.mags.find(_.mag == readInstruction.bucket.mag)
 
-    zarrMagOpt match {
+    magLocatorOpt match {
       case None => Fox.empty
-      case Some(zarrMag) =>
+      case Some(magLocator) =>
         remoteSourceDescriptorServiceOpt match {
           case Some(remoteSourceDescriptorService: RemoteSourceDescriptorService) =>
             for {
-              magPath: VaultPath <- if (zarrMag.isRemote) {
-                remoteSourceDescriptorService.vaultPathFor(zarrMag)
-              } else localPathFrom(readInstruction, zarrMag.pathWithFallback)
+              magPath: VaultPath <- remoteSourceDescriptorService.vaultPathFor(readInstruction.baseDir,
+                                                                               readInstruction.dataSource.id,
+                                                                               readInstruction.dataLayer.name,
+                                                                               magLocator)
               chunkContentsCache <- sharedChunkContentsCache.toFox
               cubeHandle <- Zarr3Array
-                .open(magPath, dataSourceId, layer.name, zarrMag.axisOrder, zarrMag.channelIndex, chunkContentsCache)
+                .open(magPath,
+                      dataSourceId,
+                      layer.name,
+                      magLocator.axisOrder,
+                      magLocator.channelIndex,
+                      chunkContentsCache)
                 .map(new ZarrCubeHandle(_))
             } yield cubeHandle
           case None => Empty