WIP

- Chunks with high z are spread around -> investigate
- It is not clear to me why the chunks has to be in a different order than xyz when given to copyRange, while the offset needs to be in xyz
- Histogram not working -> not yet investigated

conf/application.conf

@@ -287,5 +287,5 @@ pidfile.path = "/dev/null"
 
 
 # uncomment these lines for faster restart during local backend development (but beware the then-missing features):
-#slick.checkSchemaOnStartup = false
-#play.modules.disabled += "play.modules.swagger.SwaggerModule"
+slick.checkSchemaOnStartup = false
+play.modules.disabled += "play.modules.swagger.SwaggerModule"

package.json

@@ -85,6 +85,7 @@
     "build-watch": "node_modules/.bin/webpack -w",
     "listening": "lsof -i:5005,7155,9000,9001,9002",
     "kill-listeners": "kill -9 $(lsof -t -i:5005,7155,9000,9001,9002)",
+    "rm-lock": "rm fossildb/data/LOCK",
     "test": "tools/test.sh test --timeout=30s",
     "test-changed": "tools/test.sh test-changed --timeout=30s",
     "test-verbose": "xvfb-run -s '-ac -screen 0 1280x1024x24' tools/test.sh test --timeout=60s --verbose",

util/src/main/scala/com/scalableminds/util/geometry/BoundingBox.scala

@@ -100,6 +100,9 @@ object BoundingBox {
     else
       None
 
+  def fromOffsetAndShape(offset: Vec3Int, shape: Vec3Int): BoundingBox =
+    BoundingBox(offset, shape.x, shape.y, shape.z)
+
   def union(bbs: List[BoundingBox]): BoundingBox =
     bbs match {
       case head :: tail =>

util/src/main/scala/com/scalableminds/util/geometry/Vec3Int.scala

@@ -14,6 +14,15 @@ case class Vec3Int(x: Int, y: Int, z: Int) {
   def *(that: Int): Vec3Int =
     Vec3Int(x * that, y * that, z * that)
 
+  def /(that: Vec3Int): Vec3Int =
+    Vec3Int(x / that.x, y / that.y, z / that.z)
+
+  def +(that: Vec3Int): Vec3Int =
+    Vec3Int(x + that.x, y + that.y, z + that.z)
+
+  def -(that: Vec3Int): Vec3Int =
+    Vec3Int(x - that.x, y - that.y, z - that.z)
+
   def scale(s: Float): Vec3Int =
     Vec3Int((x * s).toInt, (y * s).toInt, (z * s).toInt)
 

webknossos-datastore/app/com/scalableminds/webknossos/datastore/dataformats/precomputed/PrecomputedBucketProvider.scala

@@ -53,7 +53,8 @@ class PrecomputedBucketProvider(layer: PrecomputedLayer)
           case None => Empty
           case Some(magPath) =>
             tryo(onError = e => logError(e))(
-              PrecomputedArray.open(magPath, precomputedMag.axisOrder, precomputedMag.channelIndex))
+              PrecomputedArray
+                .open(magPath, precomputedMag.axisOrder, precomputedMag.channelIndex, readInstruction.bucket.mag))
               .map(new PrecomputedCubeHandle(_))
         }
     }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkReader.scala

@@ -27,8 +27,8 @@ class ChunkReader(val header: DatasetHeader, val store: FileSystemStore, val typ
   lazy val chunkSize: Int = header.chunkSize.toList.product
 
   @throws[IOException]
-  def read(path: String): Future[MultiArray] =
-    typedChunkReader.read(readBytes(path))
+  def read(path: String, chunkShape: Array[Int]): Future[MultiArray] =
+    typedChunkReader.read(readBytes(path), chunkShape)
 
   protected def readBytes(path: String): Option[Array[Byte]] =
     Using.Manager { use =>
@@ -44,28 +44,27 @@ class ChunkReader(val header: DatasetHeader, val store: FileSystemStore, val typ
 abstract class TypedChunkReader {
   val header: DatasetHeader
 
-  var chunkShape: Array[Int] = header.chunkShapeOrdered
   def ma2DataType: MADataType
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray]
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray]
 
-  def createFilled(dataType: MADataType): MultiArray =
+  def createFilled(dataType: MADataType, chunkShape: Array[Int]): MultiArray =
     MultiArrayUtils.createFilledArray(dataType, chunkShape, header.fillValueNumber)
 }
 
 class ByteChunkReader(val header: DatasetHeader) extends TypedChunkReader {
   val ma2DataType: MADataType = MADataType.BYTE
 
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray] =
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray] =
     Future.successful(bytes.map { result =>
       MultiArray.factory(ma2DataType, chunkShape, result)
-    }.getOrElse(createFilled(ma2DataType)))
+    }.getOrElse(createFilled(ma2DataType, chunkShape)))
 }
 
 class DoubleChunkReader(val header: DatasetHeader) extends TypedChunkReader {
 
   val ma2DataType: MADataType = MADataType.DOUBLE
 
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray] =
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray] =
     Future.successful(Using.Manager { use =>
       bytes.map { result =>
         val typedStorage = new Array[Double](chunkShape.product)
@@ -74,15 +73,15 @@ class DoubleChunkReader(val header: DatasetHeader) extends TypedChunkReader {
         iis.setByteOrder(header.byteOrder)
         iis.readFully(typedStorage, 0, typedStorage.length)
         MultiArray.factory(ma2DataType, chunkShape, typedStorage)
-      }.getOrElse(createFilled(ma2DataType))
+      }.getOrElse(createFilled(ma2DataType, chunkShape))
     }.get)
 }
 
 class ShortChunkReader(val header: DatasetHeader) extends TypedChunkReader with LazyLogging {
 
   val ma2DataType: MADataType = MADataType.SHORT
 
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray] =
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray] =
     Future.successful(Using.Manager { use =>
       bytes.map { result =>
         val typedStorage = new Array[Short](chunkShape.product)
@@ -91,15 +90,15 @@ class ShortChunkReader(val header: DatasetHeader) extends TypedChunkReader with
         iis.setByteOrder(header.byteOrder)
         iis.readFully(typedStorage, 0, typedStorage.length)
         MultiArray.factory(ma2DataType, chunkShape, typedStorage)
-      }.getOrElse(createFilled(ma2DataType))
+      }.getOrElse(createFilled(ma2DataType, chunkShape))
     }.get)
 }
 
 class IntChunkReader(val header: DatasetHeader) extends TypedChunkReader {
 
   val ma2DataType: MADataType = MADataType.INT
 
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray] =
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray] =
     Future.successful(Using.Manager { use =>
       bytes.map { result =>
         val typedStorage = new Array[Int](chunkShape.product)
@@ -108,15 +107,15 @@ class IntChunkReader(val header: DatasetHeader) extends TypedChunkReader {
         iis.setByteOrder(header.byteOrder)
         iis.readFully(typedStorage, 0, typedStorage.length)
         MultiArray.factory(ma2DataType, chunkShape, typedStorage)
-      }.getOrElse(createFilled(ma2DataType))
+      }.getOrElse(createFilled(ma2DataType, chunkShape))
     }.get)
 }
 
 class LongChunkReader(val header: DatasetHeader) extends TypedChunkReader {
 
   val ma2DataType: MADataType = MADataType.LONG
 
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray] =
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray] =
     Future.successful(Using.Manager { use =>
       bytes.map { result =>
         val typedStorage = new Array[Long](chunkShape.product)
@@ -125,15 +124,15 @@ class LongChunkReader(val header: DatasetHeader) extends TypedChunkReader {
         iis.setByteOrder(header.byteOrder)
         iis.readFully(typedStorage, 0, typedStorage.length)
         MultiArray.factory(ma2DataType, chunkShape, typedStorage)
-      }.getOrElse(createFilled(ma2DataType))
+      }.getOrElse(createFilled(ma2DataType, chunkShape))
     }.get)
 }
 
 class FloatChunkReader(val header: DatasetHeader) extends TypedChunkReader {
 
   val ma2DataType: MADataType = MADataType.FLOAT
 
-  def read(bytes: Option[Array[Byte]]): Future[MultiArray] =
+  def read(bytes: Option[Array[Byte]], chunkShape: Array[Int]): Future[MultiArray] =
     Future.successful(Using.Manager { use =>
       bytes.map { result =>
         val typedStorage = new Array[Float](chunkShape.product)
@@ -142,6 +141,6 @@ class FloatChunkReader(val header: DatasetHeader) extends TypedChunkReader {
         iis.setByteOrder(header.byteOrder)
         iis.readFully(typedStorage, 0, typedStorage.length)
         MultiArray.factory(ma2DataType, chunkShape, typedStorage)
-      }.getOrElse(createFilled(ma2DataType))
+      }.getOrElse(createFilled(ma2DataType, chunkShape))
     }.get)
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/ChunkUtils.scala

@@ -40,4 +40,29 @@ object ChunkUtils extends LazyLogging {
     }
     chunkIndices.toList
   }
+/*
+  def calculatePrecomputedChunks(arrayShape: Array[Int],
+                                 arrayChunkSize: Array[Int],
+                                 selectedShape: Array[Int],
+                                 selectedOffset: Array[Int],
+                                 mag: Vec3Int): Iterable[BoundingBox] = {
+
+    val offset = (selectedOffset, mag.toList).zipped.map((o, m) => o / m)
+    val requested = BoundingBox.fromOffsetAndShape(Vec3Int.fromList(offset.toList).getOrElse(Vec3Int(0, 0, 0)),
+                                                   Vec3Int.fromList(selectedShape.toList).getOrElse(Vec3Int(0, 0, 0)))
+    val boundingBox = BoundingBox(Vec3Int(0, 0, 0), arrayShape(0), arrayShape(1), arrayShape(2))
+    val inside = requested.intersection(boundingBox)
+
+    val chunkSize = Vec3Int.fromList(arrayChunkSize.toList).getOrElse(Vec3Int(0, 0, 0))
+
+    inside match {
+      case Some(inside) => {
+        val aligned = (inside - boundingBox.topLeft).div(chunkSize) * chunkSize + boundingBox.topLeft
+        aligned
+          .range(chunkSize)
+          .flatMap(chunkOffset => BoundingBox.fromOffsetAndShape(chunkOffset, chunkSize).intersection(boundingBox))
+      }
+      case _ => List()
+    }
+  }*/
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/Compressor.scala

@@ -15,7 +15,7 @@ import java.nio.ByteBuffer
 import java.util
 import java.util.zip.{Deflater, DeflaterOutputStream, Inflater, InflaterInputStream}
 import javax.imageio.ImageIO
-import javax.imageio.ImageIO.{createImageInputStream, createImageOutputStream}
+import javax.imageio.ImageIO.{createImageInputStream}
 import javax.imageio.stream.ImageInputStream
 
 sealed trait CompressionSetting

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetArray.scala

@@ -25,7 +25,7 @@ class DatasetArray(relativePath: DatasetPath,
     ChunkReader.create(store, header)
 
   // cache currently limited to 1 GB per array
-  private lazy val chunkContentsCache: Cache[String, MultiArray] = {
+  protected lazy val chunkContentsCache: Cache[String, MultiArray] = {
     val maxSizeBytes = 1000L * 1000 * 1000
     val maxEntries = maxSizeBytes / header.bytesPerChunk
     AlfuCache(maxEntries.toInt)
@@ -58,7 +58,7 @@ class DatasetArray(relativePath: DatasetPath,
   // This function will internally adapt to the array's axis order so that XYZ data in fortran-order is returned.
   @throws[IOException]
   @throws[InvalidRangeException]
-  private def readAsFortranOrder(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext): Fox[Object] = {
+  protected def readAsFortranOrder(shape: Array[Int], offset: Array[Int])(implicit ec: ExecutionContext): Fox[Object] = {
     val chunkIndices = ChunkUtils.computeChunkIndices(axisOrder.permuteIndicesReverse(header.datasetShape),
                                                       axisOrder.permuteIndicesReverse(header.chunkSize),
                                                       shape,
@@ -78,7 +78,8 @@ class DatasetArray(relativePath: DatasetPath,
           offsetInChunk = computeOffsetInChunk(chunkIndex, offset)
           sourceChunkInCOrder: MultiArray = MultiArrayUtils.axisOrderXYZView(sourceChunk,
                                                                              axisOrder,
-                                                                             flip = header.order != ArrayOrder.C)
+                                                                             flip = header.order != ArrayOrder.C,
+                                                                             header.shiftAxisOrderRight)
           _ = MultiArrayUtils.copyRange(offsetInChunk, sourceChunkInCOrder, targetInCOrder)
         } yield ()
       }
@@ -88,12 +89,13 @@ class DatasetArray(relativePath: DatasetPath,
     }
   }
 
-  private def getSourceChunkDataWithCache(chunkIndex: Array[Int]): Future[MultiArray] = {
+  protected def getSourceChunkDataWithCache(chunkIndex: Array[Int]): Future[MultiArray] = {
     val chunkFilename = getChunkFilename(chunkIndex)
     val chunkFilePath = relativePath.resolve(chunkFilename)
     val storeKey = chunkFilePath.storeKey
+    val chunkShape = header.chunkSizeAtIndex(chunkIndex)
 
-    chunkContentsCache.getOrLoad(storeKey, chunkReader.read)
+    chunkContentsCache.getOrLoad(storeKey, key => chunkReader.read(key, chunkShape))
   }
 
   protected def getChunkFilename(chunkIndex: Array[Int]): String =
@@ -118,9 +120,9 @@ class DatasetArray(relativePath: DatasetPath,
   private def isZeroOffset(offset: Array[Int]): Boolean =
     util.Arrays.equals(offset, new Array[Int](offset.length))
 
-  private def computeOffsetInChunk(chunkIndex: Array[Int], globalOffset: Array[Int]): Array[Int] =
+  protected def computeOffsetInChunk(chunkIndex: Array[Int], globalOffset: Array[Int]): Array[Int] =
     chunkIndex.indices.map { dim =>
-      globalOffset(dim) - (chunkIndex(dim) * axisOrder.permuteIndicesReverse(header.chunkSize)(dim))
+      globalOffset(dim) - (chunkIndex(dim) * header.chunkSize(dim))
     }.toArray
 
   override def toString: String =

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/DatasetHeader.scala

@@ -9,13 +9,19 @@ import java.nio.ByteOrder
 
 trait DatasetHeader {
   def datasetShape: Array[Int] // shape of the entire array
+
   def chunkSize: Array[Int] // shape of each chunk
+
   def dimension_separator: DimensionSeparator
+
   def dataType: String
+
   def fill_value: Either[String, Number]
+
   def order: ArrayOrder
 
   def resolvedDataType: ArrayDataType
+
   def compressorImpl: Compressor
 
   lazy val byteOrder: ByteOrder = ByteOrder.BIG_ENDIAN
@@ -40,4 +46,8 @@ trait DatasetHeader {
       Some(BoundingBox(Vec3Int.zeros, datasetShape(axisOrder.x), datasetShape(axisOrder.y), datasetShape(axisOrder.z)))
 
   lazy val rank: Int = datasetShape.length
+
+  def shiftAxisOrderRight: Boolean = false
+
+  def chunkSizeAtIndex(chunkIndex: Array[Int]): Array[Int] = chunkSize
 }

webknossos-datastore/app/com/scalableminds/webknossos/datastore/datareaders/FileSystemStore.scala

@@ -1,7 +1,9 @@
 package com.scalableminds.webknossos.datastore.datareaders
 
+import com.typesafe.scalalogging.LazyLogging
 import net.liftweb.util.Helpers.tryo
 
+import java.io.FileNotFoundException
 import java.nio.file.{FileSystem, Files, Path}
 
 class FileSystemStore(val internalRoot: Path) {
@@ -12,7 +14,8 @@ class FileSystemStore(val internalRoot: Path) {
 }
 
 class GoogleCloudFileSystemStore(override val internalRoot: Path, fs: FileSystem)
-    extends FileSystemStore(internalRoot) {
+    extends FileSystemStore(internalRoot)
+    with LazyLogging {
 
   private def normalizedInternalRoot = {
     def prefix = internalRoot.getParent.toString // This part uses "/"
@@ -23,6 +26,17 @@ class GoogleCloudFileSystemStore(override val internalRoot: Path, fs: FileSystem
 
   override def readBytes(key: String): Option[Array[Byte]] = {
     val path = s"$normalizedInternalRoot%2F$key?alt=media"
-    tryo(Files.readAllBytes(fs.getPath(path))).toOption
+    try {
+      Some(Files.readAllBytes(fs.getPath(path)))
+    } catch {
+      case e: FileNotFoundException => {
+        logger.info(s"Could not read data at ${path}")
+        None
+      }
+      case _ => {
+        logger.info(s"Could not read data at ${path}")
+        None
+      }
+    }
   }
 }