-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce data vault as storage backend abstraction (#6899)
- Loading branch information
Showing
56 changed files
with
689 additions
and
3,740 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| package backend | ||
|
|
||
| import org.scalatestplus.play.PlaySpec | ||
|
|
||
| import java.net.URI | ||
| import com.scalableminds.webknossos.datastore.datavault.{DataVault, GoogleCloudDataVault, HttpsDataVault, VaultPath} | ||
| import com.scalableminds.webknossos.datastore.storage.RemoteSourceDescriptor | ||
|
|
||
| import scala.collection.immutable.NumericRange | ||
|
|
||
| class DataVaultTestSuite extends PlaySpec { | ||
|
|
||
| "Data vault" when { | ||
| "using Range requests" when { | ||
| val range: NumericRange[Long] = Range.Long(0, 1024, 1) | ||
| val dataKey = "32_32_40/15360-15424_8384-8448_3520-3584" // when accessed via range request, the response body is 1024 bytes long, otherwise 124.8 KB | ||
|
|
||
| "with HTTP Vault" should { | ||
| "return correct response" in { | ||
| val uri = new URI("http://storage.googleapis.com/") | ||
| val vaultPath = new VaultPath(uri, HttpsDataVault.create(RemoteSourceDescriptor(uri, None))) | ||
| val bytes = | ||
| (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey").readBytes(Some(range)).get | ||
|
|
||
| assert(bytes.length == range.length) | ||
| assert(bytes.take(10).sameElements(Array(-1, -40, -1, -32, 0, 16, 74, 70, 73, 70))) | ||
| } | ||
| } | ||
|
|
||
| "with Google Cloud Storage Vault" should { | ||
| "return correct response" in { | ||
| val uri = new URI("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig") | ||
| val vaultPath = new VaultPath(uri, GoogleCloudDataVault.create(RemoteSourceDescriptor(uri, None))) | ||
| val bytes = (vaultPath / dataKey).readBytes(Some(range)).get | ||
|
|
||
| assert(bytes.length == range.length) | ||
| assert(bytes.take(10).sameElements(Array(-1, -40, -1, -32, 0, 16, 74, 70, 73, 70))) | ||
| } | ||
| } | ||
| } | ||
| "using regular requests" when { | ||
| val dataKey = "32_32_40/15360-15424_8384-8448_3520-3584" | ||
| val dataLength = 127808 | ||
|
|
||
| "with HTTP Vault" should { | ||
| "return correct response" in { | ||
| val uri = new URI("http://storage.googleapis.com/") | ||
| val vaultPath = new VaultPath(uri, HttpsDataVault.create(RemoteSourceDescriptor(uri, None))) | ||
| val bytes = (vaultPath / s"neuroglancer-fafb-data/fafb_v14/fafb_v14_orig/$dataKey").readBytes().get | ||
|
|
||
| assert(bytes.length == dataLength) | ||
| assert(bytes.take(10).sameElements(Array(-1, -40, -1, -32, 0, 16, 74, 70, 73, 70))) | ||
| } | ||
| } | ||
|
|
||
| "with Google Cloud Storage Vault" should { | ||
| "return correct response" in { | ||
| val uri = new URI("gs://neuroglancer-fafb-data/fafb_v14/fafb_v14_orig") | ||
| val vaultPath = new VaultPath(uri, GoogleCloudDataVault.create(RemoteSourceDescriptor(uri, None))) | ||
| val bytes = (vaultPath / dataKey).readBytes().get | ||
|
|
||
| assert(bytes.length == dataLength) | ||
| assert(bytes.take(10).sameElements(Array(-1, -40, -1, -32, 0, 16, 74, 70, 73, 70))) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| "using vault path" when { | ||
| class MockDataVault extends DataVault { | ||
| override def readBytes(path: VaultPath, range: Option[NumericRange[Long]]): Array[Byte] = ??? | ||
| } | ||
|
|
||
| "Uri has no trailing slash" should { | ||
| val someUri = new URI("protocol://host/a/b") | ||
| val somePath = new VaultPath(someUri, new MockDataVault) | ||
|
|
||
| "resolve child" in { | ||
| val childPath = somePath / "c" | ||
| assert(childPath.toUri.toString == s"${someUri.toString}/c") | ||
| } | ||
|
|
||
| "get parent" in { | ||
| assert((somePath / "..").toString == "protocol://host/a/") | ||
| } | ||
|
|
||
| "get directory" in { | ||
| assert((somePath / ".").toString == s"${someUri.toString}/") | ||
| } | ||
|
|
||
| "handle sequential parameters" in { | ||
| assert((somePath / "c" / "d" / "e").toString == "protocol://host/a/b/c/d/e") | ||
| } | ||
|
|
||
| "resolve relative to host with starting slash in parameter" in { | ||
| assert((somePath / "/x").toString == "protocol://host/x") | ||
| } | ||
|
|
||
| "resolving path respects trailing slash" in { | ||
| assert((somePath / "x/").toString == "protocol://host/a/b/x/") | ||
| assert((somePath / "x").toString == "protocol://host/a/b/x") | ||
| } | ||
| } | ||
| "Uri has trailing slash" should { | ||
| val trailingSlashUri = new URI("protocol://host/a/b/") | ||
| val trailingSlashPath = new VaultPath(trailingSlashUri, new MockDataVault) | ||
| "resolve child" in { | ||
| val childPath = trailingSlashPath / "c" | ||
| assert(childPath.toUri.toString == s"${trailingSlashUri.toString}c") | ||
| } | ||
|
|
||
| "get parent" in { | ||
| assert((trailingSlashPath / "..").toString == "protocol://host/a/") | ||
| } | ||
|
|
||
| "get directory" in { | ||
| assert((trailingSlashPath / ".").toString == s"${trailingSlashUri.toString}") | ||
| } | ||
| } | ||
|
|
||
| } | ||
| } | ||
| } |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.