diff --git a/.circleci/config.yml b/.circleci/config.yml index f35317647f1..a70616eb2d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,6 +118,7 @@ jobs: - run: name: Run end-to-end tests command: | + mkdir -p binaryData/Organization_X && chmod 777 binaryData/Organization_X for i in {1..3}; do # retry .circleci/not-on-master.sh docker-compose run e2e-tests && s=0 && break || s=$? done diff --git a/conf/messages b/conf/messages index 44eff4e660d..4f59c9f0e84 100644 --- a/conf/messages +++ b/conf/messages @@ -73,10 +73,6 @@ oidc.disabled=OIDC is disabled oidc.configuration.invalid=OIDC configuration is invalid oidc.authentication.failed=Failed to register / log in via Single-Sign-On (SSO with OIDC) -braintracing.new=An account on braintracing.org was created for you. You can use the same credentials as on WEBKNOSSOS to login. -braintracing.error=We could not automatically create an account for you on braintracing.org. Please do it on your own. -braintracing.exists=Great, you already have an account on braintracing.org. Please double check that you have uploaded all requested information. - dataset=Dataset dataset.notFound=Dataset {0} does not exist or could not be accessed dataset.notFoundConsiderLogin=Dataset {0} does not exist or could not be accessed. You may need to log in. diff --git a/docker-compose.yml b/docker-compose.yml index 76f802426bb..a1f0e549c0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -201,7 +201,7 @@ services: -Ddatastore.redis.address=redis -Ddatastore.watchFileSystem.enabled=false" volumes: - - ./binaryData/Connectomics department:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X + - ./binaryData/Organization_X:/home/${USER_NAME:-sbt-user}/webknossos/binaryData/Organization_X screenshot-tests: image: scalableminds/puppeteer:master diff --git a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts index ab2e5f0856f..a82653b9462 100644 --- a/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts +++ b/frontend/javascripts/test/backend-snapshot-tests/datasets.e2e.ts @@ -1,5 +1,11 @@ import _ from "lodash"; -import { tokenUserA, setCurrToken, resetDatabase, writeTypeCheckingFile } from "test/e2e-setup"; +import { + tokenUserA, + setCurrToken, + resetDatabase, + writeTypeCheckingFile, + replaceVolatileValues, +} from "test/e2e-setup"; import type { APIDataset } from "types/api_flow_types"; import * as api from "admin/admin_rest_api"; import test from "ava"; @@ -15,6 +21,7 @@ async function getFirstDataset(): Promise { test.before("Reset database and change token", async () => { resetDatabase(); setCurrToken(tokenUserA); + await api.triggerDatasetCheck("http://localhost:9000"); }); test.serial("getDatasets", async (t) => { let datasets = await api.getDatasets(); @@ -29,19 +36,19 @@ test.serial("getDatasets", async (t) => { writeTypeCheckingFile(datasets, "dataset", "APIDatasetCompact", { isArray: true, }); - t.snapshot(datasets); + t.snapshot(replaceVolatileValues(datasets)); }); test("getActiveDatasets", async (t) => { let datasets = await api.getActiveDatasetsOfMyOrganization(); datasets = _.sortBy(datasets, (d) => d.name); - t.snapshot(datasets); + t.snapshot(replaceVolatileValues(datasets)); }); test("getDatasetAccessList", async (t) => { const dataset = await getFirstDataset(); const accessList = _.sortBy(await api.getDatasetAccessList(dataset), (user) => user.id); - t.snapshot(accessList); + t.snapshot(replaceVolatileValues(accessList)); }); test("updateDatasetTeams", async (t) => { const [dataset, newTeams] = await Promise.all([getFirstDataset(), api.getEditableTeams()]); @@ -49,7 +56,7 @@ test("updateDatasetTeams", async (t) => { dataset, newTeams.map((team) => team.id), ); - t.snapshot(updatedDataset); + t.snapshot(replaceVolatileValues(updatedDataset)); // undo the Change await api.updateDatasetTeams( dataset, @@ -62,3 +69,42 @@ test("updateDatasetTeams", async (t) => { // await api.revokeDatasetSharingToken(dataset.name); // t.pass(); // }); + +test("Zarr streaming", async (t) => { + const zattrsResp = await fetch("/data/zarr/Organization_X/test-dataset/segmentation/.zattrs", { + headers: new Headers(), + }); + const zattrs = await zattrsResp.text(); + t.snapshot(zattrs); + + const rawDataResponse = await fetch( + "/data/zarr/Organization_X/test-dataset/segmentation/1/0.1.1.0", + { + headers: new Headers(), + }, + ); + const bytes = await rawDataResponse.arrayBuffer(); + const base64 = btoa(String.fromCharCode(...new Uint8Array(bytes.slice(-128)))); + t.snapshot(base64); +}); + +test("Zarr 3 streaming", async (t) => { + const zarrJsonResp = await fetch( + "/data/zarr3_experimental/Organization_X/test-dataset/segmentation/zarr.json", + { + headers: new Headers(), + }, + ); + const zarrJson = await zarrJsonResp.text(); + t.snapshot(zarrJson); + + const rawDataResponse = await fetch( + "/data/zarr3_experimental/Organization_X/test-dataset/segmentation/1/0.1.1.0", + { + headers: new Headers(), + }, + ); + const bytes = await rawDataResponse.arrayBuffer(); + const base64 = btoa(String.fromCharCode(...new Uint8Array(bytes.slice(-128)))); + t.snapshot(base64); +}); diff --git a/frontend/javascripts/test/e2e-setup.ts b/frontend/javascripts/test/e2e-setup.ts index 9e8dee48c81..b1330eb25e1 100644 --- a/frontend/javascripts/test/e2e-setup.ts +++ b/frontend/javascripts/test/e2e-setup.ts @@ -39,6 +39,7 @@ const volatileKeys: Array = [ "lastActivity", "tracingTime", "tracingId", + "sortingKey", ]; export function replaceVolatileValues(obj: ArbitraryObject | null | undefined) { if (obj == null) return obj; @@ -130,7 +131,7 @@ export async function writeTypeCheckingFile( const fullTypeAnnotation = options.isArray ? `Array<${typeString}>` : typeString; fs.writeFileSync( `frontend/javascripts/test/snapshots/type-check/test-type-checking-${name}.ts`, - ` + ` import type { ${typeString} } from "types/api_flow_types"; const a: ${fullTypeAnnotation} = ${JSON.stringify(object)}`, ); diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md index 719d80174c7..f54066d7901 100644 --- a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md +++ b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.md @@ -11,10 +11,10 @@ Generated by [AVA](https://avajs.dev). [ { colorLayerNames: [], - created: 1460379470082, + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', - id: '570b9f4e4bb848d0885ee711', + id: 'id', isActive: false, isEditable: true, isUnreported: true, @@ -27,10 +27,10 @@ Generated by [AVA](https://avajs.dev). }, { colorLayerNames: [], - created: 1460379470080, + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', - id: '570b9f4e4bb848d0885ee713', + id: 'id', isActive: false, isEditable: true, isUnreported: true, @@ -43,10 +43,10 @@ Generated by [AVA](https://avajs.dev). }, { colorLayerNames: [], - created: 1460379470079, + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', - id: '570b9f4e4bb848d0885ee712', + id: 'id', isActive: false, isEditable: true, isUnreported: true, @@ -58,51 +58,43 @@ Generated by [AVA](https://avajs.dev). tags: [], }, { - colorLayerNames: [ - 'color_1', - 'color_2', - 'color_3', - ], - created: 1508495293763, + colorLayerNames: [], + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', - id: '59e9cfbdba632ac2ab8b23b3', - isActive: true, + id: 'id', + isActive: false, isEditable: true, - isUnreported: false, + isUnreported: true, lastUsedByUser: 0, name: 'confocal-multi_knossos', owningOrganization: 'Organization_X', segmentationLayerNames: [], - status: '', + status: 'No longer available on datastore.', tags: [], }, { - colorLayerNames: [ - 'color', - ], - created: 1508495293789, + colorLayerNames: [], + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', - id: '59e9cfbdba632ac2ab8b23b5', - isActive: true, + id: 'id', + isActive: false, isEditable: true, - isUnreported: false, + isUnreported: true, lastUsedByUser: 0, name: 'l4_sample', owningOrganization: 'Organization_X', - segmentationLayerNames: [ - 'segmentation', - ], - status: '', + segmentationLayerNames: [], + status: 'No longer available on datastore.', tags: [], }, { colorLayerNames: [], - created: 1460379603792, + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', - id: '570b9fd34bb848d0885ee716', + id: 'id', isActive: false, isEditable: true, isUnreported: true, @@ -113,276 +105,51 @@ Generated by [AVA](https://avajs.dev). status: 'No longer available on datastore.', tags: [], }, - ] - -## getActiveDatasets - -> Snapshot 1 - - [ { - allowedTeams: [ - { - id: '570b9f4b2a7c0e3b008da6ec', - name: 'team_X1', - organization: 'Organization_X', - }, - ], - allowedTeamsCumulative: [ - { - id: '570b9f4b2a7c0e3b008da6ec', - name: 'team_X1', - organization: 'Organization_X', - }, - ], - created: 1508495293763, - dataSource: { - dataLayers: [ - { - boundingBox: { - depth: 256, - height: 512, - topLeft: [ - 0, - 0, - 0, - ], - width: 512, - }, - category: 'color', - elementClass: 'uint8', - name: 'color_1', - resolutions: [ - [ - 1, - 1, - 1, - ], - [ - 2, - 2, - 2, - ], - [ - 4, - 4, - 4, - ], - [ - 8, - 8, - 8, - ], - [ - 16, - 16, - 16, - ], - ], - }, - { - boundingBox: { - depth: 256, - height: 512, - topLeft: [ - 0, - 0, - 0, - ], - width: 512, - }, - category: 'color', - elementClass: 'uint8', - name: 'color_2', - resolutions: [ - [ - 1, - 1, - 1, - ], - [ - 2, - 2, - 2, - ], - [ - 4, - 4, - 4, - ], - [ - 8, - 8, - 8, - ], - [ - 16, - 16, - 16, - ], - ], - }, - { - boundingBox: { - depth: 256, - height: 512, - topLeft: [ - 0, - 0, - 0, - ], - width: 512, - }, - category: 'color', - elementClass: 'uint8', - name: 'color_3', - resolutions: [ - [ - 1, - 1, - 1, - ], - [ - 2, - 2, - 2, - ], - [ - 4, - 4, - 4, - ], - [ - 8, - 8, - 8, - ], - [ - 16, - 16, - 16, - ], - ], - }, - ], - id: { - name: 'confocal-multi_knossos', - team: 'Organization_X', - }, - scale: { - factor: [ - 22, - 22, - 44.599998474121094, - ], - unit: 'nanometer', - }, - }, - dataStore: { - allowsUpload: true, - isScratch: false, - jobsEnabled: false, - jobsSupportedByAvailableWorkers: [], - name: 'localhost', - url: 'http://localhost:9000', - }, - description: null, + colorLayerNames: [], + created: 'created', displayName: null, folderId: '570b9f4e4bb848d0885ea917', + id: 'id', isActive: true, isEditable: true, - isPublic: false, isUnreported: false, lastUsedByUser: 0, - logoUrl: '/assets/images/mpi-logos.svg', - metadata: [ - { - key: 'key', - type: 'number', - value: 4, - }, - ], - name: 'confocal-multi_knossos', + name: 'test-dataset', owningOrganization: 'Organization_X', - publication: null, - sortingKey: 1508495293763, + segmentationLayerNames: [ + 'segmentation', + ], + status: '', tags: [], - usedStorageBytes: 0, }, + ] + +## getActiveDatasets + +> Snapshot 1 + + [ { - allowedTeams: [ - { - id: '570b9f4b2a7c0e3b008da6ec', - name: 'team_X1', - organization: 'Organization_X', - }, - ], - allowedTeamsCumulative: [ - { - id: '570b9f4b2a7c0e3b008da6ec', - name: 'team_X1', - organization: 'Organization_X', - }, - ], - created: 1508495293789, + allowedTeams: [], + allowedTeamsCumulative: [], + created: 'created', dataSource: { dataLayers: [ { boundingBox: { - depth: 1024, - height: 1024, + depth: 100, + height: 100, topLeft: [ - 3072, - 3072, - 512, + 50, + 50, + 25, ], - width: 1024, - }, - category: 'color', - elementClass: 'uint8', - name: 'color', - resolutions: [ - [ - 1, - 1, - 1, - ], - [ - 2, - 2, - 1, - ], - [ - 4, - 4, - 1, - ], - [ - 8, - 8, - 2, - ], - [ - 16, - 16, - 4, - ], - ], - }, - { - boundingBox: { - depth: 1024, - height: 1024, - topLeft: [ - 3072, - 3072, - 512, - ], - width: 1024, + width: 100, }, category: 'segmentation', elementClass: 'uint32', - largestSegmentId: 2504697, + largestSegmentId: 176, name: 'segmentation', resolutions: [ [ @@ -390,37 +157,14 @@ Generated by [AVA](https://avajs.dev). 1, 1, ], - [ - 2, - 2, - 1, - ], - [ - 4, - 4, - 1, - ], - [ - 8, - 8, - 2, - ], - [ - 16, - 16, - 4, - ], ], }, ], - id: { - name: 'l4_sample', - team: 'Organization_X', - }, + id: 'id', scale: { factor: [ - 11.239999771118164, - 11.239999771118164, + 11.24, + 11.24, 28, ], unit: 'nanometer', @@ -444,10 +188,10 @@ Generated by [AVA](https://avajs.dev). lastUsedByUser: 0, logoUrl: '/assets/images/mpi-logos.svg', metadata: [], - name: 'l4_sample', + name: 'test-dataset', owningOrganization: 'Organization_X', publication: null, - sortingKey: 1508495293789, + sortingKey: 'sortingKey', tags: [], usedStorageBytes: 0, }, @@ -461,24 +205,24 @@ Generated by [AVA](https://avajs.dev). { email: 'user_A@scalableminds.com', firstName: 'user_A', - id: '570b9f4d2a7c0e4d008da6ef', + id: 'id', isAdmin: true, isAnonymous: false, isDatasetManager: true, lastName: 'last_A', teams: [ { - id: '570b9f4b2a7c0e3b008da6ec', + id: 'id', isTeamManager: true, name: 'team_X1', }, { - id: '59882b370d889b84020efd3f', + id: 'id', isTeamManager: false, name: 'team_X3', }, { - id: '59882b370d889b84020efd6f', + id: 'id', isTeamManager: true, name: 'team_X4', }, @@ -487,35 +231,19 @@ Generated by [AVA](https://avajs.dev). { email: 'user_B@scalableminds.com', firstName: 'user_B', - id: '670b9f4d2a7c0e4d008da6ef', + id: 'id', isAdmin: false, isAnonymous: false, isDatasetManager: true, lastName: 'last_B', teams: [ { - id: '570b9f4b2a7c0e3b008da6ec', + id: 'id', isTeamManager: true, name: 'team_X1', }, ], }, - { - email: 'user_C@scalableminds.com', - firstName: 'user_C', - id: '770b9f4d2a7c0e4d008da6ef', - isAdmin: false, - isAnonymous: false, - isDatasetManager: false, - lastName: 'last_C', - teams: [ - { - id: '570b9f4b2a7c0e3b008da6ec', - isTeamManager: false, - name: 'team_X1', - }, - ], - }, ] ## updateDatasetTeams @@ -528,3 +256,23 @@ Generated by [AVA](https://avajs.dev). '59882b370d889b84020efd6f', '69882b370d889b84020efd4f', ] + +## Zarr streaming + +> Snapshot 1 + + '{"multiscales":[{"version":"0.4","name":"segmentation","axes":[{"name":"c","type":"channel"},{"name":"x","type":"space","unit":"nanometer"},{"name":"y","type":"space","unit":"nanometer"},{"name":"z","type":"space","unit":"nanometer"}],"datasets":[{"path":"1","coordinateTransformations":[{"type":"scale","scale":[1,11.24,11.24,28]}]}]}]}' + +> Snapshot 2 + + 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAA=' + +## Zarr 3 streaming + +> Snapshot 1 + + '{"zarr_format":3,"node_type":"group","attributes":{"ome":{"version":"0.5","multiscales":[{"name":"segmentation","axes":[{"name":"c","type":"channel"},{"name":"x","type":"space","unit":"nanometer"},{"name":"y","type":"space","unit":"nanometer"},{"name":"z","type":"space","unit":"nanometer"}],"datasets":[{"path":"1","coordinateTransformations":[{"type":"scale","scale":[1,11.24,11.24,28]}]}]}]}}}' + +> Snapshot 2 + + 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAABAAAAAQAAAAEAAAA=' diff --git a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap index 054bb7f37c2..a47da3e0e5c 100644 Binary files a/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap and b/frontend/javascripts/test/snapshots/public-test/test-bundle/test/backend-snapshot-tests/datasets.e2e.js.snap differ diff --git a/test/dataset/.gitignore b/test/dataset/.gitignore new file mode 100644 index 00000000000..867373c7958 --- /dev/null +++ b/test/dataset/.gitignore @@ -0,0 +1 @@ +!test-dataset.zip diff --git a/test/dataset/test-dataset.zip b/test/dataset/test-dataset.zip new file mode 100644 index 00000000000..16e1a3924d2 Binary files /dev/null and b/test/dataset/test-dataset.zip differ diff --git a/test/e2e/End2EndSpec.scala b/test/e2e/End2EndSpec.scala index 9a85437e684..1de30f63de7 100644 --- a/test/e2e/End2EndSpec.scala +++ b/test/e2e/End2EndSpec.scala @@ -1,5 +1,6 @@ package e2e +import com.scalableminds.util.io.ZipIO import com.typesafe.scalalogging.LazyLogging import org.scalatestplus.play.guice._ import org.specs2.main.Arguments @@ -8,6 +9,8 @@ import play.api.inject.guice.GuiceApplicationBuilder import play.api.libs.ws.{WSClient, WSResponse} import play.api.test.WithServer +import java.io.File +import java.nio.file.Paths import scala.concurrent.Await import scala.concurrent.duration._ import scala.sys.process._ @@ -27,6 +30,8 @@ class End2EndSpec(arguments: Arguments) extends Specification with GuiceFakeAppl "pass the e2e tests" in new WithServer(app = application, port = testPort) { + ensureTestDataset() + val resp: WSResponse = Await.result(ws.url(s"http://localhost:$testPort").get(), 2 seconds) resp.status === 200 @@ -43,4 +48,41 @@ class End2EndSpec(arguments: Arguments) extends Specification with GuiceFakeAppl customArgumentsMap.groupBy(_(0).substring(2)).view.mapValues(_(0).last).toMap } + private def ensureTestDataset(): Unit = { + val testDatasetPath = "test/dataset/test-dataset.zip" + val dataDirectory = new File("binaryData/Organization_X") + if (!dataDirectory.exists()) { + dataDirectory.mkdirs() + } + val testDatasetZip = new File(testDatasetPath) + if (!testDatasetZip.exists()) { + throw new Exception("Test dataset zip file does not exist.") + } + // Skip unzipping if the test dataset is already present + if (!dataDirectory.listFiles().exists(_.getName == "test-dataset")) + ZipIO.unzipToFolder( + testDatasetZip, + Paths.get(dataDirectory.toPath.toString, "test-dataset"), + includeHiddenFiles = true, + hiddenFilesWhitelist = List(), + truncateCommonPrefix = true, + excludeFromPrefix = None + ) + + // Test if the dataset was unzipped successfully + if (!dataDirectory.listFiles().exists(_.getName == "test-dataset")) { + throw new Exception("Test dataset was not unzipped successfully.") + } + val testFile = new File(dataDirectory, "test-dataset/datasource-properties.json") + if (!testFile.exists()) { + throw new Exception("Required file does not exist.") + } + val testFileSource = scala.io.Source.fromFile(testFile) + val testFileContent = try testFileSource.mkString + finally testFileSource.close() + if (testFileContent.isEmpty) { + throw new Exception("Required file is empty.") + } + } + }