Merge remote-tracking branch 'origin/master' into isosurface

* origin/master:
  Optimize performance for the list request /api/datasets (#3441)
  add annotation dataset foreign key  (#3482)
  thumbnails: correctly use zoom value if specified (#3487)
  Store Meshes in Postgres (#3367)
  fix alpha return (#3483)
  Added script to apply all new evolutions (#3427)
  Simple fix to speed up dataset gallery (#3480)
  better errors for screenshot tests, fix imports, refresh screenshots (#3479)
  (Backend only) Add project priority to progress report json (#3476)
  Handle missing write access on datastore (#3411)
  Re-introduce "Flightmode improvements"" (#3473)
  Circleci-notify: linkify PR number (#3469)
  Revert "Flightmode improvements" (#3472)
  also flow-ignore binaryData when using symlinks (#3471)
  Flightmode improvements (#3392)
  Circleci custom notification (#3465)
  enable /api/switch cross-organization (#3464)

.circleci/config.yml

@@ -212,10 +212,12 @@ jobs:
               docker push scalableminds/webknossos-dev:$CIRCLE_BRANCH
             fi
             docker logout
-
       - run:
           name: Report coverage
           command: .circleci/not-on-master.sh docker-compose run base yarn coverage
+      - run:
+          name: Send Slack notification (master only)
+          command: .circleci/slack-notification.sh
 
   nightly:
     docker:

.circleci/slack-notification.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [ "${CIRCLE_BRANCH}" == "master" ] ; then
+    author=${CIRCLE_USERNAME}
+    author=${author/fm3/<@florian>}
+    author=${author/jstriebel/<@jonathan>}
+    author=${author/daniel-wer/<@daniel>}
+    author=${author/georgwiese/<@georg>}
+    author=${author/hotzenklotz/<@tom>}
+    author=${author/jfrohnhofen/<@johannes>}
+    author=${author/MichaelBuessemeyer/<@michael>}
+    author=${author/normanrz/<@norman>}
+    author=${author/philippotto/<@philipp>}
+    author=${author/rschwanhold/<@robert>}
+    author=${author/tmbo/<@tmbo>}
+    author=${author/valentin-pinkau/<@valentin>}
+    channel="webknossos-bots"
+    commitmsg=$(git log --format=%s -n 1)
+    pullregex="(.*)#([0-9]+)(.*)"
+    while [[ $commitmsg =~ $pullregex ]]
+    do
+        commitmsg="${BASH_REMATCH[1]}#<https://github.com/scalableminds/webknossos/issues/${BASH_REMATCH[2]}|${BASH_REMATCH[2]}>${BASH_REMATCH[3]}"
+    done
+    buildlink="<https://circleci.com/gh/scalableminds/webknossos/${CIRCLE_BUILD_NUM}|${CIRCLE_BUILD_NUM}>"
+    mesg="${author} your ${CIRCLE_BRANCH} build ${buildlink} “${commitmsg}” is almost finished."
+    user="circleci-notify"
+    token="${SLACK_NOTIFY_TOKEN:-}"
+    res=$(curl -s \
+        -X POST \
+        -d "token=${token}" \
+        -d "channel=${channel}" \
+        -d "text=${mesg}" \
+        -d "username=${user}" \
+        -d "icon_url=https://a.slack-edge.com/41b0a/img/plugins/circleci/service_48.png" \
+        https://slack.com/api/chat.postMessage
+    )
+    if [[ "$(echo ${res} | jq '.ok')" == "false" ]]; then
+        echo "[WARN] Error sending Slack notification: $(echo ${res} | jq -r '.error')."
+    fi
+    echo "[INFO] Sent Slack notification to ${channel}."
+fi

.flowconfig

@@ -8,6 +8,7 @@
 <PROJECT_ROOT>/test/db/.*
 <PROJECT_ROOT>/userBinaryData/.*
 <PROJECT_ROOT>/binaryData/.*
+.*/binaryData/.*
 <PROJECT_ROOT>/conf/.*
 <PROJECT_ROOT>/lib/.*
 <PROJECT_ROOT>/public/.*

CHANGELOG.md

@@ -16,13 +16,15 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md).
 ### Changed
 
 - Improved support for datasets with a large skew in scale (e.g., [600, 600, 35]). [#3398](https://github.com/scalableminds/webknossos/pull/3398)
+- Improved performance for flight mode. [#3392](https://github.com/scalableminds/webknossos/pull/3392)
 
 ### Fixed
 
 - Fixed a bug where the initial onboarding setup failed if automatic initial data was disabled. [#3421](https://github.com/scalableminds/webknossos/pull/3421)
 - Fixed a permission issue in the try setup
 - Fixed a bug where the guessed bounding box for datasets that do not start at (0,0,0) was too large [#3437](https://github.com/scalableminds/webknossos/pull/3437)
 - Fixed a bug where dataset list refresh failed when datasets for non-existing organizations were reported. [#3438](https://github.com/scalableminds/webknossos/pull/3438)
+- Editing team access rights for datasets now works even if the datastore has no disk write access. [#3411](https://github.com/scalableminds/webknossos/pull/3411)
 - Fixed a bug where the form values when editing TaskTypes were missing. [#3451](https://github.com/scalableminds/webknossos/pull/3451)
 - Fixed a bug which caused RGB data to not render correctly. [#3455](https://github.com/scalableminds/webknossos/pull/3455)
 

MIGRATIONS.md

@@ -6,10 +6,11 @@ User-facing changes are documented in the [changelog](CHANGELOG.md).
 
 ## Unreleased
 - If additional dataset directories were watched using the config key `additionalFolders`, those symlinks are no longer updated. Consider setting up additional datastores for these directories respectively.
--
-### Postgres Evolutions:
-- [033-tasktype-recommendedConfiguration.sql](033-tasktype-recommendedConfiguration.sql)
 
+### Postgres Evolutions:
+- [033-tasktype-recommendedConfiguration.sql](conf/evolutions/033-tasktype-recommendedConfiguration.sql)
+- [034-meshes.sql](conf/evolutions/034-meshes.sql)
+- [035-add-annotation-dataset-foreign-key.sql](conf/evolutions/035-add-annotation-dataset-foreign-key.sql)
 
 ## [18.11.0](https://github.com/scalableminds/webknossos/releases/tag/18.11.0) - 2018-10-29
 - Some config keys have changed, if you overwrite them in your setup, please adapt: the `oxalis` prefix is renamed to `webKnossos` so the new keys are `webKnossos.user.time.tracingPauseInSeconds`, `webKnossos.tasks.maxOpenPerUser`, `webKnossos.newOrganizationMailingList` as well as `datastore.webKnossos.uri`, `datastore.webKnossos.secured`, `datastore.webKnossos.pingIntervalMinutes` for the data store.

app/assets/javascripts/admin/admin_rest_api.js

@@ -38,6 +38,8 @@ import {
   type DatasetConfig,
   type ExperienceDomainList,
   type HybridServerTracing,
+  type MeshMetaData,
+  type RemoteMeshMetaData,
   type ServerSkeletonTracing,
   type ServerTracing,
   type ServerVolumeTracing,
@@ -945,3 +947,43 @@ export function setMaintenance(bool: boolean): Promise<void> {
   return Request.triggerRequest("/api/maintenance", { method: bool ? "POST" : "DELETE" });
 }
 window.setMaintenance = setMaintenance;
+
+// ### Meshes
+
+type MeshMetaDataForCreation = $Diff<MeshMetaData, {| id: string |}>;
+
+export async function createMesh(
+  metadata: MeshMetaDataForCreation,
+  data: ArrayBuffer,
+): Promise<MeshMetaData> {
+  const mesh = await createMeshMetaData(metadata);
+  await updateMeshData(mesh.id, data);
+  return mesh;
+}
+
+function createMeshMetaData(metadata: MeshMetaDataForCreation): Promise<MeshMetaData> {
+  return Request.sendJSONReceiveJSON("/api/meshes", { method: "POST", data: metadata });
+}
+
+export async function updateMeshMetaData(metadata: RemoteMeshMetaData): Promise<void> {
+  return Request.sendJSONReceiveJSON(`/api/meshes/${metadata.id}`, {
+    method: "PUT",
+    data: metadata,
+  });
+}
+
+export async function updateMeshData(id: string, data: ArrayBuffer): Promise<void> {
+  return Request.sendJSONReceiveJSON(`/api/meshes/${id}/data`, { method: "PUT", data });
+}
+
+export function deleteMesh(id: string): Promise<void> {
+  return Request.triggerRequest(`/api/meshes/${id}`, { method: "DELETE" });
+}
+
+export function getMeshMetaData(id: string): Promise<MeshMetaData> {
+  return Request.receiveJSON(`/api/meshes/${id}`);
+}
+
+export function getMeshData(id: string): Promise<ArrayBuffer> {
+  return Request.receiveArraybuffer(`/api/meshes/${id}/data`);
+}

app/assets/javascripts/admin/api_flow_types.js

@@ -283,12 +283,31 @@ export type APIAnnotationTypeCompact = {
   +typ: APITracingType,
 };
 
+export type LocalMeshMetaData = {|
+  isVisible?: boolean,
+  isLoaded?: boolean,
+  isLoading?: boolean,
+|};
+
+export type RemoteMeshMetaData = {|
+  annotationId: string,
+  position: Vector3,
+  description: string,
+  id: string,
+|};
+
+export type MeshMetaData = {|
+  ...LocalMeshMetaData,
+  ...RemoteMeshMetaData,
+|};
+
 type APIAnnotationTypeBase = APIAnnotationTypeCompact & {
   +dataStore: APIDataStore,
   +tracingStore: APITracingStore,
   +restrictions: APIRestrictions,
   +settings: APISettings,
   +user?: APIUserBase,
+  +meshes: Array<MeshMetaData>,
 };
 
 export type APIAnnotation = APIAnnotationTypeBase & {

app/assets/javascripts/dashboard/dataset/dataset_import_view.js

@@ -232,14 +232,13 @@ class DatasetImportView extends React.PureComponent<Props, State> {
           }),
         );
       }
+      await updateDatasetTeams(dataset, teamIds);
 
       const dataSource = JSON.parse(formValues.dataSourceJson);
       if (this.state.dataset != null && !this.state.dataset.isForeign) {
         await updateDatasetDatasource(this.props.datasetId.name, dataset.dataStore.url, dataSource);
       }
 
-      await updateDatasetTeams(dataset, teamIds);
-
       const verb = this.props.isEditingMode ? "updated" : "imported";
       Toast.success(`Successfully ${verb} ${this.props.datasetId.name}`);
       datasetCache.clear();

app/assets/javascripts/dashboard/gallery_dataset_view.js

@@ -70,7 +70,7 @@ class GalleryDatasetView extends React.PureComponent<Props, State> {
         {groupedDatasets.map(([organization, datasets]) => (
           <DatasetPanel
             showOrganizationHeader={hasMultipleOrganizations}
-            croppedDatasetCount={hasMultipleOrganizations ? croppedDatasetCount : null}
+            croppedDatasetCount={croppedDatasetCount}
             className="dataset-panel"
             key={organization}
             organizationName={this.state.organizationNameMap[organization] || organization}

app/assets/javascripts/libs/input.js

@@ -324,6 +324,9 @@ export class InputMouse {
     _.extend(this, BackboneEvents);
     this.targetSelector = targetSelector;
     this.domElement = document.querySelector(targetSelector);
+    if (!this.domElement) {
+      throw new Error(`Input couldn't be attached to the following selector ${targetSelector}`);
+    }
     this.id = id;
 
     this.leftMouseButton = new InputMouseButton("left", 1, this, this.id);

app/assets/javascripts/libs/request.js

@@ -56,7 +56,10 @@ class Request {
       return options;
     }
 
-    let body = _.isString(options.data) ? options.data : JSON.stringify(options.data);
+    let body =
+      _.isString(options.data) || _.isArrayBuffer(options.data)
+        ? options.data
+        : JSON.stringify(options.data);
 
     if (options.compress) {
       body = await compress(body);

app/assets/javascripts/oxalis/constants.js

@@ -37,9 +37,17 @@ export const OrthoViews = {
   PLANE_XZ: "PLANE_XZ",
   TDView: "TDView",
 };
-export const ArbitraryViewport = "arbitraryViewport";
 export type OrthoView = $Keys<typeof OrthoViews>;
 export type OrthoViewMap<T> = { [key: OrthoView]: T };
+
+export const ArbitraryViewport = "arbitraryViewport";
+export const ArbitraryViews = {
+  arbitraryViewport: "arbitraryViewport",
+  TDView: "TDView",
+};
+export type ArbitraryView = $Keys<typeof ArbitraryViews>;
+export type ArbitraryViewMap<T> = { [key: ArbitraryView]: T };
+
 export type Viewport = OrthoView | typeof ArbitraryViewport;
 export const OrthoViewValues: Array<OrthoView> = Object.keys(OrthoViews);
 export const OrthoViewIndices = {

app/assets/javascripts/oxalis/controller.js

@@ -19,10 +19,10 @@ import { setViewModeAction, updateUserSettingAction } from "oxalis/model/actions
 import { wkReadyAction } from "oxalis/model/actions/actions";
 import ApiLoader from "oxalis/api/api_loader";
 import ArbitraryController from "oxalis/controller/viewmodes/arbitrary_controller";
+import { initializeSceneController } from "oxalis/controller/scene_controller";
 import Model from "oxalis/model";
 import NewTaskDescriptionModal from "oxalis/view/new_task_description_modal";
 import PlaneController from "oxalis/controller/viewmodes/plane_controller";
-import SceneController from "oxalis/controller/scene_controller";
 import Store, {
   type OxalisState,
   type TraceOrViewCommand,
@@ -120,7 +120,7 @@ class Controller extends React.PureComponent<Props, State> {
     window.onbeforeunload = beforeUnload;
 
     UrlManager.startUrlUpdater();
-    SceneController.initialize();
+    initializeSceneController();
 
     // FPS stats
     this.stats = new Stats();

app/assets/javascripts/oxalis/controller/combinations/skeletontracing_plane_controller.js

@@ -40,9 +40,9 @@ import {
 } from "oxalis/model/actions/skeletontracing_actions";
 import { setDirectionAction } from "oxalis/model/actions/flycam_actions";
 import type PlaneView from "oxalis/view/plane_view";
-import SceneController from "oxalis/controller/scene_controller";
 import Store from "oxalis/store";
 import api from "oxalis/api/internal_api";
+import getSceneController from "oxalis/controller/scene_controller_provider";
 
 const OrthoViewToNumber: OrthoViewMap<number> = {
   [OrthoViews.PLANE_XY]: 0,
@@ -131,6 +131,7 @@ function onClick(
     // do nothing
     return;
   }
+  const SceneController = getSceneController();
 
   // render the clicked viewport with picking enabled
   // we need a dedicated pickingScene, since we only want to render all nodes and no planes / bounding box / edges etc.

app/assets/javascripts/oxalis/controller/scene_controller.js

@@ -7,6 +7,7 @@ import BackboneEvents from "backbone-events-standalone";
 import * as THREE from "three";
 import _ from "lodash";
 
+import type { MeshMetaData } from "admin/api_flow_types";
 import { V3 } from "libs/mjs";
 import { getBoundaries } from "oxalis/model/accessors/dataset_accessor";
 import {
@@ -37,9 +38,10 @@ import constants, {
   OrthoViews,
   type Vector3,
 } from "oxalis/constants";
-import parseStlBuffer from "libs/parse_stl_buffer";
 import window from "libs/window";
 
+import { setSceneController } from "./scene_controller_provider";
+
 const CUBE_COLOR = 0x999999;
 
 class SceneController {
@@ -56,6 +58,7 @@ class SceneController {
   renderer: THREE.WebGLRenderer;
   scene: THREE.Scene;
   rootGroup: THREE.Object3D;
+  stlMeshes: { [key: string]: THREE.Mesh };
 
   // This class collects all the meshes displayed in the Skeleton View and updates position and scale of each
   // element depending on the provided flycam.
@@ -68,6 +71,7 @@ class SceneController {
       [OrthoViews.PLANE_XZ]: true,
     };
     this.planeShift = [0, 0, 0];
+    this.stlMeshes = {};
   }
 
   initialize() {
@@ -116,12 +120,35 @@ class SceneController {
     window.removeBucketMesh = (mesh: THREE.LineSegments) => this.rootNode.remove(mesh);
   }
 
-  addSTL(stlBuffer: ArrayBuffer): void {
-    const geometry = parseStlBuffer(stlBuffer);
+  addSTL(meshMetaData: MeshMetaData, geometry: THREE.Geometry): void {
+    const { id, position } = meshMetaData;
+    if (this.stlMeshes[id] != null) {
+      console.warn(`Mesh with id ${id} has already been added to the scene.`);
+      return;
+    }
     geometry.computeVertexNormals();
 
     const meshMaterial = new THREE.MeshNormalMaterial();
-    this.rootGroup.add(new THREE.Mesh(geometry, meshMaterial));
+    const mesh = new THREE.Mesh(geometry, meshMaterial);
+    this.rootGroup.add(mesh);
+    this.stlMeshes[id] = mesh;
+    this.updateMeshPostion(id, position);
+  }
+
+  removeSTL(id: string): void {
+    this.rootGroup.remove(this.stlMeshes[id]);
+  }
+
+  setMeshVisibility(id: string, visibility: boolean): void {
+    this.stlMeshes[id].visible = visibility;
+  }
+
+  updateMeshPostion(id: string, position: Vector3): void {
+    const [x, y, z] = position;
+    const mesh = this.stlMeshes[id];
+    mesh.position.x = x;
+    mesh.position.y = y;
+    mesh.position.z = z;
   }
 
   createMeshes(): void {
@@ -370,4 +397,14 @@ class SceneController {
   }
 }
 
-export default new SceneController();
+export type SceneControllerType = SceneController;
+
+export function initializeSceneController() {
+  const controller = new SceneController();
+  setSceneController(controller);
+  controller.initialize();
+}
+
+// Please use scene_controller_provider to get a reference to SceneController. This avoids
+// problems with circular dependencies.
+export default {};

app/assets/javascripts/oxalis/controller/scene_controller_provider.js

@@ -0,0 +1,17 @@
+// @flow
+
+import type { SceneControllerType } from "./scene_controller";
+
+let sceneController: ?SceneControllerType = null;
+
+export default function getSceneController(): SceneControllerType {
+  if (!sceneController) {
+    throw new Error("SceneController was not initialized yet");
+  }
+
+  return sceneController;
+}
+
+export function setSceneController(c: SceneControllerType): void {
+  sceneController = c;
+}