diff --git a/CHANGELOG.md b/CHANGELOG.md index 50962649b94..ea3e47a02d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.md). ### Changed - Improved support for datasets with a large skew in scale (e.g., [600, 600, 35]). [#3398](https://github.com/scalableminds/webknossos/pull/3398) +- Improved performance for flight mode. [#3392](https://github.com/scalableminds/webknossos/pull/3392) ### Fixed diff --git a/app/assets/javascripts/libs/input.js b/app/assets/javascripts/libs/input.js index d504f718ec0..3cac88bb4ea 100644 --- a/app/assets/javascripts/libs/input.js +++ b/app/assets/javascripts/libs/input.js @@ -324,6 +324,9 @@ export class InputMouse { _.extend(this, BackboneEvents); this.targetSelector = targetSelector; this.domElement = document.querySelector(targetSelector); + if (!this.domElement) { + throw new Error(`Input couldn't be attached to the following selector ${targetSelector}`); + } this.id = id; this.leftMouseButton = new InputMouseButton("left", 1, this, this.id); diff --git a/app/assets/javascripts/oxalis/constants.js b/app/assets/javascripts/oxalis/constants.js index c5009c0f248..aa881dae7a8 100644 --- a/app/assets/javascripts/oxalis/constants.js +++ b/app/assets/javascripts/oxalis/constants.js @@ -37,9 +37,17 @@ export const OrthoViews = { PLANE_XZ: "PLANE_XZ", TDView: "TDView", }; -export const ArbitraryViewport = "arbitraryViewport"; export type OrthoView = $Keys; export type OrthoViewMap = { [key: OrthoView]: T }; + +export const ArbitraryViewport = "arbitraryViewport"; +export const ArbitraryViews = { + arbitraryViewport: "arbitraryViewport", + TDView: "TDView", +}; +export type ArbitraryView = $Keys; +export type ArbitraryViewMap = { [key: ArbitraryView]: T }; + export type Viewport = OrthoView | typeof ArbitraryViewport; export const OrthoViewValues: Array = Object.keys(OrthoViews); export const OrthoViewIndices = { diff --git a/app/assets/javascripts/oxalis/controller/td_controller.js b/app/assets/javascripts/oxalis/controller/td_controller.js new file mode 100644 index 00000000000..3d64583a49a --- /dev/null +++ b/app/assets/javascripts/oxalis/controller/td_controller.js @@ -0,0 +1,216 @@ +// @flow +import * as React from "react"; +import * as Utils from "libs/utils"; +import { InputMouse } from "libs/input"; +import { getViewportScale, getInputCatcherRect } from "oxalis/model/accessors/view_mode_accessor"; +import CameraController from "oxalis/controller/camera_controller"; +import { voxelToNm } from "oxalis/model/scaleinfo"; +import TrackballControls from "libs/trackball_controls"; +import Store from "oxalis/store"; +import type { OxalisState, Flycam, CameraData, Tracing } from "oxalis/store"; +import * as THREE from "three"; +import { OrthoViews, type Vector3 } from "oxalis/constants"; +import type { Point2, OrthoViewMap } from "oxalis/constants"; +import { connect } from "react-redux"; +import { + setViewportAction, + setTDCameraAction, + zoomTDViewAction, + moveTDViewXAction, + moveTDViewYAction, + moveTDViewByVectorAction, +} from "oxalis/model/actions/view_mode_actions"; +import PlaneView from "oxalis/view/plane_view"; +import { getPosition } from "oxalis/model/accessors/flycam_accessor"; +import * as skeletonController from "oxalis/controller/combinations/skeletontracing_plane_controller"; + +export function threeCameraToCameraData(camera: THREE.OrthographicCamera): CameraData { + const { position, up, near, far, lookAt, left, right, top, bottom } = camera; + const objToArr = ({ x, y, z }) => [x, y, z]; + return { + left, + right, + top, + bottom, + near, + far, + position: objToArr(position), + up: objToArr(up), + lookAt: objToArr(lookAt), + }; +} + +type OwnProps = {| + cameras: OrthoViewMap, + planeView?: PlaneView, + tracing?: Tracing, +|}; + +type Props = { + ...OwnProps, + flycam: Flycam, + scale: Vector3, +}; + +class TDController extends React.PureComponent { + controls: TrackballControls; + mouseController: InputMouse; + oldNmPos: Vector3; + isStarted: boolean; + + componentDidMount() { + const { dataset, flycam } = Store.getState(); + this.oldNmPos = voxelToNm(dataset.dataSource.scale, getPosition(flycam)); + this.isStarted = true; + + this.initMouse(); + } + + componentWillUnmount() { + this.isStarted = false; + if (this.mouseController != null) { + this.mouseController.destroy(); + } + if (this.controls != null) { + this.controls.destroy(); + } + } + + initMouse(): void { + const tdView = OrthoViews.TDView; + const inputcatcherSelector = `#inputcatcher_${tdView}`; + Utils.waitForSelector(inputcatcherSelector).then(view => { + if (!this.isStarted) { + return; + } + this.mouseController = new InputMouse( + inputcatcherSelector, + this.getTDViewMouseControls(), + tdView, + ); + this.initTrackballControls(view); + }); + } + + initTrackballControls(view): void { + const pos = voxelToNm(this.props.scale, getPosition(this.props.flycam)); + const tdCamera = this.props.cameras[OrthoViews.TDView]; + this.controls = new TrackballControls(tdCamera, view, new THREE.Vector3(...pos), () => { + // write threeJS camera into store + Store.dispatch(setTDCameraAction(threeCameraToCameraData(tdCamera))); + }); + + this.controls.noZoom = true; + this.controls.noPan = true; + this.controls.staticMoving = true; + + this.controls.target.set(...pos); + + // This is necessary, since we instantiated this.controls now. This should be removed + // when the workaround with requestAnimationFrame(initInputHandlers) is removed. + this.forceUpdate(); + } + + updateControls = () => this.controls.update(true); + + getTDViewMouseControls(): Object { + const baseControls = { + leftDownMove: (delta: Point2) => this.moveTDView(delta), + scroll: (value: number) => this.zoomTDView(Utils.clamp(-1, value, 1), true), + over: () => { + Store.dispatch(setViewportAction(OrthoViews.TDView)); + // Fix the rotation target of the TrackballControls + this.setTargetAndFixPosition(); + }, + pinch: delta => this.zoomTDView(delta, true), + }; + + const skeletonControls = + this.props.tracing != null && + this.props.tracing.skeleton != null && + this.props.planeView != null + ? skeletonController.getTDViewMouseControls(this.props.planeView) + : {}; + + return { + ...baseControls, + ...skeletonControls, + }; + } + + setTargetAndFixPosition(): void { + const position = getPosition(this.props.flycam); + const nmPosition = voxelToNm(this.props.scale, position); + + this.controls.target.set(...nmPosition); + this.controls.update(); + + // The following code is a dirty hack. If someone figures out + // how the trackball control's target can be set without affecting + // the camera position, go ahead. + // As the previous step will also move the camera, we need to + // fix this by offsetting the viewport + + const invertedDiff = []; + for (let i = 0; i <= 2; i++) { + invertedDiff.push(this.oldNmPos[i] - nmPosition[i]); + } + + if (invertedDiff.every(el => el === 0)) return; + + this.oldNmPos = nmPosition; + + const nmVector = new THREE.Vector3(...invertedDiff); + // moves camera by the nm vector + const camera = this.props.cameras[OrthoViews.TDView]; + + const rotation = THREE.Vector3.prototype.multiplyScalar.call(camera.rotation.clone(), -1); + // reverse euler order + rotation.order = rotation.order + .split("") + .reverse() + .join(""); + + nmVector.applyEuler(rotation); + + Store.dispatch(moveTDViewByVectorAction(nmVector.x, nmVector.y)); + } + + zoomTDView(value: number, zoomToMouse: boolean = true): void { + let zoomToPosition; + if (zoomToMouse && this.mouseController) { + zoomToPosition = this.mouseController.position; + } + const { width } = getInputCatcherRect(OrthoViews.TDView); + Store.dispatch(zoomTDViewAction(value, zoomToPosition, width)); + } + + moveTDView(delta: Point2): void { + const scale = getViewportScale(OrthoViews.TDView); + Store.dispatch(moveTDViewXAction((delta.x / scale) * -1)); + Store.dispatch(moveTDViewYAction((delta.y / scale) * -1)); + } + + render() { + if (!this.controls) { + return null; + } + + return ( + + ); + } +} + +export function mapStateToProps(state: OxalisState, ownProps: OwnProps): Props { + return { + ...ownProps, + flycam: state.flycam, + scale: state.dataset.dataSource.scale, + }; +} + +export default connect(mapStateToProps)(TDController); diff --git a/app/assets/javascripts/oxalis/controller/viewmodes/arbitrary_controller.js b/app/assets/javascripts/oxalis/controller/viewmodes/arbitrary_controller.js index 7e2e0272b84..c8445a731c6 100644 --- a/app/assets/javascripts/oxalis/controller/viewmodes/arbitrary_controller.js +++ b/app/assets/javascripts/oxalis/controller/viewmodes/arbitrary_controller.js @@ -28,8 +28,8 @@ import { toggleInactiveTreesAction, } from "oxalis/model/actions/skeletontracing_actions"; import { - updateUserSettingAction, setFlightmodeRecordingAction, + updateUserSettingAction, } from "oxalis/model/actions/settings_actions"; import { yawFlycamAction, @@ -44,6 +44,7 @@ import Crosshair from "oxalis/geometries/crosshair"; import Model from "oxalis/model"; import SceneController from "oxalis/controller/scene_controller"; import Store from "oxalis/store"; +import TDController from "oxalis/controller/td_controller"; import Toast from "libs/toast"; import * as Utils from "libs/utils"; import api from "oxalis/api/internal_api"; @@ -53,10 +54,10 @@ import messages from "messages"; const arbitraryViewportSelector = "#inputcatcher_arbitraryViewport"; -type Props = { +type Props = {| onRender: () => void, viewMode: Mode, -}; +|}; class ArbitraryController extends React.PureComponent { // See comment in Controller class on general controller architecture. @@ -68,7 +69,7 @@ class ArbitraryController extends React.PureComponent { crosshair: Crosshair; lastNodeMatrix: Matrix4x4; input: { - mouse?: InputMouse, + mouseController: ?InputMouse, keyboard?: InputKeyboard, keyboardLoopDelayed?: InputKeyboard, keyboardNoLoop?: InputKeyboardNoLoop, @@ -81,7 +82,9 @@ class ArbitraryController extends React.PureComponent { componentDidMount() { _.extend(this, BackboneEvents); - this.input = {}; + this.input = { + mouseController: null, + }; this.storePropertyUnsubscribers = []; this.start(); } @@ -92,7 +95,7 @@ class ArbitraryController extends React.PureComponent { initMouse(): void { Utils.waitForSelector(arbitraryViewportSelector).then(() => { - this.input.mouse = new InputMouse(arbitraryViewportSelector, { + this.input.mouseController = new InputMouse(arbitraryViewportSelector, { leftDownMove: (delta: Point2) => { if (this.props.viewMode === constants.MODE_ARBITRARY) { Store.dispatch( @@ -221,6 +224,7 @@ class ArbitraryController extends React.PureComponent { // Rotate view by 180 deg r: () => { Store.dispatch(yawFlycamAction(Math.PI)); + window.needsRerender = true; }, // Delete active node and recenter last node @@ -323,6 +327,7 @@ class ArbitraryController extends React.PureComponent { this.crosshair.setVisibility(Store.getState().userConfiguration.displayCrosshair); this.arbitraryView.addGeometry(this.plane); + this.arbitraryView.setArbitraryPlane(this.plane); this.arbitraryView.addGeometry(this.crosshair); this.bindToEvents(); @@ -337,6 +342,7 @@ class ArbitraryController extends React.PureComponent { this.arbitraryView.draw(); this.isStarted = true; + this.forceUpdate(); } unsubscribeStoreListeners() { @@ -365,7 +371,7 @@ class ArbitraryController extends React.PureComponent { }; destroyInput() { - Utils.__guard__(this.input.mouse, x => x.destroy()); + Utils.__guard__(this.input.mouseController, x => x.destroy()); Utils.__guard__(this.input.keyboard, x => x.destroy()); Utils.__guard__(this.input.keyboardLoopDelayed, x => x.destroy()); Utils.__guard__(this.input.keyboardNoLoop, x => x.destroy()); @@ -434,7 +440,10 @@ class ArbitraryController extends React.PureComponent { } render() { - return null; + if (!this.arbitraryView) { + return null; + } + return ; } } diff --git a/app/assets/javascripts/oxalis/controller/viewmodes/plane_controller.js b/app/assets/javascripts/oxalis/controller/viewmodes/plane_controller.js index 20d363ac484..a38ea16466c 100644 --- a/app/assets/javascripts/oxalis/controller/viewmodes/plane_controller.js +++ b/app/assets/javascripts/oxalis/controller/viewmodes/plane_controller.js @@ -7,11 +7,11 @@ import { connect } from "react-redux"; import BackboneEvents from "backbone-events-standalone"; import Clipboard from "clipboard-js"; import * as React from "react"; -import * as THREE from "three"; import _ from "lodash"; import { InputKeyboard, InputKeyboardNoLoop, InputMouse, type ModifierKeys } from "libs/input"; import { document } from "libs/window"; +import { getBaseVoxel, getBaseVoxelFactors } from "oxalis/model/scaleinfo"; import { getPosition, getRequestLogZoomStep, @@ -30,30 +30,20 @@ import { setBrushSizeAction, setMousePositionAction, } from "oxalis/model/actions/volumetracing_actions"; -import { - setViewportAction, - setTDCameraAction, - zoomTDViewAction, - moveTDViewXAction, - moveTDViewYAction, - moveTDViewByVectorAction, -} from "oxalis/model/actions/view_mode_actions"; +import { setViewportAction, zoomTDViewAction } from "oxalis/model/actions/view_mode_actions"; import { updateUserSettingAction } from "oxalis/model/actions/settings_actions"; -import { voxelToNm, getBaseVoxel, getBaseVoxelFactors } from "oxalis/model/scaleinfo"; -import CameraController from "oxalis/controller/camera_controller"; import Dimensions from "oxalis/model/dimensions"; import Model from "oxalis/model"; import PlaneView from "oxalis/view/plane_view"; import SceneController from "oxalis/controller/scene_controller"; -import Store, { type CameraData, type Flycam, type OxalisState, type Tracing } from "oxalis/store"; +import Store, { type OxalisState, type Tracing } from "oxalis/store"; +import TDController from "oxalis/controller/td_controller"; import Toast from "libs/toast"; -import TrackballControls from "libs/trackball_controls"; import * as Utils from "libs/utils"; import api from "oxalis/api/internal_api"; import constants, { type OrthoView, type OrthoViewMap, - OrthoViewValues, OrthoViewValuesWithoutTDView, OrthoViews, type Point2, @@ -83,8 +73,6 @@ type OwnProps = { }; type Props = OwnProps & { - flycam: Flycam, - scale: Vector3, tracing: Tracing, }; @@ -102,9 +90,7 @@ class PlaneController extends React.PureComponent { }; storePropertyUnsubscribers: Array; isStarted: boolean; - oldNmPos: Vector3; zoomPos: Vector3; - controls: TrackballControls; // Copied from backbone events (TODO: handle this better) listenTo: Function; stopListening: Function; @@ -121,13 +107,10 @@ class PlaneController extends React.PureComponent { }; this.isStarted = false; - const state = Store.getState(); - this.oldNmPos = voxelToNm(state.dataset.dataSource.scale, getPosition(state.flycam)); - this.planeView = new PlaneView(); + this.forceUpdate(); Store.dispatch(setViewportAction(OrthoViews.PLANE_XY)); - this.start(); } @@ -136,41 +119,29 @@ class PlaneController extends React.PureComponent { } initMouse(): void { - OrthoViewValues.forEach(id => { - const inputcatcherSelector = `#inputcatcher_${OrthoViews[id]}`; - Utils.waitForSelector(inputcatcherSelector).then(() => { - this.input.mouseControllers[id] = new InputMouse( - inputcatcherSelector, - id !== OrthoViews.TDView ? this.getPlaneMouseControls(id) : this.getTDViewMouseControls(), - id, - ); + // Workaround: We are only waiting for tdview since this + // introduces the necessary delay to attach the events to the + // newest input catchers. We should refactor the + // InputMouse handling so that this is not necessary anymore. + // See: https://github.com/scalableminds/webknossos/issues/3475 + const tdSelector = `#inputcatcher_${OrthoViews.TDView}`; + Utils.waitForSelector(tdSelector).then(() => { + OrthoViewValuesWithoutTDView.forEach(id => { + const inputcatcherSelector = `#inputcatcher_${OrthoViews[id]}`; + Utils.waitForSelector(inputcatcherSelector).then(el => { + if (!document.body.contains(el)) { + console.error("el is not attached anymore"); + } + this.input.mouseControllers[id] = new InputMouse( + inputcatcherSelector, + this.getPlaneMouseControls(id), + id, + ); + }); }); }); } - getTDViewMouseControls(): Object { - const baseControls = { - leftDownMove: (delta: Point2) => this.moveTDView(delta), - scroll: (value: number) => this.zoomTDView(Utils.clamp(-1, value, 1), true), - over: () => { - Store.dispatch(setViewportAction(OrthoViews.TDView)); - // Fix the rotation target of the TrackballControls - this.setTargetAndFixPosition(); - }, - pinch: delta => this.zoomTDView(delta, true), - }; - - const skeletonControls = - this.props.tracing.skeleton != null - ? skeletonController.getTDViewMouseControls(this.planeView) - : {}; - - return { - ...baseControls, - ...skeletonControls, - }; - } - getPlaneMouseControls(planeId: OrthoView): Object { const baseControls = { leftDownMove: (delta: Point2) => { @@ -209,65 +180,6 @@ class PlaneController extends React.PureComponent { }; } - setTargetAndFixPosition(): void { - const position = getPosition(this.props.flycam); - const nmPosition = voxelToNm(this.props.scale, position); - - this.controls.target.set(...nmPosition); - this.controls.update(); - - // The following code is a dirty hack. If someone figures out - // how the trackball control's target can be set without affecting - // the camera position, go ahead. - // As the previous step will also move the camera, we need to - // fix this by offsetting the viewport - - const invertedDiff = []; - for (let i = 0; i <= 2; i++) { - invertedDiff.push(this.oldNmPos[i] - nmPosition[i]); - } - - if (invertedDiff.every(el => el === 0)) return; - - this.oldNmPos = nmPosition; - - const nmVector = new THREE.Vector3(...invertedDiff); - // moves camera by the nm vector - const camera = this.planeView.getCameras()[OrthoViews.TDView]; - - const rotation = THREE.Vector3.prototype.multiplyScalar.call(camera.rotation.clone(), -1); - // reverse euler order - rotation.order = rotation.order - .split("") - .reverse() - .join(""); - - nmVector.applyEuler(rotation); - - Store.dispatch(moveTDViewByVectorAction(nmVector.x, nmVector.y)); - } - - initTrackballControls(): void { - Utils.waitForSelector("#inputcatcher_TDView").then(view => { - const pos = voxelToNm(this.props.scale, getPosition(this.props.flycam)); - const tdCamera = this.planeView.getCameras()[OrthoViews.TDView]; - this.controls = new TrackballControls(tdCamera, view, new THREE.Vector3(...pos), () => { - // write threeJS camera into store - Store.dispatch(setTDCameraAction(threeCameraToCameraData(tdCamera))); - }); - - this.controls.noZoom = true; - this.controls.noPan = true; - this.controls.staticMoving = true; - - this.controls.target.set(...pos); - - // This is necessary, since we instantiated this.controls now. This should be removed - // when the workaround with requestAnimationFrame(initInputHandlers) is removed. - this.forceUpdate(); - }); - } - initKeyboard(): void { // avoid scrolling while pressing space document.addEventListener("keydown", (event: KeyboardEvent) => { @@ -398,25 +310,14 @@ class PlaneController extends React.PureComponent { this.planeView.start(); this.initKeyboard(); + this.initMouse(); this.init(); this.isStarted = true; - - // Workaround: defer mouse initialization to make sure DOM elements have - // actually been rendered by React (InputCatchers Component) - // DOM Elements get deleted when switching between ortho and arbitrary mode - - Utils.waitForSelector("#inputcatcher_TDView").then(() => { - if (this.isStarted) { - this.initTrackballControls(); - this.initMouse(); - } - }); } stop(): void { if (this.isStarted) { this.destroyInput(); - this.controls.destroy(); } SceneController.stopPlaneMode(); @@ -475,7 +376,7 @@ class PlaneController extends React.PureComponent { if (OrthoViewValuesWithoutTDView.includes(activeViewport)) { this.zoomPlanes(value, zoomToMouse); } else { - this.zoomTDView(value, zoomToMouse); + this.zoomTDView(value); } } @@ -491,21 +392,12 @@ class PlaneController extends React.PureComponent { } } - zoomTDView(value: number, zoomToMouse: boolean = true): void { - let zoomToPosition; - if (zoomToMouse) { - zoomToPosition = this.input.mouseControllers[OrthoViews.TDView].position; - } + zoomTDView(value: number): void { + const zoomToPosition = null; const { width } = getInputCatcherRect(OrthoViews.TDView); Store.dispatch(zoomTDViewAction(value, zoomToPosition, width)); } - moveTDView(delta: Point2): void { - const scale = getViewportScale(OrthoViews.TDView); - Store.dispatch(moveTDViewXAction((delta.x / scale) * -1)); - Store.dispatch(moveTDViewYAction((delta.y / scale) * -1)); - } - finishZoom = (): void => { // Move the plane so that the mouse is at the same position as // before the zoom @@ -600,8 +492,6 @@ class PlaneController extends React.PureComponent { this.unsubscribeStoreListeners(); } - updateControls = () => this.controls.update(true); - createToolDependentHandler(skeletonHandler: ?Function, volumeHandler: ?Function): Function { return (...args) => { if (skeletonHandler && volumeHandler) { @@ -620,35 +510,20 @@ class PlaneController extends React.PureComponent { } render() { - if (!this.controls) { + if (!this.planeView) { return null; } return ( - ); } } -function threeCameraToCameraData(camera: THREE.OrthographicCamera): CameraData { - const { position, up, near, far, lookAt, left, right, top, bottom } = camera; - const objToArr = ({ x, y, z }) => [x, y, z]; - return { - left, - right, - top, - bottom, - near, - far, - position: objToArr(position), - up: objToArr(up), - lookAt: objToArr(lookAt), - }; -} - export function calculateGlobalPos(clickPos: Point2): Vector3 { let position; const state = Store.getState(); @@ -697,8 +572,6 @@ export function calculateGlobalPos(clickPos: Point2): Vector3 { export function mapStateToProps(state: OxalisState, ownProps: OwnProps): Props { return { - flycam: state.flycam, - scale: state.dataset.dataSource.scale, onRender: ownProps.onRender, tracing: state.tracing, }; diff --git a/app/assets/javascripts/oxalis/geometries/arbitrary_plane.js b/app/assets/javascripts/oxalis/geometries/arbitrary_plane.js index 2dc23030924..982ab4a52c9 100644 --- a/app/assets/javascripts/oxalis/geometries/arbitrary_plane.js +++ b/app/assets/javascripts/oxalis/geometries/arbitrary_plane.js @@ -2,14 +2,17 @@ * arbitrary_plane.js * @flow */ - import * as THREE from "three"; +import _ from "lodash"; import { getZoomedMatrix } from "oxalis/model/accessors/flycam_accessor"; import PlaneMaterialFactory from "oxalis/geometries/materials/plane_material_factory"; import SceneController from "oxalis/controller/scene_controller"; +// Importing throttled_store, would result in flickering when zooming out, +// since the plane is not updated fast enough import Store from "oxalis/store"; import constants, { OrthoViews, type Vector4 } from "oxalis/constants"; +import shaderEditor from "oxalis/model/helpers/shader_editor"; // Let's set up our trianglesplane. // It serves as a "canvas" where the brain images @@ -24,15 +27,22 @@ import constants, { OrthoViews, type Vector4 } from "oxalis/constants"; // attached to bend surface. // The result is then projected on a flat surface. +const renderDebuggerPlane = true; + +type ArbitraryMeshes = {| + mainPlane: THREE.Mesh, + debuggerPlane: ?THREE.Mesh, +|}; + class ArbitraryPlane { - mesh: THREE.Mesh; + meshes: ArbitraryMeshes; isDirty: boolean; stopStoreListening: () => void; materialFactory: PlaneMaterialFactory; constructor() { this.isDirty = true; - this.mesh = this.createMesh(); + this.meshes = this.createMeshes(); this.stopStoreListening = Store.subscribe(() => { this.isDirty = true; @@ -46,68 +56,138 @@ class ArbitraryPlane { updateAnchorPoints(anchorPoint: ?Vector4, fallbackAnchorPoint: ?Vector4): void { if (anchorPoint) { - this.mesh.material.setAnchorPoint(anchorPoint); + this.meshes.mainPlane.material.setAnchorPoint(anchorPoint); } if (fallbackAnchorPoint) { - this.mesh.material.setFallbackAnchorPoint(fallbackAnchorPoint); + this.meshes.mainPlane.material.setFallbackAnchorPoint(fallbackAnchorPoint); } } setPosition = ({ x, y, z }: THREE.Vector3) => { - this.mesh.material.setGlobalPosition([x, y, z]); + this.meshes.mainPlane.material.setGlobalPosition([x, y, z]); }; addToScene(scene: THREE.Scene) { - scene.add(this.mesh); + _.values(this.meshes).forEach(mesh => { + if (mesh) { + scene.add(mesh); + } + }); } update() { if (this.isDirty) { - const { mesh } = this; - const matrix = getZoomedMatrix(Store.getState().flycam); - mesh.matrix.set( - matrix[0], - matrix[4], - matrix[8], - matrix[12], - matrix[1], - matrix[5], - matrix[9], - matrix[13], - matrix[2], - matrix[6], - matrix[10], - matrix[14], - matrix[3], - matrix[7], - matrix[11], - matrix[15], - ); - - mesh.matrix.multiply(new THREE.Matrix4().makeRotationY(Math.PI)); - mesh.matrixWorldNeedsUpdate = true; + const updateMesh = mesh => { + if (!mesh) { + return; + } + mesh.matrix.set( + matrix[0], + matrix[4], + matrix[8], + matrix[12], + matrix[1], + matrix[5], + matrix[9], + matrix[13], + matrix[2], + matrix[6], + matrix[10], + matrix[14], + matrix[3], + matrix[7], + matrix[11], + matrix[15], + ); + + mesh.matrix.multiply(new THREE.Matrix4().makeRotationY(Math.PI)); + mesh.matrixWorldNeedsUpdate = true; + }; + + _.values(this.meshes).forEach(updateMesh); this.isDirty = false; SceneController.update(this); } } - createMesh() { + createMeshes(): ArbitraryMeshes { + const adaptPlane = _plane => { + _plane.rotation.x = Math.PI; + _plane.matrixAutoUpdate = false; + _plane.material.side = THREE.DoubleSide; + return _plane; + }; + this.materialFactory = new PlaneMaterialFactory(OrthoViews.PLANE_XY, false, 4); const textureMaterial = this.materialFactory.setup().getMaterial(); - const plane = new THREE.Mesh( - new THREE.PlaneGeometry(constants.VIEWPORT_WIDTH, constants.VIEWPORT_WIDTH, 1, 1), - textureMaterial, + const mainPlane = adaptPlane( + new THREE.Mesh( + new THREE.PlaneGeometry(constants.VIEWPORT_WIDTH, constants.VIEWPORT_WIDTH, 1, 1), + textureMaterial, + ), ); - plane.rotation.x = Math.PI; - plane.matrixAutoUpdate = false; - plane.doubleSided = true; + const debuggerPlane = renderDebuggerPlane ? adaptPlane(this.createDebuggerPlane()) : null; - return plane; + return { + mainPlane, + debuggerPlane, + }; + } + + createDebuggerPlane() { + const debuggerMaterial = new THREE.ShaderMaterial({ + uniforms: this.materialFactory.uniforms, + vertexShader: ` + uniform float sphericalCapRadius; + varying vec3 vNormal; + varying float isBorder; + + void main() { + vec3 centerVertex = vec3(0.0, 0.0, -sphericalCapRadius); + vec3 _position = position; + _position += centerVertex; + _position = _position * (sphericalCapRadius / length(_position)); + _position -= centerVertex; + + isBorder = mod(floor(position.x * 1.0), 2.0) + mod(floor(position.y * 1.0), 2.0) > 0.0 ? 1.0 : 0.0; + + gl_Position = projectionMatrix * + modelViewMatrix * + vec4(_position,1.0); + vNormal = normalize((modelViewMatrix * vec4(_position, 1.0)).xyz); + } + `, + fragmentShader: ` + varying mediump vec3 vNormal; + varying float isBorder; + void main() { + mediump vec3 light = vec3(0.5, 0.2, 1.0); + + // ensure it's normalized + light = normalize(light); + + // calculate the dot product of + // the light to the vertex normal + mediump float dProd = max(0.0, dot(vNormal, light)); + + gl_FragColor = 1.0 - isBorder < 0.001 ? vec4(vec3(dProd, 1.0, 0.0), 0.9) : vec4(0.0); + } + `, + }); + debuggerMaterial.transparent = true; + + shaderEditor.addMaterial(99, debuggerMaterial); + + const debuggerPlane = new THREE.Mesh( + new THREE.PlaneGeometry(constants.VIEWPORT_WIDTH, constants.VIEWPORT_WIDTH, 50, 50), + debuggerMaterial, + ); + return debuggerPlane; } } diff --git a/app/assets/javascripts/oxalis/geometries/materials/plane_material_factory.js b/app/assets/javascripts/oxalis/geometries/materials/plane_material_factory.js index d3af093fb44..1e0c9440d5b 100644 --- a/app/assets/javascripts/oxalis/geometries/materials/plane_material_factory.js +++ b/app/assets/javascripts/oxalis/geometries/materials/plane_material_factory.js @@ -155,6 +155,10 @@ class PlaneMaterialFactory extends AbstractPlaneMaterialFactory { type: "v3", value: new THREE.Vector3(0, 0, 0), }, + renderBucketIndices: { + type: "b", + value: false, + }, }); for (const dataLayer of Model.getAllLayers()) { diff --git a/app/assets/javascripts/oxalis/model.js b/app/assets/javascripts/oxalis/model.js index 633d5b5754e..0499dcb634a 100644 --- a/app/assets/javascripts/oxalis/model.js +++ b/app/assets/javascripts/oxalis/model.js @@ -13,6 +13,7 @@ import { saveNowAction } from "oxalis/model/actions/save_actions"; import ConnectionInfo from "oxalis/model/data_connection_info"; import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; import DataLayer from "oxalis/model/data_layer"; +import type LayerRenderingManager from "oxalis/model/bucket_data_handling/layer_rendering_manager"; import type PullQueue from "oxalis/model/bucket_data_handling/pullqueue"; import Store, { type TraceOrViewCommand, type TracingTypeTracing } from "oxalis/store"; import * as Utils from "libs/utils"; @@ -90,6 +91,13 @@ export class OxalisModel { return this.dataLayers[name].pullQueue; } + getLayerRenderingManagerByName(name: string): LayerRenderingManager { + if (!this.dataLayers[name]) { + throw new Error(`Layer with name ${name} was not found.`); + } + return this.dataLayers[name].layerRenderingManager; + } + stateSaved() { const state = Store.getState(); const storeStateSaved = @@ -115,5 +123,7 @@ export class OxalisModel { }; } +const model = new OxalisModel(); + // export the model as a singleton -export default new OxalisModel(); +export default model; diff --git a/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket.js b/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket.js index 1b9f0a48d52..683ec2f896f 100644 --- a/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket.js +++ b/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket.js @@ -37,6 +37,7 @@ export class DataBucket { BYTE_OFFSET: number; visualizedMesh: ?Object; visualizationColor: number; + neededAtPickerTick: ?number; state: BucketStateEnumType; dirty: boolean; @@ -334,6 +335,10 @@ export class DataBucket { } } + setNeededAtPickerTick(tick: number) { + this.neededAtPickerTick = tick; + } + // The following three methods can be used for debugging purposes. // The bucket will be rendered in the 3D scene as a wireframe geometry. visualize() { diff --git a/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket_picker_strategies/flight_bucket_picker.js b/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket_picker_strategies/flight_bucket_picker.js index d76d9a8a2ac..b533ef4ac53 100644 --- a/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket_picker_strategies/flight_bucket_picker.js +++ b/app/assets/javascripts/oxalis/model/bucket_data_handling/bucket_picker_strategies/flight_bucket_picker.js @@ -2,19 +2,45 @@ import PriorityQueue from "js-priority-queue"; import { M4x4, type Matrix4x4, V3 } from "libs/mjs"; -import { getMatrixScale } from "oxalis/model/reducers/flycam_reducer"; import { getPosition } from "oxalis/model/accessors/flycam_accessor"; import { getResolutions } from "oxalis/model/accessors/dataset_accessor"; import { globalPositionToBucketPosition, - bucketPositionToGlobalAddress, + globalPositionToBucketPositionFloat, + zoomedAddressToAnotherZoomStep, } from "oxalis/model/helpers/position_converter"; import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; import Store from "oxalis/store"; -import * as Utils from "libs/utils"; -import constants, { type Vector3 } from "oxalis/constants"; +import constants, { type Vector3, type Vector4 } from "oxalis/constants"; + +const aggregatePerDimension = (aggregateFn, buckets): Vector3 => + // $FlowFixMe + [0, 1, 2].map(dim => aggregateFn(...buckets.map(pos => pos[dim]))); + +const getBBox = buckets => ({ + cornerMin: aggregatePerDimension(Math.min, buckets), + cornerMax: aggregatePerDimension(Math.max, buckets), +}); + +function createDistinctBucketAdder(buckets: Array) { + const bucketLookUp = []; + const maybeAddBucket = (bucketPos: Vector4) => { + const [x, y, z] = bucketPos; + /* eslint-disable-next-line */ + bucketLookUp[x] = bucketLookUp[x] || []; + const lookupX = bucketLookUp[x]; + /* eslint-disable-next-line */ + lookupX[y] = lookupX[y] || []; + const lookupY = lookupX[y]; + + if (!lookupY[z]) { + lookupY[z] = true; + buckets.push(bucketPos); + } + }; -import { getFallbackBuckets } from "./oblique_bucket_picker"; + return maybeAddBucket; +} export default function determineBucketsForFlight( cube: DataCube, @@ -24,124 +50,110 @@ export default function determineBucketsForFlight( fallbackZoomStep: number, isFallbackAvailable: boolean, ): void { - const queryMatrix = M4x4.scale1(1, matrix); - - const enlargementFactor = 1.0; - const enlargedExtent = constants.VIEWPORT_WIDTH * enlargementFactor; - const enlargedHalfExtent = enlargedExtent / 2; - const { sphericalCapRadius } = Store.getState().userConfiguration; - const cameraVertex = [0, 0, -sphericalCapRadius]; const resolutions = getResolutions(Store.getState().dataset); + const centerPosition = getPosition(Store.getState().flycam); + const queryMatrix = M4x4.scale1(1, matrix); + const width = constants.VIEWPORT_WIDTH; + const halfWidth = width / 2; + const cameraVertex = [0, 0, -sphericalCapRadius]; - // This array holds the four corners and the center point of the rendered plane - const planePoints = M4x4.transformVectorsAffine( - queryMatrix, - [ - [-enlargedHalfExtent, -enlargedHalfExtent, 0], - [enlargedHalfExtent, -enlargedHalfExtent, 0], - [0, 0, 0], - [-enlargedHalfExtent, enlargedHalfExtent, 0], - [enlargedHalfExtent, enlargedHalfExtent, 0], - ].map(vec => { - V3.sub(vec, cameraVertex, vec); - V3.scale(vec, sphericalCapRadius / V3.length(vec), vec); - V3.add(vec, cameraVertex, vec); - return vec; - }), - ).map((position: Vector3) => globalPositionToBucketPosition(position, resolutions, logZoomStep)); - - const cameraPosition = M4x4.transformVectorsAffine(queryMatrix, [cameraVertex])[0]; - - const { scale } = Store.getState().dataset.dataSource; - const matrixScale = getMatrixScale(scale); - - const inverseScale = V3.divide3([1, 1, 1], matrixScale); - - const aggregatePerDimension = aggregateFn => - [0, 1, 2].map(dim => aggregateFn(...planePoints.map(pos => pos[dim]))); - - const boundingBoxBuckets = { - cornerMin: aggregatePerDimension(Math.min), - cornerMax: aggregatePerDimension(Math.max), + const transformToSphereCap = _vec => { + const vec = V3.sub(_vec, cameraVertex); + V3.scale(vec, sphericalCapRadius / V3.length(vec), vec); + V3.add(vec, cameraVertex, vec); + return vec; }; + const transformAndApplyMatrix = vec => + M4x4.transformPointsAffine(queryMatrix, transformToSphereCap(vec)); let traversedBuckets = []; + const maybeAddBucket = createDistinctBucketAdder(traversedBuckets); - const { zoomStep } = Store.getState().flycam; - const squaredRadius = (zoomStep * sphericalCapRadius) ** 2; - const tolerance = 1; - - // iterate over all buckets within bounding box - for ( - let x = boundingBoxBuckets.cornerMin[0] - tolerance; - x <= boundingBoxBuckets.cornerMax[0] + tolerance; - x++ - ) { - for ( - let y = boundingBoxBuckets.cornerMin[1] - tolerance; - y <= boundingBoxBuckets.cornerMax[1] + tolerance; - y++ - ) { - for ( - let z = boundingBoxBuckets.cornerMin[2] - tolerance; - z <= boundingBoxBuckets.cornerMax[2] + tolerance; - z++ - ) { - const pos = bucketPositionToGlobalAddress([x, y, z, logZoomStep], resolutions); - const nextPos = bucketPositionToGlobalAddress( - [x + 1, y + 1, z + 1, logZoomStep], - resolutions, - ); - - const closest = [0, 1, 2].map(dim => - Utils.clamp(pos[dim], cameraPosition[dim], nextPos[dim]), - ); - - const farthest = [0, 1, 2].map( - dim => - Math.abs(pos[dim] - cameraPosition[dim]) > Math.abs(nextPos[dim] - cameraPosition[dim]) - ? pos[dim] - : nextPos[dim], - ); - - const closestDist = V3.scaledSquaredDist(cameraPosition, closest, inverseScale); - const farthestDist = V3.scaledSquaredDist(cameraPosition, farthest, inverseScale); - - const collisionTolerance = 0.05; - const doesCollide = - (1 - collisionTolerance) * closestDist <= squaredRadius && - (1 + collisionTolerance) * farthestDist >= squaredRadius; - - if (doesCollide) { - traversedBuckets.push([x, y, z]); + const cameraPosition = M4x4.transformVectorsAffine(queryMatrix, [cameraVertex])[0]; + const cameraDirection = V3.sub(centerPosition, cameraPosition); + V3.scale(cameraDirection, 1 / Math.abs(V3.length(cameraDirection)), cameraDirection); + + const iterStep = 10; + for (let y = -halfWidth; y <= halfWidth; y += iterStep) { + const xOffset = y % iterStep; + for (let x = -halfWidth - xOffset; x <= halfWidth + xOffset; x += iterStep) { + const z = 0; + const transformedVec = transformAndApplyMatrix([x, y, z]); + + const bucketPos = globalPositionToBucketPositionFloat( + transformedVec, + resolutions, + logZoomStep, + ); + + // $FlowFixMe + const flooredBucketPos: Vector4 = bucketPos.map(Math.floor); + maybeAddBucket(flooredBucketPos); + + const neighbourThreshold = 3; + bucketPos.forEach((pos, idx) => { + // $FlowFixMe + const newNeighbour: Vector4 = flooredBucketPos.slice(); + const rest = (pos % 1) * constants.BUCKET_WIDTH; + if (rest < neighbourThreshold) { + // Pick the previous neighbor + newNeighbour[idx]--; + maybeAddBucket(newNeighbour); + } else if (rest > constants.BUCKET_WIDTH - neighbourThreshold) { + // Pick the next neighbor + newNeighbour[idx]++; + maybeAddBucket(newNeighbour); } - } + }); } } - traversedBuckets = traversedBuckets.map(addr => [...addr, logZoomStep]); - - const fallbackBuckets = getFallbackBuckets( - traversedBuckets, - resolutions, - fallbackZoomStep, - isFallbackAvailable, + // This array holds the four corners and the center point of the rendered plane + const planePointsGlobal = [ + [-halfWidth, -halfWidth, 0], // 0 bottom left + [halfWidth, -halfWidth, 0], // 1 bottom right + [0, 0, 0], + [-halfWidth, halfWidth, 0], // 3 top left + [halfWidth, halfWidth, 0], // 4 top right + ].map(transformAndApplyMatrix); + + const planeBuckets = planePointsGlobal.map((position: Vector3) => + globalPositionToBucketPosition(position, resolutions, logZoomStep), ); - traversedBuckets = traversedBuckets.concat(fallbackBuckets); + const traverseFallbackBBox = boundingBoxBuckets => { + const tolerance = 1; + const fallbackBuckets = []; + // use all fallback buckets in bbox + const min = zoomedAddressToAnotherZoomStep( + [...boundingBoxBuckets.cornerMin, logZoomStep], + resolutions, + fallbackZoomStep, + ); + const max = zoomedAddressToAnotherZoomStep( + [...boundingBoxBuckets.cornerMax, logZoomStep], + resolutions, + fallbackZoomStep, + ); + for (let x = min[0] - tolerance; x <= max[0] + tolerance; x++) { + for (let y = min[1] - tolerance; y <= max[1] + tolerance; y++) { + for (let z = min[2] - tolerance; z <= max[2] + tolerance; z++) { + fallbackBuckets.push([x, y, z, fallbackZoomStep]); + } + } + } + return fallbackBuckets; + }; - const centerAddress = globalPositionToBucketPosition( - getPosition(Store.getState().flycam), - resolutions, - logZoomStep, - ); + const fallbackBuckets = isFallbackAvailable ? traverseFallbackBBox(getBBox(planeBuckets)) : []; + traversedBuckets = traversedBuckets.concat(fallbackBuckets); for (const bucketAddress of traversedBuckets) { const bucket = cube.getOrCreateBucket(bucketAddress); if (bucket.type !== "null") { - const priority = V3.sub(bucketAddress, centerAddress).reduce((a, b) => a + Math.abs(b), 0); + const priority = 0; bucketQueue.queue({ bucket, priority }); } } diff --git a/app/assets/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js b/app/assets/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js index eb79d539695..91b24ed022b 100644 --- a/app/assets/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js +++ b/app/assets/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js @@ -91,6 +91,7 @@ export default class LayerRenderingManager { name: string; isSegmentation: boolean; needsRefresh: boolean = false; + currentBucketPickerTick: number = 0; constructor( name: string, @@ -193,6 +194,7 @@ export default class LayerRenderingManager { this.lastSphericalCapRadius = sphericalCapRadius; this.lastIsInvisible = isInvisible; this.needsRefresh = false; + this.currentBucketPickerTick++; const bucketQueue = new PriorityQueue({ // small priorities take precedence diff --git a/app/assets/javascripts/oxalis/model/bucket_data_handling/prefetch_strategy_arbitrary.js b/app/assets/javascripts/oxalis/model/bucket_data_handling/prefetch_strategy_arbitrary.js index 07201771661..41a762f8e87 100644 --- a/app/assets/javascripts/oxalis/model/bucket_data_handling/prefetch_strategy_arbitrary.js +++ b/app/assets/javascripts/oxalis/model/bucket_data_handling/prefetch_strategy_arbitrary.js @@ -1,9 +1,10 @@ // @flow import { AbstractPrefetchStrategy } from "oxalis/model/bucket_data_handling/prefetch_strategy_plane"; -import type { BoundingBoxType } from "oxalis/constants"; -import { M4x4, type Matrix4x4 } from "libs/mjs"; +import type { BoundingBoxType, Vector3 } from "oxalis/constants"; +import { M4x4, type Matrix4x4, V3 } from "libs/mjs"; import type { PullQueueItem } from "oxalis/model/bucket_data_handling/pullqueue"; +import { globalPositionToBucketPosition } from "oxalis/model/helpers/position_converter"; import PolyhedronRasterizer from "oxalis/model/bucket_data_handling/polyhedron_rasterizer"; export class PrefetchStrategyArbitrary extends AbstractPrefetchStrategy { @@ -14,12 +15,12 @@ export class PrefetchStrategyArbitrary extends AbstractPrefetchStrategy { name = "ARBITRARY"; prefetchPolyhedron: PolyhedronRasterizer.Master = PolyhedronRasterizer.Master.squareFrustum( - 5, - 5, + 7, + 7, -0.5, - 4, - 4, - 2, + 10, + 10, + 20, ); getExtentObject( @@ -51,7 +52,12 @@ export class PrefetchStrategyArbitrary extends AbstractPrefetchStrategy { matrix[14] += 1; } - prefetch(matrix: Matrix4x4, zoomStep: number): Array { + prefetch( + matrix: Matrix4x4, + zoomStep: number, + position: Vector3, + resolutions: Array, + ): Array { const pullQueue = []; const matrix0 = M4x4.clone(matrix); @@ -66,7 +72,13 @@ export class PrefetchStrategyArbitrary extends AbstractPrefetchStrategy { const bucketY = testAddresses[i++]; const bucketZ = testAddresses[i++]; - pullQueue.push({ bucket: [bucketX, bucketY, bucketZ, zoomStep], priority: 0 }); + const positionBucket = globalPositionToBucketPosition(position, resolutions, zoomStep); + const distanceToPosition = V3.length(V3.sub([bucketX, bucketY, bucketZ], positionBucket)); + + pullQueue.push({ + bucket: [bucketX, bucketY, bucketZ, zoomStep], + priority: 1 + distanceToPosition, + }); } return pullQueue; } diff --git a/app/assets/javascripts/oxalis/model/bucket_data_handling/pullqueue.js b/app/assets/javascripts/oxalis/model/bucket_data_handling/pullqueue.js index e395fecfede..18dc89e4a35 100644 --- a/app/assets/javascripts/oxalis/model/bucket_data_handling/pullqueue.js +++ b/app/assets/javascripts/oxalis/model/bucket_data_handling/pullqueue.js @@ -15,6 +15,7 @@ import { import { requestWithFallback } from "oxalis/model/bucket_data_handling/wkstore_adapter"; import ConnectionInfo from "oxalis/model/data_connection_info"; import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; +import Model from "oxalis/model"; import Store, { type DataStoreInfo, type DataLayerType } from "oxalis/store"; export type PullQueueItem = { @@ -36,6 +37,8 @@ const createPriorityQueue = () => }); const BATCH_SIZE = 3; +// If ${maximumPickerTickCount} bucket picker ticks didn't select a bucket, that bucket is discarded from the pullqueue +const maximumPickerTickCount = 5; class PullQueue { cube: DataCube; @@ -67,15 +70,26 @@ class PullQueue { pull(): Array> { // Starting to download some buckets + const layerRenderingManager = Model.getLayerRenderingManagerByName(this.layerName); + const { currentBucketPickerTick } = layerRenderingManager; + const promises = []; while (this.batchCount < PullQueueConstants.BATCH_LIMIT && this.priorityQueue.length > 0) { const batch = []; while (batch.length < BATCH_SIZE && this.priorityQueue.length > 0) { const address = this.priorityQueue.dequeue().bucket; const bucket = this.cube.getOrCreateBucket(address); + if (bucket.type === "data" && bucket.needsRequest()) { - batch.push(address); - bucket.pull(); + const isOutdated = + bucket.neededAtPickerTick != null && + currentBucketPickerTick - bucket.neededAtPickerTick > maximumPickerTickCount; + if (!isOutdated) { + batch.push(address); + bucket.pull(); + } else { + bucket.unvisualize(); + } } } @@ -152,6 +166,7 @@ class PullQueue { this.maybeWhitenEmptyBucket(bucketData); if (bucket.type === "data") { bucket.receiveData(bucketData); + bucket.setVisualizationColor(0x00ff00); if (zoomStep === this.cube.MAX_UNSAMPLED_ZOOM_STEP) { const higherAddress = zoomedAddressToAnotherZoomStep( bucketAddress, @@ -175,30 +190,24 @@ class PullQueue { } } - clearNormalPriorities(): void { - // The following code removes all items from the priorityQueue which are not PRIORITY_HIGHEST - - const newQueue = createPriorityQueue(); - while (this.priorityQueue.length > 0) { - const item = this.priorityQueue.dequeue(); - if (item.priority === PullQueueConstants.PRIORITY_HIGHEST) { - newQueue.queue(item); - } else if (item.priority > PullQueueConstants.PRIORITY_HIGHEST) { - // Since dequeuing is ordered, we will only receive priorities which are - // not PRIORITY_HIGHEST - break; + add(item: PullQueueItem, currentBucketPickerTick?: number): void { + const bucket = this.cube.getOrCreateBucket(item.bucket); + if (bucket.type === "data") { + if (currentBucketPickerTick == null) { + const layerRenderingManager = Model.getLayerRenderingManagerByName(this.layerName); + currentBucketPickerTick = layerRenderingManager.currentBucketPickerTick; } + bucket.setNeededAtPickerTick(currentBucketPickerTick); } - this.priorityQueue = newQueue; - } - add(item: PullQueueItem): void { this.priorityQueue.queue(item); } addAll(items: Array): void { + const layerRenderingManager = Model.getLayerRenderingManagerByName(this.layerName); + const { currentBucketPickerTick } = layerRenderingManager; for (const item of items) { - this.priorityQueue.queue(item); + this.add(item, currentBucketPickerTick); } } diff --git a/app/assets/javascripts/oxalis/model/bucket_data_handling/texture_bucket_manager.js b/app/assets/javascripts/oxalis/model/bucket_data_handling/texture_bucket_manager.js index e251d4a1cb0..1d6f4f4f040 100644 --- a/app/assets/javascripts/oxalis/model/bucket_data_handling/texture_bucket_manager.js +++ b/app/assets/javascripts/oxalis/model/bucket_data_handling/texture_bucket_manager.js @@ -24,6 +24,9 @@ import window from "libs/window"; const lookUpBufferWidth = constants.LOOK_UP_TEXTURE_WIDTH; +// DEBUG flag for visualizing buckets which are passed to the GPU +const visualizeBucketsOnGPU = false; + // At the moment, we only store one float f per bucket. // If f >= 0, f denotes the index in the data texture where the bucket is stored. // If f == -1, the bucket is not yet committed @@ -95,6 +98,9 @@ export default class TextureBucketManager { if (unusedIndex == null) { return; } + if (visualizeBucketsOnGPU) { + bucket.unvisualize(); + } this.activeBucketToIndexMap.delete(bucket); this.committedBucketSet.delete(bucket); this.freeIndexSet.add(unusedIndex); @@ -110,6 +116,7 @@ export default class TextureBucketManager { fallbackAnchorPoint: Vector4, ): void { this.currentAnchorPoint = anchorPoint; + window.currentAnchorPoint = anchorPoint; this.fallbackAnchorPoint = fallbackAnchorPoint; // Find out which buckets are not needed anymore const freeBucketSet = new Set(this.activeBucketToIndexMap.keys()); @@ -175,6 +182,10 @@ export default class TextureBucketManager { const dataTextureIndex = Math.floor(_index / bucketsPerTexture); const indexInDataTexture = _index % bucketsPerTexture; + if (visualizeBucketsOnGPU) { + bucket.visualize(); + } + this.dataTextures[dataTextureIndex].update( bucket.getData(), 0, diff --git a/app/assets/javascripts/oxalis/model/helpers/position_converter.js b/app/assets/javascripts/oxalis/model/helpers/position_converter.js index 1d887f4771a..c06e229c560 100644 --- a/app/assets/javascripts/oxalis/model/helpers/position_converter.js +++ b/app/assets/javascripts/oxalis/model/helpers/position_converter.js @@ -24,6 +24,24 @@ export function globalPositionToBucketPosition( ]; } +export function globalPositionToBucketPositionFloat( + [x, y, z]: Vector3, + resolutions: Array, + resolutionIndex: number, +): Vector4 { + const resolution = + resolutionIndex < resolutions.length + ? resolutions[resolutionIndex] + : upsampleResolution(resolutions, resolutionIndex); + + return [ + x / (constants.BUCKET_WIDTH * resolution[0]), + y / (constants.BUCKET_WIDTH * resolution[1]), + z / (constants.BUCKET_WIDTH * resolution[2]), + resolutionIndex, + ]; +} + export function upsampleResolution(resolutions: Array, resolutionIndex: number): Vector3 { const lastResolutionIndex = resolutions.length - 1; const lastResolution = resolutions[lastResolutionIndex]; diff --git a/app/assets/javascripts/oxalis/model/helpers/shader_editor.js b/app/assets/javascripts/oxalis/model/helpers/shader_editor.js index 7f7de81ffff..54d64ffc873 100644 --- a/app/assets/javascripts/oxalis/model/helpers/shader_editor.js +++ b/app/assets/javascripts/oxalis/model/helpers/shader_editor.js @@ -12,14 +12,15 @@ export default { }, }; -window._setupShaderEditor = viewport => { +window._setupShaderEditor = (viewport, _shaderType) => { const outer = document.createElement("div"); const input = document.createElement("textarea"); - input.value = window.materials[viewport].fragmentShader; + const shaderType = _shaderType || "fragmentShader"; + input.value = window.materials[viewport][shaderType]; const button = document.createElement("button"); const buttonContainer = document.createElement("div"); function overrideShader() { - window.materials[viewport].fragmentShader = input.value; + window.materials[viewport][shaderType] = input.value; window.materials[viewport].needsUpdate = true; window.needsRerender = true; } diff --git a/app/assets/javascripts/oxalis/model/sagas/prefetch_saga.js b/app/assets/javascripts/oxalis/model/sagas/prefetch_saga.js index 902995437a8..4f97cab5560 100644 --- a/app/assets/javascripts/oxalis/model/sagas/prefetch_saga.js +++ b/app/assets/javascripts/oxalis/model/sagas/prefetch_saga.js @@ -27,6 +27,9 @@ const DIRECTION_VECTOR_SMOOTHER = 0.125; const prefetchStrategiesArbitrary = [new PrefetchStrategyArbitrary()]; const prefetchStrategiesPlane = [new PrefetchStrategySkeleton(), new PrefetchStrategyVolume()]; +// DEBUG flag for visualizing buckets which are prefetched +const visualizePrefetchedBuckets = false; + export function* watchDataRelevantChanges(): Saga { yield* take("WK_READY"); @@ -114,7 +117,6 @@ export function* prefetchForPlaneMode(layer: DataLayer, previousProperties: Obje strategy.inVelocityRange(layer.connectionInfo.bandwidth) && strategy.inRoundTripTimeRange(layer.connectionInfo.roundTripTime) ) { - layer.pullQueue.clearNormalPriorities(); const buckets = strategy.prefetch( layer.cube, position, @@ -140,11 +142,13 @@ export function* prefetchForArbitraryMode( layer: DataLayerType, previousProperties: Object, ): Saga { + const position = yield* select(state => getPosition(state.flycam)); const matrix = yield* select(state => state.flycam.currentMatrix); const zoomStep = yield* select(state => getRequestLogZoomStep(state)); const tracingTypes = yield* select(getTracingTypes); + const resolutions = yield* select(state => getResolutions(state.dataset)); const { lastMatrix, lastZoomStep } = previousProperties; - const { connectionInfo, pullQueue } = Model.dataLayers[layer.name]; + const { connectionInfo, pullQueue, cube } = Model.dataLayers[layer.name]; if (matrix !== lastMatrix || zoomStep !== lastZoomStep) { for (const strategy of prefetchStrategiesArbitrary) { @@ -153,8 +157,15 @@ export function* prefetchForArbitraryMode( strategy.inVelocityRange(connectionInfo.bandwidth) && strategy.inRoundTripTimeRange(connectionInfo.roundTripTime) ) { - pullQueue.clearNormalPriorities(); - const buckets = strategy.prefetch(matrix, zoomStep); + const buckets = strategy.prefetch(matrix, zoomStep, position, resolutions); + if (visualizePrefetchedBuckets) { + for (const item of buckets) { + const bucket = cube.getOrCreateBucket(item.bucket); + if (bucket.type !== "null") { + bucket.visualize(); + } + } + } pullQueue.addAll(buckets); break; } diff --git a/app/assets/javascripts/oxalis/shaders/coords.glsl.js b/app/assets/javascripts/oxalis/shaders/coords.glsl.js index f0720f110cb..b85a01cfd0c 100644 --- a/app/assets/javascripts/oxalis/shaders/coords.glsl.js +++ b/app/assets/javascripts/oxalis/shaders/coords.glsl.js @@ -1,6 +1,7 @@ // @flow +import { isFlightMode, getW } from "oxalis/shaders/utils.glsl"; + import type { ShaderModule } from "./shader_module_system"; -import { getW } from "./utils.glsl"; export const getResolution: ShaderModule = { code: ` @@ -44,7 +45,7 @@ export const getRelativeCoords: ShaderModule = { }; export const getWorldCoordUVW: ShaderModule = { - requirements: [getW], + requirements: [getW, isFlightMode], code: ` vec3 getWorldCoordUVW() { vec3 worldCoordUVW = transDim(worldCoord.xyz); diff --git a/app/assets/javascripts/oxalis/shaders/main_data_fragment.glsl.js b/app/assets/javascripts/oxalis/shaders/main_data_fragment.glsl.js index a1a4e6f238f..94c32d106bf 100644 --- a/app/assets/javascripts/oxalis/shaders/main_data_fragment.glsl.js +++ b/app/assets/javascripts/oxalis/shaders/main_data_fragment.glsl.js @@ -85,6 +85,7 @@ const int dataTextureCountPerLayer = <%= dataTextureCountPerLayer %>; uniform float sphericalCapRadius; uniform float viewMode; uniform float alpha; +uniform bool renderBucketIndices; uniform bool highlightHoveredCellId; uniform vec3 bboxMin; uniform vec3 bboxMax; @@ -145,6 +146,11 @@ void main() { vec3 coords = getRelativeCoords(worldCoordUVW, zoomStep); vec3 bucketPosition = div(floor(coords), bucketWidth); + if (renderBucketIndices) { + gl_FragColor = vec4(bucketPosition, zoomStep) / 255.; + // gl_FragColor = vec4(0.5, 1.0, 1.0, 1.0); + return; + } vec3 offsetInBucket = mod(floor(coords), bucketWidth); <% if (hasSegmentation) { %> diff --git a/app/assets/javascripts/oxalis/shaders/texture_access.glsl.js b/app/assets/javascripts/oxalis/shaders/texture_access.glsl.js index 865c0872b3f..80111d7746c 100644 --- a/app/assets/javascripts/oxalis/shaders/texture_access.glsl.js +++ b/app/assets/javascripts/oxalis/shaders/texture_access.glsl.js @@ -131,7 +131,14 @@ const getColorFor: ShaderModule = { if (bucketAddress == -2.0) { // The bucket is out of bounds. Render black - return vec4(0.0, 0.0, 0.0, 0.0); + // In flight mode, it can happen that buckets were not passed to the GPU + // since the approximate implementation of the bucket picker missed the bucket. + // We simply handle this case as if the bucket was not yet loaded which means + // that fallback data is loaded. + // The downside is that data which does exist, will be rendered gray instead of black. + // Issue to track progress: #3446 + float alpha = isFlightMode() ? -1.0 : 0.0; + return vec4(0.0, 0.0, 0.0, -1.0); } if (bucketAddress < 0. || isNan(bucketAddress)) { diff --git a/app/assets/javascripts/oxalis/view/arbitrary_view.js b/app/assets/javascripts/oxalis/view/arbitrary_view.js index 8f9ee88e5f5..2f4aeb8d699 100644 --- a/app/assets/javascripts/oxalis/view/arbitrary_view.js +++ b/app/assets/javascripts/oxalis/view/arbitrary_view.js @@ -11,7 +11,9 @@ import { getDesiredLayoutRect } from "oxalis/view/layouting/golden_layout_adapte import { getInputCatcherRect } from "oxalis/model/accessors/view_mode_accessor"; import { getZoomedMatrix } from "oxalis/model/accessors/flycam_accessor"; import { listenToStoreProperty } from "oxalis/model/helpers/listener_helpers"; -import Constants, { ArbitraryViewport } from "oxalis/constants"; +import { show3DViewportInArbitrary } from "oxalis/view/layouting/default_layout_configs"; +import type ArbitraryPlane from "oxalis/geometries/arbitrary_plane"; +import Constants, { ArbitraryViewport, type OrthoViewMap, OrthoViews } from "oxalis/constants"; import SceneController from "oxalis/controller/scene_controller"; import Store from "oxalis/store"; import app from "app"; @@ -23,6 +25,8 @@ class ArbitraryView { // Copied form backbone events (TODO: handle this better) trigger: Function; unbindChangedScaleListener: () => void; + cameras: OrthoViewMap; + plane: ArbitraryPlane; animate: () => void; setClippingDistance: (value: number) => void; @@ -36,6 +40,8 @@ class ArbitraryView { camDistance: number; camera: THREE.PerspectiveCamera = null; + tdCamera: THREE.OrthographicCamera = null; + geometries: Array = []; group: THREE.Object3D; cameraPosition: Array; @@ -54,6 +60,22 @@ class ArbitraryView { this.camera = new THREE.PerspectiveCamera(45, 1, 50, 1000); this.camera.matrixAutoUpdate = false; + const tdCamera = new THREE.OrthographicCamera(0, 0, 0, 0); + tdCamera.position.copy(new THREE.Vector3(10, 10, -10)); + tdCamera.up = new THREE.Vector3(0, 0, -1); + tdCamera.matrixAutoUpdate = true; + + this.tdCamera = tdCamera; + + const dummyCamera = new THREE.PerspectiveCamera(45, 1, 50, 1000); + + this.cameras = { + TDView: tdCamera, + PLANE_XY: dummyCamera, + PLANE_YZ: dummyCamera, + PLANE_XZ: dummyCamera, + }; + this.cameraPosition = [0, 0, this.camDistance]; this.needsRerender = true; @@ -68,6 +90,10 @@ class ArbitraryView { }); } + getCameras(): OrthoViewMap { + return this.cameras; + } + start(): void { if (!this.isRunning) { this.isRunning = true; @@ -152,10 +178,24 @@ class ArbitraryView { clearCanvas(renderer); - const { left, top, width, height } = getInputCatcherRect(ArbitraryViewport); - if (width > 0 && height > 0) { - setupRenderArea(renderer, left, top, Math.min(width, height), width, height, 0xffffff); - renderer.render(scene, camera); + const renderViewport = (viewport, _camera) => { + const { left, top, width, height } = getInputCatcherRect(viewport); + if (width > 0 && height > 0) { + setupRenderArea(renderer, left, top, Math.min(width, height), width, height, 0xffffff); + renderer.render(scene, _camera); + } + }; + + if (this.plane.meshes.debuggerPlane != null) { + this.plane.meshes.debuggerPlane.visible = false; + } + renderViewport(ArbitraryViewport, camera); + if (show3DViewportInArbitrary) { + if (this.plane.meshes.debuggerPlane != null) { + this.plane.meshes.debuggerPlane.visible = true; + } + + renderViewport(OrthoViews.TDView, this.tdCamera); } this.needsRerender = false; @@ -169,6 +209,82 @@ class ArbitraryView { this.needsRerender = true; } + renderToTexture(): Uint8Array { + const { renderer, scene } = SceneController; + + renderer.autoClear = true; + let { width, height } = getInputCatcherRect(ArbitraryViewport); + width = Math.round(width); + height = Math.round(height); + + const { camera } = this; + + renderer.setViewport(0, 0, width, height); + renderer.setScissorTest(false); + renderer.setClearColor(0x222222, 1); + + const renderTarget = new THREE.WebGLRenderTarget(width, height); + const buffer = new Uint8Array(width * height * 4); + this.plane.materialFactory.uniforms.renderBucketIndices.value = true; + renderer.render(scene, camera, renderTarget); + renderer.readRenderTargetPixels(renderTarget, 0, 0, width, height, buffer); + this.plane.materialFactory.uniforms.renderBucketIndices.value = false; + + return buffer; + } + + getRenderedBucketsDebug = () => { + // This method can be used to determine which buckets were used during rendering. + // It returns an array with bucket indices which were used by the fragment shader. + // Code similar to the following will render buckets wireframes in red, if there were + // passed to the GPU but were not used. + // It can be used within the orthogonal bucket picker for example. + // // @flow + // import * as Utils from "libs/utils"; + // import type { Vector4 } from "oxalis/constants"; + + // const makeBucketId = ([x, y, z], logZoomStep) => [x, y, z, logZoomStep].join(","); + // const unpackBucketId = (str): Vector4 => + // str + // .split(",") + // .map(el => parseInt(el)) + // .map((el, idx) => (idx < 3 ? el : 0)); + // function diff(traversedBuckets, lastRenderedBuckets) { + // const bucketDiff = Utils.diffArrays(traversedBuckets.map(makeBucketId), lastRenderedBuckets.map(makeBucketId)); + // + // bucketDiff.onlyA.forEach(bucketAddress => { + // const bucket = cube.getOrCreateBucket(unpackBucketId(bucketAddress)); + // if (bucket.type !== "null") bucket.setVisualizationColor(0xff0000); + // }); + // bucketDiff.both.forEach(bucketAddress => { + // const bucket = cube.getOrCreateBucket(unpackBucketId(bucketAddress)); + // if (bucket.type !== "null") bucket.setVisualizationColor(0x00ff00); + // }); + // } + // diff(traversedBuckets, getRenderedBucketsDebug()); + + const buffer = this.renderToTexture(); + let index = 0; + + const usedBucketSet = new Set(); + const usedBuckets = []; + + while (index < buffer.length) { + const bucketAddress = buffer + .subarray(index, index + 4) + .map((el, idx) => (idx < 3 ? window.currentAnchorPoint[idx] + el : el)); + index += 4; + + const id = bucketAddress.join(","); + if (!usedBucketSet.has(id)) { + usedBucketSet.add(id); + usedBuckets.push(bucketAddress); + } + } + + return usedBuckets; + }; + addGeometry(geometry: THREE.Geometry): void { // Adds a new Three.js geometry to the scene. // This provides the public interface to the GeometryFactory. @@ -177,6 +293,10 @@ class ArbitraryView { geometry.addToScene(this.group); } + setArbitraryPlane(p: ArbitraryPlane) { + this.plane = p; + } + resizeImpl = (): void => { // Call this after the canvas was resized to fix the viewport const { width, height } = getDesiredLayoutRect(); diff --git a/app/assets/javascripts/oxalis/view/layouting/default_layout_configs.js b/app/assets/javascripts/oxalis/view/layouting/default_layout_configs.js index 7a63e970da0..88dba8110b8 100644 --- a/app/assets/javascripts/oxalis/view/layouting/default_layout_configs.js +++ b/app/assets/javascripts/oxalis/view/layouting/default_layout_configs.js @@ -17,6 +17,7 @@ export const currentLayoutVersion = 6; export const layoutHeaderHeight = 20; export const headerHeight = 55; const dummyExtent = 500; +export const show3DViewportInArbitrary = false; const LayoutSettings = { showPopoutIcon: false, @@ -99,7 +100,11 @@ const unmemoizedGetDefaultLayouts = () => { const OrthoLayout = createLayout(Row(...OrthoViewsGrid, SkeletonRightHandColumn)); const OrthoLayoutView = createLayout(Row(...OrthoViewsGrid, NonSkeletonRightHandColumn)); const VolumeTracingView = createLayout(Row(...OrthoViewsGrid, NonSkeletonRightHandColumn)); - const ArbitraryLayout = createLayout(Row(Panes.arbitraryViewport, SkeletonRightHandColumn)); + + const arbitraryPanes = [Panes.arbitraryViewport, SkeletonRightHandColumn].concat( + show3DViewportInArbitrary ? [Panes.td] : [], + ); + const ArbitraryLayout = createLayout(Row(...arbitraryPanes)); return { OrthoLayout, OrthoLayoutView, VolumeTracingView, ArbitraryLayout }; }; diff --git a/app/assets/javascripts/oxalis/view/right-menu/dataset_info_tab_view.js b/app/assets/javascripts/oxalis/view/right-menu/dataset_info_tab_view.js index 0a4038a1e4d..56ebf95f92f 100644 --- a/app/assets/javascripts/oxalis/view/right-menu/dataset_info_tab_view.js +++ b/app/assets/javascripts/oxalis/view/right-menu/dataset_info_tab_view.js @@ -314,20 +314,19 @@ class DatasetInfoTabView extends React.PureComponent {

Viewport Width: {formatNumberToLength(zoomLevel)}

Dataset Resolution: {formatScale(this.props.dataset.dataSource.scale)}

-

- - - - - - - - - - -
Dataset Extent:{formatExtentWithLength(extentInVoxel, x => `${x}`)} Voxel³
- {formatExtentWithLength(extent, formatNumberToLength)}
-

+ + + + + + + + + + + +
Dataset Extent:{formatExtentWithLength(extentInVoxel, x => `${x}`)} Voxel³
+ {formatExtentWithLength(extent, formatNumberToLength)}
{this.getTracingStatistics()} {this.getKeyboardShortcuts(isDatasetViewMode)} diff --git a/app/assets/javascripts/oxalis/view/right-menu/mapping_info_view.js b/app/assets/javascripts/oxalis/view/right-menu/mapping_info_view.js index bc508b08756..57363736143 100644 --- a/app/assets/javascripts/oxalis/view/right-menu/mapping_info_view.js +++ b/app/assets/javascripts/oxalis/view/right-menu/mapping_info_view.js @@ -47,6 +47,7 @@ const hasSegmentation = () => Model.getSegmentationLayer() != null; class MappingInfoView extends React.Component { componentDidMount() { + this.isMounted = true; if (!hasSegmentation()) { return; } @@ -56,6 +57,7 @@ class MappingInfoView extends React.Component { } componentWillUnmount() { + this.isMounted = false; if (!hasSegmentation()) { return; } @@ -64,7 +66,12 @@ class MappingInfoView extends React.Component { cube.off("volumeLabeled", this._forceUpdate); } + isMounted: boolean = false; + _forceUpdate = _.throttle(() => { + if (!this.isMounted) { + return; + } this.forceUpdate(); }, 100); diff --git a/app/assets/javascripts/test/model/binary/pullqueue.spec.js b/app/assets/javascripts/test/model/binary/pullqueue.spec.js index c34fabb9468..fba0910bdc3 100644 --- a/app/assets/javascripts/test/model/binary/pullqueue.spec.js +++ b/app/assets/javascripts/test/model/binary/pullqueue.spec.js @@ -12,6 +12,9 @@ const RequestMock = { mockRequire("oxalis/model/sagas/root_saga", function*() { yield; }); +mockRequire("oxalis/model", { + getLayerRenderingManagerByName: () => ({ currentBucketPickerTick: 0 }), +}); mockRequire("libs/request", RequestMock); const WkstoreAdapterMock = { requestWithFallback: sinon.stub() }; mockRequire("oxalis/model/bucket_data_handling/wkstore_adapter", WkstoreAdapterMock); @@ -66,9 +69,9 @@ test.beforeEach(t => { const buckets = [new DataBucket(8, [0, 0, 0, 0], null), new DataBucket(8, [1, 1, 1, 1], null)]; for (const bucket of buckets) { - pullQueue.add({ bucket: bucket.zoomedAddress, priority: 0 }); cube.getBucket.withArgs(bucket.zoomedAddress).returns(bucket); cube.getOrCreateBucket.withArgs(bucket.zoomedAddress).returns(bucket); + pullQueue.add({ bucket: bucket.zoomedAddress, priority: 0 }); } t.context = { buckets, pullQueue };