From 477178f6c67a016a7d2b5f0a96da2fd33f124715 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Tue, 24 May 2022 11:44:43 +0200 Subject: [PATCH 01/18] rework the volume interpolation feature to be shortcut-bound and support depths > 2 --- .../recommended_configuration_view.tsx | 1 - frontend/javascripts/libs/utils.ts | 10 ++ frontend/javascripts/messages.ts | 1 - frontend/javascripts/oxalis/default_state.ts | 2 - .../model/accessors/volumetracing_accessor.ts | 32 +++- .../javascripts/oxalis/model/dimensions.ts | 20 ++- .../model/reducers/volumetracing_reducer.ts | 8 +- .../reducers/volumetracing_reducer_helpers.ts | 13 +- .../sagas/volume/volume_interpolation_saga.ts | 37 +++-- .../oxalis/model/sagas/volumetracing_saga.tsx | 149 ++---------------- .../oxalis/model/volumetracing/volumelayer.ts | 4 + frontend/javascripts/oxalis/store.ts | 4 +- .../oxalis/view/action-bar/toolbar_view.tsx | 97 +++++++----- .../test/fixtures/volumetracing_object.ts | 2 +- .../reducers/volumetracing_reducer.spec.ts | 4 +- .../types/schemas/user_settings.schema.ts | 2 - 16 files changed, 168 insertions(+), 218 deletions(-) diff --git a/frontend/javascripts/admin/tasktype/recommended_configuration_view.tsx b/frontend/javascripts/admin/tasktype/recommended_configuration_view.tsx index 3b551c9a659..d691ebc8508 100644 --- a/frontend/javascripts/admin/tasktype/recommended_configuration_view.tsx +++ b/frontend/javascripts/admin/tasktype/recommended_configuration_view.tsx @@ -49,7 +49,6 @@ function getRecommendedConfigByCategory() { }, volume: { brushSize: 50, - isVolumeInterpolationEnabled: false, }, }; } diff --git a/frontend/javascripts/libs/utils.ts b/frontend/javascripts/libs/utils.ts index 135060e3825..c8c39f88d53 100644 --- a/frontend/javascripts/libs/utils.ts +++ b/frontend/javascripts/libs/utils.ts @@ -915,3 +915,13 @@ export function coalesce(obj: { [key: string]: T }, field: T): T | null { } return null; } + +export function pluralize(str: string, count: number, optPluralForm: string | null = null): string { + if (count < 2) { + return str; + } + if (optPluralForm != null) { + return optPluralForm; + } + return `${str}s`; +} diff --git a/frontend/javascripts/messages.ts b/frontend/javascripts/messages.ts index 13e32ff44cd..2a9575c0780 100644 --- a/frontend/javascripts/messages.ts +++ b/frontend/javascripts/messages.ts @@ -43,7 +43,6 @@ export const settings: Partial> = gpuMemoryFactor: "Hardware Utilization", overwriteMode: "Volume Annotation Overwrite Mode", useLegacyBindings: "Classic Controls", - isVolumeInterpolationEnabled: "Volume Interpolation", }; export const settingsTooltips: Partial> = { loadingStrategy: `You can choose between loading the best quality first diff --git a/frontend/javascripts/oxalis/default_state.ts b/frontend/javascripts/oxalis/default_state.ts index a91d9a4b16b..3a02c554602 100644 --- a/frontend/javascripts/oxalis/default_state.ts +++ b/frontend/javascripts/oxalis/default_state.ts @@ -76,8 +76,6 @@ const defaultState: OxalisState = { overwriteMode: OverwriteModeEnum.OVERWRITE_ALL, fillMode: FillModeEnum._2D, useLegacyBindings: false, - isVolumeInterpolationEnabled: false, - volumeInterpolationDepth: 2, }, temporaryConfiguration: { viewMode: Constants.MODE_PLANE_TRACING, diff --git a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts index 8183fcad2cb..502aebee0fa 100644 --- a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts @@ -27,12 +27,13 @@ import { getVisibleSegmentationLayer, } from "oxalis/model/accessors/dataset_accessor"; import { getMaxZoomStepDiff } from "oxalis/model/bucket_data_handling/loading_strategy_logic"; -import { getRequestLogZoomStep } from "oxalis/model/accessors/flycam_accessor"; +import { getFlooredPosition, getRequestLogZoomStep } from "oxalis/model/accessors/flycam_accessor"; import { reuseInstanceOnEquality } from "oxalis/model/accessors/accessor_helpers"; export function getVolumeTracings(tracing: Tracing): Array { return tracing.volumes; } + export function getVolumeTracingById(tracing: Tracing, tracingId: string): VolumeTracing { const volumeTracing = tracing.volumes.find((t) => t.tracingId === tracingId); @@ -42,10 +43,12 @@ export function getVolumeTracingById(tracing: Tracing, tracingId: string): Volum return volumeTracing; } + export function getVolumeTracingLayers(dataset: APIDataset): Array { const layers = getSegmentationLayers(dataset); return layers.filter((layer) => layer.tracingId != null); } + export function getVolumeTracingByLayerName( tracing: Tracing, layerName: string, @@ -55,14 +58,17 @@ export function getVolumeTracingByLayerName( const volumeTracing = tracing.volumes.find((t) => t.tracingId === layerName); return volumeTracing; } + export function hasVolumeTracings(tracing: Tracing): boolean { return tracing.volumes.length > 0; } + export function getVolumeDescriptors( annotation: APIAnnotation | APIAnnotationCompact | HybridTracing, ): Array { return annotation.annotationLayers.filter((layer) => layer.typ === "Volume"); } + export function getVolumeDescriptorById( annotation: APIAnnotation | APIAnnotationCompact | HybridTracing, tracingId: string, @@ -77,6 +83,7 @@ export function getVolumeDescriptorById( return descriptors[0]; } + export function getReadableNameByVolumeTracingId( tracing: APIAnnotation | APIAnnotationCompact | HybridTracing, tracingId: string, @@ -115,10 +122,12 @@ export function getServerVolumeTracings( ); return volumeTracings; } + export function getActiveCellId(volumeTracing: VolumeTracing): number { const { activeCellId } = volumeTracing; return activeCellId; } + export function getContourTracingMode(volumeTracing: VolumeTracing): ContourMode { const { contourTracingMode } = volumeTracing; return contourTracingMode; @@ -137,6 +146,7 @@ const MAG_THRESHOLDS_FOR_ZOOM: Partial> = { export function isVolumeTool(tool: AnnotationTool): boolean { return VolumeTools.indexOf(tool) > -1; } + export function isVolumeAnnotationDisallowedForZoom(tool: AnnotationTool, state: OxalisState) { if (state.tracing.volumes.length === 0) { return true; @@ -171,12 +181,14 @@ export function getMaximumBrushSize(state: OxalisState) { // we double the maximum brush size. return MAX_BRUSH_SIZE_FOR_MAG1 * 2 ** lowestExistingResolutionIndex; } + export function isSegmentationMissingForZoomstep( state: OxalisState, maxZoomStepForSegmentation: number, ): boolean { return getRequestLogZoomStep(state) > maxZoomStepForSegmentation; } + export function getRequestedOrVisibleSegmentationLayer( state: OxalisState, layerName: string | null | undefined, @@ -230,9 +242,11 @@ export function getRequestedOrDefaultSegmentationTracingLayer( return getTracingForSegmentationLayer(state, visibleLayer); } + export function getActiveSegmentationTracing(state: OxalisState): VolumeTracing | null | undefined { return getRequestedOrDefaultSegmentationTracingLayer(state, null); } + export function getActiveSegmentationTracingLayer( state: OxalisState, ): APISegmentationLayer | null | undefined { @@ -244,6 +258,7 @@ export function getActiveSegmentationTracingLayer( return getSegmentationLayerForTracing(state, tracing); } + export function enforceActiveVolumeTracing(state: OxalisState): VolumeTracing { const tracing = getActiveSegmentationTracing(state); @@ -253,6 +268,7 @@ export function enforceActiveVolumeTracing(state: OxalisState): VolumeTracing { return tracing; } + export function getRequestedOrVisibleSegmentationLayerEnforced( state: OxalisState, layerName: string | null | undefined, @@ -268,6 +284,7 @@ export function getRequestedOrVisibleSegmentationLayerEnforced( "No segmentation layer is currently visible. Pass a valid layerName (you may want to use `getSegmentationLayerName`)", ); } + export function getNameOfRequestedOrVisibleSegmentationLayer( state: OxalisState, layerName: string | null | undefined, @@ -275,6 +292,7 @@ export function getNameOfRequestedOrVisibleSegmentationLayer( const layer = getRequestedOrVisibleSegmentationLayer(state, layerName); return layer != null ? layer.name : null; } + export function getSegmentsForLayer( state: OxalisState, layerName: string | null | undefined, @@ -291,6 +309,7 @@ export function getSegmentsForLayer( return state.localSegmentationData[layer.name].segments; } + export function getVisibleSegments(state: OxalisState): SegmentMap | null | undefined { const layer = getVisibleSegmentationLayer(state); @@ -387,9 +406,20 @@ function _getRenderableResolutionForActiveSegmentationTracing(state: OxalisState export const getRenderableResolutionForActiveSegmentationTracing = reuseInstanceOnEquality( _getRenderableResolutionForActiveSegmentationTracing, ); + export function getMappingInfoForVolumeTracing( state: OxalisState, tracingId: string | null | undefined, ): ActiveMappingInfo { return getMappingInfo(state.temporaryConfiguration.activeMappingByLayer, tracingId); } + +export function getPreviousCentroidInDim( + state: OxalisState, + volumeTracing: VolumeTracing, + dim: 0 | 1 | 2, +): Vector3 | undefined { + const position = getFlooredPosition(state.flycam); + + return volumeTracing.lastCentroids.find((el) => Math.floor(el[dim]) != position[dim]); +} diff --git a/frontend/javascripts/oxalis/model/dimensions.ts b/frontend/javascripts/oxalis/model/dimensions.ts index e4033b69da4..b1779f74656 100644 --- a/frontend/javascripts/oxalis/model/dimensions.ts +++ b/frontend/javascripts/oxalis/model/dimensions.ts @@ -1,5 +1,5 @@ -import type { OrthoView, Vector3 } from "oxalis/constants"; -import { OrthoViews } from "oxalis/constants"; +import { OrthoView, OrthoViews, OrthoViewsToName, Vector3 } from "oxalis/constants"; + export type DimensionIndices = 0 | 1 | 2; export type DimensionMap = [DimensionIndices, DimensionIndices, DimensionIndices]; // This is a class with static methods dealing with dimensions and @@ -69,6 +69,22 @@ const Dimensions = { } }, + dimensionNameForIndex(dim: DimensionIndices): string { + switch (dim) { + case 2: + return "Z"; + + case 0: + return "X"; + + case 1: + return "Y"; + + default: + throw new Error(`Unrecognized dimension: ${dim}`); + } + }, + roundCoordinate(coordinate: Vector3): Vector3 { return [Math.floor(coordinate[0]), Math.floor(coordinate[1]), Math.floor(coordinate[2])]; }, diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts index b3276f806ce..97eeed0ad77 100644 --- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts +++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer.ts @@ -123,12 +123,13 @@ function handleUpdateSegment(state: OxalisState, action: UpdateSegmentAction) { // If oldSegment exists, its creationTime will be // used by ...oldSegment creationTime: action.timestamp, + name: null, ...oldSegment, ...segment, somePosition, id: segmentId, }; - // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ somePosition: Vector3; id: num... Remove this comment to see the full error message + const newSegmentMap = segments.set(segmentId, newSegment); if (updateInfo.type === "UPDATE_VOLUME_TRACING") { @@ -150,7 +151,7 @@ export function serverVolumeToClientVolumeTracing(tracing: ServerVolumeTracing): const userBoundingBoxes = convertUserBoundingBoxesFromServerToFrontend(tracing.userBoundingBoxes); const volumeTracing = { createdTimestamp: tracing.createdTimestamp, - type: "volume", + type: "volume" as "volume", segments: new DiffableMap( tracing.segments.map((segment) => [ segment.segmentId, @@ -162,7 +163,7 @@ export function serverVolumeToClientVolumeTracing(tracing: ServerVolumeTracing): ]), ), activeCellId: 0, - lastCentroid: null, + lastCentroids: [], contourTracingMode: ContourModeEnum.DRAW, contourList: [], maxCellId, @@ -172,7 +173,6 @@ export function serverVolumeToClientVolumeTracing(tracing: ServerVolumeTracing): fallbackLayer: tracing.fallbackLayer, userBoundingBoxes, }; - // @ts-expect-error ts-migrate(2322) FIXME: Type '{ createdTimestamp: number; type: string; se... Remove this comment to see the full error message return volumeTracing; } diff --git a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts index cf42e0d319d..4a64a78a1bd 100644 --- a/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts +++ b/frontend/javascripts/oxalis/model/reducers/volumetracing_reducer_helpers.ts @@ -43,6 +43,8 @@ export function createCellReducer(state: OxalisState, volumeTracing: VolumeTraci activeCellId: id, }); } + +const MAXIMUM_CENTROID_COUNT = 50; export function updateDirectionReducer( state: OxalisState, volumeTracing: VolumeTracing, @@ -50,16 +52,17 @@ export function updateDirectionReducer( ) { let newState = state; - if (volumeTracing.lastCentroid != null) { + const lastCentroid = volumeTracing.lastCentroids[0]; + if (lastCentroid != null) { newState = setDirectionReducer(state, [ - centroid[0] - volumeTracing.lastCentroid[0], - centroid[1] - volumeTracing.lastCentroid[1], - centroid[2] - volumeTracing.lastCentroid[2], + centroid[0] - lastCentroid[0], + centroid[1] - lastCentroid[1], + centroid[2] - lastCentroid[2], ]); } return updateVolumeTracing(newState, volumeTracing.tracingId, { - lastCentroid: centroid, + lastCentroids: [centroid].concat(volumeTracing.lastCentroids).slice(0, MAXIMUM_CENTROID_COUNT), }); } export function addToLayerReducer( diff --git a/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts b/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts index 032fe3d56d9..db5aaf8b5d8 100644 --- a/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts @@ -16,15 +16,19 @@ import { getFlooredPosition, getRequestLogZoomStep } from "oxalis/model/accessor import { enforceActiveVolumeTracing, getActiveSegmentationTracingLayer, + getPreviousCentroidInDim, isVolumeAnnotationDisallowedForZoom, } from "oxalis/model/accessors/volumetracing_accessor"; +import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; import Dimensions from "oxalis/model/dimensions"; import type { Saga } from "oxalis/model/sagas/effect-generators"; import { select } from "oxalis/model/sagas/effect-generators"; -import VolumeLayer, { VoxelBuffer2D } from "oxalis/model/volumetracing/volumelayer"; +import { VoxelBuffer2D } from "oxalis/model/volumetracing/volumelayer"; import { call } from "typed-redux-saga"; import { createVolumeLayer, getBoundingBoxForViewport, labelWithVoxelBuffer2D } from "./helpers"; +export const MAXIMUM_INTERPOLATION_DEPTH = 8; + const isEqual = cwise({ args: ["array", "scalar"], body: function body(a: number, b: number) { @@ -99,7 +103,7 @@ function signedDist(arr: ndarray.NdArray) { } export default function* maybeInterpolateSegmentationLayer( - layer: VolumeLayer, + drawnBoundingBoxMag1: BoundingBox | null, isDrawing: boolean, activeTool: AnnotationTool, ): Saga { @@ -110,13 +114,11 @@ export default function* maybeInterpolateSegmentationLayer( return; } - const isVolumeInterpolationEnabled = yield* select( - (state) => - state.userConfiguration.isVolumeInterpolationEnabled && - state.tracing.restrictions.volumeInterpolationAllowed, + const isVolumeInterpolationAllowed = yield* select( + (state) => state.tracing.restrictions.volumeInterpolationAllowed, ); - if (!isVolumeInterpolationEnabled) { + if (!isVolumeInterpolationAllowed) { return; } @@ -156,20 +158,29 @@ export default function* maybeInterpolateSegmentationLayer( return; } - // Annotate only every n-th slice while the remaining ones are interpolated automatically. - const interpolationDepth = yield* select( - (store) => store.userConfiguration.volumeInterpolationDepth, + const previousCentroid = yield* select((store) => + getPreviousCentroidInDim(store, volumeTracing, thirdDim), ); + if (previousCentroid == null) { + console.warn("no last centroid"); + return; + } + const interpolationDepth = Math.abs(V3.floor(V3.sub(previousCentroid, position))[thirdDim]); - const drawnBoundingBoxMag1 = layer.getLabeledBoundingBox(); - if (drawnBoundingBoxMag1 == null) { + if (interpolationDepth < 2 || interpolationDepth > 8) { + console.warn("interpolation depth too small or too high", interpolationDepth); return; } + const viewportBoxMag1 = yield* call(getBoundingBoxForViewport, position, activeViewport); + if (drawnBoundingBoxMag1 == null) { + drawnBoundingBoxMag1 = viewportBoxMag1; + // return; + } + const transpose = (vector: Vector3) => Dimensions.transDim(vector, activeViewport); const uvSize = V3.scale3(drawnBoundingBoxMag1.getSize(), transpose([1, 1, 0])); - const viewportBoxMag1 = yield* call(getBoundingBoxForViewport, position, activeViewport); const relevantBoxMag1 = drawnBoundingBoxMag1 // Increase the drawn region by a factor of 2 (use half the size as a padding on each size) .paddedWithMargins(V3.scale(uvSize, 0.5)) diff --git a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx index 048d2fb80d2..7eb3f59b2b8 100644 --- a/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx +++ b/frontend/javascripts/oxalis/model/sagas/volumetracing_saga.tsx @@ -26,7 +26,6 @@ import { CONTOUR_COLOR_DELETE, CONTOUR_COLOR_NORMAL } from "oxalis/geometries/co import Model from "oxalis/model"; import { getBoundaries, getResolutionInfo } from "oxalis/model/accessors/dataset_accessor"; import { - getFlooredPosition, getPosition, getRequestLogZoomStep, getRotation, @@ -69,13 +68,11 @@ import { } from "oxalis/model/actions/volumetracing_actions"; import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; import { markVolumeTransactionEnd } from "oxalis/model/bucket_data_handling/bucket"; -import DataLayer from "oxalis/model/data_layer"; import Dimensions from "oxalis/model/dimensions"; import type { Saga } from "oxalis/model/sagas/effect-generators"; import { select, take } from "oxalis/model/sagas/effect-generators"; import listenToMinCut from "oxalis/model/sagas/min_cut_saga"; import { takeEveryUnlessBusy } from "oxalis/model/sagas/saga_helpers"; -import { getHalfViewportExtents } from "oxalis/model/sagas/saga_selectors"; import type { UpdateAction } from "oxalis/model/sagas/update_actions"; import { createSegmentVolumeAction, @@ -85,8 +82,7 @@ import { updateUserBoundingBoxes, updateVolumeTracing, } from "oxalis/model/sagas/update_actions"; -import VolumeLayer, { getFast3DCoordinateHelper } from "oxalis/model/volumetracing/volumelayer"; -import { applyVoxelMap } from "oxalis/model/volumetracing/volume_annotation_sampling"; +import VolumeLayer from "oxalis/model/volumetracing/volumelayer"; import type { Flycam, SegmentMap, VolumeTracing } from "oxalis/store"; import { getBBoxNameForPartialFloodfill } from "oxalis/view/right-border-tabs/bounding_box_tab"; import React from "react"; @@ -283,7 +279,12 @@ export function* editVolumeLayerAsync(): Saga { ), ); - yield* call(maybeInterpolateSegmentationLayer, currentLayer, isDrawing, activeTool); + // yield* call( + // maybeInterpolateSegmentationLayer, + // currentLayer.getLabeledBoundingBox(), + // isDrawing, + // activeTool, + // ); yield* put(finishAnnotationStrokeAction(volumeTracing.tracingId)); } @@ -323,140 +324,8 @@ function* getBoundingBoxForFloodFill( }; } -function* copySegmentationLayer(action: Action): Saga { - if (action.type !== "COPY_SEGMENTATION_LAYER") { - throw new Error("Satisfy typescript"); - } - - const allowUpdate = yield* select((state) => state.tracing.restrictions.allowUpdate); - if (!allowUpdate) return; - const activeViewport = yield* select((state) => state.viewModeData.plane.activeViewport); - - if (activeViewport === "TDView") { - // Cannot copy labels from 3D view - return; - } - - // Disable copy-segmentation for the same zoom steps where the trace tool is forbidden, too, - // to avoid large performance lags. - const isResolutionTooLow = yield* select((state) => - isVolumeAnnotationDisallowedForZoom(AnnotationToolEnum.TRACE, state), - ); - - if (isResolutionTooLow) { - Toast.warning( - 'The "copy segmentation"-feature is not supported at this zoom level. Please zoom in further.', - ); - return; - } - - const volumeTracing = yield* select(enforceActiveVolumeTracing); - const segmentationLayer: DataLayer = yield* call( - [Model, Model.getSegmentationTracingLayer], - volumeTracing.tracingId, - ); - const { cube } = segmentationLayer; - const requestedZoomStep = yield* select((state) => getRequestLogZoomStep(state)); - const resolutionInfo = yield* call(getResolutionInfo, segmentationLayer.resolutions); - const labeledZoomStep = resolutionInfo.getClosestExistingIndex(requestedZoomStep); - const dimensionIndices = Dimensions.getIndices(activeViewport); - const position = yield* select((state) => getFlooredPosition(state.flycam)); - const [halfViewportExtentX, halfViewportExtentY] = yield* call( - getHalfViewportExtents, - activeViewport, - ); - const activeCellId = volumeTracing.activeCellId; - const labeledVoxelMapOfCopiedVoxel: LabeledVoxelsMap = new Map(); - - function copyVoxelLabel(voxelTemplateAddress: Vector3, voxelTargetAddress: Vector3) { - const templateLabelValue = cube.getDataValue(voxelTemplateAddress, null, labeledZoomStep); - - // Only copy voxels from the previous layer which belong to the current cell - if (templateLabelValue === activeCellId) { - const currentLabelValue = cube.getDataValue(voxelTargetAddress, null, labeledZoomStep); - - // Do not overwrite already labeled voxels - if (currentLabelValue === 0) { - const bucket = cube.getOrCreateBucket( - cube.positionToZoomedAddress(voxelTargetAddress, labeledZoomStep), - ); - - if (bucket.type === "null") { - return; - } - - const labeledVoxelInBucket = cube.getVoxelOffset(voxelTargetAddress, labeledZoomStep); - const labelMapOfBucket = - labeledVoxelMapOfCopiedVoxel.get(bucket.zoomedAddress) || - new Uint8Array(Constants.BUCKET_WIDTH ** 2).fill(0); - const labeledVoxel2D = [ - labeledVoxelInBucket[dimensionIndices[0]], - labeledVoxelInBucket[dimensionIndices[1]], - ]; - labelMapOfBucket[labeledVoxel2D[0] * Constants.BUCKET_WIDTH + labeledVoxel2D[1]] = 1; - labeledVoxelMapOfCopiedVoxel.set(bucket.zoomedAddress, labelMapOfBucket); - } - } - } - - const thirdDim = dimensionIndices[2]; - const labeledResolution = resolutionInfo.getResolutionByIndexOrThrow(labeledZoomStep); - const directionInverter = action.source === "nextLayer" ? 1 : -1; - let direction = 1; - const useDynamicSpaceDirection = yield* select( - (state) => state.userConfiguration.dynamicSpaceDirection, - ); - - if (useDynamicSpaceDirection) { - const spaceDirectionOrtho = yield* select((state) => state.flycam.spaceDirectionOrtho); - direction = spaceDirectionOrtho[thirdDim]; - } - - const [tx, ty, tz] = Dimensions.transDim(position, activeViewport); - const z = tz; - // When using this tool in more coarse resolutions, the distance to the previous/next slice might be larger than 1 - const previousZ = z + direction * directionInverter * labeledResolution[thirdDim]; - for (let x = tx - halfViewportExtentX; x < tx + halfViewportExtentX; x++) { - for (let y = ty - halfViewportExtentY; y < ty + halfViewportExtentY; y++) { - copyVoxelLabel( - Dimensions.transDim([x, y, previousZ], activeViewport), - Dimensions.transDim([x, y, z], activeViewport), - ); - } - } - - if (labeledVoxelMapOfCopiedVoxel.size === 0) { - const dimensionLabels = ["x", "y", "z"]; - Toast.warning( - `Did not copy any voxels from slice ${dimensionLabels[thirdDim]}=${previousZ}.` + - ` Either no voxels with cell id ${activeCellId} were found or all of the respective voxels were already labeled in the current slice.`, - ); - } - - // applyVoxelMap assumes get3DAddress to be local to the corresponding bucket (so in the labeled resolution as well) - const zInLabeledResolution = - Math.floor(tz / labeledResolution[thirdDim]) % Constants.BUCKET_WIDTH; - applyVoxelMap( - labeledVoxelMapOfCopiedVoxel, - cube, - activeCellId, - getFast3DCoordinateHelper(activeViewport, zInLabeledResolution), - 1, - thirdDim, - false, - 0, - ); - applyLabeledVoxelMapToAllMissingResolutions( - labeledVoxelMapOfCopiedVoxel, - labeledZoomStep, - dimensionIndices, - resolutionInfo, - cube, - activeCellId, - z, - false, - ); - yield* put(finishAnnotationStrokeAction(volumeTracing.tracingId)); +function* copySegmentationLayer(_action: Action): Saga { + yield* call(maybeInterpolateSegmentationLayer, null, true, AnnotationToolEnum.TRACE); } const FLOODFILL_PROGRESS_KEY = "FLOODFILL_PROGRESS_KEY"; diff --git a/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts b/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts index c5ff8dcd250..0b515e54dc2 100644 --- a/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts +++ b/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts @@ -544,6 +544,10 @@ class VolumeLayer { } const area = sumArea / 2; + if (area == 0) { + return contourList[0]; + } + const cx = sumCx / 6 / area; const cy = sumCy / 6 / area; const zoomedPosition = this.get3DCoordinate([cx, cy]); diff --git a/frontend/javascripts/oxalis/store.ts b/frontend/javascripts/oxalis/store.ts index f098bf843f7..2a41395c97e 100644 --- a/frontend/javascripts/oxalis/store.ts +++ b/frontend/javascripts/oxalis/store.ts @@ -217,7 +217,7 @@ export type VolumeTracing = TracingBase & { readonly segments: SegmentMap; readonly maxCellId: number; readonly activeCellId: number; - readonly lastCentroid: Vector3 | null | undefined; + readonly lastCentroids: Vector3[]; // lastCentroids[0] is the newest readonly contourTracingMode: ContourMode; // Stores points of the currently drawn region in global coordinates readonly contourList: Array; @@ -304,8 +304,6 @@ export type UserConfiguration = { readonly overwriteMode: OverwriteMode; readonly fillMode: FillMode; readonly useLegacyBindings: boolean; - readonly isVolumeInterpolationEnabled: boolean; - readonly volumeInterpolationDepth: number; }; export type RecommendedConfiguration = Partial< UserConfiguration & diff --git a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx index bf4da3ddd86..b956acd8aea 100644 --- a/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/toolbar_view.tsx @@ -1,12 +1,15 @@ import { Radio, Tooltip, Badge, Space, Popover, RadioChangeEvent, Button } from "antd"; -import { CaretDownOutlined, CaretUpOutlined, ExportOutlined } from "@ant-design/icons"; +import { ExportOutlined } from "@ant-design/icons"; import { useSelector, useDispatch } from "react-redux"; import React, { useEffect, useState } from "react"; import { LogSliderSetting } from "oxalis/view/components/setting_input_views"; import { addUserBoundingBoxAction } from "oxalis/model/actions/annotation_actions"; import { convertCellIdToCSS } from "oxalis/view/left-border-tabs/mapping_settings_view"; -import { createCellAction } from "oxalis/model/actions/volumetracing_actions"; +import { + copySegmentationLayerAction, + createCellAction, +} from "oxalis/model/actions/volumetracing_actions"; import { createTreeAction, setMergerModeEnabledAction, @@ -16,6 +19,7 @@ import { getActiveSegmentationTracing, getMappingInfoForVolumeTracing, getMaximumBrushSize, + getPreviousCentroidInDim, getRenderableResolutionForActiveSegmentationTracing, } from "oxalis/model/accessors/volumetracing_accessor"; import { getActiveTree } from "oxalis/model/accessors/skeletontracing_accessor"; @@ -24,7 +28,7 @@ import { adaptActiveToolToShortcuts, } from "oxalis/model/accessors/tool_accessor"; import { setToolAction } from "oxalis/model/actions/ui_actions"; -import { toNullable } from "libs/utils"; +import { pluralize, toNullable } from "libs/utils"; import { updateUserSettingAction } from "oxalis/model/actions/settings_actions"; import { usePrevious, useKeyPress } from "libs/react_hooks"; import { userSettings } from "types/schemas/user_settings.schema"; @@ -47,6 +51,9 @@ import Store, { OxalisState, VolumeTracing } from "oxalis/store"; import Dimensions from "oxalis/model/dimensions"; import features from "features"; +import { V3 } from "libs/mjs"; +import { getFlooredPosition } from "oxalis/model/accessors/flycam_accessor"; +import { MAXIMUM_INTERPOLATION_DEPTH } from "oxalis/model/sagas/volume/volume_interpolation_saga"; const narrowButtonStyle = { paddingLeft: 10, @@ -230,57 +237,65 @@ function OverwriteModeSwitch({ } function VolumeInterpolationButton() { + const dispatch = useDispatch(); const isAllowed = useSelector( (state: OxalisState) => state.tracing.restrictions.volumeInterpolationAllowed, ); - const isEnabled = useSelector( - (state: OxalisState) => state.userConfiguration.isVolumeInterpolationEnabled, - ); - const spaceDirectionOrtho = useSelector((state: OxalisState) => state.flycam.spaceDirectionOrtho); const activeViewport = useSelector( (state: OxalisState) => state.viewModeData.plane.activeViewport, ); - const onChange = () => { - Store.dispatch(updateUserSettingAction("isVolumeInterpolationEnabled", !isEnabled)); + const onClick = () => { + dispatch(copySegmentationLayerAction()); }; - let directionIcon = null; - if (isEnabled && isAllowed && activeViewport !== OrthoViews.TDView) { - const thirdDim = Dimensions.thirdDimensionForPlane(activeViewport); - directionIcon = - spaceDirectionOrtho[thirdDim] > 0 ? ( - - ) : ( - - ); + const thirdDim = + activeViewport !== OrthoViews.TDView ? Dimensions.thirdDimensionForPlane(activeViewport) : 2; + const previousCentroid = useSelector((state: OxalisState) => { + const volumeTracing = getActiveSegmentationTracing(state); + if (!volumeTracing) { + return null; + } + return getPreviousCentroidInDim(state, volumeTracing, thirdDim); + }); + let isPossible = false; + let tooltipAddendum = + "Not available because all recent label actions were performed on the current slice."; + const position = useSelector((state: OxalisState) => getFlooredPosition(state.flycam)); + + if (previousCentroid != null) { + const interpolationDepth = Math.abs(V3.floor(V3.sub(previousCentroid, position))[thirdDim]); + isPossible = true; + tooltipAddendum = `Labels ${interpolationDepth - 1} ${pluralize( + "slice", + interpolationDepth - 1, + )} along ${Dimensions.dimensionNameForIndex(thirdDim)}`; + + if (activeViewport === OrthoViews.TDView) { + isPossible = false; + tooltipAddendum = "Not available for the 3D viewport"; + } else if (interpolationDepth > MAXIMUM_INTERPOLATION_DEPTH) { + isPossible = false; + tooltipAddendum = `Not available since the last label action is too many slices away (the maximum distance is ${MAXIMUM_INTERPOLATION_DEPTH})`; + } } return ( - - - - ); + const iconEl = faIcon != null && !this.props.loading ? : null; + const button = + // Differentiate via children != null, since antd uses a different styling for buttons + // with a single icon child (.ant-btn-icon-only will be assigned) + children != null ? ( + + ) : ( + + ); return title != null ? {button} : button; } } From 26f02b173031b98d0138b08f46856fb38eec5230 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 25 May 2022 16:47:39 +0200 Subject: [PATCH 17/18] properly regard the depth for volume interpolation depending on how the current mag effectively rounds the coordinates; fix wrong coordinate system for centroid when brushing with a single click --- frontend/javascripts/libs/mjs.ts | 9 +++ .../model/accessors/volumetracing_accessor.ts | 14 +++- .../sagas/volume/volume_interpolation_saga.ts | 68 ++++++++++++++----- .../oxalis/model/volumetracing/volumelayer.ts | 2 +- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/frontend/javascripts/libs/mjs.ts b/frontend/javascripts/libs/mjs.ts index df4bf34c397..1403ef445bd 100644 --- a/frontend/javascripts/libs/mjs.ts +++ b/frontend/javascripts/libs/mjs.ts @@ -282,6 +282,15 @@ const V3 = { toArray(vec: ArrayLike): Vector3 { return [vec[0], vec[1], vec[2]]; }, + + roundElementToResolution(vec: Vector3, resolution: Vector3, index: 0 | 1 | 2): Vector3 { + // Rounds the element at the position referenced by index so that it's divisible by the + // resolution element. + // For example: roundElementToResolution([11, 12, 13], [4, 4, 2], 2) == [11, 12, 12] + const res: Vector3 = [vec[0], vec[1], vec[2]]; + res[index] = Math.floor(res[index] / resolution[index]) * resolution[index]; + return res; + }, }; export { M4x4, V2, V3 }; diff --git a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts index 620a6c112ec..d5524778912 100644 --- a/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts +++ b/frontend/javascripts/oxalis/model/accessors/volumetracing_accessor.ts @@ -30,6 +30,7 @@ import { import { getMaxZoomStepDiff } from "oxalis/model/bucket_data_handling/loading_strategy_logic"; import { getFlooredPosition, getRequestLogZoomStep } from "oxalis/model/accessors/flycam_accessor"; import { reuseInstanceOnEquality } from "oxalis/model/accessors/accessor_helpers"; +import { V3 } from "libs/mjs"; export function getVolumeTracings(tracing: Tracing): Array { return tracing.volumes; @@ -422,9 +423,16 @@ export function getLastLabelAction(volumeTracing: VolumeTracing): LabelAction | export function getLabelActionFromPreviousSlice( state: OxalisState, volumeTracing: VolumeTracing, + resolution: Vector3, dim: 0 | 1 | 2, ): LabelAction | undefined { - const position = getFlooredPosition(state.flycam); - - return volumeTracing.lastLabelActions.find((el) => Math.floor(el.centroid[dim]) != position[dim]); + // Gets the last label action which was performed on a different slice. + // Note that in coarser mags (e.g., 8-8-2), the comparison of the coordinates + // is done while respecting how the coordinates are clipped due to that resolution. + const adapt = (vec: Vector3) => V3.roundElementToResolution(vec, resolution, dim); + const position = adapt(getFlooredPosition(state.flycam)); + + return volumeTracing.lastLabelActions.find( + (el) => Math.floor(adapt(el.centroid)[dim]) != position[dim], + ); } diff --git a/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts b/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts index 11c52998d87..28d9a69217b 100644 --- a/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts @@ -35,21 +35,51 @@ export const MAXIMUM_INTERPOLATION_DEPTH = 8; export function getInterpolationInfo(state: OxalisState, explanationPrefix: string) { const isAllowed = state.tracing.restrictions.volumeInterpolationAllowed; const volumeTracing = getActiveSegmentationTracing(state); - const mostRecentLabelAction = volumeTracing != null ? getLastLabelAction(volumeTracing) : null; + if (!volumeTracing) { + // Return dummy values, since the feature should be disabled, anyway + return { + tooltipTitle: "Volume Interpolation", + disabledExplanation: "Only available when a volume annotation exists.", + isDisabled: true, + activeViewport: OrthoViews.PLANE_XY, + previousCentroid: null, + labeledResolution: [1, 1, 1] as Vector3, + labeledZoomStep: 0, + }; + } + const mostRecentLabelAction = getLastLabelAction(volumeTracing); const activeViewport = mostRecentLabelAction?.plane || OrthoViews.PLANE_XY; - const thirdDim = Dimensions.thirdDimensionForPlane(activeViewport); - const previousCentroid = volumeTracing - ? getLabelActionFromPreviousSlice(state, volumeTracing, thirdDim)?.centroid - : null; + + const requestedZoomStep = getRequestLogZoomStep(state); + const segmentationLayer = Model.getSegmentationTracingLayer(volumeTracing.tracingId); + const resolutionInfo = getResolutionInfo(segmentationLayer.resolutions); + const labeledZoomStep = resolutionInfo.getClosestExistingIndex(requestedZoomStep); + const labeledResolution = resolutionInfo.getResolutionByIndexOrThrow(labeledZoomStep); + + const previousCentroid = getLabelActionFromPreviousSlice( + state, + volumeTracing, + labeledResolution, + thirdDim, + )?.centroid; let disabledExplanation = null; let tooltipAddendum = ""; if (previousCentroid != null) { const position = getFlooredPosition(state.flycam); - const interpolationDepth = Math.abs(V3.floor(V3.sub(previousCentroid, position))[thirdDim]); + // Note that in coarser mags (e.g., 8-8-2), the comparison of the coordinates + // is done while respecting how the coordinates are clipped due to that resolution. + // For example, in mag 8-8-2, the z distance needs to be divided by two, since it is measured + // in global coordinates. + const adapt = (vec: Vector3) => V3.roundElementToResolution(vec, labeledResolution, thirdDim); + const interpolationDepth = Math.floor( + Math.abs( + V3.sub(adapt(previousCentroid), adapt(position))[thirdDim] / labeledResolution[thirdDim], + ), + ); if (interpolationDepth > MAXIMUM_INTERPOLATION_DEPTH) { disabledExplanation = `${explanationPrefix} last labeled slice is too many slices away (distance > ${MAXIMUM_INTERPOLATION_DEPTH}).`; @@ -72,7 +102,15 @@ export function getInterpolationInfo(state: OxalisState, explanationPrefix: stri ? `Interpolate current segment between last labeled and current slice (V) – ${tooltipAddendum}` : "Volume Interpolation was disabled for this annotation."; const isDisabled = !(isAllowed && isPossible); - return { tooltipTitle, disabledExplanation, isDisabled, activeViewport, previousCentroid }; + return { + tooltipTitle, + disabledExplanation, + isDisabled, + activeViewport, + previousCentroid, + labeledResolution, + labeledZoomStep, + }; } const isEqual = cwise({ @@ -200,24 +238,22 @@ export default function* maybeInterpolateSegmentationLayer(): Saga { return; } - const { activeViewport, previousCentroid, disabledExplanation } = yield* select((state) => + const { + activeViewport, + previousCentroid, + disabledExplanation, + labeledResolution, + labeledZoomStep, + } = yield* select((state) => getInterpolationInfo(state, "Could not interpolate segment because"), ); const volumeTracing = yield* select(enforceActiveVolumeTracing); - const segmentationLayer = yield* call( - [Model, Model.getSegmentationTracingLayer], - volumeTracing.tracingId, - ); - const requestedZoomStep = yield* select((state) => getRequestLogZoomStep(state)); - const resolutionInfo = yield* call(getResolutionInfo, segmentationLayer.resolutions); - const labeledZoomStep = resolutionInfo.getClosestExistingIndex(requestedZoomStep); const [firstDim, secondDim, thirdDim] = Dimensions.getIndices(activeViewport); const position = yield* select((state) => getFlooredPosition(state.flycam)); const activeCellId = volumeTracing.activeCellId; - const labeledResolution = resolutionInfo.getResolutionByIndexOrThrow(labeledZoomStep); const spaceDirectionOrtho = yield* select((state) => state.flycam.spaceDirectionOrtho); const directionFactor = spaceDirectionOrtho[thirdDim]; diff --git a/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts b/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts index 0b515e54dc2..8defdc018c2 100644 --- a/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts +++ b/frontend/javascripts/oxalis/model/volumetracing/volumelayer.ts @@ -545,7 +545,7 @@ class VolumeLayer { const area = sumArea / 2; if (area == 0) { - return contourList[0]; + return zoomedPositionToGlobalPosition(contourList[0], this.activeResolution); } const cx = sumCx / 6 / area; From 6756ab70d260a1906865aa643fb7f8af5405f1a2 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 25 May 2022 16:53:39 +0200 Subject: [PATCH 18/18] fix wrong (not mag-adapted) interpolationDepth in saga --- .../oxalis/model/sagas/volume/volume_interpolation_saga.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts b/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts index 28d9a69217b..6ec55eb9717 100644 --- a/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts +++ b/frontend/javascripts/oxalis/model/sagas/volume/volume_interpolation_saga.ts @@ -35,6 +35,7 @@ export const MAXIMUM_INTERPOLATION_DEPTH = 8; export function getInterpolationInfo(state: OxalisState, explanationPrefix: string) { const isAllowed = state.tracing.restrictions.volumeInterpolationAllowed; const volumeTracing = getActiveSegmentationTracing(state); + let interpolationDepth = 0; if (!volumeTracing) { // Return dummy values, since the feature should be disabled, anyway return { @@ -45,6 +46,7 @@ export function getInterpolationInfo(state: OxalisState, explanationPrefix: stri previousCentroid: null, labeledResolution: [1, 1, 1] as Vector3, labeledZoomStep: 0, + interpolationDepth, }; } const mostRecentLabelAction = getLastLabelAction(volumeTracing); @@ -75,7 +77,7 @@ export function getInterpolationInfo(state: OxalisState, explanationPrefix: stri // For example, in mag 8-8-2, the z distance needs to be divided by two, since it is measured // in global coordinates. const adapt = (vec: Vector3) => V3.roundElementToResolution(vec, labeledResolution, thirdDim); - const interpolationDepth = Math.floor( + interpolationDepth = Math.floor( Math.abs( V3.sub(adapt(previousCentroid), adapt(position))[thirdDim] / labeledResolution[thirdDim], ), @@ -110,6 +112,7 @@ export function getInterpolationInfo(state: OxalisState, explanationPrefix: stri previousCentroid, labeledResolution, labeledZoomStep, + interpolationDepth, }; } @@ -244,6 +247,7 @@ export default function* maybeInterpolateSegmentationLayer(): Saga { disabledExplanation, labeledResolution, labeledZoomStep, + interpolationDepth, } = yield* select((state) => getInterpolationInfo(state, "Could not interpolate segment because"), ); @@ -270,7 +274,6 @@ export default function* maybeInterpolateSegmentationLayer(): Saga { } return; } - const interpolationDepth = Math.abs(V3.floor(V3.sub(previousCentroid, position))[thirdDim]); const viewportBoxMag1 = yield* call(getBoundingBoxForViewport, position, activeViewport);