From 9ded08e581335675681946f24f6e00a2253532a7 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 16 Feb 2022 09:38:57 +0100 Subject: [PATCH 01/18] draft: dynamically allocate buckets so that no hard limit exists --- .../model/bucket_data_handling/bucket.js | 1 + .../model/bucket_data_handling/data_cube.js | 87 +++++++++++++------ .../bucket_data_handling/wkstore_adapter.js | 2 + 3 files changed, 63 insertions(+), 27 deletions(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js b/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js index b4a7d5b1bb0..d71d8898df6 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js @@ -230,6 +230,7 @@ export class DataBucket { this.data = null; // Remove all event handlers (see https://github.com/ai/nanoevents#remove-all-listeners) this.emitter.events = {}; + this.trigger("bucketCollected"); } needsRequest(): boolean { diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index 004bb9cc136..fcd3c690eeb 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -81,6 +81,8 @@ class DataCube { resolutionInfo: ResolutionInfo; layerName: string; + dirtyBucketsCount: number; + // The cube stores the buckets in a separate array for each zoomStep. For each // zoomStep the cube-array contains the boundaries and an array holding the buckets. // The bucket-arrays are initialized large enough to hold the whole cube. Thus no @@ -108,13 +110,41 @@ class DataCube { this.resolutionInfo = resolutionInfo; this.layerName = layerName; + this.dirtyBucketsCount = 0; + + if (isSegmentation) { + setInterval(() => { + console.log("#######"); + + let dirtyBucketsCount = 0; + for (let i = 0; i < this.buckets.length; i++) { + if (this.buckets[i].dirty) { + dirtyBucketsCount++; + } + } + + let weirdDirtyState = 0; + for (let i = 0; i < this.buckets.length; i++) { + if (this.buckets[i].dirty && this.buckets[i].dirtyCount === 0) { + weirdDirtyState++; + } + } + if (weirdDirtyState > 0) { + console.log("weirdDirtyState", weirdDirtyState); + } + + console.log("dirtyBucketsCount", dirtyBucketsCount); + console.log("this.buckets.length", this.buckets.length); + }, 2000); + } + _.extend(this, BackboneEvents); this.cubes = []; if (isSegmentation) { - this.MAXIMUM_BUCKET_COUNT *= 2; + this.MAXIMUM_BUCKET_COUNT = Math.floor(this.MAXIMUM_BUCKET_COUNT / 5); } - this.buckets = new Array(this.MAXIMUM_BUCKET_COUNT); + this.buckets = []; // Initializing the cube-arrays with boundaries const cubeBoundary = [ @@ -280,44 +310,48 @@ class DataCube { } markBucketsAsUnneeded(): void { - for (let i = 0; i < this.bucketCount; i++) { + for (let i = 0; i < this.buckets.length; i++) { this.buckets[i].markAsUnneeded(); } } addBucketToGarbageCollection(bucket: DataBucket): void { - if (this.bucketCount >= this.MAXIMUM_BUCKET_COUNT) { - for (let i = 0; i < this.bucketCount; i++) { - this.bucketIterator = ++this.bucketIterator % this.MAXIMUM_BUCKET_COUNT; + if (this.buckets.length >= this.MAXIMUM_BUCKET_COUNT) { + let foundCollectibleBucket = false; + + for (let i = 0; i < this.buckets.length; i++) { + this.bucketIterator = (this.bucketIterator + 1) % this.buckets.length; if (this.buckets[this.bucketIterator].shouldCollect()) { + foundCollectibleBucket = true; break; } } - if (!this.buckets[this.bucketIterator].shouldCollect()) { - if (process.env.BABEL_ENV === "test") { - throw new Error("Bucket was forcefully evicted/garbage-collected."); - } - const errorMessage = - "A bucket was forcefully garbage-collected. This indicates that too many buckets are currently in RAM."; - console.error(errorMessage); - ErrorHandling.notify(new Error(errorMessage), { - elementClass: this.elementClass, - isSegmentation: this.isSegmentation, - resolutionInfo: this.resolutionInfo, - }); + if (foundCollectibleBucket) { + this.collectBucket(this.buckets[this.bucketIterator]); + } else { + // if (process.env.BABEL_ENV === "test") { + // throw new Error("Bucket was forcefully evicted/garbage-collected."); + // } + // const errorMessage = + // "A bucket was forcefully garbage-collected. This indicates that too many buckets are currently in RAM."; + // console.error(errorMessage); + // ErrorHandling.notify(new Error(errorMessage), { + // elementClass: this.elementClass, + // isSegmentation: this.isSegmentation, + // resolutionInfo: this.resolutionInfo, + // }); + + // Effectively, push to this.buckets by setting the iterator to + // a new index. + this.bucketIterator = this.buckets.length; } - - this.collectBucket(this.buckets[this.bucketIterator]); - this.bucketCount--; } - this.bucketCount++; - if (this.buckets[this.bucketIterator]) { - this.buckets[this.bucketIterator].trigger("bucketCollected"); - } this.buckets[this.bucketIterator] = bucket; - this.bucketIterator = ++this.bucketIterator % this.MAXIMUM_BUCKET_COUNT; + // Note that bucketIterator is allowed to point to the next free + // slot (i.e., bucketIterator == this.buckets.length). + this.bucketIterator = (this.bucketIterator + 1) % (this.buckets.length + 1); } collectAllBuckets(): void { @@ -326,7 +360,6 @@ class DataCube { for (const bucket of this.buckets) { if (bucket != null) { this.collectBucket(bucket); - bucket.trigger("bucketCollected"); } } this.buckets = []; diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js index 30636230bc0..eb5eb1fff52 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js @@ -222,6 +222,7 @@ function sliceBufferIntoPieces( export async function sendToStore(batch: Array, tracingId: string): Promise { const items = []; + console.time("sendToStore for " + batch.length); for (const bucket of batch) { const data = bucket.getData(); const bucketInfo = createSendBucketInfo( @@ -233,5 +234,6 @@ export async function sendToStore(batch: Array, tracingId: string): const compressedBase64 = await byteArrayToLz4Base64(byteArray); items.push(updateBucket(bucketInfo, compressedBase64)); } + console.timeEnd("sendToStore for " + batch.length); Store.dispatch(pushSaveQueueTransaction(items, "volume", tracingId)); } From 3d9c7a404f3e91612e2158281f22958af7dd1d50 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Thu, 17 Feb 2022 12:16:26 +0100 Subject: [PATCH 02/18] use multiple compression workers; show bucket stats at save button; expose comlinks transfer function --- frontend/javascripts/oxalis/model.js | 14 +++++ .../model/bucket_data_handling/pushqueue.js | 9 +++- .../bucket_data_handling/wkstore_adapter.js | 53 +++++++++++++------ .../oxalis/model/sagas/update_actions.js | 2 +- .../oxalis/view/action-bar/save_button.js | 30 +++++++++-- .../oxalis/workers/comlink_wrapper.js | 9 ++-- 6 files changed, 90 insertions(+), 27 deletions(-) diff --git a/frontend/javascripts/oxalis/model.js b/frontend/javascripts/oxalis/model.js index 13d10028cf0..856a24a2ae5 100644 --- a/frontend/javascripts/oxalis/model.js +++ b/frontend/javascripts/oxalis/model.js @@ -264,6 +264,20 @@ export class OxalisModel { return storeStateSaved && pushQueuesSaved; } + getTotalPushQueueSize() { + const compressingBuckets = _.sum( + Object.values(this.dataLayers).map(dataLayer => dataLayer.pushQueue.taskQueue.tasks.length), + ); + const pendingBuckets = _.sum( + Object.values(this.dataLayers).map(dataLayer => dataLayer.pushQueue.queue.size), + ); + + return { + compressingBuckets, + pendingBuckets, + }; + } + forceSave = () => { // In contrast to the save function, this method will trigger exactly one saveNowAction // regardless of what the current save state is diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js b/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js index dedf12fb41e..85a4a4f5953 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js @@ -9,7 +9,12 @@ import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; import Toast from "libs/toast"; const BATCH_SIZE = 32; -const DEBOUNCE_TIME = 1000; +// Only process the PushQueue after there was no user interaction +// for PUSH_DEBOUNCE_TIME milliseconds... +const PUSH_DEBOUNCE_TIME = 1000; +// ...unless a timeout of PUSH_DEBOUNCE_MAX_WAIT_TIME milliseconds +// is exceeded. Then, initiate a push. +const PUSH_DEBOUNCE_MAX_WAIT_TIME = 30000; class PushQueue { dataSetName: string; @@ -92,7 +97,7 @@ class PushQueue { } }; - push = _.debounce(this.pushImpl, DEBOUNCE_TIME); + push = _.debounce(this.pushImpl, PUSH_DEBOUNCE_TIME, { maxWait: PUSH_DEBOUNCE_MAX_WAIT_TIME }); pushBatch(batch: Array): Promise { return sendToStore(batch, this.cube.layerName); diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js index eb5eb1fff52..d60301f2037 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js @@ -1,5 +1,6 @@ // @flow +import _ from "lodash"; import type { DataBucket } from "oxalis/model/bucket_data_handling/bucket"; import { bucketPositionToGlobalAddress } from "oxalis/model/helpers/position_converter"; import { createWorker } from "oxalis/workers/comlink_wrapper"; @@ -13,7 +14,7 @@ import { } from "oxalis/model/accessors/dataset_accessor"; import { parseAsMaybe } from "libs/utils"; import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions"; -import { updateBucket } from "oxalis/model/sagas/update_actions"; +import { updateBucket, type UpdateBucketUpdateAction } from "oxalis/model/sagas/update_actions"; import ByteArrayToLz4Base64Worker from "oxalis/workers/byte_array_to_lz4_base64.worker"; import DecodeFourBitWorker from "oxalis/workers/decode_four_bit.worker"; import ErrorHandling from "libs/error_handling"; @@ -23,7 +24,18 @@ import constants, { type Vector3, type Vector4, MappingStatusEnum } from "oxalis import window from "libs/window"; const decodeFourBit = createWorker(DecodeFourBitWorker); -const byteArrayToLz4Base64 = createWorker(ByteArrayToLz4Base64Worker); + +const workers = _.range(0, 2).map(idx => createWorker(ByteArrayToLz4Base64Worker)); +let workerIdx = 0; +let maximumSize = 0; +const byteArrayToLz4Base64 = array => { + if (array.length > maximumSize) { + maximumSize = array.length; + console.log("new maximumSize", maximumSize); + } + workerIdx = (workerIdx + 1) % workers.length; + return workers[workerIdx](array); +}; export const REQUEST_TIMEOUT = 60000; @@ -221,19 +233,28 @@ function sliceBufferIntoPieces( } export async function sendToStore(batch: Array, tracingId: string): Promise { - const items = []; - console.time("sendToStore for " + batch.length); - for (const bucket of batch) { - const data = bucket.getData(); - const bucketInfo = createSendBucketInfo( - bucket.zoomedAddress, - getResolutions(Store.getState().dataset), - ); - const byteArray = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - // eslint-disable-next-line no-await-in-loop - const compressedBase64 = await byteArrayToLz4Base64(byteArray); - items.push(updateBucket(bucketInfo, compressedBase64)); - } - console.timeEnd("sendToStore for " + batch.length); + const then = performance.now(); + + const items = await Promise.all( + batch.map( + async (bucket): Promise => { + const data = bucket.getCopyOfData(); + const bucketInfo = createSendBucketInfo( + bucket.zoomedAddress, + getResolutions(Store.getState().dataset), + ); + const byteArray = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + // eslint-disable-next-line no-await-in-loop + const compressedBase64 = await byteArrayToLz4Base64(byteArray); + console.log("byteArray", byteArray); + return updateBucket(bucketInfo, compressedBase64); + }, + ), + ); + const duration = performance.now() - then; + const durationPerItem = duration / batch.length; + + console.log("durationPerItem", durationPerItem); + Store.dispatch(pushSaveQueueTransaction(items, "volume", tracingId)); } diff --git a/frontend/javascripts/oxalis/model/sagas/update_actions.js b/frontend/javascripts/oxalis/model/sagas/update_actions.js index 6f18bfd57fd..1bbcd20a3b4 100644 --- a/frontend/javascripts/oxalis/model/sagas/update_actions.js +++ b/frontend/javascripts/oxalis/model/sagas/update_actions.js @@ -134,7 +134,7 @@ type UpdateUserBoundingBoxesAction = {| boundingBoxes: Array, |}, |}; -type UpdateBucketUpdateAction = {| +export type UpdateBucketUpdateAction = {| name: "updateBucket", value: SendBucketInfo & { base64Data: string, diff --git a/frontend/javascripts/oxalis/view/action-bar/save_button.js b/frontend/javascripts/oxalis/view/action-bar/save_button.js index 1f5377e3df3..07f522785dc 100644 --- a/frontend/javascripts/oxalis/view/action-bar/save_button.js +++ b/frontend/javascripts/oxalis/view/action-bar/save_button.js @@ -52,6 +52,10 @@ class SaveButton extends React.PureComponent { state = { isStateSaved: false, showUnsavedWarning: false, + saveInfo: { + compressingBuckets: 0, + pendingBuckets: 0, + }, }; componentDidMount() { @@ -74,9 +78,15 @@ class SaveButton extends React.PureComponent { reportUnsavedDurationThresholdExceeded(); } + const { compressingBuckets, pendingBuckets } = Model.getTotalPushQueueSize(); + this.setState({ isStateSaved, showUnsavedWarning, + saveInfo: { + compressingBuckets, + pendingBuckets, + }, }); }; @@ -107,11 +117,21 @@ class SaveButton extends React.PureComponent { className={this.props.className} style={{ background: showUnsavedWarning ? "var(--ant-error)" : null }} > - {this.shouldShowProgress() ? ( - {Math.floor((progressFraction || 0) * 100)} % - ) : ( - Save - )} + +
Pending buckets: {this.state.saveInfo.pendingBuckets}
+
Compressing buckets: {this.state.saveInfo.compressingBuckets}
+ + } + > + {this.shouldShowProgress() ? ( + {Math.floor((progressFraction || 0) * 100)} % + ) : ( + Save + )} + {showUnsavedWarning ? ( (t: T): Promise { // $FlowExpectedError[incompatible-return] return t; } + +export const transfer = _transfer; From e1d5065ee144fc8cf107a0b4dc59667723163eca Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Thu, 17 Feb 2022 12:16:41 +0100 Subject: [PATCH 03/18] try to integrate lz4-wasm --- package.json | 1 + webpack.config.js | 3 +++ yarn.lock | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/package.json b/package.json index b9a27458995..52702599e49 100644 --- a/package.json +++ b/package.json @@ -159,6 +159,7 @@ "jsonschema": "^1.2.4", "jszip": "^3.7.0", "lodash": "^4.17.21", + "lz4-wasm": "^0.7.5", "lz4js": "^0.2.0", "memoize-one": "^4.0.2", "mini-css-extract-plugin": "^2.5.2", diff --git a/webpack.config.js b/webpack.config.js index fb4b0b97a58..686d7c83f02 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -62,6 +62,9 @@ module.exports = function(env = {}) { }; return { + experiments: { + syncWebAssembly: true, + }, entry: { main: "main.js", light: "style_light.js", diff --git a/yarn.lock b/yarn.lock index 1dcb2b17709..fca440dfd4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8371,6 +8371,11 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" +lz4-wasm@^0.7.5: + version "0.7.5" + resolved "https://registry.yarnpkg.com/lz4-wasm/-/lz4-wasm-0.7.5.tgz#1f852bb82d193eb25428048077a5b68173f97018" + integrity sha512-OoZg/zRi5D/ccDAVrsvSmRrbRQeaDJLXJSAIGS90iCEFQJJpeQjXRj39pNDQHIxgqwT3StkthDdqlOKJxXGZZg== + lz4js@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/lz4js/-/lz4js-0.2.0.tgz#09f1a397cb2158f675146c3351dde85058cb322f" From 6c6a23b4668ba5cfa5c34f05581fc8a5c871ed31 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Thu, 17 Feb 2022 12:16:48 +0100 Subject: [PATCH 04/18] Revert "try to integrate lz4-wasm" This reverts commit e1d5065ee144fc8cf107a0b4dc59667723163eca. --- package.json | 1 - webpack.config.js | 3 --- yarn.lock | 5 ----- 3 files changed, 9 deletions(-) diff --git a/package.json b/package.json index 52702599e49..b9a27458995 100644 --- a/package.json +++ b/package.json @@ -159,7 +159,6 @@ "jsonschema": "^1.2.4", "jszip": "^3.7.0", "lodash": "^4.17.21", - "lz4-wasm": "^0.7.5", "lz4js": "^0.2.0", "memoize-one": "^4.0.2", "mini-css-extract-plugin": "^2.5.2", diff --git a/webpack.config.js b/webpack.config.js index 686d7c83f02..fb4b0b97a58 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -62,9 +62,6 @@ module.exports = function(env = {}) { }; return { - experiments: { - syncWebAssembly: true, - }, entry: { main: "main.js", light: "style_light.js", diff --git a/yarn.lock b/yarn.lock index fca440dfd4a..1dcb2b17709 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8371,11 +8371,6 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lz4-wasm@^0.7.5: - version "0.7.5" - resolved "https://registry.yarnpkg.com/lz4-wasm/-/lz4-wasm-0.7.5.tgz#1f852bb82d193eb25428048077a5b68173f97018" - integrity sha512-OoZg/zRi5D/ccDAVrsvSmRrbRQeaDJLXJSAIGS90iCEFQJJpeQjXRj39pNDQHIxgqwT3StkthDdqlOKJxXGZZg== - lz4js@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/lz4js/-/lz4js-0.2.0.tgz#09f1a397cb2158f675146c3351dde85058cb322f" From 48bb6169ef1453c09e619b4f2ee3157e82dcc2fa Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 18 Feb 2022 14:12:11 +0100 Subject: [PATCH 05/18] create WorkerPool abstraction --- frontend/javascripts/libs/task_pool.js | 5 ++++ frontend/javascripts/libs/worker_pool.js | 29 ++++++++++++++++++ .../bucket_data_handling/wkstore_adapter.js | 30 ++++++++----------- 3 files changed, 47 insertions(+), 17 deletions(-) create mode 100644 frontend/javascripts/libs/worker_pool.js diff --git a/frontend/javascripts/libs/task_pool.js b/frontend/javascripts/libs/task_pool.js index 36e03f01e5f..3c6918f554b 100644 --- a/frontend/javascripts/libs/task_pool.js +++ b/frontend/javascripts/libs/task_pool.js @@ -1,5 +1,10 @@ // @flow +/* + Given an array of async tasks, processTaskWithPool + allows to execute at most ${poolSize} tasks concurrently. + */ + export default function processTaskWithPool( tasks: Array<() => Promise>, poolSize: number, diff --git a/frontend/javascripts/libs/worker_pool.js b/frontend/javascripts/libs/worker_pool.js new file mode 100644 index 00000000000..8959c77b63a --- /dev/null +++ b/frontend/javascripts/libs/worker_pool.js @@ -0,0 +1,29 @@ +// @flow + +import _ from "lodash"; + +export default class WorkerPool { + // This class can be used to instantiate multiple web workers + // which are then used for computation in a simple round-robin manner. + // + // Example: + // const compressionPool = new WorkerPool( + // () => createWorker(ByteArrayToLz4Base64Worker), + // COMPRESSION_WORKER_COUNT, + // ); + // const promise1 = compressionPool.submit(data1); + // const promise2 = compressionPool.submit(data2); + + workers: Array<(...args: Array

) => R>; + currentWorkerIdx: number; + + constructor(workerFn: () => (...args: Array

) => R, count: number) { + this.workers = _.range(0, count).map(idx => workerFn()); + this.currentWorkerIdx = 0; + } + + submit(...args: Array

): R { + this.currentWorkerIdx = (this.currentWorkerIdx + 1) % this.workers.length; + return this.workers[this.currentWorkerIdx](...args); + } +} diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js index d60301f2037..5505897dab4 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js @@ -1,41 +1,38 @@ // @flow import _ from "lodash"; + import type { DataBucket } from "oxalis/model/bucket_data_handling/bucket"; import { bucketPositionToGlobalAddress } from "oxalis/model/helpers/position_converter"; import { createWorker } from "oxalis/workers/comlink_wrapper"; import { doWithToken } from "admin/admin_rest_api"; -import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor"; import { getResolutions, isSegmentationLayer, getByteCountFromLayer, getMappingInfo, } from "oxalis/model/accessors/dataset_accessor"; +import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor"; import { parseAsMaybe } from "libs/utils"; import { pushSaveQueueTransaction } from "oxalis/model/actions/save_actions"; -import { updateBucket, type UpdateBucketUpdateAction } from "oxalis/model/sagas/update_actions"; +import { updateBucket, type UpdateAction } from "oxalis/model/sagas/update_actions"; import ByteArrayToLz4Base64Worker from "oxalis/workers/byte_array_to_lz4_base64.worker"; import DecodeFourBitWorker from "oxalis/workers/decode_four_bit.worker"; import ErrorHandling from "libs/error_handling"; import Request from "libs/request"; import Store, { type DataLayerType, type VolumeTracing } from "oxalis/store"; +import WorkerPool from "libs/worker_pool"; import constants, { type Vector3, type Vector4, MappingStatusEnum } from "oxalis/constants"; import window from "libs/window"; const decodeFourBit = createWorker(DecodeFourBitWorker); -const workers = _.range(0, 2).map(idx => createWorker(ByteArrayToLz4Base64Worker)); -let workerIdx = 0; -let maximumSize = 0; -const byteArrayToLz4Base64 = array => { - if (array.length > maximumSize) { - maximumSize = array.length; - console.log("new maximumSize", maximumSize); - } - workerIdx = (workerIdx + 1) % workers.length; - return workers[workerIdx](array); -}; +const COMPRESSION_WORKER_COUNT = 2; + +const compressionPool = new WorkerPool( + () => createWorker(ByteArrayToLz4Base64Worker), + COMPRESSION_WORKER_COUNT, +); export const REQUEST_TIMEOUT = 60000; @@ -235,9 +232,9 @@ function sliceBufferIntoPieces( export async function sendToStore(batch: Array, tracingId: string): Promise { const then = performance.now(); - const items = await Promise.all( + const items: Array = await Promise.all( batch.map( - async (bucket): Promise => { + async (bucket): Promise => { const data = bucket.getCopyOfData(); const bucketInfo = createSendBucketInfo( bucket.zoomedAddress, @@ -245,8 +242,7 @@ export async function sendToStore(batch: Array, tracingId: string): ); const byteArray = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); // eslint-disable-next-line no-await-in-loop - const compressedBase64 = await byteArrayToLz4Base64(byteArray); - console.log("byteArray", byteArray); + const compressedBase64 = await compressionPool.submit(byteArray); return updateBucket(bucketInfo, compressedBase64); }, ), From 0361f769bcf41c6d1a00101fbddf74b16d977e10 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 18 Feb 2022 15:53:47 +0100 Subject: [PATCH 06/18] clean up UI progress for compressing buckets --- frontend/javascripts/libs/utils.js | 5 +++ frontend/javascripts/oxalis/model.js | 17 +++++--- .../model/bucket_data_handling/pushqueue.js | 43 +++++++++++-------- .../oxalis/view/action-bar/save_button.js | 26 ++++++----- .../fetch_buffer_with_headers.worker.js | 13 +++--- 5 files changed, 63 insertions(+), 41 deletions(-) diff --git a/frontend/javascripts/libs/utils.js b/frontend/javascripts/libs/utils.js index 9683f7b6ade..45d9fefc971 100644 --- a/frontend/javascripts/libs/utils.js +++ b/frontend/javascripts/libs/utils.js @@ -24,6 +24,11 @@ export function mod(x: number, n: number) { return ((x % n) + n) % n; } +export function values(o: { [K]: V }): Array { + // $FlowIssue[incompatible-return] remove once https://github.com/facebook/flow/issues/2221 is fixed + return Object.values(o); +} + export function map2(fn: (A, number) => B, tuple: [A, A]): [B, B] { const [x, y] = tuple; return [fn(x, 0), fn(y, 1)]; diff --git a/frontend/javascripts/oxalis/model.js b/frontend/javascripts/oxalis/model.js index 856a24a2ae5..1e6245668f9 100644 --- a/frontend/javascripts/oxalis/model.js +++ b/frontend/javascripts/oxalis/model.js @@ -6,6 +6,7 @@ import {} from "oxalis/model/actions/settings_actions"; import _ from "lodash"; +import { COMPRESSING_BATCH_SIZE } from "oxalis/model/bucket_data_handling/pushqueue"; import { type Vector3 } from "oxalis/constants"; import type { Versions } from "oxalis/view/version_view"; import { getActiveSegmentationTracingLayer } from "oxalis/model/accessors/volumetracing_accessor"; @@ -264,17 +265,19 @@ export class OxalisModel { return storeStateSaved && pushQueuesSaved; } - getTotalPushQueueSize() { - const compressingBuckets = _.sum( - Object.values(this.dataLayers).map(dataLayer => dataLayer.pushQueue.taskQueue.tasks.length), + getPushQueueStats() { + const compressingBucketCount = _.sum( + Utils.values(this.dataLayers).map( + dataLayer => dataLayer.pushQueue.compressionTaskQueue.tasks.length * COMPRESSING_BATCH_SIZE, + ), ); - const pendingBuckets = _.sum( - Object.values(this.dataLayers).map(dataLayer => dataLayer.pushQueue.queue.size), + const waitingForCompressionBucketCount = _.sum( + Utils.values(this.dataLayers).map(dataLayer => dataLayer.pushQueue.pendingQueue.size), ); return { - compressingBuckets, - pendingBuckets, + compressingBucketCount, + waitingForCompressionBucketCount, }; } diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js b/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js index 85a4a4f5953..caa47ea69e5 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/pushqueue.js @@ -8,7 +8,7 @@ import AsyncTaskQueue from "libs/async_task_queue"; import type DataCube from "oxalis/model/bucket_data_handling/data_cube"; import Toast from "libs/toast"; -const BATCH_SIZE = 32; +export const COMPRESSING_BATCH_SIZE = 32; // Only process the PushQueue after there was no user interaction // for PUSH_DEBOUNCE_TIME milliseconds... const PUSH_DEBOUNCE_TIME = 1000; @@ -19,25 +19,30 @@ const PUSH_DEBOUNCE_MAX_WAIT_TIME = 30000; class PushQueue { dataSetName: string; cube: DataCube; - taskQueue: AsyncTaskQueue; + compressionTaskQueue: AsyncTaskQueue; sendData: boolean; - queue: Set; + // The pendingQueue contains all buckets which are marked as + // "should be snapshotted and saved". That queue is processed + // in a debounced manner and sent to the `compressionTaskQueue`. + // The `compressionTaskQueue` compresses the bucket data and + // sends it to the save queue. + pendingQueue: Set; constructor(cube: DataCube, sendData: boolean = true) { this.cube = cube; - this.taskQueue = new AsyncTaskQueue(Infinity); + this.compressionTaskQueue = new AsyncTaskQueue(Infinity); this.sendData = sendData; - this.queue = new Set(); + this.pendingQueue = new Set(); const autoSaveFailureMessage = "Auto-Save failed!"; - this.taskQueue.on("failure", () => { + this.compressionTaskQueue.on("failure", () => { console.error("PushQueue failure"); if (document.body != null) { document.body.classList.add("save-error"); } Toast.error(autoSaveFailureMessage, { sticky: true }); }); - this.taskQueue.on("success", () => { + this.compressionTaskQueue.on("success", () => { if (document.body != null) { document.body.classList.remove("save-error"); } @@ -47,15 +52,15 @@ class PushQueue { stateSaved(): boolean { return ( - this.queue.size === 0 && + this.pendingQueue.size === 0 && this.cube.temporalBucketManager.getCount() === 0 && - !this.taskQueue.isBusy() + !this.compressionTaskQueue.isBusy() ); } insert(bucket: DataBucket): void { - if (!this.queue.has(bucket)) { - this.queue.add(bucket); + if (!this.pendingQueue.has(bucket)) { + this.pendingQueue.add(bucket); bucket.dirtyCount++; } @@ -63,11 +68,11 @@ class PushQueue { } clear(): void { - this.queue.clear(); + this.pendingQueue.clear(); } print(): void { - this.queue.forEach(e => console.log(e)); + this.pendingQueue.forEach(e => console.log(e)); } pushImpl = async () => { @@ -76,22 +81,22 @@ class PushQueue { return; } - while (this.queue.size) { - let batchSize = Math.min(BATCH_SIZE, this.queue.size); + while (this.pendingQueue.size) { + let batchSize = Math.min(COMPRESSING_BATCH_SIZE, this.pendingQueue.size); const batch = []; - for (const bucket of this.queue) { + for (const bucket of this.pendingQueue) { if (batchSize <= 0) break; - this.queue.delete(bucket); + this.pendingQueue.delete(bucket); batch.push(bucket); batchSize--; } // fire and forget - this.taskQueue.scheduleTask(() => this.pushBatch(batch)); + this.compressionTaskQueue.scheduleTask(() => this.pushBatch(batch)); } try { // wait here - await this.taskQueue.join(); + await this.compressionTaskQueue.join(); } catch (error) { alert("We've encountered a permanent issue while saving. Please try to reload the page."); } diff --git a/frontend/javascripts/oxalis/view/action-bar/save_button.js b/frontend/javascripts/oxalis/view/action-bar/save_button.js index 07f522785dc..1051c307b3c 100644 --- a/frontend/javascripts/oxalis/view/action-bar/save_button.js +++ b/frontend/javascripts/oxalis/view/action-bar/save_button.js @@ -31,6 +31,10 @@ type Props = {| ...OwnProps, ...StateProps |}; type State = { isStateSaved: boolean, showUnsavedWarning: boolean, + saveInfo: { + compressingBucketCount: number, + waitingForCompressionBucketCount: number, + }, }; const SAVE_POLLING_INTERVAL = 1000; // 1s @@ -53,8 +57,8 @@ class SaveButton extends React.PureComponent { isStateSaved: false, showUnsavedWarning: false, saveInfo: { - compressingBuckets: 0, - pendingBuckets: 0, + compressingBucketCount: 0, + waitingForCompressionBucketCount: 0, }, }; @@ -78,14 +82,14 @@ class SaveButton extends React.PureComponent { reportUnsavedDurationThresholdExceeded(); } - const { compressingBuckets, pendingBuckets } = Model.getTotalPushQueueSize(); + const { compressingBucketCount, waitingForCompressionBucketCount } = Model.getPushQueueStats(); this.setState({ isStateSaved, showUnsavedWarning, saveInfo: { - compressingBuckets, - pendingBuckets, + compressingBucketCount, + waitingForCompressionBucketCount, }, }); }; @@ -108,6 +112,10 @@ class SaveButton extends React.PureComponent { const { progressFraction } = this.props; const { showUnsavedWarning } = this.state; + const totalBucketsToCompress = + this.state.saveInfo.waitingForCompressionBucketCount + + this.state.saveInfo.compressingBucketCount; + return ( { style={{ background: showUnsavedWarning ? "var(--ant-error)" : null }} > -

Pending buckets: {this.state.saveInfo.pendingBuckets}
-
Compressing buckets: {this.state.saveInfo.compressingBuckets}
- + totalBucketsToCompress > 0 + ? `${totalBucketsToCompress} buckets remaining to compress...` + : null } > {this.shouldShowProgress() ? ( diff --git a/frontend/javascripts/oxalis/workers/fetch_buffer_with_headers.worker.js b/frontend/javascripts/oxalis/workers/fetch_buffer_with_headers.worker.js index 3e7c9b2a97b..115270cf806 100644 --- a/frontend/javascripts/oxalis/workers/fetch_buffer_with_headers.worker.js +++ b/frontend/javascripts/oxalis/workers/fetch_buffer_with_headers.worker.js @@ -2,7 +2,7 @@ import handleStatus from "libs/handle_http_status"; -import { expose } from "./comlink_wrapper"; +import { expose, transfer } from "./comlink_wrapper"; function fetchBufferWithHeaders( url: RequestInfo, @@ -17,10 +17,13 @@ function fetchBufferWithHeaders( for (const [key, value] of headers.entries()) { headerObject[key] = value; } - return { - buffer, - headers: headerObject, - }; + return transfer( + { + buffer, + headers: headerObject, + }, + [buffer], + ); }); } From cfa0d09a736f8c8b9c637f52eebfdde3a3a4629e Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 18 Feb 2022 15:54:49 +0100 Subject: [PATCH 07/18] fix mocked transfer --- frontend/javascripts/oxalis/workers/comlink_wrapper.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/workers/comlink_wrapper.js b/frontend/javascripts/oxalis/workers/comlink_wrapper.js index 6b9875bdc5b..2c97e68f000 100644 --- a/frontend/javascripts/oxalis/workers/comlink_wrapper.js +++ b/frontend/javascripts/oxalis/workers/comlink_wrapper.js @@ -17,7 +17,7 @@ function importComlink() { wrap: null, transferHandlers: new Map(), _expose: null, - _transfer: null, + _transfer:

(element: P, transferrables: Array): P => element, }; } } From 187c6ce0038e9d601889f9c1c6318824cc8279ef Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 18 Feb 2022 16:11:40 +0100 Subject: [PATCH 08/18] clean up dynamic allocation and warn user about unusually high allocations --- frontend/javascripts/libs/utils.js | 8 +-- frontend/javascripts/libs/worker_pool.js | 2 +- .../model/bucket_data_handling/data_cube.js | 51 +++++++++++-------- .../bucket_data_handling/wkstore_adapter.js | 3 -- .../oxalis/workers/comlink_wrapper.js | 2 +- 5 files changed, 37 insertions(+), 29 deletions(-) diff --git a/frontend/javascripts/libs/utils.js b/frontend/javascripts/libs/utils.js index 45d9fefc971..dd5a3f65806 100644 --- a/frontend/javascripts/libs/utils.js +++ b/frontend/javascripts/libs/utils.js @@ -526,8 +526,8 @@ export function filterWithSearchQueryOR>( _.some(properties, fieldName => { const value = typeof fieldName === "function" ? fieldName(model) : model[fieldName]; if (value != null && (typeof value === "string" || value instanceof Object)) { - const values = getRecursiveValues(value); - return _.some(values, v => v != null && v.toString().match(regexp)); + const recursiveValues = getRecursiveValues(value); + return _.some(recursiveValues, v => v != null && v.toString().match(regexp)); } else { return false; } @@ -557,8 +557,8 @@ export function filterWithSearchQueryAND>( _.some(properties, fieldName => { const value = typeof fieldName === "function" ? fieldName(model) : model[fieldName]; if (value !== null && (typeof value === "string" || value instanceof Object)) { - const values = getRecursiveValues(value); - return _.some(values, v => v != null && v.toString().match(pattern)); + const recursiveValues = getRecursiveValues(value); + return _.some(recursiveValues, v => v != null && v.toString().match(pattern)); } else { return false; } diff --git a/frontend/javascripts/libs/worker_pool.js b/frontend/javascripts/libs/worker_pool.js index 8959c77b63a..50058a1547d 100644 --- a/frontend/javascripts/libs/worker_pool.js +++ b/frontend/javascripts/libs/worker_pool.js @@ -18,7 +18,7 @@ export default class WorkerPool { currentWorkerIdx: number; constructor(workerFn: () => (...args: Array

) => R, count: number) { - this.workers = _.range(0, count).map(idx => workerFn()); + this.workers = _.range(0, count).map(_idx => workerFn()); this.currentWorkerIdx = 0; } diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index fcd3c690eeb..a9d2d56b5a5 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -6,7 +6,6 @@ import BackboneEvents from "backbone-events-standalone"; import _ from "lodash"; -import ErrorHandling from "libs/error_handling"; import { type Bucket, DataBucket, @@ -15,23 +14,28 @@ import { NullBucket, type BucketDataArray, } from "oxalis/model/bucket_data_handling/bucket"; +import { type ElementClass } from "types/api_flow_types"; +import { type ProgressCallback } from "libs/progress_callback"; +import { V3 } from "libs/mjs"; import { VoxelNeighborQueue2D, VoxelNeighborQueue3D } from "oxalis/model/volumetracing/volumelayer"; +import { areBoundingBoxesOverlappingOrTouching } from "libs/utils"; import { getResolutions, ResolutionInfo, getMappingInfo, } from "oxalis/model/accessors/dataset_accessor"; import { getSomeTracing } from "oxalis/model/accessors/tracing_accessor"; -import { V3 } from "libs/mjs"; import { globalPositionToBucketPosition } from "oxalis/model/helpers/position_converter"; import { listenToStoreProperty } from "oxalis/model/helpers/listener_helpers"; import ArbitraryCubeAdapter from "oxalis/model/bucket_data_handling/arbitrary_cube_adapter"; import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; +import Dimensions, { type DimensionMap } from "oxalis/model/dimensions"; +import ErrorHandling from "libs/error_handling"; import PullQueue, { PullQueueConstants } from "oxalis/model/bucket_data_handling/pullqueue"; import PushQueue from "oxalis/model/bucket_data_handling/pushqueue"; import Store, { type Mapping } from "oxalis/store"; import TemporalBucketManager from "oxalis/model/bucket_data_handling/temporal_bucket_manager"; -import Dimensions, { type DimensionMap } from "oxalis/model/dimensions"; +import Toast from "libs/toast"; import constants, { type Vector3, type Vector4, @@ -39,9 +43,13 @@ import constants, { type LabelMasksByBucketAndW, MappingStatusEnum, } from "oxalis/constants"; -import { type ElementClass } from "types/api_flow_types"; -import { areBoundingBoxesOverlappingOrTouching } from "libs/utils"; -import { type ProgressCallback } from "libs/progress_callback"; + +const warnAboutTooManyAllocations = _.once(() => { + const msg = + "webKnossos needed to allocate an unusually large amount of image data. It is advised to save your work and reload the page."; + ErrorHandling.notify(new Error(msg)); + Toast.warning(msg, { sticky: true }); +}); class CubeEntry { data: Map; @@ -142,7 +150,8 @@ class DataCube { this.cubes = []; if (isSegmentation) { - this.MAXIMUM_BUCKET_COUNT = Math.floor(this.MAXIMUM_BUCKET_COUNT / 5); + // undo + this.MAXIMUM_BUCKET_COUNT = Math.floor(this.MAXIMUM_BUCKET_COUNT / 10); } this.buckets = []; @@ -330,19 +339,21 @@ class DataCube { if (foundCollectibleBucket) { this.collectBucket(this.buckets[this.bucketIterator]); } else { - // if (process.env.BABEL_ENV === "test") { - // throw new Error("Bucket was forcefully evicted/garbage-collected."); - // } - // const errorMessage = - // "A bucket was forcefully garbage-collected. This indicates that too many buckets are currently in RAM."; - // console.error(errorMessage); - // ErrorHandling.notify(new Error(errorMessage), { - // elementClass: this.elementClass, - // isSegmentation: this.isSegmentation, - // resolutionInfo: this.resolutionInfo, - // }); - - // Effectively, push to this.buckets by setting the iterator to + const warnMessage = `More than ${this.buckets.length} needed to be allocated.`; + if (this.buckets.length % 100 === 0) { + console.warn(warnMessage); + ErrorHandling.notify(new Error(warnMessage), { + elementClass: this.elementClass, + isSegmentation: this.isSegmentation, + resolutionInfo: this.resolutionInfo, + }); + } + + if (this.buckets.length > (2 * constants.MAXIMUM_BUCKET_COUNT_PER_LAYER) / 10) { + warnAboutTooManyAllocations(); + } + + // Effectively, push to `this.buckets` by setting the iterator to // a new index. this.bucketIterator = this.buckets.length; } diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js index 5505897dab4..8be96c20bb6 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js @@ -1,7 +1,5 @@ // @flow -import _ from "lodash"; - import type { DataBucket } from "oxalis/model/bucket_data_handling/bucket"; import { bucketPositionToGlobalAddress } from "oxalis/model/helpers/position_converter"; import { createWorker } from "oxalis/workers/comlink_wrapper"; @@ -241,7 +239,6 @@ export async function sendToStore(batch: Array, tracingId: string): getResolutions(Store.getState().dataset), ); const byteArray = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); - // eslint-disable-next-line no-await-in-loop const compressedBase64 = await compressionPool.submit(byteArray); return updateBucket(bucketInfo, compressedBase64); }, diff --git a/frontend/javascripts/oxalis/workers/comlink_wrapper.js b/frontend/javascripts/oxalis/workers/comlink_wrapper.js index 2c97e68f000..aee8fdc0c0f 100644 --- a/frontend/javascripts/oxalis/workers/comlink_wrapper.js +++ b/frontend/javascripts/oxalis/workers/comlink_wrapper.js @@ -17,7 +17,7 @@ function importComlink() { wrap: null, transferHandlers: new Map(), _expose: null, - _transfer:

(element: P, transferrables: Array): P => element, + _transfer:

(element: P, _transferrables: Array): P => element, }; } } From 4f0765826ed893209554c0ae92c7b5f18fe984db Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Fri, 18 Feb 2022 16:30:22 +0100 Subject: [PATCH 09/18] more slight refactoring and better comments --- .../oxalis/model/bucket_data_handling/bucket.js | 14 +++++++++++--- .../oxalis/model/bucket_data_handling/data_cube.js | 5 +---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js b/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js index d71d8898df6..e56da2ea9c7 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js @@ -129,11 +129,18 @@ export class DataBucket { elementClass: ElementClass; visualizedMesh: ?Object; visualizationColor: number; + // If dirty, the bucket's data was potentially edited and needs to be + // saved to the server. + dirty: boolean; + // `dirtyCount` reflects how many pending snapshots of the bucket exist. + // A pending snapshot is a snapshot which was either + // - not yet saved (after successful saving the dirtyCount is decremented) or + // - not yet created by the PushQueue, since the PushQueue creates the snapshots + // in a debounced manner dirtyCount: number = 0; pendingOperations: Array<(BucketDataArray) => void> = []; state: BucketStateEnumType; - dirty: boolean; accessed: boolean; data: ?BucketDataArray; temporalBucketManager: TemporalBucketManager; @@ -360,10 +367,11 @@ export class DataBucket { return data; } - setData(newData: BucketDataArray) { + setData(newData: BucketDataArray, newPendingOperations: Array<(BucketDataArray) => void>) { this.data = newData; + this.pendingOperations = newPendingOperations; this.dirty = true; - this.trigger("bucketLabeled"); + this.endDataMutation(); } uint8ToTypedBuffer(arrayBuffer: ?Uint8Array) { diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index a9d2d56b5a5..bb780b5b89c 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -692,10 +692,7 @@ class DataCube { if (bucket.type === "null") { return; } - bucket.setData(data); - bucket.pendingOperations = newPendingOperations; - - this.pushQueue.insert(bucket); + bucket.setData(data, newPendingOperations); } triggerPushQueue() { From bfe21bc07c8e671fa0091dc3ba34cb0803c06ac2 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 21 Feb 2022 10:18:05 +0100 Subject: [PATCH 10/18] adapt tests --- .../model/bucket_data_handling/data_cube.js | 21 +++++------ .../bucket_data_handling/wkstore_adapter.js | 6 ---- .../test/model/binary/cube.spec.js | 36 +++++++++++++------ ...js => bucket_eviction_with_saving.spec.js} | 0 ...=> bucket_eviction_without_saving.spec.js} | 13 +++---- 5 files changed, 42 insertions(+), 34 deletions(-) rename frontend/javascripts/test/sagas/volumetracing/{bucket_eviction_expect_success.spec.js => bucket_eviction_with_saving.spec.js} (100%) rename frontend/javascripts/test/sagas/volumetracing/{bucket_eviction_expect_failure.spec.js => bucket_eviction_without_saving.spec.js} (68%) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index bb780b5b89c..7187cbcc84d 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -73,12 +73,11 @@ const FLOODFILL_VOXEL_THRESHOLD = 5 * 1000000; const USE_FLOODFILL_VOXEL_THRESHOLD = false; class DataCube { - MAXIMUM_BUCKET_COUNT = constants.MAXIMUM_BUCKET_COUNT_PER_LAYER; + BUCKET_COUNT_SOFT_LIMIT = constants.MAXIMUM_BUCKET_COUNT_PER_LAYER; arbitraryCube: ArbitraryCubeAdapter; upperBoundary: Vector3; buckets: Array; bucketIterator: number = 0; - bucketCount: number = 0; cubes: Array; boundingBox: BoundingBox; pullQueue: PullQueue; @@ -93,9 +92,12 @@ class DataCube { // The cube stores the buckets in a separate array for each zoomStep. For each // zoomStep the cube-array contains the boundaries and an array holding the buckets. - // The bucket-arrays are initialized large enough to hold the whole cube. Thus no - // expanding is necessary. bucketCount keeps track of how many buckets are currently - // in the cube. + // The bucket array is initialized as an empty array and grows dynamically. If the + // length exceeds BUCKET_COUNT_SOFT_LIMIT, it is tried to garbage-collect an older + // bucket when placing a new one. If this does not succeed (happens if all buckets + // in a volume annotation layer are dirty), the array grows further. + // If the array grows beyond 2 * BUCKET_COUNT_SOFT_LIMIT, the user is warned about + // this. // // Each bucket consists of an access-value, the zoomStep and the actual data. // The access-values are used for garbage collection. When a bucket is accessed, its @@ -120,7 +122,7 @@ class DataCube { this.dirtyBucketsCount = 0; - if (isSegmentation) { + if (isSegmentation && process.env.BABEL_ENV !== "test") { setInterval(() => { console.log("#######"); @@ -149,9 +151,9 @@ class DataCube { _.extend(this, BackboneEvents); this.cubes = []; - if (isSegmentation) { + if (isSegmentation && process.env.BABEL_ENV !== "test") { // undo - this.MAXIMUM_BUCKET_COUNT = Math.floor(this.MAXIMUM_BUCKET_COUNT / 10); + this.BUCKET_COUNT_SOFT_LIMIT = Math.floor(this.BUCKET_COUNT_SOFT_LIMIT / 10); } this.buckets = []; @@ -325,7 +327,7 @@ class DataCube { } addBucketToGarbageCollection(bucket: DataBucket): void { - if (this.buckets.length >= this.MAXIMUM_BUCKET_COUNT) { + if (this.buckets.length >= this.BUCKET_COUNT_SOFT_LIMIT) { let foundCollectibleBucket = false; for (let i = 0; i < this.buckets.length; i++) { @@ -374,7 +376,6 @@ class DataCube { } } this.buckets = []; - this.bucketCount = 0; this.bucketIterator = 0; } diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js index 8be96c20bb6..8754840b7a6 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/wkstore_adapter.js @@ -228,8 +228,6 @@ function sliceBufferIntoPieces( } export async function sendToStore(batch: Array, tracingId: string): Promise { - const then = performance.now(); - const items: Array = await Promise.all( batch.map( async (bucket): Promise => { @@ -244,10 +242,6 @@ export async function sendToStore(batch: Array, tracingId: string): }, ), ); - const duration = performance.now() - then; - const durationPerItem = duration / batch.length; - - console.log("durationPerItem", durationPerItem); Store.dispatch(pushSaveQueueTransaction(items, "volume", tracingId)); } diff --git a/frontend/javascripts/test/model/binary/cube.spec.js b/frontend/javascripts/test/model/binary/cube.spec.js index 7f8918b79c0..d9f643d5b3e 100644 --- a/frontend/javascripts/test/model/binary/cube.spec.js +++ b/frontend/javascripts/test/model/binary/cube.spec.js @@ -100,16 +100,16 @@ test("GetBucket should return a NullBucket on getBucket()", t => { const { cube } = t.context; const bucket = cube.getBucket([0, 0, 0, 0]); t.is(bucket.type, "null"); - t.is(cube.bucketCount, 0); + t.is(cube.buckets.length, 0); }); test("GetBucket should create a new bucket on getOrCreateBucket()", t => { const { cube } = t.context; - t.is(cube.bucketCount, 0); + t.is(cube.buckets.length, 0); const bucket = cube.getOrCreateBucket([0, 0, 0, 0]); t.is(bucket.type, "data"); - t.is(cube.bucketCount, 1); + t.is(cube.buckets.length, 1); }); test("GetBucket should only create one bucket on getOrCreateBucket()", t => { @@ -117,7 +117,7 @@ test("GetBucket should only create one bucket on getOrCreateBucket()", t => { const bucket1 = cube.getOrCreateBucket([0, 0, 0, 0]); const bucket2 = cube.getOrCreateBucket([0, 0, 0, 0]); t.is(bucket1, bucket2); - t.is(cube.bucketCount, 1); + t.is(cube.buckets.length, 1); }); test("Voxel Labeling should request buckets when temporal buckets are created", t => { @@ -199,23 +199,21 @@ test("getDataValue() should return the mapping value if available", async t => { t.is(cube.getDataValue([1, 1, 1], mapping), 43); }); -test("Garbage Collection should only keep 3 buckets", t => { +test("Garbage Collection should only keep 3 buckets when possible", t => { const { cube } = t.context; - cube.MAXIMUM_BUCKET_COUNT = 3; - cube.buckets = new Array(cube.MAXIMUM_BUCKET_COUNT); + cube.BUCKET_COUNT_SOFT_LIMIT = 3; cube.getOrCreateBucket([0, 0, 0, 0]); cube.getOrCreateBucket([1, 1, 1, 0]); cube.getOrCreateBucket([2, 2, 2, 0]); cube.getOrCreateBucket([3, 3, 3, 0]); - t.is(cube.bucketCount, 3); + t.is(cube.buckets.length, 3); }); test("Garbage Collection should not collect buckets with shouldCollect() == false", t => { const { cube } = t.context; - cube.MAXIMUM_BUCKET_COUNT = 3; - cube.buckets = new Array(cube.MAXIMUM_BUCKET_COUNT); + cube.BUCKET_COUNT_SOFT_LIMIT = 3; const b1 = cube.getOrCreateBucket([0, 0, 0, 0]); b1.markAsPulled(); @@ -230,6 +228,24 @@ test("Garbage Collection should not collect buckets with shouldCollect() == fals t.deepEqual(addresses, [[0, 0, 0, 0], [3, 3, 3, 0], [2, 2, 2, 0]]); }); +test("Garbage Collection should grow beyond soft limit if necessary", t => { + const { cube } = t.context; + cube.BUCKET_COUNT_SOFT_LIMIT = 3; + + const b1 = cube.getOrCreateBucket([0, 0, 0, 0]); + const b2 = cube.getOrCreateBucket([1, 1, 1, 0]); + const b3 = cube.getOrCreateBucket([2, 2, 2, 0]); + + // No bucket may be collected. + [b1, b2, b3].map(b => b.markAsPulled()); + + // Allocate a 4th one which should still be possible (will exceed BUCKET_COUNT_SOFT_LIMIT) + const b4 = cube.getOrCreateBucket([3, 3, 3, 0]); + + const addresses = cube.buckets.map(b => b.zoomedAddress); + t.deepEqual(addresses, [[0, 0, 0, 0], [1, 1, 1, 0], [2, 2, 2, 0], [3, 3, 3, 0]]); +}); + test("getVoxelIndexByVoxelOffset should return the correct index of a position within a bucket", t => { const { cube } = t.context; let index = cube.getVoxelIndexByVoxelOffset([0, 0, 0]); diff --git a/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_expect_success.spec.js b/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_with_saving.spec.js similarity index 100% rename from frontend/javascripts/test/sagas/volumetracing/bucket_eviction_expect_success.spec.js rename to frontend/javascripts/test/sagas/volumetracing/bucket_eviction_with_saving.spec.js diff --git a/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_expect_failure.spec.js b/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.js similarity index 68% rename from frontend/javascripts/test/sagas/volumetracing/bucket_eviction_expect_failure.spec.js rename to frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.js index 4df9f2d832a..a103f1655a5 100644 --- a/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_expect_failure.spec.js +++ b/frontend/javascripts/test/sagas/volumetracing/bucket_eviction_without_saving.spec.js @@ -25,7 +25,7 @@ test.beforeEach(async t => { }); test.serial( - "Brushing/Tracing should crash when too many buckets are labeled at once without saving inbetween", + "Brushing/Tracing should not crash when a lot of buckets are labeled at once without saving inbetween", async t => { await t.context.api.tracing.save(); @@ -34,16 +34,13 @@ test.serial( 0, 0, ); - // webKnossos will start to evict buckets forcefully if too many are dirty at the same time. - // This is not ideal, but usually handled by the fact that buckets are regularly saved to the - // backend and then marked as not dirty. - // This test provokes that webKnossos crashes (a hard crash is only done during testing; in dev/prod - // a soft warning is emitted via the devtools). - // The corresponding sibling test checks that saving inbetween does not make webKnossos crash. + // In earlier versions of webKnossos, buckets could be evicted forcefully when + // too many were dirty at the same time. This led to a crash in earlier versions. + // Now, the code should not crash, anymore. t.plan(2); t.false(hasRootSagaCrashed()); const failedSagaPromise = waitForCondition(hasRootSagaCrashed, 500); await Promise.race([testLabelingManyBuckets(t, false), failedSagaPromise]); - t.true(hasRootSagaCrashed()); + t.false(hasRootSagaCrashed()); }, ); From b05e00fb27e5c76d83655ca122ed9381807fc266 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 21 Feb 2022 10:28:44 +0100 Subject: [PATCH 11/18] update changelog --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 3727085a8fc..1ddd0a289e0 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -29,6 +29,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Changed a number of API routes from GET to POST to avoid unwanted side effects. [#6023](https://github.com/scalableminds/webknossos/pull/6023) - Removed unused datastore route `checkInbox` (use `checkInboxBlocking` instead). [#6023](https://github.com/scalableminds/webknossos/pull/6023) - Migrated to Google Analytics 4. [#6031](https://github.com/scalableminds/webknossos/pull/6031) +- Improved stability and speed of volume annotations when annotating large areas. [#6055](https://github.com/scalableminds/webknossos/pull/6055) ### Fixed - Fixed volume-related bugs which could corrupt the volume data in certain scenarios. [#5955](https://github.com/scalableminds/webknossos/pull/5955) From 2dbd48a7ebc0f6199ee80e125553d18dab42d7fa Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Mon, 21 Feb 2022 10:41:20 +0100 Subject: [PATCH 12/18] linting --- frontend/javascripts/test/model/binary/cube.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/test/model/binary/cube.spec.js b/frontend/javascripts/test/model/binary/cube.spec.js index d9f643d5b3e..a303bb401f3 100644 --- a/frontend/javascripts/test/model/binary/cube.spec.js +++ b/frontend/javascripts/test/model/binary/cube.spec.js @@ -240,7 +240,7 @@ test("Garbage Collection should grow beyond soft limit if necessary", t => { [b1, b2, b3].map(b => b.markAsPulled()); // Allocate a 4th one which should still be possible (will exceed BUCKET_COUNT_SOFT_LIMIT) - const b4 = cube.getOrCreateBucket([3, 3, 3, 0]); + cube.getOrCreateBucket([3, 3, 3, 0]); const addresses = cube.buckets.map(b => b.zoomedAddress); t.deepEqual(addresses, [[0, 0, 0, 0], [1, 1, 1, 0], [2, 2, 2, 0], [3, 3, 3, 0]]); From 465387657620ad510a87ccc069c53d11563edc2a Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Tue, 22 Feb 2022 09:58:31 +0100 Subject: [PATCH 13/18] fix that bucketCollected event was never consumed due to too early clearing of handlers --- .../javascripts/oxalis/model/bucket_data_handling/bucket.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js b/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js index 225d9484f6c..0f6c94061e6 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/bucket.js @@ -235,9 +235,9 @@ export class DataBucket { // so that at least the big memory hog is tamed (unfortunately, // this doesn't help against references which point directly to this.data) this.data = null; + this.trigger("bucketCollected"); // Remove all event handlers (see https://github.com/ai/nanoevents#remove-all-listeners) this.emitter.events = {}; - this.trigger("bucketCollected"); } needsRequest(): boolean { From 5f2f7bf2cfacad09d2a91b04c3ed99d7ae67cfb0 Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Tue, 22 Feb 2022 14:00:28 +0100 Subject: [PATCH 14/18] fix that required buckets could be GCed during bucket picking --- .../oxalis/model/bucket_data_handling/data_cube.js | 6 ++---- .../model/bucket_data_handling/layer_rendering_manager.js | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index 7ae41ffb073..74c16910ade 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -341,7 +341,7 @@ class DataCube { if (foundCollectibleBucket) { this.collectBucket(this.buckets[this.bucketIterator]); } else { - const warnMessage = `More than ${this.buckets.length} needed to be allocated.`; + const warnMessage = `More than ${this.buckets.length} buckets needed to be allocated.`; if (this.buckets.length % 100 === 0) { console.warn(warnMessage); ErrorHandling.notify(new Error(warnMessage), { @@ -371,9 +371,7 @@ class DataCube { this.pullQueue.clear(); this.pullQueue.abortRequests(); for (const bucket of this.buckets) { - if (bucket != null) { - this.collectBucket(bucket); - } + this.collectBucket(bucket); } this.buckets = []; this.bucketIterator = 0; diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js b/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js index 4ee2eade9f7..a4f1f3cdf57 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/layer_rendering_manager.js @@ -89,6 +89,8 @@ function consumeBucketsFromArrayBuffer( const bucket = cube.getOrCreateBucket(bucketAddress); if (bucket.type !== "null") { + // This tells the bucket collection, that the buckets are necessary for rendering + bucket.markAsNeeded(); bucketsWithPriorities.push({ bucket, priority }); } @@ -237,6 +239,7 @@ export default class LayerRenderingManager { pickingPromise.then( buffer => { + this.cube.markBucketsAsUnneeded(); const bucketsWithPriorities = consumeBucketsFromArrayBuffer( buffer, this.cube, @@ -244,9 +247,6 @@ export default class LayerRenderingManager { ); const buckets = bucketsWithPriorities.map(({ bucket }) => bucket); - this.cube.markBucketsAsUnneeded(); - // This tells the bucket collection, that the buckets are necessary for rendering - buckets.forEach(b => b.markAsNeeded()); this.textureBucketManager.setActiveBuckets( buckets, From 2b8b5e04bc3f85a64a71396aca6b89e4caf8703a Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Tue, 22 Feb 2022 14:07:18 +0100 Subject: [PATCH 15/18] rename buckets to items in user-facing code --- frontend/javascripts/oxalis/view/action-bar/save_button.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/view/action-bar/save_button.js b/frontend/javascripts/oxalis/view/action-bar/save_button.js index b3550edbc12..f3dbe2748b3 100644 --- a/frontend/javascripts/oxalis/view/action-bar/save_button.js +++ b/frontend/javascripts/oxalis/view/action-bar/save_button.js @@ -129,7 +129,7 @@ class SaveButton extends React.PureComponent { 0 - ? `${totalBucketsToCompress} buckets remaining to compress...` + ? `${totalBucketsToCompress} items remaining to compress...` : null } > From 8d2caaa78cbf67c64c3176952b00afae6815c1dc Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 23 Feb 2022 12:00:28 +0100 Subject: [PATCH 16/18] remove debugging related code --- .../model/bucket_data_handling/data_cube.js | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index 74c16910ade..abdbbdaf6a7 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -88,8 +88,6 @@ class DataCube { resolutionInfo: ResolutionInfo; layerName: string; - dirtyBucketsCount: number; - // The cube stores the buckets in a separate array for each zoomStep. For each // zoomStep the cube-array contains the boundaries and an array holding the buckets. // The bucket array is initialized as an empty array and grows dynamically. If the @@ -120,41 +118,9 @@ class DataCube { this.resolutionInfo = resolutionInfo; this.layerName = layerName; - this.dirtyBucketsCount = 0; - - if (isSegmentation && process.env.BABEL_ENV !== "test") { - setInterval(() => { - console.log("#######"); - - let dirtyBucketsCount = 0; - for (let i = 0; i < this.buckets.length; i++) { - if (this.buckets[i].dirty) { - dirtyBucketsCount++; - } - } - - let weirdDirtyState = 0; - for (let i = 0; i < this.buckets.length; i++) { - if (this.buckets[i].dirty && this.buckets[i].dirtyCount === 0) { - weirdDirtyState++; - } - } - if (weirdDirtyState > 0) { - console.log("weirdDirtyState", weirdDirtyState); - } - - console.log("dirtyBucketsCount", dirtyBucketsCount); - console.log("this.buckets.length", this.buckets.length); - }, 2000); - } - _.extend(this, BackboneEvents); this.cubes = []; - if (isSegmentation && process.env.BABEL_ENV !== "test") { - // undo - this.BUCKET_COUNT_SOFT_LIMIT = Math.floor(this.BUCKET_COUNT_SOFT_LIMIT / 10); - } this.buckets = []; // Initializing the cube-arrays with boundaries @@ -351,7 +317,7 @@ class DataCube { }); } - if (this.buckets.length > (2 * constants.MAXIMUM_BUCKET_COUNT_PER_LAYER) / 10) { + if (this.buckets.length > 2 * constants.MAXIMUM_BUCKET_COUNT_PER_LAYER) { warnAboutTooManyAllocations(); } From 653e853b4f15f04391fa159dcaafc1fef2ccfeed Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 23 Feb 2022 12:02:57 +0100 Subject: [PATCH 17/18] update changelog --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 62998dd0438..3c12574c2f8 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -18,6 +18,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released ### Fixed - Fixed a bug where deactivated users would still be listed as allowed to access the datasets of their team. [#6070](https://github.com/scalableminds/webknossos/pull/6070) +- Fix occasionally "disappearing" data. [#6055](https://github.com/scalableminds/webknossos/pull/6055) ### Removed From 36e46f589d16ad696ba4e2832c17b68d9a86be4e Mon Sep 17 00:00:00 2001 From: Philipp Otto Date: Wed, 23 Feb 2022 12:51:01 +0100 Subject: [PATCH 18/18] use this.BUCKET_COUNT_SOFT_LIMIT in data_cube.js --- .../javascripts/oxalis/model/bucket_data_handling/data_cube.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js index abdbbdaf6a7..50f65450f07 100644 --- a/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js +++ b/frontend/javascripts/oxalis/model/bucket_data_handling/data_cube.js @@ -317,7 +317,7 @@ class DataCube { }); } - if (this.buckets.length > 2 * constants.MAXIMUM_BUCKET_COUNT_PER_LAYER) { + if (this.buckets.length > 2 * this.BUCKET_COUNT_SOFT_LIMIT) { warnAboutTooManyAllocations(); }