-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Transactional saving of volume annotations (#7264)
* create volume transactions according to debounced pushqueue.push; increase efficiency of compressing-worker by increasing payload size * make PushQueue.push more robust by avoiding concurrent execution of it (by implementing createDebouncedAbortableCallable * Revert "Revert "temporarily disable most CI checks"" This reverts commit d69a7cf. * don't use AsyncTaskQueue in pushqueue anymore * remove AsyncTaskQueue implementation + specs * implement small AsyncFifoResolver to prevent theoretical race condition * ensure that the save saga consumes N items from the save queue where N is the size of the queue at the time the auto-save-interval kicked in * fix tests * fix accidentally skipped tests; improve linting rule to avoid this; fix broken segment group test * harden error handling in PushQueue * move some lib modules into libs/async * warn user when pushqueue is starving * Apply suggestions from code review * clean up a bit * tune batch count constants for volume tracings; also show downloading buckets in save button tooltip * fix race condition in AsyncFifoResolver * fix incorrect dtype in comment * update changelog * improve comment * rename pendingQueue to pendingBuckets * fix incorrect method name
- Loading branch information
1 parent
2485549
commit 58f5703
Showing
38 changed files
with
688 additions
and
462 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * This class can be used to await promises | ||
| * in the order they were passed to | ||
| * orderedWaitFor. | ||
| * | ||
| * This enables scheduling of asynchronous work | ||
| * concurrently while ensuring that the results | ||
| * are processed in the order they were requested | ||
| * (instead of the order in which they finished). | ||
| * | ||
| * Example: | ||
| * const resolver = new AsyncFifoResolver(); | ||
| * const promise1Done = resolver.orderedWaitFor(promise1); | ||
| * const promise2Done = resolver.orderedWaitFor(promise2); | ||
| * | ||
| * Even if promise2 resolves before promise1, promise2Done | ||
| * will resolve *after* promise1Done. | ||
| */ | ||
|
|
||
| export class AsyncFifoResolver<T> { | ||
| queue: Promise<T>[]; | ||
| constructor() { | ||
| this.queue = []; | ||
| } | ||
|
|
||
| async orderedWaitFor(promise: Promise<T>): Promise<T> { | ||
| this.queue.push(promise); | ||
| const promiseCountToAwait = this.queue.length; | ||
| const retVals = await Promise.all(this.queue); | ||
| // Note that this.queue can have changed during the await. | ||
| // Find the index of the promise and trim the queue accordingly. | ||
| this.queue = this.queue.slice(this.queue.indexOf(promise) + 1); | ||
| return retVals[promiseCountToAwait - 1]; | ||
| } | ||
| } |
95 changes: 95 additions & 0 deletions
95
frontend/javascripts/libs/async/debounced_abortable_saga.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| import { call, type Saga } from "oxalis/model/sagas/effect-generators"; | ||
| import { buffers, Channel, channel, runSaga } from "redux-saga"; | ||
| import { delay, race, take } from "redux-saga/effects"; | ||
|
|
||
| /* | ||
| * This function takes a saga and a debounce threshold | ||
| * and returns a function F that will trigger the given saga | ||
| * in a debounced manner. | ||
| * In contrast to a normal debouncing mechanism, the saga | ||
| * will be cancelled if F is called while the saga is running. | ||
| * Note that this means that concurrent executions of the saga | ||
| * are impossible that way (by design). | ||
| * | ||
| * Also note that the performance of this debouncing mechanism | ||
| * is slower than a standard _.debounce. Also see | ||
| * debounced_abortable_saga.spec.ts for a small benchmark. | ||
| */ | ||
| export function createDebouncedAbortableCallable<T, C>( | ||
| fn: (param1: T) => Saga<void>, | ||
| debounceThreshold: number, | ||
| context: C, | ||
| ) { | ||
| // The communication with the saga is done via a channel. | ||
| // That way, we can expose a normal function that | ||
| // does the triggering by filling the channel. | ||
|
|
||
| // Only the most recent invocation should survive. | ||
| // Therefore, create a sliding buffer with size 1. | ||
| const buffer = buffers.sliding<T>(1); | ||
| const triggerChannel: Channel<T> = channel<T>(buffer); | ||
|
|
||
| const _task = runSaga( | ||
| {}, | ||
| debouncedAbortableSagaRunner, | ||
| debounceThreshold, | ||
| triggerChannel, | ||
| // @ts-expect-error TS thinks fn doesnt match, but it does. | ||
| fn, | ||
| context, | ||
| ); | ||
|
|
||
| return (msg: T) => { | ||
| triggerChannel.put(msg); | ||
| }; | ||
| } | ||
|
|
||
| export function createDebouncedAbortableParameterlessCallable<C>( | ||
| fn: () => Saga<void>, | ||
| debounceThreshold: number, | ||
| context: C, | ||
| ) { | ||
| const wrappedFn = createDebouncedAbortableCallable(fn, debounceThreshold, context); | ||
| const dummyParameter = {}; | ||
| return () => { | ||
| wrappedFn(dummyParameter); | ||
| }; | ||
| } | ||
|
|
||
| function* debouncedAbortableSagaRunner<T, C>( | ||
| debounceThreshold: number, | ||
| triggerChannel: Channel<T>, | ||
| abortableFn: (param: T) => Saga<void>, | ||
| context: C, | ||
| ): Saga<void> { | ||
| while (true) { | ||
| // Wait for a trigger-call by consuming | ||
| // the channel. | ||
| let msg = yield take(triggerChannel); | ||
|
|
||
| // Repeatedly try to execute abortableFn (each try | ||
| // might be cancelled due to new initiation-requests) | ||
| while (true) { | ||
| const { debounced, latestMessage } = yield race({ | ||
| debounced: delay(debounceThreshold), | ||
| latestMessage: take(triggerChannel), | ||
| }); | ||
|
|
||
| if (latestMessage) { | ||
| msg = latestMessage; | ||
| } | ||
|
|
||
| if (debounced) { | ||
| const { abortingMessage } = yield race({ | ||
| finished: call([context, abortableFn], msg), | ||
| abortingMessage: take(triggerChannel), | ||
| }); | ||
| if (abortingMessage) { | ||
| msg = abortingMessage; | ||
| } else { | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
6 changes: 3 additions & 3 deletions
6
frontend/javascripts/libs/worker_pool.ts → frontend/javascripts/libs/webworker_pool.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.