From 52e612da5bae792edf8877de039cb3ace6dc4c6b Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Thu, 7 Nov 2024 12:02:52 +0100 Subject: [PATCH 1/5] Remove requirements of bounding boxes to have the same x/y dimension as well as the same dimensions overall for model training --- .../oxalis/view/jobs/train_ai_model.tsx | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx index 1153c9eaf0..2092b6e1f6 100644 --- a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx +++ b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx @@ -34,7 +34,6 @@ import _ from "lodash"; import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; import { formatVoxels } from "libs/format_utils"; import * as Utils from "libs/utils"; -import { V3 } from "libs/mjs"; import type { APIAnnotation, APIDataset, ServerVolumeTracing } from "types/api_flow_types"; import type { Vector3 } from "oxalis/constants"; import { serverVolumeToClientVolumeTracing } from "oxalis/model/reducers/volumetracing_reducer"; @@ -66,7 +65,7 @@ enum AiModelCategory { const ExperimentalWarning = () => ( @@ -424,25 +423,7 @@ function areInvalidBoundingBoxesIncluded(userBoundingBoxes: UserBoundingBox[]): invalidBBoxesReason: "At least one bounding box must be defined.", }; } - const getSize = (bbox: UserBoundingBox) => V3.sub(bbox.boundingBox.max, bbox.boundingBox.min); - - const size = getSize(userBoundingBoxes[0]); - // width must equal height - if (size[0] !== size[1]) { - return { - areSomeBBoxesInvalid: true, - invalidBBoxesReason: "The bounding box width must equal its height.", - }; - } - // all bounding boxes must have the same size - const areSizesIdentical = userBoundingBoxes.every((bbox) => V3.isEqual(getSize(bbox), size)); - if (areSizesIdentical) { - return { areSomeBBoxesInvalid: false, invalidBBoxesReason: null }; - } - return { - areSomeBBoxesInvalid: true, - invalidBBoxesReason: "All bounding boxes must have the same size.", - }; + return { areSomeBBoxesInvalid: false, invalidBBoxesReason: null }; } function AnnotationsCsvInput({ From 985e1b7b86bc137715b398e7d7d64c2de3b80671 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Wed, 20 Nov 2024 20:44:59 +0100 Subject: [PATCH 2/5] update changelog --- CHANGELOG.unreleased.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index f069e844ba..4d167f03e0 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -17,6 +17,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Reading image files on datastore filesystem is now done asynchronously. [#8126](https://github.com/scalableminds/webknossos/pull/8126) - Improved error messages for starting jobs on datasets from other organizations. [#8181](https://github.com/scalableminds/webknossos/pull/8181) - Removed bounding box size restriction for inferral jobs for super users. [#8200](https://github.com/scalableminds/webknossos/pull/8200) +- Allowed to train an AI model using differently sized bounding boxes. We recommend all bounding boxes to have equal dimensions or to have dimensions which are multiples of the smallest bounding box. [#8222](https://github.com/scalableminds/webknossos/pull/8222) ### Fixed - Fix performance bottleneck when deleting a lot of trees at once. [#8176](https://github.com/scalableminds/webknossos/pull/8176) From 7d7945c7050d557a9d44aafdad3949264000f78d Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Mon, 25 Nov 2024 16:16:10 +0100 Subject: [PATCH 3/5] Show warnings if bounding boxes for CNN training are suboptimal. Make errors and warnings more prominent by using Alerts. Include annotation ID and topleft + size for offending boxes. --- .../oxalis/view/jobs/train_ai_model.tsx | 154 ++++++++++++++---- 1 file changed, 126 insertions(+), 28 deletions(-) diff --git a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx index 2092b6e1f6..3c630cd596 100644 --- a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx +++ b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx @@ -35,9 +35,10 @@ import BoundingBox from "oxalis/model/bucket_data_handling/bounding_box"; import { formatVoxels } from "libs/format_utils"; import * as Utils from "libs/utils"; import type { APIAnnotation, APIDataset, ServerVolumeTracing } from "types/api_flow_types"; -import type { Vector3 } from "oxalis/constants"; +import type { Vector3, Vector6 } from "oxalis/constants"; import { serverVolumeToClientVolumeTracing } from "oxalis/model/reducers/volumetracing_reducer"; import { convertUserBoundingBoxesFromServerToFrontend } from "oxalis/model/reducers/reducer_helpers"; +import { computeArrayFromBoundingBox } from "libs/utils"; const { TextArea } = Input; const FormItem = Form.Item; @@ -66,7 +67,7 @@ const ExperimentalWarning = () => ( @@ -216,19 +217,29 @@ export function TrainAiModelTab userBoundingBoxes); + const userBoundingBoxes = annotationInfos.flatMap(({ userBoundingBoxes, annotation }) => + userBoundingBoxes.map((box) => ({ + ...box, + annotationId: "id" in annotation ? annotation.id : annotation.annotationId, + })), + ); const bboxesVoxelCount = _.sum( (userBoundingBoxes || []).map((bbox) => new BoundingBox(bbox.boundingBox).getVolume()), ); - const { areSomeAnnotationsInvalid, invalidAnnotationsReason } = - areInvalidAnnotationsIncluded(annotationInfos); - const { areSomeBBoxesInvalid, invalidBBoxesReason } = - areInvalidBoundingBoxesIncluded(userBoundingBoxes); - const invalidReasons = [invalidAnnotationsReason, invalidBBoxesReason] - .filter((reason) => reason) - .join("\n"); + const { hasAnnotationErrors, errors: annotationErrors } = + checkAnnotationsForErrorsAndWarnings(annotationInfos); + const { + hasBBoxErrors, + hasBBoxWarnings, + errors: bboxErrors, + warnings: bboxWarnings, + } = checkBoundingBoxesForErrorsAndWarnings(userBoundingBoxes); + const hasErrors = hasAnnotationErrors || hasBBoxErrors; + const hasWarnings = hasBBoxWarnings; + const errors = [...annotationErrors, ...bboxErrors]; + const warnings = bboxWarnings; return (
) : null} + + {hasErrors + ? errors.map((error) => ( + + )) + : null} + {hasWarnings + ? warnings.map((warning) => ( + + )) + : null} + - + @@ -384,16 +425,16 @@ export function CollapsibleWorkflowYamlEditor({ ); } -function areInvalidAnnotationsIncluded( +function checkAnnotationsForErrorsAndWarnings( annotationsWithDatasets: Array>, ): { - areSomeAnnotationsInvalid: boolean; - invalidAnnotationsReason: string | null; + hasAnnotationErrors: boolean; + errors: string[]; } { if (annotationsWithDatasets.length === 0) { return { - areSomeAnnotationsInvalid: true, - invalidAnnotationsReason: "At least one annotation must be defined.", + hasAnnotationErrors: true, + errors: ["At least one annotation must be defined."], }; } const annotationsWithoutBoundingBoxes = annotationsWithDatasets.filter( @@ -406,24 +447,81 @@ function areInvalidAnnotationsIncluded( "id" in annotation ? annotation.id : annotation.annotationId, ); return { - areSomeAnnotationsInvalid: true, - invalidAnnotationsReason: `All annotations must have at least one bounding box. Annotations without bounding boxes are: ${annotationIds.join(", ")}`, + hasAnnotationErrors: true, + errors: [ + `All annotations must have at least one bounding box. Annotations without bounding boxes are: ${annotationIds.join(", ")}`, + ], }; } - return { areSomeAnnotationsInvalid: false, invalidAnnotationsReason: null }; + return { hasAnnotationErrors: false, errors: [] }; } -function areInvalidBoundingBoxesIncluded(userBoundingBoxes: UserBoundingBox[]): { - areSomeBBoxesInvalid: boolean; - invalidBBoxesReason: string | null; +function checkBoundingBoxesForErrorsAndWarnings( + userBoundingBoxes: (UserBoundingBox & { annotationId: string })[], +): { + hasBBoxErrors: boolean; + hasBBoxWarnings: boolean; + errors: string[]; + warnings: string[]; } { + let hasBBoxErrors = false; + let hasBBoxWarnings = false; + const errors = []; + const warnings = []; if (userBoundingBoxes.length === 0) { - return { - areSomeBBoxesInvalid: true, - invalidBBoxesReason: "At least one bounding box must be defined.", - }; + hasBBoxErrors = true; + errors.push("At least one bounding box must be defined."); } - return { areSomeBBoxesInvalid: false, invalidBBoxesReason: null }; + // Find smallest bounding box dimensions + const minDimensions = userBoundingBoxes.reduce( + (min, { boundingBox: box }) => ({ + x: Math.min(min.x, box.max[0] - box.min[0]), + y: Math.min(min.y, box.max[1] - box.min[1]), + z: Math.min(min.z, box.max[2] - box.min[2]), + }), + { x: Number.POSITIVE_INFINITY, y: Number.POSITIVE_INFINITY, z: Number.POSITIVE_INFINITY }, + ); + + // Validate minimum size and multiple requirements + type BoundingBoxWithAnnotationId = { boundingBox: Vector6; annotationId: string }; + const tooSmallBoxes: BoundingBoxWithAnnotationId[] = []; + const nonMultipleBoxes: BoundingBoxWithAnnotationId[] = []; + userBoundingBoxes.map(({ boundingBox: box, annotationId }) => { + const arrayBox = computeArrayFromBoundingBox(box); + const [_x, _y, _z, width, height, depth] = arrayBox; + if (width < 10 || height < 10 || depth < 10) { + tooSmallBoxes.push({ boundingBox: arrayBox, annotationId }); + } + + if ( + width % minDimensions.x !== 0 || + height % minDimensions.y !== 0 || + depth % minDimensions.z !== 0 + ) { + nonMultipleBoxes.push({ boundingBox: arrayBox, annotationId }); + } + }); + + const boxWithIdToString = ({ boundingBox, annotationId }: BoundingBoxWithAnnotationId) => + boundingBox.join(", ") + ` (${annotationId})`; + + if (tooSmallBoxes.length > 0) { + hasBBoxWarnings = true; + const tooSmallBoxesStrings = tooSmallBoxes.map(boxWithIdToString); + warnings.push( + `The following bounding boxes are not at least 10 Vx in each dimension which is suboptimal for the training:\n${tooSmallBoxesStrings.join("\n")}`, + ); + } + + if (nonMultipleBoxes.length > 0) { + hasBBoxWarnings = true; + const nonMultipleBoxesStrings = nonMultipleBoxes.map(boxWithIdToString); + warnings.push( + `The minimum bounding box dimensions are ${minDimensions.x} x ${minDimensions.y} x ${minDimensions.z}. The following bounding boxes have dimensions which are not a multiple of the minimum dimensions which is suboptimal for the training:\n${nonMultipleBoxesStrings.join("\n")}`, + ); + } + + return { hasBBoxErrors, hasBBoxWarnings, errors, warnings }; } function AnnotationsCsvInput({ From 103dc38d2e70cc8b9cdd6442e55526dd7334a5ec Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Mon, 25 Nov 2024 16:19:45 +0100 Subject: [PATCH 4/5] Change map to forEach --- frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx index 3c630cd596..da78f34149 100644 --- a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx +++ b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx @@ -486,7 +486,7 @@ function checkBoundingBoxesForErrorsAndWarnings( type BoundingBoxWithAnnotationId = { boundingBox: Vector6; annotationId: string }; const tooSmallBoxes: BoundingBoxWithAnnotationId[] = []; const nonMultipleBoxes: BoundingBoxWithAnnotationId[] = []; - userBoundingBoxes.map(({ boundingBox: box, annotationId }) => { + userBoundingBoxes.forEach(({ boundingBox: box, annotationId }) => { const arrayBox = computeArrayFromBoundingBox(box); const [_x, _y, _z, width, height, depth] = arrayBox; if (width < 10 || height < 10 || depth < 10) { From 177c890f205156fe06a34c4118e797c33c516303 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Tue, 26 Nov 2024 20:11:10 +0100 Subject: [PATCH 5/5] Include bounding box name in warnings --- .../oxalis/view/jobs/train_ai_model.tsx | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx index da78f34149..29d115fcf3 100644 --- a/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx +++ b/frontend/javascripts/oxalis/view/jobs/train_ai_model.tsx @@ -449,7 +449,7 @@ function checkAnnotationsForErrorsAndWarnings { + userBoundingBoxes.forEach(({ boundingBox: box, name, annotationId }) => { const arrayBox = computeArrayFromBoundingBox(box); const [_x, _y, _z, width, height, depth] = arrayBox; if (width < 10 || height < 10 || depth < 10) { - tooSmallBoxes.push({ boundingBox: arrayBox, annotationId }); + tooSmallBoxes.push({ boundingBox: arrayBox, name, annotationId }); } if ( @@ -498,12 +498,12 @@ function checkBoundingBoxesForErrorsAndWarnings( height % minDimensions.y !== 0 || depth % minDimensions.z !== 0 ) { - nonMultipleBoxes.push({ boundingBox: arrayBox, annotationId }); + nonMultipleBoxes.push({ boundingBox: arrayBox, name, annotationId }); } }); - const boxWithIdToString = ({ boundingBox, annotationId }: BoundingBoxWithAnnotationId) => - boundingBox.join(", ") + ` (${annotationId})`; + const boxWithIdToString = ({ boundingBox, name, annotationId }: BoundingBoxWithAnnotationId) => + `'${name}' of annotation ${annotationId}: ${boundingBox.join(", ")}`; if (tooSmallBoxes.length > 0) { hasBBoxWarnings = true;