Intelligent scissors with OpenCV javascript (#2689)

* Some UI implementations * Added opencv wrapper * Updated Opencv wrapper * Moved initialization stub * Added threshold * Setup interaction with canvas * Fixed couple of issues * Added threshold, changing size via ctrl * tmp * Aborted host change * Fixed threshold * Aborted host * Some fixes * Using ready label selector * Raw implementation * Added additional arguments * Fixed some minor issues * Removed unused file * Fixed tool reset * Added short instructions to update opencv.js * Fixed corner case * Added error handler, opencv version, updated cvat_proxy & mod_wsgi * OpenCV returned back * Using dinamic function instead of script * Updated changelog & version
cvat-ai · Feb 3, 2021 · e0fc323 · e0fc323
1 parent 3d4fad4
commit e0fc323
Show file tree

Hide file tree

Showing 35 changed files with 1,073 additions and 87 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - CVAT-3D: support lidar data on the server side (<https://github.com/openvinotoolkit/cvat/pull/2534>)
+- Intelligent scissors with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/2689>)
 
 ### Changed
 

diff --git a/cvat-canvas/package-lock.json b/cvat-canvas/package-lock.json
diff --git a/cvat-canvas/package.json b/cvat-canvas/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-canvas",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "description": "Part of Computer Vision Annotation Tool which presents its canvas library",
   "main": "src/canvas.ts",
   "scripts": {

diff --git a/cvat-canvas/src/scss/canvas.scss b/cvat-canvas/src/scss/canvas.scss
@@ -51,6 +51,10 @@ polyline.cvat_shape_drawing_opacity {
     stroke: red;
 }
 
+.cvat_canvas_threshold {
+    stroke: red;
+}
+
 .cvat_canvas_shape_grouping {
     @extend .cvat_shape_action_dasharray;
     @extend .cvat_shape_action_opacity;

diff --git a/cvat-canvas/src/typescript/canvasModel.ts b/cvat-canvas/src/typescript/canvasModel.ts
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -76,6 +76,10 @@ export interface InteractionData {
     crosshair?: boolean;
     minPosVertices?: number;
     minNegVertices?: number;
+    enableNegVertices?: boolean;
+    enableThreshold?: boolean;
+    enableSliding?: boolean;
+    allowRemoveOnlyLast?: boolean;
 }
 
 export interface InteractionResult {

diff --git a/cvat-canvas/src/typescript/canvasView.ts b/cvat-canvas/src/typescript/canvasView.ts
@@ -165,6 +165,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
         shapes: InteractionResult[] | null,
         shapesUpdated: boolean = true,
         isDone: boolean = false,
+        threshold: number | null = null,
     ): void {
         const { zLayer } = this.controller;
         if (Array.isArray(shapes)) {
@@ -176,6 +177,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
                     isDone,
                     shapes,
                     zOrder: zLayer || 0,
+                    threshold,
                 },
             });
 
@@ -1050,6 +1052,7 @@ export class CanvasViewImpl implements CanvasView, Listener {
         });
 
         this.content.addEventListener('wheel', (event): void => {
+            if (event.ctrlKey) return;
             const { offset } = this.controller.geometry;
             const point = translateToSVG(this.content, [event.clientX, event.clientY]);
             self.controller.zoom(point[0] - offset, point[1] - offset, event.deltaY > 0 ? -1 : 1);

diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts
@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Intel Corporation
+// Copyright (C) 2020-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -24,6 +24,8 @@ export class InteractionHandlerImpl implements InteractionHandler {
     private interactionShapes: SVG.Shape[];
     private currentInteractionShape: SVG.Shape | null;
     private crosshair: Crosshair;
+    private threshold: SVG.Rect | null;
+    private thresholdRectSize: number;
 
     private prepareResult(): InteractionResult[] {
         return this.interactionShapes.map(
@@ -63,12 +65,21 @@ export class InteractionHandlerImpl implements InteractionHandler {
             return enabled && !ctrlKey && !!interactionShapes.length;
         }
 
-        const minimumVerticesAchieved =
-            (typeof minPosVertices === 'undefined' || minPosVertices <= positiveShapes.length) &&
-            (typeof minNegVertices === 'undefined' || minPosVertices <= negativeShapes.length);
+        const minPosVerticesAchieved = typeof minPosVertices === 'undefined' || minPosVertices <= positiveShapes.length;
+        const minNegVerticesAchieved = typeof minNegVertices === 'undefined' || minPosVertices <= negativeShapes.length;
+        const minimumVerticesAchieved = minPosVerticesAchieved && minNegVerticesAchieved;
         return enabled && !ctrlKey && minimumVerticesAchieved && shapesWereUpdated;
     }
 
+    private addThreshold(): void {
+        const { x, y } = this.cursorPosition;
+        this.threshold = this.canvas
+            .rect(this.thresholdRectSize, this.thresholdRectSize)
+            .fill('none')
+            .addClass('cvat_canvas_threshold');
+        this.threshold.center(x, y);
+    }
+
     private addCrosshair(): void {
         const { x, y } = this.cursorPosition;
         this.crosshair.show(this.canvas, x, y, this.geometry.scale);
@@ -80,9 +91,12 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
     private interactPoints(): void {
         const eventListener = (e: MouseEvent): void => {
-            if ((e.button === 0 || e.button === 2) && !e.altKey) {
+            if ((e.button === 0 || (e.button === 2 && this.interactionData.enableNegVertices)) && !e.altKey) {
                 e.preventDefault();
                 const [cx, cy] = translateToSVG((this.canvas.node as any) as SVGSVGElement, [e.clientX, e.clientY]);
+                if (!this.isWithingFrame(cx, cy)) return;
+                if (!this.isWithinThreshold(cx, cy)) return;
+
                 this.currentInteractionShape = this.canvas
                     .circle((consts.BASE_POINT_SIZE * 2) / this.geometry.scale)
                     .center(cx, cy)
@@ -101,6 +115,12 @@ export class InteractionHandlerImpl implements InteractionHandler {
 
                 const self = this.currentInteractionShape;
                 self.on('mouseenter', (): void => {
+                    if (this.interactionData.allowRemoveOnlyLast) {
+                        if (this.interactionShapes.indexOf(self) !== this.interactionShapes.length - 1) {
+                            return;
+                        }
+                    }
+
                     self.attr({
                         'stroke-width': consts.POINTS_SELECTED_STROKE_WIDTH / this.geometry.scale,
                     });
@@ -166,6 +186,10 @@ export class InteractionHandlerImpl implements InteractionHandler {
         if (this.interactionData.crosshair) {
             this.addCrosshair();
         }
+
+        if (this.interactionData.enableThreshold) {
+            this.addThreshold();
+        }
     }
 
     private startInteraction(): void {
@@ -183,6 +207,11 @@ export class InteractionHandlerImpl implements InteractionHandler {
             this.removeCrosshair();
         }
 
+        if (this.threshold) {
+            this.threshold.remove();
+            this.threshold = null;
+        }
+
         this.canvas.off('mousedown.interaction');
         this.interactionShapes.forEach((shape: SVG.Shape): SVG.Shape => shape.remove());
         this.interactionShapes = [];
@@ -192,14 +221,39 @@ export class InteractionHandlerImpl implements InteractionHandler {
         }
     }
 
+    private isWithinThreshold(x: number, y: number): boolean {
+        const [prev] = this.interactionShapes.slice(-1);
+        if (!this.interactionData.enableThreshold || !prev) {
+            return true;
+        }
+
+        const [prevCx, prevCy] = [(prev as SVG.Circle).cx(), (prev as SVG.Circle).cy()];
+        const xDiff = Math.abs(prevCx - x);
+        const yDiff = Math.abs(prevCy - y);
+
+        return xDiff < this.thresholdRectSize / 2 && yDiff < this.thresholdRectSize / 2;
+    }
+
+    private isWithingFrame(x: number, y: number): boolean {
+        const { offset, image } = this.geometry;
+        const { width, height } = image;
+        const [imageX, imageY] = [Math.round(x - offset), Math.round(y - offset)];
+        return imageX >= 0 && imageX < width && imageY >= 0 && imageY < height;
+    }
+
     public constructor(
-        onInteraction: (shapes: InteractionResult[] | null, shapesUpdated?: boolean, isDone?: boolean) => void,
+        onInteraction: (
+            shapes: InteractionResult[] | null,
+            shapesUpdated?: boolean,
+            isDone?: boolean,
+            threshold?: number,
+        ) => void,
         canvas: SVG.Container,
         geometry: Geometry,
     ) {
         this.onInteraction = (shapes: InteractionResult[] | null, shapesUpdated?: boolean, isDone?: boolean): void => {
             this.shapesWereUpdated = false;
-            onInteraction(shapes, shapesUpdated, isDone);
+            onInteraction(shapes, shapesUpdated, isDone, this.threshold ? this.thresholdRectSize / 2 : null);
         };
         this.canvas = canvas;
         this.geometry = geometry;
@@ -208,6 +262,8 @@ export class InteractionHandlerImpl implements InteractionHandler {
         this.interactionData = { enabled: false };
         this.currentInteractionShape = null;
         this.crosshair = new Crosshair();
+        this.threshold = null;
+        this.thresholdRectSize = 300;
         this.cursorPosition = {
             x: 0,
             y: 0,
@@ -219,6 +275,43 @@ export class InteractionHandlerImpl implements InteractionHandler {
             if (this.crosshair) {
                 this.crosshair.move(x, y);
             }
+            if (this.threshold) {
+                this.threshold.center(x, y);
+            }
+
+            if (this.interactionData.enableSliding && this.interactionShapes.length) {
+                if (this.isWithingFrame(x, y)) {
+                    if (this.interactionData.enableThreshold && !this.isWithinThreshold(x, y)) return;
+                    this.onInteraction(
+                        [
+                            ...this.prepareResult(),
+                            {
+                                points: [x - this.geometry.offset, y - this.geometry.offset],
+                                shapeType: 'points',
+                                button: 0,
+                            },
+                        ],
+                        true,
+                        false,
+                    );
+                }
+            }
+        });
+
+        this.canvas.on('wheel.interaction', (e: WheelEvent): void => {
+            if (e.ctrlKey) {
+                if (this.threshold) {
+                    const { x, y } = this.cursorPosition;
+                    e.preventDefault();
+                    if (e.deltaY > 0) {
+                        this.thresholdRectSize *= 6 / 5;
+                    } else {
+                        this.thresholdRectSize *= 5 / 6;
+                    }
+                    this.threshold.size(this.thresholdRectSize, this.thresholdRectSize);
+                    this.threshold.center(x, y);
+                }
+            }
         });
 
         document.body.addEventListener('keyup', (e: KeyboardEvent): void => {

diff --git a/cvat-core/src/ml-model.js b/cvat-core/src/ml-model.js
@@ -1,4 +1,4 @@
-// Copyright (C) 2019-2020 Intel Corporation
+// Copyright (C) 2019-2021 Intel Corporation
 //
 // SPDX-License-Identifier: MIT
 
@@ -17,6 +17,7 @@ class MLModel {
         this._params = {
             canvas: {
                 minPosVertices: data.min_pos_points,
+                enableNegVertices: true,
             },
         };
     }

diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json
diff --git a/cvat-ui/package.json b/cvat-ui/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cvat-ui",
-  "version": "1.13.8",
+  "version": "1.14.0",
   "description": "CVAT single-page application",
   "main": "src/index.tsx",
   "scripts": {

diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts
@@ -18,6 +18,7 @@ import {
     ContextMenuType,
     Workspace,
     Model,
+    OpenCVTool,
 } from 'reducers/interfaces';
 
 import getCore from 'cvat-core-wrapper';
@@ -1354,7 +1355,10 @@ export function pasteShapeAsync(): ThunkAction {
     };
 }
 
-export function interactWithCanvas(activeInteractor: Model, activeLabelID: number): AnyAction {
+export function interactWithCanvas(
+    activeInteractor: Model | OpenCVTool,
+    activeLabelID: number,
+): AnyAction {
     return {
         type: AnnotationActionTypes.INTERACT_WITH_CANVAS,
         payload: {

diff --git a/cvat-ui/src/assets/opencv.svg b/cvat-ui/src/assets/opencv.svg
diff --git a/...src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/...src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx
@@ -15,6 +15,7 @@ import MoveControl from './move-control';
 import FitControl from './fit-control';
 import ResizeControl from './resize-control';
 import ToolsControl from './tools-control';
+import OpenCVControl from './opencv-control';
 import DrawRectangleControl from './draw-rectangle-control';
 import DrawPolygonControl from './draw-polygon-control';
 import DrawPolylineControl from './draw-polyline-control';
@@ -90,6 +91,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
                 ActiveControl.DRAW_RECTANGLE,
                 ActiveControl.DRAW_CUBOID,
                 ActiveControl.AI_TOOLS,
+                ActiveControl.OPENCV_TOOLS,
             ].includes(activeControl);
 
             if (!drawing) {
@@ -103,7 +105,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
                     repeatDrawShape();
                 }
             } else {
-                if (activeControl === ActiveControl.AI_TOOLS) {
+                if ([ActiveControl.AI_TOOLS, ActiveControl.OPENCV_TOOLS].includes(activeControl)) {
                     // separated API method
                     canvasInstance.interact({ enabled: false });
                     return;
@@ -187,6 +189,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element {
 
             <hr />
             <ToolsControl />
+            <OpenCVControl />
             <DrawRectangleControl
                 canvasInstance={canvasInstance}
                 isDrawing={activeControl === ActiveControl.DRAW_RECTANGLE}