From 181481d77f86c9a5603651a0481b0e2ebb8b3f81 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2020 14:15:32 +0300 Subject: [PATCH 01/32] Bump opencv-python from 4.3.0.38 to 4.4.0.40 in /cvat/requirements (#2021) Bumps [opencv-python](https://github.com/skvark/opencv-python) from 4.3.0.38 to 4.4.0.40. - [Release notes](https://github.com/skvark/opencv-python/releases) - [Commits](https://github.com/skvark/opencv-python/commits) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 33b84dfe9aaa..7a9da4d3b009 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -40,7 +40,7 @@ matplotlib==3.0.3 scikit-image==0.15.0 tensorflow==2.2.0 keras==2.4.2 -opencv-python==4.3.0.38 +opencv-python==4.4.0.40 h5py==2.10.0 imgaug==0.4.0 django-cors-headers==3.4.0 From 2fdd8d8bf53ca401169884ebe597e2c2c9536e71 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 12 Aug 2020 14:16:33 +0300 Subject: [PATCH 02/32] Fix cli (#2022) --- datumaro/datumaro/cli/contexts/project/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datumaro/datumaro/cli/contexts/project/__init__.py b/datumaro/datumaro/cli/contexts/project/__init__.py index 9731d48bda52..e6d5809b5416 100644 --- a/datumaro/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/datumaro/cli/contexts/project/__init__.py @@ -465,7 +465,7 @@ def build_merge_parser(parser_ctor=argparse.ArgumentParser): """, formatter_class=MultilineFormatter) - parser.add_argument('other_project', + parser.add_argument('other_project_dir', help="Path to a project") parser.add_argument('-o', '--output-dir', dest='dst_dir', default=None, help="Output directory (default: current project's dir)") From b9f47fd3200485f05476a4be95ef877f837652fe Mon Sep 17 00:00:00 2001 From: Dmitry Kruchinin <33020454+dvkruchinin@users.noreply.github.com> Date: Wed, 12 Aug 2020 15:35:32 +0300 Subject: [PATCH 03/32] Cypress test for issue 1919 (#2013) * Cypress test for issue 1919 * Increase value of "defaultCommandTimeout" to 10000ms * Appying comments. Co-authored-by: Dmitry Kruchinin --- tests/cypress.json | 1 + .../integration/issue_1919_check_text_attr.js | 55 +++++++++++++++++++ tests/cypress/support/commands.js | 18 ++++++ 3 files changed, 74 insertions(+) create mode 100644 tests/cypress/integration/issue_1919_check_text_attr.js diff --git a/tests/cypress.json b/tests/cypress.json index 4f32312dc758..51422aa13a7f 100644 --- a/tests/cypress.json +++ b/tests/cypress.json @@ -3,6 +3,7 @@ "baseUrl": "http://localhost:8080", "viewportWidth": 1300, "viewportHeight": 960, + "defaultCommandTimeout": 10000, "testFiles": [ "auth_page.js", "issue_*.js" diff --git a/tests/cypress/integration/issue_1919_check_text_attr.js b/tests/cypress/integration/issue_1919_check_text_attr.js new file mode 100644 index 000000000000..ea6fca779eac --- /dev/null +++ b/tests/cypress/integration/issue_1919_check_text_attr.js @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +/// + +context('Check label attribute changes', () => { + + const issueId = '1919' + const labelName = `Issue ${issueId}` + const taskName = `New annotation task for ${labelName}` + const attrName = `Attr for ${labelName}` + const textDefaultValue = 'Some default value for type Text' + const image = `image_${issueId}.png` + const newLabelAttrValue = 'New attribute value' + const width = 800 + const height = 800 + const posX = 10 + const posY = 10 + const color = 'gray' + + before(() => { + cy.visit('auth/login') + cy.login() + cy.imageGenerator('cypress/fixtures', image, width, height, color, posX, posY, labelName) + cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, image) + cy.openTaskJob(taskName) + cy.createShape(309, 431, 616, 671) + }) + + describe(`Testing issue "${issueId}"`, () => { + it('Open object menu', () => { + cy.get('#cvat_canvas_shape_1').trigger('mousemove').rightclick() + }) + it('Open object menu details', () => { + cy.get('.cvat-canvas-context-menu') + .contains('Details') + .click() + }) + it('Clear field of text attribute and write new value', () => { + cy.get('.cvat-canvas-context-menu') + .find('.cvat-object-item-text-attribute') + .should('have.value', textDefaultValue) + .clear() + .type(newLabelAttrValue) + }) + it('Check what value of right panel is changed too', () => { + cy.get('#cvat-objects-sidebar-state-item-1') + .find('.cvat-object-item-text-attribute') + .should('have.value', newLabelAttrValue) + }) + }) +}) diff --git a/tests/cypress/support/commands.js b/tests/cypress/support/commands.js index 1601e60b36da..d8c338c87c21 100644 --- a/tests/cypress/support/commands.js +++ b/tests/cypress/support/commands.js @@ -12,6 +12,7 @@ require('../plugins/imageGenerator/imageGeneratorCommand') Cypress.Commands.add('login', (username='admin', password='12qwaszx') => { cy.get('[placeholder="Username"]').type(username) cy.get('[placeholder="Password"]').type(password) + cy.get('[type="submit"]').click() }) Cypress.Commands.add('createAnnotationTask', (taksName='New annotation task', @@ -37,6 +38,23 @@ Cypress.Commands.add('createAnnotationTask', (taksName='New annotation task', cy.url().should('include', '/tasks?page=') }) +Cypress.Commands.add('openTask', (taskName) => { + cy.contains('strong', taskName) + .parents('.cvat-tasks-list-item') + .contains('a', 'Open') + .click() +}) + +Cypress.Commands.add('openJob', () => { + cy.contains('a', 'Job #').click() + cy.url().should('include', '/jobs') +}) + +Cypress.Commands.add('openTaskJob', (taskName) => { + cy.openTask(taskName) + cy.openJob() +}) + Cypress.Commands.add('createShape', (ferstX, ferstY, lastX, lastY) => { cy.get(':nth-child(8) > svg').trigger('mousemove').click() cy.get(':nth-child(6) > :nth-child(1) > .ant-btn').click() From 0856be00e97370a9a102c3b0f505b5e39f821e86 Mon Sep 17 00:00:00 2001 From: Dmitry Kruchinin <33020454+dvkruchinin@users.noreply.github.com> Date: Wed, 12 Aug 2020 15:37:56 +0300 Subject: [PATCH 04/32] Cypress test for issue 1870 (#2016) * Cypress test for issue 1870 * Applying comments. Refactoring the test code. Co-authored-by: Dmitry Kruchinin --- .../issue_1870_cursor_not_jump_to_end.js | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/cypress/integration/issue_1870_cursor_not_jump_to_end.js diff --git a/tests/cypress/integration/issue_1870_cursor_not_jump_to_end.js b/tests/cypress/integration/issue_1870_cursor_not_jump_to_end.js new file mode 100644 index 000000000000..e44ea21ce4bd --- /dev/null +++ b/tests/cypress/integration/issue_1870_cursor_not_jump_to_end.js @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +/// + +context('Checks that the cursor doesn\'t automatically jump to the end of a word when the attribute value changes', () => { + + const issueId = '1870' + const labelName = `Issue ${issueId}` + const taskName = `New annotation task for ${labelName}` + const attrName = `Attr for ${labelName}` + const textDefaultValue = 'text' + const image = `image_${issueId}.png` + const newLabelAttrValue = 'teeext' + const width = 800 + const height = 800 + const posX=10 + const posY=10 + const color='gray' + + before(() => { + cy.visit('auth/login') + cy.login() + cy.imageGenerator('cypress/fixtures', image, width, height, color, posX, posY, labelName) + cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, image) + cy.openTaskJob(taskName) + cy.createShape(309, 431, 616, 671) + }) + + describe(`Testing issue "${issueId}"`, () => { + it('Enter 2 characters in the middle of the word attribute value and check the result', () => { + cy.get('#cvat-objects-sidebar-state-item-1') + .find('.ant-collapse-item') + .click() + .find('.cvat-object-item-text-attribute') + .type('{leftarrow}{leftarrow}ee') + .should('have.value', newLabelAttrValue) + }) + }) +}) From d66449241a3507e1586e456703f6a8338d60c59e Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 13 Aug 2020 10:50:03 +0300 Subject: [PATCH 05/32] Batch of fixes for Safari (#2025) * Styles fixes * Fixed imageBitmap & styles on annotation page * Increased version of cvat-data --- cvat-data/package-lock.json | 2 +- cvat-data/package.json | 2 +- cvat-data/src/js/cvat-data.js | 21 ++++++++++++++++++- cvat-data/src/js/unzip_imgs.worker.js | 15 ++++++++++--- .../annotation-page/annotation-page.tsx | 6 +++--- .../standard-workspace/styles.scss | 2 +- .../components/create-task-page/styles.scss | 5 +++-- cvat-ui/src/components/cvat-app.tsx | 2 +- cvat-ui/src/components/feedback/styles.scss | 3 ++- .../models-page/deployed-models-list.tsx | 4 ++-- .../src/components/models-page/styles.scss | 4 +++- .../src/components/models-page/top-bar.tsx | 2 +- cvat-ui/src/components/task-page/styles.scss | 4 +++- cvat-ui/src/components/tasks-page/styles.scss | 3 +++ 14 files changed, 56 insertions(+), 19 deletions(-) diff --git a/cvat-data/package-lock.json b/cvat-data/package-lock.json index 4a96cca3b4f4..f392ce8e80b8 100644 --- a/cvat-data/package-lock.json +++ b/cvat-data/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-data", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-data/package.json b/cvat-data/package.json index eb78de78c58c..5c42155e7538 100644 --- a/cvat-data/package.json +++ b/cvat-data/package.json @@ -1,6 +1,6 @@ { "name": "cvat-data", - "version": "1.0.0", + "version": "1.0.1", "description": "", "main": "src/js/cvat-data.js", "devDependencies": { diff --git a/cvat-data/src/js/cvat-data.js b/cvat-data/src/js/cvat-data.js index 8484e9ac7581..86a0a6d008ca 100644 --- a/cvat-data/src/js/cvat-data.js +++ b/cvat-data/src/js/cvat-data.js @@ -295,7 +295,26 @@ class FrameProvider { worker.terminate(); }; - worker.onmessage = (event) => { + worker.onmessage = async (event) => { + if (event.data.isRaw) { + // safary doesn't support createImageBitmap + // there is a way to polyfill it with using document.createElement + // but document.createElement doesn't work in worker + // so, we get raw data and decode it here, no other way + + const createImageBitmap = async function(blob) { + return new Promise((resolve,reject) => { + let img = document.createElement('img'); + img.addEventListener('load', function() { + resolve(this); + }); + img.src = URL.createObjectURL(blob); + }); + }; + + event.data.data = await createImageBitmap(event.data.data); + } + this._frames[event.data.index] = event.data.data; if (this._decodingBlocks[`${start}:${end}`].resolveCallback) { diff --git a/cvat-data/src/js/unzip_imgs.worker.js b/cvat-data/src/js/unzip_imgs.worker.js index f704e0982678..68d0b2a5957d 100644 --- a/cvat-data/src/js/unzip_imgs.worker.js +++ b/cvat-data/src/js/unzip_imgs.worker.js @@ -20,13 +20,22 @@ onmessage = (e) => { const fileIndex = index++; if (fileIndex <= end) { _zip.file(relativePath).async('blob').then((fileData) => { - createImageBitmap(fileData).then((img) => { + if (self.createImageBitmap) { + createImageBitmap(fileData).then((img) => { + postMessage({ + fileName: relativePath, + index: fileIndex, + data: img, + }); + }); + } else { postMessage({ fileName: relativePath, index: fileIndex, - data: img, + data: fileData, + isRaw: true, }); - }); + } }); } }); diff --git a/cvat-ui/src/components/annotation-page/annotation-page.tsx b/cvat-ui/src/components/annotation-page/annotation-page.tsx index 8854dd5da894..d98d4284556f 100644 --- a/cvat-ui/src/components/annotation-page/annotation-page.tsx +++ b/cvat-ui/src/components/annotation-page/annotation-page.tsx @@ -80,17 +80,17 @@ export default function AnnotationPageComponent(props: Props): JSX.Element { { workspace === Workspace.STANDARD && ( - + )} { workspace === Workspace.ATTRIBUTE_ANNOTATION && ( - + )} { workspace === Workspace.TAG_ANNOTATION && ( - + )} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss index c729d25956bf..a9514add5d2a 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss @@ -98,7 +98,7 @@ padding: 10px; border-radius: 5px; background: $background-color-2; - width: 250px; + width: 270px; > div { margin-top: 5px; diff --git a/cvat-ui/src/components/create-task-page/styles.scss b/cvat-ui/src/components/create-task-page/styles.scss index ccf664b2fcd4..a9b03632b13c 100644 --- a/cvat-ui/src/components/create-task-page/styles.scss +++ b/cvat-ui/src/components/create-task-page/styles.scss @@ -8,8 +8,9 @@ text-align: center; padding-top: 40px; overflow-y: auto; - height: 100%; - padding-bottom: 40px; + height: 90%; + position: fixed; + width: 100%; > div > span { font-size: 36px; diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index 8b7322709f66..50d7ce045ffa 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -273,7 +273,7 @@ class CVATApplication extends React.PureComponent - + diff --git a/cvat-ui/src/components/feedback/styles.scss b/cvat-ui/src/components/feedback/styles.scss index 884958736766..39415886a4b0 100644 --- a/cvat-ui/src/components/feedback/styles.scss +++ b/cvat-ui/src/components/feedback/styles.scss @@ -3,10 +3,11 @@ // SPDX-License-Identifier: MIT .cvat-feedback-button { - position: absolute; + position: fixed; bottom: 20px; right: 20px; padding: 0; + height: auto; > i { font-size: 40px; diff --git a/cvat-ui/src/components/models-page/deployed-models-list.tsx b/cvat-ui/src/components/models-page/deployed-models-list.tsx index c9d9ec6e0021..93e301be2ef0 100644 --- a/cvat-ui/src/components/models-page/deployed-models-list.tsx +++ b/cvat-ui/src/components/models-page/deployed-models-list.tsx @@ -35,10 +35,10 @@ export default function DeployedModelsListComponent(props: Props): JSX.Element { Type - + Description - + Labels diff --git a/cvat-ui/src/components/models-page/styles.scss b/cvat-ui/src/components/models-page/styles.scss index f566e92f265b..ddcaef8a92fa 100644 --- a/cvat-ui/src/components/models-page/styles.scss +++ b/cvat-ui/src/components/models-page/styles.scss @@ -6,8 +6,10 @@ .cvat-models-page { padding-top: 30px; - height: 100%; + height: 90%; overflow: auto; + position: fixed; + width: 100%; > div:nth-child(1) { margin-bottom: 10px; diff --git a/cvat-ui/src/components/models-page/top-bar.tsx b/cvat-ui/src/components/models-page/top-bar.tsx index 584fa83c6732..d0ede58340af 100644 --- a/cvat-ui/src/components/models-page/top-bar.tsx +++ b/cvat-ui/src/components/models-page/top-bar.tsx @@ -9,7 +9,7 @@ import Text from 'antd/lib/typography/Text'; export default function TopBarComponent(): JSX.Element { return ( - + Models diff --git a/cvat-ui/src/components/task-page/styles.scss b/cvat-ui/src/components/task-page/styles.scss index b13a2c44a035..fd218c50e436 100644 --- a/cvat-ui/src/components/task-page/styles.scss +++ b/cvat-ui/src/components/task-page/styles.scss @@ -6,7 +6,9 @@ .cvat-task-details-wrapper { overflow-y: auto; - height: 100%; + height: 90%; + width: 100%; + position: fixed; .cvat-task-details { width: 100%; diff --git a/cvat-ui/src/components/tasks-page/styles.scss b/cvat-ui/src/components/tasks-page/styles.scss index a5a88ca519d7..623b763393c5 100644 --- a/cvat-ui/src/components/tasks-page/styles.scss +++ b/cvat-ui/src/components/tasks-page/styles.scss @@ -6,7 +6,10 @@ .cvat-tasks-page { padding-top: 15px; + padding-bottom: 40px; height: 100%; + position: fixed; + width: 100%; > div:nth-child(1) { padding-bottom: 10px; From 8f323cfd5c579473e24818e004708dfbdfed829b Mon Sep 17 00:00:00 2001 From: Dmitry Kruchinin <33020454+dvkruchinin@users.noreply.github.com> Date: Thu, 13 Aug 2020 11:13:13 +0300 Subject: [PATCH 06/32] Cypress test for issue 1429 (#2017) * Cypress test for issue 1429 * Rebase. Code refactoring. Co-authored-by: Dmitry Kruchinin --- .../integration/issue_1429_check_new_label.js | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/cypress/integration/issue_1429_check_new_label.js diff --git a/tests/cypress/integration/issue_1429_check_new_label.js b/tests/cypress/integration/issue_1429_check_new_label.js new file mode 100644 index 000000000000..955103a3b520 --- /dev/null +++ b/tests/cypress/integration/issue_1429_check_new_label.js @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +/// + +context('Check if the new label reflects in the options', () => { + + const issueId = '1429' + const labelName = `Issue ${issueId}` + const taskName = `New annotation task for ${labelName}` + const attrName = `Attr for ${labelName}` + const textDefaultValue = 'Some default value for type Text' + const image = `image_${issueId}.png` + const newLabelName = `New ${labelName}` + const width = 800 + const height = 800 + const posX = 10 + const posY = 10 + const color = 'gray' + + before(() => { + cy.visit('auth/login') + cy.login() + cy.imageGenerator('cypress/fixtures', image, width, height, color, posX, posY, labelName) + cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, image) + }) + + describe(`Testing issue "${issueId}"`, () => { + it('Open a task. Open a job', () => { + cy.openTaskJob(taskName) + }) + it('Return to task page using browser button "previous page"', () => { + cy.go('back') + cy.url().should('include', '/tasks').and('not.contain', '/jobs') + }) + it('Add new label', () => { + cy.contains('button', 'Add label').click() + cy.get('[placeholder="Label name"]').type(newLabelName) + cy.contains('button', 'Done').click() + }) + it('Open the job again', () => { + cy.openJob() + }) + it('Create a shape', () => { + cy.createShape(309, 431, 616, 671) + }) + it('Checking for the new label', () => { + cy.get('#cvat-objects-sidebar-state-item-1') + .find('.ant-select-selection') + .click() + cy.get('.ant-select-dropdown-menu-item') + .should('contain', newLabelName) + }) + }) +}) From cf465b51eb5f5b13beeb22fce99ca2652070f2ce Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Sun, 16 Aug 2020 20:01:26 +0300 Subject: [PATCH 07/32] Added html meta description for cvat.org (#2035) * Added html meta description for cvat.org * Update index.html Changed tabs to spaces --- cvat-ui/src/index.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cvat-ui/src/index.html b/cvat-ui/src/index.html index b115e93bf13a..5a7b61a070fb 100644 --- a/cvat-ui/src/index.html +++ b/cvat-ui/src/index.html @@ -8,8 +8,10 @@ - + + + Computer Vision Annotation Tool From e8c44bc8e2c2f74cb2fa21dd99e305008b813a32 Mon Sep 17 00:00:00 2001 From: Dmitry Kruchinin <33020454+dvkruchinin@users.noreply.github.com> Date: Sun, 16 Aug 2020 20:03:40 +0300 Subject: [PATCH 08/32] Cypress test for issue 1439 (#2028) Co-authored-by: Dmitry Kruchinin --- .../issue_1439_blocked_object_info.js | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/cypress/integration/issue_1439_blocked_object_info.js diff --git a/tests/cypress/integration/issue_1439_blocked_object_info.js b/tests/cypress/integration/issue_1439_blocked_object_info.js new file mode 100644 index 000000000000..a729f78c5a14 --- /dev/null +++ b/tests/cypress/integration/issue_1439_blocked_object_info.js @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2020 Intel Corporation + * + * SPDX-License-Identifier: MIT + */ + +/// + +context('Information about a blocked object disappears if hover the cursor over another object', () => { + + const issueId = '1439' + const labelName = `Issue ${issueId}` + const taskName = `New annotation task for ${labelName}` + const attrName = `Attr for ${labelName}` + const textDefaultValue = 'Some default value for type Text' + const image = `image_${issueId}.png` + const width = 800 + const height = 800 + const posX = 10 + const posY = 10 + const color = 'gray' + + before(() => { + cy.visit('auth/login') + cy.login() + cy.imageGenerator('cypress/fixtures', image, width, height, color, posX, posY, labelName) + cy.createAnnotationTask(taskName, labelName, attrName, textDefaultValue, image) + cy.openTaskJob(taskName) + }) + + describe(`Testing issue "${issueId}"`, () => { + it('Create multiple objects', () => { + cy.createShape(309, 431, 409, 531) + cy.createShape(200, 300, 300, 400) + }) + it('Lock all objects', () => { + cy.get('.cvat-objects-sidebar-states-header') + .find('.anticon-unlock') + .click() + }) + it('Mousemove to 1st object', () => { + cy.get('#cvat_canvas_shape_1').trigger('mousemove') + }) + it('Mousemove to 2nd object', () => { + cy.get('#cvat_canvas_shape_2').trigger('mousemove') + }) + it('Information about 1st object not exist', () => { + cy.get('#cvat_canvas_text_content') + .contains(`${labelName} 1`) + .should('not.exist') + }) + }) +}) From 90cc36eb1c47123cfafcd91ae32eb8634fd30bd0 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 17 Aug 2020 10:34:44 +0300 Subject: [PATCH 09/32] More classes to cypress testing (#2038) --- .../attribute-switcher.tsx | 2 +- .../object-switcher.tsx | 2 +- .../attribute-annotation-workspace/styles.scss | 3 ++- .../controls-side-bar/draw-cuboid-control.tsx | 1 + .../controls-side-bar/draw-points-control.tsx | 1 + .../controls-side-bar/draw-polygon-control.tsx | 1 + .../controls-side-bar/draw-polyline-control.tsx | 1 + .../controls-side-bar/draw-rectangle-control.tsx | 1 + .../annotation-page/top-bar/player-buttons.tsx | 14 ++++++++------ 9 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-switcher.tsx b/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-switcher.tsx index 23675d0b2558..66357b2f47e6 100644 --- a/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-switcher.tsx +++ b/cvat-ui/src/components/annotation-page/attribute-annotation-workspace/attribute-annotation-sidebar/attribute-switcher.tsx @@ -27,7 +27,7 @@ function AttributeSwitcher(props: Props): JSX.Element { const title = `${currentAttribute} [${currentIndex + 1}/${attributesCount}]`; return ( -
+
+ ); + notification.info({ message: 'The task has been created', + btn, }); this.basicConfigurationComponent.resetFields(); @@ -252,3 +264,5 @@ export default class CreateTaskContent extends React.PureComponent ); } } + +export default withRouter(CreateTaskContent); diff --git a/cvat-ui/src/components/create-task-page/create-task-page.tsx b/cvat-ui/src/components/create-task-page/create-task-page.tsx index 15c5f18c47a6..39549ea68dd8 100644 --- a/cvat-ui/src/components/create-task-page/create-task-page.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-page.tsx @@ -16,6 +16,7 @@ interface Props { onCreate: (data: CreateTaskData) => void; status: string; error: string; + taskId: number | null; installedGit: boolean; } @@ -23,6 +24,7 @@ export default function CreateTaskPage(props: Props): JSX.Element { const { error, status, + taskId, onCreate, installedGit, } = props; @@ -66,6 +68,7 @@ export default function CreateTaskPage(props: Props): JSX.Element { Create a new task props.history.push('/tasks?page=1') + (event: React.MouseEvent): void => { + event.preventDefault(); + props.history.push('/tasks?page=1'); + } } > Tasks @@ -184,8 +188,12 @@ function HeaderContainer(props: Props): JSX.Element { className='cvat-header-button' type='link' value='models' + href='/models' onClick={ - (): void => props.history.push('/models') + (event: React.MouseEvent): void => { + event.preventDefault(); + props.history.push('/models'); + } } > Models @@ -195,8 +203,10 @@ function HeaderContainer(props: Props): JSX.Element { + )} + + + + + + + + + + )} + title={( + + + + Select color + + + + + + + + + + )} + placement={placement || 'left'} + overlayClassName='cvat-label-color-picker' + trigger='click' + visible={typeof visible === 'boolean' ? visible : pickerVisible} + onVisibleChange={changeVisible} + > + {children} + + ); +} + +export default React.forwardRef(ColorPicker); diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/label-item.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/label-item.tsx index 96a99a7ef298..e5067658d029 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/label-item.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/label-item.tsx @@ -5,32 +5,26 @@ import React from 'react'; import { Row, Col } from 'antd/lib/grid'; import Icon from 'antd/lib/icon'; -import Popover from 'antd/lib/popover'; import Button from 'antd/lib/button'; import Text from 'antd/lib/typography/Text'; -import ColorChanger from 'components/annotation-page/standard-workspace/objects-side-bar/color-changer'; interface Props { labelName: string; labelColor: string; - labelColors: string[]; visible: boolean; statesHidden: boolean; statesLocked: boolean; - changeColorShortcut: string; hideStates(): void; showStates(): void; lockStates(): void; unlockStates(): void; - changeColor(color: string): void; } function LabelItemComponent(props: Props): JSX.Element { const { labelName, labelColor, - labelColors, visible, statesHidden, statesLocked, @@ -38,8 +32,6 @@ function LabelItemComponent(props: Props): JSX.Element { showStates, lockStates, unlockStates, - changeColor, - changeColorShortcut, } = props; return ( @@ -51,19 +43,7 @@ function LabelItemComponent(props: Props): JSX.Element { style={{ display: visible ? 'flex' : 'none' }} > - - )} - > - + + + + )} @@ -491,6 +500,37 @@ class LabelForm extends React.PureComponent { ); } + private renderChangeColorButton(): JSX.Element { + const { label, form } = this.props; + + return ( + + + { + form.getFieldDecorator('labelColor', { + initialValue: (label && label.color) ? label.color : undefined, + })( + + + + + , + ) + } + + + ); + } + public render(): JSX.Element { const { label, @@ -511,6 +551,8 @@ class LabelForm extends React.PureComponent { { this.renderLabelNameInput() } + { this.renderChangeColorButton() } + { this.renderNewAttributeButton() } { attributeItems.length > 0 diff --git a/cvat-ui/src/components/labels-editor/labels-editor.tsx b/cvat-ui/src/components/labels-editor/labels-editor.tsx index 89245b96b986..8c0ad072981e 100644 --- a/cvat-ui/src/components/labels-editor/labels-editor.tsx +++ b/cvat-ui/src/components/labels-editor/labels-editor.tsx @@ -64,6 +64,7 @@ export default class LabelsEditor return { name: label.name, id: label.id || idGenerator(), + color: label.color, attributes: label.attributes.map((attr: any): Attribute => ( { id: attr.id || idGenerator(), @@ -198,6 +199,7 @@ export default class LabelsEditor return { name: label.name, id: label.id < 0 ? undefined : label.id, + color: label.color, attributes: label.attributes.map((attr: Attribute): any => ( { name: attr.name, diff --git a/cvat-ui/src/components/labels-editor/styles.scss b/cvat-ui/src/components/labels-editor/styles.scss index ba03ef1a9348..1e34e9929347 100644 --- a/cvat-ui/src/components/labels-editor/styles.scss +++ b/cvat-ui/src/components/labels-editor/styles.scss @@ -87,3 +87,21 @@ textarea.ant-input.cvat-raw-labels-viewer { .cvat-delete-attribute-button:hover > i { color: $danger-icon-color; } + +.cvat-new-attribute-button { + width: 100%; +} + +.cvat-change-task-label-color-button { + width: 100%; + + .ant-badge-status-text { + margin-left: 15px; + } +} + +.cvat-change-task-label-color-badge .ant-badge-status-dot { + width: 15px; + height: 15px; + border-radius: unset; +} diff --git a/cvat-ui/src/consts.ts b/cvat-ui/src/consts.ts index e5df33c7ff1e..0c1da29d4a4f 100644 --- a/cvat-ui/src/consts.ts +++ b/cvat-ui/src/consts.ts @@ -14,6 +14,7 @@ const GITHUB_IMAGE_URL = 'https://raw.githubusercontent.com/opencv/cvat/develop/ const SHARE_MOUNT_GUIDE_URL = 'https://github.com/opencv/cvat/blob/master/cvat/apps/documentation/installation.md#share-path'; const NUCLIO_GUIDE = 'https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/installation.md#semi-automatic-and-automatic-annotation'; const CANVAS_BACKGROUND_COLORS = ['#ffffff', '#f1f1f1', '#e5e5e5', '#d8d8d8', '#CCCCCC', '#B3B3B3', '#999999']; +const NEW_LABEL_COLOR = '#b3b3b3'; export default { UNDEFINED_ATTRIBUTE_VALUE, @@ -27,5 +28,6 @@ export default { GITHUB_IMAGE_URL, SHARE_MOUNT_GUIDE_URL, CANVAS_BACKGROUND_COLORS, + NEW_LABEL_COLOR, NUCLIO_GUIDE, }; diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/label-item.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/label-item.tsx index 955df16dabe5..65c6027e07d9 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/label-item.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/label-item.tsx @@ -5,10 +5,7 @@ import React from 'react'; import { connect } from 'react-redux'; -import { - changeLabelColorAsync, - updateAnnotationsAsync, -} from 'actions/annotation-actions'; +import { updateAnnotationsAsync } from 'actions/annotation-actions'; import LabelItemComponent from 'components/annotation-page/standard-workspace/objects-side-bar/label-item'; import { CombinedState } from 'reducers/interfaces'; @@ -22,8 +19,6 @@ interface StateToProps { label: any; labelName: string; labelColor: string; - labelColors: string[]; - changeColorShortcut: string; objectStates: any[]; jobInstance: any; frameNumber: any; @@ -31,7 +26,6 @@ interface StateToProps { interface DispatchToProps { updateAnnotations(states: any[]): void; - changeLabelColor(label: any, color: string): void; } function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { @@ -49,10 +43,6 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { number: frameNumber, }, }, - colors: labelColors, - }, - shortcuts: { - normalizedKeyMap, }, } = state; @@ -63,11 +53,9 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { label, labelColor: label.color, labelName: label.name, - labelColors, objectStates, jobInstance, frameNumber, - changeColorShortcut: normalizedKeyMap.CHANGE_OBJECT_COLOR, }; } @@ -76,12 +64,6 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { updateAnnotations(states: any[]): void { dispatch(updateAnnotationsAsync(states)); }, - changeLabelColor( - label: any, - color: string, - ): void { - dispatch(changeLabelColorAsync(label, color)); - }, }; } @@ -151,15 +133,6 @@ class LabelItemContainer extends React.PureComponent { this.switchLock(false); }; - private changeColor = (color: string): void => { - const { - changeLabelColor, - label, - } = this.props; - - changeLabelColor(label, color); - }; - private switchHidden(value: boolean): void { const { updateAnnotations, @@ -196,24 +169,19 @@ class LabelItemContainer extends React.PureComponent { const { labelName, labelColor, - labelColors, - changeColorShortcut, } = this.props; return ( ); } diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx index 34e8facf9948..e0b29e369c01 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/object-item.tsx @@ -15,7 +15,6 @@ import { } from 'reducers/interfaces'; import { collapseObjectItems, - changeLabelColorAsync, updateAnnotationsAsync, changeFrameAsync, removeObjectAsync, @@ -43,7 +42,6 @@ interface StateToProps { activated: boolean; colorBy: ColorBy; ready: boolean; - colors: string[]; activeControl: ActiveControl; minZLayer: number; maxZLayer: number; @@ -58,7 +56,6 @@ interface DispatchToProps { removeObject: (sessionInstance: any, objectState: any) => void; copyShape: (objectState: any) => void; propagateObject: (objectState: any) => void; - changeLabelColor(label: any, color: string): void; changeGroupColor(group: number, color: string): void; } @@ -88,7 +85,6 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { ready, activeControl, }, - colors, }, settings: { shapes: { @@ -115,7 +111,6 @@ function mapStateToProps(state: CombinedState, own: OwnProps): StateToProps { ready, activeControl, colorBy, - colors, jobInstance, frameNumber, activated: activatedStateID === own.clientID, @@ -149,12 +144,6 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { propagateObject(objectState: any): void { dispatch(propagateObjectAction(objectState)); }, - changeLabelColor( - label: any, - color: string, - ): void { - dispatch(changeLabelColorAsync(label, color)); - }, changeGroupColor(group: number, color: string): void { dispatch(changeGroupColorAsync(group, color)); }, @@ -273,7 +262,6 @@ class ObjectItemContainer extends React.PureComponent { const { objectState, colorBy, - changeLabelColor, changeGroupColor, } = this.props; @@ -282,8 +270,6 @@ class ObjectItemContainer extends React.PureComponent { this.commit(); } else if (colorBy === ColorBy.GROUP) { changeGroupColor(objectState.group.id, color); - } else if (colorBy === ColorBy.LABEL) { - changeLabelColor(objectState.label, color); } }; @@ -375,7 +361,6 @@ class ObjectItemContainer extends React.PureComponent { attributes, activated, colorBy, - colors, normalizedKeyMap, } = this.props; @@ -399,10 +384,10 @@ class ObjectItemContainer extends React.PureComponent { attrValues={{ ...objectState.attributes }} labelID={objectState.label.id} color={stateColor} - colors={colors} attributes={attributes} normalizedKeyMap={normalizedKeyMap} labels={labels} + colorBy={colorBy} collapsed={collapsed} activate={this.activate} remove={this.remove} diff --git a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/objects-list.tsx b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/objects-list.tsx index 43e6e2659ef4..14e445deee97 100644 --- a/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/objects-list.tsx +++ b/cvat-ui/src/containers/annotation-page/standard-workspace/objects-side-bar/objects-list.tsx @@ -15,7 +15,6 @@ import { copyShape as copyShapeAction, propagateObject as propagateObjectAction, changeGroupColorAsync, - changeLabelColorAsync, } from 'actions/annotation-actions'; import { Canvas } from 'cvat-canvas-wrapper'; import { @@ -53,7 +52,6 @@ interface DispatchToProps { propagateObject: (objectState: any) => void; changeFrame(frame: number): void; changeGroupColor(group: number, color: string): void; - changeLabelColor(label: any, color: string): void; } function mapStateToProps(state: CombinedState): StateToProps { @@ -155,9 +153,6 @@ function mapDispatchToProps(dispatch: any): DispatchToProps { changeGroupColor(group: number, color: string): void { dispatch(changeGroupColorAsync(group, color)); }, - changeLabelColor(label: any, color: string): void { - dispatch(changeLabelColorAsync(label, color)); - }, }; } @@ -278,7 +273,6 @@ class ObjectsListContainer extends React.PureComponent { jobInstance, updateAnnotations, changeGroupColor, - changeLabelColor, removeObject, copyShape, propagateObject, @@ -399,15 +393,11 @@ class ObjectsListContainer extends React.PureComponent { return; } - if (colorBy === ColorBy.LABEL) { - const colorID = (colors.indexOf(state.label.color) + 1) % colors.length; - changeLabelColor(state.label, colors[colorID]); - return; + if (colorBy === ColorBy.INSTANCE) { + const colorID = (colors.indexOf(state.color) + 1) % colors.length; + state.color = colors[colorID]; + updateAnnotations([state]); } - - const colorID = (colors.indexOf(state.color) + 1) % colors.length; - state.color = colors[colorID]; - updateAnnotations([state]); } }, TO_BACKGROUND: (event: KeyboardEvent | undefined) => { diff --git a/cvat-ui/src/icons.tsx b/cvat-ui/src/icons.tsx index 1eed125bf327..fbf4860723d5 100644 --- a/cvat-ui/src/icons.tsx +++ b/cvat-ui/src/icons.tsx @@ -41,6 +41,7 @@ import SVGBackgroundIcon from './assets/background-icon.svg'; import SVGForegroundIcon from './assets/foreground-icon.svg'; import SVGCubeIcon from './assets/cube-icon.svg'; import SVGResetPerspectiveIcon from './assets/reset-perspective.svg'; +import SVGColorizeIcon from './assets/colorize-icon.svg'; export const CVATLogo = React.memo( (): JSX.Element => , @@ -153,3 +154,6 @@ export const CubeIcon = React.memo( export const ResetPerspectiveIcon = React.memo( (): JSX.Element => , ); +export const ColorizeIcon = React.memo( + (): JSX.Element => , +); diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index 91756d9b6402..4976977dcdc6 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -634,31 +634,6 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { }, }; } - case AnnotationActionTypes.CHANGE_LABEL_COLOR_SUCCESS: { - const { - label, - states, - history, - } = action.payload; - - const { instance: job } = state.job; - const labels = [...job.task.labels]; - const index = labels.indexOf(label); - labels[index] = label; - - return { - ...state, - job: { - ...state.job, - labels, - }, - annotations: { - ...state.annotations, - states, - history, - }, - }; - } case AnnotationActionTypes.ACTIVATE_OBJECT: { const { activatedStateID, diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index e4da70868d95..a869011f0177 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -555,21 +555,6 @@ export default function (state = defaultState, action: AnyAction): Notifications }, }; } - case AnnotationActionTypes.CHANGE_LABEL_COLOR_FAILED: { - return { - ...state, - errors: { - ...state.errors, - annotation: { - ...state.errors.annotation, - changingLabelColor: { - message: 'Could not change label color', - reason: action.payload.error.toString(), - }, - }, - }, - }; - } case AnnotationActionTypes.UPDATE_ANNOTATIONS_FAILED: { return { ...state, diff --git a/cvat-ui/src/reducers/settings-reducer.ts b/cvat-ui/src/reducers/settings-reducer.ts index 3bd4cf90c5f0..eb4fd7555552 100644 --- a/cvat-ui/src/reducers/settings-reducer.ts +++ b/cvat-ui/src/reducers/settings-reducer.ts @@ -18,7 +18,7 @@ import { const defaultState: SettingsState = { shapes: { - colorBy: ColorBy.INSTANCE, + colorBy: ColorBy.LABEL, opacity: 3, selectedOpacity: 30, blackBorders: false, diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 64921555021f..30ec8bd90786 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -152,6 +152,7 @@ def _init_meta(self): ("labels", [ ("label", OrderedDict([ ("name", db_label.name), + ("color", db_label.color), ("attributes", [ ("attribute", OrderedDict([ ("name", db_attr.name), diff --git a/cvat/apps/dataset_manager/formats/mask.py b/cvat/apps/dataset_manager/formats/mask.py index 268eb347489f..b1307a9ceadf 100644 --- a/cvat/apps/dataset_manager/formats/mask.py +++ b/cvat/apps/dataset_manager/formats/mask.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: MIT -import os.path as osp from tempfile import TemporaryDirectory from pyunpack import Archive @@ -10,11 +9,10 @@ from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor, import_dm_annotations) from cvat.apps.dataset_manager.util import make_zip_archive -from datumaro.cli.util import make_file_name from datumaro.components.project import Dataset -from datumaro.util.mask_tools import generate_colormap from .registry import dm_env, exporter, importer +from .utils import make_colormap @exporter(name='Segmentation mask', ext='ZIP', version='1.1') @@ -41,44 +39,3 @@ def _import(src_file, task_data): masks_to_polygons = dm_env.transforms.get('masks_to_polygons') dataset = dataset.transform(masks_to_polygons) import_dm_annotations(dataset, task_data) - - -DEFAULT_COLORMAP_CAPACITY = 2000 -DEFAULT_COLORMAP_PATH = osp.join(osp.dirname(__file__), 'predefined_colors.txt') -def parse_default_colors(file_path=None): - if file_path is None: - file_path = DEFAULT_COLORMAP_PATH - - colors = {} - with open(file_path) as f: - for line in f: - line = line.strip() - if not line or line[0] == '#': - continue - _, label, color = line.split(':') - colors[label] = tuple(map(int, color.split(','))) - return colors - -def normalize_label(label): - label = make_file_name(label) # basically, convert to ASCII lowercase - label = label.replace('-', '_') - return label - -def make_colormap(task_data): - labels = sorted([label['name'] - for _, label in task_data.meta['task']['labels']]) - if 'background' in labels: - labels.remove('background') - labels.insert(0, 'background') - - predefined = parse_default_colors() - - # NOTE: using pop() to avoid collisions - colormap = {k: predefined.pop(normalize_label(k), None) for k in labels} - - random_labels = [k for k in labels if not colormap[k]] - if random_labels: - colors = generate_colormap(DEFAULT_COLORMAP_CAPACITY + len(random_labels)) - for i, label in enumerate(random_labels): - colormap[label] = colors[DEFAULT_COLORMAP_CAPACITY + i] - return {l: [c, [], []] for l, c in colormap.items()} \ No newline at end of file diff --git a/cvat/apps/dataset_manager/formats/utils.py b/cvat/apps/dataset_manager/formats/utils.py new file mode 100644 index 000000000000..2e30ab79200d --- /dev/null +++ b/cvat/apps/dataset_manager/formats/utils.py @@ -0,0 +1,79 @@ +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os.path as osp +from pyhash import murmur3_32 + +from datumaro.cli.util import make_file_name + +hasher = murmur3_32() + +def get_color_from_index(index): + def get_bit(number, index): + return (number >> index) & 1 + + color = [0, 0, 0] + + for j in range(7, -1, -1): + for c in range(3): + color[c] |= get_bit(index, c) << j + index >>= 3 + + return tuple(color) + +DEFAULT_COLORMAP_CAPACITY = 2000 +DEFAULT_COLORMAP_PATH = osp.join(osp.dirname(__file__), 'predefined_colors.txt') +def parse_default_colors(file_path=None): + if file_path is None: + file_path = DEFAULT_COLORMAP_PATH + + colors = {} + with open(file_path) as f: + for line in f: + line = line.strip() + if not line or line[0] == '#': + continue + _, label, color = line.split(':') + colors[label] = tuple(map(int, color.split(','))) + return colors + +def normalize_label(label): + label = make_file_name(label) # basically, convert to ASCII lowercase + label = label.replace('-', '_') + return label + +def rgb2hex(color): + return '#{0:02x}{1:02x}{2:02x}'.format(*color) + +def hex2rgb(color): + return tuple(int(color.lstrip('#')[i:i+2], 16) for i in (0, 2, 4)) + +def make_colormap(task_data): + labels = [label for _, label in task_data.meta['task']['labels']] + label_names = [label['name'] for label in labels] + + if 'background' not in label_names: + labels.insert(0, { + 'name': 'background', + 'color': '#000000', + } + ) + + return {label['name']: [hex2rgb(label['color']), [], []] for label in labels} + + +def get_label_color(label_name, label_names): + predefined = parse_default_colors() + normalized_names = [normalize_label(l_name) for l_name in label_names] + normalized_name = normalize_label(label_name) + + color = predefined.get(normalized_name, None) + offset = hasher(normalized_name) + normalized_names.count(normalized_name) + + if color is None: + color = get_color_from_index(DEFAULT_COLORMAP_CAPACITY + offset) + elif normalized_names.count(normalized_name): + color = get_color_from_index(DEFAULT_COLORMAP_CAPACITY + offset - 1) + + return rgb2hex(color) diff --git a/cvat/apps/engine/migrations/0028_labelcolor.py b/cvat/apps/engine/migrations/0028_labelcolor.py new file mode 100644 index 000000000000..a725eb9aa290 --- /dev/null +++ b/cvat/apps/engine/migrations/0028_labelcolor.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.13 on 2020-08-11 11:26 +from django.db import migrations, models +from cvat.apps.dataset_manager.formats.utils import get_label_color + +def alter_label_colors(apps, schema_editor): + Label = apps.get_model('engine', 'Label') + Task = apps.get_model('engine', 'Task') + + for task in Task.objects.all(): + labels = Label.objects.filter(task_id=task.id).order_by('id') + label_names = list() + for label in labels: + label.color = get_label_color(label.name, label_names) + label_names.append(label.name) + label.save() + +class Migration(migrations.Migration): + + dependencies = [ + ('engine', '0027_auto_20200719_1552'), + ] + + operations = [ + migrations.AddField( + model_name='label', + name='color', + field=models.CharField(default='', max_length=8), + ), + migrations.RunPython( + code=alter_label_colors, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index d410ef7c823f..f3fd13b420e9 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -240,6 +240,7 @@ class Meta: class Label(models.Model): task = models.ForeignKey(Task, on_delete=models.CASCADE) name = SafeCharField(max_length=64) + color = models.CharField(default='', max_length=8) def __str__(self): return self.name diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index a878e9e67663..f211d28b2b1d 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -11,6 +11,7 @@ from cvat.apps.engine import models from cvat.apps.engine.log import slogger +from cvat.apps.dataset_manager.formats.utils import get_label_color class AttributeSerializer(serializers.ModelSerializer): @@ -37,10 +38,11 @@ def to_representation(self, instance): class LabelSerializer(serializers.ModelSerializer): attributes = AttributeSerializer(many=True, source='attributespec_set', default=[]) + color = serializers.CharField(allow_blank=True, required=False) class Meta: model = models.Label - fields = ('id', 'name', 'attributes') + fields = ('id', 'name', 'color', 'attributes') class JobCommitSerializer(serializers.ModelSerializer): class Meta: @@ -250,8 +252,12 @@ class Meta: def create(self, validated_data): labels = validated_data.pop('label_set') db_task = models.Task.objects.create(**validated_data) + label_names = list() for label in labels: attributes = label.pop('attributespec_set') + if not label.get('color', None): + label['color'] = get_label_color(label['name'], label_names) + label_names.append(label['name']) db_label = models.Label.objects.create(task=db_task, **label) for attr in attributes: models.AttributeSpec.objects.create(label=db_label, **attr) @@ -286,6 +292,14 @@ def update(self, instance, validated_data): else: slogger.task[instance.id].info("{} label was updated" .format(db_label.name)) + if not label.get('color', None): + label_names = [l.name for l in + models.Label.objects.filter(task_id=instance.id).exclude(id=db_label.id).order_by('id') + ] + db_label.color = get_label_color(db_label.name, label_names) + else: + db_label.color = label.get('color', db_label.color) + db_label.save() for attr in attributes: (db_attr, created) = models.AttributeSpec.objects.get_or_create( label=db_label, name=attr['name'], defaults=attr) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 3889fed5cd4a..46e90ff1ac08 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -10,6 +10,7 @@ Pillow==7.2.0 numpy==1.18.5 python-ldap==3.3.1 pytz==2020.1 +pyhash==0.9.3 pyunpack==0.2.1 rcssmin==1.0.6 redis==3.5.3 From 4b32a9706089dba54c22bca33a3a4527ea4d9bd9 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 21:39:13 +0300 Subject: [PATCH 20/32] Bump rq from 1.4.2 to 1.5.1 in /cvat/requirements (#2065) Bumps [rq](https://github.com/nvie/rq) from 1.4.2 to 1.5.1. - [Release notes](https://github.com/nvie/rq/releases) - [Changelog](https://github.com/rq/rq/blob/master/CHANGES.md) - [Commits](https://github.com/nvie/rq/compare/v1.4.2...v1.5.1) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 46e90ff1ac08..6187a5a203a7 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -16,7 +16,7 @@ rcssmin==1.0.6 redis==3.5.3 rjsmin==1.1.0 requests==2.24.0 -rq==1.4.2 +rq==1.5.1 rq-scheduler==0.10.0 scipy==1.4.1 sqlparse==0.3.1 From 35e8a127a85ee6077d17ac8d9315f770fb5a4c85 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 21:39:39 +0300 Subject: [PATCH 21/32] Bump shapely from 1.7.0 to 1.7.1 in /cvat/requirements (#2064) Bumps [shapely](https://github.com/Toblerity/Shapely) from 1.7.0 to 1.7.1. - [Release notes](https://github.com/Toblerity/Shapely/releases) - [Changelog](https://github.com/Toblerity/Shapely/blob/master/CHANGES.txt) - [Commits](https://github.com/Toblerity/Shapely/compare/1.7.0...1.7.1) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 6187a5a203a7..5db238b3053e 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -32,7 +32,7 @@ Markdown==3.2.2 djangorestframework==3.11.1 Pygments==2.6.1 drf-yasg==1.17.1 -Shapely==1.7.0 +Shapely==1.7.1 pdf2image==1.13.1 pascal_voc_writer==0.1.4 django-rest-auth[with_social]==0.9.5 From 082fc7a4dda2d77ba198323448b095b7d1bc7d61 Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Fri, 21 Aug 2020 21:40:22 +0300 Subject: [PATCH 22/32] Bump pylint from 2.5.3 to 2.6.0 in /cvat/requirements (#2063) Bumps [pylint](https://github.com/PyCQA/pylint) from 2.5.3 to 2.6.0. - [Release notes](https://github.com/PyCQA/pylint/releases) - [Changelog](https://github.com/PyCQA/pylint/blob/master/ChangeLog) - [Commits](https://github.com/PyCQA/pylint/compare/pylint-2.5.3...pylint-2.6.0) Signed-off-by: dependabot-preview[bot] Co-authored-by: dependabot-preview[bot] <27856297+dependabot-preview[bot]@users.noreply.github.com> --- cvat/requirements/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/requirements/development.txt b/cvat/requirements/development.txt index 92b2272d56ea..7efae6777993 100644 --- a/cvat/requirements/development.txt +++ b/cvat/requirements/development.txt @@ -3,7 +3,7 @@ astroid==2.4.2 isort==4.3.21 lazy-object-proxy==1.5.1 mccabe==0.6.1 -pylint==2.5.3 +pylint==2.6.0 pylint-django==2.3.0 pylint-plugin-utils==0.6 rope==0.17.0 From 3fb3f677ebf95ed90ff404fb9ffe52f3c8fe55b3 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Fri, 21 Aug 2020 22:58:24 +0300 Subject: [PATCH 23/32] [Datumaro] Update documentation (#2059) * Update docs * fix indent * update design --- datumaro/CONTRIBUTING.md | 2 +- datumaro/README.md | 12 +- .../datumaro/cli/contexts/project/__init__.py | 2 +- datumaro/datumaro/plugins/transforms.py | 9 +- datumaro/docs/design.md | 24 +- datumaro/docs/user_manual.md | 514 ++++++++++++++++-- 6 files changed, 495 insertions(+), 68 deletions(-) diff --git a/datumaro/CONTRIBUTING.md b/datumaro/CONTRIBUTING.md index b26152987c20..97373b287d72 100644 --- a/datumaro/CONTRIBUTING.md +++ b/datumaro/CONTRIBUTING.md @@ -129,7 +129,7 @@ Plugins reside in plugin directories: - `/.datumaro/plugins` for project-specific components A plugin is a python file or package with any name, which exports some symbols. -To export a symbol put it to `exports` list of the module like this: +To export a symbol, put it to `exports` list of the module like this: ``` python class MyComponent1: ... diff --git a/datumaro/README.md b/datumaro/README.md index a355f4507436..2d83cc4df6d6 100644 --- a/datumaro/README.md +++ b/datumaro/README.md @@ -1,13 +1,13 @@ -# Dataset Framework (Datumaro) +# Dataset Management Framework (Datumaro) A framework to build, transform, and analyze datasets. ``` CVAT annotations -- ---> Annotation tool -... \ / + \ / COCO-like dataset -----> Datumaro ---> dataset ------> Model training -... / \ + / \ VOC-like dataset -- ---> Publication etc. ``` @@ -55,12 +55,12 @@ VOC-like dataset -- ---> Publication etc. - Dataset building operations: - Merging multiple datasets into one - Dataset filtering with custom conditions, for instance: - - remove all annotations except polygons of a certain class + - remove polygons of a certain class - remove images without a specific class - - remove occluded annotations from images + - remove `occluded` annotations from images - keep only vertically-oriented images - remove small area bounding boxes from annotations - - Annotation conversions, for instance + - Annotation conversions, for instance: - polygons to instance masks and vise-versa - apply a custom colormap for mask annotations - rename or remove dataset labels diff --git a/datumaro/datumaro/cli/contexts/project/__init__.py b/datumaro/datumaro/cli/contexts/project/__init__.py index e6d5809b5416..63c84076b9ff 100644 --- a/datumaro/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/datumaro/cli/contexts/project/__init__.py @@ -579,7 +579,7 @@ def build_transform_parser(parser_ctor=argparse.ArgumentParser): |n Examples:|n - Convert instance polygons to masks:|n - |s|stransform -n polygons_to_masks + |s|stransform -t polygons_to_masks """ % ', '.join(builtins), formatter_class=MultilineFormatter) diff --git a/datumaro/datumaro/plugins/transforms.py b/datumaro/datumaro/plugins/transforms.py index 368a891a02d2..82493610b71e 100644 --- a/datumaro/datumaro/plugins/transforms.py +++ b/datumaro/datumaro/plugins/transforms.py @@ -409,6 +409,13 @@ def transform_item(self, item): .format(item=item)) class RemapLabels(Transform, CliPlugin): + """ + Changes labels in the dataset.|n + Examples:|n + - Rename 'person' to 'car' and 'cat' to 'dog', keep 'bus', remove others:|n + |s|sremap_labels -l person:car -l bus:bus -l cat:dog --default delete + """ + DefaultAction = Enum('DefaultAction', ['keep', 'delete']) @staticmethod @@ -428,7 +435,7 @@ def build_cmdline_parser(cls, **kwargs): parser.add_argument('--default', choices=[a.name for a in cls.DefaultAction], default=cls.DefaultAction.keep.name, - help="Action for unspecified labels") + help="Action for unspecified labels (default: %(default)s)") return parser def __init__(self, extractor, mapping, default=None): diff --git a/datumaro/docs/design.md b/datumaro/docs/design.md index 7d89e8ebe382..528b2adf7549 100644 --- a/datumaro/docs/design.md +++ b/datumaro/docs/design.md @@ -49,6 +49,8 @@ Datumaro is: - Versioning (for images, annotations, subsets, sources etc., comparison) - Documentation generation - Provision of iterators for user code +- Dataset downloading +- Dataset generation - Dataset building (export in a specific format, indexation, statistics, documentation) - Dataset exporting to other formats - Dataset debugging (run inference, generate dataset slices, compute statistics) @@ -111,17 +113,17 @@ can be downloaded by user to be operated on with Datumaro CLI. - [ ] with TensorBoard - Calculation of statistics for datasets - - [ ] Pixel mean, std - - [ ] Object counts (detection scenario) - - [ ] Image-Class distribution (classification scenario) - - [ ] Pixel-Class distribution (segmentation scenario) - - [ ] Image clusters + - [x] Pixel mean, std + - [x] Object counts (detection scenario) + - [x] Image-Class distribution (classification scenario) + - [x] Pixel-Class distribution (segmentation scenario) + - [ ] Image similarity clusters - [ ] Custom statistics - Dataset building - [x] Composite dataset building - - [ ] Annotation remapping - - [ ] Subset splitting + - [x] Class remapping + - [x] Subset splitting - [x] Dataset filtering (`extract`) - [x] Dataset merging (`merge`) - [ ] Dataset item editing (`edit`) @@ -129,7 +131,7 @@ can be downloaded by user to be operated on with Datumaro CLI. - Dataset comparison (`diff`) - [x] Annotation-annotation comparison - [x] Annotation-inference comparison - - [ ] Annotation quality estimation (for CVAT) + - [x] Annotation quality estimation (for CVAT) - Provide a simple method to check annotation quality with a model and generate summary @@ -142,9 +144,9 @@ can be downloaded by user to be operated on with Datumaro CLI. - [x] Task export - [x] Datumaro project export - [x] Dataset export - - [ ] Original raw data (images, a video file) can be downloaded (exported) + - [x] Original raw data (images, a video file) can be downloaded (exported) together with annotations or just have links - on CVAT server (in the future support S3, etc) + on CVAT server (in future, support S3, etc) - [x] Be able to use local files instead of remote links - [ ] Specify cache directory - [x] Use case "annotate for model training" @@ -154,7 +156,7 @@ can be downloaded by user to be operated on with Datumaro CLI. - convert to a training format - train a DL model - [x] Use case "annotate - reannotate problematic images - merge" - - [ ] Use case "annotate and estimate quality" + - [x] Use case "annotate and estimate quality" - create a task - annotate - estimate quality of annotations diff --git a/datumaro/docs/user_manual.md b/datumaro/docs/user_manual.md index 65284fa1166c..e2e798e2b166 100644 --- a/datumaro/docs/user_manual.md +++ b/datumaro/docs/user_manual.md @@ -1,22 +1,28 @@ -# Quick start guide +# User manual ## Contents - [Installation](#installation) - [Interfaces](#interfaces) -- [Supported dataset formats and annotations](#formats-support) +- [Supported dataset formats and annotations](#supported-formats) - [Command line workflow](#command-line-workflow) +- [Command reference](#command-reference) + - [Convert datasets](#convert-datasets) - [Create a project](#create-project) - [Add and remove data](#add-and-remove-data) - [Import a project](#import-project) - [Extract a subproject](#extract-subproject) - - [Merge projects](#merge-project) + - [Update project (merge)](#update-project) + - [Merge projects](#merge-projects) - [Export a project](#export-project) - [Compare projects](#compare-projects) - - [Get project info](#get-project-info) + - [Obtaining project info](#get-project-info) + - [Obtaining project statistics](#get-project-statistics) - [Register a model](#register-model) - [Run inference](#run-inference) - [Run inference explanation](#explain-inference) + - [Transform a project](#transform-project) +- [Extending](#extending) - [Links](#links) ## Installation @@ -36,7 +42,7 @@ python -m virtualenv venv . venv/bin/activate ``` -Install Datumaro: +Install: ``` bash pip install 'git+https://github.com/opencv/cvat#egg=datumaro&subdirectory=datumaro' ``` @@ -68,57 +74,93 @@ As a python library: import datumaro ``` -## Formats support +## Supported Formats List of supported formats: -- COCO (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`*) +- MS COCO (`image_info`, `instances`, `person_keypoints`, `captions`, `labels`*) - [Format specification](http://cocodataset.org/#format-data) + - [Dataset example](../tests/assets/coco_dataset) - `labels` are our extension - like `instances` with only `category_id` - PASCAL VOC (`classification`, `detection`, `segmentation` (class, instances), `action_classification`, `person_layout`) - [Format specification](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/htmldoc/index.html) + - [Dataset example](../tests/assets/voc_dataset) - YOLO (`bboxes`) - [Format specification](https://github.com/AlexeyAB/darknet#how-to-train-pascal-voc-data) + - [Dataset example](../tests/assets/yolo_dataset) - TF Detection API (`bboxes`, `masks`) - Format specifications: [bboxes](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/using_your_own_dataset.md), [masks](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/instance_segmentation.md) + - [Dataset example](../tests/assets/tf_detection_api_dataset) +- MOT sequences + - [Format specification](https://arxiv.org/pdf/1906.04567.pdf) + - [Dataset example](../tests/assets/mot_dataset) - CVAT - [Format specification](https://github.com/opencv/cvat/blob/develop/cvat/apps/documentation/xml_format.md) + - [Dataset example](../tests/assets/cvat_dataset) +- LabelMe + - [Format specification](http://labelme.csail.mit.edu/Release3.0) + - [Dataset example](../tests/assets/labelme_dataset) List of supported annotation types: - Labels - Bounding boxes - Polygons - Polylines +- (Segmentation) Masks - (Key-)Points - Captions -- Masks ## Command line workflow -> **Note**: command invocation syntax is subject to change, -> **always refer to command --help output** - -The key object is the Project. The Project is a combination of -a Project's own dataset, a number of external data sources and an environment. +The key object is a project, so most CLI commands operate on projects. However, there +are few commands operating on datasets directly. A project is a combination of +a project's own dataset, a number of external data sources and an environment. An empty Project can be created by `project create` command, an existing dataset can be imported with `project import` command. A typical way to obtain projects is to export tasks in CVAT UI. +If you want to interact with models, you need to add them to project first. + +## Command reference + +> **Note**: command invocation syntax is subject to change, +> **always refer to command --help output** + Available CLI commands: ![CLI design doc](images/cli_design.png) -If you want to interact with models, you need to add them to project first. +### Convert datasets + +This command allows to convert a dataset from one format into another. In fact, this +command is a combination of `project import` and `project export` and just provides a simpler +way to obtain the same result when no extra options is needed. A list of supported +formats can be found in the `--help` output of this command. + +Usage: + +``` bash +datum convert --help + +datum convert \ + -i \ + -if \ + -o \ + -f \ + -- [extra parameters for output format] +``` + +Example: convert a VOC-like dataset to a COCO-like one: + +``` bash +datum convert --input-format voc --input-path \ + --output-format coco +``` ### Import project This command creates a Project from an existing dataset. -Supported formats are listed in the command help. -In Datumaro dataset formats are supported by Extractors and Importers. -An Extractor produces a list of dataset items corresponding -to the dataset. An Importer creates a Project from the -data source location. It is possible to add a custom Extractor and Importer. -To do this, you need to put an Extractor and Importer implementation scripts to -`/.datumaro/extractors` and `/.datumaro/importers`. +Supported formats are listed in the command help. Check [extending tips](#extending) +for information on extra format support. Usage: @@ -178,7 +220,7 @@ datum project create -o my_dataset/ ### Add and remove data -A Project can be attached to a number of external Data Sources. Each Source +A Project can contain a number of external Data Sources. Each Data Source describes a way to produce dataset items. A Project combines dataset items from all the sources and its own dataset into one composite dataset. You can manage project sources by commands in the `source` command line context. @@ -195,11 +237,8 @@ is used in COCO format: ``` -In Datumaro dataset formats are supported by Extractors. -An Extractor produces a list of dataset items corresponding -to the dataset. It is possible to add a custom Extractor. -To do this, you need to put an Extractor -definition script to `/.datumaro/extractors`. +Supported formats are listed in the command help. Check [extending tips](#extending) +for information on extra format support. Usage: @@ -237,16 +276,16 @@ This command allows to create a sub-Project from a Project. The new project includes only items satisfying some condition. [XPath](https://devhints.io/xpath) is used as query format. -There are several filtering modes available ('-m/--mode' parameter). +There are several filtering modes available (`-m/--mode` parameter). Supported modes: -- 'i', 'items' -- 'a', 'annotations' -- 'i+a', 'a+i', 'items+annotations', 'annotations+items' +- `i`, `items` +- `a`, `annotations` +- `i+a`, `a+i`, `items+annotations`, `annotations+items` -When filtering annotations, use the 'items+annotations' +When filtering annotations, use the `items+annotations` mode to point that annotation-less dataset items should be removed. To select an annotation, write an XPath that -returns 'annotation' elements (see examples). +returns `annotation` elements (see examples). Usage: @@ -259,7 +298,7 @@ datum project extract \ -e '' ``` -Example: extract a dataset with only images which width < height +Example: extract a dataset with only images which `width` < `height` ``` bash datum project extract \ @@ -274,7 +313,7 @@ Example: extract a dataset with only large annotations of class `cat` and any no datum project extract \ -p test_project \ -o test_project-extract \ - --mode annotations -e '/item/annotation[(label="cat" and area > 999.5) or label!="person"]' + --mode annotations -e '/item/annotation[(label="cat" and area > 99.5) or label!="person"]' ``` Example: extract a dataset with only occluded annotations, remove empty images @@ -321,9 +360,9 @@ Item representations are available with `--dry-run` parameter: ``` -### Merge projects +### Update project -This command combines multiple Projects into one. +This command updates items in a project from another one (check [Merge Projects](#merge-projects) for complex merging). Usage: @@ -346,16 +385,40 @@ datum project merge \ second_project ``` +### Merge projects + +This command merges items from 2 or more projects and checks annotations for errors. + +Spatial annotations are compared by distance and intersected, labels and attributes +are selected by voting. +Merge conflicts, missing items and annotations, other errors are saved into a `.json` file. + +Usage: + +``` bash +datum merge --help + +datum merge +``` + +Example: merge 4 (partially-)intersecting projects, +- consider voting succeeded when there are 3+ same votes +- consider shapes intersecting when IoU >= 0.6 +- check annotation groups to have `person`, `hand`, `head` and `foot` (`?` for optional) + +``` bash +datum merge project1/ project2/ project3/ project4/ \ + --quorum 3 \ + -iou 0.6 \ + --groups 'person,hand?,head,foot?' +``` + ### Export project -This command exports a Project in some format. +This command exports a Project as a dataset in some format. -Supported formats are listed in the command help. -In Datumaro dataset formats are supported by Converters. -A Converter produces a dataset of a specific format -from dataset items. It is possible to add a custom Converter. -To do this, you need to put a Converter -definition script to /.datumaro/converters. +Supported formats are listed in the command help. Check [extending tips](#extending) +for information on extra format support. Usage: @@ -366,17 +429,17 @@ datum project export \ -p \ -o \ -f \ - [-- ] + -- [additional format parameters] ``` -Example: save project as VOC-like dataset, include images +Example: save project as VOC-like dataset, include images, convert images to `PNG` ``` bash datum project export \ -p test_project \ -o test_project-export \ -f voc \ - -- --save-images + -- --save-images --image-ext='.png' ``` ### Get project info @@ -398,7 +461,7 @@ Example: datum project info -p /test_project Project: - name: test_project2 + name: test_project location: /test_project Sources: source 'instances_minival2014': @@ -419,6 +482,282 @@ Dataset: labels: person, bicycle, car, motorcycle (and 76 more) ``` +### Get project statistics + +This command computes various project statistics, such as: +- image mean and std. dev. +- class and attribute balance +- mask pixel balance +- segment area distribution + +Usage: + +``` bash +datum project stats --help + +datum project stats \ + -p +``` + +Example: + +
+ +``` bash +datum project stats -p /test_project + +{ + "annotations": { + "labels": { + "attributes": { + "gender": { + "count": 358, + "distribution": { + "female": [ + 149, + 0.41620111731843573 + ], + "male": [ + 209, + 0.5837988826815642 + ] + }, + "values count": 2, + "values present": [ + "female", + "male" + ] + }, + "view": { + "count": 340, + "distribution": { + "__undefined__": [ + 4, + 0.011764705882352941 + ], + "front": [ + 54, + 0.1588235294117647 + ], + "left": [ + 14, + 0.041176470588235294 + ], + "rear": [ + 235, + 0.6911764705882353 + ], + "right": [ + 33, + 0.09705882352941177 + ] + }, + "values count": 5, + "values present": [ + "__undefined__", + "front", + "left", + "rear", + "right" + ] + } + }, + "count": 2038, + "distribution": { + "car": [ + 340, + 0.16683022571148184 + ], + "cyclist": [ + 194, + 0.09519136408243375 + ], + "head": [ + 354, + 0.17369970559371933 + ], + "ignore": [ + 100, + 0.04906771344455348 + ], + "left_hand": [ + 238, + 0.11678115799803729 + ], + "person": [ + 358, + 0.17566241413150147 + ], + "right_hand": [ + 77, + 0.037782139352306184 + ], + "road_arrows": [ + 326, + 0.15996074582924436 + ], + "traffic_sign": [ + 51, + 0.025024533856722278 + ] + } + }, + "segments": { + "area distribution": [ + { + "count": 1318, + "max": 11425.1, + "min": 0.0, + "percent": 0.9627465303140978 + }, + { + "count": 1, + "max": 22850.2, + "min": 11425.1, + "percent": 0.0007304601899196494 + }, + { + "count": 0, + "max": 34275.3, + "min": 22850.2, + "percent": 0.0 + }, + { + "count": 0, + "max": 45700.4, + "min": 34275.3, + "percent": 0.0 + }, + { + "count": 0, + "max": 57125.5, + "min": 45700.4, + "percent": 0.0 + }, + { + "count": 0, + "max": 68550.6, + "min": 57125.5, + "percent": 0.0 + }, + { + "count": 0, + "max": 79975.7, + "min": 68550.6, + "percent": 0.0 + }, + { + "count": 0, + "max": 91400.8, + "min": 79975.7, + "percent": 0.0 + }, + { + "count": 0, + "max": 102825.90000000001, + "min": 91400.8, + "percent": 0.0 + }, + { + "count": 50, + "max": 114251.0, + "min": 102825.90000000001, + "percent": 0.036523009495982466 + } + ], + "avg. area": 5411.624543462382, + "pixel distribution": { + "car": [ + 13655, + 0.0018431496518735067 + ], + "cyclist": [ + 939005, + 0.12674674030446592 + ], + "head": [ + 0, + 0.0 + ], + "ignore": [ + 5501200, + 0.7425510702956085 + ], + "left_hand": [ + 0, + 0.0 + ], + "person": [ + 954654, + 0.12885903974805205 + ], + "right_hand": [ + 0, + 0.0 + ], + "road_arrows": [ + 0, + 0.0 + ], + "traffic_sign": [ + 0, + 0.0 + ] + } + } + }, + "annotations by type": { + "bbox": { + "count": 548 + }, + "caption": { + "count": 0 + }, + "label": { + "count": 0 + }, + "mask": { + "count": 0 + }, + "points": { + "count": 669 + }, + "polygon": { + "count": 821 + }, + "polyline": { + "count": 0 + } + }, + "annotations count": 2038, + "dataset": { + "image mean": [ + 107.06903686941979, + 79.12831698580979, + 52.95829558185416 + ], + "image std": [ + 49.40237673503467, + 43.29600731496902, + 35.47373007603151 + ], + "images count": 100 + }, + "images count": 100, + "subsets": {}, + "unannotated images": [ + "img00051", + "img00052", + "img00053", + "img00054", + "img00055", + ], + "unannotated images count": 5 +} +``` + +
+ ### Register model Supported models: @@ -557,6 +896,85 @@ datum explain \ -s 1000 --progressive ``` +### Transform Project + +This command allows to modify images or annotations in a project all at once. + +``` bash +datum project transform --help + +datum project transform \ + -p \ + -o \ + -t \ + -- [extra transform options] +``` + +Example: split a dataset randomly to `train` and `test` subsets, ratio is 2:1 + +``` bash +datum project transform -t random_split -- --subset train:.67 --subset test:.33 +``` + +Example: convert polygons to masks, masks to boxes etc.: + +``` bash +datum project transform -t boxes_to_masks +datum project transform -t masks_to_polygons +datum project transform -t polygons_to_masks +datum project transform -t shapes_to_boxes +``` + +Example: remap dataset labels, `person` to `car` and `cat` to `dog`, keep `bus`, remove others + +``` bash +datum project transform -t remap_labels -- \ + -l person:car -l bus:bus -l cat:dog \ + --default delete +``` + +Example: rename dataset items by a regular expression +- Replace `pattern` with `replacement` +- Remove `frame_` from item ids + +``` bash +datum project transform -t rename -- -e '|pattern|replacement|' +datum project transform -t rename -- -e '|frame_(\d+)|\\1|' +``` + +## Extending + +There are few ways to extend and customize Datumaro behaviour, which is supported by plugins. +Check [our contribution guide](../CONTRIBUTING.md) for details on plugin implementation. +In general, a plugin is a Python code file. It must be put into a plugin directory: +- `/.datumaro/plugins` for project-specific plugins +- `/plugins` for global plugins + +### Dataset Formats + +Dataset reading is supported by Extractors and Importers. +An Extractor produces a list of dataset items corresponding +to the dataset. An Importer creates a project from the data source location. +It is possible to add custom Extractors and Importers. To do this, you need +to put an Extractor and Importer implementation scripts to a plugin directory. + +Dataset writing is supported by Converters. +A Converter produces a dataset of a specific format from dataset items. +It is possible to add custom Converters. To do this, you need to put a Converter +implementation script to a plugin directory. + +### Dataset Conversions ("Transforms") + +A Transform is a function for altering a dataset and producing a new one. It can update +dataset items, annotations, classes, and other properties. +A list of available transforms for dataset conversions can be extended by adding a Transform +implementation script into a plugin directory. + +### Model launchers + +A list of available launchers for model execution can be extended by adding a Launcher +implementation script into a plugin directory. + ## Links - [TensorFlow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) - [How to convert model to OpenVINO format](https://docs.openvinotoolkit.org/latest/_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models.html) From 1907b7924a4156dcba13aa9debec63a229707582 Mon Sep 17 00:00:00 2001 From: Andrey Zhavoronkov Date: Mon, 24 Aug 2020 13:46:10 +0300 Subject: [PATCH 24/32] Added django-rq to the proxy match pattern. (#2069) * added django-rq to the match pattern. Remoed unused locations from the config. * updated changelog * updated installation guide --- CHANGELOG.md | 1 + cvat/apps/documentation/installation.md | 15 +-------------- cvat_proxy/conf.d/cvat.conf.template | 15 +-------------- 3 files changed, 3 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d8ea49e03c4..47964aca6098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Issue loading openvino models for semi-automatic and automatic annotation () - Basic functions of CVAT works without activated nuclio dashboard - Fixed error with creating task with labels with the same name () +- Django RQ dashboard view () ### Security - diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md index 1741a16b1430..dc37a02525a3 100644 --- a/cvat/apps/documentation/installation.md +++ b/cvat/apps/documentation/installation.md @@ -569,26 +569,13 @@ server { proxy_set_header Host $http_host; proxy_pass_header Set-Cookie; - location ~* /api/.*|git/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.* { + location ~* /api/.*|git/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? { proxy_pass http://cvat:8080; } - # workaround for match location by arguments - location = / { - error_page 418 = @annotation_ui; - - if ( $query_string ~ "^id=\d+.*" ) { return 418; } - proxy_pass http://cvat_ui; - } - location / { proxy_pass http://cvat_ui; } - - # old annotation ui, will be removed in the future. - location @annotation_ui { - proxy_pass http://cvat:8080; - } } ``` diff --git a/cvat_proxy/conf.d/cvat.conf.template b/cvat_proxy/conf.d/cvat.conf.template index 287f72849d7e..bd46d38a8f40 100644 --- a/cvat_proxy/conf.d/cvat.conf.template +++ b/cvat_proxy/conf.d/cvat.conf.template @@ -12,24 +12,11 @@ server { proxy_set_header Host $http_host; proxy_pass_header Set-Cookie; - location ~* /api/.*|git/.*|analytics/.*|static/.*|admin|admin/.*|documentation/.* { + location ~* /api/.*|git/.*|analytics/.*|static/.*|admin(?:/(.*))?.*|documentation/.*|django-rq(?:/(.*))? { proxy_pass http://cvat:8080; } - # workaround for match location by arguments - location = / { - error_page 418 = @annotation_ui; - - if ( $query_string ~ "^id=\d+.*" ) { return 418; } - proxy_pass http://cvat_ui; - } - location / { proxy_pass http://cvat_ui; } - - # old annotation ui, will be removed in the future. - location @annotation_ui { - proxy_pass http://cvat:8080; - } } From 18cdcd5daae16294a353139bdfe2d3e9304b14b2 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 24 Aug 2020 13:46:50 +0300 Subject: [PATCH 25/32] Added link to admin page (#2068) * Added link to admin page * Updated version * Updated changelog --- CHANGELOG.md | 1 + cvat-ui/package-lock.json | 2 +- cvat-ui/package.json | 2 +- cvat-ui/src/components/cvat-app.tsx | 4 +- cvat-ui/src/components/header/header.tsx | 204 +++++++++++++----- .../login-page/cookie-policy-drawer.tsx | 2 +- .../register-page/register-form.tsx | 14 +- cvat-ui/src/containers/header/header.tsx | 97 --------- 8 files changed, 169 insertions(+), 157 deletions(-) delete mode 100644 cvat-ui/src/containers/header/header.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 47964aca6098..3c2cf34ba1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Datumaro] Dataset statistics () - Ability to change label color in tasks and predefined labels () - [Datumaro] Multi-dataset merge (https://github.com/opencv/cvat/pull/1695) +- Link to django admin page from UI () ### Changed - Shape coordinates are rounded to 2 digits in dumped annotations () diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index f8f63da46401..f5f4b8dc3441 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.8.0", + "version": "1.8.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index d47c1da670be..6c8da4ea27cf 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.8.0", + "version": "1.8.1", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index 50d7ce045ffa..517cf192dc20 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -21,7 +21,7 @@ import ModelsPageContainer from 'containers/models-page/models-page'; import AnnotationPageContainer from 'containers/annotation-page/annotation-page'; import LoginPageContainer from 'containers/login-page/login-page'; import RegisterPageContainer from 'containers/register-page/register-page'; -import HeaderContainer from 'containers/header/header'; +import Header from 'components/header/header'; import { customWaViewHit } from 'utils/enviroment'; import getCore from 'cvat-core-wrapper'; @@ -272,7 +272,7 @@ class CVATApplication extends React.PureComponent - +
diff --git a/cvat-ui/src/components/header/header.tsx b/cvat-ui/src/components/header/header.tsx index 87db4c51296c..379e7c70daa3 100644 --- a/cvat-ui/src/components/header/header.tsx +++ b/cvat-ui/src/components/header/header.tsx @@ -3,9 +3,9 @@ // SPDX-License-Identifier: MIT import './styles.scss'; -import React, { MouseEvent } from 'react'; -import { RouteComponentProps } from 'react-router'; -import { withRouter } from 'react-router-dom'; +import React from 'react'; +import { connect } from 'react-redux'; +import { useHistory } from 'react-router'; import { Row, Col } from 'antd/lib/grid'; import Layout from 'antd/lib/layout'; import Icon from 'antd/lib/icon'; @@ -15,50 +15,129 @@ import Dropdown from 'antd/lib/dropdown'; import Modal from 'antd/lib/modal'; import Text from 'antd/lib/typography/Text'; -import { CVATLogo, AccountIcon } from 'icons'; +import getCore from 'cvat-core-wrapper'; import consts from 'consts'; + +import { CVATLogo, AccountIcon } from 'icons'; import ChangePasswordDialog from 'components/change-password-modal/change-password-modal'; +import { switchSettingsDialog as switchSettingsDialogAction } from 'actions/settings-actions'; +import { logoutAsync, authActions } from 'actions/auth-actions'; +import { SupportedPlugins, CombinedState } from 'reducers/interfaces'; import SettingsModal from './settings-modal/settings-modal'; -interface HeaderContainerProps { - onLogout: () => void; - switchSettingsDialog: (show: boolean) => void; - switchChangePasswordDialog: (show: boolean) => void; - logoutFetching: boolean; - changePasswordFetching: boolean; - installedAnalytics: boolean; - serverHost: string; - username: string; - toolName: string; - serverVersion: string; - serverDescription: string; - coreVersion: string; - canvasVersion: string; - uiVersion: string; +const core = getCore(); + +interface Tool { + name: string; + description: string; + server: { + host: string; + version: string; + }; + core: { + version: string; + }; + canvas: { + version: string; + }; + ui: { + version: string; + }; +} + +interface StateToProps { + user: any; + tool: Tool; switchSettingsShortcut: string; settingsDialogShown: boolean; changePasswordDialogShown: boolean; + changePasswordFetching: boolean; + logoutFetching: boolean; + installedAnalytics: boolean; renderChangePasswordItem: boolean; } -type Props = HeaderContainerProps & RouteComponentProps; +interface DispatchToProps { + onLogout: () => void; + switchSettingsDialog: (show: boolean) => void; + switchChangePasswordDialog: (show: boolean) => void; +} + +function mapStateToProps(state: CombinedState): StateToProps { + const { + auth: { + user, + fetching: logoutFetching, + fetching: changePasswordFetching, + showChangePasswordDialog: changePasswordDialogShown, + allowChangePassword: renderChangePasswordItem, + }, + plugins: { + list, + }, + about: { + server, + packageVersion, + }, + shortcuts: { + normalizedKeyMap, + }, + settings: { + showDialog: settingsDialogShown, + }, + } = state; + + return { + user, + tool: { + name: server.name as string, + description: server.description as string, + server: { + host: core.config.backendAPI.slice(0, -7), + version: server.version as string, + }, + canvas: { + version: packageVersion.canvas, + }, + core: { + version: packageVersion.core, + }, + ui: { + version: packageVersion.ui, + }, + }, + switchSettingsShortcut: normalizedKeyMap.SWITCH_SETTINGS, + settingsDialogShown, + changePasswordDialogShown, + changePasswordFetching, + logoutFetching, + installedAnalytics: list[SupportedPlugins.ANALYTICS], + renderChangePasswordItem, + }; +} + +function mapDispatchToProps(dispatch: any): DispatchToProps { + return { + onLogout: (): void => dispatch(logoutAsync()), + switchSettingsDialog: (show: boolean): void => dispatch(switchSettingsDialogAction(show)), + switchChangePasswordDialog: (show: boolean): void => ( + dispatch(authActions.switchChangePasswordDialog(show)) + ), + }; +} + +type Props = StateToProps & DispatchToProps; function HeaderContainer(props: Props): JSX.Element { const { + user, + tool, installedAnalytics, - username, - toolName, - serverHost, - serverVersion, - serverDescription, - coreVersion, - canvasVersion, - uiVersion, - onLogout, logoutFetching, changePasswordFetching, settingsDialogShown, switchSettingsShortcut, + onLogout, switchSettingsDialog, switchChangePasswordDialog, renderChangePasswordItem, @@ -72,20 +151,22 @@ function HeaderContainer(props: Props): JSX.Element { GITHUB_URL, } = consts; - function aboutModal(): void { + const history = useHistory(); + + function showAboutModal(): void { Modal.info({ - title: `${toolName}`, + title: `${tool.name}`, content: (

- {`${serverDescription}`} + {`${tool.description}`}

Server version: - {` ${serverVersion}`} + {` ${tool.server.version}`}

@@ -93,7 +174,7 @@ function HeaderContainer(props: Props): JSX.Element { Core version: - {` ${coreVersion}`} + {` ${tool.core.version}`}

@@ -101,7 +182,7 @@ function HeaderContainer(props: Props): JSX.Element { Canvas version: - {` ${canvasVersion}`} + {` ${tool.canvas.version}`}

@@ -109,7 +190,7 @@ function HeaderContainer(props: Props): JSX.Element { UI version: - {` ${uiVersion}`} + {` ${tool.ui.version}`}

@@ -131,16 +212,27 @@ function HeaderContainer(props: Props): JSX.Element { const menu = ( + {user.isStaff && ( + { + // false positive + // eslint-disable-next-line + window.open(`${tool.server.host}/admin`, '_blank'); + }} + > + + Admin page + + )} + switchSettingsDialog(true) - } + onClick={() => switchSettingsDialog(true)} > Settings - aboutModal()}> + About @@ -178,7 +270,7 @@ function HeaderContainer(props: Props): JSX.Element { onClick={ (event: React.MouseEvent): void => { event.preventDefault(); - props.history.push('/tasks?page=1'); + history.push('/tasks?page=1'); } } > @@ -192,7 +284,7 @@ function HeaderContainer(props: Props): JSX.Element { onClick={ (event: React.MouseEvent): void => { event.preventDefault(); - props.history.push('/models'); + history.push('/models'); } } > @@ -203,13 +295,13 @@ function HeaderContainer(props: Props): JSX.Element {