Merge annotations and dataset_manager apps

.travis.yml

@@ -12,7 +12,8 @@ before_script:
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml build
 
 script:
-  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
-  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test --pattern="_tests.py" cvat/apps/dataset_manager'
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test cvat/apps'
+  # FIXME: Git package and application name conflict in PATH
+  - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test --pattern="_test*.py" cvat/apps/engine/tests cvat/apps/dataset_manager/tests utils/cli'
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'python3 manage.py test datumaro/'
   - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm install && cd ../cvat-core && npm install && npm run test && npm run coveralls'

.vscode/settings.json

@@ -33,5 +33,6 @@
         "./datumaro",
     ],
     "licenser.license": "Custom",
-    "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT"
+    "licenser.customHeader": "Copyright (C) @YEAR@ Intel Corporation\n\nSPDX-License-Identifier: MIT",
+    "files.trimTrailingWhitespace": true
 }

CHANGELOG.md

@@ -6,19 +6,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [1.0.0-beta.2] - Unreleased
 ### Added
+- Added `datumaro_project` export format (https://github.com/opencv/cvat/pull/1352)
 - Re-Identification algorithm to merging bounding boxes automatically to the new UI (https://github.com/opencv/cvat/pull/1406)
 - Methods ``import`` and ``export`` to import/export raw annotations for Job and Task in ``cvat-core`` (https://github.com/opencv/cvat/pull/1406)
 
 ### Changed
--
+- Downloaded file name in annotations export became more informative (https://github.com/opencv/cvat/pull/1352)
+- Added auto trimming for trailing whitespaces style enforsement (https://github.com/opencv/cvat/pull/1352)
+- REST API: updated `GET /task/<id>/annotations`: parameters are `format`, `filename` (now optional), `action` (optional) (https://github.com/opencv/cvat/pull/1352)
+- REST API: removed `dataset/formats`, changed format of `annotation/formats` (https://github.com/opencv/cvat/pull/1352)
+- Exported annotations are stored for N hours insted of indefinitely (https://github.com/opencv/cvat/pull/1352)
+- Formats: CVAT format now accepts ZIP and XML (https://github.com/opencv/cvat/pull/1352)
+- Formats: COCO format now accepts ZIP and JSON (https://github.com/opencv/cvat/pull/1352)
+- Formats: most of formats renamed, no extension in title (https://github.com/opencv/cvat/pull/1352)
+- Formats: definitions are changed, are not stored in DB anymore (https://github.com/opencv/cvat/pull/1352)
 
 ### Deprecated
 -
 
 ### Removed
--
+- `annotation` application is replaced with `dataset_manager` (https://github.com/opencv/cvat/pull/1352)
 
 ### Fixed
+- Categories for empty projects with no sources are taken from own dataset (https://github.com/opencv/cvat/pull/1352)
+- Added directory removal on error during `extract` command (https://github.com/opencv/cvat/pull/1352)
+- Added debug error message on incorrect XPath (https://github.com/opencv/cvat/pull/1352)
+- Exporting frame stepped task (https://github.com/opencv/cvat/issues/1294, https://github.com/opencv/cvat/issues/1334)
 - Auto annotation, TF annotation and Auto segmentation apps (https://github.com/opencv/cvat/pull/1409)
 
 ### Security

CONTRIBUTING.md

@@ -73,6 +73,7 @@ for development
     - [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint)
     - [vscode-remark-lint](https://marketplace.visualstudio.com/items?itemName=drewbourne.vscode-remark-lint)
     - [licenser](https://marketplace.visualstudio.com/items?itemName=ymotongpoo.licenser)
+    - [Trailing Spaces](https://marketplace.visualstudio.com/items?itemName=shardulm94.trailing-spaces)
 
 -   Reload Visual Studio Code from virtual environment
 

README.md

@@ -34,21 +34,22 @@ CVAT is free, online, interactive video and image annotation tool for computer v
 
 ## Supported annotation formats
 
-Format selection is possible after clicking on the Upload annotation / Dump annotation button.
+Format selection is possible after clicking on the Upload annotation and Dump annotation buttons.
 [Datumaro](datumaro/README.md) dataset framework allows additional dataset transformations
-via its command line tool.
+via its command line tool and Python library.
 
-| Annotation format                                                                          | Dumper | Loader |
+| Annotation format                                                                          | Import | Export |
 | ------------------------------------------------------------------------------------------ | ------ | ------ |
-| [CVAT XML v1.1 for images](cvat/apps/documentation/xml_format.md#annotation)               | X      | X      |
-| [CVAT XML v1.1 for a video](cvat/apps/documentation/xml_format.md#interpolation)           | X      | X      |
-| [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/)                                      | X      | X      |
+| [CVAT for images](cvat/apps/documentation/xml_format.md#annotation)                        | X      | X      |
+| [CVAT for a video](cvat/apps/documentation/xml_format.md#interpolation)                    | X      | X      |
+| [Datumaro](datumaro/README.md)                                                             |        | X      |
+| [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)                                      | X      | X      |
+| Segmentation masks from [PASCAL VOC](http://host.robots.ox.ac.uk/pascal/VOC/)              | X      | X      |
 | [YOLO](https://pjreddie.com/darknet/yolo/)                                                 | X      | X      |
 | [MS COCO Object Detection](http://cocodataset.org/#format-data)                            | X      | X      |
-| PNG class mask + instance mask as in [Pascal VOC](http://host.robots.ox.ac.uk/pascal/VOC/) | X      | X      |
 | [TFrecord](https://www.tensorflow.org/tutorials/load_data/tf_records)                      | X      | X      |
 | [MOT](https://motchallenge.net/)                                                           | X      | X      |
-| [LabelMe](http://labelme.csail.mit.edu/Release3.0)                                         | X      | X      |
+| [LabelMe 3.0](http://labelme.csail.mit.edu/Release3.0)                                     | X      | X      |
 
 ## Links
 - [Intel AI blog: New Computer Vision Tool Accelerates Annotation of Digital Images and Video](https://www.intel.ai/introducing-cvat)

cvat-core/src/annotation-format.js

@@ -12,9 +12,8 @@
     class Loader {
         constructor(initialData) {
             const data = {
-                display_name: initialData.display_name,
-                format: initialData.format,
-                handler: initialData.handler,
+                name: initialData.name,
+                format: initialData.ext,
                 version: initialData.version,
             };
 
@@ -27,7 +26,7 @@
                         * @readonly
                         * @instance
                     */
-                    get: () => data.display_name,
+                    get: () => data.name,
                 },
                 format: {
                     /**
@@ -39,16 +38,6 @@
                     */
                     get: () => data.format,
                 },
-                handler: {
-                    /**
-                        * @name handler
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.Loader
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.handler,
-                },
                 version: {
                     /**
                         * @name version
@@ -71,9 +60,8 @@
     class Dumper {
         constructor(initialData) {
             const data = {
-                display_name: initialData.display_name,
-                format: initialData.format,
-                handler: initialData.handler,
+                name: initialData.name,
+                format: initialData.ext,
                 version: initialData.version,
             };
 
@@ -86,7 +74,7 @@
                         * @readonly
                         * @instance
                     */
-                    get: () => data.display_name,
+                    get: () => data.name,
                 },
                 format: {
                     /**
@@ -98,16 +86,6 @@
                     */
                     get: () => data.format,
                 },
-                handler: {
-                    /**
-                        * @name handler
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.Dumper
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.handler,
-                },
                 version: {
                     /**
                         * @name version
@@ -127,108 +105,41 @@
         * @memberof module:API.cvat.classes
         * @hideconstructor
     */
-    class AnnotationFormat {
+    class AnnotationFormats {
         constructor(initialData) {
             const data = {
-                created_date: initialData.created_date,
-                updated_date: initialData.updated_date,
-                id: initialData.id,
-                owner: initialData.owner,
-                name: initialData.name,
-                handler_file: initialData.handler_file,
+                exporters: initialData.exporters.map((el) => new Dumper(el)),
+                importers: initialData.importers.map((el) => new Loader(el)),
             };
 
-            data.dumpers = initialData.dumpers.map((el) => new Dumper(el));
-            data.loaders = initialData.loaders.map((el) => new Loader(el));
-
             // Now all fields are readonly
             Object.defineProperties(this, {
-                id: {
-                    /**
-                        * @name id
-                        * @type {integer}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.id,
-                },
-                owner: {
-                    /**
-                        * @name owner
-                        * @type {integer}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.owner,
-                },
-                name: {
-                    /**
-                        * @name name
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.name,
-                },
-                createdDate: {
-                    /**
-                        * @name createdDate
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.created_date,
-                },
-                updatedDate: {
-                    /**
-                        * @name updatedDate
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.updated_date,
-                },
-                handlerFile: {
-                    /**
-                        * @name handlerFile
-                        * @type {string}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
-                        * @readonly
-                        * @instance
-                    */
-                    get: () => data.handler_file,
-                },
                 loaders: {
                     /**
                         * @name loaders
                         * @type {module:API.cvat.classes.Loader[]}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
+                        * @memberof module:API.cvat.classes.AnnotationFormats
                         * @readonly
                         * @instance
                     */
-                    get: () => [...data.loaders],
+                    get: () => [...data.importers],
                 },
                 dumpers: {
                     /**
                         * @name dumpers
                         * @type {module:API.cvat.classes.Dumper[]}
-                        * @memberof module:API.cvat.classes.AnnotationFormat
+                        * @memberof module:API.cvat.classes.AnnotationFormats
                         * @readonly
                         * @instance
                     */
-                    get: () => [...data.dumpers],
+                    get: () => [...data.exporters],
                 },
             });
         }
     }
 
     module.exports = {
-        AnnotationFormat,
+        AnnotationFormats,
         Loader,
         Dumper,
     };

cvat-core/src/api-implementation.js

@@ -26,7 +26,7 @@
     } = require('./enums');
 
     const User = require('./user');
-    const { AnnotationFormat } = require('./annotation-format.js');
+    const { AnnotationFormats } = require('./annotation-format.js');
     const { ArgumentError } = require('./exceptions');
     const { Task } = require('./session');
 
@@ -66,12 +66,7 @@
 
         cvat.server.formats.implementation = async () => {
             const result = await serverProxy.server.formats();
-            return result.map((el) => new AnnotationFormat(el));
-        };
-
-        cvat.server.datasetFormats.implementation = async () => {
-            const result = await serverProxy.server.datasetFormats();
-            return result;
+            return new AnnotationFormats(result);
         };
 
         cvat.server.register.implementation = async (username, firstName, lastName,

cvat-core/src/api.js

@@ -109,7 +109,7 @@ function build() {
                 * @method formats
                 * @async
                 * @memberof module:API.cvat.server
-                * @returns {module:API.cvat.classes.AnnotationFormat[]}
+                * @returns {module:API.cvat.classes.AnnotationFormats}
                 * @throws {module:API.cvat.exceptions.PluginError}
                 * @throws {module:API.cvat.exceptions.ServerError}
             */
@@ -118,20 +118,6 @@ function build() {
                     .apiWrapper(cvat.server.formats);
                 return result;
             },
-            /**
-                * Method returns available dataset export formats
-                * @method exportFormats
-                * @async
-                * @memberof module:API.cvat.server
-                * @returns {module:String[]}
-                * @throws {module:API.cvat.exceptions.PluginError}
-                * @throws {module:API.cvat.exceptions.ServerError}
-            */
-            async datasetFormats() {
-                const result = await PluginRegistry
-                    .apiWrapper(cvat.server.datasetFormats);
-                return result;
-            },
             /**
                 * Method allows to register on a server
                 * @method register

cvat-core/src/server-proxy.js

@@ -154,22 +154,6 @@
                 return response.data;
             }
 
-            async function datasetFormats() {
-                const { backendAPI } = config;
-
-                let response = null;
-                try {
-                    response = await Axios.get(`${backendAPI}/server/dataset/formats`, {
-                        proxy: config.proxy,
-                    });
-                    response = JSON.parse(response.data);
-                } catch (errorData) {
-                    throw generateError(errorData);
-                }
-
-                return response;
-            }
-
             async function register(username, firstName, lastName, email, password1, password2) {
                 let response = null;
                 try {
@@ -618,8 +602,11 @@
             async function dumpAnnotations(id, name, format) {
                 const { backendAPI } = config;
                 const filename = name.replace(/\//g, '_');
-                const baseURL = `${backendAPI}/tasks/${id}/annotations/${encodeURIComponent(filename)}`;
+                const baseURL = `${backendAPI}/tasks/${id}/annotations`;
                 let query = `format=${encodeURIComponent(format)}`;
+                if (name) {
+                    query += `&filename=${encodeURIComponent(filename)}`;
+                }
                 let url = `${baseURL}?${query}`;
 
                 return new Promise((resolve, reject) => {
@@ -664,7 +651,6 @@
                         about,
                         share,
                         formats,
-                        datasetFormats,
                         exception,
                         login,
                         logout,