Add dataset export facility (#813)

* Add datumaro django application
* Add cvat task datumaro bindings
* Add REST api for task export
* Add scheduler service
* Updated CHANGELOG.md

.travis.yml

@@ -13,4 +13,5 @@ before_script:
 
 script:
   - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli'
+  - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/'
   - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls'

.vscode/launch.json

@@ -71,6 +71,22 @@
             "env": {},
             "console": "internalConsole"
         },
+        {
+            "name": "server: RQ - scheduler",
+            "type": "python",
+            "request": "launch",
+            "stopOnEntry": false,
+            "justMyCode": false,
+            "pythonPath": "${config:python.pythonPath}",
+            "program": "${workspaceRoot}/manage.py",
+            "args": [
+                "rqscheduler",
+            ],
+            "django": true,
+            "cwd": "${workspaceFolder}",
+            "env": {},
+            "console": "internalConsole"
+        },
         {
             "name": "server: RQ - low",
             "type": "python",
@@ -177,6 +193,7 @@
                 "server: django",
                 "server: RQ - default",
                 "server: RQ - low",
+                "server: RQ - scheduler",
                 "server: git",
             ]
         }

CHANGELOG.md

@@ -15,6 +15,8 @@ https://github.com/opencv/cvat/issues/750).
 - Auto segmentation using Mask_RCNN component (Keras+Tensorflow Mask R-CNN Segmentation)
 - Added MOT CSV format support
 - Ability to dump/load annotations in LabelMe format from UI
+- REST API to export an annotation task (images + annotations)
+- Datumaro is an experimental framework to build, analyze, debug and visualize datasets for DL algorithms
 
 ### Changed
 -

Dockerfile

@@ -154,6 +154,10 @@ COPY utils ${HOME}/utils
 COPY cvat/ ${HOME}/cvat
 COPY cvat-core/ ${HOME}/cvat-core
 COPY tests ${HOME}/tests
+COPY datumaro/ ${HOME}/datumaro
+
+RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 pip3 install --no-cache-dir
+
 # Binary option is necessary to correctly apply the patch on Windows platform.
 # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message
 RUN patch --binary -p1 < ${HOME}/cvat/apps/engine/static/engine/js/3rdparty.patch

cvat/apps/dataset_manager/bindings.py

@@ -0,0 +1,176 @@
+from collections import OrderedDict
+import os
+import os.path as osp
+
+from django.db import transaction
+
+from cvat.apps.annotation.annotation import Annotation
+from cvat.apps.engine.annotation import TaskAnnotation
+from cvat.apps.engine.models import Task, ShapeType
+
+import datumaro.components.extractor as datumaro
+from datumaro.util.image import lazy_image
+
+
+class CvatImagesDirExtractor(datumaro.Extractor):
+    _SUPPORTED_FORMATS = ['.png', '.jpg']
+
+    def __init__(self, url):
+        super().__init__()
+
+        items = []
+        for (dirpath, _, filenames) in os.walk(url):
+            for name in filenames:
+                path = osp.join(dirpath, name)
+                if self._is_image(path):
+                    item_id = Task.get_image_frame(path)
+                    item = datumaro.DatasetItem(
+                        id=item_id, image=lazy_image(path))
+                    items.append((item.id, item))
+
+        items = sorted(items, key=lambda e: e[0])
+        items = OrderedDict(items)
+        self._items = items
+
+        self._subsets = None
+
+    def __iter__(self):
+        for item in self._items.values():
+            yield item
+
+    def __len__(self):
+        return len(self._items)
+
+    def subsets(self):
+        return self._subsets
+
+    def get(self, item_id, subset=None, path=None):
+        if path or subset:
+            raise KeyError()
+        return self._items[item_id]
+
+    def _is_image(self, path):
+        for ext in self._SUPPORTED_FORMATS:
+            if osp.isfile(path) and path.endswith(ext):
+                return True
+        return False
+
+
+class CvatTaskExtractor(datumaro.Extractor):
+    def __init__(self, url, db_task, user):
+        self._db_task = db_task
+        self._categories = self._load_categories()
+
+        cvat_annotations = TaskAnnotation(db_task.id, user)
+        with transaction.atomic():
+            cvat_annotations.init_from_db()
+        cvat_annotations = Annotation(cvat_annotations.ir_data, db_task)
+
+        dm_annotations = []
+
+        for cvat_anno in cvat_annotations.group_by_frame():
+            dm_anno = self._read_cvat_anno(cvat_anno)
+            dm_item = datumaro.DatasetItem(
+                id=cvat_anno.frame, annotations=dm_anno)
+            dm_annotations.append((dm_item.id, dm_item))
+
+        dm_annotations = sorted(dm_annotations, key=lambda e: e[0])
+        self._items = OrderedDict(dm_annotations)
+
+        self._subsets = None
+
+    def __iter__(self):
+        for item in self._items.values():
+            yield item
+
+    def __len__(self):
+        return len(self._items)
+
+    def subsets(self):
+        return self._subsets
+
+    def get(self, item_id, subset=None, path=None):
+        if path or subset:
+            raise KeyError()
+        return self._items[item_id]
+
+    def _load_categories(self):
+        categories = {}
+        label_categories = datumaro.LabelCategories()
+
+        db_labels = self._db_task.label_set.all()
+        for db_label in db_labels:
+            db_attributes = db_label.attributespec_set.all()
+            label_categories.add(db_label.name)
+
+            for db_attr in db_attributes:
+                label_categories.attributes.add(db_attr.name)
+
+        categories[datumaro.AnnotationType.label] = label_categories
+
+        return categories
+
+    def categories(self):
+        return self._categories
+
+    def _read_cvat_anno(self, cvat_anno):
+        item_anno = []
+
+        categories = self.categories()
+        label_cat = categories[datumaro.AnnotationType.label]
+
+        label_map = {}
+        label_attrs = {}
+        db_labels = self._db_task.label_set.all()
+        for db_label in db_labels:
+            label_map[db_label.name] = label_cat.find(db_label.name)[0]
+
+            attrs = {}
+            db_attributes = db_label.attributespec_set.all()
+            for db_attr in db_attributes:
+                attrs[db_attr.name] = db_attr.default_value
+            label_attrs[db_label.name] = attrs
+        map_label = lambda label_db_name: label_map[label_db_name]
+
+        for tag_obj in cvat_anno.tags:
+            anno_group = tag_obj.group
+            if isinstance(anno_group, int):
+                anno_group = anno_group
+            anno_label = map_label(tag_obj.label)
+            anno_attr = dict(label_attrs[tag_obj.label])
+            for attr in tag_obj.attributes:
+                anno_attr[attr.name] = attr.value
+
+            anno = datumaro.LabelObject(label=anno_label,
+                attributes=anno_attr, group=anno_group)
+            item_anno.append(anno)
+
+        for shape_obj in cvat_anno.labeled_shapes:
+            anno_group = shape_obj.group
+            if isinstance(anno_group, int):
+                anno_group = anno_group
+            anno_label = map_label(shape_obj.label)
+            anno_attr = dict(label_attrs[shape_obj.label])
+            for attr in shape_obj.attributes:
+                anno_attr[attr.name] = attr.value
+
+            anno_points = shape_obj.points
+            if shape_obj.type == ShapeType.POINTS:
+                anno = datumaro.PointsObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj.type == ShapeType.POLYLINE:
+                anno = datumaro.PolyLineObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj.type == ShapeType.POLYGON:
+                anno = datumaro.PolygonObject(anno_points,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            elif shape_obj.type == ShapeType.RECTANGLE:
+                x0, y0, x1, y1 = anno_points
+                anno = datumaro.BboxObject(x0, y0, x1 - x0, y1 - y0,
+                    label=anno_label, attributes=anno_attr, group=anno_group)
+            else:
+                raise Exception("Unknown shape type '%s'" % (shape_obj.type))
+
+            item_anno.append(anno)
+
+        return item_anno

cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py

@@ -0,0 +1,120 @@
+from collections import OrderedDict
+import getpass
+import json
+import os, os.path as osp
+import requests
+
+from datumaro.components.config import (Config,
+    SchemaBuilder as _SchemaBuilder,
+)
+import datumaro.components.extractor as datumaro
+from datumaro.util.image import lazy_image, load_image
+
+from cvat.utils.cli.core import CLI as CVAT_CLI, CVAT_API_V1
+
+
+CONFIG_SCHEMA = _SchemaBuilder() \
+    .add('task_id', int) \
+    .add('server_host', str) \
+    .add('server_port', int) \
+    .build()
+
+class cvat_rest_api_task_images(datumaro.Extractor):
+    def _image_local_path(self, item_id):
+        task_id = self._config.task_id
+        return osp.join(self._cache_dir,
+            'task_{}_frame_{:06d}.jpg'.format(task_id, item_id))
+
+    def _make_image_loader(self, item_id):
+        return lazy_image(item_id,
+            lambda item_id: self._image_loader(item_id, self))
+
+    def _is_image_cached(self, item_id):
+        return osp.isfile(self._image_local_path(item_id))
+
+    def _download_image(self, item_id):
+        self._connect()
+        os.makedirs(self._cache_dir, exist_ok=True)
+        self._cvat_cli.tasks_frame(task_id=self._config.task_id,
+            frame_ids=[item_id], outdir=self._cache_dir)
+
+    def _connect(self):
+        if self._session is not None:
+            return
+
+        session = None
+        try:
+            print("Enter credentials for '%s:%s':" % \
+                (self._config.server_host, self._config.server_port))
+            username = input('User: ')
+            password = getpass.getpass()
+
+            session = requests.Session()
+            session.auth = (username, password)
+
+            api = CVAT_API_V1(self._config.server_host,
+                self._config.server_port)
+            cli = CVAT_CLI(session, api)
+
+            self._session = session
+            self._cvat_cli = cli
+        except Exception:
+            if session is not None:
+                session.close()
+
+    def __del__(self):
+        if hasattr(self, '_session'):
+            if self._session is not None:
+                self._session.close()
+
+    @staticmethod
+    def _image_loader(item_id, extractor):
+        if not extractor._is_image_cached(item_id):
+            extractor._download_image(item_id)
+        local_path = extractor._image_local_path(item_id)
+        return load_image(local_path)
+
+    def __init__(self, url):
+        super().__init__()
+
+        local_dir = url
+        self._local_dir = local_dir
+        self._cache_dir = osp.join(local_dir, 'images')
+
+        with open(osp.join(url, 'config.json'), 'r') as config_file:
+            config = json.load(config_file)
+            config = Config(config, schema=CONFIG_SCHEMA)
+        self._config = config
+
+        with open(osp.join(url, 'images_meta.json'), 'r') as images_file:
+            images_meta = json.load(images_file)
+            image_list = images_meta['images']
+
+        items = []
+        for entry in image_list:
+            item_id = entry['id']
+            item = datumaro.DatasetItem(
+                id=item_id, image=self._make_image_loader(item_id))
+            items.append((item.id, item))
+
+        items = sorted(items, key=lambda e: e[0])
+        items = OrderedDict(items)
+        self._items = items
+
+        self._cvat_cli = None
+        self._session = None
+
+    def __iter__(self):
+        for item in self._items.values():
+            yield item
+
+    def __len__(self):
+        return len(self._items)
+
+    def subsets(self):
+        return None
+
+    def get(self, item_id, subset=None, path=None):
+        if path or subset:
+            raise KeyError()
+        return self._items[item_id]