Add MOTS png format (#2198)

* Add mots format

* fix upload

* update docs

* update changelog

* Update datumaro dependency

* fix header

* update dm dependency

* Support importing with outside property in mot and mots

* fix track exporting

Co-authored-by: Boris Sekachev <[email protected]>

CHANGELOG.md

@@ -27,16 +27,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -   Ability to prepare meta information manually (<https://github.com/openvinotoolkit/cvat/pull/2217>)
 -   Ability to upload prepared meta information along with a video when creating a task (<https://github.com/openvinotoolkit/cvat/pull/2217>)
 -   Optional chaining plugin for cvat-canvas and cvat-ui (<https://github.com/openvinotoolkit/cvat/pull/2249>)
+-   MOTS png mask format support (<https://github.com/openvinotoolkit/cvat/pull/2198>)
 
 ### Changed
-
 -   UI models (like DEXTR) were redesigned to be more interactive (<https://github.com/opencv/cvat/pull/2054>)
 -   Used Ubuntu:20.04 as a base image for CVAT Dockerfile (<https://github.com/opencv/cvat/pull/2101>)
 -   Right colors of label tags in label mapping when a user runs automatic detection (<https://github.com/openvinotoolkit/cvat/pull/2162>)
 -   Nuclio became an optional component of CVAT (<https://github.com/openvinotoolkit/cvat/pull/2192>)
 -   A key to remove a point from a polyshape [Ctrl => Alt] (<https://github.com/openvinotoolkit/cvat/pull/2204>)
 -   Updated `docker-compose` file version from `2.3` to `3.3`(<https://github.com/openvinotoolkit/cvat/pull/2235>)
 -   Added auto inference of url schema from host in CLI, if provided (<https://github.com/openvinotoolkit/cvat/pull/2240>)
+-   Track frames in skips between annotation is presented in MOT and MOTS formats are marked `outside` (<https://github.com/openvinotoolkit/cvat/pull/2198>)
 
 ### Deprecated
 
@@ -47,7 +48,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -
 
 ### Fixed
-
 -   Fixed multiple errors which arises when polygon is of length 5 or less (<https://github.com/opencv/cvat/pull/2100>)
 -   Fixed task creation from PDF (<https://github.com/opencv/cvat/pull/2141>)
 -   Fixed CVAT format import for frame stepped tasks (<https://github.com/openvinotoolkit/cvat/pull/2151>)
@@ -60,6 +60,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -   Fixed use case when logs could be saved twice or more times #2202 (<https://github.com/openvinotoolkit/cvat/pull/2203>)
 -   Fixed issues from #2112 (<https://github.com/openvinotoolkit/cvat/pull/2217>)
 -   Git application name (renamed to dataset_repo) (<https://github.com/openvinotoolkit/cvat/pull/2243>)
+-   A problem in exporting of tracks, where tracks could be truncated (<https://github.com/openvinotoolkit/cvat/issues/2129>)
+
 
 ### Security
 

cvat/apps/dataset_manager/annotation.py

@@ -729,7 +729,6 @@ def interpolate(shape0, shape1):
         if track.get("interpolated_shapes"):
             return track["interpolated_shapes"]
 
-        # TODO: should be return an iterator?
         shapes = []
         curr_frame = track["shapes"][0]["frame"]
         prev_shape = {}
@@ -747,9 +746,7 @@ def interpolate(shape0, shape1):
             curr_frame = shape["frame"]
             prev_shape = shape
 
-        # TODO: Need to modify a client and a database (append "outside" shapes for polytracks)
-        if not prev_shape["outside"] and (prev_shape["type"] == ShapeType.RECTANGLE
-               or prev_shape["type"] == ShapeType.POINTS or prev_shape["type"] == ShapeType.CUBOID):
+        if not prev_shape["outside"]:
             shape = copy(prev_shape)
             shape["frame"] = end_frame
             shapes.extend(interpolate(prev_shape, shape))

cvat/apps/dataset_manager/formats/README.md

@@ -13,6 +13,7 @@
   - [CVAT](#cvat)
   - [LabelMe](#labelme)
   - [MOT](#mot)
+  - [MOTS](#mots)
   - [COCO](#coco)
   - [PASCAL VOC and mask](#voc)
   - [YOLO](#yolo)
@@ -708,8 +709,8 @@ Downloaded file: a zip archive of the following structure:
 ``` bash
 taskname.zip/
 ├── img1/
-|   ├── imgage1.jpg
-|   └── imgage2.jpg
+|   ├── image1.jpg
+|   └── image2.jpg
 └── gt/
     ├── labels.txt
     └── gt.txt
@@ -742,6 +743,38 @@ taskname.zip/
 
 - supported annotations: Rectangle tracks
 
+### [MOTS PNG](https://www.vision.rwth-aachen.de/page/mots)<a id="mots" />
+
+#### MOTS PNG Dumper
+
+Downloaded file: a zip archive of the following structure:
+
+``` bash
+taskname.zip/
+└── <any_subset_name>/
+    |   images/
+    |   ├── image1.jpg
+    |   └── image2.jpg
+    └── instances/
+        ├── labels.txt
+        ├── image1.png
+        └── image2.png
+
+# labels.txt
+cat
+dog
+person
+...
+```
+
+- supported annotations: Rectangle and Polygon tracks
+
+#### MOTS PNG Loader
+
+Uploaded file: a zip archive of the structure above
+
+- supported annotations: Polygon tracks
+
 ### [LabelMe](http://labelme.csail.mit.edu/Release3.0)<a id="labelme" />
 
 #### LabelMe Dumper

cvat/apps/dataset_manager/formats/mot.py

@@ -79,6 +79,20 @@ def _import(src_file, task_data):
         for track in tracks.values():
             # MOT annotations do not require frames to be ordered
             track.shapes.sort(key=lambda t: t.frame)
+
+            # insert outside=True in skips between the frames track is visible
+            prev_shape_idx = 0
+            prev_shape = track.shapes[0]
+            for shape in track.shapes[1:]:
+                has_skip = task_data.frame_step < shape.frame - prev_shape.frame
+                if has_skip and not prev_shape.outside:
+                    prev_shape = prev_shape._replace(outside=True,
+                            frame=prev_shape.frame + task_data.frame_step)
+                    prev_shape_idx += 1
+                    track.shapes.insert(prev_shape_idx, prev_shape)
+                prev_shape = shape
+                prev_shape_idx += 1
+
             # Append a shape with outside=True to finish the track
             last_shape = track.shapes[-1]
             if last_shape.frame + task_data.frame_step <= \

cvat/apps/dataset_manager/formats/mots.py

@@ -0,0 +1,102 @@
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from tempfile import TemporaryDirectory
+
+from pyunpack import Archive
+
+from cvat.apps.dataset_manager.bindings import (CvatTaskDataExtractor,
+    find_dataset_root, match_dm_item)
+from cvat.apps.dataset_manager.util import make_zip_archive
+from datumaro.components.extractor import AnnotationType, Transform
+from datumaro.components.project import Dataset
+
+from .registry import dm_env, exporter, importer
+
+
+class KeepTracks(Transform):
+    def transform_item(self, item):
+        return item.wrap(annotations=[a for a in item.annotations
+            if 'track_id' in a.attributes])
+
+@exporter(name='MOTS PNG', ext='ZIP', version='1.0')
+def _export(dst_file, task_data, save_images=False):
+    extractor = CvatTaskDataExtractor(task_data, include_images=save_images)
+    envt = dm_env.transforms
+    extractor = extractor.transform(KeepTracks) # can only export tracks
+    extractor = extractor.transform(envt.get('polygons_to_masks'))
+    extractor = extractor.transform(envt.get('boxes_to_masks'))
+    extractor = extractor.transform(envt.get('merge_instance_segments'))
+    extractor = Dataset.from_extractors(extractor) # apply lazy transforms
+    with TemporaryDirectory() as temp_dir:
+        dm_env.converters.get('mots_png').convert(extractor,
+            save_dir=temp_dir, save_images=save_images)
+
+        make_zip_archive(temp_dir, dst_file)
+
+@importer(name='MOTS PNG', ext='ZIP', version='1.0')
+def _import(src_file, task_data):
+    with TemporaryDirectory() as tmp_dir:
+        Archive(src_file.name).extractall(tmp_dir)
+
+        dataset = dm_env.make_importer('mots')(tmp_dir).make_dataset()
+        masks_to_polygons = dm_env.transforms.get('masks_to_polygons')
+        dataset = dataset.transform(masks_to_polygons)
+
+        tracks = {}
+        label_cat = dataset.categories()[AnnotationType.label]
+
+        root_hint = find_dataset_root(dataset, task_data)
+
+        for item in dataset:
+            frame_number = task_data.abs_frame_id(
+                match_dm_item(item, task_data, root_hint=root_hint))
+
+            for ann in item.annotations:
+                if ann.type != AnnotationType.polygon:
+                    continue
+
+                track_id = ann.attributes['track_id']
+                shape = task_data.TrackedShape(
+                    type='polygon',
+                    points=ann.points,
+                    occluded=ann.attributes.get('occluded') == True,
+                    outside=False,
+                    keyframe=True,
+                    z_order=ann.z_order,
+                    frame=frame_number,
+                    attributes=[],
+                    source='manual',
+                )
+
+                # build trajectories as lists of shapes in track dict
+                if track_id not in tracks:
+                    tracks[track_id] = task_data.Track(
+                        label_cat.items[ann.label].name, 0, 'manual', [])
+                tracks[track_id].shapes.append(shape)
+
+        for track in tracks.values():
+            track.shapes.sort(key=lambda t: t.frame)
+
+            # insert outside=True in skips between the frames track is visible
+            prev_shape_idx = 0
+            prev_shape = track.shapes[0]
+            for shape in track.shapes[1:]:
+                has_skip = task_data.frame_step < shape.frame - prev_shape.frame
+                if has_skip and not prev_shape.outside:
+                    prev_shape = prev_shape._replace(outside=True,
+                            frame=prev_shape.frame + task_data.frame_step)
+                    prev_shape_idx += 1
+                    track.shapes.insert(prev_shape_idx, prev_shape)
+                prev_shape = shape
+                prev_shape_idx += 1
+
+            # Append a shape with outside=True to finish the track
+            last_shape = track.shapes[-1]
+            if last_shape.frame + task_data.frame_step <= \
+                    int(task_data.meta['task']['stop_frame']):
+                track.shapes.append(last_shape._replace(outside=True,
+                    frame=last_shape.frame + task_data.frame_step)
+                )
+            task_data.add_track(track)

cvat/apps/dataset_manager/formats/registry.py

@@ -87,6 +87,7 @@ def make_exporter(name):
 import cvat.apps.dataset_manager.formats.labelme
 import cvat.apps.dataset_manager.formats.mask
 import cvat.apps.dataset_manager.formats.mot
+import cvat.apps.dataset_manager.formats.mots
 import cvat.apps.dataset_manager.formats.pascal_voc
 import cvat.apps.dataset_manager.formats.tfrecord
 import cvat.apps.dataset_manager.formats.yolo

cvat/apps/dataset_manager/tests/test_annotation.py

@@ -8,6 +8,17 @@
 
 
 class TrackManagerTest(TestCase):
+    def _check_interpolation(self, track):
+        interpolated = TrackManager.get_interpolated_shapes(track, 0, 7)
+
+        self.assertEqual(len(interpolated), 6)
+        self.assertTrue(interpolated[0]["keyframe"])
+        self.assertFalse(interpolated[1]["keyframe"])
+        self.assertTrue(interpolated[2]["keyframe"])
+        self.assertTrue(interpolated[3]["keyframe"])
+        self.assertFalse(interpolated[4]["keyframe"])
+        self.assertFalse(interpolated[5]["keyframe"])
+
     def test_point_interpolation(self):
         track = {
             "frame": 0,
@@ -32,14 +43,18 @@ def test_point_interpolation(self):
                     "occluded": False,
                     "outside": True
                 },
+                {
+                    "frame": 4,
+                    "attributes": [],
+                    "points": [3.0, 4.0, 5.0, 6.0],
+                    "type": "points",
+                    "occluded": False,
+                    "outside": False
+                },
             ]
         }
 
-        interpolated = TrackManager.get_interpolated_shapes(track, 0, 2)
-
-        self.assertEqual(len(interpolated), 3)
-        self.assertTrue(interpolated[0]["keyframe"])
-        self.assertFalse(interpolated[1]["keyframe"])
+        self._check_interpolation(track)
 
     def test_polygon_interpolation(self):
         track = {
@@ -65,14 +80,18 @@ def test_polygon_interpolation(self):
                     "occluded": False,
                     "outside": True
                 },
+                {
+                    "frame": 4,
+                    "attributes": [],
+                    "points": [3.0, 4.0, 5.0, 6.0, 7.0, 6.0, 4.0, 5.0],
+                    "type": "polygon",
+                    "occluded": False,
+                    "outside": False
+                },
             ]
         }
 
-        interpolated = TrackManager.get_interpolated_shapes(track, 0, 2)
-
-        self.assertEqual(len(interpolated), 3)
-        self.assertTrue(interpolated[0]["keyframe"])
-        self.assertFalse(interpolated[1]["keyframe"])
+        self._check_interpolation(track)
 
     def test_bbox_interpolation(self):
         track = {
@@ -98,14 +117,18 @@ def test_bbox_interpolation(self):
                     "occluded": False,
                     "outside": True
                 },
+                {
+                    "frame": 4,
+                    "attributes": [],
+                    "points": [3.0, 4.0, 5.0, 6.0],
+                    "type": "rectangle",
+                    "occluded": False,
+                    "outside": False
+                },
             ]
         }
 
-        interpolated = TrackManager.get_interpolated_shapes(track, 0, 2)
-
-        self.assertEqual(len(interpolated), 3)
-        self.assertTrue(interpolated[0]["keyframe"])
-        self.assertFalse(interpolated[1]["keyframe"])
+        self._check_interpolation(track)
 
     def test_line_interpolation(self):
         track = {
@@ -131,11 +154,15 @@ def test_line_interpolation(self):
                     "occluded": False,
                     "outside": True
                 },
+                {
+                    "frame": 4,
+                    "attributes": [],
+                    "points": [3.0, 4.0, 5.0, 6.0],
+                    "type": "polyline",
+                    "occluded": False,
+                    "outside": False
+                },
             ]
         }
 
-        interpolated = TrackManager.get_interpolated_shapes(track, 0, 2)
-
-        self.assertEqual(len(interpolated), 3)
-        self.assertTrue(interpolated[0]["keyframe"])
-        self.assertFalse(interpolated[1]["keyframe"])
+        self._check_interpolation(track)

cvat/apps/dataset_manager/tests/test_formats.py

@@ -267,6 +267,7 @@ def test_export_formats_query(self):
             'Datumaro 1.0',
             'LabelMe 3.0',
             'MOT 1.1',
+            'MOTS PNG 1.0',
             'PASCAL VOC 1.1',
             'Segmentation mask 1.1',
             'TFRecord 1.0',
@@ -282,6 +283,7 @@ def test_import_formats_query(self):
             'CVAT 1.1',
             'LabelMe 3.0',
             'MOT 1.1',
+            'MOTS PNG 1.0',
             'PASCAL VOC 1.1',
             'Segmentation mask 1.1',
             'TFRecord 1.0',
@@ -316,6 +318,7 @@ def test_empty_images_are_exported(self):
             ('Datumaro 1.0', 'datumaro_project'),
             ('LabelMe 3.0', 'label_me'),
             # ('MOT 1.1', 'mot_seq'), # does not support
+            # ('MOTS PNG 1.0', 'mots_png'), # does not support
             ('PASCAL VOC 1.1', 'voc'),
             ('Segmentation mask 1.1', 'voc'),
             ('TFRecord 1.0', 'tf_detection_api'),