diff --git a/.travis.yml b/.travis.yml index f600822d28d2..87631e76006a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,4 +13,5 @@ before_script: script: - docker exec -it cvat /bin/bash -c 'python3 manage.py test cvat/apps utils/cli' + - docker exec -it cvat /bin/bash -c 'python3 manage.py test datumaro/' - docker exec -it cvat /bin/bash -c 'cd cvat-core && npm install && npm run test && npm run coveralls' diff --git a/.vscode/launch.json b/.vscode/launch.json index de8015b16b89..1fefab7fee5b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -71,6 +71,22 @@ "env": {}, "console": "internalConsole" }, + { + "name": "server: RQ - scheduler", + "type": "python", + "request": "launch", + "stopOnEntry": false, + "justMyCode": false, + "pythonPath": "${config:python.pythonPath}", + "program": "${workspaceRoot}/manage.py", + "args": [ + "rqscheduler", + ], + "django": true, + "cwd": "${workspaceFolder}", + "env": {}, + "console": "internalConsole" + }, { "name": "server: RQ - low", "type": "python", @@ -177,6 +193,7 @@ "server: django", "server: RQ - default", "server: RQ - low", + "server: RQ - scheduler", "server: git", ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index b4daaa00ee99..6b5d465adabf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ https://github.com/opencv/cvat/issues/750). - Auto segmentation using Mask_RCNN component (Keras+Tensorflow Mask R-CNN Segmentation) - Added MOT CSV format support - Ability to dump/load annotations in LabelMe format from UI +- REST API to export an annotation task (images + annotations) +- Datumaro is an experimental framework to build, analyze, debug and visualize datasets for DL algorithms ### Changed - diff --git a/Dockerfile b/Dockerfile index 001be81efa06..cb5da2f40539 100644 --- a/Dockerfile +++ b/Dockerfile @@ -154,6 +154,10 @@ COPY utils ${HOME}/utils COPY cvat/ ${HOME}/cvat COPY cvat-core/ ${HOME}/cvat-core COPY tests ${HOME}/tests +COPY datumaro/ ${HOME}/datumaro + +RUN sed -r "s/^(.*)#.*$/\1/g" ${HOME}/datumaro/requirements.txt | xargs -n 1 -L 1 pip3 install --no-cache-dir + # Binary option is necessary to correctly apply the patch on Windows platform. # https://unix.stackexchange.com/questions/239364/how-to-fix-hunk-1-failed-at-1-different-line-endings-message RUN patch --binary -p1 < ${HOME}/cvat/apps/engine/static/engine/js/3rdparty.patch diff --git a/cvat/apps/dataset_manager/__init__.py b/cvat/apps/dataset_manager/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py new file mode 100644 index 000000000000..b6949470afbc --- /dev/null +++ b/cvat/apps/dataset_manager/bindings.py @@ -0,0 +1,176 @@ +from collections import OrderedDict +import os +import os.path as osp + +from django.db import transaction + +from cvat.apps.annotation.annotation import Annotation +from cvat.apps.engine.annotation import TaskAnnotation +from cvat.apps.engine.models import Task, ShapeType + +import datumaro.components.extractor as datumaro +from datumaro.util.image import lazy_image + + +class CvatImagesDirExtractor(datumaro.Extractor): + _SUPPORTED_FORMATS = ['.png', '.jpg'] + + def __init__(self, url): + super().__init__() + + items = [] + for (dirpath, _, filenames) in os.walk(url): + for name in filenames: + path = osp.join(dirpath, name) + if self._is_image(path): + item_id = Task.get_image_frame(path) + item = datumaro.DatasetItem( + id=item_id, image=lazy_image(path)) + items.append((item.id, item)) + + items = sorted(items, key=lambda e: e[0]) + items = OrderedDict(items) + self._items = items + + self._subsets = None + + def __iter__(self): + for item in self._items.values(): + yield item + + def __len__(self): + return len(self._items) + + def subsets(self): + return self._subsets + + def get(self, item_id, subset=None, path=None): + if path or subset: + raise KeyError() + return self._items[item_id] + + def _is_image(self, path): + for ext in self._SUPPORTED_FORMATS: + if osp.isfile(path) and path.endswith(ext): + return True + return False + + +class CvatTaskExtractor(datumaro.Extractor): + def __init__(self, url, db_task, user): + self._db_task = db_task + self._categories = self._load_categories() + + cvat_annotations = TaskAnnotation(db_task.id, user) + with transaction.atomic(): + cvat_annotations.init_from_db() + cvat_annotations = Annotation(cvat_annotations.ir_data, db_task) + + dm_annotations = [] + + for cvat_anno in cvat_annotations.group_by_frame(): + dm_anno = self._read_cvat_anno(cvat_anno) + dm_item = datumaro.DatasetItem( + id=cvat_anno.frame, annotations=dm_anno) + dm_annotations.append((dm_item.id, dm_item)) + + dm_annotations = sorted(dm_annotations, key=lambda e: e[0]) + self._items = OrderedDict(dm_annotations) + + self._subsets = None + + def __iter__(self): + for item in self._items.values(): + yield item + + def __len__(self): + return len(self._items) + + def subsets(self): + return self._subsets + + def get(self, item_id, subset=None, path=None): + if path or subset: + raise KeyError() + return self._items[item_id] + + def _load_categories(self): + categories = {} + label_categories = datumaro.LabelCategories() + + db_labels = self._db_task.label_set.all() + for db_label in db_labels: + db_attributes = db_label.attributespec_set.all() + label_categories.add(db_label.name) + + for db_attr in db_attributes: + label_categories.attributes.add(db_attr.name) + + categories[datumaro.AnnotationType.label] = label_categories + + return categories + + def categories(self): + return self._categories + + def _read_cvat_anno(self, cvat_anno): + item_anno = [] + + categories = self.categories() + label_cat = categories[datumaro.AnnotationType.label] + + label_map = {} + label_attrs = {} + db_labels = self._db_task.label_set.all() + for db_label in db_labels: + label_map[db_label.name] = label_cat.find(db_label.name)[0] + + attrs = {} + db_attributes = db_label.attributespec_set.all() + for db_attr in db_attributes: + attrs[db_attr.name] = db_attr.default_value + label_attrs[db_label.name] = attrs + map_label = lambda label_db_name: label_map[label_db_name] + + for tag_obj in cvat_anno.tags: + anno_group = tag_obj.group + if isinstance(anno_group, int): + anno_group = anno_group + anno_label = map_label(tag_obj.label) + anno_attr = dict(label_attrs[tag_obj.label]) + for attr in tag_obj.attributes: + anno_attr[attr.name] = attr.value + + anno = datumaro.LabelObject(label=anno_label, + attributes=anno_attr, group=anno_group) + item_anno.append(anno) + + for shape_obj in cvat_anno.labeled_shapes: + anno_group = shape_obj.group + if isinstance(anno_group, int): + anno_group = anno_group + anno_label = map_label(shape_obj.label) + anno_attr = dict(label_attrs[shape_obj.label]) + for attr in shape_obj.attributes: + anno_attr[attr.name] = attr.value + + anno_points = shape_obj.points + if shape_obj.type == ShapeType.POINTS: + anno = datumaro.PointsObject(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group) + elif shape_obj.type == ShapeType.POLYLINE: + anno = datumaro.PolyLineObject(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group) + elif shape_obj.type == ShapeType.POLYGON: + anno = datumaro.PolygonObject(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group) + elif shape_obj.type == ShapeType.RECTANGLE: + x0, y0, x1, y1 = anno_points + anno = datumaro.BboxObject(x0, y0, x1 - x0, y1 - y0, + label=anno_label, attributes=anno_attr, group=anno_group) + else: + raise Exception("Unknown shape type '%s'" % (shape_obj.type)) + + item_anno.append(anno) + + return item_anno \ No newline at end of file diff --git a/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py b/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py new file mode 100644 index 000000000000..276caa62f8bf --- /dev/null +++ b/cvat/apps/dataset_manager/export_templates/extractors/cvat_rest_api_task_images.py @@ -0,0 +1,120 @@ +from collections import OrderedDict +import getpass +import json +import os, os.path as osp +import requests + +from datumaro.components.config import (Config, + SchemaBuilder as _SchemaBuilder, +) +import datumaro.components.extractor as datumaro +from datumaro.util.image import lazy_image, load_image + +from cvat.utils.cli.core import CLI as CVAT_CLI, CVAT_API_V1 + + +CONFIG_SCHEMA = _SchemaBuilder() \ + .add('task_id', int) \ + .add('server_host', str) \ + .add('server_port', int) \ + .build() + +class cvat_rest_api_task_images(datumaro.Extractor): + def _image_local_path(self, item_id): + task_id = self._config.task_id + return osp.join(self._cache_dir, + 'task_{}_frame_{:06d}.jpg'.format(task_id, item_id)) + + def _make_image_loader(self, item_id): + return lazy_image(item_id, + lambda item_id: self._image_loader(item_id, self)) + + def _is_image_cached(self, item_id): + return osp.isfile(self._image_local_path(item_id)) + + def _download_image(self, item_id): + self._connect() + os.makedirs(self._cache_dir, exist_ok=True) + self._cvat_cli.tasks_frame(task_id=self._config.task_id, + frame_ids=[item_id], outdir=self._cache_dir) + + def _connect(self): + if self._session is not None: + return + + session = None + try: + print("Enter credentials for '%s:%s':" % \ + (self._config.server_host, self._config.server_port)) + username = input('User: ') + password = getpass.getpass() + + session = requests.Session() + session.auth = (username, password) + + api = CVAT_API_V1(self._config.server_host, + self._config.server_port) + cli = CVAT_CLI(session, api) + + self._session = session + self._cvat_cli = cli + except Exception: + if session is not None: + session.close() + + def __del__(self): + if hasattr(self, '_session'): + if self._session is not None: + self._session.close() + + @staticmethod + def _image_loader(item_id, extractor): + if not extractor._is_image_cached(item_id): + extractor._download_image(item_id) + local_path = extractor._image_local_path(item_id) + return load_image(local_path) + + def __init__(self, url): + super().__init__() + + local_dir = url + self._local_dir = local_dir + self._cache_dir = osp.join(local_dir, 'images') + + with open(osp.join(url, 'config.json'), 'r') as config_file: + config = json.load(config_file) + config = Config(config, schema=CONFIG_SCHEMA) + self._config = config + + with open(osp.join(url, 'images_meta.json'), 'r') as images_file: + images_meta = json.load(images_file) + image_list = images_meta['images'] + + items = [] + for entry in image_list: + item_id = entry['id'] + item = datumaro.DatasetItem( + id=item_id, image=self._make_image_loader(item_id)) + items.append((item.id, item)) + + items = sorted(items, key=lambda e: e[0]) + items = OrderedDict(items) + self._items = items + + self._cvat_cli = None + self._session = None + + def __iter__(self): + for item in self._items.values(): + yield item + + def __len__(self): + return len(self._items) + + def subsets(self): + return None + + def get(self, item_id, subset=None, path=None): + if path or subset: + raise KeyError() + return self._items[item_id] diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py new file mode 100644 index 000000000000..40a480750fb0 --- /dev/null +++ b/cvat/apps/dataset_manager/task.py @@ -0,0 +1,351 @@ +from datetime import timedelta +import json +import os +import os.path as osp +import shutil +import sys +import tempfile +from urllib.parse import urlsplit + +from django.utils import timezone +import django_rq + +from cvat.apps.engine.log import slogger +from cvat.apps.engine.models import Task, ShapeType +from .util import current_function_name, make_zip_archive + +_DATUMARO_REPO_PATH = osp.join(__file__[:__file__.rfind('cvat/')], 'datumaro') +sys.path.append(_DATUMARO_REPO_PATH) +from datumaro.components.project import Project +import datumaro.components.extractor as datumaro +from .bindings import CvatImagesDirExtractor, CvatTaskExtractor + + +_MODULE_NAME = __package__ + '.' + osp.splitext(osp.basename(__file__))[0] +def log_exception(logger=None, exc_info=True): + if logger is None: + logger = slogger + logger.exception("[%s @ %s]: exception occurred" % \ + (_MODULE_NAME, current_function_name(2)), + exc_info=exc_info) + +_TASK_IMAGES_EXTRACTOR = '_cvat_task_images' +_TASK_ANNO_EXTRACTOR = '_cvat_task_anno' +_TASK_IMAGES_REMOTE_EXTRACTOR = 'cvat_rest_api_task_images' + +def get_export_cache_dir(db_task): + return osp.join(db_task.get_task_dirname(), 'export_cache') + +class TaskProject: + @staticmethod + def _get_datumaro_project_dir(db_task): + return osp.join(db_task.get_task_dirname(), 'datumaro') + + @staticmethod + def create(db_task): + task_project = TaskProject(db_task) + task_project._create() + return task_project + + @staticmethod + def load(db_task): + task_project = TaskProject(db_task) + task_project._load() + task_project._init_dataset() + return task_project + + @staticmethod + def from_task(db_task, user): + task_project = TaskProject(db_task) + task_project._import_from_task(user) + return task_project + + def __init__(self, db_task): + self._db_task = db_task + self._project_dir = self._get_datumaro_project_dir(db_task) + self._project = None + self._dataset = None + + def _create(self): + self._project = Project.generate(self._project_dir) + self._project.add_source('task_%s' % self._db_task.id, { + 'url': self._db_task.get_data_dirname(), + 'format': _TASK_IMAGES_EXTRACTOR, + }) + self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR, + CvatImagesDirExtractor) + + self._init_dataset() + self._dataset.define_categories(self._generate_categories()) + + self.save() + + def _load(self): + self._project = Project.load(self._project_dir) + self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR, + CvatImagesDirExtractor) + + def _import_from_task(self, user): + self._project = Project.generate(self._project_dir) + + self._project.add_source('task_%s_images' % self._db_task.id, { + 'url': self._db_task.get_data_dirname(), + 'format': _TASK_IMAGES_EXTRACTOR, + }) + self._project.env.extractors.register(_TASK_IMAGES_EXTRACTOR, + CvatImagesDirExtractor) + + self._project.add_source('task_%s_anno' % self._db_task.id, { + 'format': _TASK_ANNO_EXTRACTOR, + }) + self._project.env.extractors.register(_TASK_ANNO_EXTRACTOR, + lambda url: CvatTaskExtractor(url, + db_task=self._db_task, user=user)) + + self._init_dataset() + + def _init_dataset(self): + self._dataset = self._project.make_dataset() + + def _generate_categories(self): + categories = {} + label_categories = datumaro.LabelCategories() + + db_labels = self._db_task.label_set.all() + for db_label in db_labels: + db_attributes = db_label.attributespec_set.all() + label_categories.add(db_label.name) + + for db_attr in db_attributes: + label_categories.attributes.add(db_attr.name) + + categories[datumaro.AnnotationType.label] = label_categories + + return categories + + def put_annotations(self, annotations): + patch = {} + + categories = self._dataset.categories() + label_cat = categories[datumaro.AnnotationType.label] + + label_map = {} + attr_map = {} + db_labels = self._db_task.label_set.all() + for db_label in db_labels: + label_map[db_label.id] = label_cat.find(db_label.name) + + db_attributes = db_label.attributespec_set.all() + for db_attr in db_attributes: + attr_map[(db_label.id, db_attr.id)] = db_attr.name + map_label = lambda label_db_id: label_map[label_db_id] + map_attr = lambda label_db_id, attr_db_id: \ + attr_map[(label_db_id, attr_db_id)] + + for tag_obj in annotations['tags']: + item_id = str(tag_obj['frame']) + item_anno = patch.get(item_id, []) + + anno_group = tag_obj['group'] + if isinstance(anno_group, int): + anno_group = [anno_group] + anno_label = map_label(tag_obj['label_id']) + anno_attr = {} + for attr in tag_obj['attributes']: + attr_name = map_attr(tag_obj['label_id'], attr['id']) + anno_attr[attr_name] = attr['value'] + + anno = datumaro.LabelObject(label=anno_label, + attributes=anno_attr, group=anno_group) + item_anno.append(anno) + + patch[item_id] = item_anno + + for shape_obj in annotations['shapes']: + item_id = str(shape_obj['frame']) + item_anno = patch.get(item_id, []) + + anno_group = shape_obj['group'] + if isinstance(anno_group, int): + anno_group = [anno_group] + anno_label = map_label(shape_obj['label_id']) + anno_attr = {} + for attr in shape_obj['attributes']: + attr_name = map_attr(shape_obj['label_id'], attr['id']) + anno_attr[attr_name] = attr['value'] + + anno_points = shape_obj['points'] + if shape_obj['type'] == ShapeType.POINTS: + anno = datumaro.PointsObject(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group) + elif shape_obj['type'] == ShapeType.POLYLINE: + anno = datumaro.PolyLineObject(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group) + elif shape_obj['type'] == ShapeType.POLYGON: + anno = datumaro.PolygonObject(anno_points, + label=anno_label, attributes=anno_attr, group=anno_group) + elif shape_obj['type'] == ShapeType.RECTANGLE: + x0, y0, x1, y1 = anno_points + anno = datumaro.BboxObject(x0, y0, x1 - x0, y1 - y0, + label=anno_label, attributes=anno_attr, group=anno_group) + else: + raise Exception("Unknown shape type '%s'" % (shape_obj['type'])) + + item_anno.append(anno) + + patch[item_id] = item_anno + + # TODO: support track annotations + + patch = [datumaro.DatasetItem(id=id_, annotations=anno) \ + for id_, ann in patch.items()] + + self._dataset.update(patch) + + def save(self, save_dir=None, save_images=False): + if self._dataset is not None: + self._dataset.save(save_dir=save_dir, save_images=save_images) + else: + self._project.save(save_dir=save_dir) + + def export(self, dst_format, save_dir, save_images=False, server_url=None): + if self._dataset is None: + self._init_dataset() + if dst_format == DEFAULT_FORMAT: + self._dataset.save(save_dir=save_dir, save_images=save_images) + elif dst_format == DEFAULT_FORMAT_REMOTE: + self._remote_export(save_dir=save_dir, server_url=server_url) + else: + self._dataset.export(output_format=dst_format, + save_dir=save_dir, save_images=save_images) + + def _remote_image_converter(self, save_dir, server_url=None): + os.makedirs(save_dir, exist_ok=True) + + db_task = self._db_task + items = [] + config = { + 'server_host': 'localhost', + 'server_port': '', + 'task_id': db_task.id, + } + if server_url: + parsed_url = urlsplit(server_url) + config['server_host'] = parsed_url.netloc + port = 80 + if parsed_url.port: + port = parsed_url.port + config['server_port'] = int(port) + + images_meta = { + 'images': items, + } + for db_image in self._db_task.image_set.all(): + frame_info = { + 'id': db_image.frame, + 'width': db_image.width, + 'height': db_image.height, + } + items.append(frame_info) + + with open(osp.join(save_dir, 'config.json'), 'w') as config_file: + json.dump(config, config_file) + with open(osp.join(save_dir, 'images_meta.json'), 'w') as images_file: + json.dump(images_meta, images_file) + + def _remote_export(self, save_dir, server_url=None): + if self._dataset is None: + self._init_dataset() + + os.makedirs(save_dir, exist_ok=True) + self._dataset.save(save_dir=save_dir, save_images=False, merge=True) + + exported_project = Project.load(save_dir) + source_name = 'task_%s_images' % self._db_task.id + exported_project.add_source(source_name, { + 'format': _TASK_IMAGES_REMOTE_EXTRACTOR, + }) + self._remote_image_converter( + osp.join(save_dir, exported_project.local_source_dir(source_name)), + server_url=server_url) + exported_project.save() + + templates_dir = osp.join(osp.dirname(__file__), + 'export_templates', 'extractors') + target_dir = osp.join( + exported_project.config.project_dir, + exported_project.config.env_dir, + exported_project.env.config.extractors_dir) + os.makedirs(target_dir, exist_ok=True) + shutil.copyfile( + osp.join(templates_dir, _TASK_IMAGES_REMOTE_EXTRACTOR + '.py'), + osp.join(target_dir, _TASK_IMAGES_REMOTE_EXTRACTOR + '.py')) + + # NOTE: put datumaro component to the archive so that + # it was available to the user + shutil.copytree(_DATUMARO_REPO_PATH, osp.join(save_dir, 'datumaro'), + ignore=lambda src, names: ['__pycache__'] + [ + n for n in names + if sum([int(n.endswith(ext)) for ext in + ['.pyx', '.pyo', '.pyd', '.pyc']]) + ]) + + +DEFAULT_FORMAT = "datumaro_project" +DEFAULT_FORMAT_REMOTE = "datumaro_project_remote" +DEFAULT_CACHE_TTL = timedelta(hours=10) +CACHE_TTL = DEFAULT_CACHE_TTL + +def export_project(task_id, user, dst_format=None, server_url=None): + try: + db_task = Task.objects.get(pk=task_id) + + if not dst_format: + dst_format = DEFAULT_FORMAT + + cache_dir = get_export_cache_dir(db_task) + save_dir = osp.join(cache_dir, dst_format) + archive_path = osp.normpath(save_dir) + '.zip' + + task_time = timezone.localtime(db_task.updated_date).timestamp() + if not (osp.exists(archive_path) and \ + task_time <= osp.getmtime(archive_path)): + os.makedirs(cache_dir, exist_ok=True) + with tempfile.TemporaryDirectory( + dir=cache_dir, prefix=dst_format + '_') as temp_dir: + project = TaskProject.from_task(db_task, user) + project.export(dst_format, save_dir=temp_dir, save_images=True, + server_url=server_url) + + os.makedirs(cache_dir, exist_ok=True) + make_zip_archive(temp_dir, archive_path) + + archive_ctime = osp.getctime(archive_path) + scheduler = django_rq.get_scheduler() + cleaning_job = scheduler.enqueue_in(time_delta=CACHE_TTL, + func=clear_export_cache, + task_id=task_id, + file_path=archive_path, file_ctime=archive_ctime) + slogger.task[task_id].info( + "The task '{}' is exported as '{}' " + "and available for downloading for next '{}'. " + "Export cache cleaning job is enqueued, " + "id '{}', start in '{}'".format( + db_task.name, dst_format, CACHE_TTL, + cleaning_job.id, CACHE_TTL)) + + return archive_path + except Exception: + log_exception(slogger.task[task_id]) + raise + +def clear_export_cache(task_id, file_path, file_ctime): + try: + if osp.exists(file_path) and osp.getctime(file_path) == file_ctime: + os.remove(file_path) + slogger.task[task_id].info( + "Export cache file '{}' successfully removed" \ + .format(file_path)) + except Exception: + log_exception(slogger.task[task_id]) + raise \ No newline at end of file diff --git a/cvat/apps/dataset_manager/util.py b/cvat/apps/dataset_manager/util.py new file mode 100644 index 000000000000..8ad9aabcf05b --- /dev/null +++ b/cvat/apps/dataset_manager/util.py @@ -0,0 +1,15 @@ +import inspect +import os, os.path as osp +import zipfile + + +def current_function_name(depth=1): + return inspect.getouterframes(inspect.currentframe())[depth].function + + +def make_zip_archive(src_path, dst_path): + with zipfile.ZipFile(dst_path, 'w') as archive: + for (dirpath, _, filenames) in os.walk(src_path): + for name in filenames: + path = osp.join(dirpath, name) + archive.write(path, osp.relpath(path, src_path)) diff --git a/cvat/apps/engine/models.py b/cvat/apps/engine/models.py index 079d5b9b05d9..11710f60c7f3 100644 --- a/cvat/apps/engine/models.py +++ b/cvat/apps/engine/models.py @@ -86,6 +86,17 @@ def get_frame_path(self, frame): return path + @staticmethod + def get_image_frame(image_path): + assert image_path.endswith('.jpg') + index = os.path.splitext(os.path.basename(image_path))[0] + + path = os.path.dirname(image_path) + d2 = os.path.basename(path) + d1 = os.path.basename(os.path.dirname(path)) + + return int(d1) * 10000 + int(d2) * 100 + int(index) + def get_frame_step(self): match = re.search("step\s*=\s*([1-9]\d*)", self.frame_filter) return int(match.group(1)) if match else 1 diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index b7c8d621e4c1..691951538d47 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT import os +import os.path as osp import re import traceback from ast import literal_eval @@ -25,6 +26,7 @@ from django_filters import rest_framework as filters import django_rq from django.db import IntegrityError +from django.utils import timezone from . import annotation, task, models @@ -45,6 +47,7 @@ from rest_framework.permissions import SAFE_METHODS from cvat.apps.annotation.models import AnnotationDumper, AnnotationLoader from cvat.apps.annotation.format import get_annotation_formats +import cvat.apps.dataset_manager.task as DatumaroTask # Server REST API @login_required @@ -443,6 +446,66 @@ def frame(self, request, pk, frame): "cannot get frame #{}".format(frame), exc_info=True) return HttpResponseBadRequest(str(e)) + @action(detail=True, methods=['GET'], serializer_class=None, + url_path='export/') + def dataset_export(self, request, pk): + """Export task as a dataset in a specific format""" + + db_task = self.get_object() + + action = request.query_params.get("action", "") + action = action.lower() + if action not in ["", "download"]: + raise serializers.ValidationError( + "Unexpected parameter 'action' specified for the request") + + dst_format = request.query_params.get("format", "") + if not dst_format: + dst_format = DatumaroTask.DEFAULT_FORMAT + dst_format = dst_format.lower() + if 100 < len(dst_format) or not re.fullmatch(r"^[\w_-]+$", dst_format): + raise serializers.ValidationError( + "Unexpected parameter 'format' specified for the request") + + rq_id = "task_dataset_export.{}.{}".format(pk, dst_format) + queue = django_rq.get_queue("default") + + rq_job = queue.fetch_job(rq_id) + if rq_job: + task_time = timezone.localtime(db_task.updated_date) + request_time = timezone.localtime(rq_job.meta.get('request_time', datetime.min)) + if request_time < task_time: + rq_job.cancel() + rq_job.delete() + else: + if rq_job.is_finished: + file_path = rq_job.return_value + if action == "download" and osp.exists(file_path): + rq_job.delete() + + timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S") + filename = "task_{}-{}-{}.zip".format( + db_task.name, timestamp, dst_format) + return sendfile(request, file_path, attachment=True, + attachment_filename=filename.lower()) + else: + if osp.exists(file_path): + return Response(status=status.HTTP_201_CREATED) + elif rq_job.is_failed: + exc_info = str(rq_job.exc_info) + rq_job.delete() + return Response(exc_info, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + else: + return Response(status=status.HTTP_202_ACCEPTED) + + ttl = DatumaroTask.CACHE_TTL.total_seconds() + queue.enqueue_call(func=DatumaroTask.export_project, + args=(pk, request.user, dst_format), job_id=rq_id, + meta={ 'request_time': timezone.localtime() }, + result_ttl=ttl, failure_ttl=ttl) + return Response(status=status.HTTP_201_CREATED) + class JobViewSet(viewsets.GenericViewSet, mixins.RetrieveModelMixin, mixins.UpdateModelMixin): queryset = Job.objects.all().order_by('id') diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index 0e3553c9a676..3c8c5d52617d 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -18,6 +18,7 @@ redis==3.2.0 requests==2.20.0 rjsmin==1.0.12 rq==1.0.0 +rq-scheduler==0.9.1 scipy==1.2.1 sqlparse==0.2.4 django-sendfile==0.3.11 diff --git a/cvat/settings/base.py b/cvat/settings/base.py index e5339d949b11..d6356464f248 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -95,6 +95,7 @@ def generate_ssh_keys(): 'cvat.apps.authentication', 'cvat.apps.documentation', 'cvat.apps.git', + 'cvat.apps.dataset_manager', 'cvat.apps.annotation', 'django_rq', 'compressor', diff --git a/datumaro/README.md b/datumaro/README.md new file mode 100644 index 000000000000..e7b58f54e90b --- /dev/null +++ b/datumaro/README.md @@ -0,0 +1,36 @@ +# Dataset framework + +A framework to prepare, manage, build, analyze datasets + +## Documentation + +-[Quick start guide](docs/quickstart.md) + +## Installation + +Python3.5+ is required. + +To install into a virtual environment do: + +``` bash +python -m pip install virtualenv +python -m virtualenv venv +. venv/bin/activate +pip install -r requirements.txt +``` + +## Execution + +The tool can be executed both as a script and as a module. + +``` bash +PYTHONPATH="..." +python -m datumaro +python path/to/datum.py +``` + +## Testing + +``` bash +python -m unittest discover -s tests +``` diff --git a/datumaro/datum.py b/datumaro/datum.py new file mode 100755 index 000000000000..d6ae4d2c85de --- /dev/null +++ b/datumaro/datum.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +import sys + +from datumaro import main + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/datumaro/datumaro/__init__.py b/datumaro/datumaro/__init__.py new file mode 100644 index 000000000000..48088c40801e --- /dev/null +++ b/datumaro/datumaro/__init__.py @@ -0,0 +1,89 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import logging as log +import sys + +from .cli import ( + project as project_module, + source as source_module, + item as item_module, + model as model_module, + # inference as inference_module, + + create_command as create_command_module, + add_command as add_command_module, + remove_command as remove_command_module, + export_command as export_command_module, + # diff_command as diff_command_module, + # build_command as build_command_module, + stats_command as stats_command_module, + explain_command as explain_command_module, +) +from .components.config import VERSION + + +KNOWN_COMMANDS = { + # contexts + 'project': project_module.main, + 'source': source_module.main, + 'item': item_module.main, + 'model': model_module.main, + # 'inference': inference_module.main, + + # shortcuts + 'create': create_command_module.main, + 'add': add_command_module.main, + 'remove': remove_command_module.main, + 'export': export_command_module.main, + # 'diff': diff_command_module.main, + # 'build': build_command_module.main, + 'stats': stats_command_module.main, + 'explain': explain_command_module.main, +} + +def get_command(name, args=None): + return KNOWN_COMMANDS[name] + +def loglevel(name): + numeric = getattr(log, name.upper(), None) + if not isinstance(numeric, int): + raise ValueError('Invalid log level: %s' % name) + return numeric + +def parse_command(input_args): + parser = argparse.ArgumentParser() + parser.add_argument('command', choices=KNOWN_COMMANDS.keys(), + help='A command to execute') + parser.add_argument('args', nargs=argparse.REMAINDER) + parser.add_argument('--version', action='version', version=VERSION) + parser.add_argument('--loglevel', type=loglevel, default='info', + help="Logging level (default: %(default)s)") + + general_args = parser.parse_args(input_args) + command_name = general_args.command + command_args = general_args.args + return general_args, command_name, command_args + +def set_up_logger(general_args): + loglevel = general_args.loglevel + log.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', + level=loglevel) + +def main(args=None): + if args is None: + args = sys.argv[1:] + + general_args, command_name, command_args = parse_command(args) + + set_up_logger(general_args) + + command = get_command(command_name, general_args) + return command(command_args) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/datumaro/datumaro/__main__.py b/datumaro/datumaro/__main__.py new file mode 100644 index 000000000000..9a055fae8f16 --- /dev/null +++ b/datumaro/datumaro/__main__.py @@ -0,0 +1,12 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import sys +from . import main + + +if __name__ == '__main__': + sys.exit(main()) + diff --git a/datumaro/datumaro/cli/__init__.py b/datumaro/datumaro/cli/__init__.py new file mode 100644 index 000000000000..a9773073830c --- /dev/null +++ b/datumaro/datumaro/cli/__init__.py @@ -0,0 +1,5 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + diff --git a/datumaro/datumaro/cli/add_command.py b/datumaro/datumaro/cli/add_command.py new file mode 100644 index 000000000000..49113084b4e1 --- /dev/null +++ b/datumaro/datumaro/cli/add_command.py @@ -0,0 +1,21 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse + +from . import source as source_module + + +def build_parser(parser=argparse.ArgumentParser()): + source_module.build_add_parser(parser). \ + set_defaults(command=source_module.add_command) + + return parser + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + + return args.command(args) diff --git a/datumaro/datumaro/cli/create_command.py b/datumaro/datumaro/cli/create_command.py new file mode 100644 index 000000000000..eb52458be0e6 --- /dev/null +++ b/datumaro/datumaro/cli/create_command.py @@ -0,0 +1,21 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse + +from . import project as project_module + + +def build_parser(parser=argparse.ArgumentParser()): + project_module.build_create_parser(parser) \ + .set_defaults(command=project_module.create_command) + + return parser + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + + return args.command(args) diff --git a/datumaro/datumaro/cli/explain_command.py b/datumaro/datumaro/cli/explain_command.py new file mode 100644 index 000000000000..195b1026b9af --- /dev/null +++ b/datumaro/datumaro/cli/explain_command.py @@ -0,0 +1,192 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import cv2 +import logging as log +import os +import os.path as osp + +from datumaro.components.project import Project +from datumaro.components.algorithms.rise import RISE +from datumaro.util.command_targets import (TargetKinds, target_selector, + ProjectTarget, SourceTarget, ImageTarget, is_project_path) +from datumaro.util.image import load_image +from .util.project import load_project + + +def build_parser(parser=argparse.ArgumentParser()): + parser.add_argument('-m', '--model', required=True, + help="Model to use for inference") + parser.add_argument('-t', '--target', default=None, + help="Inference target - image, source, project " + "(default: current dir)") + parser.add_argument('-d', '--save-dir', default=None, + help="Directory to save output (default: display only)") + + method_sp = parser.add_subparsers(dest='algorithm') + + rise_parser = method_sp.add_parser('rise') + rise_parser.add_argument('-s', '--max-samples', default=None, type=int, + help="Number of algorithm iterations (default: mask size ^ 2)") + rise_parser.add_argument('--mw', '--mask-width', + dest='mask_width', default=7, type=int, + help="Mask width (default: %(default)s)") + rise_parser.add_argument('--mh', '--mask-height', + dest='mask_height', default=7, type=int, + help="Mask height (default: %(default)s)") + rise_parser.add_argument('--prob', default=0.5, type=float, + help="Mask pixel inclusion probability (default: %(default)s)") + rise_parser.add_argument('--iou', '--iou-thresh', + dest='iou_thresh', default=0.9, type=float, + help="IoU match threshold for detections (default: %(default)s)") + rise_parser.add_argument('--nms', '--nms-iou-thresh', + dest='nms_iou_thresh', default=0.0, type=float, + help="IoU match threshold in Non-maxima suppression (default: no NMS)") + rise_parser.add_argument('--conf', '--det-conf-thresh', + dest='det_conf_thresh', default=0.0, type=float, + help="Confidence threshold for detections (default: do not filter)") + rise_parser.add_argument('-b', '--batch-size', default=1, type=int, + help="Inference batch size (default: %(default)s)") + rise_parser.add_argument('--progressive', action='store_true', + help="Visualize results during computations") + + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + parser.set_defaults(command=explain_command) + + return parser + +def explain_command(args): + from matplotlib import cm + + project = load_project(args.project_dir) + + model = project.make_executable_model(args.model) + + if str(args.algorithm).lower() != 'rise': + raise NotImplementedError() + + rise = RISE(model, + max_samples=args.max_samples, + mask_width=args.mask_width, + mask_height=args.mask_height, + prob=args.prob, + iou_thresh=args.iou_thresh, + nms_thresh=args.nms_iou_thresh, + det_conf_thresh=args.det_conf_thresh, + batch_size=args.batch_size) + + if args.target[0] == TargetKinds.image: + image_path = args.target[1] + image = load_image(image_path) + if model.preferred_input_size() is not None: + h, w = model.preferred_input_size() + image = cv2.resize(image, (w, h)) + + log.info("Running inference explanation for '%s'" % image_path) + heatmap_iter = rise.apply(image, progressive=args.progressive) + + image = image / 255.0 + file_name = osp.splitext(osp.basename(image_path))[0] + if args.progressive: + for i, heatmaps in enumerate(heatmap_iter): + for j, heatmap in enumerate(heatmaps): + hm_painted = cm.jet(heatmap)[:, :, 2::-1] + disp = (image + hm_painted) / 2 + cv2.imshow('heatmap-%s' % j, hm_painted) + cv2.imshow(file_name + '-heatmap-%s' % j, disp) + cv2.waitKey(10) + print("Iter", i, "of", args.max_samples, end='\r') + else: + heatmaps = next(heatmap_iter) + + if args.save_dir is not None: + log.info("Saving inference heatmaps at '%s'" % args.save_dir) + os.makedirs(args.save_dir, exist_ok=True) + + for j, heatmap in enumerate(heatmaps): + save_path = osp.join(args.save_dir, + file_name + '-heatmap-%s.png' % j) + cv2.imwrite(save_path, heatmap * 255.0) + else: + for j, heatmap in enumerate(heatmaps): + disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2 + cv2.imshow(file_name + '-heatmap-%s' % j, disp) + cv2.waitKey(0) + elif args.target[0] == TargetKinds.source or \ + args.target[0] == TargetKinds.project: + if args.target[0] == TargetKinds.source: + source_name = args.target[1] + dataset = project.make_source_project(source_name).make_dataset() + log.info("Running inference explanation for '%s'" % source_name) + else: + project_name = project.config.project_name + dataset = project.make_dataset() + log.info("Running inference explanation for '%s'" % project_name) + + for item in dataset: + image = item.image + if image is None: + log.warn( + "Dataset item %s does not have image data. Skipping." % \ + (item.id)) + continue + + if model.preferred_input_size() is not None: + h, w = model.preferred_input_size() + image = cv2.resize(image, (w, h)) + heatmap_iter = rise.apply(image) + + image = image / 255.0 + file_name = osp.splitext(osp.basename(image_path))[0] + heatmaps = next(heatmap_iter) + + if args.save_dir is not None: + log.info("Saving inference heatmaps at '%s'" % args.save_dir) + os.makedirs(args.save_dir, exist_ok=True) + + for j, heatmap in enumerate(heatmaps): + save_path = osp.join(args.save_dir, + file_name + '-heatmap-%s.png' % j) + cv2.imwrite(save_path, heatmap * 255.0) + + if args.progressive: + for j, heatmap in enumerate(heatmaps): + disp = (image + cm.jet(heatmap)[:, :, 2::-1]) / 2 + cv2.imshow(file_name + '-heatmap-%s' % j, disp) + cv2.waitKey(0) + else: + raise NotImplementedError() + + return 0 + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + if 'command' not in args: + parser.print_help() + return 1 + + project_path = args.project_dir + if is_project_path(project_path): + project = Project.load(project_path) + else: + project = None + try: + args.target = target_selector( + ProjectTarget(is_default=True, project=project), + SourceTarget(project=project), + ImageTarget() + )(args.target) + if args.target[0] == TargetKinds.project: + if is_project_path(args.target[1]): + args.project_dir = osp.dirname(osp.abspath(args.target[1])) + except argparse.ArgumentTypeError as e: + print(e) + parser.print_help() + return 1 + + return args.command(args) diff --git a/datumaro/datumaro/cli/export_command.py b/datumaro/datumaro/cli/export_command.py new file mode 100644 index 000000000000..3bd3efe68d58 --- /dev/null +++ b/datumaro/datumaro/cli/export_command.py @@ -0,0 +1,69 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import os.path as osp + +from datumaro.components.project import Project +from datumaro.util.command_targets import (TargetKinds, target_selector, + ProjectTarget, SourceTarget, ImageTarget, ExternalDatasetTarget, + is_project_path +) + +from . import project as project_module +from . import source as source_module +from . import item as item_module + + +def export_external_dataset(target, params): + raise NotImplementedError() + +def build_parser(parser=argparse.ArgumentParser()): + parser.add_argument('target', nargs='?', default=None) + parser.add_argument('params', nargs=argparse.REMAINDER) + + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + + return parser + +def process_command(target, params, args): + project_dir = args.project_dir + target_kind, target_value = target + if target_kind == TargetKinds.project: + return project_module.main(['export', '-p', target_value] + params) + elif target_kind == TargetKinds.source: + return source_module.main(['export', '-p', project_dir, '-n', target_value] + params) + elif target_kind == TargetKinds.item: + return item_module.main(['export', '-p', project_dir, target_value] + params) + elif target_kind == TargetKinds.external_dataset: + return export_external_dataset(target_value, params) + return 1 + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + + project_path = args.project_dir + if is_project_path(project_path): + project = Project.load(project_path) + else: + project = None + try: + args.target = target_selector( + ProjectTarget(is_default=True, project=project), + SourceTarget(project=project), + ExternalDatasetTarget(), + ImageTarget() + )(args.target) + if args.target[0] == TargetKinds.project: + if is_project_path(args.target[1]): + args.project_dir = osp.dirname(osp.abspath(args.target[1])) + except argparse.ArgumentTypeError as e: + print(e) + parser.print_help() + return 1 + + return process_command(args.target, args.params, args) diff --git a/datumaro/datumaro/cli/inference/__init__.py b/datumaro/datumaro/cli/inference/__init__.py new file mode 100644 index 000000000000..f5d48b7cff38 --- /dev/null +++ b/datumaro/datumaro/cli/inference/__init__.py @@ -0,0 +1,33 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse + + +def run_command(args): + return 0 + +def build_run_parser(parser): + return parser + +def build_parser(parser=argparse.ArgumentParser()): + command_parsers = parser.add_subparsers(dest='command') + + build_run_parser(command_parsers.add_parser('run')). \ + set_defaults(command=run_command) + + return parser + +def process_command(command, args): + return 0 + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + if 'command' not in args: + parser.print_help() + return 1 + + return args.command(args) diff --git a/datumaro/datumaro/cli/item/__init__.py b/datumaro/datumaro/cli/item/__init__.py new file mode 100644 index 000000000000..6082932a1570 --- /dev/null +++ b/datumaro/datumaro/cli/item/__init__.py @@ -0,0 +1,38 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse + + +def build_export_parser(parser): + return parser + +def build_stats_parser(parser): + return parser + +def build_diff_parser(parser): + return parser + +def build_edit_parser(parser): + return parser + +def build_parser(parser=argparse.ArgumentParser()): + command_parsers = parser.add_subparsers(dest='command_name') + + build_export_parser(command_parsers.add_parser('export')) + build_stats_parser(command_parsers.add_parser('stats')) + build_diff_parser(command_parsers.add_parser('diff')) + build_edit_parser(command_parsers.add_parser('edit')) + + return parser + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + if 'command' not in args: + parser.print_help() + return 1 + + return args.command(args) diff --git a/datumaro/datumaro/cli/model/__init__.py b/datumaro/datumaro/cli/model/__init__.py new file mode 100644 index 000000000000..b168248f1c4a --- /dev/null +++ b/datumaro/datumaro/cli/model/__init__.py @@ -0,0 +1,127 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import logging as log +import os +import os.path as osp +import shutil + +from ..util.project import load_project + + +def add_command(args): + project = load_project(args.project_dir) + + log.info("Adding '%s' model to '%s' project" % \ + (args.launcher, project.config.project_name)) + + options = args.launcher_args_extractor(args) + + if args.launcher == 'openvino' and args.copy: + config = project.config + env_config = project.env.config + + model_dir_rel = osp.join( + config.env_dir, env_config.models_dir, args.name) + model_dir = osp.join( + config.project_dir, model_dir_rel) + + os.makedirs(model_dir, exist_ok=True) + + shutil.copy(options.description, + osp.join(model_dir, osp.basename(options.description))) + options.description = \ + osp.join(model_dir_rel, osp.basename(options.description)) + + shutil.copy(options.weights, + osp.join(model_dir, osp.basename(options.weights))) + options.weights = \ + osp.join(model_dir_rel, osp.basename(options.weights)) + + shutil.copy(options.interpretation_script, + osp.join(model_dir, osp.basename(options.interpretation_script))) + options.interpretation_script = \ + osp.join(model_dir_rel, osp.basename(options.interpretation_script)) + + project.add_model(args.name, { + 'launcher': args.launcher, + 'options': vars(options), + }) + + project.save() + + return 0 + +def build_openvino_add_parser(parser): + parser.add_argument('-d', '--description', required=True, + help="Path to the model description file (.xml)") + parser.add_argument('-w', '--weights', required=True, + help="Path to the model weights file (.bin)") + parser.add_argument('-i', '--interpretation-script', required=True, + help="Path to the network output interpretation script (.py)") + parser.add_argument('--plugins-path', default=None, + help="Path to the custom Inference Engine plugins directory") + parser.add_argument('--copy', action='store_true', + help="Copy the model data to the project") + return parser + +def openvino_args_extractor(args): + my_args = argparse.Namespace() + my_args.description = args.description + my_args.weights = args.weights + my_args.interpretation_script = args.interpretation_script + my_args.plugins_path = args.plugins_path + return my_args + +def build_add_parser(parser): + parser.add_argument('name', + help="Name of the model to be added") + launchers_sp = parser.add_subparsers(dest='launcher') + + build_openvino_add_parser(launchers_sp.add_parser('openvino')) \ + .set_defaults(launcher_args_extractor=openvino_args_extractor) + + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + + +def remove_command(args): + project = load_project(args.project_dir) + + project.remove_model(args.name) + project.save() + + return 0 + +def build_remove_parser(parser): + parser.add_argument('name', + help="Name of the model to be removed") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + + return parser + + +def build_parser(parser=argparse.ArgumentParser()): + command_parsers = parser.add_subparsers(dest='command_name') + + build_add_parser(command_parsers.add_parser('add')) \ + .set_defaults(command=add_command) + + build_remove_parser(command_parsers.add_parser('remove')) \ + .set_defaults(command=remove_command) + + return parser + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + if 'command' not in args: + parser.print_help() + return 1 + + return args.command(args) diff --git a/datumaro/datumaro/cli/project/__init__.py b/datumaro/datumaro/cli/project/__init__.py new file mode 100644 index 000000000000..1f4db9786685 --- /dev/null +++ b/datumaro/datumaro/cli/project/__init__.py @@ -0,0 +1,283 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import logging as log +import os +import os.path as osp + +from datumaro.components.project import Project +from datumaro.components.comparator import Comparator +from .diff import DiffVisualizer +from ..util.project import make_project_path, load_project + + +def build_create_parser(parser): + parser.add_argument('-d', '--dest', default='.', dest='dst_dir', + help="Save directory for the new project (default: current dir") + parser.add_argument('-n', '--name', default=None, + help="Name of the new project (default: same as project dir)") + parser.add_argument('--overwrite', action='store_true', + help="Overwrite existing files in the save directory") + return parser + +def create_command(args): + project_dir = osp.abspath(args.dst_dir) + project_path = make_project_path(project_dir) + if not args.overwrite and osp.isfile(project_path): + log.error("Project file '%s' already exists" % (project_path)) + return 1 + + project_name = args.name + if project_name is None: + project_name = osp.basename(project_dir) + + log.info("Creating project at '%s'" % (project_dir)) + + Project.generate(project_dir, { + 'project_name': project_name, + }) + + log.info("Project has been created at '%s'" % (project_dir)) + + return 0 + +def build_import_parser(parser): + import datumaro.components.importers as importers_module + importers_list = [name for name, cls in importers_module.items] + + parser.add_argument('source_path', + help="Path to import a project from") + parser.add_argument('-f', '--format', required=True, + help="Source project format (options: %s)" % (', '.join(importers_list))) + parser.add_argument('-d', '--dest', default='.', dest='dst_dir', + help="Directory to save the new project to (default: current dir)") + parser.add_argument('extra_args', nargs=argparse.REMAINDER, + help="Additional arguments for importer") + parser.add_argument('-n', '--name', default=None, + help="Name of the new project (default: same as project dir)") + parser.add_argument('--overwrite', action='store_true', + help="Overwrite existing files in the save directory") + return parser + +def import_command(args): + project_dir = osp.abspath(args.dst_dir) + project_path = make_project_path(project_dir) + if not args.overwrite and osp.isfile(project_path): + log.error("Project file '%s' already exists" % (project_path)) + return 1 + + project_name = args.name + if project_name is None: + project_name = osp.basename(project_dir) + + log.info("Importing project from '%s' as '%s'" % \ + (args.source_path, args.format)) + + source_path = osp.abspath(args.source_path) + project = Project.import_from(source_path, args.format) + project.config.project_name = project_name + project.config.project_dir = project_dir + project = project.make_dataset() + project.save(merge=True, save_images=False) + + log.info("Project has been created at '%s'" % (project_dir)) + + return 0 + +def build_build_parser(parser): + return parser + +def build_export_parser(parser): + parser.add_argument('-e', '--filter', default=None, + help="Filter expression for dataset items. Examples: " + "extract images with width < height: " + "'/item[image/width < image/height]'; " + "extract images with large-area bboxes: " + "'/item[annotation/type=\"bbox\" and annotation/area>2000]'" + ) + parser.add_argument('-d', '--dest', dest='dst_dir', required=True, + help="Directory to save output") + parser.add_argument('-f', '--output-format', required=True, + help="Output format") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + parser.add_argument('--save-images', action='store_true', + help="Save images") + return parser + +def export_command(args): + project = load_project(args.project_dir) + + dst_dir = osp.abspath(args.dst_dir) + os.makedirs(dst_dir, exist_ok=False) + + project.make_dataset().export( + save_dir=dst_dir, + output_format=args.output_format, + filter_expr=args.filter, + save_images=args.save_images) + log.info("Project exported to '%s' as '%s'" % \ + (dst_dir, args.output_format)) + + return 0 + +def build_stats_parser(parser): + parser.add_argument('name') + return parser + +def build_docs_parser(parser): + return parser + +def build_extract_parser(parser): + parser.add_argument('-e', '--filter', default=None, + help="Filter expression for dataset items. Examples: " + "extract images with width < height: " + "'/item[image/width < image/height]'; " + "extract images with large-area bboxes: " + "'/item[annotation/type=\"bbox\" and annotation/area>2000]'" + ) + parser.add_argument('-d', '--dest', dest='dst_dir', required=True, + help="Output directory") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def extract_command(args): + project = load_project(args.project_dir) + + dst_dir = osp.abspath(args.dst_dir) + os.makedirs(dst_dir, exist_ok=False) + + project.make_dataset().extract(filter_expr=args.filter, save_dir=dst_dir) + log.info("Subproject extracted to '%s'" % (dst_dir)) + + return 0 + +def build_merge_parser(parser): + parser.add_argument('other_project_dir', + help="Directory of the project to get data updates from") + parser.add_argument('-d', '--dest', dest='dst_dir', default=None, + help="Output directory (default: current project's dir)") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def merge_command(args): + first_project = load_project(args.project_dir) + second_project = load_project(args.other_project_dir) + + first_dataset = first_project.make_dataset() + first_dataset.update(second_project.make_dataset()) + + dst_dir = args.dst_dir + first_dataset.save(save_dir=dst_dir) + + if dst_dir is None: + dst_dir = first_project.config.project_dir + dst_dir = osp.abspath(dst_dir) + log.info("Merge result saved to '%s'" % (dst_dir)) + + return 0 + +def build_diff_parser(parser): + parser.add_argument('other_project_dir', + help="Directory of the second project to be compared") + parser.add_argument('-d', '--dest', default=None, dest='dst_dir', + help="Directory to save comparison results (default: do not save)") + parser.add_argument('-f', '--output-format', + default=DiffVisualizer.DEFAULT_FORMAT, + choices=[f.name for f in DiffVisualizer.Format], + help="Output format (default: %(default)s)") + parser.add_argument('--iou-thresh', default=0.5, type=float, + help="IoU match threshold for detections (default: %(default)s)") + parser.add_argument('--conf-thresh', default=0.5, type=float, + help="Confidence threshold for detections (default: %(default)s)") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the first project to be compared (default: current dir)") + return parser + +def diff_command(args): + first_project = load_project(args.project_dir) + second_project = load_project(args.other_project_dir) + + comparator = Comparator( + iou_threshold=args.iou_thresh, + conf_threshold=args.conf_thresh) + + save_dir = args.dst_dir + if save_dir is not None: + log.info("Saving diff to '%s'" % save_dir) + os.makedirs(osp.abspath(save_dir)) + visualizer = DiffVisualizer(save_dir=save_dir, comparator=comparator, + output_format=args.output_format) + visualizer.save_dataset_diff( + first_project.make_dataset(), + second_project.make_dataset()) + + return 0 + +def build_transform_parser(parser): + parser.add_argument('-d', '--dest', dest='dst_dir', required=True, + help="Directory to save output") + parser.add_argument('-m', '--model', dest='model_name', required=True, + help="Model to apply to the project") + parser.add_argument('-f', '--output-format', required=True, + help="Output format") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def transform_command(args): + project = load_project(args.project_dir) + + dst_dir = osp.abspath(args.dst_dir) + os.makedirs(dst_dir, exist_ok=False) + project.make_dataset().transform( + save_dir=dst_dir, + model_name=args.model_name) + + log.info("Transform results saved to '%s'" % (dst_dir)) + + return 0 + + +def build_parser(parser=argparse.ArgumentParser()): + command_parsers = parser.add_subparsers(dest='command_name') + + build_create_parser(command_parsers.add_parser('create')) \ + .set_defaults(command=create_command) + + build_import_parser(command_parsers.add_parser('import')) \ + .set_defaults(command=import_command) + + build_export_parser(command_parsers.add_parser('export')) \ + .set_defaults(command=export_command) + + build_extract_parser(command_parsers.add_parser('extract')) \ + .set_defaults(command=extract_command) + + build_merge_parser(command_parsers.add_parser('merge')) \ + .set_defaults(command=merge_command) + + build_build_parser(command_parsers.add_parser('build')) + build_stats_parser(command_parsers.add_parser('stats')) + build_docs_parser(command_parsers.add_parser('docs')) + build_diff_parser(command_parsers.add_parser('diff')) \ + .set_defaults(command=diff_command) + + build_transform_parser(command_parsers.add_parser('transform')) \ + .set_defaults(command=transform_command) + + return parser + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + if 'command' not in args: + parser.print_help() + return 1 + + return args.command(args) diff --git a/datumaro/datumaro/cli/project/diff.py b/datumaro/datumaro/cli/project/diff.py new file mode 100644 index 000000000000..8fdae82bc21d --- /dev/null +++ b/datumaro/datumaro/cli/project/diff.py @@ -0,0 +1,274 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import Counter +import cv2 +from enum import Enum +import numpy as np +import os +import os.path as osp + +_formats = ['simple'] + +import warnings +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + import tensorboardX as tb + _formats.append('tensorboard') + +from datumaro.components.extractor import AnnotationType + + +Format = Enum('Formats', _formats) + +class DiffVisualizer: + Format = Format + DEFAULT_FORMAT = Format.simple + + _UNMATCHED_LABEL = -1 + + + def __init__(self, comparator, save_dir, output_format=DEFAULT_FORMAT): + self.comparator = comparator + + if isinstance(output_format, str): + output_format = Format[output_format] + assert output_format in Format + self.output_format = output_format + + self.save_dir = save_dir + if output_format is Format.tensorboard: + logdir = osp.join(self.save_dir, 'logs', 'diff') + self.file_writer = tb.SummaryWriter(logdir) + if output_format is Format.simple: + self.label_diff_writer = None + + self.categories = {} + + self.label_confusion_matrix = Counter() + self.bbox_confusion_matrix = Counter() + + def save_dataset_diff(self, extractor_a, extractor_b): + if self.save_dir: + os.makedirs(self.save_dir, exist_ok=True) + + if len(extractor_a) != len(extractor_b): + print("Datasets have different lengths: %s vs %s" % \ + (len(extractor_a), len(extractor_b))) + + self.categories = {} + + label_mismatch = self.comparator. \ + compare_dataset_labels(extractor_a, extractor_b) + if label_mismatch is None: + print("Datasets have no label information") + elif len(label_mismatch) != 0: + print("Datasets have mismatching labels:") + for a_label, b_label in label_mismatch: + if a_label is None: + print(" > %s" % b_label.name) + elif b_label is None: + print(" < %s" % a_label.name) + else: + print(" %s != %s" % (a_label.name, b_label.name)) + else: + self.categories.update(extractor_a.categories()) + self.categories.update(extractor_b.categories()) + + self.label_confusion_matrix = Counter() + self.bbox_confusion_matrix = Counter() + + if self.output_format is Format.tensorboard: + self.file_writer.reopen() + + for i, (item_a, item_b) in enumerate(zip(extractor_a, extractor_b)): + if item_a.id != item_b.id or not item_a.id or not item_b.id: + print("Dataset items #%s '%s' '%s' do not match" % \ + (i + 1, item_a.id, item_b.id)) + continue + + label_diff = self.comparator.compare_item_labels(item_a, item_b) + self.update_label_confusion(label_diff) + + bbox_diff = self.comparator.compare_item_bboxes(item_a, item_b) + self.update_bbox_confusion(bbox_diff) + + self.save_item_label_diff(item_a, item_b, label_diff) + self.save_item_bbox_diff(item_a, item_b, bbox_diff) + + if len(self.label_confusion_matrix) != 0: + self.save_conf_matrix(self.label_confusion_matrix, + 'labels_confusion.png') + if len(self.bbox_confusion_matrix) != 0: + self.save_conf_matrix(self.bbox_confusion_matrix, + 'bbox_confusion.png') + + if self.output_format is Format.tensorboard: + self.file_writer.flush() + self.file_writer.close() + elif self.output_format is Format.simple: + if self.label_diff_writer: + self.label_diff_writer.flush() + self.label_diff_writer.close() + + def update_label_confusion(self, label_diff): + matches, a_unmatched, b_unmatched = label_diff + for label in matches: + self.label_confusion_matrix[(label, label)] += 1 + for a_label in a_unmatched: + self.label_confusion_matrix[(a_label, self._UNMATCHED_LABEL)] += 1 + for b_label in b_unmatched: + self.label_confusion_matrix[(self._UNMATCHED_LABEL, b_label)] += 1 + + def update_bbox_confusion(self, bbox_diff): + matches, mispred, a_unmatched, b_unmatched = bbox_diff + for a_bbox, b_bbox in matches: + self.bbox_confusion_matrix[(a_bbox.label, b_bbox.label)] += 1 + for a_bbox, b_bbox in mispred: + self.bbox_confusion_matrix[(a_bbox.label, b_bbox.label)] += 1 + for a_bbox in a_unmatched: + self.bbox_confusion_matrix[(a_bbox.label, self._UNMATCHED_LABEL)] += 1 + for b_bbox in b_unmatched: + self.bbox_confusion_matrix[(self._UNMATCHED_LABEL, b_bbox.label)] += 1 + + @classmethod + def draw_text_with_background(cls, frame, text, origin, + font=cv2.FONT_HERSHEY_SIMPLEX, scale=1.0, + color=(0, 0, 0), thickness=1, bgcolor=(1, 1, 1)): + text_size, baseline = cv2.getTextSize(text, font, scale, thickness) + cv2.rectangle(frame, + tuple((origin + (0, baseline)).astype(int)), + tuple((origin + (text_size[0], -text_size[1])).astype(int)), + bgcolor, cv2.FILLED) + cv2.putText(frame, text, + tuple(origin.astype(int)), + font, scale, color, thickness) + return text_size, baseline + + def draw_detection_roi(self, frame, x, y, w, h, label, conf, color): + cv2.rectangle(frame, (x, y), (x + w, y + h), color, 2) + + text = '%s %.2f%%' % (label, 100.0 * conf) + text_scale = 0.5 + font = cv2.FONT_HERSHEY_SIMPLEX + text_size = cv2.getTextSize(text, font, text_scale, 1) + line_height = np.array([0, text_size[0][1]]) + self.draw_text_with_background(frame, text, + np.array([x, y]) - line_height * 0.5, + font, scale=text_scale, color=[255 - c for c in color]) + + def get_label(self, label_id): + cat = self.categories.get(AnnotationType.label) + if cat is None: + return str(label_id) + return cat.items[label_id].name + + def draw_bbox(self, img, shape, color): + x, y, w, h = shape.get_bbox() + self.draw_detection_roi(img, int(x), int(y), int(w), int(h), + self.get_label(shape.label), shape.attributes.get('score', 1), + color) + + def get_label_diff_file(self): + if self.label_diff_writer is None: + self.label_diff_writer = \ + open(osp.join(self.save_dir, 'label_diff.txt'), 'w') + return self.label_diff_writer + + def save_item_label_diff(self, item_a, item_b, diff): + _, a_unmatched, b_unmatched = diff + + if 0 < len(a_unmatched) + len(b_unmatched): + if self.output_format is Format.simple: + f = self.get_label_diff_file() + f.write(item_a.id + '\n') + for a_label in a_unmatched: + f.write(' >%s\n' % self.get_label(a_label)) + for b_label in b_unmatched: + f.write(' <%s\n' % self.get_label(b_label)) + elif self.output_format is Format.tensorboard: + tag = item_a.id + for a_label in a_unmatched: + self.file_writer.add_text(tag, + '>%s\n' % self.get_label(a_label)) + for b_label in b_unmatched: + self.file_writer.add_text(tag, + '<%s\n' % self.get_label(b_label)) + + def save_item_bbox_diff(self, item_a, item_b, diff): + _, mispred, a_unmatched, b_unmatched = diff + + if 0 < len(a_unmatched) + len(b_unmatched) + len(mispred): + img_a = item_a.image.copy() + img_b = img_a.copy() + for a_bbox, b_bbox in mispred: + self.draw_bbox(img_a, a_bbox, (0, 255, 0)) + self.draw_bbox(img_b, b_bbox, (0, 0, 255)) + for a_bbox in a_unmatched: + self.draw_bbox(img_a, a_bbox, (255, 255, 0)) + for b_bbox in b_unmatched: + self.draw_bbox(img_b, b_bbox, (255, 255, 0)) + + img = np.hstack([img_a, img_b]) + + path = osp.join(self.save_dir, 'diff_%s' % item_a.id) + + if self.output_format is Format.simple: + cv2.imwrite(path + '.png', img) + elif self.output_format is Format.tensorboard: + self.save_as_tensorboard(img, path) + + def save_as_tensorboard(self, img, name): + img = img[:, :, ::-1] # to RGB + img = np.transpose(img, (2, 0, 1)) # to (C, H, W) + img = img.astype(dtype=np.uint8) + self.file_writer.add_image(name, img) + + def save_conf_matrix(self, conf_matrix, filename): + import matplotlib.pyplot as plt + + classes = None + label_categories = self.categories.get(AnnotationType.label) + if label_categories is not None: + classes = { id: c.name for id, c in enumerate(label_categories.items) } + if classes is None: + classes = { c: 'label_%s' % c for c, _ in conf_matrix } + classes[self._UNMATCHED_LABEL] = 'unmatched' + + class_idx = { id: i for i, id in enumerate(classes.keys()) } + matrix = np.zeros((len(classes), len(classes)), dtype=int) + for idx_pair in conf_matrix: + index = (class_idx[idx_pair[0]], class_idx[idx_pair[1]]) + matrix[index] = conf_matrix[idx_pair] + + labels = [label for id, label in classes.items()] + + fig = plt.figure() + fig.add_subplot(111) + table = plt.table( + cellText=matrix, + colLabels=labels, + rowLabels=labels, + loc ='center') + table.auto_set_font_size(False) + table.set_fontsize(8) + table.scale(3, 3) + # Removing ticks and spines enables you to get the figure only with table + plt.tick_params(axis='x', which='both', bottom=False, top=False, labelbottom=False) + plt.tick_params(axis='y', which='both', right=False, left=False, labelleft=False) + for pos in ['right','top','bottom','left']: + plt.gca().spines[pos].set_visible(False) + + for idx_pair in conf_matrix: + i = class_idx[idx_pair[0]] + j = class_idx[idx_pair[1]] + if conf_matrix[idx_pair] != 0: + if i != j: + table._cells[(i + 1, j)].set_facecolor('#FF0000') + else: + table._cells[(i + 1, j)].set_facecolor('#00FF00') + + plt.savefig(osp.join(self.save_dir, filename), + bbox_inches='tight', pad_inches=0.05) diff --git a/datumaro/datumaro/cli/remove_command.py b/datumaro/datumaro/cli/remove_command.py new file mode 100644 index 000000000000..f419cd3a7e14 --- /dev/null +++ b/datumaro/datumaro/cli/remove_command.py @@ -0,0 +1,21 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse + +from . import source as source_module + + +def build_parser(parser=argparse.ArgumentParser()): + source_module.build_add_parser(parser). \ + set_defaults(command=source_module.remove_command) + + return parser + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + + return args.command(args) diff --git a/datumaro/datumaro/cli/source/__init__.py b/datumaro/datumaro/cli/source/__init__.py new file mode 100644 index 000000000000..cd17e2e89caf --- /dev/null +++ b/datumaro/datumaro/cli/source/__init__.py @@ -0,0 +1,219 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import logging as log +import os +import os.path as osp +import shutil + +from ..util.project import load_project + + +def build_create_parser(parser): + parser.add_argument('-n', '--name', required=True, + help="Name of the source to be created") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def create_command(args): + project = load_project(args.project_dir) + config = project.config + + name = args.name + + if project.env.git.has_submodule(name): + log.fatal("Source '%s' already exists" % (name)) + return 1 + + try: + project.get_source(name) + log.fatal("Source '%s' already exists" % (name)) + return 1 + except KeyError: + pass + + dst_dir = osp.join(config.project_dir, config.sources_dir, name) + project.env.git.init(dst_dir) + + project.add_source(name, { 'url': name }) + project.save() + + log.info("Source '%s' has been added to the project, location: '%s'" \ + % (name, dst_dir)) + + return 0 + +def build_import_parser(parser): + sp = parser.add_subparsers(dest='source_type') + + repo_parser = sp.add_parser('repo') + repo_parser.add_argument('url', + help="URL of the source git repository") + repo_parser.add_argument('-b', '--branch', default='master', + help="Branch of the source repository (default: %(default)s)") + repo_parser.add_argument('--checkout', action='store_true', + help="Do branch checkout") + + dir_parser = sp.add_parser('dir') + dir_parser.add_argument('url', + help="Path to the source directory") + dir_parser.add_argument('--copy', action='store_true', + help="Copy data to the project") + + parser.add_argument('-f', '--format', default=None, + help="Name of the source dataset format (default: 'project')") + parser.add_argument('-n', '--name', default=None, + help="Name of the source to be imported") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def import_command(args): + project = load_project(args.project_dir) + + if args.source_type == 'repo': + name = args.name + if name is None: + name = osp.splitext(osp.basename(args.url))[0] + + if project.env.git.has_submodule(name): + log.fatal("Submodule '%s' already exists" % (name)) + return 1 + + try: + project.get_source(name) + log.fatal("Source '%s' already exists" % (name)) + return 1 + except KeyError: + pass + + dst_dir = project.local_source_dir(name) + project.env.git.create_submodule(name, dst_dir, + url=args.url, branch=args.branch, no_checkout=not args.checkout) + + source = { 'url': args.url } + if args.format: + source['format'] = args.format + project.add_source(name, source) + project.save() + + log.info("Source '%s' has been added to the project, location: '%s'" \ + % (name, dst_dir)) + elif args.source_type == 'dir': + url = osp.abspath(args.url) + if not osp.exists(url): + log.fatal("Source path '%s' does not exist" % url) + return 1 + + name = args.name + if name is None: + name = osp.splitext(osp.basename(url))[0] + + try: + project.get_source(name) + log.fatal("Source '%s' already exists" % (name)) + return 1 + except KeyError: + pass + + dst_dir = url + if args.copy: + dst_dir = project.local_source_dir(name) + log.info("Copying from '%s' to '%s'" % (url, dst_dir)) + shutil.copytree(url, dst_dir) + url = name + + source = { 'url': url } + if args.format: + source['format'] = args.format + project.add_source(name, source) + project.save() + + log.info("Source '%s' has been added to the project, location: '%s'" \ + % (name, dst_dir)) + + return 0 + +def build_remove_parser(parser): + parser.add_argument('-n', '--name', required=True, + help="Name of the source to be removed") + parser.add_argument('--force', action='store_true', + help="Ignore possible errors during removal") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def remove_command(args): + project = load_project(args.project_dir) + + name = args.name + if name is None: + log.fatal("Expected source name") + return + + if project.env.git.has_submodule(name): + if args.force: + log.warning("Forcefully removing the '%s' source..." % (name)) + + project.env.git.remove_submodule(name, force=args.force) + + project.remove_source(name) + project.save() + + log.info("Source '%s' has been removed from the project" % (name)) + + return 0 + +def build_export_parser(parser): + parser.add_argument('-n', '--name', required=True, + help="Source dataset to be extracted") + parser.add_argument('-d', '--dest', dest='dst_dir', required=True, + help="Directory to save output") + parser.add_argument('-f', '--output-format', required=True, + help="Output format (default: %(default)s)") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + return parser + +def export_command(args): + project = load_project(args.project_dir) + + dst_dir = osp.abspath(args.dst_dir) + os.makedirs(dst_dir, exist_ok=False) + + source_project = project.make_source_project(args.name) + source_project.make_dataset().export( + save_dir=args.dst_dir, + output_format=args.output_format) + log.info("Source '%s' exported to '%s' as '%s'" % \ + (args.name, dst_dir, args.output_format)) + + return 0 + +def build_parser(parser=argparse.ArgumentParser()): + command_parsers = parser.add_subparsers(dest='command_name') + + build_create_parser(command_parsers.add_parser('create')) \ + .set_defaults(command=create_command) + build_import_parser(command_parsers.add_parser('import')) \ + .set_defaults(command=import_command) + build_remove_parser(command_parsers.add_parser('remove')) \ + .set_defaults(command=remove_command) + build_export_parser(command_parsers.add_parser('export')) \ + .set_defaults(command=export_command) + + return parser + + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + if 'command' not in args: + parser.print_help() + return 1 + + return args.command(args) diff --git a/datumaro/datumaro/cli/stats_command.py b/datumaro/datumaro/cli/stats_command.py new file mode 100644 index 000000000000..333883dedcef --- /dev/null +++ b/datumaro/datumaro/cli/stats_command.py @@ -0,0 +1,69 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import os.path as osp + +from datumaro.components.project import Project +from datumaro.util.command_targets import (TargetKinds, target_selector, + ProjectTarget, SourceTarget, ExternalDatasetTarget, ImageTarget, + is_project_path +) + +from . import project as project_module +from . import source as source_module +from . import item as item_module + + +def compute_external_dataset_stats(target, params): + raise NotImplementedError() + +def build_parser(parser=argparse.ArgumentParser()): + parser.add_argument('target', nargs='?', default=None) + parser.add_argument('params', nargs=argparse.REMAINDER) + + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to operate on (default: current dir)") + + return parser + +def process_command(target, params, args): + project_dir = args.project_dir + target_kind, target_value = target + if target_kind == TargetKinds.project: + return project_module.main(['stats', '-p', target_value] + params) + elif target_kind == TargetKinds.source: + return source_module.main(['stats', '-p', project_dir, target_value] + params) + elif target_kind == TargetKinds.item: + return item_module.main(['stats', '-p', project_dir, target_value] + params) + elif target_kind == TargetKinds.external_dataset: + return compute_external_dataset_stats(target_value, params) + return 1 + +def main(args=None): + parser = build_parser() + args = parser.parse_args(args) + + project_path = args.project_dir + if is_project_path(project_path): + project = Project.load(project_path) + else: + project = None + try: + args.target = target_selector( + ProjectTarget(is_default=True, project=project), + SourceTarget(project=project), + ExternalDatasetTarget(), + ImageTarget() + )(args.target) + if args.target[0] == TargetKinds.project: + if is_project_path(args.target[1]): + args.project_dir = osp.dirname(osp.abspath(args.target[1])) + except argparse.ArgumentTypeError as e: + print(e) + parser.print_help() + return 1 + + return process_command(args.target, args.params, args) diff --git a/datumaro/datumaro/cli/util/__init__.py b/datumaro/datumaro/cli/util/__init__.py new file mode 100644 index 000000000000..a9773073830c --- /dev/null +++ b/datumaro/datumaro/cli/util/__init__.py @@ -0,0 +1,5 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + diff --git a/datumaro/datumaro/cli/util/project.py b/datumaro/datumaro/cli/util/project.py new file mode 100644 index 000000000000..6e1f5e650bdc --- /dev/null +++ b/datumaro/datumaro/cli/util/project.py @@ -0,0 +1,20 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os.path as osp + +from datumaro.components.project import Project, \ + PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG + + +def make_project_path(project_dir, project_filename=None): + if project_filename is None: + project_filename = DEFAULT_CONFIG.project_filename + return osp.join(project_dir, project_filename) + +def load_project(project_dir, project_filename=None): + if project_filename: + project_dir = osp.join(project_dir, project_filename) + return Project.load(project_dir) \ No newline at end of file diff --git a/datumaro/datumaro/components/__init__.py b/datumaro/datumaro/components/__init__.py new file mode 100644 index 000000000000..a9773073830c --- /dev/null +++ b/datumaro/datumaro/components/__init__.py @@ -0,0 +1,5 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + diff --git a/datumaro/datumaro/components/algorithms/__init__.py b/datumaro/datumaro/components/algorithms/__init__.py new file mode 100644 index 000000000000..a9773073830c --- /dev/null +++ b/datumaro/datumaro/components/algorithms/__init__.py @@ -0,0 +1,5 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + diff --git a/datumaro/datumaro/components/algorithms/rise.py b/datumaro/datumaro/components/algorithms/rise.py new file mode 100644 index 000000000000..e61737d31749 --- /dev/null +++ b/datumaro/datumaro/components/algorithms/rise.py @@ -0,0 +1,219 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +# pylint: disable=unused-variable + +import cv2 +import numpy as np +from math import ceil + +from datumaro.components.extractor import * + + +def flatmatvec(mat): + return np.reshape(mat, (len(mat), -1)) + +def expand(array, axis=None): + if axis is None: + axis = len(array.shape) + return np.expand_dims(array, axis=axis) + +class RISE: + """ + Implements RISE: Randomized Input Sampling for + Explanation of Black-box Models algorithm + See explanations at: https://arxiv.org/pdf/1806.07421.pdf + """ + + def __init__(self, model, + max_samples=None, mask_width=7, mask_height=7, prob=0.5, + iou_thresh=0.9, nms_thresh=0.0, det_conf_thresh=0.0, + batch_size=1): + self.model = model + self.max_samples = max_samples + self.mask_height = mask_height + self.mask_width = mask_width + self.prob = prob + self.iou_thresh = iou_thresh + self.nms_thresh = nms_thresh + self.det_conf_thresh = det_conf_thresh + self.batch_size = batch_size + + @staticmethod + def split_outputs(annotations): + labels = [] + bboxes = [] + for r in annotations: + if r.type is AnnotationType.label: + labels.append(r) + elif r.type is AnnotationType.bbox: + bboxes.append(r) + return labels, bboxes + + @staticmethod + def nms(boxes, iou_thresh=0.5): + indices = np.argsort([b.attributes['score'] for b in boxes]) + ious = np.array([[a.iou(b) for b in boxes] for a in boxes]) + + predictions = [] + while len(indices) != 0: + i = len(indices) - 1 + pred_idx = indices[i] + to_remove = [i] + predictions.append(boxes[pred_idx]) + for i, box_idx in enumerate(indices[:i]): + if iou_thresh < ious[pred_idx, box_idx]: + to_remove.append(i) + indices = np.delete(indices, to_remove) + + return predictions + + def normalize_hmaps(self, heatmaps, counts): + eps = np.finfo(heatmaps.dtype).eps + mhmaps = flatmatvec(heatmaps) + mhmaps /= expand(counts * self.prob + eps) + mhmaps -= expand(np.min(mhmaps, axis=1)) + mhmaps /= expand(np.max(mhmaps, axis=1) + eps) + return np.reshape(mhmaps, heatmaps.shape) + + def apply(self, image, progressive=False): + assert len(image.shape) == 3, \ + "Expected an input image in (H, W, C) format" + assert image.shape[2] in [3, 4], \ + "Expected BGR or BGRA input" + image = image[:, :, :3].astype(np.float32) + + model = self.model + iou_thresh = self.iou_thresh + + image_size = np.array((image.shape[:2])) + mask_size = np.array((self.mask_height, self.mask_width)) + cell_size = np.ceil(image_size / mask_size) + upsampled_size = np.ceil((mask_size + 1) * cell_size) + + rng = lambda shape=None: np.random.rand(*shape) + samples = np.prod(image_size) + if self.max_samples is not None: + samples = min(self.max_samples, samples) + batch_size = self.batch_size + + result = next(iter(model.launch(expand(image, 0)))) + result_labels, result_bboxes = self.split_outputs(result) + if 0 < self.det_conf_thresh: + result_bboxes = [b for b in result_bboxes \ + if self.det_conf_thresh <= b.attributes['score']] + if 0 < self.nms_thresh: + result_bboxes = self.nms(result_bboxes, self.nms_thresh) + + predicted_labels = set() + if len(result_labels) != 0: + predicted_label = max(result_labels, + key=lambda r: r.attributes['score']).label + predicted_labels.add(predicted_label) + if len(result_bboxes) != 0: + for bbox in result_bboxes: + predicted_labels.add(bbox.label) + predicted_labels = { label: idx \ + for idx, label in enumerate(predicted_labels) } + + predicted_bboxes = result_bboxes + + heatmaps_count = len(predicted_labels) + len(predicted_bboxes) + heatmaps = np.zeros((heatmaps_count, *image_size), dtype=np.float32) + total_counts = np.zeros(heatmaps_count, dtype=np.int32) + confs = np.zeros(heatmaps_count, dtype=np.float32) + + heatmap_id = 0 + + label_heatmaps = None + label_total_counts = None + label_confs = None + if len(predicted_labels) != 0: + step = len(predicted_labels) + label_heatmaps = heatmaps[heatmap_id : heatmap_id + step] + label_total_counts = total_counts[heatmap_id : heatmap_id + step] + label_confs = confs[heatmap_id : heatmap_id + step] + heatmap_id += step + + bbox_heatmaps = None + bbox_total_counts = None + bbox_confs = None + if len(predicted_bboxes) != 0: + step = len(predicted_bboxes) + bbox_heatmaps = heatmaps[heatmap_id : heatmap_id + step] + bbox_total_counts = total_counts[heatmap_id : heatmap_id + step] + bbox_confs = confs[heatmap_id : heatmap_id + step] + heatmap_id += step + + ups_mask = np.empty(upsampled_size.astype(int), dtype=np.float32) + masks = np.empty((batch_size, *image_size), dtype=np.float32) + + full_batch_inputs = np.empty((batch_size, *image.shape), dtype=np.float32) + current_heatmaps = np.empty_like(heatmaps) + for b in range(ceil(samples / batch_size)): + batch_pos = b * batch_size + current_batch_size = min(samples - batch_pos, batch_size) + + batch_masks = masks[: current_batch_size] + for i in range(current_batch_size): + mask = (rng(mask_size) < self.prob).astype(np.float32) + cv2.resize(mask, (int(upsampled_size[1]), int(upsampled_size[0])), + ups_mask) + + offsets = np.round(rng((2,)) * cell_size) + mask = ups_mask[ + int(offsets[0]):int(image_size[0] + offsets[0]), + int(offsets[1]):int(image_size[1] + offsets[1]) ] + batch_masks[i] = mask + + batch_inputs = full_batch_inputs[:current_batch_size] + np.multiply(expand(batch_masks), expand(image, 0), out=batch_inputs) + + results = model.launch(batch_inputs) + for mask, result in zip(batch_masks, results): + result_labels, result_bboxes = self.split_outputs(result) + + confs.fill(0) + if len(predicted_labels) != 0: + for r in result_labels: + idx = predicted_labels.get(r.label, None) + if idx is not None: + label_total_counts[idx] += 1 + label_confs[idx] += r.attributes['score'] + for r in result_bboxes: + idx = predicted_labels.get(r.label, None) + if idx is not None: + label_total_counts[idx] += 1 + label_confs[idx] += r.attributes['score'] + + if len(predicted_bboxes) != 0 and len(result_bboxes) != 0: + if 0 < self.det_conf_thresh: + result_bboxes = [b for b in result_bboxes \ + if self.det_conf_thresh <= b.attributes['score']] + if 0 < self.nms_thresh: + result_bboxes = self.nms(result_bboxes, self.nms_thresh) + + for detection in result_bboxes: + for pred_idx, pred in enumerate(predicted_bboxes): + if pred.label != detection.label: + continue + + iou = pred.iou(detection) + assert 0 <= iou and iou <= 1 + if iou < iou_thresh: + continue + + bbox_total_counts[pred_idx] += 1 + + conf = detection.attributes['score'] + bbox_confs[pred_idx] += conf + + np.multiply.outer(confs, mask, out=current_heatmaps) + heatmaps += current_heatmaps + + if progressive: + yield self.normalize_hmaps(heatmaps.copy(), total_counts) + + yield self.normalize_hmaps(heatmaps, total_counts) \ No newline at end of file diff --git a/datumaro/datumaro/components/comparator.py b/datumaro/datumaro/components/comparator.py new file mode 100644 index 000000000000..842a3963a989 --- /dev/null +++ b/datumaro/datumaro/components/comparator.py @@ -0,0 +1,113 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from itertools import zip_longest +import numpy as np + +from datumaro.components.extractor import AnnotationType, LabelCategories + + +class Comparator: + def __init__(self, + iou_threshold=0.5, conf_threshold=0.9): + self.iou_threshold = iou_threshold + self.conf_threshold = conf_threshold + + @staticmethod + def iou(box_a, box_b): + return box_a.iou(box_b) + + # pylint: disable=no-self-use + def compare_dataset_labels(self, extractor_a, extractor_b): + a_label_cat = extractor_a.categories().get(AnnotationType.label) + b_label_cat = extractor_b.categories().get(AnnotationType.label) + if not a_label_cat and not b_label_cat: + return None + if not a_label_cat: + a_label_cat = LabelCategories() + if not b_label_cat: + b_label_cat = LabelCategories() + + mismatches = [] + for a_label, b_label in zip_longest(a_label_cat.items, b_label_cat.items): + if a_label != b_label: + mismatches.append((a_label, b_label)) + return mismatches + # pylint: enable=no-self-use + + def compare_item_labels(self, item_a, item_b): + conf_threshold = self.conf_threshold + + a_labels = set([ann.label for ann in item_a.annotations \ + if ann.type is AnnotationType.label and \ + conf_threshold < ann.attributes.get('score', 1)]) + b_labels = set([ann.label for ann in item_b.annotations \ + if ann.type is AnnotationType.label and \ + conf_threshold < ann.attributes.get('score', 1)]) + + a_unmatched = a_labels - b_labels + b_unmatched = b_labels - a_labels + matches = a_labels & b_labels + + return matches, a_unmatched, b_unmatched + + def compare_item_bboxes(self, item_a, item_b): + iou_threshold = self.iou_threshold + conf_threshold = self.conf_threshold + + a_boxes = [ann for ann in item_a.annotations \ + if ann.type is AnnotationType.bbox and \ + conf_threshold < ann.attributes.get('score', 1)] + b_boxes = [ann for ann in item_b.annotations \ + if ann.type is AnnotationType.bbox and \ + conf_threshold < ann.attributes.get('score', 1)] + a_boxes.sort(key=lambda ann: 1 - ann.attributes.get('score', 1)) + b_boxes.sort(key=lambda ann: 1 - ann.attributes.get('score', 1)) + + # a_matches: indices of b_boxes matched to a bboxes + # b_matches: indices of a_boxes matched to b bboxes + a_matches = -np.ones(len(a_boxes), dtype=int) + b_matches = -np.ones(len(b_boxes), dtype=int) + + iou_matrix = np.array([ + [self.iou(a, b) for b in b_boxes] for a in a_boxes + ]) + + # matches: boxes we succeeded to match completely + # mispred: boxes we succeeded to match, having label mismatch + matches = [] + mispred = [] + + for a_idx, a_bbox in enumerate(a_boxes): + if len(b_boxes) == 0: + break + matched_b = a_matches[a_idx] + iou_max = max(iou_matrix[a_idx, matched_b], iou_threshold) + for b_idx, b_bbox in enumerate(b_boxes): + if 0 <= b_matches[b_idx]: # assign a_bbox with max conf + continue + iou = iou_matrix[a_idx, b_idx] + if iou < iou_max: + continue + iou_max = iou + matched_b = b_idx + + if matched_b < 0: + continue + a_matches[a_idx] = matched_b + b_matches[matched_b] = a_idx + + b_bbox = b_boxes[matched_b] + + if a_bbox.label == b_bbox.label: + matches.append( (a_bbox, b_bbox) ) + else: + mispred.append( (a_bbox, b_bbox) ) + + # *_umatched: boxes of (*) we failed to match + a_unmatched = [a_boxes[i] for i, m in enumerate(a_matches) if m < 0] + b_unmatched = [b_boxes[i] for i, m in enumerate(b_matches) if m < 0] + + return matches, mispred, a_unmatched, b_unmatched diff --git a/datumaro/datumaro/components/config.py b/datumaro/datumaro/components/config.py new file mode 100644 index 000000000000..330d01f5bf2e --- /dev/null +++ b/datumaro/datumaro/components/config.py @@ -0,0 +1,238 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import yaml + + +class Schema: + class Item: + def __init__(self, ctor, internal=False): + self.ctor = ctor + self.internal = internal + + def __call__(self, *args, **kwargs): + return self.ctor(*args, **kwargs) + + def __init__(self, items=None, fallback=None): + self._items = {} + if items is not None: + self._items.update(items) + self._fallback = fallback + + def _get_items(self, allow_fallback=True): + all_items = {} + + if allow_fallback and self._fallback is not None: + all_items.update(self._fallback) + all_items.update(self._items) + + return all_items + + def items(self, allow_fallback=True): + return self._get_items(allow_fallback=allow_fallback).items() + + def keys(self, allow_fallback=True): + return self._get_items(allow_fallback=allow_fallback).keys() + + def values(self, allow_fallback=True): + return self._get_items(allow_fallback=allow_fallback).values() + + def __contains__(self, key): + return key in self.keys() + + def __len__(self): + return len(self._get_items()) + + def __iter__(self): + return iter(self._get_items()) + + def __getitem__(self, key): + default = object() + value = self.get(key, default=default) + if value is default: + raise KeyError('Key "%s" does not exist' % (key)) + return value + + def get(self, key, default=None): + found = self._items.get(key, default) + if found is not default: + return found + + if self._fallback is not None: + return self._fallback.get(key, default) + +class SchemaBuilder: + def __init__(self): + self._items = {} + + def add(self, name, ctor=str, internal=False): + if name in self._items: + raise KeyError('Key "%s" already exists' % (name)) + + self._items[name] = Schema.Item(ctor, internal=internal) + return self + + def build(self): + return Schema(self._items) + +class Config: + def __init__(self, config=None, fallback=None, schema=None, mutable=True): + # schema should be established first + self.__dict__['_schema'] = schema + self.__dict__['_mutable'] = True + + self.__dict__['_config'] = {} + if fallback is not None: + for k, v in fallback.items(allow_fallback=False): + self.set(k, v) + if config is not None: + self.update(config) + + self.__dict__['_mutable'] = mutable + + def _items(self, allow_fallback=True, allow_internal=True): + all_config = {} + if allow_fallback and self._schema is not None: + for key, item in self._schema.items(): + all_config[key] = item() + all_config.update(self._config) + + if not allow_internal and self._schema is not None: + for key, item in self._schema.items(): + if item.internal: + all_config.pop(key) + return all_config + + def items(self, allow_fallback=True, allow_internal=True): + return self._items( + allow_fallback=allow_fallback, + allow_internal=allow_internal + ).items() + + def keys(self, allow_fallback=True, allow_internal=True): + return self._items( + allow_fallback=allow_fallback, + allow_internal=allow_internal + ).keys() + + def values(self, allow_fallback=True, allow_internal=True): + return self._items( + allow_fallback=allow_fallback, + allow_internal=allow_internal + ).values() + + def __contains__(self, key): + return key in self.keys() + + def __len__(self): + return len(self.items()) + + def __iter__(self): + return iter(zip(self.keys(), self.values())) + + def __getitem__(self, key): + default = object() + value = self.get(key, default=default) + if value is default: + raise KeyError('Key "%s" does not exist' % (key)) + return value + + def __setitem__(self, key, value): + return self.set(key, value) + + def __getattr__(self, key): + return self.get(key) + + def __setattr__(self, key, value): + return self.set(key, value) + + def __eq__(self, other): + try: + for k, my_v in self.items(allow_internal=False): + other_v = other[k] + if my_v != other_v: + return False + return True + except Exception: + return False + + def update(self, other): + for k, v in other.items(): + self.set(k, v) + + def remove(self, key): + if not self._mutable: + raise Exception("Cannot set value of immutable object") + + self._config.pop(key, None) + + def get(self, key, default=None): + found = self._config.get(key, default) + if found is not default: + return found + + if self._schema is not None: + found = self._schema.get(key, default) + if found is not default: + # ignore mutability + found = found() + self._config[key] = found + return found + + return found + + def set(self, key, value): + if not self._mutable: + raise Exception("Cannot set value of immutable object") + + if self._schema is not None: + if key not in self._schema: + raise Exception("Can not set key '%s' - schema mismatch" % (key)) + + schema_entry = self._schema[key] + schema_entry_instance = schema_entry() + if not isinstance(value, type(schema_entry_instance)): + if isinstance(value, dict) and \ + isinstance(schema_entry_instance, Config): + schema_entry_instance.update(value) + value = schema_entry_instance + else: + raise Exception("Can not set key '%s' - schema mismatch" % (key)) + + self._config[key] = value + return value + + @staticmethod + def parse(path): + with open(path, 'r') as f: + return Config(yaml.safe_load(f)) + + @staticmethod + def yaml_representer(dumper, value): + return dumper.represent_data( + value._items(allow_internal=False, allow_fallback=False)) + + def dump(self, path): + with open(path, 'w+') as f: + yaml.dump(self, f) + +yaml.add_multi_representer(Config, Config.yaml_representer) + + +class DefaultConfig(Config): + def __init__(self, default=None): + super().__init__() + self.__dict__['_default'] = default + + def set(self, key, value): + if key not in self.keys(allow_fallback=False): + value = self._default(value) + return super().set(key, value) + else: + return super().set(key, value) + + +VERSION = '0.1.0' +DEFAULT_FORMAT = 'datumaro' \ No newline at end of file diff --git a/datumaro/datumaro/components/config_model.py b/datumaro/datumaro/components/config_model.py new file mode 100644 index 000000000000..d02ed41b8af0 --- /dev/null +++ b/datumaro/datumaro/components/config_model.py @@ -0,0 +1,83 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from datumaro.components.config import Config, \ + DefaultConfig as _DefaultConfig, \ + SchemaBuilder as _SchemaBuilder + + +SOURCE_SCHEMA = _SchemaBuilder() \ + .add('url', str) \ + .add('format', str) \ + .add('options', str) \ + .build() + +class Source(Config): + def __init__(self, config=None): + super().__init__(config, schema=SOURCE_SCHEMA) + + +MODEL_SCHEMA = _SchemaBuilder() \ + .add('launcher', str) \ + .add('model_dir', str, internal=True) \ + .add('options', dict) \ + .build() + +class Model(Config): + def __init__(self, config=None): + super().__init__(config, schema=MODEL_SCHEMA) + + +ENV_SCHEMA = _SchemaBuilder() \ + .add('models_dir', str) \ + .add('importers_dir', str) \ + .add('launchers_dir', str) \ + .add('converters_dir', str) \ + .add('extractors_dir', str) \ + \ + .add('models', lambda: _DefaultConfig( + lambda v=None: Model(v))) \ + .build() + +ENV_DEFAULT_CONFIG = Config({ + 'models_dir': 'models', + 'importers_dir': 'importers', + 'launchers_dir': 'launchers', + 'converters_dir': 'converters', + 'extractors_dir': 'extractors', +}, mutable=False, schema=ENV_SCHEMA) + + +PROJECT_SCHEMA = _SchemaBuilder() \ + .add('project_name', str) \ + .add('format_version', int) \ + \ + .add('sources_dir', str) \ + .add('dataset_dir', str) \ + .add('build_dir', str) \ + .add('subsets', list) \ + .add('sources', lambda: _DefaultConfig( + lambda v=None: Source(v))) \ + .add('filter', str) \ + \ + .add('project_filename', str, internal=True) \ + .add('project_dir', str, internal=True) \ + .add('env_filename', str, internal=True) \ + .add('env_dir', str, internal=True) \ + .build() + +PROJECT_DEFAULT_CONFIG = Config({ + 'project_name': 'undefined', + 'format_version': 1, + + 'sources_dir': 'sources', + 'dataset_dir': 'dataset', + 'build_dir': 'build', + + 'project_filename': 'config.yaml', + 'project_dir': '', + 'env_filename': 'datumaro.yaml', + 'env_dir': '.datumaro', +}, mutable=False, schema=PROJECT_SCHEMA) diff --git a/datumaro/datumaro/components/converter.py b/datumaro/datumaro/components/converter.py new file mode 100644 index 000000000000..7d2aece0e349 --- /dev/null +++ b/datumaro/datumaro/components/converter.py @@ -0,0 +1,8 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +class Converter: + def __call__(self, extractor, save_dir): + raise NotImplementedError() diff --git a/datumaro/datumaro/components/converters/__init__.py b/datumaro/datumaro/components/converters/__init__.py new file mode 100644 index 000000000000..8b7ab56db90b --- /dev/null +++ b/datumaro/datumaro/components/converters/__init__.py @@ -0,0 +1,43 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from datumaro.components.converters.datumaro import DatumaroConverter + +from datumaro.components.converters.ms_coco import ( + CocoConverter, + CocoImageInfoConverter, + CocoCaptionsConverter, + CocoInstancesConverter, + CocoPersonKeypointsConverter, + CocoLabelsConverter, +) + +from datumaro.components.converters.voc import ( + VocConverter, + VocClassificationConverter, + VocDetectionConverter, + VocLayoutConverter, + VocActionConverter, + VocSegmentationConverter, +) + + +items = [ + ('datumaro', DatumaroConverter), + + ('coco', CocoConverter), + ('coco_images', CocoImageInfoConverter), + ('coco_captions', CocoCaptionsConverter), + ('coco_instances', CocoInstancesConverter), + ('coco_person_kp', CocoPersonKeypointsConverter), + ('coco_labels', CocoLabelsConverter), + + ('voc', VocConverter), + ('voc_cls', VocClassificationConverter), + ('voc_det', VocDetectionConverter), + ('voc_segm', VocSegmentationConverter), + ('voc_action', VocActionConverter), + ('voc_layout', VocLayoutConverter), +] diff --git a/datumaro/datumaro/components/converters/datumaro.py b/datumaro/datumaro/components/converters/datumaro.py new file mode 100644 index 000000000000..9e17f82bd368 --- /dev/null +++ b/datumaro/datumaro/components/converters/datumaro.py @@ -0,0 +1,294 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +# pylint: disable=no-self-use + +import cv2 +import json +import os +import os.path as osp + +from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, + AnnotationType, Annotation, + LabelObject, MaskObject, PointsObject, PolygonObject, + PolyLineObject, BboxObject, CaptionObject, + LabelCategories, MaskCategories, PointsCategories +) +from datumaro.components.formats.datumaro import DatumaroPath +from datumaro.util.mask_tools import apply_colormap + + +def _cast(value, type_conv, default=None): + if value is None: + return default + try: + return type_conv(value) + except Exception: + return default + +class _SubsetWriter: + def __init__(self, name, converter): + self._name = name + self._converter = converter + + self._data = { + 'info': {}, + 'categories': {}, + 'items': [], + } + + self._next_mask_id = 1 + + @property + def categories(self): + return self._data['categories'] + + @property + def items(self): + return self._data['items'] + + def write_item(self, item): + annotations = [] + self.items.append({ + 'id': item.id, + 'path': item.path, + 'annotations': annotations, + }) + + for ann in item.annotations: + if isinstance(ann, LabelObject): + converted_ann = self._convert_label_object(ann) + elif isinstance(ann, MaskObject): + converted_ann = self._convert_mask_object(ann) + elif isinstance(ann, PointsObject): + converted_ann = self._convert_points_object(ann) + elif isinstance(ann, PolyLineObject): + converted_ann = self._convert_polyline_object(ann) + elif isinstance(ann, PolygonObject): + converted_ann = self._convert_polygon_object(ann) + elif isinstance(ann, BboxObject): + converted_ann = self._convert_bbox_object(ann) + elif isinstance(ann, CaptionObject): + converted_ann = self._convert_caption_object(ann) + else: + raise NotImplementedError() + annotations.append(converted_ann) + + def write_categories(self, categories): + for ann_type, desc in categories.items(): + if isinstance(desc, LabelCategories): + converted_desc = self._convert_label_categories(desc) + elif isinstance(desc, MaskCategories): + converted_desc = self._convert_mask_categories(desc) + elif isinstance(desc, PointsCategories): + converted_desc = self._convert_points_categories(desc) + else: + raise NotImplementedError() + self.categories[ann_type.name] = converted_desc + + def write(self, save_dir): + with open(osp.join(save_dir, '%s.json' % (self._name)), 'w') as f: + json.dump(self._data, f) + + def _convert_annotation(self, obj): + assert isinstance(obj, Annotation) + + ann_json = { + 'id': _cast(obj.id, int), + 'type': _cast(obj.type.name, str), + 'attributes': obj.attributes, + 'group': _cast(obj.group, int, None), + } + return ann_json + + def _convert_label_object(self, obj): + converted = self._convert_annotation(obj) + + converted.update({ + 'label_id': _cast(obj.label, int), + }) + return converted + + def _save_mask(self, mask): + mask_id = None + if mask is None: + return mask_id + + if self._converter._apply_colormap: + categories = self._converter._extractor.categories() + categories = categories[AnnotationType.mask] + colormap = categories.colormap + + mask = apply_colormap(mask, colormap) + + mask_id = self._next_mask_id + self._next_mask_id += 1 + + filename = '%d%s' % (mask_id, DatumaroPath.MASK_EXT) + masks_dir = osp.join(self._converter._annotations_dir, + DatumaroPath.MASKS_DIR) + os.makedirs(masks_dir, exist_ok=True) + path = osp.join(masks_dir, filename) + cv2.imwrite(path, mask) + return mask_id + + def _convert_mask_object(self, obj): + converted = self._convert_annotation(obj) + + mask = obj.image + mask_id = None + if mask is not None: + mask_id = self._save_mask(mask) + + converted.update({ + 'label_id': _cast(obj.label, int), + 'mask_id': _cast(mask_id, int), + }) + return converted + + def _convert_polyline_object(self, obj): + converted = self._convert_annotation(obj) + + converted.update({ + 'label_id': _cast(obj.label, int), + 'points': [float(p) for p in obj.get_points()], + }) + return converted + + def _convert_polygon_object(self, obj): + converted = self._convert_annotation(obj) + + converted.update({ + 'label_id': _cast(obj.label, int), + 'points': [float(p) for p in obj.get_points()], + }) + return converted + + def _convert_bbox_object(self, obj): + converted = self._convert_annotation(obj) + + converted.update({ + 'label_id': _cast(obj.label, int), + 'bbox': [float(p) for p in obj.get_bbox()], + }) + return converted + + def _convert_points_object(self, obj): + converted = self._convert_annotation(obj) + + converted.update({ + 'label_id': _cast(obj.label, int), + 'points': [float(p) for p in obj.points], + 'visibility': [int(v.value) for v in obj.visibility], + }) + return converted + + def _convert_caption_object(self, obj): + converted = self._convert_annotation(obj) + + converted.update({ + 'caption': _cast(obj.caption, str), + }) + return converted + + def _convert_label_categories(self, obj): + converted = { + 'labels': [], + } + for label in obj.items: + converted['labels'].append({ + 'name': _cast(label.name, str), + 'parent': _cast(label.parent, str), + }) + return converted + + def _convert_mask_categories(self, obj): + converted = { + 'colormap': [], + } + for label_id, color in obj.colormap.items(): + converted['colormap'].append({ + 'label_id': int(label_id), + 'r': int(color[0]), + 'g': int(color[1]), + 'b': int(color[2]), + }) + return converted + + def _convert_points_categories(self, obj): + converted = { + 'items': [], + } + for label_id, item in obj.items.items(): + converted['items'].append({ + 'label_id': int(label_id), + 'labels': [_cast(label, str) for label in item.labels], + 'adjacent': [int(v) for v in item.adjacent], + }) + return converted + +class _Converter: + def __init__(self, extractor, save_dir, + save_images=False, apply_colormap=False): + self._extractor = extractor + self._save_dir = save_dir + self._save_images = save_images + self._apply_colormap = apply_colormap + + def convert(self): + os.makedirs(self._save_dir, exist_ok=True) + + images_dir = osp.join(self._save_dir, DatumaroPath.IMAGES_DIR) + os.makedirs(images_dir, exist_ok=True) + self._images_dir = images_dir + + annotations_dir = osp.join(self._save_dir, DatumaroPath.ANNOTATIONS_DIR) + os.makedirs(annotations_dir, exist_ok=True) + self._annotations_dir = annotations_dir + + subsets = self._extractor.subsets() + if len(subsets) == 0: + subsets = [ None ] + subsets = [n if n else DEFAULT_SUBSET_NAME for n in subsets] + subsets = { name: _SubsetWriter(name, self) for name in subsets } + + for subset, writer in subsets.items(): + writer.write_categories(self._extractor.categories()) + + for item in self._extractor: + subset = item.subset + if not subset: + subset = DEFAULT_SUBSET_NAME + writer = subsets[subset] + + if self._save_images: + self._save_image(item) + writer.write_item(item) + + for subset, writer in subsets.items(): + writer.write(annotations_dir) + + def _save_image(self, item): + image = item.image + if image is None: + return + + image_path = osp.join(self._images_dir, + str(item.id) + DatumaroPath.IMAGE_EXT) + cv2.imwrite(image_path, image) + +class DatumaroConverter(Converter): + def __init__(self, save_images=False, apply_colormap=False): + super().__init__() + self._save_images = save_images + self._apply_colormap = apply_colormap + + def __call__(self, extractor, save_dir): + converter = _Converter(extractor, save_dir, + apply_colormap=self._apply_colormap, + save_images=self._save_images) + converter.convert() \ No newline at end of file diff --git a/datumaro/datumaro/components/converters/ms_coco.py b/datumaro/datumaro/components/converters/ms_coco.py new file mode 100644 index 000000000000..d8d2da899c97 --- /dev/null +++ b/datumaro/datumaro/components/converters/ms_coco.py @@ -0,0 +1,386 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import cv2 +import json +import numpy as np +import os +import os.path as osp + +import pycocotools.mask as mask_utils + +from datumaro.components.converter import Converter +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, AnnotationType, PointsObject, BboxObject +) +from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath +from datumaro.util import find +import datumaro.util.mask_tools as mask_tools + + +def _cast(value, type_conv, default=None): + if value is None: + return default + try: + return type_conv(value) + except Exception: + return default + +class _TaskConverter: + def __init__(self): + self._min_ann_id = 1 + + data = { + 'licenses': [], + 'info': {}, + 'categories': [], + 'images': [], + 'annotations': [] + } + + data['licenses'].append({ + 'name': '', + 'id': 0, + 'url': '' + }) + + data['info'] = { + 'contributor': '', + 'date_created': '', + 'description': '', + 'url': '', + 'version': '', + 'year': '' + } + self._data = data + + def is_empty(self): + return len(self._data['annotations']) == 0 + + def save_image_info(self, item, filename): + if item.has_image: + h, w, _ = item.image.shape + else: + h = 0 + w = 0 + + self._data['images'].append({ + 'id': _cast(item.id, int, 0), + 'width': int(w), + 'height': int(h), + 'file_name': filename, + 'license': 0, + 'flickr_url': '', + 'coco_url': '', + 'date_captured': 0, + }) + + def save_categories(self, dataset): + raise NotImplementedError() + + def save_annotations(self, item): + raise NotImplementedError() + + def write(self, path): + next_id = self._min_ann_id + for ann in self.annotations: + if ann['id'] is None: + ann['id'] = next_id + next_id += 1 + + with open(path, 'w') as outfile: + json.dump(self._data, outfile) + + @property + def annotations(self): + return self._data['annotations'] + + @property + def categories(self): + return self._data['categories'] + + def _get_ann_id(self, annotation): + ann_id = annotation.id + if ann_id: + self._min_ann_id = max(ann_id, self._min_ann_id) + return ann_id + +class _InstancesConverter(_TaskConverter): + def save_categories(self, dataset): + label_categories = dataset.categories().get(AnnotationType.label) + if label_categories is None: + return + + for idx, cat in enumerate(label_categories.items): + self.categories.append({ + 'id': 1 + idx, + 'name': cat.name, + 'supercategory': cat.parent, + }) + + def save_annotations(self, item): + for ann in item.annotations: + if ann.type != AnnotationType.bbox: + continue + + is_crowd = ann.attributes.get('is_crowd', False) + segmentation = None + if ann.group is not None: + if is_crowd: + segmentation = find(item.annotations, lambda x: \ + x.group == ann.group and x.type == AnnotationType.mask) + if segmentation is not None: + binary_mask = np.array(segmentation.image, dtype=np.bool) + binary_mask = np.asfortranarray(binary_mask, dtype=np.uint8) + segmentation = mask_utils.encode(binary_mask) + area = mask_utils.area(segmentation) + segmentation = mask_tools.convert_mask_to_rle(binary_mask) + else: + segmentation = find(item.annotations, lambda x: \ + x.group == ann.group and x.type == AnnotationType.polygon) + if segmentation is not None: + area = ann.area() + segmentation = [segmentation.get_points()] + if segmentation is None: + is_crowd = False + segmentation = [ann.get_polygon()] + area = ann.area() + + elem = { + 'id': self._get_ann_id(ann), + 'image_id': _cast(item.id, int, 0), + 'category_id': _cast(ann.label, int, -1) + 1, + 'segmentation': segmentation, + 'area': float(area), + 'bbox': ann.get_bbox(), + 'iscrowd': int(is_crowd), + } + if 'score' in ann.attributes: + elem['score'] = float(ann.attributes['score']) + + self.annotations.append(elem) + +class _ImageInfoConverter(_TaskConverter): + def is_empty(self): + return len(self._data['images']) == 0 + + def save_categories(self, dataset): + pass + + def save_annotations(self, item): + pass + +class _CaptionsConverter(_TaskConverter): + def save_categories(self, dataset): + pass + + def save_annotations(self, item): + for ann in item.annotations: + if ann.type != AnnotationType.caption: + continue + + elem = { + 'id': self._get_ann_id(ann), + 'image_id': _cast(item.id, int, 0), + 'category_id': 0, # NOTE: workaround for a bug in cocoapi + 'caption': ann.caption, + } + if 'score' in ann.attributes: + elem['score'] = float(ann.attributes['score']) + + self.annotations.append(elem) + +class _KeypointsConverter(_TaskConverter): + def save_categories(self, dataset): + label_categories = dataset.categories().get(AnnotationType.label) + if label_categories is None: + return + points_categories = dataset.categories().get(AnnotationType.points) + if points_categories is None: + return + + for idx, kp_cat in points_categories.items.items(): + label_cat = label_categories.items[idx] + + cat = { + 'id': 1 + idx, + 'name': label_cat.name, + 'supercategory': label_cat.parent, + 'keypoints': [str(l) for l in kp_cat.labels], + 'skeleton': [int(i) for i in kp_cat.adjacent], + } + self.categories.append(cat) + + def save_annotations(self, item): + for ann in item.annotations: + if ann.type != AnnotationType.points: + continue + + elem = { + 'id': self._get_ann_id(ann), + 'image_id': _cast(item.id, int, 0), + 'category_id': _cast(ann.label, int, -1) + 1, + } + if 'score' in ann.attributes: + elem['score'] = float(ann.attributes['score']) + + keypoints = [] + points = ann.get_points() + visibility = ann.visibility + for index in range(0, len(points), 2): + kp = points[index : index + 2] + state = visibility[index // 2].value + keypoints.extend([*kp, state]) + + num_visible = len([v for v in visibility \ + if v == PointsObject.Visibility.visible]) + + bbox = find(item.annotations, lambda x: \ + x.group == ann.group and \ + x.type == AnnotationType.bbox and + x.label == ann.label) + if bbox is None: + bbox = BboxObject(*ann.get_bbox()) + elem.update({ + 'segmentation': bbox.get_polygon(), + 'area': bbox.area(), + 'bbox': bbox.get_bbox(), + 'iscrowd': 0, + 'keypoints': keypoints, + 'num_keypoints': num_visible, + }) + + self.annotations.append(elem) + +class _LabelsConverter(_TaskConverter): + def save_categories(self, dataset): + label_categories = dataset.categories().get(AnnotationType.label) + if label_categories is None: + return + + for idx, cat in enumerate(label_categories.items): + self.categories.append({ + 'id': 1 + idx, + 'name': cat.name, + 'supercategory': cat.parent, + }) + + def save_annotations(self, item): + for ann in item.annotations: + if ann.type != AnnotationType.label: + continue + + elem = { + 'id': self._get_ann_id(ann), + 'image_id': _cast(item.id, int, 0), + 'category_id': int(ann.label) + 1, + } + if 'score' in ann.attributes: + elem['score'] = float(ann.attributes['score']) + + self.annotations.append(elem) + +class _Converter: + _TASK_CONVERTER = { + CocoAnnotationType.image_info: _ImageInfoConverter, + CocoAnnotationType.instances: _InstancesConverter, + CocoAnnotationType.person_keypoints: _KeypointsConverter, + CocoAnnotationType.captions: _CaptionsConverter, + CocoAnnotationType.labels: _LabelsConverter, + } + + def __init__(self, extractor, save_dir, save_images=False, task=None): + if not task: + task = list(self._TASK_CONVERTER.keys()) + elif task in CocoAnnotationType: + task = [task] + self._task = task + self._extractor = extractor + self._save_dir = save_dir + self._save_images = save_images + + def make_dirs(self): + self._images_dir = osp.join(self._save_dir, CocoPath.IMAGES_DIR) + os.makedirs(self._images_dir, exist_ok=True) + + self._ann_dir = osp.join(self._save_dir, CocoPath.ANNOTATIONS_DIR) + os.makedirs(self._ann_dir, exist_ok=True) + + def make_task_converter(self, task): + return self._TASK_CONVERTER[task]() + + def make_task_converters(self): + return { + task: self.make_task_converter(task) for task in self._task + } + + def save_image(self, item, subset_name, filename): + path = osp.join(self._images_dir, subset_name, filename) + cv2.imwrite(path, item.image) + + return path + + def convert(self): + self.make_dirs() + + subsets = self._extractor.subsets() + if len(subsets) == 0: + subsets = [ None ] + + for subset_name in subsets: + if subset_name: + subset = self._extractor.get_subset(subset_name) + else: + subset_name = DEFAULT_SUBSET_NAME + subset = self._extractor + + task_converters = self.make_task_converters() + for task_conv in task_converters.values(): + task_conv.save_categories(subset) + for item in subset: + filename = '' + if item.has_image: + filename = str(item.id) + CocoPath.IMAGE_EXT + if self._save_images: + self.save_image(item, subset_name, filename) + for task_conv in task_converters.values(): + task_conv.save_image_info(item, filename) + task_conv.save_annotations(item) + + for task, task_conv in task_converters.items(): + if not task_conv.is_empty(): + task_conv.write(osp.join(self._ann_dir, + '%s_%s.json' % (task.name, subset_name))) + +class CocoConverter(Converter): + def __init__(self, task=None, save_images=False): + super().__init__() + self._task = task + self._save_images = save_images + + def __call__(self, extractor, save_dir): + converter = _Converter(extractor, save_dir, + save_images=self._save_images, task=self._task) + converter.convert() + +def CocoInstancesConverter(save_images=False): + return CocoConverter(CocoAnnotationType.instances, + save_images=save_images) + +def CocoImageInfoConverter(save_images=False): + return CocoConverter(CocoAnnotationType.image_info, + save_images=save_images) + +def CocoPersonKeypointsConverter(save_images=False): + return CocoConverter(CocoAnnotationType.person_keypoints, + save_images=save_images) + +def CocoCaptionsConverter(save_images=False): + return CocoConverter(CocoAnnotationType.captions, + save_images=save_images) + +def CocoLabelsConverter(save_images=False): + return CocoConverter(CocoAnnotationType.labels, + save_images=save_images) \ No newline at end of file diff --git a/datumaro/datumaro/components/converters/voc.py b/datumaro/datumaro/components/converters/voc.py new file mode 100644 index 000000000000..c513150e7f9e --- /dev/null +++ b/datumaro/datumaro/components/converters/voc.py @@ -0,0 +1,370 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import cv2 +from collections import OrderedDict, defaultdict +import os +import os.path as osp +from lxml import etree as ET + +from datumaro.components.converter import Converter +from datumaro.components.extractor import DEFAULT_SUBSET_NAME, AnnotationType +from datumaro.components.formats.voc import VocLabel, VocAction, \ + VocBodyPart, VocPose, VocTask, VocPath, VocColormap, VocInstColormap +from datumaro.util import find +from datumaro.util.mask_tools import apply_colormap + + +def _write_xml_bbox(bbox, parent_elem): + x, y, w, h = bbox + bbox_elem = ET.SubElement(parent_elem, 'bndbox') + ET.SubElement(bbox_elem, 'xmin').text = str(x) + ET.SubElement(bbox_elem, 'ymin').text = str(y) + ET.SubElement(bbox_elem, 'xmax').text = str(x + w) + ET.SubElement(bbox_elem, 'ymax').text = str(y + h) + return bbox_elem + +class _Converter: + _LABELS = set([entry.name for entry in VocLabel]) + _BODY_PARTS = set([entry.name for entry in VocBodyPart]) + _ACTIONS = set([entry.name for entry in VocAction]) + + def __init__(self, task, extractor, save_dir, + apply_colormap=True, save_images=False): + + assert not task or task in VocTask + self._task = task + self._extractor = extractor + self._save_dir = save_dir + self._apply_colormap = apply_colormap + self._save_images = save_images + + self._label_categories = extractor.categories() \ + .get(AnnotationType.label) + self._mask_categories = extractor.categories() \ + .get(AnnotationType.mask) + + def convert(self): + self.init_dirs() + self.save_subsets() + + def init_dirs(self): + save_dir = self._save_dir + subsets_dir = osp.join(save_dir, VocPath.SUBSETS_DIR) + cls_subsets_dir = osp.join(subsets_dir, + VocPath.TASK_DIR[VocTask.classification]) + action_subsets_dir = osp.join(subsets_dir, + VocPath.TASK_DIR[VocTask.action_classification]) + layout_subsets_dir = osp.join(subsets_dir, + VocPath.TASK_DIR[VocTask.person_layout]) + segm_subsets_dir = osp.join(subsets_dir, + VocPath.TASK_DIR[VocTask.segmentation]) + ann_dir = osp.join(save_dir, VocPath.ANNOTATIONS_DIR) + img_dir = osp.join(save_dir, VocPath.IMAGES_DIR) + segm_dir = osp.join(save_dir, VocPath.SEGMENTATION_DIR) + inst_dir = osp.join(save_dir, VocPath.INSTANCES_DIR) + images_dir = osp.join(save_dir, VocPath.IMAGES_DIR) + + os.makedirs(subsets_dir, exist_ok=True) + os.makedirs(ann_dir, exist_ok=True) + os.makedirs(img_dir, exist_ok=True) + os.makedirs(segm_dir, exist_ok=True) + os.makedirs(inst_dir, exist_ok=True) + os.makedirs(images_dir, exist_ok=True) + + self._subsets_dir = subsets_dir + self._cls_subsets_dir = cls_subsets_dir + self._action_subsets_dir = action_subsets_dir + self._layout_subsets_dir = layout_subsets_dir + self._segm_subsets_dir = segm_subsets_dir + self._ann_dir = ann_dir + self._img_dir = img_dir + self._segm_dir = segm_dir + self._inst_dir = inst_dir + self._images_dir = images_dir + + def get_label(self, label_id): + return self._label_categories.items[label_id].name + + def save_subsets(self): + subsets = self._extractor.subsets() + if len(subsets) == 0: + subsets = [ None ] + + for subset_name in subsets: + if subset_name: + subset = self._extractor.get_subset(subset_name) + else: + subset_name = DEFAULT_SUBSET_NAME + subset = self._extractor + + class_lists = OrderedDict() + clsdet_list = OrderedDict() + action_list = OrderedDict() + layout_list = OrderedDict() + segm_list = OrderedDict() + + for item in subset: + item_id = str(item.id) + if self._save_images: + data = item.image + if data is not None: + cv2.imwrite(osp.join(self._images_dir, + str(item_id) + VocPath.IMAGE_EXT), + data) + + labels = [] + bboxes = [] + masks = [] + for a in item.annotations: + if a.type == AnnotationType.label: + labels.append(a) + elif a.type == AnnotationType.bbox: + bboxes.append(a) + elif a.type == AnnotationType.mask: + masks.append(a) + + if len(bboxes) != 0: + root_elem = ET.Element('annotation') + if '_' in item_id: + folder = item_id[ : item_id.find('_')] + else: + folder = '' + ET.SubElement(root_elem, 'folder').text = folder + ET.SubElement(root_elem, 'filename').text = \ + item_id + VocPath.IMAGE_EXT + + if item.has_image: + h, w, c = item.image.shape + size_elem = ET.SubElement(root_elem, 'size') + ET.SubElement(size_elem, 'width').text = str(w) + ET.SubElement(size_elem, 'height').text = str(h) + ET.SubElement(size_elem, 'depth').text = str(c) + + item_segmented = 0 < len(masks) + if item_segmented: + ET.SubElement(root_elem, 'segmented').text = '1' + + objects_with_parts = [] + objects_with_actions = defaultdict(dict) + + main_bboxes = [] + layout_bboxes = [] + for bbox in bboxes: + label = self.get_label(bbox.label) + if label in self._LABELS: + main_bboxes.append(bbox) + elif label in self._BODY_PARTS: + layout_bboxes.append(bbox) + + for new_obj_id, obj in enumerate(main_bboxes): + attr = obj.attributes + + obj_elem = ET.SubElement(root_elem, 'object') + ET.SubElement(obj_elem, 'name').text = self.get_label(obj.label) + + pose = attr.get('pose') + if pose is not None: + ET.SubElement(obj_elem, 'pose').text = VocPose[pose].name + + truncated = attr.get('truncated') + if truncated is not None: + ET.SubElement(obj_elem, 'truncated').text = '%d' % truncated + + difficult = attr.get('difficult') + if difficult is not None: + ET.SubElement(obj_elem, 'difficult').text = '%d' % difficult + + bbox = obj.get_bbox() + if bbox is not None: + _write_xml_bbox(bbox, obj_elem) + + for part in VocBodyPart: + part_bbox = find(layout_bboxes, lambda x: \ + obj.id == x.group and \ + self.get_label(x.label) == part.name) + if part_bbox is not None: + part_elem = ET.SubElement(obj_elem, 'part') + ET.SubElement(part_elem, 'name').text = part.name + _write_xml_bbox(part_bbox.get_bbox(), part_elem) + + objects_with_parts.append(new_obj_id) + + actions = [x for x in labels + if obj.id == x.group and \ + self.get_label(x.label) in self._ACTIONS] + if len(actions) != 0: + actions_elem = ET.SubElement(obj_elem, 'actions') + for action in VocAction: + presented = find(actions, lambda x: \ + self.get_label(x.label) == action.name) is not None + ET.SubElement(actions_elem, action.name).text = \ + '%d' % presented + + objects_with_actions[new_obj_id][action] = presented + + if self._task in [None, + VocTask.detection, + VocTask.person_layout, + VocTask.action_classification]: + with open(osp.join(self._ann_dir, item_id + '.xml'), 'w') as f: + f.write(ET.tostring(root_elem, + encoding='unicode', pretty_print=True)) + + clsdet_list[item_id] = True + layout_list[item_id] = objects_with_parts + action_list[item_id] = objects_with_actions + + for label_obj in labels: + label = self.get_label(label_obj.label) + if label not in self._LABELS: + continue + class_list = class_lists.get(item_id, set()) + class_list.add(label_obj.label) + class_lists[item_id] = class_list + + clsdet_list[item_id] = True + + for mask_obj in masks: + if mask_obj.attributes.get('class') == True: + self.save_segm(osp.join(self._segm_dir, + item_id + VocPath.SEGM_EXT), + mask_obj, self._mask_categories.colormap) + if mask_obj.attributes.get('instances') == True: + self.save_segm(osp.join(self._inst_dir, + item_id + VocPath.SEGM_EXT), + mask_obj, VocInstColormap) + + segm_list[item_id] = True + + if len(item.annotations) == 0: + clsdet_list[item_id] = None + layout_list[item_id] = None + action_list[item_id] = None + segm_list[item_id] = None + + if self._task in [None, + VocTask.classification, + VocTask.detection, + VocTask.action_classification, + VocTask.person_layout]: + self.save_clsdet_lists(subset_name, clsdet_list) + if self._task in [None, VocTask.classification]: + self.save_class_lists(subset_name, class_lists) + if self._task in [None, VocTask.action_classification]: + self.save_action_lists(subset_name, action_list) + if self._task in [None, VocTask.person_layout]: + self.save_layout_lists(subset_name, layout_list) + if self._task in [None, VocTask.segmentation]: + self.save_segm_lists(subset_name, segm_list) + + def save_action_lists(self, subset_name, action_list): + os.makedirs(self._action_subsets_dir, exist_ok=True) + + ann_file = osp.join(self._action_subsets_dir, subset_name + '.txt') + with open(ann_file, 'w') as f: + for item in action_list: + f.write('%s\n' % item) + + if len(action_list) == 0: + return + + for action in VocAction: + ann_file = osp.join(self._action_subsets_dir, + '%s_%s.txt' % (action.name, subset_name)) + with open(ann_file, 'w') as f: + for item, objs in action_list.items(): + if not objs: + continue + for obj_id, obj_actions in objs.items(): + presented = obj_actions[action] + f.write('%s %s % d\n' % \ + (item, 1 + obj_id, 1 if presented else -1)) + + def save_class_lists(self, subset_name, class_lists): + os.makedirs(self._cls_subsets_dir, exist_ok=True) + + if len(class_lists) == 0: + return + + for label in VocLabel: + ann_file = osp.join(self._cls_subsets_dir, + '%s_%s.txt' % (label.name, subset_name)) + with open(ann_file, 'w') as f: + for item, item_labels in class_lists.items(): + if not item_labels: + continue + presented = label.value in item_labels + f.write('%s % d\n' % \ + (item, 1 if presented else -1)) + + def save_clsdet_lists(self, subset_name, clsdet_list): + os.makedirs(self._cls_subsets_dir, exist_ok=True) + + ann_file = osp.join(self._cls_subsets_dir, subset_name + '.txt') + with open(ann_file, 'w') as f: + for item in clsdet_list: + f.write('%s\n' % item) + + def save_segm_lists(self, subset_name, segm_list): + os.makedirs(self._segm_subsets_dir, exist_ok=True) + + ann_file = osp.join(self._segm_subsets_dir, subset_name + '.txt') + with open(ann_file, 'w') as f: + for item in segm_list: + f.write('%s\n' % item) + + def save_layout_lists(self, subset_name, layout_list): + os.makedirs(self._layout_subsets_dir, exist_ok=True) + + ann_file = osp.join(self._layout_subsets_dir, subset_name + '.txt') + with open(ann_file, 'w') as f: + for item, item_layouts in layout_list.items(): + if item_layouts: + for obj_id in item_layouts: + f.write('%s % d\n' % (item, 1 + obj_id)) + else: + f.write('%s\n' % (item)) + + def save_segm(self, path, annotation, colormap): + data = annotation.image + if self._apply_colormap: + if colormap is None: + colormap = VocColormap + data = apply_colormap(data, colormap) + cv2.imwrite(path, data) + +class VocConverter(Converter): + def __init__(self, task=None, save_images=False, apply_colormap=False): + super().__init__() + self._task = task + self._save_images = save_images + self._apply_colormap = apply_colormap + + def __call__(self, extractor, save_dir): + converter = _Converter(self._task, extractor, save_dir, + apply_colormap=self._apply_colormap, + save_images=self._save_images) + converter.convert() + +def VocClassificationConverter(save_images=False): + return VocConverter(VocTask.classification, + save_images=save_images) + +def VocDetectionConverter(save_images=False): + return VocConverter(VocTask.detection, + save_images=save_images) + +def VocLayoutConverter(save_images=False): + return VocConverter(VocTask.person_layout, + save_images=save_images) + +def VocActionConverter(save_images=False): + return VocConverter(VocTask.action_classification, + save_images=save_images) + +def VocSegmentationConverter(save_images=False, apply_colormap=True): + return VocConverter(VocTask.segmentation, + save_images=save_images, apply_colormap=apply_colormap) diff --git a/datumaro/datumaro/components/dataset_filter.py b/datumaro/datumaro/components/dataset_filter.py new file mode 100644 index 000000000000..7b27c49914fe --- /dev/null +++ b/datumaro/datumaro/components/dataset_filter.py @@ -0,0 +1,193 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from lxml import etree as ET +from datumaro.components.extractor import (DatasetItem, Annotation, + LabelObject, MaskObject, PointsObject, PolygonObject, + PolyLineObject, BboxObject, CaptionObject, +) + + +def _cast(value, type_conv, default=None): + if value is None: + return default + try: + return type_conv(value) + except Exception: + return default + +class DatasetItemEncoder: + def encode_item(self, item): + item_elem = ET.Element('item') + ET.SubElement(item_elem, 'id').text = str(item.id) + ET.SubElement(item_elem, 'subset').text = str(item.subset) + + # Dataset wrapper-specific + ET.SubElement(item_elem, 'source').text = \ + str(getattr(item, 'source', None)) + ET.SubElement(item_elem, 'extractor').text = \ + str(getattr(item, 'extractor', None)) + + image = item.image + if image is not None: + item_elem.append(self.encode_image(image)) + + for ann in item.annotations: + item_elem.append(self.encode_object(ann)) + + return item_elem + + @classmethod + def encode_image(cls, image): + image_elem = ET.Element('image') + + h, w, c = image.shape + ET.SubElement(image_elem, 'width').text = str(w) + ET.SubElement(image_elem, 'height').text = str(h) + ET.SubElement(image_elem, 'depth').text = str(c) + + return image_elem + + @classmethod + def encode_annotation(cls, annotation): + assert isinstance(annotation, Annotation) + ann_elem = ET.Element('annotation') + ET.SubElement(ann_elem, 'id').text = str(annotation.id) + ET.SubElement(ann_elem, 'type').text = str(annotation.type.name) + + for k, v in annotation.attributes.items(): + ET.SubElement(ann_elem, k).text = str(v) + + ET.SubElement(ann_elem, 'group').text = str(annotation.group) + + return ann_elem + + @classmethod + def encode_label_object(cls, obj): + ann_elem = cls.encode_annotation(obj) + + ET.SubElement(ann_elem, 'label_id').text = str(obj.label) + + return ann_elem + + @classmethod + def encode_mask_object(cls, obj): + ann_elem = cls.encode_annotation(obj) + + ET.SubElement(ann_elem, 'label_id').text = str(obj.label) + + mask = obj.image + if mask is not None: + ann_elem.append(cls.encode_image(mask)) + + return ann_elem + + @classmethod + def encode_bbox_object(cls, obj): + ann_elem = cls.encode_annotation(obj) + + ET.SubElement(ann_elem, 'label_id').text = str(obj.label) + ET.SubElement(ann_elem, 'x').text = str(obj.x) + ET.SubElement(ann_elem, 'y').text = str(obj.y) + ET.SubElement(ann_elem, 'w').text = str(obj.w) + ET.SubElement(ann_elem, 'h').text = str(obj.h) + ET.SubElement(ann_elem, 'area').text = str(obj.area()) + + return ann_elem + + @classmethod + def encode_points_object(cls, obj): + ann_elem = cls.encode_annotation(obj) + + ET.SubElement(ann_elem, 'label_id').text = str(obj.label) + + x, y, w, h = obj.get_bbox() + area = w * h + bbox_elem = ET.SubElement(ann_elem, 'bbox') + ET.SubElement(bbox_elem, 'x').text = str(x) + ET.SubElement(bbox_elem, 'y').text = str(y) + ET.SubElement(bbox_elem, 'w').text = str(w) + ET.SubElement(bbox_elem, 'h').text = str(h) + ET.SubElement(bbox_elem, 'area').text = str(area) + + points = ann_elem.points + for i in range(0, len(points), 2): + point_elem = ET.SubElement(ann_elem, 'point') + ET.SubElement(point_elem, 'x').text = str(points[i * 2]) + ET.SubElement(point_elem, 'y').text = str(points[i * 2 + 1]) + ET.SubElement(point_elem, 'visible').text = \ + str(ann_elem.visibility[i // 2].name) + + return ann_elem + + @classmethod + def encode_polyline_object(cls, obj): + ann_elem = cls.encode_annotation(obj) + + ET.SubElement(ann_elem, 'label_id').text = str(obj.label) + + x, y, w, h = obj.get_bbox() + area = w * h + bbox_elem = ET.SubElement(ann_elem, 'bbox') + ET.SubElement(bbox_elem, 'x').text = str(x) + ET.SubElement(bbox_elem, 'y').text = str(y) + ET.SubElement(bbox_elem, 'w').text = str(w) + ET.SubElement(bbox_elem, 'h').text = str(h) + ET.SubElement(bbox_elem, 'area').text = str(area) + + points = ann_elem.points + for i in range(0, len(points), 2): + point_elem = ET.SubElement(ann_elem, 'point') + ET.SubElement(point_elem, 'x').text = str(points[i * 2]) + ET.SubElement(point_elem, 'y').text = str(points[i * 2 + 1]) + + return ann_elem + + @classmethod + def encode_caption_object(cls, obj): + ann_elem = cls.encode_annotation(obj) + + ET.SubElement(ann_elem, 'caption').text = str(obj.caption) + + return ann_elem + + def encode_object(self, o): + if isinstance(o, LabelObject): + return self.encode_label_object(o) + if isinstance(o, MaskObject): + return self.encode_mask_object(o) + if isinstance(o, BboxObject): + return self.encode_bbox_object(o) + if isinstance(o, PointsObject): + return self.encode_points_object(o) + if isinstance(o, PolyLineObject): + return self.encode_polyline_object(o) + if isinstance(o, PolygonObject): + return self.encode_polygon_object(o) + if isinstance(o, CaptionObject): + return self.encode_caption_object(o) + if isinstance(o, Annotation): # keep after derived classes + return self.encode_annotation(o) + + if isinstance(o, DatasetItem): + return self.encode_item(o) + + return None + +class XPathDatasetFilter: + def __init__(self, filter_text=None): + self._filter = None + if filter_text is not None: + self._filter = ET.XPath(filter_text) + self._encoder = DatasetItemEncoder() + + def __call__(self, item): + encoded_item = self._serialize_item(item) + if self._filter is None: + return True + return bool(self._filter(encoded_item)) + + def _serialize_item(self, item): + return self._encoder.encode_item(item) \ No newline at end of file diff --git a/datumaro/datumaro/components/extractor.py b/datumaro/datumaro/components/extractor.py new file mode 100644 index 000000000000..b51302f5c31b --- /dev/null +++ b/datumaro/datumaro/components/extractor.py @@ -0,0 +1,549 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import namedtuple +from enum import Enum +import numpy as np + + +AnnotationType = Enum('AnnotationType', + [ + 'label', + 'mask', + 'points', + 'polygon', + 'polyline', + 'bbox', + 'caption', + ]) + +class Annotation: + # pylint: disable=redefined-builtin + def __init__(self, id=None, type=None, attributes=None, group=None): + if id is not None: + id = int(id) + self.id = id + + assert type in AnnotationType + self.type = type + + if attributes is None: + attributes = {} + else: + attributes = dict(attributes) + self.attributes = attributes + + if group is not None: + group = int(group) + self.group = group + # pylint: enable=redefined-builtin + + def __eq__(self, other): + if not isinstance(other, Annotation): + return False + return \ + (self.id == other.id) and \ + (self.type == other.type) and \ + (self.attributes == other.attributes) and \ + (self.group == other.group) + +class Categories: + def __init__(self, attributes=None): + if attributes is None: + attributes = set() + self.attributes = attributes + + def __eq__(self, other): + if not isinstance(other, Categories): + return False + return \ + (self.attributes == other.attributes) + +class LabelCategories(Categories): + Category = namedtuple('Category', ['name', 'parent']) + + def __init__(self, items=None, attributes=None): + super().__init__(attributes=attributes) + + if items is None: + items = [] + self.items = items + + self._indices = {} + self._reindex() + + def _reindex(self): + indices = {} + for index, item in enumerate(self.items): + assert item.name not in self._indices + indices[item.name] = index + self._indices = indices + + def add(self, name, parent=None): + assert name not in self._indices + + index = len(self.items) + self.items.append(self.Category(name, parent)) + self._indices[name] = index + + def find(self, name): + index = self._indices.get(name) + if index: + return index, self.items[index] + return index, None + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.items == other.items) + +class LabelObject(Annotation): + # pylint: disable=redefined-builtin + def __init__(self, label=None, + id=None, attributes=None, group=None): + super().__init__(id=id, type=AnnotationType.label, + attributes=attributes, group=group) + self.label = label + # pylint: enable=redefined-builtin + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.label == other.label) + +class MaskCategories(Categories): + def __init__(self, colormap=None, inverse_colormap=None, attributes=None): + super().__init__(attributes=attributes) + + # colormap: label id -> color + if colormap is None: + colormap = {} + self.colormap = colormap + self._inverse_colormap = inverse_colormap + + @property + def inverse_colormap(self): + from datumaro.util.mask_tools import invert_colormap + if self._inverse_colormap is None: + if self.colormap is not None: + try: + self._inverse_colormap = invert_colormap(self.colormap) + except Exception: + pass + return self._inverse_colormap + + def __eq__(self, other): + if not super().__eq__(other): + return False + for label_id, my_color in self.colormap.items(): + other_color = other.colormap.get(label_id) + if not np.array_equal(my_color, other_color): + return False + return True + +class MaskObject(Annotation): + # pylint: disable=redefined-builtin + def __init__(self, image=None, label=None, + id=None, attributes=None, group=None): + super().__init__(id=id, type=AnnotationType.mask, + attributes=attributes, group=group) + self._image = image + self._label = label + # pylint: enable=redefined-builtin + + @property + def label(self): + return self._label + + @property + def image(self): + if callable(self._image): + return self._image() + return self._image + + def painted_data(self, colormap): + raise NotImplementedError() + + def area(self): + raise NotImplementedError() + + def extract(self, class_id): + raise NotImplementedError() + + def bbox(self): + raise NotImplementedError() + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.label == other.label) and \ + (np.all(self.image == other.image)) + +def compute_iou(bbox_a, bbox_b): + aX, aY, aW, aH = bbox_a + bX, bY, bW, bH = bbox_b + in_right = min(aX + aW, bX + bW) + in_left = max(aX, bX) + in_top = max(aY, bY) + in_bottom = min(aY + aH, bY + bH) + + in_w = max(0, in_right - in_left) + in_h = max(0, in_bottom - in_top) + intersection = in_w * in_h + + a_area = aW * aH + b_area = bW * bH + union = a_area + b_area - intersection + + return intersection / max(1.0, union) + +class ShapeObject(Annotation): + # pylint: disable=redefined-builtin + def __init__(self, type, points=None, label=None, + id=None, attributes=None, group=None): + super().__init__(id=id, type=type, + attributes=attributes, group=group) + self.points = points + self.label = label + # pylint: enable=redefined-builtin + + def area(self): + raise NotImplementedError() + + def get_polygon(self): + raise NotImplementedError() + + def get_bbox(self): + points = self.get_points() + if not self.points: + return None + + xs = [p for p in points[0::2]] + ys = [p for p in points[1::2]] + x0 = min(xs) + x1 = max(xs) + y0 = min(ys) + y1 = max(ys) + return [x0, y0, x1 - x0, y1 - y0] + + def get_points(self): + return self.points + + def get_mask(self): + raise NotImplementedError() + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.points == other.points) and \ + (self.label == other.label) + +class PolyLineObject(ShapeObject): + # pylint: disable=redefined-builtin + def __init__(self, points=None, + label=None, id=None, attributes=None, group=None): + super().__init__(type=AnnotationType.polyline, + points=points, label=label, + id=id, attributes=attributes, group=group) + # pylint: enable=redefined-builtin + + def get_polygon(self): + return self.get_points() + + def area(self): + return 0 + +class PolygonObject(ShapeObject): + # pylint: disable=redefined-builtin + def __init__(self, points=None, + label=None, id=None, attributes=None, group=None): + super().__init__(type=AnnotationType.polygon, + points=points, label=label, + id=id, attributes=attributes, group=group) + # pylint: enable=redefined-builtin + + def get_polygon(self): + return self.get_points() + +class BboxObject(ShapeObject): + # pylint: disable=redefined-builtin + def __init__(self, x=0, y=0, w=0, h=0, + label=None, id=None, attributes=None, group=None): + super().__init__(type=AnnotationType.bbox, + points=[x, y, x + w, y + h], label=label, + id=id, attributes=attributes, group=group) + # pylint: enable=redefined-builtin + + @property + def x(self): + return self.points[0] + + @property + def y(self): + return self.points[1] + + @property + def w(self): + return self.points[2] - self.points[0] + + @property + def h(self): + return self.points[3] - self.points[1] + + def area(self): + return self.w * self.h + + def get_bbox(self): + return [self.x, self.y, self.w, self.h] + + def get_polygon(self): + x, y, w, h = self.get_bbox() + return [ + x, y, + x + w, y, + x + w, y + h, + x, y + h + ] + + def iou(self, other): + return compute_iou(self.get_bbox(), other.get_bbox()) + +class PointsCategories(Categories): + Category = namedtuple('Category', ['labels', 'adjacent']) + + def __init__(self, items=None, attributes=None): + super().__init__(attributes=attributes) + + if items is None: + items = {} + self.items = items + + def add(self, label_id, labels=None, adjacent=None): + if labels is None: + labels = [] + if adjacent is None: + adjacent = [] + self.items[label_id] = self.Category(labels, set(adjacent)) + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.items == other.items) + +class PointsObject(ShapeObject): + Visibility = Enum('Visibility', [ + ('absent', 0), + ('hidden', 1), + ('visible', 2), + ]) + + # pylint: disable=redefined-builtin + def __init__(self, points=None, visibility=None, label=None, + id=None, attributes=None, group=None): + if points is not None: + assert len(points) % 2 == 0 + + if visibility is not None: + assert len(visibility) == len(points) // 2 + for i, v in enumerate(visibility): + if not isinstance(v, self.Visibility): + visibility[i] = self.Visibility(v) + else: + visibility = [] + for _ in range(len(points) // 2): + visibility.append(self.Visibility.absent) + + super().__init__(type=AnnotationType.points, + points=points, label=label, + id=id, attributes=attributes, group=group) + + self.visibility = visibility + # pylint: enable=redefined-builtin + + def area(self): + return 0 + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.visibility == other.visibility) + +class CaptionObject(Annotation): + # pylint: disable=redefined-builtin + def __init__(self, caption=None, + id=None, attributes=None, group=None): + super().__init__(id=id, type=AnnotationType.caption, + attributes=attributes, group=group) + + if caption is None: + caption = '' + self.caption = caption + # pylint: enable=redefined-builtin + + def __eq__(self, other): + if not super().__eq__(other): + return False + return \ + (self.caption == other.caption) + +class DatasetItem: + # pylint: disable=redefined-builtin + def __init__(self, id, annotations=None, + subset=None, path=None, image=None): + assert id is not None + if not isinstance(id, str): + id = str(id) + assert len(id) != 0 + self._id = id + + if subset is None: + subset = '' + assert isinstance(subset, str) + self._subset = subset + + if path is None: + path = [] + self._path = path + + if annotations is None: + annotations = [] + self._annotations = annotations + + self._image = image + # pylint: enable=redefined-builtin + + @property + def id(self): + return self._id + + @property + def subset(self): + return self._subset + + @property + def path(self): + return self._path + + @property + def annotations(self): + return self._annotations + + @property + def image(self): + if callable(self._image): + return self._image() + return self._image + + @property + def has_image(self): + return self._image is not None + + def __eq__(self, other): + if not isinstance(other, __class__): + return False + return \ + (self.id == other.id) and \ + (self.subset == other.subset) and \ + (self.annotations == other.annotations) and \ + (self.image == other.image) + +class IExtractor: + def __iter__(self): + raise NotImplementedError() + + def __len__(self): + raise NotImplementedError() + + def subsets(self): + raise NotImplementedError() + + def get_subset(self, name): + raise NotImplementedError() + + def categories(self): + raise NotImplementedError() + + def select(self, pred): + raise NotImplementedError() + + def get(self, item_id, subset=None, path=None): + raise NotImplementedError() + +class _DatasetFilter: + def __init__(self, iterable, predicate): + self.iterable = iterable + self.predicate = predicate + + def __iter__(self): + return filter(self.predicate, self.iterable) + +class _ExtractorBase(IExtractor): + def __init__(self, length=None): + self._length = length + self._subsets = None + + def _init_cache(self): + subsets = set() + length = -1 + for length, item in enumerate(self): + subsets.add(item.subset) + length += 1 + + if self._length is None: + self._length = length + if self._subsets is None: + self._subsets = subsets + + def __len__(self): + if self._length is None: + self._init_cache() + return self._length + + def subsets(self): + if self._subsets is None: + self._init_cache() + return list(self._subsets) + + def get_subset(self, name): + if name in self.subsets(): + return self.select(lambda item: item.subset == name) + else: + raise Exception("Unknown subset '%s' requested" % name) + +class DatasetIteratorWrapper(_ExtractorBase): + def __init__(self, iterable, categories): + super().__init__(length=None) + self._iterable = iterable + self._categories = categories + + def __iter__(self): + return iter(self._iterable) + + def categories(self): + return self._categories + + def select(self, pred): + return DatasetIteratorWrapper( + _DatasetFilter(self, pred), self.categories()) + +class Extractor(_ExtractorBase): + def __init__(self, length=None): + super().__init__(length=None) + + def categories(self): + return {} + + def select(self, pred): + return DatasetIteratorWrapper( + _DatasetFilter(self, pred), self.categories()) + + +DEFAULT_SUBSET_NAME = 'default' \ No newline at end of file diff --git a/datumaro/datumaro/components/extractors/__init__.py b/datumaro/datumaro/components/extractors/__init__.py new file mode 100644 index 000000000000..2907a53e9fdf --- /dev/null +++ b/datumaro/datumaro/components/extractors/__init__.py @@ -0,0 +1,50 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from datumaro.components.extractors.datumaro import DatumaroExtractor + +from datumaro.components.extractors.ms_coco import ( + CocoImageInfoExtractor, + CocoCaptionsExtractor, + CocoInstancesExtractor, + CocoLabelsExtractor, + CocoPersonKeypointsExtractor, +) + +from datumaro.components.extractors.voc import ( + VocClassificationExtractor, + VocDetectionExtractor, + VocSegmentationExtractor, + VocLayoutExtractor, + VocActionExtractor, + VocComp_1_2_Extractor, + VocComp_3_4_Extractor, + VocComp_5_6_Extractor, + VocComp_7_8_Extractor, + VocComp_9_10_Extractor, +) + + +items = [ + ('datumaro', DatumaroExtractor), + + ('coco_images', CocoImageInfoExtractor), + ('coco_captions', CocoCaptionsExtractor), + ('coco_instances', CocoInstancesExtractor), + ('coco_person_kp', CocoPersonKeypointsExtractor), + ('coco_labels', CocoLabelsExtractor), + + ('voc_cls', VocClassificationExtractor), + ('voc_det', VocDetectionExtractor), + ('voc_segm', VocSegmentationExtractor), + ('voc_layout', VocLayoutExtractor), + ('voc_action', VocActionExtractor), + + ('voc_comp_1_2', VocComp_1_2_Extractor), + ('voc_comp_3_4', VocComp_3_4_Extractor), + ('voc_comp_5_6', VocComp_5_6_Extractor), + ('voc_comp_7_8', VocComp_7_8_Extractor), + ('voc_comp_9_10', VocComp_9_10_Extractor), +] \ No newline at end of file diff --git a/datumaro/datumaro/components/extractors/datumaro.py b/datumaro/datumaro/components/extractors/datumaro.py new file mode 100644 index 000000000000..429cb38e094d --- /dev/null +++ b/datumaro/datumaro/components/extractors/datumaro.py @@ -0,0 +1,214 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import defaultdict +import json +import os.path as osp + +from datumaro.components.extractor import (Extractor, DatasetItem, + DEFAULT_SUBSET_NAME, + AnnotationType, + LabelObject, MaskObject, PointsObject, PolygonObject, + PolyLineObject, BboxObject, CaptionObject, + LabelCategories, MaskCategories, PointsCategories +) +from datumaro.components.formats.datumaro import DatumaroPath +from datumaro.util import dir_items +from datumaro.util.image import lazy_image +from datumaro.util.mask_tools import lazy_mask + + +class DatumaroExtractor(Extractor): + class Subset(Extractor): + def __init__(self, name, parent): + super().__init__() + self._parent = parent + self._name = name + self.items = [] + + def __iter__(self): + for item in self.items: + yield self._parent._get(item, self._name) + + def __len__(self): + return len(self.items) + + def categories(self): + return self._parent.categories() + + def __init__(self, path): + super().__init__() + + assert osp.isdir(path) + self._path = path + + annotations = defaultdict(list) + found_subsets = self._find_subsets(path) + parsed_anns = None + subsets = {} + for subset_name, subset_path in found_subsets.items(): + if subset_name == DEFAULT_SUBSET_NAME: + subset_name = None + subset = self.Subset(subset_name, self) + with open(subset_path, 'r') as f: + parsed_anns = json.load(f) + + for index, _ in enumerate(parsed_anns['items']): + subset.items.append(index) + + annotations[subset_name] = parsed_anns + subsets[subset_name] = subset + self._annotations = dict(annotations) + self._subsets = subsets + + self._categories = {} + if parsed_anns is not None: + self._categories = self._load_categories(parsed_anns) + + @staticmethod + def _load_categories(parsed): + categories = {} + + parsed_label_cat = parsed['categories'].get(AnnotationType.label.name) + if parsed_label_cat: + label_categories = LabelCategories() + for item in parsed_label_cat['labels']: + label_categories.add(item['name'], parent=item['parent']) + + categories[AnnotationType.label] = label_categories + + parsed_mask_cat = parsed['categories'].get(AnnotationType.mask.name) + if parsed_mask_cat: + colormap = {} + for item in parsed_mask_cat['colormap']: + colormap[int(item['label_id'])] = \ + (item['r'], item['g'], item['b']) + + mask_categories = MaskCategories(colormap=colormap) + categories[AnnotationType.mask] = mask_categories + + parsed_points_cat = parsed['categories'].get(AnnotationType.points.name) + if parsed_points_cat: + point_categories = PointsCategories() + for item in parsed_points_cat['items']: + point_categories.add(int(item['label_id']), + item['labels'], adjacent=item['adjacent']) + + categories[AnnotationType.points] = point_categories + + return categories + + def _get(self, index, subset_name): + item = self._annotations[subset_name]['items'][index] + + item_id = item.get('id') + + image_path = osp.join(self._path, DatumaroPath.IMAGES_DIR, + item_id + DatumaroPath.IMAGE_EXT) + image = None + if osp.isfile(image_path): + image = lazy_image(image_path) + + annotations = self._load_annotations(item) + + return DatasetItem(id=item_id, subset=subset_name, + annotations=annotations, image=image) + + def _load_annotations(self, item): + parsed = item['annotations'] + loaded = [] + + for ann in parsed: + ann_id = ann.get('id') + ann_type = AnnotationType[ann['type']] + attributes = ann.get('attributes') + group = ann.get('group') + + if ann_type == AnnotationType.label: + label_id = ann.get('label_id') + loaded.append(LabelObject(label=label_id, + id=ann_id, attributes=attributes, group=group)) + + elif ann_type == AnnotationType.mask: + label_id = ann.get('label_id') + mask_id = str(ann.get('mask_id')) + + mask_path = osp.join(self._path, DatumaroPath.ANNOTATIONS_DIR, + DatumaroPath.MASKS_DIR, mask_id + DatumaroPath.MASK_EXT) + mask = None + + if osp.isfile(mask_path): + mask_cat = self._categories.get(AnnotationType.mask) + if mask_cat is not None: + mask = lazy_mask(mask_path, mask_cat.inverse_colormap) + else: + mask = lazy_image(mask_path) + + loaded.append(MaskObject(label=label_id, image=mask, + id=ann_id, attributes=attributes, group=group)) + + elif ann_type == AnnotationType.polyline: + label_id = ann.get('label_id') + points = ann.get('points') + loaded.append(PolyLineObject(points, label=label_id, + id=ann_id, attributes=attributes, group=group)) + + elif ann_type == AnnotationType.polygon: + label_id = ann.get('label_id') + points = ann.get('points') + loaded.append(PolygonObject(points, label=label_id, + id=ann_id, attributes=attributes, group=group)) + + elif ann_type == AnnotationType.bbox: + label_id = ann.get('label_id') + x, y, w, h = ann.get('bbox') + loaded.append(BboxObject(x, y, w, h, label=label_id, + id=ann_id, attributes=attributes, group=group)) + + elif ann_type == AnnotationType.points: + label_id = ann.get('label_id') + points = ann.get('points') + loaded.append(PointsObject(points, label=label_id, + id=ann_id, attributes=attributes, group=group)) + + elif ann_type == AnnotationType.caption: + caption = ann.get('caption') + loaded.append(CaptionObject(caption, + id=ann_id, attributes=attributes, group=group)) + + else: + raise NotImplementedError() + + return loaded + + def categories(self): + return self._categories + + def __iter__(self): + for subset_name, subset in self._subsets.items(): + for index in subset.items: + yield self._get(index, subset_name) + + def __len__(self): + length = 0 + for subset in self._subsets.values(): + length += len(subset) + return length + + def subsets(self): + return list(self._subsets) + + def get_subset(self, name): + return self._subsets[name] + + @staticmethod + def _find_subsets(path): + anno_dir = osp.join(path, DatumaroPath.ANNOTATIONS_DIR) + if not osp.isdir(anno_dir): + raise Exception('Datumaro dataset not found at "%s"' % path) + + return { name: osp.join(anno_dir, name + '.json') + for name in dir_items(anno_dir, '.json', truncate_ext=True) + } \ No newline at end of file diff --git a/datumaro/datumaro/components/extractors/ms_coco.py b/datumaro/datumaro/components/extractors/ms_coco.py new file mode 100644 index 000000000000..1997b31ad505 --- /dev/null +++ b/datumaro/datumaro/components/extractors/ms_coco.py @@ -0,0 +1,297 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import numpy as np +import os.path as osp + +from pycocotools.coco import COCO +import pycocotools.mask as mask_utils + +from datumaro.components.extractor import (Extractor, DatasetItem, + AnnotationType, + LabelObject, MaskObject, PointsObject, PolygonObject, + BboxObject, CaptionObject, + LabelCategories, PointsCategories +) +from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath +from datumaro.util.image import lazy_image + + +class RleMask(MaskObject): + # pylint: disable=redefined-builtin + def __init__(self, rle=None, label=None, + id=None, attributes=None, group=None): + lazy_decode = lambda: mask_utils.decode(rle).astype(np.bool) + super().__init__(image=lazy_decode, label=label, + id=id, attributes=attributes, group=group) + + self._rle = rle + # pylint: enable=redefined-builtin + + def area(self): + return mask_utils.area(self._rle) + + def bbox(self): + return mask_utils.toBbox(self._rle) + + def __eq__(self, other): + if not isinstance(other, __class__): + return super().__eq__(other) + return self._rle == other._rle + + +class CocoExtractor(Extractor): + class Subset(Extractor): + def __init__(self, name, parent): + super().__init__() + self._name = name + self._parent = parent + self.loaders = {} + self.items = set() + + def __iter__(self): + for img_id in self.items: + yield self._parent._get(img_id, self._name) + + def __len__(self): + return len(self.items) + + def categories(self): + return self._parent.categories() + + def __init__(self, path, task): + super().__init__() + + rootpath = path.rsplit(CocoPath.ANNOTATIONS_DIR, maxsplit=1)[0] + self._path = rootpath + self._task = task + self._subsets = {} + + subset_name = osp.splitext(osp.basename(path))[0] \ + .rsplit('_', maxsplit=1)[1] + subset = CocoExtractor.Subset(subset_name, self) + loader = self._make_subset_loader(path) + subset.loaders[task] = loader + for img_id in loader.getImgIds(): + subset.items.add(img_id) + self._subsets[subset_name] = subset + + self._load_categories() + + @staticmethod + def _make_subset_loader(path): + # COCO API has an 'unclosed file' warning + coco_api = COCO() + with open(path, 'r') as f: + import json + dataset = json.load(f) + + coco_api.dataset = dataset + coco_api.createIndex() + return coco_api + + def _load_categories(self): + loaders = {} + + for subset in self._subsets.values(): + loaders.update(subset.loaders) + + self._categories = {} + + label_loader = loaders.get(CocoAnnotationType.labels) + instances_loader = loaders.get(CocoAnnotationType.instances) + person_kp_loader = loaders.get(CocoAnnotationType.person_keypoints) + + if label_loader is None and instances_loader is not None: + label_loader = instances_loader + if label_loader is None and person_kp_loader is not None: + label_loader = person_kp_loader + if label_loader is not None: + label_categories, label_map = \ + self._load_label_categories(label_loader) + self._categories[AnnotationType.label] = label_categories + self._label_map = label_map + + if person_kp_loader is not None: + person_kp_categories = \ + self._load_person_kp_categories(person_kp_loader) + self._categories[AnnotationType.points] = person_kp_categories + + # pylint: disable=no-self-use + def _load_label_categories(self, loader): + catIds = loader.getCatIds() + cats = loader.loadCats(catIds) + + categories = LabelCategories() + label_map = {} + for idx, cat in enumerate(cats): + label_map[cat['id']] = idx + categories.add(name=cat['name'], parent=cat['supercategory']) + + return categories, label_map + # pylint: enable=no-self-use + + def _load_person_kp_categories(self, loader): + catIds = loader.getCatIds() + cats = loader.loadCats(catIds) + + categories = PointsCategories() + for cat in cats: + label_id, _ = self._categories[AnnotationType.label].find(cat['name']) + categories.add(label_id=label_id, + labels=cat['keypoints'], adjacent=cat['skeleton']) + + return categories + + def categories(self): + return self._categories + + def __iter__(self): + for subset_name, subset in self._subsets.items(): + for img_id in subset.items: + yield self._get(img_id, subset_name) + + def __len__(self): + length = 0 + for subset in self._subsets.values(): + length += len(subset) + return length + + def subsets(self): + return list(self._subsets) + + def get_subset(self, name): + return self._subsets[name] + + def _get(self, img_id, subset): + file_name = None + image_info = None + image = None + annotations = [] + for ann_type, loader in self._subsets[subset].loaders.items(): + if image is None: + image_info = loader.loadImgs(img_id)[0] + file_name = image_info['file_name'] + if file_name != '': + file_path = osp.join( + self._path, CocoPath.IMAGES_DIR, subset, file_name) + if osp.exists(file_path): + image = lazy_image(file_path) + + annIds = loader.getAnnIds(imgIds=img_id) + anns = loader.loadAnns(annIds) + + for ann in anns: + self._parse_annotation(ann, ann_type, annotations, image_info) + return DatasetItem(id=img_id, subset=subset, + image=image, annotations=annotations) + + def _parse_label(self, ann): + cat_id = ann.get('category_id') + if cat_id in [0, None]: + return None + return self._label_map[cat_id] + + def _parse_annotation(self, ann, ann_type, parsed_annotations, + image_info=None): + ann_id = ann.get('id') + attributes = {} + if 'score' in ann: + attributes['score'] = ann['score'] + + if ann_type is CocoAnnotationType.instances: + x, y, w, h = ann['bbox'] + label_id = self._parse_label(ann) + group = None + + is_crowd = bool(ann['iscrowd']) + attributes['is_crowd'] = is_crowd + + segmentation = ann.get('segmentation') + if segmentation is not None: + group = ann_id + + if isinstance(segmentation, list): + # polygon -- a single object might consist of multiple parts + for polygon_points in segmentation: + parsed_annotations.append(PolygonObject( + points=polygon_points, label=label_id, + group=group + )) + + # we merge all parts into one mask RLE code + img_h = image_info['height'] + img_w = image_info['width'] + rles = mask_utils.frPyObjects(segmentation, img_h, img_w) + rle = mask_utils.merge(rles) + elif isinstance(segmentation['counts'], list): + # uncompressed RLE + img_h, img_w = segmentation['size'] + rle = mask_utils.frPyObjects([segmentation], img_h, img_w)[0] + else: + # compressed RLE + rle = segmentation + + parsed_annotations.append(RleMask(rle=rle, label=label_id, + group=group + )) + + parsed_annotations.append( + BboxObject(x, y, w, h, label=label_id, + id=ann_id, attributes=attributes, group=group) + ) + elif ann_type is CocoAnnotationType.labels: + label_id = self._parse_label(ann) + parsed_annotations.append( + LabelObject(label=label_id, + id=ann_id, attributes=attributes) + ) + elif ann_type is CocoAnnotationType.person_keypoints: + keypoints = ann['keypoints'] + points = [p for i, p in enumerate(keypoints) if i % 3 != 2] + visibility = keypoints[2::3] + bbox = ann.get('bbox') + label_id = self._parse_label(ann) + group = None + if bbox is not None: + group = ann_id + parsed_annotations.append( + PointsObject(points, visibility, label=label_id, + id=ann_id, attributes=attributes, group=group) + ) + if bbox is not None: + parsed_annotations.append( + BboxObject(*bbox, label=label_id, group=group) + ) + elif ann_type is CocoAnnotationType.captions: + caption = ann['caption'] + parsed_annotations.append( + CaptionObject(caption, + id=ann_id, attributes=attributes) + ) + else: + raise NotImplementedError() + + return parsed_annotations + +class CocoImageInfoExtractor(CocoExtractor): + def __init__(self, path): + super().__init__(path, task=CocoAnnotationType.image_info) + +class CocoCaptionsExtractor(CocoExtractor): + def __init__(self, path): + super().__init__(path, task=CocoAnnotationType.captions) + +class CocoInstancesExtractor(CocoExtractor): + def __init__(self, path): + super().__init__(path, task=CocoAnnotationType.instances) + +class CocoPersonKeypointsExtractor(CocoExtractor): + def __init__(self, path): + super().__init__(path, task=CocoAnnotationType.person_keypoints) + +class CocoLabelsExtractor(CocoExtractor): + def __init__(self, path): + super().__init__(path, task=CocoAnnotationType.labels) \ No newline at end of file diff --git a/datumaro/datumaro/components/extractors/voc.py b/datumaro/datumaro/components/extractors/voc.py new file mode 100644 index 000000000000..1963f2d365cf --- /dev/null +++ b/datumaro/datumaro/components/extractors/voc.py @@ -0,0 +1,705 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import defaultdict +from itertools import chain +import os +import os.path as osp +from xml.etree import ElementTree as ET + +from datumaro.components.extractor import (Extractor, DatasetItem, + AnnotationType, LabelObject, MaskObject, BboxObject, + LabelCategories, MaskCategories +) +from datumaro.components.formats.voc import VocLabel, VocAction, \ + VocBodyPart, VocTask, VocPath, VocColormap, VocInstColormap +from datumaro.util import dir_items +from datumaro.util.image import lazy_image +from datumaro.util.mask_tools import lazy_mask, invert_colormap + + +_inverse_inst_colormap = invert_colormap(VocInstColormap) + +# pylint: disable=pointless-statement +def _make_voc_categories(): + categories = {} + + label_categories = LabelCategories() + for label in chain(VocLabel, VocAction, VocBodyPart): + label_categories.add(label.name) + categories[AnnotationType.label] = label_categories + + def label_id(class_index): + class_label = VocLabel(class_index).name + label_id, _ = label_categories.find(class_label) + return label_id + colormap = { label_id(idx): tuple(color) \ + for idx, color in VocColormap.items() } + mask_categories = MaskCategories(colormap) + mask_categories.inverse_colormap # init inverse colormap + categories[AnnotationType.mask] = mask_categories + + return categories +# pylint: enable=pointless-statement + +class VocExtractor(Extractor): + class Subset(Extractor): + def __init__(self, name, parent): + super().__init__() + self._parent = parent + self._name = name + self.items = [] + + def __iter__(self): + for item in self.items: + yield self._parent._get(item, self._name) + + def __len__(self): + return len(self.items) + + def categories(self): + return self._parent.categories() + + def _load_subsets(self, subsets_dir): + dir_files = dir_items(subsets_dir, '.txt', truncate_ext=True) + subset_names = [s for s in dir_files if '_' not in s] + + subsets = {} + for subset_name in subset_names: + subset = __class__.Subset(subset_name, self) + + with open(osp.join(subsets_dir, subset_name + '.txt'), 'r') as f: + subset.items = [line.split()[0] for line in f] + + subsets[subset_name] = subset + return subsets + + def _load_cls_annotations(self, subsets_dir, subset_names): + dir_files = dir_items(subsets_dir, '.txt', truncate_ext=True) + + label_annotations = defaultdict(list) + label_anno_files = [s for s in dir_files \ + if '_' in s and s[s.rfind('_') + 1:] in subset_names] + for ann_file in label_anno_files: + with open(osp.join(subsets_dir, ann_file + '.txt'), 'r') as f: + label = ann_file[:ann_file.rfind('_')] + label_id = VocLabel[label].value + for line in f: + item, present = line.split() + if present == '1': + label_annotations[item].append(label_id) + + self._annotations[VocTask.classification] = dict(label_annotations) + + def _load_det_annotations(self): + det_anno_dir = osp.join(self._path, VocPath.ANNOTATIONS_DIR) + det_anno_items = dir_items(det_anno_dir, '.xml', truncate_ext=True) + det_annotations = dict() + for ann_item in det_anno_items: + with open(osp.join(det_anno_dir, ann_item + '.xml'), 'r') as f: + ann_file_data = f.read() + ann_file_root = ET.fromstring(ann_file_data) + item = ann_file_root.find('filename').text + item = osp.splitext(item)[0] + det_annotations[item] = ann_file_data + + self._annotations[VocTask.detection] = det_annotations + + def _load_categories(self): + self._categories = _make_voc_categories() + + def __init__(self, path, task): + super().__init__() + + self._path = path + self._subsets = {} + self._categories = {} + self._annotations = {} + self._task = task + + self._load_categories() + + def __len__(self): + length = 0 + for subset in self._subsets.values(): + length += len(subset) + return length + + def subsets(self): + return list(self._subsets) + + def get_subset(self, name): + return self._subsets[name] + + def categories(self): + return self._categories + + def __iter__(self): + for subset_name, subset in self._subsets.items(): + for item in subset.items: + yield self._get(item, subset_name) + + def _get(self, item, subset_name): + image = None + image_path = osp.join(self._path, VocPath.IMAGES_DIR, + item + VocPath.IMAGE_EXT) + if osp.isfile(image_path): + image = lazy_image(image_path) + + annotations = self._get_annotations(item) + + return DatasetItem(annotations=annotations, + id=item, subset=subset_name, image=image) + + def _get_label_id(self, label): + label_id, _ = self._categories[AnnotationType.label].find(label) + assert label_id is not None + return label_id + + def _get_annotations(self, item): + item_annotations = [] + + if self._task is VocTask.segmentation: + segm_path = osp.join(self._path, VocPath.SEGMENTATION_DIR, + item + VocPath.SEGM_EXT) + if osp.isfile(segm_path): + inverse_cls_colormap = \ + self._categories[AnnotationType.mask].inverse_colormap + item_annotations.append(MaskObject( + image=lazy_mask(segm_path, inverse_cls_colormap), + attributes={ 'class': True } + )) + + inst_path = osp.join(self._path, VocPath.INSTANCES_DIR, + item + VocPath.SEGM_EXT) + if osp.isfile(inst_path): + item_annotations.append(MaskObject( + image=lazy_mask(inst_path, _inverse_inst_colormap), + attributes={ 'instances': True } + )) + + cls_annotations = self._annotations.get(VocTask.classification) + if cls_annotations is not None and \ + self._task is VocTask.classification: + item_labels = cls_annotations.get(item) + if item_labels is not None: + for label in item_labels: + label_id = self._get_label_id(VocLabel(label).name) + item_annotations.append(LabelObject(label_id)) + + det_annotations = self._annotations.get(VocTask.detection) + if det_annotations is not None: + det_annotations = det_annotations.get(item) + if det_annotations is not None: + root_elem = ET.fromstring(det_annotations) + + for obj_id, object_elem in enumerate(root_elem.findall('object')): + attributes = {} + group = None + + obj_label_id = None + label_elem = object_elem.find('name') + if label_elem is not None: + obj_label_id = self._get_label_id(label_elem.text) + + obj_bbox = self._parse_bbox(object_elem) + + if obj_label_id is None or obj_bbox is None: + continue + + difficult_elem = object_elem.find('difficult') + if difficult_elem is not None: + attributes['difficult'] = (difficult_elem.text == '1') + + truncated_elem = object_elem.find('truncated') + if truncated_elem is not None: + attributes['truncated'] = (truncated_elem.text == '1') + + occluded_elem = object_elem.find('occluded') + if occluded_elem is not None: + attributes['occluded'] = (occluded_elem.text == '1') + + pose_elem = object_elem.find('pose') + if pose_elem is not None: + attributes['pose'] = pose_elem.text + + point_elem = object_elem.find('point') + if point_elem is not None: + point_x = point_elem.find('x') + point_y = point_elem.find('y') + point = [float(point_x.text), float(point_y.text)] + attributes['point'] = point + + actions_elem = object_elem.find('actions') + if actions_elem is not None and \ + self._task is VocTask.action_classification: + for action in VocAction: + action_elem = actions_elem.find(action.name) + if action_elem is None or action_elem.text != '1': + continue + + act_label_id = self._get_label_id(action.name) + assert group in [None, obj_id] + group = obj_id + item_annotations.append(LabelObject(act_label_id, + group=obj_id)) + + if self._task is VocTask.person_layout: + for part_elem in object_elem.findall('part'): + part = part_elem.find('name').text + part_label_id = self._get_label_id(part) + bbox = self._parse_bbox(part_elem) + group = obj_id + item_annotations.append(BboxObject( + *bbox, label=part_label_id, + group=obj_id)) + + if self._task in [VocTask.action_classification, VocTask.person_layout]: + if group is None: + continue + + item_annotations.append(BboxObject(*obj_bbox, label=obj_label_id, + attributes=attributes, id=obj_id, group=group)) + + return item_annotations + + @staticmethod + def _parse_bbox(object_elem): + try: + bbox_elem = object_elem.find('bndbox') + xmin = int(bbox_elem.find('xmin').text) + xmax = int(bbox_elem.find('xmax').text) + ymin = int(bbox_elem.find('ymin').text) + ymax = int(bbox_elem.find('ymax').text) + return [xmin, ymin, xmax - xmin, ymax - ymin] + except Exception: + return None + +class VocClassificationExtractor(VocExtractor): + _ANNO_DIR = 'Main' + + def __init__(self, path): + super().__init__(path, task=VocTask.classification) + + subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR) + subsets = self._load_subsets(subsets_dir) + self._subsets = subsets + + self._load_cls_annotations(subsets_dir, subsets) + +class VocDetectionExtractor(VocExtractor): + _ANNO_DIR = 'Main' + + def __init__(self, path): + super().__init__(path, task=VocTask.detection) + + subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR) + subsets = self._load_subsets(subsets_dir) + self._subsets = subsets + + self._load_det_annotations() + +class VocSegmentationExtractor(VocExtractor): + _ANNO_DIR = 'Segmentation' + + def __init__(self, path): + super().__init__(path, task=VocTask.segmentation) + + subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR) + subsets = self._load_subsets(subsets_dir) + self._subsets = subsets + +class VocLayoutExtractor(VocExtractor): + _ANNO_DIR = 'Layout' + + def __init__(self, path): + super().__init__(path, task=VocTask.person_layout) + + subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR) + subsets = self._load_subsets(subsets_dir) + self._subsets = subsets + + self._load_det_annotations() + +class VocActionExtractor(VocExtractor): + _ANNO_DIR = 'Action' + + def __init__(self, path): + super().__init__(path, task=VocTask.action_classification) + + subsets_dir = osp.join(path, VocPath.SUBSETS_DIR, self._ANNO_DIR) + subsets = self._load_subsets(subsets_dir) + self._subsets = subsets + + self._load_det_annotations() + + +class VocResultsExtractor(Extractor): + class Subset(Extractor): + def __init__(self, name, parent): + super().__init__() + self._parent = parent + self._name = name + self.items = [] + + def __iter__(self): + for item in self.items: + yield self._parent._get(item, self._name) + + def __len__(self): + return len(self.items) + + def categories(self): + return self._parent.categories() + + _SUPPORTED_TASKS = { + VocTask.classification: { + 'dir': 'Main', + 'mark': 'cls', + 'ext': '.txt', + 'path' : ['%(comp)s_cls_%(subset)s_%(label)s.txt'], + 'comp': ['comp1', 'comp2'], + }, + VocTask.detection: { + 'dir': 'Main', + 'mark': 'det', + 'ext': '.txt', + 'path': ['%(comp)s_det_%(subset)s_%(label)s.txt'], + 'comp': ['comp3', 'comp4'], + }, + VocTask.segmentation: { + 'dir': 'Segmentation', + 'mark': ['cls', 'inst'], + 'ext': '.png', + 'path': ['%(comp)s_%(subset)s_cls', '%(item)s.png'], + 'comp': ['comp5', 'comp6'], + }, + VocTask.person_layout: { + 'dir': 'Layout', + 'mark': 'layout', + 'ext': '.xml', + 'path': ['%(comp)s_layout_%(subset)s.xml'], + 'comp': ['comp7', 'comp8'], + }, + VocTask.action_classification: { + 'dir': 'Action', + 'mark': 'action', + 'ext': '.txt', + 'path': ['%(comp)s_action_%(subset)s_%(label)s.txt'], + 'comp': ['comp9', 'comp10'], + }, + } + + def _parse_txt_ann(self, path, subsets, annotations, task): + task_desc = self._SUPPORTED_TASKS[task] + task_dir = osp.join(path, task_desc['dir']) + ann_ext = task_desc['ext'] + if not osp.isdir(task_dir): + return + + ann_files = dir_items(task_dir, ann_ext, truncate_ext=True) + + for ann_file in ann_files: + ann_parts = filter(None, ann_file.strip().split('_')) + if len(ann_parts) != 4: + continue + _, mark, subset_name, label = ann_parts + if mark != task_desc['mark']: + continue + + label_id = VocLabel[label].value + anns = defaultdict(list) + with open(osp.join(task_dir, ann_file + ann_ext), 'r') as f: + for line in f: + line_parts = line.split() + item = line_parts[0] + anns[item].append((label_id, *line_parts[1:])) + + subset = VocResultsExtractor.Subset(subset_name, self) + subset.items = list(anns) + + subsets[subset_name] = subset + annotations[subset_name] = dict(anns) + + def _parse_classification(self, path, subsets, annotations): + self._parse_txt_ann(path, subsets, annotations, + VocTask.classification) + + def _parse_detection(self, path, subsets, annotations): + self._parse_txt_ann(path, subsets, annotations, + VocTask.detection) + + def _parse_action(self, path, subsets, annotations): + self._parse_txt_ann(path, subsets, annotations, + VocTask.action_classification) + + def _load_categories(self): + self._categories = _make_voc_categories() + + def _get_label_id(self, label): + label_id = self._categories[AnnotationType.label].find(label) + assert label_id is not None + return label_id + + def __init__(self, path): + super().__init__() + + self._path = path + self._subsets = {} + self._annotations = {} + + self._load_categories() + + def __len__(self): + length = 0 + for subset in self._subsets.values(): + length += len(subset) + return length + + def subsets(self): + return list(self._subsets) + + def get_subset(self, name): + return self._subsets[name] + + def categories(self): + return self._categories + + def __iter__(self): + for subset_name, subset in self._subsets.items(): + for item in subset.items: + yield self._get(item, subset_name) + + def _get(self, item, subset_name): + image = None + image_path = osp.join(self._path, VocPath.IMAGES_DIR, + item + VocPath.IMAGE_EXT) + if osp.isfile(image_path): + image = lazy_image(image_path) + + annotations = self._get_annotations(item, subset_name) + + return DatasetItem(annotations=annotations, + id=item, subset=subset_name, image=image) + + def _get_annotations(self, item, subset_name): + raise NotImplementedError() + +class VocComp_1_2_Extractor(VocResultsExtractor): + def __init__(self, path): + super().__init__(path) + + subsets = {} + annotations = defaultdict(dict) + + self._parse_classification(path, subsets, annotations) + + self._subsets = subsets + self._annotations = dict(annotations) + + def _get_annotations(self, item, subset_name): + annotations = [] + + cls_ann = self._annotations[subset_name].get(item) + if cls_ann is not None: + for desc in cls_ann: + label_id, conf = desc + label_id = self._get_label_id(VocLabel(int(label_id)).name) + annotations.append(LabelObject( + label_id, + attributes={ 'score': float(conf) } + )) + + return annotations + +class VocComp_3_4_Extractor(VocResultsExtractor): + def __init__(self, path): + super().__init__(path) + + subsets = {} + annotations = defaultdict(dict) + + self._parse_detection(path, subsets, annotations) + + self._subsets = subsets + self._annotations = dict(annotations) + + def _get_annotations(self, item, subset_name): + annotations = [] + + det_ann = self._annotations[subset_name].get(item) + if det_ann is not None: + for desc in det_ann: + label_id, conf, left, top, right, bottom = desc + label_id = self._get_label_id(VocLabel(int(label_id)).name) + annotations.append(BboxObject( + x=float(left), y=float(top), + w=float(right) - float(left), h=float(bottom) - float(top), + label=label_id, + attributes={ 'score': float(conf) } + )) + + return annotations + +class VocComp_5_6_Extractor(VocResultsExtractor): + def __init__(self, path): + super().__init__(path) + + subsets = {} + annotations = defaultdict(dict) + + task_dir = osp.join(path, 'Segmentation') + if not osp.isdir(task_dir): + return + + ann_files = os.listdir(task_dir) + + for ann_dir in ann_files: + ann_parts = filter(None, ann_dir.strip().split('_')) + if len(ann_parts) != 4: + continue + _, subset_name, mark = ann_parts + if mark not in ['cls', 'inst']: + continue + + item_dir = osp.join(task_dir, ann_dir) + items = dir_items(item_dir, '.png', truncate_ext=True) + items = { name: osp.join(item_dir, item + '.png') \ + for name, item in items } + + subset = VocResultsExtractor.Subset(subset_name, self) + subset.items = list(items) + + subsets[subset_name] = subset + annotations[subset_name][mark] = items + + self._subsets = subsets + self._annotations = dict(annotations) + + def _get_annotations(self, item, subset_name): + annotations = [] + + segm_ann = self._annotations[subset_name] + cls_image_path = segm_ann.get(item) + if cls_image_path and osp.isfile(cls_image_path): + inverse_cls_colormap = \ + self._categories[AnnotationType.mask].inverse_colormap + annotations.append(MaskObject( + image=lazy_mask(cls_image_path, inverse_cls_colormap), + attributes={ 'class': True } + )) + + inst_ann = self._annotations[subset_name] + inst_image_path = inst_ann.get(item) + if inst_image_path and osp.isfile(inst_image_path): + annotations.append(MaskObject( + image=lazy_mask(inst_image_path, _inverse_inst_colormap), + attributes={ 'instances': True } + )) + + return annotations + +class VocComp_7_8_Extractor(VocResultsExtractor): + def __init__(self, path): + super().__init__(path) + + subsets = {} + annotations = defaultdict(dict) + + task = VocTask.person_layout + task_desc = self._SUPPORTED_TASKS[task] + task_dir = osp.join(path, task_desc['dir']) + if not osp.isdir(task_dir): + return + + ann_ext = task_desc['ext'] + ann_files = dir_items(task_dir, ann_ext, truncate_ext=True) + + for ann_file in ann_files: + ann_parts = filter(None, ann_file.strip().split('_')) + if len(ann_parts) != 4: + continue + _, mark, subset_name, _ = ann_parts + if mark != task_desc['mark']: + continue + + layouts = {} + root = ET.parse(osp.join(task_dir, ann_file + ann_ext)) + root_elem = root.getroot() + for layout_elem in root_elem.findall('layout'): + item = layout_elem.find('image').text + obj_id = int(layout_elem.find('object').text) + conf = float(layout_elem.find('confidence').text) + parts = [] + for part_elem in layout_elem.findall('part'): + label_id = VocBodyPart[part_elem.find('class').text].value + bbox_elem = part_elem.find('bndbox') + xmin = float(bbox_elem.find('xmin').text) + xmax = float(bbox_elem.find('xmax').text) + ymin = float(bbox_elem.find('ymin').text) + ymax = float(bbox_elem.find('ymax').text) + bbox = [xmin, ymin, xmax - xmin, ymax - ymin] + parts.append((label_id, bbox)) + layouts[item] = [obj_id, conf, parts] + + subset = VocResultsExtractor.Subset(subset_name, self) + subset.items = list(layouts) + + subsets[subset_name] = subset + annotations[subset_name] = layouts + + self._subsets = subsets + self._annotations = dict(annotations) + + def _get_annotations(self, item, subset_name): + annotations = [] + + layout_ann = self._annotations[subset_name].get(item) + if layout_ann is not None: + for desc in layout_ann: + obj_id, conf, parts = desc + attributes = { + 'score': conf, + 'object_id': obj_id, + } + + for part in parts: + part_id, bbox = part + label_id = self._get_label_id(VocBodyPart(part_id).name) + annotations.append(BboxObject( + *bbox, label=label_id, + attributes=attributes)) + + return annotations + +class VocComp_9_10_Extractor(VocResultsExtractor): + def __init__(self, path): + super().__init__(path) + + subsets = {} + annotations = defaultdict(dict) + + self._parse_action(path, subsets, annotations) + + self._subsets = subsets + self._annotations = dict(annotations) + + def _get_annotations(self, item, subset_name): + annotations = [] + + action_ann = self._annotations[subset_name].get(item) + if action_ann is not None: + for desc in action_ann: + action_id, obj_id, conf = desc + label_id = self._get_label_id(VocAction(int(action_id)).name) + annotations.append(LabelObject( + label_id, + attributes={ + 'score': conf, + 'object_id': int(obj_id), + } + )) + + return annotations \ No newline at end of file diff --git a/datumaro/datumaro/components/formats/__init__.py b/datumaro/datumaro/components/formats/__init__.py new file mode 100644 index 000000000000..a9773073830c --- /dev/null +++ b/datumaro/datumaro/components/formats/__init__.py @@ -0,0 +1,5 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + diff --git a/datumaro/datumaro/components/formats/datumaro.py b/datumaro/datumaro/components/formats/datumaro.py new file mode 100644 index 000000000000..ef587b9bbbe9 --- /dev/null +++ b/datumaro/datumaro/components/formats/datumaro.py @@ -0,0 +1,12 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +class DatumaroPath: + IMAGES_DIR = 'images' + ANNOTATIONS_DIR = 'annotations' + MASKS_DIR = 'masks' + + IMAGE_EXT = '.jpg' + MASK_EXT = '.png' diff --git a/datumaro/datumaro/components/formats/ms_coco.py b/datumaro/datumaro/components/formats/ms_coco.py new file mode 100644 index 000000000000..bb3bb82bafdb --- /dev/null +++ b/datumaro/datumaro/components/formats/ms_coco.py @@ -0,0 +1,23 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from enum import Enum + + +CocoAnnotationType = Enum('CocoAnnotationType', [ + 'instances', + 'person_keypoints', + 'captions', + 'labels', # extension, does not exist in original COCO format + 'image_info', + 'panoptic', + 'stuff', +]) + +class CocoPath: + IMAGES_DIR = 'images' + ANNOTATIONS_DIR = 'annotations' + + IMAGE_EXT = '.jpg' \ No newline at end of file diff --git a/datumaro/datumaro/components/formats/voc.py b/datumaro/datumaro/components/formats/voc.py new file mode 100644 index 000000000000..b7acaffde6a4 --- /dev/null +++ b/datumaro/datumaro/components/formats/voc.py @@ -0,0 +1,103 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from enum import Enum +import numpy as np + + +VocTask = Enum('VocTask', [ + 'classification', + 'detection', + 'segmentation', + 'action_classification', + 'person_layout', +]) + +VocLabel = Enum('VocLabel', [ + ('aeroplane', 0), + ('bicycle', 1), + ('bird', 2), + ('boat', 3), + ('bottle', 4), + ('bus', 5), + ('car', 6), + ('cat', 7), + ('chair', 8), + ('cow', 9), + ('diningtable', 10), + ('dog', 11), + ('horse', 12), + ('motorbike', 13), + ('person', 14), + ('pottedplant', 15), + ('sheep', 16), + ('sofa', 17), + ('train', 18), + ('tvmonitor', 19), +]) + +VocPose = Enum('VocPose', [ + 'Unspecified', + 'Left', + 'Right', + 'Frontal', + 'Rear', +]) + +VocBodyPart = Enum('VocBodyPart', [ + 'head', + 'hand', + 'foot', +]) + +VocAction = Enum('VocAction', [ + 'other', + 'jumping', + 'phoning', + 'playinginstrument', + 'reading', + 'ridingbike', + 'ridinghorse', + 'running', + 'takingphoto', + 'usingcomputer', + 'walking', +]) + +def generate_colormap(length=256): + def get_bit(number, index): + return (number >> index) & 1 + + colormap = np.zeros((length, 3), dtype=int) + indices = np.arange(length, dtype=int) + + for j in range(7, -1, -1): + for c in range(3): + colormap[:, c] |= get_bit(indices, c) << j + indices >>= 3 + + return { + id: tuple(color) for id, color in enumerate(colormap) + } + +VocColormap = generate_colormap(len(VocLabel)) +VocInstColormap = generate_colormap(256) + +class VocPath: + IMAGES_DIR = 'JPEGImages' + ANNOTATIONS_DIR = 'Annotations' + SEGMENTATION_DIR = 'SegmentationClass' + INSTANCES_DIR = 'SegmentationObject' + SUBSETS_DIR = 'ImageSets' + IMAGE_EXT = '.jpg' + SEGM_EXT = '.png' + + TASK_DIR = { + VocTask.classification: 'Main', + VocTask.detection: 'Main', + VocTask.segmentation: 'Segmentation', + VocTask.action_classification: 'Action', + VocTask.person_layout: 'Layout', + } \ No newline at end of file diff --git a/datumaro/datumaro/components/importers/__init__.py b/datumaro/datumaro/components/importers/__init__.py new file mode 100644 index 000000000000..4383b467b660 --- /dev/null +++ b/datumaro/datumaro/components/importers/__init__.py @@ -0,0 +1,24 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from datumaro.components.importers.datumaro import DatumaroImporter + +from datumaro.components.importers.ms_coco import ( + CocoImporter, +) + +from datumaro.components.importers.voc import ( + VocImporter, + VocResultsImporter, +) + +items = [ + ('datumaro', DatumaroImporter), + + ('ms_coco', CocoImporter), + + ('voc', VocImporter), + ('voc_results', VocResultsImporter), +] \ No newline at end of file diff --git a/datumaro/datumaro/components/importers/datumaro.py b/datumaro/datumaro/components/importers/datumaro.py new file mode 100644 index 000000000000..40939b90cf18 --- /dev/null +++ b/datumaro/datumaro/components/importers/datumaro.py @@ -0,0 +1,25 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os.path as osp + + +class DatumaroImporter: + EXTRACTOR_NAME = 'datumaro' + + def __call__(self, path): + from datumaro.components.project import Project # cyclic import + project = Project() + + if not osp.exists(path): + raise Exception("Failed to find 'datumaro' dataset at '%s'" % path) + + source_name = osp.splitext(osp.basename(path))[0] + project.add_source(source_name, { + 'url': path, + 'format': self.EXTRACTOR_NAME, + }) + + return project \ No newline at end of file diff --git a/datumaro/datumaro/components/importers/ms_coco.py b/datumaro/datumaro/components/importers/ms_coco.py new file mode 100644 index 000000000000..b3af4bfa5af2 --- /dev/null +++ b/datumaro/datumaro/components/importers/ms_coco.py @@ -0,0 +1,69 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import defaultdict +import os +import os.path as osp + +from datumaro.components.formats.ms_coco import CocoAnnotationType, CocoPath + + +class CocoImporter: + _COCO_EXTRACTORS = { + CocoAnnotationType.instances: 'coco_instances', + CocoAnnotationType.person_keypoints: 'coco_person_kp', + CocoAnnotationType.captions: 'coco_captions', + CocoAnnotationType.labels: 'coco_labels', + CocoAnnotationType.image_info: 'coco_images', + } + + def __init__(self, task_filter=None): + self._task_filter = task_filter + + def __call__(self, path): + from datumaro.components.project import Project # cyclic import + project = Project() + + subsets = self.find_subsets(path) + + if len(subsets) == 0: + raise Exception("Failed to find 'coco' dataset at '%s'" % path) + + for ann_files in subsets.values(): + for ann_type, ann_file in ann_files.items(): + source_name = osp.splitext(osp.basename(ann_file))[0] + project.add_source(source_name, { + 'url': ann_file, + 'format': self._COCO_EXTRACTORS[ann_type], + }) + + return project + + @staticmethod + def find_subsets(dataset_dir): + ann_dir = os.path.join(dataset_dir, CocoPath.ANNOTATIONS_DIR) + if not osp.isdir(ann_dir): + raise NotADirectoryError( + 'COCO annotations directory not found at "%s"' % ann_dir) + + subsets = defaultdict(dict) + for ann_file in os.listdir(ann_dir): + subset_path = osp.join(ann_dir, ann_file) + if not subset_path.endswith('.json'): + continue + + name_parts = osp.splitext(ann_file)[0].rsplit('_', maxsplit=1) + ann_type = name_parts[0] + try: + ann_type = CocoAnnotationType[ann_type] + except KeyError: + raise Exception( + 'Unknown subset type %s, only known are: %s' % \ + (ann_type, + ', '.join([e.name for e in CocoAnnotationType]) + )) + subset_name = name_parts[1] + subsets[subset_name][ann_type] = subset_path + return dict(subsets) \ No newline at end of file diff --git a/datumaro/datumaro/components/importers/voc.py b/datumaro/datumaro/components/importers/voc.py new file mode 100644 index 000000000000..432cf374141e --- /dev/null +++ b/datumaro/datumaro/components/importers/voc.py @@ -0,0 +1,77 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os +import os.path as osp + +from datumaro.components.formats.voc import VocTask, VocPath +from datumaro.util import find + + +class VocImporter: + _TASKS = [ + (VocTask.classification, 'voc_cls', 'Main'), + (VocTask.detection, 'voc_det', 'Main'), + (VocTask.segmentation, 'voc_segm', 'Segmentation'), + (VocTask.person_layout, 'voc_layout', 'Layout'), + (VocTask.action_classification, 'voc_action', 'Action'), + ] + + def __call__(self, path): + from datumaro.components.project import Project # cyclic import + project = Project() + + for task, extractor_type, task_dir in self._TASKS: + task_dir = osp.join(path, VocPath.SUBSETS_DIR, task_dir) + if not osp.isdir(task_dir): + continue + + project.add_source(task.name, { + 'url': path, + 'format': extractor_type, + }) + + if len(project.config.sources) == 0: + raise Exception("Failed to find 'voc' dataset at '%s'" % path) + + return project + + +class VocResultsImporter: + _TASKS = [ + ('comp1', 'voc_comp_1_2', 'Main'), + ('comp2', 'voc_comp_1_2', 'Main'), + ('comp3', 'voc_comp_3_4', 'Main'), + ('comp4', 'voc_comp_3_4', 'Main'), + ('comp5', 'voc_comp_5_6', 'Segmentation'), + ('comp6', 'voc_comp_5_6', 'Segmentation'), + ('comp7', 'voc_comp_7_8', 'Layout'), + ('comp8', 'voc_comp_7_8', 'Layout'), + ('comp9', 'voc_comp_9_10', 'Action'), + ('comp10', 'voc_comp_9_10', 'Action'), + ] + + def __call__(self, path): + from datumaro.components.project import Project # cyclic import + project = Project() + + for task_name, extractor_type, task_dir in self._TASKS: + task_dir = osp.join(path, task_dir) + if not osp.isdir(task_dir): + continue + dir_items = os.listdir(task_dir) + if not find(dir_items, lambda x: x == task_name): + continue + + project.add_source(task_name, { + 'url': task_dir, + 'format': extractor_type, + }) + + if len(project.config.sources) == 0: + raise Exception("Failed to find 'voc_results' dataset at '%s'" % \ + path) + + return project \ No newline at end of file diff --git a/datumaro/datumaro/components/launcher.py b/datumaro/datumaro/components/launcher.py new file mode 100644 index 000000000000..0e11e8cf076e --- /dev/null +++ b/datumaro/datumaro/components/launcher.py @@ -0,0 +1,95 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import numpy as np + +from datumaro.components.extractor import DatasetItem, Extractor + + +# pylint: disable=no-self-use +class Launcher: + def __init__(self): + pass + + def launch(self, inputs): + raise NotImplementedError() + + def preferred_input_size(self): + return None + + def get_categories(self): + return None +# pylint: enable=no-self-use + +class InferenceWrapper(Extractor): + class ItemWrapper(DatasetItem): + def __init__(self, item, annotations, path=None): + super().__init__(id=item.id) + self._annotations = annotations + self._item = item + self._path = path + + @DatasetItem.id.getter + def id(self): + return self._item.id + + @DatasetItem.subset.getter + def subset(self): + return self._item.subset + + @DatasetItem.path.getter + def path(self): + return self._path + + @DatasetItem.annotations.getter + def annotations(self): + return self._annotations + + @DatasetItem.image.getter + def image(self): + return self._item.image + + def __init__(self, extractor, launcher, batch_size=1): + super().__init__() + self._extractor = extractor + self._launcher = launcher + self._batch_size = batch_size + + def __iter__(self): + stop = False + data_iter = iter(self._extractor) + while not stop: + batch_items = [] + try: + for _ in range(self._batch_size): + item = next(data_iter) + batch_items.append(item) + except StopIteration: + stop = True + if len(batch_items) == 0: + break + + inputs = np.array([item.image for item in batch_items]) + inference = self._launcher.launch(inputs) + + for item, annotations in zip(batch_items, inference): + yield self.ItemWrapper(item, annotations) + + def __len__(self): + return len(self._extractor) + + def subsets(self): + return self._extractor.subsets() + + def get_subset(self, name): + subset = self._extractor.get_subset(name) + return InferenceWrapper(subset, + self._launcher, self._batch_size) + + def categories(self): + launcher_override = self._launcher.get_categories() + if launcher_override is not None: + return launcher_override + return self._extractor.categories() \ No newline at end of file diff --git a/datumaro/datumaro/components/launchers/__init__.py b/datumaro/datumaro/components/launchers/__init__.py new file mode 100644 index 000000000000..8d613a2ac53a --- /dev/null +++ b/datumaro/datumaro/components/launchers/__init__.py @@ -0,0 +1,13 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +items = [ +] + +try: + from datumaro.components.launchers.openvino import OpenVinoLauncher + items.append(('openvino', OpenVinoLauncher)) +except ImportError: + pass diff --git a/datumaro/datumaro/components/launchers/openvino.py b/datumaro/datumaro/components/launchers/openvino.py new file mode 100644 index 000000000000..28155539b98f --- /dev/null +++ b/datumaro/datumaro/components/launchers/openvino.py @@ -0,0 +1,189 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +# pylint: disable=exec-used + +import cv2 +import os +import os.path as osp +import numpy as np +import subprocess +import platform + +from openvino.inference_engine import IENetwork, IEPlugin + +from datumaro.components.launcher import Launcher + + +class InterpreterScript: + def __init__(self, path): + with open(path, 'r') as f: + script = f.read() + + context = {} + exec(script, context, context) + + process_outputs = context['process_outputs'] + assert callable(process_outputs) + self.__dict__['process_outputs'] = process_outputs + + get_categories = context.get('get_categories') + assert callable(get_categories) or get_categories is None + self.__dict__['get_categories'] = get_categories + + @staticmethod + def get_categories(): + return None + + @staticmethod + def process_outputs(inputs, outputs): + return [] + +class OpenVinoLauncher(Launcher): + _DEFAULT_IE_PLUGINS_PATH = "/opt/intel/openvino_2019.1.144/deployment_tools/inference_engine/lib/intel64" + _IE_PLUGINS_PATH = os.getenv("IE_PLUGINS_PATH", _DEFAULT_IE_PLUGINS_PATH) + + @staticmethod + def _check_instruction_set(instruction): + return instruction == str.strip( + subprocess.check_output( + 'lscpu | grep -o "{}" | head -1'.format(instruction), shell=True + ).decode('utf-8') + ) + + @staticmethod + def make_plugin(device='cpu', plugins_path=_IE_PLUGINS_PATH): + if plugins_path is None or not osp.isdir(plugins_path): + raise Exception('Inference engine plugins directory "%s" not found' % \ + (plugins_path)) + + plugin = IEPlugin(device='CPU', plugin_dirs=[plugins_path]) + if (OpenVinoLauncher._check_instruction_set('avx2')): + plugin.add_cpu_extension(os.path.join(plugins_path, + 'libcpu_extension_avx2.so')) + elif (OpenVinoLauncher._check_instruction_set('sse4')): + plugin.add_cpu_extension(os.path.join(plugins_path, + 'libcpu_extension_sse4.so')) + elif platform.system() == 'Darwin': + plugin.add_cpu_extension(os.path.join(plugins_path, + 'libcpu_extension.dylib')) + else: + raise Exception('Inference engine requires support of avx2 or sse4') + + return plugin + + @staticmethod + def make_network(model, weights): + return IENetwork.from_ir(model=model, weights=weights) + + def __init__(self, description, weights, interpretation_script, + plugins_path=None, model_dir=None, **kwargs): + if model_dir is None: + model_dir = '' + if not osp.isfile(description): + description = osp.join(model_dir, description) + if not osp.isfile(description): + raise Exception('Failed to open model description file "%s"' % \ + (description)) + + if not osp.isfile(weights): + weights = osp.join(model_dir, weights) + if not osp.isfile(weights): + raise Exception('Failed to open model weights file "%s"' % \ + (weights)) + + if not osp.isfile(interpretation_script): + interpretation_script = \ + osp.join(model_dir, interpretation_script) + if not osp.isfile(interpretation_script): + raise Exception('Failed to open model interpretation script file "%s"' % \ + (interpretation_script)) + + self._interpreter_script = InterpreterScript(interpretation_script) + + if plugins_path is None: + plugins_path = OpenVinoLauncher._IE_PLUGINS_PATH + + plugin = OpenVinoLauncher.make_plugin(plugins_path=plugins_path) + network = OpenVinoLauncher.make_network(description, weights) + self._network = network + self._plugin = plugin + self._load_executable_net() + + def _load_executable_net(self, batch_size=1): + network = self._network + plugin = self._plugin + + supported_layers = plugin.get_supported_layers(network) + not_supported_layers = [l for l in network.layers.keys() if l not in supported_layers] + if len(not_supported_layers) != 0: + raise Exception('Following layers are not supported by the plugin' + ' for the specified device {}:\n {}'. format( \ + plugin.device, ", ".join(not_supported_layers))) + + iter_inputs = iter(network.inputs) + self._input_blob_name = next(iter_inputs) + self._output_blob_name = next(iter(network.outputs)) + + # NOTE: handling for the inclusion of `image_info` in OpenVino2019 + self._require_image_info = 'image_info' in network.inputs + if self._input_blob_name == 'image_info': + self._input_blob_name = next(iter_inputs) + + input_type = network.inputs[self._input_blob_name] + self._input_layout = input_type if isinstance(input_type, list) else input_type.shape + + self._input_layout[0] = batch_size + network.reshape({self._input_blob_name: self._input_layout}) + self._batch_size = batch_size + + self._net = plugin.load(network=network, num_requests=1) + + def infer(self, inputs): + assert len(inputs.shape) == 4, \ + "Expected an input image in (N, H, W, C) format, got %s" % \ + (inputs.shape) + assert inputs.shape[3] == 3, \ + "Expected BGR input" + + n, c, h, w = self._input_layout + if inputs.shape[1:3] != (h, w): + resized_inputs = np.empty((n, h, w, c), dtype=inputs.dtype) + for inp, resized_input in zip(inputs, resized_inputs): + cv2.resize(inp, (w, h), resized_input) + inputs = resized_inputs + inputs = inputs.transpose((0, 3, 1, 2)) # NHWC to NCHW + inputs = {self._input_blob_name: inputs} + if self._require_image_info: + info = np.zeros([1, 3]) + info[0, 0] = h + info[0, 1] = w + info[0, 2] = 1.0 # scale + inputs['image_info'] = info + + results = self._net.infer(inputs) + if len(results) == 1: + return results[self._output_blob_name] + else: + return results + + def launch(self, inputs): + batch_size = len(inputs) + if self._batch_size < batch_size: + self._load_executable_net(batch_size) + + outputs = self.infer(inputs) + results = self.process_outputs(inputs, outputs) + return results + + def get_categories(self): + return self._interpreter_script.get_categories() + + def process_outputs(self, inputs, outputs): + return self._interpreter_script.process_outputs(inputs, outputs) + + def preferred_input_size(self): + _, _, h, w = self._input_layout + return (h, w) diff --git a/datumaro/datumaro/components/project.py b/datumaro/datumaro/components/project.py new file mode 100644 index 000000000000..e51c2baabbc2 --- /dev/null +++ b/datumaro/datumaro/components/project.py @@ -0,0 +1,712 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import OrderedDict, defaultdict +import git +import importlib +from functools import reduce +import logging as log +import os +import os.path as osp +import sys + +from datumaro.components.config import Config, DEFAULT_FORMAT +from datumaro.components.config_model import * +from datumaro.components.extractor import * +from datumaro.components.launcher import * +from datumaro.components.dataset_filter import XPathDatasetFilter + + +def import_foreign_module(name, path): + module = None + default_path = sys.path.copy() + try: + sys.path = [ osp.abspath(path), ] + default_path + sys.modules.pop(name, None) # remove from cache + module = importlib.import_module(name) + sys.modules.pop(name) # remove from cache + except ImportError as e: + log.warn("Failed to import module '%s': %s" % (name, e)) + finally: + sys.path = default_path + return module + + +class Registry: + def __init__(self, config=None, item_type=None): + self.item_type = item_type + + self.items = {} + + if config is not None: + self.load(config) + + def load(self, config): + pass + + def register(self, name, value): + if self.item_type: + value = self.item_type(value) + self.items[name] = value + return value + + def unregister(self, name): + return self.items.pop(name, None) + + def get(self, key): + return self.items[key] # returns a class / ctor + + +class ModelRegistry(Registry): + def __init__(self, config=None): + super().__init__(config, item_type=Model) + + def load(self, config): + # TODO: list default dir, insert values + if 'models' in config: + for name, model in config.models.items(): + self.register(name, model) + + +class SourceRegistry(Registry): + def __init__(self, config=None): + super().__init__(config, item_type=Source) + + def load(self, config): + # TODO: list default dir, insert values + if 'sources' in config: + for name, source in config.sources.items(): + self.register(name, source) + + +class ModuleRegistry(Registry): + def __init__(self, config=None, builtin=None, local=None): + super().__init__(config) + + if builtin is not None: + for k, v in builtin: + self.register(k, v) + if local is not None: + for k, v in local: + self.register(k, v) + + +class GitWrapper: + def __init__(self, config=None): + self.repo = None + + if config is not None: + self.init(config.project_dir) + + @staticmethod + def _git_dir(base_path): + return osp.join(base_path, '.git') + + def init(self, path): + spawn = not osp.isdir(GitWrapper._git_dir(path)) + self.repo = git.Repo.init(path=path) + if spawn: + author = git.Actor("Nobody", "nobody@example.com") + self.repo.index.commit('Initial commit', author=author) + return self.repo + + def get_repo(self): + return self.repo + + def is_initialized(self): + return self.repo is not None + + def create_submodule(self, name, dst_dir, **kwargs): + self.repo.create_submodule(name, dst_dir, **kwargs) + + def has_submodule(self, name): + return name in [submodule.name for submodule in self.repo.submodules] + + def remove_submodule(self, name, **kwargs): + return self.repo.submodule(name).remove(**kwargs) + +def load_project_as_dataset(url): + # symbol forward declaration + raise NotImplementedError() + +class Environment: + PROJECT_EXTRACTOR_NAME = 'project' + + def __init__(self, config=None): + config = Config(config, + fallback=PROJECT_DEFAULT_CONFIG, schema=PROJECT_SCHEMA) + + env_dir = osp.join(config.project_dir, config.env_dir) + env_config_path = osp.join(env_dir, config.env_filename) + env_config = Config(fallback=ENV_DEFAULT_CONFIG, schema=ENV_SCHEMA) + if osp.isfile(env_config_path): + env_config.update(Config.parse(env_config_path)) + + self.config = env_config + + self.models = ModelRegistry(env_config) + self.sources = SourceRegistry(config) + + import datumaro.components.importers as builtin_importers + builtin_importers = builtin_importers.items + custom_importers = self._get_custom_module_items( + env_dir, env_config.importers_dir) + self.importers = ModuleRegistry(config, + builtin=builtin_importers, local=custom_importers) + + import datumaro.components.extractors as builtin_extractors + builtin_extractors = builtin_extractors.items + custom_extractors = self._get_custom_module_items( + env_dir, env_config.extractors_dir) + self.extractors = ModuleRegistry(config, + builtin=builtin_extractors, local=custom_extractors) + self.extractors.register(self.PROJECT_EXTRACTOR_NAME, + load_project_as_dataset) + + import datumaro.components.launchers as builtin_launchers + builtin_launchers = builtin_launchers.items + custom_launchers = self._get_custom_module_items( + env_dir, env_config.launchers_dir) + self.launchers = ModuleRegistry(config, + builtin=builtin_launchers, local=custom_launchers) + + import datumaro.components.converters as builtin_converters + builtin_converters = builtin_converters.items + custom_converters = self._get_custom_module_items( + env_dir, env_config.converters_dir) + if custom_converters is not None: + custom_converters = custom_converters.items + self.converters = ModuleRegistry(config, + builtin=builtin_converters, local=custom_converters) + + self.statistics = ModuleRegistry(config) + self.visualizers = ModuleRegistry(config) + self.git = GitWrapper(config) + + def _get_custom_module_items(self, module_dir, module_name): + items = None + + module = None + if osp.exists(osp.join(module_dir, module_name)): + module = import_foreign_module(module_name, module_dir) + if module is not None: + if hasattr(module, 'items'): + items = module.items + else: + items = self._find_custom_module_items( + osp.join(module_dir, module_name)) + + return items + + @staticmethod + def _find_custom_module_items(module_dir): + files = [p for p in os.listdir(module_dir) + if p.endswith('.py') and p != '__init__.py'] + + all_items = [] + for f in files: + name = osp.splitext(f)[0] + module = import_foreign_module(name, module_dir) + + items = [] + if hasattr(module, 'items'): + items = module.items + else: + if hasattr(module, name): + items = [ (name, getattr(module, name)) ] + else: + log.warn("Failed to import custom module '%s'." + " Custom module is expected to provide 'items' " + "list or have an item matching its file name." + " Skipping this module." % \ + (module_dir + '.' + name)) + + all_items.extend(items) + + return all_items + + def save(self, path): + self.config.dump(path) + + def make_extractor(self, name, *args, **kwargs): + return self.extractors.get(name)(*args, **kwargs) + + def make_importer(self, name, *args, **kwargs): + return self.importers.get(name)(*args, **kwargs) + + def make_launcher(self, name, *args, **kwargs): + return self.launchers.get(name)(*args, **kwargs) + + def make_converter(self, name, *args, **kwargs): + return self.converters.get(name)(*args, **kwargs) + + def register_model(self, name, model): + self.config.models[name] = model + self.models.register(name, model) + + def unregister_model(self, name): + self.config.models.remove(name) + self.models.unregister(name) + + +class Subset(Extractor): + def __init__(self, parent): + self._parent = parent + self.items = OrderedDict() + + def __iter__(self): + for item in self.items.values(): + yield item + + def __len__(self): + return len(self.items) + + def categories(self): + return self._parent.categories() + +class DatasetItemWrapper(DatasetItem): + def __init__(self, item, path, annotations, image=None): + self._item = item + self._path = path + self._annotations = annotations + self._image = image + + @DatasetItem.id.getter + def id(self): + return self._item.id + + @DatasetItem.subset.getter + def subset(self): + return self._item.subset + + @DatasetItem.path.getter + def path(self): + return self._path + + @DatasetItem.annotations.getter + def annotations(self): + return self._annotations + + @DatasetItem.has_image.getter + def has_image(self): + if self._image is not None: + return True + return self._item.has_image + + @DatasetItem.image.getter + def image(self): + if self._image is not None: + if callable(self._image): + return self._image() + return self._image + return self._item.image + +class ProjectDataset(Extractor): + def __init__(self, project): + super().__init__() + + self._project = project + config = self.config + env = self.env + + dataset_filter = None + if config.filter: + dataset_filter = XPathDatasetFilter(config.filter) + self._filter = dataset_filter + + sources = {} + for s_name, source in config.sources.items(): + s_format = source.format + if not s_format: + s_format = env.PROJECT_EXTRACTOR_NAME + options = {} + options.update(source.options) + + url = source.url + if not source.url: + url = osp.join(config.project_dir, config.sources_dir, s_name) + sources[s_name] = env.make_extractor(s_format, + url, **options) + self._sources = sources + + own_source = None + own_source_dir = osp.join(config.project_dir, config.dataset_dir) + if osp.isdir(own_source_dir): + own_source = env.make_extractor(DEFAULT_FORMAT, own_source_dir) + + # merge categories + # TODO: implement properly with merging and annotations remapping + categories = {} + for source in self._sources.values(): + categories.update(source.categories()) + for source in self._sources.values(): + for cat_type, source_cat in source.categories().items(): + assert categories[cat_type] == source_cat + if own_source is not None and len(own_source) != 0: + categories.update(own_source.categories()) + self._categories = categories + + # merge items + subsets = defaultdict(lambda: Subset(self)) + for source_name, source in self._sources.items(): + for item in source: + if dataset_filter and not dataset_filter(item): + continue + + existing_item = subsets[item.subset].items.get(item.id) + if existing_item is not None: + image = None + if existing_item.has_image: + # TODO: think of image comparison + image = lambda: existing_item.image + + path = existing_item.path + if item.path != path: + path = None + item = DatasetItemWrapper(item=item, path=path, + image=image, annotations=self._merge_anno( + existing_item.annotations, item.annotations)) + else: + s_config = config.sources[source_name] + if s_config and \ + s_config.format != self.env.PROJECT_EXTRACTOR_NAME: + # NOTE: consider imported sources as our own dataset + path = None + else: + path = item.path + if path is None: + path = [] + path = [source_name] + path + item = DatasetItemWrapper(item=item, path=path, + annotations=item.annotations) + + subsets[item.subset].items[item.id] = item + + # override with our items, fallback to existing images + if own_source is not None: + for item in own_source: + if dataset_filter and not dataset_filter(item): + continue + + if not item.has_image: + existing_item = subsets[item.subset].items.get(item.id) + if existing_item is not None: + image = None + if existing_item.has_image: + # TODO: think of image comparison + image = lambda: existing_item.image + item = DatasetItemWrapper(item=item, path=None, + annotations=item.annotations, image=image) + + subsets[item.subset].items[item.id] = item + + # TODO: implement subset remapping when needed + subsets_filter = config.subsets + if len(subsets_filter) != 0: + subsets = { k: v for k, v in subsets.items() if k in subsets_filter} + self._subsets = dict(subsets) + + self._length = None + + @staticmethod + def _merge_anno(a, b): + from itertools import chain + merged = [] + for item in chain(a, b): + found = False + for elem in merged: + if elem == item: + found = True + break + if not found: + merged.append(item) + + return merged + + def iterate_own(self): + return self.select(lambda item: not item.path) + + def __iter__(self): + for subset in self._subsets.values(): + for item in subset: + if self._filter and not self._filter(item): + continue + yield item + + def __len__(self): + if self._length is None: + self._length = reduce(lambda s, x: s + len(x), + self._subsets.values(), 0) + return self._length + + def get_subset(self, name): + return self._subsets[name] + + def subsets(self): + return list(self._subsets) + + def categories(self): + return self._categories + + def define_categories(self, categories): + assert not self._categories + self._categories = categories + + def get(self, item_id, subset=None, path=None): + if path: + source = path[0] + rest_path = path[1:] + return self._sources[source].get( + item_id=item_id, subset=subset, path=rest_path) + return self._subsets[subset].items[item_id] + + def put(self, item, item_id=None, subset=None, path=None): + if path is None: + path = item.path + if path: + source = path[0] + rest_path = path[1:] + # TODO: reverse remapping + self._sources[source].put(item, + item_id=item_id, subset=subset, path=rest_path) + + if item_id is None: + item_id = item.id + if subset is None: + subset = item.subset + + item = DatasetItemWrapper(item=item, path=path, + annotations=item.annotations) + if item.subset not in self._subsets: + self._subsets[item.subset] = Subset(self) + self._subsets[subset].items[item_id] = item + self._length = None + + return item + + def build(self, tasks=None): + pass + + def docs(self): + pass + + def transform(self, model_name, save_dir=None): + project = Project(self.config) + project.config.remove('sources') + + if save_dir is None: + save_dir = self.config.project_dir + project.config.project_dir = save_dir + + dataset = project.make_dataset() + launcher = self._project.make_executable_model(model_name) + inference = InferenceWrapper(self, launcher) + dataset.update(inference) + + dataset.save(merge=True) + + def export(self, save_dir, output_format, + filter_expr=None, **converter_kwargs): + save_dir = osp.abspath(save_dir) + os.makedirs(save_dir, exist_ok=True) + + dataset = self + if filter_expr: + dataset_filter = XPathDatasetFilter(filter_expr) + dataset = dataset.select(dataset_filter) + + converter = self.env.make_converter(output_format, **converter_kwargs) + converter(dataset, save_dir) + + def extract(self, save_dir, filter_expr=None): + project = Project(self.config) + if filter_expr: + XPathDatasetFilter(filter_expr) + project.set_filter(filter_expr) + project.save(save_dir) + + def update(self, items): + for item in items: + if self._filter and not self._filter(item): + continue + self.put(item) + return self + + def save(self, save_dir=None, merge=False, recursive=True, + save_images=False, apply_colormap=True): + if save_dir is None: + assert self.config.project_dir + save_dir = self.config.project_dir + project = self._project + else: + merge = True + + if merge: + project = Project(Config(self.config)) + project.config.remove('sources') + + save_dir = osp.abspath(save_dir) + os.makedirs(save_dir, exist_ok=True) + + dataset_save_dir = osp.join(save_dir, project.config.dataset_dir) + os.makedirs(dataset_save_dir, exist_ok=True) + + converter_kwargs = { + 'save_images': save_images, + 'apply_colormap': apply_colormap, + } + + if merge: + # merge and save the resulting dataset + converter = self.env.make_converter( + DEFAULT_FORMAT, **converter_kwargs) + converter(self, dataset_save_dir) + else: + if recursive: + # children items should already be updated + # so we just save them recursively + for source in self._sources.values(): + if isinstance(source, ProjectDataset): + source.save(**converter_kwargs) + + converter = self.env.make_converter( + DEFAULT_FORMAT, **converter_kwargs) + converter(self.iterate_own(), dataset_save_dir) + + project.save(save_dir) + + @property + def env(self): + return self._project.env + + @property + def config(self): + return self._project.config + + @property + def sources(self): + return self._sources + +class Project: + @staticmethod + def load(path): + path = osp.abspath(path) + if osp.isdir(path): + path = osp.join(path, PROJECT_DEFAULT_CONFIG.project_filename) + config = Config.parse(path) + config.project_dir = osp.dirname(path) + config.project_filename = osp.basename(path) + return Project(config) + + def save(self, save_dir=None): + config = self.config + if save_dir is None: + assert config.project_dir + save_dir = osp.abspath(config.project_dir) + config_path = osp.join(save_dir, config.project_filename) + + env_dir = osp.join(save_dir, config.env_dir) + os.makedirs(env_dir, exist_ok=True) + self.env.save(osp.join(env_dir, config.env_filename)) + + config.dump(config_path) + + @staticmethod + def generate(save_dir, config=None): + project = Project(config) + project.save(save_dir) + project.config.project_dir = save_dir + return project + + @staticmethod + def import_from(path, dataset_format, env=None, **kwargs): + if env is None: + env = Environment() + importer = env.make_importer(dataset_format) + return importer(path, **kwargs) + + def __init__(self, config=None): + self.config = Config(config, + fallback=PROJECT_DEFAULT_CONFIG, schema=PROJECT_SCHEMA) + self.env = Environment(self.config) + + def make_dataset(self): + return ProjectDataset(self) + + def add_source(self, name, value=Source()): + if isinstance(value, (dict, Config)): + value = Source(value) + self.config.sources[name] = value + self.env.sources.register(name, value) + + def remove_source(self, name): + self.config.sources.remove(name) + self.env.sources.unregister(name) + + def get_source(self, name): + return self.config.sources[name] + + def get_subsets(self): + return self.config.subsets + + def set_subsets(self, value): + if not value: + self.config.remove('subsets') + else: + self.config.subsets = value + + def add_model(self, name, value=Model()): + if isinstance(value, (dict, Config)): + value = Model(value) + self.env.register_model(name, value) + + def get_model(self, name): + return self.env.models.get(name) + + def remove_model(self, name): + self.env.unregister_model(name) + + def make_executable_model(self, name): + model = self.get_model(name) + model.model_dir = self.local_model_dir(name) + return self.env.make_launcher(model.launcher, + **model.options, model_dir=model.model_dir) + + def make_source_project(self, name): + source = self.get_source(name) + + config = Config(self.config) + config.remove('sources') + config.remove('subsets') + config.remove('filter') + project = Project(config) + project.add_source(name, source) + return project + + def get_filter(self): + if 'filter' in self.config: + return self.config.filter + return '' + + def set_filter(self, value=None): + if not value: + self.config.remove('filter') + else: + # check filter + XPathDatasetFilter(value) + self.config.filter = value + + def local_model_dir(self, model_name): + return osp.join( + self.config.env_dir, self.env.config.models_dir, model_name) + + def local_source_dir(self, source_name): + return osp.join(self.config.sources_dir, source_name) + +# pylint: disable=function-redefined +def load_project_as_dataset(url): + # implement the function declared above + return Project.load(url).make_dataset() +# pylint: enable=function-redefined \ No newline at end of file diff --git a/datumaro/datumaro/util/__init__.py b/datumaro/datumaro/util/__init__.py new file mode 100644 index 000000000000..87c800fe515c --- /dev/null +++ b/datumaro/datumaro/util/__init__.py @@ -0,0 +1,20 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import os + + +def find(iterable, pred=lambda x: True, default=None): + return next((x for x in iterable if pred(x)), default) + +def dir_items(path, ext, truncate_ext=False): + items = [] + for f in os.listdir(path): + ext_pos = f.rfind(ext) + if ext_pos != -1: + if truncate_ext: + f = f[:ext_pos] + items.append(f) + return items \ No newline at end of file diff --git a/datumaro/datumaro/util/command_targets.py b/datumaro/datumaro/util/command_targets.py new file mode 100644 index 000000000000..8c7c2dd92717 --- /dev/null +++ b/datumaro/datumaro/util/command_targets.py @@ -0,0 +1,110 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import argparse +import cv2 +from enum import Enum + +from datumaro.components.project import Project + + +TargetKinds = Enum('TargetKinds', + ['project', 'source', 'external_dataset', 'inference', 'image']) + +def is_project_name(value, project): + return value == project.config.project_name + +def is_project_path(value): + if value: + try: + Project.load(value) + return True + except Exception: + pass + return False + +def is_project(value, project=None): + if is_project_path(value): + return True + elif project is not None: + return is_project_name(value, project) + + return False + +def is_source(value, project=None): + if project is not None: + try: + project.get_source(value) + return True + except KeyError: + pass + + return False + +def is_external_source(value): + return False + +def is_inference_path(value): + return False + +def is_image_path(value): + return cv2.imread(value) is not None + + +class Target: + def __init__(self, kind, test, is_default=False, name=None): + self.kind = kind + self.test = test + self.is_default = is_default + self.name = name + + def _get_fields(self): + return [self.kind, self.test, self.is_default, self.name] + + def __str__(self): + return self.name or str(self.kind) + + def __len__(self): + return len(self._get_fields()) + + def __iter__(self): + return iter(self._get_fields()) + +def ProjectTarget(kind=TargetKinds.project, test=None, + is_default=False, name='project name or path', + project=None): + if test is None: + test = lambda v: is_project(v, project=project) + return Target(kind, test, is_default, name) + +def SourceTarget(kind=TargetKinds.source, test=None, + is_default=False, name='source name', + project=None): + if test is None: + test = lambda v: is_source(v, project=project) + return Target(kind, test, is_default, name) + +def ExternalDatasetTarget(kind=TargetKinds.external_dataset, + test=is_external_source, + is_default=False, name='external dataset path'): + return Target(kind, test, is_default, name) + +def InferenceTarget(kind=TargetKinds.inference, test=is_inference_path, + is_default=False, name='inference path'): + return Target(kind, test, is_default, name) + +def ImageTarget(kind=TargetKinds.image, test=is_image_path, + is_default=False, name='image path'): + return Target(kind, test, is_default, name) + + +def target_selector(*targets): + def selector(value): + for (kind, test, is_default, _) in targets: + if (is_default and (value == '' or value is None)) or test(value): + return (kind, value) + raise argparse.ArgumentTypeError('Value should be one of: %s' \ + % (', '.join([str(t) for t in targets]))) + return selector diff --git a/datumaro/datumaro/util/image.py b/datumaro/datumaro/util/image.py new file mode 100644 index 000000000000..8e1794aab333 --- /dev/null +++ b/datumaro/datumaro/util/image.py @@ -0,0 +1,30 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import cv2 +import numpy as np + + +def load_image(path): + """ + Reads an image in the HWC Grayscale/BGR(A) float [0; 255] format. + """ + image = cv2.imread(path) + image = image.astype(np.float32) + + assert len(image.shape) == 3 + assert image.shape[2] in [1, 3, 4] + return image + +class lazy_image: + def __init__(self, path, loader=load_image): + self.path = path + self.loader = loader + self.image = None + + def __call__(self): + if self.image is None: + self.image = self.loader(self.path) + return self.image \ No newline at end of file diff --git a/datumaro/datumaro/util/mask_tools.py b/datumaro/datumaro/util/mask_tools.py new file mode 100644 index 000000000000..d9c7fe9235d7 --- /dev/null +++ b/datumaro/datumaro/util/mask_tools.py @@ -0,0 +1,96 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from itertools import groupby +import numpy as np + +from datumaro.util.image import lazy_image, load_image + + +def generate_colormap(length=256): + def get_bit(number, index): + return (number >> index) & 1 + + colormap = np.zeros((length, 3), dtype=int) + indices = np.arange(length, dtype=int) + + for j in range(7, -1, -1): + for c in range(3): + colormap[:, c] |= get_bit(indices, c) << j + indices >>= 3 + + return { + id: tuple(color) for id, color in enumerate(colormap) + } + +def invert_colormap(colormap): + return { + tuple(a): index for index, a in colormap.items() + } + +_default_colormap = generate_colormap() +_default_unpaint_colormap = invert_colormap(_default_colormap) + +def _default_unpaint_colormap_fn(r, g, b): + return _default_unpaint_colormap[(r, g, b)] + +def unpaint_mask(painted_mask, colormap=None): + # expect HWC BGR [0; 255] image + # expect RGB->index colormap + assert len(painted_mask.shape) == 3 + if colormap is None: + colormap = _default_unpaint_colormap_fn + if callable(colormap): + map_fn = lambda a: colormap(int(a[2]), int(a[1]), int(a[0])) + else: + map_fn = lambda a: colormap[(int(a[2]), int(a[1]), int(a[0]))] + + unpainted_mask = np.apply_along_axis(map_fn, + 1, np.reshape(painted_mask, (-1, 3))) + unpainted_mask = np.reshape(unpainted_mask, (painted_mask.shape[:2])) + return unpainted_mask.astype(int) + + +def apply_colormap(mask, colormap=None): + # expect HW [0; max_index] mask + # expect index->RGB colormap + assert len(mask.shape) == 2 + + if colormap is None: + colormap = _default_colormap + if callable(colormap): + map_fn = lambda p: colormap(int(p[0]))[::-1] + else: + map_fn = lambda p: colormap[int(p[0])][::-1] + painted_mask = np.apply_along_axis(map_fn, 1, np.reshape(mask, (-1, 1))) + + painted_mask = np.reshape(painted_mask, (*mask.shape, 3)) + return painted_mask.astype(np.float32) + + +def load_mask(path, colormap=None): + mask = load_image(path) + if colormap is not None: + if len(mask.shape) == 3 and mask.shape[2] != 1: + mask = unpaint_mask(mask, colormap=colormap) + return mask + +def lazy_mask(path, colormap=None): + return lazy_image(path, lambda path: load_mask(path, colormap)) + + +def convert_mask_to_rle(binary_mask): + counts = [] + for i, (value, elements) in enumerate( + groupby(binary_mask.ravel(order='F'))): + # decoding starts from 0 + if i == 0 and value == 1: + counts.append(0) + counts.append(len(list(elements))) + + return { + 'counts': counts, + 'size': list(binary_mask.shape) + } \ No newline at end of file diff --git a/datumaro/datumaro/util/test_utils.py b/datumaro/datumaro/util/test_utils.py new file mode 100644 index 000000000000..9219f5cfa01b --- /dev/null +++ b/datumaro/datumaro/util/test_utils.py @@ -0,0 +1,39 @@ + +# Copyright (C) 2019 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import inspect +import os +import os.path as osp +import shutil + + +def current_function_name(depth=1): + return inspect.getouterframes(inspect.currentframe())[depth].function + +class FileRemover: + def __init__(self, path, is_dir=False, ignore_errors=False): + self.path = path + self.is_dir = is_dir + self.ignore_errors = ignore_errors + + def __enter__(self): + return self + + # pylint: disable=redefined-builtin + def __exit__(self, type=None, value=None, traceback=None): + if self.is_dir: + shutil.rmtree(self.path, ignore_errors=self.ignore_errors) + else: + os.remove(self.path) + # pylint: enable=redefined-builtin + +class TestDir(FileRemover): + def __init__(self, path=None, ignore_errors=False): + if path is None: + path = osp.abspath('temp_%s' % current_function_name(2)) + + os.makedirs(path, exist_ok=ignore_errors) + + super().__init__(path, is_dir=True, ignore_errors=ignore_errors) \ No newline at end of file diff --git a/datumaro/docs/cli_design.mm b/datumaro/docs/cli_design.mm new file mode 100644 index 000000000000..4c7b188cf5fa --- /dev/null +++ b/datumaro/docs/cli_design.mm @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/datumaro/docs/design.md b/datumaro/docs/design.md new file mode 100644 index 000000000000..69d4d198816a --- /dev/null +++ b/datumaro/docs/design.md @@ -0,0 +1,228 @@ +# Datumaro + + + +## Table of contents + +- [Concept](#concept) +- [Design](#design) +- [RC 1 vision](#rc-1-vision) + +## Concept + +Datumaro is: +- a tool to build composite datasets and iterate over them +- a tool to create and maintain datasets + - Version control of annotations and images + - Publication (with removal of sensitive information) + - Editing + - Joining and splitting + - Exporting, format changing + - Image preprocessing +- a dataset storage +- a tool to debug datasets + - A network can be used to generate + informative data subsets (e.g. with false-positives) + to be analyzed further + +### Requirements + +- User interfaces + - a library + - a console tool with visualization means +- Targets: single datasets, composite datasets, single images / videos +- Built-in support for well-known annotation formats and datasets: + CVAT, COCO, PASCAL VOC, Cityscapes, ImageNet +- Extensibility with user-provided components +- Lightweightness - it should be easy to start working with Datumaro + - Minimal dependency on environment and configuration + - It should be easier to use Datumaro than writing own code + for computation of statistics or dataset manipulations + +### Functionality and ideas + +- Blur sensitive areas on dataset images +- Dataset annotation filters, relabelling etc. +- Dataset augmentation +- Calculation of statistics: + - Mean & std, custom stats +- "Edit" command to modify annotations +- Versioning (for images, annotations, subsets, sources etc., comparison) +- Documentation generation +- Provision of iterators for user code +- Dataset building (export in a specific format, indexation, statistics, documentation) +- Dataset exporting to other formats +- Dataset debugging (run inference, generate dataset slices, compute statistics) +- "Explainable AI" - highlight network attention areas ([paper](https://arxiv.org/abs/1901.04592)) + - Black-box approach + - Classification, Detection, Segmentation, Captioning + - White-box approach + +### Research topics + +- exploration of network prediction uncertainty (aka Bayessian approach) + Use case: explanation of network "quality", "stability", "certainty" +- adversarial attacks on networks +- dataset minification / reduction + Use case: removal of redundant information to reach the same network quality with lesser training time +- dataset expansion and filtration of additions + Use case: add only important data +- guidance for key frame selection for tracking ([paper](https://arxiv.org/abs/1903.11779)) + Use case: more effective annotation, better predictions + +## Design + +### Command-line + +Use Docker as an example. Basically, the interface is partitioned +on contexts and shortcuts. Contexts are semantically grouped commands, +related to a single topic or target. Shortcuts are handy shorter +alternatives for the most used commands and also special commands, +which are hard to be put into specific context. + +![cli-design-image](images/cli_design.png) + +- [FreeMind tool link](http://freemind.sourceforge.net/wiki/index.php/Main_Page) + +### High-level architecture + +- Using MVVM UI pattern + +![mvvm-image](images/mvvm.png) + +### Datumaro project and environment structure + + +``` +├── [datumaro module] +└── [project folder] + ├── .datumaro/ + │   ├── config.yml + │   ├── .git/ + │   ├── importers/ + │   │   ├── custom_format_importer1.py + │   │   └── ... + │   ├── statistics/ + │   │   ├── custom_statistic1.py + │   │   └── ... + │   ├── visualizers/ + │   │ ├── custom_visualizer1.py + │   │ └── ... + │   └── extractors/ + │   ├── custom_extractor1.py + │   └── ... + └── sources/ + ├── source1 + └── ... +``` + + +## RC 1 vision + +In the first version Datumaro should be a project manager for CVAT. +It should only consume data from CVAT. The collected dataset +can be downloaded by user to be operated on with Datumaro CLI. + + +``` + User + | + v + +------------------+ + | CVAT | + +--------v---------+ +------------------+ +--------------+ + | Datumaro module | ----> | Datumaro project | <---> | Datumaro CLI | <--- User + +------------------+ +------------------+ +--------------+ +``` + + +### Interfaces + +- [x] Python API for user code + - [ ] Installation as a package +- [x] A command-line tool for dataset manipulations + +### Features + +- Dataset format support (reading, exporting) + - [x] Own format + - [x] COCO + - [x] PASCAL VOC + - [ ] Cityscapes + - [ ] ImageNet + - [ ] CVAT + +- Dataset visualization (`show`) + - [ ] Ability to visualize a dataset + - [ ] with TensorBoard + +- Calculation of statistics for datasets + - [ ] Pixel mean, std + - [ ] Object counts (detection scenario) + - [ ] Image-Class distribution (classification scenario) + - [ ] Pixel-Class distribution (segmentation scenario) + - [ ] Image clusters + - [ ] Custom statistics + +- Dataset building + - [x] Composite dataset building + - [ ] Annotation remapping + - [ ] Subset splitting + - [x] Dataset filtering (`extract`) + - [x] Dataset merging (`merge`) + - [ ] Dataset item editing (`edit`) + +- Dataset comparison (`diff`) + - [x] Annotation-annotation comparison + - [x] Annotation-inference comparison + - [ ] Annotation quality estimation (for CVAT) + - Provide a simple method to check + annotation quality with a model and generate summary + +- Dataset and model debugging + - [x] Inference explanation (`explain`) + - [x] Black-box approach ([RISE paper](https://arxiv.org/abs/1806.07421)) + - [x] Ability to run a model on a dataset and read the results + +- CVAT-integration features + - [x] Task export + - [x] Datumaro project export + - [x] Dataset export + - [ ] Original raw data (images, a video file) can be downloaded (exported) + together with annotations or just have links + on CVAT server (in the future support S3, etc) + - [x] Be able to use local files instead of remote links + - [ ] Specify cache directory + - [x] Use case "annotate for model training" + - create a task + - annotate + - export the task + - convert to a training format + - train a DL model + - [ ] Use case "annotate and estimate quality" + - create a task + - annotate + - estimate quality of annotations + +### Optional features + +- Dataset publishing + - [ ] Versioning (for annotations, subsets, sources, etc.) + - [ ] Blur sensitive areas on images + - [ ] Tracking of legal information + - [ ] Documentation generation + +- Dataset building + - [ ] Dataset minification / Extraction of the most representative subset + - Use case: generate low-precision calibration dataset + +- Dataset and model debugging + - [ ] Training visualization + - [ ] Inference explanation (`explain`) + - [ ] White-box approach + +### Properties + +- Lightweightness +- Modularity +- Extensibility diff --git a/datumaro/docs/images/cli_design.png b/datumaro/docs/images/cli_design.png new file mode 100644 index 000000000000..702728c442a7 Binary files /dev/null and b/datumaro/docs/images/cli_design.png differ diff --git a/datumaro/docs/images/mvvm.png b/datumaro/docs/images/mvvm.png new file mode 100644 index 000000000000..88257123ac7d Binary files /dev/null and b/datumaro/docs/images/mvvm.png differ diff --git a/datumaro/docs/quickstart.md b/datumaro/docs/quickstart.md new file mode 100644 index 000000000000..d5fb98a6ab5a --- /dev/null +++ b/datumaro/docs/quickstart.md @@ -0,0 +1,325 @@ +# Quick start guide + +## Installation + +### Prerequisites + +- Python (3.5+) +- OpenVINO (optional) + +### Installation steps + +Download the project to any directory. + +Set up a virtual environment: + +``` bash +python -m pip install virtualenv +python -m virtualenv venv +. venv/bin/activate +while read -r p; do pip install $p; done < requirements.txt +``` + +## Usage + +The directory containing the project should be in the +`PYTHONPATH` environment variable. The other way is to invoke +commands from that directory. + +As a python module: + +``` bash +python -m datumaro --help +``` + +As a standalone python script: + +``` bash +python datum.py --help +``` + +As a python library: + +``` python +import datumaro +``` + +## Workflow + +> **Note**: command invocation **syntax is subject to change, refer to --help output** + +The key object is the project. It can be created or imported with +`project create` and `project import` commands. The project is a combination of +dataset and environment. + +If you want to interact with models, you should add them to project first. + +Implemented commands ([CLI design doc](images/cli_design.png)): +- project create +- project import +- project diff +- project transform +- source add +- explain + +### Create a project + +Usage: + +``` bash +python datum.py project create --help + +python datum.py project create \ + -d +``` + +Example: + +``` bash +python datum.py project create -d /home/my_dataset +``` + +### Import a project + +This command creates a project from an existing dataset. Supported formats: +- MS COCO +- Custom formats via custom `importers` and `extractors` + +Usage: + +``` bash +python -m datumaro project import --help + +python -m datumaro project import \ + \ + -d \ + -t +``` + +Example: + +``` bash +python -m datumaro project import \ + /home/coco_dir \ + -d /home/project_dir \ + -t ms_coco +``` + +An _MS COCO_-like dataset should have the following directory structure: + + +``` +COCO/ +├── annotations/ +│   ├── instances_val2017.json +│   ├── instances_train2017.json +├── images/ +│   ├── val2017 +│   ├── train2017 +``` + + +Everything after the last `_` is considered as a subset name. + +### Register a model + +Supported models: +- OpenVINO +- Custom models via custom `launchers` + +Usage: + +``` bash +python -m datumaro model add --help +``` + +Example: register OpenVINO model + +A model consists of a graph description and weights. There is also a script +used to convert model outputs to internal data structures. + +``` bash +python -m datumaro model add \ + openvino \ + -d -w -i +``` + +Interpretation script for an OpenVINO detection model (`convert.py`): + +``` python +from datumaro.components.extractor import * + +max_det = 10 +conf_thresh = 0.1 + +def process_outputs(inputs, outputs): + # inputs = model input, array or images, shape = (N, C, H, W) + # outputs = model output, shape = (N, 1, K, 7) + # results = conversion result, [ [ Annotation, ... ], ... ] + results = [] + for input, output in zip(inputs, outputs): + input_height, input_width = input.shape[:2] + detections = output[0] + image_results = [] + for i, det in enumerate(detections): + label = int(det[1]) + conf = det[2] + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + image_results.append(BboxObject(x, y, w, h, + label=label, attributes={'score': conf} )) + + results.append(image_results[:max_det]) + + return results + +def get_categories(): + # Optionally, provide output categories - label map etc. + # Example: + label_categories = LabelCategories() + label_categories.add('person') + label_categories.add('car') + return { AnnotationType.label: label_categories } +``` + +### Run a model inference + +This command сreates a new project from the current project. The new +one annotations are the model outputs. + +Usage: + +``` bash +python -m datumaro project transform --help + +python -m datumaro project transform \ + -m \ + -d +``` + +Example: + +``` bash +python -m datumaro project import <...> +python -m datumaro model add mymodel <...> +python -m datumaro project transform -m mymodel -d ../mymodel_inference +``` + +### Compare datasets + +The command compares two datasets and saves the results in the +specified directory. The current project is considered to be +"ground truth". + +``` bash +python -m datumaro project diff --help + +python -m datumaro project diff -d +``` + +Example: compare a dataset with model inference + +``` bash +python -m datumaro project import <...> +python -m datumaro model add mymodel <...> +python -m datumaro project transform <...> -d ../inference +python -m datumaro project diff ../inference -d ../diff +``` + +### Run inference explanation + +Usage: + +``` bash +python -m datumaro explain --help + +python -m datumaro explain \ + -m \ + -d \ + -t \ + \ + +``` + +Example: run inference explanation on a single image with visualization + +``` bash +python -m datumaro project create <...> +python -m datumaro model add mymodel <...> +python -m datumaro explain \ + -m mymodel \ + -t 'image.png' \ + rise \ + -s 1000 --progressive +``` + +### Extract data subset based on filter + +This command allows to create a subprject form a project, which +would include only items satisfying some condition. XPath is used as a query +format. + +Usage: + +``` bash +python -m datumaro project extract --help + +python -m datumaro project extract \ + -p \ + -d \ + -f '' +``` + +Example: + +``` bash +python -m datumaro project extract \ + -p ../test_project \ + -d ../test_project-extract \ + -f '/item[image/width < image/height]' +``` + +Item representation: + +``` xml + + 290768 + minival2014 + + 612 + 612 + 3 + + + 80154 + bbox + 39 + 264.59 + 150.25 + 11.199999999999989 + 42.31 + 473.87199999999956 + + + 669839 + bbox + 41 + 163.58 + 191.75 + 76.98999999999998 + 73.63 + 5668.773699999998 + + ... + +``` + +## Links +- [TensorFlow detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) +- [How to convert model to OpenVINO format](https://docs.openvinotoolkit.org/latest/_docs_MO_DG_prepare_model_convert_model_tf_specific_Convert_Object_Detection_API_Models.html) +- [Model convert script for this model](https://github.com/opencv/cvat/blob/3e09503ba6c6daa6469a6c4d275a5a8b168dfa2c/components/tf_annotation/install.sh#L23) diff --git a/datumaro/requirements.txt b/datumaro/requirements.txt new file mode 100644 index 000000000000..652ec3eab366 --- /dev/null +++ b/datumaro/requirements.txt @@ -0,0 +1,11 @@ +Cython>=0.27.3 # include before pycocotools +GitPython>=2.1.11 +lxml>=4.4.1 +matplotlib<3.1 # 3.1+ requires python3.6, but we have 3.5 in cvat +opencv-python>=4.1.0.25 +Pillow>=6.1.0 +pycocotools>=2.0.0 +PyYAML>=5.1.1 +requests>=2.20.0 +tensorboard>=1.12.0 +tensorboardX>=1.8 diff --git a/datumaro/test.py b/datumaro/test.py new file mode 100644 index 000000000000..184bbff53262 --- /dev/null +++ b/datumaro/test.py @@ -0,0 +1,5 @@ +import unittest + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/datumaro/tests/__init__.py b/datumaro/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/datumaro/tests/test_RISE.py b/datumaro/tests/test_RISE.py new file mode 100644 index 000000000000..a7560f1ff0e3 --- /dev/null +++ b/datumaro/tests/test_RISE.py @@ -0,0 +1,230 @@ +from collections import namedtuple +import cv2 +import numpy as np + +from unittest import TestCase + +from datumaro.components.extractor import LabelObject, BboxObject +from datumaro.components.launcher import Launcher +from datumaro.components.algorithms.rise import RISE + + +class RiseTest(TestCase): + def test_rise_can_be_applied_to_classification_model(self): + class TestLauncher(Launcher): + def __init__(self, class_count, roi, **kwargs): + self.class_count = class_count + self.roi = roi + + def launch(self, inputs): + for inp in inputs: + yield self._process(inp) + + def _process(self, image): + roi = self.roi + roi_area = (roi[1] - roi[0]) * (roi[3] - roi[2]) + if 0.5 * roi_area < np.sum(image[roi[0]:roi[1], roi[2]:roi[3], 0]): + cls = 0 + else: + cls = 1 + + cls_conf = 0.5 + other_conf = (1.0 - cls_conf) / (self.class_count - 1) + + return [ + LabelObject(i, attributes={ + 'score': cls_conf if cls == i else other_conf }) \ + for i in range(self.class_count) + ] + + roi = [70, 90, 7, 90] + model = TestLauncher(class_count=3, roi=roi) + + rise = RISE(model, max_samples=(7 * 7) ** 2, mask_width=7, mask_height=7) + + image = np.ones((100, 100, 3)) + heatmaps = next(rise.apply(image)) + + self.assertEqual(1, len(heatmaps)) + + heatmap = heatmaps[0] + self.assertEqual(image.shape[:2], heatmap.shape) + + h_sum = np.sum(heatmap) + h_area = np.prod(heatmap.shape) + roi_sum = np.sum(heatmap[roi[0]:roi[1], roi[2]:roi[3]]) + roi_area = (roi[1] - roi[0]) * (roi[3] - roi[2]) + roi_den = roi_sum / roi_area + hrest_den = (h_sum - roi_sum) / (h_area - roi_area) + self.assertLess(hrest_den, roi_den) + + def test_rise_can_be_applied_to_detection_model(self): + ROI = namedtuple('ROI', + ['threshold', 'x', 'y', 'w', 'h', 'label']) + + class TestLauncher(Launcher): + def __init__(self, rois, class_count, fp_count=4, pixel_jitter=20, **kwargs): + self.rois = rois + self.roi_base_sums = [None, ] * len(rois) + self.class_count = class_count + self.fp_count = fp_count + self.pixel_jitter = pixel_jitter + + @staticmethod + def roi_value(roi, image): + return np.sum( + image[roi.y:roi.y + roi.h, roi.x:roi.x + roi.w, :]) + + def launch(self, inputs): + for inp in inputs: + yield self._process(inp) + + def _process(self, image): + detections = [] + for i, roi in enumerate(self.rois): + roi_sum = self.roi_value(roi, image) + roi_base_sum = self.roi_base_sums[i] + first_run = roi_base_sum is None + if first_run: + roi_base_sum = roi_sum + self.roi_base_sums[i] = roi_base_sum + + cls_conf = roi_sum / roi_base_sum + + if roi.threshold < roi_sum / roi_base_sum: + cls = roi.label + detections.append( + BboxObject(roi.x, roi.y, roi.w, roi.h, + label=cls, attributes={'score': cls_conf}) + ) + + if first_run: + continue + for j in range(self.fp_count): + if roi.threshold < cls_conf: + cls = roi.label + else: + cls = (i + j) % self.class_count + box = [roi.x, roi.y, roi.w, roi.h] + offset = (np.random.rand(4) - 0.5) * self.pixel_jitter + detections.append( + BboxObject(*(box + offset), + label=cls, attributes={'score': cls_conf}) + ) + + return detections + + rois = [ + ROI(0.3, 10, 40, 30, 10, 0), + ROI(0.5, 70, 90, 7, 10, 0), + ROI(0.7, 5, 20, 40, 60, 2), + ROI(0.9, 30, 20, 10, 40, 1), + ] + model = model = TestLauncher(class_count=3, rois=rois) + + rise = RISE(model, max_samples=(7 * 7) ** 2, mask_width=7, mask_height=7) + + image = np.ones((100, 100, 3)) + heatmaps = next(rise.apply(image)) + heatmaps_class_count = len(set([roi.label for roi in rois])) + self.assertEqual(heatmaps_class_count + len(rois), len(heatmaps)) + + # roi_image = image.copy() + # for i, roi in enumerate(rois): + # cv2.rectangle(roi_image, (roi.x, roi.y), (roi.x + roi.w, roi.y + roi.h), (32 * i) * 3) + # cv2.imshow('img', roi_image) + + for c in range(heatmaps_class_count): + class_roi = np.zeros(image.shape[:2]) + for i, roi in enumerate(rois): + if roi.label != c: + continue + class_roi[roi.y:roi.y + roi.h, roi.x:roi.x + roi.w] \ + += roi.threshold + + heatmap = heatmaps[c] + + roi_pixels = heatmap[class_roi != 0] + h_sum = np.sum(roi_pixels) + h_area = np.sum(roi_pixels != 0) + h_den = h_sum / h_area + + rest_pixels = heatmap[class_roi == 0] + r_sum = np.sum(rest_pixels) + r_area = np.sum(rest_pixels != 0) + r_den = r_sum / r_area + + # print(r_den, h_den) + # cv2.imshow('class %s' % c, heatmap) + self.assertLess(r_den, h_den) + + for i, roi in enumerate(rois): + heatmap = heatmaps[heatmaps_class_count + i] + h_sum = np.sum(heatmap) + h_area = np.prod(heatmap.shape) + roi_sum = np.sum(heatmap[roi.y:roi.y + roi.h, roi.x:roi.x + roi.w]) + roi_area = roi.h * roi.w + roi_den = roi_sum / roi_area + hrest_den = (h_sum - roi_sum) / (h_area - roi_area) + # print(hrest_den, h_den) + # cv2.imshow('roi %s' % i, heatmap) + self.assertLess(hrest_den, roi_den) + # cv2.waitKey(0) + + @staticmethod + def DISABLED_test_roi_nms(): + ROI = namedtuple('ROI', + ['conf', 'x', 'y', 'w', 'h', 'label']) + + class_count = 3 + noisy_count = 3 + rois = [ + ROI(0.3, 10, 40, 30, 10, 0), + ROI(0.5, 70, 90, 7, 10, 0), + ROI(0.7, 5, 20, 40, 60, 2), + ROI(0.9, 30, 20, 10, 40, 1), + ] + pixel_jitter = 10 + + detections = [] + for i, roi in enumerate(rois): + detections.append( + BboxObject(roi.x, roi.y, roi.w, roi.h, + label=roi.label, attributes={'score': roi.conf}) + ) + + for j in range(noisy_count): + cls_conf = roi.conf * j / noisy_count + cls = (i + j) % class_count + box = [roi.x, roi.y, roi.w, roi.h] + offset = (np.random.rand(4) - 0.5) * pixel_jitter + detections.append( + BboxObject(*(box + offset), + label=cls, attributes={'score': cls_conf}) + ) + + image = np.zeros((100, 100, 3)) + for i, det in enumerate(detections): + roi = ROI(det.attributes['score'], *det.get_bbox(), det.label) + p1 = (int(roi.x), int(roi.y)) + p2 = (int(roi.x + roi.w), int(roi.y + roi.h)) + c = (0, 1 * (i % (1 + noisy_count) == 0), 1) + cv2.rectangle(image, p1, p2, c) + cv2.putText(image, 'd%s-%s-%.2f' % (i, roi.label, roi.conf), + p1, cv2.FONT_HERSHEY_SIMPLEX, 0.25, c) + cv2.imshow('nms_image', image) + cv2.waitKey(0) + + nms_boxes = RISE.nms(detections, iou_thresh=0.25) + print(len(detections), len(nms_boxes)) + + for i, det in enumerate(nms_boxes): + roi = ROI(det.attributes['score'], *det.get_bbox(), det.label) + p1 = (int(roi.x), int(roi.y)) + p2 = (int(roi.x + roi.w), int(roi.y + roi.h)) + c = (0, 1, 0) + cv2.rectangle(image, p1, p2, c) + cv2.putText(image, 'p%s-%s-%.2f' % (i, roi.label, roi.conf), + p1, cv2.FONT_HERSHEY_SIMPLEX, 0.25, c) + cv2.imshow('nms_image', image) + cv2.waitKey(0) \ No newline at end of file diff --git a/datumaro/tests/test_coco_format.py b/datumaro/tests/test_coco_format.py new file mode 100644 index 000000000000..1d42bc77ce79 --- /dev/null +++ b/datumaro/tests/test_coco_format.py @@ -0,0 +1,389 @@ +import json +import numpy as np +import os +import os.path as osp +from PIL import Image + +from unittest import TestCase + +from datumaro.components.project import Project +from datumaro.components.extractor import ( + DEFAULT_SUBSET_NAME, + Extractor, DatasetItem, + AnnotationType, LabelObject, MaskObject, PointsObject, PolygonObject, + BboxObject, CaptionObject, + LabelCategories, PointsCategories +) +from datumaro.components.converters.ms_coco import ( + CocoConverter, + CocoImageInfoConverter, + CocoCaptionsConverter, + CocoInstancesConverter, + CocoPersonKeypointsConverter, + CocoLabelsConverter, +) +from datumaro.util import find +from datumaro.util.test_utils import TestDir + + +class CocoImporterTest(TestCase): + @staticmethod + def generate_annotation(): + annotation = { + 'licenses': [], + 'info': {}, + 'categories': [], + 'images': [], + 'annotations': [] + } + annotation['licenses'].append({ + 'name': '', + 'id': 0, + 'url': '' + }) + annotation['info'] = { + 'contributor': '', + 'date_created': '', + 'description': '', + 'url': '', + 'version': '', + 'year': '' + } + annotation['licenses'].append({ + 'name': '', + 'id': 0, + 'url': '' + }) + annotation['categories'].append({'id': 0, 'name': 'TEST', 'supercategory': ''}) + annotation['images'].append({ + "id": 0, + "width": 10, + "height": 5, + "file_name": '000000000001.jpg', + "license": 0, + "flickr_url": '', + "coco_url": '', + "date_captured": 0 + }) + annotation['annotations'].append({ + "id": 0, + "image_id": 0, + "category_id": 0, + "segmentation": [[0, 0, 1, 0, 1, 2, 0, 2]], + "area": 2, + "bbox": [0, 0, 1, 2], + "iscrowd": 0 + }) + annotation['annotations'].append({ + "id": 1, + "image_id": 0, + "category_id": 0, + "segmentation": { + "counts": [ + 0, 10, + 5, 5, + 5, 5, + 0, 10, + 10, 0], + "size": [10, 5]}, + "area": 30, + "bbox": [0, 0, 10, 4], + "iscrowd": 0 + }) + return annotation + + def COCO_dataset_generate(self, path): + img_dir = osp.join(path, 'images', 'val') + ann_dir = osp.join(path, 'annotations') + os.makedirs(img_dir) + os.makedirs(ann_dir) + a = np.random.rand(100, 100, 3) * 255 + im_out = Image.fromarray(a.astype('uint8')).convert('RGB') + im_out.save(osp.join(img_dir, '000000000001.jpg')) + annotation = self.generate_annotation() + with open(osp.join(ann_dir, 'instances_val.json'), 'w') as outfile: + json.dump(annotation, outfile) + + def test_can_import(self): + with TestDir() as temp_dir: + self.COCO_dataset_generate(temp_dir.path) + project = Project.import_from(temp_dir.path, 'ms_coco') + dataset = project.make_dataset() + + self.assertListEqual(['val'], sorted(dataset.subsets())) + self.assertEqual(1, len(dataset)) + + item = next(iter(dataset)) + self.assertTrue(item.has_image) + self.assertEqual(5, len(item.annotations)) + + ann_0 = find(item.annotations, lambda x: x.id == 0) + ann_0_poly = find(item.annotations, lambda x: \ + x.group == ann_0.id and x.type == AnnotationType.polygon) + ann_0_mask = find(item.annotations, lambda x: \ + x.group == ann_0.id and x.type == AnnotationType.mask) + self.assertFalse(ann_0 is None) + self.assertFalse(ann_0_poly is None) + self.assertFalse(ann_0_mask is None) + + ann_1 = find(item.annotations, lambda x: x.id == 1) + ann_1_mask = find(item.annotations, lambda x: \ + x.group == ann_1.id and x.type == AnnotationType.mask) + self.assertFalse(ann_1 is None) + self.assertFalse(ann_1_mask is None) + +class CocoConverterTest(TestCase): + def _test_save_and_load(self, source_dataset, converter_type, test_dir): + converter = converter_type() + converter(source_dataset, test_dir.path) + + project = Project.import_from(test_dir.path, 'ms_coco') + parsed_dataset = project.make_dataset() + + source_subsets = [s if s else DEFAULT_SUBSET_NAME + for s in source_dataset.subsets()] + self.assertListEqual( + sorted(source_subsets), + sorted(parsed_dataset.subsets()), + ) + + self.assertEqual(len(source_dataset), len(parsed_dataset)) + + for item_a in source_dataset: + item_b = find(parsed_dataset, lambda x: x.id == item_a.id) + self.assertFalse(item_b is None) + self.assertEqual(len(item_a.annotations), len(item_b.annotations)) + for ann_a in item_a.annotations: + ann_b = find(item_b.annotations, lambda x: \ + x.id == ann_a.id if ann_a.id else \ + x.type == ann_a.type and x.group == ann_a.group) + self.assertEqual(ann_a, ann_b) + + def test_can_save_and_load_captions(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=0, subset='train', + annotations=[ + CaptionObject('hello', id=1), + CaptionObject('world', id=2), + ]), + DatasetItem(id=1, subset='train', + annotations=[ + CaptionObject('test', id=3), + ]), + + DatasetItem(id=2, subset='val', + annotations=[ + CaptionObject('word', id=1), + ] + ), + ] + return iter(items) + + def subsets(self): + return ['train', 'val'] + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoCaptionsConverter, test_dir) + + def test_can_save_and_load_instances(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=0, subset='train', image=np.ones((4, 4, 3)), + annotations=[ + BboxObject(0, 1, 2, 3, label=2, group=1, + attributes={ 'is_crowd': False }, id=1), + PolygonObject([0, 1, 2, 1, 2, 3, 0, 3], + label=2, group=1), + MaskObject(np.array([[0, 0, 0, 0], [1, 1, 0, 0], + [1, 1, 0, 0], [0, 0, 0, 0]], + # does not include lower row + dtype=np.bool), + label=2, group=1), + ]), + DatasetItem(id=1, subset='train', + annotations=[ + BboxObject(0, 1, 3, 3, label=4, group=3, + attributes={ 'is_crowd': True }, id=3), + MaskObject(np.array([[0, 0, 0, 0], [1, 0, 1, 0], + [1, 1, 0, 0], [0, 0, 1, 0]], + dtype=np.bool), + label=4, group=3), + ]), + + DatasetItem(id=2, subset='val', + annotations=[ + BboxObject(0, 1, 3, 2, label=4, group=3, + attributes={ 'is_crowd': True }, id=3), + MaskObject(np.array([[0, 0, 0, 0], [1, 0, 1, 0], + [1, 1, 0, 0], [0, 0, 0, 0]], + dtype=np.bool), + label=4, group=3), + ]), + ] + return iter(items) + + def subsets(self): + return ['train', 'val'] + + def categories(self): + label_categories = LabelCategories() + for i in range(10): + label_categories.add(str(i)) + return { + AnnotationType.label: label_categories, + } + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoInstancesConverter, test_dir) + + def test_can_save_and_load_images(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=0, subset='train'), + DatasetItem(id=1, subset='train'), + + DatasetItem(id=2, subset='val'), + DatasetItem(id=3, subset='val'), + DatasetItem(id=4, subset='val'), + + DatasetItem(id=5, subset='test'), + ] + return iter(items) + + def subsets(self): + return ['train', 'val', 'test'] + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoImageInfoConverter, test_dir) + + def test_can_save_and_load_labels(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=0, subset='train', + annotations=[ + LabelObject(4, id=1), + LabelObject(9, id=2), + ]), + DatasetItem(id=1, subset='train', + annotations=[ + LabelObject(4, id=4), + ]), + + DatasetItem(id=2, subset='val', + annotations=[ + LabelObject(2, id=1), + ]), + ] + return iter(items) + + def subsets(self): + return ['train', 'val'] + + def categories(self): + label_categories = LabelCategories() + for i in range(10): + label_categories.add(str(i)) + return { + AnnotationType.label: label_categories, + } + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoLabelsConverter, test_dir) + + def test_can_save_and_load_keypoints(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=0, subset='train', + annotations=[ + PointsObject([1, 2, 0, 2, 4, 1], [0, 1, 2], + label=3, group=1, id=1), + BboxObject(1, 2, 3, 4, label=3, group=1), + PointsObject([5, 6, 0, 7], group=2, id=2), + BboxObject(1, 2, 3, 4, group=2), + ]), + DatasetItem(id=1, subset='train', + annotations=[ + PointsObject([1, 2, 0, 2, 4, 1], label=5, + group=3, id=3), + BboxObject(1, 2, 3, 4, label=5, group=3), + ]), + + DatasetItem(id=2, subset='val', + annotations=[ + PointsObject([0, 2, 0, 2, 4, 1], label=2, + group=3, id=3), + BboxObject(0, 2, 4, 4, label=2, group=3), + ]), + ] + return iter(items) + + def subsets(self): + return ['train', 'val'] + + def categories(self): + label_categories = LabelCategories() + points_categories = PointsCategories() + for i in range(10): + label_categories.add(str(i)) + points_categories.add(i, []) + + return { + AnnotationType.label: label_categories, + AnnotationType.points: points_categories, + } + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoPersonKeypointsConverter, test_dir) + + def test_can_save_dataset_with_no_subsets(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=1, annotations=[ + LabelObject(2, id=1), + ]), + + DatasetItem(id=2, image=np.zeros((5, 5, 3)), annotations=[ + LabelObject(3, id=3), + BboxObject(0, 0, 5, 5, label=3, + attributes={ 'is_crowd': False }, id=4, group=4), + PolygonObject([0, 0, 4, 0, 4, 4], + label=3, group=4), + MaskObject(np.array([ + [0, 1, 1, 1, 0], + [0, 0, 1, 1, 0], + [0, 0, 0, 1, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0]], + # only internal fragment (without the border), + # but not everywhere... + dtype=np.bool), + label=3, group=4), + ]), + ] + + for item in items: + yield item + + def categories(self): + label_cat = LabelCategories() + for label in range(10): + label_cat.add('label_' + str(label)) + return { + AnnotationType.label: label_cat, + } + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + CocoConverter, test_dir) \ No newline at end of file diff --git a/datumaro/tests/test_command_targets.py b/datumaro/tests/test_command_targets.py new file mode 100644 index 000000000000..d029b92397a8 --- /dev/null +++ b/datumaro/tests/test_command_targets.py @@ -0,0 +1,131 @@ +import cv2 +import numpy as np +import os.path as osp + +from unittest import TestCase + +from datumaro.components.project import Project +from datumaro.util.command_targets import ProjectTarget, \ + ImageTarget, SourceTarget +from datumaro.util.test_utils import current_function_name, TestDir + + +class CommandTargetsTest(TestCase): + def test_image_false_when_no_file(self): + path = '%s.jpg' % current_function_name() + target = ImageTarget() + + status = target.test(path) + + self.assertFalse(status) + + def test_image_false_when_false(self): + with TestDir() as test_dir: + path = osp.join(test_dir.path, 'test.jpg') + with open(path, 'w+') as f: + f.write('qwerty123') + + target = ImageTarget() + + status = target.test(path) + + self.assertFalse(status) + + def test_image_true_when_true(self): + with TestDir() as test_dir: + path = osp.join(test_dir.path, 'test.jpg') + image = np.random.random_sample([10, 10, 3]) + cv2.imwrite(path, image) + + target = ImageTarget() + + status = target.test(path) + + self.assertTrue(status) + + def test_project_false_when_no_file(self): + path = '%s.jpg' % current_function_name() + target = ProjectTarget() + + status = target.test(path) + + self.assertFalse(status) + + def test_project_false_when_no_name(self): + target = ProjectTarget(project=Project()) + + status = target.test('') + + self.assertFalse(status) + + def test_project_true_when_project_file(self): + with TestDir() as test_dir: + path = osp.join(test_dir.path, 'test.jpg') + Project().save(path) + + target = ProjectTarget() + + status = target.test(path) + + self.assertTrue(status) + + def test_project_true_when_project_name(self): + project_name = 'qwerty' + project = Project({ + 'project_name': project_name + }) + target = ProjectTarget(project=project) + + status = target.test(project_name) + + self.assertTrue(status) + + def test_project_false_when_not_project_name(self): + project_name = 'qwerty' + project = Project({ + 'project_name': project_name + }) + target = ProjectTarget(project=project) + + status = target.test(project_name + '123') + + self.assertFalse(status) + + def test_project_true_when_not_project_file(self): + with TestDir() as test_dir: + path = osp.join(test_dir.path, 'test.jpg') + with open(path, 'w+') as f: + f.write('wqererw') + + target = ProjectTarget() + + status = target.test(path) + + self.assertFalse(status) + + def test_source_false_when_no_project(self): + target = SourceTarget() + + status = target.test('qwerty123') + + self.assertFalse(status) + + def test_source_true_when_source_exists(self): + source_name = 'qwerty' + project = Project() + project.add_source(source_name) + target = SourceTarget(project=project) + + status = target.test(source_name) + + self.assertTrue(status) + + def test_source_false_when_source_doesnt_exist(self): + source_name = 'qwerty' + project = Project() + project.add_source(source_name) + target = SourceTarget(project=project) + + status = target.test(source_name + '123') + + self.assertFalse(status) \ No newline at end of file diff --git a/datumaro/tests/test_datumaro_format.py b/datumaro/tests/test_datumaro_format.py new file mode 100644 index 000000000000..63907be6f449 --- /dev/null +++ b/datumaro/tests/test_datumaro_format.py @@ -0,0 +1,101 @@ +from itertools import zip_longest +import numpy as np + +from unittest import TestCase + +from datumaro.components.project import Project +from datumaro.components.extractor import (Extractor, DatasetItem, + AnnotationType, LabelObject, MaskObject, PointsObject, PolygonObject, + PolyLineObject, BboxObject, CaptionObject, + LabelCategories, MaskCategories, PointsCategories +) +from datumaro.components.converters.datumaro import DatumaroConverter +from datumaro.util.test_utils import TestDir +from datumaro.util.mask_tools import generate_colormap + + +class DatumaroConverterTest(TestCase): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=100, subset='train', + annotations=[ + CaptionObject('hello', id=1), + CaptionObject('world', id=2, group=5), + LabelObject(2, id=3, attributes={ + 'x': 1, + 'y': '2', + }), + BboxObject(1, 2, 3, 4, label=4, id=4, attributes={ + 'score': 10.0, + }), + BboxObject(5, 6, 7, 8, id=5, group=5), + PointsObject([1, 2, 2, 0, 1, 1], label=0, id=5), + MaskObject(label=3, id=5, image=np.ones((2, 3))), + ]), + DatasetItem(id=21, subset='train', + annotations=[ + CaptionObject('test'), + LabelObject(2), + BboxObject(1, 2, 3, 4, 5, id=42, group=42) + ]), + + DatasetItem(id=2, subset='val', + annotations=[ + PolyLineObject([1, 2, 3, 4, 5, 6, 7, 8], id=11), + PolygonObject([1, 2, 3, 4, 5, 6, 7, 8], id=12), + ]), + + DatasetItem(id=42, subset='test'), + ] + return iter(items) + + def subsets(self): + return ['train', 'val', 'test'] + + def categories(self): + label_categories = LabelCategories() + for i in range(5): + label_categories.add('cat' + str(i)) + + mask_categories = MaskCategories( + generate_colormap(len(label_categories.items))) + + points_categories = PointsCategories() + for index, _ in enumerate(label_categories.items): + points_categories.add(index, ['cat1', 'cat2'], adjacent=[0, 1]) + + return { + AnnotationType.label: label_categories, + AnnotationType.mask: mask_categories, + AnnotationType.points: points_categories, + } + + def test_can_save_and_load(self): + with TestDir() as test_dir: + source_dataset = self.TestExtractor() + + converter = DatumaroConverter( + save_images=True, apply_colormap=True) + converter(source_dataset, test_dir.path) + + project = Project.import_from(test_dir.path, 'datumaro') + parsed_dataset = project.make_dataset() + + self.assertListEqual( + sorted(source_dataset.subsets()), + sorted(parsed_dataset.subsets()), + ) + + self.assertEqual(len(source_dataset), len(parsed_dataset)) + + for subset_name in source_dataset.subsets(): + source_subset = source_dataset.get_subset(subset_name) + parsed_subset = parsed_dataset.get_subset(subset_name) + for idx, (item_a, item_b) in enumerate( + zip_longest(source_subset, parsed_subset)): + self.assertEqual(item_a, item_b, str(idx)) + + self.assertEqual( + source_dataset.categories(), + parsed_dataset.categories()) \ No newline at end of file diff --git a/datumaro/tests/test_diff.py b/datumaro/tests/test_diff.py new file mode 100644 index 000000000000..5f0655f1228f --- /dev/null +++ b/datumaro/tests/test_diff.py @@ -0,0 +1,142 @@ +from unittest import TestCase + +from datumaro.components.extractor import DatasetItem, LabelObject, BboxObject +from datumaro.components.comparator import Comparator + + +class DiffTest(TestCase): + def test_no_bbox_diff_with_same_item(self): + detections = 3 + anns = [ + BboxObject(i * 10, 10, 10, 10, label=i, + attributes={'score': (1.0 + i) / detections}) \ + for i in range(detections) + ] + item = DatasetItem(id=0, annotations=anns) + + iou_thresh = 0.5 + conf_thresh = 0.5 + comp = Comparator( + iou_threshold=iou_thresh, conf_threshold=conf_thresh) + + result = comp.compare_item_bboxes(item, item) + + matches, mispred, a_greater, b_greater = result + self.assertEqual(0, len(mispred)) + self.assertEqual(0, len(a_greater)) + self.assertEqual(0, len(b_greater)) + self.assertEqual(len([it for it in item.annotations \ + if conf_thresh < it.attributes['score']]), + len(matches)) + for a_bbox, b_bbox in matches: + self.assertLess(iou_thresh, a_bbox.iou(b_bbox)) + self.assertEqual(a_bbox.label, b_bbox.label) + self.assertLess(conf_thresh, a_bbox.attributes['score']) + self.assertLess(conf_thresh, b_bbox.attributes['score']) + + def test_can_find_bbox_with_wrong_label(self): + detections = 3 + class_count = 2 + item1 = DatasetItem(id=1, annotations=[ + BboxObject(i * 10, 10, 10, 10, label=i, + attributes={'score': (1.0 + i) / detections}) \ + for i in range(detections) + ]) + item2 = DatasetItem(id=2, annotations=[ + BboxObject(i * 10, 10, 10, 10, label=(i + 1) % class_count, + attributes={'score': (1.0 + i) / detections}) \ + for i in range(detections) + ]) + + iou_thresh = 0.5 + conf_thresh = 0.5 + comp = Comparator( + iou_threshold=iou_thresh, conf_threshold=conf_thresh) + + result = comp.compare_item_bboxes(item1, item2) + + matches, mispred, a_greater, b_greater = result + self.assertEqual(len([it for it in item1.annotations \ + if conf_thresh < it.attributes['score']]), + len(mispred)) + self.assertEqual(0, len(a_greater)) + self.assertEqual(0, len(b_greater)) + self.assertEqual(0, len(matches)) + for a_bbox, b_bbox in mispred: + self.assertLess(iou_thresh, a_bbox.iou(b_bbox)) + self.assertEqual((a_bbox.label + 1) % class_count, b_bbox.label) + self.assertLess(conf_thresh, a_bbox.attributes['score']) + self.assertLess(conf_thresh, b_bbox.attributes['score']) + + def test_can_find_missing_boxes(self): + detections = 3 + class_count = 2 + item1 = DatasetItem(id=1, annotations=[ + BboxObject(i * 10, 10, 10, 10, label=i, + attributes={'score': (1.0 + i) / detections}) \ + for i in range(detections) if i % 2 == 0 + ]) + item2 = DatasetItem(id=2, annotations=[ + BboxObject(i * 10, 10, 10, 10, label=(i + 1) % class_count, + attributes={'score': (1.0 + i) / detections}) \ + for i in range(detections) if i % 2 == 1 + ]) + + iou_thresh = 0.5 + conf_thresh = 0.5 + comp = Comparator( + iou_threshold=iou_thresh, conf_threshold=conf_thresh) + + result = comp.compare_item_bboxes(item1, item2) + + matches, mispred, a_greater, b_greater = result + self.assertEqual(0, len(mispred)) + self.assertEqual(len([it for it in item1.annotations \ + if conf_thresh < it.attributes['score']]), + len(a_greater)) + self.assertEqual(len([it for it in item2.annotations \ + if conf_thresh < it.attributes['score']]), + len(b_greater)) + self.assertEqual(0, len(matches)) + + def test_no_label_diff_with_same_item(self): + detections = 3 + anns = [ + LabelObject(i, attributes={'score': (1.0 + i) / detections}) \ + for i in range(detections) + ] + item = DatasetItem(id=1, annotations=anns) + + conf_thresh = 0.5 + comp = Comparator(conf_threshold=conf_thresh) + + result = comp.compare_item_labels(item, item) + + matches, a_greater, b_greater = result + self.assertEqual(0, len(a_greater)) + self.assertEqual(0, len(b_greater)) + self.assertEqual(len([it for it in item.annotations \ + if conf_thresh < it.attributes['score']]), + len(matches)) + + def test_can_find_wrong_label(self): + item1 = DatasetItem(id=1, annotations=[ + LabelObject(0), + LabelObject(1), + LabelObject(2), + ]) + item2 = DatasetItem(id=2, annotations=[ + LabelObject(2), + LabelObject(3), + LabelObject(4), + ]) + + conf_thresh = 0.5 + comp = Comparator(conf_threshold=conf_thresh) + + result = comp.compare_item_labels(item1, item2) + + matches, a_greater, b_greater = result + self.assertEqual(2, len(a_greater)) + self.assertEqual(2, len(b_greater)) + self.assertEqual(1, len(matches)) \ No newline at end of file diff --git a/datumaro/tests/test_project.py b/datumaro/tests/test_project.py new file mode 100644 index 000000000000..1d9df96f14ab --- /dev/null +++ b/datumaro/tests/test_project.py @@ -0,0 +1,450 @@ +import os +import os.path as osp + +from unittest import TestCase + +from datumaro.components.project import Project, Environment +from datumaro.components.project import Source, Model +from datumaro.components.launcher import Launcher, InferenceWrapper +from datumaro.components.converter import Converter +from datumaro.components.extractor import Extractor, DatasetItem, LabelObject +from datumaro.components.config import Config, DefaultConfig, SchemaBuilder +from datumaro.components.dataset_filter import XPathDatasetFilter +from datumaro.util.test_utils import TestDir + + +class ProjectTest(TestCase): + def test_project_generate(self): + src_config = Config({ + 'project_name': 'test_project', + 'format_version': 1, + }) + + with TestDir() as test_dir: + project_path = test_dir.path + Project.generate(project_path, src_config) + + self.assertTrue(osp.isdir(project_path)) + + result_config = Project.load(project_path).config + self.assertEqual( + src_config.project_name, result_config.project_name) + self.assertEqual( + src_config.format_version, result_config.format_version) + + @staticmethod + def test_default_ctor_is_ok(): + Project() + + @staticmethod + def test_empty_config_is_ok(): + Project(Config()) + + def test_add_source(self): + source_name = 'source' + origin = Source({ + 'url': 'path', + 'format': 'ext' + }) + project = Project() + + project.add_source(source_name, origin) + + added = project.get_source(source_name) + self.assertIsNotNone(added) + self.assertEqual(added, origin) + + def test_added_source_can_be_saved(self): + source_name = 'source' + origin = Source({ + 'url': 'path', + }) + project = Project() + project.add_source(source_name, origin) + + saved = project.config + + self.assertEqual(origin, saved.sources[source_name]) + + def test_added_source_can_be_dumped(self): + source_name = 'source' + origin = Source({ + 'url': 'path', + }) + project = Project() + project.add_source(source_name, origin) + + with TestDir() as test_dir: + project.save(test_dir.path) + + loaded = Project.load(test_dir.path) + loaded = loaded.get_source(source_name) + self.assertEqual(origin, loaded) + + def test_can_import_with_custom_importer(self): + class TestImporter: + def __call__(self, path, subset=None): + return Project({ + 'project_filename': path, + 'subsets': [ subset ] + }) + + path = 'path' + importer_name = 'test_importer' + + env = Environment() + env.importers.register(importer_name, TestImporter) + + project = Project.import_from(path, importer_name, env, + subset='train') + + self.assertEqual(path, project.config.project_filename) + self.assertListEqual(['train'], project.config.subsets) + + def test_can_dump_added_model(self): + model_name = 'model' + + project = Project() + saved = Model({ 'launcher': 'name' }) + project.add_model(model_name, saved) + + with TestDir() as test_dir: + project.save(test_dir.path) + + loaded = Project.load(test_dir.path) + loaded = loaded.get_model(model_name) + self.assertEqual(saved, loaded) + + def test_can_have_project_source(self): + with TestDir() as test_dir: + Project.generate(test_dir.path) + + project2 = Project() + project2.add_source('project1', { + 'url': test_dir.path, + }) + dataset = project2.make_dataset() + + self.assertTrue('project1' in dataset.sources) + + def test_can_batch_launch_custom_model(self): + class TestExtractor(Extractor): + def __init__(self, url, n=0): + super().__init__(length=n) + self.n = n + + def __iter__(self): + for i in range(self.n): + yield DatasetItem(id=i, subset='train', image=i) + + def subsets(self): + return ['train'] + + class TestLauncher(Launcher): + def __init__(self, **kwargs): + pass + + def launch(self, inputs): + for i, inp in enumerate(inputs): + yield [ LabelObject(attributes={'idx': i, 'data': inp}) ] + + model_name = 'model' + launcher_name = 'custom_launcher' + + project = Project() + project.env.launchers.register(launcher_name, TestLauncher) + project.add_model(model_name, { 'launcher': launcher_name }) + model = project.make_executable_model(model_name) + extractor = TestExtractor('', n=5) + + batch_size = 3 + executor = InferenceWrapper(extractor, model, batch_size=batch_size) + + for item in executor: + self.assertEqual(1, len(item.annotations)) + self.assertEqual(int(item.id) % batch_size, + item.annotations[0].attributes['idx']) + self.assertEqual(int(item.id), + item.annotations[0].attributes['data']) + + def test_can_do_transform_with_custom_model(self): + class TestExtractorSrc(Extractor): + def __init__(self, url, n=2): + super().__init__(length=n) + self.n = n + + def __iter__(self): + for i in range(self.n): + yield DatasetItem(id=i, subset='train', image=i, + annotations=[ LabelObject(i) ]) + + def subsets(self): + return ['train'] + + class TestLauncher(Launcher): + def __init__(self, **kwargs): + pass + + def launch(self, inputs): + for inp in inputs: + yield [ LabelObject(inp) ] + + class TestConverter(Converter): + def __call__(self, extractor, save_dir): + for item in extractor: + with open(osp.join(save_dir, '%s.txt' % item.id), 'w+') as f: + f.write(str(item.subset) + '\n') + f.write(str(item.annotations[0].label) + '\n') + + class TestExtractorDst(Extractor): + def __init__(self, url): + super().__init__() + self.items = [osp.join(url, p) for p in sorted(os.listdir(url))] + + def __iter__(self): + for path in self.items: + with open(path, 'r') as f: + index = osp.splitext(osp.basename(path))[0] + subset = f.readline()[:-1] + label = int(f.readline()[:-1]) + assert(subset == 'train') + yield DatasetItem(id=index, subset=subset, + annotations=[ LabelObject(label) ]) + + def __len__(self): + return len(self.items) + + def subsets(self): + return ['train'] + + + model_name = 'model' + launcher_name = 'custom_launcher' + extractor_name = 'custom_extractor' + + project = Project() + project.env.launchers.register(launcher_name, TestLauncher) + project.env.extractors.register(extractor_name, TestExtractorSrc) + project.env.converters.register(extractor_name, TestConverter) + project.add_model(model_name, { 'launcher': launcher_name }) + project.add_source('source', { 'format': extractor_name }) + + with TestDir() as test_dir: + project.make_dataset().transform(model_name, test_dir.path) + + result = Project.load(test_dir.path) + result.env.extractors.register(extractor_name, TestExtractorDst) + it = iter(result.make_dataset()) + item1 = next(it) + item2 = next(it) + self.assertEqual(0, item1.annotations[0].label) + self.assertEqual(1, item2.annotations[0].label) + + def test_source_datasets_can_be_merged(self): + class TestExtractor(Extractor): + def __init__(self, url, n=0, s=0): + super().__init__(length=n) + self.n = n + self.s = s + + def __iter__(self): + for i in range(self.n): + yield DatasetItem(id=self.s + i, subset='train') + + def subsets(self): + return ['train'] + + e_name1 = 'e1' + e_name2 = 'e2' + n1 = 2 + n2 = 4 + + project = Project() + project.env.extractors.register(e_name1, lambda p: TestExtractor(p, n=n1)) + project.env.extractors.register(e_name2, lambda p: TestExtractor(p, n=n2, s=n1)) + project.add_source('source1', { 'format': e_name1 }) + project.add_source('source2', { 'format': e_name2 }) + + dataset = project.make_dataset() + + self.assertEqual(n1 + n2, len(dataset)) + + def test_project_filter_can_be_applied(self): + class TestExtractor(Extractor): + def __init__(self, url, n=10): + super().__init__(length=n) + self.n = n + + def __iter__(self): + for i in range(self.n): + yield DatasetItem(id=i, subset='train') + + def subsets(self): + return ['train'] + + e_type = 'type' + project = Project() + project.env.extractors.register(e_type, TestExtractor) + project.add_source('source', { 'format': e_type }) + project.set_filter('/item[id < 5]') + + dataset = project.make_dataset() + + self.assertEqual(5, len(dataset)) + + def test_project_own_dataset_can_be_modified(self): + project = Project() + dataset = project.make_dataset() + + item = DatasetItem(id=1) + dataset.put(item) + + self.assertEqual(item, next(iter(dataset))) + + def test_project_compound_child_can_be_modified_recursively(self): + with TestDir() as test_dir: + child1 = Project({ + 'project_dir': osp.join(test_dir.path, 'child1'), + }) + child1.save() + + child2 = Project({ + 'project_dir': osp.join(test_dir.path, 'child2'), + }) + child2.save() + + parent = Project() + parent.add_source('child1', { + 'url': child1.config.project_dir + }) + parent.add_source('child2', { + 'url': child2.config.project_dir + }) + dataset = parent.make_dataset() + + item1 = DatasetItem(id='ch1', path=['child1']) + item2 = DatasetItem(id='ch2', path=['child2']) + dataset.put(item1) + dataset.put(item2) + + self.assertEqual(2, len(dataset)) + self.assertEqual(1, len(dataset.sources['child1'])) + self.assertEqual(1, len(dataset.sources['child2'])) + + def test_project_can_merge_item_annotations(self): + class TestExtractor(Extractor): + def __init__(self, url, v=None): + super().__init__() + self.v = v + + def __iter__(self): + v1_item = DatasetItem(id=1, subset='train', annotations=[ + LabelObject(2, id=3), + LabelObject(3, attributes={ 'x': 1 }), + ]) + + v2_item = DatasetItem(id=1, subset='train', annotations=[ + LabelObject(3, attributes={ 'x': 1 }), + LabelObject(4, id=4), + ]) + + if self.v == 1: + yield v1_item + else: + yield v2_item + + def subsets(self): + return ['train'] + + project = Project() + project.env.extractors.register('t1', lambda p: TestExtractor(p, v=1)) + project.env.extractors.register('t2', lambda p: TestExtractor(p, v=2)) + project.add_source('source1', { 'format': 't1' }) + project.add_source('source2', { 'format': 't2' }) + + merged = project.make_dataset() + + self.assertEqual(1, len(merged)) + + item = next(iter(merged)) + self.assertEqual(3, len(item.annotations)) + +class DatasetFilterTest(TestCase): + class TestExtractor(Extractor): + def __init__(self, url, n=0): + super().__init__(length=n) + self.n = n + + def __iter__(self): + for i in range(self.n): + yield DatasetItem(id=i, subset='train') + + def subsets(self): + return ['train'] + + def test_xpathfilter_can_be_applied(self): + extractor = self.TestExtractor('', n=4) + dataset_filter = XPathDatasetFilter('/item[id > 1]') + + filtered = extractor.select(dataset_filter) + + self.assertEqual(2, len(filtered)) + +class ConfigTest(TestCase): + def test_can_produce_multilayer_config_from_dict(self): + schema_low = SchemaBuilder() \ + .add('options', dict) \ + .build() + schema_mid = SchemaBuilder() \ + .add('desc', lambda: Config(schema=schema_low)) \ + .build() + schema_top = SchemaBuilder() \ + .add('container', lambda: DefaultConfig( + lambda v: Config(v, schema=schema_mid))) \ + .build() + + value = 1 + source = Config({ + 'container': { + 'elem': { + 'desc': { + 'options': { + 'k': value + } + } + } + } + }, schema=schema_top) + + self.assertEqual(value, source.container['elem'].desc.options['k']) + +class ExtractorTest(TestCase): + def test_custom_extractor_can_be_created(self): + class CustomExtractor(Extractor): + def __init__(self, url): + super().__init__() + + def __iter__(self): + return iter([ + DatasetItem(id=0, subset='train'), + DatasetItem(id=1, subset='train'), + DatasetItem(id=2, subset='train'), + + DatasetItem(id=3, subset='test'), + ]) + + def subsets(self): + return ['train', 'test'] + + extractor_name = 'ext1' + project = Project() + project.env.extractors.register(extractor_name, CustomExtractor) + project.add_source('src1', { + 'url': 'path', + 'format': extractor_name, + }) + project.set_subsets(['train']) + + dataset = project.make_dataset() + + self.assertEqual(3, len(dataset)) diff --git a/datumaro/tests/test_voc_format.py b/datumaro/tests/test_voc_format.py new file mode 100644 index 000000000000..841ce7264afb --- /dev/null +++ b/datumaro/tests/test_voc_format.py @@ -0,0 +1,487 @@ +import cv2 +from itertools import zip_longest +import numpy as np +import os +import os.path as osp +from xml.etree import ElementTree as ET +import shutil + +from unittest import TestCase + +from datumaro.components.extractor import (Extractor, DatasetItem, + AnnotationType, BboxObject, LabelCategories, +) +import datumaro.components.formats.voc as VOC +from datumaro.components.extractors.voc import ( + VocClassificationExtractor, + VocDetectionExtractor, + VocSegmentationExtractor, + VocLayoutExtractor, + VocActionExtractor, +) +from datumaro.components.converters.voc import ( + VocConverter, + VocClassificationConverter, + VocDetectionConverter, + VocLayoutConverter, + VocActionConverter, + VocSegmentationConverter, +) +from datumaro.components.importers.voc import VocImporter +from datumaro.util import find +from datumaro.util.test_utils import TestDir + + +class VocTest(TestCase): + def test_colormap_generator(self): + reference = [ + [ 0, 0, 0], + [128, 0, 0], + [ 0, 128, 0], + [128, 128, 0], + [ 0, 0, 128], + [128, 0, 128], + [ 0, 128, 128], + [128, 128, 128], + [ 64, 0, 0], + [192, 0, 0], + [ 64, 128, 0], + [192, 128, 0], + [ 64, 0, 128], + [192, 0, 128], + [ 64, 128, 128], + [192, 128, 128], + [ 0, 64, 0], + [128, 64, 0], + [ 0, 192, 0], + [128, 192, 0], + ] + + self.assertTrue(np.array_equal(reference, list(VOC.VocColormap.values()))) + +def get_label(extractor, label_id): + return extractor.categories()[AnnotationType.label].items[label_id].name + +def generate_dummy_voc(path): + cls_subsets_dir = osp.join(path, 'ImageSets', 'Main') + action_subsets_dir = osp.join(path, 'ImageSets', 'Action') + layout_subsets_dir = osp.join(path, 'ImageSets', 'Layout') + segm_subsets_dir = osp.join(path, 'ImageSets', 'Segmentation') + ann_dir = osp.join(path, 'Annotations') + img_dir = osp.join(path, 'JPEGImages') + segm_dir = osp.join(path, 'SegmentationClass') + inst_dir = osp.join(path, 'SegmentationObject') + + os.makedirs(cls_subsets_dir) + os.makedirs(ann_dir) + os.makedirs(img_dir) + os.makedirs(segm_dir) + os.makedirs(inst_dir) + + subsets = { + 'train': ['2007_000001'], + 'test': ['2007_000002'], + } + + # Subsets + for subset_name, subset in subsets.items(): + for item in subset: + with open(osp.join(cls_subsets_dir, subset_name + '.txt'), 'w') as f: + for item in subset: + f.write('%s\n' % item) + shutil.copytree(cls_subsets_dir, action_subsets_dir) + shutil.copytree(cls_subsets_dir, layout_subsets_dir) + shutil.copytree(cls_subsets_dir, segm_subsets_dir) + + # Classification + subset_name = 'train' + subset = subsets[subset_name] + for label in VOC.VocLabel: + with open(osp.join(cls_subsets_dir, '%s_%s.txt' % \ + (label.name, subset_name)), 'w') as f: + for item in subset: + presence = label.value % 2 + f.write('%s %2d\n' % (item, 1 if presence else -1)) + + # Detection + Action + Layout + subset_name = 'train' + subset = subsets[subset_name] + for item in subset: + root_elem = ET.Element('annotation') + ET.SubElement(root_elem, 'folder').text = 'VOC' + item.split('_')[0] + ET.SubElement(root_elem, 'filename').text = item + '.jpg' + + size_elem = ET.SubElement(root_elem, 'size') + ET.SubElement(size_elem, 'width').text = '10' + ET.SubElement(size_elem, 'height').text = '20' + ET.SubElement(size_elem, 'depth').text = '3' + + ET.SubElement(root_elem, 'segmented').text = '1' + + obj1_elem = ET.SubElement(root_elem, 'object') + ET.SubElement(obj1_elem, 'name').text = VOC.VocLabel(1).name + ET.SubElement(obj1_elem, 'pose').text = VOC.VocPose(1).name + ET.SubElement(obj1_elem, 'truncated').text = '1' + ET.SubElement(obj1_elem, 'difficult').text = '0' + obj1bb_elem = ET.SubElement(obj1_elem, 'bndbox') + ET.SubElement(obj1bb_elem, 'xmin').text = '1' + ET.SubElement(obj1bb_elem, 'ymin').text = '2' + ET.SubElement(obj1bb_elem, 'xmax').text = '3' + ET.SubElement(obj1bb_elem, 'ymax').text = '4' + + obj2_elem = ET.SubElement(root_elem, 'object') + ET.SubElement(obj2_elem, 'name').text = VOC.VocLabel.person.name + obj2bb_elem = ET.SubElement(obj2_elem, 'bndbox') + ET.SubElement(obj2bb_elem, 'xmin').text = '4' + ET.SubElement(obj2bb_elem, 'ymin').text = '5' + ET.SubElement(obj2bb_elem, 'xmax').text = '6' + ET.SubElement(obj2bb_elem, 'ymax').text = '7' + obj2head_elem = ET.SubElement(obj2_elem, 'part') + ET.SubElement(obj2head_elem, 'name').text = VOC.VocBodyPart(1).name + obj2headbb_elem = ET.SubElement(obj2head_elem, 'bndbox') + ET.SubElement(obj2headbb_elem, 'xmin').text = '5' + ET.SubElement(obj2headbb_elem, 'ymin').text = '6' + ET.SubElement(obj2headbb_elem, 'xmax').text = '7' + ET.SubElement(obj2headbb_elem, 'ymax').text = '8' + obj2act_elem = ET.SubElement(obj2_elem, 'actions') + for act in VOC.VocAction: + ET.SubElement(obj2act_elem, act.name).text = '%s' % (act.value % 2) + + with open(osp.join(ann_dir, item + '.xml'), 'w') as f: + f.write(ET.tostring(root_elem, encoding='unicode')) + + # Segmentation + Instances + subset_name = 'train' + subset = subsets[subset_name] + for item in subset: + cv2.imwrite(osp.join(segm_dir, item + '.png'), + np.ones([10, 20, 3]) * VOC.VocColormap[2]) + cv2.imwrite(osp.join(inst_dir, item + '.png'), + np.ones([10, 20, 3]) * VOC.VocColormap[2]) + + # Test images + subset_name = 'test' + subset = subsets[subset_name] + for item in subset: + cv2.imwrite(osp.join(img_dir, item + '.jpg'), + np.ones([10, 20, 3])) + + return subsets + +class VocExtractorTest(TestCase): + def test_can_load_voc_cls(self): + with TestDir() as test_dir: + generated_subsets = generate_dummy_voc(test_dir.path) + + extractor = VocClassificationExtractor(test_dir.path) + + self.assertEqual(len(generated_subsets), len(extractor.subsets())) + + subset_name = 'train' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + count = 0 + for label in VOC.VocLabel: + if label.value % 2 == 1: + count += 1 + ann = find(item.annotations, + lambda x: x.type == AnnotationType.label and \ + x.label == label.value) + self.assertFalse(ann is None) + self.assertEqual(count, len(item.annotations)) + + subset_name = 'test' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + self.assertEqual(0, len(item.annotations)) + + def test_can_load_voc_det(self): + with TestDir() as test_dir: + generated_subsets = generate_dummy_voc(test_dir.path) + + extractor = VocDetectionExtractor(test_dir.path) + + self.assertEqual(len(generated_subsets), len(extractor.subsets())) + + subset_name = 'train' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + obj1 = find(item.annotations, + lambda x: x.type == AnnotationType.bbox and \ + get_label(extractor, x.label) == VOC.VocLabel(1).name) + self.assertFalse(obj1 is None) + self.assertListEqual([1, 2, 2, 2], obj1.get_bbox()) + self.assertDictEqual( + { + 'pose': VOC.VocPose(1).name, + 'truncated': True, + 'difficult': False, + }, + obj1.attributes) + + obj2 = find(item.annotations, + lambda x: x.type == AnnotationType.bbox and \ + get_label(extractor, x.label) == VOC.VocLabel.person.name) + self.assertFalse(obj2 is None) + self.assertListEqual([4, 5, 2, 2], obj2.get_bbox()) + + self.assertEqual(2, len(item.annotations)) + + subset_name = 'test' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + self.assertEqual(0, len(item.annotations)) + + def test_can_load_voc_segm(self): + with TestDir() as test_dir: + generated_subsets = generate_dummy_voc(test_dir.path) + + extractor = VocSegmentationExtractor(test_dir.path) + + self.assertEqual(len(generated_subsets), len(extractor.subsets())) + + subset_name = 'train' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + cls_mask = find(item.annotations, + lambda x: x.type == AnnotationType.mask and \ + x.attributes.get('class') == True) + self.assertFalse(cls_mask is None) + self.assertFalse(cls_mask.image is None) + + inst_mask = find(item.annotations, + lambda x: x.type == AnnotationType.mask and \ + x.attributes.get('instances') == True) + self.assertFalse(inst_mask is None) + self.assertFalse(inst_mask.image is None) + + self.assertEqual(2, len(item.annotations)) + + subset_name = 'test' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + self.assertEqual(0, len(item.annotations)) + + def test_can_load_voc_layout(self): + with TestDir() as test_dir: + generated_subsets = generate_dummy_voc(test_dir.path) + + extractor = VocLayoutExtractor(test_dir.path) + + self.assertEqual(len(generated_subsets), len(extractor.subsets())) + + subset_name = 'train' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + obj2 = find(item.annotations, + lambda x: x.type == AnnotationType.bbox and \ + get_label(extractor, x.label) == VOC.VocLabel.person.name) + self.assertFalse(obj2 is None) + self.assertListEqual([4, 5, 2, 2], obj2.get_bbox()) + + obj2head = find(item.annotations, + lambda x: x.type == AnnotationType.bbox and \ + get_label(extractor, x.label) == VOC.VocBodyPart(1).name) + self.assertTrue(obj2.id == obj2head.group) + self.assertListEqual([5, 6, 2, 2], obj2head.get_bbox()) + + self.assertEqual(2, len(item.annotations)) + + subset_name = 'test' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + self.assertEqual(0, len(item.annotations)) + + def test_can_load_voc_action(self): + with TestDir() as test_dir: + generated_subsets = generate_dummy_voc(test_dir.path) + + extractor = VocActionExtractor(test_dir.path) + + self.assertEqual(len(generated_subsets), len(extractor.subsets())) + + subset_name = 'train' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + obj2 = find(item.annotations, + lambda x: x.type == AnnotationType.bbox and \ + get_label(extractor, x.label) == VOC.VocLabel.person.name) + self.assertFalse(obj2 is None) + self.assertListEqual([4, 5, 2, 2], obj2.get_bbox()) + + count = 1 + for action in VOC.VocAction: + if action.value % 2 == 1: + count += 1 + ann = find(item.annotations, + lambda x: x.type == AnnotationType.label and \ + get_label(extractor, x.label) == action.name) + self.assertFalse(ann is None) + self.assertTrue(obj2.id == ann.group) + self.assertEqual(count, len(item.annotations)) + + subset_name = 'test' + generated_subset = generated_subsets[subset_name] + for id_ in generated_subset: + parsed_subset = extractor.get_subset(subset_name) + self.assertEqual(len(generated_subset), len(parsed_subset)) + + item = find(parsed_subset, lambda x: x.id == id_) + self.assertFalse(item is None) + + self.assertEqual(0, len(item.annotations)) + +class VocConverterTest(TestCase): + def _test_can_save_voc(self, extractor_type, converter_type, test_dir): + dummy_dir = osp.join(test_dir, 'dummy') + generate_dummy_voc(dummy_dir) + gen_extractor = extractor_type(dummy_dir) + + conv_dir = osp.join(test_dir, 'converted') + converter = converter_type() + converter(gen_extractor, conv_dir) + + conv_extractor = extractor_type(conv_dir) + for item_a, item_b in zip_longest(gen_extractor, conv_extractor): + self.assertEqual(item_a.id, item_b.id) + self.assertEqual(len(item_a.annotations), len(item_b.annotations)) + for ann_a, ann_b in zip(item_a.annotations, item_b.annotations): + self.assertEqual(ann_a.type, ann_b.type) + + def test_can_save_voc_cls(self): + with TestDir() as test_dir: + self._test_can_save_voc( + VocClassificationExtractor, VocClassificationConverter, + test_dir.path) + + def test_can_save_voc_det(self): + with TestDir() as test_dir: + self._test_can_save_voc( + VocDetectionExtractor, VocDetectionConverter, + test_dir.path) + + def test_can_save_voc_segm(self): + with TestDir() as test_dir: + self._test_can_save_voc( + VocSegmentationExtractor, VocSegmentationConverter, + test_dir.path) + + def test_can_save_voc_layout(self): + with TestDir() as test_dir: + self._test_can_save_voc( + VocLayoutExtractor, VocLayoutConverter, + test_dir.path) + + def test_can_save_voc_action(self): + with TestDir() as test_dir: + self._test_can_save_voc( + VocActionExtractor, VocActionConverter, + test_dir.path) + + def test_can_save_dataset_with_no_subsets(self): + class TestExtractor(Extractor): + def __iter__(self): + items = [ + DatasetItem(id=1, annotations=[ + BboxObject(2, 3, 4, 5, label=2, id=1), + BboxObject(2, 3, 4, 5, label=3, id=2), + ]), + + DatasetItem(id=2, annotations=[ + BboxObject(5, 4, 6, 5, label=3, id=1), + ]), + ] + + for item in items: + yield item + + def categories(self): + label_cat = LabelCategories() + for label in VOC.VocLabel: + label_cat.add(label.name) + return { + AnnotationType.label: label_cat, + } + + with TestDir() as test_dir: + src_extractor = TestExtractor() + converter = VocConverter() + + converter(src_extractor, test_dir.path) + + dst_extractor = VocImporter()(test_dir.path).make_dataset() + + self.assertEqual(len(src_extractor), len(dst_extractor)) + for item_a, item_b in zip_longest(src_extractor, dst_extractor): + self.assertEqual(item_a.id, item_b.id) + self.assertEqual(len(item_a.annotations), len(item_b.annotations)) + for ann_a, ann_b in zip(item_a.annotations, item_b.annotations): + self.assertEqual(ann_a.type, ann_b.type) + +class VocImporterTest(TestCase): + def test_can_import(self): + with TestDir() as test_dir: + dummy_dir = osp.join(test_dir.path, 'dummy') + subsets = generate_dummy_voc(dummy_dir) + + dataset = VocImporter()(dummy_dir).make_dataset() + + self.assertEqual(len(VOC.VocTask), len(dataset.sources)) + self.assertEqual(set(subsets), set(dataset.subsets())) + self.assertEqual( + sum([len(s) for _, s in subsets.items()]), + len(dataset)) \ No newline at end of file diff --git a/supervisord.conf b/supervisord.conf index 554ae6fa7fb7..d5e222d2a99b 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -41,6 +41,12 @@ command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" numprocs=1 +[program:rqscheduler] +command=%(ENV_HOME)s/wait-for-it.sh redis:6379 -t 0 -- bash -ic \ + "/usr/bin/python3 /usr/local/bin/rqscheduler --host redis -i 30" +environment=SSH_AUTH_SOCK="/tmp/ssh-agent.sock" +numprocs=1 + [program:runserver] ; Here need to run a couple of commands to initialize DB and copy static files. ; We cannot initialize DB on build because the DB should be online. Also some