diff --git a/.travis.yml b/.travis.yml index 24843e8ee6..6b9f252237 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ matrix: install: - pip install -e ./ - pip install tensorflow + - pip install pandas script: - python -m unittest discover -v diff --git a/CHANGELOG.md b/CHANGELOG.md index 56ef01f423..5c9b6c138e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,33 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 24/03/2021 - Release v0.1.7 +### Added +- OpenVINO plugin examples () +- Dataset validation for classification and detection datasets () +- Arbitrary image extensions in formats (import and export) () +- Ability to set a custom subset name for an imported dataset () +- CLI support for NDR() + +### Changed +- Common ICDAR format is split into 3 sub-formats () + +### Deprecated +- + +### Removed +- + +### Fixed +- The ability to work with file names containing Cyrillic and spaces () +- Image reading and saving in ICDAR formats () +- Unnecessary image loading on dataset saving () +- Allowed spaces in ICDAR captions () +- Saving of masks in VOC when masks are not requested () + +### Security +- + ## 03/02/2021 - Release v0.1.6.1 (hotfix) ### Added - @@ -34,6 +61,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `LFW` dataset format () - Support of polygons' and masks' confusion matrices and mismathing classes in `diff` command () - Add near duplicate image removal plugin () +- Sampler Plugin that analyzes inference result from the given dataset and selects samples for annotation() ### Changed - OpenVINO model launcher is updated for OpenVINO r2021.1 () diff --git a/README.md b/README.md index 5a4e582943..65aa2817c2 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,11 @@ CVAT annotations ---> Publication, statistics etc. - for detection task, based on bboxes - for re-identification task, based on labels, avoiding having same IDs in training and test splits + - Sampling a dataset + - analyzes inference result from the given dataset + and selects the ‘best’ and the ‘least amount of’ samples for annotation. + - Select the sample that best suits model training. + - sampling with Entropy based algorithm - Dataset quality checking - Simple checking for errors - Comparison with model infernece diff --git a/datumaro/cli/__main__.py b/datumaro/cli/__main__.py index 005804cae0..2ecf9f7a78 100644 --- a/datumaro/cli/__main__.py +++ b/datumaro/cli/__main__.py @@ -77,6 +77,7 @@ def make_parser(): ('stats', commands.stats, "Compute project statistics"), ('info', commands.info, "Print project info"), ('explain', commands.explain, "Run Explainable AI algorithm for model"), + ('validate', commands.validate, "Validate project") ] # Argparse doesn't support subparser groups: diff --git a/datumaro/cli/commands/__init__.py b/datumaro/cli/commands/__init__.py index 2d87b945bb..9324f12252 100644 --- a/datumaro/cli/commands/__init__.py +++ b/datumaro/cli/commands/__init__.py @@ -9,5 +9,5 @@ explain, export, merge, convert, transform, filter, diff, ediff, stats, - info + info, validate ) diff --git a/datumaro/cli/commands/validate.py b/datumaro/cli/commands/validate.py new file mode 100644 index 0000000000..13794187a7 --- /dev/null +++ b/datumaro/cli/commands/validate.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +# pylint: disable=unused-import + +from ..contexts.project import build_validate_parser as build_parser diff --git a/datumaro/cli/contexts/project/__init__.py b/datumaro/cli/contexts/project/__init__.py index 26c97bd350..44e9c82529 100644 --- a/datumaro/cli/contexts/project/__init__.py +++ b/datumaro/cli/contexts/project/__init__.py @@ -10,14 +10,14 @@ import shutil from enum import Enum -from datumaro.components.cli_plugin import CliPlugin from datumaro.components.dataset_filter import DatasetItemEncoder from datumaro.components.extractor import AnnotationType from datumaro.components.operations import (DistanceComparator, - ExactComparator, compute_ann_statistics, compute_image_statistics, mean_std) + ExactComparator, compute_ann_statistics, compute_image_statistics) from datumaro.components.project import \ PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG from datumaro.components.project import Environment, Project +from datumaro.components.validator import validate_annotations, TaskType from datumaro.util import error_rollback from ...util import (CliException, MultilineFormatter, add_subparser, @@ -791,6 +791,51 @@ def print_extractor_info(extractor, indent=''): return 0 +def build_validate_parser(parser_ctor=argparse.ArgumentParser): + parser = parser_ctor(help="Validate project", + description=""" + Validates project based on specified task type and stores + results like statistics, reports and summary in JSON file. + """, + formatter_class=MultilineFormatter) + + parser.add_argument('task_type', + choices=[task_type.name for task_type in TaskType], + help="Task type for validation") + parser.add_argument('-s', '--subset', dest='subset_name', default=None, + help="Subset to validate (default: None)") + parser.add_argument('-p', '--project', dest='project_dir', default='.', + help="Directory of the project to validate (default: current dir)") + parser.set_defaults(command=validate_command) + + return parser + +def validate_command(args): + project = load_project(args.project_dir) + task_type = args.task_type + subset_name = args.subset_name + dst_file_name = 'validation_results' + + dataset = project.make_dataset() + if subset_name is not None: + dataset = dataset.get_subset(subset_name) + dst_file_name += f'-{subset_name}' + validation_results = validate_annotations(dataset, task_type) + + def _convert_tuple_keys_to_str(d): + for key, val in list(d.items()): + if isinstance(key, tuple): + d[str(key)] = val + d.pop(key) + if isinstance(val, dict): + _convert_tuple_keys_to_str(val) + + _convert_tuple_keys_to_str(validation_results) + + dst_file = generate_next_file_name(dst_file_name, ext='.json') + log.info("Writing project validation results to '%s'" % dst_file) + with open(dst_file, 'w') as f: + json.dump(validation_results, f, indent=4, sort_keys=True) def build_parser(parser_ctor=argparse.ArgumentParser): parser = parser_ctor( @@ -814,5 +859,6 @@ def build_parser(parser_ctor=argparse.ArgumentParser): add_subparser(subparsers, 'transform', build_transform_parser) add_subparser(subparsers, 'info', build_info_parser) add_subparser(subparsers, 'stats', build_stats_parser) + add_subparser(subparsers, 'validate', build_validate_parser) return parser diff --git a/datumaro/components/converter.py b/datumaro/components/converter.py index 24e41462eb..a63f7bd1f7 100644 --- a/datumaro/components/converter.py +++ b/datumaro/components/converter.py @@ -57,15 +57,23 @@ def _find_image_ext(self, item): return self._image_ext or src_ext or self._default_image_ext - def _make_image_filename(self, item): - return item.id + self._find_image_ext(item) + def _make_image_filename(self, item, *, name=None, subdir=None): + name = name or item.id + subdir = subdir or '' + return osp.join(subdir, name + self._find_image_ext(item)) + + def _save_image(self, item, path=None, *, + name=None, subdir=None, basedir=None): + assert not ((subdir or name or basedir) and path), \ + "Can't use both subdir or name or basedir and path arguments" - def _save_image(self, item, path=None): if not item.image.has_data: log.warning("Item '%s' has no image", item.id) return - path = path or self._make_image_filename(item) + basedir = basedir or self._save_dir + path = path or osp.join(basedir, + self._make_image_filename(item, name=name, subdir=subdir)) path = osp.abspath(path) src_ext = item.image.ext.lower() diff --git a/datumaro/components/errors.py b/datumaro/components/errors.py index 7fd831e3e0..3d8da0629b 100644 --- a/datumaro/components/errors.py +++ b/datumaro/components/errors.py @@ -15,7 +15,7 @@ class DatasetError(DatumaroError): @attrs class RepeatedItemError(DatasetError): def __str__(self): - return "Item %s is repeated in the source sequence." % (self.item_id) + return "Item %s is repeated in the source sequence." % (self.item_id, ) @attrs class MismatchingImageInfoError(DatasetError): @@ -89,4 +89,219 @@ class FailedAttrVotingError(MergeError): def __str__(self): return "Item %s: attribute voting failed " \ "for ann %s, votes %s, sources %s" % \ - (self.item_id, self.ann, self.votes, self.sources) \ No newline at end of file + (self.item_id, self.ann, self.votes, self.sources) + +@attrs +class DatasetValidationError(DatumaroError): + severity = attrib() + + def to_dict(self): + return { + 'anomaly_type': self.__class__.__name__, + 'description': str(self), + 'severity': self.severity.name, + } + +@attrs +class DatasetItemValidationError(DatasetValidationError): + item_id = attrib() + subset = attrib() + + def to_dict(self): + dict_repr = super().to_dict() + dict_repr['item_id'] = self.item_id + dict_repr['subset'] = self.subset + return dict_repr + +@attrs +class MissingLabelCategories(DatasetValidationError): + def __str__(self): + return "Metadata (ex. LabelCategories) should be defined" \ + " to validate a dataset." + +@attrs +class MissingLabelAnnotation(DatasetItemValidationError): + def __str__(self): + return "Item needs a label, but not found." + +@attrs +class MultiLabelAnnotations(DatasetItemValidationError): + def __str__(self): + return 'Item needs a single label but multiple labels are found.' + +@attrs +class MissingAttribute(DatasetItemValidationError): + label_name = attrib() + attr_name = attrib() + + def __str__(self): + return f"Item needs the attribute '{self.attr_name}' " \ + f"for the label '{self.label_name}'." + +@attrs +class UndefinedLabel(DatasetItemValidationError): + label_name = attrib() + + def __str__(self): + return f"Item has the label '{self.label_name}' which " \ + "is not defined in metadata." + +@attrs +class UndefinedAttribute(DatasetItemValidationError): + label_name = attrib() + attr_name = attrib() + + def __str__(self): + return f"Item has the attribute '{self.attr_name}' for the " \ + f"label '{self.label_name}' which is not defined in metadata." + +@attrs +class LabelDefinedButNotFound(DatasetValidationError): + label_name = attrib() + + def __str__(self): + return f"The label '{self.label_name}' is defined in " \ + "metadata, but not found in the dataset." + +@attrs +class AttributeDefinedButNotFound(DatasetValidationError): + label_name = attrib() + attr_name = attrib() + + def __str__(self): + return f"The attribute '{self.attr_name}' for the label " \ + f"'{self.label_name}' is defined in metadata, but not " \ + "found in the dataset." + +@attrs +class OnlyOneLabel(DatasetValidationError): + label_name = attrib() + + def __str__(self): + return f"The dataset has only one label '{self.label_name}'." + +@attrs +class OnlyOneAttributeValue(DatasetValidationError): + label_name = attrib() + attr_name = attrib() + value = attrib() + + def __str__(self): + return "The dataset has the only attribute value " \ + f"'{self.value}' for the attribute '{self.attr_name}' for the " \ + f"label '{self.label_name}'." + +@attrs +class FewSamplesInLabel(DatasetValidationError): + label_name = attrib() + count = attrib() + + def __str__(self): + return f"The number of samples in the label '{self.label_name}'" \ + f" might be too low. Found '{self.count}' samples." + +@attrs +class FewSamplesInAttribute(DatasetValidationError): + label_name = attrib() + attr_name = attrib() + attr_value = attrib() + count = attrib() + + def __str__(self): + return "The number of samples for attribute = value " \ + f"'{self.attr_name} = {self.attr_value}' for the label " \ + f"'{self.label_name}' might be too low. " \ + f"Found '{self.count}' samples." + +@attrs +class ImbalancedLabels(DatasetValidationError): + def __str__(self): + return 'There is an imbalance in the label distribution.' + +@attrs +class ImbalancedAttribute(DatasetValidationError): + label_name = attrib() + attr_name = attrib() + + def __str__(self): + return "There is an imbalance in the distribution of attribute" \ + f" '{self. attr_name}' for the label '{self.label_name}'." + +@attrs +class ImbalancedBboxDistInLabel(DatasetValidationError): + label_name = attrib() + prop = attrib() + + def __str__(self): + return f"Values of bbox '{self.prop}' are not evenly " \ + f"distributed for '{self.label_name}' label." + +@attrs +class ImbalancedBboxDistInAttribute(DatasetValidationError): + label_name = attrib() + attr_name = attrib() + attr_value = attrib() + prop = attrib() + + def __str__(self): + return f"Values of bbox '{self.prop}' are not evenly " \ + f"distributed for '{self.attr_name}' = '{self.attr_value}' for " \ + f"the '{self.label_name}' label." + +@attrs +class MissingBboxAnnotation(DatasetItemValidationError): + def __str__(self): + return 'Item needs one or more bounding box annotations, ' \ + 'but not found.' + +@attrs +class NegativeLength(DatasetItemValidationError): + ann_id = attrib() + prop = attrib() + val = attrib() + + def __str__(self): + return f"Bounding box annotation '{self.ann_id}' in " \ + "the item should have a positive value of " \ + f"'{self.prop}' but got '{self.val}'." + +@attrs +class InvalidValue(DatasetItemValidationError): + ann_id = attrib() + prop = attrib() + + def __str__(self): + return f"Bounding box annotation '{self.ann_id}' in " \ + 'the item has an inf or a NaN value of ' \ + f"bounding box '{self.prop}'." + +@attrs +class FarFromLabelMean(DatasetItemValidationError): + label_name = attrib() + ann_id = attrib() + prop = attrib() + mean = attrib() + val = attrib() + + def __str__(self): + return f"Bounding box annotation '{self.ann_id}' in " \ + f"the item has a value of bounding box '{self.prop}' that " \ + "is too far from the label average. (mean of " \ + f"'{self.label_name}' label: {self.mean}, got '{self.val}')." + +@attrs +class FarFromAttrMean(DatasetItemValidationError): + label_name = attrib() + ann_id = attrib() + attr_name = attrib() + attr_value = attrib() + prop = attrib() + mean = attrib() + val = attrib() + + def __str__(self): + return f"Bounding box annotation '{self.ann_id}' in the " \ + f"item has a value of bounding box '{self.prop}' that " \ + "is too far from the attribute average. (mean of " \ + f"'{self.attr_name}' = '{self.attr_value}' for the " \ + f"'{self.label_name}' label: {self.mean}, got '{self.val}')." diff --git a/datumaro/components/validator.py b/datumaro/components/validator.py new file mode 100644 index 0000000000..84fba8d35c --- /dev/null +++ b/datumaro/components/validator.py @@ -0,0 +1,902 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from copy import deepcopy +from enum import Enum +from typing import Union + +import numpy as np + +from datumaro.components.dataset import IDataset +from datumaro.components.errors import (MissingLabelCategories, + MissingLabelAnnotation, MultiLabelAnnotations, MissingAttribute, + UndefinedLabel, UndefinedAttribute, LabelDefinedButNotFound, + AttributeDefinedButNotFound, OnlyOneLabel, FewSamplesInLabel, + FewSamplesInAttribute, ImbalancedLabels, ImbalancedAttribute, + ImbalancedBboxDistInLabel, ImbalancedBboxDistInAttribute, + MissingBboxAnnotation, NegativeLength, InvalidValue, FarFromLabelMean, + FarFromAttrMean, OnlyOneAttributeValue) +from datumaro.components.extractor import AnnotationType, LabelCategories +from datumaro.util import parse_str_enum_value + + +Severity = Enum('Severity', ['warning', 'error']) + +TaskType = Enum('TaskType', ['classification', 'detection']) + + +class _Validator: + """ + A base class for task-specific validators. + + ... + + Attributes + ---------- + task_type : str or TaskType + task type (ie. classification, detection etc.) + ann_type : str or AnnotationType + annotation type to validate (default is AnnotationType.label) + far_from_mean_thr : float + constant used to define mean +/- k * stdev (default is None) + + Methods + ------- + compute_statistics(dataset): + Computes various statistics of the dataset based on task type. + generate_reports(stats): + Abstract method that must be implemented in a subclass. + """ + + def __init__(self, task_type=None, ann_type=None, far_from_mean_thr=None): + task_type = parse_str_enum_value(task_type, TaskType, + default=TaskType.classification) + ann_type = parse_str_enum_value(ann_type, AnnotationType, + default=AnnotationType.label) + + self.task_type = task_type + self.ann_type = ann_type + self.far_from_mean_thr = far_from_mean_thr + + def compute_statistics(self, dataset): + """ + Computes various statistics of the dataset based on task type. + + Parameters + ---------- + dataset : IDataset object + + Returns + ------- + stats (dict): A dict object containing statistics of the dataset. + """ + + defined_attr_template = { + 'items_missing_attribute': [], + 'distribution': {} + } + + undefined_attr_template = { + 'items_with_undefined_attr': [], + 'distribution': {} + } + + undefined_label_template = { + 'count': 0, + 'items_with_undefined_label': [], + } + + stats = { + 'label_distribution': { + 'defined_labels': {}, + 'undefined_labels': {}, + }, + 'attribute_distribution': { + 'defined_attributes': {}, + 'undefined_attributes': {} + }, + } + + label_dist = stats['label_distribution'] + attr_dist = stats['attribute_distribution'] + defined_label_dist = label_dist['defined_labels'] + defined_attr_dist = attr_dist['defined_attributes'] + undefined_label_dist = label_dist['undefined_labels'] + undefined_attr_dist = attr_dist['undefined_attributes'] + + label_categories = dataset.categories().get(AnnotationType.label, + LabelCategories()) + base_valid_attrs = label_categories.attributes + + if self.task_type == TaskType.classification: + stats['total_label_count'] = 0 + stats['items_missing_label'] = [] + stats['items_with_multiple_labels'] = [] + + elif self.task_type == TaskType.detection: + bbox_info_template = { + 'items_far_from_mean': {}, + 'mean': None, + 'stdev': None, + 'min': None, + 'max': None, + 'median': None, + 'histogram': { + 'bins': [], + 'counts': [], + }, + 'distribution': np.array([]) + } + + bbox_template = { + 'width': deepcopy(bbox_info_template), + 'height': deepcopy(bbox_info_template), + 'area(wxh)': deepcopy(bbox_info_template), + 'ratio(w/h)': deepcopy(bbox_info_template), + 'short': deepcopy(bbox_info_template), + 'long': deepcopy(bbox_info_template) + } + + stats['total_bbox_count'] = 0 + stats['items_missing_bbox'] = [] + stats['items_with_negative_length'] = {} + stats['items_with_invalid_value'] = {} + stats['bbox_distribution_in_label'] = {} + stats['bbox_distribution_in_attribute'] = {} + stats['bbox_distribution_in_dataset_item'] = {} + + bbox_dist_by_label = stats['bbox_distribution_in_label'] + bbox_dist_by_attr = stats['bbox_distribution_in_attribute'] + bbox_dist_in_item = stats['bbox_distribution_in_dataset_item'] + items_w_neg_len = stats['items_with_negative_length'] + items_w_invalid_val = stats['items_with_invalid_value'] + _k = self.far_from_mean_thr + + def _update_prop_distributions(ann_bbox_info, target_stats): + for prop, val in ann_bbox_info.items(): + prop_stats = target_stats[prop] + prop_dist = prop_stats['distribution'] + prop_stats['distribution'] = np.append(prop_dist, val) + + def _generate_ann_bbox_info(_x, _y, _w, _h, area, + ratio, _short, _long): + return { + 'x': _x, + 'y': _y, + 'width': _w, + 'height': _h, + 'area(wxh)': area, + 'ratio(w/h)': ratio, + 'short': _short, + 'long': _long, + } + + def _update_bbox_stats_by_label(item, ann, bbox_label_stats): + bbox_has_error = False + + _x, _y, _w, _h = ann.get_bbox() + area = ann.get_area() + + if _h != 0 and _h != float('inf'): + ratio = _w / _h + else: + ratio = float('nan') + + _short = _w if _w < _h else _h + _long = _w if _w > _h else _h + + ann_bbox_info = _generate_ann_bbox_info( + _x, _y, _w, _h, area, ratio, _short, _long) + + for prop, val in ann_bbox_info.items(): + if val == float('inf') or np.isnan(val): + bbox_has_error = True + anns_w_invalid_val = items_w_invalid_val.setdefault( + (item.id, item.subset), {}) + invalid_props = anns_w_invalid_val.setdefault( + ann.id, []) + invalid_props.append(prop) + + for prop in ['width', 'height']: + val = ann_bbox_info[prop] + if val < 1: + bbox_has_error = True + anns_w_neg_len = items_w_neg_len.setdefault( + (item.id, item.subset), {}) + neg_props = anns_w_neg_len.setdefault(ann.id, {}) + neg_props[prop] = val + + if not bbox_has_error: + ann_bbox_info.pop('x') + ann_bbox_info.pop('y') + _update_prop_distributions(ann_bbox_info, bbox_label_stats) + + return ann_bbox_info, bbox_has_error + + def _compute_prop_stats_from_dist(): + for label_name, bbox_stats in bbox_dist_by_label.items(): + prop_stats_list = list(bbox_stats.values()) + bbox_attr_label = bbox_dist_by_attr.get(label_name, {}) + for vals in bbox_attr_label.values(): + for val_stats in vals.values(): + prop_stats_list += list(val_stats.values()) + + for prop_stats in prop_stats_list: + prop_dist = prop_stats.pop('distribution', []) + if len(prop_dist) > 0: + prop_stats['mean'] = np.mean(prop_dist) + prop_stats['stdev'] = np.std(prop_dist) + prop_stats['min'] = np.min(prop_dist) + prop_stats['max'] = np.max(prop_dist) + prop_stats['median'] = np.median(prop_dist) + + counts, bins = np.histogram(prop_dist) + prop_stats['histogram']['bins'] = bins.tolist() + prop_stats['histogram']['counts'] = counts.tolist() + + def _is_valid_bbox(item, ann): + is_bbox = ann.type == self.ann_type + has_defined_label = 0 <= ann.label < len(label_categories) + if not is_bbox or not has_defined_label: + return False + + bbox_has_neg_len = ann.id in items_w_neg_len.get( + (item.id, item.subset), {}) + bbox_has_invalid_val = ann.id in items_w_invalid_val.get( + (item.id, item.subset), {}) + return not (bbox_has_neg_len or bbox_has_invalid_val) + + def _far_from_mean(val, mean, stdev): + return val > mean + (_k * stdev) or val < mean - (_k * stdev) + + def _update_props_far_from_mean(item, ann): + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + label_name = label_categories[ann.label].name + bbox_label_stats = bbox_dist_by_label[label_name] + + _x, _y, _w, _h = ann.get_bbox() + area = ann.get_area() + ratio = _w / _h + _short = _w if _w < _h else _h + _long = _w if _w > _h else _h + + ann_bbox_info = _generate_ann_bbox_info( + _x, _y, _w, _h, area, ratio, _short, _long) + ann_bbox_info.pop('x') + ann_bbox_info.pop('y') + + for prop, val in ann_bbox_info.items(): + prop_stats = bbox_label_stats[prop] + items_far_from_mean = prop_stats['items_far_from_mean'] + mean = prop_stats['mean'] + stdev = prop_stats['stdev'] + + if _far_from_mean(val, mean, stdev): + bboxs_far_from_mean = items_far_from_mean.setdefault( + (item.id, item.subset), {}) + bboxs_far_from_mean[ann.id] = val + + for attr, value in ann.attributes.items(): + if attr in valid_attrs: + bbox_attr_stats = bbox_dist_by_attr[label_name][attr] + bbox_val_stats = bbox_attr_stats[str(value)] + + for prop, val in ann_bbox_info.items(): + prop_stats = bbox_val_stats[prop] + items_far_from_mean = \ + prop_stats['items_far_from_mean'] + mean = prop_stats['mean'] + stdev = prop_stats['stdev'] + + if _far_from_mean(val, mean, stdev): + bboxs_far_from_mean = \ + items_far_from_mean.setdefault( + (item.id, item.subset), {}) + bboxs_far_from_mean[ann.id] = val + + for category in label_categories: + defined_label_dist[category.name] = 0 + + for item in dataset: + ann_count = [ann.type == self.ann_type \ + for ann in item.annotations].count(True) + + if self.task_type == TaskType.classification: + if ann_count == 0: + stats['items_missing_label'].append((item.id, item.subset)) + elif ann_count > 1: + stats['items_with_multiple_labels'].append( + (item.id, item.subset)) + stats['total_label_count'] += ann_count + + elif self.task_type == TaskType.detection: + if ann_count < 1: + stats['items_missing_bbox'].append((item.id, item.subset)) + stats['total_bbox_count'] += ann_count + bbox_dist_in_item[(item.id, item.subset)] = ann_count + + for ann in item.annotations: + if ann.type == self.ann_type: + if not 0 <= ann.label < len(label_categories): + label_name = ann.label + + label_stats = undefined_label_dist.setdefault( + ann.label, deepcopy(undefined_label_template)) + label_stats['items_with_undefined_label'].append( + (item.id, item.subset)) + + label_stats['count'] += 1 + valid_attrs = set() + missing_attrs = set() + else: + label_name = label_categories[ann.label].name + defined_label_dist[label_name] += 1 + + defined_attr_stats = defined_attr_dist.setdefault( + label_name, {}) + + valid_attrs = base_valid_attrs.union( + label_categories[ann.label].attributes) + ann_attrs = getattr(ann, 'attributes', {}).keys() + missing_attrs = valid_attrs.difference(ann_attrs) + + for attr in valid_attrs: + defined_attr_stats.setdefault( + attr, deepcopy(defined_attr_template)) + + if self.task_type == TaskType.detection: + bbox_label_stats = bbox_dist_by_label.setdefault( + label_name, deepcopy(bbox_template)) + ann_bbox_info, bbox_has_error = \ + _update_bbox_stats_by_label( + item, ann, bbox_label_stats) + + for attr in missing_attrs: + attr_dets = defined_attr_stats[attr] + attr_dets['items_missing_attribute'].append( + (item.id, item.subset)) + + for attr, value in ann.attributes.items(): + if attr not in valid_attrs: + undefined_attr_stats = \ + undefined_attr_dist.setdefault( + label_name, {}) + attr_dets = undefined_attr_stats.setdefault( + attr, deepcopy(undefined_attr_template)) + attr_dets['items_with_undefined_attr'].append( + (item.id, item.subset)) + else: + attr_dets = defined_attr_stats[attr] + + if self.task_type == TaskType.detection and \ + ann.type == self.ann_type: + bbox_attr_label = bbox_dist_by_attr.setdefault( + label_name, {}) + bbox_attr_stats = bbox_attr_label.setdefault( + attr, {}) + bbox_val_stats = bbox_attr_stats.setdefault( + str(value), deepcopy(bbox_template)) + + if not bbox_has_error: + _update_prop_distributions( + ann_bbox_info, bbox_val_stats) + + attr_dets['distribution'].setdefault(str(value), 0) + attr_dets['distribution'][str(value)] += 1 + + if self.task_type == TaskType.detection: + _compute_prop_stats_from_dist() + + for item in dataset: + for ann in item.annotations: + if _is_valid_bbox(item, ann): + _update_props_far_from_mean(item, ann) + + return stats + + def _check_missing_label_categories(self, stats): + validation_reports = [] + + if len(stats['label_distribution']['defined_labels']) == 0: + validation_reports += self._generate_validation_report( + MissingLabelCategories, Severity.error) + + return validation_reports + + def _check_missing_attribute(self, label_name, attr_name, attr_dets): + validation_reports = [] + + items_missing_attr = attr_dets['items_missing_attribute'] + for item_id, item_subset in items_missing_attr: + details = (item_subset, label_name, attr_name) + validation_reports += self._generate_validation_report( + MissingAttribute, Severity.warning, item_id, *details) + + return validation_reports + + def _check_undefined_label(self, label_name, label_stats): + validation_reports = [] + + items_with_undefined_label = label_stats['items_with_undefined_label'] + for item_id, item_subset in items_with_undefined_label: + details = (item_subset, label_name) + validation_reports += self._generate_validation_report( + UndefinedLabel, Severity.error, item_id, *details) + + return validation_reports + + def _check_undefined_attribute(self, label_name, attr_name, attr_dets): + validation_reports = [] + + items_with_undefined_attr = attr_dets['items_with_undefined_attr'] + for item_id, item_subset in items_with_undefined_attr: + details = (item_subset, label_name, attr_name) + validation_reports += self._generate_validation_report( + UndefinedAttribute, Severity.error, item_id, *details) + + return validation_reports + + def _check_label_defined_but_not_found(self, stats): + validation_reports = [] + count_by_defined_labels = stats['label_distribution']['defined_labels'] + labels_not_found = [label_name \ + for label_name, count in count_by_defined_labels.items() \ + if count == 0] + + for label_name in labels_not_found: + validation_reports += self._generate_validation_report( + LabelDefinedButNotFound, Severity.warning, label_name) + + return validation_reports + + def _check_attribute_defined_but_not_found(self, label_name, attr_stats): + validation_reports = [] + attrs_not_found = [attr_name \ + for attr_name, attr_dets in attr_stats.items() \ + if len(attr_dets['distribution']) == 0] + + for attr_name in attrs_not_found: + details = (label_name, attr_name) + validation_reports += self._generate_validation_report( + AttributeDefinedButNotFound, Severity.warning, *details) + + return validation_reports + + def _check_only_one_label(self, stats): + validation_reports = [] + count_by_defined_labels = stats['label_distribution']['defined_labels'] + labels_found = [label_name \ + for label_name, count in count_by_defined_labels.items() \ + if count > 0] + + if len(labels_found) == 1: + validation_reports += self._generate_validation_report( + OnlyOneLabel, Severity.warning, labels_found[0]) + + return validation_reports + + def _check_only_one_attribute_value(self, label_name, attr_name, attr_dets): + validation_reports = [] + values = list(attr_dets['distribution'].keys()) + + if len(values) == 1: + details = (label_name, attr_name, values[0]) + validation_reports += self._generate_validation_report( + OnlyOneAttributeValue, Severity.warning, *details) + + return validation_reports + + def _check_few_samples_in_label(self, stats, thr): + validation_reports = [] + defined_label_dist = stats['label_distribution']['defined_labels'] + labels_with_few_samples = [(label_name, count) \ + for label_name, count in defined_label_dist.items() \ + if 0 < count < thr] + + for label_name, count in labels_with_few_samples: + validation_reports += self._generate_validation_report( + FewSamplesInLabel, Severity.warning, label_name, count) + + return validation_reports + + def _check_few_samples_in_attribute(self, label_name, + attr_name, attr_dets, thr): + validation_reports = [] + attr_values_with_few_samples = [(attr_value, count) \ + for attr_value, count in attr_dets['distribution'].items() \ + if count < thr] + + for attr_value, count in attr_values_with_few_samples: + details = (label_name, attr_name, attr_value, count) + validation_reports += self._generate_validation_report( + FewSamplesInAttribute, Severity.warning, *details) + + return validation_reports + + def _check_imbalanced_labels(self, stats, thr): + validation_reports = [] + + defined_label_dist = stats['label_distribution']['defined_labels'] + count_by_defined_labels = [count \ + for label, count in defined_label_dist.items()] + + if len(count_by_defined_labels) == 0: + return validation_reports + + count_max = np.max(count_by_defined_labels) + count_min = np.min(count_by_defined_labels) + balance = count_max / count_min if count_min > 0 else float('inf') + if balance > thr: + validation_reports += self._generate_validation_report( + ImbalancedLabels, Severity.warning) + + return validation_reports + + def _check_imbalanced_attribute(self, label_name, attr_name, + attr_dets, thr): + validation_reports = [] + + count_by_defined_attr = list(attr_dets['distribution'].values()) + if len(count_by_defined_attr) == 0: + return validation_reports + + count_max = np.max(count_by_defined_attr) + count_min = np.min(count_by_defined_attr) + balance = count_max / count_min if count_min > 0 else float('inf') + if balance > thr: + validation_reports += self._generate_validation_report( + ImbalancedAttribute, Severity.warning, label_name, attr_name) + + return validation_reports + + def generate_reports(self, stats): + raise NotImplementedError('Should be implemented in a subclass.') + + def _generate_validation_report(self, error, *args, **kwargs): + return [error(*args, **kwargs)] + + +class ClassificationValidator(_Validator): + """ + A validator class for classification tasks. + """ + + def __init__(self): + super().__init__(TaskType.classification, AnnotationType.label) + + def _check_missing_label_annotation(self, stats): + validation_reports = [] + + items_missing_label = stats['items_missing_label'] + for item_id, item_subset in items_missing_label: + validation_reports += self._generate_validation_report( + MissingLabelAnnotation, Severity.warning, item_id, item_subset) + + return validation_reports + + def _check_multi_label_annotations(self, stats): + validation_reports = [] + + items_with_multiple_labels = stats['items_with_multiple_labels'] + for item_id, item_subset in items_with_multiple_labels: + validation_reports += self._generate_validation_report( + MultiLabelAnnotations, Severity.error, item_id, item_subset) + + return validation_reports + + def generate_reports(self, stats): + """ + Validates the dataset for classification tasks based on its statistics. + + Parameters + ---------- + dataset : IDataset object + stats: Dict object + + Returns + ------- + reports (list): List of validation reports (DatasetValidationError). + """ + + reports = [] + + reports += self._check_missing_label_categories(stats) + reports += self._check_missing_label_annotation(stats) + reports += self._check_multi_label_annotations(stats) + reports += self._check_label_defined_but_not_found(stats) + reports += self._check_only_one_label(stats) + reports += self._check_few_samples_in_label(stats, 2) + reports += self._check_imbalanced_labels(stats, 5) + + label_dist = stats['label_distribution'] + attr_dist = stats['attribute_distribution'] + defined_attr_dist = attr_dist['defined_attributes'] + undefined_label_dist = label_dist['undefined_labels'] + undefined_attr_dist = attr_dist['undefined_attributes'] + + defined_labels = defined_attr_dist.keys() + for label_name in defined_labels: + attr_stats = defined_attr_dist[label_name] + + reports += self._check_attribute_defined_but_not_found( + label_name, attr_stats) + + for attr_name, attr_dets in attr_stats.items(): + reports += self._check_few_samples_in_attribute( + label_name, attr_name, attr_dets, 2) + reports += self._check_imbalanced_attribute( + label_name, attr_name, attr_dets, 5) + reports += self._check_only_one_attribute_value( + label_name, attr_name, attr_dets) + reports += self._check_missing_attribute( + label_name, attr_name, attr_dets) + + for label_name, label_stats in undefined_label_dist.items(): + reports += self._check_undefined_label(label_name, label_stats) + + for label_name, attr_stats in undefined_attr_dist.items(): + for attr_name, attr_dets in attr_stats.items(): + reports += self._check_undefined_attribute( + label_name, attr_name, attr_dets) + + return reports + + +class DetectionValidator(_Validator): + """ + A validator class for detection tasks. + """ + + DEFAULT_FAR_FROM_MEAN = 2.0 + + def __init__(self): + super().__init__(TaskType.detection, AnnotationType.bbox, + far_from_mean_thr=self.DEFAULT_FAR_FROM_MEAN) + + def _check_imbalanced_bbox_dist_in_label(self, label_name, bbox_label_stats, + thr, topk_ratio): + validation_reports = [] + + for prop, prop_stats in bbox_label_stats.items(): + value_counts = prop_stats['histogram']['counts'] + n_bucket = len(value_counts) + topk = int(np.around(n_bucket * topk_ratio)) + + if topk > 0: + topk_values = np.sort(value_counts)[-topk:] + ratio = np.sum(topk_values) / np.sum(value_counts) + if ratio > thr: + details = (label_name, prop) + validation_reports += self._generate_validation_report( + ImbalancedBboxDistInLabel, Severity.warning, *details) + + return validation_reports + + def _check_imbalanced_bbox_dist_in_attr(self, label_name, attr_name, + bbox_attr_stats, thr, topk_ratio): + validation_reports = [] + + for attr_value, value_stats in bbox_attr_stats.items(): + for prop, prop_stats in value_stats.items(): + value_counts = prop_stats['histogram']['counts'] + n_bucket = len(value_counts) + topk = int(np.around(n_bucket * topk_ratio)) + + if topk > 0: + topk_values = np.sort(value_counts)[-topk:] + ratio = np.sum(topk_values) / np.sum(value_counts) + if ratio > thr: + details = (label_name, attr_name, attr_value, prop) + validation_reports += self._generate_validation_report( + ImbalancedBboxDistInAttribute, + Severity.warning, + *details + ) + + return validation_reports + + def _check_missing_bbox_annotation(self, stats): + validation_reports = [] + + items_missing_bbox = stats['items_missing_bbox'] + for item_id, item_subset in items_missing_bbox: + validation_reports += self._generate_validation_report( + MissingBboxAnnotation, Severity.warning, item_id, item_subset) + + return validation_reports + + def _check_negative_length(self, stats): + validation_reports = [] + + items_w_neg_len = stats['items_with_negative_length'] + for item_dets, anns_w_neg_len in items_w_neg_len.items(): + item_id, item_subset = item_dets + for ann_id, props in anns_w_neg_len.items(): + for prop, val in props.items(): + val = round(val, 2) + details = (item_subset, ann_id, prop, val) + validation_reports += self._generate_validation_report( + NegativeLength, Severity.error, item_id, *details) + + return validation_reports + + def _check_invalid_value(self, stats): + validation_reports = [] + + items_w_invalid_val = stats['items_with_invalid_value'] + for item_dets, anns_w_invalid_val in items_w_invalid_val.items(): + item_id, item_subset = item_dets + for ann_id, props in anns_w_invalid_val.items(): + for prop in props: + details = (item_subset, ann_id, prop) + validation_reports += self._generate_validation_report( + InvalidValue, Severity.error, item_id, *details) + + return validation_reports + + def _check_far_from_label_mean(self, label_name, bbox_label_stats): + validation_reports = [] + + for prop, prop_stats in bbox_label_stats.items(): + items_far_from_mean = prop_stats['items_far_from_mean'] + if prop_stats['mean'] is not None: + mean = round(prop_stats['mean'], 2) + + for item_dets, anns_far_from_mean in items_far_from_mean.items(): + item_id, item_subset = item_dets + for ann_id, val in anns_far_from_mean.items(): + val = round(val, 2) + details = (item_subset, label_name, ann_id, prop, mean, val) + validation_reports += self._generate_validation_report( + FarFromLabelMean, Severity.warning, item_id, *details) + + return validation_reports + + def _check_far_from_attr_mean(self, label_name, attr_name, bbox_attr_stats): + validation_reports = [] + + for attr_value, value_stats in bbox_attr_stats.items(): + for prop, prop_stats in value_stats.items(): + items_far_from_mean = prop_stats['items_far_from_mean'] + if prop_stats['mean'] is not None: + mean = round(prop_stats['mean'], 2) + + for item_dets, anns_far_from_mean in items_far_from_mean.items(): + item_id, item_subset = item_dets + for ann_id, val in anns_far_from_mean.items(): + val = round(val, 2) + details = (item_subset, label_name, ann_id, attr_name, + attr_value, prop, mean, val) + validation_reports += self._generate_validation_report( + FarFromAttrMean, + Severity.warning, + item_id, + *details + ) + + return validation_reports + + def generate_reports(self, stats): + """ + Validates the dataset for detection tasks based on its statistics. + + Parameters + ---------- + dataset : IDataset object + stats : Dict object + + Returns + ------- + reports (list): List of validation reports (DatasetValidationError). + """ + + reports = [] + + reports += self._check_missing_label_categories(stats) + reports += self._check_missing_bbox_annotation(stats) + reports += self._check_label_defined_but_not_found(stats) + reports += self._check_only_one_label(stats) + reports += self._check_few_samples_in_label(stats, 2) + reports += self._check_imbalanced_labels(stats, 5) + reports += self._check_negative_length(stats) + reports += self._check_invalid_value(stats) + + label_dist = stats['label_distribution'] + attr_dist = stats['attribute_distribution'] + defined_attr_dist = attr_dist['defined_attributes'] + undefined_label_dist = label_dist['undefined_labels'] + undefined_attr_dist = attr_dist['undefined_attributes'] + + bbox_dist_by_label = stats['bbox_distribution_in_label'] + bbox_dist_by_attr = stats['bbox_distribution_in_attribute'] + + defined_labels = defined_attr_dist.keys() + for label_name in defined_labels: + attr_stats = defined_attr_dist[label_name] + + reports += self._check_attribute_defined_but_not_found( + label_name, attr_stats) + + for attr_name, attr_dets in attr_stats.items(): + reports += self._check_few_samples_in_attribute( + label_name, attr_name, attr_dets, 2) + reports += self._check_imbalanced_attribute( + label_name, attr_name, attr_dets, 5) + reports += self._check_only_one_attribute_value( + label_name, attr_name, attr_dets) + reports += self._check_missing_attribute( + label_name, attr_name, attr_dets) + + bbox_label_stats = bbox_dist_by_label[label_name] + bbox_attr_label = bbox_dist_by_attr.get(label_name, {}) + + reports += self._check_far_from_label_mean( + label_name, bbox_label_stats) + reports += self._check_imbalanced_bbox_dist_in_label( + label_name, bbox_label_stats, 1, 0.25) + + for attr_name, bbox_attr_stats in bbox_attr_label.items(): + reports += self._check_far_from_attr_mean( + label_name, attr_name, bbox_attr_stats) + reports += self._check_imbalanced_bbox_dist_in_attr( + label_name, attr_name, bbox_attr_stats, 1, 0.25) + + for label_name, label_stats in undefined_label_dist.items(): + reports += self._check_undefined_label(label_name, label_stats) + + for label_name, attr_stats in undefined_attr_dist.items(): + for attr_name, attr_dets in attr_stats.items(): + reports += self._check_undefined_attribute( + label_name, attr_name, attr_dets) + + return reports + + +def validate_annotations(dataset: IDataset, task_type: Union[str, TaskType]): + """ + Returns the validation results of a dataset based on task type. + + Args: + dataset (IDataset): Dataset to be validated + task_type (str or TaskType): Type of the task + (classification, detection etc.) + + Raises: + ValueError + + Returns: + validation_results (dict): + Dict with validation statistics, reports and summary. + + """ + + validation_results = {} + + task_type = parse_str_enum_value(task_type, TaskType) + if task_type == TaskType.classification: + validator = ClassificationValidator() + elif task_type == TaskType.detection: + validator = DetectionValidator() + + if not isinstance(dataset, IDataset): + raise TypeError("Invalid dataset type '%s'" % type(dataset)) + + # generate statistics + stats = validator.compute_statistics(dataset) + validation_results['statistics'] = stats + + # generate validation reports and summary + reports = validator.generate_reports(stats) + reports = list(map(lambda r : r.to_dict(), reports)) + + summary = { + 'errors': sum(map(lambda r : r['severity'] == 'error', reports)), + 'warnings': sum(map(lambda r : r['severity'] == 'warning', reports)) + } + + validation_results['validation_reports'] = reports + validation_results['summary'] = summary + + return validation_results diff --git a/datumaro/plugins/camvid_format.py b/datumaro/plugins/camvid_format.py index ace780148b..76de818196 100644 --- a/datumaro/plugins/camvid_format.py +++ b/datumaro/plugins/camvid_format.py @@ -3,21 +3,22 @@ # # SPDX-License-Identifier: MIT +import logging as log import os import os.path as osp from collections import OrderedDict from enum import Enum -from glob import glob import numpy as np + from datumaro.components.converter import Converter from datumaro.components.extractor import (AnnotationType, CompiledMask, DatasetItem, Importer, LabelCategories, Mask, MaskCategories, SourceExtractor) from datumaro.util import find, str_to_bool +from datumaro.util.annotation_util import make_label_id_mapping from datumaro.util.image import save_image -from datumaro.util.mask_tools import lazy_mask, paint_mask, generate_colormap - +from datumaro.util.mask_tools import generate_colormap, lazy_mask, paint_mask CamvidLabelMap = OrderedDict([ ('Void', (0, 0, 0)), @@ -57,7 +58,8 @@ class CamvidPath: LABELMAP_FILE = 'label_colors.txt' SEGM_DIR = "annot" - IMAGE_EXT = '.png' + IMAGE_EXT = '.jpg' + MASK_EXT = '.png' def parse_label_map(path): @@ -133,11 +135,14 @@ def make_camvid_categories(label_map=None): class CamvidExtractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): assert osp.isfile(path), path self._path = path self._dataset_dir = osp.dirname(path) - super().__init__(subset=osp.splitext(osp.basename(path))[0]) + + if not subset: + subset = osp.splitext(osp.basename(path))[0] + super().__init__(subset=subset) self._categories = self._load_categories(self._dataset_dir) self._items = list(self._load_items(path).values()) @@ -154,33 +159,46 @@ def _load_categories(self, path): def _load_items(self, path): items = {} + + labels = self._categories[AnnotationType.label]._indices + labels = { labels[label_name]: label_name + for label_name in labels } + with open(path, encoding='utf-8') as f: for line in f: - objects = line.split() + line = line.strip() + objects = line.split('\"') + if 1 < len(objects): + if len(objects) == 5: + objects[0] = objects[1] + objects[1] = objects[3] + else: + raise Exception("Line %s: unexpected number " + "of quotes in filename" % line) + else: + objects = line.split() image = objects[0] - item_id = ('/'.join(image.split('/')[2:]))[:-len(CamvidPath.IMAGE_EXT)] - image_path = osp.join(self._dataset_dir, - (image, image[1:])[image[0] == '/']) + item_id = osp.splitext(osp.join(*image.split('/')[2:]))[0] + image_path = osp.join(self._dataset_dir, image.lstrip('/')) + item_annotations = [] if 1 < len(objects): gt = objects[1] - gt_path = osp.join(self._dataset_dir, - (gt, gt[1:]) [gt[0] == '/']) - inverse_cls_colormap = \ - self._categories[AnnotationType.mask].inverse_colormap - mask = lazy_mask(gt_path, inverse_cls_colormap) - # loading mask through cache - mask = mask() + gt_path = osp.join(self._dataset_dir, gt.lstrip('/')) + mask = lazy_mask(gt_path, + self._categories[AnnotationType.mask].inverse_colormap) + mask = mask() # loading mask through cache + classes = np.unique(mask) - labels = self._categories[AnnotationType.label]._indices - labels = { labels[label_name]: label_name - for label_name in labels } for label_id in classes: if labels[label_id] in self._labels: image = self._lazy_extract_mask(mask, label_id) - item_annotations.append(Mask(image=image, label=label_id)) + item_annotations.append( + Mask(image=image, label=label_id)) + items[item_id] = DatasetItem(id=item_id, subset=self._subset, image=image_path, annotations=item_annotations) + return items @staticmethod @@ -198,7 +216,7 @@ def find_sources(cls, path): LabelmapType = Enum('LabelmapType', ['camvid', 'source']) class CamvidConverter(Converter): - DEFAULT_IMAGE_EXT = '.png' + DEFAULT_IMAGE_EXT = CamvidPath.IMAGE_EXT @classmethod def build_cmdline_parser(cls, **kwargs): @@ -221,12 +239,15 @@ def __init__(self, extractor, save_dir, self._load_categories(label_map) def apply(self): - subset_dir = self._save_dir - os.makedirs(subset_dir, exist_ok=True) + os.makedirs(self._save_dir, exist_ok=True) for subset_name, subset in self._extractor.subsets().items(): segm_list = {} for item in subset: + image_path = self._make_image_filename(item, subdir=subset_name) + if self._save_images: + self._save_image(item, osp.join(self._save_dir, image_path)) + masks = [a for a in item.annotations if a.type == AnnotationType.mask] @@ -235,17 +256,13 @@ def apply(self): instance_labels=[self._label_id_mapping(m.label) for m in masks]) - self.save_segm(osp.join(subset_dir, - subset_name + CamvidPath.SEGM_DIR, - item.id + CamvidPath.IMAGE_EXT), + mask_path = osp.join(subset_name + CamvidPath.SEGM_DIR, + item.id + CamvidPath.MASK_EXT) + self.save_segm(osp.join(self._save_dir, mask_path), compiled_mask.class_mask) - segm_list[item.id] = True + segm_list[item.id] = (image_path, mask_path) else: - segm_list[item.id] = False - - if self._save_images: - self._save_image(item, osp.join(subset_dir, subset_name, - item.id + CamvidPath.IMAGE_EXT)) + segm_list[item.id] = (image_path, '') self.save_segm_lists(subset_name, segm_list) self.save_label_map() @@ -262,15 +279,14 @@ def save_segm_lists(self, subset_name, segm_list): return ann_file = osp.join(self._save_dir, subset_name + '.txt') - with open(ann_file, 'w') as f: - for item in segm_list: - if segm_list[item]: - path_mask = '/%s/%s' % (subset_name + CamvidPath.SEGM_DIR, - item + CamvidPath.IMAGE_EXT) - else: - path_mask = '' - f.write('/%s/%s %s\n' % (subset_name, - item + CamvidPath.IMAGE_EXT, path_mask)) + with open(ann_file, 'w', encoding='utf-8') as f: + for (image_path, mask_path) in segm_list.values(): + image_path = '/' + image_path.replace('\\', '/') + mask_path = mask_path.replace('\\', '/') + if 1 < len(image_path.split()) or 1 < len(mask_path.split()): + image_path = '\"' + image_path + '\"' + mask_path = '\"' + mask_path + '\"' + f.write('%s %s\n' % (image_path, mask_path)) def save_label_map(self): path = osp.join(self._save_dir, CamvidPath.LABELMAP_FILE) @@ -320,20 +336,24 @@ def _load_categories(self, label_map_source): self._label_id_mapping = self._make_label_id_map() def _make_label_id_map(self): - source_labels = { - id: label.name for id, label in - enumerate(self._extractor.categories().get( - AnnotationType.label, LabelCategories()).items) - } - target_labels = { - label.name: id for id, label in - enumerate(self._categories[AnnotationType.label].items) - } - id_mapping = { - src_id: target_labels.get(src_label, 0) - for src_id, src_label in source_labels.items() - } - - def map_id(src_id): - return id_mapping.get(src_id, 0) + map_id, id_mapping, src_labels, dst_labels = make_label_id_mapping( + self._extractor.categories().get(AnnotationType.label), + self._categories[AnnotationType.label]) + + void_labels = [src_label for src_id, src_label in src_labels.items() + if src_label not in dst_labels] + if void_labels: + log.warning("The following labels are remapped to background: %s" % + ', '.join(void_labels)) + log.debug("Saving segmentations with the following label mapping: \n%s" % + '\n'.join(["#%s '%s' -> #%s '%s'" % + ( + src_id, src_label, id_mapping[src_id], + self._categories[AnnotationType.label] \ + .items[id_mapping[src_id]].name + ) + for src_id, src_label in src_labels.items() + ]) + ) + return map_id diff --git a/datumaro/plugins/coco_format/extractor.py b/datumaro/plugins/coco_format/extractor.py index 4f94776b1d..29b97f7e27 100644 --- a/datumaro/plugins/coco_format/extractor.py +++ b/datumaro/plugins/coco_format/extractor.py @@ -21,11 +21,12 @@ class _CocoExtractor(SourceExtractor): - def __init__(self, path, task, merge_instance_polygons=False): + def __init__(self, path, task, merge_instance_polygons=False, subset=None): assert osp.isfile(path), path - subset = osp.splitext(osp.basename(path))[0].rsplit('_', maxsplit=1) - subset = subset[1] if len(subset) == 2 else None + if not subset: + subset = osp.splitext(osp.basename(path))[0].rsplit('_', maxsplit=1) + subset = subset[1] if len(subset) == 2 else None super().__init__(subset=subset) rootpath = '' diff --git a/datumaro/plugins/cvat_format/converter.py b/datumaro/plugins/cvat_format/converter.py index d327d7a20f..c0611e7dbb 100644 --- a/datumaro/plugins/cvat_format/converter.py +++ b/datumaro/plugins/cvat_format/converter.py @@ -11,7 +11,8 @@ from datumaro.components.converter import Converter from datumaro.components.dataset import ItemStatus -from datumaro.components.extractor import AnnotationType, DatasetItem +from datumaro.components.extractor import (AnnotationType, DatasetItem, + LabelCategories) from datumaro.util import cast, pairs from .format import CvatPath @@ -190,7 +191,8 @@ def _write_item(self, item, index): self._writer.close_image() def _write_meta(self): - label_cat = self._extractor.categories()[AnnotationType.label] + label_cat = self._extractor.categories().get( + AnnotationType.label, LabelCategories()) meta = OrderedDict([ ("task", OrderedDict([ ("id", ""), @@ -222,6 +224,8 @@ def _write_meta(self): self._writer.write_meta(meta) def _get_label(self, label_id): + if label_id is None: + return "" label_cat = self._extractor.categories()[AnnotationType.label] return label_cat.items[label_id] diff --git a/datumaro/plugins/cvat_format/extractor.py b/datumaro/plugins/cvat_format/extractor.py index 59a8f4ed94..466ab96a95 100644 --- a/datumaro/plugins/cvat_format/extractor.py +++ b/datumaro/plugins/cvat_format/extractor.py @@ -19,7 +19,7 @@ class CvatExtractor(SourceExtractor): _SUPPORTED_SHAPES = ('box', 'polygon', 'polyline', 'points') - def __init__(self, path): + def __init__(self, path, subset=None): assert osp.isfile(path), path rootpath = osp.dirname(path) images_dir = '' @@ -28,7 +28,9 @@ def __init__(self, path): self._images_dir = images_dir self._path = path - super().__init__(subset=osp.splitext(osp.basename(path))[0]) + if not subset: + subset = osp.splitext(osp.basename(path))[0] + super().__init__(subset=subset) items, categories = self._parse(path) self._items = list(self._load_items(items).values()) diff --git a/datumaro/plugins/datumaro_format/converter.py b/datumaro/plugins/datumaro_format/converter.py index 46bb812c97..9146fe8451 100644 --- a/datumaro/plugins/datumaro_format/converter.py +++ b/datumaro/plugins/datumaro_format/converter.py @@ -62,9 +62,10 @@ def write_item(self, item): self._context._save_image(item, path) item_desc['image'] = { - 'size': item.image.size, 'path': path, } + if item.image.has_size: # avoid occasional loading + item_desc['image']['size'] = item.image.size self.items.append(item_desc) for ann in item.annotations: diff --git a/datumaro/plugins/icdar_format/converter.py b/datumaro/plugins/icdar_format/converter.py index c7aeee19ef..e7759b2eba 100644 --- a/datumaro/plugins/icdar_format/converter.py +++ b/datumaro/plugins/icdar_format/converter.py @@ -10,73 +10,80 @@ from datumaro.util.image import save_image from datumaro.util.mask_tools import paint_mask -from .format import IcdarPath, IcdarTask +from .format import IcdarPath -class _WordRecognitionConverter: - def __init__(self): - self.annotations = '' +class IcdarWordRecognitionConverter(Converter): + DEFAULT_IMAGE_EXT = IcdarPath.IMAGE_EXT - def save_annotations(self, item, path): - self.annotations += '%s, ' % (item.id + IcdarPath.IMAGE_EXT) - for ann in item.annotations: - if ann.type != AnnotationType.caption: - continue - self.annotations += '\"%s\"' % ann.caption - self.annotations += '\n' + def apply(self): + for subset_name, subset in self._extractor.subsets().items(): + annotation = '' + for item in subset: + image_filename = self._make_image_filename(item) + if self._save_images and item.has_image: + self._save_image(item, osp.join(self._save_dir, + subset_name, IcdarPath.IMAGES_DIR, image_filename)) + + annotation += '%s, ' % image_filename + for ann in item.annotations: + if ann.type != AnnotationType.caption: + continue + annotation += '\"%s\"' % ann.caption + annotation += '\n' - def write(self, path): - file = osp.join(path, 'gt.txt') - os.makedirs(osp.dirname(file), exist_ok=True) - with open(file, 'w') as f: - f.write(self.annotations) + if len(annotation): + anno_file = osp.join(self._save_dir, subset_name, 'gt.txt') + os.makedirs(osp.dirname(anno_file), exist_ok=True) + with open(anno_file, 'w', encoding='utf-8') as f: + f.write(annotation) - def is_empty(self): - return len(self.annotations) == 0 +class IcdarTextLocalizationConverter(Converter): + DEFAULT_IMAGE_EXT = IcdarPath.IMAGE_EXT -class _TextLocalizationConverter: - def __init__(self): - self.annotations = {} + def apply(self): + for subset_name, subset in self._extractor.subsets().items(): + for item in subset: + if self._save_images and item.has_image: + self._save_image(item, + subdir=osp.join(subset_name, IcdarPath.IMAGES_DIR)) + + annotation = '' + for ann in item.annotations: + if ann.type == AnnotationType.bbox: + annotation += ' '.join(str(p) for p in ann.points) + if ann.attributes and 'text' in ann.attributes: + annotation += ' \"%s\"' % ann.attributes['text'] + elif ann.type == AnnotationType.polygon: + annotation += ','.join(str(p) for p in ann.points) + if ann.attributes and 'text' in ann.attributes: + annotation += ',\"%s\"' % ann.attributes['text'] + annotation += '\n' + + anno_file = osp.join(self._save_dir, subset_name, + osp.dirname(item.id), 'gt_' + osp.basename(item.id) + '.txt') + os.makedirs(osp.dirname(anno_file), exist_ok=True) + with open(anno_file, 'w', encoding='utf-8') as f: + f.write(annotation) + +class IcdarTextSegmentationConverter(Converter): + DEFAULT_IMAGE_EXT = IcdarPath.IMAGE_EXT + + def apply(self): + for subset_name, subset in self._extractor.subsets().items(): + for item in subset: + self._save_item(subset_name, subset, item) + + def _save_item(self, subset_name, subset, item): + if self._save_images and item.has_image: + self._save_image(item, + subdir=osp.join(subset_name, IcdarPath.IMAGES_DIR)) - def save_annotations(self, item, path): - annotation = '' - for ann in item.annotations: - if ann.type == AnnotationType.bbox: - annotation += ' '.join(str(p) for p in ann.points) - if ann.attributes and 'text' in ann.attributes: - annotation += ' \"%s\"' % ann.attributes['text'] - elif ann.type == AnnotationType.polygon: - annotation += ','.join(str(p) for p in ann.points) - if ann.attributes and 'text' in ann.attributes: - annotation += ',\"%s\"' % ann.attributes['text'] - annotation += '\n' - self.annotations[item.id] = annotation - - def write(self, path): - os.makedirs(path, exist_ok=True) - for item in self.annotations: - file = osp.join(path, 'gt_' + item + '.txt') - with open(file, 'w') as f: - f.write(self.annotations[item]) - - def is_empty(self): - return len(self.annotations) == 0 - -class _TextSegmentationConverter: - def __init__(self): - self.annotations = {} - - def save_annotations(self, item, path): annotation = '' colormap = [(255, 255, 255)] - anns = [a for a in item.annotations - if a.type == AnnotationType.mask] + anns = [a for a in item.annotations if a.type == AnnotationType.mask] if anns: - is_not_index = len([p for p in anns if 'index' not in p.attributes]) - if is_not_index: - raise Exception("Item %s: a mask must have" - "'index' attribute" % item.id) - anns = sorted(anns, key=lambda a: a.attributes['index']) + anns = sorted(anns, key=lambda a: int(a.attributes.get('index', 0))) group = anns[0].group for ann in anns: if ann.group != group or (not ann.group and anns[0].group != 0): @@ -95,7 +102,7 @@ def save_annotations(self, item, path): annotation += ' '.join(p for p in color) else: raise Exception("Item %s: a mask must have " - "an RGB color attribute, e. g. '10 7 50'" % item.id) + "an RGB color attribute, e.g. '10 7 50'" % item.id) if 'center' in ann.attributes: annotation += ' %s' % ann.attributes['center'] else: @@ -111,84 +118,11 @@ def save_annotations(self, item, path): instance_labels=[m.attributes['index'] + 1 for m in anns]) mask = paint_mask(mask.class_mask, { i: colormap[i] for i in range(len(colormap)) }) - save_image(osp.join(path, item.id + '_GT' + IcdarPath.GT_EXT), - mask, create_dir=True) - self.annotations[item.id] = annotation - - def write(self, path): - os.makedirs(path, exist_ok=True) - for item in self.annotations: - file = osp.join(path, item + '_GT' + '.txt') - with open(file, 'w') as f: - f.write(self.annotations[item]) - - def is_empty(self): - return len(self.annotations) == 0 - - -class IcdarConverter(Converter): - DEFAULT_IMAGE_EXT = IcdarPath.IMAGE_EXT - - _TASK_CONVERTER = { - IcdarTask.word_recognition: _WordRecognitionConverter, - IcdarTask.text_localization: _TextLocalizationConverter, - IcdarTask.text_segmentation: _TextSegmentationConverter, - } - - def __init__(self, extractor, save_dir, tasks=None, **kwargs): - super().__init__(extractor, save_dir, **kwargs) - - assert tasks is None or isinstance(tasks, (IcdarTask, list, str)) - if isinstance(tasks, IcdarTask): - tasks = [tasks] - elif isinstance(tasks, str): - tasks = [IcdarTask[tasks]] - elif tasks: - for i, t in enumerate(tasks): - if isinstance(t, str): - tasks[i] = IcdarTask[t] - else: - assert t in IcdarTask, t - self._tasks = tasks - - def _make_task_converter(self, task): - if task not in self._TASK_CONVERTER: - raise NotImplementedError() - return self._TASK_CONVERTER[task]() - - def _make_task_converters(self): - return { task: self._make_task_converter(task) - for task in (self._tasks or self._TASK_CONVERTER) } - - def apply(self): - for subset_name, subset in self._extractor.subsets().items(): - task_converters = self._make_task_converters() - for item in subset: - for task, task_conv in task_converters.items(): - if item.has_image and self._save_images: - self._save_image(item, osp.join( - self._save_dir, subset_name, IcdarPath.IMAGES_DIR, - item.id + IcdarPath.IMAGE_EXT)) - task_conv.save_annotations(item, osp.join(self._save_dir, - IcdarPath.TASK_DIR[task], subset_name)) - - for task, task_conv in task_converters.items(): - if task_conv.is_empty() and not self._tasks: - continue - task_conv.write(osp.join(self._save_dir, - IcdarPath.TASK_DIR[task], subset_name)) - -class IcdarWordRecognitionConverter(IcdarConverter): - def __init__(self, *args, **kwargs): - kwargs['tasks'] = IcdarTask.word_recognition - super().__init__(*args, **kwargs) - -class IcdarTextLocalizationConverter(IcdarConverter): - def __init__(self, *args, **kwargs): - kwargs['tasks'] = IcdarTask.text_localization - super().__init__(*args, **kwargs) - -class IcdarTextSegmentationConverter(IcdarConverter): - def __init__(self, *args, **kwargs): - kwargs['tasks'] = IcdarTask.text_segmentation - super().__init__(*args, **kwargs) + save_image(osp.join(self._save_dir, subset_name, + item.id + '_GT' + IcdarPath.GT_EXT), mask, create_dir=True) + + anno_file = osp.join(self._save_dir, subset_name, + item.id + '_GT' + '.txt') + os.makedirs(osp.dirname(anno_file), exist_ok=True) + with open(anno_file, 'w', encoding='utf-8') as f: + f.write(annotation) \ No newline at end of file diff --git a/datumaro/plugins/icdar_format/extractor.py b/datumaro/plugins/icdar_format/extractor.py index 41a4854f58..bc6f4e91c4 100644 --- a/datumaro/plugins/icdar_format/extractor.py +++ b/datumaro/plugins/icdar_format/extractor.py @@ -2,20 +2,21 @@ # # SPDX-License-Identifier: MIT -from glob import glob +from glob import iglob import os.path as osp import numpy as np from datumaro.components.extractor import (Bbox, Caption, DatasetItem, Importer, Mask, MaskCategories, Polygon, SourceExtractor) +from datumaro.util.image import find_images from datumaro.util.mask_tools import lazy_mask from .format import IcdarPath, IcdarTask class _IcdarExtractor(SourceExtractor): - def __init__(self, path, task): + def __init__(self, path, task, subset=None): self._path = path self._task = task @@ -23,7 +24,11 @@ def __init__(self, path, task): if not osp.isfile(path): raise FileNotFoundError( "Can't read annotation file '%s'" % path) - super().__init__(subset=osp.basename(osp.dirname(path))) + + if not subset: + subset = osp.basename(osp.dirname(path)) + super().__init__(subset=subset) + self._dataset_dir = osp.dirname(osp.dirname(path)) self._items = list(self._load_recognition_items().values()) @@ -31,28 +36,42 @@ def __init__(self, path, task): if not osp.isdir(path): raise NotADirectoryError( "Can't open folder with annotation files '%s'" % path) - super().__init__(subset=osp.basename(path)) + + if not subset: + subset = osp.basename(path) + super().__init__(subset=subset) + self._dataset_dir = osp.dirname(path) + if task is IcdarTask.text_localization: self._items = list(self._load_localization_items().values()) else: self._items = list(self._load_segmentation_items().values()) - def _load_recognition_items(self): items = {} + with open(self._path, encoding='utf-8') as f: for line in f: line = line.strip() objects = line.split(', ') if len(objects) == 2: image = objects[0] - captions = objects[1].split() + objects = objects[1].split('\"') + if 1 < len(objects): + if len(objects) % 2: + captions = [objects[2 * i + 1] + for i in range(int(len(objects) / 2))] + else: + raise Exception("Line %s: unexpected number " + "of quotes in filename" % line) + else: + captions = objects[0].split() else: image = objects[0][:-1] captions = [] - item_id = image[:-len(IcdarPath.IMAGE_EXT)] + item_id = osp.splitext(image)[0] image_path = osp.join(osp.dirname(self._path), IcdarPath.IMAGES_DIR, image) if item_id not in items: @@ -61,40 +80,57 @@ def _load_recognition_items(self): annotations = items[item_id].annotations for caption in captions: - if caption[0] == '\"' and caption[-1] == '\"': - caption = caption[1:-1] annotations.append(Caption(caption)) + return items def _load_localization_items(self): items = {} - for path in glob(osp.join(self._path, '*.txt')): - item_id = osp.splitext(osp.basename(path))[0] - if item_id.startswith('gt_'): - item_id = item_id[3:] - image_path = osp.join(self._path, IcdarPath.IMAGES_DIR, - item_id + IcdarPath.IMAGE_EXT) + image_dir = osp.join(self._path, IcdarPath.IMAGES_DIR) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + + for path in iglob(osp.join(self._path, '**', '*.txt'), recursive=True): + item_id = osp.splitext(osp.relpath(path, self._path))[0] + if osp.basename(item_id).startswith('gt_'): + item_id = osp.join(osp.dirname(item_id), osp.basename(item_id)[3:]) + item_id = item_id.replace('\\', '/') + if item_id not in items: items[item_id] = DatasetItem(item_id, subset=self._subset, - image=image_path) + image=images.get(item_id)) annotations = items[item_id].annotations with open(path, encoding='utf-8') as f: for line in f: line = line.strip() - objects = line.split() + objects = line.split('\"') + if 1 < len(objects): + if len(objects) == 3: + text = objects[1] + else: + raise Exception("Line %s: unexpected number " + "of quotes in filename" % line) + else: + text = '' + objects = objects[0].split() if len(objects) == 1: - objects = line.split(',') + objects = objects[0].split(',') if 8 <= len(objects): points = [float(p) for p in objects[:8]] + attributes = {} - if len(objects) == 9: + if 0 < len(text): + attributes['text'] = text + elif len(objects) == 9: text = objects[8] - if text[0] == '\"' and text[-1] == '\"': - text = text[1:-1] attributes['text'] = text + annotations.append( Polygon(points, attributes=attributes)) elif 4 <= len(objects): @@ -102,12 +138,14 @@ def _load_localization_items(self): y = float(objects[1]) w = float(objects[2]) - x h = float(objects[3]) - y + attributes = {} - if len(objects) == 5: + if 0 < len(text): + attributes['text'] = text + elif len(objects) == 5: text = objects[4] - if text[0] == '\"' and text[-1] == '\"': - text = text[1:-1] attributes['text'] = text + annotations.append( Bbox(x, y, w, h, attributes=attributes)) return items @@ -115,15 +153,22 @@ def _load_localization_items(self): def _load_segmentation_items(self): items = {} - for path in glob(osp.join(self._path, '*.txt')): - item_id = osp.splitext(osp.basename(path))[0] + image_dir = osp.join(self._path, IcdarPath.IMAGES_DIR) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + + for path in iglob(osp.join(self._path, '**', '*.txt'), recursive=True): + item_id = osp.splitext(osp.relpath(path, self._path))[0] + item_id = item_id.replace('\\', '/') if item_id.endswith('_GT'): item_id = item_id[:-3] - image_path = osp.join(self._path, IcdarPath.IMAGES_DIR, - item_id + IcdarPath.IMAGE_EXT) + if item_id not in items: items[item_id] = DatasetItem(item_id, subset=self._subset, - image=image_path) + image=images.get(item_id)) annotations = items[item_id].annotations colors = [(255, 255, 255)] @@ -149,7 +194,8 @@ def _load_segmentation_items(self): objects[9] = '\" \"' objects.pop() if len(objects) != 10: - continue + raise Exception("Line %s contains the wrong number " + "of arguments, e.g. '241 73 144 1 4 0 3 1 4 \"h\"" % line) centers.append(objects[3] + ' ' + objects[4]) groups.append(group) @@ -179,7 +225,8 @@ def _load_segmentation_items(self): i = int(label_id) annotations.append(Mask(group=groups[i], image=self._lazy_extract_mask(mask, label_id), - attributes={ 'index': i - 1, 'color': ' '.join(str(p) for p in colors[i]), + attributes={ 'index': i - 1, + 'color': ' '.join(str(p) for p in colors[i]), 'text': chars[i], 'center': centers[i] } )) return items @@ -203,30 +250,18 @@ def __init__(self, path, **kwargs): kwargs['task'] = IcdarTask.text_segmentation super().__init__(path, **kwargs) -class IcdarImporter(Importer): - _TASKS = [ - (IcdarTask.word_recognition, 'icdar_word_recognition', 'word_recognition'), - (IcdarTask.text_localization, 'icdar_text_localization', 'text_localization'), - (IcdarTask.text_segmentation, 'icdar_text_segmentation', 'text_segmentation'), - ] +class IcdarWordRecognitionImporter(Importer): + @classmethod + def find_sources(cls, path): + return cls._find_sources_recursive(path, '.txt', 'icdar_word_recognition') + +class IcdarTextLocalizationImporter(Importer): + @classmethod + def find_sources(cls, path): + return cls._find_sources_recursive(path, '', 'icdar_text_localization') + +class IcdarTextSegmentationImporter(Importer): @classmethod def find_sources(cls, path): - sources = [] - paths = [path] - if osp.basename(path) not in IcdarPath.TASK_DIR.values(): - paths = [p for p in glob(osp.join(path, '**')) - if osp.basename(p) in IcdarPath.TASK_DIR.values()] - for path in paths: - for task, extractor_type, task_dir in cls._TASKS: - if not osp.isdir(path) or osp.basename(path) != task_dir: - continue - if task is IcdarTask.word_recognition: - ext = '.txt' - elif task is IcdarTask.text_localization or \ - task is IcdarTask.text_segmentation: - ext = '' - sources += cls._find_sources_recursive(path, ext, - extractor_type, file_filter=lambda p: - osp.basename(p) != IcdarPath.VOCABULARY_FILE) - return sources + return cls._find_sources_recursive(path, '', 'icdar_text_segmentation') diff --git a/datumaro/plugins/icdar_format/format.py b/datumaro/plugins/icdar_format/format.py index 00f9493691..fb52a83eaf 100644 --- a/datumaro/plugins/icdar_format/format.py +++ b/datumaro/plugins/icdar_format/format.py @@ -15,10 +15,3 @@ class IcdarPath: IMAGE_EXT = '.png' GT_EXT = '.bmp' IMAGES_DIR = 'images' - VOCABULARY_FILE = 'vocabulary.txt' - - TASK_DIR = { - IcdarTask.word_recognition: 'word_recognition', - IcdarTask.text_localization: 'text_localization', - IcdarTask.text_segmentation: 'text_segmentation', - } diff --git a/datumaro/plugins/image_dir.py b/datumaro/plugins/image_dir_format.py similarity index 53% rename from datumaro/plugins/image_dir.py rename to datumaro/plugins/image_dir_format.py index f8a45baa67..3cca401a43 100644 --- a/datumaro/plugins/image_dir.py +++ b/datumaro/plugins/image_dir_format.py @@ -9,7 +9,7 @@ from datumaro.components.extractor import DatasetItem, SourceExtractor, Importer from datumaro.components.converter import Converter -from datumaro.util.os_util import walk +from datumaro.util.image import find_images class ImageDirImporter(Importer): @@ -20,21 +20,16 @@ def find_sources(cls, path): return [{ 'url': path, 'format': 'image_dir' }] class ImageDirExtractor(SourceExtractor): - IMAGE_EXT_FORMATS = {'.jpg', '.jpeg', '.png', '.ppm', '.bmp', - '.pgm', '.tif', '.tiff'} - - def __init__(self, url, max_depth=10): - super().__init__() + def __init__(self, url, subset=None, max_depth=None, exts=None): + super().__init__(subset=subset) assert osp.isdir(url), url - for dirpath, _, filenames in walk(url, max_depth=max_depth): - for name in filenames: - if not osp.splitext(name)[-1] in self.IMAGE_EXT_FORMATS: - continue - path = osp.join(dirpath, name) - item_id = osp.relpath(osp.splitext(path)[0], url) - self._items.append(DatasetItem(id=item_id, image=path)) + for path in find_images(url, exts=exts, + recursive=True, max_depth=max_depth): + item_id = osp.relpath(osp.splitext(path)[0], url) + self._items.append(DatasetItem(id=item_id, subset=self._subset, + image=path)) class ImageDirConverter(Converter): DEFAULT_IMAGE_EXT = '.jpg' @@ -44,7 +39,6 @@ def apply(self): for item in self._extractor: if item.has_image: - self._save_image(item, - osp.join(self._save_dir, self._make_image_filename(item))) + self._save_image(item) else: log.debug("Item '%s' has no image info", item.id) \ No newline at end of file diff --git a/datumaro/plugins/imagenet_format.py b/datumaro/plugins/imagenet_format.py index 9702262008..9254662d06 100644 --- a/datumaro/plugins/imagenet_format.py +++ b/datumaro/plugins/imagenet_format.py @@ -1,9 +1,7 @@ - # Copyright (C) 2020 Intel Corporation # # SPDX-License-Identifier: MIT -from glob import glob import logging as log import os import os.path as osp @@ -12,13 +10,11 @@ LabelCategories, AnnotationType, SourceExtractor, Importer ) from datumaro.components.converter import Converter +from datumaro.util.image import find_images class ImagenetPath: - DEFAULT_IMAGE_EXT = '.jpg' - IMAGE_EXT_FORMATS = {'.jpg', '.jpeg', '.png', '.ppm', '.bmp', - '.pgm', '.tif', '.tiff'} - IMAGES_DIR_NO_LABEL = 'no_label' + IMAGE_DIR_NO_LABEL = 'no_label' class ImagenetExtractor(SourceExtractor): @@ -31,29 +27,31 @@ def __init__(self, path, subset=None): def _load_categories(self, path): label_cat = LabelCategories() - for images_dir in sorted(os.listdir(path)): - if images_dir != ImagenetPath.IMAGES_DIR_NO_LABEL: - label_cat.add(images_dir) + for dirname in sorted(os.listdir(path)): + if dirname != ImagenetPath.IMAGE_DIR_NO_LABEL: + label_cat.add(dirname) return { AnnotationType.label: label_cat } def _load_items(self, path): items = {} - for image_path in glob(osp.join(path, '*', '*')): - if not osp.isfile(image_path) or \ - osp.splitext(image_path)[-1] not in \ - ImagenetPath.IMAGE_EXT_FORMATS: - continue + + for image_path in find_images(path, recursive=True, max_depth=1): label = osp.basename(osp.dirname(image_path)) - image_name = osp.splitext(osp.basename(image_path))[0][len(label) + 1:] + image_name = osp.splitext(osp.basename(image_path))[0] + if image_name.startswith(label + '_'): + image_name = image_name[len(label) + 1:] + item = items.get(image_name) if item is None: item = DatasetItem(id=image_name, subset=self._subset, image=image_path) + items[image_name] = item annotations = item.annotations - if label != ImagenetPath.IMAGES_DIR_NO_LABEL: + + if label != ImagenetPath.IMAGE_DIR_NO_LABEL: label = self._categories[AnnotationType.label].find(label)[0] annotations.append(Label(label=label)) - items[image_name] = item + return items @@ -66,27 +64,27 @@ def find_sources(cls, path): class ImagenetConverter(Converter): - DEFAULT_IMAGE_EXT = ImagenetPath.DEFAULT_IMAGE_EXT + DEFAULT_IMAGE_EXT = '.jpg' def apply(self): if 1 < len(self._extractor.subsets()): - log.warning("ImageNet format supports exporting only a single " + log.warning("ImageNet format only supports exporting a single " "subset, subset information will not be used.") subset_dir = self._save_dir extractor = self._extractor labels = {} for item in self._extractor: - image_name = item.id - labels[image_name] = [p.label for p in item.annotations - if p.type == AnnotationType.label] - for label in labels[image_name]: + labels = set(p.label for p in item.annotations + if p.type == AnnotationType.label) + + for label in labels: label_name = extractor.categories()[AnnotationType.label][label].name self._save_image(item, osp.join(subset_dir, label_name, '%s_%s' % (label_name, self._make_image_filename(item)))) - if not labels[image_name]: + if not labels: self._save_image(item, osp.join(subset_dir, - ImagenetPath.IMAGES_DIR_NO_LABEL, - ImagenetPath.IMAGES_DIR_NO_LABEL + '_' - + self._make_image_filename(item))) + ImagenetPath.IMAGE_DIR_NO_LABEL, + ImagenetPath.IMAGE_DIR_NO_LABEL + '_' + \ + self._make_image_filename(item))) diff --git a/datumaro/plugins/imagenet_txt_format.py b/datumaro/plugins/imagenet_txt_format.py index 36ee68a7c5..3a1578431d 100644 --- a/datumaro/plugins/imagenet_txt_format.py +++ b/datumaro/plugins/imagenet_txt_format.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: MIT -from glob import glob import os import os.path as osp @@ -11,18 +10,20 @@ LabelCategories, AnnotationType, SourceExtractor, Importer ) from datumaro.components.converter import Converter +from datumaro.util.image import find_images class ImagenetTxtPath: - DEFAULT_IMAGE_EXT = '.jpg' - IMAGE_EXT_FORMAT = ['.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif'] LABELS_FILE = 'synsets.txt' IMAGE_DIR = 'images' class ImagenetTxtExtractor(SourceExtractor): - def __init__(self, path, labels=None, image_dir=None): + def __init__(self, path, labels=None, image_dir=None, subset=None): assert osp.isfile(path), path - super().__init__(subset=osp.splitext(osp.basename(path))[0]) + + if not subset: + subset = osp.splitext(osp.basename(path))[0] + super().__init__(subset=subset) if not image_dir: image_dir = ImagenetTxtPath.IMAGE_DIR @@ -33,8 +34,8 @@ def __init__(self, path, labels=None, image_dir=None): labels = self._parse_labels(labels) else: assert all(isinstance(e, str) for e in labels) - self._categories = self._load_categories(labels) + self._items = list(self._load_items(path).values()) @staticmethod @@ -47,25 +48,39 @@ def _load_categories(self, labels): def _load_items(self, path): items = {} + + image_dir = self.image_dir + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + with open(path, encoding='utf-8') as f: for line in f: - item = line.split() - item_id = item[0] - label_ids = [int(id) for id in item[1:]] + item = line.split('\"') + if 1 < len(item): + if len(item) == 3: + item_id = item[1] + label_ids = [int(id) for id in item[2].split()] + else: + raise Exception("Line %s: unexpected number " + "of quotes in filename" % line) + else: + item = line.split() + item_id = item[0] + label_ids = [int(id) for id in item[1:]] + anno = [] for label in label_ids: assert 0 <= label and \ label < len(self._categories[AnnotationType.label]), \ "Image '%s': unknown label id '%s'" % (item_id, label) anno.append(Label(label)) - image_path = osp.join(self.image_dir, item_id + - ImagenetTxtPath.DEFAULT_IMAGE_EXT) - for path in glob(osp.join(self.image_dir, item_id + '*')): - if osp.splitext(path)[1] in ImagenetTxtPath.IMAGE_EXT_FORMAT: - image_path = path - break + items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path, annotations=anno) + image=images.get(item_id), annotations=anno) + return items @@ -78,7 +93,7 @@ def find_sources(cls, path): class ImagenetTxtConverter(Converter): - DEFAULT_IMAGE_EXT = ImagenetTxtPath.DEFAULT_IMAGE_EXT + DEFAULT_IMAGE_EXT = '.jpg' def apply(self): subset_dir = self._save_dir @@ -87,22 +102,28 @@ def apply(self): extractor = self._extractor for subset_name, subset in self._extractor.subsets().items(): annotation_file = osp.join(subset_dir, '%s.txt' % subset_name) + labels = {} for item in subset: - labels[item.id] = [str(p.label) for p in item.annotations - if p.type == AnnotationType.label] + labels[item.id] = set(p.label for p in item.annotations + if p.type == AnnotationType.label) if self._save_images and item.has_image: - self._save_image(item, - osp.join(self._save_dir, ImagenetTxtPath.IMAGE_DIR, - self._make_image_filename(item))) + self._save_image(item, subdir=ImagenetTxtPath.IMAGE_DIR) + + annotation = '' + for item_id, item_labels in labels.items(): + if 1 < len(item_id.split()): + item_id = '\"' + item_id + '\"' + annotation += '%s %s\n' % ( + item_id, ' '.join(str(l) for l in item_labels)) with open(annotation_file, 'w', encoding='utf-8') as f: - f.writelines(['%s %s\n' % (item_id, ' '.join(labels[item_id])) - for item_id in labels]) + f.write(annotation) labels_file = osp.join(subset_dir, ImagenetTxtPath.LABELS_FILE) with open(labels_file, 'w', encoding='utf-8') as f: - f.write('\n'.join(l.name - for l in extractor.categories()[AnnotationType.label]) + f.writelines(l.name + '\n' + for l in extractor.categories().get( + AnnotationType.label, LabelCategories()) ) diff --git a/datumaro/plugins/labelme_format.py b/datumaro/plugins/labelme_format.py index a3a83147b6..5580dbc77b 100644 --- a/datumaro/plugins/labelme_format.py +++ b/datumaro/plugins/labelme_format.py @@ -22,9 +22,9 @@ class LabelMePath: IMAGE_EXT = '.jpg' class LabelMeExtractor(SourceExtractor): - def __init__(self, path, subset_name=None): + def __init__(self, path, subset=None): assert osp.isdir(path), path - super().__init__(subset=subset_name) + super().__init__(subset=subset) items, categories = self._parse(path) self._categories = categories @@ -243,7 +243,7 @@ def has_annotations(d): d = osp.join(path, d) if osp.isdir(d) and has_annotations(d): subset_paths.append({'url': d, 'format': cls.EXTRACTOR, - 'options': {'subset_name': subset} + 'options': {'subset': subset} }) return subset_paths diff --git a/datumaro/plugins/lfw_format.py b/datumaro/plugins/lfw_format.py index 3d16a2949f..5799ad87e5 100644 --- a/datumaro/plugins/lfw_format.py +++ b/datumaro/plugins/lfw_format.py @@ -9,6 +9,7 @@ from datumaro.components.converter import Converter from datumaro.components.extractor import (AnnotationType, DatasetItem, Importer, Points, SourceExtractor) +from datumaro.util.image import find_images class LfwPath: @@ -19,47 +20,68 @@ class LfwPath: PATTERN = re.compile(r'([\w]+)_([-\d]+)') class LfwExtractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isfile(path): - raise NotADirectoryError("Can't read annotation file '%s'" % path) - super().__init__(subset=osp.basename(osp.dirname(path))) + raise FileNotFoundError("Can't read annotation file '%s'" % path) + + if not subset: + subset = osp.basename(osp.dirname(path)) + super().__init__(subset=subset) + self._dataset_dir = osp.dirname(osp.dirname(path)) self._items = list(self._load_items(path).values()) def _load_items(self, path): items = {} + images_dir = osp.join(self._dataset_dir, self._subset, LfwPath.IMAGES_DIR) + if osp.isdir(images_dir): + images = { osp.splitext(osp.relpath(p, images_dir))[0]: p + for p in find_images(images_dir, recursive=True) } + else: + images = {} + with open(path, encoding='utf-8') as f: for line in f: - pair = line.strip().split() + pair = line.strip().split('\t') if len(pair) == 3: - image1 = self.get_image_name(pair[0], pair[1]) - image2 = self.get_image_name(pair[0], pair[2]) + if pair[0] == '-': + image1 = pair[1] + image2 = pair[2] + else: + image1 = self.get_image_name(pair[0], pair[1]) + image2 = self.get_image_name(pair[0], pair[2]) if image1 not in items: items[image1] = DatasetItem(id=image1, subset=self._subset, - image=osp.join(images_dir, image1 + LfwPath.IMAGE_EXT), + image=images.get(image1), attributes={'positive_pairs': [], 'negative_pairs': []}) if image2 not in items: items[image2] = DatasetItem(id=image2, subset=self._subset, - image=osp.join(images_dir, image2 + LfwPath.IMAGE_EXT), + image=images.get(image2), attributes={'positive_pairs': [], 'negative_pairs': []}) - attributes = items[image1].attributes - attributes['positive_pairs'].append(image2) + # pairs form a directed graph + items[image1].attributes['positive_pairs'].append(image2) elif len(pair) == 4: - image1 = self.get_image_name(pair[0], pair[1]) - image2 = self.get_image_name(pair[2], pair[3]) + if pair[0] == '-': + image1 = pair[1] + else: + image1 = self.get_image_name(pair[0], pair[1]) + if pair[2] == '-': + image2 = pair[3] + else: + image2 = self.get_image_name(pair[2], pair[3]) if image1 not in items: items[image1] = DatasetItem(id=image1, subset=self._subset, - image=osp.join(images_dir, image1 + LfwPath.IMAGE_EXT), + image=images.get(image1), attributes={'positive_pairs': [], 'negative_pairs': []}) if image2 not in items: items[image2] = DatasetItem(id=image2, subset=self._subset, - image=osp.join(images_dir, image2 + LfwPath.IMAGE_EXT), + image=images.get(image2), attributes={'positive_pairs': [], 'negative_pairs': []}) - attributes = items[image1].attributes - attributes['negative_pairs'].append(image2) + # pairs form a directed graph + items[image1].attributes['negative_pairs'].append(image2) landmarks_file = osp.join(self._dataset_dir, self._subset, LfwPath.LANDMARKS_FILE) @@ -68,9 +90,7 @@ def _load_items(self, path): for line in f: line = line.split('\t') - item_id = line[0] - if item_id.endswith(LfwPath.IMAGE_EXT): - item_id = item_id[:-len(LfwPath.IMAGE_EXT)] + item_id = osp.splitext(line[0])[0] if item_id not in items: items[item_id] = DatasetItem(id=item_id, subset=self._subset, image=osp.join(images_dir, line[0]), @@ -78,6 +98,7 @@ def _load_items(self, path): annotations = items[item_id].annotations annotations.append(Points([float(p) for p in line[1:]])) + return items @staticmethod @@ -90,29 +111,44 @@ def find_sources(cls, path): return cls._find_sources_recursive(path, LfwPath.PAIRS_FILE, 'lfw') class LfwConverter(Converter): - DEFAULT_IMAGE_EXT = '.jpg' + DEFAULT_IMAGE_EXT = LfwPath.IMAGE_EXT def apply(self): for subset_name, subset in self._extractor.subsets().items(): positive_pairs = [] negative_pairs = [] landmarks = [] - for item in subset: - if item.has_image and self._save_images: - self._save_image(item, osp.join(self._save_dir, subset_name, - LfwPath.IMAGES_DIR, item.id + LfwPath.IMAGE_EXT)) - person1, num1 = LfwPath.PATTERN.search(item.id).groups() - num1 = int(num1) + for item in subset: + if self._save_images and item.has_image: + self._save_image(item, + subdir=osp.join(subset_name, LfwPath.IMAGES_DIR)) + + search = LfwPath.PATTERN.search(item.id) + if search: + person1, num1 = search.groups() + num1 = int(num1) + else: + person1 = '-' + num1 = item.id if 'positive_pairs' in item.attributes: for pair in item.attributes['positive_pairs']: - num2 = LfwPath.PATTERN.search(pair).groups()[1] - num2 = int(num2) + search = LfwPath.PATTERN.search(pair) + if search: + num2 = search.groups()[1] + num2 = int(num2) + else: + num2 = pair positive_pairs.append('%s\t%s\t%s' % (person1, num1, num2)) if 'negative_pairs' in item.attributes: for pair in item.attributes['negative_pairs']: - person2, num2 = LfwPath.PATTERN.search(pair).groups() - num2 = int(num2) + search = LfwPath.PATTERN.search(pair) + if search: + person2, num2 = search.groups() + num2 = int(num2) + else: + person2 = '-' + num2 = pair negative_pairs.append('%s\t%s\t%s\t%s' % \ (person1, num1, person2, num2)) diff --git a/datumaro/plugins/market1501_format.py b/datumaro/plugins/market1501_format.py index 8f4e26cc70..2493b495bc 100644 --- a/datumaro/plugins/market1501_format.py +++ b/datumaro/plugins/market1501_format.py @@ -2,80 +2,98 @@ # # SPDX-License-Identifier: MIT +import os import os.path as osp import re from distutils.util import strtobool -from glob import glob +from itertools import chain from datumaro.components.converter import Converter from datumaro.components.extractor import (DatasetItem, Importer, SourceExtractor) +from datumaro.util.image import find_images class Market1501Path: QUERY_DIR = 'query' BBOX_DIR = 'bounding_box_' IMAGE_EXT = '.jpg' - PATTERN = re.compile(r'([-\d]+)_c(\d)') - IMAGE_NAMES = 'images_' + PATTERN = re.compile(r'^(-?\d+)_c(\d+)(?:s\d+_\d+_00(.*))?') + LIST_PREFIX = 'images_' + UNKNOWN_ID = -1 class Market1501Extractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isdir(path): raise NotADirectoryError( "Can't open folder with annotation files '%s'" % path) - subset = '' - for dirname in glob(osp.join(path, '*')): - if osp.basename(dirname).startswith(Market1501Path.BBOX_DIR): - subset = osp.basename(dirname).replace(Market1501Path.BBOX_DIR, '') - if osp.basename(dirname).startswith(Market1501Path.IMAGE_NAMES): - subset = osp.basename(dirname).replace(Market1501Path.IMAGE_NAMES, '') - subset = osp.splitext(subset)[0] - break + if not subset: + subset = '' + for p in os.listdir(path): + pf = osp.join(path, p) + + if p.startswith(Market1501Path.BBOX_DIR) and osp.isdir(pf): + subset = p.replace(Market1501Path.BBOX_DIR, '') + break + + if p.startswith(Market1501Path.LIST_PREFIX) and osp.isfile(pf): + subset = p.replace(Market1501Path.LIST_PREFIX, '') + subset = osp.splitext(subset)[0] + break super().__init__(subset=subset) self._path = path self._items = list(self._load_items(path).values()) - def _load_items(self, path): + def _load_items(self, rootdir): items = {} - paths = glob(osp.join(path, Market1501Path.QUERY_DIR, '*')) - paths += glob(osp.join(path, Market1501Path.BBOX_DIR + self._subset, '*')) - - anno_file = osp.join(path, - Market1501Path.IMAGE_NAMES + self._subset + '.txt') - if len(paths) == 0 and osp.isfile(anno_file): + paths = [] + anno_file = osp.join(rootdir, + Market1501Path.LIST_PREFIX + self._subset + '.txt') + if osp.isfile(anno_file): with open(anno_file, encoding='utf-8') as f: for line in f: - paths.append(line.strip()) + paths.append(osp.join(rootdir, line.strip())) + else: + paths = list(chain( + find_images(osp.join(rootdir, + Market1501Path.QUERY_DIR), + recursive=True), + find_images(osp.join(rootdir, + Market1501Path.BBOX_DIR + self._subset), + recursive=True), + )) for image_path in paths: - if osp.splitext(image_path)[-1] != Market1501Path.IMAGE_EXT: - continue - - item_id = osp.splitext(osp.basename(image_path))[0] - pid, camid = -1, -1 - search = Market1501Path.PATTERN.search(image_path) + item_id = osp.splitext(osp.normpath(image_path))[0] + if osp.isabs(image_path): + item_id = osp.relpath(item_id, rootdir) + subdir, item_id = item_id.split(os.sep, maxsplit=1) + + pid = Market1501Path.UNKNOWN_ID + camid = Market1501Path.UNKNOWN_ID + search = Market1501Path.PATTERN.search(osp.basename(item_id)) if search: - pid, camid = map(int, search.groups()) - if 19 < len(item_id): - item_id = item_id[19:] - items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path) - - if pid == -1: - continue - - attributes = items[item_id].attributes - camid -= 1 - attributes['person_id'] = pid - attributes['camera_id'] = camid - if osp.basename(osp.dirname(image_path)) == Market1501Path.QUERY_DIR: - attributes['query'] = True - else: - attributes['query'] = False + pid, camid = map(int, search.groups()[0:2]) + camid -= 1 # make ids 0-based + custom_name = search.groups()[2] + if custom_name: + item_id = osp.join(osp.dirname(item_id), custom_name) + + item = items.get(item_id) + if item is None: + item = DatasetItem(id=item_id, subset=self._subset, + image=image_path) + items[item_id] = item + + if pid != Market1501Path.UNKNOWN_ID or \ + camid != Market1501Path.UNKNOWN_ID: + attributes = item.attributes + attributes['query'] = subdir == Market1501Path.QUERY_DIR + attributes['person_id'] = pid + attributes['camera_id'] = camid return items class Market1501Importer(Importer): @@ -86,20 +104,23 @@ def find_sources(cls, path): return [{ 'url': path, 'format': 'market1501' }] class Market1501Converter(Converter): - DEFAULT_IMAGE_EXT = '.jpg' + DEFAULT_IMAGE_EXT = Market1501Path.IMAGE_EXT def apply(self): for subset_name, subset in self._extractor.subsets().items(): annotation = '' + for item in subset: image_name = item.id if Market1501Path.PATTERN.search(image_name) == None: if 'person_id' in item.attributes and \ 'camera_id' in item.attributes: image_pattern = '{:04d}_c{}s1_000000_00{}' - pid = int(item.attributes.get('person_id')) - camid = int(item.attributes.get('camera_id')) + 1 - image_name = image_pattern.format(pid, camid, item.id) + pid = int(item.attributes['person_id']) + camid = int(item.attributes['camera_id']) + 1 + dirname, basename = osp.split(item.id) + image_name = osp.join(dirname, + image_pattern.format(pid, camid, basename)) dirname = Market1501Path.BBOX_DIR + subset_name if 'query' in item.attributes: @@ -108,15 +129,15 @@ def apply(self): query = strtobool(query) if query: dirname = Market1501Path.QUERY_DIR - image_path = osp.join(self._save_dir, dirname, - image_name + Market1501Path.IMAGE_EXT) - if item.has_image and self._save_images: - self._save_image(item, image_path) - else: - annotation += '%s\n' % image_path - - if 0 < len(annotation): - annotation_file = osp.join(self._save_dir, - Market1501Path.IMAGE_NAMES + subset_name + '.txt') - with open(annotation_file, 'w') as f: - f.write(annotation) + + image_path = self._make_image_filename(item, + name=image_name, subdir=dirname) + if self._save_images and item.has_image: + self._save_image(item, osp.join(self._save_dir, image_path)) + + annotation += '%s\n' % image_path + + annotation_file = osp.join(self._save_dir, + Market1501Path.LIST_PREFIX + subset_name + '.txt') + with open(annotation_file, 'w') as f: + f.write(annotation) diff --git a/datumaro/plugins/mot_format.py b/datumaro/plugins/mot_format.py index 2fbc28001c..8008f25d2b 100644 --- a/datumaro/plugins/mot_format.py +++ b/datumaro/plugins/mot_format.py @@ -18,7 +18,7 @@ ) from datumaro.components.converter import Converter from datumaro.util import cast -from datumaro.util.image import Image +from datumaro.util.image import Image, find_images MotLabel = Enum('MotLabel', [ @@ -59,8 +59,9 @@ class MotPath: class MotSeqExtractor(SourceExtractor): - def __init__(self, path, labels=None, occlusion_threshold=0, is_gt=None): - super().__init__() + def __init__(self, path, labels=None, occlusion_threshold=0, is_gt=None, + subset=None): + super().__init__(subset=subset) assert osp.isfile(path) seq_root = osp.dirname(osp.dirname(path)) @@ -132,14 +133,10 @@ def _load_items(self, path): ) ) elif osp.isdir(self._image_dir): - for p in os.listdir(self._image_dir): - if p.endswith(MotPath.IMAGE_EXT): - frame_id = int(osp.splitext(p)[0]) - items[frame_id] = DatasetItem( - id=frame_id, - subset=self._subset, - image=osp.join(self._image_dir, p), - ) + for p in find_images(self._image_dir): + frame_id = int(osp.splitext(osp.relpath(p, self._image_dir))[0]) + items[frame_id] = DatasetItem(id=frame_id, subset=self._subset, + image=p) with open(path, newline='', encoding='utf-8') as csv_file: # NOTE: Different MOT files have different count of fields @@ -214,9 +211,8 @@ class MotSeqGtConverter(Converter): def apply(self): extractor = self._extractor - images_dir = osp.join(self._save_dir, MotPath.IMAGE_DIR) - os.makedirs(images_dir, exist_ok=True) - self._images_dir = images_dir + image_dir = osp.join(self._save_dir, MotPath.IMAGE_DIR) + os.makedirs(image_dir, exist_ok=True) anno_dir = osp.join(self._save_dir, 'gt') os.makedirs(anno_dir, exist_ok=True) @@ -259,8 +255,8 @@ def apply(self): if self._save_images: if item.has_image and item.image.has_data: - self._save_image(item, osp.join(self._images_dir, - '%06d%s' % (frame_id, self._find_image_ext(item)))) + self._save_image(item, subdir=image_dir, + name='%06d' % frame_id) else: log.debug("Item '%s' has no image", item.id) diff --git a/datumaro/plugins/mots_format.py b/datumaro/plugins/mots_format.py index 57bcbec475..522378d905 100644 --- a/datumaro/plugins/mots_format.py +++ b/datumaro/plugins/mots_format.py @@ -5,7 +5,7 @@ # Implements MOTS format https://www.vision.rwth-aachen.de/page/mots from enum import Enum -from glob import glob +from glob import iglob import logging as log import numpy as np import os @@ -15,7 +15,7 @@ DatasetItem, AnnotationType, Mask, LabelCategories ) from datumaro.components.converter import Converter -from datumaro.util.image import load_image, save_image +from datumaro.util.image import find_images, load_image, save_image from datumaro.util.mask_tools import merge_masks @@ -40,9 +40,9 @@ def detect_dataset(path): return [{'url': path, 'format': 'mots_png'}] return [] - def __init__(self, path, subset_name=None): + def __init__(self, path, subset=None): assert osp.isdir(path), path - super().__init__(subset=subset_name) + super().__init__(subset=subset) self._images_dir = osp.join(path, 'images') self._anno_dir = osp.join(path, MotsPath.MASKS_DIR) self._categories = self._parse_categories( @@ -59,11 +59,18 @@ def _parse_categories(self, path): def _parse_items(self): items = [] - for p in sorted(p for p in - glob(self._anno_dir + '/**/*.png', recursive=True)): + + image_dir = self._images_dir + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + + for p in sorted(iglob(self._anno_dir + '/**/*.png', recursive=True)): item_id = osp.splitext(osp.relpath(p, self._anno_dir))[0] items.append(DatasetItem(id=item_id, subset=self._subset, - image=osp.join(self._images_dir, item_id + MotsPath.IMAGE_EXT), + image=images.get(item_id), annotations=self._parse_annotations(p))) return items @@ -100,7 +107,7 @@ def find_sources(cls, path): for p in os.listdir(path): detected = MotsPngExtractor.detect_dataset(osp.join(path, p)) for s in detected: - s.setdefault('options', {})['subset_name'] = p + s.setdefault('options', {})['subset'] = p subsets.extend(detected) return subsets @@ -111,7 +118,7 @@ class MotsPngConverter(Converter): def apply(self): for subset_name, subset in self._extractor.subsets().items(): subset_dir = osp.join(self._save_dir, subset_name) - images_dir = osp.join(subset_dir, MotsPath.IMAGE_DIR) + image_dir = osp.join(subset_dir, MotsPath.IMAGE_DIR) anno_dir = osp.join(subset_dir, MotsPath.MASKS_DIR) os.makedirs(anno_dir, exist_ok=True) @@ -120,8 +127,7 @@ def apply(self): if self._save_images: if item.has_image and item.image.has_data: - self._save_image(item, - osp.join(images_dir, self._make_image_filename(item))) + self._save_image(item, subdir=image_dir) else: log.debug("Item '%s' has no image", item.id) diff --git a/datumaro/plugins/ndr.py b/datumaro/plugins/ndr.py index ea27f56e86..df82e17935 100644 --- a/datumaro/plugins/ndr.py +++ b/datumaro/plugins/ndr.py @@ -1,26 +1,68 @@ # Copyright (C) 2020 Intel Corporation # # SPDX-License-Identifier: MIT + from enum import Enum +import logging as log import cv2 import numpy as np from scipy.linalg import orth from datumaro.components.extractor import Transform, DEFAULT_SUBSET_NAME +from datumaro.components.cli_plugin import CliPlugin +from datumaro.util import parse_str_enum_value + + +Algorithm = Enum("Algorithm", ["gradient"]) # other algorithms will be added -AlgoList = Enum("AlgoList", ["gradient"]) # other algorithms will be added +OverSamplingMethod = Enum("OverSamplingMethod", ["random", "similarity"]) -class NDR(Transform): +UnderSamplingMethod = Enum("UnderSamplingMethod", ["uniform", "inverse"]) + +class NDR(Transform, CliPlugin): """ Near-duplicated image removal |n Removes near-duplicated images in subset |n + Example: control number of outputs to 100 after NDR |n + |s|s%(prog)s \ |n + |s|s|s|s--working_subset train \ |n + |s|s|s|s--algorithm gradient \ |n + |s|s|s|s--num_cut 100 \ |n + |s|s|s|s--over_sample random \ |n + |s|s|s|s--under_sample uniform """ + @classmethod + def build_cmdline_parser(cls, **kwargs): + parser = super().build_cmdline_parser(**kwargs) + parser.add_argument('-w', '--working_subset', default=None, + help="Name of the subset to operate (default: %(default)s)") + parser.add_argument('-d', '--duplicated_subset', default='duplicated', + help="Name of the subset for the removed data " + "after NDR runs (default: %(default)s)") + parser.add_argument('-a', '--algorithm', default=Algorithm.gradient.name, + choices=[algo.name for algo in Algorithm], + help="Name of the algorithm to use (default: %(default)s)") + parser.add_argument('-k', '--num_cut', default=None, type=int, + help="Number of outputs you want") + parser.add_argument('-e', '--over_sample', + default=OverSamplingMethod.random.name, + choices=[method.name for method in OverSamplingMethod], + help="Specify the strategy when num_cut is bigger " + "than length of the result (default: %(default)s)") + parser.add_argument('-u', '--under_sample', + default=UnderSamplingMethod.uniform.name, + choices=[method.name for method in UnderSamplingMethod], + help="Specify the strategy when num_cut is smaller " + "than length of the result (default: %(default)s)") + parser.add_argument('-s', '--seed', type=int, help="Random seed") + return parser + def __init__(self, extractor, - working_subset=None, duplicated_subset="duplicated", - algorithm='gradient', num_cut=None, - over_sample='random', under_sample='uniform', + working_subset, duplicated_subset='duplicated', + algorithm=None, num_cut=None, + over_sample=None, under_sample=None, seed=None, **kwargs): """ Near-duplicated image removal @@ -42,11 +84,14 @@ def __init__(self, extractor, over_sample: "random" or "similarity" specify the strategy when num_cut > length of the result after removal if random, sample from removed data randomly - if similarity, select from removed data with ascending order of similarity + if similarity, select from removed data with ascending + order of similarity under_sample: "uniform" or "inverse" specify the strategy when num_cut < length of the result after removal if uniform, sample data with uniform distribution - if inverse, sample data with reciprocal of the number of data which have same hash key + if inverse, sample data with reciprocal of the number + of data which have same hash key + Algorithm Specific for gradient block_shape: tuple, (h, w) for the robustness, this function will operate on blocks @@ -71,12 +116,16 @@ def __init__(self, extractor, # parameter validation before main runs if working_subset == duplicated_subset: raise ValueError("working_subset == duplicated_subset") - if algorithm not in [algo_name.name for algo_name in AlgoList]: - raise ValueError("Invalid algorithm name") - if over_sample not in ("random", "similarity"): - raise ValueError("Invalid over_sample") - if under_sample not in ("uniform", "inverse"): - raise ValueError("Invalid under_sample") + + algorithm = parse_str_enum_value(algorithm, Algorithm, + default=Algorithm.gradient, + unknown_member_error="Unknown algorithm '{value}'.") + over_sample = parse_str_enum_value(over_sample, OverSamplingMethod, + default=OverSamplingMethod.random, + unknown_member_error="Unknown oversampling method '{value}'.") + under_sample = parse_str_enum_value(under_sample, UnderSamplingMethod, + default=UnderSamplingMethod.uniform, + unknown_member_error="Unknown undersampling method '{value}'.") if seed: self.seed = seed @@ -103,12 +152,39 @@ def _remove(self): for item in working_subset: if item.image.has_data: having_image.append(item) - all_imgs.append(item.image.data) + img = item.image.data + # Not handle empty image, as utils/image.py if check empty + if len(img.shape) == 2: + img = np.stack((img,)*3, axis=-1) + elif len(img.shape) == 3: + if img.shape[2] == 1: + img = np.stack((img[:,:,0],)*3, axis=-1) + elif img.shape[2] == 4: + img = img[...,:3] + elif img.shape[2] == 3: + pass + else: + raise ValueError("Item %s: invalid image shape: " + "unexpected number of channels (%s)" % \ + (item.id, img.shape[2])) + else: + raise ValueError("Item %s: invalid image shape: " + "unexpected number of dimensions (%s)" % \ + (item.id, len(img.shape))) + + if self.algorithm == Algorithm.gradient: + # Caculate gradient + img = self._cgrad_feature(img) + else : + raise NotImplementedError() + all_imgs.append(img) + else: + log.debug("Skipping item %s: no image data available", item.id) if self.num_cut and self.num_cut > len(all_imgs): raise ValueError("The number of images is smaller than the cut you want") - if self.algorithm == AlgoList.gradient.name: + if self.algorithm == Algorithm.gradient: all_key, fidx, kept_index, key_counter, removed_index_with_sim = \ self._gradient_based(all_imgs, **self.algorithm_specific) else: @@ -132,12 +208,8 @@ def _gradient_based(self, all_imgs, block_shape=(4, 4), if hash_dim <= 0: raise ValueError("hash_dim should be positive") - # Caculate gradient - all_clr = np.array( - [self._cgrad_feature(img, out_wh=block_shape) for img in all_imgs]) - # Compute hash keys from all the features - all_clr = np.reshape(all_clr, (len(all_imgs), -1)) + all_clr = np.reshape(np.array(all_imgs), (len(all_imgs), -1)) all_key = self._project(all_clr, hash_dim) # Remove duplication using hash @@ -176,11 +248,11 @@ def _keep_cut(self, num_cut, all_key, fidx, kept_index, key_counter, removed_index_with_similarity, over_sample, under_sample): if num_cut and num_cut > len(kept_index): - if over_sample == "random": + if over_sample == OverSamplingMethod.random: selected_index = np.random.choice( list(set(fidx) - set(kept_index)), size=num_cut - len(kept_index), replace=False) - elif over_sample == "similarity": + elif over_sample == OverSamplingMethod.similarity: removed_index_with_similarity = [[key, value] \ for key, value in removed_index_with_similarity.items()] removed_index_with_similarity.sort(key=lambda x: x[1]) @@ -188,13 +260,15 @@ def _keep_cut(self, num_cut, all_key, fidx, for index, _ in removed_index_with_similarity[:num_cut - len(kept_index)]] kept_index.extend(selected_index) elif num_cut and num_cut < len(kept_index): - if under_sample == "uniform": + if under_sample == UnderSamplingMethod.uniform: prob = None - elif under_sample == "inverse": - # if inverse - probability with inverse of the collision(number of same hash key) + elif under_sample == UnderSamplingMethod.inverse: + # if inverse - probability with inverse + # of the collision(number of same hash key) # [x1, x2, y1, y2, y3, y4, z1, z2, z3]. x, y and z for hash key # i.e. there are 4 elements which have hash key y. - # then the occurence will be [2, 4, 3] and reverse of them will be [1/2, 1/4, 1/3] + # then the occurence will be [2, 4, 3] and reverse of them + # will be [1/2, 1/4, 1/3] # Normalizing them by dividing with sum, we get [6/13, 3/13, 4/13] # Then the key x will be sampled with probability 6/13 # and each point, x1 and x2, will share same prob. 3/13 diff --git a/datumaro/plugins/openvino/README.md b/datumaro/plugins/openvino/README.md new file mode 100644 index 0000000000..a8f37d3ef9 --- /dev/null +++ b/datumaro/plugins/openvino/README.md @@ -0,0 +1,90 @@ +# OpenVINO™ Inference Interpreter +Interpreter samples to parse OpenVINO™ inference outputs. + +## Models supported from interpreter samples +There are detection and image classification examples. + +- Detection (SSD-based) + - Intel Pre-trained Models > Object Detection + - [face-detection-0200](https://docs.openvinotoolkit.org/latest/omz_models_intel_face_detection_0200_description_face_detection_0200.html) + - [face-detection-0202](https://docs.openvinotoolkit.org/latest/omz_models_intel_face_detection_0202_description_face_detection_0202.html) + - [face-detection-0204](https://docs.openvinotoolkit.org/latest/omz_models_intel_face_detection_0204_description_face_detection_0204.html) + - [person-detection-0200](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_detection_0200_description_person_detection_0200.html) + - [person-detection-0201](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_detection_0201_description_person_detection_0201.html) + - [person-detection-0202](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_detection_0202_description_person_detection_0202.html) + - [person-vehicle-bike-detection-2000](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_vehicle_bike_detection_2000_description_person_vehicle_bike_detection_2000.html) + - [person-vehicle-bike-detection-2001](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_vehicle_bike_detection_2001_description_person_vehicle_bike_detection_2001.html) + - [person-vehicle-bike-detection-2002](https://docs.openvinotoolkit.org/latest/omz_models_intel_person_vehicle_bike_detection_2002_description_person_vehicle_bike_detection_2002.html) + - [vehicle-detection-0200](https://docs.openvinotoolkit.org/latest/omz_models_intel_vehicle_detection_0200_description_vehicle_detection_0200.html) + - [vehicle-detection-0201](https://docs.openvinotoolkit.org/latest/omz_models_intel_vehicle_detection_0201_description_vehicle_detection_0201.html) + - [vehicle-detection-0202](https://docs.openvinotoolkit.org/latest/omz_models_intel_vehicle_detection_0202_description_vehicle_detection_0202.html) + + - Public Pre-Trained Models(OMZ) > Object Detection + - [ssd_mobilenet_v1_coco](https://docs.openvinotoolkit.org/latest/omz_models_public_ssd_mobilenet_v1_coco_ssd_mobilenet_v1_coco.html) + - [ssd_mobilenet_v2_coco](https://docs.openvinotoolkit.org/latest/omz_models_public_ssd_mobilenet_v2_coco_ssd_mobilenet_v2_coco.html) + +- Image Classification + - Public Pre-Trained Models(OMZ) > Classification + - [mobilenet-v2-pytorch](https://docs.openvinotoolkit.org/latest/omz_models_public_mobilenet_v2_pytorch_mobilenet_v2_pytorch.html) + +You can find more OpenVINO™ Trained Models [here](https://docs.openvinotoolkit.org/latest/omz_models_intel_index.html) +To run the inference with OpenVINO™, the model format should be Intermediate Representation(IR). +For the Caffe/TensorFlow/MXNet/Kaldi/ONNX models, please see the [Model Conversion Instruction](https://docs.openvinotoolkit.org/latest/openvino_docs_MO_DG_prepare_model_convert_model_Converting_Model.html) + +You need to implement your own interpreter samples to support the other OpenVINO™ Trained Models. + +## Model download +- Prerequisites + - OpenVINO™ (To install OpenVINO™, please see the [OpenVINO™ Installation Instruction](https://docs.openvinotoolkit.org/latest/openvino_docs_install_guides_installing_openvino_linux.html)) + - OpenVINO™ models (To download OpenVINO™ models, please see the [Model Downloader Instruction](https://docs.openvinotoolkit.org/latest/omz_tools_downloader_README.html)) + - PASCAL VOC 2012 dataset (To download VOC 2012 dataset, please go [VOC2012 download](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#devkit)) + + ```bash + # cd /deployment_tools/open_model_zoo/tools/downloader + # ./downloader.py --name + # + # Examples + cd /opt/intel/openvino/deployment_tools/open_model_zoo/tools/downloader + ./downloader.py --name face-detection-0200 + ``` + +## Model inference +- Prerequisites: + - OpenVINO™ (To install OpenVINO™, please see the [OpenVINO™ Installation Instruction](https://docs.openvinotoolkit.org/latest/openvino_docs_install_guides_installing_openvino_linux.html)) + - Datumaro (To install Datumaro, please see the [User Manual](docs/user_manual.md)) + - OpenVINO™ models (To download OpenVINO™ models, please see the [Model Downloader Instruction](https://docs.openvinotoolkit.org/latest/omz_tools_downloader_README.html)) + - PASCAL VOC 2012 dataset (To download VOC 2012 dataset, please go [VOC2012 download](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/#devkit)) + +- To run the inference with OpenVINO™ models and the interpreter samples, please follow the instructions below. + + ```bash + # source /bin/setupvars.sh + # datum create -o + # datum model add -l -p --copy -- -d -w -i + # datum add path -p -f + # datum model run -p -m model-0 + # + # Examples + # Detection> ssd_mobilenet_v2_coco + source /opt/intel/openvino/bin/setupvars.sh + cd datumaro/plugins/openvino + datum create -o proj_ssd_mobilenet_v2_coco_detection + datum model add -l openvino -p proj_ssd_mobilenet_v2_coco_detection --copy -- \ + --output-layers=do_ExpandDims_conf/sigmoid \ + -d model/ssd_mobilenet_v2_coco.xml \ + -w model/ssd_mobilenet_v2_coco.bin \ + -i samples/ssd_mobilenet_coco_detection_interp.py + datum add path -p proj_ssd_mobilenet_v2_coco_detection -f voc VOCdevkit/ + datum model run -p proj_ssd_mobilenet_v2_coco_detection -m model-0 + + # Classification> mobilenet-v2-pytorch + source /opt/intel/openvino/bin/setupvars.sh + cd datumaro/plugins/openvino + datum create -o proj_mobilenet_v2_classification + datum model add -l openvino -p proj_mobilenet_v2_classification --copy -- \ + -d model/mobilenet-v2-pytorch.xml \ + -w model/mobilenet-v2-pytorch.bin \ + -i samples/mobilenet_v2_pytorch_interp.py + datum add path -p proj_mobilenet_v2_classification -f voc VOCdevkit/ + datum model run -p proj_mobilenet_v2_classification -m model-0 + ``` \ No newline at end of file diff --git a/datumaro/plugins/openvino/__init__.py b/datumaro/plugins/openvino/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/datumaro/plugins/openvino_launcher.py b/datumaro/plugins/openvino/launcher.py similarity index 100% rename from datumaro/plugins/openvino_launcher.py rename to datumaro/plugins/openvino/launcher.py diff --git a/datumaro/plugins/openvino/samples/coco.class b/datumaro/plugins/openvino/samples/coco.class new file mode 100644 index 0000000000..9682e2b88c --- /dev/null +++ b/datumaro/plugins/openvino/samples/coco.class @@ -0,0 +1,91 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +trafficlight +firehydrant +streetsign +stopsign +parkingmeter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +hat +backpack +umbrella +shoe +eyeglasses +handbag +tie +suitcase +frisbee +skis +snowboard +sportsball +kite +baseballbat +baseballglove +skateboard +surfboard +tennisracket +bottle +plate +wineglass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hotdog +pizza +donut +cake +chair +couch +pottedplant +bed +mirror +diningtable +window +desk +toilet +door +tv +laptop +mouse +remote +keyboard +cellphone +microwave +oven +toaster +sink +refrigerator +blender +book +clock +vase +scissors +teddybear +hairdrier +toothbrush +hairbrush \ No newline at end of file diff --git a/datumaro/plugins/openvino/samples/imagenet.class b/datumaro/plugins/openvino/samples/imagenet.class new file mode 100644 index 0000000000..2571d1b729 --- /dev/null +++ b/datumaro/plugins/openvino/samples/imagenet.class @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane2 +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot2 +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue \ No newline at end of file diff --git a/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py b/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py new file mode 100644 index 0000000000..43ce43fd21 --- /dev/null +++ b/datumaro/plugins/openvino/samples/mobilenet_v2_pytorch_interp.py @@ -0,0 +1,38 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import Label, LabelCategories, AnnotationType +from datumaro.util.annotation_util import softmax + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, output in zip(inputs, outputs): + image_results = [] + output = softmax(output).tolist() + label = output.index(max(output)) + image_results.append(Label(label=label, attributes={"scores": output})) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + + with open("samples/imagenet.class", "r") as file: + for line in file.readlines(): + label = line.strip() + label_categories.add(label) + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py new file mode 100644 index 0000000000..abb4604f8d --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_face_detection_interp.py @@ -0,0 +1,83 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("face") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py new file mode 100644 index 0000000000..3b3e5de252 --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_mobilenet_coco_detection_interp.py @@ -0,0 +1,90 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.3 +model_class_num = 91 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, confs, detections in zip( + inputs, outputs["do_ExpandDims_conf/sigmoid"], outputs["DetectionOutput"] + ): + + input_height, input_width = input.shape[:2] + + confs = confs[0].reshape(-1, model_class_num) + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + + with open("samples/coco.class", "r") as file: + for line in file.readlines(): + label = line.strip() + label_categories.add(label) + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py new file mode 100644 index 0000000000..3888944df4 --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_person_detection_interp.py @@ -0,0 +1,83 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("person") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py new file mode 100644 index 0000000000..a2de43dd6d --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_person_vehicle_bike_detection_interp.py @@ -0,0 +1,85 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("vehicle") + label_categories.add("person") + label_categories.add("bike") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py b/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py new file mode 100644 index 0000000000..2866133542 --- /dev/null +++ b/datumaro/plugins/openvino/samples/ssd_vehicle_detection_interp.py @@ -0,0 +1,83 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + + +from datumaro.components.extractor import * + +conf_thresh = 0.02 + + +def _match_confs(confs, detections): + matches = [-1] * len(detections) + + queries = {} + for i, det in enumerate(detections): + queries.setdefault(int(det[1]), []).append((det[2], i)) + + found_count = 0 + for i, v in enumerate(confs): + if found_count == len(detections): + break + + for cls_id, query in queries.items(): + if found_count == len(detections): + break + + for q_id, (conf, det_idx) in enumerate(query): + if v[cls_id] == conf: + matches[det_idx] = i + query.pop(q_id) + found_count += 1 + break + + return matches + + +def process_outputs(inputs, outputs): + # inputs = model input; array or images; shape = (B, H, W, C) + # outputs = model output; shape = (1, 1, N, 7); N is the number of detected bounding boxes. + # det = [image_id, label(class id), conf, x_min, y_min, x_max, y_max] + # results = conversion result; [[ Annotation, ... ], ... ] + + results = [] + for input, detections in zip(inputs, outputs["detection_out"]): + + input_height, input_width = input.shape[:2] + + confs = outputs["Softmax_189/Softmax_"] + detections = detections[0] + + conf_ids = _match_confs(confs, detections) + + image_results = [] + for i, det in enumerate(detections): + image_id = int(det[0]) + label = int(det[1]) + conf = float(det[2]) + det_confs = confs[conf_ids[i]] + + if conf <= conf_thresh: + continue + + x = max(int(det[3] * input_width), 0) + y = max(int(det[4] * input_height), 0) + w = min(int(det[5] * input_width - x), input_width) + h = min(int(det[6] * input_height - y), input_height) + + image_results.append(Bbox(x, y, w, h, label=label, + attributes={ 'score': conf, 'scores': list(map(float, det_confs)) } + )) + + results.append(image_results) + + return results + + +def get_categories(): + # output categories - label map etc. + + label_categories = LabelCategories() + label_categories.add("vehicle") + + return {AnnotationType.label: label_categories} diff --git a/datumaro/plugins/sampler/__init__.py b/datumaro/plugins/sampler/__init__.py new file mode 100644 index 0000000000..0aa5e58c75 --- /dev/null +++ b/datumaro/plugins/sampler/__init__.py @@ -0,0 +1,3 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT diff --git a/datumaro/plugins/sampler/algorithm/algorithm.py b/datumaro/plugins/sampler/algorithm/algorithm.py new file mode 100644 index 0000000000..5dd562f062 --- /dev/null +++ b/datumaro/plugins/sampler/algorithm/algorithm.py @@ -0,0 +1,24 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from enum import Enum + + +SamplingMethod = Enum("SamplingMethod", + ["topk", "lowk", "randk", "mixk", "randtopk"]) + +Algorithm = Enum("Algorithm", ["entropy"]) + +class InferenceResultAnalyzer: + """ + Basic interface for IRA (Inference Result Analyzer) + """ + + def __init__(self, dataset, inference): + self.data = dataset + self.inference = inference + self.sampling_method = SamplingMethod + + def get_sample(self, method: str, k: int): + raise NotImplementedError() diff --git a/datumaro/plugins/sampler/algorithm/entropy.py b/datumaro/plugins/sampler/algorithm/entropy.py new file mode 100644 index 0000000000..d26a9081cb --- /dev/null +++ b/datumaro/plugins/sampler/algorithm/entropy.py @@ -0,0 +1,191 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +import logging as log +import math +import re + +import pandas as pd + +from .algorithm import InferenceResultAnalyzer + + +class SampleEntropy(InferenceResultAnalyzer): + """ + Entropy is a class that inherits an Sampler, + calculates an uncertainty score based on an entropy, + and get samples based on that score. + """ + + def __init__(self, data, inference): + """ + Constructor function + Args: + data: Receive the data format in pd.DataFrame format. + ImageID is an essential element for data. + inference: + Receive the inference format in the form of pd.DataFrame. + ImageID and ClassProbability are essential for inferences. + """ + super().__init__(data, inference) + + # check the existence of "ImageID" in data & inference + if 'ImageID' not in data: + raise Exception("Invalid Data, ImageID not found in data") + if 'ImageID' not in inference: + raise Exception("Invalid Data, ImageID not found in inference") + + # check the existence of "ClassProbability" in inference + self.num_classes = 0 + for head in list(inference): + if re.match(r"ClassProbability\d+", head): + self.num_classes += 1 + + if self.num_classes == 0: + raise Exception( + "Invalid data, Inference do not have ClassProbability values") + + # rank: The inference DataFrame, sorted according to the score. + self.rank = self._rank_images().sort_values(by='rank') + + def get_sample(self, method: str, k: int, n: int = 3) -> pd.DataFrame: + """ + A function that extracts sample data and returns it. + Args: + method: + - 'topk': It extracts the k sample data with the + highest uncertainty. + - 'lowk': It extracts the k sample data with the + lowest uncertainty. + - 'randomk': Extract and return random k sample data. + k: number of sample data + n: Parameters to be used in the randtopk method, Variable to first + extract data of multiple n of k. + Returns: + Extracted sample data : pd.DataFrame + """ + temp_rank = self.rank + + # 1. k value check + if not isinstance(k, int) or k <= 0: + raise ValueError( + f"Invalid value {k}. k must have an integer greater than zero." + ) + + # 2. Select a sample according to the method + if k <= len(temp_rank): + if method == self.sampling_method.topk.name: + temp_rank = temp_rank[:k] + elif method == self.sampling_method.lowk.name: + temp_rank = temp_rank[-k:] + elif method == self.sampling_method.randk.name: + return self.data.sample(n=k).reset_index(drop=True) + elif method in {self.sampling_method.mixk.name, + self.sampling_method.randtopk.name}: + return self._get_sample_mixed(method=method, k=k, n=n) + else: + raise ValueError(f"Unknown sampling method '{method}'") + else: + log.warning( + "The number of samples is greater than the size of the" + "selected subset." + ) + + columns = list(self.data.columns) + merged_df = pd.merge(temp_rank, self.data, how='inner', on=['ImageID']) + return merged_df[columns].reset_index(drop=True) + + def _get_sample_mixed(self, method: str, k: int, n: int = 3) -> pd.DataFrame: + """ + A function that extracts sample data and returns it. + Args: + method: + - 'mixk': Return top-k and low-k halves based on uncertainty. + - 'randomtopk': Randomly extract n*k and return k + with high uncertainty. + k: number of sample data + n: Number to extract n * k from total data according to n, + and top-k from it + Returns: + Extracted sample data : pd.DataFrame + """ + temp_rank = self.rank + + # Select a sample according to the method + if k <= len(temp_rank): + if method == self.sampling_method.mixk.name: + if k % 2 == 0: + temp_rank = pd.concat([temp_rank[: k // 2], temp_rank[-(k // 2) :]]) + else: + temp_rank = pd.concat( + [temp_rank[: (k // 2) + 1], temp_rank[-(k // 2) :]] + ) + elif method == self.sampling_method.randtopk.name: + if n * k <= len(temp_rank): + temp_rank = temp_rank.sample(n=n * k).sort_values(by='rank') + else: + log.warning(msg="n * k exceeds the length of the inference") + temp_rank = temp_rank[:k] + + columns = list(self.data.columns) + merged_df = pd.merge(temp_rank, self.data, how='inner', on=['ImageID']) + return merged_df[columns].reset_index(drop=True) + + def _rank_images(self) -> pd.DataFrame: + """ + A internal function that ranks the inference data based on uncertainty. + Returns: + inference data sorted by uncertainty. pd.DataFrame + """ + # 1. Load Inference + inference, res = None, None + if self.inference is not None: + inference = pd.DataFrame(self.inference) + else: + raise Exception("Invalid Data, Failed to load inference result") + + # 2. If the reference data frame does not contain an uncertify score, calculate it + if 'Uncertainty' not in inference: + inference = self._calculate_uncertainty_from_classprob(inference) + + # 3. Check that Uncertainty values are in place. + na_df = inference.isna().sum() + if 'Uncertainty' in na_df and na_df['Uncertainty'] > 0: + raise Exception("Some inference results do not have Uncertainty values") + + # 4. Ranked based on Uncertainty score + res = inference[['ImageID', 'Uncertainty']].groupby('ImageID').mean() + res['rank'] = res['Uncertainty'].rank(ascending=False, method='first') + res = res.reset_index() + + return res + + def _calculate_uncertainty_from_classprob( + self, inference: pd.DataFrame) -> pd.DataFrame: + """ + A function that calculates uncertainty based on entropy through + ClassProbability values. + Args: + inference: Inference data where uncertainty has not been calculated + Returns: + inference data with uncertainty variable + """ + + # Calculate Entropy (Uncertainty Score) + uncertainty = [] + for i in range(len(inference)): + entropy = 0 + for j in range(self.num_classes): + p = inference.loc[i][f'ClassProbability{j+1}'] + if p < 0 or p > 1: + raise Exception( + "Invalid data, Math domain Error! p is between 0 and 1" + ) + entropy -= p * math.log(p + 1e-14, math.e) + + uncertainty.append(entropy) + + inference['Uncertainty'] = uncertainty + + return inference diff --git a/datumaro/plugins/sampler/sampler.py b/datumaro/plugins/sampler/sampler.py new file mode 100644 index 0000000000..2fefa0e1dc --- /dev/null +++ b/datumaro/plugins/sampler/sampler.py @@ -0,0 +1,184 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import defaultdict + +import pandas as pd + +from datumaro.components.extractor import Transform +from datumaro.components.cli_plugin import CliPlugin + +from .algorithm.algorithm import SamplingMethod, Algorithm + + +class Sampler(Transform, CliPlugin): + """ + Sampler that analyzes model inference results on the dataset |n + and picks the best sample for training.|n + |n + Notes:|n + - Each image's inference result must contain the probability for + all classes.|n + - Requesting a sample larger than the number of all images will + return all images.|n + |n + Example: select the most relevant data subset of 20 images |n + |s|sbased on model certainty, put the result into 'sample' subset + |s|sand put all the rest into 'unsampled' subset, use 'train' subset |n + |s|sas input. |n + |s|s%(prog)s \ |n + |s|s|s|s--algorithm entropy \ |n + |s|s|s|s--subset_name train \ |n + |s|s|s|s--sample_name sample \ |n + |s|s|s|s--unsampled_name unsampled \ |n + |s|s|s|s--sampling_method topk -k 20 + """ + + @classmethod + def build_cmdline_parser(cls, **kwargs): + parser = super().build_cmdline_parser(**kwargs) + parser.add_argument("-k", "--count", type=int, required=True, + help="Number of items to sample") + parser.add_argument("-a", "--algorithm", + default=Algorithm.entropy.name, + choices=[t.name for t in Algorithm], + help="Sampling algorithm (one of {}; default: %(default)s)".format( + ', '.join(t.name for t in Algorithm))) + parser.add_argument("-i", "--input_subset", default=None, + help="Subset name to select sample from (default: %(default)s)") + parser.add_argument("-o", "--sampled_subset", default="sample", + help="Subset name to put sampled data to (default: %(default)s)") + parser.add_argument("-u", "--unsampled_subset", default="unsampled", + help="Subset name to put the rest data to (default: %(default)s)") + parser.add_argument("-m", "--sampling_method", + default=SamplingMethod.topk.name, + choices=[t.name for t in SamplingMethod], + help="Sampling method (one of {}; default: %(default)s)".format( + ', '.join(t.name for t in SamplingMethod))) + parser.add_argument("-d", "--output_file", + help="A .csv file path to dump sampling results file path") + return parser + + def __init__(self, extractor, algorithm, input_subset, sampled_subset, + unsampled_subset, sampling_method, count, output_file): + """ + Parameters + ---------- + extractor : Extractor, Dataset + algorithm : str + Specifying the algorithm to calculate the uncertainty + for sample selection. default: 'entropy' + subset_name : str + The name of the subset to which you want to select a sample. + sample_name : str + Subset name of the selected sample, default: 'sample' + sampling_method : str + Method of sampling, 'topk' or 'lowk' or 'randk' + num_sample : int + Number of samples extracted + output_file : str + Path of sampler result, Use when user want to save results + """ + super().__init__(extractor) + + # Get Parameters + self.input_subset = input_subset + self.sampled_subset = sampled_subset + self.unsampled_subset = unsampled_subset + self.algorithm = algorithm + self.sampling_method = sampling_method + self.count = count + self.output_file = output_file + + # Use the --output_file option to save the sample list as a csv file + if output_file and not output_file.endswith(".csv"): + raise ValueError("The output file must have the '.csv' extension") + + @staticmethod + def _load_inference_from_subset(extractor, subset_name): + # 1. Get Dataset from subset name + if subset_name in extractor.subsets(): + subset = extractor.get_subset(subset_name) + else: + raise Exception(f"Unknown subset '{subset_name}'") + + data_df = defaultdict(list) + infer_df = defaultdict(list) + + # 2. Fill the data_df and infer_df to fit the sampler algorithm + # input format. + for item in subset: + data_df['ImageID'].append(item.id) + + if not item.has_image or item.image.size is None: + raise Exception(f"Item {item.id} does not have image info") + + width, height = item.image.size + data_df['Width'].append(width) + data_df['Height'].append(height) + data_df['ImagePath'].append(item.image.path) + + if not item.annotations: + raise Exception(f"Item {item.id} does not have annotations") + + for annotation in item.annotations: + if 'scores' not in annotation.attributes: + raise Exception(f"Item {item.id} - an annotation " + "does not have 'scores' attribute") + probs = annotation.attributes['scores'] + + infer_df['ImageID'].append(item.id) + + for prob_idx, prob in enumerate(probs): + infer_df[f"ClassProbability{prob_idx+1}"].append(prob) + + data_df = pd.DataFrame(data_df) + infer_df = pd.DataFrame(infer_df) + + return data_df, infer_df + + @staticmethod + def _calculate_uncertainty(algorithm, data, inference): + # Checking and creating algorithms + if algorithm == Algorithm.entropy.name: + from .algorithm.entropy import SampleEntropy + + # Data delivery, uncertainty score calculations also proceed. + sampler = SampleEntropy(data, inference) + else: + raise Exception(f"Unknown algorithm '{algorithm}', available " + f"algorithms: {[a.name for a in Algorithm]}") + return sampler + + def _get_sample_subset(self, image): + if image.subset == self.input_subset: + # 1. Returns the sample subset if the id belongs to samples. + if image.id in self.sample_id: + return self.sampled_subset + else: + return self.unsampled_subset + else: + # 2. Returns the existing subset name if it is not a sample + return image.subset + + def __iter__(self): + # Import data into a subset name and convert it + # to a format that will be used in the sampler algorithm with the inference result. + data_df, infer_df = self._load_inference_from_subset( + self._extractor, self.input_subset) + + sampler = self._calculate_uncertainty(self.algorithm, data_df, infer_df) + self.result = sampler.get_sample(method=self.sampling_method, + k=self.count) + + if self.output_file is not None: + self.result.to_csv(self.output_file, index=False) + + self.sample_id = self.result['ImageID'].to_list() + + # Transform properties for each data + for item in self._extractor: + # After checking whether each item belongs to a sample, + # rename the subset + yield self.wrap_item(item, subset=self._get_sample_subset(item)) diff --git a/datumaro/plugins/tf_detection_api_format/extractor.py b/datumaro/plugins/tf_detection_api_format/extractor.py index 75d560453b..9001a61cbf 100644 --- a/datumaro/plugins/tf_detection_api_format/extractor.py +++ b/datumaro/plugins/tf_detection_api_format/extractor.py @@ -22,7 +22,7 @@ def clamp(value, _min, _max): return max(min(_max, value), _min) class TfDetectionApiExtractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): assert osp.isfile(path), path images_dir = '' root_dir = osp.dirname(osp.abspath(path)) @@ -32,7 +32,9 @@ def __init__(self, path): if not osp.isdir(images_dir): images_dir = '' - super().__init__(subset=osp.splitext(osp.basename(path))[0]) + if not subset: + subset = osp.splitext(osp.basename(path))[0] + super().__init__(subset=subset) items, labels = self._parse_tfrecord_file(path, self._subset, images_dir) self._items = items diff --git a/datumaro/plugins/vgg_face2_format.py b/datumaro/plugins/vgg_face2_format.py index c38478193b..33b4102160 100644 --- a/datumaro/plugins/vgg_face2_format.py +++ b/datumaro/plugins/vgg_face2_format.py @@ -9,6 +9,7 @@ from datumaro.components.converter import Converter from datumaro.components.extractor import (AnnotationType, Bbox, DatasetItem, Importer, Label, LabelCategories, Points, SourceExtractor) +from datumaro.util.image import find_images class VggFace2Path: @@ -20,15 +21,16 @@ class VggFace2Path: IMAGES_DIR_NO_LABEL = 'no_label' class VggFace2Extractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isfile(path): raise Exception("Can't read .csv annotation file '%s'" % path) self._path = path self._dataset_dir = osp.dirname(osp.dirname(path)) - subset = osp.splitext(osp.basename(path))[0] - if subset.startswith(VggFace2Path.LANDMARKS_FILE): - subset = subset.split('_')[2] + if not subset: + subset = osp.splitext(osp.basename(path))[0] + if subset.startswith(VggFace2Path.LANDMARKS_FILE): + subset = subset.split('_')[2] super().__init__(subset=subset) self._categories = self._load_categories() @@ -68,7 +70,14 @@ def _split_item_path(path): items = {} - with open(path) as content: + image_dir = osp.join(self._dataset_dir, self._subset) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + + with open(path, encoding='utf-8') as content: landmarks_table = list(csv.DictReader(content)) for row in landmarks_table: item_id = row['NAME_ID'] @@ -77,10 +86,8 @@ def _split_item_path(path): item_id, label = _split_item_path(item_id) if item_id not in items: - image_path = osp.join(self._dataset_dir, self._subset, - row['NAME_ID'] + VggFace2Path.IMAGE_EXT) items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path) + image=images.get(row['NAME_ID'])) annotations = items[item_id].annotations if [a for a in annotations if a.type == AnnotationType.points]: @@ -96,7 +103,7 @@ def _split_item_path(path): bboxes_path = osp.join(self._dataset_dir, VggFace2Path.ANNOTATION_DIR, VggFace2Path.BBOXES_FILE + self._subset + '.csv') if osp.isfile(bboxes_path): - with open(bboxes_path) as content: + with open(bboxes_path, encoding='utf-8') as content: bboxes_table = list(csv.DictReader(content)) for row in bboxes_table: item_id = row['NAME_ID'] @@ -105,10 +112,8 @@ def _split_item_path(path): item_id, label = _split_item_path(item_id) if item_id not in items: - image_path = osp.join(self._dataset_dir, self._subset, - row['NAME_ID'] + VggFace2Path.IMAGE_EXT) items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path) + image=images.get(row['NAME_ID'])) annotations = items[item_id].annotations if [a for a in annotations if a.type == AnnotationType.bbox]: @@ -129,7 +134,7 @@ def find_sources(cls, path): not osp.basename(p).startswith(VggFace2Path.BBOXES_FILE)) class VggFace2Converter(Converter): - DEFAULT_IMAGE_EXT = '.jpg' + DEFAULT_IMAGE_EXT = VggFace2Path.IMAGE_EXT def apply(self): save_dir = self._save_dir @@ -148,7 +153,6 @@ def apply(self): label_categories = self._extractor.categories()[AnnotationType.label] for subset_name, subset in self._extractor.subsets().items(): - subset_dir = osp.join(save_dir, subset_name) bboxes_table = [] landmarks_table = [] for item in subset: @@ -157,13 +161,11 @@ def apply(self): if getattr(p, 'label') != None) if labels: for label in labels: - self._save_image(item, osp.join(subset_dir, - label_categories[label].name + '/' \ - + item.id + VggFace2Path.IMAGE_EXT)) + self._save_image(item, subdir=osp.join(subset_name, + label_categories[label].name)) else: - self._save_image(item, osp.join(subset_dir, - VggFace2Path.IMAGES_DIR_NO_LABEL, - item.id + VggFace2Path.IMAGE_EXT)) + self._save_image(item, subdir=osp.join(subset_name, + VggFace2Path.IMAGES_DIR_NO_LABEL)) landmarks = [a for a in item.annotations if a.type == AnnotationType.points] @@ -224,7 +226,7 @@ def apply(self): landmarks_path = osp.join(save_dir, VggFace2Path.ANNOTATION_DIR, VggFace2Path.LANDMARKS_FILE + subset_name + '.csv') os.makedirs(osp.dirname(landmarks_path), exist_ok=True) - with open(landmarks_path, 'w', newline='') as file: + with open(landmarks_path, 'w', encoding='utf-8', newline='') as file: columns = ['NAME_ID', 'P1X', 'P1Y', 'P2X', 'P2Y', 'P3X', 'P3Y', 'P4X', 'P4Y', 'P5X', 'P5Y'] writer = csv.DictWriter(file, fieldnames=columns) @@ -235,7 +237,7 @@ def apply(self): bboxes_path = osp.join(save_dir, VggFace2Path.ANNOTATION_DIR, VggFace2Path.BBOXES_FILE + subset_name + '.csv') os.makedirs(osp.dirname(bboxes_path), exist_ok=True) - with open(bboxes_path, 'w', newline='') as file: + with open(bboxes_path, 'w', encoding='utf-8', newline='') as file: columns = ['NAME_ID', 'X', 'Y', 'W', 'H'] writer = csv.DictWriter(file, fieldnames=columns) writer.writeheader() diff --git a/datumaro/plugins/voc_format/converter.py b/datumaro/plugins/voc_format/converter.py index a022d04265..abb109ff9a 100644 --- a/datumaro/plugins/voc_format/converter.py +++ b/datumaro/plugins/voc_format/converter.py @@ -17,6 +17,7 @@ from datumaro.components.extractor import (AnnotationType, CompiledMask, DatasetItem, LabelCategories) from datumaro.util import find, str_to_bool +from datumaro.util.annotation_util import make_label_id_mapping from datumaro.util.image import save_image from datumaro.util.mask_tools import paint_mask, remap_mask @@ -296,7 +297,7 @@ def save_subsets(self): VocTask.action_classification}: ann_path = osp.join(self._ann_dir, item.id + '.xml') os.makedirs(osp.dirname(ann_path), exist_ok=True) - with open(ann_path, 'w') as f: + with open(ann_path, 'w', encoding='utf-8') as f: f.write(ET.tostring(root_elem, encoding='unicode', pretty_print=True)) @@ -314,7 +315,7 @@ def save_subsets(self): clsdet_list[item.id] = True - if masks: + if masks and VocTask.segmentation in self._tasks: compiled_mask = CompiledMask.from_instance_masks(masks, instance_labels=[self._label_id_mapping(m.label) for m in masks]) @@ -350,7 +351,7 @@ def save_subsets(self): @staticmethod def _get_filtered_lines(path, patch, subset, items=None): lines = {} - with open(path) as f: + with open(path, encoding='utf-8') as f: for line in f: item, text, _ = line.split(maxsplit=1) + ['', ''] if not patch or patch.updated_items.get((item, subset)) != \ @@ -367,7 +368,7 @@ def save_action_lists(self, subset_name, action_list): items = {k: True for k in action_list} if self._patch and osp.isfile(ann_file): self._get_filtered_lines(ann_file, self._patch, subset_name, items) - with open(ann_file, 'w') as f: + with open(ann_file, 'w', encoding='utf-8') as f: for item in items: f.write('%s\n' % item) @@ -392,7 +393,7 @@ def _write_item(f, item, objs, action): if self._patch and osp.isfile(ann_file): lines = self._get_filtered_lines(ann_file, None, subset_name) - with open(ann_file, 'w') as f: + with open(ann_file, 'w', encoding='utf-8') as f: for item in items: if item in action_list: _write_item(f, item, action_list[item], action) @@ -418,7 +419,7 @@ def _write_item(f, item, item_labels): lines = self._get_filtered_lines(ann_file, self._patch, subset_name, items) - with open(ann_file, 'w') as f: + with open(ann_file, 'w', encoding='utf-8') as f: for item in items: if item in class_lists: _write_item(f, item, class_lists[item]) @@ -433,7 +434,7 @@ def save_clsdet_lists(self, subset_name, clsdet_list): if self._patch and osp.isfile(ann_file): self._get_filtered_lines(ann_file, self._patch, subset_name, items) - with open(ann_file, 'w') as f: + with open(ann_file, 'w', encoding='utf-8') as f: for item in items: f.write('%s\n' % item) @@ -445,12 +446,14 @@ def save_segm_lists(self, subset_name, segm_list): if self._patch and osp.isfile(ann_file): self._get_filtered_lines(ann_file, self._patch, subset_name, items) - with open(ann_file, 'w') as f: + with open(ann_file, 'w', encoding='utf-8') as f: for item in items: f.write('%s\n' % item) def save_layout_lists(self, subset_name, layout_list): def _write_item(f, item, item_layouts): + if 1 < len(item.split()): + item = '\"' + item + '\"' if item_layouts: for obj_id in item_layouts: f.write('%s % d\n' % (item, 1 + obj_id)) @@ -465,7 +468,7 @@ def _write_item(f, item, item_layouts): if self._patch and osp.isfile(ann_file): self._get_filtered_lines(ann_file, self._patch, subset_name, items) - with open(ann_file, 'w') as f: + with open(ann_file, 'w', encoding='utf-8') as f: for item in items: if item in layout_list: _write_item(f, item, layout_list[item]) @@ -562,22 +565,12 @@ def _get_actions(self, label): return label_desc[2] def _make_label_id_map(self): - source_labels = { - id: label.name for id, label in - enumerate(self._extractor.categories().get( - AnnotationType.label, LabelCategories()).items) - } - target_labels = { - label.name: id for id, label in - enumerate(self._categories[AnnotationType.label].items) - } - id_mapping = { - src_id: target_labels.get(src_label, 0) - for src_id, src_label in source_labels.items() - } + map_id, id_mapping, src_labels, dst_labels = make_label_id_mapping( + self._extractor.categories().get(AnnotationType.label), + self._categories[AnnotationType.label]) - void_labels = [src_label for src_id, src_label in source_labels.items() - if src_label not in target_labels] + void_labels = [src_label for src_id, src_label in src_labels.items() + if src_label not in dst_labels] if void_labels: log.warning("The following labels are remapped to background: %s" % ', '.join(void_labels)) @@ -588,12 +581,10 @@ def _make_label_id_map(self): self._categories[AnnotationType.label] \ .items[id_mapping[src_id]].name ) - for src_id, src_label in source_labels.items() + for src_id, src_label in src_labels.items() ]) ) - def map_id(src_id): - return id_mapping.get(src_id, 0) return map_id def _remap_mask(self, mask): diff --git a/datumaro/plugins/voc_format/extractor.py b/datumaro/plugins/voc_format/extractor.py index 655d72b893..993b825350 100644 --- a/datumaro/plugins/voc_format/extractor.py +++ b/datumaro/plugins/voc_format/extractor.py @@ -12,8 +12,8 @@ from datumaro.components.extractor import (SourceExtractor, DatasetItem, AnnotationType, Label, Mask, Bbox, CompiledMask ) -from datumaro.util import dir_items -from datumaro.util.image import Image +from datumaro.util.os_util import dir_items +from datumaro.util.image import Image, find_images from datumaro.util.mask_tools import lazy_mask, invert_colormap from .format import ( @@ -24,10 +24,11 @@ _inverse_inst_colormap = invert_colormap(VocInstColormap) class _VocExtractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, task): assert osp.isfile(path), path self._path = path self._dataset_dir = osp.dirname(osp.dirname(osp.dirname(path))) + self._task = task super().__init__(subset=osp.splitext(osp.basename(path))[0]) @@ -41,7 +42,7 @@ def __init__(self, path): self._categories[AnnotationType.label].items )) )) - self._items = self._load_subset_list(path) + self._items = { item: None for item in self._load_subset_list(path) } def _get_label_id(self, label): label_id, _ = self._categories[AnnotationType.label].find(label) @@ -56,21 +57,44 @@ def _load_categories(dataset_path): label_map = parse_label_map(label_map_path) return make_voc_categories(label_map) - @staticmethod - def _load_subset_list(subset_path): - with open(subset_path) as f: - return [line.split()[0] for line in f] + def _load_subset_list(self, subset_path): + subset_list = [] + with open(subset_path, encoding='utf-8') as f: + for line in f: + if self._task == VocTask.person_layout: + objects = line.split('\"') + if 1 < len(objects): + if len(objects) == 3: + line = objects[1] + else: + raise Exception("Line %s: unexpected number " + "of quotes in filename" % line) + else: + line = line.split()[0] + else: + line = line.strip() + subset_list.append(line) + return subset_list class VocClassificationExtractor(_VocExtractor): + def __init__(self, path): + super().__init__(path, VocTask.classification) + def __iter__(self): raw_anns = self._load_annotations() + + image_dir = osp.join(self._dataset_dir, VocPath.IMAGES_DIR) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + for item_id in self._items: log.debug("Reading item '%s'" % item_id) - image = osp.join(self._dataset_dir, VocPath.IMAGES_DIR, - item_id + VocPath.IMAGE_EXT) anns = self._parse_annotations(raw_anns, item_id) yield DatasetItem(id=item_id, subset=self._subset, - image=image, annotations=anns) + image=images.get(item_id), annotations=anns) def _load_annotations(self): annotations = defaultdict(list) @@ -78,11 +102,11 @@ def _load_annotations(self): anno_files = [s for s in dir_items(task_dir, '.txt') if s.endswith('_' + osp.basename(self._path))] for ann_filename in anno_files: - with open(osp.join(task_dir, ann_filename)) as f: + with open(osp.join(task_dir, ann_filename), encoding='utf-8') as f: label = ann_filename[:ann_filename.rfind('_')] label_id = self._get_label_id(label) for line in f: - item, present = line.split() + item, present = line.rsplit(maxsplit=1) if present == '1': annotations[item].append(label_id) @@ -94,8 +118,7 @@ def _parse_annotations(raw_anns, item_id): class _VocXmlExtractor(_VocExtractor): def __init__(self, path, task): - super().__init__(path) - self._task = task + super().__init__(path, task) def __iter__(self): anno_dir = osp.join(self._dataset_dir, VocPath.ANNOTATIONS_DIR) @@ -230,14 +253,22 @@ def __init__(self, path): super().__init__(path, task=VocTask.action_classification) class VocSegmentationExtractor(_VocExtractor): + def __init__(self, path): + super().__init__(path, task=VocTask.segmentation) + def __iter__(self): + image_dir = osp.join(self._dataset_dir, VocPath.IMAGES_DIR) + if osp.isdir(image_dir): + images = { osp.splitext(osp.relpath(p, image_dir))[0]: p + for p in find_images(image_dir, recursive=True) } + else: + images = {} + for item_id in self._items: log.debug("Reading item '%s'" % item_id) - image = osp.join(self._dataset_dir, VocPath.IMAGES_DIR, - item_id + VocPath.IMAGE_EXT) anns = self._load_annotations(item_id) yield DatasetItem(id=item_id, subset=self._subset, - image=image, annotations=anns) + image=images.get(item_id), annotations=anns) @staticmethod def _lazy_extract_mask(mask, c): diff --git a/datumaro/plugins/widerface_format.py b/datumaro/plugins/widerface_format.py index 87005b66ad..f5e0008f60 100644 --- a/datumaro/plugins/widerface_format.py +++ b/datumaro/plugins/widerface_format.py @@ -23,15 +23,16 @@ class WiderFacePath: 'occluded', 'pose', 'invalid'] class WiderFaceExtractor(SourceExtractor): - def __init__(self, path): + def __init__(self, path, subset=None): if not osp.isfile(path): raise Exception("Can't read annotation file '%s'" % path) self._path = path self._dataset_dir = osp.dirname(osp.dirname(path)) - subset = osp.splitext(osp.basename(path))[0] - if re.fullmatch(r'wider_face_\S+_bbx_gt', subset): - subset = subset.split('_')[2] + if not subset: + subset = osp.splitext(osp.basename(path))[0] + if re.fullmatch(r'wider_face_\S+_bbx_gt', subset): + subset = subset.split('_')[2] super().__init__(subset=subset) self._categories = self._load_categories() @@ -62,18 +63,21 @@ def _load_categories(self): def _load_items(self, path): items = {} - with open(path, 'r') as f: + with open(path, 'r', encoding='utf-8') as f: lines = f.readlines() - image_ids = [image_id for image_id, line in enumerate(lines) - if WiderFacePath.IMAGE_EXT in line] + line_ids = [line_idx for line_idx, line in enumerate(lines) + if ('/' in line or '\\' in line) and '.' in line] \ + # a heuristic for paths + + for line_idx in line_ids: + image_path = lines[line_idx].strip() + item_id = osp.splitext(image_path)[0] - for image_id in image_ids: - image = lines[image_id] image_path = osp.join(self._dataset_dir, WiderFacePath.SUBSET_DIR + self._subset, - WiderFacePath.IMAGES_DIR, image[:-1]) - item_id = image[:-(len(WiderFacePath.IMAGE_EXT) + 1)] + WiderFacePath.IMAGES_DIR, image_path) + annotations = [] if '/' in item_id: label_name = item_id.split('/')[0] @@ -85,8 +89,15 @@ def _load_items(self, path): annotations.append(Label(label=label)) item_id = item_id[len(item_id.split('/')[0]) + 1:] - bbox_count = lines[image_id + 1] - bbox_lines = lines[image_id + 2 : image_id + int(bbox_count) + 2] + items[item_id] = DatasetItem(id=item_id, subset=self._subset, + image=image_path, annotations=annotations) + + try: + bbox_count = int(lines[line_idx + 1]) # can be the next image + except ValueError: + continue + + bbox_lines = lines[line_idx + 2 : line_idx + bbox_count + 2] for bbox in bbox_lines: bbox_list = bbox.split() if 4 <= len(bbox_list): @@ -111,8 +122,6 @@ def _load_items(self, path): attributes=attributes, label=label )) - items[item_id] = DatasetItem(id=item_id, subset=self._subset, - image=image_path, annotations=annotations) return items class WiderFaceImporter(Importer): @@ -122,7 +131,7 @@ def find_sources(cls, path): dirname=WiderFacePath.ANNOTATIONS_DIR) class WiderFaceConverter(Converter): - DEFAULT_IMAGE_EXT = '.jpg' + DEFAULT_IMAGE_EXT = WiderFacePath.IMAGE_EXT def apply(self): save_dir = self._save_dir @@ -143,12 +152,12 @@ def apply(self): labels = [a.label for a in item.annotations if a.type == AnnotationType.label] if labels: - image_path = '%s--%s/%s' % ( - labels[0], label_categories[labels[0]].name, - item.id + WiderFacePath.IMAGE_EXT) + image_path = self._make_image_filename(item, + subdir='%s--%s' % ( + labels[0], label_categories[labels[0]].name)) else: - image_path = '%s/%s' % (WiderFacePath.IMAGES_DIR_NO_LABEL, - item.id + WiderFacePath.IMAGE_EXT) + image_path = self._make_image_filename(item, + subdir=WiderFacePath.IMAGES_DIR_NO_LABEL) wider_annotation += image_path + '\n' if item.has_image and self._save_images: self._save_image(item, osp.join(save_dir, subset_dir, @@ -178,5 +187,5 @@ def apply(self): annotation_path = osp.join(save_dir, WiderFacePath.ANNOTATIONS_DIR, 'wider_face_' + subset_name + '_bbx_gt.txt') os.makedirs(osp.dirname(annotation_path), exist_ok=True) - with open(annotation_path, 'w') as f: + with open(annotation_path, 'w', encoding='utf-8') as f: f.write(wider_annotation) diff --git a/datumaro/plugins/yolo_format/converter.py b/datumaro/plugins/yolo_format/converter.py index 351636b5d8..fb71b8f172 100644 --- a/datumaro/plugins/yolo_format/converter.py +++ b/datumaro/plugins/yolo_format/converter.py @@ -39,7 +39,7 @@ def apply(self): label_categories = extractor.categories()[AnnotationType.label] label_ids = {label.name: idx for idx, label in enumerate(label_categories.items)} - with open(osp.join(save_dir, 'obj.names'), 'w') as f: + with open(osp.join(save_dir, 'obj.names'), 'w', encoding='utf-8') as f: f.writelines('%s\n' % l[0] for l in sorted(label_ids.items(), key=lambda x: x[1])) @@ -88,15 +88,15 @@ def apply(self): annotation_path = osp.join(subset_dir, '%s.txt' % item.id) os.makedirs(osp.dirname(annotation_path), exist_ok=True) - with open(annotation_path, 'w') as f: + with open(annotation_path, 'w', encoding='utf-8') as f: f.write(yolo_annotation) subset_list_name = '%s.txt' % subset_name subset_lists[subset_name] = subset_list_name - with open(osp.join(save_dir, subset_list_name), 'w') as f: + with open(osp.join(save_dir, subset_list_name), 'w', encoding='utf-8') as f: f.writelines('%s\n' % s for s in image_paths.values()) - with open(osp.join(save_dir, 'obj.data'), 'w') as f: + with open(osp.join(save_dir, 'obj.data'), 'w', encoding='utf-8') as f: f.write('classes = %s\n' % len(label_ids)) for subset_name, subset_list_name in subset_lists.items(): diff --git a/datumaro/plugins/yolo_format/extractor.py b/datumaro/plugins/yolo_format/extractor.py index 54774f08cb..33ab8eb7ff 100644 --- a/datumaro/plugins/yolo_format/extractor.py +++ b/datumaro/plugins/yolo_format/extractor.py @@ -10,7 +10,7 @@ from datumaro.components.extractor import (SourceExtractor, Extractor, DatasetItem, AnnotationType, Bbox, LabelCategories, Importer ) -from datumaro.util import split_path +from datumaro.util.os_util import split_path from datumaro.util.image import Image from .format import YoloPath @@ -52,14 +52,14 @@ def __init__(self, config_path, image_info=None): if isinstance(image_info, str): if not osp.isfile(image_info): raise Exception("Can't read image meta file '%s'" % image_info) - with open(image_info) as f: + with open(image_info, encoding='utf-8') as f: image_info = {} for line in f: - image_name, h, w = line.strip().split() + image_name, h, w = line.strip().rsplit(maxsplit=2) image_info[image_name] = (int(h), int(w)) self._image_info = image_info - with open(config_path, 'r') as f: + with open(config_path, 'r', encoding='utf-8') as f: config_lines = f.readlines() subsets = OrderedDict() @@ -89,7 +89,7 @@ def __init__(self, config_path, image_info=None): raise Exception("Not found '%s' subset list file" % subset_name) subset = YoloExtractor.Subset(subset_name, self) - with open(list_path, 'r') as f: + with open(list_path, 'r', encoding='utf-8') as f: subset.items = OrderedDict( (self.name_from_path(p), self.localize_path(p)) for p in f @@ -107,7 +107,7 @@ def __init__(self, config_path, image_info=None): @staticmethod def localize_path(path): path = osp.normpath(path).strip() - default_base = osp.join('data', '') + default_base = 'data' + osp.sep if path.startswith(default_base): # default path path = path[len(default_base) : ] return path @@ -117,9 +117,9 @@ def name_from_path(cls, path): path = cls.localize_path(path) parts = split_path(path) if 1 < len(parts) and not osp.isabs(path): - # NOTE: when path is like [data/]/ + # NOTE: when path is like [data/]_obj/ # drop everything but - # can be , so no just basename() + # can be , so not just basename() path = osp.join(*parts[1:]) return osp.splitext(path)[0] @@ -143,7 +143,7 @@ def _get(self, item_id, subset_name): @staticmethod def _parse_annotations(anno_path, image): lines = [] - with open(anno_path, 'r') as f: + with open(anno_path, 'r', encoding='utf-8') as f: for line in f: line = line.strip() if line: @@ -151,10 +151,10 @@ def _parse_annotations(anno_path, image): annotations = [] if lines: - size = image.size # use image info as late as possible - if size is None: + # use image info as late as possible + if image.size is None: raise Exception("Can't find image info for '%s'" % image.path) - image_height, image_width = size + image_height, image_width = image.size for line in lines: label_id, xc, yc, w, h = line.split() label_id = int(label_id) @@ -174,7 +174,7 @@ def _parse_annotations(anno_path, image): def _load_categories(names_path): label_categories = LabelCategories() - with open(names_path, 'r') as f: + with open(names_path, 'r', encoding='utf-8') as f: for label in f: label_categories.add(label.strip()) diff --git a/datumaro/util/__init__.py b/datumaro/util/__init__.py index b7e56890ae..ad16d2347d 100644 --- a/datumaro/util/__init__.py +++ b/datumaro/util/__init__.py @@ -4,42 +4,17 @@ # SPDX-License-Identifier: MIT import attr -import os -import os.path as osp from contextlib import ExitStack from functools import partial, wraps from itertools import islice +from distutils.util import strtobool as str_to_bool # pylint: disable=unused-import +NOTSET = object() + def find(iterable, pred=lambda x: True, default=None): return next((x for x in iterable if pred(x)), default) -def dir_items(path, ext, truncate_ext=False): - items = [] - for f in os.listdir(path): - ext_pos = f.rfind(ext) - if ext_pos != -1: - if truncate_ext: - f = f[:ext_pos] - items.append(f) - return items - -def split_path(path): - path = osp.normpath(path) - parts = [] - - while True: - path, part = osp.split(path) - if part: - parts.append(part) - else: - if path: - parts.append(path) - break - parts.reverse() - - return parts - def cast(value, type_conv, default=None): if value is None: return default @@ -83,18 +58,33 @@ def take_by(iterable, count): yield batch -def str_to_bool(s): - t = s.lower() - if t in {'true', '1', 'ok', 'yes', 'y'}: - return True - elif t in {'false', '0', 'no', 'n'}: - return False - else: - raise ValueError("Can't convert value '%s' to bool" % s) - def filter_dict(d, exclude_keys): return { k: v for k, v in d.items() if k not in exclude_keys } +def parse_str_enum_value(value, enum_class, default=NOTSET, + unknown_member_error=None): + if value is None and default is not NOTSET: + value = default + elif isinstance(value, str): + try: + value = enum_class[value] + except KeyError: + raise ValueError((unknown_member_error or + "Unknown element of {cls} '{value}'. " + "The only known are: {available}") \ + .format( + cls=enum_class.__name__, + value=value, + available=', '.join(e.name for e in enum_class) + ) + ) + elif isinstance(value, enum_class): + pass + else: + raise TypeError("Expected value type string or %s, but got %s" % \ + (enum_class.__name__, type(value).__name__)) + return value + def optional_arg_decorator(fn): @wraps(fn) def wrapped_decorator(*args, **kwargs): diff --git a/datumaro/util/annotation_util.py b/datumaro/util/annotation_util.py index 3daa313f3f..a9e50306dd 100644 --- a/datumaro/util/annotation_util.py +++ b/datumaro/util/annotation_util.py @@ -6,7 +6,8 @@ import numpy as np -from datumaro.components.extractor import _Shape, Mask, AnnotationType, RleMask +from datumaro.components.extractor import (LabelCategories, _Shape, Mask, + AnnotationType, RleMask) from datumaro.util.mask_tools import mask_to_rle @@ -210,3 +211,19 @@ def smooth_line(points, segments): new_points[new_segment] = prev_p * (1 - r) + next_p * r return new_points, step + +def make_label_id_mapping( + src_labels: LabelCategories, dst_labels: LabelCategories, fallback=0): + source_labels = { id: label.name + for id, label in enumerate(src_labels or LabelCategories().items) + } + target_labels = { label.name: id + for id, label in enumerate(dst_labels or LabelCategories().items) + } + id_mapping = { src_id: target_labels.get(src_label, fallback) + for src_id, src_label in source_labels.items() + } + + def map_id(src_id): + return id_mapping.get(src_id, fallback) + return map_id, id_mapping, source_labels, target_labels diff --git a/datumaro/util/image.py b/datumaro/util/image.py index e2f086e74b..9500232263 100644 --- a/datumaro/util/image.py +++ b/datumaro/util/image.py @@ -7,6 +7,7 @@ from enum import Enum from io import BytesIO +from typing import Iterator, Iterable, Union import numpy as np import os import os.path as osp @@ -21,6 +22,7 @@ _IMAGE_BACKEND = _IMAGE_BACKENDS.PIL from datumaro.util.image_cache import ImageCache as _ImageCache +from datumaro.util.os_util import walk def load_image(path, dtype=np.float32): @@ -153,6 +155,37 @@ def decode_image(image_bytes, dtype=np.float32): assert image.shape[2] in {3, 4} return image +IMAGE_EXTENSIONS = {'.jpg', '.jpeg', '.jpe', '.jp2', + '.png', '.bmp', '.dib', '.tif', '.tiff', '.tga', '.webp', '.pfm', + '.sr', '.ras', '.exr', '.hdr', '.pic', + '.pbm', '.pgm', '.ppm', '.pxm', '.pnm', +} + +def find_images(dirpath: str, exts: Union[str, Iterable[str]] = None, + recursive: bool = False, max_depth: int = None) -> Iterator[str]: + if isinstance(exts, str): + exts = ['.' + exts.lower().lstrip('.')] + elif exts is None: + exts = IMAGE_EXTENSIONS + else: + exts = list('.' + e.lower().lstrip('.') for e in exts) + + def _check_image_ext(filename: str): + dotpos = filename.rfind('.') + if 0 < dotpos: # exclude '.ext' cases too + ext = filename[dotpos:].lower() + if ext in exts: + return True + return False + + for d, _, filenames in walk(dirpath, + max_depth=max_depth if recursive else 0): + for filename in filenames: + if not _check_image_ext(filename): + continue + + yield osp.join(d, filename) + class lazy_image: def __init__(self, path, loader=None, cache=None): @@ -217,6 +250,9 @@ def __init__(self, data=None, path=None, loader=None, cache=None, data = lazy_image(path, loader=loader, cache=cache) self._data = data + if not self._size and isinstance(data, np.ndarray): + self._size = data.shape[:2] + @property def path(self): return self._path @@ -228,13 +264,22 @@ def ext(self): @property def data(self): if callable(self._data): - return self._data() - return self._data + data = self._data() + else: + data = self._data + + if self._size is None and data is not None: + self._size = data.shape[:2] + return data @property def has_data(self): return self._data is not None + @property + def has_size(self): + return self._size is not None or isinstance(self._data, np.ndarray) + @property def size(self): if self._size is None: diff --git a/datumaro/util/os_util.py b/datumaro/util/os_util.py index c090dced14..094329206a 100644 --- a/datumaro/util/os_util.py +++ b/datumaro/util/os_util.py @@ -9,6 +9,8 @@ import sys +DEFAULT_MAX_DEPTH = 10 + def check_instruction_set(instruction): return instruction == str.strip( # Let's ignore a warning from bandit about using shell=True. @@ -34,10 +36,39 @@ def import_foreign_module(name, path, package=None): return module def walk(path, max_depth=None): + if max_depth is None: + max_depth = DEFAULT_MAX_DEPTH + baselevel = path.count(osp.sep) for dirpath, dirnames, filenames in os.walk(path, topdown=True): curlevel = dirpath.count(osp.sep) if baselevel + max_depth <= curlevel: dirnames.clear() # topdown=True allows to modify the list - yield dirpath, dirnames, filenames \ No newline at end of file + yield dirpath, dirnames, filenames + +def dir_items(path, ext, truncate_ext=False): + items = [] + for f in os.listdir(path): + ext_pos = f.rfind(ext) + if ext_pos != -1: + if truncate_ext: + f = f[:ext_pos] + items.append(f) + return items + +def split_path(path): + path = osp.normpath(path) + parts = [] + + while True: + path, part = osp.split(path) + if part: + parts.append(part) + else: + if path: + parts.append(path) + break + parts.reverse() + + return parts diff --git a/datumaro/util/test_utils.py b/datumaro/util/test_utils.py index 63bd4222c0..8c5cf05af2 100644 --- a/datumaro/util/test_utils.py +++ b/datumaro/util/test_utils.py @@ -6,9 +6,17 @@ import inspect import os import os.path as osp -import shutil import tempfile +try: + # Use rmtree from GitPython to avoid the problem with removal of + # readonly files on Windows, which Git uses extensively + # It double checks if a file cannot be removed because of readonly flag + from git.util import rmtree, rmfile +except ImportError: + from shutil import rmtree + from os import remove as rmfile + from datumaro.components.extractor import AnnotationType from datumaro.components.dataset import Dataset from datumaro.util import find @@ -18,10 +26,9 @@ def current_function_name(depth=1): return inspect.getouterframes(inspect.currentframe())[depth].function class FileRemover: - def __init__(self, path, is_dir=False, ignore_errors=False): + def __init__(self, path, is_dir=False): self.path = path self.is_dir = is_dir - self.ignore_errors = ignore_errors def __enter__(self): return self.path @@ -29,20 +36,30 @@ def __enter__(self): # pylint: disable=redefined-builtin def __exit__(self, type=None, value=None, traceback=None): if self.is_dir: - shutil.rmtree(self.path, ignore_errors=self.ignore_errors) + rmtree(self.path) else: - os.remove(self.path) + rmfile(self.path) # pylint: enable=redefined-builtin class TestDir(FileRemover): - def __init__(self, path=None, ignore_errors=False): + """ + Creates a temporary directory for a test. Uses the name of + the test function to name the directory. + + Usage: + + with TestDir() as test_dir: + ... + """ + + def __init__(self, path=None): if path is None: path = osp.abspath('temp_%s-' % current_function_name(2)) path = tempfile.mkdtemp(dir=os.getcwd(), prefix=path) else: - os.makedirs(path, exist_ok=ignore_errors) + os.makedirs(path, exist_ok=False) - super().__init__(path, is_dir=True, ignore_errors=ignore_errors) + super().__init__(path, is_dir=True) def compare_categories(test, expected, actual): test.assertEqual( @@ -91,7 +108,7 @@ def compare_datasets(test, expected, actual, ignored_attrs=None, item_b = find(actual, lambda x: x.id == item_a.id and \ x.subset == item_a.subset) test.assertFalse(item_b is None, item_a.id) - test.assertEqual(item_a.attributes, item_b.attributes) + test.assertEqual(item_a.attributes, item_b.attributes, item_a.id) if (require_images and item_a.has_image and item_a.image.has_data) or \ item_a.has_image and item_a.image.has_data and \ item_b.has_image and item_b.image.has_data: @@ -127,7 +144,7 @@ def compare_datasets_strict(test, expected, actual): (idx, item_a, item_b)) def test_save_and_load(test, source_dataset, converter, test_dir, importer, - target_dataset=None, importer_args=None, compare=None): + target_dataset=None, importer_args=None, compare=None, **kwargs): converter(source_dataset, test_dir) if importer_args is None: @@ -139,4 +156,4 @@ def test_save_and_load(test, source_dataset, converter, test_dir, importer, if not compare: compare = compare_datasets - compare(test, expected=target_dataset, actual=parsed_dataset) \ No newline at end of file + compare(test, expected=target_dataset, actual=parsed_dataset, **kwargs) \ No newline at end of file diff --git a/datumaro/version.py b/datumaro/version.py index f756c42212..95aaa20678 100644 --- a/datumaro/version.py +++ b/datumaro/version.py @@ -1 +1 @@ -VERSION = '0.1.6.1' \ No newline at end of file +VERSION = '0.1.7' \ No newline at end of file diff --git a/docs/user_manual.md b/docs/user_manual.md index 6266c323d8..06585d36fd 100644 --- a/docs/user_manual.md +++ b/docs/user_manual.md @@ -5,6 +5,7 @@ - [Installation](#installation) - [Interfaces](#interfaces) - [Supported dataset formats and annotations](#supported-formats) +- [Supported data formats](#data-formats) - [Command line workflow](#command-line-workflow) - [Project structure](#project-structure) - [Command reference](#command-reference) @@ -140,6 +141,55 @@ List of supported annotation types: - (Key-)Points - Captions +## Data formats + +Datumaro only works with 2d RGB(A) images. + +To create an unlabelled dataset from an arbitrary directory with images use +`ImageDir` format: + +```bash +datum create -o +datum add path -p -f image_dir +``` + +or if you work with Datumaro API: + +For using with a project: + +```python +from datumaro.components.project import Project + +project = Project() +project.add_source('source1', { + 'format': 'image_dir', + 'url': 'directory/path/' +}) +dataset = project.make_dataset() +``` + +And for using as a dataset: + +```python +from datumaro.components.dataset import Dataset + +dataset = Dataset.import_from('directory/path/', 'image_dir') +``` + +This will search for images in the directory recursively and add +them as dataset entries with names like `//`. +The list of formats matches the list of supported image formats in OpenCV. +``` +.jpg, .jpeg, .jpe, .jp2, .png, .bmp, .dib, .tif, .tiff, .tga, .webp, .pfm, +.sr, .ras, .exr, .hdr, .pic, .pbm, .pgm, .ppm, .pxm, .pnm +``` + +After addition into a project, images can be split into subsets and renamed +with transformations, filtered, joined with existing annotations etc. + +To use a video as an input, one should either [create an Extractor plugin](../docs/developer_guide.md#plugins), +which splits a video into frames, or split the video manually and import images. + ## Command line workflow The key object is a project, so most CLI commands operate on projects. @@ -850,6 +900,7 @@ datum model add \ ``` Interpretation script for an OpenVINO detection model (`convert.py`): +You can find OpenVINO™ model interpreter samples in datumaro/plugins/openvino/samples. [Instruction](datumaro/plugins/openvino/README.md) ``` python from datumaro.components.extractor import * @@ -1023,6 +1074,43 @@ datum transform -t rename -- -e '|pattern|replacement|' datum transform -t rename -- -e '|frame_(\d+)|\\1|' ``` +Example: Sampling dataset items, subset `train` is divided into `sampled`(sampled_subset) and `unsampled` +- `train` has 100 data, and 20 samples are selected. There are `sampled`(20 samples) and 80 `unsampled`(80 datas) subsets. +- Remove `train` subset (if sample_name=`train` or unsample_name=`train`, still remain) +- There are five methods of sampling the m option. + - `topk`: Return the k with high uncertainty data + - `lowk`: Return the k with low uncertainty data + - `randk`: Return the random k data + - `mixk`: Return half to topk method and the rest to lowk method + - `randtopk`: First, select 3 times the number of k randomly, and return the topk among them. + +``` bash +datum transform -t sampler -- \ + -a entropy \ + -subset_name train \ + -sample_name sampled \ + -unsample_name unsampled \ + -m topk \ + -k 20 +``` + +Example : Control number of outputs to 100 after NDR +- There are two methods in NDR e option + - `random`: sample from removed data randomly + - `similarity`: sample from removed data with ascending +- There are two methods in NDR u option + - `uniform`: sample data with uniform distribution + - `inverse`: sample data with reciprocal of the number + +```bash +datum transform -t ndr -- \ + -w train \ + -a gradient \ + -k 100 \ + -e random \ + -u uniform +``` + ## Extending There are few ways to extend and customize Datumaro behaviour, which is supported by plugins. diff --git a/requirements.txt b/requirements.txt index 6bc3c7ee79..5cfc7dd4f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ pycocotools>=2.0.0 PyYAML>=5.3.1 scikit-image>=0.15.0 tensorboardX>=1.8 +pandas>=1.1.5 diff --git a/tests/assets/sampler/inference.csv b/tests/assets/sampler/inference.csv new file mode 100644 index 0000000000..e08065831a --- /dev/null +++ b/tests/assets/sampler/inference.csv @@ -0,0 +1,31 @@ +ImageID,ClassProbability1,ClassProbability2,ClassProbability3,Uncertainty +1,0.975242317,0.024469912,0.000287826,0.117586322 +2,0.999715984,0.000281501,2.53E-06,0.002618015 +3,0.999299884,0.000691595,8.50E-06,0.005831472 +4,0.971567273,0.027958876,0.000473852,0.131661266 +5,0.999411225,0.000576135,1.26E-05,0.005028461 +6,0.999715269,0.00027976,4.95E-06,0.002634019 +7,0.978483677,0.021343108,0.00017317,0.104890488 +8,0.984344006,0.015289294,0.000366639,0.082351737 +9,0.974284053,0.025472108,0.000243954,0.120898865 +10,0.964820206,0.034958012,0.000221764,0.153654948 +11,0.996293604,0.003278826,0.000427532,0.02577186 +12,0.999689937,0.000307999,2.14E-06,0.002828279 +13,0.997596323,0.000604421,0.001799274,0.018252373 +14,0.999696493,0.000294724,8.87E-06,0.002802743 +15,0.999686837,0.000309912,3.27E-06,0.002858304 +16,0.999234438,0.000750318,1.53E-05,0.006333055 +17,0.999581277,0.000413273,5.49E-06,0.003705278 +18,0.999384761,0.000604751,1.05E-05,0.005217474 +19,0.999574125,0.000416982,8.93E-06,0.003774712 +20,0.999575078,0.000411838,1.31E-05,0.003782649 +21,0.999712646,0.000286349,9.24E-07,0.002636151 +22,0.998748422,0.001103578,0.000147974,0.010070177 +23,0.999729574,0.000268848,1.53E-06,0.002501184 +24,0.999636412,0.000354998,8.59E-06,0.003283583 +25,0.999675989,0.000322926,1.11E-06,0.002934833 +26,0.970380008,0.029310413,0.000309611,0.135138899 +27,0.979150653,0.019359451,0.001489813,0.106692567 +28,0.999622822,0.000374233,3.02E-06,0.003368486 +29,0.999201596,0.000615866,0.000182658,0.006923281 +30,0.999691606,0.0002986,9.82E-06,0.002845172 diff --git a/tests/test_camvid_format.py b/tests/test_camvid_format.py index 85e0b6e7d9..9bf3b1b234 100644 --- a/tests/test_camvid_format.py +++ b/tests/test_camvid_format.py @@ -9,6 +9,7 @@ Extractor, LabelCategories, Mask) from datumaro.components.dataset import Dataset from datumaro.plugins.camvid_format import CamvidConverter, CamvidImporter +from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -77,10 +78,10 @@ def test_can_detect_camvid(self): class CamvidConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='camvid', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_camvid_segm(self): class TestExtractor(TestExtractorBase): @@ -145,6 +146,22 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(CamvidConverter.convert, label_map='camvid'), test_dir) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='кириллица с пробелом', + image=np.ones((1, 5, 3)), annotations=[ + Mask(image=np.array([[1, 0, 0, 1, 0]]), label=0), + Mask(image=np.array([[0, 1, 1, 0, 1]]), label=3), + ] + ), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(CamvidConverter.convert, label_map='camvid'), test_dir) + def test_can_save_with_no_masks(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -227,3 +244,57 @@ def categories(self): self._test_save_and_load(SrcExtractor(), partial(CamvidConverter.convert, label_map='source'), test_dir, target_dataset=DstExtractor()) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + class SrcExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='a/b/c/2', image=Image( + path='a/b/c/2.bmp', data=np.ones((1, 5, 3)) + ), + annotations=[ + Mask(np.array([[0, 0, 0, 1, 0]]), + label=self._label('a')), + Mask(np.array([[0, 1, 1, 0, 0]]), + label=self._label('b')), + ]) + ]) + + def categories(self): + label_map = OrderedDict() + label_map['a'] = None + label_map['b'] = None + return Camvid.make_camvid_categories(label_map) + + class DstExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='a/b/c/2', image=Image( + path='a/b/c/2.bmp', data=np.ones((1, 5, 3)) + ), + annotations=[ + Mask(np.array([[1, 0, 0, 0, 1]]), + label=self._label('background')), + Mask(np.array([[0, 0, 0, 1, 0]]), + label=self._label('a')), + Mask(np.array([[0, 1, 1, 0, 0]]), + label=self._label('b')), + ]) + ]) + + def categories(self): + label_map = OrderedDict() + label_map['background'] = None + label_map['a'] = None + label_map['b'] = None + return Camvid.make_camvid_categories(label_map) + + with TestDir() as test_dir: + self._test_save_and_load(SrcExtractor(), + partial(CamvidConverter.convert, save_images=True), + test_dir, require_images=True, + target_dataset=DstExtractor()) \ No newline at end of file diff --git a/tests/test_coco_format.py b/tests/test_coco_format.py index c2ee51bd8e..b884009b22 100644 --- a/tests/test_coco_format.py +++ b/tests/test_coco_format.py @@ -148,10 +148,10 @@ def test_can_detect(self): class CocoConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='coco', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_and_load_captions(self): expected_dataset = Dataset.from_iterable([ @@ -424,6 +424,16 @@ def test_can_save_and_load_images(self): self._test_save_and_load(expected_dataset, CocoImageInfoConverter.convert, test_dir) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + expected_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', subset='train', + attributes={'id': 1}), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(expected_dataset, + CocoImageInfoConverter.convert, test_dir) + def test_can_save_and_load_labels(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id=1, subset='train', @@ -544,7 +554,21 @@ def test_relative_paths(self): with TestDir() as test_dir: self._test_save_and_load(expected_dataset, - partial(CocoImageInfoConverter.convert, save_images=True), test_dir) + partial(CocoImageInfoConverter.convert, save_images=True), + test_dir, require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3))), attributes={'id': 1}), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3))), attributes={'id': 2}), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(expected, + partial(CocoConverter.convert, save_images=True), + test_dir, require_images=True) def test_preserve_coco_ids(self): expected_dataset = Dataset.from_iterable([ @@ -554,7 +578,8 @@ def test_preserve_coco_ids(self): with TestDir() as test_dir: self._test_save_and_load(expected_dataset, - partial(CocoImageInfoConverter.convert, save_images=True), test_dir) + partial(CocoImageInfoConverter.convert, save_images=True), + test_dir, require_images=True) def test_annotation_attributes(self): expected_dataset = Dataset.from_iterable([ diff --git a/tests/test_cvat_format.py b/tests/test_cvat_format.py index 4caeaeed0d..ced0930f04 100644 --- a/tests/test_cvat_format.py +++ b/tests/test_cvat_format.py @@ -142,10 +142,10 @@ def test_can_load_video(self): class CvatConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='cvat', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_and_load(self): label_categories = LabelCategories() @@ -242,7 +242,7 @@ def test_relative_paths(self): DatasetItem(id='1', image=np.ones((4, 2, 3))), DatasetItem(id='subdir1/1', image=np.ones((2, 6, 3))), DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3))), - ], categories={ AnnotationType.label: LabelCategories() }) + ]) target_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((4, 2, 3)), @@ -251,22 +251,65 @@ def test_relative_paths(self): attributes={'frame': 1}), DatasetItem(id='subdir2/1', image=np.ones((5, 4, 3)), attributes={'frame': 2}), + ], categories=[]) + + with TestDir() as test_dir: + self._test_save_and_load(source_dataset, + partial(CvatConverter.convert, save_images=True), test_dir, + target_dataset=target_dataset, require_images=True) + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + label_categories = LabelCategories() + for i in range(10): + label_categories.add(str(i)) + label_categories.items[2].attributes.update(['a1', 'a2', 'empty']) + label_categories.attributes.update(['occluded']) + + source_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', + subset='s1', image=np.zeros((5, 10, 3)), + annotations=[ + Label(1), + ] + ), ], categories={ - AnnotationType.label: LabelCategories() + AnnotationType.label: label_categories, + }) + + target_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', + subset='s1', image=np.zeros((5, 10, 3)), + annotations=[ + Label(1), + ], attributes={'frame': 0} + ), + ], categories={ + AnnotationType.label: label_categories, }) with TestDir() as test_dir: self._test_save_and_load(source_dataset, partial(CvatConverter.convert, save_images=True), test_dir, - target_dataset=target_dataset) + target_dataset=target_dataset, require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem('q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3))), attributes={'frame': 1}), + DatasetItem('a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3))), attributes={'frame': 2}), + ], categories=[]) + + with TestDir() as test_dir: + self._test_save_and_load(expected, + partial(CvatConverter.convert, save_images=True), + test_dir, require_images=True) def test_preserve_frame_ids(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), attributes={'frame': 40}), - ], categories={ - AnnotationType.label: LabelCategories() - }) + ], categories=[]) with TestDir() as test_dir: self._test_save_and_load(expected_dataset, @@ -276,12 +319,12 @@ def test_reindex(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), attributes={'frame': 40}), - ], categories={ AnnotationType.label: LabelCategories() }) + ]) expected_dataset = Dataset.from_iterable([ DatasetItem(id='some/name1', image=np.ones((4, 2, 3)), attributes={'frame': 0}), - ], categories={ AnnotationType.label: LabelCategories() }) + ], categories=[]) with TestDir() as test_dir: self._test_save_and_load(source_dataset, @@ -295,7 +338,7 @@ def test_inplace_save_writes_only_updated_data(self): DatasetItem(1, subset='a'), DatasetItem(2, subset='b'), DatasetItem(3, subset='c', image=np.ones((2, 2, 3))), - ], categories=[]) + ]) dataset.export(path, 'cvat', save_images=True) os.unlink(osp.join(path, 'a.xml')) os.unlink(osp.join(path, 'b.xml')) diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 6b5ec423f1..ac5e6b65a6 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -564,6 +564,25 @@ def test_flushes_changes_on_save(self): self.assertFalse(dataset.is_modified) + def test_does_not_load_images_on_saving(self): + # Issue https://github.com/openvinotoolkit/datumaro/issues/177 + # Missing image metadata (size etc.) can lead to image loading on + # dataset save without image saving + + called = False + def test_loader(): + nonlocal called + called = True + + dataset = Dataset.from_iterable([ + DatasetItem(1, image=test_loader) + ]) + + with TestDir() as test_dir: + dataset.save(test_dir) + + self.assertFalse(called) + class DatasetItemTest(TestCase): def test_ctor_requires_id(self): diff --git a/tests/test_datumaro_format.py b/tests/test_datumaro_format.py index fa063fcfda..047950e6d3 100644 --- a/tests/test_datumaro_format.py +++ b/tests/test_datumaro_format.py @@ -102,6 +102,29 @@ def test_relative_paths(self): self._test_save_and_load(test_dataset, partial(DatumaroConverter.convert, save_images=True), test_dir) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + test_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', image=np.ones((4, 2, 3))), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(test_dataset, + partial(DatumaroConverter.convert, save_images=True), + test_dir) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3))), attributes={'frame': 1}), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3))), attributes={'frame': 2}), + ]) + + with TestDir() as test_dir: + self._test_save_and_load(expected, + partial(DatumaroConverter.convert, save_images=True), + test_dir) + def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset diff --git a/tests/test_icdar_format.py b/tests/test_icdar_format.py index 5583446531..7559d6fe37 100644 --- a/tests/test_icdar_format.py +++ b/tests/test_icdar_format.py @@ -1,14 +1,19 @@ import os.path as osp +from functools import partial from unittest import TestCase import numpy as np + from datumaro.components.extractor import (Bbox, Caption, DatasetItem, Mask, Polygon) from datumaro.components.project import Dataset from datumaro.plugins.icdar_format.converter import ( IcdarTextLocalizationConverter, IcdarTextSegmentationConverter, IcdarWordRecognitionConverter) -from datumaro.plugins.icdar_format.extractor import IcdarImporter +from datumaro.plugins.icdar_format.extractor import ( + IcdarWordRecognitionImporter, IcdarTextLocalizationImporter, + IcdarTextSegmentationImporter) +from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -16,10 +21,18 @@ DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'icdar_dataset') class IcdarImporterTest(TestCase): - def test_can_detect(self): - self.assertTrue(IcdarImporter.detect( + def test_can_detect_word_recognition(self): + self.assertTrue(IcdarWordRecognitionImporter.detect( osp.join(DUMMY_DATASET_DIR, 'word_recognition'))) + def test_can_detect_text_localization(self): + self.assertTrue(IcdarTextLocalizationImporter.detect( + osp.join(DUMMY_DATASET_DIR, 'text_localization'))) + + def test_can_detect_text_segmentation(self): + self.assertTrue(IcdarTextSegmentationImporter.detect( + osp.join(DUMMY_DATASET_DIR, 'text_segmentation'))) + def test_can_import_captions(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='word_1', subset='train', @@ -37,7 +50,8 @@ def test_can_import_captions(self): ]) dataset = Dataset.import_from( - osp.join(DUMMY_DATASET_DIR, 'word_recognition'), 'icdar') + osp.join(DUMMY_DATASET_DIR, 'word_recognition'), + 'icdar_word_recognition') compare_datasets(self, expected_dataset, dataset) @@ -60,7 +74,8 @@ def test_can_import_bboxes(self): ]) dataset = Dataset.import_from( - osp.join(DUMMY_DATASET_DIR, 'text_localization'), 'icdar') + osp.join(DUMMY_DATASET_DIR, 'text_localization'), + 'icdar_text_localization') compare_datasets(self, expected_dataset, dataset) @@ -89,48 +104,50 @@ def test_can_import_masks(self): ]) dataset = Dataset.import_from( - osp.join(DUMMY_DATASET_DIR, 'text_segmentation'), 'icdar') + osp.join(DUMMY_DATASET_DIR, 'text_segmentation'), + 'icdar_text_segmentation') compare_datasets(self, expected_dataset, dataset) class IcdarConverterTest(TestCase): - def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + def _test_save_and_load(self, source_dataset, converter, test_dir, importer, + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, - importer='icdar', - target_dataset=target_dataset, importer_args=importer_args) + importer, + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_and_load_captions(self): expected_dataset = Dataset.from_iterable([ - DatasetItem(id=1, subset='train', - annotations=[ - Caption('caption_0'), + DatasetItem(id='a/b/1', subset='train', + image=np.ones((10, 15, 3)), annotations=[ + Caption('caption 0'), ]), DatasetItem(id=2, subset='train', - annotations=[ + image=np.ones((10, 15, 3)), annotations=[ Caption('caption_1'), ]), ]) with TestDir() as test_dir: self._test_save_and_load(expected_dataset, - IcdarWordRecognitionConverter.convert, test_dir) + partial(IcdarWordRecognitionConverter.convert, save_images=True), + test_dir, 'icdar_word_recognition') def test_can_save_and_load_bboxes(self): expected_dataset = Dataset.from_iterable([ - DatasetItem(id=1, subset='train', - annotations=[ + DatasetItem(id='a/b/1', subset='train', + image=np.ones((10, 15, 3)), annotations=[ Bbox(1, 3, 6, 10), - Bbox(0, 1, 3, 5, attributes={'text': 'word_0'}), + Bbox(0, 1, 3, 5, attributes={'text': 'word 0'}), ]), DatasetItem(id=2, subset='train', - annotations=[ + image=np.ones((10, 15, 3)), annotations=[ Polygon([0, 0, 3, 0, 4, 7, 1, 8], - attributes={'text': 'word_1'}), + attributes={'text': 'word 1'}), Polygon([1, 2, 5, 3, 6, 8, 0, 7]), ]), DatasetItem(id=3, subset='train', - annotations=[ + image=np.ones((10, 15, 3)), annotations=[ Polygon([2, 2, 8, 3, 7, 10, 2, 9], attributes={'text': 'word_2'}), Bbox(0, 2, 5, 9, attributes={'text': 'word_3'}), @@ -139,12 +156,13 @@ def test_can_save_and_load_bboxes(self): with TestDir() as test_dir: self._test_save_and_load(expected_dataset, - IcdarTextLocalizationConverter.convert, test_dir) + partial(IcdarTextLocalizationConverter.convert, save_images=True), + test_dir, 'icdar_text_localization') def test_can_save_and_load_masks(self): expected_dataset = Dataset.from_iterable([ - DatasetItem(id=1, subset='train', - annotations=[ + DatasetItem(id='a/b/1', subset='train', + image=np.ones((10, 15, 3)), annotations=[ Mask(image=np.array([[0, 0, 0, 1, 1]]), group=1, attributes={ 'index': 1, 'color': '82 174 214', 'text': 'j', 'center': '0 3' }), @@ -153,7 +171,7 @@ def test_can_save_and_load_masks(self): 'center': '0 1' }), ]), DatasetItem(id=2, subset='train', - annotations=[ + image=np.ones((10, 15, 3)), annotations=[ Mask(image=np.array([[0, 0, 0, 0, 0, 1]]), group=0, attributes={ 'index': 3, 'color': '183 6 28', 'text': ' ', 'center': '0 5' }), @@ -171,7 +189,8 @@ def test_can_save_and_load_masks(self): with TestDir() as test_dir: self._test_save_and_load(expected_dataset, - IcdarTextSegmentationConverter.convert, test_dir) + partial(IcdarTextSegmentationConverter.convert, save_images=True), + test_dir, 'icdar_text_segmentation') def test_can_save_and_load_with_no_subsets(self): expected_dataset = Dataset.from_iterable([ @@ -183,4 +202,39 @@ def test_can_save_and_load_with_no_subsets(self): with TestDir() as test_dir: self._test_save_and_load(expected_dataset, - IcdarTextLocalizationConverter.convert, test_dir) + IcdarTextLocalizationConverter.convert, test_dir, + 'icdar_text_localization') + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + expected_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', + image=np.ones((8, 8, 3))), + ]) + + for importer, converter in [ + ('icdar_word_recognition', IcdarWordRecognitionConverter), + ('icdar_text_localization', IcdarTextLocalizationConverter), + ('icdar_text_segmentation', IcdarTextSegmentationConverter), + ]: + with self.subTest(subformat=converter), TestDir() as test_dir: + self._test_save_and_load(expected_dataset, + partial(converter.convert, save_images=True), + test_dir, importer, require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3)))), + ]) + + for importer, converter in [ + ('icdar_word_recognition', IcdarWordRecognitionConverter), + ('icdar_text_localization', IcdarTextLocalizationConverter), + ('icdar_text_segmentation', IcdarTextSegmentationConverter), + ]: + with self.subTest(subformat=converter), TestDir() as test_dir: + self._test_save_and_load(expected, + partial(converter.convert, save_images=True), + test_dir, importer, require_images=True) \ No newline at end of file diff --git a/tests/test_image_dir_format.py b/tests/test_image_dir_format.py index f7f21b0888..1b056e8f3e 100644 --- a/tests/test_image_dir_format.py +++ b/tests/test_image_dir_format.py @@ -1,11 +1,14 @@ import numpy as np +import os +import os.path as osp from unittest import TestCase from datumaro.components.project import Dataset from datumaro.components.extractor import DatasetItem -from datumaro.plugins.image_dir import ImageDirConverter -from datumaro.util.test_utils import TestDir, test_save_and_load +from datumaro.plugins.image_dir_format import ImageDirConverter +from datumaro.util.image import Image, save_image +from datumaro.util.test_utils import TestDir, compare_datasets, test_save_and_load class ImageDirFormatTest(TestCase): @@ -17,7 +20,7 @@ def test_can_load(self): with TestDir() as test_dir: test_save_and_load(self, dataset, ImageDirConverter.convert, - test_dir, importer='image_dir') + test_dir, importer='image_dir', require_images=True) def test_relative_paths(self): dataset = Dataset.from_iterable([ @@ -28,4 +31,40 @@ def test_relative_paths(self): with TestDir() as test_dir: test_save_and_load(self, dataset, ImageDirConverter.convert, - test_dir, importer='image_dir') \ No newline at end of file + test_dir, importer='image_dir') + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', image=np.ones((4, 2, 3))), + ]) + + with TestDir() as test_dir: + test_save_and_load(self, dataset, ImageDirConverter.convert, + test_dir, importer='image_dir') + + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3)))), + ]) + + with TestDir() as test_dir: + test_save_and_load(self, dataset, ImageDirConverter.convert, + test_dir, importer='image_dir', require_images=True) + + def test_can_save_and_load_image_with_custom_extension(self): + expected = Dataset.from_iterable([ + DatasetItem(id='a/3', image=Image(path='a/3.qq', + data=np.zeros((3, 4, 3)))), + ]) + + with TestDir() as test_dir: + image_path = osp.join(test_dir, 'a', '3.jpg') + save_image(image_path, expected.get('a/3').image.data, + create_dir=True) + os.rename(image_path, osp.join(test_dir, 'a', '3.qq')) + + actual = Dataset.import_from(test_dir, 'image_dir', exts='qq') + compare_datasets(self, expected, actual, require_images=True) \ No newline at end of file diff --git a/tests/test_imagenet_format.py b/tests/test_imagenet_format.py index 2b4ef79fb1..9a4da64a3f 100644 --- a/tests/test_imagenet_format.py +++ b/tests/test_imagenet_format.py @@ -8,6 +8,7 @@ LabelCategories, AnnotationType ) from datumaro.plugins.imagenet_format import ImagenetConverter, ImagenetImporter +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets class ImagenetFormatTest(TestCase): @@ -21,17 +22,9 @@ def test_can_save_and_load(self): image=np.ones((10, 10, 3)), annotations=[Label(1)] ), - DatasetItem(id='3', - image=np.ones((10, 10, 3)), - annotations=[Label(0)] - ), - DatasetItem(id='4', - image=np.ones((8, 8, 3)), - annotations=[Label(2)] - ), ], categories={ AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(label) for label in range(3)), + 'label_' + str(label) for label in range(2)), }) with TestDir() as test_dir: @@ -46,33 +39,33 @@ def test_can_save_and_load_with_multiple_labels(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((8, 8, 3)), - annotations=[Label(0), Label(1)] + annotations=[Label(0), Label(1), Label(2)] ), DatasetItem(id='2', - image=np.ones((10, 10, 3)), - annotations=[Label(0), Label(1)] - ), - DatasetItem(id='3', - image=np.ones((10, 10, 3)), - annotations=[Label(0), Label(2)] + image=np.ones((8, 8, 3)) ), - DatasetItem(id='4', + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label_' + str(label) for label in range(3)), + }) + + with TestDir() as test_dir: + ImagenetConverter.convert(source_dataset, test_dir, save_images=True) + + parsed_dataset = Dataset.import_from(test_dir, 'imagenet') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id="кириллица с пробелом", image=np.ones((8, 8, 3)), - annotations=[Label(2), Label(4)] - ), - DatasetItem(id='5', - image=np.ones((10, 10, 3)), - annotations=[Label(3), Label(4)] - ), - DatasetItem(id='6', - image=np.ones((10, 10, 3)), - ), - DatasetItem(id='7', - image=np.ones((8, 8, 3)) + annotations=[Label(0), Label(1)] ), ], categories={ AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(label) for label in range(5)), + 'label_' + str(label) for label in range(2)), }) with TestDir() as test_dir: @@ -83,6 +76,21 @@ def test_can_save_and_load_with_multiple_labels(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='a', image=Image(path='a.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='b', image=Image(path='b.bmp', + data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + ImagenetConverter.convert(dataset, test_dir, save_images=True) + + parsed_dataset = Dataset.import_from(test_dir, 'imagenet') + + compare_datasets(self, dataset, parsed_dataset, + require_images=True) DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'imagenet_dataset') diff --git a/tests/test_imagenet_txt_format.py b/tests/test_imagenet_txt_format.py index 4f5dda37c5..2c4d231f58 100644 --- a/tests/test_imagenet_txt_format.py +++ b/tests/test_imagenet_txt_format.py @@ -9,6 +9,7 @@ ) from datumaro.plugins.imagenet_txt_format import \ ImagenetTxtConverter, ImagenetTxtImporter +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -18,24 +19,12 @@ def test_can_save_and_load(self): DatasetItem(id='1', subset='train', annotations=[Label(0)] ), - DatasetItem(id='2', subset='train', - annotations=[Label(0)] - ), - DatasetItem(id='3', subset='train', image=np.zeros((8, 8, 3)), - annotations=[Label(0)] - ), - DatasetItem(id='4', subset='train', - annotations=[Label(1)] - ), - DatasetItem(id='5', subset='train', image=np.zeros((4, 8, 3)), + DatasetItem(id='2', subset='train', image=np.zeros((8, 8, 3)), annotations=[Label(1)] ), - DatasetItem(id='6', subset='train', - annotations=[Label(5)] - ), ], categories={ AnnotationType.label: LabelCategories.from_iterable( - 'label_' + str(label) for label in range(10)), + 'label_' + str(label) for label in range(4)), }) with TestDir() as test_dir: @@ -50,12 +39,9 @@ def test_can_save_and_load(self): def test_can_save_and_load_with_multiple_labels(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', - annotations=[Label(1), Label(3)] - ), - DatasetItem(id='2', subset='train', image=np.zeros((8, 6, 3)), - annotations=[Label(0)] + annotations=[Label(1), Label(2), Label(3)] ), - DatasetItem(id='3', subset='train', image=np.zeros((2, 8, 3)), + DatasetItem(id='2', subset='train', image=np.zeros((2, 8, 3)), ), ], categories={ AnnotationType.label: LabelCategories.from_iterable( @@ -90,6 +76,41 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset, require_images=True) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + dataset = Dataset.from_iterable([ + DatasetItem(id="кириллица с пробелом", + image=np.ones((8, 8, 3)), + annotations=[Label(0), Label(1)] + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label_' + str(label) for label in range(2)), + }) + + with TestDir() as test_dir: + ImagenetTxtConverter.convert(dataset, test_dir, save_images=True) + + parsed_dataset = Dataset.import_from(test_dir, 'imagenet_txt') + + compare_datasets(self, dataset, parsed_dataset, + require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='a/1', image=Image(path='a/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='b/c/d/2', image=Image(path='b/c/d/2.bmp', + data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + ImagenetTxtConverter.convert(dataset, test_dir, save_images=True) + + parsed_dataset = Dataset.import_from(test_dir, 'imagenet_txt') + + compare_datasets(self, dataset, parsed_dataset, + require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'imagenet_txt_dataset') class ImagenetTxtImporterTest(TestCase): diff --git a/tests/test_labelme_format.py b/tests/test_labelme_format.py index 244a590b07..3a514d3d9f 100644 --- a/tests/test_labelme_format.py +++ b/tests/test_labelme_format.py @@ -8,16 +8,17 @@ AnnotationType, Bbox, Mask, Polygon, LabelCategories ) from datumaro.plugins.labelme_format import LabelMeImporter, LabelMeConverter +from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) class LabelMeConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='label_me', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ @@ -85,8 +86,63 @@ def test_can_save_and_load(self): self._test_save_and_load( source_dataset, partial(LabelMeConverter.convert, save_images=True), - test_dir, target_dataset=target_dataset) + test_dir, target_dataset=target_dataset, require_images=True) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='a/1', image=Image(path='a/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='b/c/d/2', image=Image(path='b/c/d/2.bmp', + data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + self._test_save_and_load(dataset, + partial(LabelMeConverter.convert, save_images=True), + test_dir, require_images=True) + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', subset='train', + image=np.ones((16, 16, 3)), + annotations=[ + Polygon([0, 4, 4, 4, 5, 6], label=3, attributes={ + 'occluded': True, + 'a1': 'qwe', + 'a2': True, + 'a3': 123, + }), + ] + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label_' + str(label) for label in range(10)), + }) + + target_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', subset='train', + image=np.ones((16, 16, 3)), + annotations=[ + Polygon([0, 4, 4, 4, 5, 6], label=0, id=0, + attributes={ + 'occluded': True, 'username': '', + 'a1': 'qwe', + 'a2': True, + 'a3': 123, + } + ), + ] + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable([ + 'label_3']), + }) + + with TestDir() as test_dir: + self._test_save_and_load( + source_dataset, + partial(LabelMeConverter.convert, save_images=True), + test_dir, target_dataset=target_dataset, require_images=True) DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'labelme_dataset') diff --git a/tests/test_lfw_format.py b/tests/test_lfw_format.py index 541cccaa02..3aa64365d1 100644 --- a/tests/test_lfw_format.py +++ b/tests/test_lfw_format.py @@ -5,6 +5,7 @@ from datumaro.components.dataset import Dataset from datumaro.components.extractor import DatasetItem, Points from datumaro.plugins.lfw_format import LfwConverter, LfwImporter +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -13,29 +14,29 @@ def test_can_save_and_load(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='name0/name0_0001', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': ['name0/name0_0002'], 'negative_pairs': [] } ), DatasetItem(id='name0/name0_0002', subset='test', image=np.ones((2, 5, 3)), - attributes = { - 'positive_pairs': [], + attributes={ + 'positive_pairs': ['name0/name0_0001'], 'negative_pairs': ['name1/name1_0001'] } ), DatasetItem(id='name1/name1_0001', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': ['name1/name1_0002'], 'negative_pairs': [] } ), DatasetItem(id='name1/name1_0002', subset='test', image=np.ones((2, 5, 3)), - attributes = { - 'positive_pairs': [], + attributes={ + 'positive_pairs': ['name1/name1_0002'], 'negative_pairs': ['name0/name0_0001'] } ), @@ -51,7 +52,7 @@ def test_can_save_and_load_with_landmarks(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='name0/name0_0001', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': ['name0/name0_0002'], 'negative_pairs': [] }, @@ -61,7 +62,7 @@ def test_can_save_and_load_with_landmarks(self): ), DatasetItem(id='name0/name0_0002', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': [], 'negative_pairs': [] }, @@ -81,14 +82,14 @@ def test_can_save_and_load_with_no_subsets(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='name0/name0_0001', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': ['name0/name0_0002'], 'negative_pairs': [] }, ), DatasetItem(id='name0/name0_0002', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': [], 'negative_pairs': [] }, @@ -101,6 +102,54 @@ def test_can_save_and_load_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', + image=np.ones((2, 5, 3)), + attributes = { + 'positive_pairs': [], + 'negative_pairs': [] + }, + ), + DatasetItem(id='name0/name0_0002', + image=np.ones((2, 5, 3)), + attributes = { + 'positive_pairs': [], + 'negative_pairs': ['кириллица с пробелом'] + }, + ), + ]) + + with TestDir() as test_dir: + LfwConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'lfw') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem(id='name0/name0_0001', image=Image( + path='name0/name0_0001.JPEG', data=np.zeros((4, 3, 3))), + attributes={ + 'positive_pairs': [], + 'negative_pairs': [] + }, + ), + DatasetItem(id='name0/name0_0002', image=Image( + path='name0/name0_0002.bmp', data=np.zeros((3, 4, 3))), + attributes={ + 'positive_pairs': ['name0/name0_0001'], + 'negative_pairs': [] + }, + ), + ]) + + with TestDir() as test_dir: + LfwConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'lfw') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'lfw_dataset') class LfwImporterTest(TestCase): @@ -111,7 +160,7 @@ def test_can_import(self): expected_dataset = Dataset.from_iterable([ DatasetItem(id='name0/name0_0001', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': [], 'negative_pairs': ['name1/name1_0001', 'name1/name1_0002'] @@ -122,7 +171,7 @@ def test_can_import(self): ), DatasetItem(id='name1/name1_0001', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': ['name1/name1_0002'], 'negative_pairs': [] }, @@ -132,7 +181,7 @@ def test_can_import(self): ), DatasetItem(id='name1/name1_0002', subset='test', image=np.ones((2, 5, 3)), - attributes = { + attributes={ 'positive_pairs': [], 'negative_pairs': [] }, diff --git a/tests/test_market1501_format.py b/tests/test_market1501_format.py index d5422acc09..9eaaa30fcf 100644 --- a/tests/test_market1501_format.py +++ b/tests/test_market1501_format.py @@ -6,6 +6,7 @@ from datumaro.components.extractor import DatasetItem from datumaro.plugins.market1501_format import (Market1501Converter, Market1501Importer) +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -62,6 +63,25 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', + image=np.ones((2, 5, 3)), + attributes = { + 'camera_id': 1, + 'person_id': 1, + 'query': True + } + ), + ]) + + with TestDir() as test_dir: + Market1501Converter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'market1501') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + def test_can_save_dataset_with_no_save_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='0001_c2s3_000001_00', @@ -88,6 +108,52 @@ def test_can_save_dataset_with_no_save_images(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem(id='q/1', image=Image( + path='q/1.JPEG', data=np.zeros((4, 3, 3))), + attributes={ + 'camera_id': 1, + 'person_id': 1, + 'query': False + }), + DatasetItem(id='a/b/c/2', image=Image( + path='a/b/c/2.bmp', data=np.zeros((3, 4, 3))), + attributes={ + 'camera_id': 1, + 'person_id': 2, + 'query': True + }), + ]) + + with TestDir() as test_dir: + Market1501Converter.convert(expected, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'market1501') + + compare_datasets(self, expected, parsed_dataset, + require_images=True) + + def test_can_save_dataset_with_no_attributes(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='test1', + subset='test', image=np.ones((2, 5, 3)), + ), + DatasetItem(id='test2', + subset='test', image=np.ones((2, 5, 3)), + attributes={ + 'camera_id': 1, + 'person_id': -1, + 'query': True + } + ), + ]) + + with TestDir() as test_dir: + Market1501Converter.convert(source_dataset, test_dir, save_images=False) + parsed_dataset = Dataset.import_from(test_dir, 'market1501') + + compare_datasets(self, source_dataset, parsed_dataset) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'market1501_dataset') class Market1501ImporterTest(TestCase): diff --git a/tests/test_mot_format.py b/tests/test_mot_format.py index 259bde298a..e5757d3cd7 100644 --- a/tests/test_mot_format.py +++ b/tests/test_mot_format.py @@ -8,16 +8,17 @@ AnnotationType, Bbox, LabelCategories ) from datumaro.plugins.mot_format import MotSeqGtConverter, MotSeqImporter +from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) class MotConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='mot_seq', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_bboxes(self): source_dataset = Dataset.from_iterable([ @@ -93,11 +94,31 @@ def test_can_save_bboxes(self): }) with TestDir() as test_dir: - self._test_save_and_load( - source_dataset, + self._test_save_and_load(source_dataset, partial(MotSeqGtConverter.convert, save_images=True), - test_dir, target_dataset=target_dataset) + test_dir, target_dataset=target_dataset, require_images=True) + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem('1', image=Image( + path='1.JPEG', data=np.zeros((4, 3, 3))), + annotations=[ + Bbox(0, 4, 4, 8, label=0, attributes={ + 'occluded': True, + 'visibility': 0.0, + 'ignored': False, + }), + ] + ), + DatasetItem('2', image=Image( + path='2.bmp', data=np.zeros((3, 4, 3))), + ), + ], categories=['a']) + + with TestDir() as test_dir: + self._test_save_and_load(expected, + partial(MotSeqGtConverter.convert, save_images=True), + test_dir, require_images=True) DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'mot_dataset') diff --git a/tests/test_mots_format.py b/tests/test_mots_format.py index f8358dda3c..534c01b016 100644 --- a/tests/test_mots_format.py +++ b/tests/test_mots_format.py @@ -7,6 +7,7 @@ from datumaro.components.extractor import DatasetItem, Mask from datumaro.components.dataset import Dataset from datumaro.plugins.mots_format import MotsPngConverter, MotsImporter +from datumaro.util.image import Image from datumaro.util.test_utils import (TestDir, compare_datasets, test_save_and_load) @@ -15,10 +16,10 @@ class MotsPngConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='mots', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_masks(self): source = Dataset.from_iterable([ @@ -66,6 +67,43 @@ def test_can_save_masks(self): partial(MotsPngConverter.convert, save_images=True), test_dir, target_dataset=target) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', subset='a', + image=np.ones((5, 1)), annotations=[ + Mask(np.array([[1, 0, 0, 0, 0]]), label=0, + attributes={'track_id': 2}), + ]), + ], categories=['a']) + + with TestDir() as test_dir: + self._test_save_and_load(source, + partial(MotsPngConverter.convert, save_images=True), + test_dir, require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + expected = Dataset.from_iterable([ + DatasetItem('q/1', image=Image( + path='q/1.JPEG', data=np.zeros((4, 3, 3))), + annotations=[ + Mask(np.array([[0, 1, 0, 0, 0]]), label=0, + attributes={'track_id': 1}), + ] + ), + DatasetItem('a/b/c/2', image=Image( + path='a/b/c/2.bmp', data=np.zeros((3, 4, 3))), + annotations=[ + Mask(np.array([[0, 1, 0, 0, 0]]), label=0, + attributes={'track_id': 1}), + ] + ), + ], categories=['a']) + + with TestDir() as test_dir: + self._test_save_and_load(expected, + partial(MotsPngConverter.convert, save_images=True), + test_dir, require_images=True) + class MotsImporterTest(TestCase): def test_can_detect(self): self.assertTrue(MotsImporter.detect(DUMMY_DATASET_DIR)) diff --git a/tests/test_ndr.py b/tests/test_ndr.py index 1e0195bc3a..cdc32fde73 100644 --- a/tests/test_ndr.py +++ b/tests/test_ndr.py @@ -5,15 +5,21 @@ from datumaro.components.project import Dataset from datumaro.components.extractor import (DatasetItem, Label, LabelCategories, AnnotationType) -from datumaro.util.image import Image import datumaro.plugins.ndr as ndr class NDRTest(TestCase): - def _generate_classification_dataset(self, config, num_duplicate): + def _generate_dataset(self, config, num_duplicate, dataset='classification'): subsets = ["train", "val", "test"] - dummy_images = [np.random.randint(0, 255, size=(224, 224, 3)) - for _ in range(num_duplicate)] + if dataset=='classification': + dummy_images = [np.random.randint(0, 255, size=(224, 224, 3)) + for _ in range(num_duplicate)] + if dataset=='invalid_channel': + dummy_images = [np.random.randint(0, 255, size=(224, 224, 2)) + for _ in range(num_duplicate)] + if dataset=='invalid_dimension': + dummy_images = [np.random.randint(0, 255, size=(224, 224, 3, 3)) + for _ in range(num_duplicate)] iterable = [] label_cat = LabelCategories() idx = 0 @@ -24,15 +30,9 @@ def _generate_classification_dataset(self, config, num_duplicate): for _ in range(num_item): idx += 1 iterable.append( - DatasetItem( - idx, - subset=subset, - annotations=[ - Label( - label_id - ) - ], - image=Image(data=dummy_images[idx % num_duplicate]), + DatasetItem(idx, subset=subset, + annotations=[Label(label_id)], + image=dummy_images[idx % num_duplicate], ) ) categories = {AnnotationType.label: label_cat} @@ -48,44 +48,54 @@ def test_ndr_with_error(self): # train : 300, val : 300, test : 300 np.random.seed(1234) with self.assertRaisesRegex(ValueError, "Invalid working_subset name"): - source = self._generate_classification_dataset(config, 3) + source = self._generate_dataset(config, 3) subset = "no_such_subset" result = ndr.NDR(source, working_subset=subset) len(result) with self.assertRaisesRegex(ValueError, "working_subset == duplicated_subset"): - source = self._generate_classification_dataset(config, 3) + source = self._generate_dataset(config, 3) result = ndr.NDR(source, working_subset="train", duplicated_subset="train") len(result) - with self.assertRaisesRegex(ValueError, "Invalid algorithm name"): - source = self._generate_classification_dataset(config, 3) + with self.assertRaisesRegex(ValueError, "Unknown algorithm"): + source = self._generate_dataset(config, 3) algorithm = "no_such_algo" result = ndr.NDR(source, working_subset="train", algorithm=algorithm) len(result) with self.assertRaisesRegex(ValueError, "The number of images is smaller than the cut you want"): - source = self._generate_classification_dataset(config, 3) + source = self._generate_dataset(config, 3) result = ndr.NDR(source, working_subset='train', num_cut=10000) len(result) - with self.assertRaisesRegex(ValueError, "Invalid over_sample"): - source = self._generate_classification_dataset(config, 10) + with self.assertRaisesRegex(ValueError, "Unknown oversampling method"): + source = self._generate_dataset(config, 10) sampling = "no_such_sampling" result = ndr.NDR(source, working_subset='train', num_cut=100, seed=12145, over_sample=sampling) len(result) - with self.assertRaisesRegex(ValueError, "Invalid under_sample"): - source = self._generate_classification_dataset(config, 10) + with self.assertRaisesRegex(ValueError, "Unknown undersampling method"): + source = self._generate_dataset(config, 10) sampling = "no_such_sampling" result = ndr.NDR(source, working_subset='train', num_cut=1, seed=12145, under_sample=sampling) len(result) + with self.assertRaisesRegex(ValueError, "unexpected number of channels"): + source = self._generate_dataset(config, 10, 'invalid_channel') + result = ndr.NDR(source, working_subset='train') + len(result) + + with self.assertRaisesRegex(ValueError, "unexpected number of dimensions"): + source = self._generate_dataset(config, 10, 'invalid_dimension') + result = ndr.NDR(source, working_subset='train') + len(result) + def test_ndr_without_cut(self): config = { "label1": 100, @@ -94,12 +104,12 @@ def test_ndr_without_cut(self): } # train : 300, val : 300, test : 300 np.random.seed(1234) - source = self._generate_classification_dataset(config, 10) + source = self._generate_dataset(config, 10) result = ndr.NDR(source, working_subset='train', seed=12145) - self.assertEqual(2, len(result.get_subset("train"))) - self.assertEqual(298, len(result.get_subset("duplicated"))) + self.assertEqual(1, len(result.get_subset("train"))) + self.assertEqual(299, len(result.get_subset("duplicated"))) self.assertEqual(300, len(result.get_subset("val"))) self.assertEqual(300, len(result.get_subset("test"))) # Check source @@ -107,7 +117,7 @@ def test_ndr_without_cut(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) - def test_ndr_under_sample(self): + def test_ndr_can_use_undersample_uniform(self): config = { "label1": 100, "label2": 100, @@ -115,7 +125,7 @@ def test_ndr_under_sample(self): } # train : 300, val : 300, test : 300 np.random.seed(1234) - source = self._generate_classification_dataset(config, 10) + source = self._generate_dataset(config, 10) result = ndr.NDR(source, working_subset='train', num_cut=1, under_sample='uniform', seed=12145) @@ -129,6 +139,16 @@ def test_ndr_under_sample(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + def test_ndr_can_use_undersample_inverse(self): + config = { + "label1": 100, + "label2": 100, + "label3": 100 + } + # train : 300, val : 300, test : 300 + np.random.seed(1234) + source = self._generate_dataset(config, 10) + result = ndr.NDR(source, working_subset='train', num_cut=1, under_sample='inverse', seed=12145) @@ -141,7 +161,7 @@ def test_ndr_under_sample(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) - def test_ndr_over_sample(self): + def test_ndr_can_use_oversample_random(self): config = { "label1": 100, "label2": 100, @@ -149,7 +169,7 @@ def test_ndr_over_sample(self): } # train : 300, val : 300, test : 300 np.random.seed(1234) - source = self._generate_classification_dataset(config, 10) + source = self._generate_dataset(config, 10) result = ndr.NDR(source, working_subset='train', num_cut=10, over_sample='random', seed=12145) @@ -163,6 +183,16 @@ def test_ndr_over_sample(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) + def test_ndr_can_use_oversample_similarity(self): + config = { + "label1": 100, + "label2": 100, + "label3": 100 + } + # train : 300, val : 300, test : 300 + np.random.seed(1234) + source = self._generate_dataset(config, 10) + result = ndr.NDR(source, working_subset='train', num_cut=10, over_sample='similarity', seed=12145) @@ -175,25 +205,45 @@ def test_ndr_over_sample(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) - def test_ndr_gradient_specific(self): - config = { - "label1": 100, - "label2": 100, - "label3": 100 - } - # train : 300, val : 300, test : 300 - np.random.seed(1234) - source = self._generate_classification_dataset(config, 10) + def test_ndr_gradient_fails_on_invalid_parameters(self): + source = self._generate_dataset({ 'label1': 5 }, 10) + with self.assertRaisesRegex(ValueError, "Invalid block_shape"): result = ndr.NDR(source, working_subset='train', over_sample='random', - block_shape=(3, 6, 6), seed=12145) + block_shape=(3, 6, 6), algorithm='gradient') len(result) with self.assertRaisesRegex(ValueError, "block_shape should be positive"): result = ndr.NDR(source, working_subset='train', over_sample='random', - block_shape=(-1, 0), seed=12145) + block_shape=(-1, 0), algorithm='gradient') + len(result) + + with self.assertRaisesRegex(ValueError, + "sim_threshold should be large than 0"): + result = ndr.NDR(source, working_subset='train', over_sample='random', + sim_threshold=0, block_shape=(8, 8), algorithm='gradient') + len(result) + + with self.assertRaisesRegex(ValueError, + "hash_dim should be smaller than feature shape"): + result = ndr.NDR(source, working_subset='train', over_sample='random', + hash_dim=1024, block_shape=(8, 8), algorithm='gradient') len(result) + with self.assertRaisesRegex(ValueError, "hash_dim should be positive"): + result = ndr.NDR(source, working_subset='train', over_sample='random', + hash_dim=-5, block_shape=(8, 8), algorithm='gradient') + len(result) + + def test_ndr_gradient_can_use_block(self): + config = { + "label1": 100, + "label2": 100, + "label3": 100 + } + # train : 300, val : 300, test : 300 + np.random.seed(1234) + source = self._generate_dataset(config, 10) result = ndr.NDR(source, working_subset='train', over_sample='random', block_shape=(8, 8), seed=12145) @@ -206,22 +256,21 @@ def test_ndr_gradient_specific(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) - with self.assertRaisesRegex(ValueError, - "hash_dim should be smaller than feature shape"): - result = ndr.NDR(source, working_subset='train', over_sample='random', - hash_dim=1024, block_shape=(8, 8), seed=12145) - len(result) - - with self.assertRaisesRegex(ValueError, "hash_dim should be positive"): - result = ndr.NDR(source, working_subset='train', over_sample='random', - hash_dim=-5, block_shape=(8, 8), seed=12145) - len(result) + def test_ndr_gradient_can_use_hash_dim(self): + config = { + "label1": 100, + "label2": 100, + "label3": 100 + } + # train : 300, val : 300, test : 300 + np.random.seed(1234) + source = self._generate_dataset(config, 10) result = ndr.NDR(source, working_subset='train', over_sample='random', hash_dim=16, seed=12145) - self.assertEqual(2, len(result.get_subset("train"))) - self.assertEqual(298, len(result.get_subset("duplicated"))) + self.assertEqual(1, len(result.get_subset("train"))) + self.assertEqual(299, len(result.get_subset("duplicated"))) self.assertEqual(300, len(result.get_subset("val"))) self.assertEqual(300, len(result.get_subset("test"))) # Check source @@ -229,17 +278,21 @@ def test_ndr_gradient_specific(self): self.assertEqual(300, len(source.get_subset("val"))) self.assertEqual(300, len(source.get_subset("test"))) - with self.assertRaisesRegex(ValueError, - "sim_threshold should be large than 0"): - result = ndr.NDR(source, working_subset='train', over_sample='random', - sim_threshold=0, block_shape=(8, 8), seed=12145) - len(result) + def test_ndr_gradient_can_use_sim_thresh(self): + config = { + "label1": 100, + "label2": 100, + "label3": 100 + } + # train : 300, val : 300, test : 300 + np.random.seed(1234) + source = self._generate_dataset(config, 10) result = ndr.NDR(source, working_subset='train', over_sample='random', sim_threshold=0.7, seed=12145) - self.assertEqual(2, len(result.get_subset("train"))) - self.assertEqual(298, len(result.get_subset("duplicated"))) + self.assertEqual(1, len(result.get_subset("train"))) + self.assertEqual(299, len(result.get_subset("duplicated"))) self.assertEqual(300, len(result.get_subset("val"))) self.assertEqual(300, len(result.get_subset("test"))) # Check source @@ -255,7 +308,7 @@ def test_ndr_seed(self): } # train : 300, val : 300, test : 300 np.random.seed(1234) - source = self._generate_classification_dataset(config, 10) + source = self._generate_dataset(config, 10) result1 = ndr.NDR(source, working_subset="train", seed=12345) result2 = ndr.NDR(source, working_subset="train", seed=12345) diff --git a/tests/test_sampler.py b/tests/test_sampler.py new file mode 100644 index 0000000000..3f3d316d28 --- /dev/null +++ b/tests/test_sampler.py @@ -0,0 +1,1101 @@ +from collections import defaultdict +from unittest import TestCase, skipIf + +from datumaro.components.project import Dataset +from datumaro.components.extractor import ( + DatasetItem, + Label, + LabelCategories, + AnnotationType, +) +from datumaro.util.image import Image + +import csv + +try: + import pandas as pd + from datumaro.plugins.sampler.sampler import Sampler + from datumaro.plugins.sampler.algorithm.entropy import SampleEntropy as entropy + has_libs = True +except ImportError: + has_libs = False + + +@skipIf(not has_libs, "pandas library is not available") +class SamplerTest(TestCase): + @staticmethod + def _get_probs(out_range=False): + probs = [] + inference_file = "tests/assets/sampler/inference.csv" + with open(inference_file) as csv_file: + csv_reader = csv.reader(csv_file) + col = 0 + for row in csv_reader: + if col == 0: + col += 1 + continue + else: + if out_range: + probs.append(list(map(lambda x: -float(x), row[1:4]))) + else: + probs.append(list(map(float, row[1:4]))) + return probs + + def _generate_classification_dataset(self, config, subset=None, + empty_scores=False, out_range=False, no_attr=False, no_img=False): + probs = self._get_probs(out_range) + if subset is None: + self.subset = ["train", "val", "test"] + else: + self.subset = subset + + iterable = [] + label_cat = LabelCategories() + idx = 0 + for label_id, label in enumerate(config.keys()): + num_item = config[label] + label_cat.add(label, attributes=None) + for _ in range(num_item): + scores = probs[idx] + idx += 1 + if empty_scores: + scores = [] + attr = {"scores": scores} + if no_attr: + attr = {} + img = Image(path=f"test/dataset/{idx}.jpg", size=(90, 90)) + if no_img: + img = None + iterable.append( + DatasetItem( + idx, + subset=self.subset[idx % len(self.subset)], + annotations=[ + Label( + label_id, + attributes=attr, + ) + ], + image=img, + ) + ) + categories = {AnnotationType.label: label_cat} + dataset = Dataset.from_iterable(iterable, categories) + return dataset + + def test_sampler_get_sample_classification(self): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config, ["train"]) + num_pre_train_subset = len(source.get_subset("train")) + + num_sample = 5 + + with self.subTest("Top-K method"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + self.assertEqual(num_sample, len(result.get_subset("sample"))) + self.assertEqual( + len(result.get_subset("unsampled")), + num_pre_train_subset - len(result.get_subset("sample")), + ) + topk_expected_result = [1, 4, 9, 10, 26] + topk_result = list(map(int, result.result["ImageID"].to_list())) + self.assertEqual(sorted(topk_result), topk_expected_result) + + with self.subTest("Low-K method"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="lowk", + count=num_sample, + output_file=None, + ) + self.assertEqual(num_sample, len(result.get_subset("sample"))) + self.assertEqual( + len(result.get_subset("unsampled")), + num_pre_train_subset - len(result.get_subset("sample")), + ) + lowk_expected_result = [2, 6, 14, 21, 23] + lowk_result = list(map(int, result.result["ImageID"].to_list())) + self.assertEqual(sorted(lowk_result), lowk_expected_result) + + with self.subTest("Rand-K method"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="randk", + count=num_sample, + output_file=None, + ) + self.assertEqual(num_sample, len(result.get_subset("sample"))) + self.assertEqual( + len(result.get_subset("unsampled")), + num_pre_train_subset - len(result.get_subset("sample")), + ) + + with self.subTest("Mix-K method"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="mixk", + count=num_sample, + output_file=None, + ) + self.assertEqual(num_sample, len(result.get_subset("sample"))) + self.assertEqual( + len(result.get_subset("unsampled")), + num_pre_train_subset - len(result.get_subset("sample")), + ) + mixk_expected_result = [2, 4, 10, 23, 26] + mixk_result = list(map(int, result.result["ImageID"].to_list())) + self.assertEqual(sorted(mixk_result), mixk_expected_result) + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="mixk", + count=6, + output_file=None, + ) + self.assertEqual(6, len(result.get_subset("sample"))) + self.assertEqual( + len(result.get_subset("unsampled")), + num_pre_train_subset - len(result.get_subset("sample")), + ) + mixk_expected_result = [2, 4, 6, 10, 23, 26] + mixk_result = list(map(int, result.result["ImageID"].to_list())) + self.assertEqual(sorted(mixk_result), mixk_expected_result) + + with self.subTest("Randtop-K method"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="randtopk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(num_sample, len(result.get_subset("sample"))) + self.assertEqual( + len(result.get_subset("unsampled")), + num_pre_train_subset - len(result.get_subset("sample")), + ) + + def test_sampler_gives_error(self): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + num_sample = 5 + + source = self._generate_classification_dataset(config) + + with self.subTest("Not found"): + with self.assertRaisesRegex(Exception, "Unknown subset"): + subset = "hello" + result = Sampler( + source, + algorithm="entropy", + input_subset=subset, + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + result = iter(result) + next(result) + + with self.assertRaisesRegex(Exception, "Unknown algorithm"): + algorithm = "hello" + result = Sampler( + source, + algorithm=algorithm, + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + result = iter(result) + next(result) + + with self.assertRaisesRegex(Exception, "Unknown sampling method"): + sampling_method = "hello" + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + result = iter(result) + next(result) + + with self.subTest("Invalid Value"): + with self.assertRaisesRegex(Exception, "Invalid value"): + k = 0 + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=k, + output_file=None, + ) + result = iter(result) + next(result) + + with self.assertRaisesRegex(Exception, "Invalid value"): + k = -1 + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=k, + output_file=None, + ) + result = iter(result) + next(result) + + with self.assertRaisesRegex(Exception, "Invalid value"): + k = "string" + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=k, + output_file=None, + ) + result = iter(result) + next(result) + + with self.assertRaisesRegex(Exception, "extension"): + output_file = "string.xml" + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=num_sample, + output_file=output_file, + ) + result = iter(result) + next(result) + + with self.assertRaisesRegex( + Exception, "Invalid Data, ImageID not found in data" + ): + sub = source.get_subset("train") + + data_df = defaultdict(list) + infer_df = defaultdict(list) + + for data in sub: + width, height = data.image.size + data_df["Width"].append(width) + data_df["Height"].append(height) + data_df["ImagePath"].append(data.image.path) + + for annotation in data.annotations: + probs = annotation.attributes["scores"] + infer_df["ImageID"].append(data.id) + + for prob_idx, prob in enumerate(probs): + infer_df[f"ClassProbability{prob_idx+1}"].append(prob) + + data_df = pd.DataFrame(data_df) + infer_df = pd.DataFrame(infer_df) + + entropy(data_df, infer_df) + + with self.assertRaisesRegex( + Exception, "Invalid Data, ImageID not found in inference" + ): + sub = source.get_subset("train") + + data_df = defaultdict(list) + infer_df = defaultdict(list) + + for data in sub: + width, height = data.image.size + data_df["ImageID"].append(data.id) + data_df["Width"].append(width) + data_df["Height"].append(height) + data_df["ImagePath"].append(data.image.path) + + for annotation in data.annotations: + probs = annotation.attributes["scores"] + + for prob_idx, prob in enumerate(probs): + infer_df[f"ClassProbability{prob_idx+1}"].append(prob) + + data_df = pd.DataFrame(data_df) + infer_df = pd.DataFrame(infer_df) + + entropy(data_df, infer_df) + + def test_sampler_get_invalid_data(self): + with self.subTest("empty dataset"): + config = { + "label1": 0, + "label2": 0, + "label3": 0, + } + + source = self._generate_classification_dataset(config) + with self.assertRaisesRegex(Exception, "Unknown subset"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=5, + output_file=None, + ) + result = iter(result) + next(result) + + with self.subTest("Dataset without Scores (Probability)"): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config, empty_scores=True) + with self.assertRaisesRegex(Exception, "ClassProbability"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=5, + output_file=None, + ) + result = iter(result) + next(result) + + with self.subTest("Out of range, probability (Less than 0 or more than 1)"): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset( + config, empty_scores=False, out_range=True + ) + with self.assertRaisesRegex(Exception, "Invalid data"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=5, + output_file=None, + ) + result = iter(result) + next(result) + + with self.subTest("No Scores Attribute Data"): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config, no_attr=True) + with self.assertRaisesRegex(Exception, "does not have 'scores'"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=5, + output_file=None, + ) + result = iter(result) + next(result) + + with self.subTest("No Image Data"): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config, no_img=True) + with self.assertRaisesRegex(Exception, "does not have image info"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method="topk", + count=5, + output_file=None, + ) + result = iter(result) + next(result) + + def test_sampler_number_of_samples(self): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config) + num_pre_train_subset = len(source.get_subset("train")) + + with self.subTest("k > num of data with top-k"): + num_sample = 500 + sampling_method = "topk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k > num of data with low-k"): + num_sample = 500 + sampling_method = "lowk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k > num of data with rand-k"): + num_sample = 500 + sampling_method = "randk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k > num of data with mix-k"): + num_sample = 500 + sampling_method = "mixk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k > num of data with randtop-k"): + num_sample = 500 + sampling_method = "randtopk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k == num of data with top-k"): + num_sample = 10 + sampling_method = "topk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k == num of data with low-k"): + num_sample = 10 + sampling_method = "lowk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k == num of data with rand-k"): + num_sample = 10 + sampling_method = "randk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k == num of data with mix-k"): + num_sample = 10 + sampling_method = "mixk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + with self.subTest("k == num of data with randtop-k"): + num_sample = 10 + sampling_method = "randtopk" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(num_pre_train_subset, len(result.get_subset("sample"))) + + num_sample = 9 + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample", + unsampled_subset="unsampled", + sampling_method=sampling_method, + count=num_sample, + output_file=None, + ) + self.assertEqual(len(result.get_subset("sample")), 9) + + def test_sampler_accumulated_sampling(self): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config) + + num_pre_train_subset = len(source.get_subset("train")) + num_pre_val_subset = len(source.get_subset("val")) + num_pre_test_subset = len(source.get_subset("test")) + + with self.subTest("Same Subset, Same number of datas 3times"): + num_sample = 3 + sample_subset_name = "sample" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), num_sample) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), num_sample * 2) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample * 2 + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), num_sample * 3) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample * 3 + ) + + with self.subTest("Same Subset, 2, 3, 4 sampling"): + sample_subset_name = "sample" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=2, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), 2) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 2) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=3, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), 5) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 5) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=4, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), 9) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 9) + + with self.subTest("Different Subset, Same number of datas 3times"): + num_sample = 3 + sample_subset_name = "sample" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), num_sample) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="val", + sampled_subset=sample_subset_name, + unsampled_subset="val", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), num_sample * 2) + self.assertEqual( + len(result.get_subset("val")), num_pre_val_subset - num_sample + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="test", + sampled_subset=sample_subset_name, + unsampled_subset="test", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), num_sample * 3) + self.assertEqual( + len(result.get_subset("test")), num_pre_test_subset - num_sample + ) + + with self.subTest("Different Subset, 2, 3, 4 sampling"): + sample_subset_name = "sample" + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset=sample_subset_name, + unsampled_subset="train", + sampling_method="topk", + count=2, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), 2) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 2) + + result = Sampler( + result, + algorithm="entropy", + input_subset="val", + sampled_subset=sample_subset_name, + unsampled_subset="val", + sampling_method="topk", + count=3, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), 5) + self.assertEqual(len(result.get_subset("val")), num_pre_val_subset - 3) + + result = Sampler( + result, + algorithm="entropy", + input_subset="test", + sampled_subset=sample_subset_name, + unsampled_subset="test", + sampling_method="topk", + count=4, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample")), 9) + self.assertEqual(len(result.get_subset("test")), num_pre_test_subset - 4) + + def test_sampler_unaccumulated_sampling(self): + config = { + "label1": 10, + "label2": 10, + "label3": 10, + } + + source = self._generate_classification_dataset(config) + + num_pre_train_subset = len(source.get_subset("train")) + num_pre_val_subset = len(source.get_subset("val")) + num_pre_test_subset = len(source.get_subset("test")) + + with self.subTest("Same Subset, Same number of datas 3times"): + num_sample = 3 + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample1", + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), num_sample) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset="sample2", + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), num_sample) + self.assertEqual(len(result.get_subset("sample2")), num_sample) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample * 2 + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset="sample3", + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), num_sample) + self.assertEqual(len(result.get_subset("sample2")), num_sample) + self.assertEqual(len(result.get_subset("sample3")), num_sample) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample * 3 + ) + + with self.subTest("Same Subset, 2, 3, 4 sampling"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample1", + unsampled_subset="train", + sampling_method="topk", + count=2, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), 2) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 2) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset="sample2", + unsampled_subset="train", + sampling_method="topk", + count=3, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), 2) + self.assertEqual(len(result.get_subset("sample2")), 3) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 5) + + result = Sampler( + result, + algorithm="entropy", + input_subset="train", + sampled_subset="sample3", + unsampled_subset="train", + sampling_method="topk", + count=4, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), 2) + self.assertEqual(len(result.get_subset("sample2")), 3) + self.assertEqual(len(result.get_subset("sample3")), 4) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 9) + + with self.subTest("Different Subset, Same number of datas 3times"): + num_sample = 3 + + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample1", + unsampled_subset="train", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), num_sample) + self.assertEqual( + len(result.get_subset("train")), num_pre_train_subset - num_sample + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="val", + sampled_subset="sample2", + unsampled_subset="val", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), num_sample) + self.assertEqual(len(result.get_subset("sample2")), num_sample) + self.assertEqual( + len(result.get_subset("val")), num_pre_val_subset - num_sample + ) + + result = Sampler( + result, + algorithm="entropy", + input_subset="test", + sampled_subset="sample3", + unsampled_subset="test", + sampling_method="topk", + count=num_sample, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), num_sample) + self.assertEqual(len(result.get_subset("sample2")), num_sample) + self.assertEqual(len(result.get_subset("sample3")), num_sample) + self.assertEqual( + len(result.get_subset("test")), num_pre_test_subset - num_sample + ) + + with self.subTest("Different Subset, 2, 3, 4 sampling"): + result = Sampler( + source, + algorithm="entropy", + input_subset="train", + sampled_subset="sample1", + unsampled_subset="train", + sampling_method="topk", + count=2, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), 2) + self.assertEqual(len(result.get_subset("train")), num_pre_train_subset - 2) + + result = Sampler( + result, + algorithm="entropy", + input_subset="val", + sampled_subset="sample2", + unsampled_subset="val", + sampling_method="topk", + count=3, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), 2) + self.assertEqual(len(result.get_subset("sample2")), 3) + self.assertEqual(len(result.get_subset("val")), num_pre_val_subset - 3) + + result = Sampler( + result, + algorithm="entropy", + input_subset="test", + sampled_subset="sample3", + unsampled_subset="test", + sampling_method="topk", + count=4, + output_file=None, + ) + + self.assertEqual(len(result.get_subset("sample1")), 2) + self.assertEqual(len(result.get_subset("sample2")), 3) + self.assertEqual(len(result.get_subset("sample3")), 4) + self.assertEqual(len(result.get_subset("test")), num_pre_test_subset - 4) + + def test_sampler_parser(self): + from argparse import ArgumentParser + + assert isinstance(Sampler.build_cmdline_parser(), ArgumentParser) diff --git a/tests/test_tfrecord_format.py b/tests/test_tfrecord_format.py index 8b63c71a1b..6db7c07eb3 100644 --- a/tests/test_tfrecord_format.py +++ b/tests/test_tfrecord_format.py @@ -37,10 +37,10 @@ def test_raises_when_crashes_on_import(self): @skipIf(import_failed, "Failed to import tensorflow") class TfrecordConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='tf_detection_api', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_bboxes(self): test_dataset = Dataset.from_iterable([ @@ -121,15 +121,34 @@ def test_can_save_dataset_with_no_subsets(self): partial(TfDetectionApiConverter.convert, save_images=True), test_dir) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + test_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', + image=np.ones((16, 16, 3)), + annotations=[ + Bbox(2, 1, 4, 4, label=2), + Bbox(4, 2, 8, 4, label=3), + ], + attributes={'source_id': ''} + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label_' + str(label) for label in range(10)), + }) + + with TestDir() as test_dir: + self._test_save_and_load( + test_dataset, + partial(TfDetectionApiConverter.convert, save_images=True), + test_dir) + def test_can_save_dataset_with_image_info(self): test_dataset = Dataset.from_iterable([ DatasetItem(id='1/q.e', image=Image(path='1/q.e', size=(10, 15)), attributes={'source_id': ''} ) - ], categories={ - AnnotationType.label: LabelCategories(), - }) + ], categories=[]) with TestDir() as test_dir: self._test_save_and_load(test_dataset, @@ -147,12 +166,27 @@ def test_can_save_dataset_with_unknown_image_formats(self): ext='qwe'), attributes={'source_id': ''} ) - ], categories={ AnnotationType.label: LabelCategories(), }) + ], categories=[]) with TestDir() as test_dir: self._test_save_and_load(test_dataset, partial(TfDetectionApiConverter.convert, save_images=True), - test_dir) + test_dir, require_images=True) + + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', subset='train', + image=Image(path='q/1.JPEG', data=np.zeros((4, 3, 3))), + attributes={'source_id': ''}), + DatasetItem('a/b/c/2', subset='valid', + image=Image(path='a/b/c/2.bmp', data=np.zeros((3, 4, 3))), + attributes={'source_id': ''}), + ], categories=[]) + + with TestDir() as test_dir: + self._test_save_and_load(dataset, + partial(TfDetectionApiConverter.convert, save_images=True), + test_dir, require_images=True) def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: diff --git a/tests/test_validator.py b/tests/test_validator.py new file mode 100644 index 0000000000..5c348103df --- /dev/null +++ b/tests/test_validator.py @@ -0,0 +1,518 @@ +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT + +from collections import Counter +from unittest import TestCase +import numpy as np + +from datumaro.components.dataset import Dataset, DatasetItem +from datumaro.components.errors import (MissingLabelCategories, + MissingLabelAnnotation, MultiLabelAnnotations, MissingAttribute, + UndefinedLabel, UndefinedAttribute, LabelDefinedButNotFound, + AttributeDefinedButNotFound, OnlyOneLabel, FewSamplesInLabel, + FewSamplesInAttribute, ImbalancedLabels, ImbalancedAttribute, + ImbalancedBboxDistInLabel, ImbalancedBboxDistInAttribute, + MissingBboxAnnotation, NegativeLength, InvalidValue, FarFromLabelMean, + FarFromAttrMean, OnlyOneAttributeValue) +from datumaro.components.extractor import Bbox, Label +from datumaro.components.validator import (ClassificationValidator, + DetectionValidator, TaskType, validate_annotations, _Validator) + +class TestValidatorTemplate(TestCase): + @classmethod + def setUpClass(cls): + cls.dataset = Dataset.from_iterable([ + DatasetItem(id=1, image=np.ones((5, 5, 3)), annotations=[ + Label(1, id=0, attributes={ 'a': 1, 'b': 7, }), + Bbox(1, 2, 3, 4, id=1, label=0, attributes={ + 'a': 1, 'b': 2, + }), + ]), + DatasetItem(id=2, image=np.ones((2, 4, 3)), annotations=[ + Label(2, id=0, attributes={ 'a': 2, 'b': 2, }), + Bbox(2, 3, 1, 4, id=1, label=0, attributes={ + 'a': 1, 'b': 1, + }), + ]), + DatasetItem(id=3), + DatasetItem(id=4, image=np.ones((2, 4, 3)), annotations=[ + Label(0, id=0, attributes={ 'b': 4, }), + Label(1, id=1, attributes={ 'a': 11, 'b': 7, }), + Bbox(1, 3, 2, 4, id=2, label=0, attributes={ + 'a': 2, 'b': 1, + }), + Bbox(3, 1, 4, 2, id=3, label=0, attributes={ + 'a': 2, 'b': 2, + }), + ]), + DatasetItem(id=5, image=np.ones((2, 4, 3)), annotations=[ + Label(0, id=0, attributes={ 'a': 20, 'b': 10 }), + Bbox(1, 2, 3, 4, id=1, label=1, attributes={ + 'a': 1, 'b': 1, + }), + ]), + DatasetItem(id=6, image=np.ones((2, 4, 3)), annotations=[ + Label(1, id=0, attributes={ 'a': 11, 'b': 2, 'c': 3}), + Bbox(2, 3, 4, 1, id=1, label=1, attributes={ + 'a': 2, 'b': 2, + }), + ]), + DatasetItem(id=7, image=np.ones((2, 4, 3)), annotations=[ + Label(1, id=0, attributes={ 'a': 1, 'b': 2, 'c': 5 }), + Bbox(1, 2, 3, 4, id=1, label=2, attributes={ + 'a': 1, 'b': 2, + }), + ]), + DatasetItem(id=8, image=np.ones((2, 4, 3)), annotations=[ + Label(2, id=0, attributes={ 'a': 7, 'b': 9, 'c': 5 }), + Bbox(2, 1, 3, 4, id=1, label=2, attributes={ + 'a': 2, 'b': 1, + }), + ]), + ], categories=[[f'label_{i}', None, { 'a', 'b' }] \ + for i in range(2)]) + + +class TestBaseValidator(TestValidatorTemplate): + @classmethod + def setUpClass(cls): + cls.validator = _Validator(TaskType.classification) + + def test_generate_reports(self): + with self.assertRaises(NotImplementedError): + self.validator.generate_reports({}) + + def test_check_missing_label_categories(self): + stats = { + 'label_distribution': { + 'defined_labels': {} + } + } + + actual_reports = self.validator._check_missing_label_categories(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], MissingLabelCategories) + + def test_check_missing_attribute(self): + label_name = 'unit' + attr_name = 'test' + attr_dets = { + 'items_missing_attribute': [(1, 'unittest')] + } + + actual_reports = self.validator._check_missing_attribute( + label_name, attr_name, attr_dets) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], MissingAttribute) + + def test_check_undefined_label(self): + label_name = 'unittest' + label_stats = { + 'items_with_undefined_label': [(1, 'unittest')] + } + + actual_reports = self.validator._check_undefined_label( + label_name, label_stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], UndefinedLabel) + + def test_check_undefined_attribute(self): + label_name = 'unit' + attr_name = 'test' + attr_dets = { + 'items_with_undefined_attr':[(1, 'unittest')] + } + + actual_reports = self.validator._check_undefined_attribute( + label_name, attr_name, attr_dets) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], UndefinedAttribute) + + def test_check_label_defined_but_not_found(self): + stats = { + 'label_distribution': { + 'defined_labels': { + 'unittest': 0 + } + } + } + + actual_reports = self.validator._check_label_defined_but_not_found( + stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], LabelDefinedButNotFound) + + def test_check_attribute_defined_but_not_found(self): + label_name = 'unit' + attr_stats = { + 'test': { + 'distribution': {} + } + } + + actual_reports = self.validator._check_attribute_defined_but_not_found( + label_name, attr_stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], AttributeDefinedButNotFound) + + def test_check_only_one_label(self): + stats = { + 'label_distribution': { + 'defined_labels': { + 'unit': 1, + 'test': 0 + } + } + } + + actual_reports = self.validator._check_only_one_label(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], OnlyOneLabel) + + def test_check_only_one_attribute_value(self): + label_name = 'unit' + attr_name = 'test' + attr_dets = { + 'distribution': { + 'mock': 1 + } + } + + actual_reports = self.validator._check_only_one_attribute_value( + label_name, attr_name, attr_dets) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], OnlyOneAttributeValue) + + def test_check_few_samples_in_label(self): + stats = { + 'label_distribution': { + 'defined_labels': { + 'unit': 1 + } + } + } + + actual_reports = self.validator._check_few_samples_in_label(stats, 2) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], FewSamplesInLabel) + + def test_check_few_samples_in_attribute(self): + label_name = 'unit' + attr_name = 'test' + attr_dets = { + 'distribution': { + 'mock': 1 + } + } + + actual_reports = self.validator._check_few_samples_in_attribute( + label_name, attr_name, attr_dets, 2) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], FewSamplesInAttribute) + + def test_check_imbalanced_labels(self): + stats = { + 'label_distribution': { + 'defined_labels': { + 'unit': 5, + 'test': 1 + } + } + } + + actual_reports = self.validator._check_imbalanced_labels(stats, 2) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], ImbalancedLabels) + + def test_check_imbalanced_attribute(self): + label_name = 'unit' + attr_name = 'test' + attr_dets = { + 'distribution': { + 'mock': 5, + 'mock_1': 1 + } + } + + actual_reports = self.validator._check_imbalanced_attribute( + label_name, attr_name, attr_dets, 2) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], ImbalancedAttribute) + + +class TestClassificationValidator(TestValidatorTemplate): + @classmethod + def setUpClass(cls): + cls.validator = ClassificationValidator() + + def test_check_missing_label_annotation(self): + stats = { + 'items_missing_label': [(1, 'unittest')] + } + + actual_reports = self.validator._check_missing_label_annotation(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], MissingLabelAnnotation) + + def test_check_multi_label_annotations(self): + stats = { + 'items_with_multiple_labels': [(1, 'unittest')] + } + + actual_reports = self.validator._check_multi_label_annotations(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], MultiLabelAnnotations) + + +class TestDetectionValidator(TestValidatorTemplate): + @classmethod + def setUpClass(cls): + cls.validator = DetectionValidator() + + def test_check_imbalanced_bbox_dist_in_label(self): + label_name = 'unittest' + bbox_label_stats = { + 'x': { + 'histogram': { + 'counts': [1] + } + } + } + + actual_reports = self.validator._check_imbalanced_bbox_dist_in_label( + label_name, bbox_label_stats, 0.9, 1) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], ImbalancedBboxDistInLabel) + + def test_check_imbalanced_bbox_dist_in_attr(self): + label_name = 'unit' + attr_name = 'test' + bbox_attr_stats = { + 'mock': { + 'x': { + 'histogram': { + 'counts': [1] + } + } + } + } + + actual_reports = self.validator._check_imbalanced_bbox_dist_in_attr( + label_name, attr_name, bbox_attr_stats, 0.9, 1) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], ImbalancedBboxDistInAttribute) + + def test_check_missing_bbox_annotation(self): + stats = { + 'items_missing_bbox': [(1, 'unittest')] + } + + actual_reports = self.validator._check_missing_bbox_annotation(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], MissingBboxAnnotation) + + def test_check_negative_length(self): + stats = { + 'items_with_negative_length': { + ('1', 'unittest'): { + 1: { + 'x': -1 + } + } + } + } + + actual_reports = self.validator._check_negative_length(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], NegativeLength) + + def test_check_invalid_value(self): + stats = { + 'items_with_invalid_value': { + ('1', 'unittest'): { + 1: ['x'] + } + } + } + + actual_reports = self.validator._check_invalid_value(stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], InvalidValue) + + + def test_check_far_from_label_mean(self): + label_name = 'unittest' + bbox_label_stats = { + 'w': { + 'items_far_from_mean': { + ('1', 'unittest'): { + 1: 100 + } + }, + 'mean': 0, + } + } + + actual_reports = self.validator._check_far_from_label_mean( + label_name, bbox_label_stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], FarFromLabelMean) + + def test_check_far_from_attr_mean(self): + label_name = 'unit' + attr_name = 'test' + bbox_attr_stats = { + 'mock': { + 'w': { + 'items_far_from_mean': { + ('1', 'unittest'): { + 1: 100 + } + }, + 'mean': 0, + } + } + } + + actual_reports = self.validator._check_far_from_attr_mean( + label_name, attr_name, bbox_attr_stats) + + self.assertTrue(len(actual_reports) == 1) + self.assertIsInstance(actual_reports[0], FarFromAttrMean) + + +class TestValidateAnnotations(TestValidatorTemplate): + def test_validate_annotations_classification(self): + actual_results = validate_annotations(self.dataset, 'classification') + + with self.subTest('Test of statistics', i = 0): + actual_stats = actual_results['statistics'] + self.assertEqual(actual_stats['total_label_count'], 8) + self.assertEqual(len(actual_stats['items_missing_label']), 1) + self.assertEqual(len(actual_stats['items_with_multiple_labels']), 1) + + label_dist = actual_stats['label_distribution'] + defined_label_dist = label_dist['defined_labels'] + self.assertEqual(len(defined_label_dist), 2) + self.assertEqual(sum(defined_label_dist.values()), 6) + + undefined_label_dist = label_dist['undefined_labels'] + undefined_label_stats = undefined_label_dist[2] + self.assertEqual(len(undefined_label_dist), 1) + self.assertEqual(undefined_label_stats['count'], 2) + self.assertEqual( + len(undefined_label_stats['items_with_undefined_label']), 2) + + attr_stats = actual_stats['attribute_distribution'] + defined_attr_dets = attr_stats['defined_attributes']['label_0']['a'] + self.assertEqual( + len(defined_attr_dets['items_missing_attribute']), 1) + self.assertEqual(defined_attr_dets['distribution'], {'20': 1}) + + undefined_attr_dets = attr_stats['undefined_attributes'][2]['c'] + self.assertEqual( + len(undefined_attr_dets['items_with_undefined_attr']), 1) + self.assertEqual(undefined_attr_dets['distribution'], {'5': 1}) + + with self.subTest('Test of validation reports', i = 1): + actual_reports = actual_results['validation_reports'] + report_types = [r['anomaly_type'] for r in actual_reports] + report_count_by_type = Counter(report_types) + + self.assertEqual(len(actual_reports), 16) + self.assertEqual(report_count_by_type['UndefinedAttribute'], 7) + self.assertEqual(report_count_by_type['FewSamplesInAttribute'], 3) + self.assertEqual(report_count_by_type['UndefinedLabel'], 2) + self.assertEqual(report_count_by_type['MissingLabelAnnotation'], 1) + self.assertEqual(report_count_by_type['MultiLabelAnnotations'], 1) + self.assertEqual(report_count_by_type['OnlyOneAttributeValue'], 1) + self.assertEqual(report_count_by_type['MissingAttribute'], 1) + + with self.subTest('Test of summary', i = 2): + actual_summary = actual_results['summary'] + expected_summary = { + 'errors': 10, + 'warnings': 6 + } + + self.assertEqual(actual_summary, expected_summary) + + def test_validate_annotations_detection(self): + actual_results = validate_annotations(self.dataset, 'detection') + + with self.subTest('Test of statistics', i = 0): + actual_stats = actual_results['statistics'] + self.assertEqual(actual_stats['total_bbox_count'], 8) + self.assertEqual(len(actual_stats['items_missing_bbox']), 1) + self.assertEqual(actual_stats['items_with_negative_length'], {}) + self.assertEqual(actual_stats['items_with_invalid_value'], {}) + + bbox_dist_by_label = actual_stats['bbox_distribution_in_label'] + label_prop_stats = bbox_dist_by_label['label_1']['width'] + self.assertEqual(label_prop_stats['items_far_from_mean'], {}) + self.assertEqual(label_prop_stats['mean'], 3.5) + self.assertEqual(label_prop_stats['stdev'], 0.5) + self.assertEqual(label_prop_stats['min'], 3.0) + self.assertEqual(label_prop_stats['max'], 4.0) + self.assertEqual(label_prop_stats['median'], 3.5) + + bbox_dist_by_attr = actual_stats['bbox_distribution_in_attribute'] + attr_prop_stats = bbox_dist_by_attr['label_0']['a']['1']['width'] + self.assertEqual(attr_prop_stats['items_far_from_mean'], {}) + self.assertEqual(attr_prop_stats['mean'], 2.0) + self.assertEqual(attr_prop_stats['stdev'], 1.0) + self.assertEqual(attr_prop_stats['min'], 1.0) + self.assertEqual(attr_prop_stats['max'], 3.0) + self.assertEqual(attr_prop_stats['median'], 2.0) + + bbox_dist_item = actual_stats['bbox_distribution_in_dataset_item'] + self.assertEqual(sum(bbox_dist_item.values()), 8) + + with self.subTest('Test of validation reports', i = 1): + actual_reports = actual_results['validation_reports'] + report_types = [r['anomaly_type'] for r in actual_reports] + report_count_by_type = Counter(report_types) + + self.assertEqual(len(actual_reports), 11) + self.assertEqual(report_count_by_type['FewSamplesInAttribute'], 4) + self.assertEqual(report_count_by_type['UndefinedAttribute'], 4) + self.assertEqual(report_count_by_type['UndefinedLabel'], 2) + self.assertEqual(report_count_by_type['MissingBboxAnnotation'], 1) + + with self.subTest('Test of summary', i = 2): + actual_summary = actual_results['summary'] + expected_summary = { + 'errors': 6, + 'warnings': 5 + } + + self.assertEqual(actual_summary, expected_summary) + + def test_validate_annotations_invalid_task_type(self): + with self.assertRaises(ValueError): + validate_annotations(self.dataset, 'INVALID') + + def test_validate_annotations_invalid_dataset_type(self): + with self.assertRaises(TypeError): + validate_annotations(object(), 'classification') diff --git a/tests/test_vgg_face2_format.py b/tests/test_vgg_face2_format.py index 38eb7aacdf..d6d232a921 100644 --- a/tests/test_vgg_face2_format.py +++ b/tests/test_vgg_face2_format.py @@ -7,6 +7,7 @@ Label, LabelCategories, Points) from datumaro.plugins.vgg_face2_format import (VggFace2Converter, VggFace2Importer) +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -71,6 +72,23 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', image=np.ones((8, 8, 3)), + annotations=[ + Points([4.23, 4.32, 5.34, 4.45, 3.54, + 3.56, 4.52, 3.51, 4.78, 3.34], label=0), + ] + ), + ], categories=['a']) + + with TestDir() as test_dir: + VggFace2Converter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'vgg_face2') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + def test_can_save_dataset_with_no_save_images(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', image=np.ones((8, 8, 3)), @@ -131,6 +149,26 @@ def test_can_save_dataset_with_wrong_number_of_points(self): compare_datasets(self, target_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem('a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3))), + annotations=[ + Bbox(0, 2, 4, 2, label=0), + Points([4.23, 4.32, 5.34, 4.45, 3.54, + 3.56, 4.52, 3.51, 4.78, 3.34], label=0), + ] + ), + ], categories=['a']) + + with TestDir() as test_dir: + VggFace2Converter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'vgg_face2') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'vgg_face2_dataset') class VggFace2ImporterTest(TestCase): diff --git a/tests/test_voc_format.py b/tests/test_voc_format.py index fddafd6470..6e3a2db0a5 100644 --- a/tests/test_voc_format.py +++ b/tests/test_voc_format.py @@ -132,10 +132,10 @@ def test_can_detect_voc(self): class VocConverterTest(TestCase): def _test_save_and_load(self, source_dataset, converter, test_dir, - target_dataset=None, importer_args=None): + target_dataset=None, importer_args=None, **kwargs): return test_save_and_load(self, source_dataset, converter, test_dir, importer='voc', - target_dataset=target_dataset, importer_args=importer_args) + target_dataset=target_dataset, importer_args=importer_args, **kwargs) def test_can_save_voc_cls(self): class TestExtractor(TestExtractorBase): @@ -401,19 +401,31 @@ def test_can_save_dataset_with_no_subsets(self): class TestExtractor(TestExtractorBase): def __iter__(self): return iter([ - DatasetItem(id=1, annotations=[ - Label(2), - Label(3), - ]), + DatasetItem(id=1), + DatasetItem(id=2), + ]) - DatasetItem(id=2, annotations=[ - Label(3), - ]), + for task in [None] + list(VOC.VocTask): + with self.subTest(subformat=task), TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc', tasks=task), + test_dir) + + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='кириллица с пробелом 1'), + DatasetItem(id='кириллица с пробелом 2', + image=np.ones([4, 5, 3])), ]) - with TestDir() as test_dir: - self._test_save_and_load(TestExtractor(), - partial(VocConverter.convert, label_map='voc'), test_dir) + for task in [None] + list(VOC.VocTask): + with self.subTest(subformat=task), TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc', tasks=task, + save_images=True), + test_dir, require_images=True) def test_can_save_dataset_with_images(self): class TestExtractor(TestExtractorBase): @@ -430,7 +442,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(VocConverter.convert, label_map='voc', save_images=True, tasks=task), - test_dir) + test_dir, require_images=True) def test_dataset_with_voc_labelmap(self): class SrcExtractor(TestExtractorBase): @@ -629,6 +641,23 @@ def __iter__(self): partial(VocConverter.convert, label_map='voc', tasks=task), test_dir) + def test_can_save_and_load_image_with_arbitrary_extension(self): + class TestExtractor(TestExtractorBase): + def __iter__(self): + return iter([ + DatasetItem(id='q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem(id='a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3)))), + ]) + + for task in [None] + list(VOC.VocTask): + with self.subTest(subformat=task), TestDir() as test_dir: + self._test_save_and_load(TestExtractor(), + partial(VocConverter.convert, label_map='voc', tasks=task, + save_images=True), + test_dir, require_images=True) + def test_relative_paths(self): class TestExtractor(TestExtractorBase): def __iter__(self): @@ -643,7 +672,7 @@ def __iter__(self): self._test_save_and_load(TestExtractor(), partial(VocConverter.convert, label_map='voc', save_images=True, tasks=task), - test_dir) + test_dir, require_images=True) def test_can_save_attributes(self): class TestExtractor(TestExtractorBase): diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py index 03f15e623c..0465f5d3f3 100644 --- a/tests/test_widerface_format.py +++ b/tests/test_widerface_format.py @@ -6,6 +6,7 @@ Label, LabelCategories) from datumaro.components.dataset import Dataset from datumaro.plugins.widerface_format import WiderFaceConverter, WiderFaceImporter +from datumaro.util.image import Image from datumaro.util.test_utils import TestDir, compare_datasets @@ -15,7 +16,7 @@ def test_can_save_and_load(self): DatasetItem(id='1', subset='train', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes = { + Bbox(0, 1, 2, 3, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), Label(0), @@ -23,13 +24,13 @@ def test_can_save_and_load(self): ), DatasetItem(id='2', subset='train', image=np.ones((10, 10, 3)), annotations=[ - Bbox(0, 2, 4, 2, attributes = { + Bbox(0, 2, 4, 2, attributes={ 'blur': '2', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '1', 'invalid': '0'}), - Bbox(3, 3, 2, 3, attributes = { + Bbox(3, 3, 2, 3, attributes={ 'blur': '0', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), - Bbox(2, 1, 2, 3, attributes = { + Bbox(2, 1, 2, 3, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '0', 'invalid': '1'}), Label(1), @@ -38,13 +39,13 @@ def test_can_save_and_load(self): DatasetItem(id='3', subset='val', image=np.ones((8, 8, 3)), annotations=[ - Bbox(0, 1.1, 5.3, 2.1, attributes = { + Bbox(0, 1.1, 5.3, 2.1, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), - Bbox(0, 2, 3, 2, attributes = { + Bbox(0, 2, 3, 2, attributes={ 'occluded': 'False'}), Bbox(0, 2, 4, 2), - Bbox(0, 7, 3, 2, attributes = { + Bbox(0, 7, 3, 2, attributes={ 'blur': '2', 'expression': '1', 'illumination': '0', 'occluded': '0', 'pose': '1', 'invalid': '0'}), ] @@ -67,7 +68,7 @@ def test_can_save_dataset_with_no_subsets(self): DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2, label=2), - Bbox(0, 1, 2, 3, label=1, attributes = { + Bbox(0, 1, 2, 3, label=1, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), ] @@ -83,15 +84,36 @@ def test_can_save_dataset_with_no_subsets(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', image=np.ones((8, 8, 3)), + annotations=[ + Bbox(0, 1, 2, 3, label=1, attributes = { + 'blur': '2', 'expression': '0', 'illumination': '0', + 'occluded': '0', 'pose': '2', 'invalid': '0'}), + ] + ), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label_' + str(i) for i in range(3)), + }) + + with TestDir() as test_dir: + WiderFaceConverter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'wider_face') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + def test_can_save_dataset_with_non_widerface_attributes(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes = { + Bbox(0, 1, 2, 3, attributes={ 'non-widerface attribute': '0', 'blur': 1, 'invalid': '1'}), - Bbox(1, 1, 2, 2, attributes = { + Bbox(1, 1, 2, 2, attributes={ 'non-widerface attribute': '0'}), ] ), @@ -101,7 +123,7 @@ def test_can_save_dataset_with_non_widerface_attributes(self): DatasetItem(id='a/b/1', image=np.ones((8, 8, 3)), annotations=[ Bbox(0, 2, 4, 2), - Bbox(0, 1, 2, 3, attributes = { + Bbox(0, 1, 2, 3, attributes={ 'blur': '1', 'invalid': '1'}), Bbox(1, 1, 2, 2), ] @@ -114,6 +136,20 @@ def test_can_save_dataset_with_non_widerface_attributes(self): compare_datasets(self, target_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', image=Image(path='q/1.JPEG', + data=np.zeros((4, 3, 3)))), + DatasetItem('a/b/c/2', image=Image(path='a/b/c/2.bmp', + data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + WiderFaceConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'wider_face') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'widerface_dataset') class WiderFaceImporterTest(TestCase): @@ -125,7 +161,7 @@ def test_can_import(self): DatasetItem(id='0_Parade_image_01', subset='train', image=np.ones((10, 15, 3)), annotations=[ - Bbox(1, 2, 2, 2, attributes = { + Bbox(1, 2, 2, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '0', 'invalid': '0'}), Label(0), @@ -134,10 +170,10 @@ def test_can_import(self): DatasetItem(id='1_Handshaking_image_02', subset='train', image=np.ones((10, 15, 3)), annotations=[ - Bbox(1, 1, 2, 2, attributes = { + Bbox(1, 1, 2, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '0', 'invalid': '0'}), - Bbox(5, 1, 2, 2, attributes = { + Bbox(5, 1, 2, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '1', 'occluded': '0', 'pose': '0', 'invalid': '0'}), Label(1), @@ -146,13 +182,13 @@ def test_can_import(self): DatasetItem(id='0_Parade_image_03', subset='val', image=np.ones((10, 15, 3)), annotations=[ - Bbox(0, 0, 1, 1, attributes = { + Bbox(0, 0, 1, 1, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), - Bbox(3, 2, 1, 2, attributes = { + Bbox(3, 2, 1, 2, attributes={ 'blur': '0', 'expression': '0', 'illumination': '0', 'occluded': '1', 'pose': '0', 'invalid': '0'}), - Bbox(5, 6, 1, 1, attributes = { + Bbox(5, 6, 1, 1, attributes={ 'blur': '2', 'expression': '0', 'illumination': '0', 'occluded': '0', 'pose': '2', 'invalid': '0'}), Label(0), diff --git a/tests/test_yolo_format.py b/tests/test_yolo_format.py index 5c46eb27a0..5449ba6626 100644 --- a/tests/test_yolo_format.py +++ b/tests/test_yolo_format.py @@ -90,6 +90,25 @@ def test_can_load_dataset_with_exact_image_info(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_dataset_with_cyrillic_and_spaces_in_filename(self): + source_dataset = Dataset.from_iterable([ + DatasetItem(id='кириллица с пробелом', subset='train', image=np.ones((8, 8, 3)), + annotations=[ + Bbox(0, 2, 4, 2, label=2), + Bbox(0, 1, 2, 3, label=4), + ]), + ], categories={ + AnnotationType.label: LabelCategories.from_iterable( + 'label_' + str(i) for i in range(10)), + }) + + with TestDir() as test_dir: + YoloConverter.convert(source_dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'yolo') + + compare_datasets(self, source_dataset, parsed_dataset, + require_images=True) + def test_relative_paths(self): source_dataset = Dataset.from_iterable([ DatasetItem(id='1', subset='train', @@ -98,9 +117,7 @@ def test_relative_paths(self): image=np.ones((2, 6, 3))), DatasetItem(id='subdir2/1', subset='train', image=np.ones((5, 4, 3))), - ], categories={ - AnnotationType.label: LabelCategories(), - }) + ], categories=[]) for save_images in {True, False}: with self.subTest(save_images=save_images): @@ -111,6 +128,20 @@ def test_relative_paths(self): compare_datasets(self, source_dataset, parsed_dataset) + def test_can_save_and_load_image_with_arbitrary_extension(self): + dataset = Dataset.from_iterable([ + DatasetItem('q/1', subset='train', + image=Image(path='q/1.JPEG', data=np.zeros((4, 3, 3)))), + DatasetItem('a/b/c/2', subset='valid', + image=Image(path='a/b/c/2.bmp', data=np.zeros((3, 4, 3)))), + ], categories=[]) + + with TestDir() as test_dir: + YoloConverter.convert(dataset, test_dir, save_images=True) + parsed_dataset = Dataset.import_from(test_dir, 'yolo') + + compare_datasets(self, dataset, parsed_dataset, require_images=True) + def test_inplace_save_writes_only_updated_data(self): with TestDir() as path: # generate initial dataset