From e37e65774e554d9a234eb3f1447fd744ba098389 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Thu, 10 Feb 2022 19:54:18 +0300 Subject: [PATCH 1/8] add scannet instance dataset with metrics --- mmdet3d/core/evaluation/__init__.py | 3 +- mmdet3d/core/evaluation/instance_seg_eval.py | 105 +++++++ .../evaluate_semantic_instance.py | 296 ++++++++++++++++++ .../core/evaluation/scannet_utils/util_3d.py | 64 ++++ mmdet3d/datasets/__init__.py | 15 +- mmdet3d/datasets/scannet_dataset.py | 148 ++++++++- .../test_datasets/test_scannet_dataset.py | 152 ++++++++- tests/test_metrics/test_instance_seg_eval.py | 75 +++++ 8 files changed, 848 insertions(+), 10 deletions(-) create mode 100644 mmdet3d/core/evaluation/instance_seg_eval.py create mode 100644 mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py create mode 100644 mmdet3d/core/evaluation/scannet_utils/util_3d.py create mode 100644 tests/test_metrics/test_instance_seg_eval.py diff --git a/mmdet3d/core/evaluation/__init__.py b/mmdet3d/core/evaluation/__init__.py index 877458a05c..b1d489f36c 100644 --- a/mmdet3d/core/evaluation/__init__.py +++ b/mmdet3d/core/evaluation/__init__.py @@ -1,10 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. from .indoor_eval import indoor_eval +from .instance_seg_eval import instance_seg_eval from .kitti_utils import kitti_eval, kitti_eval_coco_style from .lyft_eval import lyft_eval from .seg_eval import seg_eval __all__ = [ 'kitti_eval_coco_style', 'kitti_eval', 'indoor_eval', 'lyft_eval', - 'seg_eval' + 'seg_eval', 'instance_seg_eval' ] diff --git a/mmdet3d/core/evaluation/instance_seg_eval.py b/mmdet3d/core/evaluation/instance_seg_eval.py new file mode 100644 index 0000000000..facb80d732 --- /dev/null +++ b/mmdet3d/core/evaluation/instance_seg_eval.py @@ -0,0 +1,105 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +from mmcv.utils import print_log +from terminaltables import AsciiTable + +from .scannet_utils.evaluate_semantic_instance import scannet_eval + + +def aggregate_predictions(masks, labels, scores, valid_class_ids): + infos = [] + for id, (mask, label, score) in enumerate(zip(masks, labels, scores)): + mask = mask.clone().numpy() + label = label.clone().numpy() + score = score.clone().numpy() + info = dict() + n_instances = mask.max() + 1 + for i in range(n_instances): + # match pred_instance['filename'] from assign_instances_for_scan + file_name = f'{id}_{i}' + info[file_name] = dict() + info[file_name]['mask'] = (mask == i).astype(np.int) + info[file_name]['label_id'] = valid_class_ids[label[i]] + info[file_name]['conf'] = score[i] + infos.append(info) + return infos + + +def rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids): + renamed_instance_masks = [] + for semantic_mask, instance_mask in zip(gt_semantic_masks, + gt_instance_masks): + semantic_mask = semantic_mask.clone().numpy() + instance_mask = instance_mask.clone().numpy() + unique = np.unique(instance_mask) + assert len(unique) < 1000 + for i in unique: + semantic_instance = semantic_mask[instance_mask == i] + semantic_unique = np.unique(semantic_instance) + assert len(semantic_unique) == 1 + if semantic_unique[0] < len(valid_class_ids): + instance_mask[ + instance_mask == + i] = 1000 * valid_class_ids[semantic_unique[0]] + i + renamed_instance_masks.append(instance_mask) + return renamed_instance_masks + + +def instance_seg_eval(gt_semantic_masks, + gt_instance_masks, + pred_instance_masks, + pred_instance_labels, + pred_instance_scores, + valid_class_ids, + class_labels, + options=None, + logger=None): + """Instance Segmentation Evaluation. + + Evaluate the result of the instance segmentation. + + Args: + gt_semantic_masks (list[torch.Tensor]): Ground truth semantic masks. + gt_instance_masks (list[torch.Tensor]): Ground truth instance masks. + pred_instance_masks (list[torch.Tensor]): Predicted instance masks. + pred_instance_labels (list[torch.Tensor]): Predicted instance labels. + pred_instance_scores (list[torch.Tensor]): Predicted instance labels. + valid_class_ids (tuple[int]): Ids of valid categories. + class_labels (tuple[str]): Names of valid categories. + options (dict): Additional options. Keys may contain: `overlaps`, + `min_region_sizes`, `distance_threshes`, `distance_confs`. + logger (logging.Logger | str, optional): The way to print the mAP + summary. See `mmdet.utils.print_log()` for details. Default: None. + + Returns: + dict[str, float]: Dict of results. + """ + assert len(valid_class_ids) == len(class_labels) + id_to_label = { + valid_class_ids[i]: class_labels[i] + for i in range(len(valid_class_ids)) + } + preds = aggregate_predictions( + masks=pred_instance_masks, + labels=pred_instance_labels, + scores=pred_instance_scores, + valid_class_ids=valid_class_ids) + gts = rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids) + metrics = scannet_eval( + preds=preds, + gts=gts, + options=options, + valid_class_ids=valid_class_ids, + class_labels=class_labels, + id_to_label=id_to_label) + header = ['classes', 'AP_0.25', 'AP_0.50', 'AP'] + rows = [] + for label, data in metrics['classes'].items(): + aps = [data['ap25%'], data['ap50%'], data['ap']] + rows.append([label] + [f'{ap:.4f}' for ap in aps]) + aps = metrics['all_ap_25%'], metrics['all_ap_50%'], metrics['all_ap'] + footer = ['Overall'] + [f'{ap:.4f}' for ap in aps] + table = AsciiTable([header] + rows + [footer]) + table.inner_footing_row_border = True + print_log('\n' + table.table, logger=logger) + return metrics diff --git a/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py b/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py new file mode 100644 index 0000000000..93d4515d88 --- /dev/null +++ b/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py @@ -0,0 +1,296 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# adapted from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py # noqa +import numpy as np +from copy import deepcopy + +from . import util_3d + + +def evaluate_matches(matches, class_labels, options): + overlaps = options['overlaps'] + min_region_sizes = [options['min_region_sizes'][0]] + dist_threshes = [options['distance_threshes'][0]] + dist_confs = [options['distance_confs'][0]] + + # results: class x overlap + ap = np.zeros((len(dist_threshes), len(class_labels), len(overlaps)), + np.float) + for di, (min_region_size, distance_thresh, distance_conf) in enumerate( + zip(min_region_sizes, dist_threshes, dist_confs)): + for oi, overlap_th in enumerate(overlaps): + pred_visited = {} + for m in matches: + # (@filaPro:) it is not used + # for p in matches[m]['pred']: + for label_name in class_labels: + for p in matches[m]['pred'][label_name]: + if 'filename' in p: + pred_visited[p['filename']] = False + for li, label_name in enumerate(class_labels): + y_true = np.empty(0) + y_score = np.empty(0) + hard_false_negatives = 0 + has_gt = False + has_pred = False + for m in matches: + pred_instances = matches[m]['pred'][label_name] + gt_instances = matches[m]['gt'][label_name] + # filter groups in ground truth + gt_instances = [ + gt for gt in gt_instances + if gt['instance_id'] >= 1000 and gt['vert_count'] >= + min_region_size and gt['med_dist'] <= distance_thresh + and gt['dist_conf'] >= distance_conf + ] + if gt_instances: + has_gt = True + if pred_instances: + has_pred = True + + cur_true = np.ones(len(gt_instances)) + cur_score = np.ones(len(gt_instances)) * (-float('inf')) + cur_match = np.zeros(len(gt_instances), dtype=np.bool) + # collect matches + for (gti, gt) in enumerate(gt_instances): + found_match = False + # (@filaPro:) it is not used + # num_pred = len(gt['matched_pred']) + for pred in gt['matched_pred']: + # greedy assignments + if pred_visited[pred['filename']]: + continue + overlap = float(pred['intersection']) / ( + gt['vert_count'] + pred['vert_count'] - + pred['intersection']) + if overlap > overlap_th: + confidence = pred['confidence'] + # if already have a prediction for this gt, + # the prediction with the lower score is automatically a false positive # noqa + if cur_match[gti]: + max_score = max(cur_score[gti], confidence) + min_score = min(cur_score[gti], confidence) + cur_score[gti] = max_score + # append false positive + cur_true = np.append(cur_true, 0) + cur_score = np.append(cur_score, min_score) + cur_match = np.append(cur_match, True) + # otherwise set score + else: + found_match = True + cur_match[gti] = True + cur_score[gti] = confidence + pred_visited[pred['filename']] = True + if not found_match: + hard_false_negatives += 1 + # remove non-matched ground truth instances + cur_true = cur_true[cur_match] + cur_score = cur_score[cur_match] + + # collect non-matched predictions as false positive + for pred in pred_instances: + found_gt = False + for gt in pred['matched_gt']: + overlap = float(gt['intersection']) / ( + gt['vert_count'] + pred['vert_count'] - + gt['intersection']) + if overlap > overlap_th: + found_gt = True + break + if not found_gt: + num_ignore = pred['void_intersection'] + for gt in pred['matched_gt']: + # group? + if gt['instance_id'] < 1000: + num_ignore += gt['intersection'] + # small ground truth instances + if gt['vert_count'] < min_region_size or gt[ + 'med_dist'] > distance_thresh or gt[ + 'dist_conf'] < distance_conf: + num_ignore += gt['intersection'] + proportion_ignore = float( + num_ignore) / pred['vert_count'] + # if not ignored append false positive + if proportion_ignore <= overlap_th: + cur_true = np.append(cur_true, 0) + confidence = pred['confidence'] + cur_score = np.append(cur_score, confidence) + + # append to overall results + y_true = np.append(y_true, cur_true) + y_score = np.append(y_score, cur_score) + + # compute average precision + if has_gt and has_pred: + # compute precision recall curve first + + # sorting and cumsum + score_arg_sort = np.argsort(y_score) + y_score_sorted = y_score[score_arg_sort] + y_true_sorted = y_true[score_arg_sort] + y_true_sorted_cumsum = np.cumsum(y_true_sorted) + + # unique thresholds + (thresholds, unique_indices) = np.unique( + y_score_sorted, return_index=True) + num_prec_recall = len(unique_indices) + 1 + + # prepare precision recall + num_examples = len(y_score_sorted) + # (@filaPro:) follow https://github.com/ScanNet/ScanNet/pull/26 ? # noqa + num_true_examples = y_true_sorted_cumsum[-1] if len( + y_true_sorted_cumsum) > 0 else 0 + precision = np.zeros(num_prec_recall) + recall = np.zeros(num_prec_recall) + + # deal with the first point + y_true_sorted_cumsum = np.append(y_true_sorted_cumsum, 0) + # deal with remaining + for idx_res, idx_scores in enumerate(unique_indices): + cumsum = y_true_sorted_cumsum[idx_scores - 1] + tp = num_true_examples - cumsum + fp = num_examples - idx_scores - tp + fn = cumsum + hard_false_negatives + p = float(tp) / (tp + fp) + r = float(tp) / (tp + fn) + precision[idx_res] = p + recall[idx_res] = r + + # first point in curve is artificial + precision[-1] = 1. + recall[-1] = 0. + + # compute average of precision-recall curve + recall_for_conv = np.copy(recall) + recall_for_conv = np.append(recall_for_conv[0], + recall_for_conv) + recall_for_conv = np.append(recall_for_conv, 0.) + + stepWidths = np.convolve(recall_for_conv, [-0.5, 0, 0.5], + 'valid') + # integrate is now simply a dot product + ap_current = np.dot(precision, stepWidths) + + elif has_gt: + ap_current = 0.0 + else: + ap_current = float('nan') + ap[di, li, oi] = ap_current + return ap + + +def compute_averages(aps, options, class_labels): + d_inf = 0 + o50 = np.where(np.isclose(options['overlaps'], 0.5)) + o25 = np.where(np.isclose(options['overlaps'], 0.25)) + oAllBut25 = np.where(np.logical_not(np.isclose(options['overlaps'], 0.25))) + avg_dict = {} + # avg_dict['all_ap'] = np.nanmean(aps[ d_inf, :, :]) + avg_dict['all_ap'] = np.nanmean(aps[d_inf, :, oAllBut25]) + avg_dict['all_ap_50%'] = np.nanmean(aps[d_inf, :, o50]) + avg_dict['all_ap_25%'] = np.nanmean(aps[d_inf, :, o25]) + avg_dict['classes'] = {} + for (li, label_name) in enumerate(class_labels): + avg_dict['classes'][label_name] = {} + # avg_dict['classes'][label_name]['ap'] = np.average(aps[d_inf, li, :]) + avg_dict['classes'][label_name]['ap'] = np.average(aps[d_inf, li, + oAllBut25]) + avg_dict['classes'][label_name]['ap50%'] = np.average(aps[d_inf, li, + o50]) + avg_dict['classes'][label_name]['ap25%'] = np.average(aps[d_inf, li, + o25]) + return avg_dict + + +def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids, + class_labels, id_to_label): + # (@filaPro:) reading gt and pred files is replaced with their data + # get gt instances + gt_instances = util_3d.get_instances(gt_ids, valid_class_ids, class_labels, + id_to_label) + # associate + gt2pred = deepcopy(gt_instances) + for label in gt2pred: + for gt in gt2pred[label]: + gt['matched_pred'] = [] + pred2gt = {} + for label in class_labels: + pred2gt[label] = [] + num_pred_instances = 0 + # mask of void labels in the groundtruth + bool_void = np.logical_not(np.in1d(gt_ids // 1000, valid_class_ids)) + # go through all prediction masks + for pred_mask_file in pred_info: + label_id = int(pred_info[pred_mask_file]['label_id']) + conf = pred_info[pred_mask_file]['conf'] + if not label_id in id_to_label: # noqa E713 + continue + label_name = id_to_label[label_id] + # read the mask + pred_mask = pred_info[pred_mask_file]['mask'] + if len(pred_mask) != len(gt_ids): + raise ValueError('len(pred_mask) != len(gt_ids)') + # convert to binary + pred_mask = np.not_equal(pred_mask, 0) + num = np.count_nonzero(pred_mask) + if num < options['min_region_sizes'][0]: + continue # skip if empty + + pred_instance = {} + pred_instance['filename'] = pred_mask_file + pred_instance['pred_id'] = num_pred_instances + pred_instance['label_id'] = label_id + pred_instance['vert_count'] = num + pred_instance['confidence'] = conf + pred_instance['void_intersection'] = np.count_nonzero( + np.logical_and(bool_void, pred_mask)) + + # matched gt instances + matched_gt = [] + # go thru all gt instances with matching label + for (gt_num, gt_inst) in enumerate(gt2pred[label_name]): + intersection = np.count_nonzero( + np.logical_and(gt_ids == gt_inst['instance_id'], pred_mask)) + if intersection > 0: + gt_copy = gt_inst.copy() + pred_copy = pred_instance.copy() + gt_copy['intersection'] = intersection + pred_copy['intersection'] = intersection + matched_gt.append(gt_copy) + gt2pred[label_name][gt_num]['matched_pred'].append(pred_copy) + pred_instance['matched_gt'] = matched_gt + num_pred_instances += 1 + pred2gt[label_name].append(pred_instance) + + return gt2pred, pred2gt + + +def scannet_eval(preds, gts, options, valid_class_ids, class_labels, + id_to_label): + options = get_options(options) + matches = {} + for i, (pred, gt) in enumerate(zip(preds, gts)): + # (@filaPro:) replace gt filename with index + matches_key = i + # assign gt to predictions + gt2pred, pred2gt = assign_instances_for_scan(pred, gt, options, + valid_class_ids, + class_labels, id_to_label) + matches[matches_key] = {} + matches[matches_key]['gt'] = gt2pred + matches[matches_key]['pred'] = pred2gt + + ap_scores = evaluate_matches(matches, class_labels, options) + avgs = compute_averages(ap_scores, options, class_labels) + return avgs + + +def get_options(options): + assert options is None or isinstance(options, dict) + _options = dict( + overlaps=np.append(np.arange(0.5, 0.95, 0.05), 0.25), + min_region_sizes=np.array([100]), + distance_threshes=np.array([float('inf')]), + distance_confs=np.array([-float('inf')])) + if options is not None: + _options.update(options) + return _options diff --git a/mmdet3d/core/evaluation/scannet_utils/util_3d.py b/mmdet3d/core/evaluation/scannet_utils/util_3d.py new file mode 100644 index 0000000000..4081a59e39 --- /dev/null +++ b/mmdet3d/core/evaluation/scannet_utils/util_3d.py @@ -0,0 +1,64 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# copied with deletions from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/util_3d.py # noqa +import json +import numpy as np + + +class Instance(object): + instance_id = 0 + label_id = 0 + vert_count = 0 + med_dist = -1 + dist_conf = 0.0 + + def __init__(self, mesh_vert_instances, instance_id): + if (instance_id == -1): + return + self.instance_id = int(instance_id) + self.label_id = int(self.get_label_id(instance_id)) + self.vert_count = int( + self.get_instance_verts(mesh_vert_instances, instance_id)) + + def get_label_id(self, instance_id): + return int(instance_id // 1000) + + def get_instance_verts(self, mesh_vert_instances, instance_id): + return (mesh_vert_instances == instance_id).sum() + + def to_json(self): + return json.dumps( + self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + + def to_dict(self): + dict = {} + dict['instance_id'] = self.instance_id + dict['label_id'] = self.label_id + dict['vert_count'] = self.vert_count + dict['med_dist'] = self.med_dist + dict['dist_conf'] = self.dist_conf + return dict + + def from_json(self, data): + self.instance_id = int(data['instance_id']) + self.label_id = int(data['label_id']) + self.vert_count = int(data['vert_count']) + if ('med_dist' in data): + self.med_dist = float(data['med_dist']) + self.dist_conf = float(data['dist_conf']) + + def __str__(self): + return '(' + str(self.instance_id) + ')' + + +def get_instances(ids, class_ids, class_labels, id2label): + instances = {} + for label in class_labels: + instances[label] = [] + instance_ids = np.unique(ids) + for id in instance_ids: + if id == 0: + continue + inst = Instance(ids, id) + if inst.label_id in class_ids: + instances[id2label[inst.label_id]].append(inst.to_dict()) + return instances diff --git a/mmdet3d/datasets/__init__.py b/mmdet3d/datasets/__init__.py index 26ba3d73c9..45d859c39b 100644 --- a/mmdet3d/datasets/__init__.py +++ b/mmdet3d/datasets/__init__.py @@ -21,7 +21,8 @@ VoxelBasedPointSampler) # yapf: enable from .s3dis_dataset import S3DISDataset, S3DISSegDataset -from .scannet_dataset import ScanNetDataset, ScanNetSegDataset +from .scannet_dataset import (ScanNetDataset, ScanNetInstanceSegDataset, + ScanNetSegDataset) from .semantickitti_dataset import SemanticKITTIDataset from .sunrgbd_dataset import SUNRGBDDataset from .utils import get_loading_pipeline @@ -35,10 +36,10 @@ 'LoadPointsFromFile', 'S3DISSegDataset', 'S3DISDataset', 'NormalizePointsColor', 'IndoorPatchPointSample', 'IndoorPointSample', 'PointSample', 'LoadAnnotations3D', 'GlobalAlignment', 'SUNRGBDDataset', - 'ScanNetDataset', 'ScanNetSegDataset', 'SemanticKITTIDataset', - 'Custom3DDataset', 'Custom3DSegDataset', 'LoadPointsFromMultiSweeps', - 'WaymoDataset', 'BackgroundPointsFilter', 'VoxelBasedPointSampler', - 'get_loading_pipeline', 'RandomDropPointsColor', 'RandomJitterPoints', - 'ObjectNameFilter', 'AffineResize', 'RandomShiftScale', - 'LoadPointsFromDict' + 'ScanNetDataset', 'ScanNetSegDataset', 'ScanNetInstanceSegDataset', + 'SemanticKITTIDataset', 'Custom3DDataset', 'Custom3DSegDataset', + 'LoadPointsFromMultiSweeps', 'WaymoDataset', 'BackgroundPointsFilter', + 'VoxelBasedPointSampler', 'get_loading_pipeline', 'RandomDropPointsColor', + 'RandomJitterPoints', 'ObjectNameFilter', 'AffineResize', + 'RandomShiftScale', 'LoadPointsFromDict' ] diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index df03986f27..e638a4552e 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -4,7 +4,7 @@ import warnings from os import path as osp -from mmdet3d.core import show_result, show_seg_result +from mmdet3d.core import instance_seg_eval, show_result, show_seg_result from mmdet3d.core.bbox import DepthInstance3DBoxes from mmdet.datasets import DATASETS from mmseg.datasets import DATASETS as SEG_DATASETS @@ -459,3 +459,149 @@ def format_results(self, results, txtfile_prefix=None): outputs.append(dict(seg_mask=pred_label)) return outputs, tmp_dir + + +@DATASETS.register_module() +@SEG_DATASETS.register_module() +class ScanNetInstanceSegDataset(Custom3DSegDataset): + CLASSES = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door', 'window', + 'bookshelf', 'picture', 'counter', 'desk', 'curtain', + 'refrigerator', 'showercurtrain', 'toilet', 'sink', 'bathtub', + 'garbagebin') + + VALID_CLASS_IDS = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, + 36, 39) + + ALL_CLASS_IDS = tuple(range(41)) + + def get_ann_info(self, index): + """Get annotation info according to the given index. + Args: + index (int): Index of the annotation data to get. + Returns: + dict: annotation information consists of the following keys: + - pts_semantic_mask_path (str): Path of semantic masks. + - pts_instance_mask_path (str): Path of instance masks. + """ + # Use index to get the annos, thus the evalhook could also use this api + info = self.data_infos[index] + + pts_instance_mask_path = osp.join(self.data_root, + info['pts_instance_mask_path']) + pts_semantic_mask_path = osp.join(self.data_root, + info['pts_semantic_mask_path']) + + anns_results = dict( + pts_instance_mask_path=pts_instance_mask_path, + pts_semantic_mask_path=pts_semantic_mask_path) + return anns_results + + def get_classes_and_palette(self, classes=None, palette=None): + """Get class names of current dataset. + + Palette is simply ignored for instance segmentation. + Args: + classes (Sequence[str] | str | None): If classes is None, use + default CLASSES defined by builtin dataset. If classes is a + string, take it as a file name. The file contains the name of + classes where each line contains one class name. If classes is + a tuple or list, override the CLASSES defined by the dataset. + Defaults to None. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, random + palette will be generated. Defaults to None. + """ + if classes is not None: + return classes, None + return self.CLASSES, None + + def _build_default_pipeline(self): + """Build the default pipeline for this dataset.""" + pipeline = [ + dict( + type='LoadPointsFromFile', + coord_type='DEPTH', + shift_height=False, + use_color=True, + load_dim=6, + use_dim=[0, 1, 2, 3, 4, 5]), + dict( + type='LoadAnnotations3D', + with_bbox_3d=False, + with_label_3d=False, + with_mask_3d=True, + with_seg_3d=True), + dict( + type='PointSegClassMapping', + valid_cat_ids=self.VALID_CLASS_IDS, + max_cat_id=40), + dict( + type='DefaultFormatBundle3D', + with_label=False, + class_names=self.CLASSES), + dict( + type='Collect3D', + keys=['points', 'pts_semantic_mask', 'pts_instance_mask']) + ] + return Compose(pipeline) + + def evaluate(self, + results, + metric=None, + options=None, + logger=None, + show=False, + out_dir=None, + pipeline=None): + """Evaluate. + + Evaluation in semantic segmentation protocol. + Args: + results (list[dict]): List of results. + metric (str | list[str]): Metrics to be evaluated. + options (dict, optional): options for instance_seg_eval. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Defaults to None. + show (bool, optional): Whether to visualize. + Defaults to False. + out_dir (str, optional): Path to save the visualization results. + Defaults to None. + pipeline (list[dict], optional): raw data loading for showing. + Default: None. + Returns: + dict: Evaluation results. + """ + assert isinstance( + results, list), f'Expect results to be list, got {type(results)}.' + assert len(results) > 0, 'Expect length of results > 0.' + assert len(results) == len(self.data_infos) + assert isinstance( + results[0], dict + ), f'Expect elements in results to be dict, got {type(results[0])}.' + + load_pipeline = self._get_pipeline(pipeline) + pred_instance_masks = [result['instance_mask'] for result in results] + pred_instance_labels = [result['instance_label'] for result in results] + pred_instance_scores = [result['instance_score'] for result in results] + gt_semantic_masks, gt_instance_masks = zip(*[ + self._extract_data( + index=i, + pipeline=load_pipeline, + key=['pts_semantic_mask', 'pts_instance_mask'], + load_annos=True) for i in range(len(self.data_infos)) + ]) + ret_dict = instance_seg_eval( + gt_semantic_masks, + gt_instance_masks, + pred_instance_masks, + pred_instance_labels, + pred_instance_scores, + valid_class_ids=self.VALID_CLASS_IDS, + class_labels=self.CLASSES, + options=options, + logger=logger) + + if show: + raise NotImplementedError('show is not implemented for now') + + return ret_dict diff --git a/tests/test_data/test_datasets/test_scannet_dataset.py b/tests/test_data/test_datasets/test_scannet_dataset.py index e66f2b0020..de47a24fb2 100644 --- a/tests/test_data/test_datasets/test_scannet_dataset.py +++ b/tests/test_data/test_datasets/test_scannet_dataset.py @@ -4,7 +4,8 @@ import pytest import torch -from mmdet3d.datasets import ScanNetDataset, ScanNetSegDataset +from mmdet3d.datasets import (ScanNetDataset, ScanNetInstanceSegDataset, + ScanNetSegDataset) def test_getitem(): @@ -681,3 +682,152 @@ def test_seg_format_results(): expected_txt_path = osp.join(tmp_dir.name, 'results', 'scene0000_00.txt') assert np.all(result_files[0]['seg_mask'] == expected_label) mmcv.check_file_exist(expected_txt_path) + + +def test_instance_seg_getitem(): + np.random.seed(0) + root_path = './tests/data/scannet/' + ann_file = './tests/data/scannet/scannet_infos.pkl' + class_names = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door', + 'window', 'bookshelf', 'picture', 'counter', 'desk', + 'curtain', 'refrigerator', 'showercurtrain', 'toilet', + 'sink', 'bathtub', 'garbagebin') + train_pipeline = [ + dict( + type='LoadPointsFromFile', + coord_type='DEPTH', + shift_height=False, + use_color=True, + load_dim=6, + use_dim=[0, 1, 2, 3, 4, 5]), + dict( + type='LoadAnnotations3D', + with_bbox_3d=False, + with_label_3d=False, + with_mask_3d=True, + with_seg_3d=True), + dict( + type='PointSegClassMapping', + valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, + 34, 36, 39), + max_cat_id=40), + dict(type='NormalizePointsColor', color_mean=None), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict( + type='Collect3D', + keys=['points', 'pts_semantic_mask', 'pts_instance_mask']) + ] + scannet_dataset = ScanNetInstanceSegDataset( + data_root=root_path, + ann_file=ann_file, + pipeline=train_pipeline, + classes=class_names, + test_mode=False) + expected_points = torch.tensor([[ + -3.4742e+00, 7.8792e-01, 1.7397e+00, 3.3725e-01, 3.5294e-01, 3.0588e-01 + ], [ + 2.7216e+00, 3.4164e+00, 2.4572e+00, 6.6275e-01, 6.2745e-01, 5.1373e-01 + ], + [ + 1.3404e+00, -1.4675e+00, -4.4059e-02, + 3.8431e-01, 3.6078e-01, 3.5686e-01 + ], + [ + -3.0335e+00, 2.7273e+00, 1.5181e+00, + 2.3137e-01, 1.6078e-01, 8.2353e-02 + ], + [ + -4.3207e-01, 1.8154e+00, 1.7455e-01, + 4.0392e-01, 3.8039e-01, 4.1961e-01 + ]]) + expected_semantic_mask = torch.tensor([11, 18, 18, 0, 4]).long() + expected_instance_mask = torch.tensor([6, 56, 10, 9, 35]).long() + + data = scannet_dataset[0] + assert torch.allclose(data['points']._data[:5], expected_points, 1e-2) + assert torch.allclose(data['pts_semantic_mask']._data[:5], + expected_semantic_mask) + assert torch.allclose(data['pts_instance_mask']._data[:5], + expected_instance_mask) + + +def test_instance_seg_evaluate(): + root_path = './tests/data/scannet' + ann_file = './tests/data/scannet/scannet_infos.pkl' + class_names = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door', + 'window', 'bookshelf', 'picture', 'counter', 'desk', + 'curtain', 'refrigerator', 'showercurtrain', 'toilet', + 'sink', 'bathtub', 'garbagebin') + test_pipeline = [ + dict( + type='LoadPointsFromFile', + coord_type='DEPTH', + shift_height=False, + use_color=True, + load_dim=6, + use_dim=[0, 1, 2, 3, 4, 5]), + dict(type='NormalizePointsColor', color_mean=None), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict(type='Collect3D', keys=['points']) + ] + scannet_dataset = ScanNetInstanceSegDataset( + data_root=root_path, + ann_file=ann_file, + pipeline=test_pipeline, + test_mode=True) + + pred_mask = torch.tensor([ + 1, -1, -1, -1, 7, 11, 2, -1, 1, 10, -1, -1, 5, -1, -1, -1, -1, 1, -1, + -1, -1, -1, 0, -1, 1, -1, 12, -1, -1, -1, 8, 5, 1, 5, 2, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 8, -1, -1, -1, + 0, 4, 3, -1, 9, -1, -1, 6, -1, -1, -1, -1, 13, -1, -1, 5, -1, 5, -1, + -1, 9, 0, 5, -1, -1, 2, 3, 4, -1, -1, -1, 2, -1, -1, -1, 5, 9, -1, 1, + -1, 4, 10, 4, -1 + ]).long() + pred_labels = torch.tensor( + [4, 11, 11, 10, 0, 3, 12, 4, 14, 1, 0, 0, 0, 5, 5]).long() + pred_scores = torch.tensor([.99 for _ in range(len(pred_labels))]) + results = [ + dict( + instance_mask=pred_mask, + instance_label=pred_labels, + instance_score=torch.tensor(pred_scores)) + ] + eval_pipeline = [ + dict( + type='LoadPointsFromFile', + coord_type='DEPTH', + shift_height=False, + use_color=True, + load_dim=6, + use_dim=[0, 1, 2, 3, 4, 5]), + dict( + type='LoadAnnotations3D', + with_bbox_3d=False, + with_label_3d=False, + with_mask_3d=True, + with_seg_3d=True), + dict( + type='PointSegClassMapping', + valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, + 34, 36, 39), + max_cat_id=40), + dict(type='NormalizePointsColor', color_mean=None), + dict(type='DefaultFormatBundle3D', class_names=class_names), + dict( + type='Collect3D', + keys=['points', 'pts_semantic_mask', 'pts_instance_mask']) + ] + # We add options here as default min_region_size + # is much bigger than test instances. + ret_dict = scannet_dataset.evaluate( + results, + pipeline=eval_pipeline, + options=dict(min_region_sizes=np.array([1]))) + assert abs(ret_dict['all_ap'] - 0.90625) < 0.001 + assert abs(ret_dict['all_ap_50%'] - 0.90625) < 0.001 + assert abs(ret_dict['all_ap_25%'] - 0.94444) < 0.001 + assert abs(ret_dict['classes']['cabinet']['ap25%'] - 1.0) < 0.001 + assert abs(ret_dict['classes']['cabinet']['ap50%'] - 0.65625) < 0.001 + assert abs(ret_dict['classes']['door']['ap25%'] - 0.5) < 0.001 + assert abs(ret_dict['classes']['door']['ap50%'] - 0.5) < 0.001 diff --git a/tests/test_metrics/test_instance_seg_eval.py b/tests/test_metrics/test_instance_seg_eval.py new file mode 100644 index 0000000000..a73483a18a --- /dev/null +++ b/tests/test_metrics/test_instance_seg_eval.py @@ -0,0 +1,75 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch + +from mmdet3d.core import instance_seg_eval + + +def test_instance_seg_eval(): + valid_class_ids = (3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, + 36, 39) + class_labels = ('cabinet', 'bed', 'chair', 'sofa', 'table', 'door', + 'window', 'bookshelf', 'picture', 'counter', 'desk', + 'curtain', 'refrigerator', 'showercurtrain', 'toilet', + 'sink', 'bathtub', 'garbagebin') + n_points_list = [3300, 3000] + gt_labels_list = [[0, 0, 0, 0, 0, 0, 14, 14, 2, 1], + [13, 13, 2, 1, 3, 3, 0, 0, 0]] + gt_instance_masks = [] + gt_semantic_masks = [] + pred_instance_masks = [] + pred_instance_labels = [] + pred_instance_scores = [] + for n_points, gt_labels in zip(n_points_list, gt_labels_list): + gt_instance_mask = np.ones(n_points, dtype=np.int) * -1 + gt_semantic_mask = np.ones(n_points, dtype=np.int) * -1 + pred_instance_mask = np.ones(n_points, dtype=np.int) * -1 + labels = [] + scores = [] + for i, gt_label in enumerate(gt_labels): + begin = i * 300 + end = begin + 300 + gt_instance_mask[begin:end] = i + gt_semantic_mask[begin:end] = gt_label + pred_instance_mask[begin:end] = i + labels.append(gt_label) + scores.append(.99) + gt_instance_masks.append(torch.tensor(gt_instance_mask)) + gt_semantic_masks.append(torch.tensor(gt_semantic_mask)) + pred_instance_masks.append(torch.tensor(pred_instance_mask)) + pred_instance_labels.append(torch.tensor(labels)) + pred_instance_scores.append(torch.tensor(scores)) + + ret_value = instance_seg_eval( + gt_semantic_masks=gt_semantic_masks, + gt_instance_masks=gt_instance_masks, + pred_instance_masks=pred_instance_masks, + pred_instance_labels=pred_instance_labels, + pred_instance_scores=pred_instance_scores, + valid_class_ids=valid_class_ids, + class_labels=class_labels) + for label in [ + 'cabinet', 'bed', 'chair', 'sofa', 'showercurtrain', 'toilet' + ]: + metrics = ret_value['classes'][label] + assert metrics['ap'] == 1.0 + assert metrics['ap50%'] == 1.0 + assert metrics['ap25%'] == 1.0 + + pred_instance_masks[1][2240:2700] = -1 + pred_instance_masks[0][2700:3000] = 8 + pred_instance_labels[0][9] = 2 + ret_value = instance_seg_eval( + gt_semantic_masks=gt_semantic_masks, + gt_instance_masks=gt_instance_masks, + pred_instance_masks=pred_instance_masks, + pred_instance_labels=pred_instance_labels, + pred_instance_scores=pred_instance_scores, + valid_class_ids=valid_class_ids, + class_labels=class_labels) + assert abs(ret_value['classes']['cabinet']['ap50%'] - 0.72916) < 0.01 + assert abs(ret_value['classes']['cabinet']['ap25%'] - 0.88888) < 0.01 + assert abs(ret_value['classes']['bed']['ap50%'] - 0.5) < 0.01 + assert abs(ret_value['classes']['bed']['ap25%'] - 0.5) < 0.01 + assert abs(ret_value['classes']['chair']['ap50%'] - 0.375) < 0.01 + assert abs(ret_value['classes']['chair']['ap25%'] - 1.0) < 0.01 From 91a2845b880fa8e589fbe1ee03ad0e0619b082bc Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Thu, 10 Feb 2022 20:06:19 +0300 Subject: [PATCH 2/8] fix import --- mmdet3d/datasets/scannet_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index efb4d9e6f1..186469e190 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -5,7 +5,7 @@ import numpy as np -from mmdet3d.core import show_result, show_seg_result, show_seg_result +from mmdet3d.core import show_result, show_seg_result from mmdet3d.core.bbox import DepthInstance3DBoxes from mmdet.datasets import DATASETS from mmseg.datasets import DATASETS as SEG_DATASETS From c5dd33a4f76bfc00f88c5105d2827fbbb6853ff6 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Thu, 10 Feb 2022 20:09:28 +0300 Subject: [PATCH 3/8] add missing import --- mmdet3d/datasets/scannet_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index 186469e190..1b3734d94d 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -5,7 +5,7 @@ import numpy as np -from mmdet3d.core import show_result, show_seg_result +from mmdet3d.core import show_result, show_seg_result, instance_seg_eval from mmdet3d.core.bbox import DepthInstance3DBoxes from mmdet.datasets import DATASETS from mmseg.datasets import DATASETS as SEG_DATASETS From 93ceb7450915292212c3fe1d55215fae1b130fbd Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Wed, 16 Feb 2022 12:40:36 +0300 Subject: [PATCH 4/8] fix comments and docstrings --- mmdet3d/core/evaluation/instance_seg_eval.py | 27 +++++++- .../evaluate_semantic_instance.py | 67 ++++++++++++++----- .../core/evaluation/scannet_utils/util_3d.py | 31 +++++++-- mmdet3d/datasets/scannet_dataset.py | 15 +++-- 4 files changed, 110 insertions(+), 30 deletions(-) diff --git a/mmdet3d/core/evaluation/instance_seg_eval.py b/mmdet3d/core/evaluation/instance_seg_eval.py index facb80d732..31f5110abe 100644 --- a/mmdet3d/core/evaluation/instance_seg_eval.py +++ b/mmdet3d/core/evaluation/instance_seg_eval.py @@ -7,6 +7,17 @@ def aggregate_predictions(masks, labels, scores, valid_class_ids): + """Maps predictions to ScanNet evaluator format. + + Args: + masks (list[torch.Tensor]): Per scene predicted instance masks. + labels (list[torch.Tensor]): Per scene predicted instance labels. + scores (list[torch.Tensor]): Per scene predicted instance scores. + valid_class_ids (tuple[int]): Ids of valid categories. + + Returns: + list[dict]: Per scene aggregated predictions. + """ infos = [] for id, (mask, label, score) in enumerate(zip(masks, labels, scores)): mask = mask.clone().numpy() @@ -26,6 +37,17 @@ def aggregate_predictions(masks, labels, scores, valid_class_ids): def rename_gt(gt_semantic_masks, gt_instance_masks, valid_class_ids): + """Maps gt instance and semantic masks to instance masks for ScanNet + evaluator. + + Args: + gt_semantic_masks (list[torch.Tensor]): Per scene gt semantic masks. + gt_instance_masks (list[torch.Tensor]): Per scene gt instance masks. + valid_class_ids (tuple[int]): Ids of valid categories. + + Returns: + list[np.array]: Per scene instance masks. + """ renamed_instance_masks = [] for semantic_mask, instance_mask in zip(gt_semantic_masks, gt_instance_masks): @@ -66,8 +88,9 @@ def instance_seg_eval(gt_semantic_masks, pred_instance_scores (list[torch.Tensor]): Predicted instance labels. valid_class_ids (tuple[int]): Ids of valid categories. class_labels (tuple[str]): Names of valid categories. - options (dict): Additional options. Keys may contain: `overlaps`, - `min_region_sizes`, `distance_threshes`, `distance_confs`. + options (dict, optional): Additional options. Keys may contain: + `overlaps`, `min_region_sizes`, `distance_threshes`, + `distance_confs`. Default: None. logger (logging.Logger | str, optional): The way to print the mAP summary. See `mmdet.utils.print_log()` for details. Default: None. diff --git a/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py b/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py index 93d4515d88..26a3a70286 100644 --- a/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py +++ b/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py @@ -7,6 +7,17 @@ def evaluate_matches(matches, class_labels, options): + """Evaluate instance segmentation from matched gt and predicted instances + for all scenes. + + Args: + matches (dict): Contains gt2pred and pred2gt infos for every scene. + class_labels (tuple[str]): Class names. + options (dict): ScanNet evaluator options. See get_options. + + Returns: + np.array: Average precision scores for all thresholds and categories. + """ overlaps = options['overlaps'] min_region_sizes = [options['min_region_sizes'][0]] dist_threshes = [options['distance_threshes'][0]] @@ -20,8 +31,6 @@ def evaluate_matches(matches, class_labels, options): for oi, overlap_th in enumerate(overlaps): pred_visited = {} for m in matches: - # (@filaPro:) it is not used - # for p in matches[m]['pred']: for label_name in class_labels: for p in matches[m]['pred'][label_name]: if 'filename' in p: @@ -53,8 +62,6 @@ def evaluate_matches(matches, class_labels, options): # collect matches for (gti, gt) in enumerate(gt_instances): found_match = False - # (@filaPro:) it is not used - # num_pred = len(gt['matched_pred']) for pred in gt['matched_pred']: # greedy assignments if pred_visited[pred['filename']]: @@ -136,7 +143,7 @@ def evaluate_matches(matches, class_labels, options): # prepare precision recall num_examples = len(y_score_sorted) - # (@filaPro:) follow https://github.com/ScanNet/ScanNet/pull/26 ? # noqa + # follow https://github.com/ScanNet/ScanNet/pull/26 ? # noqa num_true_examples = y_true_sorted_cumsum[-1] if len( y_true_sorted_cumsum) > 0 else 0 precision = np.zeros(num_prec_recall) @@ -179,21 +186,30 @@ def evaluate_matches(matches, class_labels, options): def compute_averages(aps, options, class_labels): + """Averages AP scores for all categories. + + Args: + aps (np.array): AP scores for all thresholds and categories. + options (dict): ScanNet evaluator options. See get_options. + class_labels (tuple[str]): Class names. + + Returns: + dict: Overall and per-category AP scores. + """ d_inf = 0 o50 = np.where(np.isclose(options['overlaps'], 0.5)) o25 = np.where(np.isclose(options['overlaps'], 0.25)) - oAllBut25 = np.where(np.logical_not(np.isclose(options['overlaps'], 0.25))) + o_all_but25 = np.where( + np.logical_not(np.isclose(options['overlaps'], 0.25))) avg_dict = {} - # avg_dict['all_ap'] = np.nanmean(aps[ d_inf, :, :]) - avg_dict['all_ap'] = np.nanmean(aps[d_inf, :, oAllBut25]) + avg_dict['all_ap'] = np.nanmean(aps[d_inf, :, o_all_but25]) avg_dict['all_ap_50%'] = np.nanmean(aps[d_inf, :, o50]) avg_dict['all_ap_25%'] = np.nanmean(aps[d_inf, :, o25]) avg_dict['classes'] = {} for (li, label_name) in enumerate(class_labels): avg_dict['classes'][label_name] = {} - # avg_dict['classes'][label_name]['ap'] = np.average(aps[d_inf, li, :]) avg_dict['classes'][label_name]['ap'] = np.average(aps[d_inf, li, - oAllBut25]) + o_all_but25]) avg_dict['classes'][label_name]['ap50%'] = np.average(aps[d_inf, li, o50]) avg_dict['classes'][label_name]['ap25%'] = np.average(aps[d_inf, li, @@ -203,7 +219,7 @@ def compute_averages(aps, options, class_labels): def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids, class_labels, id_to_label): - # (@filaPro:) reading gt and pred files is replaced with their data + """Assign gt and predicted instances for a single scene.""" # get gt instances gt_instances = util_3d.get_instances(gt_ids, valid_class_ids, class_labels, id_to_label) @@ -216,7 +232,7 @@ def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids, for label in class_labels: pred2gt[label] = [] num_pred_instances = 0 - # mask of void labels in the groundtruth + # mask of void labels in the ground truth bool_void = np.logical_not(np.in1d(gt_ids // 1000, valid_class_ids)) # go through all prediction masks for pred_mask_file in pred_info: @@ -246,7 +262,7 @@ def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids, # matched gt instances matched_gt = [] - # go thru all gt instances with matching label + # go through all gt instances with matching label for (gt_num, gt_inst) in enumerate(gt2pred[label_name]): intersection = np.count_nonzero( np.logical_and(gt_ids == gt_inst['instance_id'], pred_mask)) @@ -266,10 +282,23 @@ def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids, def scannet_eval(preds, gts, options, valid_class_ids, class_labels, id_to_label): + """Evaluate instance segmentation in ScanNet protocol. + + Args: + preds (list[dict]): Per scene predictions of mask, label and + confidence. + gts (list[np.array]): Per scene ground truth instance masks. + options (dict): ScanNet evaluator options. See get_options. + valid_class_ids (tuple[int]): Ids of valid categories. + class_labels (tuple[str]): Class names. + id_to_label (dict[int, str]): Mapping of valid class id to class label. + + Returns: + dict: Overall and per-category AP scores. + """ options = get_options(options) matches = {} for i, (pred, gt) in enumerate(zip(preds, gts)): - # (@filaPro:) replace gt filename with index matches_key = i # assign gt to predictions gt2pred, pred2gt = assign_instances_for_scan(pred, gt, options, @@ -284,7 +313,15 @@ def scannet_eval(preds, gts, options, valid_class_ids, class_labels, return avgs -def get_options(options): +def get_options(options=None): + """Set ScanNet evaluator options. + + Args: + options (dict, optional): Not default options. Default: None. + + Returns: + dict: Updated options with all 4 keys. + """ assert options is None or isinstance(options, dict) _options = dict( overlaps=np.append(np.arange(0.5, 0.95, 0.05), 0.25), diff --git a/mmdet3d/core/evaluation/scannet_utils/util_3d.py b/mmdet3d/core/evaluation/scannet_utils/util_3d.py index 4081a59e39..fd9291de94 100644 --- a/mmdet3d/core/evaluation/scannet_utils/util_3d.py +++ b/mmdet3d/core/evaluation/scannet_utils/util_3d.py @@ -1,10 +1,16 @@ # Copyright (c) OpenMMLab. All rights reserved. -# copied with deletions from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/util_3d.py # noqa +# adapted from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/util_3d.py # noqa import json import numpy as np -class Instance(object): +class Instance: + """Single instance for ScanNet evaluator. + + Args: + mesh_vert_instances (np.array): Instance ids for each point. + instance_id: Id of single instance. + """ instance_id = 0 label_id = 0 vert_count = 0 @@ -12,17 +18,19 @@ class Instance(object): dist_conf = 0.0 def __init__(self, mesh_vert_instances, instance_id): - if (instance_id == -1): + if instance_id == -1: return self.instance_id = int(instance_id) self.label_id = int(self.get_label_id(instance_id)) self.vert_count = int( self.get_instance_verts(mesh_vert_instances, instance_id)) - def get_label_id(self, instance_id): + @staticmethod + def get_label_id(instance_id): return int(instance_id // 1000) - def get_instance_verts(self, mesh_vert_instances, instance_id): + @staticmethod + def get_instance_verts(mesh_vert_instances, instance_id): return (mesh_vert_instances == instance_id).sum() def to_json(self): @@ -42,7 +50,7 @@ def from_json(self, data): self.instance_id = int(data['instance_id']) self.label_id = int(data['label_id']) self.vert_count = int(data['vert_count']) - if ('med_dist' in data): + if 'med_dist' in data: self.med_dist = float(data['med_dist']) self.dist_conf = float(data['dist_conf']) @@ -51,6 +59,17 @@ def __str__(self): def get_instances(ids, class_ids, class_labels, id2label): + """Transform gt instance mask to Instance objects. + + Args: + ids (np.array): Instance ids for each point. + class_ids: (tuple[int]): Ids of valid categories. + class_labels (tuple[str]): Class names. + id2label: (dict[int, str]): Mapping of valid class id to class label. + + Returns: + dict [str, list]: Instance objects grouped by class label. + """ instances = {} for label in class_labels: instances[label] = [] diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index 1b3734d94d..88db58b31c 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -1,11 +1,10 @@ # Copyright (c) OpenMMLab. All rights reserved. +import numpy as np import tempfile import warnings from os import path as osp -import numpy as np - -from mmdet3d.core import show_result, show_seg_result, instance_seg_eval +from mmdet3d.core import instance_seg_eval, show_result, show_seg_result from mmdet3d.core.bbox import DepthInstance3DBoxes from mmdet.datasets import DATASETS from mmseg.datasets import DATASETS as SEG_DATASETS @@ -477,8 +476,10 @@ class ScanNetInstanceSegDataset(Custom3DSegDataset): def get_ann_info(self, index): """Get annotation info according to the given index. + Args: index (int): Index of the annotation data to get. + Returns: dict: annotation information consists of the following keys: - pts_semantic_mask_path (str): Path of semantic masks. @@ -498,9 +499,9 @@ def get_ann_info(self, index): return anns_results def get_classes_and_palette(self, classes=None, palette=None): - """Get class names of current dataset. + """Get class names of current dataset. Palette is simply ignored for + instance segmentation. - Palette is simply ignored for instance segmentation. Args: classes (Sequence[str] | str | None): If classes is None, use default CLASSES defined by builtin dataset. If classes is a @@ -554,9 +555,8 @@ def evaluate(self, show=False, out_dir=None, pipeline=None): - """Evaluate. + """Evaluation in instance segmentation protocol. - Evaluation in semantic segmentation protocol. Args: results (list[dict]): List of results. metric (str | list[str]): Metrics to be evaluated. @@ -569,6 +569,7 @@ def evaluate(self, Defaults to None. pipeline (list[dict], optional): raw data loading for showing. Default: None. + Returns: dict: Evaluation results. """ From d3f463bb2fe4c4214aa45e1f043a98afcf6318e9 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Wed, 23 Feb 2022 11:17:39 +0300 Subject: [PATCH 5/8] merge v1.0.0.dev0 to scannet_instance_dataset (#1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump to v0.18.0 (#1148) * Update README & getting_started & changelog & version.py * resolve comments * Update the required maximum version of mmcv-full * Update changelog.md * Update getting_started.md * Add highlight in changelog.md * update imvoxelnet citation (#1153) * deprecate the support for "python setup.py test" (#1164) * add OpenMMLab website and platform links (#1176) * [Fix] Update regnet metafile (#1184) * update regnet metafile * Add Architechture in RegNet metafile * fix axis=0 rotation in master branch (#1182) * [Enhance] support semantic seg in Flip3D augmentation (#1181) * support semantic seg in flip * change box_dtype_3d to bbox3d_fields * add point cloud annotation in doc (#1185) * [Fix] Fix bug in non-distributed multi-gpu training/testing (#1197) * Fix bug in non-distributed multi-gpu training/testing * add deprecated warning to void BC-breaking * Add missing explanation of cam_intrinsic in the nuScenes dataset doc (#1193) * [Fix] Fix corner bug in different coordinates (#1212) * fix corner bug in cam coord * fix corner bugs in depth & lidar coord * Bump to v0.18.1 (#1218) * bump to v0.18.1 * fix comments * update bazel version (#1223) * [Fix] fix pointpillars bug on kitti in master branch (#1163) * fix pointpillars bug on kitti in master branch * replace with AlignedAnchorGenerator * Update the installation of MMCV (#1226) * [Fix] Fix a potential overflow bug when post_max_size is set too large in the circle_nms (#1225) * [Fix] overflow bug * [Fix] typo Co-authored-by: zeyuzeng Co-authored-by: zeyu-hello * Fix PointRCNN bugs (#1224) * fix bug to allow pointnet to train in fp16 (#1207) * fix bug to allow pointnet to train in fp16 * remove unused import * fix lint * fix lint for gather_points_cuda.cu Co-authored-by: peng * [Fix] Recheck import sorting (#1242) * fix missed mmcv * reinstall pre-commit and recheck * fix inference_demo.ipynb bug (#1236) * [Enhance] upgrade PointPillars performace on dev branch (#1166) * upgrade PointPillars performace on dev branch * update DynamicPillarFeatureNet * fix comments * change to AlignedAnchor3DRangeGenerator * change to AlignedAnchor3DRangeGenerator * fix * replace with AlignedAnchorGenerator * fix lint * update using isort * Bump to v1.0.0.rc0 (#928) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import (#839) * [Enhance] refactor iou_neg_piecewise_sampler.py (#842) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import * refactor iou_neg_piecewise_sampler.py * add docstring * modify docstring Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: THU17cyz * [Feature] Add roipooling cuda ops (#843) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import * add roipooling cuda ops * add roi extractor * add test_roi_extractor unittest * Modify setup.py to install roipooling ops * modify docstring * remove enlarge bbox in roipoint pooling * add_roipooling_ops * modify docstring Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: THU17cyz * [Refactor] Refactor code structure and docstrings (#803) * refactor points_in_boxes * Merge same functions of three boxes * More docstring fixes and unify x/y/z size * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Remove None in function param type * Fix unittest * Add comments for NMS functions * Merge methods of Points * Add unittest * Add optional and default value * Fix box conversion and add unittest * Fix comments * Add unit test * Indent * Fix CI * Remove useless \\ * Remove useless \\ * Remove useless \\ * Remove useless \\ * Remove useless \\ * Add unit test for box bev * More unit tests and refine docstrings in box_np_ops * Fix comment * Add deprecation warning * [Feature] PointXYZWHLRBBoxCoder (#856) * support PointBasedBoxCoder * fix unittest bug * support unittest in gpu * support unittest in gpu * modified docstring * add args * add args * [Enhance] Change Groupfree3D config (#855) * All mods * PointSample * PointSample * [Doc] Add tutorials/data_pipeline Chinese version (#827) * [Doc] Add tutorials/data_pipeline Chinese version * refine doc * Use the absolute link * Use the absolute link Co-authored-by: Tai-Wang * [Doc] Add Chinese doc for `scannet_det.md` (#836) * Part * Complete * Fix comments * Fix comments * [Doc] Add Chinese doc for `waymo_det.md` (#859) * Add complete translation * Refinements * Fix comments * Fix a minor typo Co-authored-by: Tai-Wang * Remove 2D annotations on Lyft (#867) * Add header for files (#869) * Add header for files * Add header for files * Add header for files * Add header for files * [fix] fix typos (#872) * Fix 3 unworking configs (#882) * [Fix] Fix `index.rst` for Chinese docs (#873) * Fix index.rst for zh docs * Change switch language * [Fix] Centerpoint head nested list transpose (#879) * FIX Transpose nested lists without Numpy * Removed unused Numpy import * [Enhance] Update PointFusion (#791) * update point fusion * remove LIDAR hardcode * move get_proj_mat_by_coord_type to utils * fix lint * remove todo * fix lint * [Doc] Add nuscenes_det.md Chinese version (#854) * add nus chinese doc * add nuScenes Chinese doc * fix typo * fix typo * fix typo * fix typo * fix typo * [Fix] Fix RegNet pretrained weight loading (#889) * Fix regnet pretrained weight loading * Remove unused file * Fix centerpoint tta (#892) * [Enhance] Add benchmark regression script (#808) * Initial commit * [Feature] Support DGCNN (v1.0.0.dev0) (#896) * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * fix typo * fix typo * fix typo * del gf&fa registry (wo reuse pointnet module) * fix typo * add benchmark and add copyright header (for DGCNN only) * fix typo * fix typo * fix typo * fix typo * fix typo * support dgcnn * Change cam rot_3d_in_axis (#906) * [Doc] Add coord sys tutorial pic and change links to dev branch (#912) * Modify link branch and add pic * Fix pic * Update v1.0.0rc0 changelog * Remove v1.0.0.rc0 changelog * Init v1.0.0.rc0 changelog * Fix minor typos in the zh-CN index.rst * Add master updates in changelog v1.0.0 * Update changelog_v1.0.md * Update changelog_v1.0.md * Update changelog_v1.0.md * Adjust the order of logs * Update the number of developers * Add v1.0.0.rc0 changelog * Delete changelog_v1.0.md * Add RoIPointPool3d back * Change openmmlab pre-commit hook rev * Add links for compatibility doc * Refine details * Update README.md * Update highlights * Update README_zh-CN.md * Update getting_started.md * Update getting_started.md * Update version.py * Fix the released version name * Fix version name * Update version name * Update version name * Update version name * Update version name Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Co-authored-by: THU17cyz Co-authored-by: Wenhao Wu <79644370+wHao-Wu@users.noreply.github.com> Co-authored-by: dingchang Co-authored-by: 谢恩泽 Co-authored-by: Robin Karlsson <34254153+robin-karlsson0@users.noreply.github.com> Co-authored-by: Danila Rukhovich Co-authored-by: ChaimZhu Co-authored-by: Wenhao Wu <79644370+wHao-Wu@users.noreply.github.com> Co-authored-by: ChaimZhu Co-authored-by: Tai-Wang Co-authored-by: Double-Z Co-authored-by: zeyuzeng Co-authored-by: zeyu-hello Co-authored-by: maskjp Co-authored-by: peng Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Co-authored-by: THU17cyz Co-authored-by: dingchang Co-authored-by: 谢恩泽 Co-authored-by: Robin Karlsson <34254153+robin-karlsson0@users.noreply.github.com> --- .github/workflows/build.yml | 8 +- .pre-commit-config.yaml | 2 +- README.md | 31 +++-- README_zh-CN.md | 31 +++-- .../models/hv_pointpillars_secfpn_kitti.py | 9 +- configs/_base_/models/point_rcnn.py | 9 +- ...pillars_secfpn_6x8_160e_kitti-3d-3class.py | 12 +- demo/inference_demo.ipynb | 64 ++++----- docs/en/changelog.md | 129 ++++++++++++++++++ docs/en/datasets/nuscenes_det.md | 2 +- docs/en/datasets/waymo_det.md | 9 +- docs/en/faq.md | 9 ++ docs/en/getting_started.md | 38 ++++-- docs/zh_cn/datasets/nuscenes_det.md | 2 +- docs/zh_cn/datasets/waymo_det.md | 9 +- docs/zh_cn/faq.md | 9 ++ docs/zh_cn/getting_started.md | 38 ++++-- .../samplers/iou_neg_piecewise_sampler.py | 4 +- mmdet3d/core/bbox/structures/depth_box3d.py | 4 +- mmdet3d/core/bbox/structures/lidar_box3d.py | 6 +- mmdet3d/core/post_processing/box3d_nms.py | 6 +- mmdet3d/datasets/kitti_dataset.py | 26 +++- mmdet3d/datasets/pipelines/dbsampler.py | 11 +- mmdet3d/datasets/pipelines/transforms_3d.py | 25 +++- .../models/dense_heads/centerpoint_head.py | 4 +- mmdet3d/models/dense_heads/point_rpn_head.py | 29 +++- .../models/voxel_encoders/pillar_encoder.py | 24 +++- mmdet3d/ops/gather_points/gather_points.py | 4 +- .../ops/gather_points/src/gather_points.cpp | 55 ++++---- .../gather_points/src/gather_points_cuda.cu | 68 ++++++--- mmdet3d/ops/group_points/group_points.py | 5 + mmdet3d/utils/setup_env.py | 3 +- mmdet3d/version.py | 2 +- setup.py | 2 - .../test_common_modules/test_pointnet_ops.py | 15 +- tests/test_utils/test_setup_env.py | 3 +- tools/data_converter/kitti_data_utils.py | 1 + tools/test.py | 23 +++- tools/train.py | 25 +++- 39 files changed, 562 insertions(+), 194 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8677ade1c..4b2cd1917e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,22 +41,22 @@ jobs: torch: [1.5.0+cu101, 1.6.0+cu101, 1.7.0+cu101, 1.8.0+cu101] include: - torch: 1.5.0+cu101 - torch_version: torch1.5.0 + torch_version: torch1.5 torchvision: 0.6.0+cu101 mmcv_link: "torch1.5.0" cuda_arch: "7.0" - torch: 1.6.0+cu101 - torch_version: torch1.6.0 + torch_version: torch1.6 mmcv_link: "torch1.6.0" torchvision: 0.7.0+cu101 cuda_arch: "7.0" - torch: 1.7.0+cu101 - torch_version: torch1.7.0 + torch_version: torch1.7 mmcv_link: "torch1.7.0" torchvision: 0.8.1+cu101 cuda_arch: "7.0" - torch: 1.8.0+cu101 - torch_version: torch1.8.0 + torch_version: torch1.8 mmcv_link: "torch1.8.0" torchvision: 0.9.0+cu101 cuda_arch: "7.0" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b141730f5c..5bd88c5a27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: - id: docformatter args: ["--in-place", "--wrap-descriptions", "79"] - repo: https://github.com/open-mmlab/pre-commit-hooks - rev: master # Use the ref you want to point at + rev: v0.2.0 # Use the ref you want to point at hooks: - id: check-algo-readme - id: check-copyright diff --git a/README.md b/README.md index d0fd737d08..178c20ae0c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,22 @@
+
 
+
+ OpenMMLab website + + + HOT + + +      + OpenMMLab platform + + + TRY IT OUT + + +
+
 
[![docs](https://img.shields.io/badge/docs-latest-blue)](https://mmdetection3d.readthedocs.io/en/latest/) @@ -8,16 +25,12 @@ [![license](https://img.shields.io/github/license/open-mmlab/mmdetection3d.svg)](https://github.com/open-mmlab/mmdetection3d/blob/master/LICENSE) -**News**: We released the codebase v0.17.3. +**News**: We released the codebase v1.0.0rc0. -In addition, we have preliminarily supported several new models on the [v1.0.0.dev0](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0) branch, including [DGCNN](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/configs/dgcnn/README.md), [SMOKE](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/configs/smoke/README.md) and [PGD](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/configs/pgd/README.md). - -Note: We are going through large refactoring to provide simpler and more unified usage of many modules. Thus, few features will be added to the master branch in the following months. +Note: We are going through large refactoring to provide simpler and more unified usage of many modules. The compatibilities of models are broken due to the unification and simplification of coordinate systems. For now, most models are benchmarked with similar performance, though few models are still being benchmarked. -You can start experiments with [v1.0.0.dev0](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0) if you are interested. Please note that our new features will only be supported in v1.0.0 branch afterward. - In the [nuScenes 3D detection challenge](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any) of the 5th AI Driving Olympics in NeurIPS 2020, we obtained the best PKL award and the second runner-up by multi-modality entry, and the best vision-only results. Code and models for the best vision-only method, [FCOS3D](https://arxiv.org/abs/2104.10956), have been released. Please stay tuned for [MoCa](https://arxiv.org/abs/2012.12741). @@ -70,11 +83,9 @@ This project is released under the [Apache 2.0 license](LICENSE). ## Changelog -v0.17.3 was released in 1/12/2021. +v1.0.0rc0 was released in 18/2/2022. Please refer to [changelog.md](docs/en/changelog.md) for details and release history. -For branch v1.0.0.dev0, please refer to [changelog_v1.0.md](https://github.com/Tai-Wang/mmdetection3d/blob/v1.0.0.dev0-changelog/docs/changelog_v1.0.md) for our latest features and more details. - ## Benchmark and model zoo Supported methods and backbones are shown in the below table. @@ -104,7 +115,7 @@ Support methods - [x] [FCOS3D (ICCVW'2021)](configs/fcos3d/README.md) - [x] [PointNet++ (NeurIPS'2017)](configs/pointnet2/README.md) - [x] [Group-Free-3D (ICCV'2021)](configs/groupfree3d/README.md) -- [x] [ImVoxelNet (Arxiv'2021)](configs/imvoxelnet/README.md) +- [x] [ImVoxelNet (WACV'2022)](configs/imvoxelnet/README.md) - [x] [PAConv (CVPR'2021)](configs/paconv/README.md) - [x] [DGCNN (TOG'2019)](configs/dgcnn/README.md) - [x] [SMOKE (CVPRW'2020)](configs/smoke/README.md) diff --git a/README_zh-CN.md b/README_zh-CN.md index fb09c00f21..85fa01f3de 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -1,5 +1,22 @@
+
 
+
+ OpenMMLab 官网 + + + HOT + + +      + OpenMMLab 开放平台 + + + TRY IT OUT + + +
+
 
[![docs](https://img.shields.io/badge/docs-latest-blue)](https://mmdetection3d.readthedocs.io/en/latest/) @@ -8,16 +25,12 @@ [![license](https://img.shields.io/github/license/open-mmlab/mmdetection3d.svg)](https://github.com/open-mmlab/mmdetection3d/blob/master/LICENSE) -**新闻**: 我们发布了版本 v0.17.3. +**新闻**: 我们发布了版本 v1.0.0rc0. -另外，我们在 [v1.0.0.dev0](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0) 分支初步支持了多个新模型，包括 [DGCNN](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/configs/dgcnn/README.md), [SMOKE](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/configs/smoke/README.md) 和 [PGD](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/configs/pgd/README.md)。 - -说明：我们正在进行大规模的重构，以提供对许多模块更简单、更统一的使用。因此，在接下来的几个月里，很少有功能会添加到主分支中。 +说明：我们正在进行大规模的重构，以提供对许多模块更简单、更统一的使用。 由于坐标系的统一和简化，模型的兼容性会受到影响。目前，大多数模型都以类似的性能对齐了精度，但仍有少数模型在进行基准测试。 -如果您感兴趣，可以开始使用 [v1.0.0.dev0](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0) 分支进行实验。请注意，我们的新功能将只支持在 v1.0.0 分支。 - 在第三届 [nuScenes 3D 检测挑战赛](https://www.nuscenes.org/object-detection?externalData=all&mapData=all&modalities=Any)（第五届 AI Driving Olympics, NeurIPS 2020）中，我们获得了最佳 PKL 奖、第三名和最好的纯视觉的结果，相关的代码和模型将会在不久后发布。 最好的纯视觉方法 [FCOS3D](https://arxiv.org/abs/2104.10956) 的代码和模型已经发布。请继续关注我们的多模态检测器 [MoCa](https://arxiv.org/abs/2012.12741)。 @@ -70,11 +83,9 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代 ## 更新日志 -最新的版本 v0.17.3 在 2021.12.01 发布。 +最新的版本 v1.0.0rc0 在 2022.2.18 发布。 如果想了解更多版本更新细节和历史信息，请阅读[更新日志](docs/zh_cn/changelog.md)。 -对于分支 [v1.0.0.dev0](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0) ，请参考 [v1.0 更新日志](https://github.com/Tai-Wang/mmdetection3d/blob/v1.0.0.dev0-changelog/docs/changelog_v1.0.md) 来了解我们的最新功能和更多细节。 - ## 基准测试和模型库 测试结果和模型可以在[模型库](docs/zh_cn/model_zoo.md)中找到。 @@ -103,7 +114,7 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代 - [x] [FCOS3D (ICCVW'2021)](configs/fcos3d/README.md) - [x] [PointNet++ (NeurIPS'2017)](configs/pointnet2/README.md) - [x] [Group-Free-3D (ICCV'2021)](configs/groupfree3d/README.md) -- [x] [ImVoxelNet (Arxiv'2021)](configs/imvoxelnet/README.md) +- [x] [ImVoxelNet (WACV'2022)](configs/imvoxelnet/README.md) - [x] [PAConv (CVPR'2021)](configs/paconv/README.md) - [x] [DGCNN (TOG'2019)](configs/dgcnn/README.md) - [x] [SMOKE (CVPRW'2020)](configs/smoke/README.md) diff --git a/configs/_base_/models/hv_pointpillars_secfpn_kitti.py b/configs/_base_/models/hv_pointpillars_secfpn_kitti.py index bb2014729f..ac46475d6e 100644 --- a/configs/_base_/models/hv_pointpillars_secfpn_kitti.py +++ b/configs/_base_/models/hv_pointpillars_secfpn_kitti.py @@ -34,12 +34,13 @@ in_channels=384, feat_channels=384, use_direction_classifier=True, + assign_per_class=True, anchor_generator=dict( - type='Anchor3DRangeGenerator', + type='AlignedAnchor3DRangeGenerator', ranges=[ - [0, -39.68, -0.6, 70.4, 39.68, -0.6], - [0, -39.68, -0.6, 70.4, 39.68, -0.6], - [0, -39.68, -1.78, 70.4, 39.68, -1.78], + [0, -39.68, -0.6, 69.12, 39.68, -0.6], + [0, -39.68, -0.6, 69.12, 39.68, -0.6], + [0, -39.68, -1.78, 69.12, 39.68, -1.78], ], sizes=[[0.8, 0.6, 1.73], [1.76, 0.6, 1.73], [3.9, 1.6, 1.56]], rotations=[0, 1.57], diff --git a/configs/_base_/models/point_rcnn.py b/configs/_base_/models/point_rcnn.py index 7a9667e733..02a1414f7d 100644 --- a/configs/_base_/models/point_rcnn.py +++ b/configs/_base_/models/point_rcnn.py @@ -91,7 +91,8 @@ pos_iou_thr=0.55, neg_iou_thr=0.55, min_pos_iou=0.55, - ignore_iof_thr=-1), + ignore_iof_thr=-1, + match_low_quality=False), dict( # for Pedestrian type='MaxIoUAssigner', iou_calculator=dict( @@ -99,7 +100,8 @@ pos_iou_thr=0.55, neg_iou_thr=0.55, min_pos_iou=0.55, - ignore_iof_thr=-1), + ignore_iof_thr=-1, + match_low_quality=False), dict( # for Cyclist type='MaxIoUAssigner', iou_calculator=dict( @@ -107,7 +109,8 @@ pos_iou_thr=0.55, neg_iou_thr=0.55, min_pos_iou=0.55, - ignore_iof_thr=-1) + ignore_iof_thr=-1, + match_low_quality=False) ], sampler=dict( type='IoUNegPiecewiseSampler', diff --git a/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-3class.py b/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-3class.py index 5c5c939de5..2611e86d3a 100644 --- a/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-3class.py +++ b/configs/pointpillars/hv_pointpillars_secfpn_6x8_160e_kitti-3d-3class.py @@ -15,21 +15,15 @@ rate=1.0, prepare=dict( filter_by_difficulty=[-1], - filter_by_min_points=dict(Car=5, Pedestrian=10, Cyclist=10)), + filter_by_min_points=dict(Car=5, Pedestrian=5, Cyclist=5)), classes=class_names, - sample_groups=dict(Car=15, Pedestrian=10, Cyclist=10)) + sample_groups=dict(Car=15, Pedestrian=15, Cyclist=15)) # PointPillars uses different augmentation hyper parameters train_pipeline = [ dict(type='LoadPointsFromFile', coord_type='LIDAR', load_dim=4, use_dim=4), dict(type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True), - dict(type='ObjectSample', db_sampler=db_sampler), - dict( - type='ObjectNoise', - num_try=100, - translation_std=[0.25, 0.25, 0.25], - global_rot_range=[0.0, 0.0], - rot_range=[-0.15707963267, 0.15707963267]), + dict(type='ObjectSample', db_sampler=db_sampler, use_ground_plane=False), dict(type='RandomFlip3D', flip_ratio_bev_horizontal=0.5), dict( type='GlobalRotScaleTrans', diff --git a/demo/inference_demo.ipynb b/demo/inference_demo.ipynb index e42cc0ba73..5f6731db1f 100644 --- a/demo/inference_demo.ipynb +++ b/demo/inference_demo.ipynb @@ -3,74 +3,74 @@ { "cell_type": "code", "execution_count": 7, + "source": [ + "from mmdet3d.apis import init_model, inference_detector, show_result_meshlab" + ], + "outputs": [], "metadata": { "pycharm": { "is_executing": false } - }, - "outputs": [], - "source": [ - "from mmdet3d.apis import init_detector, inference_detector, show_result_meshlab" - ] + } }, { "cell_type": "code", "execution_count": 8, - "metadata": { - "pycharm": { - "is_executing": false - } - }, - "outputs": [], "source": [ "config_file = '../configs/second/hv_second_secfpn_6x8_80e_kitti-3d-car.py'\n", "# download the checkpoint from model zoo and put it in `checkpoints/`\n", "checkpoint_file = '../work_dirs/second/epoch_40.pth'" - ] + ], + "outputs": [], + "metadata": { + "pycharm": { + "is_executing": false + } + } }, { "cell_type": "code", "execution_count": 9, + "source": [ + "# build the model from a config file and a checkpoint file\n", + "model = init_model(config_file, checkpoint_file, device='cuda:0')" + ], + "outputs": [], "metadata": { "pycharm": { "is_executing": false } - }, - "outputs": [], - "source": [ - "# build the model from a config file and a checkpoint file\n", - "model = init_detector(config_file, checkpoint_file, device='cuda:0')" - ] + } }, { "cell_type": "code", "execution_count": 10, - "metadata": { - "pycharm": { - "is_executing": false - } - }, - "outputs": [], "source": [ "# test a single sample\n", "pcd = 'kitti_000008.bin'\n", "result, data = inference_detector(model, pcd)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, + ], + "outputs": [], "metadata": { "pycharm": { "is_executing": false } - }, - "outputs": [], + } + }, + { + "cell_type": "code", + "execution_count": 11, "source": [ "# show the results\n", "out_dir = './'\n", "show_result_meshlab(data, result, out_dir)" - ] + ], + "outputs": [], + "metadata": { + "pycharm": { + "is_executing": false + } + } } ], "metadata": { diff --git a/docs/en/changelog.md b/docs/en/changelog.md index a248295c49..54a4601ee5 100644 --- a/docs/en/changelog.md +++ b/docs/en/changelog.md @@ -1,5 +1,134 @@ ## Changelog +### v1.0.0rc0 (18/2/2022) + +#### Compatibility + +- We refactor our three coordinate systems to make their rotation directions and origins more consistent, and further remove unnecessary hacks in different datasets and models. Therefore, please re-generate data infos or convert the old version to the new one with our provided scripts. We will also provide updated checkpoints in the next version. Please refer to the [compatibility documentation](https://github.com/open-mmlab/mmdetection3d/blob/v1.0.0.dev0/docs/en/compatibility.md) for more details. +- Unify the camera keys for consistent transformation between coordinate systems on different datasets. The modification changes the key names to `lidar2img`, `depth2img`, `cam2img`, etc., for easier understanding. Customized codes using legacy keys may be influenced. +- The next release will begin to move files of CUDA ops to [MMCV](https://github.com/open-mmlab/mmcv). It will influence the way to import related functions. We will not break the compatibility but will raise a warning first and please prepare to migrate it. + +#### Highlights + +- Support new monocular 3D detectors: [PGD](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0/configs/pgd), [SMOKE](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0/configs/smoke), [MonoFlex](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0/configs/monoflex) +- Support a new LiDAR-based detector: [PointRCNN](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0/configs/point_rcnn) +- Support a new backbone: [DGCNN](https://github.com/open-mmlab/mmdetection3d/tree/v1.0.0.dev0/configs/dgcnn) +- Support 3D object detection on the S3DIS dataset +- Support compilation on Windows +- Full benchmark for PAConv on S3DIS +- Further enhancement for documentation, especially on the Chinese documentation + +#### New Features + +- Support 3D object detection on the S3DIS dataset (#835) +- Support PointRCNN (#842, #843, #856, #974, #1022, #1109, #1125) +- Support DGCNN (#896) +- Support PGD (#938, #940, #948, #950, #964, #1014, #1065, #1070, #1157) +- Support SMOKE (#939, #955, #959, #975, #988, #999, #1029) +- Support MonoFlex (#1026, #1044, #1114, #1115, #1183) +- Support CPU Training (#1196) + +#### Improvements + +- Support point sampling based on distance metric (#667, #840) +- Refactor coordinate systems (#677, #774, #803, #899, #906, #912, #968, #1001) +- Unify camera keys in PointFusion and transformations between different systems (#791, #805) +- Refine documentation (#792, #827, #829, #836, #849, #854, #859, #1111, #1113, #1116, #1121, #1132, #1135, #1185, #1193, #1226) +- Add a script to support benchmark regression (#808) +- Benchmark PAConvCUDA on S3DIS (#847) +- Support to download pdf and epub documentation (#850) +- Change the `repeat` setting in Group-Free-3D configs to reduce training epochs (#855) +- Support KITTI AP40 evaluation metric (#927) +- Add the mmdet3d2torchserve tool for SECOND (#977) +- Add code-spell pre-commit hook and fix typos (#995) +- Support the latest numba version (#1043) +- Set a default seed to use when the random seed is not specified (#1072) +- Distribute mix-precision models to each algorithm folder (#1074) +- Add abstract and a representative figure for each algorithm (#1086) +- Upgrade pre-commit hook (#1088, #1217) +- Support augmented data and ground truth visualization (#1092) +- Add local yaw property for `CameraInstance3DBoxes` (#1130) +- Lock the required numba version to 0.53.0 (#1159) +- Support the usage of plane information for KITTI dataset (#1162) +- Deprecate the support for "python setup.py test" (#1164) +- Reduce the number of multi-process threads to accelerate training (#1168) +- Support 3D flip augmentation for semantic segmentation (#1181) +- Update README format for each model (#1195) + +#### Bug Fixes + +- Fix compiling errors on Windows (#766) +- Fix the deprecated nms setting in the ImVoteNet config (#828) +- Use the latest `wrap_fp16_model` import from mmcv (#861) +- Remove 2D annotations generation on Lyft (#867) +- Update index files for the Chinese documentation to be consistent with the English version (#873) +- Fix the nested list transpose in the CenterPoint head (#879) +- Fix deprecated pretrained model loading for RegNet (#889) +- Fix the incorrect dimension indices of rotations and testing config in the CenterPoint test time augmentation (#892) +- Fix and improve visualization tools (#956, #1066, #1073) +- Fix PointPillars FLOPs calculation error (#1075) +- Fix missing dimension information in the SUN RGB-D data generation (#1120) +- Fix incorrect anchor range settings in the PointPillars [config](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/_base_/models/hv_pointpillars_secfpn_kitti.py) for KITTI (#1163) +- Fix incorrect model information in the RegNet metafile (#1184) +- Fix bugs in non-distributed multi-gpu training and testing (#1197) +- Fix a potential assertion error when generating corners from an empty box (#1212) +- Upgrade bazel version according to the requirement of Waymo Devkit (#1223) + +#### Contributors + +A total of 12 developers contributed to this release. + +@THU17cyz, @wHao-Wu, @wangruohui, @Wuziyi616, @filaPro, @ZwwWayne, @Tai-Wang, @DCNSW, @xieenze, @robin-karlsson0, @ZCMax, @Otteri + +### v0.18.1 (1/2/2022) + +#### Improvements + +- Support Flip3D augmentation in semantic segmentation task (#1182) +- Update regnet metafile (#1184) +- Add point cloud annotation tools introduction in FAQ (#1185) +- Add missing explanations of `cam_intrinsic` in the nuScenes dataset doc (#1193) + +#### Bug Fixes + +- Deprecate the support for "python setup.py test" (#1164) +- Fix the rotation matrix while rotation axis=0 (#1182) +- Fix the bug in non-distributed multi-gpu training/testing (#1197) +- Fix a potential bug when generating corners for empty bounding boxes (#1212) + +#### Contributors + +A total of 4 developers contributed to this release. + +@ZwwWayne, @ZCMax, @Tai-Wang, @wHao-Wu + +### v0.18.0 (1/1/2022) + +#### Highlights + +- Update the required minimum version of mmdet and mmseg + +#### Improvements + +- Use the official markdownlint hook and add codespell hook for pre-committing (#1088) +- Improve CI operation (#1095, #1102, #1103) +- Use shared menu content from OpenMMLab's theme and remove duplicated contents from config (#1111) +- Refactor the structure of documentation (#1113, #1121) +- Update the required minimum version of mmdet and mmseg (#1147) + +#### Bug Fixes + +- Fix symlink failure on Windows (#1096) +- Fix the upper bound of mmcv version in the mminstall requirements (#1104) +- Fix API documentation compilation and mmcv build errors (#1116) +- Fix figure links and pdf documentation compilation (#1132, #1135) + +#### Contributors + +A total of 4 developers contributed to this release. + +@ZwwWayne, @ZCMax, @Tai-Wang, @wHao-Wu + ### v0.17.3 (1/12/2021) #### Improvements diff --git a/docs/en/datasets/nuscenes_det.md b/docs/en/datasets/nuscenes_det.md index b2192dbf27..76995c7edd 100644 --- a/docs/en/datasets/nuscenes_det.md +++ b/docs/en/datasets/nuscenes_det.md @@ -77,7 +77,7 @@ Next, we will elaborate on the details recorded in these info files. - info['sweeps'][i]['sensor2lidar_translation']: The translation from the current sensor (for collecting the sweep data) to lidar. (1x3 list) - info['sweeps'][i]['sensor2lidar_rotation']: The rotation from the current sensor (for collecting the sweep data) to lidar. (1x4 list in the quaternion format) - info['cams']: Cameras calibration information. It contains six keys corresponding to each camera: `'CAM_FRONT'`, `'CAM_FRONT_RIGHT'`, `'CAM_FRONT_LEFT'`, `'CAM_BACK'`, `'CAM_BACK_LEFT'`, `'CAM_BACK_RIGHT'`. - Each dictionary contains detailed information following the above way for each sweep data (has the same keys for each information as above). + Each dictionary contains detailed information following the above way for each sweep data (has the same keys for each information as above). In addition, each camera has a key `'cam_intrinsic'` for recording the intrinsic parameters when projecting 3D points to each image plane. - info['lidar2ego_translation']: The translation from lidar to ego vehicle. (1x3 list) - info['lidar2ego_rotation']: The rotation from lidar to ego vehicle. (1x4 list in the quaternion format) - info['ego2global_translation']: The translation from the ego vehicle to global coordinates. (1x3 list) diff --git a/docs/en/datasets/waymo_det.md b/docs/en/datasets/waymo_det.md index 84a955b995..d86d003b2f 100644 --- a/docs/en/datasets/waymo_det.md +++ b/docs/en/datasets/waymo_det.md @@ -104,16 +104,21 @@ Considering there are many similar frames in the original dataset, we can basica For evaluation on Waymo, please follow the [instruction](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md/) to build the binary file `compute_detection_metrics_main` for metrics computation and put it into `mmdet3d/core/evaluation/waymo_utils/`. Basically, you can follow the commands below to install `bazel` and build the file. ```shell + # download the code and enter the base directory git clone https://github.com/waymo-research/waymo-open-dataset.git waymo-od cd waymo-od git checkout remotes/origin/master + # use the Bazel build system sudo apt-get install --assume-yes pkg-config zip g++ zlib1g-dev unzip python3 python3-pip - wget https://github.com/bazelbuild/bazel/releases/download/0.28.0/bazel-0.28.0-installer-linux-x86_64.sh - sudo bash bazel-0.28.0-installer-linux-x86_64.sh + BAZEL_VERSION=3.1.0 + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh + sudo bash bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh sudo apt install build-essential + # configure .bazelrc ./configure.sh + # delete previous bazel outputs and reset internal caches bazel clean bazel build waymo_open_dataset/metrics/tools/compute_detection_metrics_main diff --git a/docs/en/faq.md b/docs/en/faq.md index 6a98ba2c2c..0cb93b57b4 100644 --- a/docs/en/faq.md +++ b/docs/en/faq.md @@ -30,3 +30,12 @@ We list some potential troubles encountered by users and developers, along with or ``pip install -e "git+https://github.com/ppwwyyxx/cocoapi#egg=pycocotools&subdirectory=PythonAPI"`` + +## How to annotate point cloud? + +MMDetection3D does not support point cloud annotation. Some open-source annotation tool are offered for reference: + +- [SUSTechPOINTS](https://github.com/naurril/SUSTechPOINTS) +- [LATTE](https://github.com/bernwang/latte) + +Besides, we improved [LATTE](https://github.com/bernwang/latte) for better use. More details can be found [here](https://arxiv.org/abs/2011.10174). diff --git a/docs/en/getting_started.md b/docs/en/getting_started.md index 6c602ce3b8..a483eaa626 100644 --- a/docs/en/getting_started.md +++ b/docs/en/getting_started.md @@ -13,20 +13,23 @@ The required versions of MMCV, MMDetection and MMSegmentation for different vers | MMDetection3D version | MMDetection version | MMSegmentation version | MMCV version | |:-------------------:|:-------------------:|:-------------------:|:-------------------:| | master | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.3 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.2 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.1 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.16.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.15.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.14.0 | mmdet>=2.10.0, <=2.11.0| mmseg==0.14.0 | mmcv-full>=1.3.1, <=1.5.0| -| 0.13.0 | mmdet>=2.10.0, <=2.11.0| Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.12.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.11.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.10.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.9.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.8.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.5.0| -| 0.7.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.5.0| +| v1.0.0rc0 | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| +| 0.18.1 | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| +| 0.18.0 | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| +| 0.17.3 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.17.2 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.17.1 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.17.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.16.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.15.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.14.0 | mmdet>=2.10.0, <=2.11.0| mmseg==0.14.0 | mmcv-full>=1.3.1, <=1.4.0| +| 0.13.0 | mmdet>=2.10.0, <=2.11.0| Not required | mmcv-full>=1.2.4, <=1.4.0| +| 0.12.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.4.0| +| 0.11.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.3.0| +| 0.10.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.3.0| +| 0.9.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.3.0| +| 0.8.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.3.0| +| 0.7.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.3.0| | 0.6.0 | mmdet>=2.4.0, <=2.11.0 | Not required | mmcv-full>=1.1.3, <=1.2.0| | 0.5.0 | 2.3.0 | Not required | mmcv-full==1.0.5| @@ -82,6 +85,13 @@ Please replace `{cu_version}` and `{torch_version}` in the url to your desired o pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu110/torch1.7.0/index.html ``` +mmcv-full is only compiled on PyTorch 1.x.0 because the compatibility usually holds between 1.x.0 and 1.x.1. If your PyTorch version is 1.x.1, you can install mmcv-full compiled with PyTorch 1.x.0 and it usually works well. + +```shell +# We can ignore the micro version of PyTorch +pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu110/torch1.7/index.html +``` + See [here](https://github.com/open-mmlab/mmcv#install-with-pip) for different versions of MMCV compatible to different PyTorch and CUDA versions. Optionally, you could also build the full version from source: diff --git a/docs/zh_cn/datasets/nuscenes_det.md b/docs/zh_cn/datasets/nuscenes_det.md index e4b6c84f45..18b426bd3d 100644 --- a/docs/zh_cn/datasets/nuscenes_det.md +++ b/docs/zh_cn/datasets/nuscenes_det.md @@ -78,7 +78,7 @@ mmdetection3d - info['sweeps'][i]['sensor2lidar_translation']：从当前传感器（用于收集扫描数据）到激光雷达的转换（1x3 列表）。 - info['sweeps'][i]['sensor2lidar_rotation']：从当前传感器（用于收集扫描数据）到激光雷达的旋转（四元数格式的 1x4 列表）。 - info['cams']：相机校准信息。它包含与每个摄像头对应的六个键值： `'CAM_FRONT'`, `'CAM_FRONT_RIGHT'`, `'CAM_FRONT_LEFT'`, `'CAM_BACK'`, `'CAM_BACK_LEFT'`, `'CAM_BACK_RIGHT'`。 - 每个字典包含每个扫描数据按照上述方式的详细信息（每个信息的关键字与上述相同）。 + 每个字典包含每个扫描数据按照上述方式的详细信息（每个信息的关键字与上述相同）。除此之外，每个相机还包含了一个键值 `'cam_intrinsic'` 用来保存 3D 点投影到图像平面上需要的内参信息。 - info['lidar2ego_translation']：从激光雷达到自车的转换（1x3 列表）。 - info['lidar2ego_rotation']：从激光雷达到自车的旋转（四元数格式的 1x4 列表）。 - info['ego2global_translation']：从自车到全局坐标的转换（1x3 列表）。 diff --git a/docs/zh_cn/datasets/waymo_det.md b/docs/zh_cn/datasets/waymo_det.md index fe5e764ad3..acadf9e1d7 100644 --- a/docs/zh_cn/datasets/waymo_det.md +++ b/docs/zh_cn/datasets/waymo_det.md @@ -104,16 +104,21 @@ mmdetection3d 为了在 Waymo 数据集上进行检测性能评估，请按照[此处指示](https://github.com/waymo-research/waymo-open-dataset/blob/master/docs/quick_start.md/)构建用于计算评估指标的二进制文件 `compute_detection_metrics_main`，并将它置于 `mmdet3d/core/evaluation/waymo_utils/` 下。您基本上可以按照下方命令安装 `bazel`，然后构建二进制文件： ```shell + # download the code and enter the base directory git clone https://github.com/waymo-research/waymo-open-dataset.git waymo-od cd waymo-od git checkout remotes/origin/master + # use the Bazel build system sudo apt-get install --assume-yes pkg-config zip g++ zlib1g-dev unzip python3 python3-pip - wget https://github.com/bazelbuild/bazel/releases/download/0.28.0/bazel-0.28.0-installer-linux-x86_64.sh - sudo bash bazel-0.28.0-installer-linux-x86_64.sh + BAZEL_VERSION=3.1.0 + wget https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh + sudo bash bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh sudo apt install build-essential + # configure .bazelrc ./configure.sh + # delete previous bazel outputs and reset internal caches bazel clean bazel build waymo_open_dataset/metrics/tools/compute_detection_metrics_main diff --git a/docs/zh_cn/faq.md b/docs/zh_cn/faq.md index 90aa67ed25..19a116e15f 100644 --- a/docs/zh_cn/faq.md +++ b/docs/zh_cn/faq.md @@ -30,3 +30,12 @@ 或者 ``pip install -e "git+https://github.com/ppwwyyxx/cocoapi#egg=pycocotools&subdirectory=PythonAPI"`` + +## 如何标注点云？ + +MMDetection3D 不支持点云标注。我们提供一些开源的标注工具供参考： + +- [SUSTechPOINTS](https://github.com/naurril/SUSTechPOINTS) +- [LATTE](https://github.com/bernwang/latte) + +此外，我们改进了 [LATTE](https://github.com/bernwang/latte) 以便更方便的标注。 更多的细节请参考 [这里](https://arxiv.org/abs/2011.10174)。 diff --git a/docs/zh_cn/getting_started.md b/docs/zh_cn/getting_started.md index 776dc65c13..cd0772ccdc 100644 --- a/docs/zh_cn/getting_started.md +++ b/docs/zh_cn/getting_started.md @@ -10,20 +10,23 @@ | MMDetection3D version | MMDetection version | MMSegmentation version | MMCV version | |:-------------------:|:-------------------:|:-------------------:|:-------------------:| | master | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.3 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.2 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.1 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.17.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.16.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.15.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| -| 0.14.0 | mmdet>=2.10.0, <=2.11.0| mmseg>=0.14.0 | mmcv-full>=1.3.1, <=1.5.0| -| 0.13.0 | mmdet>=2.10.0, <=2.11.0| Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.12.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.11.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.10.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.9.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.5.0| -| 0.8.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.5.0| -| 0.7.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.5.0| +| v1.0.0rc0 | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| +| 0.18.1 | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| +| 0.18.0 | mmdet>=2.19.0, <=3.0.0| mmseg>=0.20.0, <=1.0.0 | mmcv-full>=1.3.8, <=1.5.0| +| 0.17.3 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.17.2 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.17.1 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.17.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.16.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.15.0 | mmdet>=2.14.0, <=3.0.0| mmseg>=0.14.1, <=1.0.0 | mmcv-full>=1.3.8, <=1.4.0| +| 0.14.0 | mmdet>=2.10.0, <=2.11.0| mmseg==0.14.0 | mmcv-full>=1.3.1, <=1.4.0| +| 0.13.0 | mmdet>=2.10.0, <=2.11.0| Not required | mmcv-full>=1.2.4, <=1.4.0| +| 0.12.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.4.0| +| 0.11.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.3.0| +| 0.10.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.3.0| +| 0.9.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.2.4, <=1.3.0| +| 0.8.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.3.0| +| 0.7.0 | mmdet>=2.5.0, <=2.11.0 | Not required | mmcv-full>=1.1.5, <=1.3.0| | 0.6.0 | mmdet>=2.4.0, <=2.11.0 | Not required | mmcv-full>=1.1.3, <=1.2.0| | 0.5.0 | 2.3.0 | Not required | mmcv-full==1.0.5| @@ -74,6 +77,13 @@ pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/{cu_version}/{ pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu110/torch1.7.0/index.html ``` +PyTorch 在 1.x.0 和 1.x.1 之间通常是兼容的，故 mmcv-full 只提供 1.x.0 的编译包。如果你的 PyTorch 版本是 1.x.1，你可以放心地安装在 1.x.0 版本编译的 mmcv-full。 + +``` +# 我们可以忽略 PyTorch 的小版本号 +pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu110/torch1.7/index.html +``` + 请参考 [MMCV](https://mmcv.readthedocs.io/en/latest/#installation) 获取不同版本的 MMCV 所兼容的的不同的 PyTorch 和 CUDA 版本。同时，也可以通过以下命令行从源码编译 MMCV： ```shell diff --git a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py index c0de845f68..cbd8483cad 100644 --- a/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py +++ b/mmdet3d/core/bbox/samplers/iou_neg_piecewise_sampler.py @@ -60,9 +60,7 @@ def _sample_neg(self, assign_result, num_expected, **kwargs): if neg_inds.numel() != 0: neg_inds = neg_inds.squeeze(1) if len(neg_inds) <= 0: - raise NotImplementedError( - 'Not support sampling the negative samples when the length ' - 'of negative samples is 0') + return neg_inds.squeeze(1) else: neg_inds_choice = neg_inds.new_zeros([0]) extend_num = 0 diff --git a/mmdet3d/core/bbox/structures/depth_box3d.py b/mmdet3d/core/bbox/structures/depth_box3d.py index 38c1ec7437..dd9278bfb4 100644 --- a/mmdet3d/core/bbox/structures/depth_box3d.py +++ b/mmdet3d/core/bbox/structures/depth_box3d.py @@ -71,7 +71,9 @@ def corners(self): (x0, y0, z0) + ----------- + --------> right x (x1, y0, z0) """ - assert len(self.tensor) != 0 + if self.tensor.numel() == 0: + return torch.empty([0, 8, 3], device=self.tensor.device) + dims = self.dims corners_norm = torch.from_numpy( np.stack(np.unravel_index(np.arange(8), [2] * 3), axis=1)).to( diff --git a/mmdet3d/core/bbox/structures/lidar_box3d.py b/mmdet3d/core/bbox/structures/lidar_box3d.py index 0d14655c63..706a6c0d58 100644 --- a/mmdet3d/core/bbox/structures/lidar_box3d.py +++ b/mmdet3d/core/bbox/structures/lidar_box3d.py @@ -69,9 +69,9 @@ def corners(self): left y<-------- + ----------- + (x0, y1, z0) (x0, y0, z0) """ - # TODO: rotation_3d_in_axis function do not support - # empty tensor currently. - assert len(self.tensor) != 0 + if self.tensor.numel() == 0: + return torch.empty([0, 8, 3], device=self.tensor.device) + dims = self.dims corners_norm = torch.from_numpy( np.stack(np.unravel_index(np.arange(8), [2] * 3), axis=1)).to( diff --git a/mmdet3d/core/post_processing/box3d_nms.py b/mmdet3d/core/post_processing/box3d_nms.py index c2bfcb0267..a8d6521d6f 100644 --- a/mmdet3d/core/post_processing/box3d_nms.py +++ b/mmdet3d/core/post_processing/box3d_nms.py @@ -219,4 +219,8 @@ def circle_nms(dets, thresh, post_max_size=83): # ovr = inter / areas[j] if dist <= thresh: suppressed[j] = 1 - return keep[:post_max_size] + + if post_max_size < len(keep): + return keep[:post_max_size] + + return keep diff --git a/mmdet3d/datasets/kitti_dataset.py b/mmdet3d/datasets/kitti_dataset.py index 2ed86cc1d7..9a864efff2 100644 --- a/mmdet3d/datasets/kitti_dataset.py +++ b/mmdet3d/datasets/kitti_dataset.py @@ -153,12 +153,33 @@ def get_ann_info(self, index): - gt_bboxes (np.ndarray): 2D ground truth bboxes. - gt_labels (np.ndarray): Labels of ground truths. - gt_names (list[str]): Class names of ground truths. + - difficulty (int): Difficulty defined by KITTI. + 0, 1, 2 represent xxxxx respectively. """ # Use index to get the annos, thus the evalhook could also use this api info = self.data_infos[index] rect = info['calib']['R0_rect'].astype(np.float32) Trv2c = info['calib']['Tr_velo_to_cam'].astype(np.float32) + if 'plane' in info: + # convert ground plane to velodyne coordinates + reverse = np.linalg.inv(rect @ Trv2c) + + (plane_norm_cam, + plane_off_cam) = (info['plane'][:3], + -info['plane'][:3] * info['plane'][3]) + plane_norm_lidar = \ + (reverse[:3, :3] @ plane_norm_cam[:, None])[:, 0] + plane_off_lidar = ( + reverse[:3, :3] @ plane_off_cam[:, None][:, 0] + + reverse[:3, 3]) + plane_lidar = np.zeros_like(plane_norm_lidar, shape=(4, )) + plane_lidar[:3] = plane_norm_lidar + plane_lidar[3] = -plane_norm_lidar.T @ plane_off_lidar + else: + plane_lidar = None + + difficulty = info['annos']['difficulty'] annos = info['annos'] # we need other objects to avoid collision when sample annos = self.remove_dontcare(annos) @@ -192,7 +213,9 @@ def get_ann_info(self, index): gt_labels_3d=gt_labels_3d, bboxes=gt_bboxes, labels=gt_labels, - gt_names=gt_names) + gt_names=gt_names, + plane=plane_lidar, + difficulty=difficulty) return anns_results def drop_arrays_by_name(self, gt_names, used_classes): @@ -319,6 +342,7 @@ def evaluate(self, If not specified, a temp file will be created. Default: None. submission_prefix (str, optional): The prefix of submission data. If not specified, the submission data will not be generated. + Default: None. show (bool, optional): Whether to visualize. Default: False. out_dir (str, optional): Path to save the visualization results. diff --git a/mmdet3d/datasets/pipelines/dbsampler.py b/mmdet3d/datasets/pipelines/dbsampler.py index d2e844e6d1..f0a7074441 100644 --- a/mmdet3d/datasets/pipelines/dbsampler.py +++ b/mmdet3d/datasets/pipelines/dbsampler.py @@ -189,7 +189,7 @@ def filter_by_min_points(db_infos, min_gt_points_dict): db_infos[name] = filtered_infos return db_infos - def sample_all(self, gt_bboxes, gt_labels, img=None): + def sample_all(self, gt_bboxes, gt_labels, img=None, ground_plane=None): """Sampling all categories of bboxes. Args: @@ -264,6 +264,15 @@ def sample_all(self, gt_bboxes, gt_labels, img=None): gt_labels = np.array([self.cat2label[s['name']] for s in sampled], dtype=np.long) + + if ground_plane is not None: + xyz = sampled_gt_bboxes[:, :3] + dz = (ground_plane[:3][None, :] * + xyz).sum(-1) + ground_plane[3] + sampled_gt_bboxes[:, 2] -= dz + for i, s_points in enumerate(s_points_list): + s_points.tensor[:, 2].sub_(dz[i]) + ret = { 'gt_labels_3d': gt_labels, diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index 276f0a9f84..6269dc217b 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -116,6 +116,10 @@ def random_flip_data_3d(self, input_dict, direction='horizontal'): updated in the result dict. """ assert direction in ['horizontal', 'vertical'] + # for semantic segmentation task, only points will be flipped. + if 'bbox3d_fields' not in input_dict: + input_dict['points'].flip(direction) + return if len(input_dict['bbox3d_fields']) == 0: # test mode input_dict['bbox3d_fields'].append('empty_box3d') input_dict['empty_box3d'] = input_dict['box_type_3d']( @@ -264,14 +268,17 @@ class ObjectSample(object): sample_2d (bool): Whether to also paste 2D image patch to the images This should be true when applying multi-modality cut-and-paste. Defaults to False. + use_ground_plane (bool): Whether to use gound plane to adjust the + 3D labels. """ - def __init__(self, db_sampler, sample_2d=False): + def __init__(self, db_sampler, sample_2d=False, use_ground_plane=False): self.sampler_cfg = db_sampler self.sample_2d = sample_2d if 'type' not in db_sampler.keys(): db_sampler['type'] = 'DataBaseSampler' self.db_sampler = build_from_cfg(db_sampler, OBJECTSAMPLERS) + self.use_ground_plane = use_ground_plane @staticmethod def remove_points_in_boxes(points, boxes): @@ -302,6 +309,11 @@ def __call__(self, input_dict): gt_bboxes_3d = input_dict['gt_bboxes_3d'] gt_labels_3d = input_dict['gt_labels_3d'] + if self.use_ground_plane and 'plane' in input_dict['ann_info']: + ground_plane = input_dict['ann_info']['plane'] + input_dict['plane'] = ground_plane + else: + ground_plane = None # change to float for blending operation points = input_dict['points'] if self.sample_2d: @@ -315,7 +327,10 @@ def __call__(self, input_dict): img=img) else: sampled_dict = self.db_sampler.sample_all( - gt_bboxes_3d.tensor.numpy(), gt_labels_3d, img=None) + gt_bboxes_3d.tensor.numpy(), + gt_labels_3d, + img=None, + ground_plane=ground_plane) if sampled_dict is not None: sampled_gt_bboxes_3d = sampled_dict['gt_bboxes_3d'] @@ -921,6 +936,12 @@ def __call__(self, results): and 'pts_semantic_mask' keys are updated in the result dict. """ points = results['points'] + # Points in Camera coord can provide the depth information. + # TODO: Need to support distance-based sampling for other coord system. + if self.sample_range is not None: + from mmdet3d.core.points import CameraPoints + assert isinstance(points, CameraPoints), \ + 'Sampling based on distance is only applicable for CAM coord' points, choices = self._points_random_sampling( points, self.num_points, diff --git a/mmdet3d/models/dense_heads/centerpoint_head.py b/mmdet3d/models/dense_heads/centerpoint_head.py index 0d043d8a6a..0529968f77 100644 --- a/mmdet3d/models/dense_heads/centerpoint_head.py +++ b/mmdet3d/models/dense_heads/centerpoint_head.py @@ -24,7 +24,7 @@ class SeparateHead(BaseModule): heads (dict): Conv information. head_conv (int, optional): Output channels. Default: 64. - final_kernal (int, optional): Kernel size for the last conv layer. + final_kernel (int, optional): Kernel size for the last conv layer. Default: 1. init_bias (float, optional): Initial bias. Default: -2.19. conv_cfg (dict, optional): Config of conv layer. @@ -137,7 +137,7 @@ class DCNSeparateHead(BaseModule): dcn_config (dict): Config of dcn layer. head_conv (int, optional): Output channels. Default: 64. - final_kernal (int, optional): Kernel size for the last conv + final_kernel (int, optional): Kernel size for the last conv layer. Default: 1. init_bias (float, optional): Initial bias. Default: -2.19. conv_cfg (dict, optional): Config of conv layer. diff --git a/mmdet3d/models/dense_heads/point_rpn_head.py b/mmdet3d/models/dense_heads/point_rpn_head.py index 50cc9d5525..48ed1324e4 100644 --- a/mmdet3d/models/dense_heads/point_rpn_head.py +++ b/mmdet3d/models/dense_heads/point_rpn_head.py @@ -272,7 +272,8 @@ def get_bboxes(self, bbox3d = self.bbox_coder.decode(bbox_preds[b], points[b, ..., :3], object_class[b]) bbox_selected, score_selected, labels, cls_preds_selected = \ - self.class_agnostic_nms(obj_scores[b], sem_scores[b], bbox3d) + self.class_agnostic_nms(obj_scores[b], sem_scores[b], bbox3d, + points[b, ..., :3], input_metas[b]) bbox = input_metas[b]['box_type_3d']( bbox_selected.clone(), box_dim=bbox_selected.shape[-1], @@ -280,7 +281,8 @@ def get_bboxes(self, results.append((bbox, score_selected, labels, cls_preds_selected)) return results - def class_agnostic_nms(self, obj_scores, sem_scores, bbox): + def class_agnostic_nms(self, obj_scores, sem_scores, bbox, points, + input_meta): """Class agnostic nms. Args: @@ -298,6 +300,29 @@ def class_agnostic_nms(self, obj_scores, sem_scores, bbox): else: nms_func = nms_normal_gpu + num_bbox = bbox.shape[0] + bbox = input_meta['box_type_3d']( + bbox.clone(), + box_dim=bbox.shape[-1], + with_yaw=True, + origin=(0.5, 0.5, 0.5)) + + if isinstance(bbox, LiDARInstance3DBoxes): + box_idx = bbox.points_in_boxes(points) + box_indices = box_idx.new_zeros([num_bbox + 1]) + box_idx[box_idx == -1] = num_bbox + box_indices.scatter_add_(0, box_idx.long(), + box_idx.new_ones(box_idx.shape)) + box_indices = box_indices[:-1] + nonempty_box_mask = box_indices >= 0 + elif isinstance(bbox, DepthInstance3DBoxes): + box_indices = bbox.points_in_boxes(points) + nonempty_box_mask = box_indices.T.sum(1) >= 0 + else: + raise NotImplementedError('Unsupported bbox type!') + + bbox = bbox.tensor[nonempty_box_mask] + if self.test_cfg.score_thr is not None: score_thr = self.test_cfg.score_thr keep = (obj_scores >= score_thr) diff --git a/mmdet3d/models/voxel_encoders/pillar_encoder.py b/mmdet3d/models/voxel_encoders/pillar_encoder.py index 45b8d53886..c91cf282ad 100644 --- a/mmdet3d/models/voxel_encoders/pillar_encoder.py +++ b/mmdet3d/models/voxel_encoders/pillar_encoder.py @@ -15,7 +15,6 @@ class PillarFeatureNet(nn.Module): The network prepares the pillar features and performs forward pass through PFNLayers. - Args: in_channels (int, optional): Number of input features, either x, y, z or x, y, z, r. Defaults to 4. @@ -54,7 +53,7 @@ def __init__(self, if with_cluster_center: in_channels += 3 if with_voxel_center: - in_channels += 2 + in_channels += 3 if with_distance: in_channels += 1 self._with_distance = with_distance @@ -84,8 +83,10 @@ def __init__(self, # Need pillar (voxel) size and x/y offset in order to calculate offset self.vx = voxel_size[0] self.vy = voxel_size[1] + self.vz = voxel_size[2] self.x_offset = self.vx / 2 + point_cloud_range[0] self.y_offset = self.vy / 2 + point_cloud_range[1] + self.z_offset = self.vz / 2 + point_cloud_range[2] self.point_cloud_range = point_cloud_range @force_fp32(out_fp16=True) @@ -97,7 +98,6 @@ def forward(self, features, num_points, coors): (N, M, C). num_points (torch.Tensor): Number of points in each pillar. coors (torch.Tensor): Coordinates of each voxel. - Returns: torch.Tensor: Features of pillars. """ @@ -114,21 +114,27 @@ def forward(self, features, num_points, coors): dtype = features.dtype if self._with_voxel_center: if not self.legacy: - f_center = torch.zeros_like(features[:, :, :2]) + f_center = torch.zeros_like(features[:, :, :3]) f_center[:, :, 0] = features[:, :, 0] - ( coors[:, 3].to(dtype).unsqueeze(1) * self.vx + self.x_offset) f_center[:, :, 1] = features[:, :, 1] - ( coors[:, 2].to(dtype).unsqueeze(1) * self.vy + self.y_offset) + f_center[:, :, 2] = features[:, :, 2] - ( + coors[:, 1].to(dtype).unsqueeze(1) * self.vz + + self.z_offset) else: - f_center = features[:, :, :2] + f_center = features[:, :, :3] f_center[:, :, 0] = f_center[:, :, 0] - ( coors[:, 3].type_as(features).unsqueeze(1) * self.vx + self.x_offset) f_center[:, :, 1] = f_center[:, :, 1] - ( coors[:, 2].type_as(features).unsqueeze(1) * self.vy + self.y_offset) + f_center[:, :, 2] = f_center[:, :, 2] - ( + coors[:, 1].type_as(features).unsqueeze(1) * self.vz + + self.z_offset) features_ls.append(f_center) if self._with_distance: @@ -177,6 +183,8 @@ class DynamicPillarFeatureNet(PillarFeatureNet): Defaults to dict(type='BN1d', eps=1e-3, momentum=0.01). mode (str, optional): The mode to gather point features. Options are 'max' or 'avg'. Defaults to 'max'. + legacy (bool, optional): Whether to use the new behavior or + the original behavior. Defaults to True. """ def __init__(self, @@ -188,7 +196,8 @@ def __init__(self, voxel_size=(0.2, 0.2, 4), point_cloud_range=(0, -40, -3, 70.4, 40, 1), norm_cfg=dict(type='BN1d', eps=1e-3, momentum=0.01), - mode='max'): + mode='max', + legacy=True): super(DynamicPillarFeatureNet, self).__init__( in_channels, feat_channels, @@ -198,7 +207,8 @@ def __init__(self, voxel_size=voxel_size, point_cloud_range=point_cloud_range, norm_cfg=norm_cfg, - mode=mode) + mode=mode, + legacy=legacy) self.fp16_enabled = False feat_channels = [self.in_channels] + list(feat_channels) pfn_layers = [] diff --git a/mmdet3d/ops/gather_points/gather_points.py b/mmdet3d/ops/gather_points/gather_points.py index f1051a73bd..1c290a6e5f 100644 --- a/mmdet3d/ops/gather_points/gather_points.py +++ b/mmdet3d/ops/gather_points/gather_points.py @@ -28,7 +28,7 @@ def forward(ctx, features: torch.Tensor, B, npoint = indices.size() _, C, N = features.size() - output = torch.cuda.FloatTensor(B, C, npoint) + output = features.new_zeros((B, C, npoint)) gather_points_ext.gather_points_wrapper(B, C, N, npoint, features, indices, output) @@ -42,7 +42,7 @@ def backward(ctx, grad_out): idx, C, N = ctx.for_backwards B, npoint = idx.size() - grad_features = torch.cuda.FloatTensor(B, C, N).zero_() + grad_features = grad_out.new_zeros((B, C, N)) grad_out_data = grad_out.data.contiguous() gather_points_ext.gather_points_grad_wrapper(B, C, N, npoint, grad_out_data, idx, diff --git a/mmdet3d/ops/gather_points/src/gather_points.cpp b/mmdet3d/ops/gather_points/src/gather_points.cpp index 01a3e4046f..9462d7f6d9 100644 --- a/mmdet3d/ops/gather_points/src/gather_points.cpp +++ b/mmdet3d/ops/gather_points/src/gather_points.cpp @@ -1,57 +1,54 @@ #include +#include #include #include #include #include + extern THCState *state; int gather_points_wrapper(int b, int c, int n, int npoints, - at::Tensor points_tensor, at::Tensor idx_tensor, - at::Tensor out_tensor); + at::Tensor& points_tensor, at::Tensor& idx_tensor, + at::Tensor& out_tensor); void gather_points_kernel_launcher(int b, int c, int n, int npoints, - const float *points, const int *idx, - float *out, cudaStream_t stream); + const at::Tensor& points_tensor, + const at::Tensor& idx_tensor, + at::Tensor& out_tensor); int gather_points_grad_wrapper(int b, int c, int n, int npoints, - at::Tensor grad_out_tensor, - at::Tensor idx_tensor, - at::Tensor grad_points_tensor); + at::Tensor& grad_out_tensor, + at::Tensor& idx_tensor, + at::Tensor& grad_points_tensor); void gather_points_grad_kernel_launcher(int b, int c, int n, int npoints, - const float *grad_out, const int *idx, - float *grad_points, - cudaStream_t stream); + const at::Tensor& grad_out_tensor, + const at::Tensor& idx_tensor, + at::Tensor& grad_points_tensor); int gather_points_wrapper(int b, int c, int n, int npoints, - at::Tensor points_tensor, at::Tensor idx_tensor, - at::Tensor out_tensor) { - const float *points = points_tensor.data_ptr(); - const int *idx = idx_tensor.data_ptr(); - float *out = out_tensor.data_ptr(); - - cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); - gather_points_kernel_launcher(b, c, n, npoints, points, idx, out, stream); + at::Tensor& points_tensor, at::Tensor& idx_tensor, + at::Tensor& out_tensor) +{ + gather_points_kernel_launcher(b, c, n, npoints, points_tensor, idx_tensor, out_tensor); return 1; } int gather_points_grad_wrapper(int b, int c, int n, int npoints, - at::Tensor grad_out_tensor, - at::Tensor idx_tensor, - at::Tensor grad_points_tensor) { - const float *grad_out = grad_out_tensor.data_ptr(); - const int *idx = idx_tensor.data_ptr(); - float *grad_points = grad_points_tensor.data_ptr(); - - cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); - gather_points_grad_kernel_launcher(b, c, n, npoints, grad_out, idx, - grad_points, stream); + at::Tensor& grad_out_tensor, + at::Tensor& idx_tensor, + at::Tensor& grad_points_tensor) +{ + gather_points_grad_kernel_launcher(b, c, n, npoints, grad_out_tensor, idx_tensor, + grad_points_tensor); return 1; } -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) +{ m.def("gather_points_wrapper", &gather_points_wrapper, "gather_points_wrapper"); m.def("gather_points_grad_wrapper", &gather_points_grad_wrapper, diff --git a/mmdet3d/ops/gather_points/src/gather_points_cuda.cu b/mmdet3d/ops/gather_points/src/gather_points_cuda.cu index c93f6436d2..1b4ec3f046 100644 --- a/mmdet3d/ops/gather_points/src/gather_points_cuda.cu +++ b/mmdet3d/ops/gather_points/src/gather_points_cuda.cu @@ -1,14 +1,21 @@ #include #include +#include +#include +#include +#include + +#include #define TOTAL_THREADS 1024 #define THREADS_PER_BLOCK 256 #define DIVUP(m, n) ((m) / (n) + ((m) % (n) > 0)) +template __global__ void gather_points_kernel(int b, int c, int n, int m, - const float *__restrict__ points, + const scalar_t *__restrict__ points, const int *__restrict__ idx, - float *__restrict__ out) { + scalar_t *__restrict__ out) { // points: (B, C, N) // idx: (B, M) // output: @@ -26,8 +33,10 @@ __global__ void gather_points_kernel(int b, int c, int n, int m, } void gather_points_kernel_launcher(int b, int c, int n, int npoints, - const float *points, const int *idx, - float *out, cudaStream_t stream) { + const at::Tensor& points_tensor, + const at::Tensor& idx_tensor, + at::Tensor& out_tensor) +{ // points: (B, C, N) // idx: (B, npoints) // output: @@ -35,23 +44,33 @@ void gather_points_kernel_launcher(int b, int c, int n, int npoints, cudaError_t err; dim3 blocks(DIVUP(npoints, THREADS_PER_BLOCK), c, - b); // blockIdx.x(col), blockIdx.y(row) + b); // blockIdx.x(col), blockIdx.y(row) dim3 threads(THREADS_PER_BLOCK); - - gather_points_kernel<<>>(b, c, n, npoints, points, - idx, out); - + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + out_tensor.scalar_type(), "gather_points_kernel", + [&] + { + const scalar_t *points = points_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + scalar_t *out = out_tensor.data_ptr(); + gather_points_kernel<<>>(b, c, n, npoints, points, + idx, out); + }); err = cudaGetLastError(); - if (cudaSuccess != err) { + if (cudaSuccess != err) + { fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err)); exit(-1); } } +template __global__ void gather_points_grad_kernel(int b, int c, int n, int m, - const float *__restrict__ grad_out, + const scalar_t *__restrict__ grad_out, const int *__restrict__ idx, - float *__restrict__ grad_points) { + scalar_t *__restrict__ grad_points) { // grad_out: (B, C, M) // idx: (B, M) // output: @@ -70,9 +89,10 @@ __global__ void gather_points_grad_kernel(int b, int c, int n, int m, } void gather_points_grad_kernel_launcher(int b, int c, int n, int npoints, - const float *grad_out, const int *idx, - float *grad_points, - cudaStream_t stream) { + const at::Tensor& grad_out_tensor, + const at::Tensor& idx_tensor, + at::Tensor& grad_points_tensor) +{ // grad_out: (B, C, npoints) // idx: (B, npoints) // output: @@ -80,14 +100,24 @@ void gather_points_grad_kernel_launcher(int b, int c, int n, int npoints, cudaError_t err; dim3 blocks(DIVUP(npoints, THREADS_PER_BLOCK), c, - b); // blockIdx.x(col), blockIdx.y(row) + b); // blockIdx.x(col), blockIdx.y(row) dim3 threads(THREADS_PER_BLOCK); - gather_points_grad_kernel<<>>( - b, c, n, npoints, grad_out, idx, grad_points); + cudaStream_t stream = at::cuda::getCurrentCUDAStream().stream(); + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + grad_points_tensor.scalar_type(), "gather_points_grad_kernel", + [&] + { + const scalar_t *grad_out = grad_out_tensor.data_ptr(); + const int *idx = idx_tensor.data_ptr(); + scalar_t *grad_points = grad_points_tensor.data_ptr(); + gather_points_grad_kernel<<>>( + b, c, n, npoints, grad_out, idx, grad_points); + }); err = cudaGetLastError(); - if (cudaSuccess != err) { + if (cudaSuccess != err) + { fprintf(stderr, "CUDA kernel failed : %s\n", cudaGetErrorString(err)); exit(-1); } diff --git a/mmdet3d/ops/group_points/group_points.py b/mmdet3d/ops/group_points/group_points.py index e92d39f7ca..961285d749 100644 --- a/mmdet3d/ops/group_points/group_points.py +++ b/mmdet3d/ops/group_points/group_points.py @@ -2,6 +2,7 @@ from typing import Tuple import torch +from mmcv.runner import force_fp32 from torch import nn as nn from torch.autograd import Function @@ -62,7 +63,9 @@ def __init__(self, if self.max_radius is None: assert not self.normalize_xyz, \ 'can not normalize grouped xyz when max_radius is None' + self.fp16_enabled = False + @force_fp32() def forward(self, points_xyz, center_xyz, features=None): """forward. @@ -143,7 +146,9 @@ class GroupAll(nn.Module): def __init__(self, use_xyz: bool = True): super().__init__() self.use_xyz = use_xyz + self.fp16_enabled = False + @force_fp32() def forward(self, xyz: torch.Tensor, new_xyz: torch.Tensor, diff --git a/mmdet3d/utils/setup_env.py b/mmdet3d/utils/setup_env.py index 282ad491e7..98bcc8853f 100644 --- a/mmdet3d/utils/setup_env.py +++ b/mmdet3d/utils/setup_env.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. -import cv2 import os import platform import warnings + +import cv2 from torch import multiprocessing as mp diff --git a/mmdet3d/version.py b/mmdet3d/version.py index f887a8eae6..9b89d23fe2 100644 --- a/mmdet3d/version.py +++ b/mmdet3d/version.py @@ -1,6 +1,6 @@ # Copyright (c) Open-MMLab. All rights reserved. -__version__ = '0.17.3' +__version__ = '1.0.0rc0' short_version = __version__ diff --git a/setup.py b/setup.py index 5023af0c6f..4e896e2b82 100644 --- a/setup.py +++ b/setup.py @@ -217,8 +217,6 @@ def add_mim_extention(): 'Programming Language :: Python :: 3.7', ], license='Apache License 2.0', - setup_requires=parse_requirements('requirements/build.txt'), - tests_require=parse_requirements('requirements/tests.txt'), install_requires=parse_requirements('requirements/runtime.txt'), extras_require={ 'all': parse_requirements('requirements.txt'), diff --git a/tests/test_models/test_common_modules/test_pointnet_ops.py b/tests/test_models/test_common_modules/test_pointnet_ops.py index 095c2d01ca..ec3a581e63 100644 --- a/tests/test_models/test_common_modules/test_pointnet_ops.py +++ b/tests/test_models/test_common_modules/test_pointnet_ops.py @@ -2,9 +2,16 @@ import pytest import torch -from mmdet3d.ops import (ball_query, furthest_point_sample, - furthest_point_sample_with_dist, gather_points, - grouping_operation, knn, three_interpolate, three_nn) +from mmdet3d.ops import ( + ball_query, + furthest_point_sample, + furthest_point_sample_with_dist, + gather_points, + grouping_operation, + knn, + three_interpolate, + three_nn, +) def test_fps(): @@ -236,6 +243,8 @@ def test_gather_points(): [-0.7172, 0.0462, -0.6227, -0.7172, -0.7172, -0.7172]]]).cuda() assert torch.allclose(output, expected_output) + output_half = gather_points(features.half(), idx) + assert torch.allclose(output_half, expected_output.half()) def test_three_interpolate(): diff --git a/tests/test_utils/test_setup_env.py b/tests/test_utils/test_setup_env.py index 08233efa71..0c070c9f0e 100644 --- a/tests/test_utils/test_setup_env.py +++ b/tests/test_utils/test_setup_env.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. -import cv2 import multiprocessing as mp import os import platform + +import cv2 from mmcv import Config from mmdet3d.utils import setup_multi_processes diff --git a/tools/data_converter/kitti_data_utils.py b/tools/data_converter/kitti_data_utils.py index 206d50d680..8e3dba6f35 100644 --- a/tools/data_converter/kitti_data_utils.py +++ b/tools/data_converter/kitti_data_utils.py @@ -4,6 +4,7 @@ from os import path as osp from pathlib import Path +import mmcv import numpy as np from skimage import io diff --git a/tools/test.py b/tools/test.py index f36e572054..cff933a89e 100644 --- a/tools/test.py +++ b/tools/test.py @@ -36,6 +36,18 @@ def parse_args(): action='store_true', help='Whether to fuse conv and bn, this will slightly increase' 'the inference speed') + parser.add_argument( + '--gpu-ids', + type=int, + nargs='+', + help='(Deprecated, please use --gpu-id) ids of gpus to use ' + '(only applicable to non-distributed training)') + parser.add_argument( + '--gpu-id', + type=int, + default=0, + help='id of gpu to use ' + '(only applicable to non-distributed testing)') parser.add_argument( '--format-only', action='store_true', @@ -152,6 +164,15 @@ def main(): for ds_cfg in cfg.data.test: ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline) + if args.gpu_ids is not None: + cfg.gpu_ids = args.gpu_ids[0:1] + warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. ' + 'Because we only support single GPU mode in ' + 'non-distributed testing. Use the first GPU ' + 'in `gpu_ids` now.') + else: + cfg.gpu_ids = [args.gpu_id] + # init distributed env first, since logger depends on the dist info. if args.launcher == 'none': distributed = False @@ -195,7 +216,7 @@ def main(): model.PALETTE = dataset.PALETTE if not distributed: - model = MMDataParallel(model, device_ids=[0]) + model = MMDataParallel(model, device_ids=cfg.gpu_ids) outputs = single_gpu_test(model, data_loader, args.show, args.show_dir) else: model = MMDistributedDataParallel( diff --git a/tools/train.py b/tools/train.py index 3dc86a5ec4..eb0ad4c01f 100644 --- a/tools/train.py +++ b/tools/train.py @@ -43,13 +43,19 @@ def parse_args(): group_gpus.add_argument( '--gpus', type=int, - help='number of gpus to use ' + help='(Deprecated, please use --gpu-id) number of gpus to use ' '(only applicable to non-distributed training)') group_gpus.add_argument( '--gpu-ids', type=int, nargs='+', - help='ids of gpus to use ' + help='(Deprecated, please use --gpu-id) ids of gpus to use ' + '(only applicable to non-distributed training)') + group_gpus.add_argument( + '--gpu-id', + type=int, + default=0, + help='number of gpus to use ' '(only applicable to non-distributed training)') parser.add_argument('--seed', type=int, default=0, help='random seed') parser.add_argument( @@ -122,10 +128,19 @@ def main(): osp.splitext(osp.basename(args.config))[0]) if args.resume_from is not None: cfg.resume_from = args.resume_from + if args.gpus is not None: + cfg.gpu_ids = range(1) + warnings.warn('`--gpus` is deprecated because we only support ' + 'single GPU mode in non-distributed training. ' + 'Use `gpus=1` now.') if args.gpu_ids is not None: - cfg.gpu_ids = args.gpu_ids - else: - cfg.gpu_ids = range(1) if args.gpus is None else range(args.gpus) + cfg.gpu_ids = args.gpu_ids[0:1] + warnings.warn('`--gpu-ids` is deprecated, please use `--gpu-id`. ' + 'Because we only support single GPU mode in ' + 'non-distributed training. Use the first GPU ' + 'in `gpu_ids` now.') + if args.gpus is None and args.gpu_ids is None: + cfg.gpu_ids = [args.gpu_id] if args.autoscale_lr: # apply the linear scaling rule (https://arxiv.org/abs/1706.02677) From 7f48ba791a39cba05d7f3d9ac2e8aaf294650d71 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Wed, 23 Feb 2022 12:31:27 +0300 Subject: [PATCH 6/8] fix lint, add docstring --- .../evaluate_semantic_instance.py | 18 ++++++++++++++++-- .../core/evaluation/scannet_utils/util_3d.py | 1 + mmdet3d/datasets/scannet_dataset.py | 3 ++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py b/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py index 26a3a70286..e4b94395fc 100644 --- a/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py +++ b/mmdet3d/core/evaluation/scannet_utils/evaluate_semantic_instance.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. # adapted from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/3d_evaluation/evaluate_semantic_instance.py # noqa -import numpy as np from copy import deepcopy +import numpy as np + from . import util_3d @@ -219,7 +220,20 @@ def compute_averages(aps, options, class_labels): def assign_instances_for_scan(pred_info, gt_ids, options, valid_class_ids, class_labels, id_to_label): - """Assign gt and predicted instances for a single scene.""" + """Assign gt and predicted instances for a single scene. + + Args: + pred_info (dict): Predicted masks, labels and scores. + gt_ids (np.array): Ground truth instance masks. + options (dict): ScanNet evaluator options. See get_options. + valid_class_ids (tuple[int]): Ids of valid categories. + class_labels (tuple[str]): Class names. + id_to_label (dict[int, str]): Mapping of valid class id to class label. + + Returns: + dict: Per class assigned gt to predicted instances. + dict: Per class assigned predicted to gt instances. + """ # get gt instances gt_instances = util_3d.get_instances(gt_ids, valid_class_ids, class_labels, id_to_label) diff --git a/mmdet3d/core/evaluation/scannet_utils/util_3d.py b/mmdet3d/core/evaluation/scannet_utils/util_3d.py index fd9291de94..527d341266 100644 --- a/mmdet3d/core/evaluation/scannet_utils/util_3d.py +++ b/mmdet3d/core/evaluation/scannet_utils/util_3d.py @@ -1,6 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. # adapted from https://github.com/ScanNet/ScanNet/blob/master/BenchmarkScripts/util_3d.py # noqa import json + import numpy as np diff --git a/mmdet3d/datasets/scannet_dataset.py b/mmdet3d/datasets/scannet_dataset.py index 88db58b31c..87084010e1 100644 --- a/mmdet3d/datasets/scannet_dataset.py +++ b/mmdet3d/datasets/scannet_dataset.py @@ -1,9 +1,10 @@ # Copyright (c) OpenMMLab. All rights reserved. -import numpy as np import tempfile import warnings from os import path as osp +import numpy as np + from mmdet3d.core import instance_seg_eval, show_result, show_seg_result from mmdet3d.core.bbox import DepthInstance3DBoxes from mmdet.datasets import DATASETS From 3a8c8a2464a2f4a1621953628655b28864692c76 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Wed, 23 Feb 2022 16:58:07 +0300 Subject: [PATCH 7/8] V1.0.0.dev0 (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bump to v0.18.0 (#1148) * Update README & getting_started & changelog & version.py * resolve comments * Update the required maximum version of mmcv-full * Update changelog.md * Update getting_started.md * Add highlight in changelog.md * update imvoxelnet citation (#1153) * deprecate the support for "python setup.py test" (#1164) * add OpenMMLab website and platform links (#1176) * [Fix] Update regnet metafile (#1184) * update regnet metafile * Add Architechture in RegNet metafile * fix axis=0 rotation in master branch (#1182) * [Enhance] support semantic seg in Flip3D augmentation (#1181) * support semantic seg in flip * change box_dtype_3d to bbox3d_fields * add point cloud annotation in doc (#1185) * [Fix] Fix bug in non-distributed multi-gpu training/testing (#1197) * Fix bug in non-distributed multi-gpu training/testing * add deprecated warning to void BC-breaking * Add missing explanation of cam_intrinsic in the nuScenes dataset doc (#1193) * [Fix] Fix corner bug in different coordinates (#1212) * fix corner bug in cam coord * fix corner bugs in depth & lidar coord * Bump to v0.18.1 (#1218) * bump to v0.18.1 * fix comments * update bazel version (#1223) * [Fix] fix pointpillars bug on kitti in master branch (#1163) * fix pointpillars bug on kitti in master branch * replace with AlignedAnchorGenerator * Update the installation of MMCV (#1226) * [Fix] Fix a potential overflow bug when post_max_size is set too large in the circle_nms (#1225) * [Fix] overflow bug * [Fix] typo Co-authored-by: zeyuzeng Co-authored-by: zeyu-hello * Fix PointRCNN bugs (#1224) * fix bug to allow pointnet to train in fp16 (#1207) * fix bug to allow pointnet to train in fp16 * remove unused import * fix lint * fix lint for gather_points_cuda.cu Co-authored-by: peng * [Fix] Recheck import sorting (#1242) * fix missed mmcv * reinstall pre-commit and recheck * fix inference_demo.ipynb bug (#1236) * [Enhance] upgrade PointPillars performace on dev branch (#1166) * upgrade PointPillars performace on dev branch * update DynamicPillarFeatureNet * fix comments * change to AlignedAnchor3DRangeGenerator * change to AlignedAnchor3DRangeGenerator * fix * replace with AlignedAnchorGenerator * fix lint * update using isort * Bump to v1.0.0.rc0 (#928) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import (#839) * [Enhance] refactor iou_neg_piecewise_sampler.py (#842) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import * refactor iou_neg_piecewise_sampler.py * add docstring * modify docstring Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: THU17cyz * [Feature] Add roipooling cuda ops (#843) * [Refactor] Main code modification for coordinate system refactor (#677) * [Enhance] Add script for data update (#774) * Fixed wrong config paths and fixed a bug in test * Fixed metafile * Coord sys refactor (main code) * Update test_waymo_dataset.py * Manually resolve conflict * Removed unused lines and fixed imports * remove coord2box and box2coord * update dir_limit_offset * Some minor improvements * Removed some \s in comments * Revert a change * Change Box3DMode to Coord3DMode where points are converted * Fix points_in_bbox function * Fix Imvoxelnet config * Revert adding a line * Fix rotation bug when batch size is 0 * Keep sign of dir_scores as before * Fix several comments * Add a comment * Fix docstring * Add data update scripts * Fix comments * fix import * add roipooling cuda ops * add roi extractor * add test_roi_extractor unittest * Modify setup.py to install roipooling ops * modify docstring * remove enlarge bbox in roipoint pooling * add_roipooling_ops * modify docstring Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: THU17cyz * [Refactor] Refactor code structure and docstrings (#803) * refactor points_in_boxes * Merge same functions of three boxes * More docstring fixes and unify x/y/z size * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Add "optional" and fix "Default" * Remove None in function param type * Fix unittest * Add comments for NMS functions * Merge methods of Points * Add unittest * Add optional and default value * Fix box conversion and add unittest * Fix comments * Add unit test * Indent * Fix CI * Remove useless \\ * Remove useless \\ * Remove useless \\ * Remove useless \\ * Remove useless \\ * Add unit test for box bev * More unit tests and refine docstrings in box_np_ops * Fix comment * Add deprecation warning * [Feature] PointXYZWHLRBBoxCoder (#856) * support PointBasedBoxCoder * fix unittest bug * support unittest in gpu * support unittest in gpu * modified docstring * add args * add args * [Enhance] Change Groupfree3D config (#855) * All mods * PointSample * PointSample * [Doc] Add tutorials/data_pipeline Chinese version (#827) * [Doc] Add tutorials/data_pipeline Chinese version * refine doc * Use the absolute link * Use the absolute link Co-authored-by: Tai-Wang * [Doc] Add Chinese doc for `scannet_det.md` (#836) * Part * Complete * Fix comments * Fix comments * [Doc] Add Chinese doc for `waymo_det.md` (#859) * Add complete translation * Refinements * Fix comments * Fix a minor typo Co-authored-by: Tai-Wang * Remove 2D annotations on Lyft (#867) * Add header for files (#869) * Add header for files * Add header for files * Add header for files * Add header for files * [fix] fix typos (#872) * Fix 3 unworking configs (#882) * [Fix] Fix `index.rst` for Chinese docs (#873) * Fix index.rst for zh docs * Change switch language * [Fix] Centerpoint head nested list transpose (#879) * FIX Transpose nested lists without Numpy * Removed unused Numpy import * [Enhance] Update PointFusion (#791) * update point fusion * remove LIDAR hardcode * move get_proj_mat_by_coord_type to utils * fix lint * remove todo * fix lint * [Doc] Add nuscenes_det.md Chinese version (#854) * add nus chinese doc * add nuScenes Chinese doc * fix typo * fix typo * fix typo * fix typo * fix typo * [Fix] Fix RegNet pretrained weight loading (#889) * Fix regnet pretrained weight loading * Remove unused file * Fix centerpoint tta (#892) * [Enhance] Add benchmark regression script (#808) * Initial commit * [Feature] Support DGCNN (v1.0.0.dev0) (#896) * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * support dgcnn * fix typo * fix typo * fix typo * del gf&fa registry (wo reuse pointnet module) * fix typo * add benchmark and add copyright header (for DGCNN only) * fix typo * fix typo * fix typo * fix typo * fix typo * support dgcnn * Change cam rot_3d_in_axis (#906) * [Doc] Add coord sys tutorial pic and change links to dev branch (#912) * Modify link branch and add pic * Fix pic * Update v1.0.0rc0 changelog * Remove v1.0.0.rc0 changelog * Init v1.0.0.rc0 changelog * Fix minor typos in the zh-CN index.rst * Add master updates in changelog v1.0.0 * Update changelog_v1.0.md * Update changelog_v1.0.md * Update changelog_v1.0.md * Adjust the order of logs * Update the number of developers * Add v1.0.0.rc0 changelog * Delete changelog_v1.0.md * Add RoIPointPool3d back * Change openmmlab pre-commit hook rev * Add links for compatibility doc * Refine details * Update README.md * Update highlights * Update README_zh-CN.md * Update getting_started.md * Update getting_started.md * Update version.py * Fix the released version name * Fix version name * Update version name * Update version name * Update version name * Update version name Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Co-authored-by: THU17cyz Co-authored-by: Wenhao Wu <79644370+wHao-Wu@users.noreply.github.com> Co-authored-by: dingchang Co-authored-by: 谢恩泽 Co-authored-by: Robin Karlsson <34254153+robin-karlsson0@users.noreply.github.com> Co-authored-by: Danila Rukhovich Co-authored-by: ChaimZhu * fix ci in dev branch (#1264) Co-authored-by: Wenhao Wu <79644370+wHao-Wu@users.noreply.github.com> Co-authored-by: ChaimZhu Co-authored-by: Tai-Wang Co-authored-by: Double-Z Co-authored-by: zeyuzeng Co-authored-by: zeyu-hello Co-authored-by: maskjp Co-authored-by: peng Co-authored-by: Yezhen Cong <52420115+THU17cyz@users.noreply.github.com> Co-authored-by: Xi Liu <75658786+xiliu8006@users.noreply.github.com> Co-authored-by: THU17cyz Co-authored-by: dingchang Co-authored-by: 谢恩泽 Co-authored-by: Robin Karlsson <34254153+robin-karlsson0@users.noreply.github.com> --- mmdet3d/ops/group_points/group_points.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mmdet3d/ops/group_points/group_points.py b/mmdet3d/ops/group_points/group_points.py index 961285d749..0430aa65e6 100644 --- a/mmdet3d/ops/group_points/group_points.py +++ b/mmdet3d/ops/group_points/group_points.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. + from typing import Tuple import torch From 27e6af29a40fd84623fc32ed80d954ea5f6c9b13 Mon Sep 17 00:00:00 2001 From: Danila Rukhovich Date: Tue, 1 Mar 2022 12:12:03 +0300 Subject: [PATCH 8/8] try revert imports --- .../test_common_modules/test_pointnet_ops.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/test_models/test_common_modules/test_pointnet_ops.py b/tests/test_models/test_common_modules/test_pointnet_ops.py index ec3a581e63..30ad69b5d6 100644 --- a/tests/test_models/test_common_modules/test_pointnet_ops.py +++ b/tests/test_models/test_common_modules/test_pointnet_ops.py @@ -2,16 +2,9 @@ import pytest import torch -from mmdet3d.ops import ( - ball_query, - furthest_point_sample, - furthest_point_sample_with_dist, - gather_points, - grouping_operation, - knn, - three_interpolate, - three_nn, -) +from mmdet3d.ops import (ball_query, furthest_point_sample, + furthest_point_sample_with_dist, gather_points, + grouping_operation, knn, three_interpolate, three_nn) def test_fps():